diff --git a/C4/Biblio.pm b/C4/Biblio.pm index fc3a986f30..f873a665e5 100755 --- a/C4/Biblio.pm +++ b/C4/Biblio.pm @@ -41,7 +41,7 @@ use vars qw($VERSION @ISA @EXPORT); # EXPORTED FUNCTIONS. # to add biblios or items -push @EXPORT, qw( &AddBiblio &AddItem ); +push @EXPORT, qw( &AddBiblio &AddItem &AddBiblioAndItems ); # to get something push @EXPORT, qw( @@ -227,6 +227,115 @@ sub AddBiblio { return ( $biblionumber, $biblioitemnumber ); } +=head2 AddBiblioAndItems + +=over 4 + +($biblionumber,$biblioitemnumber, $itemnumber_ref) = AddBiblioAndItems($record, $frameworkcode); + +=back + +Efficiently add a biblio record and create item records from its +embedded item fields. This routine is suitable for batch jobs. + +The goal of this API is to have a similar effect to using AddBiblio +and AddItems in succession, but without inefficient repeated +parsing of the MARC XML bib record. + +=cut + +sub AddBiblioAndItems { + my ( $record, $frameworkcode ) = @_; + my ($biblionumber,$biblioitemnumber,$error); + my @itemnumbers = (); + my $dbh = C4::Context->dbh; + + # transform the data into koha-table style data + # FIXME - this paragraph copied from AddBiblio + my $olddata = FasterTransformMarcToKoha( $dbh, $record, $frameworkcode ); + ($biblionumber,$error) = _koha_add_biblio( $dbh, $olddata, $frameworkcode ); + $olddata->{'biblionumber'} = $biblionumber; + ($biblioitemnumber,$error) = _koha_add_biblioitem( $dbh, $olddata ); + + # FIXME - this paragraph copied from AddBiblio + _koha_marc_update_bib_ids($record, $frameworkcode, $biblionumber, $biblioitemnumber); + + # now we loop through the item tags and start creating items + my ($itemtag, $itemsubfield) = &GetMarcFromKohaField("items.itemnumber",''); + foreach my $item_field ($record->field($itemtag)) { + # we take the item field and stick it into a new + # MARC record -- this is required so far because (FIXME) + # TransformMarcToKoha requires a MARC::Record, not a MARC::Field + # and there is no TransformMarcFieldToKoha + my $temp_item_marc = MARC::Record->new(); + $temp_item_marc->append_fields($item_field); + + # add biblionumber and biblioitemnumber + my $item = &FasterTransformMarcToKoha( $dbh, $temp_item_marc, $frameworkcode, 'items' ); + $item->{'biblionumber'} = $biblionumber; + $item->{'biblioitemnumber'} = $biblioitemnumber; + + # figure out what item type to use -- biblioitem-level or item-level + my $itemtype; + if (C4::Context->preference('item-level_itypes')) { + $itemtype = $item->{'itype'}; + } else { + $itemtype = $olddata->{'itemtype'}; + } + + # FIXME - notforloan stuff copied from AddItem + my $sth = $dbh->prepare("SELECT notforloan FROM itemtypes WHERE itemtype=?"); + $sth->execute($itemtype); + my $notforloan = $sth->fetchrow; + ##Change the notforloan field if $notforloan found + if ( $notforloan > 0 ) { + $item->{'notforloan'} = $notforloan; + &MARCitemchange( $temp_item_marc, "items.notforloan", $notforloan ); + } + + # FIXME - dateaccessioned stuff copied from AddItem + if ( !$item->{'dateaccessioned'} || $item->{'dateaccessioned'} eq '' ) { + + # find today's date + my ( $sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst ) = + localtime(time); + $year += 1900; + $mon += 1; + my $date = + "$year-" . sprintf( "%0.2d", $mon ) . "-" . sprintf( "%0.2d", $mday ); + $item->{'dateaccessioned'} = $date; + &MARCitemchange( $temp_item_marc, "items.dateaccessioned", $date ); + } + + my ( $itemnumber, $error ) = &_koha_new_items( $dbh, $item, $item->{barcode} ); + warn $error if $error; + push @itemnumbers, $itemnumber; # FIXME not checking error + + # FIXME - not copied from AddItem + # FIXME - AddItems equiv code about passing $sth to TransformKohaToMarcOneField is stupid + &MARCitemchange( $temp_item_marc, "items.itemnumber", $itemnumber ); + + &logaction(C4::Context->userenv->{'number'},"CATALOGUING","ADD",$itemnumber,"item") + if C4::Context->preference("CataloguingLog"); + + $item_field->replace_with($temp_item_marc->field($itemtag)); + } + + # now add the record + # FIXME - this paragraph copied from AddBiblio -- however, moved since + # since we need to create the items row and plug in the itemnumbers in the + # MARC + $biblionumber = ModBiblioMarc( $record, $biblionumber, $frameworkcode ); + + # FIXME - when using this API, do we log both bib and item add, or just + # bib + &logaction(C4::Context->userenv->{'number'},"CATALOGUING","ADD",$biblionumber,"biblio") + if C4::Context->preference("CataloguingLog"); + + return ( $biblionumber, $biblioitemnumber, \@itemnumbers); + +} + =head2 AddItem =over 2 @@ -2505,6 +2614,122 @@ sub TransformMarcToKoha { } +# cache inverted MARC field map +our $inverted_field_map; + +=head2 FasterTransformMarcToKoha + +=over 4 + + $result = FasterTransformMarcToKoha( $dbh, $record, $frameworkcode ) + +=back + +Extract data from a MARC bib record into a hashref representing +Koha biblio, biblioitems, and items fields. This function will +replace TransformMarcToKoha once it has been tested. + +=cut +sub FasterTransformMarcToKoha { + my ( $dbh, $record, $frameworkcode, $limit_table ) = @_; + + my $result; + + unless (defined $inverted_field_map) { + $inverted_field_map = _get_inverted_marc_field_map(); + } + + my %tables = (); + if ($limit_table eq 'items') { + $tables{'items'} = 1; + } else { + $tables{'items'} = 1; + $tables{'biblio'} = 1; + $tables{'biblioitems'} = 1; + } + + # traverse through record + MARCFIELD: foreach my $field ($record->fields()) { + my $tag = $field->tag(); + next MARCFIELD unless exists $inverted_field_map->{$frameworkcode}->{$tag}; + if ($field->is_control_field()) { + my $kohafields = $inverted_field_map->{$frameworkcode}->{$tag}->{list}; + ENTRY: foreach my $entry (@{ $kohafields }) { + my ($subfield, $table, $column) = @{ $entry }; + next ENTRY unless exists $tables{$table}; + my $key = _disambiguate($table, $column); + if ($result->{$key}) { + unless (($key eq "biblionumber" or $key eq "biblioitemnumber") and ($field->data() eq "")) { + $result->{$key} .= " | " . $field->data(); + } + } else { + $result->{$key} = $field->data(); + } + } + } else { + # deal with subfields + MARCSUBFIELD: foreach my $sf ($field->subfields()) { + my $code = $sf->[0]; + next MARCSUBFIELD unless exists $inverted_field_map->{$frameworkcode}->{$tag}->{sfs}->{$code}; + my $value = $sf->[1]; + SFENTRY: foreach my $entry (@{ $inverted_field_map->{$frameworkcode}->{$tag}->{sfs}->{$code} }) { + my ($table, $column) = @{ $entry }; + next SFENTRY unless exists $tables{$table}; + my $key = _disambiguate($table, $column); + if ($result->{$key}) { + unless (($key eq "biblionumber" or $key eq "biblioitemnumber") and ($value eq "")) { + $result->{$key} .= " | " . $value; + } + } else { + $result->{$key} = $value; + } + } + } + } + } + + # modify copyrightdate to keep only the 1st year found + my $temp = $result->{'copyrightdate'}; + $temp =~ m/c(\d\d\d\d)/; # search cYYYY first + if ( $1 > 0 ) { + $result->{'copyrightdate'} = $1; + } + else { # if no cYYYY, get the 1st date. + $temp =~ m/(\d\d\d\d)/; + $result->{'copyrightdate'} = $1; + } + + # modify publicationyear to keep only the 1st year found + $temp = $result->{'publicationyear'}; + $temp =~ m/c(\d\d\d\d)/; # search cYYYY first + if ( $1 > 0 ) { + $result->{'publicationyear'} = $1; + } + else { # if no cYYYY, get the 1st date. + $temp =~ m/(\d\d\d\d)/; + $result->{'publicationyear'} = $1; + } + return $result; +} + +sub _get_inverted_marc_field_map { + my $relations = C4::Context->marcfromkohafield; + + my $field_map = {}; + my $relations = C4::Context->marcfromkohafield; + + foreach my $frameworkcode (keys %{ $relations }) { + foreach my $kohafield (keys %{ $relations->{$frameworkcode} }) { + my $tag = $relations->{$frameworkcode}->{$kohafield}->[0]; + my $subfield = $relations->{$frameworkcode}->{$kohafield}->[1]; + my ($table, $column) = split /[.]/, $kohafield, 2; + push @{ $field_map->{$frameworkcode}->{$tag}->{list} }, [ $subfield, $table, $column ]; + push @{ $field_map->{$frameworkcode}->{$tag}->{sfs}->{$subfield} }, [ $table, $column ]; + } + } + return $field_map; +} + =head2 _disambiguate =over 4 diff --git a/misc/migration_tools/bulkmarcimport.pl b/misc/migration_tools/bulkmarcimport.pl index e22e7aeab4..f920a4d6b2 100755 --- a/misc/migration_tools/bulkmarcimport.pl +++ b/misc/migration_tools/bulkmarcimport.pl @@ -221,109 +221,34 @@ $commitnum = $commit; } -#1st of all, find item MARC tag. -my ($tagfield,$tagsubfield) = &GetMarcFromKohaField("items.itemnumber",''); -# $dbh->do("lock tables biblio write, biblioitems write, items write, marc_biblio write, marc_subfield_table write, marc_blob_subfield write, marc_word write, marc_subfield_structure write, stopwords write"); +my $dbh = C4::Context->dbh(); +$dbh->{AutoCommit} = 0; while ( my $record = $batch->next() ) { -# warn "=>".$record->as_formatted; -# warn "I:".$i; -# warn "NUM:".$number; $i++; print "."; print "\r$i" unless $i % 100; -# if ($i==$number) { -# z3950_extended_services('commit',set_service_options('commit')); -# print "COMMIT OPERATION SUCCESSFUL\n"; -# -# my $timeneeded = gettimeofday - $starttime; -# die "$i MARC records imported in $timeneeded seconds\n"; -# } -# # perform the commit operation ever so often -# if ($i==$commit) { -# z3950_extended_services('commit',set_service_options('commit')); -# $commit+=$commitnum; -# print "COMMIT OPERATION SUCCESSFUL\n"; -# } - #now, parse the record, extract the item fields, and store them in somewhere else. - - ## create an empty record object to populate - my $newRecord = MARC::Record->new(); - $newRecord->leader($record->leader()); - - # go through each field in the existing record - foreach my $oldField ( $record->fields() ) { - - # just reproduce tags < 010 in our new record - # - # Fields are not necessarily only numeric in the actual world of records - # nor in what I would recommend for additonal safe non-interfering local - # use fields. The following regular expression match is much safer than - # a numeric evaluation. -- thd - if ( $oldField->tag() =~ m/^00/ ) { - $newRecord->append_fields( $oldField ); - next(); - } - - # store our new subfield data in this list - my @newSubfields = (); - - # go through each subfield code/data pair - foreach my $pair ( $oldField->subfields() ) { - #$pair->[1] =~ s/\[1] =~ s/\>//g; - push( @newSubfields, $pair->[0], $pair->[1] ); #char_decode($pair->[1],$char_encoding) ); - } - - # add the new field to our new record - my $newField = MARC::Field->new( - $oldField->tag(), - $oldField->indicator(1), - $oldField->indicator(2), - @newSubfields - ); - - $newRecord->append_fields( $newField ); - - } - - warn "$i ==>".$newRecord->as_formatted() if $verbose eq 2; - my @fields = $newRecord->field($tagfield); - my @items; - my $nbitems=0; - - foreach my $field (@fields) { - my $item = MARC::Record->new(); - $item->append_fields($field); - push @items,$item; - $newRecord->delete_field($field); - $nbitems++; - } - print "$i : $nbitems items found\n" if $verbose; - # now, create biblio and items with Addbiblio call. unless ($test_parameter) { - my ( $bibid, $oldbibitemnum ); - eval { ( $bibid, $oldbibitemnum ) = AddBiblio( $newRecord, '' ); }; + # FIXME add back dup barcode check + my ( $bibid, $oldbibitemnum, $itemnumbers_ref ); + eval { ( $bibid, $oldbibitemnum, $itemnumbers_ref ) = AddBiblioAndItems( $record, '' ); }; warn $@ if $@; - if ( $@ ) { - warn "ERROR: Adding biblio $bibid failed\n" if $verbose - } else { - warn "ADDED biblio NB $bibid in DB\n" if $verbose; - for ( my $it = 0 ; $it <= $#items ; $it++ ) { - # FIXME - duplicate barcode check needs to become part of AddItem() - my $itemhash = TransformMarcToKoha($dbh, $items[$it]); - my $duplicate_barcode = exists($itemhash->{'barcode'}) && GetItemnumberFromBarcode($itemhash->{'barcode'}); - if ($duplicate_barcode) { - warn "ERROR: cannot add item $itemhash->{'barcode'} for biblio $bibid: duplicate barcode\n" if $verbose; - } else { - eval { AddItem( $items[$it], $bibid, $oldbibitemnum ); }; - warn "ERROR: Adding item $it, rec $i failed\n" if ($@); - } - } - } - } + if ( $@ ) { + warn "ERROR: Adding biblio and or items $bibid failed\n" if $verbose + } + $dbh->commit() if (0 == $i % $commitnum); + } +# # FIXME - duplicate barcode check needs to become part of AddItem() +# my $itemhash = TransformMarcToKoha($dbh, $items[$it]); +# my $duplicate_barcode = exists($itemhash->{'barcode'}) && GetItemnumberFromBarcode($itemhash->{'barcode'}); +# if ($duplicate_barcode) { +# warn "ERROR: cannot add item $itemhash->{'barcode'} for biblio $bibid: duplicate barcode\n" if $verbose; +# } else { +# eval { AddItem( $items[$it], $bibid, $oldbibitemnum ); }; +# warn "ERROR: Adding item $it, rec $i failed\n" if ($@); last if $i == $number; } +$dbh->commit(); if ($fk_off) {