bulk MARC record import - speed improved
Changes to improve speed of MARC bib and item
imports:
[1] Turn off autocommit and commit database
transactions in larger batches.
[2] Introduce a new C4::Biblio function (AddBiblioAndItems)
to combine AddBiblio and AddItems -- this is faster
because we are not parsing the MARC XML of the biblio
every time we add an item.
[3] Introduce FasterTransformMarcToKoha, which is much
faster than TransformMarcToKoha. The new version,
which will replace the old one once it has been
fully tested, scans through each field in the
MARC record just once, instead of potentially
dozens of times.
[4] Remove code in bulkmarcexport that moved the
item tags to separate MARC::Record objects.
Signed-off-by: Chris Cormack <crc@liblime.com>
Signed-off-by: Joshua Ferraro <jmf@liblime.com>
This commit is contained in:
Galen Charlton
Joshua Ferraro
parent
2143002750
commit
7d47666f7e
227
C4/Biblio.pm
227
C4/Biblio.pm
|
|
@ -41,7 +41,7 @@ use vars qw($VERSION @ISA @EXPORT);
|
|||
# EXPORTED FUNCTIONS.
|
||||
|
||||
# to add biblios or items
|
||||
push @EXPORT, qw( &AddBiblio &AddItem );
|
||||
push @EXPORT, qw( &AddBiblio &AddItem &AddBiblioAndItems );
|
||||
|
||||
# to get something
|
||||
push @EXPORT, qw(
|
||||
|
|
@ -227,6 +227,115 @@ sub AddBiblio {
|
|||
return ( $biblionumber, $biblioitemnumber );
|
||||
}
|
||||
|
||||
=head2 AddBiblioAndItems
|
||||
|
||||
=over 4
|
||||
|
||||
($biblionumber,$biblioitemnumber, $itemnumber_ref) = AddBiblioAndItems($record, $frameworkcode);
|
||||
|
||||
=back
|
||||
|
||||
Efficiently add a biblio record and create item records from its
|
||||
embedded item fields. This routine is suitable for batch jobs.
|
||||
|
||||
The goal of this API is to have a similar effect to using AddBiblio
|
||||
and AddItems in succession, but without inefficient repeated
|
||||
parsing of the MARC XML bib record.
|
||||
|
||||
=cut
|
||||
|
||||
sub AddBiblioAndItems {
|
||||
my ( $record, $frameworkcode ) = @_;
|
||||
my ($biblionumber,$biblioitemnumber,$error);
|
||||
my @itemnumbers = ();
|
||||
my $dbh = C4::Context->dbh;
|
||||
|
||||
# transform the data into koha-table style data
|
||||
# FIXME - this paragraph copied from AddBiblio
|
||||
my $olddata = FasterTransformMarcToKoha( $dbh, $record, $frameworkcode );
|
||||
($biblionumber,$error) = _koha_add_biblio( $dbh, $olddata, $frameworkcode );
|
||||
$olddata->{'biblionumber'} = $biblionumber;
|
||||
($biblioitemnumber,$error) = _koha_add_biblioitem( $dbh, $olddata );
|
||||
|
||||
# FIXME - this paragraph copied from AddBiblio
|
||||
_koha_marc_update_bib_ids($record, $frameworkcode, $biblionumber, $biblioitemnumber);
|
||||
|
||||
# now we loop through the item tags and start creating items
|
||||
my ($itemtag, $itemsubfield) = &GetMarcFromKohaField("items.itemnumber",'');
|
||||
foreach my $item_field ($record->field($itemtag)) {
|
||||
# we take the item field and stick it into a new
|
||||
# MARC record -- this is required so far because (FIXME)
|
||||
# TransformMarcToKoha requires a MARC::Record, not a MARC::Field
|
||||
# and there is no TransformMarcFieldToKoha
|
||||
my $temp_item_marc = MARC::Record->new();
|
||||
$temp_item_marc->append_fields($item_field);
|
||||
|
||||
# add biblionumber and biblioitemnumber
|
||||
my $item = &FasterTransformMarcToKoha( $dbh, $temp_item_marc, $frameworkcode, 'items' );
|
||||
$item->{'biblionumber'} = $biblionumber;
|
||||
$item->{'biblioitemnumber'} = $biblioitemnumber;
|
||||
|
||||
# figure out what item type to use -- biblioitem-level or item-level
|
||||
my $itemtype;
|
||||
if (C4::Context->preference('item-level_itypes')) {
|
||||
$itemtype = $item->{'itype'};
|
||||
} else {
|
||||
$itemtype = $olddata->{'itemtype'};
|
||||
}
|
||||
|
||||
# FIXME - notforloan stuff copied from AddItem
|
||||
my $sth = $dbh->prepare("SELECT notforloan FROM itemtypes WHERE itemtype=?");
|
||||
$sth->execute($itemtype);
|
||||
my $notforloan = $sth->fetchrow;
|
||||
##Change the notforloan field if $notforloan found
|
||||
if ( $notforloan > 0 ) {
|
||||
$item->{'notforloan'} = $notforloan;
|
||||
&MARCitemchange( $temp_item_marc, "items.notforloan", $notforloan );
|
||||
}
|
||||
|
||||
# FIXME - dateaccessioned stuff copied from AddItem
|
||||
if ( !$item->{'dateaccessioned'} || $item->{'dateaccessioned'} eq '' ) {
|
||||
|
||||
# find today's date
|
||||
my ( $sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst ) =
|
||||
localtime(time);
|
||||
$year += 1900;
|
||||
$mon += 1;
|
||||
my $date =
|
||||
"$year-" . sprintf( "%0.2d", $mon ) . "-" . sprintf( "%0.2d", $mday );
|
||||
$item->{'dateaccessioned'} = $date;
|
||||
&MARCitemchange( $temp_item_marc, "items.dateaccessioned", $date );
|
||||
}
|
||||
|
||||
my ( $itemnumber, $error ) = &_koha_new_items( $dbh, $item, $item->{barcode} );
|
||||
warn $error if $error;
|
||||
push @itemnumbers, $itemnumber; # FIXME not checking error
|
||||
|
||||
# FIXME - not copied from AddItem
|
||||
# FIXME - AddItems equiv code about passing $sth to TransformKohaToMarcOneField is stupid
|
||||
&MARCitemchange( $temp_item_marc, "items.itemnumber", $itemnumber );
|
||||
|
||||
&logaction(C4::Context->userenv->{'number'},"CATALOGUING","ADD",$itemnumber,"item")
|
||||
if C4::Context->preference("CataloguingLog");
|
||||
|
||||
$item_field->replace_with($temp_item_marc->field($itemtag));
|
||||
}
|
||||
|
||||
# now add the record
|
||||
# FIXME - this paragraph copied from AddBiblio -- however, moved since
|
||||
# since we need to create the items row and plug in the itemnumbers in the
|
||||
# MARC
|
||||
$biblionumber = ModBiblioMarc( $record, $biblionumber, $frameworkcode );
|
||||
|
||||
# FIXME - when using this API, do we log both bib and item add, or just
|
||||
# bib
|
||||
&logaction(C4::Context->userenv->{'number'},"CATALOGUING","ADD",$biblionumber,"biblio")
|
||||
if C4::Context->preference("CataloguingLog");
|
||||
|
||||
return ( $biblionumber, $biblioitemnumber, \@itemnumbers);
|
||||
|
||||
}
|
||||
|
||||
=head2 AddItem
|
||||
|
||||
=over 2
|
||||
|
|
@ -2505,6 +2614,122 @@ sub TransformMarcToKoha {
|
|||
}
|
||||
|
||||
|
||||
# cache inverted MARC field map
|
||||
our $inverted_field_map;
|
||||
|
||||
=head2 FasterTransformMarcToKoha
|
||||
|
||||
=over 4
|
||||
|
||||
$result = FasterTransformMarcToKoha( $dbh, $record, $frameworkcode )
|
||||
|
||||
=back
|
||||
|
||||
Extract data from a MARC bib record into a hashref representing
|
||||
Koha biblio, biblioitems, and items fields. This function will
|
||||
replace TransformMarcToKoha once it has been tested.
|
||||
|
||||
=cut
|
||||
sub FasterTransformMarcToKoha {
|
||||
my ( $dbh, $record, $frameworkcode, $limit_table ) = @_;
|
||||
|
||||
my $result;
|
||||
|
||||
unless (defined $inverted_field_map) {
|
||||
$inverted_field_map = _get_inverted_marc_field_map();
|
||||
}
|
||||
|
||||
my %tables = ();
|
||||
if ($limit_table eq 'items') {
|
||||
$tables{'items'} = 1;
|
||||
} else {
|
||||
$tables{'items'} = 1;
|
||||
$tables{'biblio'} = 1;
|
||||
$tables{'biblioitems'} = 1;
|
||||
}
|
||||
|
||||
# traverse through record
|
||||
MARCFIELD: foreach my $field ($record->fields()) {
|
||||
my $tag = $field->tag();
|
||||
next MARCFIELD unless exists $inverted_field_map->{$frameworkcode}->{$tag};
|
||||
if ($field->is_control_field()) {
|
||||
my $kohafields = $inverted_field_map->{$frameworkcode}->{$tag}->{list};
|
||||
ENTRY: foreach my $entry (@{ $kohafields }) {
|
||||
my ($subfield, $table, $column) = @{ $entry };
|
||||
next ENTRY unless exists $tables{$table};
|
||||
my $key = _disambiguate($table, $column);
|
||||
if ($result->{$key}) {
|
||||
unless (($key eq "biblionumber" or $key eq "biblioitemnumber") and ($field->data() eq "")) {
|
||||
$result->{$key} .= " | " . $field->data();
|
||||
}
|
||||
} else {
|
||||
$result->{$key} = $field->data();
|
||||
}
|
||||
}
|
||||
} else {
|
||||
# deal with subfields
|
||||
MARCSUBFIELD: foreach my $sf ($field->subfields()) {
|
||||
my $code = $sf->[0];
|
||||
next MARCSUBFIELD unless exists $inverted_field_map->{$frameworkcode}->{$tag}->{sfs}->{$code};
|
||||
my $value = $sf->[1];
|
||||
SFENTRY: foreach my $entry (@{ $inverted_field_map->{$frameworkcode}->{$tag}->{sfs}->{$code} }) {
|
||||
my ($table, $column) = @{ $entry };
|
||||
next SFENTRY unless exists $tables{$table};
|
||||
my $key = _disambiguate($table, $column);
|
||||
if ($result->{$key}) {
|
||||
unless (($key eq "biblionumber" or $key eq "biblioitemnumber") and ($value eq "")) {
|
||||
$result->{$key} .= " | " . $value;
|
||||
}
|
||||
} else {
|
||||
$result->{$key} = $value;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# modify copyrightdate to keep only the 1st year found
|
||||
my $temp = $result->{'copyrightdate'};
|
||||
$temp =~ m/c(\d\d\d\d)/; # search cYYYY first
|
||||
if ( $1 > 0 ) {
|
||||
$result->{'copyrightdate'} = $1;
|
||||
}
|
||||
else { # if no cYYYY, get the 1st date.
|
||||
$temp =~ m/(\d\d\d\d)/;
|
||||
$result->{'copyrightdate'} = $1;
|
||||
}
|
||||
|
||||
# modify publicationyear to keep only the 1st year found
|
||||
$temp = $result->{'publicationyear'};
|
||||
$temp =~ m/c(\d\d\d\d)/; # search cYYYY first
|
||||
if ( $1 > 0 ) {
|
||||
$result->{'publicationyear'} = $1;
|
||||
}
|
||||
else { # if no cYYYY, get the 1st date.
|
||||
$temp =~ m/(\d\d\d\d)/;
|
||||
$result->{'publicationyear'} = $1;
|
||||
}
|
||||
return $result;
|
||||
}
|
||||
|
||||
sub _get_inverted_marc_field_map {
|
||||
my $relations = C4::Context->marcfromkohafield;
|
||||
|
||||
my $field_map = {};
|
||||
my $relations = C4::Context->marcfromkohafield;
|
||||
|
||||
foreach my $frameworkcode (keys %{ $relations }) {
|
||||
foreach my $kohafield (keys %{ $relations->{$frameworkcode} }) {
|
||||
my $tag = $relations->{$frameworkcode}->{$kohafield}->[0];
|
||||
my $subfield = $relations->{$frameworkcode}->{$kohafield}->[1];
|
||||
my ($table, $column) = split /[.]/, $kohafield, 2;
|
||||
push @{ $field_map->{$frameworkcode}->{$tag}->{list} }, [ $subfield, $table, $column ];
|
||||
push @{ $field_map->{$frameworkcode}->{$tag}->{sfs}->{$subfield} }, [ $table, $column ];
|
||||
}
|
||||
}
|
||||
return $field_map;
|
||||
}
|
||||
|
||||
=head2 _disambiguate
|
||||
|
||||
=over 4
|
||||
|
|
|
|||
|
|
@ -221,109 +221,34 @@ $commitnum = $commit;
|
|||
|
||||
}
|
||||
|
||||
#1st of all, find item MARC tag.
|
||||
my ($tagfield,$tagsubfield) = &GetMarcFromKohaField("items.itemnumber",'');
|
||||
# $dbh->do("lock tables biblio write, biblioitems write, items write, marc_biblio write, marc_subfield_table write, marc_blob_subfield write, marc_word write, marc_subfield_structure write, stopwords write");
|
||||
my $dbh = C4::Context->dbh();
|
||||
$dbh->{AutoCommit} = 0;
|
||||
while ( my $record = $batch->next() ) {
|
||||
# warn "=>".$record->as_formatted;
|
||||
# warn "I:".$i;
|
||||
# warn "NUM:".$number;
|
||||
$i++;
|
||||
print ".";
|
||||
print "\r$i" unless $i % 100;
|
||||
# if ($i==$number) {
|
||||
# z3950_extended_services('commit',set_service_options('commit'));
|
||||
# print "COMMIT OPERATION SUCCESSFUL\n";
|
||||
#
|
||||
# my $timeneeded = gettimeofday - $starttime;
|
||||
# die "$i MARC records imported in $timeneeded seconds\n";
|
||||
# }
|
||||
# # perform the commit operation ever so often
|
||||
# if ($i==$commit) {
|
||||
# z3950_extended_services('commit',set_service_options('commit'));
|
||||
# $commit+=$commitnum;
|
||||
# print "COMMIT OPERATION SUCCESSFUL\n";
|
||||
# }
|
||||
#now, parse the record, extract the item fields, and store them in somewhere else.
|
||||
|
||||
## create an empty record object to populate
|
||||
my $newRecord = MARC::Record->new();
|
||||
$newRecord->leader($record->leader());
|
||||
|
||||
# go through each field in the existing record
|
||||
foreach my $oldField ( $record->fields() ) {
|
||||
|
||||
# just reproduce tags < 010 in our new record
|
||||
#
|
||||
# Fields are not necessarily only numeric in the actual world of records
|
||||
# nor in what I would recommend for additonal safe non-interfering local
|
||||
# use fields. The following regular expression match is much safer than
|
||||
# a numeric evaluation. -- thd
|
||||
if ( $oldField->tag() =~ m/^00/ ) {
|
||||
$newRecord->append_fields( $oldField );
|
||||
next();
|
||||
}
|
||||
|
||||
# store our new subfield data in this list
|
||||
my @newSubfields = ();
|
||||
|
||||
# go through each subfield code/data pair
|
||||
foreach my $pair ( $oldField->subfields() ) {
|
||||
#$pair->[1] =~ s/\<//g;
|
||||
#$pair->[1] =~ s/\>//g;
|
||||
push( @newSubfields, $pair->[0], $pair->[1] ); #char_decode($pair->[1],$char_encoding) );
|
||||
}
|
||||
|
||||
# add the new field to our new record
|
||||
my $newField = MARC::Field->new(
|
||||
$oldField->tag(),
|
||||
$oldField->indicator(1),
|
||||
$oldField->indicator(2),
|
||||
@newSubfields
|
||||
);
|
||||
|
||||
$newRecord->append_fields( $newField );
|
||||
|
||||
}
|
||||
|
||||
warn "$i ==>".$newRecord->as_formatted() if $verbose eq 2;
|
||||
my @fields = $newRecord->field($tagfield);
|
||||
my @items;
|
||||
my $nbitems=0;
|
||||
|
||||
foreach my $field (@fields) {
|
||||
my $item = MARC::Record->new();
|
||||
$item->append_fields($field);
|
||||
push @items,$item;
|
||||
$newRecord->delete_field($field);
|
||||
$nbitems++;
|
||||
}
|
||||
print "$i : $nbitems items found\n" if $verbose;
|
||||
# now, create biblio and items with Addbiblio call.
|
||||
|
||||
unless ($test_parameter) {
|
||||
my ( $bibid, $oldbibitemnum );
|
||||
eval { ( $bibid, $oldbibitemnum ) = AddBiblio( $newRecord, '' ); };
|
||||
# FIXME add back dup barcode check
|
||||
my ( $bibid, $oldbibitemnum, $itemnumbers_ref );
|
||||
eval { ( $bibid, $oldbibitemnum, $itemnumbers_ref ) = AddBiblioAndItems( $record, '' ); };
|
||||
warn $@ if $@;
|
||||
if ( $@ ) {
|
||||
warn "ERROR: Adding biblio $bibid failed\n" if $verbose
|
||||
} else {
|
||||
warn "ADDED biblio NB $bibid in DB\n" if $verbose;
|
||||
for ( my $it = 0 ; $it <= $#items ; $it++ ) {
|
||||
# FIXME - duplicate barcode check needs to become part of AddItem()
|
||||
my $itemhash = TransformMarcToKoha($dbh, $items[$it]);
|
||||
my $duplicate_barcode = exists($itemhash->{'barcode'}) && GetItemnumberFromBarcode($itemhash->{'barcode'});
|
||||
if ($duplicate_barcode) {
|
||||
warn "ERROR: cannot add item $itemhash->{'barcode'} for biblio $bibid: duplicate barcode\n" if $verbose;
|
||||
} else {
|
||||
eval { AddItem( $items[$it], $bibid, $oldbibitemnum ); };
|
||||
warn "ERROR: Adding item $it, rec $i failed\n" if ($@);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if ( $@ ) {
|
||||
warn "ERROR: Adding biblio and or items $bibid failed\n" if $verbose
|
||||
}
|
||||
$dbh->commit() if (0 == $i % $commitnum);
|
||||
}
|
||||
# # FIXME - duplicate barcode check needs to become part of AddItem()
|
||||
# my $itemhash = TransformMarcToKoha($dbh, $items[$it]);
|
||||
# my $duplicate_barcode = exists($itemhash->{'barcode'}) && GetItemnumberFromBarcode($itemhash->{'barcode'});
|
||||
# if ($duplicate_barcode) {
|
||||
# warn "ERROR: cannot add item $itemhash->{'barcode'} for biblio $bibid: duplicate barcode\n" if $verbose;
|
||||
# } else {
|
||||
# eval { AddItem( $items[$it], $bibid, $oldbibitemnum ); };
|
||||
# warn "ERROR: Adding item $it, rec $i failed\n" if ($@);
|
||||
last if $i == $number;
|
||||
}
|
||||
$dbh->commit();
|
||||
|
||||
|
||||
if ($fk_off) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue