gitea
/
Koha-clms
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

speed boost: command-line stage and commit biblios 

Turned off autocommit and commit every 100 records.

Signed-off-by: Chris Cormack <crc@liblime.com>
Signed-off-by: Joshua Ferraro <jmf@liblime.com>
This commit is contained in:
Galen Charlton 2008-01-03 12:36:41 -06:00 committed by Joshua Ferraro
parent c2a0ed8077
commit 991cdc31bc
2 changed files with 13 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
misc/commit_biblios_file.pl
View File

@ -39,12 +39,15 @@ if ($list_batches) {
# in future, probably should tie to a real user account
C4::Context->set_userenv(0, 'batch', 0, 'batch', 'batch', 'batch', 'batch', 'batch');
my $dbh = C4::Context->dbh;
$dbh->{AutoCommit} = 0;
if ($batch_number =~ /^\d+$/ and $batch_number > 0) {
my $batch = GetImportBatch($batch_number);
die "$0: import batch $batch_number does not exist in database\n" unless defined $batch;
die "$0: import batch $batch_number status is '" . $batch->{'import_status'} . "', and therefore cannot be imported\n"
unless $batch->{'import_status'} eq "staged" or $batch->{'import_status'} eq "reverted";
process_batch($batch_number);
$dbh->commit();
} else {
die "$0: please specify a numeric batch ID\n";
}
@ -71,7 +74,7 @@ sub process_batch {
print "... importing MARC records -- please wait\n";
my ($num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored) =
BatchCommitBibRecords($import_batch_id, 100, \&print_progress);
BatchCommitBibRecords($import_batch_id, 100, \&print_progress_and_commit);
print "... finished importing MARC records\n";
print <<_SUMMARY_;
@ -90,9 +93,10 @@ duplicate of one already in the database.
_SUMMARY_
}
sub print_progress {
sub print_progress_and_commit {
my $recs = shift;
print "... processed $recs records\n";
$dbh->commit();
}
sub print_usage {

10
misc/stage_biblios_file.pl
View File

@ -39,7 +39,10 @@ unless (-r $input_file) {
die "$0: cannot open input file $input_file: $!\n";
}
my $dbh = C4::Context->dbh;
$dbh->{AutoCommit} = 0;
process_batch($input_file, $match_bibs, $add_items, $batch_comment);
$dbh->commit();
exit 0;
@ -62,7 +65,7 @@ sub process_batch {
print "... staging MARC records -- please wait\n";
my ($batch_id, $num_valid, $num_items, @import_errors) =
BatchStageMarcRecords($marc_flavor, $marc_records, $input_file, $batch_comment, '', $add_items, 0,
100, \&print_progress);
100, \&print_progress_and_commit);
print "... finished staging MARC records\n";
my $num_with_matches = 0;
@ -72,7 +75,7 @@ sub process_batch {
$matcher->add_simple_required_check('245', 'a', -1, 0, '',
'245', 'a', -1, 0, '');
print "... looking for matches with records already in database\n";
$num_with_matches = BatchFindBibDuplicates($batch_id, $matcher, 10, 100, \&print_progress);
$num_with_matches = BatchFindBibDuplicates($batch_id, $matcher, 10, 100, \&print_progress_and_commit);
print "... finished looking for matches\n";
}
@ -102,8 +105,9 @@ _SUMMARY_
print "\n";
}
sub print_progress {
sub print_progress_and_commit {
my $recs = shift;
$dbh->commit();
print "... processed $recs records\n";
}