diff --git a/C4/ImportBatch.pm b/C4/ImportBatch.pm index bf4e1862a1..9f9c7f645e 100644 --- a/C4/ImportBatch.pm +++ b/C4/ImportBatch.pm @@ -54,6 +54,10 @@ BEGIN { SetImportBatchStatus GetImportBatchOverlayAction SetImportBatchOverlayAction + GetImportBatchNoMatchAction + SetImportBatchNoMatchAction + GetImportBatchItemAction + SetImportBatchItemAction GetImportBatchMatcher SetImportBatchMatcher GetImportRecordOverlayStatus @@ -261,6 +265,12 @@ sub BatchStageMarcRecords { } my $batch_id = AddImportBatch('create_new', 'staging', 'batch', $file_name, $comments); + if ($parse_items) { + SetImportBatchItemAction($batch_id, 'always_add'); + } else { + SetImportBatchItemAction($batch_id, 'ignore'); + } + my @invalid_records = (); my $num_valid = 0; my $num_items = 0; @@ -347,8 +357,7 @@ my $num_with_matches = BatchFindBibDuplicates($batch_id, $matcher, $max_matches, =back Goes through the records loaded in the batch and attempts to -find duplicates for each one. Sets the overlay action to -"replace" if it was "create_new", and sets the overlay status +find duplicates for each one. Sets the matching status of each record to "no_match" or "auto_match" as appropriate. The $max_matches parameter is optional; if it is not supplied, @@ -379,10 +388,6 @@ sub BatchFindBibDuplicates { } my $dbh = C4::Context->dbh; - my $old_overlay_action = GetImportBatchOverlayAction($batch_id); - if ($old_overlay_action eq "create_new") { - SetImportBatchOverlayAction($batch_id, 'replace'); - } my $sth = $dbh->prepare("SELECT import_record_id, marc FROM import_records @@ -448,6 +453,8 @@ sub BatchCommitBibRecords { # FIXME biblio only at the moment SetImportBatchStatus('importing'); my $overlay_action = GetImportBatchOverlayAction($batch_id); + my $nomatch_action = GetImportBatchNoMatchAction($batch_id); + my $item_action = GetImportBatchItemAction($batch_id); my $dbh = C4::Context->dbh; my $sth = $dbh->prepare("SELECT import_record_id, status, overlay_status, marc, encoding FROM import_records @@ -472,20 +479,26 @@ sub BatchCommitBibRecords { $marc_record->delete_field($item_field); } - if ($overlay_action eq 'create_new' or - ($overlay_action eq 'replace' and $rowref->{'overlay_status'} eq 'no_match')) { + # decide what what to do with the bib and item records + my ($bib_result, $item_result, $bib_match) = + _get_commit_action($overlay_action, $nomatch_action, $item_action, + $rowref->{'overlay_status'}, $rowref->{'import_record_id'}); + + if ($bib_result eq 'create_new') { $num_added++; my ($biblionumber, $biblioitemnumber) = AddBiblio($marc_record, ''); my $sth = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"); $sth->execute($biblionumber, $rowref->{'import_record_id'}); $sth->finish(); - my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $biblionumber); - $num_items_added += $bib_items_added; - $num_items_errored += $bib_items_errored; + if ($item_result eq 'create_new') { + my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $biblionumber); + $num_items_added += $bib_items_added; + $num_items_errored += $bib_items_errored; + } SetImportRecordStatus($rowref->{'import_record_id'}, 'imported'); - } else { + } elsif ($bib_result eq 'replace') { $num_updated++; - my $biblionumber = GetBestRecordMatch($rowref->{'import_record_id'}); + my $biblionumber = $bib_match; my ($count, $oldbiblio) = GetBiblio($biblionumber); my $oldxml = GetXmlBiblio($biblionumber); @@ -503,11 +516,27 @@ sub BatchCommitBibRecords { my $sth2 = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"); $sth2->execute($biblionumber, $rowref->{'import_record_id'}); $sth2->finish(); - my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $biblionumber); - $num_items_added += $bib_items_added; - $num_items_errored += $bib_items_errored; + if ($item_result eq 'create_new') { + my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $biblionumber); + $num_items_added += $bib_items_added; + $num_items_errored += $bib_items_errored; + } SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'match_applied'); SetImportRecordStatus($rowref->{'import_record_id'}, 'imported'); + } elsif ($bib_result eq 'ignore') { + $num_ignored++; + my $biblionumber = $bib_match; + if (defined $biblionumber and $item_result eq 'create_new') { + my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $biblionumber); + $num_items_added += $bib_items_added; + $num_items_errored += $bib_items_errored; + # still need to record the matched biblionumber so that the + # items can be reverted + my $sth2 = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"); + $sth2->execute($biblionumber, $rowref->{'import_record_id'}); + SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'match_applied'); + } + SetImportRecordStatus($rowref->{'import_record_id'}, 'ignored'); } } $sth->finish(); @@ -588,6 +617,7 @@ sub BatchRevertBibRecords { # FIXME biblio only at the moment SetImportBatchStatus('reverting'); my $overlay_action = GetImportBatchOverlayAction($batch_id); + my $nomatch_action = GetImportBatchNoMatchAction($batch_id); my $dbh = C4::Context->dbh; my $sth = $dbh->prepare("SELECT import_record_id, status, overlay_status, marcxml_old, encoding, matched_biblionumber FROM import_records @@ -598,8 +628,10 @@ sub BatchRevertBibRecords { if ($rowref->{'status'} eq 'error' or $rowref->{'status'} eq 'reverted') { $num_ignored++; } - if ($overlay_action eq 'create_new' or - ($overlay_action eq 'replace' and $rowref->{'overlay_status'} eq 'no_match')) { + + my $bib_result = _get_revert_action($overlay_action, $rowref->{'overlay_status'}, $rowref->{'status'}); + + if ($bib_result eq 'delete') { $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'}); my $error = DelBiblio($rowref->{'matched_biblionumber'}); if (defined $error) { @@ -608,7 +640,7 @@ sub BatchRevertBibRecords { $num_deleted++; SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted'); } - } else { + } elsif ($bib_result eq 'restore') { $num_reverted++; my $old_record = MARC::Record->new_from_xml(StripNonXmlChars($rowref->{'marcxml_old'}), 'UTF-8', $rowref->{'encoding'}); my $biblionumber = $rowref->{'matched_biblionumber'}; @@ -616,8 +648,12 @@ sub BatchRevertBibRecords { $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'}); ModBiblio($old_record, $biblionumber, $oldbiblio->{'frameworkcode'}); SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted'); + } elsif ($bib_result eq 'ignore') { + $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'}); + SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted'); } } + $sth->finish(); SetImportBatchStatus($batch_id, 'reverted'); return ($num_deleted, $num_errors, $num_reverted, $num_items_deleted, $num_ignored); @@ -905,6 +941,92 @@ sub SetImportBatchOverlayAction { } +=head2 GetImportBatchNoMatchAction + +=over 4 + +my $nomatch_action = GetImportBatchNoMatchAction($batch_id); + +=back + +=cut + +sub GetImportBatchNoMatchAction { + my ($batch_id) = @_; + + my $dbh = C4::Context->dbh; + my $sth = $dbh->prepare("SELECT nomatch_action FROM import_batches WHERE import_batch_id = ?"); + $sth->execute($batch_id); + my ($nomatch_action) = $sth->fetchrow_array(); + $sth->finish(); + return $nomatch_action; + +} + + +=head2 SetImportBatchNoMatchAction + +=over 4 + +SetImportBatchNoMatchAction($batch_id, $new_nomatch_action); + +=back + +=cut + +sub SetImportBatchNoMatchAction { + my ($batch_id, $new_nomatch_action) = @_; + + my $dbh = C4::Context->dbh; + my $sth = $dbh->prepare("UPDATE import_batches SET nomatch_action = ? WHERE import_batch_id = ?"); + $sth->execute($new_nomatch_action, $batch_id); + $sth->finish(); + +} + +=head2 GetImportBatchItemAction + +=over 4 + +my $item_action = GetImportBatchItemAction($batch_id); + +=back + +=cut + +sub GetImportBatchItemAction { + my ($batch_id) = @_; + + my $dbh = C4::Context->dbh; + my $sth = $dbh->prepare("SELECT item_action FROM import_batches WHERE import_batch_id = ?"); + $sth->execute($batch_id); + my ($item_action) = $sth->fetchrow_array(); + $sth->finish(); + return $item_action; + +} + + +=head2 SetImportBatchItemAction + +=over 4 + +SetImportBatchItemAction($batch_id, $new_item_action); + +=back + +=cut + +sub SetImportBatchItemAction { + my ($batch_id, $new_item_action) = @_; + + my $dbh = C4::Context->dbh; + my $sth = $dbh->prepare("UPDATE import_batches SET item_action = ? WHERE import_batch_id = ?"); + $sth->execute($new_item_action, $batch_id); + $sth->finish(); + +} + =head2 GetImportBatchMatcher =over 4 @@ -1190,6 +1312,46 @@ sub _update_batch_record_counts { } +sub _get_commit_action { + my ($overlay_action, $nomatch_action, $item_action, $overlay_status, $import_record_id) = @_; + + my ($bib_result, $bib_match, $item_result); + + if ($overlay_status ne 'no_match') { + $bib_match = GetBestRecordMatch($import_record_id); + if ($overlay_action eq 'replace') { + $bib_result = defined($bib_match) ? 'replace' : 'create_new'; + } elsif ($overlay_action eq 'create_new') { + $bib_result = 'create_new'; + } elsif ($overlay_action eq 'ignore') { + $bib_result = 'ignore'; + } + $item_result = ($item_action eq 'always_add' or $item_action eq 'add_only_for_matches') ? 'create_new' : 'ignore'; + } else { + $bib_result = $nomatch_action; + $item_result = ($item_action eq 'always_add' or $item_action eq 'add_only_for_new') ? 'create_new' : 'ignore'; + } + + return ($bib_result, $item_result, $bib_match); +} + +sub _get_revert_action { + my ($overlay_action, $overlay_status, $status) = @_; + + my $bib_result; + + if ($status eq 'ignored') { + $bib_result = 'ignore'; + } else { + if ($overlay_action eq 'create_new') { + $bib_result = 'delete'; + } else { + $bib_result = ($overlay_status eq 'match_applied') ? 'restore' : 'delete'; + } + } + return $bib_result; +} + 1; __END__ diff --git a/installer/data/mysql/kohastructure.sql b/installer/data/mysql/kohastructure.sql index 964614ddbe..ff55985fdd 100644 --- a/installer/data/mysql/kohastructure.sql +++ b/installer/data/mysql/kohastructure.sql @@ -872,7 +872,9 @@ CREATE TABLE `import_batches` ( `num_biblios` int(11) NOT NULL default 0, `num_items` int(11) NOT NULL default 0, `upload_timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP, - `overlay_action` enum('replace', 'create_new', 'use_template') NOT NULL default 'create_new', + `overlay_action` enum('replace', 'create_new', 'use_template', 'ignore') NOT NULL default 'create_new', + `nomatch_action` enum('create_new', 'ignore') NOT NULL default 'create_new', + `item_action` enum('always_add', 'add_only_for_matches', 'add_only_for_new', 'ignore') NOT NULL default 'always_add', `import_status` enum('staging', 'staged', 'importing', 'imported', 'reverting', 'reverted', 'cleaned') NOT NULL default 'staging', `batch_type` enum('batch', 'z3950') NOT NULL default 'batch', `file_name` varchar(100), @@ -898,7 +900,7 @@ CREATE TABLE `import_records` ( `marcxml_old` longtext NOT NULL, `record_type` enum('biblio', 'auth', 'holdings') NOT NULL default 'biblio', `overlay_status` enum('no_match', 'auto_match', 'manual_match', 'match_applied') NOT NULL default 'no_match', - `status` enum('error', 'staged', 'imported', 'reverted', 'items_reverted') NOT NULL default 'staged', + `status` enum('error', 'staged', 'imported', 'reverted', 'items_reverted', 'ignored') NOT NULL default 'staged', `import_error` mediumtext, `encoding` varchar(40) NOT NULL default '', `z3950random` varchar(40) default NULL, @@ -954,7 +956,7 @@ CREATE TABLE `import_items` ( `import_record_id` int(11) NOT NULL, `itemnumber` int(11) default NULL, `branchcode` varchar(10) default NULL, - `status` enum('error', 'staged', 'imported', 'reverted') NOT NULL default 'staged', + `status` enum('error', 'staged', 'imported', 'reverted', 'ignored') NOT NULL default 'staged', `marcxml` longtext NOT NULL, `import_error` mediumtext, PRIMARY KEY (`import_items_id`), diff --git a/installer/data/mysql/updatedatabase.pl b/installer/data/mysql/updatedatabase.pl index 0399292e94..592170d022 100755 --- a/installer/data/mysql/updatedatabase.pl +++ b/installer/data/mysql/updatedatabase.pl @@ -1397,6 +1397,26 @@ if (C4::Context->preference("Version") < TransformToNum($DBversion)) { SetVersion ($DBversion); } +$DBversion = "3.00.00.076"; +if (C4::Context->preference("Version") < TransformToNum($DBversion)) { + $dbh->do("ALTER TABLE import_batches + ADD COLUMN nomatch_action enum('create_new', 'ignore') NOT NULL default 'create_new' AFTER overlay_action"); + $dbh->do("ALTER TABLE import_batches + ADD COLUMN item_action enum('always_add', 'add_only_for_matches', 'add_only_for_new', 'ignore') + NOT NULL default 'always_add' AFTER nomatch_action"); + $dbh->do("ALTER TABLE import_batches + MODIFY overlay_action enum('replace', 'create_new', 'use_template', 'ignore') + NOT NULL default 'create_new'"); + $dbh->do("ALTER TABLE import_records + MODIFY status enum('error', 'staged', 'imported', 'reverted', 'items_reverted', + 'ignored') NOT NULL default 'staged'"); + $dbh->do("ALTER TABLE import_items + MODIFY status enum('error', 'staged', 'imported', 'reverted', 'ignored') NOT NULL default 'staged'"); + + print "Upgrade to $DBversion done (changes to import_batches and import_records) "; + SetVersion ($DBversion); +} + =item DropAllForeignKeys($table) Drop all foreign keys of the table $table diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/tools-item-action.inc b/koha-tmpl/intranet-tmpl/prog/en/includes/tools-item-action.inc new file mode 100644 index 0000000000..7540b5fec6 --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/tools-item-action.inc @@ -0,0 +1,26 @@ + diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/tools-nomatch-action.inc b/koha-tmpl/intranet-tmpl/prog/en/includes/tools-nomatch-action.inc new file mode 100644 index 0000000000..87d2ac097c --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/tools-nomatch-action.inc @@ -0,0 +1,14 @@ + diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/tools-overlay-action.inc b/koha-tmpl/intranet-tmpl/prog/en/includes/tools-overlay-action.inc new file mode 100644 index 0000000000..d12f497dde --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/tools-overlay-action.inc @@ -0,0 +1,20 @@ + diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/manage-marc-import.tmpl b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/manage-marc-import.tmpl index 190bcaf98b..ea0a5e5f4f 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/manage-marc-import.tmpl +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/manage-marc-import.tmpl @@ -39,23 +39,33 @@ -

-

-

-

-

+

+ +
+
" method="post"> " /> " /> +
    +
  • +
  • +
  • + +
  • +
  • + +
  • +
  • + +
  • +
+ + +

Failed to apply different matching rule

+ +

Applied different matching rule. Number of records matched now + +

+ + + +

Changed action if matching record found

+ + +

Changed action if no match found

+ + +

Changed item processing option

+ +
+
" method="post"> @@ -97,15 +138,16 @@ +

Completed import of records

+ -
Number of records added
Number of records updated
Number of records ignored
Number of items added
Number of items ignored because of duplicate barcode
Number of records ignored
@@ -118,15 +160,6 @@ Number of records ignored - - -

Failed to apply different matching rule

- -

Applied different matching rule. Number of records matched now - -

- -
diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/stage-marc-import.tmpl b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/stage-marc-import.tmpl index 00c5a50d41..36906b1404 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/stage-marc-import.tmpl +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/stage-marc-import.tmpl @@ -100,7 +100,14 @@ function CheckForm(f) { - + +
  • + +
  • +
  • + +
  • +
    Check for embedded item record data? @@ -118,6 +125,11 @@ function CheckForm(f) { +
      +
    1. + +
    2. +
    diff --git a/kohaversion.pl b/kohaversion.pl index 2c726b4086..5d26b41a49 100644 --- a/kohaversion.pl +++ b/kohaversion.pl @@ -10,7 +10,7 @@ use strict; sub kohaversion { - our $VERSION = "3.00.00.075"; + our $VERSION = "3.00.00.076"; # version needs to be set this way # so that it can be picked up by Makefile.PL # during install diff --git a/misc/stage_biblios_file.pl b/misc/stage_biblios_file.pl index cf140c26f8..c54915076a 100755 --- a/misc/stage_biblios_file.pl +++ b/misc/stage_biblios_file.pl @@ -74,16 +74,20 @@ sub process_batch { my $num_with_matches = 0; if ($match_bibs) { - my $matcher = C4::Matcher->fetch($match_bibs) ; - if( ! defined $matcher) { - $matcher = C4::Matcher->new('biblio'); - $matcher->add_simple_matchpoint('isbn', 1000, '020', 'a', -1, 0, ''); - $matcher->add_simple_required_check('245', 'a', -1, 0, '', + my $matcher = C4::Matcher->fetch($match_bibs) ; + if (! defined $matcher) { + $matcher = C4::Matcher->new('biblio'); + $matcher->add_simple_matchpoint('isbn', 1000, '020', 'a', -1, 0, ''); + $matcher->add_simple_required_check('245', 'a', -1, 0, '', '245', 'a', -1, 0, ''); - } else { - SetImportBatchMatcher($batch_id, $match_bibs); - } - print "... looking for matches with records already in database\n"; + } else { + SetImportBatchMatcher($batch_id, $match_bibs); + } + # set default record overlay behavior + SetImportBatchOverlayAction($batch_id, 'replace'); + SetImportBatchNoMatchAction($batch_id, 'create_new'); + SetImportBatchItemAction($batch_id, 'always_add'); + print "... looking for matches with records already in database\n"; $num_with_matches = BatchFindBibDuplicates($batch_id, $matcher, 10, 100, \&print_progress_and_commit); print "... finished looking for matches\n"; } @@ -137,8 +141,8 @@ Parameters: --match-bibs use this option to match bibs in the file with bibs already in the database for future overlay. - If isn't defined, a default - MARC21 ISBN & title match rule will be applied. + If isn't defined, a default + MARC21 ISBN & title match rule will be applied. --add-items use this option to specify that item data is embedded in the MARC bibs and should be parsed. diff --git a/t/lib/KohaTest/ImportBatch.pm b/t/lib/KohaTest/ImportBatch.pm new file mode 100644 index 0000000000..82c703f153 --- /dev/null +++ b/t/lib/KohaTest/ImportBatch.pm @@ -0,0 +1,106 @@ +package KohaTest::ImportBatch; +use base qw(KohaTest); + +use strict; +use warnings; + +use Test::More; + +use C4::ImportBatch; +use C4::Matcher; +sub testing_class { 'C4::ImportBatch' }; + + +sub routines : Test( 1 ) { + my $self = shift; + my @routines = qw( + GetZ3950BatchId + GetImportRecordMarc + AddImportBatch + GetImportBatch + AddBiblioToBatch + ModBiblioInBatch + BatchStageMarcRecords + AddItemsToImportBiblio + BatchFindBibDuplicates + BatchCommitBibRecords + BatchCommitItems + BatchRevertBibRecords + BatchRevertItems + GetAllImportBatches + GetImportBatchRangeDesc + GetItemNumbersFromImportBatch + GetNumberOfNonZ3950ImportBatches + GetImportBibliosRange + GetBestRecordMatch + GetImportBatchStatus + SetImportBatchStatus + GetImportBatchOverlayAction + SetImportBatchOverlayAction + GetImportBatchNoMatchAction + SetImportBatchNoMatchAction + GetImportBatchItemAction + SetImportBatchItemAction + GetImportBatchItemAction + SetImportBatchItemAction + GetImportBatchMatcher + SetImportBatchMatcher + GetImportRecordOverlayStatus + SetImportRecordOverlayStatus + GetImportRecordStatus + SetImportRecordStatus + GetImportRecordMatches + SetImportRecordMatches + _create_import_record + _update_import_record_marc + _add_biblio_fields + _update_biblio_fields + _parse_biblio_fields + _update_batch_record_counts + _get_commit_action + _get_revert_action + ); + + can_ok($self->testing_class, @routines); +} + +sub startup_50_add_matcher : Test( startup => 1 ) { + my $self = shift; + # create test MARC21 ISBN matcher + my $matcher = C4::Matcher->new('biblio'); + $matcher->threshold(1000); + $matcher->code('TESTISBN'); + $matcher->description('test MARC21 ISBN matcher'); + $matcher->add_simple_matchpoint('isbn', 1000, '020', 'a', -1, 0, ''); + my $matcher_id = $matcher->store(); + like($matcher_id, qr/^\d+$/, "store new matcher and get back ID"); + + $self->{'matcher_id'} = $matcher_id; +} + +sub shutdown_50_remove_matcher : Test( shutdown => 6) { + my $self = shift; + my @matchers = C4::Matcher::GetMatcherList(); + cmp_ok(scalar(@matchers), ">=", 1, "at least one matcher present"); + my $matcher_id; + my $testisbn_count = 0; + # look for TESTISBN + foreach my $matcher (@matchers) { + if ($matcher->{'code'} eq 'TESTISBN') { + $testisbn_count++; + $matcher_id = $matcher->{'matcher_id'}; + } + } + ok($testisbn_count == 1, "only one TESTISBN matcher"); + like($matcher_id, qr/^\d+$/, "matcher ID is valid"); + my $matcher = C4::Matcher->fetch($matcher_id); + ok(defined($matcher), "got back a matcher"); + ok($matcher_id == $matcher->{'id'}, "got back the correct matcher"); + C4::Matcher->delete($matcher_id); + my $matcher2 = C4::Matcher->fetch($matcher_id); + ok(not(defined($matcher2)), "matcher removed"); + + delete $self->{'matcher_id'}; +} + +1; diff --git a/t/lib/KohaTest/ImportBatch/BatchStageCommitRevert.pm b/t/lib/KohaTest/ImportBatch/BatchStageCommitRevert.pm new file mode 100644 index 0000000000..94f811528c --- /dev/null +++ b/t/lib/KohaTest/ImportBatch/BatchStageCommitRevert.pm @@ -0,0 +1,252 @@ +package KohaTest::ImportBatch::BatchStageCommitRevert; +use base qw( KohaTest::ImportBatch ); + +use strict; +use warnings; + +use Test::More; + +use C4::ImportBatch; +use C4::Matcher; +use C4::Biblio; + +# define test records for various batches +sub startup_60_make_test_records : Test( startup ) { + my $self = shift; + $self->{'batches'} = { + 'batch1' => { + marc => _make_marc_batch([ + ['isbn001', 'title 1', ['batch-item-1'] ], + ['isbn002', 'title 2', [] ], + ['isbn003', 'title 3', ['batch-item-2','batch-item-3'] ], + ['isbn004', 'title 4', [ 'batch-item-4' ] ], + ['isbn005', 'title 5', [ 'batch-item-5', 'batch-item-6', 'batch-item-7' ] ], + ]), + args => { + parse_items => 1, + overlay_action => 'create_new', + nomatch_action => 'create_new', + item_action => 'always_add', + }, + results => { + num_bibs => 5, + num_items => 7, + num_invalid => 0, + num_matches => 0, + num_added => 5, + num_updated => 0, + num_items_added => 7, + num_items_errored => 0, + num_ignored => 0, + }, + }, + 'batch2' => { + marc => _make_marc_batch([ + ['isbn001', 'overlay title 1', ['batch-item-8'] ], + ['isbn002', 'overlay title 2', ['batch-item-9'] ], + ['isbn006', 'title 6', ['batch-item-10'] ], + ]), + args => { + parse_items => 1, + overlay_action => 'replace', + nomatch_action => 'create_new', + item_action => 'always_add', + }, + results => { + num_bibs => 3, + num_items => 3, + num_invalid => 0, + num_matches => 2, + num_added => 1, + num_updated => 2, + num_items_added => 3, + num_items_errored => 0, + num_ignored => 0, + }, + }, + 'batch3' => { + marc => _make_marc_batch([ + ['isbn007', 'title 7', ['batch-item-11'] ], + ['isbn006', 'overlay title 6', ['batch-item-12'] ], + ]), + args => { + parse_items => 1, + overlay_action => 'ignore', + nomatch_action => 'ignore', + item_action => 'always_add', + }, + results => { + num_bibs => 2, + num_items => 2, + num_invalid => 0, + num_matches => 1, + num_added => 0, + num_updated => 0, + num_items_added => 1, + num_items_errored => 0, + num_ignored => 2, + }, + }, + 'batch4' => { + marc => _make_marc_batch([ + ['isbn008', 'title 8', ['batch-item-13'] ], # not loading this item + ]), + args => { + parse_items => 0, + overlay_action => undef, + nomatch_action => 'create_new', + item_action => 'ignore', + }, + results => { + num_bibs => 1, + num_items => 0, + num_invalid => 0, + num_matches => 0, + num_added => 1, + num_updated => 0, + num_items_added => 0, + num_items_errored => 0, + num_ignored => 0, + }, + }, + 'batch5' => { + marc => _make_marc_batch([ + ['isbn009', 'title 9', ['batch-item-1'] ], # trigger dup barcode error + 'junkjunkjunkjunk', # trigger invalid bib + ]), + args => { + parse_items => 1, + overlay_action => undef, + nomatch_action => undef, + item_action => undef, + }, + results => { + num_bibs => 1, + num_items => 1, + num_invalid => 1, + num_matches => 0, + num_added => 1, + num_updated => 0, + num_items_added => 0, + num_items_errored => 1, + num_ignored => 0, + }, + }, + 'batch6' => { + marc => _make_marc_batch([ + ['isbn001', 'match title 1', ['batch-item-14', 'batch-item-15'] ], + ['isbn010', 'title 10', ['batch-item-16', 'batch-item-17'] ], + ]), + args => { + parse_items => 1, + overlay_action => 'ignore', + nomatch_action => 'create_new', + item_action => 'always_add', + }, + results => { + num_bibs => 2, + num_items => 4, + num_invalid => 0, + num_matches => 1, + num_added => 1, + num_updated => 0, + num_items_added => 4, + num_items_errored => 0, + num_ignored => 1, + }, + }, + }; + +} + +sub _make_marc_batch { + my $defs = shift; + my @marc = (); + foreach my $rec (@$defs) { + if (ref($rec) eq 'ARRAY') { + my $isbn = $rec->[0]; + my $title = $rec->[1]; + my $items = $rec->[2]; + my $bib = MARC::Record->new(); + $bib->leader(' nam a22 7a 4500'); + $bib->append_fields(MARC::Field->new('020', ' ', ' ', a => $isbn), + MARC::Field->new('245', ' ', ' ', a => $title)); + foreach my $barcode (@$items) { + my ($itemtag, $toss, $barcodesf, $branchsf); + ($itemtag, $toss) = GetMarcFromKohaField('items.itemnumber', ''); + ($toss, $barcodesf) = GetMarcFromKohaField('items.barcode', ''); + ($toss, $branchsf) = GetMarcFromKohaField('items.homebranch', ''); + $bib->append_fields(MARC::Field->new($itemtag, ' ', ' ', $barcodesf => $barcode, $branchsf => 'CPL')); + # FIXME: define branch in KohaTest + } + push @marc, $bib->as_usmarc(); + } else { + push @marc, $rec; + } + } + return join('', @marc); +} + +sub stage_commit_batches : Test( 75 ) { + my $self = shift; + + my $matcher = C4::Matcher->fetch($self->{'matcher_id'}); + ok(ref($matcher) eq 'C4::Matcher', "retrieved matcher"); + + for my $batch_key (sort keys %{ $self->{'batches'} }) { + my $batch = $self->{'batches'}->{$batch_key}; + my $args = $batch->{'args'}; + my $results = $batch->{'results'}; + my ($batch_id, $num_bibs, $num_items, @invalid) = + BatchStageMarcRecords('MARC21', $batch->{marc}, "$batch_key.mrc", "$batch_key comments", + '', $args->{'parse_items'}, 0); + like($batch_id, qr/^\d+$/, "staged $batch_key"); + cmp_ok($num_bibs, "==", $results->{'num_bibs'}, "$batch_key: correct number of bibs"); + cmp_ok($num_items, "==", $results->{'num_items'}, "$batch_key: correct number of items"); + cmp_ok(scalar(@invalid), "==", $results->{'num_invalid'}, "$batch_key: correct number of invalid bibs"); + + my $num_matches = BatchFindBibDuplicates($batch_id, $matcher, 10); + cmp_ok($num_matches, "==", $results->{'num_matches'}, "$batch_key: correct number of bib matches"); + + if (defined $args->{'overlay_action'}) { + if ($args->{'overlay_action'} eq 'create_new') { + cmp_ok(GetImportBatchOverlayAction($batch_id), "eq", 'create_new', "$batch_key: verify default overlay action"); + } else { + SetImportBatchOverlayAction($batch_id, $args->{'overlay_action'}); + cmp_ok(GetImportBatchOverlayAction($batch_id), "eq", $args->{'overlay_action'}, + "$batch_key: changed overlay action"); + } + } + if (defined $args->{'nomatch_action'}) { + if ($args->{'nomatch_action'} eq 'create_new') { + cmp_ok(GetImportBatchNoMatchAction($batch_id), "eq", 'create_new', "$batch_key: verify default nomatch action"); + } else { + SetImportBatchNoMatchAction($batch_id, $args->{'nomatch_action'}); + cmp_ok(GetImportBatchNoMatchAction($batch_id), "eq", $args->{'nomatch_action'}, + "$batch_key: changed nomatch action"); + } + } + if (defined $args->{'item_action'}) { + if ($args->{'item_action'} eq 'create_new') { + cmp_ok(GetImportBatchItemAction($batch_id), "eq", 'always_add', "$batch_key: verify default item action"); + } else { + SetImportBatchItemAction($batch_id, $args->{'item_action'}); + cmp_ok(GetImportBatchItemAction($batch_id), "eq", $args->{'item_action'}, + "$batch_key: changed item action"); + } + } + + my ($num_added, $num_updated, $num_items_added, + $num_items_errored, $num_ignored) = BatchCommitBibRecords($batch_id); + cmp_ok($num_added, "==", $results->{'num_added'}, "$batch_key: added correct number of bibs"); + cmp_ok($num_updated, "==", $results->{'num_updated'}, "$batch_key: updated correct number of bibs"); + cmp_ok($num_items_added, "==", $results->{'num_items_added'}, "$batch_key: added correct number of items"); + cmp_ok($num_items_errored, "==", $results->{'num_items_errored'}, "$batch_key: correct number of item add errors"); + cmp_ok($num_ignored, "==", $results->{'num_ignored'}, "$batch_key: ignored correct number of bibs"); + + $self->reindex_marc(); + } + +} + +1; diff --git a/tools/manage-marc-import.pl b/tools/manage-marc-import.pl index 2501a6bc6c..eaf9815ada 100755 --- a/tools/manage-marc-import.pl +++ b/tools/manage-marc-import.pl @@ -99,7 +99,11 @@ if ($op eq "") { } elsif ($op eq "redo-matching") { my $new_matcher_id = $input->param('new_matcher_id'); my $current_matcher_id = $input->param('current_matcher_id'); - redo_matching($template, $import_batch_id, $new_matcher_id, $current_matcher_id); + my $overlay_action = $input->param('overlay_action'); + my $nomatch_action = $input->param('nomatch_action'); + my $item_action = $input->param('item_action'); + redo_matching($template, $import_batch_id, $new_matcher_id, $current_matcher_id, + $overlay_action, $nomatch_action, $item_action); import_biblios_list($template, $import_batch_id, $offset, $results_per_page); } @@ -108,10 +112,34 @@ output_html_with_http_headers $input, $cookie, $template->output; exit 0; sub redo_matching { - my ($template, $import_batch_id, $new_matcher_id, $current_matcher_id) = @_; + my ($template, $import_batch_id, $new_matcher_id, $current_matcher_id, $overlay_action, $nomatch_action, $item_action) = @_; my $rematch_failed = 0; return if not defined $new_matcher_id and not defined $current_matcher_id; - return if $new_matcher_id == $current_matcher_id; + my $old_overlay_action = GetImportBatchOverlayAction($import_batch_id); + my $old_nomatch_action = GetImportBatchNoMatchAction($import_batch_id); + my $old_item_action = GetImportBatchItemAction($import_batch_id); + return if $new_matcher_id == $current_matcher_id and + $old_overlay_action eq $overlay_action and + $old_nomatch_action eq $nomatch_action and + $old_item_action eq $item_action; + + if ($old_overlay_action ne $overlay_action) { + SetImportBatchOverlayAction($import_batch_id, $overlay_action); + $template->param('changed_overlay_action' => 1); + } + if ($old_nomatch_action ne $nomatch_action) { + SetImportBatchNoMatchAction($import_batch_id, $nomatch_action); + $template->param('changed_nomatch_action' => 1); + } + if ($old_item_action ne $item_action) { + SetImportBatchItemAction($import_batch_id, $item_action); + $template->param('changed_item_action' => 1); + } + + if ($new_matcher_id == $current_matcher_id) { + return; + } + my $num_with_matches = 0; if (defined $new_matcher_id and $new_matcher_id ne "") { my $matcher = C4::Matcher->fetch($new_matcher_id); @@ -123,7 +151,8 @@ sub redo_matching { } } else { $num_with_matches = BatchFindBibDuplicates($import_batch_id, undef); - SetImportBatchMatcher($import_batch_id, undef); + SetImportBatchMatcher($import_batch_id, undef); + SetImportBatchOverlayAction('create_new'); } $template->param(rematch_failed => $rematch_failed); $template->param(rematch_attempted => 1); @@ -314,6 +343,15 @@ sub import_biblios_list { $template->param(num_results => $num_biblios); $template->param(results_per_page => $results_per_page); $template->param(import_batch_id => $import_batch_id); + my $overlay_action = GetImportBatchOverlayAction($import_batch_id); + $template->param("overlay_action_${overlay_action}" => 1); + $template->param(overlay_action => $overlay_action); + my $nomatch_action = GetImportBatchNoMatchAction($import_batch_id); + $template->param("nomatch_action_${nomatch_action}" => 1); + $template->param(nomatch_action => $nomatch_action); + my $item_action = GetImportBatchItemAction($import_batch_id); + $template->param("item_action_${item_action}" => 1); + $template->param(item_action => $item_action); batch_info($template, $batch); } diff --git a/tools/stage-marc-import.pl b/tools/stage-marc-import.pl index b926da643d..9c09d8c06d 100755 --- a/tools/stage-marc-import.pl +++ b/tools/stage-marc-import.pl @@ -50,7 +50,10 @@ my $fileID=$input->param('uploadedfileid'); my $runinbackground = $input->param('runinbackground'); my $completedJobID = $input->param('completedJobID'); my $matcher_id = $input->param('matcher'); +my $overlay_action = $input->param('overlay_action'); +my $nomatch_action = $input->param('nomatch_action'); my $parse_items = $input->param('parse_items'); +my $item_action = $input->param('item_action'); my $comments = $input->param('comments'); my $syntax = $input->param('syntax'); my ($template, $loggedinuser, $cookie) @@ -140,8 +143,12 @@ if ($completedJobID) { if (defined $matcher) { $checked_matches = 1; $matcher_code = $matcher->code(); - $num_with_matches = BatchFindBibDuplicates($batch_id, $matcher, 10, 50, matching_progress_callback($job, $dbh)); + $num_with_matches = BatchFindBibDuplicates($batch_id, $matcher, + 10, 50, matching_progress_callback($job, $dbh)); SetImportBatchMatcher($batch_id, $matcher_id); + SetImportBatchOverlayAction($batch_id, $overlay_action); + SetImportBatchNoMatchAction($batch_id, $nomatch_action); + SetImportBatchItemAction($batch_id, $item_action); $dbh->commit(); } else { $matcher_failed = 1;