Bug 5635: bulkmarcimport new parameters & features
See the script's documentation for more details New parameters are: - authtypes - filter - insert - update - all Signed-off-by: Pascale Nalon <pascale.nalon@gmail.com> This patch is live in Mines ParisTech since 2012-07-24. Signing off Signed-off-by: Katrin Fischer <Katrin.Fischer.83@web.de> - Moved the sign-off from bugzilla to the commit message. - All tests and QA script pass. - Amended commit message to list new parameters. - Verified this patch works on a UNIMARC installation. - Verified normal import still works correct on a MARC21 installation. Signed-off-by: Jared Camins-Esakov <jcamins@cpbibliography.com>
This commit is contained in:
Stéphane Delaune
Jared Camins-Esakov
parent
34fb8a87f2
commit
cefa7c21e2
|
|
@ -379,9 +379,9 @@ Get the record and tries to guess the adequate authtypecode from its content.
|
|||
=cut
|
||||
|
||||
sub GuessAuthTypeCode {
|
||||
my ($record) = @_;
|
||||
my ($record, $heading_fields) = @_;
|
||||
return unless defined $record;
|
||||
my $heading_fields = {
|
||||
$heading_fields //= {
|
||||
"MARC21"=>{
|
||||
'100'=>{authtypecode=>'PERSO_NAME'},
|
||||
'110'=>{authtypecode=>'CORPO_NAME'},
|
||||
|
|
|
|||
|
|
@ -24,15 +24,18 @@ use C4::Koha;
|
|||
use C4::Debug;
|
||||
use C4::Charset;
|
||||
use C4::Items;
|
||||
use YAML;
|
||||
use Unicode::Normalize;
|
||||
use Time::HiRes qw(gettimeofday);
|
||||
use Getopt::Long;
|
||||
use IO::File;
|
||||
use Pod::Usage;
|
||||
|
||||
binmode STDOUT, ':encoding(UTF-8)';
|
||||
use open qw( :std :encoding(UTF-8) );
|
||||
binmode( STDOUT, ":encoding(UTF-8)" );
|
||||
my ( $input_marc_file, $number, $offset) = ('',0,0);
|
||||
my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile);
|
||||
my ( $insert, $filters, $update, $all, $yamlfile, $authtypes );
|
||||
my $cleanisbn = 1;
|
||||
my ($sourcetag,$sourcesubfield,$idmapfl, $dedup_barcode);
|
||||
my $framework = '';
|
||||
|
|
@ -46,7 +49,7 @@ GetOptions(
|
|||
'o|offset:f' => \$offset,
|
||||
'h' => \$version,
|
||||
'd' => \$delete,
|
||||
't' => \$test_parameter,
|
||||
't|test' => \$test_parameter,
|
||||
's' => \$skip_marc8_conversion,
|
||||
'c:s' => \$char_encoding,
|
||||
'v:s' => \$verbose,
|
||||
|
|
@ -56,22 +59,37 @@ GetOptions(
|
|||
'k|keepids:s' => \$keepids,
|
||||
'b|biblios' => \$biblios,
|
||||
'a|authorities' => \$authorities,
|
||||
'authtypes:s' => \$authtypes,
|
||||
'filter=s@' => \$filters,
|
||||
'insert' => \$insert,
|
||||
'update' => \$update,
|
||||
'all' => \$all,
|
||||
'match=s@' => \$match,
|
||||
'i|isbn' => \$isbn_check,
|
||||
'x:s' => \$sourcetag,
|
||||
'y:s' => \$sourcesubfield,
|
||||
'idmap:s' => \$idmapfl,
|
||||
'cleanisbn!' => \$cleanisbn,
|
||||
'yaml:s' => \$yamlfile,
|
||||
'dedupbarcode' => \$dedup_barcode,
|
||||
'framework=s' => \$framework,
|
||||
);
|
||||
$biblios=!$authorities||$biblios;
|
||||
$biblios ||= !$authorities;
|
||||
$insert ||= !$update;
|
||||
|
||||
if ($all) {
|
||||
$insert = 1;
|
||||
$update = 1;
|
||||
}
|
||||
|
||||
if ($version || ($input_marc_file eq '')) {
|
||||
pod2usage( -verbose => 2 );
|
||||
exit;
|
||||
}
|
||||
|
||||
my $dbh = C4::Context->dbh;
|
||||
my $heading_fields=get_heading_fields();
|
||||
|
||||
if (defined $idmapfl) {
|
||||
open(IDMAP,">$idmapfl") or die "cannot open $idmapfl \n";
|
||||
}
|
||||
|
|
@ -81,8 +99,6 @@ if ((not defined $sourcesubfield) && (not defined $sourcetag)){
|
|||
$sourcesubfield="a";
|
||||
}
|
||||
|
||||
my $dbh = C4::Context->dbh;
|
||||
|
||||
# save the CataloguingLog property : we don't want to log a bulkmarcimport. It will slow the import &
|
||||
# will create problems in the action_logs table, that can't handle more than 1 entry per second per user.
|
||||
my $CataloguingLog = C4::Context->preference('CataloguingLog');
|
||||
|
|
@ -142,7 +158,7 @@ $batch->warnings_off();
|
|||
$batch->strict_off();
|
||||
my $i=0;
|
||||
my $commitnum = $commit ? $commit : 50;
|
||||
|
||||
my $yamlhash;
|
||||
|
||||
# Skip file offset
|
||||
if ( $offset ) {
|
||||
|
|
@ -199,6 +215,7 @@ RECORD: while ( ) {
|
|||
next RECORD;
|
||||
}
|
||||
}
|
||||
SetUTF8Flag($record);
|
||||
my $isbn;
|
||||
# remove trailing - in isbn (only for biblios, of course)
|
||||
if ($biblios && $cleanisbn) {
|
||||
|
|
@ -212,45 +229,76 @@ RECORD: while ( ) {
|
|||
}
|
||||
my $id;
|
||||
# search for duplicates (based on Local-number)
|
||||
if ($match){
|
||||
require C4::Search;
|
||||
my $query=build_query($match,$record);
|
||||
my $server=($authorities?'authorityserver':'biblioserver');
|
||||
my ($error, $results,$totalhits)=C4::Search::SimpleSearch( $query, 0, 3, [$server] );
|
||||
die "unable to search the database for duplicates : $error" if (defined $error);
|
||||
#warn "$query $server : $totalhits";
|
||||
if ( @{$results} == 1 ){
|
||||
my $marcrecord = MARC::File::USMARC::decode($results->[0]);
|
||||
$id=GetRecordId($marcrecord,$tagid,$subfieldid);
|
||||
}
|
||||
elsif ( @{$results} > 1){
|
||||
$debug && warn "more than one match for $query";
|
||||
}
|
||||
else {
|
||||
$debug && warn "nomatch for $query";
|
||||
}
|
||||
my $originalid;
|
||||
$originalid = GetRecordId( $record, $tagid, $subfieldid );
|
||||
if ($match) {
|
||||
require C4::Search;
|
||||
my $query = build_query( $match, $record );
|
||||
my $server = ( $authorities ? 'authorityserver' : 'biblioserver' );
|
||||
$debug && warn $query;
|
||||
my ( $error, $results, $totalhits ) = C4::Search::SimpleSearch( $query, 0, 3, [$server] );
|
||||
die "unable to search the database for duplicates : $error" if ( defined $error );
|
||||
$debug && warn "$query $server : $totalhits";
|
||||
if ( $results && scalar(@$results) == 1 ) {
|
||||
my $marcrecord = MARC::File::USMARC::decode( $results->[0] );
|
||||
SetUTF8Flag($marcrecord);
|
||||
$id = GetRecordId( $marcrecord, $tagid, $subfieldid );
|
||||
if ( $authorities && $marcFlavour ) {
|
||||
#Skip if authority in database is the same as the on in database
|
||||
if ( $marcrecord->field('005')->data >= $record->field('005')->data ) {
|
||||
if ($yamlfile) {
|
||||
$yamlhash->{$originalid}->{'authid'} = $id;
|
||||
|
||||
# we recover all subfields of the heading authorities
|
||||
my @subfields;
|
||||
foreach my $field ( $marcrecord->field("2..") ) {
|
||||
push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
|
||||
}
|
||||
$yamlhash->{$originalid}->{'subfields'} = \@subfields;
|
||||
}
|
||||
next;
|
||||
}
|
||||
}
|
||||
} elsif ( $results && scalar(@$results) > 1 ) {
|
||||
$debug && warn "more than one match for $query";
|
||||
} else {
|
||||
$debug && warn "nomatch for $query";
|
||||
}
|
||||
}
|
||||
my $originalid;
|
||||
if ($keepids){
|
||||
$originalid=GetRecordId($record,$tagid,$subfieldid);
|
||||
if ($originalid){
|
||||
my $storeidfield;
|
||||
if (length($keepids)==3){
|
||||
$storeidfield=MARC::Field->new($keepids,$originalid);
|
||||
}
|
||||
else {
|
||||
$storeidfield=MARC::Field->new(substr($keepids,0,3),"","",substr($keepids,3,1),$originalid);
|
||||
}
|
||||
$record->insert_fields_ordered($storeidfield);
|
||||
$record->delete_field($record->field($tagid));
|
||||
}
|
||||
if ($keepids && $originalid) {
|
||||
my $storeidfield;
|
||||
if ( length($keepids) == 3 ) {
|
||||
$storeidfield = MARC::Field->new( $keepids, $originalid );
|
||||
} else {
|
||||
$storeidfield = MARC::Field->new( substr( $keepids, 0, 3 ), "", "", substr( $keepids, 3, 1 ), $originalid );
|
||||
}
|
||||
$record->insert_fields_ordered($storeidfield);
|
||||
$record->delete_field( $record->field($tagid) );
|
||||
}
|
||||
foreach my $stringfilter (@$filters) {
|
||||
if ( length($stringfilter) == 3 ) {
|
||||
foreach my $field ( $record->field($stringfilter) ) {
|
||||
$record->delete_field($field);
|
||||
$debug && warn "removed : ", $field->as_string;
|
||||
}
|
||||
} elsif ($stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/) {
|
||||
my $removetag = $1;
|
||||
my $removesubfield = $2;
|
||||
my $removematch = $3;
|
||||
if ( ( $removetag > "010" ) && $removesubfield ) {
|
||||
foreach my $field ( $record->field($removetag) ) {
|
||||
$field->delete_subfield( code => "$removesubfield", match => $removematch );
|
||||
$debug && warn "Potentially removed : ", $field->subfield($removesubfield);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
unless ($test_parameter) {
|
||||
if ($authorities){
|
||||
use C4::AuthoritiesMarc;
|
||||
my $authtypecode=GuessAuthTypeCode($record);
|
||||
my $authtypecode=GuessAuthTypeCode($record, $heading_fields);
|
||||
my $authid= ($id?$id:GuessAuthId($record));
|
||||
if ($authid && GetAuthority($authid)){
|
||||
if ($authid && GetAuthority($authid) && $update ){
|
||||
## Authority has an id and is in database : Replace
|
||||
eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) };
|
||||
if ($@){
|
||||
|
|
@ -283,6 +331,14 @@ RECORD: while ( ) {
|
|||
printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile);
|
||||
}
|
||||
}
|
||||
if ($yamlfile) {
|
||||
$yamlhash->{$originalid}->{'authid'} = $authid;
|
||||
my @subfields;
|
||||
foreach my $field ( $record->field("2..") ) {
|
||||
push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
|
||||
}
|
||||
$yamlhash->{$originalid}->{'subfields'} = \@subfields;
|
||||
}
|
||||
}
|
||||
else {
|
||||
my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref );
|
||||
|
|
@ -306,20 +362,33 @@ RECORD: while ( ) {
|
|||
}
|
||||
# create biblio, unless we already have it ( either match or isbn )
|
||||
if ($biblionumber) {
|
||||
eval{$biblioitemnumber=GetBiblioData($biblionumber)->{biblioitemnumber};}
|
||||
}
|
||||
else
|
||||
{
|
||||
eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio($record, $framework, { defer_marc_save => 1 }) };
|
||||
eval{$biblioitemnumber=GetBiblioData($biblionumber)->{biblioitemnumber};};
|
||||
if ($update) {
|
||||
eval { ( $biblionumber, $biblioitemnumber ) = ModBiblio( $record, $biblionumber, GetFrameworkcode($biblionumber) ) };
|
||||
if ($@) {
|
||||
warn "ERROR: Edit biblio $biblionumber failed: $@\n";
|
||||
printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ERROR" } ) if ($logfile);
|
||||
next RECORD;
|
||||
} else {
|
||||
printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ok" } ) if ($logfile);
|
||||
}
|
||||
} else {
|
||||
printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "warning : already in database" } ) if ($logfile);
|
||||
}
|
||||
} else {
|
||||
if ($insert) {
|
||||
eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio( $record, '', { defer_marc_save => 1 } ) };
|
||||
if ($@) {
|
||||
warn "ERROR: Adding biblio $biblionumber failed: $@\n";
|
||||
printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ERROR" } ) if ($logfile);
|
||||
next RECORD;
|
||||
} else {
|
||||
printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ok" } ) if ($logfile);
|
||||
}
|
||||
} else {
|
||||
printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "warning : not in database" } ) if ($logfile);
|
||||
}
|
||||
}
|
||||
if ( $@ ) {
|
||||
warn "ERROR: Adding biblio $biblionumber failed: $@\n";
|
||||
printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ERROR"}) if ($logfile);
|
||||
next RECORD;
|
||||
}
|
||||
else{
|
||||
printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ok"}) if ($logfile);
|
||||
}
|
||||
eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); };
|
||||
my $error_adding = $@;
|
||||
# Work on a clone so that if there are real errors, we can maybe
|
||||
|
|
@ -384,6 +453,7 @@ RECORD: while ( ) {
|
|||
if ($#{ $errors_ref } > -1) {
|
||||
report_item_errors($biblionumber, $errors_ref);
|
||||
}
|
||||
$yamlhash->{$originalid} = $biblionumber if ($yamlfile);
|
||||
}
|
||||
$dbh->commit() if (0 == $i % $commitnum);
|
||||
}
|
||||
|
|
@ -407,6 +477,10 @@ if ($logfile){
|
|||
print $loghandle "$i MARC records done in $timeneeded seconds\n";
|
||||
$loghandle->close;
|
||||
}
|
||||
if ($yamlfile) {
|
||||
open my $yamlfileout, q{>}, "$yamlfile" or die "cannot open $yamlfile \n";
|
||||
print $yamlfileout Dump($yamlhash);
|
||||
}
|
||||
exit 0;
|
||||
|
||||
sub GetRecordId{
|
||||
|
|
@ -445,14 +519,16 @@ sub build_query {
|
|||
sub build_simplequery {
|
||||
my $element=shift;
|
||||
my $record=shift;
|
||||
my ($index,$recorddata)=split /,/,$element;
|
||||
my ($tag,$subfields) =($1,$2) if ($recorddata=~/(\d{3})(.*)/);
|
||||
my @searchstrings;
|
||||
my @searchstrings;
|
||||
my ($index,$recorddata)=split /,/,$element;
|
||||
if ($recorddata=~/(\d{3})(.*)/) {
|
||||
my ($tag,$subfields) =($1,$2);
|
||||
foreach my $field ($record->field($tag)){
|
||||
if (length($field->as_string("$subfields"))>0){
|
||||
push @searchstrings,"$index:\"".$field->as_string("$subfields")."\"";
|
||||
}
|
||||
}
|
||||
}
|
||||
my $QParser;
|
||||
$QParser = C4::Context->queryparser if (C4::Context->preference('UseQueryParser'));
|
||||
my $op;
|
||||
|
|
@ -478,9 +554,21 @@ sub report_item_errors {
|
|||
}
|
||||
sub printlog{
|
||||
my $logelements=shift;
|
||||
print $loghandle join (";",@$logelements{qw<id op status>}),"\n";
|
||||
print $loghandle join( ";", map { defined $_ ? $_ : "" } @$logelements{qw<id op status>} ), "\n";
|
||||
}
|
||||
sub get_heading_fields{
|
||||
my $headingfields;
|
||||
if ($authtypes){
|
||||
$headingfields=YAML::LoadFile($authtypes);
|
||||
$headingfields={C4::Context->preference('marcflavour')=>$headingfields};
|
||||
$debug && warn YAML::Dump($headingfields);
|
||||
}
|
||||
unless ($headingfields){
|
||||
$headingfields=$dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}});
|
||||
$headingfields={C4::Context->preference('marcflavour')=>$headingfields};
|
||||
}
|
||||
return $headingfields;
|
||||
}
|
||||
|
||||
|
||||
=head1 NAME
|
||||
|
||||
|
|
@ -542,7 +630,7 @@ The I<NUMBER> of records to wait before performing a 'commit' operation
|
|||
|
||||
File logs actions done for each record and their status into file
|
||||
|
||||
=item B<-t>
|
||||
=item B<-t, -test>
|
||||
|
||||
Test mode: parses the file, saying what he would do, but doing nothing.
|
||||
|
||||
|
|
@ -565,6 +653,32 @@ biblioitems, items
|
|||
|
||||
Input file I<FORMAT>: I<MARCXML> or I<ISO2709> (defaults to ISO2709)
|
||||
|
||||
=item B<-authtypes>
|
||||
|
||||
file yamlfile with authoritiesTypes and distinguishable record field in order
|
||||
to store the correct authtype
|
||||
|
||||
=item B<-yaml>
|
||||
|
||||
yaml file format a yaml file with ids
|
||||
|
||||
=item B<-filter>
|
||||
|
||||
list of fields that will not be imported. Can be any from 000 to 999 or field,
|
||||
subfield and subfield's matching value such as 200avalue
|
||||
|
||||
=item B<-insert>
|
||||
|
||||
if set, only insert when possible
|
||||
|
||||
=item B<-update>
|
||||
|
||||
if set, only updates (any biblio should have a matching record)
|
||||
|
||||
=item B<-all>
|
||||
|
||||
if set, do whatever is required
|
||||
|
||||
=item B<-k, -keepids>=<FIELD>
|
||||
|
||||
Field store ids in I<FIELD> (usefull for authorities, where 001 contains the
|
||||
|
|
|
|||
Loading…
Reference in New Issue