gitea
/
Koha-clms
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Bug 40777: Move all biblio tests to the new package 

Signed-off-by: David Nind <david@davidnind.com>
Signed-off-by: Victor Grousset/tuxayo <victor@tuxayo.net>
Signed-off-by: Lucas Gass <lucas@bywatersolutions.com>
This commit is contained in:
Jonathan Druart 2025-10-21 13:45:44 +02:00 committed by Lucas Gass
parent fba31a1035
commit a4c0746370
No known key found for this signature in database
GPG Key ID: 60B447398B0D36CA
3 changed files with 349 additions and 243 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

307
Koha/Database/DataInconsistency.pm
View File

@ -1,9 +1,16 @@
package Koha::Database::DataInconsistency;
use Modern::Perl;
use C4::Context;
use C4::Biblio;
use Koha::Biblioitems;
use Koha::Biblio::Metadatas;
use Koha::BiblioFrameworks;
use Koha::Items;
use Koha::ItemTypes;
use Koha::I18N qw( __x );
sub item_library {
sub invalid_item_library {
my ( $self, $items ) = @_;
$items = $items->search( { -or => { homebranch => undef, holdingbranch => undef } } );
@ -32,11 +39,303 @@ sub item_library {
return @errors;
}
sub for_biblio {
my ( $self, $biblio ) = @_;
sub ids {
my ( $self, $object ) = @_;
if ( $object->can('_resultset') ) {
# It's a Koha::Objects
return [ $object->get_column('biblionumber') ];
} else {
# It's a single Koha::Object
return [ $object->biblionumber ];
}
}
sub no_item_type {
my ( $self, $biblios ) = @_;
my $ids = $self->ids($biblios);
my @errors;
push @errors, $self->item_library( $biblio->items );
if ( C4::Context->preference('item-level_itypes') ) {
my $items_without_itype = Koha::Items->search(
{
biblionumber => $ids,
-or => [ itype => undef, itype => '' ]
}
);
if ( $items_without_itype->count ) {
while ( my $item = $items_without_itype->next ) {
if ( defined $item->biblioitem->itemtype && $item->biblioitem->itemtype ne '' ) {
push @errors, __x(
"Item with itemnumber={itemnumber} does not have a itype value, biblio's item type will be used ({itemtype})",
itemnumber => $item->itemnumber,
itemtype => $item->biblioitem->itemtype,
);
} else {
push @errors, __x(
"Item with itemnumber={itemnumber} does not have a itype value, additionally no item type defined for biblionumber={biblionumber}",
itemnumber => $item->itemnumber,
biblionumber => $item->biblioitem->biblionumber,
);
}
}
}
} else {
my $biblioitems_without_itemtype = Koha::Biblioitems->search( { biblionumber => $ids, itemtype => undef } );
if ( $biblioitems_without_itemtype->count ) {
while ( my $biblioitem = $biblioitems_without_itemtype->next ) {
push @errors, __x(
"Biblioitem with biblioitemnumber={biblioitemnumber} does not have a itemtype value",
biblioitemnumber => $biblioitem->biblioitemnumber,
);
}
}
}
return @errors;
}
sub invalid_item_type {
my ( $self, $biblios ) = @_;
my $ids = $self->ids($biblios);
my @errors;
my @itemtypes = Koha::ItemTypes->search->get_column('itemtype');
if ( C4::Context->preference('item-level_itypes') ) {
my $items_with_invalid_itype = Koha::Items->search(
{
biblionumber => $ids,
-and => [ itype => { not_in => \@itemtypes }, itype => { '!=' => '' } ]
}
);
if ( $items_with_invalid_itype->count ) {
while ( my $item = $items_with_invalid_itype->next ) {
push @errors, __x(
"Item with itemnumber={itemnumber}, biblionumber={biblionumber} does not have a valid itype value ({itype})",
itemnumber => $item->itemnumber,
biblionumber => $item->biblionumber,
itype => $item->itype,
);
}
}
} else {
my $biblioitems_with_invalid_itemtype =
Koha::Biblioitems->search( { biblionumber => $ids, itemtype => { not_in => \@itemtypes } } );
if ( $biblioitems_with_invalid_itemtype->count ) {
while ( my $biblioitem = $biblioitems_with_invalid_itemtype->next ) {
push @errors, __x(
"Biblioitem with biblioitemnumber={biblioitemnumber} does not have a valid itemtype value",
biblioitemnumber => $biblioitem->biblioitemnumber,
);
}
}
}
return @errors;
}
sub errors_in_marc {
my ( $self, $biblios ) = @_;
my $ids = $self->ids($biblios);
my @item_fields_in_marc;
my $errors;
my ( $item_tag, $item_subfield ) = C4::Biblio::GetMarcFromKohaField("items.itemnumber");
my $search_string = q{ExtractValue(metadata,'count(//datafield[@tag="} . $item_tag . q{"])')>0};
my $biblio_metadatas_with_item_fields =
Koha::Biblio::Metadatas->search( { biblionumber => $ids } )->search( \$search_string );
if ( $biblio_metadatas_with_item_fields->count ) {
while ( my $biblio_metadata_with_item_fields = $biblio_metadatas_with_item_fields->next ) {
push @item_fields_in_marc,
{
biblionumber => $biblio_metadata_with_item_fields->biblionumber,
};
}
}
my ( $biblio_tag, $biblio_subfield ) = C4::Biblio::GetMarcFromKohaField("biblio.biblionumber");
my ( $biblioitem_tag, $biblioitem_subfield ) = C4::Biblio::GetMarcFromKohaField("biblioitems.biblioitemnumber");
$biblios = Koha::Biblios->search( { biblionumber => $ids } );
while ( my $biblio = $biblios->next ) {
my $record = eval { $biblio->metadata->record; };
if ($@) {
push @{ $errors->{decoding_errors} }, $@;
next;
}
my ( $biblionumber, $biblioitemnumber );
if ( $biblio_tag < 10 ) {
my $biblio_control_field = $record->field($biblio_tag);
$biblionumber = $biblio_control_field->data if $biblio_control_field;
} else {
$biblionumber = $record->subfield( $biblio_tag, $biblio_subfield );
}
if ( $biblioitem_tag < 10 ) {
my $biblioitem_control_field = $record->field($biblioitem_tag);
$biblioitemnumber = $biblioitem_control_field->data if $biblioitem_control_field;
} else {
$biblioitemnumber = $record->subfield( $biblioitem_tag, $biblioitem_subfield );
}
if ( $biblionumber != $biblio->biblionumber ) {
push @{ $errors->{ids_not_in_marc} },
__x(
q{Biblionumber {biblionumber} has '{biblionumber_in_marc}' in {biblio_tag}${biblio_subfield}},
biblionumber => $biblio->biblionumber,
biblionumber_in_marc => $biblionumber,
biblio_tag => $biblio_tag,
biblio_subfield => $biblio_subfield,
);
}
if ( $biblioitemnumber != $biblio->biblioitem->biblioitemnumber ) {
push @{ $errors->{ids_not_in_marc} },
__x(
q{Biblionumber {biblionumber} has biblioitemnumber '{biblioitemnumber}' but should be '{biblioitemnumber_in_marc}' in {biblioitem_tag}${biblioitem_subfield}},
biblionumber => $biblio->biblionumber,
biblioitemnumber => $biblio->biblioitem->biblioitemnumber,
biblioitemnumber_in_marc => $biblionumber,
biblioitem_tag => $biblioitem_tag,
biblioitem_subfield => $biblioitem_subfield,
);
}
}
}
if (@item_fields_in_marc) {
for my $biblionumber (@item_fields_in_marc) {
push @{ $errors->{item_fields_in_marc} },
__x(
q{Biblionumber {biblionumber} has item fields ({item_tag}) in the marc record},
biblionumber => $biblionumber->{biblionumber},
item_tag => $item_tag,
);
}
}
return $errors;
}
sub nonexistent_AV {
my ( $self, $biblios ) = @_;
my $ids = $self->ids($biblios);
my @errors;
my @framework_codes = Koha::BiblioFrameworks->search()->get_column('frameworkcode');
push @framework_codes, ""; # The default is not stored in frameworks, we need to force it
my $invalid_av_per_framework = {};
foreach my $frameworkcode (@framework_codes) {
# We are only checking fields that are mapped to DB fields
my $msss = Koha::MarcSubfieldStructures->search(
{
frameworkcode => $frameworkcode,
authorised_value => { '!=' => [ -and => ( undef, '' ) ] },
kohafield => { '!=' => [ -and => ( undef, '' ) ] }
}
);
while ( my $mss = $msss->next ) {
my $kohafield = $mss->kohafield;
my $av = $mss->authorised_value;
next if grep { $_ eq $av } qw( branches itemtypes cn_source ); # internal categories
my $avs = Koha::AuthorisedValues->search_by_koha_field(
{
frameworkcode => $frameworkcode,
kohafield => $kohafield,
}
);
my $tmp_kohafield = $kohafield;
if ( $tmp_kohafield =~ /^biblioitems/ ) {
$tmp_kohafield =~ s|biblioitems|biblioitem|;
} else {
$tmp_kohafield =~ s|items|me|;
}
# replace items.attr with me.attr
# We are only checking biblios with items
my $items = Koha::Items->search(
{
'me.biblionumber' => $ids,
$tmp_kohafield => {
-not_in => [ $avs->get_column('authorised_value'), '' ],
'!=' => undef,
},
'biblio.frameworkcode' => $frameworkcode
},
{ join => [ 'biblioitem', 'biblio' ] }
);
if ( $items->count ) {
$invalid_av_per_framework->{$frameworkcode}->{$av} =
{ items => $items, kohafield => $kohafield };
}
}
}
if (%$invalid_av_per_framework) {
for my $frameworkcode ( keys %$invalid_av_per_framework ) {
while ( my ( $av_category, $v ) = each %{ $invalid_av_per_framework->{$frameworkcode} } ) {
my $items = $v->{items};
my $kohafield = $v->{kohafield};
my ( $table, $column ) = split '\.', $kohafield;
my $output;
while ( my $i = $items->next ) {
my $value =
$table eq 'items' ? $i->$column
: $table eq 'biblio' ? $i->biblio->$column
: $i->biblioitem->$column;
$output .= " {" . $i->itemnumber . " => " . $value . "}\n";
}
push @errors, __x(
"The Framework *{frameworkcode}* is using the authorised value's category *{av_category}*, "
. "but the following {kohafield} do not have a value defined ({itemnumber => value }):\n{output}",
frameworkcode => $frameworkcode,
av_category => $av_category,
kohafield => $kohafield,
output => $output,
);
}
}
}
return @errors;
}
sub empty_title {
my ( $self, $biblios ) = @_;
my $ids = $self->ids($biblios);
my @errors;
$biblios = Koha::Biblios->search(
{
biblionumber => $ids,
-or => [
title => '',
title => undef,
]
}
);
if ( $biblios->count ) {
while ( my $biblio = $biblios->next ) {
push @errors,
__x(
"Biblio with biblionumber={biblionumber} does not have title defined",
biblionumber => $biblio->biblionumber
);
}
}
return @errors;
}
sub for_biblio {
my ( $self, $biblio ) = @_;
my @invalid_item_library = $self->invalid_item_library( $biblio->items );
my @no_item_type = $self->no_item_type($biblio);
my @invalid_item_type = $self->invalid_item_type($biblio);
my $errors_in_marc = $self->errors_in_marc($biblio);
my @nonexistent_AV = $self->nonexistent_AV($biblio);
my @empty_title = $self->empty_title($biblio);
return {
invalid_item_library => \@invalid_item_library,
no_item_type => \@no_item_type,
invalid_item_type => \@invalid_item_type,
decoding_errors => $errors_in_marc->{decoding_errors} || [],
ids_not_in_marc => $errors_in_marc->{ids_not_in_marc} || [],
item_fields_in_marc => $errors_in_marc->{item_fields_in_marc} || [],
nonexistent_AV => \@nonexistent_AV,
empty_title => \@empty_title,
};
}
1;

4
catalogue/detail.pl
View File

@ -593,10 +593,10 @@ $template->param( found1 => scalar $query->param('found1') );
$template->param( biblio => $biblio );
if ( $query->param('audit') ) {
my @audit_errors = Koha::Database::DataInconsistency->for_biblio($biblio);
my $audit_errors = Koha::Database::DataInconsistency->for_biblio($biblio);
$template->param(
auditing => 1,
audit_errors => \@audit_errors,
audit_errors => [ map { scalar @{ $audit_errors->{$_} } ? $audit_errors->{$_} : () } keys %$audit_errors ],
);
}

281
misc/maintenance/search_for_data_inconsistencies.pl
View File

@ -31,12 +31,10 @@ use Koha::Database::DataInconsistency;
{
my $items = Koha::Items->search;
my @errors = Koha::Database::DataInconsistency->item_library($items);
my @errors = Koha::Database::DataInconsistency->invalid_item_library($items);
if (@errors) {
new_section("Not defined items.homebranch and/or items.holdingbranch");
for my $error (@errors) {
new_item($error);
}
new_item($_) for @errors;
}
if (@errors) { new_hint("Edit these items and set valid homebranch and/or holdingbranch") }
}
@ -56,270 +54,79 @@ use Koha::Database::DataInconsistency;
}
{
if ( C4::Context->preference('item-level_itypes') ) {
my $items_without_itype = Koha::Items->search( { -or => [ itype => undef, itype => '' ] } );
if ( $items_without_itype->count ) {
my $biblios = Koha::Biblios->search;
my @errors = Koha::Database::DataInconsistency->no_item_type($biblios);
if (@errors) {
if ( C4::Context->preference('item-level_itypes') ) {
new_section("Items do not have itype defined");
while ( my $item = $items_without_itype->next ) {
if ( defined $item->biblioitem->itemtype && $item->biblioitem->itemtype ne '' ) {
new_item(
sprintf
"Item with itemnumber=%s does not have a itype value, biblio's item type will be used (%s)",
$item->itemnumber, $item->biblioitem->itemtype
);
} else {
new_item(
sprintf
"Item with itemnumber=%s does not have a itype value, additionally no item type defined for biblionumber=%s",
$item->itemnumber, $item->biblioitem->biblionumber
);
}
}
new_item($_) for @errors;
new_hint("The system preference item-level_itypes expects item types to be defined at item level");
}
} else {
my $biblioitems_without_itemtype = Koha::Biblioitems->search( { itemtype => undef } );
if ( $biblioitems_without_itemtype->count ) {
} else {
new_section("Biblioitems do not have itemtype defined");
while ( my $biblioitem = $biblioitems_without_itemtype->next ) {
new_item(
sprintf "Biblioitem with biblioitemnumber=%s does not have a itemtype value",
$biblioitem->biblioitemnumber
);
}
new_item($_) for @errors;
new_hint("The system preference item-level_itypes expects item types to be defined at biblio level");
}
}
my @itemtypes = Koha::ItemTypes->search->get_column('itemtype');
if ( C4::Context->preference('item-level_itypes') ) {
my $items_with_invalid_itype =
Koha::Items->search( { -and => [ itype => { not_in => \@itemtypes }, itype => { '!=' => '' } ] } );
if ( $items_with_invalid_itype->count ) {
$biblios = Koha::Biblios->search;
@errors = Koha::Database::DataInconsistency->invalid_item_type($biblios);
if (@errors) {
if ( C4::Context->preference('item-level_itypes') ) {
new_section("Items have invalid itype defined");
while ( my $item = $items_with_invalid_itype->next ) {
new_item(
sprintf "Item with itemnumber=%s, biblionumber=%s does not have a valid itype value (%s)",
$item->itemnumber, $item->biblionumber, $item->itype
);
}
new_item($_) for @errors;
new_hint(
"The items must have a itype value that is defined in the item types of Koha (Home › Administration › Item types administration)"
);
}
} else {
my $biblioitems_with_invalid_itemtype = Koha::Biblioitems->search( { itemtype => { not_in => \@itemtypes } } );
if ( $biblioitems_with_invalid_itemtype->count ) {
} else {
new_section("Biblioitems do not have itemtype defined");
while ( my $biblioitem = $biblioitems_with_invalid_itemtype->next ) {
new_item(
sprintf "Biblioitem with biblioitemnumber=%s does not have a valid itemtype value",
$biblioitem->biblioitemnumber
);
}
new_item($_) for @errors;
new_hint(
"The biblioitems must have a itemtype value that is defined in the item types of Koha (Home › Administration › Item types administration)"
);
}
}
my @item_fields_in_marc;
my ( $item_tag, $item_subfield ) = C4::Biblio::GetMarcFromKohaField("items.itemnumber");
my $search_string = q{ExtractValue(metadata,'count(//datafield[@tag="} . $item_tag . q{"])')>0};
my $biblio_metadatas_with_item_fields = Koha::Biblio::Metadatas->search( \$search_string );
if ( $biblio_metadatas_with_item_fields->count ) {
while ( my $biblio_metadata_with_item_fields = $biblio_metadatas_with_item_fields->next ) {
push @item_fields_in_marc,
{
biblionumber => $biblio_metadata_with_item_fields->biblionumber,
};
$biblios = Koha::Biblios->search;
my $errors = Koha::Database::DataInconsistency->errors_in_marc($biblios);
if ( $errors && %$errors ) {
if ( exists $errors->{decoding_errors} ) {
new_section("Bibliographic records have invalid MARCXML");
new_item($_) for @{ $errors->{decoding_errors} };
new_hint(
"The bibliographic records must have a valid MARCXML or you will face encoding issues or wrong displays"
);
}
if ( exists $errors->{ids_not_in_marc} ) {
new_section("Bibliographic records have MARCXML without biblionumber or biblioitemnumber");
new_item($_) for @{ $errors->{ids_not_in_marc} };
new_hint("The bibliographic records must have the biblionumber and biblioitemnumber in MARCXML");
}
if ( exists $errors->{item_fields_in_marc} ) {
new_section("Bibliographic records have item fields in the MARC");
new_item($_) for @{ $errors->{item_fields_in_marc} };
new_hint("You can fix these by running misc/maintenance/touch_all_biblios.pl");
}
}
}
my ( @decoding_errors, @ids_not_in_marc );
{
my $biblios = Koha::Biblios->search;
my ( $biblio_tag, $biblio_subfield ) = C4::Biblio::GetMarcFromKohaField("biblio.biblionumber");
my ( $biblioitem_tag, $biblioitem_subfield ) = C4::Biblio::GetMarcFromKohaField("biblioitems.biblioitemnumber");
while ( my $biblio = $biblios->next ) {
my $record = eval { $biblio->metadata->record; };
if ($@) {
push @decoding_errors, $@;
next;
}
my ( $biblionumber, $biblioitemnumber );
if ( $biblio_tag < 10 ) {
my $biblio_control_field = $record->field($biblio_tag);
$biblionumber = $biblio_control_field->data if $biblio_control_field;
} else {
$biblionumber = $record->subfield( $biblio_tag, $biblio_subfield );
}
if ( $biblioitem_tag < 10 ) {
my $biblioitem_control_field = $record->field($biblioitem_tag);
$biblioitemnumber = $biblioitem_control_field->data if $biblioitem_control_field;
} else {
$biblioitemnumber = $record->subfield( $biblioitem_tag, $biblioitem_subfield );
}
if ( $biblionumber != $biblio->biblionumber ) {
push @ids_not_in_marc,
{
biblionumber => $biblio->biblionumber,
biblionumber_in_marc => $biblionumber,
};
}
if ( $biblioitemnumber != $biblio->biblioitem->biblioitemnumber ) {
push @ids_not_in_marc,
{
biblionumber => $biblio->biblionumber,
biblioitemnumber => $biblio->biblioitem->biblioitemnumber,
biblioitemnumber_in_marc => $biblionumber,
};
}
}
if (@decoding_errors) {
new_section("Bibliographic records have invalid MARCXML");
new_item($_) for @decoding_errors;
new_hint(
"The bibliographic records must have a valid MARCXML or you will face encoding issues or wrong displays");
}
if (@ids_not_in_marc) {
new_section("Bibliographic records have MARCXML without biblionumber or biblioitemnumber");
for my $id (@ids_not_in_marc) {
if ( exists $id->{biblioitemnumber} ) {
new_item(
sprintf(
q{Biblionumber %s has biblioitemnumber '%s' but should be '%s' in %s$%s},
$id->{biblionumber},
$id->{biblioitemnumber},
$id->{biblioitemnumber_in_marc},
$biblioitem_tag,
$biblioitem_subfield,
)
);
} else {
new_item(
sprintf(
q{Biblionumber %s has '%s' in %s$%s},
$id->{biblionumber},
$id->{biblionumber_in_marc},
$biblio_tag,
$biblio_subfield,
)
);
}
}
new_hint("The bibliographic records must have the biblionumber and biblioitemnumber in MARCXML");
}
if (@item_fields_in_marc) {
new_section("Bibliographic records have item fields in the MARC");
for my $biblionumber (@item_fields_in_marc) {
new_item(
sprintf(
q{Biblionumber %s has item fields (%s) in the marc record},
$biblionumber->{biblionumber},
$item_tag,
)
);
}
new_hint("You can fix these by running misc/maintenance/touch_all_biblios.pl");
}
}
{
my @framework_codes = Koha::BiblioFrameworks->search()->get_column('frameworkcode');
push @framework_codes, ""; # The default is not stored in frameworks, we need to force it
my $invalid_av_per_framework = {};
foreach my $frameworkcode (@framework_codes) {
# We are only checking fields that are mapped to DB fields
my $msss = Koha::MarcSubfieldStructures->search(
{
frameworkcode => $frameworkcode,
authorised_value => { '!=' => [ -and => ( undef, '' ) ] },
kohafield => { '!=' => [ -and => ( undef, '' ) ] }
}
);
while ( my $mss = $msss->next ) {
my $kohafield = $mss->kohafield;
my $av = $mss->authorised_value;
next if grep { $_ eq $av } qw( branches itemtypes cn_source ); # internal categories
my $avs = Koha::AuthorisedValues->search_by_koha_field(
{
frameworkcode => $frameworkcode,
kohafield => $kohafield,
}
);
my $tmp_kohafield = $kohafield;
if ( $tmp_kohafield =~ /^biblioitems/ ) {
$tmp_kohafield =~ s|biblioitems|biblioitem|;
} else {
$tmp_kohafield =~ s|items|me|;
}
# replace items.attr with me.attr
# We are only checking biblios with items
my $items = Koha::Items->search(
{
$tmp_kohafield => {
-not_in => [ $avs->get_column('authorised_value'), '' ],
'!=' => undef,
},
'biblio.frameworkcode' => $frameworkcode
},
{ join => [ 'biblioitem', 'biblio' ] }
);
if ( $items->count ) {
$invalid_av_per_framework->{$frameworkcode}->{$av} =
{ items => $items, kohafield => $kohafield };
}
}
}
if (%$invalid_av_per_framework) {
my @errors = Koha::Database::DataInconsistency->nonexistent_AV($biblios);
if (@errors) {
new_section('Wrong values linked to authorised values');
for my $frameworkcode ( keys %$invalid_av_per_framework ) {
while ( my ( $av_category, $v ) = each %{ $invalid_av_per_framework->{$frameworkcode} } ) {
my $items = $v->{items};
my $kohafield = $v->{kohafield};
my ( $table, $column ) = split '\.', $kohafield;
my $output;
while ( my $i = $items->next ) {
my $value =
$table eq 'items' ? $i->$column
: $table eq 'biblio' ? $i->biblio->$column
: $i->biblioitem->$column;
$output .= " {" . $i->itemnumber . " => " . $value . "}\n";
}
new_item(
sprintf(
"The Framework *%s* is using the authorised value's category *%s*, "
. "but the following %s do not have a value defined ({itemnumber => value }):\n%s",
$frameworkcode, $av_category, $kohafield, $output
)
);
}
}
new_item($_) for @errors;
}
}
{
my $biblios = Koha::Biblios->search(
{
-or => [
title => '',
title => undef,
]
}
);
if ( $biblios->count ) {
my $biblios = Koha::Biblios->search;
my @errors = Koha::Database::DataInconsistency->empty_title($biblios);
if (@errors) {
my ( $title_tag, $title_subtag ) = C4::Biblio::GetMarcFromKohaField('biblio.title');
my $title_field = $title_tag // '';
$title_field .= '$' . $title_subtag if $title_subtag;
new_section("Biblio without title $title_field");
while ( my $biblio = $biblios->next ) {
new_item( sprintf "Biblio with biblionumber=%s does not have title defined", $biblio->biblionumber );
}
new_item($_) for @errors;
new_hint("Edit these bibliographic records to define a title");
}
}