Bug 35086: Also split chunks when indexing from background job
The es background indexer is designed to combine background jobs when started based on the 'batch_size' option. While this is helpful for combining individual updates, it can be problematic when there are several large batch modifications, or when worker has stopped and is restarted. This patch uses the same logic as in the indexer to split the chunks that are sent directly for indexing. To test: 1 - Follow test plan on previous patch 2 - Confirm items are correctly indexed and jobs marked Signed-off-by: David Nind <david@davidnind.com> Signed-off-by: Jonathan Druart <jonathan.druart@bugs.koha-community.org> Signed-off-by: Katrin Fischer <katrin.fischer@bsz-bw.de> (cherry picked from commit7496a603cd) Signed-off-by: Fridolin Somers <fridolin.somers@biblibre.com> (cherry picked from commit2f8ae2ec21) Signed-off-by: Lucas Gass <lucas@bywatersolutions.com>
This commit is contained in:
parent
19f442c18f
commit
eb2617409a
|
|
@ -55,6 +55,7 @@ use JSON qw( decode_json );
|
|||
use Try::Tiny;
|
||||
use Pod::Usage;
|
||||
use Getopt::Long;
|
||||
use List::MoreUtils qw( natatime );
|
||||
|
||||
use C4::Context;
|
||||
use Koha::Logger;
|
||||
|
|
@ -95,8 +96,11 @@ if ( $conn ) {
|
|||
}
|
||||
);
|
||||
}
|
||||
my $biblio_indexer = Koha::SearchEngine::Indexer->new({ index => $Koha::SearchEngine::BIBLIOS_INDEX });
|
||||
my $auth_indexer = Koha::SearchEngine::Indexer->new({ index => $Koha::SearchEngine::AUTHORITIES_INDEX });
|
||||
my $biblio_indexer = Koha::SearchEngine::Indexer->new( { index => $Koha::SearchEngine::BIBLIOS_INDEX } );
|
||||
my $auth_indexer = Koha::SearchEngine::Indexer->new( { index => $Koha::SearchEngine::AUTHORITIES_INDEX } );
|
||||
my $config = $biblio_indexer->get_elasticsearch_params;
|
||||
my $at_a_time = $config->{chunk_size} // 5000;
|
||||
|
||||
my @jobs = ();
|
||||
|
||||
while (1) {
|
||||
|
|
@ -177,18 +181,24 @@ sub commit {
|
|||
}
|
||||
|
||||
if (@auth_records) {
|
||||
try {
|
||||
$auth_indexer->update_index( \@auth_records );
|
||||
} catch {
|
||||
$logger->warn( sprintf "Update of elastic index failed with: %s", $_ );
|
||||
};
|
||||
my $auth_chunks = natatime $at_a_time, @auth_records;
|
||||
while ( ( my @auth_chunk = $auth_chunks->() ) ) {
|
||||
try {
|
||||
$auth_indexer->update_index( \@auth_chunk );
|
||||
} catch {
|
||||
$logger->warn( sprintf "Update of elastic index failed with: %s", $_ );
|
||||
};
|
||||
}
|
||||
}
|
||||
if (@bib_records) {
|
||||
try {
|
||||
$biblio_indexer->update_index( \@bib_records );
|
||||
} catch {
|
||||
$logger->warn( sprintf "Update of elastic index failed with: %s", $_ );
|
||||
};
|
||||
my $biblio_chunks = natatime $at_a_time, @bib_records;
|
||||
while ( ( my @bib_chunk = $biblio_chunks->() ) ) {
|
||||
try {
|
||||
$biblio_indexer->update_index( \@bib_chunk );
|
||||
} catch {
|
||||
$logger->warn( sprintf "Update of elastic index failed with: %s", $_ );
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
# Finish
|
||||
|
|
|
|||
Loading…
Reference in New Issue