From 23ff8a7b1d376a26f45df0309a8ad75cff6469ab Mon Sep 17 00:00:00 2001 From: Tomas Cohen Arazi Date: Mon, 21 Apr 2025 15:16:32 -0300 Subject: [PATCH] Bug 39694: Make `es_indexer_daemon.pl` use batch_size to limit DB poll size This patch makes the indexer poll the DB in configurable batches. Pretty much like Rabbit does. The only change is that it will now pass a LIMIT to the query. To test: 0. Have KTD started with ES (skip if you have an instance running): $ ktd --es7 up -d 1. Stop the indexer and rabbit: $ ktd --shell k$ sudo koha-es-indexer --stop kohadev k$ sudo service rabbitmq-server stop 2. On a separate terminal, watch the queue: k$ watch -n 0.5 'echo "SELECT COUNT(*) FROM background_jobs WHERE status=\"new\"" | koha-mysql kohadev' => SUCCESS: It should be empty 3. Add a bunch of things to index (this works with default KTD data): k$ for i in $(seq 1 438; seq 1 438) ; do echo "INSERT INTO background_jobs (status,queue,data) VALUES ('new','elastic_index','{\"record_server\":\"biblioserver\",\"record_ids\":[$i]}')" | koha-mysql kohadev ; done 4. Start the ES indexer: k$ sudo koha-es-indexer --start kohadev 5. Look at the queue size => FAIL: It shrinks too fast [1] 6. Apply this patch 7. Restart the indexer k$ perl misc/workers/es_indexer_daemon.pl & 8. Repeat 3 9. Look at the queue size: => SUCCESS: It shrinks 'slowly' 10. Sign off :-D [1] This is a problem in certain scenarios, not in all of them. This is the best I come up with in order to test the behavior change. Anyone can set a new size of their taste. Signed-off-by: Tomas Cohen Arazi Signed-off-by: Martin Renvoize Signed-off-by: Katrin Fischer (cherry picked from commit 20b600d45042ef8c676aa8631a864428c4a3bff6) Signed-off-by: Fridolin Somers --- misc/workers/es_indexer_daemon.pl | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/misc/workers/es_indexer_daemon.pl b/misc/workers/es_indexer_daemon.pl index 9e900b7195..8edf55200a 100755 --- a/misc/workers/es_indexer_daemon.pl +++ b/misc/workers/es_indexer_daemon.pl @@ -64,9 +64,8 @@ use Koha::BackgroundJobs; use Koha::SearchEngine; use Koha::SearchEngine::Indexer; - -my ( $help, $batch_size ); - +my $help; +my $batch_size = 10; my $not_found_retries = {}; my $max_retries = $ENV{MAX_RETRIES} || 10; @@ -77,8 +76,6 @@ GetOptions( pod2usage(0) if $help; -$batch_size //= 10; - warn "Not using Elasticsearch" unless C4::Context->preference('SearchEngine') eq 'Elasticsearch'; my $logger = Koha::Logger->get({ interface => 'worker' }); @@ -171,7 +168,9 @@ while (1) { } else { @jobs = Koha::BackgroundJobs->search( - { status => 'new', queue => 'elastic_index' } )->as_list; + { status => 'new', queue => 'elastic_index' }, + { rows => $batch_size } + )->as_list; commit(@jobs); @jobs = (); sleep 10;