Bug 32440: Support purging statistics by type in cleanup_database.pl
Cleanup_database can delete all statistics entries more than X days old. If one is using pseudonymization to create pseudonymized_transactions data, then one may wish to use that cleanup_database function to delete statistics entries that have been duplicated in pseudonymized_transactions. However, not all types of transactions in statistics are duplicated in pseudonymized transactions. Pseudonymized_transactions currently only includes checkouts, returns, and renewals. This patch adds two additional parameters to cleanup_database.pl: 1. --statistics-type Defines the types of statistics to purge. Will purge all types if parameter is omitted. Repeatable. 2. --statistics-type-pseudo Grabs values from @Koha::Statistic::pseudonymization_types. At the time of writing this patch, they are: renew, issue, return and onsite_checkout To test: 1. prove t/db_dependent/Koha/Statistic.t 2. Before applying this patch: 3. Create some statistics entry by checking out, renewing and checking in items. 4. perl misc/cronjobs/cleanup_database.pl --statistics 1 --verbose Observe: Purging statistics older than 1 days n statistics would have been removed Where n is the amount of statistics rows matching your test environment 5. Apply patch 6. Repeat step 4, observe same result 7. perl misc/cronjobs/cleanup_database.pl --statistics 1 --statistics-type-pseudo --verbose Observe: Purging statistics older than 1 days with types "onsite_checkout,renew,return,issue". n statistics would have been removed Where n is the amount of statistics rows matching your test environment (order of listed types does not matter) 8. perl misc/cronjobs/cleanup_database.pl --statistics 1 --statistics-type test1 --statistics-type test2 --verbose Observe: Purging statistics older than 1 days with types "test1,test2". n statistics would have been removed Where n is the amount of statistics rows matching your test environment (order of listed types does not matter) 9. perl misc/cronjobs/cleanup_database.pl --statistics 1 --statistics-type-pseudo --statistics-type test1 --verbose Observe: Purging statistics older than 1 days with types "test1,onsite_checkout,renew,return,issue". n statistics would have been removed Where n is the amount of statistics rows matching your test environment (order of listed types does not matter) 10. Try previous cleanup_database.pl commands with --confirm flag and make sure correct rows are purged Signed-off-by: Andrew Fuerste Henry <andrew@bywatersolutions.com> Signed-off-by: Marcel de Rooy <m.de.rooy@rijksmuseum.nl> Signed-off-by: Katrin Fischer <katrin.fischer@bsz-bw.de>
This commit is contained in:
Lari Taskula
Katrin Fischer
parent
e2221f5b04
commit
b4662eb60a
|
|
@ -149,7 +149,7 @@ Generate a pesudonymized version of the statistic.
|
|||
sub pseudonymize {
|
||||
my ($self) = @_;
|
||||
|
||||
return unless ( $self->borrowernumber && grep { $_ eq $self->type } qw(renew issue return onsite_checkout) );
|
||||
return unless ( $self->borrowernumber && grep { $_ eq $self->type } @Koha::Statistic::pseudonymization_types );
|
||||
|
||||
# FIXME When getting the object from svc/renewal we get a DateTime object
|
||||
# normally we just fetch from DB to clear this, but statistics has no primary key
|
||||
|
|
@ -161,6 +161,14 @@ sub pseudonymize {
|
|||
|
||||
}
|
||||
|
||||
=head2 Pseudonymization types
|
||||
|
||||
=head3 @pseudonymization_types
|
||||
|
||||
=cut
|
||||
|
||||
our @pseudonymization_types = qw(renew issue return onsite_checkout);
|
||||
|
||||
=head2 Internal methods
|
||||
|
||||
=head3 _type
|
||||
|
|
|
|||
|
|
@ -54,11 +54,12 @@ use Koha::Old::Patrons;
|
|||
use Koha::Patron::Debarments qw( DelDebarment );
|
||||
use Koha::Patron::Messages;
|
||||
use Koha::PseudonymizedTransactions;
|
||||
use Koha::Statistics;
|
||||
use Koha::UploadedFiles;
|
||||
|
||||
sub usage {
|
||||
print STDERR <<USAGE;
|
||||
Usage: $0 [-h|--help] [--confirm] [--sessions] [--sessdays DAYS] [-v|--verbose] [--zebraqueue DAYS] [-m|--mail] [--merged] [--import DAYS] [--logs DAYS] [--searchhistory DAYS] [--restrictions DAYS] [--all-restrictions] [--fees DAYS] [--temp-uploads] [--temp-uploads-days DAYS] [--uploads-missing 0|1 ] [--statistics DAYS] [--deleted-catalog DAYS] [--deleted-patrons DAYS] [--old-issues DAYS] [--old-reserves DAYS] [--transfers DAYS] [--labels DAYS] [--cards DAYS] [--bg-days DAYS [--bg-type TYPE] ] [--edifact-messages DAYS]
|
||||
Usage: $0 [-h|--help] [--confirm] [--sessions] [--sessdays DAYS] [-v|--verbose] [--zebraqueue DAYS] [-m|--mail] [--merged] [--import DAYS] [--logs DAYS] [--searchhistory DAYS] [--restrictions DAYS] [--all-restrictions] [--fees DAYS] [--temp-uploads] [--temp-uploads-days DAYS] [--uploads-missing 0|1 ] [--statistics DAYS] [--statistics-type TYPE] [--deleted-catalog DAYS] [--deleted-patrons DAYS] [--old-issues DAYS] [--old-reserves DAYS] [--transfers DAYS] [--labels DAYS] [--cards DAYS] [--bg-days DAYS [--bg-type TYPE] ] [--edifact-messages DAYS]
|
||||
|
||||
-h --help prints this help message, and exits, ignoring all
|
||||
other options
|
||||
|
|
@ -107,6 +108,9 @@ Usage: $0 [-h|--help] [--confirm] [--sessions] [--sessdays DAYS] [-v|--verbose]
|
|||
--oauth-tokens Delete expired OAuth2 tokens
|
||||
--statistics DAYS Purge statistics entries more than DAYS days old.
|
||||
This table is used to build reports, make sure you are aware of the consequences of this before using it!
|
||||
--statistics-type TYPES What type of statistics to purge. Will purge all types if parameter is omitted. Repeatable.
|
||||
--statistics-type-pseudo Purges statistic types of "@Koha::Statistic::pseudonymization_types", unless more types are defined
|
||||
with --statistics-type.
|
||||
--deleted-catalog DAYS Purge catalog records deleted more then DAYS days ago
|
||||
(from tables deleteditems, deletedbiblioitems, deletedbiblio_metadata and deletedbiblio).
|
||||
--deleted-patrons DAYS Purge patrons deleted more than DAYS days ago.
|
||||
|
|
@ -155,6 +159,8 @@ my $temp_uploads_days;
|
|||
my $uploads_missing;
|
||||
my $oauth_tokens;
|
||||
my $pStatistics;
|
||||
my @statistics_types;
|
||||
my $statistics_type_pseudo;
|
||||
my $pDeletedCatalog;
|
||||
my $pDeletedPatrons;
|
||||
my $pOldIssues;
|
||||
|
|
@ -205,6 +211,8 @@ GetOptions(
|
|||
'uploads-missing:i' => \$uploads_missing,
|
||||
'oauth-tokens' => \$oauth_tokens,
|
||||
'statistics:i' => \$pStatistics,
|
||||
'statistics-type:s' => \@statistics_types,
|
||||
'statistics-type-pseudo:i' => \$statistics_type_pseudo,
|
||||
'deleted-catalog:i' => \$pDeletedCatalog,
|
||||
'deleted-patrons:i' => \$pDeletedPatrons,
|
||||
'old-issues:i' => \$pOldIssues,
|
||||
|
|
@ -235,6 +243,9 @@ $jobs_days = DEFAULT_JOBS_PURGEDAYS if defined($jobs_days)
|
|||
@jobs_types = (DEFAULT_JOBS_PURGETYPES) if $jobs_days && @jobs_types == 0;
|
||||
$edifact_msg_days = DEFAULT_EDIFACT_MSG_PURGEDAYS if defined($edifact_msg_days) && $edifact_msg_days == 0;
|
||||
|
||||
@statistics_types = ( @statistics_types, @Koha::Statistic::pseudonymization_types ) if defined($statistics_type_pseudo);
|
||||
@statistics_types = keys %{ { map { $_, 1 } @statistics_types } }; # remove duplicate types
|
||||
|
||||
# Choose the number of days at which to purge unaccepted list invites:
|
||||
# - DAYS defined in the list-invites parameter is prioritised first
|
||||
# - PurgeListShareInvitesOlderThan system preference is prioritised second
|
||||
|
|
@ -602,11 +613,21 @@ if ($oauth_tokens) {
|
|||
}
|
||||
|
||||
if ($pStatistics) {
|
||||
print "Purging statistics older than $pStatistics days.\n" if $verbose;
|
||||
my $statistics =
|
||||
Koha::Statistics->filter_by_last_update( { timestamp_column_name => 'datetime', days => $pStatistics } );
|
||||
my $count = $statistics->count;
|
||||
$statistics->delete if $confirm;
|
||||
print "Purging statistics older than $pStatistics days" if $verbose;
|
||||
my $statistics_search_params = {};
|
||||
|
||||
if ( scalar(@statistics_types) ) {
|
||||
print " with types \"" . join( ',', @statistics_types ) . "\".\n" if $verbose;
|
||||
$statistics_search_params->{type} = \@statistics_types;
|
||||
|
||||
}
|
||||
print "\n" if $verbose;
|
||||
|
||||
my $statistics = Koha::Statistics->search($statistics_search_params);
|
||||
my $filtered_statistics =
|
||||
$statistics->filter_by_last_update( { timestamp_column_name => 'datetime', days => $pStatistics } );
|
||||
my $count = $filtered_statistics->count;
|
||||
$filtered_statistics->delete if $confirm;
|
||||
if ($verbose) {
|
||||
say $confirm
|
||||
? sprintf( "Done with purging %d statistics", $count )
|
||||
|
|
|
|||
|
|
@ -61,8 +61,7 @@ subtest 'new() tests' => sub {
|
|||
|
||||
subtest 'pseudonymize() tests' => sub {
|
||||
|
||||
my @pseudonymization_types = qw(renew issue return onsite_checkout);
|
||||
plan tests => scalar(@pseudonymization_types) + 2;
|
||||
plan tests => scalar(@Koha::Statistic::pseudonymization_types) + 2;
|
||||
|
||||
$schema->storage->txn_begin;
|
||||
|
||||
|
|
@ -73,10 +72,10 @@ subtest 'pseudonymize() tests' => sub {
|
|||
|
||||
my @statistics = ();
|
||||
|
||||
ok( scalar(@pseudonymization_types) > 0, 'some pseudonymization_types are defined' );
|
||||
ok( scalar(@Koha::Statistic::pseudonymization_types) > 0, 'some pseudonymization_types are defined' );
|
||||
|
||||
my $sub_days = 0; # TestBuilder does not handle many rows of Koha::Statistics very intuitively
|
||||
foreach my $type (@pseudonymization_types) {
|
||||
foreach my $type (@Koha::Statistic::pseudonymization_types) {
|
||||
push(
|
||||
@statistics,
|
||||
$builder->build_object(
|
||||
|
|
@ -121,4 +120,4 @@ subtest 'pseudonymize() tests' => sub {
|
|||
}
|
||||
|
||||
$schema->storage->txn_rollback;
|
||||
};
|
||||
};
|
||||
|
|
|
|||
Loading…
Reference in New Issue