gitea
/
Koha-clms
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Bug 2627: Allow Unicode data in import_borrowers, expand feedback on errors. 

Date fields are now checked against syspref and iso regexps before attempting
to convert or insert them.  The problem characters were non-ASCII diacriticals.
Note: this may rely on improvements in the current 0.54 version of Text::CSV and
Text::CSV_XS, rather than the Koha minimum of 0.01 and 0.32, respectively.

Signed-off-by: Galen Charlton <galen.charlton@liblime.com>
This commit is contained in:
Joe Atzberger 2008-09-18 19:02:45 -05:00 committed by Galen Charlton
parent 20c82796aa
commit c9fd091c14
2 changed files with 92 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
koha-tmpl/intranet-tmpl/prog/en/modules/tools/import_borrowers.tmpl
View File

@ -3,6 +3,7 @@
<!-- TMPL_INCLUDE NAME="doc-head-close.inc" -->
<style type="text/css">
.yui-u fieldset.rows label.widelabel { width: 12em; }
.line_error { width: 100%; }
code { background-color: yellow; }
</style>
</head>
@ -13,12 +14,11 @@
<div id="breadcrumbs"><a href="/cgi-bin/koha/mainpage.pl">Home</a> &rsaquo; <a href="/cgi-bin/koha/tools/tools-home.pl">Tools</a> &rsaquo; <a href="/cgi-bin/koha/tools/import_borrowers.pl">Import Patrons</a><!-- TMPL_IF name="uploadborrowers" --> &rsaquo; Results<!-- /TMPL_IF --></div>
<div id="doc3" class="yui-t2">
<div id="bd">
<div id="yui-main">
<div class="yui-b">
<div class="yui-g">
<div class="yui-u first">
<div id="bd">
<div id="yui-main">
<div class="yui-b">
<div class="yui-g">
<div class="yui-u first">
<h1>Import Patrons</h1>
<!-- TMPL_IF name="uploadborrowers" -->
<h5>Import results :</h5>
@ -30,6 +30,23 @@
<li><!-- TMPL_VAR name="total" --> records parsed</li>
<li><a href="/cgi-bin/koha/tools/tools-home.pl">Back to Tools</a></li>
</ul>
<!-- TMPL_IF NAME="FEEDBACK" -->
<br /><br />
<div>
<h5>Feedback:</h5>
<ul class="feedback">
<!-- TMPL_LOOP NAME="FEEDBACK" -->
<li>
<!-- TMPL_IF NAME="filename" -->Parsing upload file <span class="filename"><!-- TMPL_VAR NAME="filename" --></span>
<!-- TMPL_ELSIF NAME="backend" -->Upload parsed using <!-- TMPL_VAR NAME="backend" -->
<!-- TMPL_ELSIF NAME="headerrow" -->These fields found: <!-- TMPL_VAR NAME="value" -->
<!-- TMPL_ELSE --><!-- TMPL_VAR NAME="name" --> : <!-- TMPL_VAR NAME="value" -->
<!-- /TMPL_IF -->
</li>
<!-- /TMPL_LOOP -->
</ul>
</div>
<!-- /TMPL_IF -->
<!-- TMPL_IF NAME="ERRORS" -->
<br /><br />
<div>
@ -38,11 +55,18 @@
<!-- TMPL_LOOP NAME="ERRORS" -->
<!-- TMPL_IF NAME="badheader" --><li>Header row could not be parsed</li><!-- /TMPL_IF -->
<!-- TMPL_LOOP NAME="missing_criticals" -->
<li>
<li class="line_error">
Line <span class="linenumber"><!-- TMPL_VAR NAME="line" --></span>
<!-- TMPL_IF NAME="badparse" -->
Line <span class="linenumber"><!-- TMPL_VAR NAME="line" --></span> could not be parsed!
could not be parsed!
<!-- TMPL_ELSIF NAME="bad_date" -->
has &quot;<!-- TMPL_VAR NAME="key" -->&quot; in unrecognized format: &quot;<!-- TMPL_VAR NAME="value" -->&quot;
<!-- TMPL_ELSE -->
Critical field &quot;<!-- TMPL_VAR NAME="key" -->&quot; missing on line <span class="linenumber"><!-- TMPL_VAR NAME="line" --></span>
Critical field &quot;<!-- TMPL_VAR NAME="key" -->&quot;
<!-- TMPL_IF NAME="branch_map" -->has unrecognized value &quot;<!-- TMPL_VAR NAME="value" -->&quot;
<!-- TMPL_ELSIF NAME="category_map" -->has unrecognized value &quot;<!-- TMPL_VAR NAME="value" -->&quot;
<!-- TMPL_ELSE -->missing
<!-- /TMPL_IF -->
(borrowernumber: <!-- TMPL_VAR NAME="borrowernumber" -->; surname: <!-- TMPL_VAR NAME="surname" -->).
<!-- /TMPL_IF -->
<br /><code><!-- TMPL_VAR NAME="lineraw" --></code>
@ -105,9 +129,10 @@
<ul>
<li><b>Download a starter CSV file with all the columns <a href="?sample=1">here</a>.</b> Values are comma-separated.</li>
<li>OR format your file in CSV format with the following fields:</li>
<ul><li>
<li><ul><li>
<!-- TMPL_LOOP name="columnkeys" -->'<!-- TMPL_VAR name="key" -->', <!-- /TMPL_LOOP -->
</li></ul>
</li>
<!-- TMPL_IF NAME="ExtendedPatronAttributes" -->
<li>If loading patron attributes, the 'patron_attributes' field should contain a comma-separated list of attribute types
and values. The attribute type code and a ':' should precede each value. For example: &quot;INSTID:12345,LANG:fr&quot;. This
@ -119,13 +144,13 @@ means that if an input record has more than one attribute, the 'patron_attribute
<li>Date formats should match your system preference, and <b>must</b> be zero-padded, e.g. '01/02/2008'.</li>
<li>You may optionally include a header row, defining which columns you are supplying in the import file.</li>
</ul>
</div>
</div>
</div>
</div>
<div class="yui-b noprint">
</div>
</div>
</div>
</div>
<div class="yui-b noprint">
<!-- TMPL_INCLUDE NAME="tools-menu.inc" -->
</div>
</div>
</div>
</div>
<!-- TMPL_INCLUDE NAME="intranet-bottom.inc" -->

64
tools/import_borrowers.pl
View File

@ -34,6 +34,8 @@
# branchcode and categorycode need to be valid
use strict;
use warnings;
use C4::Auth;
use C4::Output;
use C4::Dates qw(format_date_in_iso);
@ -44,9 +46,14 @@ use C4::Members::Attributes;
use C4::Members::AttributeTypes;
use Text::CSV;
use CGI;
# Text::CSV::Unicode, even in binary mode, fails to parse lines with these diacriticals:
# ė
# č
my @errors;
use CGI;
# use encoding 'utf8'; # don't do this
my (@errors, @feedback);
my $extended = C4::Context->preference('ExtendedPatronAttributes');
my @columnkeys = C4::Members->columns;
if ($extended) {
@ -55,7 +62,8 @@ if ($extended) {
my $columnkeystpl = [ map { {'key' => $_} } grep {$_ ne 'borrowernumber' && $_ ne 'cardnumber'} @columnkeys ]; # ref. to array of hashrefs.
my $input = CGI->new();
my $csv = Text::CSV->new();
my $csv = Text::CSV->new({binary => 1}); # binary needed for non-ASCII Unicode
# push @feedback, {feedback=>1, name=>'backend', value=>$csv->backend, backend=>$csv->backend};
my ( $template, $loggedinuser, $cookie ) = get_template_and_user({
template_name => "tools/import_borrowers.tmpl",
@ -77,8 +85,8 @@ if ($input->param('sample')) {
print $csv->string, "\n";
exit 1;
}
my $uploadborrowers = $input->param('uploadborrowers');
my $matchpoint = $input->param('matchpoint');
my $uploadborrowers = $input->param('uploadborrowers');
my $matchpoint = $input->param('matchpoint');
if ($matchpoint) {
$matchpoint =~ s/^patron_attribute_//;
}
@ -89,6 +97,12 @@ $template->param( SCRIPT_NAME => $ENV{'SCRIPT_NAME'} );
($extended) and $template->param(ExtendedPatronAttributes => 1);
if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
push @feedback, {feedback=>1, name=>'filename', value=>$uploadborrowers, filename=>$uploadborrowers};
my $handle = $input->upload('uploadborrowers');
my $uploadinfo = $input->uploadInfo($uploadborrowers);
foreach (keys %$uploadinfo) {
push @feedback, {feedback=>1, name=>$_, value=>$uploadinfo->{$_}, $_=>$uploadinfo->{$_}};
}
my $imported = 0;
my $alreadyindb = 0;
my $overwritten = 0;
@ -97,7 +111,7 @@ if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
my %defaults = $input->Vars;
# use header line to construct key to column map
my $borrowerline = <$uploadborrowers>;
my $borrowerline = <$handle>;
my $status = $csv->parse($borrowerline);
($status) or push @errors, {badheader=>1,line=>$., lineraw=>$borrowerline};
my @csvcolumns = $csv->fields();
@ -113,9 +127,13 @@ if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
$matchpoint_attr_type = C4::Members::AttributeTypes->fetch($matchpoint);
}
push @feedback, {feedback=>1, name=>'headerrow', value=>join(', ', @csvcolumns)};
my $today_iso = C4::Dates->new()->output('iso');
my @criticals = qw(cardnumber surname categorycode); # there probably should be others
my @errors;
LINE: while ( my $borrowerline = <$uploadborrowers> ) {
my @bad_dates; # I've had a few.
my $date_re = C4::Dates->new->regexp('syspref');
my $iso_re = C4::Dates->new->regexp('iso');
LINE: while ( my $borrowerline = <$handle> ) {
my %borrower;
my @missing_criticals;
my $patron_attributes;
@ -141,8 +159,18 @@ if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
}
}
#warn join(':',%borrower);
push @missing_criticals, {key=>'categorycode' , line=>$. , lineraw=>$borrowerline } unless( GetBorrowercategory($borrower{categorycode}) );
push @missing_criticals, {key=>'branchcode' , line=>$. , lineraw=>$borrowerline } unless( GetBranchName($borrower{branchcode}) );
if ($borrower{categorycode}) {
push @missing_criticals, {key=>'categorycode', line=>$. , lineraw=>$borrowerline, value=>$borrower{categorycode}, category_map=>1}
unless GetBorrowercategory($borrower{categorycode});
} else {
push @missing_criticals, {key=>'categorycode', line=>$. , lineraw=>$borrowerline};
}
if ($borrower{branchcode}) {
push @missing_criticals, {key=>'branchcode', line=>$. , lineraw=>$borrowerline, value=>$borrower{branchcode}, branch_map=>1}
unless GetBranchName($borrower{branchcode});
} else {
push @missing_criticals, {key=>'branchcode', line=>$. , lineraw=>$borrowerline};
}
if (@missing_criticals) {
foreach (@missing_criticals) {
$_->{borrowernumber} = $borrower{borrowernumber} || 'UNDEF';
@ -162,12 +190,19 @@ if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
# FIXME error handling
$patron_attributes = [ map { map { my @arr = split /:/, $_, 2; { code => $arr[0], value => $arr[1] } } $_ } @list ];
}
# FIXME date handling. Popular spreadsheet applications make it difficult to force date outputs to be zero-padded, but we require it.
# Popular spreadsheet applications make it difficult to force date outputs to be zero-padded, but we require it.
foreach (qw(dateofbirth dateenrolled dateexpiry)) {
my $tempdate = $borrower{$_} or next;
$borrower{$_} = format_date_in_iso($tempdate) || '';
if ($tempdate =~ /$date_re/) {
$borrower{$_} = format_date_in_iso($tempdate);
} elsif ($tempdate =~ /$iso_re/) {
$borrower{$_} = $tempdate;
} else {
$borrower{$_} = '';
push @missing_criticals, {key=>$_, line=>$. , lineraw=>$borrowerline, bad_date=>1};
}
}
$borrower{dateenrolled} = C4::Dates->new()->output('iso') unless $borrower{dateenrolled};
$borrower{dateenrolled} = $today_iso unless $borrower{dateenrolled};
$borrower{dateexpiry} = GetExpiryDate($borrower{categorycode},$borrower{dateenrolled}) unless $borrower{dateexpiry};
my $borrowernumber;
my $member;
@ -231,7 +266,8 @@ if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
}
}
}
(@errors) and $template->param(ERRORS=>\@errors);
(@errors ) and $template->param( ERRORS=>\@errors );
(@feedback) and $template->param(FEEDBACK=>\@feedback);
$template->param(
'uploadborrowers' => 1,
'imported' => $imported,