MARC import: part 3 of large file support
* Introduced C4::UploadedFile to handle management and progress tracking of uploaded files. * Modified stage-marc-import.pl to handle new upload mechanism Signed-off-by: Joshua Ferraro <jmf@liblime.com>
This commit is contained in:
parent
296e4909e0
commit
e9f36cc227
|
|
@ -0,0 +1,317 @@
|
|||
package C4::UploadedFile;
|
||||
|
||||
# Copyright (C) 2007 LibLime
|
||||
# Galen Charlton <galen.charlton@liblime.com>
|
||||
#
|
||||
# This file is part of Koha.
|
||||
#
|
||||
# Koha is free software; you can redistribute it and/or modify it under the
|
||||
# terms of the GNU General Public License as published by the Free Software
|
||||
# Foundation; either version 2 of the License, or (at your option) any later
|
||||
# version.
|
||||
#
|
||||
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
|
||||
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
|
||||
# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along with
|
||||
# Koha; if not, write to the Free Software Foundation, Inc., 59 Temple Place,
|
||||
# Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
use strict;
|
||||
use C4::Context;
|
||||
use C4::Auth qw/get_session/;
|
||||
use IO::File;
|
||||
|
||||
use vars qw($VERSION);
|
||||
|
||||
# set the version for version checking
|
||||
$VERSION = 3.00;
|
||||
|
||||
=head1 NAME
|
||||
|
||||
C4::UploadedFile - manage files uploaded by the user
|
||||
for later processing.
|
||||
|
||||
=head1 SYNOPSIS
|
||||
|
||||
=over 4
|
||||
|
||||
# create and store data
|
||||
my $uploaded_file = C4::UploadedFile->new($sessionID);
|
||||
my $fileID = $uploaded_file->id();
|
||||
$uploaded_file->name('c:\temp\file.mrc');
|
||||
$uploaded_file->max_size(1024);
|
||||
while ($have_more_data) {
|
||||
$uploaded_file->stash($data, $bytes_read);
|
||||
}
|
||||
$uploaded_file->done();
|
||||
|
||||
# check status of current file upload
|
||||
my $progress = C4::UploadedFile->upload_progress($sessionID);
|
||||
|
||||
# get file handle for reading uploaded file
|
||||
my $uploaded_file = C4::UploadedFile->fetch($fileID);
|
||||
my $fh = $uploaded_file->fh();
|
||||
|
||||
=back
|
||||
|
||||
Stores files uploaded by the user from their web browser. The
|
||||
uploaded files are temporary and at present are not guaranteed
|
||||
to survive beyond the life of the user's session.
|
||||
|
||||
This module allows for tracking the progress of the file
|
||||
currently being uploaded.
|
||||
|
||||
TODO: implement secure persistant storage of uploaded files.
|
||||
|
||||
=cut
|
||||
|
||||
=head1 METHODS
|
||||
|
||||
=cut
|
||||
|
||||
=head2 new
|
||||
|
||||
=over 4
|
||||
|
||||
my $uploaded_file = C4::UploadedFile->new($sessionID);
|
||||
|
||||
=back
|
||||
|
||||
Creates a new object to represent the uploaded file. Requires
|
||||
the current session ID.
|
||||
|
||||
=cut
|
||||
|
||||
sub new {
|
||||
my $class = shift;
|
||||
my $sessionID = shift;
|
||||
|
||||
my $self = {};
|
||||
|
||||
$self->{'sessionID'} = $sessionID;
|
||||
$self->{'fileID'} = Digest::MD5::md5_hex(Digest::MD5::md5_hex(time().{}.rand().{}.$$));
|
||||
# FIXME - make staging area configurable
|
||||
my $TEMPROOT = "/tmp";
|
||||
my $OUTPUTDIR = "$TEMPROOT/$sessionID";
|
||||
mkdir $OUTPUTDIR;
|
||||
my $tmp_file_name = "$OUTPUTDIR/$self->{'fileID'}";
|
||||
my $fh = new IO::File $tmp_file_name, "w";
|
||||
unless (defined $fh) {
|
||||
return undef;
|
||||
}
|
||||
$fh->binmode(); # Windows compatibility
|
||||
$self->{'fh'} = $fh;
|
||||
$self->{'tmp_file_name'} = $tmp_file_name;
|
||||
my $session = get_session($sessionID);
|
||||
$session->param("$self->{'fileID'}.uploaded_tmpfile", $tmp_file_name);
|
||||
$session->param('current_upload', $self->{'fileID'});
|
||||
$session->flush();
|
||||
$self->{'session'} = $session;
|
||||
$self->{'name'} = '';
|
||||
$self->{'max_size'} = 0;
|
||||
$self->{'progress'} = 0;
|
||||
|
||||
bless $self, $class;
|
||||
|
||||
return $self;
|
||||
|
||||
}
|
||||
|
||||
=head2 id
|
||||
|
||||
=over 4
|
||||
|
||||
my $fileID = $uploaded_file->id();
|
||||
|
||||
=back
|
||||
|
||||
=cut
|
||||
|
||||
sub id {
|
||||
my $self = shift;
|
||||
return $self->{'fileID'};
|
||||
}
|
||||
|
||||
=head2 name
|
||||
|
||||
=over 4
|
||||
|
||||
my $name = $uploaded_file->name();
|
||||
$uploaded_file->name($name);
|
||||
|
||||
=back
|
||||
|
||||
Accessor method for the name by which the file is to be known.
|
||||
|
||||
=cut
|
||||
|
||||
sub name {
|
||||
my $self = shift;
|
||||
if (@_) {
|
||||
$self->{'name'} = shift;
|
||||
$self->{'session'}->param("$self->{'fileID'}.uploaded_filename", $self->{'name'});
|
||||
$self->{'session'}->flush();
|
||||
} else {
|
||||
return $self->{'name'};
|
||||
}
|
||||
}
|
||||
|
||||
=head2 max_size
|
||||
|
||||
=over 4
|
||||
|
||||
my $max_size = $uploaded_file->max_size();
|
||||
$uploaded_file->max_size($max_size);
|
||||
|
||||
=back
|
||||
|
||||
Accessor method for the maximum size of the uploaded file.
|
||||
|
||||
=cut
|
||||
|
||||
sub max_size {
|
||||
my $self = shift;
|
||||
@_ ? $self->{'max_size'} = shift : $self->{'max_size'};
|
||||
}
|
||||
|
||||
=head2 stash
|
||||
|
||||
=over 4
|
||||
|
||||
$uploaded_file->stash($dataref, $bytes_read);
|
||||
|
||||
=back
|
||||
|
||||
Write C<$dataref> to the temporary file. C<$bytes_read> represents
|
||||
the number of bytes (out of C<$max_size>) transmitted so far.
|
||||
|
||||
=cut
|
||||
|
||||
sub stash {
|
||||
my $self = shift;
|
||||
my $dataref = shift;
|
||||
my $bytes_read = shift;
|
||||
|
||||
my $fh = $self->{'fh'};
|
||||
print $fh $$dataref;
|
||||
|
||||
my $percentage = int(($bytes_read / $self->{'max_size'}) * 100);
|
||||
if ($percentage > $self->{'progress'}) {
|
||||
$self->{'progress'} = $percentage;
|
||||
$self->{'session'}->param("$self->{'fileID'}.uploadprogress", $self->{'progress'});
|
||||
$self->{'session'}->flush();
|
||||
}
|
||||
}
|
||||
|
||||
=head2 done
|
||||
|
||||
=over 4
|
||||
|
||||
$uploaded_file->done();
|
||||
|
||||
=back
|
||||
|
||||
Indicates that all of the bytes have been uploaded.
|
||||
|
||||
=cut
|
||||
|
||||
sub done {
|
||||
my $self = shift;
|
||||
$self->{'session'}->param("$self->{'fileID'}.uploadprogress", 'done');
|
||||
$self->{'session'}->flush();
|
||||
$self->{'fh'}->close();
|
||||
}
|
||||
|
||||
=head2 upload_progress
|
||||
|
||||
=over 4
|
||||
|
||||
my $upload_progress = C4::UploadFile->upload_progress($sessionID);
|
||||
|
||||
=back
|
||||
|
||||
Returns (as an integer from 0 to 100) the percentage
|
||||
progress of the current file upload.
|
||||
|
||||
=cut
|
||||
|
||||
sub upload_progress {
|
||||
my ($class, $sessionID) = shift;
|
||||
|
||||
my $session = get_session($sessionID);
|
||||
|
||||
my $fileID = $session->param('current_upload');
|
||||
|
||||
my $reported_progress = 0;
|
||||
if (defined $fileID and $fileID ne "") {
|
||||
my $progress = $session->param("$fileID.uploadprogress");
|
||||
if (defined $progress) {
|
||||
if ($progress eq "done") {
|
||||
$reported_progress = 100;
|
||||
} else {
|
||||
$reported_progress = $progress;
|
||||
}
|
||||
}
|
||||
}
|
||||
return $reported_progress;
|
||||
}
|
||||
|
||||
=head2 fetch
|
||||
|
||||
=over 4
|
||||
|
||||
my $uploaded_file = C4::UploadedFile->fetch($sessionID, $fileID);
|
||||
|
||||
=back
|
||||
|
||||
Retrieves an uploaded file object from the current session.
|
||||
|
||||
=cut
|
||||
|
||||
sub fetch {
|
||||
my $class = shift;
|
||||
my $sessionID = shift;
|
||||
my $fileID = shift;
|
||||
|
||||
my $self = {};
|
||||
|
||||
$self->{'sessionID'} = $sessionID;
|
||||
$self->{'fileID'} = $fileID;
|
||||
my $session = get_session($sessionID);
|
||||
$self->{'session'} = $session;
|
||||
$self->{'tmp_file_name'} = $session->param("$self->{'fileID'}.uploaded_tmpfile");
|
||||
$self->{'name'} = $session->param("$self->{'fileID'}.uploaded_filename");
|
||||
my $fh = new IO::File $self->{'tmp_file_name'}, "r";
|
||||
$self->{'fh'} = $fh;
|
||||
|
||||
bless $self, $class;
|
||||
|
||||
return $self;
|
||||
}
|
||||
|
||||
=head2 fh
|
||||
|
||||
=over
|
||||
|
||||
my $fh = $uploaded_file->fh();
|
||||
|
||||
=back
|
||||
|
||||
Returns an IO::File handle to read the uploaded file.
|
||||
|
||||
=cut
|
||||
|
||||
sub fh {
|
||||
my $self = shift;
|
||||
return $self->{'fh'};
|
||||
}
|
||||
|
||||
=head1 AUTHOR
|
||||
|
||||
Koha Development Team <info@koha.org>
|
||||
|
||||
Galen Charlton <galen.charlton@liblime.com>
|
||||
|
||||
=cut
|
||||
|
|
@ -28,6 +28,7 @@ use strict;
|
|||
|
||||
# standard or CPAN modules used
|
||||
use CGI;
|
||||
use CGI::Cookie;
|
||||
use MARC::File::USMARC;
|
||||
|
||||
# Koha modules used
|
||||
|
|
@ -38,27 +39,12 @@ use C4::Output;
|
|||
use C4::Biblio;
|
||||
use C4::ImportBatch;
|
||||
use C4::Matcher;
|
||||
|
||||
#------------------
|
||||
# Constants
|
||||
|
||||
my $includes = C4::Context->config('includes') ||
|
||||
"/usr/local/www/hdl/htdocs/includes";
|
||||
|
||||
# HTML colors for alternating lines
|
||||
my $lc1='#dddddd';
|
||||
my $lc2='#ddaaaa';
|
||||
|
||||
#-------------
|
||||
#-------------
|
||||
# Initialize
|
||||
|
||||
my $userid=$ENV{'REMOTE_USER'};
|
||||
use C4::UploadedFile;
|
||||
|
||||
my $input = new CGI;
|
||||
my $dbh = C4::Context->dbh;
|
||||
|
||||
my $uploadmarc=$input->param('uploadmarc');
|
||||
my $fileID=$input->param('uploadedfileid');
|
||||
my $matcher_id = $input->param('matcher');
|
||||
my $parse_items = $input->param('parse_items');
|
||||
my $comments = $input->param('comments');
|
||||
|
|
@ -73,15 +59,19 @@ my ($template, $loggedinuser, $cookie)
|
|||
});
|
||||
|
||||
$template->param(SCRIPT_NAME => $ENV{'SCRIPT_NAME'},
|
||||
uploadmarc => $uploadmarc);
|
||||
my $filename = $uploadmarc;
|
||||
if ($uploadmarc && length($uploadmarc)>0) {
|
||||
uploadmarc => $fileID);
|
||||
|
||||
if ($fileID) {
|
||||
my %cookies = parse CGI::Cookie($cookie);
|
||||
my $uploaded_file = C4::UploadedFile->fetch($cookies{'CGISESSID'}->value, $fileID);
|
||||
my $fh = $uploaded_file->fh();
|
||||
my $marcrecord='';
|
||||
while (<$uploadmarc>) {
|
||||
while (<$fh>) {
|
||||
$marcrecord.=$_;
|
||||
}
|
||||
|
||||
# FIXME branch code
|
||||
my $filename = $uploaded_file->name();
|
||||
my ($batch_id, $num_valid, $num_items, @import_errors) = BatchStageMarcRecords($syntax, $marcrecord, $filename,
|
||||
$comments, '', $parse_items, 0);
|
||||
my $num_with_matches = 0;
|
||||
|
|
|
|||
|
|
@ -24,10 +24,10 @@ use IO::File;
|
|||
use CGI;
|
||||
use CGI::Session;
|
||||
use C4::Context;
|
||||
use C4::Auth qw/get_session check_cookie_auth/;
|
||||
use C4::Auth qw/check_cookie_auth/;
|
||||
use C4::UploadedFile;
|
||||
use CGI::Cookie; # need to check cookies before
|
||||
# having CGI parse the POST request
|
||||
use Digest::MD5;
|
||||
|
||||
my %cookies = fetch CGI::Cookie;
|
||||
my %cookies = fetch CGI::Cookie;
|
||||
|
|
@ -39,23 +39,7 @@ if ($auth_status ne "ok") {
|
|||
exit 0;
|
||||
}
|
||||
|
||||
my $session = get_session($sessionID);
|
||||
|
||||
my $query = CGI->new;
|
||||
my $fileid = $session->param('current_upload');
|
||||
|
||||
my $reported_progress = 0;
|
||||
if (defined $fileid and $fileid ne "") {
|
||||
my $progress = $session->param("$fileid.uploadprogress");
|
||||
if (defined $progress) {
|
||||
if ($progress eq "done") {
|
||||
$reported_progress = 100;
|
||||
} else {
|
||||
$reported_progress = $progress;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
my $reported_progress = C4::UploadedFile->upload_progress($sessionID);
|
||||
|
||||
my $reply = CGI->new("");
|
||||
print $reply->header(-type => 'text/html');
|
||||
|
|
|
|||
|
|
@ -24,20 +24,10 @@ use IO::File;
|
|||
use CGI;
|
||||
use CGI::Session;
|
||||
use C4::Context;
|
||||
use C4::Auth qw/get_session check_cookie_auth/;
|
||||
use C4::Auth qw/check_cookie_auth/;
|
||||
use CGI::Cookie; # need to check cookies before
|
||||
# having CGI parse the POST request
|
||||
use Digest::MD5;
|
||||
|
||||
my %cookies = fetch CGI::Cookie;
|
||||
my ($auth_status, $sessionID) = check_cookie_auth($cookies{'CGISESSID'}->value, { tools => 1 });
|
||||
if ($auth_status ne "ok") {
|
||||
$auth_status = 'denied' if $auth_status eq 'failed';
|
||||
send_reply($auth_status, "", "");
|
||||
exit 0;
|
||||
}
|
||||
|
||||
my $session = get_session($sessionID);
|
||||
use C4::UploadedFile;
|
||||
|
||||
# upload-file.pl must authenticate the user
|
||||
# before processing the POST request,
|
||||
|
|
@ -45,68 +35,49 @@ my $session = get_session($sessionID);
|
|||
# not authorized. Consequently, unlike
|
||||
# most of the other CGI scripts, upload-file.pl
|
||||
# requires that the session cookie already
|
||||
# have been created., $fileid, $tmp_file_name
|
||||
# have been created.
|
||||
|
||||
my $fileid = Digest::MD5::md5_hex(Digest::MD5::md5_hex(time().{}.rand().{}.$$));
|
||||
|
||||
# FIXME - make staging area configurable
|
||||
my $TEMPROOT = "/tmp";
|
||||
my $OUTPUTDIR = "$TEMPROOT/$sessionID";
|
||||
mkdir $OUTPUTDIR;
|
||||
my $tmp_file_name = "$OUTPUTDIR/$fileid";
|
||||
|
||||
my $fh = new IO::File $tmp_file_name, "w";
|
||||
unless (defined $fh) {
|
||||
# FIXME - failed to create file for some reason
|
||||
send_reply('failed', '', '');
|
||||
my %cookies = fetch CGI::Cookie;
|
||||
my ($auth_status, $sessionID) = check_cookie_auth($cookies{'CGISESSID'}->value, { tools => 1 });
|
||||
if ($auth_status ne "ok") {
|
||||
$auth_status = 'denied' if $auth_status eq 'failed';
|
||||
send_reply($auth_status, "");
|
||||
exit 0;
|
||||
}
|
||||
$fh->binmode(); # for Windows compatibility
|
||||
$session->param("$fileid.uploaded_tmpfile", $tmp_file_name);
|
||||
$session->param('current_upload', $fileid);
|
||||
$session->flush();
|
||||
|
||||
my $progress = 0;
|
||||
my $uploaded_file = C4::UploadedFile->new($sessionID);
|
||||
unless (defined $uploaded_file) {
|
||||
# FIXME - failed to create file for some reason
|
||||
send_reply('failed', '');
|
||||
exit 0;
|
||||
}
|
||||
$uploaded_file->max_size($ENV{'CONTENT_LENGTH'}); # may not be the file size, exactly
|
||||
|
||||
my $first_chunk = 1;
|
||||
my $max_size = $ENV{'CONTENT_LENGTH'}; # may not be the file size, exactly
|
||||
|
||||
my $query;
|
||||
$|++;
|
||||
$query = new CGI \&upload_hook, $session;
|
||||
clean_up();
|
||||
send_reply('done', $fileid, $tmp_file_name);
|
||||
$query = new CGI \&upload_hook;
|
||||
$uploaded_file->done();
|
||||
send_reply('done', $uploaded_file->id());
|
||||
|
||||
# FIXME - if possible, trap signal caused by user cancelling upload
|
||||
# FIXME - something is wrong during cleanup: \t(in cleanup) Can't call method "commit" on unblessed reference at /usr/local/share/perl/5.8.8/CGI/Session/Driver/DBI.pm line 130 during global destruction.
|
||||
exit 0;
|
||||
|
||||
sub clean_up {
|
||||
$session->param("$fileid.uploadprogress", 'done');
|
||||
$session->flush();
|
||||
}
|
||||
|
||||
sub upload_hook {
|
||||
my ($file_name, $buffer, $bytes_read, $session) = @_;
|
||||
print $fh $buffer;
|
||||
# stash received file name
|
||||
$uploaded_file->stash(\$buffer, $bytes_read);
|
||||
if ($first_chunk) {
|
||||
$session->param("$fileid.uploaded_filename", $file_name);
|
||||
$session->flush();
|
||||
$uploaded_file->name($file_name);
|
||||
$first_chunk = 0;
|
||||
}
|
||||
my $percentage = int(($bytes_read / $max_size) * 100);
|
||||
if ($percentage > $progress) {
|
||||
$progress = $percentage;
|
||||
$session->param("$fileid.uploadprogress", $progress);
|
||||
$session->flush();
|
||||
}
|
||||
}
|
||||
|
||||
sub send_reply {
|
||||
my ($upload_status, $fileid, $tmp_file_name) = @_;
|
||||
my ($upload_status, $fileid) = @_;
|
||||
|
||||
my $reply = CGI->new("");
|
||||
print $reply->header(-type => 'text/html');
|
||||
# response will be sent back as JSON
|
||||
print "{ status: '$upload_status', fileid: '$fileid', tmp_file_name: '$tmp_file_name' }";
|
||||
print "{ status: '$upload_status', fileid: '$fileid' }";
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue