Bug 7698: Add CHR/ICU Zebra tokenization choice to installation
Word search with multi-part facets works properly only with Zebra ICU
tokenization. This patch add a new question to Koha command line
installer:
Zebra has two methods to perform records tokenization
and characters normalization: CHR and ICU. ICU is
recommended for catalogs containing non-Latin
characters. (chr, icu) [chr]
How to test:
- perl ./Makefile.PL
- Try each possible value for new parameter
- Take a look at zebradb/etc/default.idx file.
Depending of the parameter you get this line:
icuchain words-icu.xml
or this one:
charmap word-phrase-utf.chr
Signed-off-by: Jared Camins-Esakov <jcamins@cpbibliography.com>
(Note: This patch was previously associated with bug 3216; I moved it to a
separate bug because including ICU is a good idea independent of the fix for
the particular issue described in bug 3216)
Signed-off-by: Paul Poulain <paul.poulain@biblibre.com>
This commit is contained in:
parent
a5e6f107ee
commit
a0316d4d27
22
Makefile.PL
22
Makefile.PL
|
|
@ -410,6 +410,7 @@ my %config_defaults = (
|
|||
'AUTH_INDEX_MODE' => 'dom',
|
||||
'ZEBRA_MARC_FORMAT' => 'marc21',
|
||||
'ZEBRA_LANGUAGE' => 'en',
|
||||
'ZEBRA_TOKENIZER' => 'chr',
|
||||
'ZEBRA_USER' => 'kohauser',
|
||||
'ZEBRA_PASS' => 'zebrastripes',
|
||||
'ZEBRA_SRU_HOST' => 'localhost',
|
||||
|
|
@ -461,6 +462,7 @@ my %valid_config_values = (
|
|||
'AUTH_INDEX_MODE' => { 'grs1' => 1, 'dom' => 1 },
|
||||
'ZEBRA_MARC_FORMAT' => { 'marc21' => 1, 'normarc' => 1, 'unimarc' => 1 }, # FIXME should generate from contents of distributation
|
||||
'ZEBRA_LANGUAGE' => { 'en' => 1, 'fr' => 1, 'nb' => 1 }, # FIXME should generate from contents of distribution
|
||||
'ZEBRA_TOKENIZER' => { chr => 1, icu => 1 },
|
||||
'RUN_DATABASE_TESTS' => { 'yes' => 1, 'no' => 1 },
|
||||
'USE_MEMCACHED' => { 'yes' => 1, 'no' => 1 },
|
||||
);
|
||||
|
|
@ -487,11 +489,12 @@ my $pl_files = {
|
|||
'rewrite-config.PL' => [
|
||||
'blib/KOHA_CONF_DIR/koha-conf.xml',
|
||||
'blib/KOHA_CONF_DIR/koha-httpd.conf',
|
||||
'blib/ZEBRA_CONF_DIR/etc/default.idx',
|
||||
'blib/MISC_DIR/koha-install-log'
|
||||
],
|
||||
'fix-perl-path.PL' => [ # this script ensures the correct shebang line for the platform installed on...
|
||||
'blib'
|
||||
]
|
||||
],
|
||||
};
|
||||
|
||||
if ($config{'INSTALL_ZEBRA'} eq "yes") {
|
||||
|
|
@ -528,6 +531,10 @@ if ($config{'INSTALL_MODE'} ne "dev") {
|
|||
);
|
||||
}
|
||||
|
||||
$config{ZEBRA_TOKENIZER_STMT} = $config{ZEBRA_TOKENIZER} eq 'icu'
|
||||
? 'icuchain words-icu.xml'
|
||||
: 'charmap word-phrase-utf.chr';
|
||||
|
||||
my %test_suite_override_dirs = (
|
||||
KOHA_CONF_DIR => ['etc'],
|
||||
ZEBRA_CONF_DIR => ['etc', 'zebradb'],
|
||||
|
|
@ -918,8 +925,8 @@ Since you've chosen to use Zebra with Koha,
|
|||
you must specify the primary MARC format of the
|
||||
records to be indexed by Zebra.
|
||||
|
||||
Koha provides Zebra configuration files for MARC 21
|
||||
and UNIMARC.
|
||||
Koha provides Zebra configuration files for MARC21,
|
||||
NORMARC and UNIMARC.
|
||||
|
||||
MARC format for Zebra indexing);
|
||||
$msg .= _add_valid_values_disp('ZEBRA_MARC_FORMAT', $valid_values);
|
||||
|
|
@ -946,6 +953,15 @@ Authorities indexing mode);
|
|||
$msg .= _add_valid_values_disp('AUTH_INDEX_MODE', $valid_values);
|
||||
$config{'AUTH_INDEX_MODE'} = _get_value('AUTH_INDEX_MODE', $msg, $defaults->{'AUTH_INDEX_MODE'}, $valid_values, $install_log_values);
|
||||
|
||||
$msg = q(
|
||||
Zebra has two methods to perform records tokenization
|
||||
and characters normalization: CHR and ICU. ICU is
|
||||
recommended for catalogs containing non-Latin
|
||||
characters.);
|
||||
|
||||
$msg .= _add_valid_values_disp('ZEBRA_TOKENIZER', $valid_values);
|
||||
$config{'ZEBRA_TOKENIZER'} = _get_value('ZEBRA_TOKENIZER', $msg, $defaults->{'ZEBRA_TOKENIZER'}, $valid_values, $install_log_values);
|
||||
|
||||
$msg = q(
|
||||
Please specify Zebra database user);
|
||||
$config{'ZEBRA_USER'} = _get_value('ZEBRA_USER', $msg, $defaults->{'ZEBRA_USER'}, $valid_values, $install_log_values);
|
||||
|
|
|
|||
|
|
@ -10,16 +10,15 @@ completeness 0
|
|||
position 1
|
||||
alwaysmatches 1
|
||||
firstinfield 1
|
||||
charmap word-phrase-utf.chr
|
||||
#firstinfield 1
|
||||
__ZEBRA_TOKENIZER_STMT__
|
||||
|
||||
# Phrase index
|
||||
# Used if completeness is 'complete {sub}field' (@attr 6=2, @attr 6=1)
|
||||
# and structure is word/phrase/word-list/free-form-text/document-text
|
||||
index p
|
||||
completeness 1
|
||||
charmap word-phrase-utf.chr
|
||||
firstinfield 1
|
||||
__ZEBRA_TOKENIZER_STMT__
|
||||
|
||||
# URX (URL) index
|
||||
# Used if structure=urx (@attr 4=104)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,12 @@
|
|||
<icu_chain locale="">
|
||||
<transliterate rule="\'>\ "/>
|
||||
<transliterate rule="[:Number:] { '-' > '' "/>
|
||||
<transform rule="[:Control:] Any-Remove"/>
|
||||
<tokenize rule="l"/>
|
||||
<transform rule="[[:WhiteSpace:][:Punctuation:]] Remove"/>
|
||||
<transform rule="NFD"/>
|
||||
<transform rule="[:Nonspacing Mark:] Remove"/>
|
||||
<transform rule="NFC"/>
|
||||
<display/>
|
||||
<casemap rule="l"/>
|
||||
</icu_chain>
|
||||
|
|
@ -120,6 +120,7 @@ $prefix = $ENV{'INSTALL_BASE'} || "/usr";
|
|||
'__ZEBRA_RUN_DIR__' => "$prefix/var/run/zebradb",
|
||||
'__ZEBRA_MARC_FORMAT__' => 'marc21',
|
||||
'__ZEBRA_LANGUAGE__' => 'en',
|
||||
'__ZEBRA_TOKENIZER_STMT__' => 'charmap word-phrase-utf.chr',
|
||||
'__ZEBRA_AUTH_CFG__' => 'zebra-authorities.cfg',
|
||||
'__AUTH_RETRIEVAL_CFG__' => 'retrieval-info-auth-grs1.xml',
|
||||
"__MERGE_SERVER_HOST__" => $myhost,
|
||||
|
|
|
|||
Loading…
Reference in New Issue