Ключевые слова:perl, rus, patch, (найти похожие документы)
Date: Sat, 28 Dec 2002 03:39:42 +0500
From: Vladimir Volovich <[email protected]>
Newsgroups: ftn.ru.perl
Subject: Разбивка русских слов для переноса на perl при помощи TeX::Hyphen
OK> Hужны русские переносы, взялся за TeX::Hyphen, там есть поддержка
OK> только german и czech. Поскольку я не силен в TeX и не понял,
OK> надо ли делать какие-либо преобразования для русского языка (имхо
OK> нет), то просто убрал все tr/// и s///, кроме cstolower, перебив
OK> ее для koi8 (про locale знаю). Взял файлы переносов из tetex
OK> (последний ryhyphen) и ... в общем слова разбиваюся примерно как
OK> по rand ;)
OK> Может кто решал подобную проблему?
привожу патчик к (не самой новой) версии TeX::Hyphen, который я
использовал (в новой версии кое-что изменилось):
- --- Hyphen.pm.orig 2000-05-07 01:33:42.000000000 +0400
+++ Hyphen.pm 2000-05-28 15:58:08.000000000 +0400
@@ -8,7 +8,9 @@
=head1 SYNOPSIS
use TeX::Hyphen;
- my $hyp = new TeX::Hyphen "hyphen.tex";
+ my $hyp = new TeX::Hyphen "hyphen.tex" \&norm_en;
+ $hyp->{LEFTMIN} = 2;
+ $hyp->{RIGHTMIN} = 3;
my $word = "representation";
my @points = $hyp->hyphenate($word);
print $hyp->visualize($word), "\n";
@@ -19,7 +21,10 @@
patterns into memory. Then you can ask it for hyphenation of a word by
calling a method of this object. If no file is specified, the default
Donald E. Knuth's F<hyphen.tex>, that is included in this module, is
-used instead.
+used instead. You could specify a function used to `normalize' a word
+before hyphenation is done (e.g. change commands like \'a, \v r
+to 8-bit form, convert the word to lowercase, etc.). By default,
+a function which only converts to lowercase is used.
Method hyphenate() returns list of places where the word can be
divided, so
@@ -33,7 +38,7 @@
should give C<rep-re-sen-ta-tion>.
-Variables I<$TeX::Hyphen::LEFTMIN> and I<$TeX::Hyphen::RIGHTMIN> can
+Variables I<$hyp->{LEFTMIN}> and I<$hyp->{RIGHTMIN}> can
be used to restrict minimal starting and ending substring where it is
not possible to hyphenate. They both default to 2 but should be
changed to match the paratemers used to generate the patterns.
@@ -47,33 +52,14 @@
=cut
use strict;
-use vars qw( $VERSION $DEBUG $LEFTMIN $RIGHTMIN );
+use vars qw( $VERSION $DEBUG );
$VERSION = '0.10';
sub Version () { $VERSION; }
$DEBUG ||= 0;
-# To protect beginning and end of the word from hyphenation
-$LEFTMIN = 2;
-$RIGHTMIN = 2;
-
-# ######################################################
-# TeX conversions done for Czech language, eg. \'a, \v r
-#
-my %BACKV = ( 'c' => 'Х', 'd' => 'О', 'e' => 'Л', 'l' => '╣',
- 'n' => 'Р', 'r' => 'Ь', 's' => '╧', 't' => '╩', 'z' => '╬',
- 'C' => 'х', 'D' => 'о', 'E' => 'л', 'L' => '╔', 'N' => 'р',
- 'R' => 'ь', 'S' => '╘', 'T' => '╚', 'Z' => '╝' );
-my %BACKAP = ( 'a' => 'А', 'e' => 'И', 'i' => 'М', 'l' => 'Е',
- 'o' => 'С', 'u' => 'З', 'y' => 'Щ', 'A' => 'а', 'E' => 'и',
- 'I' => 'м', 'L' => 'е', 'O' => 'с', 'U' => 'з', 'Y' => 'щ');
-sub cstolower
- {
- my $e = shift;
- $e =~ tr/[A-Z]адхоилкме╔рстужь╘╚зышэщ╛╝/[a-z]АДХОИЛКМЕ╣РСТУЖЬ╧╩ЗЫШЭЩ╪╬/;
- $e;
- }
+sub norm_en { $_=lc; }
# #############################################################
# Constructor. Parameter specifies file with patterns.
@@ -82,13 +68,17 @@
#
sub new
{
- my ($class, $file) = @_;
+ my ($class, $file, $norm) = @_;
if (not defined $file)
{ *FILE = \*DATA; }
else
{ open FILE, $file or return; }
+ $norm = \&norm_en if (not defined $norm);
my $self = {};
bless $self, $class;
+ $self->{norm} = $norm;
+ $self->{LEFTMIN} = 2;
+ $self->{RIGHTMIN} = 3;
local ($/) = "\n";
my ($tag, $value);
@@ -106,15 +96,13 @@
my $begin = 0;
my $end = 0;
+ &$norm;
$begin = 1 if s!^\.!!;
$end = 1 if s!\.$!!;
- s!\\v\s+(.)!$BACKV{$+}!g;
- s!\\'(.)!$BACKAP{$+}!g;
s!(\D)(?=\D)!$+0!g;
s!^(?=\D)!0!;
($tag = $_) =~ s!\d!!g;
($value = $_) =~ s!\D!!g;
- $tag = cstolower($tag);
if ($begin and $end)
{ $bothhyphen->{$tag} = $value; }
@@ -133,11 +121,8 @@
chomp;
- s!\\v\s+(.)!$BACKV{$+}!g;
- s!\\'(.)!$BACKAP{$+}!g;
-
+ &$norm;
($tag = $_) =~ s!-!!g;
- $tag = cstolower($tag);
($value = '0' . $_) =~ s!0!g;
$value =~ s![^-]-!1!g;
$value =~ s![^01]!0!g;
@@ -173,9 +158,13 @@
sub hyphenate
{
my ($self, $word) = (shift, shift);
+ my $norm = $self->{norm};
+ my $LEFTMIN = $self->{LEFTMIN};
+ my $RIGHTMIN = $self->{RIGHTMIN};
print STDERR "Hyphenate `$word'\n" if $DEBUG;
+ $_=$word; &$norm; $word=$_;
my $exact = $self->{exact};
if (defined(my $res = $exact->{$word}))
{
и тестовый пример (hyphenate.pl):
#!/usr/bin/perl
use TeX::Hyphen;
$TeX::Hyphen::DEBUG=10; # comment this out after testing
my $hyp = new TeX::Hyphen `kpsewhich ruhyph$ARGV[0].tex`, \&norm_ru;
$hyp->{LEFTMIN} = 2;
$hyp->{RIGHTMIN} = 2;
shift;
sub norm_ru {
tr/A-ZЁёЮ-Ъ/a-zеею-ъ/;
}
while (<>) {
s/([A-Za-zЁёЮ-Ъю-ъ]+)/$hyp->visualize($1)/ego;
print;
}
использование:
./hyphenate.pl al < file.txt
Если не заработает - напишите, что выдастся на экран при попытке
перенести какое-нибудь слово (с DEBUG=10)...
Best,
v.
