# $File: //member/autrijus/Locale-Maketext-Lexicon/lib/Locale/Maketext/Lexicon.pm $ $Author: autrijus $ # $Revision: #49 $ $Change: 11058 $ $DateTime: 2004/08/22 19:23:52 $ package Locale::Maketext::Lexicon; $Locale::Maketext::Lexicon::VERSION = '0.41'; use strict; =head1 NAME Locale::Maketext::Lexicon - Use other catalog formats in Maketext =head1 VERSION This document describes version 0.41 of Locale::Maketext::Lexicon, released August 25, 2004. =head1 SYNOPSIS As part of a localization class, automatically glob for available lexicons: package Hello::L10N; use base 'Locale::Maketext'; use Locale::Maketext::Lexicon { '*' => [Gettext => '/usr/local/share/locale/*/LC_MESSAGES/hello.mo'], _decode => 1, # decode lexicon entries into utf8-strings }; Explicitly specify languages, during compile- or run-time: package Hello::L10N; use base 'Locale::Maketext'; use Locale::Maketext::Lexicon { de => [Gettext => 'hello_de.po'], fr => [ Gettext => 'hello_fr.po', Gettext => 'local/hello/fr.po', ], }; # ... incrementally add new lexicons Locale::Maketext::Lexicon->import({ de => [Gettext => 'local/hello/de.po'], }) Alternatively, as part of a localization subclass: package Hello::L10N::de; use base 'Hello::L10N'; use Locale::Maketext::Lexicon (Gettext => \*DATA); __DATA__ # Some sample data msgid "" msgstr "" "Project-Id-Version: Hello 1.3.22.1\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=iso8859-1\n" "Content-Transfer-Encoding: 8bit\n" #: Hello.pm:10 msgid "Hello, World!" msgstr "Hallo, Welt!" #: Hello.pm:11 msgid "You have %quant(%1,piece) of mail." msgstr "Sie haben %quant(%1,Poststueck,Poststuecken)." =head1 DESCRIPTION This module provides lexicon-handling modules to read from other localization formats, such as I<Gettext>, I<Msgcat>, and so on. If you are unfamiliar with the concept of lexicon modules, please consult L<Locale::Maketext> and L<http://www.autrijus.org/webl10n/> first. A command-line utility L<xgettext.pl> is also installed with this module, for extracting translatable strings from source files. =head2 The C<import> function The C<import()> function accepts two forms of arguments: =over 4 =item (I<format> => I<source> ... ) This form takes any number of argument pairs (usually one); I<source> may be a file name, a filehandle, or an array reference. For each such pair, it pass the contents specified by the second argument to B<Locale::Maketext::Lexicon::I<format>>->parse as a plain list, and export its return value as the C<%Lexicon> hash in the calling package. In the case that there are multiple such pairs, the lexicon defined by latter ones overrides earlier ones. =item { I<language> => [ I<format>, I<source> ... ] ... } This form accepts a hash reference. It will export a C<%Lexicon> into the subclasses specified by each I<language>, using the process described above. It is designed to alleviate the need to set up a separate subclass for each localized language, and just use the catalog files. This module will convert the I<language> arguments into lowercase, and replace all C<-> with C<_>, so C<zh_TW> and C<zh-tw> will both map to the C<zh_tw> subclass. If I<language> begins with C<_>, it is taken as an option that controls how lexicons are parsed. See L</Options> for a list of available options. The C<*> is a special I<language>; it must be used in conjunction with a filename that also contains C<*>; all matched files with a valid language code in the place of C<*> will be automatically prepared as a lexicon subclass. If there is multiple C<*> in the filename, the last one is used as the language name. =back =head2 Options =over 4 =item C<_decode> If set to a true value, source entries will be converted into utf8-strings (available in Perl 5.6.1 or later). This feature needs the B<Encode> or B<Encode::compat> module. Currently, only the C<Gettext> backend supports this option. =item C<_encoding> This option only has effect when C<_decode> is set to true. It specifies an encoding to store lexicon entries, instead of utf8-strings. If C<_encoding> is set to C<locale>, the encoding from the current locale setting is used. =head2 Subclassing format handlers If you wish to override how sources specified in different data types are handled, please use a subclass that overrides C<lexicon_get_I<TYPE>>. XXX: not documented well enough yet. Patches welcome. =head1 NOTES If you want to implement a new C<Lexicon::*> backend module, please note that C<parse()> takes an array containing the B<source strings> from the specified filehandle or filename, which are I<not> C<chomp>ed. Although if the source is an array reference, its elements will probably not contain any newline characters anyway. The C<parse()> function should return a hash reference, which will be assigned to the I<typeglob> (C<*Lexicon>) of the language module. All it amounts to is that if the returned reference points to a tied hash, the C<%Lexicon> will be aliased to the same tied hash if it was not initialized previously. =cut our %Opts; sub option { shift if ref($_[0]); $Opts{lc $_[0]} } sub set_option { shift if ref($_[0]); $Opts{lc $_[0]} = $_[1] } sub encoding { my $encoding = option(@_, 'encoding') or return; return $encoding unless lc($encoding) eq 'locale'; no warnings 'uninitialized'; my ($country_language, $locale_encoding); local $@; eval { require I18N::Langinfo; $locale_encoding = I18N::Langinfo::langinfo(I18N::Langinfo::CODESET()); } or eval { require Win32::Console; $locale_encoding = 'cp'.Win32::Console::OutputCP(); }; if (!$locale_encoding) { foreach my $key (qw( LANGUAGE LC_ALL LC_MESSAGES LANG )) { $ENV{$key} =~ /^([^.]+)\.([^.:]+)/ or next; ($country_language, $locale_encoding) = ($1, $2); last; } } if (defined $locale_encoding && lc($locale_encoding) eq 'euc' && defined $country_language) { if ($country_language =~ /^ja_JP|japan(?:ese)?$/i) { $locale_encoding = 'euc-jp'; } elsif ($country_language =~ /^ko_KR|korean?$/i) { $locale_encoding = 'euc-kr'; } elsif ($country_language =~ /^zh_CN|chin(?:a|ese)?$/i) { $locale_encoding = 'euc-cn'; } elsif ($country_language =~ /^zh_TW|taiwan(?:ese)?$/i) { $locale_encoding = 'euc-tw'; } } return $locale_encoding; } sub import { my $class = shift; return unless @_; my %entries; if (UNIVERSAL::isa($_[0], 'HASH')) { # a hashref with $lang as keys, [$format, $src ...] as values %entries = %{$_[0]}; } elsif (@_ % 2) { %entries = ( '' => [ @_ ] ); } # expand the wildcard entry if (my $wild_entry = delete $entries{'*'}) { while (my ($format, $src) = splice(@$wild_entry, 0, 2)) { next if ref($src); # XXX: implement globbing for the 'Tie' backend my $pattern = quotemeta($src); $pattern =~ s/\\\*(?=[^*]+$)/\([-\\w]+\)/g or next; $pattern =~ s/\\\*/.*?/g; $pattern =~ s/\\\?/./g; $pattern =~ s/\\\[/[/g; $pattern =~ s/\\\]/]/g; $pattern =~ s[\\\{(.*?)\\\\}][ '(?:'.join('|', split(/,/, $1)).')' ]eg; require File::Glob; foreach my $file (File::Glob::bsd_glob($src)) { $file =~ /$pattern/ or next; push @{$entries{$1}}, ($format => $file) if $1; } delete $entries{$1} unless !defined($1) or exists $entries{$1} and @{$entries{$1}}; } } %Opts = (); foreach my $key (grep /^_/, keys %entries) { set_option(lc(substr($key, 1)) => delete($entries{$key})); } my $OptsRef = { %Opts }; while (my ($lang, $entry) = each %entries) { my $export = caller; if (length $lang) { # normalize language tag to Maketext's subclass convention $lang = lc($lang); $lang =~ s/-/_/g; $export .= "::$lang"; } my @pairs = @{$entry||[]} or die "no format specified"; while (my ($format, $src) = splice(@pairs, 0, 2)) { if (defined($src) and !ref($src) and $src =~ /\*/) { unshift(@pairs, $format => $_) for File::Glob::bsd_glob($src); next; } my @content = $class->lexicon_get($src, scalar caller, $lang); no strict 'refs'; eval "use $class\::$format; 1" or die $@; if (defined %{"$export\::Lexicon"}) { if (ref(tied %{"$export\::Lexicon"}) eq __PACKAGE__) { tied(%{"$export\::Lexicon"})->(); } # be very careful not to pollute the possibly tied lexicon *{"$export\::Lexicon"} = { %{"$export\::Lexicon"}, %{"$class\::$format"->parse(@content)}, }; } else { my $promise; bless($promise = sub { $_[0] = $promise; return "$export\::Lexicon"; }, __PACKAGE__); tie %{"$export\::Lexicon"}, __PACKAGE__, sub { local %Opts = %$OptsRef; *{"$export\::Lexicon"} = "$class\::$format"->parse(@content); return $promise->($_[0]); }; } push(@{"$export\::ISA"}, scalar caller) if length $lang; } } } sub TIEHASH { my ($class, $promise) = @_; return bless($promise, $class); } { no strict 'refs'; sub _force { $_[0]->($_[0]) } sub FETCH { _force($_[0])->{$_[1]} } sub EXISTS { exists _force($_[0])->{$_[1]} } sub DELETE { delete _force($_[0])->{$_[1]} } sub SCALAR { scalar %{_force($_[0])} } sub STORE { _force($_[0])->{$_[1]} = $_[2] } sub CLEAR { %{_force($_[0])->{$_[1]}} = () } sub NEXTKEY { each %{_force($_[0])} } sub FIRSTKEY { my $hash = _force($_[0]); my $a = scalar keys %$hash; each %$hash; } } sub lexicon_get { my ($class, $src, $caller, $lang) = @_; return unless defined $src; foreach my $type (qw(ARRAY HASH SCALAR GLOB), ref($src)) { next unless UNIVERSAL::isa($src, $type); my $method = 'lexicon_get_' . lc($type); die "cannot handle source $type for $src: no $method defined" unless $class->can($method); return $class->$method($src, $caller, $lang); } # default handler return $class->lexicon_get_($src, $caller, $lang); } # for scalarrefs and arrayrefs we just dereference the $src sub lexicon_get_scalar { ${$_[1]} } sub lexicon_get_array { @{$_[1]} } sub lexicon_get_hash { my ($class, $src, $caller, $lang) = @_; return map { $_ => $src->{$_} } sort keys %$src; } sub lexicon_get_glob { my ($class, $src, $caller, $lang) = @_; no strict 'refs'; # be extra magical and check for DATA section if (eof($src) and $src eq \*{"$caller\::DATA"} or $src eq \*{"main\::DATA"}) { # okay, the *DATA isn't initiated yet. let's read. # my $level = 1; my $fh; while (my($pkg, $filename) = caller($level++)) { next unless $pkg eq $caller or $pkg eq 'main'; next unless -e $filename; require FileHandle; $fh = FileHandle->new; $fh->open($filename) or die "Can't open $filename: $!"; last; } while (<$fh>) { # okay, this isn't foolproof, but good enough last if /^__DATA__$/; } return <$fh>; } # fh containing the lines my $pos = tell($src); my @lines = <$src>; seek($src, $pos, 0); return @lines; } # assume filename - search path, open and return its contents sub lexicon_get_ { my ($class, $src, $caller, $lang) = @_; require FileHandle; require File::Spec; my $fh = FileHandle->new; my @path = split('::', $caller); push @path, $lang if length $lang; $src = (grep { -e } map { my @subpath = @path[0..$_]; map { File::Spec->catfile($_, @subpath, $src) } @INC; } -1 .. $#path)[-1] unless -e $src; die "cannot find $_[1] (called by $_[2]) in \@INC" unless -e $src; $fh->open($src) or die $!; binmode($fh); return <$fh>; } 1; =head1 ACKNOWLEDGMENTS Thanks to Jesse Vincent for suggesting this module to be written. Thanks also to Sean M. Burke for coming up with B<Locale::Maketext> in the first place, and encouraging me to experiment with alternative Lexicon syntaxes. Thanks also to Yi Ma Mao for providing the MO file parsing subroutine, as well as inspiring me to implement file globbing and transcoding support. See the F<AUTHORS> file in the distribution for a list of people who have sent helpful patches, ideas or comments. =head1 SEE ALSO L<xgettext.pl> for extracting translatable strings from common template systems and perl source files. L<Locale::Maketext>, L<Locale::Maketext::Lexicon::Auto>, L<Locale::Maketext::Lexicon::Gettext>, L<Locale::Maketext::Lexicon::Msgcat>, L<Locale::Maketext::Lexicon::Tie> =head1 AUTHORS Autrijus Tang E<lt>autrijus@autrijus.orgE<gt> =head1 COPYRIGHT Copyright 2002, 2003, 2004 by Autrijus Tang E<lt>autrijus@autrijus.orgE<gt>. This program is free software; you can redistribute it and/or modify it under the same terms as Perl itself. See L<http://www.perl.com/perl/misc/Artistic.html> =cut