support more omlgc.
[chise/omega.git] / inCHISE
1 #!/usr/bin/perl -w -CSD
2
3 # ver.0.2
4
5 use strict;
6 use vars qw($omegadb_path
7             $opt_protrude $opt_allow_unify
8             $opt_use_kage_for_Ext_B
9             %opt_order %order %order_map
10             $opt_in_cs $opt_out_cs
11             $opt_help $usage
12             $in_cs $out_cs $i @chars
13             $char $char_id $out_char
14             $char_unified @chars_unified
15             $ids $ids_argc %ids $idsdb $geta
16             $idsdata_file $ids_start $font_start
17             @CDP @HZK @GT
18             );
19 use Getopt::Long;
20 use utf8;
21 use Fcntl ':flock';
22 use Chise_utils ':all';
23 require 5.008;
24
25 ### Options ###
26
27 #$opt_order{'UniMulti'}='jcgk';
28 $opt_order{'UniMulti'}='jGcgkHC';
29 $opt_order{'UniCNS'}='c';
30 $opt_order{'UniGB'}='g';
31 $opt_order{'UniJIS'}='j';
32 $opt_order{'UniKS'}='k';
33
34 $opt_allow_unify=1; # 1=true, 0=false.
35 $opt_protrude=0;# 1=true, 0=false.
36
37 # currently does not work.
38 $opt_use_kage_for_Ext_B=0;# 1=true, 0=false.
39
40 ### End ###
41
42 my $strictly_forbidden_after = '「【『[(〈“‘‘(〔{《{\[\(\x{3016}{「';
43 #       \x{3016} | # white 【
44
45 my $forbidden_after = "\x{0000}";
46
47 # ¥¥$$〒♯##¢¢££@@§
48 my $slightly_forbidden_after = '¥¥$$〒♯##¢¢££@@§';
49
50 # $strictly_forbidden_before
51 # All these characters are allowed to protrude
52 # in the right margin
53 my $strictly_forbidden_before=
54     '!,.:;?、。!,.:;?。\)#}’”〉》」』】〕\x{3017})]}}」\]';
55 ###       \x{3017} | # white 】
56
57 my $forbidden_before
58     = 'ー々ぁぃぅぇぉゃゅょっゎァィゥェォャュョッヮヵヶ';
59
60 my $slightly_forbidden_before
61     = '\x{000a}\#\-‐−‰′″℃゛゜ゝゞヽヾ"%-゙゚';
62
63 my $asian = '\x{1100}-\x{11FF}\x{2E80}-\x{D7AF}\x{F900}-\x{FAFF}\x{FE30}-\x{FE4F}\x{FF00}-\x{FFFFFF}';
64
65 my $space = '\x{0020}\x{0009}\x{000A}\x{000C}\x{000D}';
66
67 my %tex_meta=('#'=>'\#',
68               '$'=>'\\textdollar{}',
69               '%'=>'\%',
70               '&'=>'\&',
71               '{'=>'\\textbraceleft{}',
72               '}'=>'\\textbraceright{}',
73               '\\'=>'\\textbackslash{}',
74               '_'=>'\\textunderscore',
75              );
76
77 my $tex_meta_re=join('|',map {quotemeta($_)} keys %tex_meta);
78
79 &GetOptions("in=s"=>\$opt_in_cs,
80             "out=s"=>\$opt_out_cs,
81             "help",\$opt_help);
82
83 $usage=<<EOF;
84 Usage: $0 -i <input coding system> -o <cmap encoding>
85     input coding system:
86       Utf8mcs, Utf8cns, Utf8gb, Utf8jis, Utf8ks, Utf8big5
87     cmap encoding:
88       UniCNS, UniGB, UniJIS, UniKS, UniMulti
89 EOF
90
91 if($opt_in_cs or $opt_out_cs){
92     $in_cs=$opt_in_cs;
93     $out_cs=$opt_out_cs;
94 }elsif(@ARGV==0){
95     ($in_cs,$out_cs)=($0=~/(Utf8.+)To(\w+)/);
96 }
97
98 # $in_cs:
99 #   Utf8mcs,Utf8cns,Utf8gb,Utf8jis,Utf8ks,Utf8big5
100 # $out_cs:
101 #   UniCNS,UniGB,UniJIS,UniKS,UniMulti
102
103 $in_cs=~s/Utf8/ucs\@/;
104
105 if($opt_help
106    or not defined($in_cs)
107    or not defined($out_cs)){
108     print $usage;
109     exit 1;
110 }
111
112 $omegadb_path=~s!/$!!;
113
114 $idsdata_file="$omegadb_path/idsdata.pl";
115 $ids_start=0x00; 
116 $font_start=0;
117
118 if(-e $idsdata_file){
119     open(IDSDATA,"+<:utf8",$idsdata_file) or die;
120     flock(IDSDATA,LOCK_EX);
121     seek(IDSDATA,0,0);
122     while(<IDSDATA>){
123         eval $_;
124     }
125     seek(IDSDATA,0,0);
126 #         require $idsdata_file;
127 }else{
128     open(IDSDATA,">:utf8",$idsdata_file) or die;
129     flock(IDSDATA,LOCK_EX);
130     seek(IDSDATA,0,0);
131 }
132
133 $ids_argc=0;
134 $ids="";
135
136 #$geta=pack("U",0x3013);
137 $geta=pack("U",0xfffd);
138
139 @GT=(#"=gt","=gt-k",
140      "=gt-pj-1","=gt-pj-2","=gt-pj-3","=gt-pj-4","=gt-pj-5",
141      "=gt-pj-6","=gt-pj-7","=gt-pj-8","=gt-pj-9","=gt-pj-10",
142      "=gt-pj-11"
143      #,"=gt-pj-k1","=gt-pj-k2"
144      );
145 @HZK=("=hanziku-1","=hanziku-2","=hanziku-3","=hanziku-4",
146       "=hanziku-5","=hanziku-6","=hanziku-7","=hanziku-8",
147       "=hanziku-9","=hanziku-10","=hanziku-11","=hanziku-12");
148 @CDP=("=big5-cdp");
149
150 %order_map=('c'=>'UniCNS',
151             'g'=>'UniGB',
152             'j'=>'UniJIS',
153             'k'=>'UniKS',
154             'G'=>'GT',
155             'H'=>'HZK',
156             'C'=>'CDP',
157             );
158
159 foreach $out_cs ('UniCNS','UniGB','UniJIS','UniKS','UniMulti'){
160     if(defined($opt_order{$out_cs})){
161         if($opt_order{$out_cs}=~/^[cgjkGHC]+$/){
162             @{$order{$out_cs}}=map {$order_map{$_}}
163             (split(//,$opt_order{$out_cs}));
164         }else{
165             print STDERR "Invalid order for $out_cs!\n";
166             exit 1;
167         }
168     }
169 }
170
171 while(<>){
172     utf8::decode($_);
173     if($in_cs ne 'ucs@mcs'){
174         s/(.)/&get_char_in_mcs($1,$in_cs)/ge;
175     }
176     s/(amp.+?;)/&de_tex_er($1)/ge;
177 #    s/(&.+?;)/&de_tex_er($1)/ge;
178     @chars=split(//);
179   CHAR:
180     for($i=0;$i<=$#chars;$i++){
181         $char=$chars[$i];
182         $char_id=unpack("U",$char);
183
184         if($char_id<=0x20){
185             print $chars[$i];
186             next CHAR;
187         }elsif($char=~m/($tex_meta_re)/o){
188             print $tex_meta{$1};
189             next CHAR;
190         }elsif(($char_id>0x20 and $char_id<=0x021f)
191                # Basic Latin
192                # Latin-1 Supplement
193                # Latin Extended-A
194                # Latin Extended-B (not all)
195                or($char_id>=0x0250 and $char_id<=0x02af)
196                # IPA Extensions
197                or($char_id>=0x0300 and $char_id<=0x033f)
198                or($char_id>=0x0360 and $char_id<=0x036f)
199                # Combining Diacritical Marks
200                or($char_id>=0x1e00 and $char_id<=0x1eff)
201                # Latin Extended Additional
202                or($char_id>=0x0370 and $char_id<=0x03ff)
203                # Greek and Coptic
204                or($char_id>=0x0400 and $char_id<=0x04ff)
205                # Cyrillic
206                or($char_id>=0x0530 and $char_id<=0x058f)
207                # Armenian
208                ){
209             print &latin_parse();
210             next CHAR;
211         }elsif($char_id>=0x2ff0 and $char_id<=0x2fff){
212             # Ideographic Description Characters
213             print &ids_parse();
214             next CHAR;
215         }else{
216             if(($out_char=&get_output_char($char,$out_cs))){
217                 print $out_char,&add_break($i);
218             }else{
219                 if($opt_allow_unify){
220                     @chars_unified=&get_chars_unified($char);
221                     if(@chars_unified>0){
222                         foreach $char_unified (@chars_unified){
223                             if(($out_char
224                                 =&get_output_char($char_unified,$out_cs))){
225                                 print $out_char,&add_break($i);
226                                 next CHAR;
227                             }
228                         }
229                     }
230                 }
231                 if($opt_use_kage_for_Ext_B){
232                     if($char_id >= 0x20000 && $char_id <=0x2a6df){
233                         # CJK Unified Ideographs Extension B
234                         if(not defined($ids{$char}) and $ids{$char}[1]>=0){
235                             $ids{$char}[0]=$font_start;
236                             $ids{$char}[1]=$ids_start;
237                             $ids_start++;
238                             if($ids_start>255){
239                                 $ids_start=0;
240                                 $font_start++;
241                             }
242                         }
243                         print "{\\fontencoding{OT1}\\fontfamily{" .
244                             sprintf("chise%03d",$ids{$char}[0]) .
245                                 "}\\selectfont\\char$ids{$char}[1]}",&add_break($i);
246                         next CHAR;
247                     }
248                 }
249                 if($ids=&get_ids($char)){
250                     print &get_macro_for_ids($ids),&add_break($i);
251                 }else{
252                     print '\rule{1ex}{1ex}',&add_break($i);
253                 }
254             }
255         }
256     }
257 }
258
259 print IDSDATA 'use utf8;',"\n";
260 foreach $ids (keys %ids){
261     print IDSDATA '$ids{\'',$ids,'\'}='
262     ,'[',join ",",@{$ids{$ids}},"];\n";
263 }
264 print IDSDATA '$font_start=',$font_start,";\n";
265 print IDSDATA '$ids_start=',$ids_start,";\n";
266 print IDSDATA "1;";
267 flock(IDSDATA,LOCK_UN);
268
269 exit 0;
270
271 sub de_tex_er{
272     my($er)=@_;
273     my($prefix,$suffix);
274     my($output_char,$atr,$value);
275     $er=~/^(amp)(.*)(;)$/
276         and $prefix=$1,$er=$2,$suffix=$3;
277     $prefix or $prefix="",$suffix or $suffix="";
278     if($er=~/^U[\+|\-]([a-fA-F\d]+)/){
279         $output_char=pack("U",hex($1));
280     }elsif($er=~/^(?:I\-)?($er_prefix_re)\-?([0-9a-fA-F]+)$/){
281         ($atr,$value)=($1,$2);
282         unless($er_alias{$atr}=~/daikanwa|gt/){
283             $value=hex($value);
284         }
285         ($output_char)=&get_chars_matching($er_alias{$atr},$value);
286     }
287     if($output_char){
288         return $output_char;
289     }else{
290         return $prefix.$er.$suffix;
291     }
292 }
293
294 sub add_break{
295     my($i)=@_;
296
297     if($i<($#chars-1)){
298         if(($chars[$i+1]=~m/[$strictly_forbidden_before]/o)
299            and($chars[$i+2]=~m/[$strictly_forbidden_before]/o)){
300             return "\\CJKunbreakablekernone ";
301         }elsif($opt_protrude){
302             if(($chars[$i+1]=~m/[$strictly_forbidden_before]/o)
303                and($chars[$i+2]=~m/[^$strictly_forbidden_before]/o)){
304                 return "\\CJKunbreakablekernone \\CJKprotrude ";
305             }
306         }
307     }
308     if(($i<$#chars)
309        and($chars[$i+1]=~m/[$strictly_forbidden_before]/o)){
310         return "\\CJKunbreakablekernone ";
311     }
312     if($chars[$i]=~m/[$strictly_forbidden_after]/o){
313         return "\\CJKunbreakablekernone ";
314     }
315     if(($i<$#chars)
316        and($chars[$i+1]=~m/[$forbidden_before]/o)){
317         return "\\CJKunbreakablekerntwo ";
318
319     }
320     if($chars[$i]=~m/[$forbidden_after]/o){
321         return "\\CJKunbreakablekerntwo ";
322     }
323     if(($i<$#chars)
324        and($chars[$i+1]=~m/[$slightly_forbidden_before]/o)){
325         return "\\CJKunbreakablekernthree ";
326     }
327     if($chars[$i]=~m/[$slightly_forbidden_after]/o){
328         return "\\CJKunbreakablekernthree ";
329     }
330     if($chars[$i]=~m/[$asian]/o){
331         return "\\CJKbreakablekern ";
332     }
333     if(($i<$#chars)and($chars[$i+1]=~m/[$asian]/o)){
334         return "\\CJKbreakablekern ";
335     }
336 }
337
338 sub latin_parse{
339     # arguments: none
340     # return: string for output with TeX macro.
341     my($char_id);
342     my $out_str=$chars[$i];
343     $i++;
344     while($i<=$#chars){
345         $char_id=unpack("U",$chars[$i]);
346         if(($char_id>0x20 and $char_id<=0x021f)
347            or($char_id>=0x0250 and $char_id<=0x02af)# IPA Extensions
348            or($char_id>=0x0300 and $char_id<=0x033f)# Combining Diacritical Marks
349            or($char_id>=0x0360 and $char_id<=0x036f)
350            or($char_id>=0x0370 and $char_id<=0x03ff)# Greek and Coptic
351            or($char_id>=0x0400 and $char_id<=0x04ff)# Cyrillic
352            or($char_id>=0x0530 and $char_id<=0x058f)# Armenian
353            ){
354             $out_str.=$chars[$i];
355         }elsif($char_id>=0x1e00 and $char_id<=0x1eff){
356             # Latin Extended Additional
357             # 0x1e00 -> 0x0600, etc.
358             $out_str.=pack("U",$char_id-0x1800);
359         }else{
360             $i--;
361             last;
362         }
363         $i++;
364     }
365     return '{\fontencoding{UT1}\fontfamily{omlgc}\selectfont '.$out_str.'}';
366 }
367
368 sub ids_parse{
369     # arguments: none
370     # return: character for output,
371     #          TeX macro for ids,
372     #          or GETA character if ids is invalid.
373     my($ids,$ids_argc)=&ids_rest("",0,$chars[$i]);
374
375     while($ids_argc>0){
376         # We are in IDS.
377         $i++;
378         if($i>$#chars){
379             print STDERR "IDS parse error: $ids\n";
380             return $geta;
381         }
382
383         ($ids,$ids_argc)=&ids_rest($ids,$ids_argc,$chars[$i]);
384         if($ids_argc==0){
385             if(($char=&get_char_for_ids($ids))
386                and($out_char=&get_output_char($char,$out_cs))){
387                 return $out_char;
388             }else{
389                 return &get_macro_for_ids($ids);
390             }
391         }
392     }
393 }
394
395 sub ids_rest{
396     # arguments: <ids>, <rest number of arguments for ids>, <character>
397     # return: ids and rest number of arguments for ids.
398     my($ids,$ids_argc,$char)=@_;
399     my($argc);
400     $argc=&ids_argc($char);
401     if($argc){
402         $ids_argc+=$ids_argc==0?$argc:$argc-1;
403     }else{
404         $ids_argc--;
405     }
406     $ids.=$char;
407     return ($ids,$ids_argc);
408 }
409
410 sub get_macro_for_ids{
411     # argument: <ids>
412     # return: TeX macro for ids
413     #          or GETA character if ids is invalid for KAGE.
414     my($ids)=@_;
415     # $ids=&normalize_ids($ids,"UniJIS");
416     return $geta if(($ids!~/[$idc]/)
417                     or($ids=~/[\x{10000}-]/));
418                     #irregular for KAGE.
419     if(not defined($ids{$ids})){
420         $ids{$ids}[0]=$font_start;
421         $ids{$ids}[1]=$ids_start;
422         $ids_start++;
423     }
424     if($ids_start>255){
425         $ids_start=0;
426         $font_start++;
427     }
428     return "{\\fontencoding{OT1}\\fontfamily{"
429         .sprintf("chise%03d",$ids{$ids}[0])
430         ."}\\selectfont\\char$ids{$ids}[1]}";
431 }
432
433 sub normalize_ids{
434     # argument: <ids>, <output coding system>
435     # return: ids or GETA character if ids is invalid for KAGE.
436     my($ids,$out_cs)=@_;
437     $out_cs=~s/Uni(.+)/'ucs@'.lc($1)/e;
438
439     my $output_ids="";
440     my($char,$output_char_id);
441     while($ids=~m/(.)/g){
442         $char=$1;
443         if($char=~/[$idc]/){
444             $output_ids.=$char;
445         }elsif($output_char_id=&get_char_attribute($char,"=$out_cs")
446            or $output_char_id=&get_char_attribute($char,"=ucs")
447            or $output_char_id=&get_char_attribute($char,"=>$out_cs")
448            or $output_char_id=&get_char_attribute($char,"=>ucs")
449            or $output_char_id=&get_char_attribute($char,"=>ucs*")
450               ){
451             $output_ids.=pack("U",$output_char_id);
452         }else{
453             return $geta;
454         }
455     }
456     return $output_ids;
457 }
458
459 sub get_output_char{
460     # argument: <char>
461     # return: character in output coding system or TeX macro or undef.
462     my($char,$out_cs)=@_;
463     my($out_char_id,$suffix);
464     my($gt,$hzk,$cdp);
465
466     foreach $out_cs (@{$order{$out_cs}}){
467         if($out_cs eq 'UniJIS'
468            and &get_char_attribute($char,"vnd-adobe-cid-unijis-utf16-h")){
469             if($out_char_id=&get_char_attribute($char,'=ucs@jis')
470                or $out_char_id=&get_char_attribute($char,'=ucs')
471                or $out_char_id=&get_char_attribute($char,'=>ucs@jis')
472                or $out_char_id=&get_char_attribute($char,'=>ucs')
473                or $out_char_id=&get_char_attribute($char,'=>ucs*')
474                ){
475                 return '{\selectjisfont\char'.$out_char_id.'}';
476             }
477         }elsif($out_cs eq 'UniGB'
478                and &get_char_attribute($char,"vnd-adobe-cid-unigb-ucs2-h")){
479             if($out_char_id=&get_char_attribute($char,'=ucs@gb')
480                or $out_char_id=&get_char_attribute($char,'=ucs')
481                or $out_char_id=&get_char_attribute($char,'=>ucs@gb')
482                or $out_char_id=&get_char_attribute($char,'=>ucs')
483                or $out_char_id=&get_char_attribute($char,'=>ucs*')
484                ){
485                 return '{\selectgbsfont\char'.$out_char_id.'}';
486             }
487         }elsif($out_cs eq 'UniCNS'
488                and &get_char_attribute($char,"vnd-adobe-cid-unicns-ucs2-h")){
489             if($out_char_id=&get_char_attribute($char,'=ucs@cns')
490                or $out_char_id=&get_char_attribute($char,'=ucs')
491                or $out_char_id=&get_char_attribute($char,'=>ucs@cns')
492                or $out_char_id=&get_char_attribute($char,'=>ucs')
493                or $out_char_id=&get_char_attribute($char,'=>ucs*')
494                ){
495                 return '{\selectcnsfont\char'.$out_char_id.'}';
496             }
497         }elsif($out_cs eq 'UniKS'
498                and &get_char_attribute($char,"vnd-adobe-cid-uniks-ucs2-h")){
499             if($out_char_id=&get_char_attribute($char,'=ucs@ks')
500                or $out_char_id=&get_char_attribute($char,'=ucs')
501                or $out_char_id=&get_char_attribute($char,'=>ucs@ks')
502                or $out_char_id=&get_char_attribute($char,'=>ucs')
503                or $out_char_id=&get_char_attribute($char,'=>ucs*')
504                ){
505                 return '{\selectksxfont\char'.$out_char_id.'}';
506             }
507         }elsif($out_cs eq 'GT'){
508             return $gt if($gt=&get_macro_for_GT($char));
509         }elsif($out_cs eq 'HZK'){
510             return $hzk if($hzk=&get_macro_for_HZK($char));
511         }elsif($out_cs eq 'CDP'){
512             return $cdp if($cdp=&get_macro_for_CDP($char));
513         }
514     }
515     return undef;
516 }
517
518 sub get_ids{
519     # argument: <character>
520     # return: ids
521     my($char)=@_;
522     my $ids="";
523     $ids=&get_char_attribute($char,"ids-aggregated")
524         or $ids=&get_char_attribute($char,"ids");
525 #    $ids=&get_char_attribute($char,"ids-decomposed")
526 #       or $ids=&get_char_attribute($char,"ids");
527 #         or $ids=&get_char_attribute($char,"ideographic-structure");
528 #    $ids=~s/[? ()]//g;
529     return $ids;
530 }
531
532 sub get_char_for_ids{
533     # argument: <ideographic description sequence>
534     # return: char or undef.
535     my($ids)=@_;
536     my($output_char);
537
538     if(($output_char)=&get_chars_matching("ids",$ids)){
539         return $output_char;
540     }else{
541         return undef;
542     }
543 }
544
545 sub get_char_in_mcs{
546     # argument: <char>, <input coding system>
547     # return:   char in ucs@mcs.
548     my($char,$in_cs)=@_;
549     my($output_char);
550
551     return $char if($in_cs eq 'ucs@mcs');
552
553     if(($output_char)=&get_chars_matching("=$in_cs",unpack("U",$char))){
554         return $output_char;
555     }else{
556         return $char;
557     }
558 }
559
560 sub get_chars_unified{
561     my($char)=@_;
562     my($chars,$ucs,$char_ucs);
563     my(@chars);
564
565     if($chars=&get_char_attribute($char,'->ucs-unified')){
566         $chars=~s/^\((.*)\)$/$1/;
567         return (split(/\s*\?/,$chars));
568     }elsif($ucs=&get_char_attribute($char,'=>ucs*')
569           or $ucs=&get_char_attribute($char,'=>ucs')){
570         $char_ucs=pack("U",$ucs);
571         if($chars=&get_char_attribute($char_ucs,'->ucs-unified')){
572             $chars=~s/^\((.*)\)$/$1/;
573             @chars=grep {not /^$char$/}
574                 (split(/\s*\?/,$chars));
575             push(@chars,$char_ucs);
576             return @chars;
577         }
578     }
579 }
580
581 sub get_macro_for_GT{
582     # argument: <char>
583     # return: TeX macro for GT fonts or undef.
584     my($char)=@_;
585     my($gt,$GT);
586     foreach (@GT){
587         if($gt=&get_char_attribute($char,$_)){
588             m/gt\-pj\-(\d+)/ and $GT=$1;
589             last;
590         }
591     }
592     if($gt){
593         return "{\\fontencoding{OT1}\\fontfamily{"
594             .sprintf("gt%02d",$GT)
595             ."}\\selectfont\\char".($gt|0x8080)."}";
596     }else{
597         return undef;
598     }
599 }
600
601 sub get_macro_for_HZK{
602     # argument: <char>
603     # return: TeX macro for Hanziku fonts or undef.
604     my($char)=@_;
605     my($hzk,$HZK);
606     foreach (@HZK){
607         if($hzk=&get_char_attribute($char,$_)){
608             m/hanziku\-(\d+)/ and $HZK=$1;
609             last;
610         }
611     }
612     if($hzk){
613         return "{\\fontencoding{OT1}\\fontfamily{".sprintf("hzk%02d",$HZK)."}\\selectfont\\char".$hzk."}";
614     }else{
615         return undef;
616     }
617 }
618
619 sub get_macro_for_CDP{
620     # argument: <char>
621     # return: TeX macro for CDP fonts or undef.
622     my($char)=@_;
623     my($cdp,$ucs);
624     foreach (@CDP){
625         if($cdp=&get_char_attribute($char,$_)){
626             last;
627         }
628     }
629     if($cdp){
630         $ucs=&get_char_attribute(&get_chars_matching("=big5-pua",$cdp),"=ucs");
631         if($ucs){
632             return "{\\fontencoding{OT1}\\fontfamily{cdp}\\selectfont\\char"
633                 .$ucs.
634                     "}";
635         }else{
636             print STDERR "This should not happen.\n";
637             print STDERR "ucs code point of CDP: $cdp not found.\n";
638         }
639     }else{
640         return undef;
641     }
642 }