1 #!/usr/bin/perl -w -CSD
6 use vars qw($omegadb_path
7 $opt_protrude $opt_allow_unify
8 $opt_use_kage_for_Ext_B
9 %opt_order %order %order_map
10 $opt_in_cs $opt_out_cs
13 $in_cs $out_cs $i @chars
14 $char $char_id $out_char
15 $char_unified @chars_unified
16 $ids $ids_argc %ids $idsdb $geta
17 $idsdata_file $ids_start $font_start
23 use Chise_utils ':all';
28 $opt_order{'UniMulti'}='jGcgk';
29 #$opt_order{'UniMulti'}='jGcgkHC';
30 $opt_order{'UniCNS'}='c';
31 $opt_order{'UniGB'}='g';
32 $opt_order{'UniJIS'}='j';
33 $opt_order{'UniKS'}='k';
35 $opt_allow_unify=1; # 1=true, 0=false.
36 $opt_protrude=0;# 1=true, 0=false.
38 # currently does not work.
39 $opt_use_kage_for_Ext_B=0;# 1=true, 0=false.
43 my $strictly_forbidden_after = '「【『[(〈‘‛“‟(〔{《{\[\(〖{「';
45 my $forbidden_after = "\x{0000}";
48 my $slightly_forbidden_after = '¥¥$$〒♯##¢¢££@@§';
50 # $strictly_forbidden_before
51 # All these characters are allowed to protrude
53 my $strictly_forbidden_before=
54 '!,.:;?、。!,.:;?。\)\]}’”〉》」』】〕〗)]}」';
57 = 'ー々ぁぃぅぇぉゃゅょっゎァィゥェォャュョッヮヵヶ';
59 my $slightly_forbidden_before
60 = '\x{000a}\#\-‐−‰′″℃゛゜ゝゞヽヾ"%-゙゚';
62 #my $asian = '\x{1100}-\x{11FF}\x{2E80}-\x{D7AF}\x{F900}-\x{FAFF}\x{FE30}-\x{FE4F}\x{FF00}-\x{FFFFFF}'; # need to be fixed.
63 my $asian = '\x{2E80}-\x{312f}\x{3190}-\x{ABFF}\x{F900}-\x{FAFF}\x{FE30}-\x{FE4F}\x{FF00}-\x{FFFFFF}'; # need to be fixed.
65 my $space = '\x{0020}\x{0009}\x{000A}\x{000C}\x{000D}';
67 my $hangul='\x{1100}-\x{11ff}\x{3130}-\x{318f}\x{AC00}-\x{D7AF}';
69 my %tex_meta=('#'=>'\#',
70 '$'=>'\\textdollar{}',
73 '{'=>'\\textbraceleft{}',
74 '}'=>'\\textbraceright{}',
75 '\\'=>'\\textbackslash{}',
76 '_'=>'\\textunderscore',
79 my $tex_meta_re=join('|',map {quotemeta($_)} keys %tex_meta);
81 &GetOptions("in=s"=>\$opt_in_cs,
82 "out=s"=>\$opt_out_cs,
86 Usage: $0 -i <input coding system> -o <cmap encoding>
88 Utf8mcs, Utf8cns, Utf8gb, Utf8jis, Utf8ks, Utf8big5
90 UniCNS, UniGB, UniJIS, UniKS, UniMulti
93 if($opt_in_cs or $opt_out_cs){
99 ,$opt_preserve_spaces)
100 =($0=~/(Utf8mcs|Utf8cns|Utf8gb|Utf8jis|Utf8ks|Utf8big5)
101 To (UniCNS|UniGB|UniJIS|UniKS|UniMulti)
106 # Utf8mcs,Utf8cns,Utf8gb,Utf8jis,Utf8ks,Utf8big5
108 # UniCNS,UniGB,UniJIS,UniKS,UniMulti
110 $in_cs=~s/Utf8/ucs\@/;
113 or not defined($in_cs)
114 or not defined($out_cs)){
119 $omegadb_path=~s!/$!!;
121 $idsdata_file="$omegadb_path/idsdata.txt";
125 if(-e $idsdata_file){
126 open(IDSDATA,"+<:utf8",$idsdata_file) or die;
127 flock(IDSDATA,LOCK_EX);
131 if(m/^START\t(\d+)\t(\d+)/){
132 $font_start=$1,$ids_start=$2;
133 }elsif(m/^(.*?)\t(\d+)\t(\d+)/){
136 die "Irregular IDS file: $idsdata_file.\n";
142 open(IDSDATA,">:utf8",$idsdata_file) or die;
143 flock(IDSDATA,LOCK_EX);
150 #$geta=pack("U",0x3013);
151 $geta=pack("U",0xfffd);
154 "=gt-pj-1","=gt-pj-2","=gt-pj-3","=gt-pj-4","=gt-pj-5",
155 "=gt-pj-6","=gt-pj-7","=gt-pj-8","=gt-pj-9","=gt-pj-10",
157 #,"=gt-pj-k1","=gt-pj-k2"
159 @HZK=("=hanziku-1","=hanziku-2","=hanziku-3","=hanziku-4",
160 "=hanziku-5","=hanziku-6","=hanziku-7","=hanziku-8",
161 "=hanziku-9","=hanziku-10","=hanziku-11","=hanziku-12");
164 %order_map=('c'=>'UniCNS',
173 foreach $out_cs ('UniCNS','UniGB','UniJIS','UniKS','UniMulti'){
174 if(defined($opt_order{$out_cs})){
175 if($opt_order{$out_cs}=~/^[cgjkGHC]+$/){
176 @{$order{$out_cs}}=map {$order_map{$_}}
177 (split(//,$opt_order{$out_cs}));
179 print STDERR "Invalid order for $out_cs!\n";
192 if($in_cs ne 'ucs@mcs'){
193 s/(.)/&get_char_in_mcs($1,$in_cs)/ge;
195 s/(amp.+?;)/&de_tex_er($1)/ge;
196 # s/(&.+?;)/&de_tex_er($1)/ge;
197 s/([$asian])[$space]+/$1/go unless($opt_preserve_spaces);
198 s/[$space]+([$asian])/$1/go unless($opt_preserve_spaces);
199 s/([$asian])\s*([^$asian$space])/$1 $2/go;
200 s/([^$asian$idc])\s*([$asian])/$1 $2/go;
201 s/\-\-\-/pack("U",0x2014)/geo;# EM DASH
202 s/\-\-/pack("U",0x2013)/geo;# EN DASH
203 # s/\`\`/pack("U",0x201f)/geo;# DOUBLE HIGH-REVERSED-9 QUOTATION MARK
204 s/\`\`/pack("U",0x201c)/geo;# _TeX compatible_
205 # s/\`/pack("U",0x201b)/geo;# SINGLE HIGH-REVERSED-9 QUOTATION MARK
206 s/\`/pack("U",0x2018)/geo;# _TeX compatible_
207 s/\'\'/pack("U",0x201d)/geo;# RIGHT DOUBLE QUOTATION MARK
208 s/\'/pack("U",0x2019)/geo;# RIGHT DOUBLE QUOTATION MARK
212 for($i=0;$i<=$#chars;$i++){
214 if($char_id=&get_char_attribute($char,'=ucs@unicode')){
215 $char=pack("U",$char_id);
217 $char_id=unpack("U",$char);
221 # add quarter space between asian and non-asian.
222 if($i>0 and $i<$#chars){
223 if(($chars[$i-1]=~/[$asian]/
224 and $chars[$i+1]=~/[^$asian]/
225 and $chars[$i+1]=~/[$strictly_forbidden_before$slightly_forbidden_before]/)
226 or($chars[$i-1]=~/[^$asian]/
227 and $chars[$i+1]=~/[$asian]/
228 and $chars[$i-1]=~/[$strictly_forbidden_after$slightly_forbidden_after]/)
231 }elsif(($chars[$i-1]=~/[$asian]/
232 and $chars[$i+1]=~/[^$asian]/)){
233 print '\unskip\kern.25ex';
234 }elsif($chars[$i-1]=~/[^$asian]/
235 and $chars[$i+1]=~/[$asian]/){
244 }elsif(($char_id>0x20 and $char_id<=0x021f)
248 # Latin Extended-B (not all)
249 or($char_id>=0x0250 and $char_id<=0x02af)
251 or($char_id>=0x0300 and $char_id<=0x033f)
252 or($char_id>=0x0360 and $char_id<=0x036f)
253 # Combining Diacritical Marks
254 or($char_id>=0x1e00 and $char_id<=0x1eff)
255 # Latin Extended Additional
256 or($char_id>=0x0370 and $char_id<=0x03ff)
258 or($char_id>=0x0400 and $char_id<=0x04ff)
260 or($char_id>=0x0530 and $char_id<=0x058f)
262 or($char_id>=0x2010 and $char_id<=0x2046)
263 # General Punctuation (partial)
265 print &latin_parse();
267 }elsif($char_id>=0x2ff0 and $char_id<=0x2fff){
268 # Ideographic Description Characters
272 if(($out_char=&get_output_char($char,$out_cs))){
273 print $out_char,&add_break($i);
275 if($opt_allow_unify){
276 @chars_unified=&get_chars_unified($char);
277 if(@chars_unified>0){
278 foreach $char_unified (@chars_unified){
280 =&get_output_char($char_unified,$out_cs))){
281 print $out_char,&add_break($i);
287 if($opt_use_kage_for_Ext_B){
288 if($char_id >= 0x20000 && $char_id <=0x2a6df){
289 # CJK Unified Ideographs Extension B
290 if(not defined($ids{$char}) and $ids{$char}[1]>=0){
291 $ids{$char}[0]=$font_start;
292 $ids{$char}[1]=$ids_start;
299 print "{\\fontencoding{OT1}\\fontfamily{",
300 sprintf("chise%03d",$ids{$char}[0]) ,
301 "}\\selectfont\\char$ids{$char}[1]}",&add_break($i);
305 if($ids=&get_ids($char)){
306 print &get_macro_for_ids($ids),&add_break($i);
308 print '\rule{1ex}{1ex}',&add_break($i);
316 print IDSDATA 'START',"\t",$font_start,"\t",$ids_start,"\n";
317 foreach $ids (keys %ids){
318 print IDSDATA $ids,"\t",join("\t",@{$ids{$ids}}),"\n";
320 flock(IDSDATA,LOCK_UN);
327 my($output_char,$atr,$value);
328 $er=~/^(amp)(.*)(;)$/
329 and $prefix=$1,$er=$2,$suffix=$3;
330 $prefix or $prefix="",$suffix or $suffix="";
331 if($er=~/^U[\+|\-]([a-fA-F\d]+)/){
332 $output_char=pack("U",hex($1));
333 }elsif($er=~/^(?:I\-)?($er_prefix_re)\-?([0-9a-fA-F]+)$/){
334 ($atr,$value)=($1,$2);
335 unless($er_alias{$atr}=~/daikanwa|gt/){
338 ($output_char)=&get_chars_matching($er_alias{$atr},$value);
343 return $prefix.$er.$suffix;
351 if(($chars[$i+1]=~m/[$strictly_forbidden_before]/o)
352 and($chars[$i+2]=~m/[$strictly_forbidden_before]/o)){
353 return "\\CJKunbreakablekernone{}";
354 }elsif($opt_protrude){
355 if(($chars[$i+1]=~m/[$strictly_forbidden_before]/o)
356 and($chars[$i+2]=~m/[^$strictly_forbidden_before]/o)){
357 return "\\CJKunbreakablekernone \\CJKprotrude{}";
362 and($chars[$i+1]eq" ")){
367 and($chars[$i+1]=~m/[$strictly_forbidden_before]/o)){
368 return "\\CJKunbreakablekernone{}";
370 if($chars[$i]=~m/[$strictly_forbidden_after]/o){
371 return "\\CJKunbreakablekernone{}";
374 and($chars[$i+1]=~m/[$forbidden_before]/o)){
375 return "\\CJKunbreakablekerntwo{}";
378 if($chars[$i]=~m/[$forbidden_after]/o){
379 return "\\CJKunbreakablekerntwo{}";
382 and($chars[$i+1]=~m/[$slightly_forbidden_before]/o)){
383 return "\\CJKunbreakablekernthree{}";
385 if($chars[$i]=~m/[$slightly_forbidden_after]/o){
386 return "\\CJKunbreakablekernthree{}";
388 if($chars[$i]=~m/[$asian]/o){
389 return "\\CJKbreakablekern{}";
391 if(($i<$#chars)and($chars[$i+1]=~m/[$asian]/o)){
392 return "\\CJKbreakablekern{}";
398 # return: string for output with TeX macro.
402 $char_id=unpack("U",$chars[$i]);
403 if($chars[$i]=~m/($tex_meta_re)/o){
404 $out_str.=$tex_meta{$chars[$i]};
405 }elsif(($char_id>0x20 and $char_id<=0x021f)
406 or($char_id>=0x0250 and $char_id<=0x02af)# IPA Extensions
407 or($char_id>=0x0300 and $char_id<=0x033f)# Combining Diacritical Marks
408 or($char_id>=0x0360 and $char_id<=0x036f)
409 or($char_id>=0x0370 and $char_id<=0x03ff)# Greek and Coptic
410 or($char_id>=0x0400 and $char_id<=0x04ff)# Cyrillic
411 or($char_id>=0x0530 and $char_id<=0x058f)# Armenian
413 $out_str.=$chars[$i];
414 }elsif($char_id>=0x1e00 and $char_id<=0x1eff){
415 # Latin Extended Additional
416 # 0x1e00 -> 0x0600, etc.
417 $out_str.=pack("U",$char_id-0x1800);
418 }elsif($char_id>=0x2010 and $char_id<=0x2046){
419 # General Punctuation (partial)
420 $out_str.=pack("U",$char_id-0x1000);
427 return '{\fontencoding{OT1}\fontfamily{omlgc}\selectfont '.$out_str.'}';###UT1?
432 # return: character for output,
434 # or GETA character if ids is invalid.
435 my($ids,$ids_argc)=&ids_rest("",0,$chars[$i]);
441 print STDERR "IDS parse error: $ids\n";
445 ($ids,$ids_argc)=&ids_rest($ids,$ids_argc,$chars[$i]);
447 if(($char=&get_char_for_ids($ids))
448 and($out_char=&get_output_char($char,$out_cs))){
451 return &get_macro_for_ids($ids);
458 # arguments: <ids>, <rest number of arguments for ids>, <character>
459 # return: ids and rest number of arguments for ids.
460 my($ids,$ids_argc,$char)=@_;
462 $argc=&ids_argc($char);
464 $ids_argc+=$ids_argc==0?$argc:$argc-1;
469 return ($ids,$ids_argc);
472 sub get_macro_for_ids{
474 # return: TeX macro for ids
475 # or GETA character if ids is invalid for KAGE.
477 # $ids=&normalize_ids($ids,"UniJIS");
478 return $geta if(($ids!~/[$idc]/)
479 or($ids=~/[\x{10000}-]/));
481 if(not defined($ids{$ids})){
482 $ids{$ids}[0]=$font_start;
483 $ids{$ids}[1]=$ids_start;
490 return "{\\fontencoding{OT1}\\fontfamily{"
491 .sprintf("chise%03d",$ids{$ids}[0])
492 ."}\\selectfont\\char$ids{$ids}[1]}\\relax{}";
496 # argument: <ids>, <output coding system>
497 # return: ids or GETA character if ids is invalid for KAGE.
499 $out_cs=~s/Uni(.+)/'ucs@'.lc($1)/e;
502 my($char,$output_char_id);
503 while($ids=~m/(.)/g){
507 }elsif($output_char_id=&get_char_attribute($char,"=$out_cs")
508 or $output_char_id=&get_char_attribute($char,"=ucs")
509 or $output_char_id=&get_char_attribute($char,"=>$out_cs")
510 or $output_char_id=&get_char_attribute($char,"=>ucs")
511 or $output_char_id=&get_char_attribute($char,"=>ucs*")
513 $output_ids.=pack("U",$output_char_id);
523 # return: character in output coding system or TeX macro or undef.
524 my($char,$out_cs)=@_;
525 my($out_char_id,$suffix);
528 foreach $out_cs (@{$order{$out_cs}}){
529 if($out_cs eq 'UniJIS'
530 and &get_char_attribute($char,"vnd-adobe-cid-unijis-utf16-h")){
531 if($out_char_id=&get_char_attribute($char,'=ucs@jis')
532 or $out_char_id=&get_char_attribute($char,'=ucs')
533 or $out_char_id=&get_char_attribute($char,'=>ucs@jis')
534 or $out_char_id=&get_char_attribute($char,'=>ucs')
535 or $out_char_id=&get_char_attribute($char,'=>ucs*')
537 return "{\\selectjisfont\\char$out_char_id}";
539 }elsif($out_cs eq 'UniGB'
540 and &get_char_attribute($char,"vnd-adobe-cid-unigb-ucs2-h")){
541 if($out_char_id=&get_char_attribute($char,'=ucs@gb')
542 or $out_char_id=&get_char_attribute($char,'=ucs')
543 or $out_char_id=&get_char_attribute($char,'=>ucs@gb')
544 or $out_char_id=&get_char_attribute($char,'=>ucs')
545 or $out_char_id=&get_char_attribute($char,'=>ucs*')
547 return "{\\selectgbsfont\\char$out_char_id}";
549 }elsif($out_cs eq 'UniCNS'
550 and &get_char_attribute($char,"vnd-adobe-cid-unicns-ucs2-h")){
551 if($out_char_id=&get_char_attribute($char,'=ucs@cns')
552 or $out_char_id=&get_char_attribute($char,'=ucs')
553 or $out_char_id=&get_char_attribute($char,'=>ucs@cns')
554 or $out_char_id=&get_char_attribute($char,'=>ucs')
555 or $out_char_id=&get_char_attribute($char,'=>ucs*')
557 return "{\\selectcnsfont\\char$out_char_id}";
559 }elsif($out_cs eq 'UniKS'
560 and &get_char_attribute($char,"vnd-adobe-cid-uniks-ucs2-h")){
561 if($out_char_id=&get_char_attribute($char,'=ucs@ks')
562 or $out_char_id=&get_char_attribute($char,'=ucs')
563 or $out_char_id=&get_char_attribute($char,'=>ucs@ks')
564 or $out_char_id=&get_char_attribute($char,'=>ucs')
565 or $out_char_id=&get_char_attribute($char,'=>ucs*')
567 return "{\\selectksxfont\\char$out_char_id}";
569 }elsif($out_cs eq 'GT'){
570 return $gt if($gt=&get_macro_for_GT($char));
571 }elsif($out_cs eq 'HZK'){
572 return $hzk if($hzk=&get_macro_for_HZK($char));
573 }elsif($out_cs eq 'CDP'){
574 return $cdp if($cdp=&get_macro_for_CDP($char));
581 # argument: <character>
585 # $ids=&get_char_attribute($char,"ids-aggregated")
586 # or $ids=&get_char_attribute($char,"ids");
587 $ids=&get_char_attribute($char,"ids-decomposed")
588 or $ids=&get_char_attribute($char,"ids")
589 or $ids=&get_char_attribute($char,"ideographic-structure");
594 sub get_char_for_ids{
595 # argument: <ideographic description sequence>
596 # return: char or undef.
600 if(($output_char)=&get_chars_matching("ids",$ids)){
608 # argument: <char>, <input coding system>
609 # return: char in ucs@mcs.
613 return $char if($in_cs eq 'ucs@mcs' or
616 if(($output_char)=&get_chars_matching("=$in_cs",unpack("U",$char))){
623 sub get_chars_unified{
625 my($chars,$ucs,$char_ucs,$char_sub);
628 if($chars=&get_char_attribute($char,'->ucs-unified')){
629 $chars=~s/^\((.*)\)$/$1/;
630 return (split(/\s*\?/,$chars));
631 }elsif($ucs=&get_char_attribute($char,'=>ucs*')
632 or $ucs=&get_char_attribute($char,'=>ucs')){
633 $char_ucs=pack("U",$ucs);
634 if($chars=&get_char_attribute($char_ucs,'->ucs-unified')){
635 $chars=~s/^\((.*)\)$/$1/;
636 @chars=grep {not /^$char$/}
637 (split(/\s*\?/,$chars));
638 push(@chars,$char_ucs);
641 }elsif($char_sub=&get_char_attribute($char,'<-subsumptive')){
642 $char_sub=~s/[? ()]//g;
643 $chars=&get_char_attribute($char_sub,'->subsumptive');
644 @chars=grep {not /^$char$/} (split(/\s*\?/,$chars));
645 push(@chars,$char_sub);
652 sub get_macro_for_GT{
654 # return: TeX macro for GT fonts or undef.
658 if($gt=&get_char_attribute($char,$_)){
659 m/gt\-pj\-(\d+)/ and $GT=$1;
664 return "{\\fontencoding{OT1}\\fontfamily{"
665 .sprintf("gt%02d",$GT)
666 ."}\\selectfont\\char".($gt|0x8080)."}";
672 sub get_macro_for_HZK{
674 # return: TeX macro for Hanziku fonts or undef.
678 if($hzk=&get_char_attribute($char,$_)){
679 m/hanziku\-(\d+)/ and $HZK=$1;
684 return "{\\fontencoding{OT1}\\fontfamily{"
685 .sprintf("hzk%02d",$HZK)
686 ."}\\selectfont\\char".$hzk."}";
692 sub get_macro_for_CDP{
694 # return: TeX macro for CDP fonts or undef.
698 if($cdp=&get_char_attribute($char,$_)){
703 $ucs=&get_char_attribute(&get_chars_matching("=big5-pua",$cdp),"=ucs");
705 return "{\\fontencoding{OT1}\\fontfamily{cdp}\\selectfont\\char$ucs}";
707 print STDERR "This should not happen.\n";
708 print STDERR "ucs code point of CDP: $cdp not found.\n";