1 #!/usr/bin/perl -w -CSD
6 use vars qw($omegadb_path
7 $opt_protrude $opt_allow_unify
8 $opt_use_kage_for_Ext_B
9 %opt_order %order %order_map
10 $opt_in_cs $opt_out_cs
12 $in_cs $out_cs $i @chars
13 $char $char_id $out_char
14 $char_unified @chars_unified
15 $ids $ids_argc %ids $idsdb $geta
16 $idsdata_file $ids_start $font_start
22 use Chise_utils ':all';
27 #$opt_order{'UniMulti'}='jcgk';
28 $opt_order{'UniMulti'}='jGcgkHC';
29 $opt_order{'UniCNS'}='c';
30 $opt_order{'UniGB'}='g';
31 $opt_order{'UniJIS'}='j';
32 $opt_order{'UniKS'}='k';
34 $opt_allow_unify=1; # 1=true, 0=false.
35 $opt_protrude=0;# 1=true, 0=false.
37 # currently does not work.
38 $opt_use_kage_for_Ext_B=0;# 1=true, 0=false.
42 my $strictly_forbidden_after = '「【『[(〈“‘‘(〔{《{\[\(\x{3016}{「';
43 # \x{3016} | # white 【
45 my $forbidden_after = "\x{0000}";
48 my $slightly_forbidden_after = '¥¥$$〒♯##¢¢££@@§';
50 # $strictly_forbidden_before
51 # All these characters are allowed to protrude
53 my $strictly_forbidden_before=
54 '!,.:;?、。!,.:;?。\)#}’”〉》」』】〕\x{3017})]}}」\]';
55 ### \x{3017} | # white 】
58 = 'ー々ぁぃぅぇぉゃゅょっゎァィゥェォャュョッヮヵヶ';
60 my $slightly_forbidden_before
61 = '\x{000a}\#\-‐−‰′″℃゛゜ゝゞヽヾ"%-゙゚';
63 my $asian = '\x{1100}-\x{11FF}\x{2E80}-\x{D7AF}\x{F900}-\x{FAFF}\x{FE30}-\x{FE4F}\x{FF00}-\x{FFFFFF}'; # need to fix.
65 my $space = '\x{0020}\x{0009}\x{000A}\x{000C}\x{000D}';
67 my %tex_meta=('#'=>'\#',
68 '$'=>'\\textdollar{}',
71 '{'=>'\\textbraceleft{}',
72 '}'=>'\\textbraceright{}',
73 '\\'=>'\\textbackslash{}',
74 '_'=>'\\textunderscore',
77 my $tex_meta_re=join('|',map {quotemeta($_)} keys %tex_meta);
79 &GetOptions("in=s"=>\$opt_in_cs,
80 "out=s"=>\$opt_out_cs,
84 Usage: $0 -i <input coding system> -o <cmap encoding>
86 Utf8mcs, Utf8cns, Utf8gb, Utf8jis, Utf8ks, Utf8big5
88 UniCNS, UniGB, UniJIS, UniKS, UniMulti
91 if($opt_in_cs or $opt_out_cs){
95 ($in_cs,$out_cs)=($0=~/(Utf8.+)To(\w+)/);
99 # Utf8mcs,Utf8cns,Utf8gb,Utf8jis,Utf8ks,Utf8big5
101 # UniCNS,UniGB,UniJIS,UniKS,UniMulti
103 $in_cs=~s/Utf8/ucs\@/;
106 or not defined($in_cs)
107 or not defined($out_cs)){
112 $omegadb_path=~s!/$!!;
114 $idsdata_file="$omegadb_path/idsdata.pl";
118 if(-e $idsdata_file){
119 open(IDSDATA,"+<:utf8",$idsdata_file) or die;
120 flock(IDSDATA,LOCK_EX);
126 # require $idsdata_file;
128 open(IDSDATA,">:utf8",$idsdata_file) or die;
129 flock(IDSDATA,LOCK_EX);
136 #$geta=pack("U",0x3013);
137 $geta=pack("U",0xfffd);
140 "=gt-pj-1","=gt-pj-2","=gt-pj-3","=gt-pj-4","=gt-pj-5",
141 "=gt-pj-6","=gt-pj-7","=gt-pj-8","=gt-pj-9","=gt-pj-10",
143 #,"=gt-pj-k1","=gt-pj-k2"
145 @HZK=("=hanziku-1","=hanziku-2","=hanziku-3","=hanziku-4",
146 "=hanziku-5","=hanziku-6","=hanziku-7","=hanziku-8",
147 "=hanziku-9","=hanziku-10","=hanziku-11","=hanziku-12");
150 %order_map=('c'=>'UniCNS',
159 foreach $out_cs ('UniCNS','UniGB','UniJIS','UniKS','UniMulti'){
160 if(defined($opt_order{$out_cs})){
161 if($opt_order{$out_cs}=~/^[cgjkGHC]+$/){
162 @{$order{$out_cs}}=map {$order_map{$_}}
163 (split(//,$opt_order{$out_cs}));
165 print STDERR "Invalid order for $out_cs!\n";
174 if($in_cs ne 'ucs@mcs'){
175 s/(.)/&get_char_in_mcs($1,$in_cs)/ge;
177 s/([$asian])\s+([$asian])/$1$2/g unless($out_cs eq 'UniKS');
178 s/([$asian])\s*([^$asian])/$1 $2/g;
179 s/([^$asian])\s*([$asian])/$1 $2/g;
180 s/\-\-\-/pack("U",0x2014)/geo;# EM DASH
181 s/\-\-/pack("U",0x2013)/geo;# EN DASH
182 s/\`\`/pack("U",0x201f)/geo;# DOUBLE HIGH-REVERSED-9 QUOTATION MARK
183 s/\`/pack("U",0x201b)/geo;# SINGLE HIGH-REVERSED-9 QUOTATION MARK
184 s/\'\'/pack("U",0x201d)/geo;# RIGHT DOUBLE QUOTATION MARK
185 s/\'/pack("U",0x2019)/geo;# RIGHT DOUBLE QUOTATION MARK
186 s/(amp.+?;)/&de_tex_er($1)/ge;
187 # s/(&.+?;)/&de_tex_er($1)/ge;
190 for($i=0;$i<=$#chars;$i++){
192 $char_id=unpack("U",$char);
195 # add quarter space between asian and non-asian.
196 if($i>0 and $i<$#chars){
197 if(($chars[$i-1]=~/[$asian]/
198 and $chars[$i+1]=~/[^$asian]/
199 and $chars[$i+1]=~/[^$strictly_forbidden_after$slightly_forbidden_after]/)
200 or($chars[$i-1]=~/[^$asian]/
201 and $chars[$i+1]=~/[$asian]/
202 and $chars[$i-1]=~/[^$strictly_forbidden_before$slightly_forbidden_before]/)
205 }elsif(($chars[$i-1]=~/[$asian]/
206 and $chars[$i+1]=~/[^$asian]/)
207 or($chars[$i-1]=~/[^$asian]/
208 and $chars[$i+1]=~/[$asian]/)){
209 print '{\selectjisfont\hspace{.25ex}}';
211 print '{\fontencoding{OT1}\fontfamily{omlgc}\selectfont{} }';###UT1?
215 }elsif($char=~m/($tex_meta_re)/o){
218 }elsif(($char_id>0x20 and $char_id<=0x021f)
222 # Latin Extended-B (not all)
223 or($char_id>=0x0250 and $char_id<=0x02af)
225 or($char_id>=0x0300 and $char_id<=0x033f)
226 or($char_id>=0x0360 and $char_id<=0x036f)
227 # Combining Diacritical Marks
228 or($char_id>=0x1e00 and $char_id<=0x1eff)
229 # Latin Extended Additional
230 or($char_id>=0x0370 and $char_id<=0x03ff)
232 or($char_id>=0x0400 and $char_id<=0x04ff)
234 or($char_id>=0x0530 and $char_id<=0x058f)
236 or($char_id>=0x2010 and $char_id<=0x2046)
237 # General Punctuation (partial)
239 print &latin_parse();
241 }elsif($char_id>=0x2ff0 and $char_id<=0x2fff){
242 # Ideographic Description Characters
246 if(($out_char=&get_output_char($char,$out_cs))){
247 print $out_char,&add_break($i);
249 if($opt_allow_unify){
250 @chars_unified=&get_chars_unified($char);
251 if(@chars_unified>0){
252 foreach $char_unified (@chars_unified){
254 =&get_output_char($char_unified,$out_cs))){
255 print $out_char,&add_break($i);
261 if($opt_use_kage_for_Ext_B){
262 if($char_id >= 0x20000 && $char_id <=0x2a6df){
263 # CJK Unified Ideographs Extension B
264 if(not defined($ids{$char}) and $ids{$char}[1]>=0){
265 $ids{$char}[0]=$font_start;
266 $ids{$char}[1]=$ids_start;
273 print "{\\fontencoding{OT1}\\fontfamily{" .
274 sprintf("chise%03d",$ids{$char}[0]) .
275 "}\\selectfont\\char$ids{$char}[1]}",&add_break($i);
279 if($ids=&get_ids($char)){
280 print &get_macro_for_ids($ids),&add_break($i);
282 print '\rule{1ex}{1ex}',&add_break($i);
289 print IDSDATA 'use utf8;',"\n";
290 foreach $ids (keys %ids){
291 print IDSDATA '$ids{\'',$ids,'\'}='
292 ,'[',join ",",@{$ids{$ids}},"];\n";
294 print IDSDATA '$font_start=',$font_start,";\n";
295 print IDSDATA '$ids_start=',$ids_start,";\n";
297 flock(IDSDATA,LOCK_UN);
304 my($output_char,$atr,$value);
305 $er=~/^(amp)(.*)(;)$/
306 and $prefix=$1,$er=$2,$suffix=$3;
307 $prefix or $prefix="",$suffix or $suffix="";
308 if($er=~/^U[\+|\-]([a-fA-F\d]+)/){
309 $output_char=pack("U",hex($1));
310 }elsif($er=~/^(?:I\-)?($er_prefix_re)\-?([0-9a-fA-F]+)$/){
311 ($atr,$value)=($1,$2);
312 unless($er_alias{$atr}=~/daikanwa|gt/){
315 ($output_char)=&get_chars_matching($er_alias{$atr},$value);
320 return $prefix.$er.$suffix;
328 if(($chars[$i+1]=~m/[$strictly_forbidden_before]/o)
329 and($chars[$i+2]=~m/[$strictly_forbidden_before]/o)){
330 return "\\CJKunbreakablekernone ";
331 }elsif($opt_protrude){
332 if(($chars[$i+1]=~m/[$strictly_forbidden_before]/o)
333 and($chars[$i+2]=~m/[^$strictly_forbidden_before]/o)){
334 return "\\CJKunbreakablekernone \\CJKprotrude ";
339 and($chars[$i+1]eq" ")){
344 and($chars[$i+1]=~m/[$strictly_forbidden_before]/o)){
345 return "\\CJKunbreakablekernone ";
347 if($chars[$i]=~m/[$strictly_forbidden_after]/o){
348 return "\\CJKunbreakablekernone ";
351 and($chars[$i+1]=~m/[$forbidden_before]/o)){
352 return "\\CJKunbreakablekerntwo ";
355 if($chars[$i]=~m/[$forbidden_after]/o){
356 return "\\CJKunbreakablekerntwo ";
359 and($chars[$i+1]=~m/[$slightly_forbidden_before]/o)){
360 return "\\CJKunbreakablekernthree ";
362 if($chars[$i]=~m/[$slightly_forbidden_after]/o){
363 return "\\CJKunbreakablekernthree ";
365 if($chars[$i]=~m/[$asian]/o){
366 return "\\CJKbreakablekern ";
368 if(($i<$#chars)and($chars[$i+1]=~m/[$asian]/o)){
369 return "\\CJKbreakablekern ";
375 # return: string for output with TeX macro.
379 $char_id=unpack("U",$chars[$i]);
380 if(($char_id>0x20 and $char_id<=0x021f)
381 or($char_id>=0x0250 and $char_id<=0x02af)# IPA Extensions
382 or($char_id>=0x0300 and $char_id<=0x033f)# Combining Diacritical Marks
383 or($char_id>=0x0360 and $char_id<=0x036f)
384 or($char_id>=0x0370 and $char_id<=0x03ff)# Greek and Coptic
385 or($char_id>=0x0400 and $char_id<=0x04ff)# Cyrillic
386 or($char_id>=0x0530 and $char_id<=0x058f)# Armenian
388 $out_str.=$chars[$i];
389 }elsif($char_id>=0x1e00 and $char_id<=0x1eff){
390 # Latin Extended Additional
391 # 0x1e00 -> 0x0600, etc.
392 $out_str.=pack("U",$char_id-0x1800);
393 }elsif($char_id>=0x2010 and $char_id<=0x2046){
394 # General Punctuation (partial)
395 $out_str.=pack("U",$char_id-0x1000);
402 return '{\fontencoding{OT1}\fontfamily{omlgc}\selectfont '.$out_str.'}';###UT1?
407 # return: character for output,
409 # or GETA character if ids is invalid.
410 my($ids,$ids_argc)=&ids_rest("",0,$chars[$i]);
416 print STDERR "IDS parse error: $ids\n";
420 ($ids,$ids_argc)=&ids_rest($ids,$ids_argc,$chars[$i]);
422 if(($char=&get_char_for_ids($ids))
423 and($out_char=&get_output_char($char,$out_cs))){
426 return &get_macro_for_ids($ids);
433 # arguments: <ids>, <rest number of arguments for ids>, <character>
434 # return: ids and rest number of arguments for ids.
435 my($ids,$ids_argc,$char)=@_;
437 $argc=&ids_argc($char);
439 $ids_argc+=$ids_argc==0?$argc:$argc-1;
444 return ($ids,$ids_argc);
447 sub get_macro_for_ids{
449 # return: TeX macro for ids
450 # or GETA character if ids is invalid for KAGE.
452 # $ids=&normalize_ids($ids,"UniJIS");
453 return $geta if(($ids!~/[$idc]/)
454 or($ids=~/[\x{10000}-]/));
456 if(not defined($ids{$ids})){
457 $ids{$ids}[0]=$font_start;
458 $ids{$ids}[1]=$ids_start;
465 return "{\\fontencoding{OT1}\\fontfamily{"
466 .sprintf("chise%03d",$ids{$ids}[0])
467 ."}\\selectfont\\char$ids{$ids}[1]}";
471 # argument: <ids>, <output coding system>
472 # return: ids or GETA character if ids is invalid for KAGE.
474 $out_cs=~s/Uni(.+)/'ucs@'.lc($1)/e;
477 my($char,$output_char_id);
478 while($ids=~m/(.)/g){
482 }elsif($output_char_id=&get_char_attribute($char,"=$out_cs")
483 or $output_char_id=&get_char_attribute($char,"=ucs")
484 or $output_char_id=&get_char_attribute($char,"=>$out_cs")
485 or $output_char_id=&get_char_attribute($char,"=>ucs")
486 or $output_char_id=&get_char_attribute($char,"=>ucs*")
488 $output_ids.=pack("U",$output_char_id);
498 # return: character in output coding system or TeX macro or undef.
499 my($char,$out_cs)=@_;
500 my($out_char_id,$suffix);
503 foreach $out_cs (@{$order{$out_cs}}){
504 if($out_cs eq 'UniJIS'
505 and &get_char_attribute($char,"vnd-adobe-cid-unijis-utf16-h")){
506 if($out_char_id=&get_char_attribute($char,'=ucs@jis')
507 or $out_char_id=&get_char_attribute($char,'=ucs')
508 or $out_char_id=&get_char_attribute($char,'=>ucs@jis')
509 or $out_char_id=&get_char_attribute($char,'=>ucs')
510 or $out_char_id=&get_char_attribute($char,'=>ucs*')
512 return '{\selectjisfont\char'.$out_char_id.'}';
514 }elsif($out_cs eq 'UniGB'
515 and &get_char_attribute($char,"vnd-adobe-cid-unigb-ucs2-h")){
516 if($out_char_id=&get_char_attribute($char,'=ucs@gb')
517 or $out_char_id=&get_char_attribute($char,'=ucs')
518 or $out_char_id=&get_char_attribute($char,'=>ucs@gb')
519 or $out_char_id=&get_char_attribute($char,'=>ucs')
520 or $out_char_id=&get_char_attribute($char,'=>ucs*')
522 return '{\selectgbsfont\char'.$out_char_id.'}';
524 }elsif($out_cs eq 'UniCNS'
525 and &get_char_attribute($char,"vnd-adobe-cid-unicns-ucs2-h")){
526 if($out_char_id=&get_char_attribute($char,'=ucs@cns')
527 or $out_char_id=&get_char_attribute($char,'=ucs')
528 or $out_char_id=&get_char_attribute($char,'=>ucs@cns')
529 or $out_char_id=&get_char_attribute($char,'=>ucs')
530 or $out_char_id=&get_char_attribute($char,'=>ucs*')
532 return '{\selectcnsfont\char'.$out_char_id.'}';
534 }elsif($out_cs eq 'UniKS'
535 and &get_char_attribute($char,"vnd-adobe-cid-uniks-ucs2-h")){
536 if($out_char_id=&get_char_attribute($char,'=ucs@ks')
537 or $out_char_id=&get_char_attribute($char,'=ucs')
538 or $out_char_id=&get_char_attribute($char,'=>ucs@ks')
539 or $out_char_id=&get_char_attribute($char,'=>ucs')
540 or $out_char_id=&get_char_attribute($char,'=>ucs*')
542 return '{\selectksxfont\char'.$out_char_id.'}';
544 }elsif($out_cs eq 'GT'){
545 return $gt if($gt=&get_macro_for_GT($char));
546 }elsif($out_cs eq 'HZK'){
547 return $hzk if($hzk=&get_macro_for_HZK($char));
548 }elsif($out_cs eq 'CDP'){
549 return $cdp if($cdp=&get_macro_for_CDP($char));
556 # argument: <character>
560 $ids=&get_char_attribute($char,"ids-aggregated")
561 or $ids=&get_char_attribute($char,"ids");
562 # $ids=&get_char_attribute($char,"ids-decomposed")
563 # or $ids=&get_char_attribute($char,"ids");
564 # or $ids=&get_char_attribute($char,"ideographic-structure");
569 sub get_char_for_ids{
570 # argument: <ideographic description sequence>
571 # return: char or undef.
575 if(($output_char)=&get_chars_matching("ids",$ids)){
583 # argument: <char>, <input coding system>
584 # return: char in ucs@mcs.
588 return $char if($in_cs eq 'ucs@mcs');
590 if(($output_char)=&get_chars_matching("=$in_cs",unpack("U",$char))){
597 sub get_chars_unified{
599 my($chars,$ucs,$char_ucs);
602 if($chars=&get_char_attribute($char,'->ucs-unified')){
603 $chars=~s/^\((.*)\)$/$1/;
604 return (split(/\s*\?/,$chars));
605 }elsif($ucs=&get_char_attribute($char,'=>ucs*')
606 or $ucs=&get_char_attribute($char,'=>ucs')){
607 $char_ucs=pack("U",$ucs);
608 if($chars=&get_char_attribute($char_ucs,'->ucs-unified')){
609 $chars=~s/^\((.*)\)$/$1/;
610 @chars=grep {not /^$char$/}
611 (split(/\s*\?/,$chars));
612 push(@chars,$char_ucs);
618 sub get_macro_for_GT{
620 # return: TeX macro for GT fonts or undef.
624 if($gt=&get_char_attribute($char,$_)){
625 m/gt\-pj\-(\d+)/ and $GT=$1;
630 return "{\\fontencoding{OT1}\\fontfamily{"
631 .sprintf("gt%02d",$GT)
632 ."}\\selectfont\\char".($gt|0x8080)."}";
638 sub get_macro_for_HZK{
640 # return: TeX macro for Hanziku fonts or undef.
644 if($hzk=&get_char_attribute($char,$_)){
645 m/hanziku\-(\d+)/ and $HZK=$1;
650 return "{\\fontencoding{OT1}\\fontfamily{".sprintf("hzk%02d",$HZK)."}\\selectfont\\char".$hzk."}";
656 sub get_macro_for_CDP{
658 # return: TeX macro for CDP fonts or undef.
662 if($cdp=&get_char_attribute($char,$_)){
667 $ucs=&get_char_attribute(&get_chars_matching("=big5-pua",$cdp),"=ucs");
669 return "{\\fontencoding{OT1}\\fontfamily{cdp}\\selectfont\\char"
673 print STDERR "This should not happen.\n";
674 print STDERR "ucs code point of CDP: $cdp not found.\n";