X-Git-Url: http://git.chise.org/gitweb/?p=chise%2Fomega.git;a=blobdiff_plain;f=inCHISE;h=b06ef5e0f7fe367c065092a42611d998e60a3c97;hp=19a8eff1cb8e1623047fb53dfd807138adeb49e8;hb=beb2ec8be86f86571f37b2dbfcdac9a9f0184de0;hpb=080920cc4a9965db24a384b68e7a5c144317a755

diff --git a/inCHISE b/inCHISE
index 19a8eff..b06ef5e 100755
--- a/inCHISE
+++ b/inCHISE
@@ -8,6 +8,7 @@ use vars qw($omegadb_path
 	    $opt_use_kage_for_Ext_B
 	    %opt_order %order %order_map
 	    $opt_in_cs $opt_out_cs
+	    $opt_preserve_spaces
 	    $opt_help $usage
 	    $in_cs $out_cs $i @chars
 	    $char $char_id $out_char
@@ -22,18 +23,16 @@ use Fcntl ':flock';
 use Chise_utils ':all';
 require 5.008;
 
-my $omegadb_path="/usr/local/lib/chise/omega";
-
 ### Options ###
 
-#$opt_order{'UniMulti'}='jcgk';
-$opt_order{'UniMulti'}='jGcgkHC';
+$opt_order{'UniMulti'}='jGcgk';
+#$opt_order{'UniMulti'}='jGcgkHC';
 $opt_order{'UniCNS'}='c';
 $opt_order{'UniGB'}='g';
 $opt_order{'UniJIS'}='j';
 $opt_order{'UniKS'}='k';
 
-$opt_allow_unify=0; # 1=true, 0=false.
+$opt_allow_unify=1; # 1=true, 0=false.
 $opt_protrude=0;# 1=true, 0=false.
 
 # currently does not work.
@@ -41,8 +40,7 @@ $opt_use_kage_for_Ext_B=0;# 1=true, 0=false.
 
 ### End ###
 
-my $strictly_forbidden_after = '「【『［（〈“‘‘（〔｛《{\[\(\x{3016}｛｢';
-#       \x{3016} | # white 【
+my $strictly_forbidden_after = '「【『［（〈‘‛“‟（〔｛《{\[\(〖｛｢';
 
 my $forbidden_after = "\x{0000}";
 
@@ -53,8 +51,7 @@ my $slightly_forbidden_after = '￥¥＄$〒♯＃#￠¢￡£＠@§';
 # All these characters are allowed to protrude
 # in the right margin
 my $strictly_forbidden_before=
-    '!,.:;?、。！，．：；？。\)#}’”〉》」』】〕\x{3017}）］｝｝」\]';
-###       \x{3017} | # white 】
+    '!,.:;?、。！，．：；？。\)\]}’”〉》」』】〕〗）］｝」';
 
 my $forbidden_before
     = 'ー々ぁぃぅぇぉゃゅょっゎァィゥェォャュョッヮヵヶ';
@@ -62,10 +59,25 @@ my $forbidden_before
 my $slightly_forbidden_before
     = '\x{000a}\#\-‐−‰′″℃゛゜ゝゞヽヾ＂％－ﾞﾟ';
 
-my $asian = '\x{1100}-\x{11FF}\x{2E80}-\x{D7AF}\x{F900}-\x{FAFF}\x{FE30}-\x{FE4F}\x{FF00}-\x{FFFFFF}';
+#my $asian = '\x{1100}-\x{11FF}\x{2E80}-\x{D7AF}\x{F900}-\x{FAFF}\x{FE30}-\x{FE4F}\x{FF00}-\x{FFFFFF}'; # need to be fixed.
+my $asian = '\x{2E80}-\x{312f}\x{3190}-\x{ABFF}\x{F900}-\x{FAFF}\x{FE30}-\x{FE4F}\x{FF00}-\x{FFFFFF}'; # need to be fixed.
 
 my $space = '\x{0020}\x{0009}\x{000A}\x{000C}\x{000D}';
 
+my $hangul='\x{1100}-\x{11ff}\x{3130}-\x{318f}\x{AC00}-\x{D7AF}';
+
+my %tex_meta=('#'=>'\#',
+	      '$'=>'\\textdollar{}',
+	      '%'=>'\%',
+	      '&'=>'\&',
+	      '{'=>'\\textbraceleft{}',
+	      '}'=>'\\textbraceright{}',
+	      '\\'=>'\\textbackslash{}',
+	      '_'=>'\\textunderscore',
+	     );
+
+my $tex_meta_re=join('|',map {quotemeta($_)} keys %tex_meta);
+
 &GetOptions("in=s"=>\$opt_in_cs,
 	    "out=s"=>\$opt_out_cs,
 	    "help",\$opt_help);
@@ -73,7 +85,7 @@ my $space = '\x{0020}\x{0009}\x{000A}\x{000C}\x{000D}';
 $usage=<<EOF;
 Usage: $0 -i <input coding system> -o <cmap encoding>
     input coding system:
-      Utf8mcs, Utf8cns, Utf8gb, Utf8jis, Utf8ks
+      Utf8mcs, Utf8cns, Utf8gb, Utf8jis, Utf8ks, Utf8big5
     cmap encoding:
       UniCNS, UniGB, UniJIS, UniKS, UniMulti
 EOF
@@ -82,11 +94,16 @@ if($opt_in_cs or $opt_out_cs){
     $in_cs=$opt_in_cs;
     $out_cs=$opt_out_cs;
 }elsif(@ARGV==0){
-    ($in_cs,$out_cs)=($0=~/(Utf8.+)To(\w+)/);
+    ($in_cs
+     ,$out_cs
+     ,$opt_preserve_spaces)
+	=($0=~/(Utf8mcs|Utf8cns|Utf8gb|Utf8jis|Utf8ks|Utf8big5)
+	  To (UniCNS|UniGB|UniJIS|UniKS|UniMulti)
+	  (Sp)?/ox);
 }
 
 # $in_cs:
-#   Utf8mcs,Utf8cns,Utf8gb,Utf8jis,Utf8ks,
+#   Utf8mcs,Utf8cns,Utf8gb,Utf8jis,Utf8ks,Utf8big5
 # $out_cs:
 #   UniCNS,UniGB,UniJIS,UniKS,UniMulti
 
@@ -101,7 +118,7 @@ if($opt_help
 
 $omegadb_path=~s!/$!!;
 
-$idsdata_file="$omegadb_path/idsdata.pl";
+$idsdata_file="$omegadb_path/idsdata.txt";
 $ids_start=0x00; 
 $font_start=0;
 
@@ -110,10 +127,17 @@ if(-e $idsdata_file){
     flock(IDSDATA,LOCK_EX);
     seek(IDSDATA,0,0);
     while(<IDSDATA>){
-	eval $_;
+	utf8::decode($_);
+	if(m/^START\t(\d+)\t(\d+)/){
+	    $font_start=$1,$ids_start=$2;
+	}elsif(m/^(.*?)\t(\d+)\t(\d+)/){
+	    $ids{$1}=[$2,$3,];
+	}else{
+	    die "Irregular IDS file: $idsdata_file.\n";
+	}
     }
     seek(IDSDATA,0,0);
-#	  require $idsdata_file;
+    truncate(IDSDATA,0);
 }else{
     open(IDSDATA,">:utf8",$idsdata_file) or die;
     flock(IDSDATA,LOCK_EX);
@@ -123,8 +147,8 @@ if(-e $idsdata_file){
 $ids_argc=0;
 $ids="";
 
-$geta=pack("U",0x3013);
-#$geta=pack("U",0xfffd);
+#$geta=pack("U",0x3013);
+$geta=pack("U",0xfffd);
 
 @GT=(#"=gt","=gt-k",
      "=gt-pj-1","=gt-pj-2","=gt-pj-3","=gt-pj-4","=gt-pj-5",
@@ -158,28 +182,86 @@ foreach $out_cs ('UniCNS','UniGB','UniJIS','UniKS','UniMulti'){
     }
 }
 
+$/="";
+
 while(<>){
     utf8::decode($_);
+
+    print '{\relax{}';
+
     if($in_cs ne 'ucs@mcs'){
 	s/(.)/&get_char_in_mcs($1,$in_cs)/ge;
     }
     s/(amp.+?;)/&de_tex_er($1)/ge;
 #    s/(&.+?;)/&de_tex_er($1)/ge;
+    s/([$asian])[$space]+/$1/go unless($opt_preserve_spaces);
+    s/[$space]+([$asian])/$1/go unless($opt_preserve_spaces);
+    s/([$asian])\s*([^$asian$space])/$1 $2/go;
+    s/([^$asian$idc])\s*([$asian])/$1 $2/go;
+    s/\-\-\-/pack("U",0x2014)/geo;# EM DASH
+    s/\-\-/pack("U",0x2013)/geo;# EN DASH
+#    s/\`\`/pack("U",0x201f)/geo;# DOUBLE HIGH-REVERSED-9 QUOTATION MARK
+    s/\`\`/pack("U",0x201c)/geo;# _TeX compatible_
+#    s/\`/pack("U",0x201b)/geo;# SINGLE HIGH-REVERSED-9 QUOTATION MARK
+    s/\`/pack("U",0x2018)/geo;# _TeX compatible_
+    s/\'\'/pack("U",0x201d)/geo;# RIGHT DOUBLE QUOTATION MARK
+    s/\'/pack("U",0x2019)/geo;# RIGHT DOUBLE QUOTATION MARK
     @chars=split(//);
+
   CHAR:
     for($i=0;$i<=$#chars;$i++){
 	$char=$chars[$i];
-	$char_id=unpack("U",$char);
+	if($char_id=&get_char_attribute($char,'=ucs@unicode')){
+	    $char=pack("U",$char_id);
+	}else{
+	    $char_id=unpack("U",$char);
+	}
 
 	if($char_id<=0x20){
-	    print $chars[$i];
+	    # add quarter space between asian and non-asian.
+	    if($i>0 and $i<$#chars){
+		if(($chars[$i-1]=~/[$asian]/
+		    and $chars[$i+1]=~/[^$asian]/
+		      and $chars[$i+1]=~/[$strictly_forbidden_before$slightly_forbidden_before]/)
+		   or($chars[$i-1]=~/[^$asian]/
+		      and $chars[$i+1]=~/[$asian]/
+		      and $chars[$i-1]=~/[$strictly_forbidden_after$slightly_forbidden_after]/)
+		   ){
+		    print '';
+		}elsif(($chars[$i-1]=~/[$asian]/
+			and $chars[$i+1]=~/[^$asian]/)){
+		    print '\unskip\kern.25ex';
+		}elsif($chars[$i-1]=~/[^$asian]/
+		       and $chars[$i+1]=~/[$asian]/){
+		    print '\kern.25ex';
+		}else{
+		    print ' ';
+		}
+	    }else{
+		print ' ';
+	    }
 	    next CHAR;
-	}elsif($char_id>0x20 and $char_id<=0x02af){
-	    # Basic Latin
-	    # Latin-1 Supplement
-	    # Latin Extended-A
-	    # Latin Extended-B
-	    # IPA Extensions
+	}elsif(($char_id>0x20 and $char_id<=0x021f)
+	       # Basic Latin
+	       # Latin-1 Supplement
+	       # Latin Extended-A
+	       # Latin Extended-B (not all)
+	       or($char_id>=0x0250 and $char_id<=0x02af)
+	       # IPA Extensions
+	       or($char_id>=0x0300 and $char_id<=0x033f)
+	       or($char_id>=0x0360 and $char_id<=0x036f)
+	       # Combining Diacritical Marks
+	       or($char_id>=0x1e00 and $char_id<=0x1eff)
+	       # Latin Extended Additional
+	       or($char_id>=0x0370 and $char_id<=0x03ff)
+               # Greek and Coptic
+	       or($char_id>=0x0400 and $char_id<=0x04ff)
+               # Cyrillic
+	       or($char_id>=0x0530 and $char_id<=0x058f)
+               # Armenian
+	       or($char_id>=0x2010 and $char_id<=0x2046)
+	       # General Punctuation (partial)
+	       ){
 	    print &latin_parse();
 	    next CHAR;
 	}elsif($char_id>=0x2ff0 and $char_id<=0x2fff){
@@ -214,8 +296,8 @@ while(<>){
 				$font_start++;
 			    }
 			}
-			print "{\\fontencoding{OT1}\\fontfamily{" .
-			    sprintf("chise%03d",$ids{$char}[0]) .
+			print "{\\fontencoding{OT1}\\fontfamily{",
+			    sprintf("chise%03d",$ids{$char}[0]) ,
 				"}\\selectfont\\char$ids{$char}[1]}",&add_break($i);
 			next CHAR;
 		    }
@@ -228,16 +310,13 @@ while(<>){
 	    }
 	}
     }
+    print '}';
 }
 
-print IDSDATA 'use utf8;',"\n";
+print IDSDATA 'START',"\t",$font_start,"\t",$ids_start,"\n";
 foreach $ids (keys %ids){
-    print IDSDATA '$ids{\'',$ids,'\'}='
-    ,'[',join ",",@{$ids{$ids}},"];\n";
+    print IDSDATA $ids,"\t",join("\t",@{$ids{$ids}}),"\n";
 }
-print IDSDATA '$font_start=',$font_start,";\n";
-print IDSDATA '$ids_start=',$ids_start,";\n";
-print IDSDATA "1;";
 flock(IDSDATA,LOCK_UN);
 
 exit 0;
@@ -271,41 +350,46 @@ sub add_break{
     if($i<($#chars-1)){
 	if(($chars[$i+1]=~m/[$strictly_forbidden_before]/o)
 	   and($chars[$i+2]=~m/[$strictly_forbidden_before]/o)){
-	    return "\\CJKunbreakablekernone ";
+	    return "\\CJKunbreakablekernone{}";
 	}elsif($opt_protrude){
 	    if(($chars[$i+1]=~m/[$strictly_forbidden_before]/o)
 	       and($chars[$i+2]=~m/[^$strictly_forbidden_before]/o)){
-		return "\\CJKunbreakablekernone \\CJKprotrude ";
+		return "\\CJKunbreakablekernone \\CJKprotrude{}";
 	    }
 	}
     }
     if(($i<$#chars)
+       and($chars[$i+1]eq" ")){
+       # preserve space.
+ 	return "";
+    }
+    if(($i<$#chars)
        and($chars[$i+1]=~m/[$strictly_forbidden_before]/o)){
-	return "\\CJKunbreakablekernone ";
+	return "\\CJKunbreakablekernone{}";
     }
     if($chars[$i]=~m/[$strictly_forbidden_after]/o){
-	return "\\CJKunbreakablekernone ";
+	return "\\CJKunbreakablekernone{}";
     }
     if(($i<$#chars)
        and($chars[$i+1]=~m/[$forbidden_before]/o)){
-	return "\\CJKunbreakablekerntwo ";
+	return "\\CJKunbreakablekerntwo{}";
 
     }
     if($chars[$i]=~m/[$forbidden_after]/o){
-	return "\\CJKunbreakablekerntwo ";
+	return "\\CJKunbreakablekerntwo{}";
     }
     if(($i<$#chars)
        and($chars[$i+1]=~m/[$slightly_forbidden_before]/o)){
-	return "\\CJKunbreakablekernthree ";
+	return "\\CJKunbreakablekernthree{}";
     }
     if($chars[$i]=~m/[$slightly_forbidden_after]/o){
-	return "\\CJKunbreakablekernthree ";
+	return "\\CJKunbreakablekernthree{}";
     }
     if($chars[$i]=~m/[$asian]/o){
-	return "\\CJKbreakablekern ";
+	return "\\CJKbreakablekern{}";
     }
     if(($i<$#chars)and($chars[$i+1]=~m/[$asian]/o)){
-	return "\\CJKbreakablekern ";
+	return "\\CJKbreakablekern{}";
     }
 }
 
@@ -313,19 +397,34 @@ sub latin_parse{
     # arguments: none
     # return: string for output with TeX macro.
     my($char_id);
-    my $out_str=$chars[$i];
-    $i++;
+    my $out_str="";
     while($i<=$#chars){
 	$char_id=unpack("U",$chars[$i]);
-	if($char_id>0x20 and $char_id<=0x02af){
+	if($chars[$i]=~m/($tex_meta_re)/o){
+	    $out_str.=$tex_meta{$chars[$i]};
+	}elsif(($char_id>0x20 and $char_id<=0x021f)
+	   or($char_id>=0x0250 and $char_id<=0x02af)# IPA Extensions
+	   or($char_id>=0x0300 and $char_id<=0x033f)# Combining Diacritical Marks
+	   or($char_id>=0x0360 and $char_id<=0x036f)
+	   or($char_id>=0x0370 and $char_id<=0x03ff)# Greek and Coptic
+	   or($char_id>=0x0400 and $char_id<=0x04ff)# Cyrillic
+	   or($char_id>=0x0530 and $char_id<=0x058f)# Armenian
+	   ){
 	    $out_str.=$chars[$i];
+	}elsif($char_id>=0x1e00 and $char_id<=0x1eff){
+	    # Latin Extended Additional
+            # 0x1e00 -> 0x0600, etc.
+	    $out_str.=pack("U",$char_id-0x1800);
+	}elsif($char_id>=0x2010 and $char_id<=0x2046){
+	    # General Punctuation (partial)
+	    $out_str.=pack("U",$char_id-0x1000);
 	}else{
 	    $i--;
 	    last;
 	}
 	$i++;
     }
-    return '{\normalfont {'.$out_str.'}}';
+    return '{\fontencoding{OT1}\fontfamily{omlgc}\selectfont '.$out_str.'}';###UT1?
 }
 
 sub ids_parse{
@@ -375,7 +474,7 @@ sub get_macro_for_ids{
     # return: TeX macro for ids
     #          or GETA character if ids is invalid for KAGE.
     my($ids)=@_;
-    $ids=&normalize_ids($ids,"UniJIS");
+    # $ids=&normalize_ids($ids,"UniJIS");
     return $geta if(($ids!~/[$idc]/)
 		    or($ids=~/[\x{10000}-]/));
                     #irregular for KAGE.
@@ -390,7 +489,7 @@ sub get_macro_for_ids{
     }
     return "{\\fontencoding{OT1}\\fontfamily{"
 	.sprintf("chise%03d",$ids{$ids}[0])
-	."}\\selectfont\\char$ids{$ids}[1]}";
+	."}\\selectfont\\char$ids{$ids}[1]}\\relax{}";
 }
 
 sub normalize_ids{
@@ -435,7 +534,7 @@ sub get_output_char{
 	       or $out_char_id=&get_char_attribute($char,'=>ucs')
 	       or $out_char_id=&get_char_attribute($char,'=>ucs*')
 	       ){
-		return '{\selectjisfont\char'.$out_char_id.'}';
+		return "{\\selectjisfont\\char$out_char_id}";
 	    }
 	}elsif($out_cs eq 'UniGB'
 	       and &get_char_attribute($char,"vnd-adobe-cid-unigb-ucs2-h")){
@@ -445,7 +544,7 @@ sub get_output_char{
 	       or $out_char_id=&get_char_attribute($char,'=>ucs')
 	       or $out_char_id=&get_char_attribute($char,'=>ucs*')
 	       ){
-		return '{\selectgbsfont\char'.$out_char_id.'}';
+		return "{\\selectgbsfont\\char$out_char_id}";
 	    }
 	}elsif($out_cs eq 'UniCNS'
 	       and &get_char_attribute($char,"vnd-adobe-cid-unicns-ucs2-h")){
@@ -455,7 +554,7 @@ sub get_output_char{
 	       or $out_char_id=&get_char_attribute($char,'=>ucs')
 	       or $out_char_id=&get_char_attribute($char,'=>ucs*')
 	       ){
-		return '{\selectcnsfont\char'.$out_char_id.'}';
+		return "{\\selectcnsfont\\char$out_char_id}";
 	    }
 	}elsif($out_cs eq 'UniKS'
 	       and &get_char_attribute($char,"vnd-adobe-cid-uniks-ucs2-h")){
@@ -465,7 +564,7 @@ sub get_output_char{
 	       or $out_char_id=&get_char_attribute($char,'=>ucs')
 	       or $out_char_id=&get_char_attribute($char,'=>ucs*')
 	       ){
-		return '{\selectksxfont\char'.$out_char_id.'}';
+		return "{\\selectksxfont\\char$out_char_id}";
 	    }
 	}elsif($out_cs eq 'GT'){
 	    return $gt if($gt=&get_macro_for_GT($char));
@@ -486,9 +585,9 @@ sub get_ids{
 #    $ids=&get_char_attribute($char,"ids-aggregated")
 #	or $ids=&get_char_attribute($char,"ids");
     $ids=&get_char_attribute($char,"ids-decomposed")
-	or $ids=&get_char_attribute($char,"ids");
-#	  or $ids=&get_char_attribute($char,"ideographic-structure");
-#    $ids=~s/[? ()]//g;
+	or $ids=&get_char_attribute($char,"ids")
+	  or $ids=&get_char_attribute($char,"ideographic-structure");
+    $ids=~s/[? ()]//g;
     return $ids;
 }
 
@@ -511,7 +610,8 @@ sub get_char_in_mcs{
     my($char,$in_cs)=@_;
     my($output_char);
 
-    return $char if($in_cs eq 'ucs@mcs');
+    return $char if($in_cs eq 'ucs@mcs' or
+		    $char le "~");
 
     if(($output_char)=&get_chars_matching("=$in_cs",unpack("U",$char))){
 	return $output_char;
@@ -522,14 +622,14 @@ sub get_char_in_mcs{
 
 sub get_chars_unified{
     my($char)=@_;
-    my($chars,$ucs,$char_ucs);
+    my($chars,$ucs,$char_ucs,$char_sub);
     my(@chars);
 
     if($chars=&get_char_attribute($char,'->ucs-unified')){
 	$chars=~s/^\((.*)\)$/$1/;
 	return (split(/\s*\?/,$chars));
     }elsif($ucs=&get_char_attribute($char,'=>ucs*')
-	  or $ucs=&get_char_attribute($char,'=>ucs')){
+	   or $ucs=&get_char_attribute($char,'=>ucs')){
 	$char_ucs=pack("U",$ucs);
 	if($chars=&get_char_attribute($char_ucs,'->ucs-unified')){
 	    $chars=~s/^\((.*)\)$/$1/;
@@ -538,6 +638,14 @@ sub get_chars_unified{
 	    push(@chars,$char_ucs);
 	    return @chars;
 	}
+    }elsif($char_sub=&get_char_attribute($char,'<-subsumptive')){
+	$char_sub=~s/[? ()]//g;
+	$chars=&get_char_attribute($char_sub,'->subsumptive');
+	@chars=grep {not /^$char$/} (split(/\s*\?/,$chars));
+	push(@chars,$char_sub);
+	return @chars;
+    }else{
+	return ();
     }
 }
 
@@ -573,7 +681,9 @@ sub get_macro_for_HZK{
 	}
     }
     if($hzk){
-	return "{\\fontencoding{OT1}\\fontfamily{".sprintf("hzk%02d",$HZK)."}\\selectfont\\char".$hzk."}";
+	return "{\\fontencoding{OT1}\\fontfamily{"
+	    .sprintf("hzk%02d",$HZK)
+	    ."}\\selectfont\\char".$hzk."}";
     }else{
 	return undef;
     }
@@ -592,9 +702,7 @@ sub get_macro_for_CDP{
     if($cdp){
 	$ucs=&get_char_attribute(&get_chars_matching("=big5-pua",$cdp),"=ucs");
 	if($ucs){
-	    return "{\\fontencoding{OT1}\\fontfamily{cdp}\\selectfont\\char"
-		.$ucs.
-		    "}";
+	    return "{\\fontencoding{OT1}\\fontfamily{cdp}\\selectfont\\char$ucs}";
 	}else{
 	    print STDERR "This should not happen.\n";
 	    print STDERR "ucs code point of CDP: $cdp not found.\n";