83a55c0b3ec59419042054297d15e89081c4e8f7
[chise/ruby.git] / chise / idsdb.rb
1 # Copyright (C) 2002-2004 Kouichirou Eto, All rights reserved.
2
3 require "chise/char"
4 require "chise/ids"
5 require "chise/qp"
6 require "chise/management"
7
8 module CHISE
9   class IDS_DB_Management
10     def initialize
11       @cd = ChiseDB.instance
12       @idsdb = IDS_DB.instance
13     end
14
15     def check_conflict_of_ids_text
16       @idsdb.each_ccs {|ccs|
17         #qp ccs
18         c = Hash.new(0)
19         h = {}
20         @idsdb.get_ccs(ccs).each_character {|char, ids|
21           c["char"] += 1
22           next if ids == char.to_s
23           next if ids.char_length == 1
24           char_id = char.char_id
25           cids = h[char_id]
26           if cids.nil? # There is no ids yet.
27             h[char_id] = ids # just set it.
28             c["good"] += 1
29           else # but, if there is already a ids?
30             if cids == ids # the two are same.
31               c["same"] += 1 # and just ignore
32             else # but, if the two are not same?
33               c["conflict"] += 1
34               puts "conflict\t#{char.to_s}\t#{ids}\t#{cids}"
35             end
36           end
37         }
38         puts "#{ccs}\t#{c['char']}\t#{c['same']}\t#{c['conflict']}\t#{c['good']}"
39       }
40     end
41
42     def store_ids_as_text
43       max = 20000
44       h = {}
45       @idsdb.each_ccs {|ccs|
46         qp ccs
47         @idsdb.get_ccs(ccs).each_character {|char, ids|
48           next if ids == char.to_s
49           next if ids.char_length == 1
50           char.ids_text = ids # just set it.
51           h[char.char_id] = ids
52 #         break if max <= h.length
53         }
54 #       break if max <= h.length
55       }
56       qp "%08X" % h.keys.max
57       qp "sync", @cd.get_feature("ids-text").sync
58       @cd.get_feature("ids-text").dump
59       qp h.length
60       qp @cd.get_feature("ids-text").to_hash.length
61     end
62
63     def store_ids_de_er
64       h = {}
65       @cd.get_feature("ids-text").each_char {|cid, ids_text|
66         char = Character.get(cid)
67         begin
68           ids = ids_text.de_er # parse Entity Reference
69         rescue => e
70           qp cid, ids_text
71           next
72         end
73         next if ids == char.to_s
74         next if ids.char_length == 1
75         char.ids_de_er = ids # set it.
76         h[char.char_id] = ids
77       }
78       qp "%08X" % h.keys.max
79       @cd.get_feature("ids-de-er").dump
80       qp h.length
81       qp @cd.get_feature("ids-de-er").to_hash.length
82     end
83
84     def check_integrity_of_ids_tree
85       h = {}
86       @cd.get_feature("ids-de-er").each_char {|cid, ids|
87         char = Character.get(cid)
88         idstree = IDS_Tree.new(ids)
89         begin
90           raise "contains self" if ids.include?(char.to_s)
91           idstree.check_integrity
92         rescue => e
93           #puts "#{cid}\t#{e.message}\t#{ids}"
94           char.ids_error = e.message
95           next
96         end
97         char.ids_org = ids # set it.
98         h[char.char_id] = ids
99       }
100       @cd.get_feature("ids-org").dump
101       qp h.length
102       qp @cd.get_feature("ids-org").to_hash.length
103       @cd.get_feature("ids-error").dump
104     end
105
106     def make_by_ids_db_org
107       h = {}
108       byids = @cd.get_by_ids_db("ids-org")
109       @cd.get_feature("ids-org").each_char {|cid, ids|
110         char = Character.get(cid)
111         byids.set_decoded_char(ids, cid)
112         h[ids] = cid
113       }
114       qp h.length
115       byids.dump
116       qp byids.to_hash.length
117     end
118
119     def store_ids_aggregated
120       h = {}
121       @cd.get_feature("ids-org").each_char {|cid, ids|
122         char = Character.get(cid)
123         #ids = char.decompose
124         #ids = char.ids
125         ag = ids.to_ids.aggregate("ids-org")
126         #puts "#{char.to_s}\t#{ids}\t#{ag}"
127         char.ids = ag # ids-aggregated
128         h[char.char_id] = ids
129       }
130       @cd.get_feature("ids").dump
131       qp h.length
132       qp @cd.get_feature("ids").to_hash.length
133     end
134
135     def store_ids_subparts
136       h = {}
137       @cd.get_feature("ids").each_char {|cid, v|
138         char = Character.get(cid)
139         pids = char.to_s # previous_ids
140         ar = []
141         i = 0 # only for infinite loop check
142         loop {
143           ids = pids.decompose
144           break if ids == pids # break if there is no possibilities.
145           ar += ids.to_a
146           i += 1
147           qp [char.to_s, pids, ids, ar] if 10 < i # something wrong.
148           pids = ids
149         }
150         str = ar.sort.uniq.join("") # can contain IDC.
151         char.ids_subparts = str
152         h[char.char_id] = str
153       }
154       @cd.get_feature("ids-subparts").dump
155       qp h.length
156       qp @cd.get_feature("ids-subparts").to_hash.length
157     end
158
159     def store_ids_contained
160       h = Hash.new
161       @cd.get_feature("ids-subparts").each_char {|cid, v|
162         char = Character.get(cid)
163         parts = char.ids_subparts
164         parts.each_char {|ch|
165           h[ch] = [] if h[ch].nil?
166           h[ch] << cid
167         }
168       }
169       h.each {|char, ar|
170         str = ar.sort.map {|cid| Character.get(cid).to_s }.join
171         char.ids_contained = str
172       }
173       @cd.get_feature("ids-contained").dump
174     end
175
176     def make_by_ids_db
177       byids = @cd.get_by_ids_db("ids")
178       @cd.get_feature("ids").each_char {|cid, ids|
179         char = Character.get(cid)
180         byids.set_decoded_char(ids, cid)
181       }
182       byids.dump
183     end
184   end
185
186   class IDS_DB
187     include Singleton
188
189     def initialize
190       @config = Config.instance
191       @path = @config.ids_dir.path
192       @dbs = {}
193     end
194     attr_reader :path
195
196     def get_ccs(ccs)
197       @dbs[ccs] = IDS_CCS_DB.new(self, ccs) if @dbs[ccs].nil?
198       @dbs[ccs]
199     end
200
201     def each_ccs
202       @path.each_entry {|f|
203         next unless /\AIDS-(.+)\.txt\Z/ =~ f
204         yield($1)
205       }
206     end
207   end
208
209   class IDS_CCS_DB
210     def initialize(idsdb, ccs)
211       @idsdb, @ccs = idsdb, ccs
212       @path = @idsdb.path+("IDS-"+ccs+".txt")
213     end
214
215     def each_line
216       @path.open {|f|
217         f.each {|line|
218           next if /\A;/ =~ line # skip comment
219           line.chomp!
220           code, picture, ids = line.split
221           raise if code.nil?
222           ids = "" if ids.nil?
223           yield(code, ids)
224         }
225       }
226     end
227
228     def each_character
229       each_line {|code, ids|
230         next if ids.nil?
231         next if ids == "" # If there is no IDS, ignore it.
232
233         er = "&"+code+";"
234         begin
235           char = Character.get(er)
236         rescue
237           #qp er
238           next
239         end
240         next if char.nil?
241         yield(char, ids)
242       }
243     end
244
245   end
246 end