1 /* Copyright (C) 2003, 2004, 2005, 2006, 2011, 2013 MORIOKA Tomohiko
2 This file is part of the CONCORD Library.
4 The CONCORD Library is free software; you can redistribute it and/or
5 modify it under the terms of the GNU Lesser General Public
6 License as published by the Free Software Foundation; either
7 version 2.1 of the License, or (at your option) any later version.
9 The CONCORD Library is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 Lesser General Public License for more details.
14 You should have received a copy of the GNU Lesser General Public
15 License along with the CONCORD Library; if not, write to the Free
16 Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
20 #include <sys/types.h>
27 #include "concord-name.h"
29 #include "concord-bdb.h"
31 COS_Feature concord_opened_feature_list = NULL;
32 COS_Feature concord_opened_feature_list_last = NULL;
33 size_t concord_opened_feature_list_length = 0;
34 size_t concord_opened_feature_list_limit = 640 /* 480 */;
36 COS_Feature concord_tenure_feature_list = NULL;
37 COS_Feature concord_tenure_feature_list_last = NULL;
38 size_t concord_tenure_feature_list_length = 0;
40 COS_Feature_INDEX concord_opened_index_list = NULL;
41 COS_Feature_INDEX concord_opened_index_list_last = NULL;
42 size_t concord_opened_index_list_length = 0;
43 size_t concord_opened_index_list_limit = 32;
47 CONCORD_String_size (const CONCORD_String s)
53 CONCORD_String_data (const CONCORD_String s)
58 CONCORD_Genre concord_ds_open_genre (CONCORD_DS ds, const char* name);
62 concord_genre_open_feature (CONCORD_Genre genre, const char* name);
66 concord_genre_open_index (CONCORD_Genre genre, const char* index);
69 concord_genre_get_feature_0 (CONCORD_Genre genre, const char* name);
73 concord_default_read_object (const unsigned char* string, size_t length);
78 COS_Object_Header header;
79 CONCORD_Backend_Type type;
81 CONCORD_NAME_TABLE* genre_names;
85 CONCORD_Object object_nil;
86 CONCORD_Object (*read_object) (const unsigned char* str, size_t length);
90 concord_default_read_object (const unsigned char* str, size_t length)
92 unsigned char* buf = malloc (length + 1);
96 strncpy ((char*)buf, (char*)str, length);
98 return (CONCORD_Object)buf;
102 concord_open_ds (CONCORD_Backend_Type type, const char* location,
103 int subtype, int modemask)
105 CONCORD_DS ds = COS_ALLOCATE_OBJECT (DS);
106 size_t len = strlen (location);
112 ds->subtype = ( (subtype != 0) ? subtype : DB_HASH );
113 ds->modemask = modemask;
114 ds->location = (char*)malloc (len + 1);
115 if (ds->location == NULL)
116 goto location_failure;
118 strcpy (ds->location, location);
120 ds->genre_names = concord_make_name_table ();
121 if (ds->genre_names == NULL)
129 ds->object_nil = NULL;
130 ds->read_object = &concord_default_read_object;
136 concord_close_ds (CONCORD_DS ds)
138 if (ds->location != NULL)
140 if (ds->genre_names != NULL)
141 concord_destroy_name_table (ds->genre_names);
147 concord_ds_location (CONCORD_DS ds)
153 concord_ds_set_object_failure (CONCORD_DS ds, CONCORD_Object object_nil)
155 ds->object_nil = object_nil;
160 concord_ds_set_read_object_function (CONCORD_DS ds,
161 CONCORD_Object (*read_object)
162 (const unsigned char* str,
165 ds->read_object = read_object;
170 concord_ds_get_genre (CONCORD_DS ds, const char* name)
174 genre = concord_name_table_get (ds->genre_names, name);
178 genre = concord_ds_open_genre (ds, name);
182 if (concord_name_table_put (ds->genre_names, name, genre))
184 concord_close_genre (genre);
191 concord_ds_foreach_genre_name (CONCORD_DS ds,
192 int (*func) (CONCORD_DS ds, char* name))
194 char* dname = ds->location;
198 if ( (dir = opendir (dname)) == NULL)
201 while ( (de = readdir (dir)) != NULL )
203 if ( (strcmp (de->d_name, ".") != 0) &&
204 (strcmp (de->d_name, "..") != 0) )
206 int i, need_to_decode = 0;
211 for (cp = (unsigned char*)de->d_name, i = 0; *cp != '\0'; i++)
222 name = (char*) alloca (i);
223 cp = (unsigned char*)de->d_name;
224 np = (unsigned char*)name;
226 while ( (ch = *cp++) != '\0')
242 if ( ('0' <= ch) && (ch <= '9') )
243 hex[index++] = ch - '0';
244 else if ( ('A' <= ch) && (ch <= 'F') )
245 hex[index++] = ch - 'A' + 10;
246 else if ( ('a' <= ch) && (ch <= 'f') )
247 hex[index++] = ch - 'a' + 10;
259 *np++ = (hex[0] << 4) | hex[1];
273 return closedir (dir);
276 return closedir (dir);
282 COS_Object_Header header;
285 CONCORD_NAME_TABLE* feature_names;
286 CONCORD_NAME_TABLE* index_names;
290 concord_ds_open_genre (CONCORD_DS ds, const char* name)
293 size_t len = strlen (name);
298 genre = COS_ALLOCATE_OBJECT (Genre);
303 genre->name = (char*)malloc (len + 1);
304 if (genre->name == NULL)
309 strcpy (genre->name, name);
311 genre->feature_names = concord_make_name_table ();
312 if (genre->feature_names == NULL)
319 genre->index_names = concord_make_name_table ();
320 if (genre->index_names == NULL)
322 free (genre->feature_names);
331 concord_close_genre (CONCORD_Genre genre)
338 if (genre->name == NULL)
346 if (genre->feature_names != NULL)
347 concord_destroy_name_table (genre->feature_names);
349 if (genre->index_names != NULL)
350 concord_destroy_name_table (genre->index_names);
357 concord_genre_get_name (CONCORD_Genre genre)
363 concord_genre_get_data_source (CONCORD_Genre genre)
369 concord_genre_foreach_feature_name (CONCORD_Genre genre,
370 int (*func) (CONCORD_Genre genre,
374 = alloca (strlen (genre->ds->location)
375 + 1 + strlen (genre->name) + sizeof ("/feature") + 1);
379 strcpy (dname, genre->ds->location);
381 strcat (dname, genre->name);
382 strcat (dname, "/feature");
384 if ( (dir = opendir (dname)) == NULL)
387 while ( (de = readdir (dir)) != NULL )
389 if ( (strcmp (de->d_name, ".") != 0) &&
390 (strcmp (de->d_name, "..") != 0) )
392 int i, need_to_decode = 0;
397 for (cp = (unsigned char*)de->d_name, i = 0; *cp != '\0'; i++)
408 name = (char*) alloca (i);
409 cp = (unsigned char*)de->d_name;
410 np = (unsigned char*)name;
412 while ( (ch = *cp++) != '\0')
428 if ( ('0' <= ch) && (ch <= '9') )
429 hex[index++] = ch - '0';
430 else if ( ('A' <= ch) && (ch <= 'F') )
431 hex[index++] = ch - 'A' + 10;
432 else if ( ('a' <= ch) && (ch <= 'f') )
433 hex[index++] = ch - 'a' + 10;
445 *np++ = (hex[0] << 4) | hex[1];
458 if (func (genre, name))
459 return closedir (dir);
462 return closedir (dir);
466 concord_genre_get_feature_0 (CONCORD_Genre genre, const char* name)
468 CONCORD_Feature feature;
470 feature = concord_name_table_get (genre->feature_names, name);
474 feature = concord_genre_open_feature (genre, name);
478 if (concord_name_table_put (genre->feature_names, name, feature))
480 concord_close_feature (feature);
487 concord_genre_get_feature (CONCORD_Genre genre, const char* name)
489 CONCORD_Genre g_feature
490 = concord_ds_get_genre (genre->ds, "feature");
492 if (g_feature != NULL)
494 CONCORD_Feature p_true_name
495 = concord_genre_get_feature_0 (g_feature, "true-name");
497 if (g_feature != NULL)
499 CONCORD_String_Tank s_true_name;
501 = concord_obj_get_feature_value_string (name,
506 char* t_name = alloca (s_true_name.size + 1);
508 strncpy (t_name, s_true_name.data, s_true_name.size);
509 t_name[s_true_name.size] = '\0';
510 return concord_genre_get_feature_0 (genre, t_name);
514 return concord_genre_get_feature_0 (genre, name);
518 concord_genre_get_index (CONCORD_Genre genre, const char* name)
522 index = concord_name_table_get (genre->index_names, name);
526 index = concord_genre_open_index (genre, name);
530 if (concord_name_table_put (genre->index_names, name, index))
532 concord_close_index (index);
540 concord_genre_open_feature (CONCORD_Genre genre, const char* feature)
542 CONCORD_Feature table;
543 size_t len = strlen (feature);
548 table = COS_ALLOCATE_OBJECT (Feature);
552 table->genre = genre;
555 table->name = (char*)malloc (len + 1);
556 if (table->name == NULL)
561 strcpy (table->name, feature);
562 table->value_table = NULL;
564 table->number_of_hits = 0;
566 table->number_of_closes = 0;
572 concord_close_feature (CONCORD_Feature feature)
579 if (feature->db == NULL)
582 status = CONCORD_BDB_close (feature->db);
584 if (feature->name == NULL)
588 free (feature->name);
596 concord_feature_get_name (CONCORD_Feature feature)
598 return feature->name;
602 concord_feature_get_genre (CONCORD_Feature feature)
604 return feature->genre;
608 concord_feature_setup_db (CONCORD_Feature feature, int writable)
617 if ((feature->access & DB_CREATE) == 0)
619 if (feature->db != NULL)
621 CONCORD_BDB_close (feature->db);
631 if (feature->db == NULL)
633 CONCORD_Genre genre = feature->genre;
635 if ( concord_opened_feature_list_length
636 >= concord_opened_feature_list_limit )
638 CONCORD_Feature top_feature = concord_opened_feature_list;
640 concord_opened_feature_list = top_feature->next;
641 if ( top_feature->number_of_hits >= 1 )
643 if ( concord_tenure_feature_list == NULL )
645 concord_tenure_feature_list = top_feature;
646 concord_tenure_feature_list_last = top_feature;
650 concord_tenure_feature_list_last->next = feature;
651 concord_tenure_feature_list_last = feature;
653 concord_tenure_feature_list_length++;
654 top_feature->next = NULL;
656 printf ("<<%02d: feature `%s' (%d hits, %d closes) was changed to tenure>>\n",
657 concord_tenure_feature_list_length,
659 top_feature->number_of_hits,
660 top_feature->number_of_closes);
666 if ( (top_feature->number_of_closes >= 2048) &&
667 (top_feature->number_of_hits >= 0) )
668 printf ("{{%02d: close feature `%s' (%d hits, %d closes)}}\n",
669 concord_opened_feature_list_length,
671 top_feature->number_of_hits,
672 top_feature->number_of_closes);
674 CONCORD_BDB_close (top_feature->db);
675 top_feature->db = NULL;
676 top_feature->next = NULL;
677 top_feature->number_of_hits = 0;
679 top_feature->number_of_closes++;
682 concord_opened_feature_list_length--;
685 = CONCORD_BDB_open (genre->ds->location, genre->name,
686 "feature", feature->name,
688 access, genre->ds->modemask);
689 if (feature->db == NULL)
691 feature->access = access;
692 if ( concord_opened_feature_list == NULL )
694 concord_opened_feature_list = feature;
695 concord_opened_feature_list_last = feature;
699 concord_opened_feature_list_last->next = feature;
700 concord_opened_feature_list_last = feature;
702 concord_opened_feature_list_length++;
706 feature->number_of_hits++;
712 concord_feature_sync (CONCORD_Feature feature)
716 if (feature->db == NULL)
719 status = CONCORD_BDB_close (feature->db);
726 concord_obj_put_feature_value_str (const char* object_id,
727 CONCORD_Feature feature,
728 unsigned char* value)
732 if (concord_feature_setup_db (feature, 1))
734 return CONCORD_BDB_put (feature->db, object_id, value);
738 concord_obj_get_feature_value_string (const char* object_id,
739 CONCORD_Feature feature,
740 CONCORD_String value)
744 if (concord_feature_setup_db (feature, 0))
746 status = CONCORD_BDB_get (feature->db, object_id, value);
751 concord_obj_get_feature_value (const char* object_id,
752 CONCORD_Feature feature)
757 if (concord_feature_setup_db (feature, 0))
758 return feature->genre->ds->object_nil;
759 status = CONCORD_BDB_get (feature->db, object_id, &valdatum);
761 return feature->genre->ds->object_nil;
762 return (*feature->genre->ds->read_object)(valdatum.data, valdatum.size);
766 concord_obj_gets_feature_value (const char* object_id,
767 CONCORD_Feature feature,
768 unsigned char* dst, size_t size)
773 if (concord_feature_setup_db (feature, 0))
775 status = CONCORD_BDB_get (feature->db, object_id, &valdatum);
778 if (size < valdatum.size)
780 strncpy ((char*)dst, valdatum.data, valdatum.size);
781 dst[valdatum.size] = '\0';
786 concord_feature_foreach_obj_string (CONCORD_Feature feature,
787 int (*func)(CONCORD_String object_id,
788 CONCORD_Feature feature,
789 CONCORD_String value))
791 CONCORD_String_Tank key, value;
795 if (concord_feature_setup_db (feature, 0))
800 status = feature->db->cursor (feature->db, NULL, &dbcp, 0);
801 for (status = dbcp->c_get (dbcp, &key, &value, DB_FIRST);
803 status = dbcp->c_get (dbcp, &key, &value, DB_NEXT))
805 int ret = func (&key, feature, &value);
810 dbcp->c_close (dbcp);
816 concord_genre_open_index (CONCORD_Genre genre, const char* index)
819 size_t len = strlen (index);
824 table = COS_ALLOCATE_OBJECT (Feature_INDEX);
828 table->genre = genre;
831 table->name = (char*)malloc (len + 1);
832 if (table->name == NULL)
837 strcpy (table->name, index);
838 table->decoding_table = NULL;
844 concord_close_index (CONCORD_INDEX table)
851 if (table->db == NULL)
854 status = CONCORD_BDB_close (table->db);
856 if (table->name == NULL)
868 concord_index_get_name (CONCORD_INDEX index)
874 concord_index_get_genre (CONCORD_INDEX index)
880 concord_index_setup_db (CONCORD_INDEX index, int writable)
889 if ((index->access & DB_CREATE) == 0)
891 if (index->db != NULL)
893 CONCORD_BDB_close (index->db);
903 if (index->db == NULL)
905 CONCORD_Genre genre = index->genre;
907 if ( concord_opened_index_list_length
908 >= concord_opened_index_list_limit )
910 CONCORD_INDEX top_index = concord_opened_index_list;
912 CONCORD_BDB_close (top_index->db);
913 top_index->db = NULL;
914 concord_opened_index_list = top_index->next;
915 top_index->next = NULL;
916 concord_opened_index_list_length--;
919 = CONCORD_BDB_open (genre->ds->location, genre->name,
920 "index", index->name,
922 access, genre->ds->modemask);
923 if (index->db == NULL)
925 index->access = access;
926 if ( concord_opened_index_list == NULL )
928 concord_opened_index_list = index;
929 concord_opened_index_list_last = index;
933 concord_opened_index_list_last->next = index;
934 concord_opened_index_list_last = index;
936 concord_opened_index_list_length++;
942 concord_index_sync (CONCORD_INDEX index)
946 if (index->db == NULL)
949 status = CONCORD_BDB_close (index->db);
956 concord_index_strid_put_obj (CONCORD_INDEX index,
957 const char* strid, char* object_id)
962 if (concord_index_setup_db (index, 1))
965 return CONCORD_BDB_put (index->db, strid, (unsigned char*)object_id);
969 concord_index_strid_get_obj_string (CONCORD_INDEX index,
971 CONCORD_String object_id)
976 if (concord_index_setup_db (index, 0))
979 return CONCORD_BDB_get (index->db, strid, object_id);