X-Git-Url: http://git.chise.org/gitweb/?p=chise%2Fxemacs-chise.git.1;a=blobdiff_plain;f=src%2Ffile-coding.c;h=763acaed80e5f4559ee1e05712203c0e3acf0423;hp=9db68c73b2ab120622de68dc3941c0d2dbbf15f3;hb=414b512c0774e67ba8e160b605447d862d3be166;hpb=3890a2e3064a7f562107c58e59d928284ec04741 diff --git a/src/file-coding.c b/src/file-coding.c index 9db68c7..763acae 100644 --- a/src/file-coding.c +++ b/src/file-coding.c @@ -46,27 +46,27 @@ Lisp_Object Vcoding_system_for_write; Lisp_Object Vfile_name_coding_system; /* Table of symbols identifying each coding category. */ -Lisp_Object coding_category_symbol[CODING_CATEGORY_LAST + 1]; +Lisp_Object coding_category_symbol[CODING_CATEGORY_LAST]; struct file_coding_dump { /* Coding system currently associated with each coding category. */ - Lisp_Object coding_category_system[CODING_CATEGORY_LAST + 1]; + Lisp_Object coding_category_system[CODING_CATEGORY_LAST]; /* Table of all coding categories in decreasing order of priority. This describes a permutation of the possible coding categories. */ - int coding_category_by_priority[CODING_CATEGORY_LAST + 1]; + int coding_category_by_priority[CODING_CATEGORY_LAST]; -#ifdef MULE +#if defined(MULE) && !defined(UTF2000) Lisp_Object ucs_to_mule_table[65536]; #endif } *fcd; static const struct lrecord_description fcd_description_1[] = { - { XD_LISP_OBJECT_ARRAY, offsetof (struct file_coding_dump, coding_category_system), CODING_CATEGORY_LAST + 1 }, -#ifdef MULE - { XD_LISP_OBJECT_ARRAY, offsetof (struct file_coding_dump, ucs_to_mule_table), 65536 }, + { XD_LISP_OBJECT_ARRAY, offsetof (struct file_coding_dump, coding_category_system), CODING_CATEGORY_LAST }, +#if defined(MULE) && !defined(UTF2000) + { XD_LISP_OBJECT_ARRAY, offsetof (struct file_coding_dump, ucs_to_mule_table), countof (fcd->ucs_to_mule_table) }, #endif { XD_END } }; @@ -176,68 +176,47 @@ EXFUN (Fcopy_coding_system, 2); #ifdef MULE struct detection_state; static int detect_coding_sjis (struct detection_state *st, - const unsigned char *src, - unsigned int n); -static void decode_coding_sjis (Lstream *decoding, - const unsigned char *src, - unsigned_char_dynarr *dst, - unsigned int n); -static void encode_coding_sjis (Lstream *encoding, - const unsigned char *src, - unsigned_char_dynarr *dst, - unsigned int n); + const Extbyte *src, Lstream_data_count n); +static void decode_coding_sjis (Lstream *decoding, const Extbyte *src, + unsigned_char_dynarr *dst, Lstream_data_count n); +static void encode_coding_sjis (Lstream *encoding, const Bufbyte *src, + unsigned_char_dynarr *dst, Lstream_data_count n); static int detect_coding_big5 (struct detection_state *st, - const unsigned char *src, - unsigned int n); -static void decode_coding_big5 (Lstream *decoding, - const unsigned char *src, - unsigned_char_dynarr *dst, unsigned int n); -static void encode_coding_big5 (Lstream *encoding, - const unsigned char *src, - unsigned_char_dynarr *dst, unsigned int n); + const Extbyte *src, Lstream_data_count n); +static void decode_coding_big5 (Lstream *decoding, const Extbyte *src, + unsigned_char_dynarr *dst, Lstream_data_count n); +static void encode_coding_big5 (Lstream *encoding, const Bufbyte *src, + unsigned_char_dynarr *dst, Lstream_data_count n); static int detect_coding_ucs4 (struct detection_state *st, - const unsigned char *src, - unsigned int n); -static void decode_coding_ucs4 (Lstream *decoding, - const unsigned char *src, - unsigned_char_dynarr *dst, unsigned int n); -static void encode_coding_ucs4 (Lstream *encoding, - const unsigned char *src, - unsigned_char_dynarr *dst, unsigned int n); + const Extbyte *src, Lstream_data_count n); +static void decode_coding_ucs4 (Lstream *decoding, const Extbyte *src, + unsigned_char_dynarr *dst, Lstream_data_count n); +static void encode_coding_ucs4 (Lstream *encoding, const Bufbyte *src, + unsigned_char_dynarr *dst, Lstream_data_count n); static int detect_coding_utf8 (struct detection_state *st, - const unsigned char *src, - unsigned int n); -static void decode_coding_utf8 (Lstream *decoding, - const unsigned char *src, - unsigned_char_dynarr *dst, unsigned int n); -static void encode_coding_utf8 (Lstream *encoding, - const unsigned char *src, - unsigned_char_dynarr *dst, unsigned int n); + const Extbyte *src, Lstream_data_count n); +static void decode_coding_utf8 (Lstream *decoding, const Extbyte *src, + unsigned_char_dynarr *dst, Lstream_data_count n); +static void encode_coding_utf8 (Lstream *encoding, const Bufbyte *src, + unsigned_char_dynarr *dst, Lstream_data_count n); static int postprocess_iso2022_mask (int mask); static void reset_iso2022 (Lisp_Object coding_system, struct iso2022_decoder *iso); static int detect_coding_iso2022 (struct detection_state *st, - const unsigned char *src, - unsigned int n); -static void decode_coding_iso2022 (Lstream *decoding, - const unsigned char *src, - unsigned_char_dynarr *dst, unsigned int n); -static void encode_coding_iso2022 (Lstream *encoding, - const unsigned char *src, - unsigned_char_dynarr *dst, unsigned int n); + const Extbyte *src, Lstream_data_count n); +static void decode_coding_iso2022 (Lstream *decoding, const Extbyte *src, + unsigned_char_dynarr *dst, Lstream_data_count n); +static void encode_coding_iso2022 (Lstream *encoding, const Bufbyte *src, + unsigned_char_dynarr *dst, Lstream_data_count n); #endif /* MULE */ -static void decode_coding_no_conversion (Lstream *decoding, - const unsigned char *src, - unsigned_char_dynarr *dst, - unsigned int n); -static void encode_coding_no_conversion (Lstream *encoding, - const unsigned char *src, - unsigned_char_dynarr *dst, - unsigned int n); -static void mule_decode (Lstream *decoding, const unsigned char *src, - unsigned_char_dynarr *dst, unsigned int n); -static void mule_encode (Lstream *encoding, const unsigned char *src, - unsigned_char_dynarr *dst, unsigned int n); +static void decode_coding_no_conversion (Lstream *decoding, const Extbyte *src, + unsigned_char_dynarr *dst, Lstream_data_count n); +static void encode_coding_no_conversion (Lstream *encoding, const Bufbyte *src, + unsigned_char_dynarr *dst, Lstream_data_count n); +static void mule_decode (Lstream *decoding, const Extbyte *src, + unsigned_char_dynarr *dst, Lstream_data_count n); +static void mule_encode (Lstream *encoding, const Bufbyte *src, + unsigned_char_dynarr *dst, Lstream_data_count n); typedef struct codesys_prop codesys_prop; struct codesys_prop @@ -454,7 +433,7 @@ eol_type_to_symbol (eol_type_t type) { switch (type) { - default: abort (); + default: ABORT (); case EOL_LF: return Qlf; case EOL_CRLF: return Qcrlf; case EOL_CR: return Qcr; @@ -732,7 +711,7 @@ nil or 'undecided JIS (the Japanese encoding commonly used for e-mail), EUC (the standard Unix encoding for Japanese and other languages), and Compound Text (the encoding used in X11). You can specify more - specific information about the conversion with the FLAGS argument. + specific information about the conversion with the PROPS argument. 'big5 Big5 (the encoding commonly used for Taiwanese). 'ccl @@ -789,12 +768,12 @@ character set. Recognized properties are: 'post-read-conversion Function called after a file has been read in, to perform the - decoding. Called with two arguments, BEG and END, denoting + decoding. Called with two arguments, START and END, denoting a region of the current buffer to be decoded. 'pre-write-conversion Function called before a file is written out, to perform the - encoding. Called with two arguments, BEG and END, denoting + encoding. Called with two arguments, START and END, denoting a region of the current buffer to be encoded. @@ -882,7 +861,6 @@ if TYPE is 'ccl: (name, type, doc_string, props)) { Lisp_Coding_System *codesys; - Lisp_Object rest, key, value; enum coding_system_type ty; int need_to_setup_eol_systems = 1; @@ -914,92 +892,118 @@ if TYPE is 'ccl: CHECK_STRING (doc_string); CODING_SYSTEM_DOC_STRING (codesys) = doc_string; - EXTERNAL_PROPERTY_LIST_LOOP (rest, key, value, props) - { - if (EQ (key, Qmnemonic)) - { - if (!NILP (value)) - CHECK_STRING (value); - CODING_SYSTEM_MNEMONIC (codesys) = value; - } + { + EXTERNAL_PROPERTY_LIST_LOOP_3 (key, value, props) + { + if (EQ (key, Qmnemonic)) + { + if (!NILP (value)) + CHECK_STRING (value); + CODING_SYSTEM_MNEMONIC (codesys) = value; + } - else if (EQ (key, Qeol_type)) - { - need_to_setup_eol_systems = NILP (value); - if (EQ (value, Qt)) - value = Qnil; - CODING_SYSTEM_EOL_TYPE (codesys) = symbol_to_eol_type (value); - } + else if (EQ (key, Qeol_type)) + { + need_to_setup_eol_systems = NILP (value); + if (EQ (value, Qt)) + value = Qnil; + CODING_SYSTEM_EOL_TYPE (codesys) = symbol_to_eol_type (value); + } - else if (EQ (key, Qpost_read_conversion)) CODING_SYSTEM_POST_READ_CONVERSION (codesys) = value; - else if (EQ (key, Qpre_write_conversion)) CODING_SYSTEM_PRE_WRITE_CONVERSION (codesys) = value; + else if (EQ (key, Qpost_read_conversion)) CODING_SYSTEM_POST_READ_CONVERSION (codesys) = value; + else if (EQ (key, Qpre_write_conversion)) CODING_SYSTEM_PRE_WRITE_CONVERSION (codesys) = value; #ifdef MULE - else if (ty == CODESYS_ISO2022) - { + else if (ty == CODESYS_ISO2022) + { #define FROB_INITIAL_CHARSET(charset_num) \ CODING_SYSTEM_ISO2022_INITIAL_CHARSET (codesys, charset_num) = \ ((EQ (value, Qt) || EQ (value, Qnil)) ? value : Fget_charset (value)) - if (EQ (key, Qcharset_g0)) FROB_INITIAL_CHARSET (0); - else if (EQ (key, Qcharset_g1)) FROB_INITIAL_CHARSET (1); - else if (EQ (key, Qcharset_g2)) FROB_INITIAL_CHARSET (2); - else if (EQ (key, Qcharset_g3)) FROB_INITIAL_CHARSET (3); + if (EQ (key, Qcharset_g0)) FROB_INITIAL_CHARSET (0); + else if (EQ (key, Qcharset_g1)) FROB_INITIAL_CHARSET (1); + else if (EQ (key, Qcharset_g2)) FROB_INITIAL_CHARSET (2); + else if (EQ (key, Qcharset_g3)) FROB_INITIAL_CHARSET (3); #define FROB_FORCE_CHARSET(charset_num) \ CODING_SYSTEM_ISO2022_FORCE_CHARSET_ON_OUTPUT (codesys, charset_num) = !NILP (value) - else if (EQ (key, Qforce_g0_on_output)) FROB_FORCE_CHARSET (0); - else if (EQ (key, Qforce_g1_on_output)) FROB_FORCE_CHARSET (1); - else if (EQ (key, Qforce_g2_on_output)) FROB_FORCE_CHARSET (2); - else if (EQ (key, Qforce_g3_on_output)) FROB_FORCE_CHARSET (3); + else if (EQ (key, Qforce_g0_on_output)) FROB_FORCE_CHARSET (0); + else if (EQ (key, Qforce_g1_on_output)) FROB_FORCE_CHARSET (1); + else if (EQ (key, Qforce_g2_on_output)) FROB_FORCE_CHARSET (2); + else if (EQ (key, Qforce_g3_on_output)) FROB_FORCE_CHARSET (3); #define FROB_BOOLEAN_PROPERTY(prop) \ CODING_SYSTEM_ISO2022_##prop (codesys) = !NILP (value) - else if (EQ (key, Qshort)) FROB_BOOLEAN_PROPERTY (SHORT); - else if (EQ (key, Qno_ascii_eol)) FROB_BOOLEAN_PROPERTY (NO_ASCII_EOL); - else if (EQ (key, Qno_ascii_cntl)) FROB_BOOLEAN_PROPERTY (NO_ASCII_CNTL); - else if (EQ (key, Qseven)) FROB_BOOLEAN_PROPERTY (SEVEN); - else if (EQ (key, Qlock_shift)) FROB_BOOLEAN_PROPERTY (LOCK_SHIFT); - else if (EQ (key, Qno_iso6429)) FROB_BOOLEAN_PROPERTY (NO_ISO6429); - else if (EQ (key, Qescape_quoted)) FROB_BOOLEAN_PROPERTY (ESCAPE_QUOTED); + else if (EQ (key, Qshort)) FROB_BOOLEAN_PROPERTY (SHORT); + else if (EQ (key, Qno_ascii_eol)) FROB_BOOLEAN_PROPERTY (NO_ASCII_EOL); + else if (EQ (key, Qno_ascii_cntl)) FROB_BOOLEAN_PROPERTY (NO_ASCII_CNTL); + else if (EQ (key, Qseven)) FROB_BOOLEAN_PROPERTY (SEVEN); + else if (EQ (key, Qlock_shift)) FROB_BOOLEAN_PROPERTY (LOCK_SHIFT); + else if (EQ (key, Qno_iso6429)) FROB_BOOLEAN_PROPERTY (NO_ISO6429); + else if (EQ (key, Qescape_quoted)) FROB_BOOLEAN_PROPERTY (ESCAPE_QUOTED); + + else if (EQ (key, Qinput_charset_conversion)) + { + codesys->iso2022.input_conv = + Dynarr_new (charset_conversion_spec); + parse_charset_conversion_specs (codesys->iso2022.input_conv, + value); + } + else if (EQ (key, Qoutput_charset_conversion)) + { + codesys->iso2022.output_conv = + Dynarr_new (charset_conversion_spec); + parse_charset_conversion_specs (codesys->iso2022.output_conv, + value); + } + else + signal_simple_error ("Unrecognized property", key); + } + else if (EQ (type, Qccl)) + { + Lisp_Object sym; + struct ccl_program test_ccl; + Extbyte *suffix; + + /* Check key first. */ + if (EQ (key, Qdecode)) + suffix = "-ccl-decode"; + else if (EQ (key, Qencode)) + suffix = "-ccl-encode"; + else + signal_simple_error ("Unrecognized property", key); + + /* If value is vector, register it as a ccl program + associated with an newly created symbol for + backward compatibility. */ + if (VECTORP (value)) + { + sym = Fintern (concat2 (Fsymbol_name (name), + build_string (suffix)), + Qnil); + Fregister_ccl_program (sym, value); + } + else + { + CHECK_SYMBOL (value); + sym = value; + } + /* check if the given ccl programs are valid. */ + if (setup_ccl_program (&test_ccl, sym) < 0) + signal_simple_error ("Invalid CCL program", value); + + if (EQ (key, Qdecode)) + CODING_SYSTEM_CCL_DECODE (codesys) = sym; + else if (EQ (key, Qencode)) + CODING_SYSTEM_CCL_ENCODE (codesys) = sym; - else if (EQ (key, Qinput_charset_conversion)) - { - codesys->iso2022.input_conv = - Dynarr_new (charset_conversion_spec); - parse_charset_conversion_specs (codesys->iso2022.input_conv, - value); - } - else if (EQ (key, Qoutput_charset_conversion)) - { - codesys->iso2022.output_conv = - Dynarr_new (charset_conversion_spec); - parse_charset_conversion_specs (codesys->iso2022.output_conv, - value); - } - else - signal_simple_error ("Unrecognized property", key); - } - else if (EQ (type, Qccl)) - { - if (EQ (key, Qdecode)) - { - CHECK_VECTOR (value); - CODING_SYSTEM_CCL_DECODE (codesys) = value; - } - else if (EQ (key, Qencode)) - { - CHECK_VECTOR (value); - CODING_SYSTEM_CCL_ENCODE (codesys) = value; - } - else - signal_simple_error ("Unrecognized property", key); - } + } #endif /* MULE */ - else - signal_simple_error ("Unrecognized property", key); - } + else + signal_simple_error ("Unrecognized property", key); + } + } if (need_to_setup_eol_systems) setup_eol_coding_systems (codesys); @@ -1071,6 +1075,7 @@ Return the coding-system symbol for which symbol ALIAS is an alias. return aliasee; else signal_simple_error ("Symbol is not a coding system alias", alias); + return Qnil; /* To keep the compiler happy */ } static Lisp_Object @@ -1217,7 +1222,7 @@ subsidiary_coding_system (Lisp_Object coding_system, eol_type_t type) case EOL_LF: new_coding_system = CODING_SYSTEM_EOL_LF (cs); break; case EOL_CR: new_coding_system = CODING_SYSTEM_EOL_CR (cs); break; case EOL_CRLF: new_coding_system = CODING_SYSTEM_EOL_CRLF (cs); break; - default: abort (); + default: ABORT (); return Qnil; } return NILP (new_coding_system) ? coding_system : new_coding_system; @@ -1255,7 +1260,7 @@ Return the type of CODING-SYSTEM. { switch (XCODING_SYSTEM_TYPE (Fget_coding_system (coding_system))) { - default: abort (); + default: ABORT (); case CODESYS_AUTODETECT: return Qundecided; #ifdef MULE case CODESYS_SHIFT_JIS: return Qshift_jis; @@ -1331,7 +1336,7 @@ Return the PROP property of CODING-SYSTEM. break; #endif /* MULE */ default: - abort (); + ABORT (); } } @@ -1399,7 +1404,7 @@ Return the PROP property of CODING-SYSTEM. unparse_charset_conversion_specs (XCODING_SYSTEM (coding_system)->iso2022.output_conv); else - abort (); + ABORT (); } else if (type == CODESYS_CCL) { @@ -1408,11 +1413,11 @@ Return the PROP property of CODING-SYSTEM. else if (EQ (prop, Qencode)) return XCODING_SYSTEM_CCL_ENCODE (coding_system); else - abort (); + ABORT (); } #endif /* MULE */ else - abort (); + ABORT (); return Qnil; /* not reached */ } @@ -1428,7 +1433,7 @@ decode_coding_category (Lisp_Object symbol) int i; CHECK_SYMBOL (symbol); - for (i = 0; i <= CODING_CATEGORY_LAST; i++) + for (i = 0; i < CODING_CATEGORY_LAST; i++) if (EQ (coding_category_symbol[i], symbol)) return i; @@ -1444,7 +1449,7 @@ Return a list of all recognized coding categories. int i; Lisp_Object list = Qnil; - for (i = CODING_CATEGORY_LAST; i >= 0; i--) + for (i = CODING_CATEGORY_LAST - 1; i >= 0; i--) list = Fcons (coding_category_symbol[i], list); return list; } @@ -1458,13 +1463,13 @@ previously. */ (list)) { - int category_to_priority[CODING_CATEGORY_LAST + 1]; + int category_to_priority[CODING_CATEGORY_LAST]; int i, j; Lisp_Object rest; /* First generate a list that maps coding categories to priorities. */ - for (i = 0; i <= CODING_CATEGORY_LAST; i++) + for (i = 0; i < CODING_CATEGORY_LAST; i++) category_to_priority[i] = -1; /* Highest priority comes from the specified list. */ @@ -1481,7 +1486,7 @@ previously. /* Now go through the existing categories by priority to retrieve the categories not yet specified and preserve their priority order. */ - for (j = 0; j <= CODING_CATEGORY_LAST; j++) + for (j = 0; j < CODING_CATEGORY_LAST; j++) { int cat = fcd->coding_category_by_priority[j]; if (category_to_priority[cat] < 0) @@ -1491,7 +1496,7 @@ previously. /* Now we need to construct the inverse of the mapping we just constructed. */ - for (i = 0; i <= CODING_CATEGORY_LAST; i++) + for (i = 0; i < CODING_CATEGORY_LAST; i++) fcd->coding_category_by_priority[category_to_priority[i]] = i; /* Phew! That was confusing. */ @@ -1506,7 +1511,7 @@ Return a list of coding categories in descending order of priority. int i; Lisp_Object list = Qnil; - for (i = CODING_CATEGORY_LAST; i >= 0; i--) + for (i = CODING_CATEGORY_LAST - 1; i >= 0; i--) list = Fcons (coding_category_symbol[fcd->coding_category_by_priority[i]], list); return list; @@ -1626,14 +1631,12 @@ mask_has_at_most_one_bit_p (int mask) } static eol_type_t -detect_eol_type (struct detection_state *st, const unsigned char *src, - unsigned int n) +detect_eol_type (struct detection_state *st, const Extbyte *src, + Lstream_data_count n) { - int c; - while (n--) { - c = *src++; + unsigned char c = *(unsigned char *)src++; if (c == '\n') { if (st->eol.just_saw_cr) @@ -1672,10 +1675,8 @@ detect_eol_type (struct detection_state *st, const unsigned char *src, static int detect_coding_type (struct detection_state *st, const Extbyte *src, - unsigned int n, int just_do_eol) + Lstream_data_count n, int just_do_eol) { - int c; - if (st->eol_type == EOL_AUTODETECT) st->eol_type = detect_eol_type (st, src, n); @@ -1686,7 +1687,7 @@ detect_coding_type (struct detection_state *st, const Extbyte *src, { for (; n; n--, src++) { - c = *src; + unsigned char c = *(unsigned char *) src; if ((c < 0x20 && !acceptable_control_char_p (c)) || c >= 0x80) { st->seen_non_ascii = 1; @@ -1760,7 +1761,7 @@ coding_system_from_mask (int mask) #endif /* Look through the coding categories by priority and find the first one that is allowed. */ - for (i = 0; i <= CODING_CATEGORY_LAST; i++) + for (i = 0; i < CODING_CATEGORY_LAST; i++) { cat = fcd->coding_category_by_priority[i]; if ((mask & (1 << cat)) && @@ -1784,6 +1785,8 @@ coding_system_from_mask (int mask) that should be unnecessary with the explicit eol-type argument. */ #define LENGTH(string_constant) (sizeof (string_constant) - 1) +/* number of leading lines to check for a coding cookie */ +#define LINES_TO_CHECK 2 void determine_real_coding_system (Lstream *stream, Lisp_Object *codesys_in_out, @@ -1805,15 +1808,15 @@ determine_real_coding_system (Lstream *stream, Lisp_Object *codesys_in_out, Extbyte buf[4096]; Lisp_Object coding_system = Qnil; Extbyte *p; - ssize_t nread = Lstream_read (stream, buf, sizeof (buf)); + Lstream_data_count nread = Lstream_read (stream, buf, sizeof (buf)); Extbyte *scan_end; + int lines_checked = 0; /* Look for initial "-*-"; mode line prefix */ for (p = buf, scan_end = buf + nread - LENGTH ("-*-coding:?-*-"); p <= scan_end - && *p != '\n' - && *p != '\r'; + && lines_checked < LINES_TO_CHECK; p++) if (*p == '-' && *(p+1) == '*' && *(p+2) == '-') { @@ -1822,8 +1825,7 @@ determine_real_coding_system (Lstream *stream, Lisp_Object *codesys_in_out, for (p = local_vars_beg, scan_end = buf + nread - LENGTH ("-*-"); p <= scan_end - && *p != '\n' - && *p != '\r'; + && lines_checked < LINES_TO_CHECK; p++) if (*p == '-' && *(p+1) == '*' && *(p+2) == '-') { @@ -1865,8 +1867,24 @@ determine_real_coding_system (Lstream *stream, Lisp_Object *codesys_in_out, } break; } + /* #### file must use standard EOLs or we miss 2d line */ + /* #### not to mention this is broken for UTF-16 DOS files */ + else if (*p == '\n' || *p == '\r') + { + lines_checked++; + /* skip past multibyte (DOS) newline */ + if (*p == '\r' && *(p+1) == '\n') p++; + } break; } + /* #### file must use standard EOLs or we miss 2d line */ + /* #### not to mention this is broken for UTF-16 DOS files */ + else if (*p == '\n' || *p == '\r') + { + lines_checked++; + /* skip past multibyte (DOS) newline */ + if (*p == '\r' && *(p+1) == '\n') p++; + } if (NILP (coding_system)) do @@ -1912,8 +1930,8 @@ determine_real_coding_system (Lstream *stream, Lisp_Object *codesys_in_out, DEFUN ("detect-coding-region", Fdetect_coding_region, 2, 3, 0, /* Detect coding system of the text in the region between START and END. -Returned a list of possible coding systems ordered by priority. -If only ASCII characters are found, it returns 'undecided or one of +Return a list of possible coding systems ordered by priority. +If only ASCII characters are found, return 'undecided or one of its subsidiary coding systems according to a detected end-of-line type. Optional arg BUFFER defaults to the current buffer. */ @@ -1938,8 +1956,8 @@ type. Optional arg BUFFER defaults to the current buffer. decst.mask = ~0; while (1) { - unsigned char random_buffer[4096]; - ssize_t nread = Lstream_read (istr, random_buffer, sizeof (random_buffer)); + Extbyte random_buffer[4096]; + Lstream_data_count nread = Lstream_read (istr, random_buffer, sizeof (random_buffer)); if (!nread) break; @@ -1958,7 +1976,7 @@ type. Optional arg BUFFER defaults to the current buffer. #ifdef MULE decst.mask = postprocess_iso2022_mask (decst.mask); #endif - for (i = CODING_CATEGORY_LAST; i >= 0; i--) + for (i = CODING_CATEGORY_LAST - 1; i >= 0; i--) { int sys = fcd->coding_category_by_priority[i]; if (decst.mask & (1 << sys)) @@ -2025,6 +2043,62 @@ do { \ /* C should be a binary character in the range 0 - 255; convert to internal format and add to Dynarr DST. */ +#ifdef UTF2000 +#define DECODE_ADD_BINARY_CHAR(c, dst) \ +do { \ + if (BYTE_ASCII_P (c)) \ + Dynarr_add (dst, c); \ + else \ + { \ + Dynarr_add (dst, (c >> 6) | 0xc0); \ + Dynarr_add (dst, (c & 0x3f) | 0x80); \ + } \ +} while (0) + +INLINE void +DECODE_ADD_UCS_CHAR(Emchar c, unsigned_char_dynarr* dst) +{ + if ( c <= 0x7f ) + { + Dynarr_add (dst, c); + } + else if ( c <= 0x7ff ) + { + Dynarr_add (dst, (c >> 6) | 0xc0); + Dynarr_add (dst, (c & 0x3f) | 0x80); + } + else if ( c <= 0xffff ) + { + Dynarr_add (dst, (c >> 12) | 0xe0); + Dynarr_add (dst, ((c >> 6) & 0x3f) | 0x80); + Dynarr_add (dst, (c & 0x3f) | 0x80); + } + else if ( c <= 0x1fffff ) + { + Dynarr_add (dst, (c >> 18) | 0xf0); + Dynarr_add (dst, ((c >> 12) & 0x3f) | 0x80); + Dynarr_add (dst, ((c >> 6) & 0x3f) | 0x80); + Dynarr_add (dst, (c & 0x3f) | 0x80); + } + else if ( c <= 0x3ffffff ) + { + Dynarr_add (dst, (c >> 24) | 0xf8); + Dynarr_add (dst, ((c >> 18) & 0x3f) | 0x80); + Dynarr_add (dst, ((c >> 12) & 0x3f) | 0x80); + Dynarr_add (dst, ((c >> 6) & 0x3f) | 0x80); + Dynarr_add (dst, (c & 0x3f) | 0x80); + } + else + { + Dynarr_add (dst, (c >> 30) | 0xfc); + Dynarr_add (dst, ((c >> 24) & 0x3f) | 0x80); + Dynarr_add (dst, ((c >> 18) & 0x3f) | 0x80); + Dynarr_add (dst, ((c >> 12) & 0x3f) | 0x80); + Dynarr_add (dst, ((c >> 6) & 0x3f) | 0x80); + Dynarr_add (dst, (c & 0x3f) | 0x80); + } +} +#else #define DECODE_ADD_BINARY_CHAR(c, dst) \ do { \ if (BYTE_ASCII_P (c)) \ @@ -2040,6 +2114,7 @@ do { \ Dynarr_add (dst, c); \ } \ } while (0) +#endif #define DECODE_OUTPUT_PARTIAL_CHAR(ch) \ do { \ @@ -2106,10 +2181,10 @@ struct decoding_stream struct detection_state decst; }; -static ssize_t decoding_reader (Lstream *stream, - unsigned char *data, size_t size); -static ssize_t decoding_writer (Lstream *stream, - const unsigned char *data, size_t size); +static Lstream_data_count decoding_reader (Lstream *stream, + unsigned char *data, Lstream_data_count size); +static Lstream_data_count decoding_writer (Lstream *stream, + const unsigned char *data, Lstream_data_count size); static int decoding_rewinder (Lstream *stream); static int decoding_seekable_p (Lstream *stream); static int decoding_flusher (Lstream *stream); @@ -2141,12 +2216,12 @@ decoding_marker (Lisp_Object stream) /* Read SIZE bytes of data and store it into DATA. We are a decoding stream so we read data from the other end, decode it, and store it into DATA. */ -static ssize_t -decoding_reader (Lstream *stream, unsigned char *data, size_t size) +static Lstream_data_count +decoding_reader (Lstream *stream, unsigned char *data, Lstream_data_count size) { struct decoding_stream *str = DECODING_STREAM_DATA (stream); unsigned char *orig_data = data; - ssize_t read_size; + Lstream_data_count read_size; int error_occurred = 0; /* We need to interface to mule_decode(), which expects to take some @@ -2162,7 +2237,7 @@ decoding_reader (Lstream *stream, unsigned char *data, size_t size) most SIZE bytes, and delete the data from the runoff. */ if (Dynarr_length (str->runoff) > 0) { - size_t chunk = min (size, (size_t) Dynarr_length (str->runoff)); + Lstream_data_count chunk = min (size, (Lstream_data_count) Dynarr_length (str->runoff)); memcpy (data, Dynarr_atp (str->runoff, 0), chunk); Dynarr_delete_many (str->runoff, 0, chunk); data += chunk; @@ -2194,7 +2269,7 @@ decoding_reader (Lstream *stream, unsigned char *data, size_t size) /* There might be some more end data produced in the translation. See the comment above. */ str->flags |= CODING_STATE_END; - mule_decode (stream, data, str->runoff, read_size); + mule_decode (stream, (Extbyte *) data, str->runoff, read_size); } if (data - orig_data == 0) @@ -2203,16 +2278,16 @@ decoding_reader (Lstream *stream, unsigned char *data, size_t size) return data - orig_data; } -static ssize_t -decoding_writer (Lstream *stream, const unsigned char *data, size_t size) +static Lstream_data_count +decoding_writer (Lstream *stream, const unsigned char *data, Lstream_data_count size) { struct decoding_stream *str = DECODING_STREAM_DATA (stream); - ssize_t retval; + Lstream_data_count retval; /* Decode all our data into the runoff, and then attempt to write it all out to the other end. Remove whatever chunk we succeeded in writing. */ - mule_decode (stream, data, str->runoff, size); + mule_decode (stream, (Extbyte *) data, str->runoff, size); retval = Lstream_write (str->other_end, Dynarr_atp (str->runoff, 0), Dynarr_length (str->runoff)); if (retval > 0) @@ -2239,6 +2314,13 @@ reset_decoding_stream (struct decoding_stream *str) } str->counter = 0; #endif /* MULE */ + if (CODING_SYSTEM_TYPE (str->codesys) == CODESYS_AUTODETECT + || CODING_SYSTEM_EOL_TYPE (str->codesys) == EOL_AUTODETECT) + { + xzero (str->decst); + str->decst.eol_type = EOL_AUTODETECT; + str->decst.mask = ~0; + } str->flags = str->ch = 0; } @@ -2357,8 +2439,8 @@ make_decoding_output_stream (Lstream *stream, Lisp_Object codesys) be used for both reading and writing. */ static void -mule_decode (Lstream *decoding, const unsigned char *src, - unsigned_char_dynarr *dst, unsigned int n) +mule_decode (Lstream *decoding, const Extbyte *src, + unsigned_char_dynarr *dst, Lstream_data_count n) { struct decoding_stream *str = DECODING_STREAM_DATA (decoding); @@ -2422,14 +2504,17 @@ mule_decode (Lstream *decoding, const unsigned char *src, break; case CODESYS_CCL: str->ccl.last_block = str->flags & CODING_STATE_END; - ccl_driver (&str->ccl, src, dst, n, 0, CCL_MODE_DECODING); + /* When applying ccl program to stream, MUST NOT set NULL + pointer to src. */ + ccl_driver (&str->ccl, (src ? (unsigned char *)src : (unsigned char*)""), + dst, n, 0, CCL_MODE_DECODING); break; case CODESYS_ISO2022: decode_coding_iso2022 (decoding, src, dst, n); break; #endif /* MULE */ default: - abort (); + ABORT (); } } @@ -2477,7 +2562,7 @@ BUFFER defaults to the current buffer if unspecified. char tempbuf[1024]; /* some random amount */ Bufpos newpos, even_newer_pos; Bufpos oldpos = lisp_buffer_stream_startpos (istr); - ssize_t size_in_bytes = Lstream_read (istr, tempbuf, sizeof (tempbuf)); + Lstream_data_count size_in_bytes = Lstream_read (istr, tempbuf, sizeof (tempbuf)); if (!size_in_bytes) break; @@ -2562,9 +2647,9 @@ struct encoding_stream #endif /* MULE */ }; -static ssize_t encoding_reader (Lstream *stream, unsigned char *data, size_t size); -static ssize_t encoding_writer (Lstream *stream, const unsigned char *data, - size_t size); +static Lstream_data_count encoding_reader (Lstream *stream, unsigned char *data, Lstream_data_count size); +static Lstream_data_count encoding_writer (Lstream *stream, const unsigned char *data, + Lstream_data_count size); static int encoding_rewinder (Lstream *stream); static int encoding_seekable_p (Lstream *stream); static int encoding_flusher (Lstream *stream); @@ -2596,12 +2681,12 @@ encoding_marker (Lisp_Object stream) /* Read SIZE bytes of data and store it into DATA. We are a encoding stream so we read data from the other end, encode it, and store it into DATA. */ -static ssize_t -encoding_reader (Lstream *stream, unsigned char *data, size_t size) +static Lstream_data_count +encoding_reader (Lstream *stream, unsigned char *data, Lstream_data_count size) { struct encoding_stream *str = ENCODING_STREAM_DATA (stream); unsigned char *orig_data = data; - ssize_t read_size; + Lstream_data_count read_size; int error_occurred = 0; /* We need to interface to mule_encode(), which expects to take some @@ -2658,11 +2743,11 @@ encoding_reader (Lstream *stream, unsigned char *data, size_t size) return data - orig_data; } -static ssize_t -encoding_writer (Lstream *stream, const unsigned char *data, size_t size) +static Lstream_data_count +encoding_writer (Lstream *stream, const unsigned char *data, Lstream_data_count size) { struct encoding_stream *str = ENCODING_STREAM_DATA (stream); - ssize_t retval; + Lstream_data_count retval; /* Encode all our data into the runoff, and then attempt to write it all out to the other end. Remove whatever chunk we succeeded @@ -2699,7 +2784,11 @@ reset_encoding_stream (struct encoding_stream *str) str->iso2022.register_right = 1; str->iso2022.current_charset = Qnil; str->iso2022.current_half = 0; +#ifdef UTF2000 + str->iso2022.current_char_boundary = 0; +#else str->iso2022.current_char_boundary = 1; +#endif break; } case CODESYS_CCL: @@ -2801,8 +2890,8 @@ make_encoding_output_stream (Lstream *stream, Lisp_Object codesys) Store the encoded data into DST. */ static void -mule_encode (Lstream *encoding, const unsigned char *src, - unsigned_char_dynarr *dst, unsigned int n) +mule_encode (Lstream *encoding, const Bufbyte *src, + unsigned_char_dynarr *dst, Lstream_data_count n) { struct encoding_stream *str = ENCODING_STREAM_DATA (encoding); @@ -2834,14 +2923,17 @@ mule_encode (Lstream *encoding, const unsigned char *src, break; case CODESYS_CCL: str->ccl.last_block = str->flags & CODING_STATE_END; - ccl_driver (&str->ccl, src, dst, n, 0, CCL_MODE_ENCODING); + /* When applying ccl program to stream, MUST NOT set NULL + pointer to src. */ + ccl_driver (&str->ccl, ((src) ? src : (unsigned char*)""), + dst, n, 0, CCL_MODE_ENCODING); break; case CODESYS_ISO2022: encode_coding_iso2022 (encoding, src, dst, n); break; #endif /* MULE */ default: - abort (); + ABORT (); } } @@ -2885,7 +2977,7 @@ text. BUFFER defaults to the current buffer if unspecified. char tempbuf[1024]; /* some random amount */ Bufpos newpos, even_newer_pos; Bufpos oldpos = lisp_buffer_stream_startpos (istr); - ssize_t size_in_bytes = Lstream_read (istr, tempbuf, sizeof (tempbuf)); + Lstream_data_count size_in_bytes = Lstream_read (istr, tempbuf, sizeof (tempbuf)); if (!size_in_bytes) break; @@ -2948,14 +3040,11 @@ text. BUFFER defaults to the current buffer if unspecified. ((c) >= 0xA1 && (c) <= 0xDF) static int -detect_coding_sjis (struct detection_state *st, const unsigned char *src, - unsigned int n) +detect_coding_sjis (struct detection_state *st, const Extbyte *src, Lstream_data_count n) { - int c; - while (n--) { - c = *src++; + unsigned char c = *(unsigned char *)src++; if (c == ISO_CODE_ESC || c == ISO_CODE_SI || c == ISO_CODE_SO) return 0; if (st->shift_jis.in_second_byte) @@ -2973,10 +3062,9 @@ detect_coding_sjis (struct detection_state *st, const unsigned char *src, /* Convert Shift-JIS data to internal format. */ static void -decode_coding_sjis (Lstream *decoding, const unsigned char *src, - unsigned_char_dynarr *dst, unsigned int n) +decode_coding_sjis (Lstream *decoding, const Extbyte *src, + unsigned_char_dynarr *dst, Lstream_data_count n) { - unsigned char c; struct decoding_stream *str = DECODING_STREAM_DATA (decoding); unsigned int flags = str->flags; unsigned int ch = str->ch; @@ -2984,7 +3072,7 @@ decode_coding_sjis (Lstream *decoding, const unsigned char *src, while (n--) { - c = *src++; + unsigned char c = *(unsigned char *)src++; if (ch) { @@ -2993,10 +3081,16 @@ decode_coding_sjis (Lstream *decoding, const unsigned char *src, { unsigned char e1, e2; - Dynarr_add (dst, LEADING_BYTE_JAPANESE_JISX0208); DECODE_SJIS (ch, c, e1, e2); +#ifdef UTF2000 + DECODE_ADD_UCS_CHAR(MAKE_CHAR(Vcharset_japanese_jisx0208, + e1 & 0x7F, + e2 & 0x7F), dst); +#else + Dynarr_add (dst, LEADING_BYTE_JAPANESE_JISX0208); Dynarr_add (dst, e1); Dynarr_add (dst, e2); +#endif } else { @@ -3012,8 +3106,13 @@ decode_coding_sjis (Lstream *decoding, const unsigned char *src, ch = c; else if (BYTE_SJIS_KATAKANA_P (c)) { +#ifdef UTF2000 + DECODE_ADD_UCS_CHAR(MAKE_CHAR(Vcharset_katakana_jisx0201, + c & 0x7F, 0), dst); +#else Dynarr_add (dst, LEADING_BYTE_KATAKANA_JISX0201); Dynarr_add (dst, c); +#endif } else DECODE_ADD_BINARY_CHAR (c, dst); @@ -3030,18 +3129,89 @@ decode_coding_sjis (Lstream *decoding, const unsigned char *src, /* Convert internally-formatted data to Shift-JIS. */ static void -encode_coding_sjis (Lstream *encoding, const unsigned char *src, - unsigned_char_dynarr *dst, unsigned int n) +encode_coding_sjis (Lstream *encoding, const Bufbyte *src, + unsigned_char_dynarr *dst, Lstream_data_count n) { - unsigned char c; struct encoding_stream *str = ENCODING_STREAM_DATA (encoding); unsigned int flags = str->flags; unsigned int ch = str->ch; eol_type_t eol_type = CODING_SYSTEM_EOL_TYPE (str->codesys); +#ifdef UTF2000 + unsigned char char_boundary = str->iso2022.current_char_boundary; +#endif while (n--) { - c = *src++; + Bufbyte c = *src++; +#ifdef UTF2000 + switch (char_boundary) + { + case 0: + if ( c >= 0xfc ) + { + ch = c & 0x01; + char_boundary = 5; + } + else if ( c >= 0xf8 ) + { + ch = c & 0x03; + char_boundary = 4; + } + else if ( c >= 0xf0 ) + { + ch = c & 0x07; + char_boundary = 3; + } + else if ( c >= 0xe0 ) + { + ch = c & 0x0f; + char_boundary = 2; + } + else if ( c >= 0xc0 ) + { + ch = c & 0x1f; + char_boundary = 1; + } + else + { + ch = 0; + if (c == '\n') + { + if (eol_type != EOL_LF && eol_type != EOL_AUTODETECT) + Dynarr_add (dst, '\r'); + if (eol_type != EOL_CR) + Dynarr_add (dst, c); + } + else + Dynarr_add (dst, c); + char_boundary = 0; + } + break; + case 1: + ch = ( ch << 6 ) | ( c & 0x3f ); + { + Lisp_Object charset; + unsigned int c1, c2, s1, s2; + + BREAKUP_CHAR (ch, charset, c1, c2); + if (EQ(charset, Vcharset_katakana_jisx0201)) + { + Dynarr_add (dst, c1 | 0x80); + } + else if (EQ(charset, Vcharset_japanese_jisx0208)) + { + ENCODE_SJIS (c1 | 0x80, c2 | 0x80, s1, s2); + Dynarr_add (dst, s1); + Dynarr_add (dst, s2); + } + } + char_boundary = 0; + break; + default: + ch = ( ch << 6 ) | ( c & 0x3f ); + char_boundary--; + } +#else if (c == '\n') { if (eol_type != EOL_LF && eol_type != EOL_AUTODETECT) @@ -3078,10 +3248,14 @@ encode_coding_sjis (Lstream *encoding, const unsigned char *src, ch = 0; } } +#endif } str->flags = flags; str->ch = ch; +#ifdef UTF2000 + str->iso2022.current_char_boundary = char_boundary; +#endif } DEFUN ("decode-shift-jis-char", Fdecode_shift_jis_char, 1, 1, 0, /* @@ -3110,16 +3284,16 @@ Return the corresponding character. } DEFUN ("encode-shift-jis-char", Fencode_shift_jis_char, 1, 1, 0, /* -Encode a JISX0208 character CHAR to SHIFT-JIS coding-system. +Encode a JISX0208 character CHARACTER to SHIFT-JIS coding-system. Return the corresponding character code in SHIFT-JIS as a cons of two bytes. */ - (ch)) + (character)) { Lisp_Object charset; int c1, c2, s1, s2; - CHECK_CHAR_COERCE_INT (ch); - BREAKUP_CHAR (XCHAR (ch), charset, c1, c2); + CHECK_CHAR_COERCE_INT (character); + BREAKUP_CHAR (XCHAR (character), charset, c1, c2); if (EQ (charset, Vcharset_japanese_jisx0208)) { ENCODE_SJIS (c1 | 0x80, c2 | 0x80, s1, s2); @@ -3234,14 +3408,11 @@ Return the corresponding character code in SHIFT-JIS as a cons of two bytes. } while (0) static int -detect_coding_big5 (struct detection_state *st, const unsigned char *src, - unsigned int n) +detect_coding_big5 (struct detection_state *st, const Extbyte *src, Lstream_data_count n) { - int c; - while (n--) { - c = *src++; + unsigned char c = *(unsigned char *)src++; if (c == ISO_CODE_ESC || c == ISO_CODE_SI || c == ISO_CODE_SO || (c >= 0x80 && c <= 0xA0)) return 0; @@ -3260,10 +3431,9 @@ detect_coding_big5 (struct detection_state *st, const unsigned char *src, /* Convert Big5 data to internal format. */ static void -decode_coding_big5 (Lstream *decoding, const unsigned char *src, - unsigned_char_dynarr *dst, unsigned int n) +decode_coding_big5 (Lstream *decoding, const Extbyte *src, + unsigned_char_dynarr *dst, Lstream_data_count n) { - unsigned char c; struct decoding_stream *str = DECODING_STREAM_DATA (decoding); unsigned int flags = str->flags; unsigned int ch = str->ch; @@ -3271,7 +3441,7 @@ decode_coding_big5 (Lstream *decoding, const unsigned char *src, while (n--) { - c = *src++; + unsigned char c = *(unsigned char *)src++; if (ch) { /* Previous character was first byte of Big5 char. */ @@ -3310,9 +3480,10 @@ decode_coding_big5 (Lstream *decoding, const unsigned char *src, /* Convert internally-formatted data to Big5. */ static void -encode_coding_big5 (Lstream *encoding, const unsigned char *src, - unsigned_char_dynarr *dst, unsigned int n) +encode_coding_big5 (Lstream *encoding, const Bufbyte *src, + unsigned_char_dynarr *dst, Lstream_data_count n) { +#ifndef UTF2000 unsigned char c; struct encoding_stream *str = ENCODING_STREAM_DATA (encoding); unsigned int flags = str->flags; @@ -3367,6 +3538,7 @@ encode_coding_big5 (Lstream *encoding, const unsigned char *src, str->flags = flags; str->ch = ch; +#endif } @@ -3387,7 +3559,7 @@ Return the corresponding character. if (BYTE_BIG5_TWO_BYTE_1_P (b1) && BYTE_BIG5_TWO_BYTE_2_P (b2)) { - int leading_byte; + Charset_ID leading_byte; Lisp_Object charset; DECODE_BIG5 (b1, b2, leading_byte, c1, c2); charset = CHARSET_BY_LEADING_BYTE (leading_byte); @@ -3398,16 +3570,16 @@ Return the corresponding character. } DEFUN ("encode-big5-char", Fencode_big5_char, 1, 1, 0, /* -Encode the Big5 character CH to BIG5 coding-system. +Encode the Big5 character CHARACTER in the BIG5 coding-system. Return the corresponding character code in Big5. */ - (ch)) + (character)) { Lisp_Object charset; int c1, c2, b1, b2; - CHECK_CHAR_COERCE_INT (ch); - BREAKUP_CHAR (XCHAR (ch), charset, c1, c2); + CHECK_CHAR_COERCE_INT (character); + BREAKUP_CHAR (XCHAR (character), charset, c1, c2); if (EQ (charset, Vcharset_chinese_big5_1) || EQ (charset, Vcharset_chinese_big5_2)) { @@ -3427,7 +3599,7 @@ Return the corresponding character code in Big5. /* */ /************************************************************************/ - +#ifndef UTF2000 DEFUN ("set-ucs-char", Fset_ucs_char, 2, 2, 0, /* Map UCS-4 code CODE to Mule character CHARACTER. @@ -3435,13 +3607,13 @@ Return T on success, NIL on failure. */ (code, character)) { - unsigned int c; + size_t c; CHECK_CHAR (character); - CHECK_INT (code); + CHECK_NATNUM (code); c = XINT (code); - if (c < sizeof (fcd->ucs_to_mule_table)) + if (c < countof (fcd->ucs_to_mule_table)) { fcd->ucs_to_mule_table[c] = character; return Qt; @@ -3453,7 +3625,7 @@ Return T on success, NIL on failure. static Lisp_Object ucs_to_char (unsigned long code) { - if (code < sizeof (fcd->ucs_to_mule_table)) + if (code < countof (fcd->ucs_to_mule_table)) { return fcd->ucs_to_mule_table[code]; } @@ -3465,7 +3637,7 @@ ucs_to_char (unsigned long code) c = code % (94 * 94); return make_char (MAKE_CHAR (CHARSET_BY_ATTRIBUTES - (CHARSET_TYPE_94X94, code / (94 * 94) + '@', + (94, 2, code / (94 * 94) + '@', CHARSET_LEFT_TO_RIGHT), c / 94 + 33, c % 94 + 33)); } @@ -3502,7 +3674,11 @@ Return the UCS code (a positive integer) corresponding to CHARACTER. { return Fget_char_table (character, mule_to_ucs_table); } +#endif +#ifdef UTF2000 +#define decode_ucs4 DECODE_ADD_UCS_CHAR +#else /* Decode a UCS-4 character into a buffer. If the lookup fails, use (U+3013) of JIS X 0208, which means correct character is not found, instead. @@ -3531,7 +3707,9 @@ decode_ucs4 (unsigned long ch, unsigned_char_dynarr *dst) Dynarr_add (dst, 46 + 128); } } +#endif +#ifndef UTF2000 static unsigned long mule_char_to_ucs4 (Lisp_Object charset, unsigned char h, unsigned char l) @@ -3575,14 +3753,14 @@ encode_ucs4 (Lisp_Object charset, Dynarr_add (dst, (code >> 8) & 255); Dynarr_add (dst, code & 255); } +#endif static int -detect_coding_ucs4 (struct detection_state *st, const unsigned char *src, - unsigned int n) +detect_coding_ucs4 (struct detection_state *st, const Extbyte *src, Lstream_data_count n) { while (n--) { - int c = *src++; + unsigned char c = *(unsigned char *)src++; switch (st->ucs4.in_byte) { case 0: @@ -3602,8 +3780,8 @@ detect_coding_ucs4 (struct detection_state *st, const unsigned char *src, } static void -decode_coding_ucs4 (Lstream *decoding, const unsigned char *src, - unsigned_char_dynarr *dst, unsigned int n) +decode_coding_ucs4 (Lstream *decoding, const Extbyte *src, + unsigned_char_dynarr *dst, Lstream_data_count n) { struct decoding_stream *str = DECODING_STREAM_DATA (decoding); unsigned int flags = str->flags; @@ -3612,7 +3790,7 @@ decode_coding_ucs4 (Lstream *decoding, const unsigned char *src, while (n--) { - unsigned char c = *src++; + unsigned char c = *(unsigned char *)src++; switch (counter) { case 0: @@ -3638,9 +3816,10 @@ decode_coding_ucs4 (Lstream *decoding, const unsigned char *src, } static void -encode_coding_ucs4 (Lstream *encoding, const unsigned char *src, - unsigned_char_dynarr *dst, unsigned int n) +encode_coding_ucs4 (Lstream *encoding, const Bufbyte *src, + unsigned_char_dynarr *dst, Lstream_data_count n) { +#ifndef UTF2000 struct encoding_stream *str = ENCODING_STREAM_DATA (encoding); unsigned int flags = str->flags; unsigned int ch = str->ch; @@ -3704,9 +3883,9 @@ encode_coding_ucs4 (Lstream *encoding, const unsigned char *src, { /* #### Bother! We don't know how to handle this yet. */ - Dynarr_add (dst, 0); - Dynarr_add (dst, 0); - Dynarr_add (dst, 0); + Dynarr_add (dst, '\0'); + Dynarr_add (dst, '\0'); + Dynarr_add (dst, '\0'); Dynarr_add (dst, '~'); } else @@ -3747,7 +3926,7 @@ encode_coding_ucs4 (Lstream *encoding, const unsigned char *src, } break; default: - abort (); + ABORT (); } } } @@ -3769,6 +3948,7 @@ encode_coding_ucs4 (Lstream *encoding, const unsigned char *src, str->iso2022.current_charset = charset; /* Verbum caro factum est! */ +#endif } @@ -3777,12 +3957,11 @@ encode_coding_ucs4 (Lstream *encoding, const unsigned char *src, /************************************************************************/ static int -detect_coding_utf8 (struct detection_state *st, const unsigned char *src, - unsigned int n) +detect_coding_utf8 (struct detection_state *st, const Extbyte *src, Lstream_data_count n) { while (n--) { - unsigned char c = *src++; + unsigned char c = *(unsigned char *)src++; switch (st->utf8.in_byte) { case 0: @@ -3812,8 +3991,8 @@ detect_coding_utf8 (struct detection_state *st, const unsigned char *src, } static void -decode_coding_utf8 (Lstream *decoding, const unsigned char *src, - unsigned_char_dynarr *dst, unsigned int n) +decode_coding_utf8 (Lstream *decoding, const Extbyte *src, + unsigned_char_dynarr *dst, Lstream_data_count n) { struct decoding_stream *str = DECODING_STREAM_DATA (decoding); unsigned int flags = str->flags; @@ -3823,7 +4002,7 @@ decode_coding_utf8 (Lstream *decoding, const unsigned char *src, while (n--) { - unsigned char c = *src++; + unsigned char c = *(unsigned char *)src++; switch (counter) { case 0: @@ -3879,6 +4058,7 @@ decode_coding_utf8 (Lstream *decoding, const unsigned char *src, str->counter = counter; } +#ifndef UTF2000 static void encode_utf8 (Lisp_Object charset, unsigned char h, unsigned char l, unsigned_char_dynarr *dst) @@ -3924,16 +4104,74 @@ encode_utf8 (Lisp_Object charset, Dynarr_add (dst, (code & 0x3f) | 0x80); } } +#endif static void -encode_coding_utf8 (Lstream *encoding, const unsigned char *src, - unsigned_char_dynarr *dst, unsigned int n) +encode_coding_utf8 (Lstream *encoding, const Bufbyte *src, + unsigned_char_dynarr *dst, Lstream_data_count n) { struct encoding_stream *str = ENCODING_STREAM_DATA (encoding); unsigned int flags = str->flags; unsigned int ch = str->ch; eol_type_t eol_type = CODING_SYSTEM_EOL_TYPE (str->codesys); unsigned char char_boundary = str->iso2022.current_char_boundary; +#ifdef UTF2000 + + while (n--) + { + unsigned char c = *src++; + switch (char_boundary) + { + case 0: + if ( c >= 0xfc ) + { + Dynarr_add (dst, c); + char_boundary = 5; + } + else if ( c >= 0xf8 ) + { + Dynarr_add (dst, c); + char_boundary = 4; + } + else if ( c >= 0xf0 ) + { + Dynarr_add (dst, c); + char_boundary = 3; + } + else if ( c >= 0xe0 ) + { + Dynarr_add (dst, c); + char_boundary = 2; + } + else if ( c >= 0xc0 ) + { + Dynarr_add (dst, c); + char_boundary = 1; + } + else + { + if (c == '\n') + { + if (eol_type != EOL_LF && eol_type != EOL_AUTODETECT) + Dynarr_add (dst, '\r'); + if (eol_type != EOL_CR) + Dynarr_add (dst, c); + } + else + Dynarr_add (dst, c); + char_boundary = 0; + } + break; + case 1: + Dynarr_add (dst, c); + char_boundary = 0; + break; + default: + Dynarr_add (dst, c); + char_boundary--; + } + } +#else /* not UTF2000 */ Lisp_Object charset = str->iso2022.current_charset; #ifdef ENABLE_COMPOSITE_CHARS @@ -4041,7 +4279,7 @@ encode_coding_utf8 (Lstream *encoding, const unsigned char *src, } break; default: - abort (); + ABORT (); } } } @@ -4057,10 +4295,13 @@ encode_coding_utf8 (Lstream *encoding, const unsigned char *src, } #endif +#endif /* not UTF2000 */ str->flags = flags; str->ch = ch; str->iso2022.current_char_boundary = char_boundary; +#ifndef UTF2000 str->iso2022.current_charset = charset; +#endif /* Verbum caro factum est! */ } @@ -4279,7 +4520,48 @@ fit_to_be_escape_quoted (unsigned char c) If CHECK_INVALID_CHARSETS is non-zero, check for designation or invocation of an invalid character set and treat that as - an unrecognized escape sequence. */ + an unrecognized escape sequence. + + ******************************************************************** + + #### Strategies for error annotation and coding orthogonalization + + We really want to separate out a number of things. Conceptually, + there is a nested syntax. + + At the top level is the ISO 2022 extension syntax, including charset + designation and invocation, and certain auxiliary controls such as the + ISO 6429 direction specification. These are octet-oriented, with the + single exception (AFAIK) of the "exit Unicode" sequence which uses the + UTF's natural width (1 byte for UTF-7 and UTF-8, 2 bytes for UCS-2 and + UTF-16, and 4 bytes for UCS-4 and UTF-32). This will be treated as a + (deprecated) special case in Unicode processing. + + The middle layer is ISO 2022 character interpretation. This will depend + on the current state of the ISO 2022 registers, and assembles octets + into the character's internal representation. + + The lowest level is translating system control conventions. At present + this is restricted to newline translation, but one could imagine doing + tab conversion or line wrapping here. "Escape from Unicode" processing + would be done at this level. + + At each level the parser will verify the syntax. In the case of a + syntax error or warning (such as a redundant escape sequence that affects + no characters), the parser will take some action, typically inserting the + erroneous octets directly into the output and creating an annotation + which can be used by higher level I/O to mark the affected region. + + This should make it possible to do something sensible about separating + newline convention processing from character construction, and about + preventing ISO 2022 escape sequences from being recognized + inappropriately. + + The basic strategy will be to have octet classification tables, and + switch processing according to the table entry. + + It's possible that, by doing the processing with tables of functions or + the like, the parser can be used for both detection and translation. */ static int parse_iso2022_esc (Lisp_Object codesys, struct iso2022_decoder *iso, @@ -4515,7 +4797,7 @@ parse_iso2022_esc (Lisp_Object codesys, struct iso2022_decoder *iso, } if (0x40 <= c && c <= 0x42) { - cs = CHARSET_BY_ATTRIBUTES (CHARSET_TYPE_94X94, c, + cs = CHARSET_BY_ATTRIBUTES (94, 2, c, *flags & CODING_STATE_R2L ? CHARSET_RIGHT_TO_LEFT : CHARSET_LEFT_TO_RIGHT); @@ -4526,7 +4808,9 @@ parse_iso2022_esc (Lisp_Object codesys, struct iso2022_decoder *iso, default: { - int type =-1; + /* int type =-1; */ + int chars = 0; + int single = 0; if (c < '0' || c > '~') return 0; /* bad final byte */ @@ -4534,24 +4818,25 @@ parse_iso2022_esc (Lisp_Object codesys, struct iso2022_decoder *iso, if (iso->esc >= ISO_ESC_2_8 && iso->esc <= ISO_ESC_2_15) { - type = ((iso->esc >= ISO_ESC_2_12) ? - CHARSET_TYPE_96 : CHARSET_TYPE_94); + chars = (iso->esc >= ISO_ESC_2_12) ? 96 : 94; + single = 1; /* single-byte */ reg = (iso->esc - ISO_ESC_2_8) & 3; } else if (iso->esc >= ISO_ESC_2_4_8 && iso->esc <= ISO_ESC_2_4_15) { - type = ((iso->esc >= ISO_ESC_2_4_12) ? - CHARSET_TYPE_96X96 : CHARSET_TYPE_94X94); + chars = (iso->esc >= ISO_ESC_2_4_12) ? 96 : 94; + single = -1; /* multi-byte */ reg = (iso->esc - ISO_ESC_2_4_8) & 3; } else { /* Can this ever be reached? -slb */ - abort(); + ABORT(); + return 0; } - cs = CHARSET_BY_ATTRIBUTES (type, c, + cs = CHARSET_BY_ATTRIBUTES (chars, single, c, *flags & CODING_STATE_R2L ? CHARSET_RIGHT_TO_LEFT : CHARSET_LEFT_TO_RIGHT); @@ -4644,8 +4929,7 @@ parse_iso2022_esc (Lisp_Object codesys, struct iso2022_decoder *iso, } static int -detect_coding_iso2022 (struct detection_state *st, const unsigned char *src, - unsigned int n) +detect_coding_iso2022 (struct detection_state *st, const Extbyte *src, Lstream_data_count n) { int mask; @@ -4675,7 +4959,7 @@ detect_coding_iso2022 (struct detection_state *st, const unsigned char *src, while (n--) { - int c = *src++; + unsigned char c = *(unsigned char *)src++; if (c >= 0xA0) { mask &= ~CODING_CATEGORY_ISO_7_MASK; @@ -4835,8 +5119,8 @@ ensure_correct_direction (int direction, Lisp_Coding_System *codesys, /* Convert ISO2022-format data to internal format. */ static void -decode_coding_iso2022 (Lstream *decoding, const unsigned char *src, - unsigned_char_dynarr *dst, unsigned int n) +decode_coding_iso2022 (Lstream *decoding, const Extbyte *src, + unsigned_char_dynarr *dst, Lstream_data_count n) { struct decoding_stream *str = DECODING_STREAM_DATA (decoding); unsigned int flags = str->flags; @@ -4856,7 +5140,7 @@ decode_coding_iso2022 (Lstream *decoding, const unsigned char *src, while (n--) { - unsigned char c = *src++; + unsigned char c = *(unsigned char *)src++; if (flags & CODING_STATE_ESCAPE) { /* Within ESC sequence */ int retval = parse_iso2022_esc (coding_system, &str->iso2022, @@ -4960,7 +5244,9 @@ decode_coding_iso2022 (Lstream *decoding, const unsigned char *src, else { /* Graphic characters */ Lisp_Object charset; - int lb; +#ifndef UTF2000 + Charset_ID lb; +#endif int reg; DECODE_HANDLE_EOL_TYPE (eol_type, c, flags, dst); @@ -5003,6 +5289,22 @@ decode_coding_iso2022 (Lstream *decoding, const unsigned char *src, charset = new_charset; } +#ifdef UTF2000 + if (XCHARSET_DIMENSION (charset) == 1) + { + DECODE_OUTPUT_PARTIAL_CHAR (ch); + DECODE_ADD_UCS_CHAR + (MAKE_CHAR (charset, c & 0x7F, 0), dst); + } + else if (ch) + { + DECODE_ADD_UCS_CHAR + (MAKE_CHAR (charset, ch & 0x7F, c & 0x7F), dst); + ch = 0; + } + else + ch = c; +#else lb = XCHARSET_LEADING_BYTE (charset); switch (XCHARSET_REP_BYTES (charset)) { @@ -5051,6 +5353,7 @@ decode_coding_iso2022 (Lstream *decoding, const unsigned char *src, else ch = c; } +#endif } if (!ch) @@ -5161,19 +5464,22 @@ ensure_shift_out (struct encoding_stream *str, unsigned_char_dynarr *dst) /* Convert internally-formatted data to ISO2022 format. */ static void -encode_coding_iso2022 (Lstream *encoding, const unsigned char *src, - unsigned_char_dynarr *dst, unsigned int n) +encode_coding_iso2022 (Lstream *encoding, const Bufbyte *src, + unsigned_char_dynarr *dst, Lstream_data_count n) { unsigned char charmask, c; unsigned char char_boundary; struct encoding_stream *str = ENCODING_STREAM_DATA (encoding); unsigned int flags = str->flags; - unsigned int ch = str->ch; + Emchar ch = str->ch; Lisp_Coding_System *codesys = str->codesys; eol_type_t eol_type = CODING_SYSTEM_EOL_TYPE (str->codesys); int i; Lisp_Object charset; int half; +#ifdef UTF2000 + unsigned int byte1, byte2; +#endif #ifdef ENABLE_COMPOSITE_CHARS /* flags for handling composite chars. We do a little switcharoo @@ -5190,6 +5496,216 @@ encode_coding_iso2022 (Lstream *encoding, const unsigned char *src, #ifdef ENABLE_COMPOSITE_CHARS back_to_square_n: #endif +#ifdef UTF2000 + while (n--) + { + c = *src++; + + switch (char_boundary) + { + case 0: + if ( c >= 0xfc ) + { + ch = c & 0x01; + char_boundary = 5; + } + else if ( c >= 0xf8 ) + { + ch = c & 0x03; + char_boundary = 4; + } + else if ( c >= 0xf0 ) + { + ch = c & 0x07; + char_boundary = 3; + } + else if ( c >= 0xe0 ) + { + ch = c & 0x0f; + char_boundary = 2; + } + else if ( c >= 0xc0 ) + { + ch = c & 0x1f; + char_boundary = 1; + } + else + { + ch = 0; + + restore_left_to_right_direction (codesys, dst, &flags, 0); + + /* Make sure G0 contains ASCII */ + if ((c > ' ' && c < ISO_CODE_DEL) || + !CODING_SYSTEM_ISO2022_NO_ASCII_CNTL (codesys)) + { + ensure_normal_shift (str, dst); + iso2022_designate (Vcharset_ascii, 0, str, dst); + } + + /* If necessary, restore everything to the default state + at end-of-line */ + if (c == '\n' && + !(CODING_SYSTEM_ISO2022_NO_ASCII_EOL (codesys))) + { + restore_left_to_right_direction (codesys, dst, &flags, 0); + + ensure_normal_shift (str, dst); + + for (i = 0; i < 4; i++) + { + Lisp_Object initial_charset = + CODING_SYSTEM_ISO2022_INITIAL_CHARSET (codesys, i); + iso2022_designate (initial_charset, i, str, dst); + } + } + if (c == '\n') + { + if (eol_type != EOL_LF && eol_type != EOL_AUTODETECT) + Dynarr_add (dst, '\r'); + if (eol_type != EOL_CR) + Dynarr_add (dst, c); + } + else + { + if (CODING_SYSTEM_ISO2022_ESCAPE_QUOTED (codesys) + && fit_to_be_escape_quoted (c)) + Dynarr_add (dst, ISO_CODE_ESC); + Dynarr_add (dst, c); + } + char_boundary = 0; + } + break; + case 1: + ch = ( ch << 6 ) | ( c & 0x3f ); + + char_boundary = 0; + if ( (0x80 <= ch) && (ch <= 0x9f) ) + { + charmask = (half == 0 ? 0x00 : 0x80); + + if (CODING_SYSTEM_ISO2022_ESCAPE_QUOTED (codesys) + && fit_to_be_escape_quoted (ch)) + Dynarr_add (dst, ISO_CODE_ESC); + /* you asked for it ... */ + Dynarr_add (dst, ch); + } + else + { + int reg; + + BREAKUP_CHAR (ch, charset, byte1, byte2); + ensure_correct_direction (XCHARSET_DIRECTION (charset), + codesys, dst, &flags, 0); + + /* Now determine which register to use. */ + reg = -1; + for (i = 0; i < 4; i++) + { + if (EQ (charset, str->iso2022.charset[i]) || + EQ (charset, + CODING_SYSTEM_ISO2022_INITIAL_CHARSET (codesys, i))) + { + reg = i; + break; + } + } + + if (reg == -1) + { + if (XCHARSET_GRAPHIC (charset) != 0) + { + if (!NILP (str->iso2022.charset[1]) && + (!CODING_SYSTEM_ISO2022_SEVEN (codesys) || + CODING_SYSTEM_ISO2022_LOCK_SHIFT (codesys))) + reg = 1; + else if (!NILP (str->iso2022.charset[2])) + reg = 2; + else if (!NILP (str->iso2022.charset[3])) + reg = 3; + else + reg = 0; + } + else + reg = 0; + } + + iso2022_designate (charset, reg, str, dst); + + /* Now invoke that register. */ + switch (reg) + { + case 0: + ensure_normal_shift (str, dst); + half = 0; + break; + + case 1: + if (CODING_SYSTEM_ISO2022_SEVEN (codesys)) + { + ensure_shift_out (str, dst); + half = 0; + } + else + half = 1; + break; + + case 2: + if (CODING_SYSTEM_ISO2022_SEVEN (str->codesys)) + { + Dynarr_add (dst, ISO_CODE_ESC); + Dynarr_add (dst, 'N'); + half = 0; + } + else + { + Dynarr_add (dst, ISO_CODE_SS2); + half = 1; + } + break; + + case 3: + if (CODING_SYSTEM_ISO2022_SEVEN (str->codesys)) + { + Dynarr_add (dst, ISO_CODE_ESC); + Dynarr_add (dst, 'O'); + half = 0; + } + else + { + Dynarr_add (dst, ISO_CODE_SS3); + half = 1; + } + break; + + default: + abort (); + } + + charmask = (half == 0 ? 0x00 : 0x80); + + switch (XCHARSET_DIMENSION (charset)) + { + case 1: + Dynarr_add (dst, byte1 | charmask); + break; + case 2: + Dynarr_add (dst, byte1 | charmask); + Dynarr_add (dst, byte2 | charmask); + break; + default: + abort (); + } + } + ch =0; + break; + default: + ch = ( ch << 6 ) | ( c & 0x3f ); + char_boundary--; + } + } +#else /* not UTF2000 */ + while (n--) { c = *src++; @@ -5339,7 +5855,7 @@ encode_coding_iso2022 (Lstream *encoding, const unsigned char *src, break; default: - abort (); + ABORT (); } } char_boundary = 0; @@ -5422,11 +5938,12 @@ encode_coding_iso2022 (Lstream *encoding, const unsigned char *src, } break; default: - abort (); + ABORT (); } } } } +#endif /* not UTF2000 */ #ifdef ENABLE_COMPOSITE_CHARS if (in_composite) @@ -5440,7 +5957,11 @@ encode_coding_iso2022 (Lstream *encoding, const unsigned char *src, } #endif /* ENABLE_COMPOSITE_CHARS */ +#ifdef UTF2000 + if ( (char_boundary == 0) && flags & CODING_STATE_END) +#else if (char_boundary && flags & CODING_STATE_END) +#endif { restore_left_to_right_direction (codesys, dst, &flags, 0); ensure_normal_shift (str, dst); @@ -5470,10 +5991,9 @@ encode_coding_iso2022 (Lstream *encoding, const unsigned char *src, contain all 256 possible byte values and that are not to be interpreted as being in any particular decoding. */ static void -decode_coding_no_conversion (Lstream *decoding, const unsigned char *src, - unsigned_char_dynarr *dst, unsigned int n) +decode_coding_no_conversion (Lstream *decoding, const Extbyte *src, + unsigned_char_dynarr *dst, Lstream_data_count n) { - unsigned char c; struct decoding_stream *str = DECODING_STREAM_DATA (decoding); unsigned int flags = str->flags; unsigned int ch = str->ch; @@ -5481,7 +6001,7 @@ decode_coding_no_conversion (Lstream *decoding, const unsigned char *src, while (n--) { - c = *src++; + unsigned char c = *(unsigned char *)src++; DECODE_HANDLE_EOL_TYPE (eol_type, c, flags, dst); DECODE_ADD_BINARY_CHAR (c, dst); @@ -5495,18 +6015,76 @@ decode_coding_no_conversion (Lstream *decoding, const unsigned char *src, } static void -encode_coding_no_conversion (Lstream *encoding, const unsigned char *src, - unsigned_char_dynarr *dst, unsigned int n) +encode_coding_no_conversion (Lstream *encoding, const Bufbyte *src, + unsigned_char_dynarr *dst, Lstream_data_count n) { unsigned char c; struct encoding_stream *str = ENCODING_STREAM_DATA (encoding); unsigned int flags = str->flags; unsigned int ch = str->ch; eol_type_t eol_type = CODING_SYSTEM_EOL_TYPE (str->codesys); +#ifdef UTF2000 + unsigned char char_boundary = str->iso2022.current_char_boundary; +#endif while (n--) { - c = *src++; + c = *src++; +#ifdef UTF2000 + switch (char_boundary) + { + case 0: + if ( c >= 0xfc ) + { + ch = c & 0x01; + char_boundary = 5; + } + else if ( c >= 0xf8 ) + { + ch = c & 0x03; + char_boundary = 4; + } + else if ( c >= 0xf0 ) + { + ch = c & 0x07; + char_boundary = 3; + } + else if ( c >= 0xe0 ) + { + ch = c & 0x0f; + char_boundary = 2; + } + else if ( c >= 0xc0 ) + { + ch = c & 0x1f; + char_boundary = 1; + } + else + { + ch = 0; + + if (c == '\n') + { + if (eol_type != EOL_LF && eol_type != EOL_AUTODETECT) + Dynarr_add (dst, '\r'); + if (eol_type != EOL_CR) + Dynarr_add (dst, c); + } + else + Dynarr_add (dst, c); + char_boundary = 0; + } + break; + case 1: + ch = ( ch << 6 ) | ( c & 0x3f ); + Dynarr_add (dst, ch & 0xff); + char_boundary = 0; + break; + default: + ch = ( ch << 6 ) | ( c & 0x3f ); + char_boundary--; + } +#else /* not UTF2000 */ if (c == '\n') { if (eol_type != EOL_LF && eol_type != EOL_AUTODETECT) @@ -5542,10 +6120,14 @@ encode_coding_no_conversion (Lstream *encoding, const unsigned char *src, untranslatable character, so ignore it */ ch = 0; } +#endif /* not UTF2000 */ } str->flags = flags; str->ch = ch; +#ifdef UTF2000 + str->iso2022.current_char_boundary = char_boundary; +#endif } @@ -5559,8 +6141,7 @@ syms_of_file_coding (void) { INIT_LRECORD_IMPLEMENTATION (coding_system); - deferror (&Qcoding_system_error, "coding-system-error", - "Coding-system error", Qio_error); + DEFERROR_STANDARD (Qcoding_system_error, Qio_error); DEFSUBR (Fcoding_system_p); DEFSUBR (Ffind_coding_system); @@ -5596,10 +6177,12 @@ syms_of_file_coding (void) DEFSUBR (Fencode_shift_jis_char); DEFSUBR (Fdecode_big5_char); DEFSUBR (Fencode_big5_char); +#ifndef UTF2000 DEFSUBR (Fset_ucs_char); DEFSUBR (Fucs_char); DEFSUBR (Fset_char_ucs); DEFSUBR (Fchar_ucs); +#endif /* not UTF2000 */ #endif /* MULE */ defsymbol (&Qcoding_systemp, "coding-system-p"); defsymbol (&Qno_conversion, "no-conversion"); @@ -5696,10 +6279,10 @@ vars_of_file_coding (void) int i; fcd = xnew (struct file_coding_dump); - dumpstruct (&fcd, &fcd_description); + dump_add_root_struct_ptr (&fcd, &fcd_description); /* Initialize to something reasonable ... */ - for (i = 0; i <= CODING_CATEGORY_LAST; i++) + for (i = 0; i < CODING_CATEGORY_LAST; i++) { fcd->coding_category_system[i] = Qnil; fcd->coding_category_by_priority[i] = i; @@ -5763,7 +6346,7 @@ complex_vars_of_file_coding (void) make_lisp_hash_table (50, HASH_TABLE_NON_WEAK, HASH_TABLE_EQ); the_codesys_prop_dynarr = Dynarr_new (codesys_prop); - dumpstruct (&the_codesys_prop_dynarr, &codesys_prop_dynarr_description); + dump_add_root_struct_ptr (&the_codesys_prop_dynarr, &codesys_prop_dynarr_description); #define DEFINE_CODESYS_PROP(Prop_Type, Sym) do \ { \ @@ -5814,6 +6397,13 @@ complex_vars_of_file_coding (void) list4 (Qeol_type, Qlf, Qmnemonic, build_string ("Binary"))); +#ifdef UTF2000 + Fmake_coding_system + (Qutf8, Qutf8, + build_string ("Coding-system of ISO/IEC 10646 UTF-8."), + list2 (Qmnemonic, build_string ("UTF8"))); +#endif + Fdefine_coding_system_alias (Qno_conversion, Qraw_text); Fdefine_coding_system_alias (Qfile_name, Qbinary); @@ -5825,14 +6415,19 @@ complex_vars_of_file_coding (void) fcd->coding_category_system[CODING_CATEGORY_NO_CONVERSION] = Fget_coding_system (Qraw_text); -#ifdef MULE +#ifdef UTF2000 + fcd->coding_category_system[CODING_CATEGORY_UTF8] + = Fget_coding_system (Qutf8); +#endif + +#if defined(MULE) && !defined(UTF2000) { - unsigned int i; + size_t i; - for (i = 0; i < 65536; i++) + for (i = 0; i < countof (fcd->ucs_to_mule_table); i++) fcd->ucs_to_mule_table[i] = Qnil; } staticpro (&mule_to_ucs_table); mule_to_ucs_table = Fmake_char_table(Qgeneric); -#endif /* MULE */ +#endif /* defined(MULE) && !defined(UTF2000) */ }