/* Code conversion functions.
Copyright (C) 1991, 1995 Free Software Foundation, Inc.
Copyright (C) 1995 Sun Microsystems, Inc.
- Copyright (C) 1999,2000 MORIOKA Tomohiko
+ Copyright (C) 1999,2000,2001,2002 MORIOKA Tomohiko
This file is part of XEmacs.
/* Synched up with: Mule 2.3. Not in FSF. */
/* Rewritten by Ben Wing <ben@xemacs.org>. */
+/* Rewritten by MORIOKA Tomohiko <tomo@m17n.org> for XEmacs UTF-2000. */
#include <config.h>
#include "lisp.h"
Lisp_Object Vcoding_system_for_write;
Lisp_Object Vfile_name_coding_system;
+Lisp_Object Vcoded_charset_entity_reference_alist;
+
/* Table of symbols identifying each coding category. */
-Lisp_Object coding_category_symbol[CODING_CATEGORY_LAST + 1];
+Lisp_Object coding_category_symbol[CODING_CATEGORY_LAST];
struct file_coding_dump {
/* Coding system currently associated with each coding category. */
- Lisp_Object coding_category_system[CODING_CATEGORY_LAST + 1];
+ Lisp_Object coding_category_system[CODING_CATEGORY_LAST];
/* Table of all coding categories in decreasing order of priority.
This describes a permutation of the possible coding categories. */
- int coding_category_by_priority[CODING_CATEGORY_LAST + 1];
+ int coding_category_by_priority[CODING_CATEGORY_LAST];
#if defined(MULE) && !defined(UTF2000)
Lisp_Object ucs_to_mule_table[65536];
} *fcd;
static const struct lrecord_description fcd_description_1[] = {
- { XD_LISP_OBJECT_ARRAY, offsetof (struct file_coding_dump, coding_category_system), CODING_CATEGORY_LAST + 1 },
+ { XD_LISP_OBJECT_ARRAY, offsetof (struct file_coding_dump, coding_category_system), CODING_CATEGORY_LAST },
#if defined(MULE) && !defined(UTF2000)
- { XD_LISP_OBJECT_ARRAY, offsetof (struct file_coding_dump, ucs_to_mule_table), 65536 },
+ { XD_LISP_OBJECT_ARRAY, offsetof (struct file_coding_dump, ucs_to_mule_table), countof (fcd->ucs_to_mule_table) },
#endif
{ XD_END }
};
#endif
#ifdef UTF2000
Lisp_Object Qdisable_composition;
+Lisp_Object Quse_entity_reference;
+Lisp_Object Qd, Qx, QX;
#endif
Lisp_Object Qencode, Qdecode;
struct detection_state;
static void
-text_encode_generic (Lstream *encoding, const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n);
+text_encode_generic (Lstream *encoding, const Bufbyte *src,
+ unsigned_char_dynarr *dst, size_t n);
static int detect_coding_sjis (struct detection_state *st,
- const unsigned char *src,
- unsigned int n);
-static void decode_coding_sjis (Lstream *decoding,
- const unsigned char *src,
- unsigned_char_dynarr *dst,
- unsigned int n);
+ const Extbyte *src, size_t n);
+static void decode_coding_sjis (Lstream *decoding, const Extbyte *src,
+ unsigned_char_dynarr *dst, size_t n);
void char_encode_shift_jis (struct encoding_stream *str, Emchar c,
unsigned_char_dynarr *dst, unsigned int *flags);
void char_finish_shift_jis (struct encoding_stream *str,
unsigned_char_dynarr *dst, unsigned int *flags);
static int detect_coding_big5 (struct detection_state *st,
- const unsigned char *src,
- unsigned int n);
-static void decode_coding_big5 (Lstream *decoding,
- const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n);
-static void encode_coding_big5 (Lstream *encoding,
- const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n);
+ const Extbyte *src, size_t n);
+static void decode_coding_big5 (Lstream *decoding, const Extbyte *src,
+ unsigned_char_dynarr *dst, size_t n);
+void char_encode_big5 (struct encoding_stream *str, Emchar c,
+ unsigned_char_dynarr *dst, unsigned int *flags);
+void char_finish_big5 (struct encoding_stream *str,
+ unsigned_char_dynarr *dst, unsigned int *flags);
+
static int detect_coding_ucs4 (struct detection_state *st,
- const unsigned char *src,
- unsigned int n);
-static void decode_coding_ucs4 (Lstream *decoding,
- const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n);
+ const Extbyte *src, size_t n);
+static void decode_coding_ucs4 (Lstream *decoding, const Extbyte *src,
+ unsigned_char_dynarr *dst, size_t n);
void char_encode_ucs4 (struct encoding_stream *str, Emchar c,
unsigned_char_dynarr *dst, unsigned int *flags);
void char_finish_ucs4 (struct encoding_stream *str,
unsigned_char_dynarr *dst, unsigned int *flags);
static int detect_coding_utf8 (struct detection_state *st,
- const unsigned char *src,
- unsigned int n);
-static void decode_coding_utf8 (Lstream *decoding,
- const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n);
+ const Extbyte *src, size_t n);
+static void decode_coding_utf8 (Lstream *decoding, const Extbyte *src,
+ unsigned_char_dynarr *dst, size_t n);
void char_encode_utf8 (struct encoding_stream *str, Emchar c,
unsigned_char_dynarr *dst, unsigned int *flags);
void char_finish_utf8 (struct encoding_stream *str,
static void reset_iso2022 (Lisp_Object coding_system,
struct iso2022_decoder *iso);
static int detect_coding_iso2022 (struct detection_state *st,
- const unsigned char *src,
- unsigned int n);
-static void decode_coding_iso2022 (Lstream *decoding,
- const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n);
+ const Extbyte *src, size_t n);
+static void decode_coding_iso2022 (Lstream *decoding, const Extbyte *src,
+ unsigned_char_dynarr *dst, size_t n);
void char_encode_iso2022 (struct encoding_stream *str, Emchar c,
unsigned_char_dynarr *dst, unsigned int *flags);
void char_finish_iso2022 (struct encoding_stream *str,
unsigned_char_dynarr *dst, unsigned int *flags);
#endif /* MULE */
-static void decode_coding_no_conversion (Lstream *decoding,
- const unsigned char *src,
- unsigned_char_dynarr *dst,
- unsigned int n);
-static void encode_coding_no_conversion (Lstream *encoding,
- const unsigned char *src,
- unsigned_char_dynarr *dst,
- unsigned int n);
-static void mule_decode (Lstream *decoding, const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n);
-static void mule_encode (Lstream *encoding, const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n);
+static void decode_coding_no_conversion (Lstream *decoding, const Extbyte *src,
+ unsigned_char_dynarr *dst, size_t n);
+static void encode_coding_no_conversion (Lstream *encoding, const Bufbyte *src,
+ unsigned_char_dynarr *dst, size_t n);
+static void mule_decode (Lstream *decoding, const Extbyte *src,
+ unsigned_char_dynarr *dst, size_t n);
+static void mule_encode (Lstream *encoding, const Bufbyte *src,
+ unsigned_char_dynarr *dst, size_t n);
typedef struct codesys_prop codesys_prop;
struct codesys_prop
{ XD_STRUCT_PTR, offsetof (Lisp_Coding_System, iso2022.output_conv), 1, &ccsd_description },
{ XD_LISP_OBJECT, offsetof (Lisp_Coding_System, ccl.decode) },
{ XD_LISP_OBJECT, offsetof (Lisp_Coding_System, ccl.encode) },
+#ifdef UTF2000
+ { XD_LISP_OBJECT, offsetof (Lisp_Coding_System, ccs_priority_list) },
+#endif
#endif
{ XD_END }
};
}
}
break;
+#ifdef UTF2000
+
+ case CODESYS_BIG5:
+ mark_object (CODING_SYSTEM_ISO2022_INITIAL_CHARSET (codesys, 0));
+ mark_object (CODING_SYSTEM_ISO2022_INITIAL_CHARSET (codesys, 1));
+ break;
+#endif
case CODESYS_CCL:
mark_object (CODING_SYSTEM_CCL_DECODE (codesys));
}
mark_object (CODING_SYSTEM_PRE_WRITE_CONVERSION (codesys));
+#ifdef UTF2000
+ mark_object (CODING_SYSTEM_CCS_PRIORITY_LIST (codesys));
+#endif
return CODING_SYSTEM_POST_READ_CONVERSION (codesys);
}
CODING_SYSTEM_TYPE (codesys) = type;
CODING_SYSTEM_MNEMONIC (codesys) = Qnil;
#ifdef MULE
+#ifdef UTF2000
+ CODING_SYSTEM_CCS_PRIORITY_LIST (codesys) = Qnil;
+#endif
if (type == CODESYS_ISO2022)
{
int i;
for (i = 0; i < 4; i++)
CODING_SYSTEM_ISO2022_INITIAL_CHARSET (codesys, i) = Qnil;
}
+#ifdef UTF2000
+ if (type == CODESYS_BIG5)
+ {
+ CODING_SYSTEM_ISO2022_INITIAL_CHARSET (codesys, 0)
+ = Vcharset_ascii;
+ CODING_SYSTEM_ISO2022_INITIAL_CHARSET (codesys, 1)
+ = Vcharset_chinese_big5;
+ CODING_SYSTEM_ISO2022_INITIAL_CHARSET (codesys, 2)
+ = Qnil;
+ CODING_SYSTEM_ISO2022_INITIAL_CHARSET (codesys, 3)
+ = Qnil;
+ }
+#endif
else if (type == CODESYS_CCL)
{
CODING_SYSTEM_CCL_DECODE (codesys) = Qnil;
JIS (the Japanese encoding commonly used for e-mail), EUC (the
standard Unix encoding for Japanese and other languages), and
Compound Text (the encoding used in X11). You can specify more
- specific information about the conversion with the FLAGS argument.
+ specific information about the conversion with the PROPS argument.
'big5
Big5 (the encoding commonly used for Taiwanese).
'ccl
If non-nil, composition/decomposition for combining characters
are disabled.
+'use-entity-reference
+ If non-nil, SGML style entity-reference is used for non-system-characters.
+
'post-read-conversion
Function called after a file has been read in, to perform the
- decoding. Called with two arguments, BEG and END, denoting
+ decoding. Called with two arguments, START and END, denoting
a region of the current buffer to be decoded.
'pre-write-conversion
Function called before a file is written out, to perform the
- encoding. Called with two arguments, BEG and END, denoting
+ encoding. Called with two arguments, START and END, denoting
a region of the current buffer to be encoded.
(name, type, doc_string, props))
{
Lisp_Coding_System *codesys;
- Lisp_Object rest, key, value;
enum coding_system_type ty;
int need_to_setup_eol_systems = 1;
CHECK_STRING (doc_string);
CODING_SYSTEM_DOC_STRING (codesys) = doc_string;
- EXTERNAL_PROPERTY_LIST_LOOP (rest, key, value, props)
- {
- if (EQ (key, Qmnemonic))
- {
- if (!NILP (value))
- CHECK_STRING (value);
- CODING_SYSTEM_MNEMONIC (codesys) = value;
- }
+ {
+ EXTERNAL_PROPERTY_LIST_LOOP_3 (key, value, props)
+ {
+ if (EQ (key, Qmnemonic))
+ {
+ if (!NILP (value))
+ CHECK_STRING (value);
+ CODING_SYSTEM_MNEMONIC (codesys) = value;
+ }
- else if (EQ (key, Qeol_type))
- {
- need_to_setup_eol_systems = NILP (value);
- if (EQ (value, Qt))
- value = Qnil;
- CODING_SYSTEM_EOL_TYPE (codesys) = symbol_to_eol_type (value);
- }
+ else if (EQ (key, Qeol_type))
+ {
+ need_to_setup_eol_systems = NILP (value);
+ if (EQ (value, Qt))
+ value = Qnil;
+ CODING_SYSTEM_EOL_TYPE (codesys) = symbol_to_eol_type (value);
+ }
- else if (EQ (key, Qpost_read_conversion))
- CODING_SYSTEM_POST_READ_CONVERSION (codesys) = value;
- else if (EQ (key, Qpre_write_conversion))
- CODING_SYSTEM_PRE_WRITE_CONVERSION (codesys) = value;
+ else if (EQ (key, Qpost_read_conversion))
+ CODING_SYSTEM_POST_READ_CONVERSION (codesys) = value;
+ else if (EQ (key, Qpre_write_conversion))
+ CODING_SYSTEM_PRE_WRITE_CONVERSION (codesys) = value;
#ifdef UTF2000
- else if (EQ (key, Qdisable_composition))
- CODING_SYSTEM_DISABLE_COMPOSITION (codesys) = !NILP (value);
+ else if (EQ (key, Qdisable_composition))
+ CODING_SYSTEM_DISABLE_COMPOSITION (codesys) = !NILP (value);
+ else if (EQ (key, Quse_entity_reference))
+ CODING_SYSTEM_USE_ENTITY_REFERENCE (codesys) = !NILP (value);
#endif
#ifdef MULE
- else if (ty == CODESYS_ISO2022)
- {
+ else if (ty == CODESYS_ISO2022)
+ {
#define FROB_INITIAL_CHARSET(charset_num) \
CODING_SYSTEM_ISO2022_INITIAL_CHARSET (codesys, charset_num) = \
((EQ (value, Qt) || EQ (value, Qnil)) ? value : Fget_charset (value))
- if (EQ (key, Qcharset_g0)) FROB_INITIAL_CHARSET (0);
- else if (EQ (key, Qcharset_g1)) FROB_INITIAL_CHARSET (1);
- else if (EQ (key, Qcharset_g2)) FROB_INITIAL_CHARSET (2);
- else if (EQ (key, Qcharset_g3)) FROB_INITIAL_CHARSET (3);
+ if (EQ (key, Qcharset_g0)) FROB_INITIAL_CHARSET (0);
+ else if (EQ (key, Qcharset_g1)) FROB_INITIAL_CHARSET (1);
+ else if (EQ (key, Qcharset_g2)) FROB_INITIAL_CHARSET (2);
+ else if (EQ (key, Qcharset_g3)) FROB_INITIAL_CHARSET (3);
#define FROB_FORCE_CHARSET(charset_num) \
CODING_SYSTEM_ISO2022_FORCE_CHARSET_ON_OUTPUT (codesys, charset_num) = !NILP (value)
- else if (EQ (key, Qforce_g0_on_output)) FROB_FORCE_CHARSET (0);
- else if (EQ (key, Qforce_g1_on_output)) FROB_FORCE_CHARSET (1);
- else if (EQ (key, Qforce_g2_on_output)) FROB_FORCE_CHARSET (2);
- else if (EQ (key, Qforce_g3_on_output)) FROB_FORCE_CHARSET (3);
+ else if (EQ (key, Qforce_g0_on_output)) FROB_FORCE_CHARSET (0);
+ else if (EQ (key, Qforce_g1_on_output)) FROB_FORCE_CHARSET (1);
+ else if (EQ (key, Qforce_g2_on_output)) FROB_FORCE_CHARSET (2);
+ else if (EQ (key, Qforce_g3_on_output)) FROB_FORCE_CHARSET (3);
#define FROB_BOOLEAN_PROPERTY(prop) \
CODING_SYSTEM_ISO2022_##prop (codesys) = !NILP (value)
- else if (EQ (key, Qshort)) FROB_BOOLEAN_PROPERTY (SHORT);
- else if (EQ (key, Qno_ascii_eol)) FROB_BOOLEAN_PROPERTY (NO_ASCII_EOL);
- else if (EQ (key, Qno_ascii_cntl)) FROB_BOOLEAN_PROPERTY (NO_ASCII_CNTL);
- else if (EQ (key, Qseven)) FROB_BOOLEAN_PROPERTY (SEVEN);
- else if (EQ (key, Qlock_shift)) FROB_BOOLEAN_PROPERTY (LOCK_SHIFT);
- else if (EQ (key, Qno_iso6429)) FROB_BOOLEAN_PROPERTY (NO_ISO6429);
- else if (EQ (key, Qescape_quoted)) FROB_BOOLEAN_PROPERTY (ESCAPE_QUOTED);
+ else if (EQ (key, Qshort)) FROB_BOOLEAN_PROPERTY (SHORT);
+ else if (EQ (key, Qno_ascii_eol)) FROB_BOOLEAN_PROPERTY (NO_ASCII_EOL);
+ else if (EQ (key, Qno_ascii_cntl)) FROB_BOOLEAN_PROPERTY (NO_ASCII_CNTL);
+ else if (EQ (key, Qseven)) FROB_BOOLEAN_PROPERTY (SEVEN);
+ else if (EQ (key, Qlock_shift)) FROB_BOOLEAN_PROPERTY (LOCK_SHIFT);
+ else if (EQ (key, Qno_iso6429)) FROB_BOOLEAN_PROPERTY (NO_ISO6429);
+ else if (EQ (key, Qescape_quoted)) FROB_BOOLEAN_PROPERTY (ESCAPE_QUOTED);
- else if (EQ (key, Qinput_charset_conversion))
- {
- codesys->iso2022.input_conv =
- Dynarr_new (charset_conversion_spec);
- parse_charset_conversion_specs (codesys->iso2022.input_conv,
- value);
- }
- else if (EQ (key, Qoutput_charset_conversion))
- {
- codesys->iso2022.output_conv =
- Dynarr_new (charset_conversion_spec);
- parse_charset_conversion_specs (codesys->iso2022.output_conv,
- value);
- }
- else
- signal_simple_error ("Unrecognized property", key);
- }
- else if (EQ (type, Qccl))
- {
- if (EQ (key, Qdecode))
- {
- CHECK_VECTOR (value);
- CODING_SYSTEM_CCL_DECODE (codesys) = value;
- }
- else if (EQ (key, Qencode))
- {
- CHECK_VECTOR (value);
- CODING_SYSTEM_CCL_ENCODE (codesys) = value;
- }
- else
- signal_simple_error ("Unrecognized property", key);
- }
+ else if (EQ (key, Qinput_charset_conversion))
+ {
+ codesys->iso2022.input_conv =
+ Dynarr_new (charset_conversion_spec);
+ parse_charset_conversion_specs (codesys->iso2022.input_conv,
+ value);
+ }
+ else if (EQ (key, Qoutput_charset_conversion))
+ {
+ codesys->iso2022.output_conv =
+ Dynarr_new (charset_conversion_spec);
+ parse_charset_conversion_specs (codesys->iso2022.output_conv,
+ value);
+ }
+ else
+ signal_simple_error ("Unrecognized property", key);
+ }
+#ifdef UTF2000
+ else if (ty == CODESYS_BIG5)
+ {
+ if (EQ (key, Qcharset_g0)) FROB_INITIAL_CHARSET (0);
+ else if (EQ (key, Qcharset_g1)) FROB_INITIAL_CHARSET (1);
+ else
+ signal_simple_error ("Unrecognized property", key);
+ }
+#endif
+ else if (EQ (type, Qccl))
+ {
+ Lisp_Object sym;
+ struct ccl_program test_ccl;
+ Extbyte *suffix;
+
+ /* Check key first. */
+ if (EQ (key, Qdecode))
+ suffix = "-ccl-decode";
+ else if (EQ (key, Qencode))
+ suffix = "-ccl-encode";
+ else
+ signal_simple_error ("Unrecognized property", key);
+
+ /* If value is vector, register it as a ccl program
+ associated with an newly created symbol for
+ backward compatibility. */
+ if (VECTORP (value))
+ {
+ sym = Fintern (concat2 (Fsymbol_name (name),
+ build_string (suffix)),
+ Qnil);
+ Fregister_ccl_program (sym, value);
+ }
+ else
+ {
+ CHECK_SYMBOL (value);
+ sym = value;
+ }
+ /* check if the given ccl programs are valid. */
+ if (setup_ccl_program (&test_ccl, sym) < 0)
+ signal_simple_error ("Invalid CCL program", value);
+
+ if (EQ (key, Qdecode))
+ CODING_SYSTEM_CCL_DECODE (codesys) = sym;
+ else if (EQ (key, Qencode))
+ CODING_SYSTEM_CCL_ENCODE (codesys) = sym;
+
+ }
#endif /* MULE */
- else
- signal_simple_error ("Unrecognized property", key);
- }
+ else
+ signal_simple_error ("Unrecognized property", key);
+ }
+ }
if (need_to_setup_eol_systems)
setup_eol_coding_systems (codesys);
return aliasee;
else
signal_simple_error ("Symbol is not a coding system alias", alias);
+ return Qnil; /* To keep the compiler happy */
}
static Lisp_Object
case EOL_LF: new_coding_system = CODING_SYSTEM_EOL_LF (cs); break;
case EOL_CR: new_coding_system = CODING_SYSTEM_EOL_CR (cs); break;
case EOL_CRLF: new_coding_system = CODING_SYSTEM_EOL_CRLF (cs); break;
- default: abort ();
+ default: abort (); return Qnil;
}
return NILP (new_coding_system) ? coding_system : new_coding_system;
else if (EQ (prop, Qpre_write_conversion))
return XCODING_SYSTEM_PRE_WRITE_CONVERSION (coding_system);
#ifdef MULE
+#ifdef UTF2000
+ else if (EQ (prop, Qdisable_composition))
+ return XCODING_SYSTEM_DISABLE_COMPOSITION (coding_system) ? Qt : Qnil;
+ else if (EQ (prop, Quse_entity_reference))
+ return XCODING_SYSTEM_USE_ENTITY_REFERENCE (coding_system) ? Qt : Qnil;
+#endif
else if (type == CODESYS_ISO2022)
{
if (EQ (prop, Qcharset_g0))
int i;
CHECK_SYMBOL (symbol);
- for (i = 0; i <= CODING_CATEGORY_LAST; i++)
+ for (i = 0; i < CODING_CATEGORY_LAST; i++)
if (EQ (coding_category_symbol[i], symbol))
return i;
int i;
Lisp_Object list = Qnil;
- for (i = CODING_CATEGORY_LAST; i >= 0; i--)
+ for (i = CODING_CATEGORY_LAST - 1; i >= 0; i--)
list = Fcons (coding_category_symbol[i], list);
return list;
}
*/
(list))
{
- int category_to_priority[CODING_CATEGORY_LAST + 1];
+ int category_to_priority[CODING_CATEGORY_LAST];
int i, j;
Lisp_Object rest;
/* First generate a list that maps coding categories to priorities. */
- for (i = 0; i <= CODING_CATEGORY_LAST; i++)
+ for (i = 0; i < CODING_CATEGORY_LAST; i++)
category_to_priority[i] = -1;
/* Highest priority comes from the specified list. */
/* Now go through the existing categories by priority to retrieve
the categories not yet specified and preserve their priority
order. */
- for (j = 0; j <= CODING_CATEGORY_LAST; j++)
+ for (j = 0; j < CODING_CATEGORY_LAST; j++)
{
int cat = fcd->coding_category_by_priority[j];
if (category_to_priority[cat] < 0)
/* Now we need to construct the inverse of the mapping we just
constructed. */
- for (i = 0; i <= CODING_CATEGORY_LAST; i++)
+ for (i = 0; i < CODING_CATEGORY_LAST; i++)
fcd->coding_category_by_priority[category_to_priority[i]] = i;
/* Phew! That was confusing. */
int i;
Lisp_Object list = Qnil;
- for (i = CODING_CATEGORY_LAST; i >= 0; i--)
+ for (i = CODING_CATEGORY_LAST - 1; i >= 0; i--)
list = Fcons (coding_category_symbol[fcd->coding_category_by_priority[i]],
list);
return list;
}
static eol_type_t
-detect_eol_type (struct detection_state *st, const unsigned char *src,
- unsigned int n)
+detect_eol_type (struct detection_state *st, const Extbyte *src,
+ size_t n)
{
- int c;
-
while (n--)
{
- c = *src++;
+ unsigned char c = *(unsigned char *)src++;
if (c == '\n')
{
if (st->eol.just_saw_cr)
static int
detect_coding_type (struct detection_state *st, const Extbyte *src,
- unsigned int n, int just_do_eol)
+ size_t n, int just_do_eol)
{
- int c;
-
if (st->eol_type == EOL_AUTODETECT)
st->eol_type = detect_eol_type (st, src, n);
{
for (; n; n--, src++)
{
- c = *src;
+ unsigned char c = *(unsigned char *) src;
if ((c < 0x20 && !acceptable_control_char_p (c)) || c >= 0x80)
{
st->seen_non_ascii = 1;
#endif
/* Look through the coding categories by priority and find
the first one that is allowed. */
- for (i = 0; i <= CODING_CATEGORY_LAST; i++)
+ for (i = 0; i < CODING_CATEGORY_LAST; i++)
{
cat = fcd->coding_category_by_priority[i];
if ((mask & (1 << cat)) &&
DEFUN ("detect-coding-region", Fdetect_coding_region, 2, 3, 0, /*
Detect coding system of the text in the region between START and END.
-Returned a list of possible coding systems ordered by priority.
-If only ASCII characters are found, it returns 'undecided or one of
+Return a list of possible coding systems ordered by priority.
+If only ASCII characters are found, return 'undecided or one of
its subsidiary coding systems according to a detected end-of-line
type. Optional arg BUFFER defaults to the current buffer.
*/
decst.mask = ~0;
while (1)
{
- unsigned char random_buffer[4096];
+ Extbyte random_buffer[4096];
ssize_t nread = Lstream_read (istr, random_buffer, sizeof (random_buffer));
if (!nread)
#ifdef MULE
decst.mask = postprocess_iso2022_mask (decst.mask);
#endif
- for (i = CODING_CATEGORY_LAST; i >= 0; i--)
+ for (i = CODING_CATEGORY_LAST - 1; i >= 0; i--)
{
int sys = fcd->coding_category_by_priority[i];
if (decst.mask & (1 << sys))
unsigned char counter;
#endif
#ifdef UTF2000
+ unsigned char er_counter;
+ unsigned char er_buf[16];
+
unsigned combined_char_count;
Emchar combined_chars[16];
Lisp_Object combining_table;
};
#ifdef UTF2000
-extern Lisp_Object Vcharacter_composition_table;
+extern Lisp_Object Qcomposition;
+
+INLINE_HEADER void
+decode_flush_er_chars (struct decoding_stream *str, unsigned_char_dynarr* dst);
+INLINE_HEADER void
+decode_flush_er_chars (struct decoding_stream *str, unsigned_char_dynarr* dst)
+{
+ if ( str->er_counter > 0)
+ {
+ Dynarr_add_many (dst, str->er_buf, str->er_counter);
+ str->er_counter = 0;
+ }
+}
+
+void decode_add_er_char (struct decoding_stream *str, Emchar character,
+ unsigned_char_dynarr* dst);
+void
+decode_add_er_char (struct decoding_stream *str, Emchar c,
+ unsigned_char_dynarr* dst)
+{
+ if (str->er_counter == 0)
+ {
+ if (CODING_SYSTEM_USE_ENTITY_REFERENCE (str->codesys)
+ && (c == '&') )
+ {
+ str->er_buf[0] = '&';
+ str->er_counter++;
+ }
+ else
+ DECODE_ADD_UCS_CHAR (c, dst);
+ }
+ else if (c == ';')
+ {
+ Lisp_Object string = make_string (str->er_buf,
+ str->er_counter);
+ Lisp_Object rest = Vcoded_charset_entity_reference_alist;
+ Lisp_Object cell;
+ Lisp_Object ret;
+ Lisp_Object pat;
+ Lisp_Object ccs;
+ int base;
+
+ while (!NILP (rest))
+ {
+ cell = Fcar (rest);
+ ccs = Fcar (cell);
+ if (NILP (ccs = Ffind_charset (ccs)))
+ continue;
+
+ cell = Fcdr (cell);
+ ret = Fcar (cell);
+ if (STRINGP (ret))
+ pat = ret;
+ else
+ continue;
+
+ cell = Fcdr (cell);
+ cell = Fcdr (cell);
+ ret = Fcar (cell);
+ if (EQ (ret, Qd))
+ {
+ pat = concat3 (build_string ("^&"),
+ pat, build_string ("\\([0-9]+\\)$"));
+ base = 10;
+ }
+ else if (EQ (ret, Qx))
+ {
+ pat = concat3 (build_string ("^&"),
+ pat, build_string ("\\([0-9a-f]+\\)$"));
+ base = 16;
+ }
+ else if (EQ (ret, QX))
+ {
+ pat = concat3 (build_string ("^&"),
+ pat, build_string ("\\([0-9A-F]+\\)$"));
+ base = 16;
+ }
+ else
+ continue;
+
+ if (!NILP (Fstring_match (pat, string, Qnil, Qnil)))
+ {
+ int code
+ = XINT (Fstring_to_number
+ (Fsubstring (string,
+ Fmatch_beginning (make_int (1)),
+ Fmatch_end (make_int (1))),
+ make_int (base)));
+
+ DECODE_ADD_UCS_CHAR (DECODE_CHAR (ccs, code), dst);
+ goto decoded;
+ }
+ rest = Fcdr (rest);
+ }
+ if (!NILP (Fstring_match (build_string ("^&MCS-\\([0-9A-F]+\\)$"),
+ string, Qnil, Qnil)))
+ {
+ int code
+ = XINT (Fstring_to_number
+ (Fsubstring (string,
+ Fmatch_beginning (make_int (1)),
+ Fmatch_end (make_int (1))),
+ make_int (16)));
+
+ DECODE_ADD_UCS_CHAR (code, dst);
+ }
+ else
+ {
+ Dynarr_add_many (dst, str->er_buf, str->er_counter);
+ Dynarr_add (dst, ';');
+ }
+ decoded:
+ str->er_counter = 0;
+ }
+ else if ( (str->er_counter >= 16) || (c >= 0x7F) )
+ {
+ Dynarr_add_many (dst, str->er_buf, str->er_counter);
+ str->er_counter = 0;
+ DECODE_ADD_UCS_CHAR (c, dst);
+ }
+ else
+ str->er_buf[str->er_counter++] = c;
+}
INLINE_HEADER void
COMPOSE_FLUSH_CHARS (struct decoding_stream *str, unsigned_char_dynarr* dst);
unsigned i;
for (i = 0; i < str->combined_char_count; i++)
- DECODE_ADD_UCS_CHAR (str->combined_chars[i], dst);
+ decode_add_er_char (str, str->combined_chars[i], dst);
str->combined_char_count = 0;
str->combining_table = Qnil;
}
-void COMPOSE_ADD_CHAR(struct decoding_stream *str, Emchar character,
- unsigned_char_dynarr* dst);
+void COMPOSE_ADD_CHAR (struct decoding_stream *str, Emchar character,
+ unsigned_char_dynarr* dst);
void
-COMPOSE_ADD_CHAR(struct decoding_stream *str,
- Emchar character, unsigned_char_dynarr* dst)
+COMPOSE_ADD_CHAR (struct decoding_stream *str,
+ Emchar character, unsigned_char_dynarr* dst)
{
if (CODING_SYSTEM_DISABLE_COMPOSITION (str->codesys))
- DECODE_ADD_UCS_CHAR (character, dst);
- else if (!CHAR_ID_TABLE_P (str->combining_table))
+ decode_add_er_char (str, character, dst);
+ else if (!CONSP (str->combining_table))
{
Lisp_Object ret
- = get_char_id_table (character, Vcharacter_composition_table);
+ = Fget_char_attribute (make_char (character), Qcomposition, Qnil);
if (NILP (ret))
- DECODE_ADD_UCS_CHAR (character, dst);
+ decode_add_er_char (str, character, dst);
else
{
str->combined_chars[0] = character;
}
else
{
- Lisp_Object ret
- = get_char_id_table (character, str->combining_table);
+ Lisp_Object ret = Fcdr (Fassq (make_char (character), str->combining_table));
if (CHARP (ret))
{
Emchar char2 = XCHARVAL (ret);
- ret = get_char_id_table (char2, Vcharacter_composition_table);
+ ret = Fget_char_attribute (make_char (character), Qcomposition, Qnil);
if (NILP (ret))
{
- DECODE_ADD_UCS_CHAR (char2, dst);
+ decode_add_er_char (str, character, dst);
str->combined_char_count = 0;
str->combining_table = Qnil;
}
str->combining_table = ret;
}
}
- else if (CHAR_ID_TABLE_P (ret))
- {
- str->combined_chars[str->combined_char_count++] = character;
- str->combining_table = ret;
- }
else
{
COMPOSE_FLUSH_CHARS (str, dst);
- DECODE_ADD_UCS_CHAR (character, dst);
+ decode_add_er_char (str, character, dst);
}
}
}
/* There might be some more end data produced in the translation.
See the comment above. */
str->flags |= CODING_STATE_END;
- mule_decode (stream, data, str->runoff, read_size);
+ mule_decode (stream, (Extbyte *) data, str->runoff, read_size);
}
if (data - orig_data == 0)
/* Decode all our data into the runoff, and then attempt to write
it all out to the other end. Remove whatever chunk we succeeded
in writing. */
- mule_decode (stream, data, str->runoff, size);
+ mule_decode (stream, (Extbyte *) data, str->runoff, size);
retval = Lstream_write (str->other_end, Dynarr_atp (str->runoff, 0),
Dynarr_length (str->runoff));
if (retval > 0)
str->counter = 0;
#endif /* MULE */
#ifdef UTF2000
+ str->er_counter = 0;
str->combined_char_count = 0;
str->combining_table = Qnil;
#endif
be used for both reading and writing. */
static void
-mule_decode (Lstream *decoding, const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n)
+mule_decode (Lstream *decoding, const Extbyte *src,
+ unsigned_char_dynarr *dst, size_t n)
{
struct decoding_stream *str = DECODING_STREAM_DATA (decoding);
break;
case CODESYS_CCL:
str->ccl.last_block = str->flags & CODING_STATE_END;
- ccl_driver (&str->ccl, src, dst, n, 0, CCL_MODE_DECODING);
+ /* When applying ccl program to stream, MUST NOT set NULL
+ pointer to src. */
+ ccl_driver (&str->ccl, (src ? (unsigned char *)src : (unsigned char*)""),
+ dst, n, 0, CCL_MODE_DECODING);
break;
case CODESYS_ISO2022:
decode_coding_iso2022 (decoding, src, dst, n);
str->encode_char = &char_encode_shift_jis;
str->finish = &char_finish_shift_jis;
break;
+ case CODESYS_BIG5:
+ str->encode_char = &char_encode_big5;
+ str->finish = &char_finish_big5;
+ break;
default:
break;
}
Store the encoded data into DST. */
static void
-mule_encode (Lstream *encoding, const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n)
+mule_encode (Lstream *encoding, const Bufbyte *src,
+ unsigned_char_dynarr *dst, size_t n)
{
struct encoding_stream *str = ENCODING_STREAM_DATA (encoding);
encode_coding_no_conversion (encoding, src, dst, n);
break;
#ifdef MULE
- case CODESYS_BIG5:
- encode_coding_big5 (encoding, src, dst, n);
- break;
case CODESYS_CCL:
str->ccl.last_block = str->flags & CODING_STATE_END;
- ccl_driver (&str->ccl, src, dst, n, 0, CCL_MODE_ENCODING);
+ /* When applying ccl program to stream, MUST NOT set NULL
+ pointer to src. */
+ ccl_driver (&str->ccl, ((src) ? src : (unsigned char*)""),
+ dst, n, 0, CCL_MODE_ENCODING);
break;
#endif /* MULE */
default:
#ifdef MULE
\f
static void
-text_encode_generic (Lstream *encoding, const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n)
+text_encode_generic (Lstream *encoding, const Bufbyte *src,
+ unsigned_char_dynarr *dst, size_t n)
{
unsigned char c;
unsigned char char_boundary;
((c) >= 0xA1 && (c) <= 0xDF)
static int
-detect_coding_sjis (struct detection_state *st, const unsigned char *src,
- unsigned int n)
+detect_coding_sjis (struct detection_state *st, const Extbyte *src, size_t n)
{
- int c;
-
while (n--)
{
- c = *src++;
+ unsigned char c = *(unsigned char *)src++;
if (c == ISO_CODE_ESC || c == ISO_CODE_SI || c == ISO_CODE_SO)
return 0;
if (st->shift_jis.in_second_byte)
/* Convert Shift-JIS data to internal format. */
static void
-decode_coding_sjis (Lstream *decoding, const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n)
+decode_coding_sjis (Lstream *decoding, const Extbyte *src,
+ unsigned_char_dynarr *dst, size_t n)
{
- unsigned char c;
struct decoding_stream *str = DECODING_STREAM_DATA (decoding);
unsigned int flags = str->flags;
unsigned int cpos = str->cpos;
while (n--)
{
- c = *src++;
+ unsigned char c = *(unsigned char *)src++;
if (cpos)
{
}
else
{
- Lisp_Object charset;
- unsigned int c1, c2, s1, s2;
+ unsigned int s1, s2;
#ifdef UTF2000
int code_point = charset_code_point (Vcharset_latin_jisx0201, ch);
if (code_point >= 0)
+ Dynarr_add (dst, code_point);
+ else if ((code_point
+ = charset_code_point (Vcharset_japanese_jisx0208_1990, ch))
+ >= 0)
{
- charset = Vcharset_latin_jisx0201;
- c1 = code_point;
- c2 = 0;
+ ENCODE_SJIS ((code_point >> 8) | 0x80,
+ (code_point & 0xFF) | 0x80, s1, s2);
+ Dynarr_add (dst, s1);
+ Dynarr_add (dst, s2);
}
+ else if ((code_point
+ = charset_code_point (Vcharset_katakana_jisx0201, ch))
+ >= 0)
+ Dynarr_add (dst, code_point | 0x80);
+ else if ((code_point
+ = charset_code_point (Vcharset_japanese_jisx0208, ch))
+ >= 0)
+ {
+ ENCODE_SJIS ((code_point >> 8) | 0x80,
+ (code_point & 0xFF) | 0x80, s1, s2);
+ Dynarr_add (dst, s1);
+ Dynarr_add (dst, s2);
+ }
+ else if ((code_point = charset_code_point (Vcharset_ascii, ch))
+ >= 0)
+ Dynarr_add (dst, code_point);
else
-#endif
- BREAKUP_CHAR (ch, charset, c1, c2);
+ Dynarr_add (dst, '?');
+#else
+ Lisp_Object charset;
+ unsigned int c1, c2;
+
+ BREAKUP_CHAR (ch, charset, c1, c2);
if (EQ(charset, Vcharset_katakana_jisx0201))
{
}
else
Dynarr_add (dst, '?');
+#endif
}
}
}
DEFUN ("encode-shift-jis-char", Fencode_shift_jis_char, 1, 1, 0, /*
-Encode a JISX0208 character CHAR to SHIFT-JIS coding-system.
+Encode a JISX0208 character CHARACTER to SHIFT-JIS coding-system.
Return the corresponding character code in SHIFT-JIS as a cons of two bytes.
*/
- (ch))
+ (character))
{
Lisp_Object charset;
int c1, c2, s1, s2;
- CHECK_CHAR_COERCE_INT (ch);
- BREAKUP_CHAR (XCHAR (ch), charset, c1, c2);
+ CHECK_CHAR_COERCE_INT (character);
+ BREAKUP_CHAR (XCHAR (character), charset, c1, c2);
if (EQ (charset, Vcharset_japanese_jisx0208))
{
ENCODE_SJIS (c1 | 0x80, c2 | 0x80, s1, s2);
contains frequently used characters and the latter contains less
frequently used characters. */
+#ifdef UTF2000
+#define BYTE_BIG5_TWO_BYTE_1_P(c) \
+ ((c) >= 0x81 && (c) <= 0xFE)
+#else
#define BYTE_BIG5_TWO_BYTE_1_P(c) \
((c) >= 0xA1 && (c) <= 0xFE)
+#endif
/* Is this the second byte of a Shift-JIS two-byte char? */
} while (0)
static int
-detect_coding_big5 (struct detection_state *st, const unsigned char *src,
- unsigned int n)
+detect_coding_big5 (struct detection_state *st, const Extbyte *src, size_t n)
{
- int c;
-
while (n--)
{
- c = *src++;
- if (c == ISO_CODE_ESC || c == ISO_CODE_SI || c == ISO_CODE_SO ||
- (c >= 0x80 && c <= 0xA0))
+ unsigned char c = *(unsigned char *)src++;
+ if (c == ISO_CODE_ESC || c == ISO_CODE_SI || c == ISO_CODE_SO
+#ifndef UTF2000
+ || (c >= 0x80 && c <= 0xA0)
+#endif
+ )
return 0;
if (st->big5.in_second_byte)
{
if (c < 0x40 || (c >= 0x80 && c <= 0xA0))
return 0;
}
- else if (c >= 0xA1)
+ else if (
+#ifdef UTF2000
+ c >= 0x81
+#else
+ c >= 0xA1
+#endif
+ )
st->big5.in_second_byte = 1;
}
return CODING_CATEGORY_BIG5_MASK;
/* Convert Big5 data to internal format. */
static void
-decode_coding_big5 (Lstream *decoding, const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n)
+decode_coding_big5 (Lstream *decoding, const Extbyte *src,
+ unsigned_char_dynarr *dst, size_t n)
{
- unsigned char c;
struct decoding_stream *str = DECODING_STREAM_DATA (decoding);
unsigned int flags = str->flags;
unsigned int cpos = str->cpos;
eol_type_t eol_type = str->eol_type;
+#ifdef UTF2000
+ Lisp_Object ccs
+ = CODING_SYSTEM_ISO2022_INITIAL_CHARSET (DECODING_STREAM_DATA
+ (decoding)->codesys, 1);
+#endif
while (n--)
{
- c = *src++;
+ unsigned char c = *(unsigned char *)src++;
if (cpos)
{
/* Previous character was first byte of Big5 char. */
if (BYTE_BIG5_TWO_BYTE_2_P (c))
{
#ifdef UTF2000
- Charset_ID b1;
- unsigned char b2, b3;
- DECODE_BIG5 (cpos, c, b1, b2, b3);
- DECODE_ADD_UCS_CHAR (MAKE_CHAR (CHARSET_BY_LEADING_BYTE (b1),
- b2 & 0x7F, b3 & 0x7F),
- dst);
+ int code_point = (cpos << 8) | c;
+ Emchar char_id = decode_defined_char (ccs, code_point);
+
+ if (char_id < 0)
+ char_id = DECODE_CHAR (Vcharset_chinese_big5, code_point);
+ DECODE_ADD_UCS_CHAR (char_id, dst);
#else
unsigned char b1, b2, b3;
DECODE_BIG5 (cpos, c, b1, b2, b3);
/* Convert internally-formatted data to Big5. */
-static void
-encode_coding_big5 (Lstream *encoding, const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n)
+void
+char_encode_big5 (struct encoding_stream *str, Emchar ch,
+ unsigned_char_dynarr *dst, unsigned int *flags)
{
-#ifndef UTF2000
- unsigned char c;
- struct encoding_stream *str = ENCODING_STREAM_DATA (encoding);
- unsigned int flags = str->flags;
- unsigned int ch = str->ch;
eol_type_t eol_type = CODING_SYSTEM_EOL_TYPE (str->codesys);
- while (n--)
+ if (ch == '\n')
{
- c = *src++;
- if (c == '\n')
- {
- if (eol_type != EOL_LF && eol_type != EOL_AUTODETECT)
- Dynarr_add (dst, '\r');
- if (eol_type != EOL_CR)
- Dynarr_add (dst, '\n');
- }
- else if (BYTE_ASCII_P (c))
+ if (eol_type != EOL_LF && eol_type != EOL_AUTODETECT)
+ Dynarr_add (dst, '\r');
+ if (eol_type != EOL_CR)
+ Dynarr_add (dst, ch);
+ }
+ else
+ {
+#ifdef UTF2000
+ int code_point;
+ Lisp_Object ccs
+ = CODING_SYSTEM_ISO2022_INITIAL_CHARSET (str->codesys, 1);
+
+ if ((code_point = charset_code_point (Vcharset_ascii, ch)) >= 0)
+ Dynarr_add (dst, code_point);
+ else if ((code_point = charset_code_point (ccs, ch)) >= 0)
{
- /* ASCII. */
- Dynarr_add (dst, c);
+ Dynarr_add (dst, code_point >> 8);
+ Dynarr_add (dst, code_point & 0xFF);
}
- else if (BUFBYTE_LEADING_BYTE_P (c))
+ else if ((code_point
+ = charset_code_point (Vcharset_chinese_big5, ch)) >= 0)
{
- if (c == LEADING_BYTE_CHINESE_BIG5_1 ||
- c == LEADING_BYTE_CHINESE_BIG5_2)
- {
- /* A recognized leading byte. */
- ch = c;
- continue; /* not done with this character. */
- }
- /* otherwise just ignore this character. */
+ Dynarr_add (dst, code_point >> 8);
+ Dynarr_add (dst, code_point & 0xFF);
}
- else if (ch == LEADING_BYTE_CHINESE_BIG5_1 ||
- ch == LEADING_BYTE_CHINESE_BIG5_2)
+ else if ((code_point
+ = charset_code_point (Vcharset_chinese_big5_1, ch)) >= 0)
{
- /* Previous char was a recognized leading byte. */
- ch = (ch << 8) | c;
- continue; /* not done with this character. */
+ unsigned int I
+ = ((code_point >> 8) - 33) * (0xFF - 0xA1)
+ + ((code_point & 0xFF) - 33);
+ unsigned char b1 = I / BIG5_SAME_ROW + 0xA1;
+ unsigned char b2 = I % BIG5_SAME_ROW;
+
+ b2 += b2 < 0x3F ? 0x40 : 0x62;
+ Dynarr_add (dst, b1);
+ Dynarr_add (dst, b2);
}
- else if (ch)
+ else if ((code_point
+ = charset_code_point (Vcharset_chinese_big5_2, ch)) >= 0)
{
- /* Encountering second byte of a Big5 character. */
+ unsigned int I
+ = ((code_point >> 8) - 33) * (0xFF - 0xA1)
+ + ((code_point & 0xFF) - 33);
unsigned char b1, b2;
- ENCODE_BIG5 (ch >> 8, ch & 0xFF, c, b1, b2);
+ I += BIG5_SAME_ROW * (0xC9 - 0xA1);
+ b1 = I / BIG5_SAME_ROW + 0xA1;
+ b2 = I % BIG5_SAME_ROW;
+ b2 += b2 < 0x3F ? 0x40 : 0x62;
Dynarr_add (dst, b1);
Dynarr_add (dst, b2);
}
-
- ch = 0;
+ else
+ Dynarr_add (dst, '?');
+#else
+#endif
}
+}
- str->flags = flags;
- str->ch = ch;
-#endif
+void
+char_finish_big5 (struct encoding_stream *str, unsigned_char_dynarr *dst,
+ unsigned int *flags)
+{
}
}
DEFUN ("encode-big5-char", Fencode_big5_char, 1, 1, 0, /*
-Encode the Big5 character CH to BIG5 coding-system.
+Encode the Big5 character CHARACTER in the BIG5 coding-system.
Return the corresponding character code in Big5.
*/
- (ch))
+ (character))
{
Lisp_Object charset;
int c1, c2, b1, b2;
- CHECK_CHAR_COERCE_INT (ch);
- BREAKUP_CHAR (XCHAR (ch), charset, c1, c2);
+ CHECK_CHAR_COERCE_INT (character);
+ BREAKUP_CHAR (XCHAR (character), charset, c1, c2);
if (EQ (charset, Vcharset_chinese_big5_1) ||
EQ (charset, Vcharset_chinese_big5_2))
{
/************************************************************************/
static int
-detect_coding_ucs4 (struct detection_state *st, const unsigned char *src,
- unsigned int n)
+detect_coding_ucs4 (struct detection_state *st, const Extbyte *src, size_t n)
{
while (n--)
{
- int c = *src++;
+ unsigned char c = *(unsigned char *)src++;
switch (st->ucs4.in_byte)
{
case 0:
}
static void
-decode_coding_ucs4 (Lstream *decoding, const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n)
+decode_coding_ucs4 (Lstream *decoding, const Extbyte *src,
+ unsigned_char_dynarr *dst, size_t n)
{
struct decoding_stream *str = DECODING_STREAM_DATA (decoding);
unsigned int flags = str->flags;
while (n--)
{
- unsigned char c = *src++;
+ unsigned char c = *(unsigned char *)src++;
switch (counter)
{
case 0:
/************************************************************************/
static int
-detect_coding_utf8 (struct detection_state *st, const unsigned char *src,
- unsigned int n)
+detect_coding_utf8 (struct detection_state *st, const Extbyte *src, size_t n)
{
while (n--)
{
- unsigned char c = *src++;
+ unsigned char c = *(unsigned char *)src++;
switch (st->utf8.in_byte)
{
case 0:
}
static void
-decode_coding_utf8 (Lstream *decoding, const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n)
+decode_output_utf8_partial_char (unsigned char counter,
+ unsigned int cpos,
+ unsigned_char_dynarr *dst)
+{
+ if (counter == 5)
+ DECODE_ADD_BINARY_CHAR ( (cpos|0xFC), dst);
+ else if (counter == 4)
+ {
+ if (cpos < (1 << 6))
+ DECODE_ADD_BINARY_CHAR ( (cpos|0xF8), dst);
+ else
+ {
+ DECODE_ADD_BINARY_CHAR ( ((cpos >> 6)|0xFC), dst);
+ DECODE_ADD_BINARY_CHAR ( ((cpos&0x3F)|0x80), dst);
+ }
+ }
+ else if (counter == 3)
+ {
+ if (cpos < (1 << 6))
+ DECODE_ADD_BINARY_CHAR ( (cpos|0xF0), dst);
+ else if (cpos < (1 << 12))
+ {
+ DECODE_ADD_BINARY_CHAR ( ((cpos >> 6)|0xF8), dst);
+ DECODE_ADD_BINARY_CHAR ( ((cpos&0x3F)|0x80), dst);
+ }
+ else
+ {
+ DECODE_ADD_BINARY_CHAR ( ( (cpos >> 12)|0xFC), dst);
+ DECODE_ADD_BINARY_CHAR ( (((cpos >> 6)&0x3F)|0x80), dst);
+ DECODE_ADD_BINARY_CHAR ( ( (cpos &0x3F)|0x80), dst);
+ }
+ }
+ else if (counter == 2)
+ {
+ if (cpos < (1 << 6))
+ DECODE_ADD_BINARY_CHAR ( (cpos|0xE0), dst);
+ else if (cpos < (1 << 12))
+ {
+ DECODE_ADD_BINARY_CHAR ( ((cpos >> 6)|0xF0), dst);
+ DECODE_ADD_BINARY_CHAR ( ((cpos&0x3F)|0x80), dst);
+ }
+ else if (cpos < (1 << 18))
+ {
+ DECODE_ADD_BINARY_CHAR ( ( (cpos >> 12)|0xF8), dst);
+ DECODE_ADD_BINARY_CHAR ( (((cpos >> 6)&0x3F)|0x80), dst);
+ DECODE_ADD_BINARY_CHAR ( ( (cpos &0x3F)|0x80), dst);
+ }
+ else
+ {
+ DECODE_ADD_BINARY_CHAR ( ( (cpos >> 18)|0xFC), dst);
+ DECODE_ADD_BINARY_CHAR ( (((cpos >> 12)&0x3F)|0x80), dst);
+ DECODE_ADD_BINARY_CHAR ( (((cpos >> 6)&0x3F)|0x80), dst);
+ DECODE_ADD_BINARY_CHAR ( ( (cpos &0x3F)|0x80), dst);
+ }
+ }
+ else
+ {
+ if (cpos < (1 << 6))
+ DECODE_ADD_BINARY_CHAR ( (cpos|0xC0), dst);
+ else if (cpos < (1 << 12))
+ {
+ DECODE_ADD_BINARY_CHAR ( ((cpos >> 6)|0xE0), dst);
+ DECODE_ADD_BINARY_CHAR ( ((cpos&0x3F)|0x80), dst);
+ }
+ else if (cpos < (1 << 18))
+ {
+ DECODE_ADD_BINARY_CHAR ( ( (cpos >> 12)|0xF0), dst);
+ DECODE_ADD_BINARY_CHAR ( (((cpos >> 6)&0x3F)|0x80), dst);
+ DECODE_ADD_BINARY_CHAR ( ( (cpos &0x3F)|0x80), dst);
+ }
+ else if (cpos < (1 << 24))
+ {
+ DECODE_ADD_BINARY_CHAR ( ( (cpos >> 18)|0xF8), dst);
+ DECODE_ADD_BINARY_CHAR ( (((cpos >> 12)&0x3F)|0x80), dst);
+ DECODE_ADD_BINARY_CHAR ( (((cpos >> 6)&0x3F)|0x80), dst);
+ DECODE_ADD_BINARY_CHAR ( ( (cpos &0x3F)|0x80), dst);
+ }
+ else
+ {
+ DECODE_ADD_BINARY_CHAR ( ( (cpos >> 24)|0xFC), dst);
+ DECODE_ADD_BINARY_CHAR ( (((cpos >> 18)&0x3F)|0x80), dst);
+ DECODE_ADD_BINARY_CHAR ( (((cpos >> 12)&0x3F)|0x80), dst);
+ DECODE_ADD_BINARY_CHAR ( (((cpos >> 6)&0x3F)|0x80), dst);
+ DECODE_ADD_BINARY_CHAR ( ( (cpos &0x3F)|0x80), dst);
+ }
+ }
+}
+
+static void
+decode_coding_utf8 (Lstream *decoding, const Extbyte *src,
+ unsigned_char_dynarr *dst, size_t n)
{
struct decoding_stream *str = DECODING_STREAM_DATA (decoding);
unsigned int flags = str->flags;
while (n--)
{
- unsigned char c = *src++;
- switch (counter)
+ unsigned char c = *(unsigned char *)src++;
+ if (counter == 0)
{
- case 0:
- if ( c >= 0xfc )
- {
- cpos = c & 0x01;
- counter = 5;
- }
- else if ( c >= 0xf8 )
+ if ( c < ' ' )
{
- cpos = c & 0x03;
- counter = 4;
- }
- else if ( c >= 0xf0 )
- {
- cpos = c & 0x07;
- counter = 3;
+ COMPOSE_FLUSH_CHARS (str, dst);
+ decode_flush_er_chars (str, dst);
+ DECODE_HANDLE_EOL_TYPE (eol_type, c, flags, dst);
+ DECODE_ADD_UCS_CHAR (c, dst);
}
- else if ( c >= 0xe0 )
+ else if ( c < 0xC0 )
+ /* decode_add_er_char (str, c, dst); */
+ COMPOSE_ADD_CHAR (str, c, dst);
+ else
{
- cpos = c & 0x0f;
- counter = 2;
+ /* decode_flush_er_chars (str, dst); */
+ if ( c < 0xE0 )
+ {
+ cpos = c & 0x1f;
+ counter = 1;
+ }
+ else if ( c < 0xF0 )
+ {
+ cpos = c & 0x0f;
+ counter = 2;
+ }
+ else if ( c < 0xF8 )
+ {
+ cpos = c & 0x07;
+ counter = 3;
+ }
+ else if ( c < 0xFC )
+ {
+ cpos = c & 0x03;
+ counter = 4;
+ }
+ else
+ {
+ cpos = c & 0x01;
+ counter = 5;
+ }
}
- else if ( c >= 0xc0 )
+ }
+ else if ( (c & 0xC0) == 0x80 )
+ {
+ cpos = ( cpos << 6 ) | ( c & 0x3f );
+ if (counter == 1)
{
- cpos = c & 0x1f;
- counter = 1;
+ /* DECODE_ADD_UCS_CHAR (cpos, dst); */
+ COMPOSE_ADD_CHAR (str, cpos, dst);
+ cpos = 0;
+ counter = 0;
}
else
- {
- DECODE_HANDLE_EOL_TYPE (eol_type, c, flags, dst);
- DECODE_ADD_UCS_CHAR (c, dst);
- }
- break;
- case 1:
- cpos = ( cpos << 6 ) | ( c & 0x3f );
- DECODE_ADD_UCS_CHAR (cpos, dst);
+ counter--;
+ }
+ else
+ {
+ COMPOSE_FLUSH_CHARS (str, dst);
+ decode_flush_er_chars (str, dst);
+ decode_output_utf8_partial_char (counter, cpos, dst);
+ DECODE_ADD_BINARY_CHAR (c, dst);
cpos = 0;
counter = 0;
- break;
- default:
- cpos = ( cpos << 6 ) | ( c & 0x3f );
- counter--;
}
label_continue_loop:;
}
if (flags & CODING_STATE_END)
- DECODE_OUTPUT_PARTIAL_CHAR (cpos);
-
+ {
+ COMPOSE_FLUSH_CHARS (str, dst);
+ decode_flush_er_chars (str, dst);
+ if (counter > 0)
+ {
+ decode_output_utf8_partial_char (counter, cpos, dst);
+ cpos = 0;
+ counter = 0;
+ }
+ }
str->flags = flags;
str->cpos = cpos;
str->counter = counter;
{
Dynarr_add (dst, ch);
}
- else if (ch <= 0x7ff)
- {
- Dynarr_add (dst, (ch >> 6) | 0xc0);
- Dynarr_add (dst, (ch & 0x3f) | 0x80);
- }
- else if (ch <= 0xffff)
- {
- Dynarr_add (dst, (ch >> 12) | 0xe0);
- Dynarr_add (dst, ((ch >> 6) & 0x3f) | 0x80);
- Dynarr_add (dst, (ch & 0x3f) | 0x80);
- }
- else if (ch <= 0x1fffff)
- {
- Dynarr_add (dst, (ch >> 18) | 0xf0);
- Dynarr_add (dst, ((ch >> 12) & 0x3f) | 0x80);
- Dynarr_add (dst, ((ch >> 6) & 0x3f) | 0x80);
- Dynarr_add (dst, (ch & 0x3f) | 0x80);
- }
- else if (ch <= 0x3ffffff)
- {
- Dynarr_add (dst, (ch >> 24) | 0xf8);
- Dynarr_add (dst, ((ch >> 18) & 0x3f) | 0x80);
- Dynarr_add (dst, ((ch >> 12) & 0x3f) | 0x80);
- Dynarr_add (dst, ((ch >> 6) & 0x3f) | 0x80);
- Dynarr_add (dst, (ch & 0x3f) | 0x80);
- }
else
{
- Dynarr_add (dst, (ch >> 30) | 0xfc);
- Dynarr_add (dst, ((ch >> 24) & 0x3f) | 0x80);
- Dynarr_add (dst, ((ch >> 18) & 0x3f) | 0x80);
- Dynarr_add (dst, ((ch >> 12) & 0x3f) | 0x80);
- Dynarr_add (dst, ((ch >> 6) & 0x3f) | 0x80);
- Dynarr_add (dst, (ch & 0x3f) | 0x80);
+ int code_point = charset_code_point (Vcharset_ucs, ch);
+
+ if ( (code_point < 0) || (code_point > 0x10FFFF) )
+ {
+ if (CODING_SYSTEM_USE_ENTITY_REFERENCE (str->codesys))
+ {
+ Lisp_Object rest = Vcoded_charset_entity_reference_alist;
+ Lisp_Object cell;
+ Lisp_Object ret;
+ Lisp_Object ccs;
+ int format_columns, idx;
+ char buf[16], format[16];
+
+ while (!NILP (rest))
+ {
+ cell = Fcar (rest);
+ ccs = Fcar (cell);
+ if (!NILP (ccs = Ffind_charset (ccs)))
+ {
+ if ( (code_point
+ = charset_code_point (ccs, ch)) >= 0 )
+ {
+ cell = Fcdr (cell);
+ ret = Fcar (cell);
+ if (STRINGP (ret)
+ && ((idx =XSTRING_LENGTH (ret)) <= 6))
+ {
+ strncpy (format, XSTRING_DATA (ret), idx);
+ }
+ else
+ continue;
+
+ cell = Fcdr (cell);
+ ret = Fcar (cell);
+ if (INTP (ret))
+ {
+ format [idx++] = '%';
+ format_columns = XINT (ret);
+ if ( (2 <= format_columns)
+ && (format_columns <= 8) )
+ {
+ format [idx++] = '0';
+ format [idx++] = '0' + format_columns;
+ }
+ }
+
+ cell = Fcdr (cell);
+ ret = Fcar (cell);
+ if (EQ (ret, Qd))
+ format [idx++] = 'd';
+ else if (EQ (ret, Qx))
+ format [idx++] = 'x';
+ else if (EQ (ret, QX))
+ format [idx++] = 'X';
+ else
+ continue;
+ format [idx++] = 0;
+
+ sprintf (buf, format, code_point);
+ Dynarr_add (dst, '&');
+ Dynarr_add_many (dst, buf, strlen (buf));
+ Dynarr_add (dst, ';');
+ return;
+ }
+ }
+ rest = Fcdr (rest);
+ }
+ sprintf (buf, "&MCS-%08X;", ch);
+ Dynarr_add_many (dst, buf, strlen (buf));
+ return;
+ }
+ else
+ code_point = ch;
+ }
+ if (code_point <= 0x7ff)
+ {
+ Dynarr_add (dst, (code_point >> 6) | 0xc0);
+ Dynarr_add (dst, (code_point & 0x3f) | 0x80);
+ }
+ else if (code_point <= 0xffff)
+ {
+ Dynarr_add (dst, (code_point >> 12) | 0xe0);
+ Dynarr_add (dst, ((code_point >> 6) & 0x3f) | 0x80);
+ Dynarr_add (dst, (code_point & 0x3f) | 0x80);
+ }
+ else if (code_point <= 0x1fffff)
+ {
+ Dynarr_add (dst, (code_point >> 18) | 0xf0);
+ Dynarr_add (dst, ((code_point >> 12) & 0x3f) | 0x80);
+ Dynarr_add (dst, ((code_point >> 6) & 0x3f) | 0x80);
+ Dynarr_add (dst, (code_point & 0x3f) | 0x80);
+ }
+ else if (code_point <= 0x3ffffff)
+ {
+ Dynarr_add (dst, (code_point >> 24) | 0xf8);
+ Dynarr_add (dst, ((code_point >> 18) & 0x3f) | 0x80);
+ Dynarr_add (dst, ((code_point >> 12) & 0x3f) | 0x80);
+ Dynarr_add (dst, ((code_point >> 6) & 0x3f) | 0x80);
+ Dynarr_add (dst, (code_point & 0x3f) | 0x80);
+ }
+ else
+ {
+ Dynarr_add (dst, (code_point >> 30) | 0xfc);
+ Dynarr_add (dst, ((code_point >> 24) & 0x3f) | 0x80);
+ Dynarr_add (dst, ((code_point >> 18) & 0x3f) | 0x80);
+ Dynarr_add (dst, ((code_point >> 12) & 0x3f) | 0x80);
+ Dynarr_add (dst, ((code_point >> 6) & 0x3f) | 0x80);
+ Dynarr_add (dst, (code_point & 0x3f) | 0x80);
+ }
}
}
}
static int
-detect_coding_iso2022 (struct detection_state *st, const unsigned char *src,
- unsigned int n)
+detect_coding_iso2022 (struct detection_state *st, const Extbyte *src, size_t n)
{
int mask;
while (n--)
{
- int c = *src++;
+ unsigned char c = *(unsigned char *)src++;
if (c >= 0xA0)
{
mask &= ~CODING_CATEGORY_ISO_7_MASK;
/* Convert ISO2022-format data to internal format. */
static void
-decode_coding_iso2022 (Lstream *decoding, const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n)
+decode_coding_iso2022 (Lstream *decoding, const Extbyte *src,
+ unsigned_char_dynarr *dst, size_t n)
{
struct decoding_stream *str = DECODING_STREAM_DATA (decoding);
unsigned int flags = str->flags;
while (n--)
{
- unsigned char c = *src++;
+ unsigned char c = *(unsigned char *)src++;
if (flags & CODING_STATE_ESCAPE)
{ /* Within ESC sequence */
int retval = parse_iso2022_esc (coding_system, &str->iso2022,
case ISO_ESC_LITERAL:
COMPOSE_FLUSH_CHARS (str, dst);
+ decode_flush_er_chars (str, dst);
DECODE_ADD_BINARY_CHAR (c, dst);
break;
/* Output the (possibly invalid) sequence */
int i;
COMPOSE_FLUSH_CHARS (str, dst);
+ decode_flush_er_chars (str, dst);
for (i = 0; i < str->iso2022.esc_bytes_index; i++)
DECODE_ADD_BINARY_CHAR (str->iso2022.esc_bytes[i], dst);
flags &= CODING_STATE_ISO2022_LOCK;
escape sequence; it could mess things up anyway.
Just add it now. */
COMPOSE_FLUSH_CHARS (str, dst);
+ decode_flush_er_chars (str, dst);
DECODE_ADD_BINARY_CHAR (c, dst);
}
}
if (counter)
{
COMPOSE_FLUSH_CHARS (str, dst);
+ decode_flush_er_chars (str, dst);
while (counter > 0)
{
counter--;
if (flags & CODING_STATE_SS2)
{
COMPOSE_FLUSH_CHARS (str, dst);
+ decode_flush_er_chars (str, dst);
DECODE_ADD_BINARY_CHAR (ISO_CODE_SS2, dst);
flags &= ~CODING_STATE_SS2;
}
if (flags & CODING_STATE_SS3)
{
COMPOSE_FLUSH_CHARS (str, dst);
+ decode_flush_er_chars (str, dst);
DECODE_ADD_BINARY_CHAR (ISO_CODE_SS3, dst);
flags &= ~CODING_STATE_SS3;
}
if (c == '\r')
{
COMPOSE_FLUSH_CHARS (str, dst);
+ decode_flush_er_chars (str, dst);
if (eol_type == EOL_CR)
Dynarr_add (dst, '\n');
else if (eol_type != EOL_CRLF || flags & CODING_STATE_CR)
if (!parse_iso2022_esc (coding_system, &str->iso2022, c, &flags, 1))
{
COMPOSE_FLUSH_CHARS (str, dst);
+ decode_flush_er_chars (str, dst);
DECODE_ADD_BINARY_CHAR (c, dst);
}
}
if (c == '\r')
{
COMPOSE_FLUSH_CHARS (str, dst);
+ decode_flush_er_chars (str, dst);
if (eol_type == EOL_CR)
Dynarr_add (dst, '\n');
else if (eol_type != EOL_CRLF || flags & CODING_STATE_CR)
to preserve it for the output. */
{
COMPOSE_FLUSH_CHARS (str, dst);
+ decode_flush_er_chars (str, dst);
while (counter > 0)
{
counter--;
if (flags & CODING_STATE_END)
{
COMPOSE_FLUSH_CHARS (str, dst);
+ decode_flush_er_chars (str, dst);
DECODE_OUTPUT_PARTIAL_CHAR (cpos);
}
str->flags = flags;
contain all 256 possible byte values and that are not to be
interpreted as being in any particular decoding. */
static void
-decode_coding_no_conversion (Lstream *decoding, const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n)
+decode_coding_no_conversion (Lstream *decoding, const Extbyte *src,
+ unsigned_char_dynarr *dst, size_t n)
{
- unsigned char c;
struct decoding_stream *str = DECODING_STREAM_DATA (decoding);
unsigned int flags = str->flags;
unsigned int cpos = str->cpos;
while (n--)
{
- c = *src++;
+ unsigned char c = *(unsigned char *)src++;
DECODE_HANDLE_EOL_TYPE (eol_type, c, flags, dst);
DECODE_ADD_BINARY_CHAR (c, dst);
}
static void
-encode_coding_no_conversion (Lstream *encoding, const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n)
+encode_coding_no_conversion (Lstream *encoding, const Bufbyte *src,
+ unsigned_char_dynarr *dst, size_t n)
{
unsigned char c;
struct encoding_stream *str = ENCODING_STREAM_DATA (encoding);
#endif /* MULE */
#ifdef UTF2000
defsymbol (&Qdisable_composition, "disable-composition");
+ defsymbol (&Quse_entity_reference, "use-entity-reference");
+ defsymbol (&Qd, "d");
+ defsymbol (&Qx, "x");
+ defsymbol (&QX, "X");
#endif
defsymbol (&Qencode, "encode");
defsymbol (&Qdecode, "decode");
int i;
fcd = xnew (struct file_coding_dump);
- dumpstruct (&fcd, &fcd_description);
+ dump_add_root_struct_ptr (&fcd, &fcd_description);
/* Initialize to something reasonable ... */
- for (i = 0; i <= CODING_CATEGORY_LAST; i++)
+ for (i = 0; i < CODING_CATEGORY_LAST; i++)
{
fcd->coding_category_system[i] = Qnil;
fcd->coding_category_by_priority[i] = i;
*/ );
Vfile_name_coding_system = Qnil;
+ DEFVAR_LISP ("coded-charset-entity-reference-alist",
+ &Vcoded_charset_entity_reference_alist /*
+Alist of coded-charset vs corresponding entity-reference.
+Each element looks like (CCS PREFIX CODE-COLUMNS CODE-TYPE).
+CCS is coded-charset.
+CODE-COLUMNS is columns of code-point of entity-reference.
+CODE-TYPE is format type of code-point of entity-reference.
+`d' means decimal value and `x' means hexadecimal value.
+*/ );
+ Vcoded_charset_entity_reference_alist = Qnil;
+
DEFVAR_BOOL ("enable-multibyte-characters", &enable_multibyte_characters /*
Non-nil means the buffer contents are regarded as multi-byte form
of characters, not a binary code. This affects the display, file I/O,
make_lisp_hash_table (50, HASH_TABLE_NON_WEAK, HASH_TABLE_EQ);
the_codesys_prop_dynarr = Dynarr_new (codesys_prop);
- dumpstruct (&the_codesys_prop_dynarr, &codesys_prop_dynarr_description);
+ dump_add_root_struct_ptr (&the_codesys_prop_dynarr, &codesys_prop_dynarr_description);
#define DEFINE_CODESYS_PROP(Prop_Type, Sym) do \
{ \
DEFINE_CODESYS_PROP (CODESYS_PROP_CCL, Qencode);
DEFINE_CODESYS_PROP (CODESYS_PROP_CCL, Qdecode);
+#ifdef UTF2000
+ DEFINE_CODESYS_PROP (CODESYS_PROP_ALL_OK, Qdisable_composition);
+ DEFINE_CODESYS_PROP (CODESYS_PROP_ALL_OK, Quse_entity_reference);
+#endif
#endif /* MULE */
/* Need to create this here or we're really screwed. */
Fmake_coding_system
#if defined(MULE) && !defined(UTF2000)
{
- unsigned int i;
+ size_t i;
- for (i = 0; i < 65536; i++)
+ for (i = 0; i < countof (fcd->ucs_to_mule_table); i++)
fcd->ucs_to_mule_table[i] = Qnil;
}
staticpro (&mule_to_ucs_table);
projects / chise / xemacs-chise.git- / blobdiff