Lisp_Object Vfile_name_coding_system;
/* Table of symbols identifying each coding category. */
-Lisp_Object coding_category_symbol[CODING_CATEGORY_LAST + 1];
+Lisp_Object coding_category_symbol[CODING_CATEGORY_LAST];
struct file_coding_dump {
/* Coding system currently associated with each coding category. */
- Lisp_Object coding_category_system[CODING_CATEGORY_LAST + 1];
+ Lisp_Object coding_category_system[CODING_CATEGORY_LAST];
/* Table of all coding categories in decreasing order of priority.
This describes a permutation of the possible coding categories. */
- int coding_category_by_priority[CODING_CATEGORY_LAST + 1];
+ int coding_category_by_priority[CODING_CATEGORY_LAST];
-#ifdef MULE
+#if defined(MULE) && !defined(UTF2000)
Lisp_Object ucs_to_mule_table[65536];
#endif
} *fcd;
static const struct lrecord_description fcd_description_1[] = {
- { XD_LISP_OBJECT_ARRAY, offsetof (struct file_coding_dump, coding_category_system), CODING_CATEGORY_LAST + 1 },
-#ifdef MULE
+ { XD_LISP_OBJECT_ARRAY, offsetof (struct file_coding_dump, coding_category_system), CODING_CATEGORY_LAST },
+#if defined(MULE) && !defined(UTF2000)
{ XD_LISP_OBJECT_ARRAY, offsetof (struct file_coding_dump, ucs_to_mule_table), countof (fcd->ucs_to_mule_table) },
#endif
{ XD_END }
#ifdef MULE
struct detection_state;
static int detect_coding_sjis (struct detection_state *st,
- const unsigned char *src,
- unsigned int n);
-static void decode_coding_sjis (Lstream *decoding,
- const unsigned char *src,
- unsigned_char_dynarr *dst,
- unsigned int n);
-static void encode_coding_sjis (Lstream *encoding,
- const unsigned char *src,
- unsigned_char_dynarr *dst,
- unsigned int n);
+ const Extbyte *src, Lstream_data_count n);
+static void decode_coding_sjis (Lstream *decoding, const Extbyte *src,
+ unsigned_char_dynarr *dst, Lstream_data_count n);
+static void encode_coding_sjis (Lstream *encoding, const Bufbyte *src,
+ unsigned_char_dynarr *dst, Lstream_data_count n);
static int detect_coding_big5 (struct detection_state *st,
- const unsigned char *src,
- unsigned int n);
-static void decode_coding_big5 (Lstream *decoding,
- const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n);
-static void encode_coding_big5 (Lstream *encoding,
- const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n);
+ const Extbyte *src, Lstream_data_count n);
+static void decode_coding_big5 (Lstream *decoding, const Extbyte *src,
+ unsigned_char_dynarr *dst, Lstream_data_count n);
+static void encode_coding_big5 (Lstream *encoding, const Bufbyte *src,
+ unsigned_char_dynarr *dst, Lstream_data_count n);
static int detect_coding_ucs4 (struct detection_state *st,
- const unsigned char *src,
- unsigned int n);
-static void decode_coding_ucs4 (Lstream *decoding,
- const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n);
-static void encode_coding_ucs4 (Lstream *encoding,
- const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n);
+ const Extbyte *src, Lstream_data_count n);
+static void decode_coding_ucs4 (Lstream *decoding, const Extbyte *src,
+ unsigned_char_dynarr *dst, Lstream_data_count n);
+static void encode_coding_ucs4 (Lstream *encoding, const Bufbyte *src,
+ unsigned_char_dynarr *dst, Lstream_data_count n);
static int detect_coding_utf8 (struct detection_state *st,
- const unsigned char *src,
- unsigned int n);
-static void decode_coding_utf8 (Lstream *decoding,
- const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n);
-static void encode_coding_utf8 (Lstream *encoding,
- const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n);
+ const Extbyte *src, Lstream_data_count n);
+static void decode_coding_utf8 (Lstream *decoding, const Extbyte *src,
+ unsigned_char_dynarr *dst, Lstream_data_count n);
+static void encode_coding_utf8 (Lstream *encoding, const Bufbyte *src,
+ unsigned_char_dynarr *dst, Lstream_data_count n);
static int postprocess_iso2022_mask (int mask);
static void reset_iso2022 (Lisp_Object coding_system,
struct iso2022_decoder *iso);
static int detect_coding_iso2022 (struct detection_state *st,
- const unsigned char *src,
- unsigned int n);
-static void decode_coding_iso2022 (Lstream *decoding,
- const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n);
-static void encode_coding_iso2022 (Lstream *encoding,
- const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n);
+ const Extbyte *src, Lstream_data_count n);
+static void decode_coding_iso2022 (Lstream *decoding, const Extbyte *src,
+ unsigned_char_dynarr *dst, Lstream_data_count n);
+static void encode_coding_iso2022 (Lstream *encoding, const Bufbyte *src,
+ unsigned_char_dynarr *dst, Lstream_data_count n);
#endif /* MULE */
-static void decode_coding_no_conversion (Lstream *decoding,
- const unsigned char *src,
- unsigned_char_dynarr *dst,
- unsigned int n);
-static void encode_coding_no_conversion (Lstream *encoding,
- const unsigned char *src,
- unsigned_char_dynarr *dst,
- unsigned int n);
-static void mule_decode (Lstream *decoding, const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n);
-static void mule_encode (Lstream *encoding, const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n);
+static void decode_coding_no_conversion (Lstream *decoding, const Extbyte *src,
+ unsigned_char_dynarr *dst, Lstream_data_count n);
+static void encode_coding_no_conversion (Lstream *encoding, const Bufbyte *src,
+ unsigned_char_dynarr *dst, Lstream_data_count n);
+static void mule_decode (Lstream *decoding, const Extbyte *src,
+ unsigned_char_dynarr *dst, Lstream_data_count n);
+static void mule_encode (Lstream *encoding, const Bufbyte *src,
+ unsigned_char_dynarr *dst, Lstream_data_count n);
typedef struct codesys_prop codesys_prop;
struct codesys_prop
{
switch (type)
{
- default: abort ();
+ default: ABORT ();
case EOL_LF: return Qlf;
case EOL_CRLF: return Qcrlf;
case EOL_CR: return Qcr;
'post-read-conversion
Function called after a file has been read in, to perform the
- decoding. Called with two arguments, BEG and END, denoting
+ decoding. Called with two arguments, START and END, denoting
a region of the current buffer to be decoded.
'pre-write-conversion
Function called before a file is written out, to perform the
- encoding. Called with two arguments, BEG and END, denoting
+ encoding. Called with two arguments, START and END, denoting
a region of the current buffer to be encoded.
}
else if (EQ (type, Qccl))
{
+ Lisp_Object sym;
+ struct ccl_program test_ccl;
+ Extbyte *suffix;
+
+ /* Check key first. */
if (EQ (key, Qdecode))
- {
- CHECK_VECTOR (value);
- CODING_SYSTEM_CCL_DECODE (codesys) = value;
- }
+ suffix = "-ccl-decode";
else if (EQ (key, Qencode))
+ suffix = "-ccl-encode";
+ else
+ signal_simple_error ("Unrecognized property", key);
+
+ /* If value is vector, register it as a ccl program
+ associated with an newly created symbol for
+ backward compatibility. */
+ if (VECTORP (value))
{
- CHECK_VECTOR (value);
- CODING_SYSTEM_CCL_ENCODE (codesys) = value;
+ sym = Fintern (concat2 (Fsymbol_name (name),
+ build_string (suffix)),
+ Qnil);
+ Fregister_ccl_program (sym, value);
}
else
- signal_simple_error ("Unrecognized property", key);
+ {
+ CHECK_SYMBOL (value);
+ sym = value;
+ }
+ /* check if the given ccl programs are valid. */
+ if (setup_ccl_program (&test_ccl, sym) < 0)
+ signal_simple_error ("Invalid CCL program", value);
+
+ if (EQ (key, Qdecode))
+ CODING_SYSTEM_CCL_DECODE (codesys) = sym;
+ else if (EQ (key, Qencode))
+ CODING_SYSTEM_CCL_ENCODE (codesys) = sym;
+
}
#endif /* MULE */
else
case EOL_LF: new_coding_system = CODING_SYSTEM_EOL_LF (cs); break;
case EOL_CR: new_coding_system = CODING_SYSTEM_EOL_CR (cs); break;
case EOL_CRLF: new_coding_system = CODING_SYSTEM_EOL_CRLF (cs); break;
- default: abort (); return Qnil;
+ default: ABORT (); return Qnil;
}
return NILP (new_coding_system) ? coding_system : new_coding_system;
{
switch (XCODING_SYSTEM_TYPE (Fget_coding_system (coding_system)))
{
- default: abort ();
+ default: ABORT ();
case CODESYS_AUTODETECT: return Qundecided;
#ifdef MULE
case CODESYS_SHIFT_JIS: return Qshift_jis;
break;
#endif /* MULE */
default:
- abort ();
+ ABORT ();
}
}
unparse_charset_conversion_specs
(XCODING_SYSTEM (coding_system)->iso2022.output_conv);
else
- abort ();
+ ABORT ();
}
else if (type == CODESYS_CCL)
{
else if (EQ (prop, Qencode))
return XCODING_SYSTEM_CCL_ENCODE (coding_system);
else
- abort ();
+ ABORT ();
}
#endif /* MULE */
else
- abort ();
+ ABORT ();
return Qnil; /* not reached */
}
int i;
CHECK_SYMBOL (symbol);
- for (i = 0; i <= CODING_CATEGORY_LAST; i++)
+ for (i = 0; i < CODING_CATEGORY_LAST; i++)
if (EQ (coding_category_symbol[i], symbol))
return i;
int i;
Lisp_Object list = Qnil;
- for (i = CODING_CATEGORY_LAST; i >= 0; i--)
+ for (i = CODING_CATEGORY_LAST - 1; i >= 0; i--)
list = Fcons (coding_category_symbol[i], list);
return list;
}
*/
(list))
{
- int category_to_priority[CODING_CATEGORY_LAST + 1];
+ int category_to_priority[CODING_CATEGORY_LAST];
int i, j;
Lisp_Object rest;
/* First generate a list that maps coding categories to priorities. */
- for (i = 0; i <= CODING_CATEGORY_LAST; i++)
+ for (i = 0; i < CODING_CATEGORY_LAST; i++)
category_to_priority[i] = -1;
/* Highest priority comes from the specified list. */
/* Now go through the existing categories by priority to retrieve
the categories not yet specified and preserve their priority
order. */
- for (j = 0; j <= CODING_CATEGORY_LAST; j++)
+ for (j = 0; j < CODING_CATEGORY_LAST; j++)
{
int cat = fcd->coding_category_by_priority[j];
if (category_to_priority[cat] < 0)
/* Now we need to construct the inverse of the mapping we just
constructed. */
- for (i = 0; i <= CODING_CATEGORY_LAST; i++)
+ for (i = 0; i < CODING_CATEGORY_LAST; i++)
fcd->coding_category_by_priority[category_to_priority[i]] = i;
/* Phew! That was confusing. */
int i;
Lisp_Object list = Qnil;
- for (i = CODING_CATEGORY_LAST; i >= 0; i--)
+ for (i = CODING_CATEGORY_LAST - 1; i >= 0; i--)
list = Fcons (coding_category_symbol[fcd->coding_category_by_priority[i]],
list);
return list;
}
static eol_type_t
-detect_eol_type (struct detection_state *st, const unsigned char *src,
- unsigned int n)
+detect_eol_type (struct detection_state *st, const Extbyte *src,
+ Lstream_data_count n)
{
- int c;
-
while (n--)
{
- c = *src++;
+ unsigned char c = *(unsigned char *)src++;
if (c == '\n')
{
if (st->eol.just_saw_cr)
static int
detect_coding_type (struct detection_state *st, const Extbyte *src,
- unsigned int n, int just_do_eol)
+ Lstream_data_count n, int just_do_eol)
{
- int c;
-
if (st->eol_type == EOL_AUTODETECT)
st->eol_type = detect_eol_type (st, src, n);
{
for (; n; n--, src++)
{
- c = *src;
+ unsigned char c = *(unsigned char *) src;
if ((c < 0x20 && !acceptable_control_char_p (c)) || c >= 0x80)
{
st->seen_non_ascii = 1;
#endif
/* Look through the coding categories by priority and find
the first one that is allowed. */
- for (i = 0; i <= CODING_CATEGORY_LAST; i++)
+ for (i = 0; i < CODING_CATEGORY_LAST; i++)
{
cat = fcd->coding_category_by_priority[i];
if ((mask & (1 << cat)) &&
that should be unnecessary with the explicit eol-type argument. */
#define LENGTH(string_constant) (sizeof (string_constant) - 1)
+/* number of leading lines to check for a coding cookie */
+#define LINES_TO_CHECK 2
void
determine_real_coding_system (Lstream *stream, Lisp_Object *codesys_in_out,
Extbyte buf[4096];
Lisp_Object coding_system = Qnil;
Extbyte *p;
- ssize_t nread = Lstream_read (stream, buf, sizeof (buf));
+ Lstream_data_count nread = Lstream_read (stream, buf, sizeof (buf));
Extbyte *scan_end;
+ int lines_checked = 0;
/* Look for initial "-*-"; mode line prefix */
for (p = buf,
scan_end = buf + nread - LENGTH ("-*-coding:?-*-");
p <= scan_end
- && *p != '\n'
- && *p != '\r';
+ && lines_checked < LINES_TO_CHECK;
p++)
if (*p == '-' && *(p+1) == '*' && *(p+2) == '-')
{
for (p = local_vars_beg,
scan_end = buf + nread - LENGTH ("-*-");
p <= scan_end
- && *p != '\n'
- && *p != '\r';
+ && lines_checked < LINES_TO_CHECK;
p++)
if (*p == '-' && *(p+1) == '*' && *(p+2) == '-')
{
}
break;
}
+ /* #### file must use standard EOLs or we miss 2d line */
+ /* #### not to mention this is broken for UTF-16 DOS files */
+ else if (*p == '\n' || *p == '\r')
+ {
+ lines_checked++;
+ /* skip past multibyte (DOS) newline */
+ if (*p == '\r' && *(p+1) == '\n') p++;
+ }
break;
}
+ /* #### file must use standard EOLs or we miss 2d line */
+ /* #### not to mention this is broken for UTF-16 DOS files */
+ else if (*p == '\n' || *p == '\r')
+ {
+ lines_checked++;
+ /* skip past multibyte (DOS) newline */
+ if (*p == '\r' && *(p+1) == '\n') p++;
+ }
if (NILP (coding_system))
do
DEFUN ("detect-coding-region", Fdetect_coding_region, 2, 3, 0, /*
Detect coding system of the text in the region between START and END.
-Returned a list of possible coding systems ordered by priority.
-If only ASCII characters are found, it returns 'undecided or one of
+Return a list of possible coding systems ordered by priority.
+If only ASCII characters are found, return 'undecided or one of
its subsidiary coding systems according to a detected end-of-line
type. Optional arg BUFFER defaults to the current buffer.
*/
decst.mask = ~0;
while (1)
{
- unsigned char random_buffer[4096];
- ssize_t nread = Lstream_read (istr, random_buffer, sizeof (random_buffer));
+ Extbyte random_buffer[4096];
+ Lstream_data_count nread = Lstream_read (istr, random_buffer, sizeof (random_buffer));
if (!nread)
break;
#ifdef MULE
decst.mask = postprocess_iso2022_mask (decst.mask);
#endif
- for (i = CODING_CATEGORY_LAST; i >= 0; i--)
+ for (i = CODING_CATEGORY_LAST - 1; i >= 0; i--)
{
int sys = fcd->coding_category_by_priority[i];
if (decst.mask & (1 << sys))
/* C should be a binary character in the range 0 - 255; convert
to internal format and add to Dynarr DST. */
+#ifdef UTF2000
+#define DECODE_ADD_BINARY_CHAR(c, dst) \
+do { \
+ if (BYTE_ASCII_P (c)) \
+ Dynarr_add (dst, c); \
+ else \
+ { \
+ Dynarr_add (dst, (c >> 6) | 0xc0); \
+ Dynarr_add (dst, (c & 0x3f) | 0x80); \
+ } \
+} while (0)
+
+INLINE void
+DECODE_ADD_UCS_CHAR(Emchar c, unsigned_char_dynarr* dst)
+{
+ if ( c <= 0x7f )
+ {
+ Dynarr_add (dst, c);
+ }
+ else if ( c <= 0x7ff )
+ {
+ Dynarr_add (dst, (c >> 6) | 0xc0);
+ Dynarr_add (dst, (c & 0x3f) | 0x80);
+ }
+ else if ( c <= 0xffff )
+ {
+ Dynarr_add (dst, (c >> 12) | 0xe0);
+ Dynarr_add (dst, ((c >> 6) & 0x3f) | 0x80);
+ Dynarr_add (dst, (c & 0x3f) | 0x80);
+ }
+ else if ( c <= 0x1fffff )
+ {
+ Dynarr_add (dst, (c >> 18) | 0xf0);
+ Dynarr_add (dst, ((c >> 12) & 0x3f) | 0x80);
+ Dynarr_add (dst, ((c >> 6) & 0x3f) | 0x80);
+ Dynarr_add (dst, (c & 0x3f) | 0x80);
+ }
+ else if ( c <= 0x3ffffff )
+ {
+ Dynarr_add (dst, (c >> 24) | 0xf8);
+ Dynarr_add (dst, ((c >> 18) & 0x3f) | 0x80);
+ Dynarr_add (dst, ((c >> 12) & 0x3f) | 0x80);
+ Dynarr_add (dst, ((c >> 6) & 0x3f) | 0x80);
+ Dynarr_add (dst, (c & 0x3f) | 0x80);
+ }
+ else
+ {
+ Dynarr_add (dst, (c >> 30) | 0xfc);
+ Dynarr_add (dst, ((c >> 24) & 0x3f) | 0x80);
+ Dynarr_add (dst, ((c >> 18) & 0x3f) | 0x80);
+ Dynarr_add (dst, ((c >> 12) & 0x3f) | 0x80);
+ Dynarr_add (dst, ((c >> 6) & 0x3f) | 0x80);
+ Dynarr_add (dst, (c & 0x3f) | 0x80);
+ }
+}
+#else
#define DECODE_ADD_BINARY_CHAR(c, dst) \
do { \
if (BYTE_ASCII_P (c)) \
Dynarr_add (dst, c); \
} \
} while (0)
+#endif
#define DECODE_OUTPUT_PARTIAL_CHAR(ch) \
do { \
struct detection_state decst;
};
-static ssize_t decoding_reader (Lstream *stream,
- unsigned char *data, size_t size);
-static ssize_t decoding_writer (Lstream *stream,
- const unsigned char *data, size_t size);
+static Lstream_data_count decoding_reader (Lstream *stream,
+ unsigned char *data, Lstream_data_count size);
+static Lstream_data_count decoding_writer (Lstream *stream,
+ const unsigned char *data, Lstream_data_count size);
static int decoding_rewinder (Lstream *stream);
static int decoding_seekable_p (Lstream *stream);
static int decoding_flusher (Lstream *stream);
/* Read SIZE bytes of data and store it into DATA. We are a decoding stream
so we read data from the other end, decode it, and store it into DATA. */
-static ssize_t
-decoding_reader (Lstream *stream, unsigned char *data, size_t size)
+static Lstream_data_count
+decoding_reader (Lstream *stream, unsigned char *data, Lstream_data_count size)
{
struct decoding_stream *str = DECODING_STREAM_DATA (stream);
unsigned char *orig_data = data;
- ssize_t read_size;
+ Lstream_data_count read_size;
int error_occurred = 0;
/* We need to interface to mule_decode(), which expects to take some
most SIZE bytes, and delete the data from the runoff. */
if (Dynarr_length (str->runoff) > 0)
{
- size_t chunk = min (size, (size_t) Dynarr_length (str->runoff));
+ Lstream_data_count chunk = min (size, (Lstream_data_count) Dynarr_length (str->runoff));
memcpy (data, Dynarr_atp (str->runoff, 0), chunk);
Dynarr_delete_many (str->runoff, 0, chunk);
data += chunk;
/* There might be some more end data produced in the translation.
See the comment above. */
str->flags |= CODING_STATE_END;
- mule_decode (stream, data, str->runoff, read_size);
+ mule_decode (stream, (Extbyte *) data, str->runoff, read_size);
}
if (data - orig_data == 0)
return data - orig_data;
}
-static ssize_t
-decoding_writer (Lstream *stream, const unsigned char *data, size_t size)
+static Lstream_data_count
+decoding_writer (Lstream *stream, const unsigned char *data, Lstream_data_count size)
{
struct decoding_stream *str = DECODING_STREAM_DATA (stream);
- ssize_t retval;
+ Lstream_data_count retval;
/* Decode all our data into the runoff, and then attempt to write
it all out to the other end. Remove whatever chunk we succeeded
in writing. */
- mule_decode (stream, data, str->runoff, size);
+ mule_decode (stream, (Extbyte *) data, str->runoff, size);
retval = Lstream_write (str->other_end, Dynarr_atp (str->runoff, 0),
Dynarr_length (str->runoff));
if (retval > 0)
be used for both reading and writing. */
static void
-mule_decode (Lstream *decoding, const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n)
+mule_decode (Lstream *decoding, const Extbyte *src,
+ unsigned_char_dynarr *dst, Lstream_data_count n)
{
struct decoding_stream *str = DECODING_STREAM_DATA (decoding);
break;
case CODESYS_CCL:
str->ccl.last_block = str->flags & CODING_STATE_END;
- ccl_driver (&str->ccl, src, dst, n, 0, CCL_MODE_DECODING);
+ /* When applying ccl program to stream, MUST NOT set NULL
+ pointer to src. */
+ ccl_driver (&str->ccl, (src ? (unsigned char *)src : (unsigned char*)""),
+ dst, n, 0, CCL_MODE_DECODING);
break;
case CODESYS_ISO2022:
decode_coding_iso2022 (decoding, src, dst, n);
break;
#endif /* MULE */
default:
- abort ();
+ ABORT ();
}
}
char tempbuf[1024]; /* some random amount */
Bufpos newpos, even_newer_pos;
Bufpos oldpos = lisp_buffer_stream_startpos (istr);
- ssize_t size_in_bytes = Lstream_read (istr, tempbuf, sizeof (tempbuf));
+ Lstream_data_count size_in_bytes = Lstream_read (istr, tempbuf, sizeof (tempbuf));
if (!size_in_bytes)
break;
#endif /* MULE */
};
-static ssize_t encoding_reader (Lstream *stream, unsigned char *data, size_t size);
-static ssize_t encoding_writer (Lstream *stream, const unsigned char *data,
- size_t size);
+static Lstream_data_count encoding_reader (Lstream *stream, unsigned char *data, Lstream_data_count size);
+static Lstream_data_count encoding_writer (Lstream *stream, const unsigned char *data,
+ Lstream_data_count size);
static int encoding_rewinder (Lstream *stream);
static int encoding_seekable_p (Lstream *stream);
static int encoding_flusher (Lstream *stream);
/* Read SIZE bytes of data and store it into DATA. We are a encoding stream
so we read data from the other end, encode it, and store it into DATA. */
-static ssize_t
-encoding_reader (Lstream *stream, unsigned char *data, size_t size)
+static Lstream_data_count
+encoding_reader (Lstream *stream, unsigned char *data, Lstream_data_count size)
{
struct encoding_stream *str = ENCODING_STREAM_DATA (stream);
unsigned char *orig_data = data;
- ssize_t read_size;
+ Lstream_data_count read_size;
int error_occurred = 0;
/* We need to interface to mule_encode(), which expects to take some
return data - orig_data;
}
-static ssize_t
-encoding_writer (Lstream *stream, const unsigned char *data, size_t size)
+static Lstream_data_count
+encoding_writer (Lstream *stream, const unsigned char *data, Lstream_data_count size)
{
struct encoding_stream *str = ENCODING_STREAM_DATA (stream);
- ssize_t retval;
+ Lstream_data_count retval;
/* Encode all our data into the runoff, and then attempt to write
it all out to the other end. Remove whatever chunk we succeeded
str->iso2022.register_right = 1;
str->iso2022.current_charset = Qnil;
str->iso2022.current_half = 0;
+#ifdef UTF2000
+ str->iso2022.current_char_boundary = 0;
+#else
str->iso2022.current_char_boundary = 1;
+#endif
break;
}
case CODESYS_CCL:
Store the encoded data into DST. */
static void
-mule_encode (Lstream *encoding, const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n)
+mule_encode (Lstream *encoding, const Bufbyte *src,
+ unsigned_char_dynarr *dst, Lstream_data_count n)
{
struct encoding_stream *str = ENCODING_STREAM_DATA (encoding);
break;
case CODESYS_CCL:
str->ccl.last_block = str->flags & CODING_STATE_END;
- ccl_driver (&str->ccl, src, dst, n, 0, CCL_MODE_ENCODING);
+ /* When applying ccl program to stream, MUST NOT set NULL
+ pointer to src. */
+ ccl_driver (&str->ccl, ((src) ? src : (unsigned char*)""),
+ dst, n, 0, CCL_MODE_ENCODING);
break;
case CODESYS_ISO2022:
encode_coding_iso2022 (encoding, src, dst, n);
break;
#endif /* MULE */
default:
- abort ();
+ ABORT ();
}
}
char tempbuf[1024]; /* some random amount */
Bufpos newpos, even_newer_pos;
Bufpos oldpos = lisp_buffer_stream_startpos (istr);
- ssize_t size_in_bytes = Lstream_read (istr, tempbuf, sizeof (tempbuf));
+ Lstream_data_count size_in_bytes = Lstream_read (istr, tempbuf, sizeof (tempbuf));
if (!size_in_bytes)
break;
((c) >= 0xA1 && (c) <= 0xDF)
static int
-detect_coding_sjis (struct detection_state *st, const unsigned char *src,
- unsigned int n)
+detect_coding_sjis (struct detection_state *st, const Extbyte *src, Lstream_data_count n)
{
- int c;
-
while (n--)
{
- c = *src++;
+ unsigned char c = *(unsigned char *)src++;
if (c == ISO_CODE_ESC || c == ISO_CODE_SI || c == ISO_CODE_SO)
return 0;
if (st->shift_jis.in_second_byte)
/* Convert Shift-JIS data to internal format. */
static void
-decode_coding_sjis (Lstream *decoding, const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n)
+decode_coding_sjis (Lstream *decoding, const Extbyte *src,
+ unsigned_char_dynarr *dst, Lstream_data_count n)
{
- unsigned char c;
struct decoding_stream *str = DECODING_STREAM_DATA (decoding);
unsigned int flags = str->flags;
unsigned int ch = str->ch;
while (n--)
{
- c = *src++;
+ unsigned char c = *(unsigned char *)src++;
if (ch)
{
{
unsigned char e1, e2;
- Dynarr_add (dst, LEADING_BYTE_JAPANESE_JISX0208);
DECODE_SJIS (ch, c, e1, e2);
+#ifdef UTF2000
+ DECODE_ADD_UCS_CHAR(MAKE_CHAR(Vcharset_japanese_jisx0208,
+ e1 & 0x7F,
+ e2 & 0x7F), dst);
+#else
+ Dynarr_add (dst, LEADING_BYTE_JAPANESE_JISX0208);
Dynarr_add (dst, e1);
Dynarr_add (dst, e2);
+#endif
}
else
{
ch = c;
else if (BYTE_SJIS_KATAKANA_P (c))
{
+#ifdef UTF2000
+ DECODE_ADD_UCS_CHAR(MAKE_CHAR(Vcharset_katakana_jisx0201,
+ c & 0x7F, 0), dst);
+#else
Dynarr_add (dst, LEADING_BYTE_KATAKANA_JISX0201);
Dynarr_add (dst, c);
+#endif
}
else
DECODE_ADD_BINARY_CHAR (c, dst);
/* Convert internally-formatted data to Shift-JIS. */
static void
-encode_coding_sjis (Lstream *encoding, const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n)
+encode_coding_sjis (Lstream *encoding, const Bufbyte *src,
+ unsigned_char_dynarr *dst, Lstream_data_count n)
{
- unsigned char c;
struct encoding_stream *str = ENCODING_STREAM_DATA (encoding);
unsigned int flags = str->flags;
unsigned int ch = str->ch;
eol_type_t eol_type = CODING_SYSTEM_EOL_TYPE (str->codesys);
+#ifdef UTF2000
+ unsigned char char_boundary = str->iso2022.current_char_boundary;
+#endif
while (n--)
{
- c = *src++;
+ Bufbyte c = *src++;
+#ifdef UTF2000
+ switch (char_boundary)
+ {
+ case 0:
+ if ( c >= 0xfc )
+ {
+ ch = c & 0x01;
+ char_boundary = 5;
+ }
+ else if ( c >= 0xf8 )
+ {
+ ch = c & 0x03;
+ char_boundary = 4;
+ }
+ else if ( c >= 0xf0 )
+ {
+ ch = c & 0x07;
+ char_boundary = 3;
+ }
+ else if ( c >= 0xe0 )
+ {
+ ch = c & 0x0f;
+ char_boundary = 2;
+ }
+ else if ( c >= 0xc0 )
+ {
+ ch = c & 0x1f;
+ char_boundary = 1;
+ }
+ else
+ {
+ ch = 0;
+ if (c == '\n')
+ {
+ if (eol_type != EOL_LF && eol_type != EOL_AUTODETECT)
+ Dynarr_add (dst, '\r');
+ if (eol_type != EOL_CR)
+ Dynarr_add (dst, c);
+ }
+ else
+ Dynarr_add (dst, c);
+ char_boundary = 0;
+ }
+ break;
+ case 1:
+ ch = ( ch << 6 ) | ( c & 0x3f );
+ {
+ Lisp_Object charset;
+ unsigned int c1, c2, s1, s2;
+
+ BREAKUP_CHAR (ch, charset, c1, c2);
+ if (EQ(charset, Vcharset_katakana_jisx0201))
+ {
+ Dynarr_add (dst, c1 | 0x80);
+ }
+ else if (EQ(charset, Vcharset_japanese_jisx0208))
+ {
+ ENCODE_SJIS (c1 | 0x80, c2 | 0x80, s1, s2);
+ Dynarr_add (dst, s1);
+ Dynarr_add (dst, s2);
+ }
+ }
+ char_boundary = 0;
+ break;
+ default:
+ ch = ( ch << 6 ) | ( c & 0x3f );
+ char_boundary--;
+ }
+#else
if (c == '\n')
{
if (eol_type != EOL_LF && eol_type != EOL_AUTODETECT)
ch = 0;
}
}
+#endif
}
str->flags = flags;
str->ch = ch;
+#ifdef UTF2000
+ str->iso2022.current_char_boundary = char_boundary;
+#endif
}
DEFUN ("decode-shift-jis-char", Fdecode_shift_jis_char, 1, 1, 0, /*
}
DEFUN ("encode-shift-jis-char", Fencode_shift_jis_char, 1, 1, 0, /*
-Encode a JISX0208 character CHAR to SHIFT-JIS coding-system.
+Encode a JISX0208 character CHARACTER to SHIFT-JIS coding-system.
Return the corresponding character code in SHIFT-JIS as a cons of two bytes.
*/
- (ch))
+ (character))
{
Lisp_Object charset;
int c1, c2, s1, s2;
- CHECK_CHAR_COERCE_INT (ch);
- BREAKUP_CHAR (XCHAR (ch), charset, c1, c2);
+ CHECK_CHAR_COERCE_INT (character);
+ BREAKUP_CHAR (XCHAR (character), charset, c1, c2);
if (EQ (charset, Vcharset_japanese_jisx0208))
{
ENCODE_SJIS (c1 | 0x80, c2 | 0x80, s1, s2);
} while (0)
static int
-detect_coding_big5 (struct detection_state *st, const unsigned char *src,
- unsigned int n)
+detect_coding_big5 (struct detection_state *st, const Extbyte *src, Lstream_data_count n)
{
- int c;
-
while (n--)
{
- c = *src++;
+ unsigned char c = *(unsigned char *)src++;
if (c == ISO_CODE_ESC || c == ISO_CODE_SI || c == ISO_CODE_SO ||
(c >= 0x80 && c <= 0xA0))
return 0;
/* Convert Big5 data to internal format. */
static void
-decode_coding_big5 (Lstream *decoding, const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n)
+decode_coding_big5 (Lstream *decoding, const Extbyte *src,
+ unsigned_char_dynarr *dst, Lstream_data_count n)
{
- unsigned char c;
struct decoding_stream *str = DECODING_STREAM_DATA (decoding);
unsigned int flags = str->flags;
unsigned int ch = str->ch;
while (n--)
{
- c = *src++;
+ unsigned char c = *(unsigned char *)src++;
if (ch)
{
/* Previous character was first byte of Big5 char. */
/* Convert internally-formatted data to Big5. */
static void
-encode_coding_big5 (Lstream *encoding, const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n)
+encode_coding_big5 (Lstream *encoding, const Bufbyte *src,
+ unsigned_char_dynarr *dst, Lstream_data_count n)
{
+#ifndef UTF2000
unsigned char c;
struct encoding_stream *str = ENCODING_STREAM_DATA (encoding);
unsigned int flags = str->flags;
str->flags = flags;
str->ch = ch;
+#endif
}
if (BYTE_BIG5_TWO_BYTE_1_P (b1) &&
BYTE_BIG5_TWO_BYTE_2_P (b2))
{
- int leading_byte;
+ Charset_ID leading_byte;
Lisp_Object charset;
DECODE_BIG5 (b1, b2, leading_byte, c1, c2);
charset = CHARSET_BY_LEADING_BYTE (leading_byte);
}
DEFUN ("encode-big5-char", Fencode_big5_char, 1, 1, 0, /*
-Encode the Big5 character CH to BIG5 coding-system.
+Encode the Big5 character CHARACTER in the BIG5 coding-system.
Return the corresponding character code in Big5.
*/
- (ch))
+ (character))
{
Lisp_Object charset;
int c1, c2, b1, b2;
- CHECK_CHAR_COERCE_INT (ch);
- BREAKUP_CHAR (XCHAR (ch), charset, c1, c2);
+ CHECK_CHAR_COERCE_INT (character);
+ BREAKUP_CHAR (XCHAR (character), charset, c1, c2);
if (EQ (charset, Vcharset_chinese_big5_1) ||
EQ (charset, Vcharset_chinese_big5_2))
{
/* */
/************************************************************************/
-
+#ifndef UTF2000
DEFUN ("set-ucs-char", Fset_ucs_char, 2, 2, 0, /*
Map UCS-4 code CODE to Mule character CHARACTER.
c = code % (94 * 94);
return make_char
(MAKE_CHAR (CHARSET_BY_ATTRIBUTES
- (CHARSET_TYPE_94X94, code / (94 * 94) + '@',
+ (94, 2, code / (94 * 94) + '@',
CHARSET_LEFT_TO_RIGHT),
c / 94 + 33, c % 94 + 33));
}
{
return Fget_char_table (character, mule_to_ucs_table);
}
+#endif
+#ifdef UTF2000
+#define decode_ucs4 DECODE_ADD_UCS_CHAR
+#else
/* Decode a UCS-4 character into a buffer. If the lookup fails, use
<GETA MARK> (U+3013) of JIS X 0208, which means correct character
is not found, instead.
Dynarr_add (dst, 46 + 128);
}
}
+#endif
+#ifndef UTF2000
static unsigned long
mule_char_to_ucs4 (Lisp_Object charset,
unsigned char h, unsigned char l)
Dynarr_add (dst, (code >> 8) & 255);
Dynarr_add (dst, code & 255);
}
+#endif
static int
-detect_coding_ucs4 (struct detection_state *st, const unsigned char *src,
- unsigned int n)
+detect_coding_ucs4 (struct detection_state *st, const Extbyte *src, Lstream_data_count n)
{
while (n--)
{
- int c = *src++;
+ unsigned char c = *(unsigned char *)src++;
switch (st->ucs4.in_byte)
{
case 0:
}
static void
-decode_coding_ucs4 (Lstream *decoding, const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n)
+decode_coding_ucs4 (Lstream *decoding, const Extbyte *src,
+ unsigned_char_dynarr *dst, Lstream_data_count n)
{
struct decoding_stream *str = DECODING_STREAM_DATA (decoding);
unsigned int flags = str->flags;
while (n--)
{
- unsigned char c = *src++;
+ unsigned char c = *(unsigned char *)src++;
switch (counter)
{
case 0:
}
static void
-encode_coding_ucs4 (Lstream *encoding, const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n)
+encode_coding_ucs4 (Lstream *encoding, const Bufbyte *src,
+ unsigned_char_dynarr *dst, Lstream_data_count n)
{
+#ifndef UTF2000
struct encoding_stream *str = ENCODING_STREAM_DATA (encoding);
unsigned int flags = str->flags;
unsigned int ch = str->ch;
{
/* #### Bother! We don't know how to
handle this yet. */
- Dynarr_add (dst, 0);
- Dynarr_add (dst, 0);
- Dynarr_add (dst, 0);
+ Dynarr_add (dst, '\0');
+ Dynarr_add (dst, '\0');
+ Dynarr_add (dst, '\0');
Dynarr_add (dst, '~');
}
else
}
break;
default:
- abort ();
+ ABORT ();
}
}
}
str->iso2022.current_charset = charset;
/* Verbum caro factum est! */
+#endif
}
\f
/************************************************************************/
static int
-detect_coding_utf8 (struct detection_state *st, const unsigned char *src,
- unsigned int n)
+detect_coding_utf8 (struct detection_state *st, const Extbyte *src, Lstream_data_count n)
{
while (n--)
{
- unsigned char c = *src++;
+ unsigned char c = *(unsigned char *)src++;
switch (st->utf8.in_byte)
{
case 0:
}
static void
-decode_coding_utf8 (Lstream *decoding, const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n)
+decode_coding_utf8 (Lstream *decoding, const Extbyte *src,
+ unsigned_char_dynarr *dst, Lstream_data_count n)
{
struct decoding_stream *str = DECODING_STREAM_DATA (decoding);
unsigned int flags = str->flags;
while (n--)
{
- unsigned char c = *src++;
+ unsigned char c = *(unsigned char *)src++;
switch (counter)
{
case 0:
str->counter = counter;
}
+#ifndef UTF2000
static void
encode_utf8 (Lisp_Object charset,
unsigned char h, unsigned char l, unsigned_char_dynarr *dst)
Dynarr_add (dst, (code & 0x3f) | 0x80);
}
}
+#endif
static void
-encode_coding_utf8 (Lstream *encoding, const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n)
+encode_coding_utf8 (Lstream *encoding, const Bufbyte *src,
+ unsigned_char_dynarr *dst, Lstream_data_count n)
{
struct encoding_stream *str = ENCODING_STREAM_DATA (encoding);
unsigned int flags = str->flags;
unsigned int ch = str->ch;
eol_type_t eol_type = CODING_SYSTEM_EOL_TYPE (str->codesys);
unsigned char char_boundary = str->iso2022.current_char_boundary;
+#ifdef UTF2000
+
+ while (n--)
+ {
+ unsigned char c = *src++;
+ switch (char_boundary)
+ {
+ case 0:
+ if ( c >= 0xfc )
+ {
+ Dynarr_add (dst, c);
+ char_boundary = 5;
+ }
+ else if ( c >= 0xf8 )
+ {
+ Dynarr_add (dst, c);
+ char_boundary = 4;
+ }
+ else if ( c >= 0xf0 )
+ {
+ Dynarr_add (dst, c);
+ char_boundary = 3;
+ }
+ else if ( c >= 0xe0 )
+ {
+ Dynarr_add (dst, c);
+ char_boundary = 2;
+ }
+ else if ( c >= 0xc0 )
+ {
+ Dynarr_add (dst, c);
+ char_boundary = 1;
+ }
+ else
+ {
+ if (c == '\n')
+ {
+ if (eol_type != EOL_LF && eol_type != EOL_AUTODETECT)
+ Dynarr_add (dst, '\r');
+ if (eol_type != EOL_CR)
+ Dynarr_add (dst, c);
+ }
+ else
+ Dynarr_add (dst, c);
+ char_boundary = 0;
+ }
+ break;
+ case 1:
+ Dynarr_add (dst, c);
+ char_boundary = 0;
+ break;
+ default:
+ Dynarr_add (dst, c);
+ char_boundary--;
+ }
+ }
+#else /* not UTF2000 */
Lisp_Object charset = str->iso2022.current_charset;
#ifdef ENABLE_COMPOSITE_CHARS
}
break;
default:
- abort ();
+ ABORT ();
}
}
}
}
#endif
+#endif /* not UTF2000 */
str->flags = flags;
str->ch = ch;
str->iso2022.current_char_boundary = char_boundary;
+#ifndef UTF2000
str->iso2022.current_charset = charset;
+#endif
/* Verbum caro factum est! */
}
If CHECK_INVALID_CHARSETS is non-zero, check for designation
or invocation of an invalid character set and treat that as
- an unrecognized escape sequence. */
+ an unrecognized escape sequence.
+
+ ********************************************************************
+
+ #### Strategies for error annotation and coding orthogonalization
+
+ We really want to separate out a number of things. Conceptually,
+ there is a nested syntax.
+
+ At the top level is the ISO 2022 extension syntax, including charset
+ designation and invocation, and certain auxiliary controls such as the
+ ISO 6429 direction specification. These are octet-oriented, with the
+ single exception (AFAIK) of the "exit Unicode" sequence which uses the
+ UTF's natural width (1 byte for UTF-7 and UTF-8, 2 bytes for UCS-2 and
+ UTF-16, and 4 bytes for UCS-4 and UTF-32). This will be treated as a
+ (deprecated) special case in Unicode processing.
+
+ The middle layer is ISO 2022 character interpretation. This will depend
+ on the current state of the ISO 2022 registers, and assembles octets
+ into the character's internal representation.
+
+ The lowest level is translating system control conventions. At present
+ this is restricted to newline translation, but one could imagine doing
+ tab conversion or line wrapping here. "Escape from Unicode" processing
+ would be done at this level.
+
+ At each level the parser will verify the syntax. In the case of a
+ syntax error or warning (such as a redundant escape sequence that affects
+ no characters), the parser will take some action, typically inserting the
+ erroneous octets directly into the output and creating an annotation
+ which can be used by higher level I/O to mark the affected region.
+
+ This should make it possible to do something sensible about separating
+ newline convention processing from character construction, and about
+ preventing ISO 2022 escape sequences from being recognized
+ inappropriately.
+
+ The basic strategy will be to have octet classification tables, and
+ switch processing according to the table entry.
+
+ It's possible that, by doing the processing with tables of functions or
+ the like, the parser can be used for both detection and translation. */
static int
parse_iso2022_esc (Lisp_Object codesys, struct iso2022_decoder *iso,
}
if (0x40 <= c && c <= 0x42)
{
- cs = CHARSET_BY_ATTRIBUTES (CHARSET_TYPE_94X94, c,
+ cs = CHARSET_BY_ATTRIBUTES (94, 2, c,
*flags & CODING_STATE_R2L ?
CHARSET_RIGHT_TO_LEFT :
CHARSET_LEFT_TO_RIGHT);
default:
{
- int type =-1;
+ /* int type =-1; */
+ int chars = 0;
+ int single = 0;
if (c < '0' || c > '~')
return 0; /* bad final byte */
if (iso->esc >= ISO_ESC_2_8 &&
iso->esc <= ISO_ESC_2_15)
{
- type = ((iso->esc >= ISO_ESC_2_12) ?
- CHARSET_TYPE_96 : CHARSET_TYPE_94);
+ chars = (iso->esc >= ISO_ESC_2_12) ? 96 : 94;
+ single = 1; /* single-byte */
reg = (iso->esc - ISO_ESC_2_8) & 3;
}
else if (iso->esc >= ISO_ESC_2_4_8 &&
iso->esc <= ISO_ESC_2_4_15)
{
- type = ((iso->esc >= ISO_ESC_2_4_12) ?
- CHARSET_TYPE_96X96 : CHARSET_TYPE_94X94);
+ chars = (iso->esc >= ISO_ESC_2_4_12) ? 96 : 94;
+ single = -1; /* multi-byte */
reg = (iso->esc - ISO_ESC_2_4_8) & 3;
}
else
{
/* Can this ever be reached? -slb */
- abort();
+ ABORT();
return 0;
}
- cs = CHARSET_BY_ATTRIBUTES (type, c,
+ cs = CHARSET_BY_ATTRIBUTES (chars, single, c,
*flags & CODING_STATE_R2L ?
CHARSET_RIGHT_TO_LEFT :
CHARSET_LEFT_TO_RIGHT);
}
static int
-detect_coding_iso2022 (struct detection_state *st, const unsigned char *src,
- unsigned int n)
+detect_coding_iso2022 (struct detection_state *st, const Extbyte *src, Lstream_data_count n)
{
int mask;
while (n--)
{
- int c = *src++;
+ unsigned char c = *(unsigned char *)src++;
if (c >= 0xA0)
{
mask &= ~CODING_CATEGORY_ISO_7_MASK;
/* Convert ISO2022-format data to internal format. */
static void
-decode_coding_iso2022 (Lstream *decoding, const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n)
+decode_coding_iso2022 (Lstream *decoding, const Extbyte *src,
+ unsigned_char_dynarr *dst, Lstream_data_count n)
{
struct decoding_stream *str = DECODING_STREAM_DATA (decoding);
unsigned int flags = str->flags;
while (n--)
{
- unsigned char c = *src++;
+ unsigned char c = *(unsigned char *)src++;
if (flags & CODING_STATE_ESCAPE)
{ /* Within ESC sequence */
int retval = parse_iso2022_esc (coding_system, &str->iso2022,
else
{ /* Graphic characters */
Lisp_Object charset;
- int lb;
+#ifndef UTF2000
+ Charset_ID lb;
+#endif
int reg;
DECODE_HANDLE_EOL_TYPE (eol_type, c, flags, dst);
charset = new_charset;
}
+#ifdef UTF2000
+ if (XCHARSET_DIMENSION (charset) == 1)
+ {
+ DECODE_OUTPUT_PARTIAL_CHAR (ch);
+ DECODE_ADD_UCS_CHAR
+ (MAKE_CHAR (charset, c & 0x7F, 0), dst);
+ }
+ else if (ch)
+ {
+ DECODE_ADD_UCS_CHAR
+ (MAKE_CHAR (charset, ch & 0x7F, c & 0x7F), dst);
+ ch = 0;
+ }
+ else
+ ch = c;
+#else
lb = XCHARSET_LEADING_BYTE (charset);
switch (XCHARSET_REP_BYTES (charset))
{
else
ch = c;
}
+#endif
}
if (!ch)
/* Convert internally-formatted data to ISO2022 format. */
static void
-encode_coding_iso2022 (Lstream *encoding, const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n)
+encode_coding_iso2022 (Lstream *encoding, const Bufbyte *src,
+ unsigned_char_dynarr *dst, Lstream_data_count n)
{
unsigned char charmask, c;
unsigned char char_boundary;
struct encoding_stream *str = ENCODING_STREAM_DATA (encoding);
unsigned int flags = str->flags;
- unsigned int ch = str->ch;
+ Emchar ch = str->ch;
Lisp_Coding_System *codesys = str->codesys;
eol_type_t eol_type = CODING_SYSTEM_EOL_TYPE (str->codesys);
int i;
Lisp_Object charset;
int half;
+#ifdef UTF2000
+ unsigned int byte1, byte2;
+#endif
#ifdef ENABLE_COMPOSITE_CHARS
/* flags for handling composite chars. We do a little switcharoo
#ifdef ENABLE_COMPOSITE_CHARS
back_to_square_n:
#endif
+#ifdef UTF2000
+ while (n--)
+ {
+ c = *src++;
+
+ switch (char_boundary)
+ {
+ case 0:
+ if ( c >= 0xfc )
+ {
+ ch = c & 0x01;
+ char_boundary = 5;
+ }
+ else if ( c >= 0xf8 )
+ {
+ ch = c & 0x03;
+ char_boundary = 4;
+ }
+ else if ( c >= 0xf0 )
+ {
+ ch = c & 0x07;
+ char_boundary = 3;
+ }
+ else if ( c >= 0xe0 )
+ {
+ ch = c & 0x0f;
+ char_boundary = 2;
+ }
+ else if ( c >= 0xc0 )
+ {
+ ch = c & 0x1f;
+ char_boundary = 1;
+ }
+ else
+ {
+ ch = 0;
+
+ restore_left_to_right_direction (codesys, dst, &flags, 0);
+
+ /* Make sure G0 contains ASCII */
+ if ((c > ' ' && c < ISO_CODE_DEL) ||
+ !CODING_SYSTEM_ISO2022_NO_ASCII_CNTL (codesys))
+ {
+ ensure_normal_shift (str, dst);
+ iso2022_designate (Vcharset_ascii, 0, str, dst);
+ }
+
+ /* If necessary, restore everything to the default state
+ at end-of-line */
+ if (c == '\n' &&
+ !(CODING_SYSTEM_ISO2022_NO_ASCII_EOL (codesys)))
+ {
+ restore_left_to_right_direction (codesys, dst, &flags, 0);
+
+ ensure_normal_shift (str, dst);
+
+ for (i = 0; i < 4; i++)
+ {
+ Lisp_Object initial_charset =
+ CODING_SYSTEM_ISO2022_INITIAL_CHARSET (codesys, i);
+ iso2022_designate (initial_charset, i, str, dst);
+ }
+ }
+ if (c == '\n')
+ {
+ if (eol_type != EOL_LF && eol_type != EOL_AUTODETECT)
+ Dynarr_add (dst, '\r');
+ if (eol_type != EOL_CR)
+ Dynarr_add (dst, c);
+ }
+ else
+ {
+ if (CODING_SYSTEM_ISO2022_ESCAPE_QUOTED (codesys)
+ && fit_to_be_escape_quoted (c))
+ Dynarr_add (dst, ISO_CODE_ESC);
+ Dynarr_add (dst, c);
+ }
+ char_boundary = 0;
+ }
+ break;
+ case 1:
+ ch = ( ch << 6 ) | ( c & 0x3f );
+
+ char_boundary = 0;
+ if ( (0x80 <= ch) && (ch <= 0x9f) )
+ {
+ charmask = (half == 0 ? 0x00 : 0x80);
+
+ if (CODING_SYSTEM_ISO2022_ESCAPE_QUOTED (codesys)
+ && fit_to_be_escape_quoted (ch))
+ Dynarr_add (dst, ISO_CODE_ESC);
+ /* you asked for it ... */
+ Dynarr_add (dst, ch);
+ }
+ else
+ {
+ int reg;
+
+ BREAKUP_CHAR (ch, charset, byte1, byte2);
+ ensure_correct_direction (XCHARSET_DIRECTION (charset),
+ codesys, dst, &flags, 0);
+
+ /* Now determine which register to use. */
+ reg = -1;
+ for (i = 0; i < 4; i++)
+ {
+ if (EQ (charset, str->iso2022.charset[i]) ||
+ EQ (charset,
+ CODING_SYSTEM_ISO2022_INITIAL_CHARSET (codesys, i)))
+ {
+ reg = i;
+ break;
+ }
+ }
+
+ if (reg == -1)
+ {
+ if (XCHARSET_GRAPHIC (charset) != 0)
+ {
+ if (!NILP (str->iso2022.charset[1]) &&
+ (!CODING_SYSTEM_ISO2022_SEVEN (codesys) ||
+ CODING_SYSTEM_ISO2022_LOCK_SHIFT (codesys)))
+ reg = 1;
+ else if (!NILP (str->iso2022.charset[2]))
+ reg = 2;
+ else if (!NILP (str->iso2022.charset[3]))
+ reg = 3;
+ else
+ reg = 0;
+ }
+ else
+ reg = 0;
+ }
+
+ iso2022_designate (charset, reg, str, dst);
+
+ /* Now invoke that register. */
+ switch (reg)
+ {
+ case 0:
+ ensure_normal_shift (str, dst);
+ half = 0;
+ break;
+
+ case 1:
+ if (CODING_SYSTEM_ISO2022_SEVEN (codesys))
+ {
+ ensure_shift_out (str, dst);
+ half = 0;
+ }
+ else
+ half = 1;
+ break;
+
+ case 2:
+ if (CODING_SYSTEM_ISO2022_SEVEN (str->codesys))
+ {
+ Dynarr_add (dst, ISO_CODE_ESC);
+ Dynarr_add (dst, 'N');
+ half = 0;
+ }
+ else
+ {
+ Dynarr_add (dst, ISO_CODE_SS2);
+ half = 1;
+ }
+ break;
+
+ case 3:
+ if (CODING_SYSTEM_ISO2022_SEVEN (str->codesys))
+ {
+ Dynarr_add (dst, ISO_CODE_ESC);
+ Dynarr_add (dst, 'O');
+ half = 0;
+ }
+ else
+ {
+ Dynarr_add (dst, ISO_CODE_SS3);
+ half = 1;
+ }
+ break;
+
+ default:
+ abort ();
+ }
+
+ charmask = (half == 0 ? 0x00 : 0x80);
+
+ switch (XCHARSET_DIMENSION (charset))
+ {
+ case 1:
+ Dynarr_add (dst, byte1 | charmask);
+ break;
+ case 2:
+ Dynarr_add (dst, byte1 | charmask);
+ Dynarr_add (dst, byte2 | charmask);
+ break;
+ default:
+ abort ();
+ }
+ }
+ ch =0;
+ break;
+ default:
+ ch = ( ch << 6 ) | ( c & 0x3f );
+ char_boundary--;
+ }
+ }
+#else /* not UTF2000 */
+
while (n--)
{
c = *src++;
break;
default:
- abort ();
+ ABORT ();
}
}
char_boundary = 0;
}
break;
default:
- abort ();
+ ABORT ();
}
}
}
}
+#endif /* not UTF2000 */
#ifdef ENABLE_COMPOSITE_CHARS
if (in_composite)
}
#endif /* ENABLE_COMPOSITE_CHARS */
+#ifdef UTF2000
+ if ( (char_boundary == 0) && flags & CODING_STATE_END)
+#else
if (char_boundary && flags & CODING_STATE_END)
+#endif
{
restore_left_to_right_direction (codesys, dst, &flags, 0);
ensure_normal_shift (str, dst);
contain all 256 possible byte values and that are not to be
interpreted as being in any particular decoding. */
static void
-decode_coding_no_conversion (Lstream *decoding, const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n)
+decode_coding_no_conversion (Lstream *decoding, const Extbyte *src,
+ unsigned_char_dynarr *dst, Lstream_data_count n)
{
- unsigned char c;
struct decoding_stream *str = DECODING_STREAM_DATA (decoding);
unsigned int flags = str->flags;
unsigned int ch = str->ch;
while (n--)
{
- c = *src++;
+ unsigned char c = *(unsigned char *)src++;
DECODE_HANDLE_EOL_TYPE (eol_type, c, flags, dst);
DECODE_ADD_BINARY_CHAR (c, dst);
}
static void
-encode_coding_no_conversion (Lstream *encoding, const unsigned char *src,
- unsigned_char_dynarr *dst, unsigned int n)
+encode_coding_no_conversion (Lstream *encoding, const Bufbyte *src,
+ unsigned_char_dynarr *dst, Lstream_data_count n)
{
unsigned char c;
struct encoding_stream *str = ENCODING_STREAM_DATA (encoding);
unsigned int flags = str->flags;
unsigned int ch = str->ch;
eol_type_t eol_type = CODING_SYSTEM_EOL_TYPE (str->codesys);
+#ifdef UTF2000
+ unsigned char char_boundary = str->iso2022.current_char_boundary;
+#endif
while (n--)
{
- c = *src++;
+ c = *src++;
+#ifdef UTF2000
+ switch (char_boundary)
+ {
+ case 0:
+ if ( c >= 0xfc )
+ {
+ ch = c & 0x01;
+ char_boundary = 5;
+ }
+ else if ( c >= 0xf8 )
+ {
+ ch = c & 0x03;
+ char_boundary = 4;
+ }
+ else if ( c >= 0xf0 )
+ {
+ ch = c & 0x07;
+ char_boundary = 3;
+ }
+ else if ( c >= 0xe0 )
+ {
+ ch = c & 0x0f;
+ char_boundary = 2;
+ }
+ else if ( c >= 0xc0 )
+ {
+ ch = c & 0x1f;
+ char_boundary = 1;
+ }
+ else
+ {
+ ch = 0;
+
+ if (c == '\n')
+ {
+ if (eol_type != EOL_LF && eol_type != EOL_AUTODETECT)
+ Dynarr_add (dst, '\r');
+ if (eol_type != EOL_CR)
+ Dynarr_add (dst, c);
+ }
+ else
+ Dynarr_add (dst, c);
+ char_boundary = 0;
+ }
+ break;
+ case 1:
+ ch = ( ch << 6 ) | ( c & 0x3f );
+ Dynarr_add (dst, ch & 0xff);
+ char_boundary = 0;
+ break;
+ default:
+ ch = ( ch << 6 ) | ( c & 0x3f );
+ char_boundary--;
+ }
+#else /* not UTF2000 */
if (c == '\n')
{
if (eol_type != EOL_LF && eol_type != EOL_AUTODETECT)
untranslatable character, so ignore it */
ch = 0;
}
+#endif /* not UTF2000 */
}
str->flags = flags;
str->ch = ch;
+#ifdef UTF2000
+ str->iso2022.current_char_boundary = char_boundary;
+#endif
}
\f
DEFSUBR (Fencode_shift_jis_char);
DEFSUBR (Fdecode_big5_char);
DEFSUBR (Fencode_big5_char);
+#ifndef UTF2000
DEFSUBR (Fset_ucs_char);
DEFSUBR (Fucs_char);
DEFSUBR (Fset_char_ucs);
DEFSUBR (Fchar_ucs);
+#endif /* not UTF2000 */
#endif /* MULE */
defsymbol (&Qcoding_systemp, "coding-system-p");
defsymbol (&Qno_conversion, "no-conversion");
int i;
fcd = xnew (struct file_coding_dump);
- dumpstruct (&fcd, &fcd_description);
+ dump_add_root_struct_ptr (&fcd, &fcd_description);
/* Initialize to something reasonable ... */
- for (i = 0; i <= CODING_CATEGORY_LAST; i++)
+ for (i = 0; i < CODING_CATEGORY_LAST; i++)
{
fcd->coding_category_system[i] = Qnil;
fcd->coding_category_by_priority[i] = i;
make_lisp_hash_table (50, HASH_TABLE_NON_WEAK, HASH_TABLE_EQ);
the_codesys_prop_dynarr = Dynarr_new (codesys_prop);
- dumpstruct (&the_codesys_prop_dynarr, &codesys_prop_dynarr_description);
+ dump_add_root_struct_ptr (&the_codesys_prop_dynarr, &codesys_prop_dynarr_description);
#define DEFINE_CODESYS_PROP(Prop_Type, Sym) do \
{ \
list4 (Qeol_type, Qlf,
Qmnemonic, build_string ("Binary")));
+#ifdef UTF2000
+ Fmake_coding_system
+ (Qutf8, Qutf8,
+ build_string ("Coding-system of ISO/IEC 10646 UTF-8."),
+ list2 (Qmnemonic, build_string ("UTF8")));
+#endif
+
Fdefine_coding_system_alias (Qno_conversion, Qraw_text);
Fdefine_coding_system_alias (Qfile_name, Qbinary);
fcd->coding_category_system[CODING_CATEGORY_NO_CONVERSION] =
Fget_coding_system (Qraw_text);
-#ifdef MULE
+#ifdef UTF2000
+ fcd->coding_category_system[CODING_CATEGORY_UTF8]
+ = Fget_coding_system (Qutf8);
+#endif
+
+#if defined(MULE) && !defined(UTF2000)
{
size_t i;
}
staticpro (&mule_to_ucs_table);
mule_to_ucs_table = Fmake_char_table(Qgeneric);
-#endif /* MULE */
+#endif /* defined(MULE) && !defined(UTF2000) */
}
projects / chise / xemacs-chise.git.1 / blobdiff