static void
text_encode_generic (Lstream *encoding, const Bufbyte *src,
- unsigned_char_dynarr *dst, size_t n);
+ unsigned_char_dynarr *dst, Lstream_data_count n);
static int detect_coding_sjis (struct detection_state *st,
- const Extbyte *src, size_t n);
+ const Extbyte *src, Lstream_data_count n);
static void decode_coding_sjis (Lstream *decoding, const Extbyte *src,
- unsigned_char_dynarr *dst, size_t n);
+ unsigned_char_dynarr *dst, Lstream_data_count n);
void char_encode_shift_jis (struct encoding_stream *str, Emchar c,
unsigned_char_dynarr *dst, unsigned int *flags);
void char_finish_shift_jis (struct encoding_stream *str,
unsigned_char_dynarr *dst, unsigned int *flags);
static int detect_coding_big5 (struct detection_state *st,
- const Extbyte *src, size_t n);
+ const Extbyte *src, Lstream_data_count n);
static void decode_coding_big5 (Lstream *decoding, const Extbyte *src,
- unsigned_char_dynarr *dst, size_t n);
+ unsigned_char_dynarr *dst, Lstream_data_count n);
void char_encode_big5 (struct encoding_stream *str, Emchar c,
unsigned_char_dynarr *dst, unsigned int *flags);
void char_finish_big5 (struct encoding_stream *str,
unsigned_char_dynarr *dst, unsigned int *flags);
static int detect_coding_ucs4 (struct detection_state *st,
- const Extbyte *src, size_t n);
+ const Extbyte *src, Lstream_data_count n);
static void decode_coding_ucs4 (Lstream *decoding, const Extbyte *src,
- unsigned_char_dynarr *dst, size_t n);
+ unsigned_char_dynarr *dst, Lstream_data_count n);
void char_encode_ucs4 (struct encoding_stream *str, Emchar c,
unsigned_char_dynarr *dst, unsigned int *flags);
void char_finish_ucs4 (struct encoding_stream *str,
unsigned_char_dynarr *dst, unsigned int *flags);
static int detect_coding_utf16 (struct detection_state *st,
- const Extbyte *src, size_t n);
+ const Extbyte *src, Lstream_data_count n);
static void decode_coding_utf16 (Lstream *decoding, const Extbyte *src,
- unsigned_char_dynarr *dst, size_t n);
+ unsigned_char_dynarr *dst, Lstream_data_count n);
void char_encode_utf16 (struct encoding_stream *str, Emchar c,
unsigned_char_dynarr *dst, unsigned int *flags);
void char_finish_utf16 (struct encoding_stream *str,
unsigned_char_dynarr *dst, unsigned int *flags);
static int detect_coding_utf8 (struct detection_state *st,
- const Extbyte *src, size_t n);
+ const Extbyte *src, Lstream_data_count n);
static void decode_coding_utf8 (Lstream *decoding, const Extbyte *src,
- unsigned_char_dynarr *dst, size_t n);
+ unsigned_char_dynarr *dst, Lstream_data_count n);
void char_encode_utf8 (struct encoding_stream *str, Emchar c,
unsigned_char_dynarr *dst, unsigned int *flags);
void char_finish_utf8 (struct encoding_stream *str,
static void reset_iso2022 (Lisp_Object coding_system,
struct iso2022_decoder *iso);
static int detect_coding_iso2022 (struct detection_state *st,
- const Extbyte *src, size_t n);
+ const Extbyte *src, Lstream_data_count n);
static void decode_coding_iso2022 (Lstream *decoding, const Extbyte *src,
- unsigned_char_dynarr *dst, size_t n);
+ unsigned_char_dynarr *dst, Lstream_data_count n);
void char_encode_iso2022 (struct encoding_stream *str, Emchar c,
unsigned_char_dynarr *dst, unsigned int *flags);
void char_finish_iso2022 (struct encoding_stream *str,
unsigned_char_dynarr *dst, unsigned int *flags);
#endif /* MULE */
static void decode_coding_no_conversion (Lstream *decoding, const Extbyte *src,
- unsigned_char_dynarr *dst, size_t n);
+ unsigned_char_dynarr *dst, Lstream_data_count n);
static void encode_coding_no_conversion (Lstream *encoding, const Bufbyte *src,
- unsigned_char_dynarr *dst, size_t n);
+ unsigned_char_dynarr *dst, Lstream_data_count n);
static void mule_decode (Lstream *decoding, const Extbyte *src,
- unsigned_char_dynarr *dst, size_t n);
+ unsigned_char_dynarr *dst, Lstream_data_count n);
static void mule_encode (Lstream *encoding, const Bufbyte *src,
- unsigned_char_dynarr *dst, size_t n);
+ unsigned_char_dynarr *dst, Lstream_data_count n);
typedef struct codesys_prop codesys_prop;
struct codesys_prop
static eol_type_t
detect_eol_type (struct detection_state *st, const Extbyte *src,
- size_t n)
+ Lstream_data_count n)
{
while (n--)
{
static int
detect_coding_type (struct detection_state *st, const Extbyte *src,
- size_t n, int just_do_eol)
+ Lstream_data_count n, int just_do_eol)
{
if (st->eol_type == EOL_AUTODETECT)
st->eol_type = detect_eol_type (st, src, n);
that should be unnecessary with the explicit eol-type argument. */
#define LENGTH(string_constant) (sizeof (string_constant) - 1)
+/* number of leading lines to check for a coding cookie */
+#define LINES_TO_CHECK 2
void
determine_real_coding_system (Lstream *stream, Lisp_Object *codesys_in_out,
Extbyte buf[4096];
Lisp_Object coding_system = Qnil;
Extbyte *p;
- ssize_t nread = Lstream_read (stream, buf, sizeof (buf));
+ Lstream_data_count nread = Lstream_read (stream, buf, sizeof (buf));
Extbyte *scan_end;
+ int lines_checked = 0;
/* Look for initial "-*-"; mode line prefix */
for (p = buf,
scan_end = buf + nread - LENGTH ("-*-coding:?-*-");
p <= scan_end
- && *p != '\n'
- && *p != '\r';
+ && lines_checked < LINES_TO_CHECK;
p++)
if (*p == '-' && *(p+1) == '*' && *(p+2) == '-')
{
for (p = local_vars_beg,
scan_end = buf + nread - LENGTH ("-*-");
p <= scan_end
- && *p != '\n'
- && *p != '\r';
+ && lines_checked < LINES_TO_CHECK;
p++)
if (*p == '-' && *(p+1) == '*' && *(p+2) == '-')
{
}
break;
}
+ /* #### file must use standard EOLs or we miss 2d line */
+ /* #### not to mention this is broken for UTF-16 DOS files */
+ else if (*p == '\n' || *p == '\r')
+ {
+ lines_checked++;
+ /* skip past multibyte (DOS) newline */
+ if (*p == '\r' && *(p+1) == '\n') p++;
+ }
break;
}
+ /* #### file must use standard EOLs or we miss 2d line */
+ /* #### not to mention this is broken for UTF-16 DOS files */
+ else if (*p == '\n' || *p == '\r')
+ {
+ lines_checked++;
+ /* skip past multibyte (DOS) newline */
+ if (*p == '\r' && *(p+1) == '\n') p++;
+ }
if (NILP (coding_system))
do
while (1)
{
Extbyte random_buffer[4096];
- ssize_t nread = Lstream_read (istr, random_buffer, sizeof (random_buffer));
+ Lstream_data_count nread = Lstream_read (istr, random_buffer, sizeof (random_buffer));
if (!nread)
break;
struct detection_state decst;
};
-static ssize_t decoding_reader (Lstream *stream,
- unsigned char *data, size_t size);
-static ssize_t decoding_writer (Lstream *stream,
- const unsigned char *data, size_t size);
+static Lstream_data_count decoding_reader (Lstream *stream,
+ unsigned char *data, Lstream_data_count size);
+static Lstream_data_count decoding_writer (Lstream *stream,
+ const unsigned char *data, Lstream_data_count size);
static int decoding_rewinder (Lstream *stream);
static int decoding_seekable_p (Lstream *stream);
static int decoding_flusher (Lstream *stream);
/* Read SIZE bytes of data and store it into DATA. We are a decoding stream
so we read data from the other end, decode it, and store it into DATA. */
-static ssize_t
-decoding_reader (Lstream *stream, unsigned char *data, size_t size)
+static Lstream_data_count
+decoding_reader (Lstream *stream, unsigned char *data, Lstream_data_count size)
{
struct decoding_stream *str = DECODING_STREAM_DATA (stream);
unsigned char *orig_data = data;
- ssize_t read_size;
+ Lstream_data_count read_size;
int error_occurred = 0;
/* We need to interface to mule_decode(), which expects to take some
most SIZE bytes, and delete the data from the runoff. */
if (Dynarr_length (str->runoff) > 0)
{
- size_t chunk = min (size, (size_t) Dynarr_length (str->runoff));
+ Lstream_data_count chunk = min (size, (Lstream_data_count) Dynarr_length (str->runoff));
memcpy (data, Dynarr_atp (str->runoff, 0), chunk);
Dynarr_delete_many (str->runoff, 0, chunk);
data += chunk;
return data - orig_data;
}
-static ssize_t
-decoding_writer (Lstream *stream, const unsigned char *data, size_t size)
+static Lstream_data_count
+decoding_writer (Lstream *stream, const unsigned char *data, Lstream_data_count size)
{
struct decoding_stream *str = DECODING_STREAM_DATA (stream);
- ssize_t retval;
+ Lstream_data_count retval;
/* Decode all our data into the runoff, and then attempt to write
it all out to the other end. Remove whatever chunk we succeeded
str->combined_char_count = 0;
str->combining_table = Qnil;
#endif
+ if (CODING_SYSTEM_TYPE (str->codesys) == CODESYS_AUTODETECT
+ || CODING_SYSTEM_EOL_TYPE (str->codesys) == EOL_AUTODETECT)
+ {
+ xzero (str->decst);
+ str->decst.eol_type = EOL_AUTODETECT;
+ str->decst.mask = ~0;
+ }
str->flags = str->cpos = 0;
}
static void
mule_decode (Lstream *decoding, const Extbyte *src,
- unsigned_char_dynarr *dst, size_t n)
+ unsigned_char_dynarr *dst, Lstream_data_count n)
{
struct decoding_stream *str = DECODING_STREAM_DATA (decoding);
char tempbuf[1024]; /* some random amount */
Bufpos newpos, even_newer_pos;
Bufpos oldpos = lisp_buffer_stream_startpos (istr);
- ssize_t size_in_bytes = Lstream_read (istr, tempbuf, sizeof (tempbuf));
+ Lstream_data_count size_in_bytes = Lstream_read (istr, tempbuf, sizeof (tempbuf));
if (!size_in_bytes)
break;
#endif /* MULE */
};
-static ssize_t encoding_reader (Lstream *stream, unsigned char *data, size_t size);
-static ssize_t encoding_writer (Lstream *stream, const unsigned char *data,
- size_t size);
+static Lstream_data_count encoding_reader (Lstream *stream, unsigned char *data, Lstream_data_count size);
+static Lstream_data_count encoding_writer (Lstream *stream, const unsigned char *data,
+ Lstream_data_count size);
static int encoding_rewinder (Lstream *stream);
static int encoding_seekable_p (Lstream *stream);
static int encoding_flusher (Lstream *stream);
/* Read SIZE bytes of data and store it into DATA. We are a encoding stream
so we read data from the other end, encode it, and store it into DATA. */
-static ssize_t
-encoding_reader (Lstream *stream, unsigned char *data, size_t size)
+static Lstream_data_count
+encoding_reader (Lstream *stream, unsigned char *data, Lstream_data_count size)
{
struct encoding_stream *str = ENCODING_STREAM_DATA (stream);
unsigned char *orig_data = data;
- ssize_t read_size;
+ Lstream_data_count read_size;
int error_occurred = 0;
/* We need to interface to mule_encode(), which expects to take some
return data - orig_data;
}
-static ssize_t
-encoding_writer (Lstream *stream, const unsigned char *data, size_t size)
+static Lstream_data_count
+encoding_writer (Lstream *stream, const unsigned char *data, Lstream_data_count size)
{
struct encoding_stream *str = ENCODING_STREAM_DATA (stream);
- ssize_t retval;
+ Lstream_data_count retval;
/* Encode all our data into the runoff, and then attempt to write
it all out to the other end. Remove whatever chunk we succeeded
static void
mule_encode (Lstream *encoding, const Bufbyte *src,
- unsigned_char_dynarr *dst, size_t n)
+ unsigned_char_dynarr *dst, Lstream_data_count n)
{
struct encoding_stream *str = ENCODING_STREAM_DATA (encoding);
char tempbuf[1024]; /* some random amount */
Bufpos newpos, even_newer_pos;
Bufpos oldpos = lisp_buffer_stream_startpos (istr);
- ssize_t size_in_bytes = Lstream_read (istr, tempbuf, sizeof (tempbuf));
+ Lstream_data_count size_in_bytes = Lstream_read (istr, tempbuf, sizeof (tempbuf));
if (!size_in_bytes)
break;
\f
static void
text_encode_generic (Lstream *encoding, const Bufbyte *src,
- unsigned_char_dynarr *dst, size_t n)
+ unsigned_char_dynarr *dst, Lstream_data_count n)
{
unsigned char c;
unsigned char char_boundary;
char_type = Qnil;
if (!NILP (ccs = Ffind_charset (ccs)))
{
- int code_point = charset_code_point (ccs, ch);
+ int code_point = charset_code_point (ccs, ch, 0);
if ( (code_point >= 0)
&& (NILP (char_type)
((c) >= 0xA1 && (c) <= 0xDF)
static int
-detect_coding_sjis (struct detection_state *st, const Extbyte *src, size_t n)
+detect_coding_sjis (struct detection_state *st, const Extbyte *src, Lstream_data_count n)
{
while (n--)
{
static void
decode_coding_sjis (Lstream *decoding, const Extbyte *src,
- unsigned_char_dynarr *dst, size_t n)
+ unsigned_char_dynarr *dst, Lstream_data_count n)
{
struct decoding_stream *str = DECODING_STREAM_DATA (decoding);
unsigned int flags = str->flags;
{
unsigned int s1, s2;
#ifdef UTF2000
- int code_point = charset_code_point (Vcharset_latin_jisx0201, ch);
+ int code_point = charset_code_point (Vcharset_latin_jisx0201, ch, 0);
if (code_point >= 0)
Dynarr_add (dst, code_point);
else if ((code_point
- = charset_code_point (Vcharset_japanese_jisx0208_1990, ch))
+ = charset_code_point (Vcharset_japanese_jisx0208_1990, ch, 0))
>= 0)
{
ENCODE_SJIS ((code_point >> 8) | 0x80,
Dynarr_add (dst, s2);
}
else if ((code_point
- = charset_code_point (Vcharset_katakana_jisx0201, ch))
+ = charset_code_point (Vcharset_katakana_jisx0201, ch, 0))
>= 0)
Dynarr_add (dst, code_point | 0x80);
else if ((code_point
- = charset_code_point (Vcharset_japanese_jisx0208, ch))
+ = charset_code_point (Vcharset_japanese_jisx0208, ch, 0))
>= 0)
{
ENCODE_SJIS ((code_point >> 8) | 0x80,
Dynarr_add (dst, s1);
Dynarr_add (dst, s2);
}
- else if ((code_point = charset_code_point (Vcharset_ascii, ch))
+ else if ((code_point = charset_code_point (Vcharset_ascii, ch, 0))
>= 0)
Dynarr_add (dst, code_point);
else
} while (0)
static int
-detect_coding_big5 (struct detection_state *st, const Extbyte *src, size_t n)
+detect_coding_big5 (struct detection_state *st, const Extbyte *src, Lstream_data_count n)
{
while (n--)
{
static void
decode_coding_big5 (Lstream *decoding, const Extbyte *src,
- unsigned_char_dynarr *dst, size_t n)
+ unsigned_char_dynarr *dst, Lstream_data_count n)
{
struct decoding_stream *str = DECODING_STREAM_DATA (decoding);
unsigned int flags = str->flags;
Lisp_Object ccs
= CODING_SYSTEM_ISO2022_INITIAL_CHARSET (str->codesys, 1);
- if ((code_point = charset_code_point (Vcharset_ascii, ch)) >= 0)
+ if ((code_point = charset_code_point (Vcharset_ascii, ch, 0)) >= 0)
Dynarr_add (dst, code_point);
- else if ((code_point = charset_code_point (ccs, ch)) >= 0)
+ else if ((code_point = charset_code_point (ccs, ch, 0)) >= 0)
{
Dynarr_add (dst, code_point >> 8);
Dynarr_add (dst, code_point & 0xFF);
}
else if ((code_point
- = charset_code_point (Vcharset_chinese_big5, ch)) >= 0)
+ = charset_code_point (Vcharset_chinese_big5, ch, 0)) >= 0)
{
Dynarr_add (dst, code_point >> 8);
Dynarr_add (dst, code_point & 0xFF);
}
else if ((code_point
- = charset_code_point (Vcharset_chinese_big5_1, ch)) >= 0)
+ = charset_code_point (Vcharset_chinese_big5_1, ch, 0)) >= 0)
{
unsigned int I
= ((code_point >> 8) - 33) * (0xFF - 0xA1)
Dynarr_add (dst, b2);
}
else if ((code_point
- = charset_code_point (Vcharset_chinese_big5_2, ch)) >= 0)
+ = charset_code_point (Vcharset_chinese_big5_2, ch, 0)) >= 0)
{
unsigned int I
= ((code_point >> 8) - 33) * (0xFF - 0xA1)
/************************************************************************/
static int
-detect_coding_ucs4 (struct detection_state *st, const Extbyte *src, size_t n)
+detect_coding_ucs4 (struct detection_state *st, const Extbyte *src, Lstream_data_count n)
{
while (n--)
{
static void
decode_coding_ucs4 (Lstream *decoding, const Extbyte *src,
- unsigned_char_dynarr *dst, size_t n)
+ unsigned_char_dynarr *dst, Lstream_data_count n)
{
struct decoding_stream *str = DECODING_STREAM_DATA (decoding);
unsigned int flags = str->flags;
/************************************************************************/
static int
-detect_coding_utf16 (struct detection_state *st, const Extbyte *src, size_t n)
+detect_coding_utf16 (struct detection_state *st, const Extbyte *src, Lstream_data_count n)
{
return CODING_CATEGORY_UTF16_MASK;
}
static void
decode_coding_utf16 (Lstream *decoding, const Extbyte *src,
- unsigned_char_dynarr *dst, size_t n)
+ unsigned_char_dynarr *dst, Lstream_data_count n)
{
struct decoding_stream *str = DECODING_STREAM_DATA (decoding);
unsigned int flags = str->flags;
unsigned int cpos = str->cpos;
- unsigned char counter = str->counter;
+ unsigned char counter = str->counter & 3;
+ unsigned char byte_order = str->counter >> 2;
eol_type_t eol_type = str->eol_type;
while (n--)
{
unsigned char c = *(unsigned char *)src++;
- if ((counter & 1) == 0)
+ if (counter == 0)
{
cpos = c;
- counter |= 1;
+ counter = 1;
}
- else
+ else if (counter == 1)
{
- int code
- = (counter & 2) == 2
- ? (cpos << 8) | c
- : (c << 8) | cpos;
+ int code;
+ if (byte_order == 0)
+ code = (c << 8) | cpos;
+ else
+ code = (cpos << 8) | c;
if (code == 0xFFFE)
{
- if ( (counter & 2) == 0 )
- {
- counter = 2;
- code = (cpos << 8) | c;
- }
+ code = ((code & 0xFF) << 8) | (code >> 8);
+ if ( byte_order == 0 )
+ byte_order = 1;
else
+ byte_order = 0;
+ }
+ if ( (0xD800 <= code) && (code <= 0xDBFF) )
+ {
+ counter = 2;
+ cpos = code;
+ }
+ else
+ {
+ counter = 0;
+ cpos = 0;
+ if (code != 0xFEFF)
{
- counter = 0;
- code = (c << 8) | cpos;
+ DECODE_HANDLE_EOL_TYPE (eol_type, code, flags, dst);
+ DECODE_ADD_UCS_CHAR (code, dst);
}
}
- counter &= 2;
+ }
+ else if (counter == 2)
+ {
+ cpos = (cpos << 8) | c;
+ counter++;
+ }
+ else
+ {
+ int x = cpos >> 8;
+ int y
+ = (byte_order == 0)
+ ? (c << 8) | (cpos & 0xFF)
+ : ((cpos & 0xFF) << 8) | c;
+
+ DECODE_ADD_UCS_CHAR ((x - 0xD800) * 0x400 + (y - 0xDC00)
+ + 0x10000, dst);
+ counter = 0;
cpos = 0;
- if (code != 0xFEFF)
- {
- DECODE_HANDLE_EOL_TYPE (eol_type, code, flags, dst);
- DECODE_ADD_UCS_CHAR (code, dst);
- }
}
label_continue_loop:;
}
str->flags = flags;
str->cpos = cpos;
- str->counter = counter;
+ str->counter = (byte_order << 2) | counter;
}
void
else
{
int y = ((ch - 0x10000) / 0x400) + 0xD800;
- int z = ((ch - 0x10000) % 0x400) + 0xD800;
+ int z = ((ch - 0x10000) % 0x400) + 0xDC00;
Dynarr_add (dst, y);
Dynarr_add (dst, y >> 8);
/************************************************************************/
static int
-detect_coding_utf8 (struct detection_state *st, const Extbyte *src, size_t n)
+detect_coding_utf8 (struct detection_state *st, const Extbyte *src, Lstream_data_count n)
{
while (n--)
{
static void
decode_coding_utf8 (Lstream *decoding, const Extbyte *src,
- unsigned_char_dynarr *dst, size_t n)
+ unsigned_char_dynarr *dst, Lstream_data_count n)
{
struct decoding_stream *str = DECODING_STREAM_DATA (decoding);
unsigned int flags = str->flags;
{
Lisp_Object ucs_ccs
= CODING_SYSTEM_ISO2022_INITIAL_CHARSET (str->codesys, 0);
- int code_point = charset_code_point (ucs_ccs, ch);
+ int code_point = charset_code_point (ucs_ccs, ch, 0);
if ( (code_point < 0) || (code_point > 0x10FFFF) )
{
}
static int
-detect_coding_iso2022 (struct detection_state *st, const Extbyte *src, size_t n)
+detect_coding_iso2022 (struct detection_state *st, const Extbyte *src, Lstream_data_count n)
{
int mask;
static void
decode_coding_iso2022 (Lstream *decoding, const Extbyte *src,
- unsigned_char_dynarr *dst, size_t n)
+ unsigned_char_dynarr *dst, Lstream_data_count n)
{
struct decoding_stream *str = DECODING_STREAM_DATA (decoding);
unsigned int flags = str->flags;
for (i = 0; i < 4; i++)
{
if ((CHARSETP (charset = str->iso2022.charset[i])
- && ((code_point = charset_code_point (charset, ch)) >= 0))
+ && ((code_point = charset_code_point (charset, ch, 0)) >= 0))
||
(CHARSETP
(charset
= CODING_SYSTEM_ISO2022_INITIAL_CHARSET (codesys, i))
- && ((code_point = charset_code_point (charset, ch)) >= 0)))
+ && ((code_point = charset_code_point (charset, ch, 0)) >= 0)))
{
reg = i;
break;
interpreted as being in any particular decoding. */
static void
decode_coding_no_conversion (Lstream *decoding, const Extbyte *src,
- unsigned_char_dynarr *dst, size_t n)
+ unsigned_char_dynarr *dst, Lstream_data_count n)
{
struct decoding_stream *str = DECODING_STREAM_DATA (decoding);
unsigned int flags = str->flags;
static void
encode_coding_no_conversion (Lstream *encoding, const Bufbyte *src,
- unsigned_char_dynarr *dst, size_t n)
+ unsigned_char_dynarr *dst, Lstream_data_count n)
{
unsigned char c;
struct encoding_stream *str = ENCODING_STREAM_DATA (encoding);
{
INIT_LRECORD_IMPLEMENTATION (coding_system);
- deferror (&Qcoding_system_error, "coding-system-error",
- "Coding-system error", Qio_error);
+ DEFERROR_STANDARD (Qcoding_system_error, Qio_error);
DEFSUBR (Fcoding_system_p);
DEFSUBR (Ffind_coding_system);
projects / chise / xemacs-chise.git- / blobdiff