From fbea6cb1b3c23e133ff8beccded2fb1fb61c2961 Mon Sep 17 00:00:00 2001
From: akr <akr>
Date: Tue, 18 Aug 1998 00:07:57 +0000
Subject: [PATCH] * ew-data.el:   - Abolish prev-token, next-token, start and
 end   properties on fragment.   - Abolish function
 `ew-tokenize-frag', `ew-add-open',   `ew-add-close',
 `ew-add-token', `ew-add-close-token' and   `ew-token-list'.
 (ew-make-anchor): Add type property.
 (ew-token-last-frag-p): New function.

* ew-parse.el: Add new token `anchor'.

* ew-scan-m.el: Use `ew-add-frag' instead of abolished
functions.

* ew-scan-s.el: Ditto.
---
 ChangeLog    |   18 ++++
 ew-data.el   |   68 ++++---------
 ew-parse.el  |  300 +++++++++++++++++++++++++++++-----------------------------
 ew-scan-m.el |   36 +++----
 ew-scan-s.el |   32 +++----
 5 files changed, 220 insertions(+), 234 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 8f4e95f..8150384 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,24 @@
 1998-08-17  Tanaka Akira      <akr@jaist.ac.jp>
 
 	* ew-data.el:
+	  - Abolish prev-token, next-token, start and end
+	  properties on fragment.
+	  - Abolish function `ew-tokenize-frag', `ew-add-open',
+	  `ew-add-close', `ew-add-token', `ew-add-close-token' and
+	  `ew-token-list'.
+	(ew-make-anchor): Add type property.
+	(ew-token-last-frag-p): New function.
+
+	* ew-parse.el: Add new token `anchor'.
+
+	* ew-scan-m.el: Use `ew-add-frag' instead of abolished
+	functions.
+
+	* ew-scan-s.el: Ditto.
+
+1998-08-17  Tanaka Akira      <akr@jaist.ac.jp>
+
+	* ew-data.el:
 	  - Require 'ew-util.
 	  - Change token prefix "ew:raw-" to "ew:".
 
diff --git a/ew-data.el b/ew-data.el
index 0121534..12fdb9d 100644
--- a/ew-data.el
+++ b/ew-data.el
@@ -5,10 +5,9 @@
 (defun ew-make-anchor (column str)
   (let ((anchor (make-symbol str)))
     (put anchor 'anchor anchor)
+    (put anchor 'type 'ew:anchor-tok)
     (put anchor 'prev-frag anchor)
     (put anchor 'next-frag anchor)
-    (put anchor 'prev-token anchor)
-    (put anchor 'next-token anchor)
     (put anchor 'column column)
     (put anchor 'line-length 0)
     anchor))
@@ -20,18 +19,9 @@
       (put frag 'line-length line-length)
       (setq frag (get frag 'prev-frag)))))
 
-(defun ew-tokenize-frag (anchor frag)
-  (put frag 'prev-token (get anchor 'prev-token))
-  (put frag 'next-token anchor)
-  (put (get anchor 'prev-token) 'next-token frag)
-  (put anchor 'prev-token frag)
-  frag)
-
 (defun ew-add-frag (anchor start end type)
   (let ((frag (make-symbol (substring (symbol-name anchor) start end))))
     (put frag 'anchor anchor)
-    (put frag 'start start)
-    (put frag 'end end)
     (put frag 'type type)
     (put frag 'prev-frag (get anchor 'prev-frag))
     (put frag 'next-frag anchor)
@@ -49,25 +39,6 @@
       (put anchor 'column (+ (get anchor 'column) (length (symbol-name frag)))))
     frag))
 
-(defun ew-add-open (anchor start end type)
-  (let ((frag (ew-add-frag anchor start end type)))
-    (put frag 'prev-open (get anchor 'prev-open))
-    (put anchor 'prev-open frag)
-    frag))
-
-(defun ew-add-close (anchor start end type)
-  (let ((frag (ew-add-frag anchor start end type)))
-    (put frag 'pair (get anchor 'prev-open))
-    (put (get anchor 'prev-open) 'pair frag)
-    (put anchor 'prev-open (get (get frag 'pair) 'prev-open))
-    frag))
-    
-(defun ew-add-token (anchor start end type)
-  (ew-tokenize-frag anchor (ew-add-frag anchor start end type)))
-
-(defun ew-add-close-token (anchor start end type)
-  (ew-tokenize-frag anchor (ew-add-close anchor start end type)))
-
 ;;; listup
 
 (defun ew-frag-list (anchor)
@@ -78,14 +49,6 @@
 	    tmp (get tmp 'prev-frag)))
     res))
 
-(defun ew-token-list (anchor)
-  (let ((res ())
-	(tmp (get anchor 'prev-token)))
-    (while (not (eq anchor tmp))
-      (setq res (cons tmp res)
-	    tmp (get tmp 'prev-token)))
-    res))
-
 (defun ew-pair-list (anchor)
   (mapcar
    (lambda (frag)
@@ -101,19 +64,14 @@
 	       (or (< 0 (match-beginning 0))
 		   (< (match-end 0) (length (symbol-name frag1)))))
       (let ((atom (symbol-name frag1))
-	    (base (get frag1 'start))
 	    (start (match-end 0))
 	    result
 	    frag)
 	(when (< 0 (match-beginning 0))
 	  (setq frag (make-symbol (substring atom 0 (match-beginning 0)))
-		result(ew-rcons* result frag))
-	  (put frag 'start base)
-	  (put frag 'end (+ base (match-beginning 0))))
+		result (ew-rcons* result frag)))
 	(setq frag (make-symbol (substring atom (match-beginning 0) (match-end 0)))
 	      result (ew-rcons* result frag))
-	(put frag 'start (+ base (match-beginning 0)))
-	(put frag 'end (+ base (match-end 0)))
 	(when (cdr result)
 	  (put frag 'prev-frag (cadr result))
 	  (put (cadr result) 'next-frag frag)
@@ -124,22 +82,16 @@
 	  (when (< start (match-beginning 0))
 	    (setq frag (make-symbol (substring atom start (match-beginning 0)))
 		  result (ew-rcons* result frag))
-	    (put frag 'start (+ base start))
-	    (put frag 'end (+ base (match-beginning 0)))
 	    (put frag 'prev-frag (cadr result))
 	    (put (cadr result) 'next-frag frag))
 	  (setq frag (make-symbol (substring atom (match-beginning 0) (match-end 0)))
 		result (ew-rcons* result frag)
 		start (match-end 0))
-	  (put frag 'start (+ base (match-beginning 0)))
-	  (put frag 'end (+ base (match-end 0)))
 	  (put frag 'prev-frag (cadr result))
 	  (put (cadr result) 'next-frag frag))
 	(when (< start (length (symbol-name frag1)))
 	  (setq frag (make-symbol (substring atom start))
 		result (ew-rcons* result frag))
-	  (put frag 'start (+ base start))
-	  (put frag 'end (get frag1 'end))
 	  (put frag 'prev-frag (cadr result))
 	  (put (cadr result) 'next-frag frag))
 	(setq frag (car result))
@@ -171,13 +123,27 @@
   (unless (ew-comment-frag-p frag2)
     (put frag2 'decode 'ew-decode-phrase))
   (setq frag2 (get frag2 'prev-frag))
-  (while (not (get frag2 'prev-token))
+  (while (not (ew-token-last-frag-p frag2))
     (unless (ew-comment-frag-p frag2)
       (put frag2 'decode 'ew-decode-phrase))
     (setq frag2 (get frag2 'prev-frag))))
 
 ;;; frag predicate
 
+(defun ew-token-last-frag-p (frag)
+  (member (get frag 'type)
+	  '(ew:anchor-tok
+	    ew:lt-tok
+	    ew:gt-tok
+	    ew:at-tok
+	    ew:comma-tok
+	    ew:semicolon-tok
+	    ew:colon-tok
+	    ew:dot-tok
+	    ew:atom-tok
+	    ew:qs-end-tok
+	    ew:dl-end-tok)))
+
 (defun ew-comment-frag-p (frag)
   (member (get frag 'type)
 	  '(ew:cm-begin-tok
diff --git a/ew-parse.el b/ew-parse.el
index 33c8941..77062f5 100644
--- a/ew-parse.el
+++ b/ew-parse.el
@@ -5,112 +5,114 @@
 
 ; *** Token Definitions ***
 (defconst ew:*eoi*-tok	0)
-(defconst ew:tag-mailbox-tok	1)
-(defconst ew:tag-mailbox*-tok	2)
-(defconst ew:tag-mailbox+-tok	3)
-(defconst ew:tag-address*-tok	4)
-(defconst ew:tag-address+-tok	5)
-(defconst ew:tag-phrase*-tok	6)
-(defconst ew:tag-phrase-msg-id*-tok	7)
-(defconst ew:lt-tok	8)
-(defconst ew:gt-tok	9)
-(defconst ew:at-tok	10)
-(defconst ew:comma-tok	11)
-(defconst ew:semicolon-tok	12)
-(defconst ew:colon-tok	13)
-(defconst ew:dot-tok	14)
-(defconst ew:atom-tok	15)
-(defconst ew:slash-tok	16)
-(defconst ew:question-tok	17)
-(defconst ew:equal-tok	18)
-(defconst ew:token-tok	19)
-(defconst ew:us-texts-tok	20)
-(defconst ew:us-wsp-tok	21)
-(defconst ew:us-fold-tok	22)
-(defconst ew:wsp-tok	23)
-(defconst ew:fold-tok	24)
-(defconst ew:qs-begin-tok	25)
-(defconst ew:qs-end-tok	26)
-(defconst ew:qs-texts-tok	27)
-(defconst ew:qs-wsp-tok	28)
-(defconst ew:qs-fold-tok	29)
-(defconst ew:qs-qfold-tok	30)
-(defconst ew:qs-qpair-tok	31)
-(defconst ew:dl-begin-tok	32)
-(defconst ew:dl-end-tok	33)
-(defconst ew:dl-texts-tok	34)
-(defconst ew:dl-wsp-tok	35)
-(defconst ew:dl-fold-tok	36)
-(defconst ew:dl-qfold-tok	37)
-(defconst ew:dl-qpair-tok	38)
-(defconst ew:cm-begin-tok	39)
-(defconst ew:cm-end-tok	40)
-(defconst ew:cm-nested-begin-tok	41)
-(defconst ew:cm-nested-end-tok	42)
-(defconst ew:cm-texts-tok	43)
-(defconst ew:cm-wsp-tok	44)
-(defconst ew:cm-fold-tok	45)
-(defconst ew:cm-qfold-tok	46)
-(defconst ew:cm-qpair-tok	47)
-(defconst ew:err-tok	48)
+(defconst ew:anchor-tok	1)
+(defconst ew:tag-mailbox-tok	2)
+(defconst ew:tag-mailbox*-tok	3)
+(defconst ew:tag-mailbox+-tok	4)
+(defconst ew:tag-address*-tok	5)
+(defconst ew:tag-address+-tok	6)
+(defconst ew:tag-phrase*-tok	7)
+(defconst ew:tag-phrase-msg-id*-tok	8)
+(defconst ew:lt-tok	9)
+(defconst ew:gt-tok	10)
+(defconst ew:at-tok	11)
+(defconst ew:comma-tok	12)
+(defconst ew:semicolon-tok	13)
+(defconst ew:colon-tok	14)
+(defconst ew:dot-tok	15)
+(defconst ew:atom-tok	16)
+(defconst ew:slash-tok	17)
+(defconst ew:question-tok	18)
+(defconst ew:equal-tok	19)
+(defconst ew:token-tok	20)
+(defconst ew:us-texts-tok	21)
+(defconst ew:us-wsp-tok	22)
+(defconst ew:us-fold-tok	23)
+(defconst ew:wsp-tok	24)
+(defconst ew:fold-tok	25)
+(defconst ew:qs-begin-tok	26)
+(defconst ew:qs-end-tok	27)
+(defconst ew:qs-texts-tok	28)
+(defconst ew:qs-wsp-tok	29)
+(defconst ew:qs-fold-tok	30)
+(defconst ew:qs-qfold-tok	31)
+(defconst ew:qs-qpair-tok	32)
+(defconst ew:dl-begin-tok	33)
+(defconst ew:dl-end-tok	34)
+(defconst ew:dl-texts-tok	35)
+(defconst ew:dl-wsp-tok	36)
+(defconst ew:dl-fold-tok	37)
+(defconst ew:dl-qfold-tok	38)
+(defconst ew:dl-qpair-tok	39)
+(defconst ew:cm-begin-tok	40)
+(defconst ew:cm-end-tok	41)
+(defconst ew:cm-nested-begin-tok	42)
+(defconst ew:cm-nested-end-tok	43)
+(defconst ew:cm-texts-tok	44)
+(defconst ew:cm-wsp-tok	45)
+(defconst ew:cm-fold-tok	46)
+(defconst ew:cm-qfold-tok	47)
+(defconst ew:cm-qpair-tok	48)
+(defconst ew:err-tok	49)
 
 (defconst ew:token-defs
   (list 
    (cons 0 "*eoi*")
-   (cons 1 "tag-mailbox")
-   (cons 2 "tag-mailbox*")
-   (cons 3 "tag-mailbox+")
-   (cons 4 "tag-address*")
-   (cons 5 "tag-address+")
-   (cons 6 "tag-phrase*")
-   (cons 7 "tag-phrase-msg-id*")
-   (cons 8 "lt")
-   (cons 9 "gt")
-   (cons 10 "at")
-   (cons 11 "comma")
-   (cons 12 "semicolon")
-   (cons 13 "colon")
-   (cons 14 "dot")
-   (cons 15 "atom")
-   (cons 16 "slash")
-   (cons 17 "question")
-   (cons 18 "equal")
-   (cons 19 "token")
-   (cons 20 "us-texts")
-   (cons 21 "us-wsp")
-   (cons 22 "us-fold")
-   (cons 23 "wsp")
-   (cons 24 "fold")
-   (cons 25 "qs-begin")
-   (cons 26 "qs-end")
-   (cons 27 "qs-texts")
-   (cons 28 "qs-wsp")
-   (cons 29 "qs-fold")
-   (cons 30 "qs-qfold")
-   (cons 31 "qs-qpair")
-   (cons 32 "dl-begin")
-   (cons 33 "dl-end")
-   (cons 34 "dl-texts")
-   (cons 35 "dl-wsp")
-   (cons 36 "dl-fold")
-   (cons 37 "dl-qfold")
-   (cons 38 "dl-qpair")
-   (cons 39 "cm-begin")
-   (cons 40 "cm-end")
-   (cons 41 "cm-nested-begin")
-   (cons 42 "cm-nested-end")
-   (cons 43 "cm-texts")
-   (cons 44 "cm-wsp")
-   (cons 45 "cm-fold")
-   (cons 46 "cm-qfold")
-   (cons 47 "cm-qpair")
-   (cons 48 "err")
+   (cons 1 "anchor")
+   (cons 2 "tag-mailbox")
+   (cons 3 "tag-mailbox*")
+   (cons 4 "tag-mailbox+")
+   (cons 5 "tag-address*")
+   (cons 6 "tag-address+")
+   (cons 7 "tag-phrase*")
+   (cons 8 "tag-phrase-msg-id*")
+   (cons 9 "lt")
+   (cons 10 "gt")
+   (cons 11 "at")
+   (cons 12 "comma")
+   (cons 13 "semicolon")
+   (cons 14 "colon")
+   (cons 15 "dot")
+   (cons 16 "atom")
+   (cons 17 "slash")
+   (cons 18 "question")
+   (cons 19 "equal")
+   (cons 20 "token")
+   (cons 21 "us-texts")
+   (cons 22 "us-wsp")
+   (cons 23 "us-fold")
+   (cons 24 "wsp")
+   (cons 25 "fold")
+   (cons 26 "qs-begin")
+   (cons 27 "qs-end")
+   (cons 28 "qs-texts")
+   (cons 29 "qs-wsp")
+   (cons 30 "qs-fold")
+   (cons 31 "qs-qfold")
+   (cons 32 "qs-qpair")
+   (cons 33 "dl-begin")
+   (cons 34 "dl-end")
+   (cons 35 "dl-texts")
+   (cons 36 "dl-wsp")
+   (cons 37 "dl-fold")
+   (cons 38 "dl-qfold")
+   (cons 39 "dl-qpair")
+   (cons 40 "cm-begin")
+   (cons 41 "cm-end")
+   (cons 42 "cm-nested-begin")
+   (cons 43 "cm-nested-end")
+   (cons 44 "cm-texts")
+   (cons 45 "cm-wsp")
+   (cons 46 "cm-fold")
+   (cons 47 "cm-qfold")
+   (cons 48 "cm-qpair")
+   (cons 49 "err")
   ))
 
 ; *** Action Table ***
 (defconst ew:action-table
   [
-     ((default . *error*) (1 . 7) (2 . 6) (3 . 5) (4 . 4) (5 . 3) (6 . 2) (7 . 1))
+     ((default . *error*) (2 . 7) (3 . 6) (4 . 5) (5 . 4) (6 . 3) (7 . 2) (8 . 1))
      ((default . -61))
      ((default . -61))
      ((default . -61))
@@ -119,13 +121,13 @@
      ((default . -61))
      ((default . -61))
      ((default . *error*) (0 . 16))
-     ((default . -51) (15 . 21) (23 . 20) (24 . 19) (25 . 18) (39 . 17))
-     ((default . -53) (15 . 21) (23 . 20) (24 . 19) (25 . 18) (39 . 17))
-     ((default . *error*) (8 . 34) (15 . 21) (23 . 20) (24 . 19) (25 . 18) (39 . 17))
-     ((default . -9) (8 . 34) (15 . 21) (23 . 20) (24 . 19) (25 . 18) (39 . 17))
-     ((default . *error*) (8 . 34) (15 . 21) (23 . 20) (24 . 19) (25 . 18) (39 . 17))
-     ((default . -30) (8 . 34) (15 . 21) (23 . 20) (24 . 19) (25 . 18) (39 . 17))
-     ((default . *error*) (8 . 34) (15 . 21) (23 . 20) (24 . 19) (25 . 18) (39 . 17))
+     ((default . -51) (16 . 21) (24 . 20) (25 . 19) (26 . 18) (40 . 17))
+     ((default . -53) (16 . 21) (24 . 20) (25 . 19) (26 . 18) (40 . 17))
+     ((default . *error*) (9 . 34) (16 . 21) (24 . 20) (25 . 19) (26 . 18) (40 . 17))
+     ((default . -9) (9 . 34) (16 . 21) (24 . 20) (25 . 19) (26 . 18) (40 . 17))
+     ((default . *error*) (9 . 34) (16 . 21) (24 . 20) (25 . 19) (26 . 18) (40 . 17))
+     ((default . -30) (9 . 34) (16 . 21) (24 . 20) (25 . 19) (26 . 18) (40 . 17))
+     ((default . *error*) (9 . 34) (16 . 21) (24 . 20) (25 . 19) (26 . 18) (40 . 17))
      ((default . -1) (0 . accept))
      ((default . -90))
      ((default . -76))
@@ -136,56 +138,56 @@
      ((default . -61))
      ((default . -48))
      ((default . -49))
-     ((default . -8) (8 . 34))
+     ((default . -8) (9 . 34))
      ((default . -57))
      ((default . -37))
-     ((default . -36) (15 . 21) (25 . 18))
+     ((default . -36) (16 . 21) (26 . 18))
      ((default . -52))
-     ((default . -54) (11 . 60))
+     ((default . -54) (12 . 60))
      ((default . -7))
      ((default . -55))
      ((default . -61))
-     ((default . -43) (10 . 63))
-     ((default . -37) (14 . -25) (10 . -25))
+     ((default . -43) (11 . 63))
+     ((default . -37) (15 . -25) (11 . -25))
      ((default . -29))
-     ((default . *error*) (8 . 34) (13 . 68))
+     ((default . *error*) (9 . 34) (14 . 68))
      ((default . -13))
-     ((default . *error*) (10 . 63) (14 . 71))
+     ((default . *error*) (11 . 63) (15 . 71))
      ((default . -14))
      ((default . -27))
      ((default . -11))
-     ((default . -6) (11 . 60))
-     ((default . -10) (11 . 60))
+     ((default . -6) (12 . 60))
+     ((default . -10) (12 . 60))
      ((default . -5))
-     ((default . *error*) (8 . 34))
-     ((default . -4) (11 . 60))
+     ((default . *error*) (9 . 34))
+     ((default . -4) (12 . 60))
      ((default . -32))
-     ((default . -31) (11 . 60))
+     ((default . -31) (12 . 60))
      ((default . -3))
      ((default . -2))
-     ((default . *error*) (40 . 83) (41 . 82) (42 . 81) (43 . 80) (44 . 79) (45 . 78) (46 . 77) (47 . 76))
-     ((default . *error*) (26 . 89) (27 . 88) (28 . 87) (29 . 86) (30 . 85) (31 . 84))
-     ((default . -74) (23 . 20) (24 . 19) (39 . 17))
-     ((default . -72) (23 . 20) (24 . 19) (39 . 17))
-     ((default . *error*) (15 . 21) (25 . 18))
-     ((default . -51) (15 . 21) (25 . 18))
+     ((default . *error*) (41 . 83) (42 . 82) (43 . 81) (44 . 80) (45 . 79) (46 . 78) (47 . 77) (48 . 76))
+     ((default . *error*) (27 . 89) (28 . 88) (29 . 87) (30 . 86) (31 . 85) (32 . 84))
+     ((default . -74) (24 . 20) (25 . 19) (40 . 17))
+     ((default . -72) (24 . 20) (25 . 19) (40 . 17))
+     ((default . *error*) (16 . 21) (26 . 18))
+     ((default . -51) (16 . 21) (26 . 18))
      ((default . -38))
      ((default . -61))
-     ((default . *error*) (15 . 21) (25 . 18))
-     ((default . -65) (23 . 20) (24 . 19) (39 . 17))
+     ((default . *error*) (16 . 21) (26 . 18))
+     ((default . -65) (24 . 20) (25 . 19) (40 . 17))
      ((default . -61))
-     ((default . *error*) (15 . 21) (32 . 96))
-     ((default . *error*) (15 . 21) (25 . 18))
-     ((default . *error*) (11 . 60) (13 . 68))
+     ((default . *error*) (16 . 21) (33 . 96))
+     ((default . *error*) (16 . 21) (26 . 18))
+     ((default . *error*) (12 . 60) (14 . 68))
      ((default . -44))
      ((default . -61))
-     ((default . -30) (8 . 34) (15 . 21) (25 . 18))
+     ((default . -30) (9 . 34) (16 . 21) (26 . 18))
      ((default . -28))
      ((default . -61))
-     ((default . *error*) (15 . 21) (25 . 18))
-     ((default . *error*) (15 . 21) (32 . 96))
-     ((default . *error*) (8 . 34) (15 . 21) (25 . 18))
-     ((default . *error*) (8 . 34) (15 . 21) (25 . 18))
+     ((default . *error*) (16 . 21) (26 . 18))
+     ((default . *error*) (16 . 21) (33 . 96))
+     ((default . *error*) (9 . 34) (16 . 21) (26 . 18))
+     ((default . *error*) (9 . 34) (16 . 21) (26 . 18))
      ((default . -97))
      ((default . -96))
      ((default . -95))
@@ -201,38 +203,38 @@
      ((default . -77))
      ((default . -75))
      ((default . -25))
-     ((default . *error*) (9 . 113))
+     ((default . *error*) (10 . 113))
      ((default . -58))
-     ((default . -68) (23 . 20) (24 . 19) (39 . 17))
+     ((default . -68) (24 . 20) (25 . 19) (40 . 17))
      ((default . -56))
-     ((default . -67) (23 . 20) (24 . 19) (39 . 17))
+     ((default . -67) (24 . 20) (25 . 19) (40 . 17))
      ((default . -83))
      ((default . -61))
      ((default . -21))
      ((default . -46))
      ((default . -19))
      ((default . -45))
-     ((default . -40) (14 . 71))
-     ((default . *error*) (9 . 113))
+     ((default . -40) (15 . 71))
+     ((default . *error*) (10 . 113))
      ((default . -39))
-     ((default . *error*) (10 . 63))
-     ((default . -70) (23 . 20) (24 . 19) (39 . 17))
-     ((default . *error*) (12 . 120))
-     ((default . -71) (23 . 20) (24 . 19) (39 . 17))
+     ((default . *error*) (11 . 63))
+     ((default . -70) (24 . 20) (25 . 19) (40 . 17))
+     ((default . *error*) (13 . 120))
+     ((default . -71) (24 . 20) (25 . 19) (40 . 17))
      ((default . -26))
-     ((default . -15) (14 . 71))
+     ((default . -15) (15 . 71))
      ((default . -12))
      ((default . -33))
      ((default . -61))
      ((default . -35))
-     ((default . *error*) (33 . 128) (34 . 127) (35 . 126) (36 . 125) (37 . 124) (38 . 123))
-     ((default . -73) (23 . 20) (24 . 19) (39 . 17))
-     ((default . *error*) (15 . 21) (32 . 96))
+     ((default . *error*) (34 . 128) (35 . 127) (36 . 126) (37 . 125) (38 . 124) (39 . 123))
+     ((default . -73) (24 . 20) (25 . 19) (40 . 17))
+     ((default . *error*) (16 . 21) (33 . 96))
      ((default . -42))
-     ((default . *error*) (15 . 21) (32 . 96))
+     ((default . *error*) (16 . 21) (33 . 96))
      ((default . -61))
      ((default . -22))
-     ((default . -66) (23 . 20) (24 . 19) (39 . 17))
+     ((default . -66) (24 . 20) (25 . 19) (40 . 17))
      ((default . -88))
      ((default . -87))
      ((default . -86))
@@ -240,8 +242,8 @@
      ((default . -84))
      ((default . -82))
      ((default . -20))
-     ((default . -41) (14 . 71))
-     ((default . -69) (23 . 20) (24 . 19) (39 . 17))
+     ((default . -41) (15 . 71))
+     ((default . -69) (24 . 20) (25 . 19) (40 . 17))
     ])
 
 ; *** Goto Table ***
diff --git a/ew-scan-m.el b/ew-scan-m.el
index 963a9e1..8dd9598 100644
--- a/ew-scan-m.el
+++ b/ew-scan-m.el
@@ -18,33 +18,33 @@
 	 (,scan
 	  str p q
 	  ([" \t"] (ew-add-frag res r p 'ew:wsp-tok))
-	  (?< (ew-add-token res r p 'ew:lt-tok))
-	  (?> (ew-add-token res r p 'ew:gt-tok))
-	  (?@ (ew-add-token res r p 'ew:at-tok))
-	  (?, (ew-add-token res r p 'ew:comma-tok))
-	  (?\; (ew-add-token res r p 'ew:semicolon-tok))
-	  (?: (ew-add-token res r p 'ew:colon-tok))
-	  (?/ (ew-add-token res r p 'ew:slash-tok))
-	  (?? (ew-add-token res r p 'ew:question-tok))
-	  (?= (ew-add-token res r p 'ew:equal-tok))
+	  (?< (ew-add-frag res r p 'ew:lt-tok))
+	  (?> (ew-add-frag res r p 'ew:gt-tok))
+	  (?@ (ew-add-frag res r p 'ew:at-tok))
+	  (?, (ew-add-frag res r p 'ew:comma-tok))
+	  (?\; (ew-add-frag res r p 'ew:semicolon-tok))
+	  (?: (ew-add-frag res r p 'ew:colon-tok))
+	  (?/ (ew-add-frag res r p 'ew:slash-tok))
+	  (?? (ew-add-frag res r p 'ew:question-tok))
+	  (?= (ew-add-frag res r p 'ew:equal-tok))
 	  ((?\r ?\n [" \t"])
 	   (ew-add-frag res r p 'ew:fold-tok))
 	  ((?\r ?\n [^ " \t"])
 	   (ew-add-frag res r (setq p q) 'ew:err-tok))
 	  ((+ [(?a ?z) (?A ?Z) (?0 ?9) "!#$%&'*+-.^_`{|}~" non-ascii])
-	   (ew-add-token res r p 'ew:token-tok))
-	  (?\" (ew-add-open res r p 'ew:qs-begin-tok)
+	   (ew-add-frag res r p 'ew:token-tok))
+	  (?\" (ew-add-frag res r p 'ew:qs-begin-tok)
 	       (setq mode 'quoted-string))
-	  (?\[ (ew-add-open res r p 'ew:dl-begin-tok)
+	  (?\[ (ew-add-frag res r p 'ew:dl-begin-tok)
 	       (setq mode 'domain-literal))
-	  (?\( (ew-add-open res r p 'ew:cm-begin-tok)
+	  (?\( (ew-add-frag res r p 'ew:cm-begin-tok)
 	       (setq mode 'comment
 		     nest 1))
 	  (() (ew-add-frag res r q 'ew:err-tok) (setq p q))))
 	((eq mode 'quoted-string)
 	 (,scan
 	  str p q
-	  (?\" (ew-add-close-token res r p 'ew:qs-end-tok)
+	  (?\" (ew-add-frag res r p 'ew:qs-end-tok)
 	       (setq mode 'token))
 	  ((?\\ ?\r ?\n [" \t"])
 	   (ew-add-frag res r p 'ew:qs-qfold-tok))
@@ -73,7 +73,7 @@
 	((eq mode 'domain-literal)
 	 (,scan
 	  str p q
-	  (?\] (ew-add-close-token res r p 'ew:dl-end-tok)
+	  (?\] (ew-add-frag res r p 'ew:dl-end-tok)
 	       (setq mode 'token))
 	  ((?\\ ?\r ?\n [" \t"])
 	   (ew-add-frag res r p 'ew:dl-qfold-tok))
@@ -102,14 +102,14 @@
 	((eq mode 'comment)
 	 (,scan
 	  str p q
-	  (?\( (ew-add-open res r p 'ew:cm-nested-begin-tok)
+	  (?\( (ew-add-frag res r p 'ew:cm-nested-begin-tok)
 	       (setq nest (1+ nest)))
 	  (?\) (setq nest (1- nest))
 	       (if (zerop nest)
 		   (progn
-		     (ew-add-close res r p 'ew:cm-end-tok)
+		     (ew-add-frag res r p 'ew:cm-end-tok)
 		     (setq mode 'token))
-		 (ew-add-close res r p 'ew:cm-nested-end-tok)))
+		 (ew-add-frag res r p 'ew:cm-nested-end-tok)))
 	  ((?\\ ?\r ?\n [" \t"])
 	   (ew-add-frag res r p 'ew:cm-qfold-tok))
 	  ((?\\ ?\r ?\n [^ " \t"])
diff --git a/ew-scan-s.el b/ew-scan-s.el
index 427bfd6..e060cdc 100644
--- a/ew-scan-s.el
+++ b/ew-scan-s.el
@@ -18,31 +18,31 @@
 	 (,scan
 	  str p q
 	  ([" \t"] (ew-add-frag res r p 'ew:wsp-tok))
-	  (?< (ew-add-token res r p 'ew:lt-tok))
-	  (?> (ew-add-token res r p 'ew:gt-tok))
-	  (?@ (ew-add-token res r p 'ew:at-tok))
-	  (?, (ew-add-token res r p 'ew:comma-tok))
-	  (?\; (ew-add-token res r p 'ew:semicolon-tok))
-	  (?: (ew-add-token res r p 'ew:colon-tok))
-	  (?. (ew-add-token res r p 'ew:dot-tok))
+	  (?< (ew-add-frag res r p 'ew:lt-tok))
+	  (?> (ew-add-frag res r p 'ew:gt-tok))
+	  (?@ (ew-add-frag res r p 'ew:at-tok))
+	  (?, (ew-add-frag res r p 'ew:comma-tok))
+	  (?\; (ew-add-frag res r p 'ew:semicolon-tok))
+	  (?: (ew-add-frag res r p 'ew:colon-tok))
+	  (?. (ew-add-frag res r p 'ew:dot-tok))
 	  ((?\r ?\n [" \t"])
 	   (ew-add-frag res r p 'ew:fold-tok))
 	  ((?\r ?\n [^ " \t"])
 	   (ew-add-frag res r (setq p q) 'ew:err-tok))
 	  ((+ [(?a ?z) (?A ?Z) (?0 ?9) "!#$%&'*+-/=?^_`{|}~" non-ascii])
-	   (ew-add-token res r p 'ew:atom-tok))
-	  (?\" (ew-add-open res r p 'ew:qs-begin-tok)
+	   (ew-add-frag res r p 'ew:atom-tok))
+	  (?\" (ew-add-frag res r p 'ew:qs-begin-tok)
 	       (setq mode 'quoted-string))
-	  (?\[ (ew-add-open res r p 'ew:dl-begin-tok)
+	  (?\[ (ew-add-frag res r p 'ew:dl-begin-tok)
 	       (setq mode 'domain-literal))
-	  (?\( (ew-add-open res r p 'ew:cm-begin-tok)
+	  (?\( (ew-add-frag res r p 'ew:cm-begin-tok)
 	       (setq mode 'comment
 		     nest 1))
 	  (() (ew-add-frag res r q 'ew:err-tok) (setq p q))))
 	((eq mode 'quoted-string)
 	 (,scan
 	  str p q
-	  (?\" (ew-add-close-token res r p 'ew:qs-end-tok)
+	  (?\" (ew-add-frag res r p 'ew:qs-end-tok)
 	       (setq mode 'token))
 	  ((?\\ ?\r ?\n [" \t"])
 	   (ew-add-frag res r p 'ew:qs-qfold-tok))
@@ -71,7 +71,7 @@
 	((eq mode 'domain-literal)
 	 (,scan
 	  str p q
-	  (?\] (ew-add-close-token res r p 'ew:dl-end-tok)
+	  (?\] (ew-add-frag res r p 'ew:dl-end-tok)
 	       (setq mode 'token))
 	  ((?\\ ?\r ?\n [" \t"])
 	   (ew-add-frag res r p 'ew:dl-qfold-tok))
@@ -100,14 +100,14 @@
 	((eq mode 'comment)
 	 (,scan
 	  str p q
-	  (?\( (ew-add-open res r p 'ew:cm-nested-begin-tok)
+	  (?\( (ew-add-frag res r p 'ew:cm-nested-begin-tok)
 	       (setq nest (1+ nest)))
 	  (?\) (setq nest (1- nest))
 	       (if (zerop nest)
 		   (progn
-		     (ew-add-close res r p 'ew:cm-end-tok)
+		     (ew-add-frag res r p 'ew:cm-end-tok)
 		     (setq mode 'token))
-		 (ew-add-close res r p 'ew:cm-nested-end-tok)))
+		 (ew-add-frag res r p 'ew:cm-nested-end-tok)))
 	  ((?\\ ?\r ?\n [" \t"])
 	   (ew-add-frag res r p 'ew:cm-qfold-tok))
 	  ((?\\ ?\r ?\n [^ " \t"])
-- 
1.7.10.4