1 ;;; eword-decode.el --- RFC 2047 based encoded-word decoder for GNU Emacs
3 ;; Copyright (C) 1995,1996,1997,1998 Free Software Foundation, Inc.
5 ;; Author: ENAMI Tsugutomo <enami@sys.ptg.sony.co.jp>
6 ;; MORIOKA Tomohiko <morioka@jaist.ac.jp>
7 ;; Tanaka Akira <akr@jaist.ac.jp>
8 ;; Maintainer: Tanaka Akira <akr@jaist.ac.jp>
10 ;; Original: 1992/07/20 ENAMI Tsugutomo's `mime.el'.
11 ;; Renamed: 1993/06/03 to tiny-mime.el
12 ;; Renamed: 1995/10/03 from tiny-mime.el (split off encoder)
13 ;; Renamed: 1997/02/22 from tm-ew-d.el
14 ;; Keywords: encoded-word, MIME, multilingual, header, mail, news
16 ;; This file is part of FLAM (Faithful Library About MIME).
18 ;; This program is free software; you can redistribute it and/or
19 ;; modify it under the terms of the GNU General Public License as
20 ;; published by the Free Software Foundation; either version 2, or (at
21 ;; your option) any later version.
23 ;; This program is distributed in the hope that it will be useful, but
24 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
25 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
26 ;; General Public License for more details.
28 ;; You should have received a copy of the GNU General Public License
29 ;; along with GNU Emacs; see the file COPYING. If not, write to the
30 ;; Free Software Foundation, Inc., 59 Temple Place - Suite 330,
31 ;; Boston, MA 02111-1307, USA.
35 (require 'std11-parse)
39 (defgroup eword-decode nil
40 "Encoded-word decoding"
47 (defcustom eword-decode-sticked-encoded-word nil
48 "*If non-nil, decode encoded-words sticked on atoms,
49 other encoded-words, etc.
50 however this behaviour violates RFC2047."
54 (defcustom eword-decode-quoted-encoded-word nil
55 "*If non-nil, decode encoded-words in quoted-string
56 however this behaviour violates RFC2047."
61 ;;; @ MIME encoded-word definition
64 (defconst eword-encoded-word-prefix-regexp
65 (concat (regexp-quote "=?")
66 "\\(" mime-charset-regexp "\\)"
70 (defconst eword-encoded-word-suffix-regexp
73 (defconst eword-encoded-text-in-unstructured-regexp "[!->@-~]+")
74 (defconst eword-encoded-word-in-unstructured-regexp
75 (concat eword-encoded-word-prefix-regexp
76 "\\(" eword-encoded-text-in-unstructured-regexp "\\)"
77 eword-encoded-word-suffix-regexp))
78 (defconst eword-after-encoded-word-in-unstructured-regexp "\\([ \t]\\|$\\)")
80 (defconst eword-encoded-text-in-phrase-regexp "[-A-Za-z0-9!*+/=_]+")
81 (defconst eword-encoded-word-in-phrase-regexp
82 (concat eword-encoded-word-prefix-regexp
83 "\\(" eword-encoded-text-in-phrase-regexp "\\)"
84 eword-encoded-word-suffix-regexp))
85 (defconst eword-after-encoded-word-in-phrase-regexp "\\([ \t(]\\|$\\)")
87 (defconst eword-encoded-text-in-comment-regexp "[]!-'*->@-[^-~]+")
88 (defconst eword-encoded-word-in-comment-regexp
89 (concat eword-encoded-word-prefix-regexp
90 "\\(" eword-encoded-text-in-comment-regexp "\\)"
91 eword-encoded-word-suffix-regexp))
92 (defconst eword-after-encoded-word-in-comment-regexp "\\([ \t()\\\\]\\|$\\)")
94 (defconst eword-encoded-text-in-quoted-string-regexp "[]!#->@-[^-~]+")
95 (defconst eword-encoded-word-in-quoted-string-regexp
96 (concat eword-encoded-word-prefix-regexp
97 "\\(" eword-encoded-text-in-quoted-string-regexp "\\)"
98 eword-encoded-word-suffix-regexp))
99 (defconst eword-after-encoded-word-in-quoted-string-regexp "\\([ \t\"\\\\]\\|$\\)")
102 (defconst eword-encoded-text-regexp eword-encoded-text-in-unstructured-regexp)
103 (defconst eword-encoded-word-regexp eword-encoded-word-in-unstructured-regexp)
109 (defconst base64-token-regexp "[A-Za-z0-9+/]")
110 (defconst base64-token-padding-regexp "[A-Za-z0-9+/=]")
112 (defconst eword-B-encoded-text-regexp
121 base64-token-padding-regexp
122 base64-token-padding-regexp
125 ;; (defconst eword-B-encoding-and-encoded-text-regexp
126 ;; (concat "\\(B\\)\\?" eword-B-encoded-text-regexp))
129 ;;; @@ Quoted-Printable
132 (defconst eword-Q-encoded-text-regexp
133 (concat "\\([^=?]\\|" quoted-printable-octet-regexp "\\)+"))
134 ;; (defconst eword-Q-encoding-and-encoded-text-regexp
135 ;; (concat "\\(Q\\)\\?" eword-Q-encoded-text-regexp))
138 ;;; @ internal utilities
141 (defun eword-decode-first-encoded-words (string
144 &optional must-unfold)
145 "Decode MIME encoded-words in beginning of STRING.
147 EWORD-REGEXP is the regexp that matches a encoded-word.
149 eword-encoded-word-in-unstructured-regexp,
150 eword-encoded-text-in-phrase-regexp,
151 eword-encoded-word-in-comment-regexp or
152 eword-encoded-word-in-quoted-string-regexp.
154 AFTER-REGEXP is the regexp that matches a after encoded-word.
156 eword-after-encoded-word-in-unstructured-regexp,
157 eword-after-encoded-text-in-phrase-regexp,
158 eword-after-encoded-word-in-comment-regexp or
159 eword-after-encoded-word-in-quoted-string-regexp.
161 If beginning of STRING matches EWORD-REGEXP with AFTER-REGEXP,
162 returns a cons cell of decoded string(sequence of characters) and
163 the rest(sequence of octets).
165 If beginning of STRING does not matches EWORD-REGEXP and AFTER-REGEXP,
168 If an encoded-word is broken or your emacs implementation can not
169 decode the charset included in it, it is returned in decoded part
170 as encoded-word form.
172 If MUST-UNFOLD is non-nil, it unfolds and eliminates line-breaks even
173 if there are in decoded encoded-words (generated by bad manner MUA
174 such as a version of Net$cape)."
175 (if eword-decode-sticked-encoded-word (setq after-regexp ""))
176 (let* ((between-ewords-regexp
177 (if eword-decode-sticked-encoded-word
180 (between-ewords-eword-after-regexp
181 (concat "\\`\\(" between-ewords-regexp "\\)"
182 "\\(" eword-regexp "\\)"
185 (concat "\\`\\(" eword-regexp "\\)" after-regexp))
186 (src string) ; sequence of octets.
187 (dst "")) ; sequence of characters.
188 (if (string-match eword-after-regexp src)
191 (ew (substring src 0 q))
192 (dw (eword-decode-encoded-word ew must-unfold)))
193 (setq dst (concat dst dw)
194 src (substring src q))
195 (if (not (string= ew dw))
199 (string-match between-ewords-eword-after-regexp src)
201 (setq p (match-end 1)
203 ew (substring src p q)
204 dw (eword-decode-encoded-word ew must-unfold))
207 (setq dst (concat dst (substring src 0 q))
208 src (substring src q))
211 (setq dst (concat dst dw)
212 src (substring src q)))))
216 (defun eword-decode-entire-string (string
221 delimiters ; list of chars.
224 (if (and code-conversion
225 (not (mime-charset-to-coding-system code-conversion)))
226 (setq code-conversion default-mime-charset))
227 (let ((equal-safe-regexp (concat "\\`=?" safe-regexp))
232 (while (< 0 (length src))
233 (let ((ch (aref src 0))
236 (eword-decode-first-encoded-words src
237 eword-regexp after-regexp must-unfold))))
238 (if (and (not (string= buf ""))
239 (or decoded (memq ch delimiters)))
240 (setq dst (concat dst
241 (std11-wrap-as-quoted-pairs
242 (decode-mime-charset-string buf code-conversion)
247 (setq dst (concat dst
248 (std11-wrap-as-quoted-pairs
252 ((memq ch delimiters)
253 (setq dst (concat dst (list ch))
254 src (substring src 1)
257 (setq buf (concat buf (list (aref src 1)))
258 src (substring src 2)
260 ((string-match "\\`[ \t\n]+" src)
261 (setq buf (concat buf (substring src 0 (match-end 0)))
262 src (substring src (match-end 0))
264 ((and (string-match equal-safe-regexp src)
266 (setq buf (concat buf (substring src 0 (match-end 0)))
267 src (substring src (match-end 0))
268 ew-enable eword-decode-sticked-encoded-word))
269 (t (error "something wrong")))))
270 (if (not (string= buf ""))
271 (setq dst (concat dst
272 (std11-wrap-as-quoted-pairs
273 (decode-mime-charset-string buf code-conversion)
281 (defun eword-decode-unstructured (string code-conversion &optional must-unfold)
282 (eword-decode-entire-string
284 eword-encoded-word-in-unstructured-regexp
285 eword-after-encoded-word-in-unstructured-regexp
292 (defun eword-decode-comment (string code-conversion &optional must-unfold)
293 (eword-decode-entire-string
295 eword-encoded-word-in-comment-regexp
296 eword-after-encoded-word-in-comment-regexp
303 (defun eword-decode-quoted-string (string code-conversion &optional must-unfold)
304 (eword-decode-entire-string
306 eword-encoded-word-in-quoted-string-regexp
307 eword-after-encoded-word-in-quoted-string-regexp
314 (defun eword-decode-string (string &optional must-unfold code-conversion)
315 "Decode MIME encoded-words in STRING.
317 STRING is unfolded before decoding.
319 If an encoded-word is broken or your emacs implementation can not
320 decode the charset included in it, it is not decoded.
322 If MUST-UNFOLD is non-nil, it unfolds and eliminates line-breaks even
323 if there are in decoded encoded-words (generated by bad manner MUA
324 such as a version of Net$cape).
326 If CODE-CONVERSION is nil, it decodes only encoded-words. If it is
327 mime-charset, it decodes non-ASCII bit patterns as the mime-charset.
328 Otherwise it decodes non-ASCII bit patterns as the
329 default-mime-charset."
330 (eword-decode-unstructured
331 (std11-unfold-string string)
339 (defun eword-decode-region (start end &optional unfolding must-unfold
341 "Decode MIME encoded-words in region between START and END.
343 If UNFOLDING is not nil, it unfolds before decoding.
345 If MUST-UNFOLD is non-nil, it unfolds and eliminates line-breaks even
346 if there are in decoded encoded-words (generated by bad manner MUA
347 such as a version of Net$cape).
349 If CODE-CONVERSION is nil, it decodes only encoded-words. If it is
350 mime-charset, it decodes non-ASCII bit patterns as the mime-charset.
351 Otherwise it decodes non-ASCII bit patterns as the
352 default-mime-charset."
356 (narrow-to-region start end)
358 (eword-decode-unfold)
360 (let ((str (eword-decode-unstructured
361 (buffer-substring (point-min) (point-max))
364 (delete-region (point-min) (point-max))
368 ;;; @ for message header
371 (defcustom eword-decode-ignored-field-list
372 '(newsgroups path lines nntp-posting-host received message-id date)
373 "*List of field-names to be ignored when decoding.
374 Each field name must be symbol."
376 :type '(repeat symbol))
378 (defcustom eword-decode-structured-field-list
379 '(reply-to resent-reply-to from resent-from sender resent-sender
380 to resent-to cc resent-cc bcc resent-bcc dcc
381 mime-version content-type content-transfer-encoding
383 "*List of field-names to decode as structured field.
384 Each field name must be symbol."
386 :type '(repeat symbol))
388 (defun eword-decode-header (&optional code-conversion separator)
389 "Decode MIME encoded-words in header fields.
390 If CODE-CONVERSION is nil, it decodes only encoded-words. If it is
391 mime-charset, it decodes non-ASCII bit patterns as the mime-charset.
392 Otherwise it decodes non-ASCII bit patterns as the
393 default-mime-charset.
394 If SEPARATOR is not nil, it is used as header separator."
396 (if (and code-conversion
397 (not (mime-charset-to-coding-system code-conversion)))
398 (setq code-conversion default-mime-charset))
401 (std11-narrow-to-header separator)
403 (let (beg p end field-name len)
404 (goto-char (point-min))
405 (while (re-search-forward std11-field-head-regexp nil t)
406 (setq beg (match-beginning 0)
408 field-name (buffer-substring beg (1- p))
409 len (string-width field-name)
410 field-name (intern (downcase field-name))
411 end (std11-field-end))
412 (cond ((memq field-name eword-decode-ignored-field-list)
415 ((memq field-name eword-decode-structured-field-list)
416 ;; Decode as structured field
417 (let ((body (buffer-substring p end)))
418 (delete-region p end)
419 (insert (eword-decode-and-fold-structured-field
423 ;; Decode as unstructured field
425 (narrow-to-region beg (1+ end))
427 (eword-decode-region beg (point-max) 'unfold nil
429 (goto-char (point-max))
431 (eword-decode-region (point-min) (point-max) t nil nil)
434 (defun eword-decode-unfold ()
435 (goto-char (point-min))
437 (while (re-search-forward std11-field-head-regexp nil t)
438 (setq beg (match-beginning 0)
439 end (std11-field-end))
440 (setq field (buffer-substring beg end))
441 (if (string-match eword-encoded-word-regexp field)
443 (narrow-to-region (goto-char beg) end)
444 (while (re-search-forward "\n\\([ \t]\\)" nil t)
445 (replace-match (match-string 1))
447 (goto-char (point-max))
452 ;;; @ encoded-word decoder
455 (defvar eword-warning-face nil "Face used for invalid encoded-word.")
457 (defun eword-decode-encoded-word (word &optional must-unfold)
458 "Decode WORD if it is an encoded-word.
460 If your emacs implementation can not decode the charset of WORD, it
461 returns WORD. Similarly the encoded-word is broken, it returns WORD.
463 If MUST-UNFOLD is non-nil, it unfolds and eliminates line-breaks even
464 if there are in decoded encoded-word (generated by bad manner MUA such
465 as a version of Net$cape)."
466 (or (if (string-match eword-encoded-word-regexp word)
468 (substring word (match-beginning 1) (match-end 1))
472 (substring word (match-beginning 2) (match-end 2))
475 (substring word (match-beginning 3) (match-end 3))
478 (eword-decode-encoded-text charset encoding text must-unfold)
481 (add-text-properties 0 (length word)
482 (and eword-warning-face
483 (list 'face eword-warning-face))
490 ;;; @ encoded-text decoder
493 (defun eword-decode-encoded-text (charset encoding string
494 &optional must-unfold)
495 "Decode STRING as an encoded-text.
497 If your emacs implementation can not decode CHARSET, it returns nil.
499 If ENCODING is not \"B\" or \"Q\", it occurs error.
500 So you should write error-handling code if you don't want break by errors.
502 If MUST-UNFOLD is non-nil, it unfolds and eliminates line-breaks even
503 if there are in decoded encoded-text (generated by bad manner MUA such
504 as a version of Net$cape)."
505 (let ((cs (mime-charset-to-coding-system charset)))
509 ((string-equal "B" encoding)
510 (if (and (string-match eword-B-encoded-text-regexp string)
511 (string-equal string (match-string 0 string)))
512 (base64-decode-string string)
513 (error "Invalid encoded-text %s" string)))
514 ((string-equal "Q" encoding)
515 (if (and (string-match eword-Q-encoded-text-regexp string)
516 (string-equal string (match-string 0 string)))
517 (q-encoding-decode-string string)
518 (error "Invalid encoded-text %s" string)))
520 (error "Invalid encoding %s" encoding)
525 (setq dest (decode-coding-string dest cs))
532 (t (char-to-string chr)))
534 (std11-unfold-string dest)
540 ;;; @ lexical analyze
543 (defvar eword-lexical-analyze-cache nil)
544 (defvar eword-lexical-analyze-cache-max 299
545 "*Max position of eword-lexical-analyze-cache.
546 It is max size of eword-lexical-analyze-cache - 1.")
548 (defcustom eword-lexical-analyzers
549 '(eword-analyze-quoted-string
550 eword-analyze-domain-literal
551 eword-analyze-comment
553 eword-analyze-special
554 eword-analyze-encoded-word
556 "*List of functions to return result of lexical analyze.
557 Each function must have two arguments: STRING and MUST-UNFOLD.
558 STRING is the target string to be analyzed.
559 If MUST-UNFOLD is not nil, each function must unfold and eliminate
560 bare-CR and bare-LF from the result even if they are included in
561 content of the encoded-word.
562 Each function must return nil if it can not analyze STRING as its
565 Previous function is preferred to next function. If a function
566 returns nil, next function is used. Otherwise the return value will
569 :type '(repeat function))
571 (defun eword-analyze-quoted-string (string &optional must-unfold)
572 (let ((p (std11-check-enclosure string ?\" ?\")))
574 (cons (cons 'quoted-string
575 (eword-decode-quoted-string
576 (substring string 0 p)
577 default-mime-charset))
578 (substring string p))
581 (defun eword-analyze-domain-literal (string &optional must-unfold)
582 (std11-analyze-domain-literal string))
584 (defun eword-analyze-comment (string &optional must-unfold)
585 (let ((len (length string)))
586 (if (and (< 0 len) (eq (aref string 0) ?\())
588 (while (and p (< p len) (eq (aref string p) ?\())
589 (setq p (std11-check-enclosure string ?\( ?\) t p)))
592 (eword-decode-comment
593 (substring string 0 p)
594 default-mime-charset))
595 (substring string p)))
598 (defun eword-analyze-spaces (string &optional must-unfold)
599 (std11-analyze-spaces string))
601 (defun eword-analyze-special (string &optional must-unfold)
602 (std11-analyze-special string))
604 (defun eword-analyze-encoded-word (string &optional must-unfold)
605 (let ((decoded (eword-decode-first-encoded-words
607 eword-encoded-word-in-phrase-regexp
608 eword-after-encoded-word-in-phrase-regexp
611 (cons (cons 'atom (car decoded)) (cdr decoded)))))
613 (defun eword-analyze-atom (string &optional must-unfold)
614 (if (let ((enable-multibyte-characters nil))
615 (string-match std11-atom-regexp string))
616 (let ((end (match-end 0)))
617 (if (and eword-decode-sticked-encoded-word
618 (string-match eword-encoded-word-in-phrase-regexp
619 (substring string 0 end))
620 (< 0 (match-beginning 0)))
621 (setq end (match-beginning 0)))
622 (cons (cons 'atom (decode-mime-charset-string
623 (substring string 0 end)
624 default-mime-charset))
625 (substring string end)
628 (defun eword-lexical-analyze-internal (string must-unfold)
630 (while (not (string-equal string ""))
632 (let ((rest eword-lexical-analyzers)
634 (while (and (setq func (car rest))
635 (null (setq r (funcall func string must-unfold)))
637 (setq rest (cdr rest)))
638 (or r `((error . ,string) . ""))
640 (setq dest (cons (car ret) dest))
641 (setq string (cdr ret))
646 (defun eword-lexical-analyze (string &optional must-unfold)
647 "Return lexical analyzed list corresponding STRING.
648 It is like std11-lexical-analyze, but it decodes non us-ascii
649 characters encoded as encoded-words or invalid \"raw\" format.
650 \"Raw\" non us-ascii characters are regarded as variable
651 `default-mime-charset'."
652 (let* ((str (copy-sequence string))
653 (key (cons str (cons default-mime-charset must-unfold)))
655 (set-text-properties 0 (length str) nil str)
656 (if (setq ret (assoc key eword-lexical-analyze-cache))
658 (setq ret (eword-lexical-analyze-internal str must-unfold))
659 (setq eword-lexical-analyze-cache
661 (last eword-lexical-analyze-cache
662 eword-lexical-analyze-cache-max)))
665 (defun eword-decode-token (token)
668 (defun eword-decode-and-fold-structured-field
669 (string start-column &optional max-column must-unfold)
670 "Decode and fold (fill) STRING as structured field body.
671 It decodes non us-ascii characters in FULL-NAME encoded as
672 encoded-words or invalid \"raw\" string. \"Raw\" non us-ascii
673 characters are regarded as variable `default-mime-charset'.
675 If an encoded-word is broken or your emacs implementation can not
676 decode the charset included in it, it is not decoded.
678 If MAX-COLUMN is omitted, `fill-column' is used.
680 If MUST-UNFOLD is non-nil, it unfolds and eliminates line-breaks even
681 if there are in decoded encoded-words (generated by bad manner MUA
682 such as a version of Net$cape)."
684 (setq max-column fill-column))
685 (let ((c start-column)
686 (tokens (eword-lexical-analyze string must-unfold))
689 (while (and (setq token (car tokens))
690 (setq tokens (cdr tokens)))
691 (let* ((type (car token)))
692 (if (eq type 'spaces)
693 (let* ((next-token (car tokens))
694 (next-str (eword-decode-token next-token))
695 (next-len (string-width next-str))
696 (next-c (+ c next-len 1)))
697 (if (< next-c max-column)
698 (setq result (concat result " " next-str)
700 (setq result (concat result "\n " next-str)
702 (setq tokens (cdr tokens))
704 (let* ((str (eword-decode-token token)))
705 (setq result (concat result str)
706 c (+ c (string-width str)))
709 (concat result (eword-decode-token token))
712 (defun eword-decode-and-unfold-structured-field (string)
713 "Decode and unfold STRING as structured field body.
714 It decodes non us-ascii characters in FULL-NAME encoded as
715 encoded-words or invalid \"raw\" string. \"Raw\" non us-ascii
716 characters are regarded as variable `default-mime-charset'.
718 If an encoded-word is broken or your emacs implementation can not
719 decode the charset included in it, it is not decoded."
720 (let ((tokens (eword-lexical-analyze string 'must-unfold))
723 (let* ((token (car tokens))
725 (setq tokens (cdr tokens))
727 (if (eq type 'spaces)
729 (concat result (eword-decode-token token))
733 (defun eword-decode-structured-field-body (string &optional must-unfold
734 start-column max-column)
735 "Decode non us-ascii characters in STRING as structured field body.
736 STRING is unfolded before decoding.
738 It decodes non us-ascii characters in FULL-NAME encoded as
739 encoded-words or invalid \"raw\" string. \"Raw\" non us-ascii
740 characters are regarded as variable `default-mime-charset'.
742 If an encoded-word is broken or your emacs implementation can not
743 decode the charset included in it, it is not decoded.
745 If MUST-UNFOLD is non-nil, it unfolds and eliminates line-breaks even
746 if there are in decoded encoded-words (generated by bad manner MUA
747 such as a version of Net$cape)."
749 ;; fold with max-column
750 (eword-decode-and-fold-structured-field
751 string start-column max-column must-unfold)
753 (mapconcat (function eword-decode-token)
754 (eword-lexical-analyze string must-unfold)
758 (defun eword-decode-unstructured-field-body (string &optional must-unfold)
759 "Decode non us-ascii characters in STRING as unstructured field body.
760 STRING is unfolded before decoding.
762 It decodes non us-ascii characters in FULL-NAME encoded as
763 encoded-words or invalid \"raw\" string. \"Raw\" non us-ascii
764 characters are regarded as variable `default-mime-charset'.
766 If an encoded-word is broken or your emacs implementation can not
767 decode the charset included in it, it is not decoded.
769 If MUST-UNFOLD is non-nil, it unfolds and eliminates line-breaks even
770 if there are in decoded encoded-words (generated by bad manner MUA
771 such as a version of Net$cape)."
772 (eword-decode-string string must-unfold default-mime-charset))
774 (defun eword-extract-address-components (string)
775 "Extract full name and canonical address from STRING.
776 Returns a list of the form (FULL-NAME CANONICAL-ADDRESS).
777 If no name can be extracted, FULL-NAME will be nil.
778 It decodes non us-ascii characters in FULL-NAME encoded as
779 encoded-words or invalid \"raw\" string. \"Raw\" non us-ascii
780 characters are regarded as variable `default-mime-charset'."
781 (let* ((structure (car (std11-parse-address
782 (eword-lexical-analyze
783 (std11-unfold-string string) 'must-unfold))))
784 (phrase (std11-full-name-string structure))
785 (address (std11-address-string structure))
787 (list phrase address)
794 (provide 'eword-decode)
796 ;;; eword-decode.el ends here