1 ;;; eword-decode.el --- RFC 2047 based encoded-word decoder for GNU Emacs
3 ;; Copyright (C) 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2003, 2004,
4 ;; 2005 Free Software Foundation, Inc.
6 ;; Author: ENAMI Tsugutomo <enami@sys.ptg.sony.co.jp>
7 ;; MORIOKA Tomohiko <tomo@m17n.org>
8 ;; TANAKA Akira <akr@m17n.org>
10 ;; Original: 1992/07/20 ENAMI Tsugutomo's `mime.el'.
11 ;; Renamed: 1993/06/03 to tiny-mime.el by MORIOKA Tomohiko
12 ;; Renamed: 1995/10/03 to tm-ew-d.el (split off encoder)
13 ;; by MORIOKA Tomohiko
14 ;; Renamed: 1997/02/22 from tm-ew-d.el by MORIOKA Tomohiko
15 ;; Keywords: encoded-word, MIME, multilingual, header, mail, news
17 ;; This file is part of FLIM (Faithful Library about Internet Message).
19 ;; This program is free software; you can redistribute it and/or
20 ;; modify it under the terms of the GNU General Public License as
21 ;; published by the Free Software Foundation; either version 2, or (at
22 ;; your option) any later version.
24 ;; This program is distributed in the hope that it will be useful, but
25 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
26 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
27 ;; General Public License for more details.
29 ;; You should have received a copy of the GNU General Public License
30 ;; along with GNU Emacs; see the file COPYING. If not, write to the
31 ;; Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
32 ;; Boston, MA 02110-1301, USA.
40 (eval-when-compile (require 'cl)) ; list*, pop
46 ;; User options are defined in mime-def.el.
49 ;;; @ MIME encoded-word definition
53 (defconst eword-encoded-text-regexp "[!->@-~]+")
55 (defconst eword-encoded-word-regexp
57 (concat (regexp-quote "=?")
59 mime-charset-regexp ; 1
63 mime-language-regexp ; 2
67 mime-encoding-regexp ; 3
71 eword-encoded-text-regexp ; 4
73 (regexp-quote "?="))))
80 (defun eword-decode-string (string &optional must-unfold)
81 "Decode MIME encoded-words in STRING.
83 STRING is unfolded before decoding.
85 If an encoded-word is broken or your emacs implementation can not
86 decode the charset included in it, it is not decoded.
88 If MUST-UNFOLD is non-nil, it unfolds and eliminates line-breaks even
89 if there are in decoded encoded-words (generated by bad manner MUA
90 such as a version of Net$cape)."
91 (setq string (std11-unfold-string string))
92 (let ((regexp (concat "[\n\t ]*\\(" eword-encoded-word-regexp "\\)"))
95 (while (setq match (string-match regexp string next))
96 (setq start (match-beginning 1)
99 (setq next (match-end 0))
100 (push (list (match-string 2 string) ;; charset
101 (match-string 3 string) ;; language
102 (match-string 4 string) ;; encoding
103 (match-string 5 string) ;; encoded-text
104 (match-string 1 string)) ;; encoded-word
106 (setq match (and (string-match regexp string next)
107 (= next (match-beginning 0)))))
108 (setq words (eword-decode-encoded-words (nreverse words) must-unfold)
109 string (concat (substring string 0 start)
111 (substring string next))
112 next (+ start (length words)))))
115 (defun eword-decode-structured-field-body (string
116 &optional start-column max-column
118 (let ((tokens (eword-lexical-analyze string start 'must-unfold))
122 (setq token (car tokens))
123 (setq result (concat result (eword-decode-token token)))
124 (setq tokens (cdr tokens)))
127 (defun eword-decode-and-unfold-structured-field-body (string
132 "Decode and unfold STRING as structured field body.
133 It decodes non us-ascii characters in FULL-NAME encoded as
134 encoded-words or invalid \"raw\" string. \"Raw\" non us-ascii
135 characters are regarded as variable `default-mime-charset'.
137 If an encoded-word is broken or your emacs implementation can not
138 decode the charset included in it, it is not decoded."
139 (let ((tokens (eword-lexical-analyze string start 'must-unfold))
142 (let* ((token (car tokens))
144 (setq tokens (cdr tokens))
146 (if (eq type 'spaces)
148 (concat result (eword-decode-token token))
152 (defun eword-decode-and-fold-structured-field-body (string
156 (if (and mime-field-decoding-max-size
157 (> (length string) mime-field-decoding-max-size))
160 (setq max-column fill-column))
161 (let ((c start-column)
162 (tokens (eword-lexical-analyze string start 'must-unfold))
165 (while (and (setq token (car tokens))
166 (setq tokens (cdr tokens)))
167 (let* ((type (car token)))
168 (if (eq type 'spaces)
169 (let* ((next-token (car tokens))
170 (next-str (eword-decode-token next-token))
171 (next-len (string-width next-str))
172 (next-c (+ c next-len 1)))
173 (if (< next-c max-column)
174 (setq result (concat result " " next-str)
176 (setq result (concat result "\n " next-str)
178 (setq tokens (cdr tokens))
180 (let* ((str (eword-decode-token token)))
181 (setq result (concat result str)
182 c (+ c (string-width str)))
185 (concat result (eword-decode-token token))
188 (defun eword-decode-unstructured-field-body (string &optional start-column
191 (decode-mime-charset-string string default-mime-charset)))
193 (defun eword-decode-and-unfold-unstructured-field-body (string
194 &optional start-column
197 (decode-mime-charset-string (std11-unfold-string string)
198 default-mime-charset)
201 (defun eword-decode-unfolded-unstructured-field-body (string
202 &optional start-column
205 (decode-mime-charset-string string default-mime-charset)
212 (defun eword-decode-region (start end &optional unfolding must-unfold)
213 "Decode MIME encoded-words in region between START and END.
215 If UNFOLDING is not nil, it unfolds before decoding.
217 If MUST-UNFOLD is non-nil, it unfolds and eliminates line-breaks even
218 if there are in decoded encoded-words (generated by bad manner MUA
219 such as a version of Net$cape)."
223 (narrow-to-region start end)
225 (eword-decode-unfold))
226 (goto-char (point-min))
227 (let ((regexp (concat "[\n\t ]*\\(" eword-encoded-word-regexp "\\)"))
229 (while (setq match (re-search-forward regexp nil t))
230 (setq start (match-beginning 1)
233 (goto-char (setq end (match-end 0)))
234 (push (list (match-string 2) ;; charset
235 (match-string 3) ;; language
236 (match-string 4) ;; encoding
237 (match-string 5) ;; encoded-text
238 (match-string 1)) ;; encoded-word
240 (setq match (looking-at regexp)))
241 (delete-region start end)
243 (eword-decode-encoded-words (nreverse words) must-unfold)))))))
245 (defun eword-decode-unfold ()
246 (goto-char (point-min))
248 (while (re-search-forward std11-field-head-regexp nil t)
249 (setq beg (match-beginning 0)
250 end (std11-field-end))
251 (setq field (buffer-substring beg end))
252 (if (string-match eword-encoded-word-regexp field)
254 (narrow-to-region (goto-char beg) end)
255 (while (re-search-forward "\n\\([ \t]\\)" nil t)
256 (replace-match (match-string 1))
258 (goto-char (point-max))
263 ;;; @ for message header
266 (defvar mime-field-decoder-alist nil)
268 (defvar mime-field-decoder-cache nil)
270 (defvar mime-update-field-decoder-cache 'mime-update-field-decoder-cache
271 "*Field decoder cache update function.")
274 (defun mime-set-field-decoder (field &rest specs)
275 "Set decoder of FIELD.
276 SPECS must be like `MODE1 DECODER1 MODE2 DECODER2 ...'.
277 Each mode must be `nil', `plain', `wide', `summary' or `nov'.
278 If mode is `nil', corresponding decoder is set up for every modes."
280 (let ((mode (pop specs))
281 (function (pop specs)))
284 (let ((cell (assq mode mime-field-decoder-alist)))
286 (setcdr cell (put-alist field function (cdr cell)))
287 (setq mime-field-decoder-alist
288 (cons (cons mode (list (cons field function)))
289 mime-field-decoder-alist))
291 (apply (function mime-set-field-decoder) field specs)
293 (mime-set-field-decoder field
301 (defmacro mime-find-field-presentation-method (name)
302 "Return field-presentation-method from NAME.
303 NAME must be `plain', `wide', `summary' or `nov'."
305 `(or (assq 'summary mime-field-decoder-cache)
311 (symbolp (car (cdr name)))
312 (null (cdr (cdr name))))
313 `(or (assq ,name mime-field-decoder-cache)
317 `(or (assq (or ,name 'summary) mime-field-decoder-cache)
318 (cons (or ,name 'summary) nil))
321 (defun mime-find-field-decoder-internal (field &optional mode)
322 "Return function to decode field-body of FIELD in MODE.
323 Optional argument MODE must be object of field-presentation-method."
324 (cdr (or (assq field (cdr mode))
326 (funcall mime-update-field-decoder-cache
329 (cdr (assq (car mode) mime-field-decoder-cache)))
333 (defun mime-find-field-decoder (field &optional mode)
334 "Return function to decode field-body of FIELD in MODE.
335 Optional argument MODE must be object or name of
336 field-presentation-method. Name of field-presentation-method must be
337 `plain', `wide', `summary' or `nov'.
338 Default value of MODE is `summary'."
340 (let ((p (cdr (mime-find-field-presentation-method mode))))
341 (if (and p (setq p (assq field p)))
343 (cdr (funcall mime-update-field-decoder-cache
344 field (or mode 'summary)))))
345 (inline (mime-find-field-decoder-internal field mode))
349 (defun mime-update-field-decoder-cache (field mode &optional function)
350 "Update field decoder cache `mime-field-decoder-cache'."
351 (cond ((eq function 'identity)
356 (cdr (assq (or mode 'summary) mime-field-decoder-alist))))
357 (setq function (cdr (or (assq field decoder-alist)
358 (assq t decoder-alist)))))
360 (let ((cell (assq mode mime-field-decoder-cache))
363 (if (setq ret (assq field (cdr cell)))
364 (setcdr ret function)
365 (setcdr cell (cons (setq ret (cons field function)) (cdr cell))))
366 (setq mime-field-decoder-cache
367 (cons (cons mode (list (setq ret (cons field function))))
368 mime-field-decoder-cache)))
372 (mime-set-field-decoder 'Archive nil nil)
373 (mime-set-field-decoder 'Content-Md5 nil nil)
374 (mime-set-field-decoder 'Control nil nil)
375 (mime-set-field-decoder 'Date nil nil)
376 (mime-set-field-decoder 'Distribution nil nil)
377 (mime-set-field-decoder 'Followup-Host nil nil)
378 (mime-set-field-decoder 'Followup-To nil nil)
379 (mime-set-field-decoder 'Lines nil nil)
380 (mime-set-field-decoder 'Message-Id nil nil)
381 (mime-set-field-decoder 'Newsgroups nil nil)
382 (mime-set-field-decoder 'Nntp-Posting-Host nil nil)
383 (mime-set-field-decoder 'Path nil nil)
384 (mime-set-field-decoder 'Posted-And-Mailed nil nil)
385 (mime-set-field-decoder 'Received nil nil)
386 (mime-set-field-decoder 'Status nil nil)
387 (mime-set-field-decoder 'X-Face nil nil)
388 (mime-set-field-decoder 'X-Face-Version nil nil)
389 (mime-set-field-decoder 'X-Info nil nil)
390 (mime-set-field-decoder 'X-Pgp-Key-Info nil nil)
391 (mime-set-field-decoder 'X-Pgp-Sig nil nil)
392 (mime-set-field-decoder 'X-Pgp-Sig-Version nil nil)
393 (mime-set-field-decoder 'Xref nil nil)
397 '(Reply-To Resent-Reply-To From Resent-From Sender Resent-Sender
398 To Resent-To Cc Resent-Cc Bcc Resent-Bcc Dcc
400 Mime-Version Content-Type Content-Transfer-Encoding
401 Content-Disposition User-Agent))
404 (setq field (pop fields))
405 (mime-set-field-decoder
407 'plain #'eword-decode-structured-field-body
408 'wide #'eword-decode-and-fold-structured-field-body
409 'summary #'eword-decode-and-unfold-structured-field-body
410 'nov #'eword-decode-and-unfold-structured-field-body)
413 ;; unstructured fields (default)
414 (mime-set-field-decoder
416 'plain #'eword-decode-unstructured-field-body
417 'wide #'eword-decode-unstructured-field-body
418 'summary #'eword-decode-and-unfold-unstructured-field-body
419 'nov #'eword-decode-unfolded-unstructured-field-body)
422 (defun mime-decode-field-body (field-body field-name
423 &optional mode max-column)
424 "Decode FIELD-BODY as FIELD-NAME in MODE, and return the result.
425 Optional argument MODE must be `plain', `wide', `summary' or `nov'.
426 Default mode is `summary'.
428 If MODE is `wide' and MAX-COLUMN is non-nil, the result is folded with
431 Non MIME encoded-word part in FILED-BODY is decoded with
432 `default-mime-charset'."
433 (let (field-name-symbol len decoder)
434 (if (symbolp field-name)
435 (setq field-name-symbol field-name
436 len (1+ (string-width (symbol-name field-name))))
437 (setq field-name-symbol (intern (capitalize field-name))
438 len (1+ (string-width field-name))))
439 (setq decoder (mime-find-field-decoder field-name-symbol mode))
441 (funcall decoder field-body len max-column)
443 (if (eq mode 'summary)
444 (std11-unfold-string field-body)
449 (defun mime-decode-header-in-region (start end
450 &optional code-conversion)
451 "Decode MIME encoded-words in region between START and END.
452 If CODE-CONVERSION is nil, it decodes only encoded-words. If it is
453 mime-charset, it decodes non-ASCII bit patterns as the mime-charset.
454 Otherwise it decodes non-ASCII bit patterns as the
455 default-mime-charset."
459 (narrow-to-region start end)
460 (let ((default-charset
462 (if (mime-charset-to-coding-system code-conversion)
464 default-mime-charset))))
466 (let ((mode-obj (mime-find-field-presentation-method 'wide))
467 beg p end field-name len field-decoder)
468 (goto-char (point-min))
469 (while (re-search-forward std11-field-head-regexp nil t)
470 (setq beg (match-beginning 0)
472 field-name (buffer-substring beg (1- p))
473 len (string-width field-name)
474 field-name (intern (capitalize field-name))
475 field-decoder (inline
476 (mime-find-field-decoder-internal
477 field-name mode-obj)))
479 (setq end (std11-field-end))
480 (let ((body (buffer-substring p end))
481 (default-mime-charset default-charset))
482 (delete-region p end)
483 (insert (funcall field-decoder body (1+ len)))
486 (eword-decode-region (point-min) (point-max) t)
490 (defun mime-decode-header-in-buffer (&optional code-conversion separator)
491 "Decode MIME encoded-words in header fields.
492 If CODE-CONVERSION is nil, it decodes only encoded-words. If it is
493 mime-charset, it decodes non-ASCII bit patterns as the mime-charset.
494 Otherwise it decodes non-ASCII bit patterns as the
495 default-mime-charset.
496 If SEPARATOR is not nil, it is used as header separator."
498 (mime-decode-header-in-region
501 (goto-char (point-min))
502 (if (re-search-forward
503 (concat "^\\(" (regexp-quote (or separator "")) "\\)?$")
510 (defalias 'eword-decode-header 'mime-decode-header-in-buffer)
511 (make-obsolete 'eword-decode-header 'mime-decode-header-in-buffer)
514 ;;; @ encoded-words decoder
517 (defvar eword-decode-allow-incomplete-encoded-text t
518 "*Non-nil means allow incomplete encoded-text in successive encoded-words.
519 Dividing of encoded-text in the place other than character boundaries
520 violates RFC2047 section 5, while we have a capability to decode it.
521 If it is non-nil, the decoder will decode B- or Q-encoding in each
522 encoded-word, concatenate them, and decode it by charset. Otherwise,
523 the decoder will fully decode each encoded-word before concatenating
526 (defun eword-decode-encoded-words (words must-unfold)
527 "Decode successive encoded-words in WORDS and return a decoded string.
528 Each element of WORDS looks like (CHARSET LANGUAGE ENCODING ENCODED-TEXT
531 If MUST-UNFOLD is non-nil, it unfolds and eliminates line-breaks even
532 if there are in decoded encoded-words (generated by bad manner MUA
533 such as a version of Net$cape)."
534 (let (word language charset encoding text rest)
536 (setq word (pop words)
537 language (nth 1 word))
538 (if (and (or (mime-charset-to-coding-system (setq charset (car word)))
540 (message "Unknown charset: %s" charset)
542 (cond ((member (setq encoding (nth 2 word)) '("B" "Q"))
544 ((member encoding '("b" "q"))
545 (setq encoding (upcase encoding)))
547 (message "Invalid encoding: %s" encoding)
551 (encoded-text-decode-string (nth 3 word) encoding))
553 (message "%s" (error-message-string err))
555 (if (and eword-decode-allow-incomplete-encoded-text
558 (string-equal (downcase charset) (downcase (caaar rest)))
559 (equal language (cdaar rest)))
560 ;; Concatenate text of which the charset is the same.
561 (setcdr (car rest) (concat (cdar rest) text))
562 (push (cons (cons charset language) text) rest))
563 ;; Don't decode encoded-word.
564 (push (cons (cons nil language) (nth 4 word)) rest)))
566 (setq word (or (and (setq charset (caaar rest))
568 (decode-mime-charset-string (cdar rest) charset)
570 (message "%s" (error-message-string err))
572 (concat (when (cdr rest) " ")
575 (not (eq (string-to-char words) ? )))
578 (setq word (mapconcat (lambda (chr)
579 (cond ((eq chr ?\n) "")
582 (t (char-to-string chr))))
583 (std11-unfold-string word)
585 (when (setq language (cdaar rest))
586 (put-text-property 0 (length word) 'mime-language language word))
587 (setq words (concat word words)
591 ;;; @ lexical analyze
594 (defvar eword-lexical-analyze-cache nil)
595 (defvar eword-lexical-analyze-cache-max 299
596 "*Max position of eword-lexical-analyze-cache.
597 It is max size of eword-lexical-analyze-cache - 1.")
599 (defvar mime-header-lexical-analyzer
600 '(eword-analyze-quoted-string
601 eword-analyze-domain-literal
602 eword-analyze-comment
604 eword-analyze-special
605 eword-analyze-encoded-word
607 "*List of functions to return result of lexical analyze.
608 Each function must have three arguments: STRING, START and MUST-UNFOLD.
609 STRING is the target string to be analyzed.
610 START is start position of STRING to analyze.
611 If MUST-UNFOLD is not nil, each function must unfold and eliminate
612 bare-CR and bare-LF from the result even if they are included in
613 content of the encoded-word.
614 Each function must return nil if it can not analyze STRING as its
617 Previous function is preferred to next function. If a function
618 returns nil, next function is used. Otherwise the return value will
621 (defun eword-analyze-quoted-string (string start &optional must-unfold)
622 (let ((p (std11-check-enclosure string ?\" ?\" nil start))
625 (setq ret (decode-mime-charset-string
626 (std11-strip-quoted-pair
627 (substring string (1+ start) (1- p)))
628 default-mime-charset))
629 (if mime-header-accept-quoted-encoded-words
630 (setq ret (eword-decode-string ret)))
631 (cons (cons 'quoted-string ret)
634 (defun eword-analyze-domain-literal (string start &optional must-unfold)
635 (std11-analyze-domain-literal string start))
637 (defun eword-analyze-comment (string from &optional must-unfold)
638 (let ((len (length string))
643 (eq (aref string i) ?\())
648 (setq chr (aref string i))
654 (setq last-str (concat last-str
655 (substring string from (1- i))
656 (char-to-string (aref string i)))
661 (setq ret (concat last-str
662 (substring string from i)))
670 (decode-mime-charset-string
671 ret default-mime-charset)
678 (if (setq ret (eword-analyze-comment string i must-unfold))
681 (substring string from i))
683 (if (string= last-str "")
684 (cons (car ret) dest)
687 (decode-mime-charset-string
688 last-str default-mime-charset)
702 (defun eword-analyze-spaces (string start &optional must-unfold)
703 (std11-analyze-spaces string start))
705 (defun eword-analyze-special (string start &optional must-unfold)
706 (std11-analyze-special string start))
708 (defun eword-analyze-encoded-word (string start &optional must-unfold)
709 (let* ((regexp (concat "[\n\t ]*\\(" eword-encoded-word-regexp "\\)"))
710 (match (and (string-match regexp string start)
711 (= start (match-beginning 0))))
714 (setq next (match-end 0))
715 (push (list (match-string 2 string) ;; charset
716 (match-string 3 string) ;; language
717 (match-string 4 string) ;; encoding
718 (match-string 5 string) ;; encoded-text
719 (match-string 1 string)) ;; encoded-word
721 (setq match (and (string-match regexp string next)
722 (= next (match-beginning 0)))))
724 (cons (cons 'atom (eword-decode-encoded-words (nreverse words)
728 (defun eword-analyze-atom (string start &optional must-unfold)
729 (if (and (string-match std11-atom-regexp string start)
730 (= (match-beginning 0) start))
731 (let ((end (match-end 0)))
732 (cons (cons 'atom (decode-mime-charset-string
733 (substring string start end)
734 default-mime-charset))
735 ;;(substring string end)
739 (defun eword-lexical-analyze-internal (string start must-unfold)
740 (let ((len (length string))
744 (let ((rest mime-header-lexical-analyzer)
746 (while (and (setq func (car rest))
748 (setq r (funcall func string start must-unfold)))
750 (setq rest (cdr rest)))
752 (cons (cons 'error (substring string start)) (1+ len)))
754 (setq dest (cons (car ret) dest)
760 (defun eword-lexical-analyze (string &optional start must-unfold)
761 "Return lexical analyzed list corresponding STRING.
762 It is like std11-lexical-analyze, but it decodes non us-ascii
763 characters encoded as encoded-words or invalid \"raw\" format.
764 \"Raw\" non us-ascii characters are regarded as variable
765 `default-mime-charset'."
766 (let ((key (substring string (or start 0)))
768 (set-text-properties 0 (length key) nil key)
769 (if (setq ret (assoc key eword-lexical-analyze-cache))
771 (setq ret (eword-lexical-analyze-internal key 0 must-unfold))
772 (setq eword-lexical-analyze-cache
774 eword-lexical-analyze-cache))
775 (if (cdr (setq cell (nthcdr eword-lexical-analyze-cache-max
776 eword-lexical-analyze-cache)))
780 (defun eword-decode-token (token)
781 (let ((type (car token))
783 (cond ((eq type 'quoted-string)
784 (std11-wrap-as-quoted-string value))
788 (setq dest (concat dest
789 (if (stringp (car value))
790 (std11-wrap-as-quoted-pairs
791 (car value) '(?( ?)))
792 (eword-decode-token (car value))
796 (concat "(" dest ")")
800 (defun eword-extract-address-components (string &optional start)
801 "Extract full name and canonical address from STRING.
802 Returns a list of the form (FULL-NAME CANONICAL-ADDRESS).
803 If no name can be extracted, FULL-NAME will be nil.
804 It decodes non us-ascii characters in FULL-NAME encoded as
805 encoded-words or invalid \"raw\" string. \"Raw\" non us-ascii
806 characters are regarded as variable `default-mime-charset'."
807 (let* ((structure (car (std11-parse-address
808 (eword-lexical-analyze
809 (std11-unfold-string string) start
811 (phrase (std11-full-name-string structure))
812 (address (std11-address-string structure))
814 (list phrase address)
821 (provide 'eword-decode)
823 ;;; eword-decode.el ends here