1 ;;; eword-decode.el --- RFC 2047 based encoded-word decoder for GNU Emacs
3 ;; Copyright (C) 1995,1996,1997,1998 Free Software Foundation, Inc.
5 ;; Author: ENAMI Tsugutomo <enami@sys.ptg.sony.co.jp>
6 ;; MORIOKA Tomohiko <morioka@jaist.ac.jp>
7 ;; Maintainer: MORIOKA Tomohiko <morioka@jaist.ac.jp>
9 ;; Original: 1992/07/20 ENAMI Tsugutomo's `mime.el'.
10 ;; Renamed: 1993/06/03 to tiny-mime.el
11 ;; Renamed: 1995/10/03 from tiny-mime.el (split off encoder)
12 ;; Renamed: 1997/02/22 from tm-ew-d.el
13 ;; Keywords: encoded-word, MIME, multilingual, header, mail, news
15 ;; This file is part of SEMI (Spadework for Emacs MIME Interfaces).
17 ;; This program is free software; you can redistribute it and/or
18 ;; modify it under the terms of the GNU General Public License as
19 ;; published by the Free Software Foundation; either version 2, or (at
20 ;; your option) any later version.
22 ;; This program is distributed in the hope that it will be useful, but
23 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
24 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
25 ;; General Public License for more details.
27 ;; You should have received a copy of the GNU General Public License
28 ;; along with GNU Emacs; see the file COPYING. If not, write to the
29 ;; Free Software Foundation, Inc., 59 Temple Place - Suite 330,
30 ;; Boston, MA 02111-1307, USA.
38 (defgroup eword-decode nil
39 "Encoded-word decoding"
43 ;;; @ MIME encoded-word definition
46 (defconst eword-encoded-text-regexp "[!->@-~]+")
47 (defconst eword-encoded-word-regexp
48 (concat (regexp-quote "=?")
56 eword-encoded-text-regexp
64 (defconst base64-token-regexp "[A-Za-z0-9+/]")
65 (defconst base64-token-padding-regexp "[A-Za-z0-9+/=]")
67 (defconst eword-B-encoded-text-regexp
76 base64-token-padding-regexp
77 base64-token-padding-regexp
80 ;; (defconst eword-B-encoding-and-encoded-text-regexp
81 ;; (concat "\\(B\\)\\?" eword-B-encoded-text-regexp))
84 ;;; @@ Quoted-Printable
87 (defconst eword-Q-encoded-text-regexp
88 (concat "\\([^=?]\\|" quoted-printable-octet-regexp "\\)+"))
89 ;; (defconst eword-Q-encoding-and-encoded-text-regexp
90 ;; (concat "\\(Q\\)\\?" eword-Q-encoded-text-regexp))
96 (defun eword-decode-string (string &optional must-unfold)
97 "Decode MIME encoded-words in STRING.
99 STRING is unfolded before decoding.
101 If an encoded-word is broken or your emacs implementation can not
102 decode the charset included in it, it is not decoded.
104 If MUST-UNFOLD is non-nil, it unfolds and eliminates line-breaks even
105 if there are in decoded encoded-words (generated by bad manner MUA
106 such as a version of Net$cape)."
107 (setq string (std11-unfold-string string))
108 (let ((dest "")(ew nil)
110 (while (and (string-match eword-encoded-word-regexp string)
111 (setq beg (match-beginning 0)
117 (string-match "^[ \t]+$" (substring string 0 beg))
119 (setq dest (concat dest (substring string 0 beg)))
124 (eword-decode-encoded-word
125 (substring string beg end) must-unfold)
127 (setq string (substring string end))
137 (defun eword-decode-region (start end &optional unfolding must-unfold)
138 "Decode MIME encoded-words in region between START and END.
140 If UNFOLDING is not nil, it unfolds before decoding.
142 If MUST-UNFOLD is non-nil, it unfolds and eliminates line-breaks even
143 if there are in decoded encoded-words (generated by bad manner MUA
144 such as a version of Net$cape)."
148 (narrow-to-region start end)
150 (eword-decode-unfold)
152 (goto-char (point-min))
153 (while (re-search-forward (concat "\\(" eword-encoded-word-regexp "\\)"
155 "\\(" eword-encoded-word-regexp "\\)")
157 (replace-match "\\1\\6")
158 (goto-char (point-min))
160 (while (re-search-forward eword-encoded-word-regexp nil t)
161 (insert (eword-decode-encoded-word
163 (buffer-substring (match-beginning 0) (match-end 0))
164 (delete-region (match-beginning 0) (match-end 0))
170 ;;; @ for message header
173 (defcustom eword-decode-ignored-field-list
174 '(Newsgroups Path Lines Nntp-Posting-Host Received Message-Id Date)
175 "*List of field-names to be ignored when decoding.
176 Each field name must be symbol."
178 :type '(repeat symbol))
180 (defcustom eword-decode-structured-field-list
181 '(Reply-To Resent-Reply-To From Resent-From Sender Resent-Sender
182 To Resent-To Cc Resent-Cc Bcc Resent-Bcc Dcc
183 Mime-Version Content-Type Content-Transfer-Encoding
185 "*List of field-names to decode as structured field.
186 Each field name must be symbol."
188 :type '(repeat symbol))
190 (defun eword-decode-header (&optional code-conversion separator)
191 "Decode MIME encoded-words in header fields.
192 If CODE-CONVERSION is nil, it decodes only encoded-words. If it is
193 mime-charset, it decodes non-ASCII bit patterns as the mime-charset.
194 Otherwise it decodes non-ASCII bit patterns as the
195 default-mime-charset.
196 If SEPARATOR is not nil, it is used as header separator."
200 (std11-narrow-to-header separator)
201 (let ((default-charset
203 (if (mime-charset-to-coding-system code-conversion)
205 default-mime-charset))))
207 (let (beg p end field-name len)
208 (goto-char (point-min))
209 (while (re-search-forward std11-field-head-regexp nil t)
210 (setq beg (match-beginning 0)
212 field-name (buffer-substring beg (1- p))
213 len (string-width field-name)
214 field-name (intern (capitalize field-name))
215 end (std11-field-end))
216 (cond ((memq field-name eword-decode-ignored-field-list)
219 ((memq field-name eword-decode-structured-field-list)
220 ;; Decode as structured field
221 (let ((body (buffer-substring p end))
222 (default-mime-charset default-charset))
223 (delete-region p end)
224 (insert (eword-decode-and-fold-structured-field
228 ;; Decode as unstructured field
230 (narrow-to-region beg (1+ end))
231 (decode-mime-charset-region p end default-charset)
233 (if (re-search-forward eword-encoded-word-regexp
235 (eword-decode-region beg (point-max) 'unfold))
237 (eword-decode-region (point-min) (point-max) t)
240 (defun eword-decode-unfold ()
241 (goto-char (point-min))
243 (while (re-search-forward std11-field-head-regexp nil t)
244 (setq beg (match-beginning 0)
245 end (std11-field-end))
246 (setq field (buffer-substring beg end))
247 (if (string-match eword-encoded-word-regexp field)
249 (narrow-to-region (goto-char beg) end)
250 (while (re-search-forward "\n\\([ \t]\\)" nil t)
251 (replace-match (match-string 1))
253 (goto-char (point-max))
257 (defun eword-visible-field-p (field-name visible-fields invisible-fields)
259 (while visible-fields
260 (let ((regexp (car visible-fields)))
261 (if (string-match regexp field-name)
264 (setq visible-fields (cdr visible-fields))
267 (while invisible-fields
268 (let ((regexp (car invisible-fields)))
269 (if (string-match regexp field-name)
272 (setq invisible-fields (cdr invisible-fields))
276 (defun mime-insert-decoded-header (entity
277 &optional invisible-fields visible-fields
279 "Insert before point a decoded header of ENTITY."
280 (let ((default-charset
282 (if (mime-charset-to-coding-system code-conversion)
284 default-mime-charset))))
286 (narrow-to-region (point)(point))
287 (let ((the-buf (current-buffer))
288 (src-buf (mime-entity-buffer entity))
289 (h-end (mime-entity-header-end entity))
290 beg p end field-name len field)
293 (goto-char (mime-entity-header-start entity))
295 (narrow-to-region (point) h-end)
296 (while (re-search-forward std11-field-head-regexp nil t)
297 (setq beg (match-beginning 0)
299 field-name (buffer-substring beg (1- p))
300 len (string-width field-name)
301 end (std11-field-end))
302 (when (eword-visible-field-p field-name
303 visible-fields invisible-fields)
304 (setq field (intern (capitalize field-name)))
309 (cond ((memq field eword-decode-ignored-field-list)
311 (insert-buffer-substring src-buf p end)
313 ((memq field eword-decode-structured-field-list)
314 ;; Decode as structured field
315 (let ((body (save-excursion
317 (buffer-substring p end)))
318 (default-mime-charset default-charset))
319 (insert (eword-decode-and-fold-structured-field
323 ;; Decode as unstructured field
324 (let ((body (save-excursion
326 (buffer-substring p end)))
327 (default-mime-charset default-charset))
328 (insert (eword-decode-unstructured-field-body
335 ;;; @ encoded-word decoder
338 (defvar eword-decode-encoded-word-error-handler
339 'eword-decode-encoded-word-default-error-handler)
341 (defvar eword-warning-face nil
342 "Face used for invalid encoded-word.")
344 (defun eword-decode-encoded-word-default-error-handler (word signal)
345 (and (add-text-properties 0 (length word)
346 (and eword-warning-face
347 (list 'face eword-warning-face))
351 (defun eword-decode-encoded-word (word &optional must-unfold)
352 "Decode WORD if it is an encoded-word.
354 If your emacs implementation can not decode the charset of WORD, it
355 returns WORD. Similarly the encoded-word is broken, it returns WORD.
357 If MUST-UNFOLD is non-nil, it unfolds and eliminates line-breaks even
358 if there are in decoded encoded-word (generated by bad manner MUA such
359 as a version of Net$cape)."
360 (or (if (string-match eword-encoded-word-regexp word)
362 (substring word (match-beginning 1) (match-end 1))
366 (substring word (match-beginning 2) (match-end 2))
369 (substring word (match-beginning 3) (match-end 3))
372 (eword-decode-encoded-text charset encoding text must-unfold)
374 (funcall eword-decode-encoded-word-error-handler word err)
380 ;;; @ encoded-text decoder
383 (defun eword-decode-encoded-text (charset encoding string
384 &optional must-unfold)
385 "Decode STRING as an encoded-text.
387 If your emacs implementation can not decode CHARSET, it returns nil.
389 If ENCODING is not \"B\" or \"Q\", it occurs error.
390 So you should write error-handling code if you don't want break by errors.
392 If MUST-UNFOLD is non-nil, it unfolds and eliminates line-breaks even
393 if there are in decoded encoded-text (generated by bad manner MUA such
394 as a version of Net$cape)."
395 (let ((cs (mime-charset-to-coding-system charset)))
399 ((string-equal "B" encoding)
400 (if (and (string-match eword-B-encoded-text-regexp string)
401 (string-equal string (match-string 0 string)))
402 (base64-decode-string string)
403 (error "Invalid encoded-text %s" string)))
404 ((string-equal "Q" encoding)
405 (if (and (string-match eword-Q-encoded-text-regexp string)
406 (string-equal string (match-string 0 string)))
407 (q-encoding-decode-string string)
408 (error "Invalid encoded-text %s" string)))
410 (error "Invalid encoding %s" encoding)
415 (setq dest (decode-coding-string dest cs))
422 (t (char-to-string chr)))
424 (std11-unfold-string dest)
430 ;;; @ lexical analyze
433 (defvar eword-lexical-analyze-cache nil)
434 (defvar eword-lexical-analyze-cache-max 299
435 "*Max position of eword-lexical-analyze-cache.
436 It is max size of eword-lexical-analyze-cache - 1.")
438 (defcustom eword-lexical-analyzers
439 '(eword-analyze-quoted-string
440 eword-analyze-domain-literal
441 eword-analyze-comment
443 eword-analyze-special
444 eword-analyze-encoded-word
446 "*List of functions to return result of lexical analyze.
447 Each function must have two arguments: STRING and MUST-UNFOLD.
448 STRING is the target string to be analyzed.
449 If MUST-UNFOLD is not nil, each function must unfold and eliminate
450 bare-CR and bare-LF from the result even if they are included in
451 content of the encoded-word.
452 Each function must return nil if it can not analyze STRING as its
455 Previous function is preferred to next function. If a function
456 returns nil, next function is used. Otherwise the return value will
459 :type '(repeat function))
461 (defun eword-analyze-quoted-string (string &optional must-unfold)
462 (let ((p (std11-check-enclosure string ?\" ?\")))
464 (cons (cons 'quoted-string
465 (decode-mime-charset-string
466 (std11-strip-quoted-pair (substring string 1 (1- p)))
467 default-mime-charset))
468 (substring string p))
471 (defun eword-analyze-domain-literal (string &optional must-unfold)
472 (std11-analyze-domain-literal string))
474 (defun eword-analyze-comment (string &optional must-unfold)
475 (let ((p (std11-check-enclosure string ?\( ?\) t)))
479 (decode-mime-charset-string
480 (std11-strip-quoted-pair (substring string 1 (1- p)))
481 default-mime-charset)
483 (substring string p))
486 (defun eword-analyze-spaces (string &optional must-unfold)
487 (std11-analyze-spaces string))
489 (defun eword-analyze-special (string &optional must-unfold)
490 (std11-analyze-special string))
492 (defun eword-analyze-encoded-word (string &optional must-unfold)
493 (if (eq (string-match eword-encoded-word-regexp string) 0)
494 (let ((end (match-end 0))
495 (dest (eword-decode-encoded-word (match-string 0 string)
498 (setq string (substring string end))
499 (while (eq (string-match `,(concat "[ \t\n]*\\("
500 eword-encoded-word-regexp
504 (setq end (match-end 0))
507 (eword-decode-encoded-word (match-string 1 string)
509 string (substring string end))
511 (cons (cons 'atom dest) string)
514 (defun eword-analyze-atom (string &optional must-unfold)
515 (if (string-match std11-atom-regexp string)
516 (let ((end (match-end 0)))
517 (cons (cons 'atom (decode-mime-charset-string
518 (substring string 0 end)
519 default-mime-charset))
520 (substring string end)
523 (defun eword-lexical-analyze-internal (string must-unfold)
525 (while (not (string-equal string ""))
527 (let ((rest eword-lexical-analyzers)
529 (while (and (setq func (car rest))
530 (null (setq r (funcall func string must-unfold)))
532 (setq rest (cdr rest)))
533 (or r `((error . ,string) . ""))
535 (setq dest (cons (car ret) dest))
536 (setq string (cdr ret))
541 (defun eword-lexical-analyze (string &optional must-unfold)
542 "Return lexical analyzed list corresponding STRING.
543 It is like std11-lexical-analyze, but it decodes non us-ascii
544 characters encoded as encoded-words or invalid \"raw\" format.
545 \"Raw\" non us-ascii characters are regarded as variable
546 `default-mime-charset'."
547 (let ((key (copy-sequence string))
549 (set-text-properties 0 (length key) nil key)
550 (if (setq ret (assoc key eword-lexical-analyze-cache))
552 (setq ret (eword-lexical-analyze-internal key must-unfold))
553 (setq eword-lexical-analyze-cache
555 (last eword-lexical-analyze-cache
556 eword-lexical-analyze-cache-max)))
559 (defun eword-decode-token (token)
560 (let ((type (car token))
562 (cond ((eq type 'quoted-string)
563 (std11-wrap-as-quoted-string value))
565 (concat "(" (std11-wrap-as-quoted-pairs value '(?( ?))) ")"))
568 (defun eword-decode-and-fold-structured-field
569 (string start-column &optional max-column must-unfold)
570 "Decode and fold (fill) STRING as structured field body.
571 It decodes non us-ascii characters in FULL-NAME encoded as
572 encoded-words or invalid \"raw\" string. \"Raw\" non us-ascii
573 characters are regarded as variable `default-mime-charset'.
575 If an encoded-word is broken or your emacs implementation can not
576 decode the charset included in it, it is not decoded.
578 If MAX-COLUMN is omitted, `fill-column' is used.
580 If MUST-UNFOLD is non-nil, it unfolds and eliminates line-breaks even
581 if there are in decoded encoded-words (generated by bad manner MUA
582 such as a version of Net$cape)."
584 (setq max-column fill-column))
585 (let ((c start-column)
586 (tokens (eword-lexical-analyze string must-unfold))
589 (while (and (setq token (car tokens))
590 (setq tokens (cdr tokens)))
591 (let* ((type (car token)))
592 (if (eq type 'spaces)
593 (let* ((next-token (car tokens))
594 (next-str (eword-decode-token next-token))
595 (next-len (string-width next-str))
596 (next-c (+ c next-len 1)))
597 (if (< next-c max-column)
598 (setq result (concat result " " next-str)
600 (setq result (concat result "\n " next-str)
602 (setq tokens (cdr tokens))
604 (let* ((str (eword-decode-token token)))
605 (setq result (concat result str)
606 c (+ c (string-width str)))
609 (concat result (eword-decode-token token))
612 (defun eword-decode-and-unfold-structured-field (string)
613 "Decode and unfold STRING as structured field body.
614 It decodes non us-ascii characters in FULL-NAME encoded as
615 encoded-words or invalid \"raw\" string. \"Raw\" non us-ascii
616 characters are regarded as variable `default-mime-charset'.
618 If an encoded-word is broken or your emacs implementation can not
619 decode the charset included in it, it is not decoded."
620 (let ((tokens (eword-lexical-analyze string 'must-unfold))
623 (let* ((token (car tokens))
625 (setq tokens (cdr tokens))
627 (if (eq type 'spaces)
629 (concat result (eword-decode-token token))
633 (defun eword-decode-structured-field-body (string &optional must-unfold
634 start-column max-column)
635 "Decode non us-ascii characters in STRING as structured field body.
636 STRING is unfolded before decoding.
638 It decodes non us-ascii characters in FULL-NAME encoded as
639 encoded-words or invalid \"raw\" string. \"Raw\" non us-ascii
640 characters are regarded as variable `default-mime-charset'.
642 If an encoded-word is broken or your emacs implementation can not
643 decode the charset included in it, it is not decoded.
645 If MUST-UNFOLD is non-nil, it unfolds and eliminates line-breaks even
646 if there are in decoded encoded-words (generated by bad manner MUA
647 such as a version of Net$cape)."
649 ;; fold with max-column
650 (eword-decode-and-fold-structured-field
651 string start-column max-column must-unfold)
653 (mapconcat (function eword-decode-token)
654 (eword-lexical-analyze string must-unfold)
658 (defun eword-decode-unstructured-field-body (string &optional must-unfold)
659 "Decode non us-ascii characters in STRING as unstructured field body.
660 STRING is unfolded before decoding.
662 It decodes non us-ascii characters in FULL-NAME encoded as
663 encoded-words or invalid \"raw\" string. \"Raw\" non us-ascii
664 characters are regarded as variable `default-mime-charset'.
666 If an encoded-word is broken or your emacs implementation can not
667 decode the charset included in it, it is not decoded.
669 If MUST-UNFOLD is non-nil, it unfolds and eliminates line-breaks even
670 if there are in decoded encoded-words (generated by bad manner MUA
671 such as a version of Net$cape)."
673 (decode-mime-charset-string string default-mime-charset)
676 (defun eword-extract-address-components (string)
677 "Extract full name and canonical address from STRING.
678 Returns a list of the form (FULL-NAME CANONICAL-ADDRESS).
679 If no name can be extracted, FULL-NAME will be nil.
680 It decodes non us-ascii characters in FULL-NAME encoded as
681 encoded-words or invalid \"raw\" string. \"Raw\" non us-ascii
682 characters are regarded as variable `default-mime-charset'."
683 (let* ((structure (car (std11-parse-address
684 (eword-lexical-analyze
685 (std11-unfold-string string) 'must-unfold))))
686 (phrase (std11-full-name-string structure))
687 (address (std11-address-string structure))
689 (list phrase address)
696 (provide 'eword-decode)
698 ;;; eword-decode.el ends here