1 ;;; eword-decode.el --- RFC 2047 based encoded-word decoder for GNU Emacs
3 ;; Copyright (C) 1995,1996,1997,1998 Free Software Foundation, Inc.
5 ;; Author: ENAMI Tsugutomo <enami@sys.ptg.sony.co.jp>
6 ;; MORIOKA Tomohiko <morioka@jaist.ac.jp>
7 ;; Maintainer: MORIOKA Tomohiko <morioka@jaist.ac.jp>
9 ;; Original: 1992/07/20 ENAMI Tsugutomo's `mime.el'.
10 ;; Renamed: 1993/06/03 to tiny-mime.el
11 ;; Renamed: 1995/10/03 from tiny-mime.el (split off encoder)
12 ;; Renamed: 1997/02/22 from tm-ew-d.el
13 ;; Keywords: encoded-word, MIME, multilingual, header, mail, news
15 ;; This file is part of SEMI (Spadework for Emacs MIME Interfaces).
17 ;; This program is free software; you can redistribute it and/or
18 ;; modify it under the terms of the GNU General Public License as
19 ;; published by the Free Software Foundation; either version 2, or (at
20 ;; your option) any later version.
22 ;; This program is distributed in the hope that it will be useful, but
23 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
24 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
25 ;; General Public License for more details.
27 ;; You should have received a copy of the GNU General Public License
28 ;; along with GNU Emacs; see the file COPYING. If not, write to the
29 ;; Free Software Foundation, Inc., 59 Temple Place - Suite 330,
30 ;; Boston, MA 02111-1307, USA.
34 (require 'std11-parse)
38 (defgroup eword-decode nil
39 "Encoded-word decoding"
42 (defconst eword-decode-version
43 `,(mapconcat #'number-to-string (cdr semi-version) "."))
46 ;;; @ MIME encoded-word definition
49 (defconst eword-encoded-text-regexp "[!->@-~]+")
50 (defconst eword-encoded-word-regexp
51 (concat (regexp-quote "=?")
59 eword-encoded-text-regexp
67 (defconst base64-token-regexp "[A-Za-z0-9+/]")
68 (defconst base64-token-padding-regexp "[A-Za-z0-9+/=]")
70 (defconst eword-B-encoded-text-regexp
79 base64-token-padding-regexp
80 base64-token-padding-regexp
83 ;; (defconst eword-B-encoding-and-encoded-text-regexp
84 ;; (concat "\\(B\\)\\?" eword-B-encoded-text-regexp))
87 ;;; @@ Quoted-Printable
90 (defconst quoted-printable-hex-chars "0123456789ABCDEF")
91 (defconst quoted-printable-octet-regexp
92 (concat "=[" quoted-printable-hex-chars
93 "][" quoted-printable-hex-chars "]"))
95 (defconst eword-Q-encoded-text-regexp
96 (concat "\\([^=?]\\|" quoted-printable-octet-regexp "\\)+"))
97 ;; (defconst eword-Q-encoding-and-encoded-text-regexp
98 ;; (concat "\\(Q\\)\\?" eword-Q-encoded-text-regexp))
104 (defun eword-decode-string (string &optional must-unfold)
105 "Decode MIME encoded-words in STRING.
107 STRING is unfolded before decoding.
109 If an encoded-word is broken or your emacs implementation can not
110 decode the charset included in it, it is not decoded.
112 If MUST-UNFOLD is non-nil, it unfolds and eliminates line-breaks even
113 if there are in decoded encoded-words (generated by bad manner MUA
114 such as a version of Net$cape)."
115 (setq string (std11-unfold-string string))
116 (let ((dest "")(ew nil)
118 (while (and (string-match eword-encoded-word-regexp string)
119 (setq beg (match-beginning 0)
125 (string-match "^[ \t]+$" (substring string 0 beg))
127 (setq dest (concat dest (substring string 0 beg)))
132 (eword-decode-encoded-word
133 (substring string beg end) must-unfold)
135 (setq string (substring string end))
145 (defun eword-decode-region (start end &optional unfolding must-unfold)
146 "Decode MIME encoded-words in region between START and END.
148 If UNFOLDING is not nil, it unfolds before decoding.
150 If MUST-UNFOLD is non-nil, it unfolds and eliminates line-breaks even
151 if there are in decoded encoded-words (generated by bad manner MUA
152 such as a version of Net$cape)."
156 (narrow-to-region start end)
158 (eword-decode-unfold)
160 (goto-char (point-min))
161 (while (re-search-forward (concat "\\(" eword-encoded-word-regexp "\\)"
163 "\\(" eword-encoded-word-regexp "\\)")
165 (replace-match "\\1\\6")
166 (goto-char (point-min))
168 (while (re-search-forward eword-encoded-word-regexp nil t)
169 (insert (eword-decode-encoded-word
171 (buffer-substring (match-beginning 0) (match-end 0))
172 (delete-region (match-beginning 0) (match-end 0))
178 ;;; @ for message header
181 (defcustom eword-decode-ignored-field-list
182 '(newsgroups path lines nntp-posting-host message-id date)
183 "*List of field-names to be ignored when decoding.
184 Each field name must be symbol."
186 :type '(repeat symbol))
188 (defcustom eword-decode-structured-field-list
189 '(reply-to resent-reply-to from resent-from sender resent-sender
190 to resent-to cc resent-cc bcc resent-bcc dcc
191 mime-version content-type content-transfer-encoding
193 "*List of field-names to decode as structured field.
194 Each field name must be symbol."
196 :type '(repeat symbol))
198 (defun eword-decode-header (&optional code-conversion separator)
199 "Decode MIME encoded-words in header fields.
200 If CODE-CONVERSION is nil, it decodes only encoded-words. If it is
201 mime-charset, it decodes non-ASCII bit patterns as the mime-charset.
202 Otherwise it decodes non-ASCII bit patterns as the
203 default-mime-charset.
204 If SEPARATOR is not nil, it is used as header separator."
208 (std11-narrow-to-header separator)
209 (let ((default-charset
211 (if (mime-charset-to-coding-system code-conversion)
213 default-mime-charset))))
215 (let (beg p end field-name len)
216 (goto-char (point-min))
217 (while (re-search-forward std11-field-head-regexp nil t)
218 (setq beg (match-beginning 0)
220 field-name (buffer-substring beg (1- p))
221 len (string-width field-name)
222 field-name (intern (downcase field-name))
223 end (std11-field-end))
224 (cond ((memq field-name eword-decode-ignored-field-list)
227 ((memq field-name eword-decode-structured-field-list)
228 ;; Decode as structured field
229 (let ((body (buffer-substring p end))
230 (default-mime-charset default-charset))
231 (delete-region p end)
232 (insert (eword-decode-and-fold-structured-field
236 ;; Decode as unstructured field
238 (narrow-to-region beg (1+ end))
239 (decode-mime-charset-region p end default-charset)
241 (if (re-search-forward eword-encoded-word-regexp
243 (eword-decode-region beg (point-max) 'unfold))
245 (eword-decode-region (point-min) (point-max) t)
248 (defun eword-decode-unfold ()
249 (goto-char (point-min))
251 (while (re-search-forward std11-field-head-regexp nil t)
252 (setq beg (match-beginning 0)
253 end (std11-field-end))
254 (setq field (buffer-substring beg end))
255 (if (string-match eword-encoded-word-regexp field)
257 (narrow-to-region (goto-char beg) end)
258 (while (re-search-forward "\n\\([ \t]\\)" nil t)
259 (replace-match (match-string 1))
261 (goto-char (point-max))
266 ;;; @ encoded-word decoder
269 (defvar eword-warning-face nil "Face used for invalid encoded-word.")
271 (defun eword-decode-encoded-word (word &optional must-unfold)
272 "Decode WORD if it is an encoded-word.
274 If your emacs implementation can not decode the charset of WORD, it
275 returns WORD. Similarly the encoded-word is broken, it returns WORD.
277 If MUST-UNFOLD is non-nil, it unfolds and eliminates line-breaks even
278 if there are in decoded encoded-word (generated by bad manner MUA such
279 as a version of Net$cape)."
280 (or (if (string-match eword-encoded-word-regexp word)
282 (substring word (match-beginning 1) (match-end 1))
286 (substring word (match-beginning 2) (match-end 2))
289 (substring word (match-beginning 3) (match-end 3))
292 (eword-decode-encoded-text charset encoding text must-unfold)
295 (add-text-properties 0 (length word)
296 (and eword-warning-face
297 (list 'face eword-warning-face))
304 ;;; @ encoded-text decoder
307 (defun eword-decode-encoded-text (charset encoding string
308 &optional must-unfold)
309 "Decode STRING as an encoded-text.
311 If your emacs implementation can not decode CHARSET, it returns nil.
313 If ENCODING is not \"B\" or \"Q\", it occurs error.
314 So you should write error-handling code if you don't want break by errors.
316 If MUST-UNFOLD is non-nil, it unfolds and eliminates line-breaks even
317 if there are in decoded encoded-text (generated by bad manner MUA such
318 as a version of Net$cape)."
319 (let ((cs (mime-charset-to-coding-system charset)))
323 ((string-equal "B" encoding)
324 (if (and (string-match eword-B-encoded-text-regexp string)
325 (string-equal string (match-string 0 string)))
326 (base64-decode-string string)
327 (error "Invalid encoded-text %s" string)))
328 ((string-equal "Q" encoding)
329 (if (and (string-match eword-Q-encoded-text-regexp string)
330 (string-equal string (match-string 0 string)))
331 (q-encoding-decode-string string)
332 (error "Invalid encoded-text %s" string)))
334 (error "Invalid encoding %s" encoding)
339 (setq dest (decode-coding-string dest cs))
346 (t (char-to-string chr)))
348 (std11-unfold-string dest)
354 ;;; @ lexical analyze
357 (defvar eword-lexical-analyze-cache nil)
358 (defvar eword-lexical-analyze-cache-max 299
359 "*Max position of eword-lexical-analyze-cache.
360 It is max size of eword-lexical-analyze-cache - 1.")
362 (defcustom eword-lexical-analyzers
363 '(eword-analyze-quoted-string
364 eword-analyze-domain-literal
365 eword-analyze-comment
367 eword-analyze-special
368 eword-analyze-encoded-word
370 "*List of functions to return result of lexical analyze.
371 Each function must have two arguments: STRING and MUST-UNFOLD.
372 STRING is the target string to be analyzed.
373 If MUST-UNFOLD is not nil, each function must unfold and eliminate
374 bare-CR and bare-LF from the result even if they are included in
375 content of the encoded-word.
376 Each function must return nil if it can not analyze STRING as its
379 Previous function is preferred to next function. If a function
380 returns nil, next function is used. Otherwise the return value will
383 :type '(repeat function))
385 (defun eword-analyze-quoted-string (string &optional must-unfold)
386 (let ((p (std11-check-enclosure string ?\" ?\")))
388 (cons (cons 'quoted-string
389 (decode-mime-charset-string
390 (std11-strip-quoted-pair (substring string 1 (1- p)))
391 default-mime-charset))
392 (substring string p))
395 (defun eword-analyze-domain-literal (string &optional must-unfold)
396 (std11-analyze-domain-literal string))
398 (defun eword-analyze-comment (string &optional must-unfold)
399 (let ((p (std11-check-enclosure string ?\( ?\) t)))
403 (decode-mime-charset-string
404 (std11-strip-quoted-pair (substring string 1 (1- p)))
405 default-mime-charset)
407 (substring string p))
410 (defun eword-analyze-spaces (string &optional must-unfold)
411 (std11-analyze-spaces string))
413 (defun eword-analyze-special (string &optional must-unfold)
414 (std11-analyze-special string))
416 (defun eword-analyze-encoded-word (string &optional must-unfold)
417 (if (eq (string-match eword-encoded-word-regexp string) 0)
418 (let ((end (match-end 0))
419 (dest (eword-decode-encoded-word (match-string 0 string)
422 (setq string (substring string end))
423 (while (eq (string-match `,(concat "[ \t\n]*\\("
424 eword-encoded-word-regexp
428 (setq end (match-end 0))
431 (eword-decode-encoded-word (match-string 1 string)
433 string (substring string end))
435 (cons (cons 'atom dest) string)
438 (defun eword-analyze-atom (string &optional must-unfold)
439 (if (string-match std11-atom-regexp string)
440 (let ((end (match-end 0)))
441 (cons (cons 'atom (decode-mime-charset-string
442 (substring string 0 end)
443 default-mime-charset))
444 (substring string end)
447 (defun eword-lexical-analyze-internal (string must-unfold)
449 (while (not (string-equal string ""))
451 (let ((rest eword-lexical-analyzers)
453 (while (and (setq func (car rest))
454 (null (setq r (funcall func string must-unfold)))
456 (setq rest (cdr rest)))
457 (or r `((error . ,string) . ""))
459 (setq dest (cons (car ret) dest))
460 (setq string (cdr ret))
465 (defun eword-lexical-analyze (string &optional must-unfold)
466 "Return lexical analyzed list corresponding STRING.
467 It is like std11-lexical-analyze, but it decodes non us-ascii
468 characters encoded as encoded-words or invalid \"raw\" format.
469 \"Raw\" non us-ascii characters are regarded as variable
470 `default-mime-charset'."
471 (let ((key (copy-sequence string))
473 (set-text-properties 0 (length key) nil key)
474 (if (setq ret (assoc key eword-lexical-analyze-cache))
476 (setq ret (eword-lexical-analyze-internal key must-unfold))
477 (setq eword-lexical-analyze-cache
479 (last eword-lexical-analyze-cache
480 eword-lexical-analyze-cache-max)))
483 (defun eword-decode-token (token)
484 (let ((type (car token))
486 (cond ((eq type 'quoted-string)
487 (std11-wrap-as-quoted-string value))
489 (concat "(" (std11-wrap-as-quoted-pairs value '(?( ?))) ")"))
492 (defun eword-decode-and-fold-structured-field
493 (string start-column &optional max-column must-unfold)
494 "Decode and fold (fill) STRING as structured field body.
495 It decodes non us-ascii characters in FULL-NAME encoded as
496 encoded-words or invalid \"raw\" string. \"Raw\" non us-ascii
497 characters are regarded as variable `default-mime-charset'.
499 If an encoded-word is broken or your emacs implementation can not
500 decode the charset included in it, it is not decoded.
502 If MAX-COLUMN is omitted, `fill-column' is used.
504 If MUST-UNFOLD is non-nil, it unfolds and eliminates line-breaks even
505 if there are in decoded encoded-words (generated by bad manner MUA
506 such as a version of Net$cape)."
508 (setq max-column fill-column))
509 (let ((c start-column)
510 (tokens (eword-lexical-analyze string must-unfold))
513 (while (and (setq token (car tokens))
514 (setq tokens (cdr tokens)))
515 (let* ((type (car token)))
516 (if (eq type 'spaces)
517 (let* ((next-token (car tokens))
518 (next-str (eword-decode-token next-token))
519 (next-len (string-width next-str))
520 (next-c (+ c next-len 1)))
521 (if (< next-c max-column)
522 (setq result (concat result " " next-str)
524 (setq result (concat result "\n " next-str)
526 (setq tokens (cdr tokens))
528 (let* ((str (eword-decode-token token)))
529 (setq result (concat result str)
530 c (+ c (string-width str)))
533 (concat result (eword-decode-token token))
536 (defun eword-decode-and-unfold-structured-field (string)
537 "Decode and unfold STRING as structured field body.
538 It decodes non us-ascii characters in FULL-NAME encoded as
539 encoded-words or invalid \"raw\" string. \"Raw\" non us-ascii
540 characters are regarded as variable `default-mime-charset'.
542 If an encoded-word is broken or your emacs implementation can not
543 decode the charset included in it, it is not decoded."
544 (let ((tokens (eword-lexical-analyze string 'must-unfold))
547 (let* ((token (car tokens))
549 (setq tokens (cdr tokens))
551 (if (eq type 'spaces)
553 (concat result (eword-decode-token token))
557 (defun eword-decode-structured-field-body (string &optional must-unfold
558 start-column max-column)
559 "Decode non us-ascii characters in STRING as structured field body.
560 STRING is unfolded before decoding.
562 It decodes non us-ascii characters in FULL-NAME encoded as
563 encoded-words or invalid \"raw\" string. \"Raw\" non us-ascii
564 characters are regarded as variable `default-mime-charset'.
566 If an encoded-word is broken or your emacs implementation can not
567 decode the charset included in it, it is not decoded.
569 If MUST-UNFOLD is non-nil, it unfolds and eliminates line-breaks even
570 if there are in decoded encoded-words (generated by bad manner MUA
571 such as a version of Net$cape)."
573 ;; fold with max-column
574 (eword-decode-and-fold-structured-field
575 string start-column max-column must-unfold)
577 (mapconcat (function eword-decode-token)
578 (eword-lexical-analyze string must-unfold)
582 (defun eword-decode-unstructured-field-body (string &optional must-unfold)
583 "Decode non us-ascii characters in STRING as unstructured field body.
584 STRING is unfolded before decoding.
586 It decodes non us-ascii characters in FULL-NAME encoded as
587 encoded-words or invalid \"raw\" string. \"Raw\" non us-ascii
588 characters are regarded as variable `default-mime-charset'.
590 If an encoded-word is broken or your emacs implementation can not
591 decode the charset included in it, it is not decoded.
593 If MUST-UNFOLD is non-nil, it unfolds and eliminates line-breaks even
594 if there are in decoded encoded-words (generated by bad manner MUA
595 such as a version of Net$cape)."
597 (decode-mime-charset-string string default-mime-charset)
600 (defun eword-extract-address-components (string)
601 "Extract full name and canonical address from STRING.
602 Returns a list of the form (FULL-NAME CANONICAL-ADDRESS).
603 If no name can be extracted, FULL-NAME will be nil.
604 It decodes non us-ascii characters in FULL-NAME encoded as
605 encoded-words or invalid \"raw\" string. \"Raw\" non us-ascii
606 characters are regarded as variable `default-mime-charset'."
607 (let* ((structure (car (std11-parse-address
608 (eword-lexical-analyze
609 (std11-unfold-string string) 'must-unfold))))
610 (phrase (std11-full-name-string structure))
611 (address (std11-address-string structure))
613 (list phrase address)
620 (provide 'eword-decode)
622 ;;; eword-decode.el ends here