1 ;;; eword-decode.el --- RFC 2047 based encoded-word decoder for GNU Emacs
3 ;; Copyright (C) 1995,1996,1997,1998 Free Software Foundation, Inc.
5 ;; Author: ENAMI Tsugutomo <enami@sys.ptg.sony.co.jp>
6 ;; MORIOKA Tomohiko <morioka@jaist.ac.jp>
7 ;; TANAKA Akira <akr@jaist.ac.jp>
9 ;; Original: 1992/07/20 ENAMI Tsugutomo's `mime.el'.
10 ;; Renamed: 1993/06/03 to tiny-mime.el
11 ;; Renamed: 1995/10/03 from tiny-mime.el (split off encoder)
12 ;; Renamed: 1997/02/22 from tm-ew-d.el
13 ;; Keywords: encoded-word, MIME, multilingual, header, mail, news
15 ;; This file is part of SEMI (Spadework for Emacs MIME Interfaces).
17 ;; This program is free software; you can redistribute it and/or
18 ;; modify it under the terms of the GNU General Public License as
19 ;; published by the Free Software Foundation; either version 2, or (at
20 ;; your option) any later version.
22 ;; This program is distributed in the hope that it will be useful, but
23 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
24 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
25 ;; General Public License for more details.
27 ;; You should have received a copy of the GNU General Public License
28 ;; along with GNU Emacs; see the file COPYING. If not, write to the
29 ;; Free Software Foundation, Inc., 59 Temple Place - Suite 330,
30 ;; Boston, MA 02111-1307, USA.
38 (defgroup eword-decode nil
39 "Encoded-word decoding"
42 (defcustom eword-max-size-to-decode 1000
43 "*Max size to decode header field."
45 :type '(choice (integer :tag "Limit (bytes)")
46 (const :tag "Don't limit" nil)))
49 ;;; @ MIME encoded-word definition
53 (defconst eword-encoded-text-regexp "[!->@-~]+")
55 (defconst eword-encoded-word-regexp
57 (concat (regexp-quote "=?")
65 eword-encoded-text-regexp
67 (regexp-quote "?="))))
73 (defun eword-decode-string (string &optional must-unfold)
74 "Decode MIME encoded-words in STRING.
76 STRING is unfolded before decoding.
78 If an encoded-word is broken or your emacs implementation can not
79 decode the charset included in it, it is not decoded.
81 If MUST-UNFOLD is non-nil, it unfolds and eliminates line-breaks even
82 if there are in decoded encoded-words (generated by bad manner MUA
83 such as a version of Net$cape)."
84 (setq string (std11-unfold-string string))
85 (let ((dest "")(ew nil)
87 (while (and (string-match eword-encoded-word-regexp string)
88 (setq beg (match-beginning 0)
94 (string-match "^[ \t]+$" (substring string 0 beg))
96 (setq dest (concat dest (substring string 0 beg)))
101 (eword-decode-encoded-word
102 (substring string beg end) must-unfold)
104 (setq string (substring string end))
110 (defun eword-decode-and-fold-structured-field
111 (string start-column &optional max-column must-unfold)
112 "Decode and fold (fill) STRING as structured field body.
113 It decodes non us-ascii characters in FULL-NAME encoded as
114 encoded-words or invalid \"raw\" string. \"Raw\" non us-ascii
115 characters are regarded as variable `default-mime-charset'.
117 If an encoded-word is broken or your emacs implementation can not
118 decode the charset included in it, it is not decoded.
120 If MAX-COLUMN is omitted, `fill-column' is used.
122 If MUST-UNFOLD is non-nil, it unfolds and eliminates line-breaks even
123 if there are in decoded encoded-words (generated by bad manner MUA
124 such as a version of Net$cape)."
125 (if (and eword-max-size-to-decode
126 (> (length string) eword-max-size-to-decode))
129 (setq max-column fill-column))
130 (let ((c start-column)
131 (tokens (eword-lexical-analyze string must-unfold))
134 (while (and (setq token (car tokens))
135 (setq tokens (cdr tokens)))
136 (let* ((type (car token)))
137 (if (eq type 'spaces)
138 (let* ((next-token (car tokens))
139 (next-str (eword-decode-token next-token))
140 (next-len (string-width next-str))
141 (next-c (+ c next-len 1)))
142 (if (< next-c max-column)
143 (setq result (concat result " " next-str)
145 (setq result (concat result "\n " next-str)
147 (setq tokens (cdr tokens))
149 (let* ((str (eword-decode-token token)))
150 (setq result (concat result str)
151 c (+ c (string-width str)))
154 (concat result (eword-decode-token token))
157 (defun eword-decode-and-unfold-structured-field (string)
158 "Decode and unfold STRING as structured field body.
159 It decodes non us-ascii characters in FULL-NAME encoded as
160 encoded-words or invalid \"raw\" string. \"Raw\" non us-ascii
161 characters are regarded as variable `default-mime-charset'.
163 If an encoded-word is broken or your emacs implementation can not
164 decode the charset included in it, it is not decoded."
165 (let ((tokens (eword-lexical-analyze string 'must-unfold))
168 (let* ((token (car tokens))
170 (setq tokens (cdr tokens))
172 (if (eq type 'spaces)
174 (concat result (eword-decode-token token))
178 (defun eword-decode-structured-field-body (string &optional must-unfold
179 start-column max-column)
180 "Decode non us-ascii characters in STRING as structured field body.
181 STRING is unfolded before decoding.
183 It decodes non us-ascii characters in FULL-NAME encoded as
184 encoded-words or invalid \"raw\" string. \"Raw\" non us-ascii
185 characters are regarded as variable `default-mime-charset'.
187 If an encoded-word is broken or your emacs implementation can not
188 decode the charset included in it, it is not decoded.
190 If MUST-UNFOLD is non-nil, it unfolds and eliminates line-breaks even
191 if there are in decoded encoded-words (generated by bad manner MUA
192 such as a version of Net$cape)."
194 ;; fold with max-column
195 (eword-decode-and-fold-structured-field
196 string start-column max-column must-unfold)
198 (mapconcat (function eword-decode-token)
199 (eword-lexical-analyze string must-unfold)
203 (defun eword-decode-unstructured-field-body (string &optional must-unfold)
204 "Decode non us-ascii characters in STRING as unstructured field body.
205 STRING is unfolded before decoding.
207 It decodes non us-ascii characters in FULL-NAME encoded as
208 encoded-words or invalid \"raw\" string. \"Raw\" non us-ascii
209 characters are regarded as variable `default-mime-charset'.
211 If an encoded-word is broken or your emacs implementation can not
212 decode the charset included in it, it is not decoded.
214 If MUST-UNFOLD is non-nil, it unfolds and eliminates line-breaks even
215 if there are in decoded encoded-words (generated by bad manner MUA
216 such as a version of Net$cape)."
218 (decode-mime-charset-string string default-mime-charset)
221 (defun eword-decode-and-unfold-unstructured-field (string)
222 "Decode and unfold STRING as unstructured field body.
223 It decodes non us-ascii characters in FULL-NAME encoded as
224 encoded-words or invalid \"raw\" string. \"Raw\" non us-ascii
225 characters are regarded as variable `default-mime-charset'.
227 If an encoded-word is broken or your emacs implementation can not
228 decode the charset included in it, it is not decoded."
230 (decode-mime-charset-string (std11-unfold-string string)
231 default-mime-charset)
238 (defun eword-decode-region (start end &optional unfolding must-unfold)
239 "Decode MIME encoded-words in region between START and END.
241 If UNFOLDING is not nil, it unfolds before decoding.
243 If MUST-UNFOLD is non-nil, it unfolds and eliminates line-breaks even
244 if there are in decoded encoded-words (generated by bad manner MUA
245 such as a version of Net$cape)."
249 (narrow-to-region start end)
251 (eword-decode-unfold)
253 (goto-char (point-min))
254 (while (re-search-forward (concat "\\(" eword-encoded-word-regexp "\\)"
256 "\\(" eword-encoded-word-regexp "\\)")
258 (replace-match "\\1\\6")
259 (goto-char (point-min))
261 (while (re-search-forward eword-encoded-word-regexp nil t)
262 (insert (eword-decode-encoded-word
264 (buffer-substring (match-beginning 0) (match-end 0))
265 (delete-region (match-beginning 0) (match-end 0))
271 ;;; @ for message header
274 (defcustom eword-decode-ignored-field-list
275 '(Newsgroups Path Lines Nntp-Posting-Host Received Message-Id Date)
276 "*List of field-names to be ignored when decoding.
277 Each field name must be symbol."
279 :type '(repeat symbol))
281 (defcustom eword-decode-structured-field-list
282 '(Reply-To Resent-Reply-To From Resent-From Sender Resent-Sender
283 To Resent-To Cc Resent-Cc Bcc Resent-Bcc Dcc
285 Mime-Version Content-Type Content-Transfer-Encoding
286 Content-Disposition User-Agent)
287 "*List of field-names to decode as structured field.
288 Each field name must be symbol."
290 :type '(repeat symbol))
292 (defun eword-decode-field-body
293 (field-body field-name &optional unfolded max-column)
294 "Decode FIELD-BODY as FIELD-NAME, and return the result.
296 If UNFOLDED is non-nil, it is assumed that FIELD-BODY is
299 If MAX-COLUMN is non-nil, the result is folded with MAX-COLUMN
300 or `fill-column' if MAX-COLUMN is t.
301 Otherwise, the result is unfolded.
303 MIME encoded-word in FIELD-BODY is recognized according to
304 `eword-decode-ignored-field-list',
305 `eword-decode-structured-field-list' and FIELD-NAME.
307 Non MIME encoded-word part in FILED-BODY is decoded with
308 `default-mime-charset'."
309 (when (eq max-column t)
310 (setq max-column fill-column))
311 (let (field-name-symbol len)
312 (if (symbolp field-name)
313 (setq field-name-symbol field-name
314 len (1+ (string-width (symbol-name field-name))))
315 (setq field-name-symbol (intern (capitalize field-name))
316 len (1+ (string-width field-name))))
317 (if (memq field-name-symbol eword-decode-ignored-field-list)
321 (std11-unfold-string field-body))
322 (if (memq field-name-symbol eword-decode-structured-field-list)
323 ;; Decode as structured field
325 (eword-decode-and-fold-structured-field
326 field-body len max-column t)
327 (eword-decode-and-unfold-structured-field field-body))
328 ;; Decode as unstructured field
330 (eword-decode-unstructured-field-body field-body len)
331 (eword-decode-unstructured-field-body
332 (std11-unfold-string field-body) len))))))
334 (defun eword-decode-header (&optional code-conversion separator)
335 "Decode MIME encoded-words in header fields.
336 If CODE-CONVERSION is nil, it decodes only encoded-words. If it is
337 mime-charset, it decodes non-ASCII bit patterns as the mime-charset.
338 Otherwise it decodes non-ASCII bit patterns as the
339 default-mime-charset.
340 If SEPARATOR is not nil, it is used as header separator."
344 (std11-narrow-to-header separator)
345 (let ((default-charset
347 (if (mime-charset-to-coding-system code-conversion)
349 default-mime-charset))))
351 (let (beg p end field-name len)
352 (goto-char (point-min))
353 (while (re-search-forward std11-field-head-regexp nil t)
354 (setq beg (match-beginning 0)
356 field-name (buffer-substring beg (1- p))
357 len (string-width field-name)
358 field-name (intern (capitalize field-name))
359 end (std11-field-end))
360 (cond ((memq field-name eword-decode-ignored-field-list)
363 ((memq field-name eword-decode-structured-field-list)
364 ;; Decode as structured field
365 (let ((body (buffer-substring p end))
366 (default-mime-charset default-charset))
367 (delete-region p end)
368 (insert (eword-decode-and-fold-structured-field
372 ;; Decode as unstructured field
374 (narrow-to-region beg (1+ end))
375 (decode-mime-charset-region p end default-charset)
377 (if (re-search-forward eword-encoded-word-regexp
379 (eword-decode-region beg (point-max) 'unfold))
381 (eword-decode-region (point-min) (point-max) t)
384 (defun eword-decode-unfold ()
385 (goto-char (point-min))
387 (while (re-search-forward std11-field-head-regexp nil t)
388 (setq beg (match-beginning 0)
389 end (std11-field-end))
390 (setq field (buffer-substring beg end))
391 (if (string-match eword-encoded-word-regexp field)
393 (narrow-to-region (goto-char beg) end)
394 (while (re-search-forward "\n\\([ \t]\\)" nil t)
395 (replace-match (match-string 1))
397 (goto-char (point-max))
402 ;;; @ encoded-word decoder
405 (defvar eword-decode-encoded-word-error-handler
406 'eword-decode-encoded-word-default-error-handler)
408 (defvar eword-warning-face nil
409 "Face used for invalid encoded-word.")
411 (defun eword-decode-encoded-word-default-error-handler (word signal)
412 (and (add-text-properties 0 (length word)
413 (and eword-warning-face
414 (list 'face eword-warning-face))
418 (defun eword-decode-encoded-word (word &optional must-unfold)
419 "Decode WORD if it is an encoded-word.
421 If your emacs implementation can not decode the charset of WORD, it
422 returns WORD. Similarly the encoded-word is broken, it returns WORD.
424 If MUST-UNFOLD is non-nil, it unfolds and eliminates line-breaks even
425 if there are in decoded encoded-word (generated by bad manner MUA such
426 as a version of Net$cape)."
427 (or (if (string-match eword-encoded-word-regexp word)
429 (substring word (match-beginning 1) (match-end 1))
433 (substring word (match-beginning 2) (match-end 2))
436 (substring word (match-beginning 3) (match-end 3))
439 (eword-decode-encoded-text charset encoding text must-unfold)
441 (funcall eword-decode-encoded-word-error-handler word err)
447 ;;; @ encoded-text decoder
450 (defun eword-decode-encoded-text (charset encoding string
451 &optional must-unfold)
452 "Decode STRING as an encoded-text.
454 If your emacs implementation can not decode CHARSET, it returns nil.
456 If ENCODING is not \"B\" or \"Q\", it occurs error.
457 So you should write error-handling code if you don't want break by errors.
459 If MUST-UNFOLD is non-nil, it unfolds and eliminates line-breaks even
460 if there are in decoded encoded-text (generated by bad manner MUA such
461 as a version of Net$cape)."
462 (let ((cs (mime-charset-to-coding-system charset)))
464 (let ((dest (encoded-text-decode-string string encoding)))
466 (setq dest (decode-mime-charset-string dest charset))
470 (cond ((eq chr ?\n) "")
472 (t (char-to-string chr)))
474 (std11-unfold-string dest)
479 ;;; @ lexical analyze
482 (defvar eword-lexical-analyze-cache nil)
483 (defvar eword-lexical-analyze-cache-max 299
484 "*Max position of eword-lexical-analyze-cache.
485 It is max size of eword-lexical-analyze-cache - 1.")
487 (defcustom eword-lexical-analyzers
488 '(eword-analyze-quoted-string
489 eword-analyze-domain-literal
490 eword-analyze-comment
492 eword-analyze-special
493 eword-analyze-encoded-word
495 "*List of functions to return result of lexical analyze.
496 Each function must have two arguments: STRING and MUST-UNFOLD.
497 STRING is the target string to be analyzed.
498 If MUST-UNFOLD is not nil, each function must unfold and eliminate
499 bare-CR and bare-LF from the result even if they are included in
500 content of the encoded-word.
501 Each function must return nil if it can not analyze STRING as its
504 Previous function is preferred to next function. If a function
505 returns nil, next function is used. Otherwise the return value will
508 :type '(repeat function))
510 (defun eword-analyze-quoted-string (string &optional must-unfold)
511 (let ((p (std11-check-enclosure string ?\" ?\")))
513 (cons (cons 'quoted-string
514 (decode-mime-charset-string
515 (std11-strip-quoted-pair (substring string 1 (1- p)))
516 default-mime-charset))
517 (substring string p))
520 (defun eword-analyze-domain-literal (string &optional must-unfold)
521 (std11-analyze-domain-literal string))
523 (defun eword-analyze-comment (string &optional must-unfold)
524 (let ((p (std11-check-enclosure string ?\( ?\) t)))
528 (decode-mime-charset-string
529 (std11-strip-quoted-pair (substring string 1 (1- p)))
530 default-mime-charset)
532 (substring string p))
535 (defun eword-analyze-spaces (string &optional must-unfold)
536 (std11-analyze-spaces string))
538 (defun eword-analyze-special (string &optional must-unfold)
539 (std11-analyze-special string))
541 (defun eword-analyze-encoded-word (string &optional must-unfold)
542 (if (eq (string-match eword-encoded-word-regexp string) 0)
543 (let ((end (match-end 0))
544 (dest (eword-decode-encoded-word (match-string 0 string)
547 (setq string (substring string end))
548 (while (eq (string-match `,(concat "[ \t\n]*\\("
549 eword-encoded-word-regexp
553 (setq end (match-end 0))
556 (eword-decode-encoded-word (match-string 1 string)
558 string (substring string end))
560 (cons (cons 'atom dest) string)
563 (defun eword-analyze-atom (string &optional must-unfold)
564 (if (string-match std11-atom-regexp string)
565 (let ((end (match-end 0)))
566 (cons (cons 'atom (decode-mime-charset-string
567 (substring string 0 end)
568 default-mime-charset))
569 (substring string end)
572 (defun eword-lexical-analyze-internal (string must-unfold)
574 (while (not (string-equal string ""))
576 (let ((rest eword-lexical-analyzers)
578 (while (and (setq func (car rest))
579 (null (setq r (funcall func string must-unfold)))
581 (setq rest (cdr rest)))
582 (or r `((error . ,string) . ""))
584 (setq dest (cons (car ret) dest))
585 (setq string (cdr ret))
590 (defun eword-lexical-analyze (string &optional must-unfold)
591 "Return lexical analyzed list corresponding STRING.
592 It is like std11-lexical-analyze, but it decodes non us-ascii
593 characters encoded as encoded-words or invalid \"raw\" format.
594 \"Raw\" non us-ascii characters are regarded as variable
595 `default-mime-charset'."
596 (let ((key (copy-sequence string))
598 (set-text-properties 0 (length key) nil key)
599 (if (setq ret (assoc key eword-lexical-analyze-cache))
601 (setq ret (eword-lexical-analyze-internal key must-unfold))
602 (setq eword-lexical-analyze-cache
604 (last eword-lexical-analyze-cache
605 eword-lexical-analyze-cache-max)))
608 (defun eword-decode-token (token)
609 (let ((type (car token))
611 (cond ((eq type 'quoted-string)
612 (std11-wrap-as-quoted-string value))
614 (concat "(" (std11-wrap-as-quoted-pairs value '(?( ?))) ")"))
617 (defun eword-extract-address-components (string)
618 "Extract full name and canonical address from STRING.
619 Returns a list of the form (FULL-NAME CANONICAL-ADDRESS).
620 If no name can be extracted, FULL-NAME will be nil.
621 It decodes non us-ascii characters in FULL-NAME encoded as
622 encoded-words or invalid \"raw\" string. \"Raw\" non us-ascii
623 characters are regarded as variable `default-mime-charset'."
624 (let* ((structure (car (std11-parse-address
625 (eword-lexical-analyze
626 (std11-unfold-string string) 'must-unfold))))
627 (phrase (std11-full-name-string structure))
628 (address (std11-address-string structure))
630 (list phrase address)
637 (provide 'eword-decode)
639 ;;; eword-decode.el ends here