Synch to No Gnus 200401082352.
[elisp/gnus.git-] / lisp / spam.el
1 ;;; spam.el --- Identifying spam
2 ;; Copyright (C) 2002, 2003, 2004 Free Software Foundation, Inc.
3
4 ;; Author: Lars Magne Ingebrigtsen <larsi@gnus.org>
5 ;; Keywords: network
6
7 ;; This file is part of GNU Emacs.
8
9 ;; GNU Emacs is free software; you can redistribute it and/or modify
10 ;; it under the terms of the GNU General Public License as published by
11 ;; the Free Software Foundation; either version 2, or (at your option)
12 ;; any later version.
13
14 ;; GNU Emacs is distributed in the hope that it will be useful,
15 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
16 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17 ;; GNU General Public License for more details.
18
19 ;; You should have received a copy of the GNU General Public License
20 ;; along with GNU Emacs; see the file COPYING.  If not, write to the
21 ;; Free Software Foundation, Inc., 59 Temple Place - Suite 330,
22 ;; Boston, MA 02111-1307, USA.
23
24 ;;; Commentary:
25
26 ;;; This module addresses a few aspects of spam control under Gnus.  Page
27 ;;; breaks are used for grouping declarations and documentation relating to
28 ;;; each particular aspect.
29
30 ;;; The integration with Gnus is not yet complete.  See various `FIXME'
31 ;;; comments, below, for supplementary explanations or discussions.
32
33 ;;; Several TODO items are marked as such
34
35 ;; TODO: spam scores, detection of spam in newsgroups, cross-server splitting,
36 ;; remote processing, training through files
37
38 ;;; Code:
39
40 (require 'path-util)
41
42 (eval-when-compile (require 'cl))
43
44 (require 'gnus-sum)
45
46 (require 'gnus-uu)                      ; because of key prefix issues
47 ;;; for the definitions of group content classification and spam processors
48 (require 'gnus)
49 (require 'message)              ;for the message-fetch-field functions
50
51 ;; for nnimap-split-download-body-default
52 (eval-when-compile (require 'nnimap))
53
54 ;; autoload query-dig
55 (eval-and-compile
56   (autoload 'query-dig "dig"))
57
58 ;; autoload spam-report
59 (eval-and-compile
60   (autoload 'spam-report-gmane "spam-report"))
61
62 ;; autoload gnus-registry
63 (eval-and-compile
64   (autoload 'gnus-registry-group-count "gnus-registry")
65   (autoload 'gnus-registry-add-group "gnus-registry")
66   (autoload 'gnus-registry-store-extra-entry "gnus-registry")
67   (autoload 'gnus-registry-fetch-extra "gnus-registry"))
68
69 ;; autoload query-dns
70 (eval-and-compile
71   (autoload 'query-dns "dns"))
72
73 ;;; Main parameters.
74
75 (defgroup spam nil
76   "Spam configuration.")
77
78 (defcustom spam-directory "~/News/spam/"
79   "Directory for spam whitelists and blacklists."
80   :type 'directory
81   :group 'spam)
82
83 (defcustom spam-move-spam-nonspam-groups-only t
84   "Whether spam should be moved in non-spam groups only.
85 When t, only ham and unclassified groups will have their spam moved
86 to the spam-process-destination.  When nil, spam will also be moved from
87 spam groups."
88   :type 'boolean
89   :group 'spam)
90
91 (defcustom spam-process-ham-in-nonham-groups nil
92   "Whether ham should be processed in non-ham groups."
93   :type 'boolean
94   :group 'spam)
95
96 (defcustom spam-log-to-registry nil
97   "Whether spam/ham processing should be logged in the registry."
98   :type 'boolean
99   :group 'spam)
100
101 (defcustom spam-split-symbolic-return nil
102   "Whether `spam-split' should work with symbols or group names."
103   :type 'boolean
104   :group 'spam)
105
106 (defcustom spam-split-symbolic-return-positive nil
107   "Whether `spam-split' should ALWAYS work with symbols or group names.
108 Do not set this if you use `spam-split' in a fancy split
109   method."
110   :type 'boolean
111   :group 'spam)
112
113 (defcustom spam-process-ham-in-spam-groups nil
114   "Whether ham should be processed in spam groups."
115   :type 'boolean
116   :group 'spam)
117
118 (defcustom spam-mark-only-unseen-as-spam t
119   "Whether only unseen articles should be marked as spam in spam groups.
120 When nil, all unread articles in a spam group are marked as
121 spam.  Set this if you want to leave an article unread in a spam group
122 without losing it to the automatic spam-marking process."
123   :type 'boolean
124   :group 'spam)
125
126 (defcustom spam-mark-ham-unread-before-move-from-spam-group nil
127   "Whether ham should be marked unread before it's moved.
128 The article is moved out of a spam group according to ham-process-destination.
129 This variable is an official entry in the international Longest Variable Name
130 Competition."
131   :type 'boolean
132   :group 'spam)
133
134 (defcustom spam-disable-spam-split-during-ham-respool nil
135   "Whether `spam-split' should be ignored while resplitting ham.
136 This is useful to prevent ham from ending up in the same spam
137 group after the resplit.  Don't set this to t if you have `spam-split' as the
138 last rule in your split configuration."
139   :type 'boolean
140   :group 'spam)
141
142 (defcustom spam-autodetect-recheck-messages nil
143   "Should spam.el recheck all meessages when autodetecting?
144 Normally this is nil, so only unseen messages will be checked."
145   :type 'boolean
146   :group 'spam)
147
148 (defcustom spam-whitelist (expand-file-name "whitelist" spam-directory)
149   "The location of the whitelist.
150 The file format is one regular expression per line.
151 The regular expression is matched against the address."
152   :type 'file
153   :group 'spam)
154
155 (defcustom spam-blacklist (expand-file-name "blacklist" spam-directory)
156   "The location of the blacklist.
157 The file format is one regular expression per line.
158 The regular expression is matched against the address."
159   :type 'file
160   :group 'spam)
161
162 (defcustom spam-use-dig t
163   "Whether `query-dig' should be used instead of `query-dns'."
164   :type 'boolean
165   :group 'spam)
166
167 (defcustom spam-use-blacklist nil
168   "Whether the blacklist should be used by `spam-split'."
169   :type 'boolean
170   :group 'spam)
171
172 (defcustom spam-blacklist-ignored-regexes nil
173   "Regular expressions that the blacklist should ignore."
174   :type '(repeat (regexp :tag "Regular expression to ignore when blacklisting"))
175   :group 'spam)
176
177 (defcustom spam-use-whitelist nil
178   "Whether the whitelist should be used by `spam-split'."
179   :type 'boolean
180   :group 'spam)
181
182 (defcustom spam-use-whitelist-exclusive nil
183   "Whether whitelist-exclusive should be used by `spam-split'.
184 Exclusive whitelisting means that all messages from senders not in the whitelist
185 are considered spam."
186   :type 'boolean
187   :group 'spam)
188
189 (defcustom spam-use-blackholes nil
190   "Whether blackholes should be used by `spam-split'."
191   :type 'boolean
192   :group 'spam)
193
194 (defcustom spam-use-hashcash nil
195   "Whether hashcash payments should be detected by `spam-split'."
196   :type 'boolean
197   :group 'spam)
198
199 (defcustom spam-use-regex-headers nil
200   "Whether a header regular expression match should be used by `spam-split'.
201 Also see the variables `spam-regex-headers-spam' and `spam-regex-headers-ham'."
202   :type 'boolean
203   :group 'spam)
204
205 (defcustom spam-use-regex-body nil
206   "Whether a body regular expression match should be used by `spam-split'.
207 Also see the variables `spam-regex-body-spam' and `spam-regex-body-ham'."
208   :type 'boolean
209   :group 'spam)
210
211 (defcustom spam-use-bogofilter-headers nil
212   "Whether bogofilter headers should be used by `spam-split'.
213 Enable this if you pre-process messages with Bogofilter BEFORE Gnus sees them."
214   :type 'boolean
215   :group 'spam)
216
217 (defcustom spam-use-bogofilter nil
218   "Whether bogofilter should be invoked by `spam-split'.
219 Enable this if you want Gnus to invoke Bogofilter on new messages."
220   :type 'boolean
221   :group 'spam)
222
223 (defcustom spam-use-BBDB nil
224   "Whether BBDB should be used by `spam-split'."
225   :type 'boolean
226   :group 'spam)
227
228 (defcustom spam-use-BBDB-exclusive nil
229   "Whether BBDB-exclusive should be used by `spam-split'.
230 Exclusive BBDB means that all messages from senders not in the BBDB are
231 considered spam."
232   :type 'boolean
233   :group 'spam)
234
235 (defcustom spam-use-ifile nil
236   "Whether ifile should be used by `spam-split'."
237   :type 'boolean
238   :group 'spam)
239
240 (defcustom spam-use-stat nil
241   "Whether `spam-stat' should be used by `spam-split'."
242   :type 'boolean
243   :group 'spam)
244
245 (defcustom spam-use-spamoracle nil
246   "Whether spamoracle should be used by `spam-split'."
247   :type 'boolean
248   :group 'spam)
249
250 (defcustom spam-install-hooks (or
251                                spam-use-dig
252                                spam-use-blacklist
253                                spam-use-whitelist
254                                spam-use-whitelist-exclusive
255                                spam-use-blackholes
256                                spam-use-hashcash
257                                spam-use-regex-headers
258                                spam-use-regex-body
259                                spam-use-bogofilter-headers
260                                spam-use-bogofilter
261                                spam-use-BBDB
262                                spam-use-BBDB-exclusive
263                                spam-use-ifile
264                                spam-use-stat
265                                spam-use-spamoracle)
266   "Whether the spam hooks should be installed.
267 Default to t if one of the spam-use-* variables is set."
268   :group 'spam
269   :type 'boolean)
270
271 (defcustom spam-split-group "spam"
272   "Group name where incoming spam should be put by `spam-split'."
273   :type 'string
274   :group 'spam)
275
276 ;;; TODO: deprecate this variable, it's confusing since it's a list of strings,
277 ;;; not regular expressions
278 (defcustom spam-junk-mailgroups (cons
279                                  spam-split-group
280                                  '("mail.junk" "poste.pourriel"))
281   "Mailgroups with spam contents.
282 All unmarked article in such group receive the spam mark on group entry."
283   :type '(repeat (string :tag "Group"))
284   :group 'spam)
285
286 (defcustom spam-blackhole-servers '("bl.spamcop.net" "relays.ordb.org"
287                                     "dev.null.dk" "relays.visi.com")
288   "List of blackhole servers."
289   :type '(repeat (string :tag "Server"))
290   :group 'spam)
291
292 (defcustom spam-blackhole-good-server-regex nil
293   "String matching IP addresses that should not be checked in the blackholes."
294   :type '(radio (const nil)
295                 (regexp :format "%t: %v\n" :size 0))
296   :group 'spam)
297
298 (defcustom spam-face 'gnus-splash-face
299   "Face for spam-marked articles."
300   :type 'face
301   :group 'spam)
302
303 (defcustom spam-regex-headers-spam '("^X-Spam-Flag: YES")
304   "Regular expression for positive header spam matches."
305   :type '(repeat (regexp :tag "Regular expression to match spam header"))
306   :group 'spam)
307
308 (defcustom spam-regex-headers-ham '("^X-Spam-Flag: NO")
309   "Regular expression for positive header ham matches."
310   :type '(repeat (regexp :tag "Regular expression to match ham header"))
311   :group 'spam)
312
313 (defcustom spam-regex-body-spam '()
314   "Regular expression for positive body spam matches."
315   :type '(repeat (regexp :tag "Regular expression to match spam body"))
316   :group 'spam)
317
318 (defcustom spam-regex-body-ham '()
319   "Regular expression for positive body ham matches."
320   :type '(repeat (regexp :tag "Regular expression to match ham body"))
321   :group 'spam)
322
323 (defgroup spam-ifile nil
324   "Spam ifile configuration."
325   :group 'spam)
326
327 (defcustom spam-ifile-path (exec-installed-p "ifile")
328   "File path of the ifile executable program."
329   :type '(choice (file :tag "Location of ifile")
330                  (const :tag "ifile is not installed"))
331   :group 'spam-ifile)
332
333 (defcustom spam-ifile-database-path nil
334   "File path of the ifile database."
335   :type '(choice (file :tag "Location of the ifile database")
336                  (const :tag "Use the default"))
337   :group 'spam-ifile)
338
339 (defcustom spam-ifile-spam-category "spam"
340   "Name of the spam ifile category."
341   :type 'string
342   :group 'spam-ifile)
343
344 (defcustom spam-ifile-ham-category nil
345   "Name of the ham ifile category.
346 If nil, the current group name will be used."
347   :type '(choice (string :tag "Use a fixed category")
348                  (const :tag "Use the current group name"))
349   :group 'spam-ifile)
350
351 (defcustom spam-ifile-all-categories nil
352   "Whether the ifile check will return all categories, or just spam.
353 Set this to t if you want to use the `spam-split' invocation of ifile as
354 your main source of newsgroup names."
355   :type 'boolean
356   :group 'spam-ifile)
357
358 (defgroup spam-bogofilter nil
359   "Spam bogofilter configuration."
360   :group 'spam)
361
362 (defcustom spam-bogofilter-path (exec-installed-p "bogofilter")
363   "File path of the Bogofilter executable program."
364   :type '(choice (file :tag "Location of bogofilter")
365                  (const :tag "Bogofilter is not installed"))
366   :group 'spam-bogofilter)
367
368 (defcustom spam-bogofilter-header "X-Bogosity"
369   "The header that Bogofilter inserts in messages."
370   :type 'string
371   :group 'spam-bogofilter)
372
373 (defcustom spam-bogofilter-spam-switch "-s"
374   "The switch that Bogofilter uses to register spam messages."
375   :type 'string
376   :group 'spam-bogofilter)
377
378 (defcustom spam-bogofilter-ham-switch "-n"
379   "The switch that Bogofilter uses to register ham messages."
380   :type 'string
381   :group 'spam-bogofilter)
382
383 (defcustom spam-bogofilter-spam-strong-switch "-S"
384   "The switch that Bogofilter uses to unregister ham messages."
385   :type 'string
386   :group 'spam-bogofilter)
387
388 (defcustom spam-bogofilter-ham-strong-switch "-N"
389   "The switch that Bogofilter uses to unregister spam messages."
390   :type 'string
391   :group 'spam-bogofilter)
392
393 (defcustom spam-bogofilter-bogosity-positive-spam-header "^\\(Yes\\|Spam\\)"
394   "The regex on `spam-bogofilter-header' for positive spam identification."
395   :type 'regexp
396   :group 'spam-bogofilter)
397
398 (defcustom spam-bogofilter-database-directory nil
399   "Directory path of the Bogofilter databases."
400   :type '(choice (directory
401                   :tag "Location of the Bogofilter database directory")
402                  (const :tag "Use the default"))
403   :group 'spam-bogofilter)
404
405 (defgroup spam-spamoracle nil
406   "Spam spamoracle configuration."
407   :group 'spam)
408
409 (defcustom spam-spamoracle-database nil
410   "Location of spamoracle database file.
411 When nil, use the default spamoracle database."
412   :type '(choice (directory :tag "Location of spamoracle database file.")
413                  (const :tag "Use the default"))
414   :group 'spam-spamoracle)
415
416 (defcustom spam-spamoracle-binary (executable-find "spamoracle")
417   "Location of the spamoracle binary."
418   :type '(choice (directory :tag "Location of the spamoracle binary")
419                  (const :tag "Use the default"))
420   :group 'spam-spamoracle)
421
422 ;;; Key bindings for spam control.
423
424 (gnus-define-keys gnus-summary-mode-map
425   "St" spam-bogofilter-score
426   "Sx" gnus-summary-mark-as-spam
427   "Mst" spam-bogofilter-score
428   "Msx" gnus-summary-mark-as-spam
429   "\M-d" gnus-summary-mark-as-spam)
430
431 (defvar spam-cache-lookups t
432   "Whether spam.el will try to cache lookups using `spam-caches'.")
433
434 (defvar spam-caches (make-hash-table
435                      :size 10
436                      :test 'equal)
437   "Cache of spam detection entries.")
438
439 (defvar spam-old-ham-articles nil
440   "List of old ham articles, generated when a group is entered.")
441
442 (defvar spam-old-spam-articles nil
443   "List of old spam articles, generated when a group is entered.")
444
445 (defvar spam-split-disabled nil
446   "If non-nil, `spam-split' is disabled, and always returns nil.")
447
448 (defvar spam-split-last-successful-check nil
449   "Internal variable.
450 `spam-split' will set this to nil or a spam-use-XYZ check if it
451 finds ham or spam.")
452
453 ;; convenience functions
454 (defun spam-clear-cache (symbol)
455   "Clear the spam-caches entry for a check."
456   (remhash symbol spam-caches))
457
458 (defun spam-xor (a b)
459   "Logical A xor B."
460   (and (or a b) (not (and a b))))
461
462 (defun spam-group-ham-mark-p (group mark &optional spam)
463   "Checks if MARK is considered a ham mark in GROUP."
464   (when (stringp group)
465     (let* ((marks (spam-group-ham-marks group spam))
466            (marks (if (symbolp mark)
467                       marks
468                     (mapcar 'symbol-value marks))))
469       (memq mark marks))))
470
471 (defun spam-group-spam-mark-p (group mark)
472   "Checks if MARK is considered a spam mark in GROUP."
473   (spam-group-ham-mark-p group mark t))
474
475 (defun spam-group-ham-marks (group &optional spam)
476   "In GROUP, get all the ham marks."
477   (when (stringp group)
478     (let* ((marks (if spam
479                       (gnus-parameter-spam-marks group)
480                     (gnus-parameter-ham-marks group)))
481            (marks (car marks))
482            (marks (if (listp (car marks)) (car marks) marks)))
483       marks)))
484
485 (defun spam-group-spam-marks (group)
486   "In GROUP, get all the spam marks."
487   (spam-group-ham-marks group t))
488
489 (defun spam-group-spam-contents-p (group)
490   "Is GROUP a spam group?"
491   (if (stringp group)
492       (or (member group spam-junk-mailgroups)
493           (memq 'gnus-group-spam-classification-spam
494                 (gnus-parameter-spam-contents group)))
495     nil))
496
497 (defun spam-group-ham-contents-p (group)
498   "Is GROUP a ham group?"
499   (if (stringp group)
500       (memq 'gnus-group-spam-classification-ham
501             (gnus-parameter-spam-contents group))
502     nil))
503
504 (defvar spam-list-of-processors
505   '((gnus-group-spam-exit-processor-report-gmane spam spam-use-gmane)
506     (gnus-group-spam-exit-processor-bogofilter   spam spam-use-bogofilter)
507     (gnus-group-spam-exit-processor-blacklist    spam spam-use-blacklist)
508     (gnus-group-spam-exit-processor-ifile        spam spam-use-ifile)
509     (gnus-group-spam-exit-processor-stat         spam spam-use-stat)
510     (gnus-group-spam-exit-processor-spamoracle   spam spam-use-spamoracle)
511     (gnus-group-ham-exit-processor-ifile         ham spam-use-ifile)
512     (gnus-group-ham-exit-processor-bogofilter    ham spam-use-bogofilter)
513     (gnus-group-ham-exit-processor-stat          ham spam-use-stat)
514     (gnus-group-ham-exit-processor-whitelist     ham spam-use-whitelist)
515     (gnus-group-ham-exit-processor-BBDB          ham spam-use-BBDB)
516     (gnus-group-ham-exit-processor-copy          ham spam-use-ham-copy)
517     (gnus-group-ham-exit-processor-spamoracle    ham spam-use-spamoracle))
518   "The `spam-list-of-processors' list.
519 This list contains pairs associating a ham/spam exit processor
520 variable with a classification and a spam-use-* variable.")
521
522 (defun spam-group-processor-p (group processor)
523   (if (and (stringp group)
524            (symbolp processor))
525       (or (member processor (nth 0 (gnus-parameter-spam-process group)))
526           (spam-group-processor-multiple-p
527            group
528            (cdr-safe (assoc processor spam-list-of-processors))))
529     nil))
530
531 (defun spam-group-processor-multiple-p (group processor-info)
532   (let* ((classification (nth 0 processor-info))
533          (check (nth 1 processor-info))
534          (parameters (nth 0 (gnus-parameter-spam-process group)))
535          found)
536     (dolist (parameter parameters)
537       (when (and (null found)
538                  (listp parameter)
539                  (eq classification (nth 0 parameter))
540                  (eq check (nth 1 parameter)))
541         (setq found t)))
542     found))
543
544 (defun spam-group-spam-processor-report-gmane-p (group)
545   (spam-group-processor-p group 'gnus-group-spam-exit-processor-report-gmane))
546
547 (defun spam-group-spam-processor-bogofilter-p (group)
548   (spam-group-processor-p group 'gnus-group-spam-exit-processor-bogofilter))
549
550 (defun spam-group-spam-processor-blacklist-p (group)
551   (spam-group-processor-p group 'gnus-group-spam-exit-processor-blacklist))
552
553 (defun spam-group-spam-processor-ifile-p (group)
554   (spam-group-processor-p group 'gnus-group-spam-exit-processor-ifile))
555
556 (defun spam-group-ham-processor-ifile-p (group)
557   (spam-group-processor-p group 'gnus-group-ham-exit-processor-ifile))
558
559 (defun spam-group-spam-processor-spamoracle-p (group)
560   (spam-group-processor-p group 'gnus-group-spam-exit-processor-spamoracle))
561
562 (defun spam-group-ham-processor-bogofilter-p (group)
563   (spam-group-processor-p group 'gnus-group-ham-exit-processor-bogofilter))
564
565 (defun spam-group-spam-processor-stat-p (group)
566   (spam-group-processor-p group 'gnus-group-spam-exit-processor-stat))
567
568 (defun spam-group-ham-processor-stat-p (group)
569   (spam-group-processor-p group 'gnus-group-ham-exit-processor-stat))
570
571 (defun spam-group-ham-processor-whitelist-p (group)
572   (spam-group-processor-p group 'gnus-group-ham-exit-processor-whitelist))
573
574 (defun spam-group-ham-processor-BBDB-p (group)
575   (spam-group-processor-p group 'gnus-group-ham-exit-processor-BBDB))
576
577 (defun spam-group-ham-processor-copy-p (group)
578   (spam-group-processor-p group 'gnus-group-ham-exit-processor-copy))
579
580 (defun spam-group-ham-processor-spamoracle-p (group)
581   (spam-group-processor-p group 'gnus-group-ham-exit-processor-spamoracle))
582
583 (defun spam-report-articles-gmane (n)
584   "Report the current message as spam.
585 Respects the process/prefix convention."
586   (interactive "P")
587   (dolist (article (gnus-summary-work-articles n))
588     (gnus-summary-remove-process-mark article)
589     (spam-report-gmane article)))
590
591 ;;; Summary entry and exit processing.
592
593 (defun spam-summary-prepare ()
594   (setq spam-old-ham-articles
595         (spam-list-articles gnus-newsgroup-articles 'ham))
596   (setq spam-old-spam-articles
597         (spam-list-articles gnus-newsgroup-articles 'spam))
598   (spam-mark-junk-as-spam-routine))
599
600 ;; The spam processors are invoked for any group, spam or ham or neither
601 (defun spam-summary-prepare-exit ()
602   (unless gnus-group-is-exiting-without-update-p
603     (gnus-message 6 "Exiting summary buffer and applying spam rules")
604
605     ;; first of all, unregister any articles that are no longer ham or spam
606     ;; we have to iterate over the processors, or else we'll be too slow
607     (dolist (classification '(spam ham))
608       (let* ((old-articles (if (eq classification 'spam)
609                                spam-old-spam-articles
610                              spam-old-ham-articles))
611              (new-articles (spam-list-articles
612                             gnus-newsgroup-articles
613                             classification))
614              (changed-articles (gnus-set-difference old-articles new-articles)))
615         ;; now that we have the changed articles, we go through the processors
616         (dolist (processor-param spam-list-of-processors)
617           (let ((processor (nth 0 processor-param))
618                 (processor-classification (nth 1 processor-param))
619                 (check (nth 2 processor-param))
620                 unregister-list)
621             (dolist (article changed-articles)
622               (let ((id (spam-fetch-field-message-id-fast article)))
623                 (when (spam-log-unregistration-needed-p
624                        id 'process classification check)
625                   (push article unregister-list))))
626             ;; call spam-register-routine with specific articles to unregister,
627             ;; when there are articles to unregister and the check is enabled
628             (when (and unregister-list (symbol-value check))
629               (spam-register-routine classification check t unregister-list))))))
630
631     ;; find all the spam processors applicable to this group
632     (dolist (processor-param spam-list-of-processors)
633       (let ((processor (nth 0 processor-param))
634             (classification (nth 1 processor-param))
635             (check (nth 2 processor-param)))
636         (when (and (eq 'spam classification)
637                    (spam-group-processor-p gnus-newsgroup-name processor))
638           (spam-register-routine classification check))))
639
640     (if spam-move-spam-nonspam-groups-only
641         (when (not (spam-group-spam-contents-p gnus-newsgroup-name))
642           (spam-mark-spam-as-expired-and-move-routine
643            (gnus-parameter-spam-process-destination gnus-newsgroup-name)))
644       (gnus-message 5 "Marking spam as expired and moving it to %s"
645                     gnus-newsgroup-name)
646       (spam-mark-spam-as-expired-and-move-routine
647        (gnus-parameter-spam-process-destination gnus-newsgroup-name)))
648
649     ;; now we redo spam-mark-spam-as-expired-and-move-routine to only
650     ;; expire spam, in case the above did not expire them
651     (gnus-message 5 "Marking spam as expired without moving it")
652     (spam-mark-spam-as-expired-and-move-routine nil)
653
654     (when (or (spam-group-ham-contents-p gnus-newsgroup-name)
655               (and (spam-group-spam-contents-p gnus-newsgroup-name)
656                    spam-process-ham-in-spam-groups)
657               spam-process-ham-in-nonham-groups)
658       ;; find all the ham processors applicable to this group
659       (dolist (processor-param spam-list-of-processors)
660         (let ((processor (nth 0 processor-param))
661               (classification (nth 1 processor-param))
662               (check (nth 2 processor-param)))
663           (when (and (eq 'ham classification)
664                      (spam-group-processor-p gnus-newsgroup-name processor))
665             (spam-register-routine classification check)))))
666
667     (when (spam-group-ham-processor-copy-p gnus-newsgroup-name)
668       (gnus-message 5 "Copying ham")
669       (spam-ham-copy-routine
670        (gnus-parameter-ham-process-destination gnus-newsgroup-name)))
671
672     ;; now move all ham articles out of spam groups
673     (when (spam-group-spam-contents-p gnus-newsgroup-name)
674       (gnus-message 5 "Moving ham messages from spam group")
675       (spam-ham-move-routine
676        (gnus-parameter-ham-process-destination gnus-newsgroup-name))))
677
678   (setq spam-old-ham-articles nil)
679   (setq spam-old-spam-articles nil))
680
681 (defun spam-mark-junk-as-spam-routine ()
682   ;; check the global list of group names spam-junk-mailgroups and the
683   ;; group parameters
684   (when (spam-group-spam-contents-p gnus-newsgroup-name)
685     (gnus-message 5 "Marking %s articles as spam"
686                   (if spam-mark-only-unseen-as-spam
687                       "unseen"
688                     "unread"))
689     (let ((articles (if spam-mark-only-unseen-as-spam
690                         gnus-newsgroup-unseen
691                       gnus-newsgroup-unreads)))
692       (dolist (article articles)
693         (gnus-summary-mark-article article gnus-spam-mark)))))
694
695 (defun spam-mark-spam-as-expired-and-move-routine (&rest groups)
696   (if (and (car-safe groups) (listp (car-safe groups)))
697       (apply 'spam-mark-spam-as-expired-and-move-routine (car groups))
698     (gnus-summary-kill-process-mark)
699     (let ((articles gnus-newsgroup-articles)
700           (backend-supports-deletions
701            (gnus-check-backend-function
702             'request-move-article gnus-newsgroup-name))
703           article tomove deletep)
704       (dolist (article articles)
705         (when (eq (gnus-summary-article-mark article) gnus-spam-mark)
706           (gnus-summary-mark-article article gnus-expirable-mark)
707           (push article tomove)))
708
709       ;; now do the actual copies
710       (dolist (group groups)
711         (when (and tomove
712                    (stringp group))
713           (dolist (article tomove)
714             (gnus-summary-set-process-mark article))
715           (when tomove
716             (if (or (not backend-supports-deletions)
717                     (> (length groups) 1))
718                 (progn
719                   (gnus-summary-copy-article nil group)
720                   (setq deletep t))
721               (gnus-summary-move-article nil group)))))
722
723       ;; now delete the articles, if there was a copy done, and the
724       ;; backend allows it
725       (when (and deletep backend-supports-deletions)
726         (dolist (article tomove)
727           (gnus-summary-set-process-mark article))
728         (when tomove
729           (let ((gnus-novice-user nil)) ; don't ask me if I'm sure
730             (gnus-summary-delete-article nil))))
731
732       (gnus-summary-yank-process-mark))))
733
734 (defun spam-ham-copy-or-move-routine (copy groups)
735   (gnus-summary-kill-process-mark)
736   (let ((todo (spam-list-articles gnus-newsgroup-articles 'ham))
737         (backend-supports-deletions
738          (gnus-check-backend-function
739           'request-move-article gnus-newsgroup-name))
740         (respool-method (gnus-find-method-for-group gnus-newsgroup-name))
741         article mark todo deletep respool)
742
743     (when (member 'respool groups)
744       (setq respool t)                  ; boolean for later
745       (setq groups '("fake"))) ; when respooling, groups are dynamic so fake it
746
747     ;; now do the actual move
748     (dolist (group groups)
749       (when (and todo (stringp group))
750         (dolist (article todo)
751           (when spam-mark-ham-unread-before-move-from-spam-group
752             (gnus-summary-mark-article article gnus-unread-mark))
753           (gnus-summary-set-process-mark article))
754
755         (if respool                        ; respooling is with a "fake" group
756             (let ((spam-split-disabled
757                    (or spam-split-disabled
758                        spam-disable-spam-split-during-ham-respool)))
759               (gnus-summary-respool-article nil respool-method))
760           (if (or (not backend-supports-deletions) ; else, we are not respooling
761                   (> (length groups) 1))
762               (progn                ; if copying, copy and set deletep
763                 (gnus-summary-copy-article nil group)
764                 (setq deletep t))
765             (gnus-summary-move-article nil group))))) ; else move articles
766
767     ;; now delete the articles, unless a) copy is t, and there was a copy done
768     ;;                                 b) a move was done to a single group
769     ;;                                 c) backend-supports-deletions is nil
770     (unless copy
771       (when (and deletep backend-supports-deletions)
772         (dolist (article todo)
773           (gnus-summary-set-process-mark article))
774         (when todo
775           (let ((gnus-novice-user nil)) ; don't ask me if I'm sure
776             (gnus-summary-delete-article nil))))))
777
778   (gnus-summary-yank-process-mark))
779
780 (defun spam-ham-copy-routine (&rest groups)
781   (if (and (car-safe groups) (listp (car-safe groups)))
782       (apply 'spam-ham-copy-routine (car groups))
783     (spam-ham-copy-or-move-routine t groups)))
784
785 (defun spam-ham-move-routine (&rest groups)
786   (if (and (car-safe groups) (listp (car-safe groups)))
787       (apply 'spam-ham-move-routine (car groups))
788     (spam-ham-copy-or-move-routine nil groups)))
789
790 (defun spam-get-article-as-string (article)
791   (let ((article-buffer (spam-get-article-as-buffer article))
792         article-string)
793     (when article-buffer
794       (save-window-excursion
795         (set-buffer article-buffer)
796         (setq article-string (buffer-string))))
797     article-string))
798
799 (defun spam-get-article-as-buffer (article)
800   (let ((article-buffer))
801     (when (numberp article)
802       (save-window-excursion
803         (gnus-summary-goto-subject article)
804         (gnus-summary-show-article t)
805         (setq article-buffer (get-buffer gnus-article-buffer))))
806     article-buffer))
807
808 ;; disabled for now
809 ;; (defun spam-get-article-as-filename (article)
810 ;;   (let ((article-filename))
811 ;;     (when (numberp article)
812 ;;       (nnml-possibly-change-directory
813 ;;        (gnus-group-real-name gnus-newsgroup-name))
814 ;;       (setq article-filename (expand-file-name
815 ;;                              (int-to-string article) nnml-current-directory)))
816 ;;     (if (file-exists-p article-filename)
817 ;;      article-filename
818 ;;       nil)))
819
820 (defun spam-fetch-field-fast (article field)
821   "Fetch a field quickly, using the internal gnus-data-list function"
822   (when (numberp article)
823     (let* ((header (assoc article (gnus-data-list nil)))
824            (data-header (if header (gnus-data-header header) nil)))
825       (cond
826        ((equal field 'from)
827         (mail-header-from data-header))
828        ((equal field 'message-id)
829         (mail-header-message-id data-header))
830        ((equal field 'subject)
831         (mail-header-subject data-header))
832        ((equal field 'references)
833         (mail-header-references data-header))
834        ((equal field 'date)
835         (mail-header-date data-header))
836        ((equal field 'xref)
837         (mail-header-xref data-header))
838        ((equal field 'extra)
839         (mail-header-extra data-header))
840        (t
841         nil)))))
842
843 (defun spam-fetch-field-from-fast (article)
844   (spam-fetch-field-fast article 'from))
845
846 (defun spam-fetch-field-subject-fast (article)
847   (spam-fetch-field-fast article 'subject))
848
849 (defun spam-fetch-field-message-id-fast (article)
850   (spam-fetch-field-fast article 'message-id))
851
852 (defun spam-insert-fake-headers (article)
853   (insert (format "From: %s\n" (spam-fetch-field-fast article 'from)))
854   (insert (format "Subject: %s\n" (spam-fetch-field-fast article 'subject)))
855   (insert (format "Message-ID: %s\n" (spam-fetch-field-fast article 'message-id)))
856   (insert (format "Date: %s\n" (spam-fetch-field-fast article 'date)))
857   (insert (format "References: %s\n" (spam-fetch-field-fast article 'references)))
858   (insert (format "Xref: %s\n" (spam-fetch-field-fast article 'xref)))
859   (when (spam-fetch-field-fast article 'extra)
860     (insert (format "%s\n" (spam-fetch-field-fast article 'extra)))))
861
862 \f
863 ;;;; Spam determination.
864
865 (defvar spam-list-of-checks
866   '((spam-use-blacklist          . spam-check-blacklist)
867     (spam-use-regex-headers      . spam-check-regex-headers)
868     (spam-use-regex-body         . spam-check-regex-body)
869     (spam-use-whitelist          . spam-check-whitelist)
870     (spam-use-BBDB               . spam-check-BBDB)
871     (spam-use-ifile              . spam-check-ifile)
872     (spam-use-spamoracle         . spam-check-spamoracle)
873     (spam-use-stat               . spam-check-stat)
874     (spam-use-blackholes         . spam-check-blackholes)
875     (spam-use-hashcash           . spam-check-hashcash)
876     (spam-use-bogofilter-headers . spam-check-bogofilter-headers)
877     (spam-use-bogofilter         . spam-check-bogofilter))
878   "The spam-list-of-checks list contains pairs associating a
879 parameter variable with a spam checking function.  If the
880 parameter variable is true, then the checking function is called,
881 and its value decides what happens.  Each individual check may
882 return nil, t, or a mailgroup name.  The value nil means that the
883 check does not yield a decision, and so, that further checks are
884 needed.  The value t means that the message is definitely not
885 spam, and that further spam checks should be inhibited.
886 Otherwise, a mailgroup name or the symbol 'spam (depending on
887 spam-split-symbolic-return) is returned where the mail should go,
888 and further checks are also inhibited.  The usual mailgroup name
889 is the value of `spam-split-group', meaning that the message is
890 definitely a spam.")
891
892 (defvar spam-list-of-statistical-checks
893   '(spam-use-ifile
894     spam-use-regex-body
895     spam-use-stat
896     spam-use-bogofilter
897     spam-use-blackholes
898     spam-use-spamoracle)
899   "The spam-list-of-statistical-checks list contains all the mail
900 splitters that need to have the full message body available.
901 Note that you should fetch extra headers if you don't like this,
902 e.g. fetch the 'Received' header for spam-use-blackholes.")
903
904 (defun spam-split (&rest specific-checks)
905   "Split this message into the `spam' group if it is spam.
906 This function can be used as an entry in the variable `nnmail-split-fancy',
907 for example like this: (: spam-split).  It can take checks as
908 parameters.  A string as a parameter will set the
909 spam-split-group to that string.
910
911 See the Info node `(gnus)Fancy Mail Splitting' for more details."
912   (interactive)
913   (setq spam-split-last-successful-check nil)
914   (unless spam-split-disabled
915     (let ((spam-split-group-choice spam-split-group))
916       (dolist (check specific-checks)
917         (when (stringp check)
918           (setq spam-split-group-choice check)
919           (setq specific-checks (delq check specific-checks))))
920
921       (let ((spam-split-group spam-split-group-choice))
922         (save-excursion
923           (save-restriction
924             (dolist (check spam-list-of-statistical-checks)
925               (when (and (symbolp check)
926                          (or (symbol-value check)
927                              (memq check specific-checks)))
928                 (widen)
929                 (gnus-message 8 "spam-split: widening the buffer (%s requires it)"
930                               (symbol-name check))
931                 (return)))
932             ;;   (progn (widen) (debug (buffer-string)))
933             (let ((list-of-checks spam-list-of-checks)
934                   decision)
935               (while (and list-of-checks (not decision))
936                 (let ((pair (pop list-of-checks)))
937                   (when (or
938                          ;; either, given specific checks, this is one of them
939                          (and specific-checks (memq (car pair) specific-checks))
940                          ;; or, given no specific checks, spam-use-CHECK is set
941                          (and (null specific-checks) (symbol-value (car pair))))
942                     (gnus-message 5 "spam-split: calling the %s function"
943                                   (symbol-name (cdr pair)))
944                     (setq decision (funcall (cdr pair)))
945                     ;; if we got a decision at all, save the current check
946                     (when decision
947                       (setq spam-split-last-successful-check (car pair)))
948
949                     (when (eq decision 'spam)
950                       (unless spam-split-symbolic-return
951                         (gnus-error
952                          5
953                          (format "spam-split got %s but %s is nil"
954                                  (symbol-name decision)
955                                  (symbol-name spam-split-symbolic-return))))))))
956               (if (eq decision t)
957                   (if spam-split-symbolic-return-positive 'ham nil)
958                 decision))))))))
959
960 (defun spam-find-spam ()
961   "This function will detect spam in the current newsgroup using spam-split."
962   (interactive)
963
964   (let* ((group gnus-newsgroup-name)
965          (autodetect (gnus-parameter-spam-autodetect group))
966          (methods (gnus-parameter-spam-autodetect-methods group))
967          (first-method (nth 0 methods))
968          (articles (if spam-autodetect-recheck-messages
969                        gnus-newsgroup-articles
970                      gnus-newsgroup-unseen))
971          article-cannot-be-faked)
972
973     (dolist (check spam-list-of-statistical-checks)
974       (when (and (symbolp check)
975                  (memq check methods))
976         (setq article-cannot-be-faked t)
977         (return)))
978
979     (when (memq 'default methods)
980       (setq article-cannot-be-faked t))
981
982     (when (and autodetect
983                (not (equal first-method 'none)))
984     (mapcar
985      (lambda (article)
986        (let ((id (spam-fetch-field-message-id-fast article))
987              (subject (spam-fetch-field-subject-fast article))
988              (sender (spam-fetch-field-from-fast article))
989              registry-lookup)
990          
991          (unless id
992            (gnus-error 5 "Article %d has no message ID!" article))
993          
994          (when (and id spam-log-to-registry)
995            (setq registry-lookup (spam-log-registration-type id 'incoming))
996            (when registry-lookup
997              (gnus-message
998               9
999               "spam-find-spam: message %s was already registered incoming"
1000               id)))
1001
1002          (let* ((spam-split-symbolic-return t)
1003                 (spam-split-symbolic-return-positive t)
1004                 (split-return
1005                  (or registry-lookup
1006                      (with-temp-buffer
1007                        (if article-cannot-be-faked
1008                            (gnus-request-article-this-buffer
1009                             article
1010                             group)
1011                          (spam-insert-fake-headers article))
1012                        (if (or (null first-method)
1013                                (equal first-method 'default))
1014                            (spam-split)
1015                          (apply 'spam-split methods))))))
1016            (if (equal split-return 'spam)
1017                (gnus-summary-mark-article article gnus-spam-mark))
1018            
1019            (when (and id split-return spam-log-to-registry)
1020              (when (zerop (gnus-registry-group-count id))
1021                (gnus-registry-add-group
1022                 id group subject sender))
1023                
1024              (unless registry-lookup
1025                (spam-log-processing-to-registry
1026                 id
1027                 'incoming
1028                 split-return
1029                 spam-split-last-successful-check
1030                 group))))))
1031     articles))))
1032
1033 (defvar spam-registration-functions
1034   ;; first the ham register, second the spam register function
1035   ;; third the ham unregister, fourth the spam unregister function
1036   '((spam-use-blacklist  nil
1037                          spam-blacklist-register-routine
1038                          nil
1039                          spam-blacklist-unregister-routine)
1040     (spam-use-whitelist  spam-whitelist-register-routine
1041                          nil
1042                          spam-whitelist-unregister-routine
1043                          nil)
1044     (spam-use-BBDB       spam-BBDB-register-routine
1045                          nil
1046                          spam-BBDB-unregister-routine
1047                          nil)
1048     (spam-use-ifile      spam-ifile-register-ham-routine
1049                          spam-ifile-register-spam-routine
1050                          spam-ifile-unregister-ham-routine
1051                          spam-ifile-unregister-spam-routine)
1052     (spam-use-spamoracle spam-spamoracle-learn-ham
1053                          spam-spamoracle-learn-spam
1054                          spam-spamoracle-unlearn-ham
1055                          spam-spamoracle-unlearn-spam)
1056     (spam-use-stat       spam-stat-register-ham-routine
1057                          spam-stat-register-spam-routine
1058                          spam-stat-unregister-ham-routine
1059                          spam-stat-unregister-spam-routine)
1060     ;; note that spam-use-gmane is not a legitimate check
1061     (spam-use-gmane      nil
1062                          spam-report-gmane-register-routine
1063                          ;; does Gmane support unregistration?
1064                          nil
1065                          nil)
1066     (spam-use-bogofilter spam-bogofilter-register-ham-routine
1067                          spam-bogofilter-register-spam-routine
1068                          spam-bogofilter-unregister-ham-routine
1069                          spam-bogofilter-unregister-spam-routine))
1070   "The spam-registration-functions list contains pairs
1071 associating a parameter variable with the ham and spam
1072 registration functions, and the ham and spam unregistration
1073 functions")
1074
1075 (defun spam-classification-valid-p (classification)
1076   (or  (eq classification 'spam)
1077        (eq classification 'ham)))
1078
1079 (defun spam-process-type-valid-p (process-type)
1080   (or  (eq process-type 'incoming)
1081        (eq process-type 'process)))
1082
1083 (defun spam-registration-check-valid-p (check)
1084   (assoc check spam-registration-functions))
1085
1086 (defun spam-unregistration-check-valid-p (check)
1087   (assoc check spam-registration-functions))
1088
1089 (defun spam-registration-function (classification check)
1090   (let ((flist (cdr-safe (assoc check spam-registration-functions))))
1091     (if (eq classification 'spam)
1092         (nth 1 flist)
1093       (nth 0 flist))))
1094
1095 (defun spam-unregistration-function (classification check)
1096   (let ((flist (cdr-safe (assoc check spam-registration-functions))))
1097     (if (eq classification 'spam)
1098         (nth 3 flist)
1099       (nth 2 flist))))
1100
1101 (defun spam-list-articles (articles classification)
1102   (let ((mark-check (if (eq classification 'spam)
1103                         'spam-group-spam-mark-p
1104                       'spam-group-ham-mark-p))
1105         list mark-cache-yes mark-cache-no)
1106     (dolist (article articles)
1107       (let ((mark (gnus-summary-article-mark article)))
1108         (unless (memq mark mark-cache-no)
1109           (if (memq mark mark-cache-yes)
1110               (push article list)
1111             ;; else, we have to actually check the mark
1112             (if (funcall mark-check
1113                          gnus-newsgroup-name
1114                          mark)
1115                 (progn
1116                   (push article list)
1117                   (push mark mark-cache-yes))
1118               (push mark mark-cache-no))))))
1119     list))
1120
1121 (defun spam-register-routine (classification
1122                               check
1123                               &optional unregister
1124                               specific-articles)
1125   (when (and (spam-classification-valid-p classification)
1126              (spam-registration-check-valid-p check))
1127     (let* ((register-function
1128             (spam-registration-function classification check))
1129            (unregister-function
1130             (spam-unregistration-function classification check))
1131            (run-function (if unregister
1132                              unregister-function
1133                            register-function))
1134            (log-function (if unregister
1135                              'spam-log-undo-registration
1136                            'spam-log-processing-to-registry))
1137            article articles)
1138
1139       (when run-function
1140         ;; make list of articles, using specific-articles if given
1141         (setq articles (or specific-articles
1142                            (spam-list-articles
1143                             gnus-newsgroup-articles
1144                             classification)))
1145         ;; process them
1146         (gnus-message 5 "%s %d %s articles with classification %s, check %s"
1147                       (if unregister "Unregistering" "Registering")
1148                       (length articles)
1149                       (if specific-articles "specific" "")
1150                       (symbol-name classification)
1151                       (symbol-name check))
1152         (funcall run-function articles)
1153         ;; now log all the registrations (or undo them, depending on unregister)
1154         (dolist (article articles)
1155           (funcall log-function
1156                    (spam-fetch-field-message-id-fast article)
1157                    'process
1158                    classification
1159                    check
1160                    gnus-newsgroup-name))))))
1161
1162 ;;; log a ham- or spam-processor invocation to the registry
1163 (defun spam-log-processing-to-registry (id type classification check group)
1164   (when spam-log-to-registry
1165     (if (and (stringp id)
1166              (stringp group)
1167              (spam-process-type-valid-p type)
1168              (spam-classification-valid-p classification)
1169              (spam-registration-check-valid-p check))
1170         (let ((cell-list (cdr-safe (gnus-registry-fetch-extra id type)))
1171               (cell (list classification check group)))
1172           (push cell cell-list)
1173           (gnus-registry-store-extra-entry
1174            id
1175            type
1176            cell-list))
1177
1178       (gnus-error 5 (format "%s called with bad ID, type, classification, check, or group"
1179                             "spam-log-processing-to-registry")))))
1180
1181 ;;; check if a ham- or spam-processor registration has been done
1182 (defun spam-log-registered-p (id type)
1183   (when spam-log-to-registry
1184     (if (and (stringp id)
1185              (spam-process-type-valid-p type))
1186         (cdr-safe (gnus-registry-fetch-extra id type))
1187       (progn
1188         (gnus-error 5 (format "%s called with bad ID, type, classification, or check"
1189                               "spam-log-registered-p"))
1190         nil))))
1191
1192 ;;; check what a ham- or spam-processor registration says
1193 ;;; returns nil if conflicting registrations are found
1194 (defun spam-log-registration-type (id type)
1195   (let ((count 0)
1196         decision)
1197     (dolist (reg (spam-log-registered-p id type))
1198       (let ((classification (nth 0 reg)))
1199         (when (spam-classification-valid-p classification)
1200           (when (and decision
1201                      (not (eq classification decision)))
1202             (setq count (+ 1 count)))
1203           (setq decision classification))))
1204     (if (< 0 count)
1205         nil
1206       decision)))
1207
1208 ;;; check if a ham- or spam-processor registration needs to be undone
1209 (defun spam-log-unregistration-needed-p (id type classification check)
1210   (when spam-log-to-registry
1211     (if (and (stringp id)
1212              (spam-process-type-valid-p type)
1213              (spam-classification-valid-p classification)
1214              (spam-registration-check-valid-p check))
1215         (let ((cell-list (cdr-safe (gnus-registry-fetch-extra id type)))
1216               found)
1217           (dolist (cell cell-list)
1218             (unless found
1219               (when (and (eq classification (nth 0 cell))
1220                          (eq check (nth 1 cell)))
1221                 (setq found t))))
1222           found)
1223       (progn
1224         (gnus-error 5 (format "%s called with bad ID, type, classification, or check"
1225                               "spam-log-unregistration-needed-p"))
1226         nil))))
1227
1228
1229 ;;; undo a ham- or spam-processor registration (the group is not used)
1230 (defun spam-log-undo-registration (id type classification check &optional group)
1231   (when (and spam-log-to-registry
1232              (spam-log-unregistration-needed-p id type classification check))
1233     (if (and (stringp id)
1234              (spam-process-type-valid-p type)
1235              (spam-classification-valid-p classification)
1236              (spam-registration-check-valid-p check))
1237         (let ((cell-list (cdr-safe (gnus-registry-fetch-extra id type)))
1238               new-cell-list found)
1239           (dolist (cell cell-list)
1240             (unless (and (eq classification (nth 0 cell))
1241                          (eq check (nth 1 cell)))
1242               (push cell new-cell-list)))
1243           (gnus-registry-store-extra-entry
1244            id
1245            type
1246            new-cell-list))
1247       (progn
1248         (gnus-error 5 (format "%s called with bad ID, type, check, or group"
1249                               "spam-log-undo-registration"))
1250         nil))))
1251
1252 ;;; set up IMAP widening if it's necessary
1253 (defun spam-setup-widening ()
1254   (dolist (check spam-list-of-statistical-checks)
1255     (when (symbol-value check)
1256       (setq nnimap-split-download-body-default t))))
1257
1258 \f
1259 ;;;; Regex body
1260
1261 (defun spam-check-regex-body ()
1262   (let ((spam-regex-headers-ham spam-regex-body-ham)
1263         (spam-regex-headers-spam spam-regex-body-spam))
1264     (spam-check-regex-headers t)))
1265
1266 \f
1267 ;;;; Regex headers
1268
1269 (defun spam-check-regex-headers (&optional body)
1270   (let ((type (if body "body" "header"))
1271         (spam-split-group (if spam-split-symbolic-return
1272                               'spam
1273                             spam-split-group))
1274         ret found)
1275     (dolist (h-regex spam-regex-headers-ham)
1276       (unless found
1277         (goto-char (point-min))
1278         (when (re-search-forward h-regex nil t)
1279           (message "Ham regex %s search positive." type)
1280           (setq found t))))
1281     (dolist (s-regex spam-regex-headers-spam)
1282       (unless found
1283         (goto-char (point-min))
1284         (when (re-search-forward s-regex nil t)
1285           (message "Spam regex %s search positive." type)
1286           (setq found t)
1287           (setq ret spam-split-group))))
1288     ret))
1289
1290 \f
1291 ;;;; Blackholes.
1292
1293 (defun spam-reverse-ip-string (ip)
1294   (when (stringp ip)
1295     (mapconcat 'identity
1296                (nreverse (split-string ip "\\."))
1297                ".")))
1298
1299 (defun spam-check-blackholes ()
1300   "Check the Received headers for blackholed relays."
1301   (let ((headers (message-fetch-field "received"))
1302         (spam-split-group (if spam-split-symbolic-return
1303                               'spam
1304                             spam-split-group))
1305         ips matches)
1306     (when headers
1307       (with-temp-buffer
1308         (insert headers)
1309         (goto-char (point-min))
1310         (gnus-message 5 "Checking headers for relay addresses")
1311         (while (re-search-forward
1312                 "\\([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\\)" nil t)
1313           (gnus-message 9 "Blackhole search found host IP %s." (match-string 1))
1314           (push (spam-reverse-ip-string (match-string 1))
1315                 ips)))
1316       (dolist (server spam-blackhole-servers)
1317         (dolist (ip ips)
1318           (unless (and spam-blackhole-good-server-regex
1319                        ;; match the good-server-regex against the reversed (again) IP string
1320                        (string-match
1321                         spam-blackhole-good-server-regex
1322                         (spam-reverse-ip-string ip)))
1323             (unless matches
1324               (let ((query-string (concat ip "." server)))
1325                 (if spam-use-dig
1326                     (let ((query-result (query-dig query-string)))
1327                       (when query-result
1328                         (gnus-message 5 "(DIG): positive blackhole check '%s'"
1329                                       query-result)
1330                         (push (list ip server query-result)
1331                               matches)))
1332                   ;; else, if not using dig.el
1333                   (when (query-dns query-string)
1334                     (gnus-message 5 "positive blackhole check")
1335                     (push (list ip server (query-dns query-string 'TXT))
1336                           matches)))))))))
1337     (when matches
1338       spam-split-group)))
1339 \f
1340 ;;;; Hashcash.
1341
1342 (condition-case nil
1343     (progn
1344       (require 'hashcash)
1345
1346       (defun spam-check-hashcash ()
1347         "Check the headers for hashcash payments."
1348         (mail-check-payment)))   ;mail-check-payment returns a boolean
1349
1350   (file-error (progn
1351                 (defalias 'mail-check-payment 'ignore)
1352                 (defalias 'spam-check-hashcash 'ignore))))
1353 \f
1354 ;;;; BBDB
1355
1356 ;;; original idea for spam-check-BBDB from Alexander Kotelnikov
1357 ;;; <sacha@giotto.sj.ru>
1358
1359 ;; all this is done inside a condition-case to trap errors
1360
1361 (condition-case nil
1362     (progn
1363       (require 'bbdb)
1364       (require 'bbdb-com)
1365
1366       ;; when the BBDB changes, we want to clear out our cache
1367       (defun spam-clear-cache-BBDB (&rest immaterial)
1368         (spam-clear-cache 'spam-use-BBDB))
1369
1370       (add-hook 'bbdb-change-hook 'spam-clear-cache-BBDB)
1371
1372       (defun spam-enter-ham-BBDB (addresses &optional remove)
1373         "Enter an address into the BBDB; implies ham (non-spam) sender"
1374         (dolist (from addresses)
1375           (when (stringp from)
1376             (let* ((parsed-address (gnus-extract-address-components from))
1377                    (name (or (nth 0 parsed-address) "Ham Sender"))
1378                    (remove-function (if remove
1379                                         'bbdb-delete-record-internal
1380                                       'ignore))
1381                    (net-address (nth 1 parsed-address))
1382                    (record (and net-address
1383                                 (bbdb-search-simple nil net-address))))
1384               (when net-address
1385                 (gnus-message 5 "%s address %s %s BBDB"
1386                               (if remove "Deleting" "Adding")
1387                               from
1388                               (if remove "from" "to"))
1389                 (if record
1390                     (funcall remove-function record)
1391                   (bbdb-create-internal name nil net-address nil nil
1392                                         "ham sender added by spam.el")))))))
1393
1394       (defun spam-BBDB-register-routine (articles &optional unregister)
1395         (let (addresses)
1396           (dolist (article articles)
1397             (when (stringp (spam-fetch-field-from-fast article))
1398               (push (spam-fetch-field-from-fast article) addresses)))
1399           ;; now do the register/unregister action
1400           (spam-enter-ham-BBDB addresses unregister)))
1401
1402       (defun spam-BBDB-unregister-routine (articles)
1403         (spam-BBDB-register-routine articles t))
1404
1405       (defun spam-check-BBDB ()
1406         "Mail from people in the BBDB is classified as ham or non-spam"
1407         (let ((who (message-fetch-field "from"))
1408               (spam-split-group (if spam-split-symbolic-return
1409                                     'spam
1410                                   spam-split-group))
1411               bbdb-cache bbdb-hashtable)
1412           (when spam-cache-lookups
1413             (setq bbdb-cache (gethash 'spam-use-BBDB spam-caches))
1414             (unless bbdb-cache
1415               (setq bbdb-cache
1416                     ;; this is the expanded (bbdb-hashtable) macro
1417                     ;; without the debugging support
1418                     (with-current-buffer (bbdb-buffer)
1419                       (save-excursion
1420                         (save-window-excursion
1421                           (bbdb-records nil t)
1422                           bbdb-hashtable))))
1423               (puthash 'spam-use-BBDB bbdb-cache spam-caches)))
1424           (when who
1425             (setq who (nth 1 (gnus-extract-address-components who)))
1426             (if
1427                 (if spam-cache-lookups
1428                     (symbol-value
1429                      (intern-soft who bbdb-cache))
1430                   (bbdb-search-simple nil who))
1431                 t
1432               (if spam-use-BBDB-exclusive
1433                   spam-split-group
1434                 nil))))))
1435
1436   (file-error (progn
1437                 (defalias 'bbdb-search-simple 'ignore)
1438                 (defalias 'bbdb-records 'ignore)
1439                 (defalias 'bbdb-buffer 'ignore)
1440                 (defalias 'spam-check-BBDB 'ignore)
1441                 (defalias 'spam-BBDB-register-routine 'ignore)
1442                 (defalias 'spam-enter-ham-BBDB 'ignore)
1443                 (defalias 'bbdb-create-internal 'ignore)
1444                 (defalias 'bbdb-delete-record-internal 'ignore)
1445                 (defalias 'bbdb-records 'ignore))))
1446
1447 \f
1448 ;;;; ifile
1449
1450 ;;; check the ifile backend; return nil if the mail was NOT classified
1451 ;;; as spam
1452
1453 (defun spam-get-ifile-database-parameter ()
1454   "Get the command-line parameter for ifile's database from
1455   spam-ifile-database-path."
1456   (if spam-ifile-database-path
1457       (format "--db-file=%s" spam-ifile-database-path)
1458     nil))
1459
1460 (defun spam-check-ifile ()
1461   "Check the ifile backend for the classification of this message."
1462   (let ((article-buffer-name (buffer-name))
1463         (spam-split-group (if spam-split-symbolic-return
1464                               'spam
1465                             spam-split-group))
1466         category return)
1467     (with-temp-buffer
1468       (let ((temp-buffer-name (buffer-name))
1469             (db-param (spam-get-ifile-database-parameter)))
1470         (save-excursion
1471           (set-buffer article-buffer-name)
1472           (apply 'call-process-region
1473                  (point-min) (point-max) spam-ifile-path
1474                  nil temp-buffer-name nil "-c"
1475                  (if db-param `(,db-param "-q") `("-q"))))
1476         ;; check the return now (we're back in the temp buffer)
1477         (goto-char (point-min))
1478         (if (not (eobp))
1479             (setq category (buffer-substring (point) (point-at-eol))))
1480         (when (not (zerop (length category))) ; we need a category here
1481           (if spam-ifile-all-categories
1482               (setq return category)
1483             ;; else, if spam-ifile-all-categories is not set...
1484             (when (string-equal spam-ifile-spam-category category)
1485               (setq return spam-split-group)))))) ; note return is nil otherwise
1486     return))
1487
1488 (defun spam-ifile-register-with-ifile (articles category &optional unregister)
1489   "Register an article, given as a string, with a category.
1490 Uses `gnus-newsgroup-name' if category is nil (for ham registration)."
1491   (let ((category (or category gnus-newsgroup-name))
1492         (add-or-delete-option (if unregister "-d" "-i"))
1493         (db (spam-get-ifile-database-parameter))
1494         parameters)
1495     (with-temp-buffer
1496       (dolist (article articles)
1497         (let ((article-string (spam-get-article-as-string article)))
1498           (when (stringp article-string)
1499             (insert article-string))))
1500       (apply 'call-process-region
1501              (point-min) (point-max) spam-ifile-path
1502              nil nil nil
1503              add-or-delete-option category
1504              (if db `(,db "-h") `("-h"))))))
1505
1506 (defun spam-ifile-register-spam-routine (articles &optional unregister)
1507   (spam-ifile-register-with-ifile articles spam-ifile-spam-category unregister))
1508
1509 (defun spam-ifile-unregister-spam-routine (articles)
1510   (spam-ifile-register-spam-routine articles t))
1511
1512 (defun spam-ifile-register-ham-routine (articles &optional unregister)
1513   (spam-ifile-register-with-ifile articles spam-ifile-ham-category unregister))
1514
1515 (defun spam-ifile-unregister-ham-routine (articles)
1516   (spam-ifile-register-ham-routine articles t))
1517
1518 \f
1519 ;;;; spam-stat
1520
1521 (condition-case nil
1522     (progn
1523       (let ((spam-stat-install-hooks nil))
1524         (require 'spam-stat))
1525
1526       (defun spam-check-stat ()
1527         "Check the spam-stat backend for the classification of this message"
1528         (let ((spam-split-group (if spam-split-symbolic-return
1529                                     'spam
1530                                   spam-split-group))
1531               (spam-stat-split-fancy-spam-group spam-split-group) ; override
1532               (spam-stat-buffer (buffer-name)) ; stat the current buffer
1533               category return)
1534           (spam-stat-split-fancy)))
1535
1536       (defun spam-stat-register-spam-routine (articles &optional unregister)
1537         (dolist (article articles)
1538           (let ((article-string (spam-get-article-as-string article)))
1539             (with-temp-buffer
1540               (insert article-string)
1541               (if unregister
1542                   (spam-stat-buffer-change-to-non-spam)
1543               (spam-stat-buffer-is-spam))))))
1544
1545       (defun spam-stat-unregister-spam-routine (articles)
1546         (spam-stat-register-spam-routine articles t))
1547
1548       (defun spam-stat-register-ham-routine (articles &optional unregister)
1549         (dolist (article articles)
1550           (let ((article-string (spam-get-article-as-string article)))
1551             (with-temp-buffer
1552               (insert article-string)
1553               (if unregister
1554                   (spam-stat-buffer-change-to-spam)
1555               (spam-stat-buffer-is-non-spam))))))
1556
1557       (defun spam-stat-unregister-ham-routine (articles)
1558         (spam-stat-register-ham-routine articles t))
1559
1560       (defun spam-maybe-spam-stat-load ()
1561         (when spam-use-stat (spam-stat-load)))
1562
1563       (defun spam-maybe-spam-stat-save ()
1564         (when spam-use-stat (spam-stat-save))))
1565
1566   (file-error (progn
1567                 (defalias 'spam-stat-load 'ignore)
1568                 (defalias 'spam-stat-save 'ignore)
1569                 (defalias 'spam-maybe-spam-stat-load 'ignore)
1570                 (defalias 'spam-maybe-spam-stat-save 'ignore)
1571                 (defalias 'spam-stat-register-ham-routine 'ignore)
1572                 (defalias 'spam-stat-unregister-ham-routine 'ignore)
1573                 (defalias 'spam-stat-register-spam-routine 'ignore)
1574                 (defalias 'spam-stat-unregister-spam-routine 'ignore)
1575                 (defalias 'spam-stat-buffer-is-spam 'ignore)
1576                 (defalias 'spam-stat-buffer-change-to-spam 'ignore)
1577                 (defalias 'spam-stat-buffer-is-non-spam 'ignore)
1578                 (defalias 'spam-stat-buffer-change-to-non-spam 'ignore)
1579                 (defalias 'spam-stat-split-fancy 'ignore)
1580                 (defalias 'spam-check-stat 'ignore))))
1581
1582 \f
1583
1584 ;;;; Blacklists and whitelists.
1585
1586 (defvar spam-whitelist-cache nil)
1587 (defvar spam-blacklist-cache nil)
1588
1589 (defun spam-kill-whole-line ()
1590   (beginning-of-line)
1591   (let ((kill-whole-line t))
1592     (kill-line)))
1593
1594 ;;; address can be a list, too
1595 (defun spam-enter-whitelist (address &optional remove)
1596   "Enter ADDRESS (list or single) into the whitelist.
1597 With a non-nil REMOVE, remove them."
1598   (interactive "sAddress: ")
1599   (spam-enter-list address spam-whitelist remove)
1600   (setq spam-whitelist-cache nil)
1601   (spam-clear-cache 'spam-use-whitelist))
1602
1603 ;;; address can be a list, too
1604 (defun spam-enter-blacklist (address &optional remove)
1605   "Enter ADDRESS (list or single) into the blacklist.
1606 With a non-nil REMOVE, remove them."
1607   (interactive "sAddress: ")
1608   (spam-enter-list address spam-blacklist remove)
1609   (setq spam-blacklist-cache nil)
1610   (spam-clear-cache 'spam-use-whitelist))
1611
1612 (defun spam-enter-list (addresses file &optional remove)
1613   "Enter ADDRESSES into the given FILE.
1614 Either the whitelist or the blacklist files can be used.  With
1615 REMOVE not nil, remove the ADDRESSES."
1616   (if (stringp addresses)
1617       (spam-enter-list (list addresses) file remove)
1618     ;; else, we have a list of addresses here
1619     (unless (file-exists-p (file-name-directory file))
1620       (make-directory (file-name-directory file) t))
1621     (save-excursion
1622       (set-buffer
1623        (find-file-noselect file))
1624       (dolist (a addresses)
1625         (when (stringp a)
1626           (goto-char (point-min))
1627           (if (re-search-forward (regexp-quote a) nil t)
1628               ;; found the address
1629               (when remove
1630                 (spam-kill-whole-line))
1631             ;; else, the address was not found
1632             (unless remove
1633               (goto-char (point-max))
1634               (unless (bobp)
1635                 (insert "\n"))
1636               (insert a "\n")))))
1637       (save-buffer))))
1638
1639 (defun spam-filelist-build-cache (type)
1640   (let ((cache (if (eq type 'spam-use-blacklist)
1641                    spam-blacklist-cache
1642                  spam-whitelist-cache))
1643         parsed-cache)
1644     (unless (gethash type spam-caches)
1645       (while cache
1646         (let ((address (pop cache)))
1647           (unless (zerop (length address)) ; 0 for a nil address too
1648             (setq address (regexp-quote address))
1649             ;; fix regexp-quote's treatment of user-intended regexes
1650             (while (string-match "\\\\\\*" address)
1651               (setq address (replace-match ".*" t t address))))
1652           (push address parsed-cache)))
1653       (puthash type parsed-cache spam-caches))))
1654
1655 (defun spam-filelist-check-cache (type from)
1656   (when (stringp from)
1657     (spam-filelist-build-cache type)
1658     (let (found)
1659       (dolist (address (gethash type spam-caches))
1660         (when (and address (string-match address from))
1661           (setq found t)
1662           (return)))
1663       found)))
1664
1665 ;;; returns t if the sender is in the whitelist, nil or
1666 ;;; spam-split-group otherwise
1667 (defun spam-check-whitelist ()
1668   ;; FIXME!  Should it detect when file timestamps change?
1669   (let ((spam-split-group (if spam-split-symbolic-return
1670                               'spam
1671                             spam-split-group)))
1672     (unless spam-whitelist-cache
1673       (setq spam-whitelist-cache (spam-parse-list spam-whitelist)))
1674     (if (spam-from-listed-p 'spam-use-whitelist)
1675         t
1676       (if spam-use-whitelist-exclusive
1677           spam-split-group
1678         nil))))
1679
1680 (defun spam-check-blacklist ()
1681   ;; FIXME!  Should it detect when file timestamps change?
1682   (let ((spam-split-group (if spam-split-symbolic-return
1683                               'spam
1684                             spam-split-group)))
1685     (unless spam-blacklist-cache
1686       (setq spam-blacklist-cache (spam-parse-list spam-blacklist)))
1687     (and (spam-from-listed-p 'spam-use-blacklist) spam-split-group)))
1688
1689 (defun spam-parse-list (file)
1690   (when (file-readable-p file)
1691     (let (contents address)
1692       (with-temp-buffer
1693         (insert-file-contents file)
1694         (while (not (eobp))
1695           (setq address (buffer-substring (point) (point-at-eol)))
1696           (forward-line 1)
1697           ;; insert the e-mail address if detected, otherwise the raw data
1698           (unless (zerop (length address))
1699             (let ((pure-address (nth 1 (gnus-extract-address-components address))))
1700               (push (or pure-address address) contents)))))
1701       (nreverse contents))))
1702
1703 (defun spam-from-listed-p (type)
1704   (let ((from (message-fetch-field "from"))
1705         found)
1706     (spam-filelist-check-cache type from)))
1707
1708 (defun spam-filelist-register-routine (articles blacklist &optional unregister)
1709   (let ((de-symbol (if blacklist 'spam-use-whitelist 'spam-use-blacklist))
1710         (declassification (if blacklist 'ham 'spam))
1711         (enter-function
1712          (if blacklist 'spam-enter-blacklist 'spam-enter-whitelist))
1713         (remove-function
1714          (if blacklist 'spam-enter-whitelist 'spam-enter-blacklist))
1715         from addresses unregister-list)
1716     (dolist (article articles)
1717       (let ((from (spam-fetch-field-from-fast article))
1718             (id (spam-fetch-field-message-id-fast article))
1719             sender-ignored)
1720         (when (stringp from)
1721           (dolist (ignore-regex spam-blacklist-ignored-regexes)
1722             (when (and (not sender-ignored)
1723                        (stringp ignore-regex)
1724                        (string-match ignore-regex from))
1725               (setq sender-ignored t)))
1726           ;; remember the messages we need to unregister, unless remove is set
1727           (when (and
1728                  (null unregister)
1729                  (spam-log-unregistration-needed-p
1730                   id 'process declassification de-symbol))
1731             (push from unregister-list))
1732           (unless sender-ignored
1733             (push from addresses)))))
1734
1735     (if unregister
1736         (funcall enter-function addresses t) ; unregister all these addresses
1737       ;; else, register normally and unregister what we need to
1738       (funcall remove-function unregister-list t)
1739       (dolist (article unregister-list)
1740         (spam-log-undo-registration
1741          (spam-fetch-field-message-id-fast article)
1742          'process
1743          declassification
1744          de-symbol))
1745       (funcall enter-function addresses nil))))
1746
1747 (defun spam-blacklist-unregister-routine (articles)
1748   (spam-blacklist-register-routine articles t))
1749
1750 (defun spam-blacklist-register-routine (articles &optional unregister)
1751   (spam-filelist-register-routine articles t unregister))
1752
1753 (defun spam-whitelist-unregister-routine (articles)
1754   (spam-whitelist-register-routine articles t))
1755
1756 (defun spam-whitelist-register-routine (articles &optional unregister)
1757   (spam-filelist-register-routine articles nil unregister))
1758
1759 \f
1760 ;;;; Spam-report glue
1761 (defun spam-report-gmane-register-routine (articles)
1762   (when articles
1763     (apply 'spam-report-gmane articles)))
1764
1765 \f
1766 ;;;; Bogofilter
1767 (defun spam-check-bogofilter-headers (&optional score)
1768   (let ((header (message-fetch-field spam-bogofilter-header))
1769         (spam-split-group (if spam-split-symbolic-return
1770                               'spam
1771                             spam-split-group)))
1772     (when header                        ; return nil when no header
1773       (if score                         ; scoring mode
1774           (if (string-match "spamicity=\\([0-9.]+\\)" header)
1775               (match-string 1 header)
1776             "0")
1777         ;; spam detection mode
1778         (when (string-match spam-bogofilter-bogosity-positive-spam-header
1779                             header)
1780           spam-split-group)))))
1781
1782 ;; return something sensible if the score can't be determined
1783 (defun spam-bogofilter-score ()
1784   "Get the Bogofilter spamicity score"
1785   (interactive)
1786   (save-window-excursion
1787     (gnus-summary-show-article t)
1788     (set-buffer gnus-article-buffer)
1789     (let ((score (or (spam-check-bogofilter-headers t)
1790                      (spam-check-bogofilter t))))
1791       (message "Spamicity score %s" score)
1792       (or score "0"))
1793     (gnus-summary-show-article)))
1794
1795 (defun spam-check-bogofilter (&optional score)
1796   "Check the Bogofilter backend for the classification of this message"
1797   (let ((article-buffer-name (buffer-name))
1798         (db spam-bogofilter-database-directory)
1799         return)
1800     (with-temp-buffer
1801       (let ((temp-buffer-name (buffer-name)))
1802         (save-excursion
1803           (set-buffer article-buffer-name)
1804           (apply 'call-process-region
1805                  (point-min) (point-max)
1806                  spam-bogofilter-path
1807                  nil temp-buffer-name nil
1808                  (if db `("-d" ,db "-v") `("-v"))))
1809         (setq return (spam-check-bogofilter-headers score))))
1810     return))
1811
1812 (defun spam-bogofilter-register-with-bogofilter (articles
1813                                                  spam
1814                                                  &optional unregister)
1815   "Register an article, given as a string, as spam or non-spam."
1816   (dolist (article articles)
1817     (let ((article-string (spam-get-article-as-string article))
1818           (db spam-bogofilter-database-directory)
1819           (switch (if unregister
1820                       (if spam
1821                           spam-bogofilter-spam-strong-switch
1822                         spam-bogofilter-ham-strong-switch)
1823                     (if spam
1824                         spam-bogofilter-spam-switch
1825                       spam-bogofilter-ham-switch))))
1826       (when (stringp article-string)
1827         (with-temp-buffer
1828           (insert article-string)
1829
1830           (apply 'call-process-region
1831                  (point-min) (point-max)
1832                  spam-bogofilter-path
1833                  nil nil nil switch
1834                  (if db `("-d" ,db "-v") `("-v"))))))))
1835
1836 (defun spam-bogofilter-register-spam-routine (articles &optional unregister)
1837   (spam-bogofilter-register-with-bogofilter articles t unregister))
1838
1839 (defun spam-bogofilter-unregister-spam-routine (articles)
1840   (spam-bogofilter-register-spam-routine articles t))
1841
1842 (defun spam-bogofilter-register-ham-routine (articles &optional unregister)
1843   (spam-bogofilter-register-with-bogofilter articles nil unregister))
1844
1845 (defun spam-bogofilter-unregister-ham-routine (articles)
1846   (spam-bogofilter-register-ham-routine articles t))
1847
1848
1849 \f
1850 ;;;; spamoracle
1851 (defun spam-check-spamoracle ()
1852   "Run spamoracle on an article to determine whether it's spam."
1853   (let ((article-buffer-name (buffer-name))
1854         (spam-split-group (if spam-split-symbolic-return
1855                               'spam
1856                             spam-split-group)))
1857     (with-temp-buffer
1858       (let ((temp-buffer-name (buffer-name)))
1859         (save-excursion
1860           (set-buffer article-buffer-name)
1861           (let ((status
1862                  (apply 'call-process-region
1863                         (point-min) (point-max)
1864                         spam-spamoracle-binary
1865                         nil temp-buffer-name nil
1866                         (if spam-spamoracle-database
1867                             `("-f" ,spam-spamoracle-database "mark")
1868                           '("mark")))))
1869             (if (eq 0 status)
1870                 (progn
1871                   (set-buffer temp-buffer-name)
1872                   (goto-char (point-min))
1873                   (when (re-search-forward "^X-Spam: yes;" nil t)
1874                     spam-split-group))
1875               (error "Error running spamoracle" status))))))))
1876
1877 (defun spam-spamoracle-learn (articles article-is-spam-p &optional unregister)
1878   "Run spamoracle in training mode."
1879   (with-temp-buffer
1880     (let ((temp-buffer-name (buffer-name)))
1881       (save-excursion
1882         (goto-char (point-min))
1883         (dolist (article articles)
1884           (insert (spam-get-article-as-string article)))
1885         (let* ((arg (if (spam-xor unregister article-is-spam-p)
1886                         "-spam"
1887                       "-good"))
1888                (status
1889                 (apply 'call-process-region
1890                        (point-min) (point-max)
1891                        spam-spamoracle-binary
1892                        nil temp-buffer-name nil
1893                        (if spam-spamoracle-database
1894                            `("-f" ,spam-spamoracle-database
1895                              "add" ,arg)
1896                          `("add" ,arg)))))
1897           (when (not (eq 0 status))
1898             (error "Error running spamoracle" status)))))))
1899
1900 (defun spam-spamoracle-learn-ham (articles &optional unregister)
1901   (spam-spamoracle-learn articles nil unregister))
1902
1903 (defun spam-spamoracle-unlearn-ham (articles &optional unregister)
1904   (spam-spamoracle-learn-ham articles t))
1905
1906 (defun spam-spamoracle-learn-spam (articles &optional unregister)
1907   (spam-spamoracle-learn articles t unregister))
1908
1909 (defun spam-spamoracle-unlearn-spam (articles &optional unregister)
1910   (spam-spamoracle-learn-spam articles t))
1911
1912 \f
1913 ;;;; Hooks
1914
1915 ;;;###autoload
1916 (defun spam-initialize ()
1917   "Install the spam.el hooks and do other initialization"
1918   (interactive)
1919   (setq spam-install-hooks t)
1920   ;; TODO: How do we redo this every time spam-face is customized?
1921   (push '((eq mark gnus-spam-mark) . spam-face)
1922         gnus-summary-highlight)
1923   ;; Add hooks for loading and saving the spam stats
1924   (add-hook 'gnus-save-newsrc-hook 'spam-maybe-spam-stat-save)
1925   (add-hook 'gnus-get-top-new-news-hook 'spam-maybe-spam-stat-load)
1926   (add-hook 'gnus-startup-hook 'spam-maybe-spam-stat-load)
1927   (add-hook 'gnus-summary-prepare-exit-hook 'spam-summary-prepare-exit)
1928   (add-hook 'gnus-summary-prepare-hook 'spam-summary-prepare)
1929   (add-hook 'gnus-get-new-news-hook 'spam-setup-widening)
1930   (add-hook 'gnus-summary-prepare-hook 'spam-find-spam))
1931
1932 (defun spam-unload-hook ()
1933   "Uninstall the spam.el hooks"
1934   (interactive)
1935   (remove-hook 'gnus-save-newsrc-hook 'spam-maybe-spam-stat-save)
1936   (remove-hook 'gnus-get-top-new-news-hook 'spam-maybe-spam-stat-load)
1937   (remove-hook 'gnus-startup-hook 'spam-maybe-spam-stat-load)
1938   (remove-hook 'gnus-summary-prepare-exit-hook 'spam-summary-prepare-exit)
1939   (remove-hook 'gnus-summary-prepare-hook 'spam-summary-prepare)
1940   (remove-hook 'gnus-get-new-news-hook 'spam-setup-widening)
1941   (remove-hook 'gnus-summary-prepare-hook 'spam-find-spam))
1942
1943 (when spam-install-hooks
1944   (spam-initialize))
1945
1946 (provide 'spam)
1947
1948 ;;; spam.el ends here