git.chise.org Git - elisp/gnus.git-/blob - lisp/spam.el

   1 ;;; spam.el --- Identifying spam
   2 ;; Copyright (C) 2002, 2003, 2004 Free Software Foundation, Inc.
   3
   4 ;; Author: Lars Magne Ingebrigtsen <larsi@gnus.org>
   5 ;; Keywords: network
   6
   7 ;; This file is part of GNU Emacs.
   8
   9 ;; GNU Emacs is free software; you can redistribute it and/or modify
  10 ;; it under the terms of the GNU General Public License as published by
  11 ;; the Free Software Foundation; either version 2, or (at your option)
  12 ;; any later version.
  13
  14 ;; GNU Emacs is distributed in the hope that it will be useful,
  15 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 ;; GNU General Public License for more details.
  18
  19 ;; You should have received a copy of the GNU General Public License
  20 ;; along with GNU Emacs; see the file COPYING.  If not, write to the
  21 ;; Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  22 ;; Boston, MA 02111-1307, USA.
  23
  24 ;;; Commentary:
  25
  26 ;;; This module addresses a few aspects of spam control under Gnus.  Page
  27 ;;; breaks are used for grouping declarations and documentation relating to
  28 ;;; each particular aspect.
  29
  30 ;;; The integration with Gnus is not yet complete.  See various `FIXME'
  31 ;;; comments, below, for supplementary explanations or discussions.
  32
  33 ;;; Several TODO items are marked as such
  34
  35 ;; TODO: spam scores, detection of spam in newsgroups, cross-server splitting,
  36 ;; remote processing, training through files
  37
  38 ;;; Code:
  39
  40 (require 'path-util)
  41
  42 (eval-when-compile (require 'cl))
  43
  44 (require 'gnus-sum)
  45
  46 (require 'gnus-uu)                      ; because of key prefix issues
  47 ;;; for the definitions of group content classification and spam processors
  48 (require 'gnus)
  49 (require 'message)              ;for the message-fetch-field functions
  50
  51 ;; for nnimap-split-download-body-default
  52 (eval-when-compile (require 'nnimap))
  53
  54 ;; autoload query-dig
  55 (eval-and-compile
  56   (autoload 'query-dig "dig"))
  57
  58 ;; autoload spam-report
  59 (eval-and-compile
  60   (autoload 'spam-report-gmane "spam-report"))
  61
  62 ;; autoload gnus-registry
  63 (eval-and-compile
  64   (autoload 'gnus-registry-group-count "gnus-registry")
  65   (autoload 'gnus-registry-add-group "gnus-registry")
  66   (autoload 'gnus-registry-store-extra-entry "gnus-registry")
  67   (autoload 'gnus-registry-fetch-extra "gnus-registry"))
  68
  69 ;; autoload query-dns
  70 (eval-and-compile
  71   (autoload 'query-dns "dns"))
  72
  73 ;;; Main parameters.
  74
  75 (defgroup spam nil
  76   "Spam configuration.")
  77
  78 (defcustom spam-directory "~/News/spam/"
  79   "Directory for spam whitelists and blacklists."
  80   :type 'directory
  81   :group 'spam)
  82
  83 (defcustom spam-move-spam-nonspam-groups-only t
  84   "Whether spam should be moved in non-spam groups only.
  85 When t, only ham and unclassified groups will have their spam moved
  86 to the spam-process-destination.  When nil, spam will also be moved from
  87 spam groups."
  88   :type 'boolean
  89   :group 'spam)
  90
  91 (defcustom spam-process-ham-in-nonham-groups nil
  92   "Whether ham should be processed in non-ham groups."
  93   :type 'boolean
  94   :group 'spam)
  95
  96 (defcustom spam-log-to-registry nil
  97   "Whether spam/ham processing should be logged in the registry."
  98   :type 'boolean
  99   :group 'spam)
 100
 101 (defcustom spam-split-symbolic-return nil
 102   "Whether `spam-split' should work with symbols or group names."
 103   :type 'boolean
 104   :group 'spam)
 105
 106 (defcustom spam-split-symbolic-return-positive nil
 107   "Whether `spam-split' should ALWAYS work with symbols or group names.
 108 Do not set this if you use `spam-split' in a fancy split
 109   method."
 110   :type 'boolean
 111   :group 'spam)
 112
 113 (defcustom spam-process-ham-in-spam-groups nil
 114   "Whether ham should be processed in spam groups."
 115   :type 'boolean
 116   :group 'spam)
 117
 118 (defcustom spam-mark-only-unseen-as-spam t
 119   "Whether only unseen articles should be marked as spam in spam groups.
 120 When nil, all unread articles in a spam group are marked as
 121 spam.  Set this if you want to leave an article unread in a spam group
 122 without losing it to the automatic spam-marking process."
 123   :type 'boolean
 124   :group 'spam)
 125
 126 (defcustom spam-mark-ham-unread-before-move-from-spam-group nil
 127   "Whether ham should be marked unread before it's moved.
 128 The article is moved out of a spam group according to ham-process-destination.
 129 This variable is an official entry in the international Longest Variable Name
 130 Competition."
 131   :type 'boolean
 132   :group 'spam)
 133
 134 (defcustom spam-disable-spam-split-during-ham-respool nil
 135   "Whether `spam-split' should be ignored while resplitting ham in a process
 136 destination.  This is useful to prevent ham from ending up in the same spam
 137 group after the resplit.  Don't set this to t if you have spam-split as the
 138 last rule in your split configuration."
 139   :type 'boolean
 140   :group 'spam)
 141
 142 (defcustom spam-autodetect-recheck-messages nil
 143   "Should spam.el recheck all meessages when autodetecting?
 144 Normally this is nil, so only unseen messages will be checked."
 145   :type 'boolean
 146   :group 'spam)
 147
 148 (defcustom spam-whitelist (expand-file-name "whitelist" spam-directory)
 149   "The location of the whitelist.
 150 The file format is one regular expression per line.
 151 The regular expression is matched against the address."
 152   :type 'file
 153   :group 'spam)
 154
 155 (defcustom spam-blacklist (expand-file-name "blacklist" spam-directory)
 156   "The location of the blacklist.
 157 The file format is one regular expression per line.
 158 The regular expression is matched against the address."
 159   :type 'file
 160   :group 'spam)
 161
 162 (defcustom spam-use-dig t
 163   "Whether `query-dig' should be used instead of `query-dns'."
 164   :type 'boolean
 165   :group 'spam)
 166
 167 (defcustom spam-use-blacklist nil
 168   "Whether the blacklist should be used by `spam-split'."
 169   :type 'boolean
 170   :group 'spam)
 171
 172 (defcustom spam-blacklist-ignored-regexes nil
 173   "Regular expressions that the blacklist should ignore."
 174   :type '(repeat (regexp :tag "Regular expression to ignore when blacklisting"))
 175   :group 'spam)
 176
 177 (defcustom spam-use-whitelist nil
 178   "Whether the whitelist should be used by `spam-split'."
 179   :type 'boolean
 180   :group 'spam)
 181
 182 (defcustom spam-use-whitelist-exclusive nil
 183   "Whether whitelist-exclusive should be used by `spam-split'.
 184 Exclusive whitelisting means that all messages from senders not in the whitelist
 185 are considered spam."
 186   :type 'boolean
 187   :group 'spam)
 188
 189 (defcustom spam-use-blackholes nil
 190   "Whether blackholes should be used by `spam-split'."
 191   :type 'boolean
 192   :group 'spam)
 193
 194 (defcustom spam-use-hashcash nil
 195   "Whether hashcash payments should be detected by `spam-split'."
 196   :type 'boolean
 197   :group 'spam)
 198
 199 (defcustom spam-use-regex-headers nil
 200   "Whether a header regular expression match should be used by `spam-split'.
 201 Also see the variables `spam-regex-headers-spam' and `spam-regex-headers-ham'."
 202   :type 'boolean
 203   :group 'spam)
 204
 205 (defcustom spam-use-regex-body nil
 206   "Whether a body regular expression match should be used by `spam-split'.
 207 Also see the variables `spam-regex-body-spam' and `spam-regex-body-ham'."
 208   :type 'boolean
 209   :group 'spam)
 210
 211 (defcustom spam-use-bogofilter-headers nil
 212   "Whether bogofilter headers should be used by `spam-split'.
 213 Enable this if you pre-process messages with Bogofilter BEFORE Gnus sees them."
 214   :type 'boolean
 215   :group 'spam)
 216
 217 (defcustom spam-use-bogofilter nil
 218   "Whether bogofilter should be invoked by `spam-split'.
 219 Enable this if you want Gnus to invoke Bogofilter on new messages."
 220   :type 'boolean
 221   :group 'spam)
 222
 223 (defcustom spam-use-BBDB nil
 224   "Whether BBDB should be used by `spam-split'."
 225   :type 'boolean
 226   :group 'spam)
 227
 228 (defcustom spam-use-BBDB-exclusive nil
 229   "Whether BBDB-exclusive should be used by `spam-split'.
 230 Exclusive BBDB means that all messages from senders not in the BBDB are
 231 considered spam."
 232   :type 'boolean
 233   :group 'spam)
 234
 235 (defcustom spam-use-ifile nil
 236   "Whether ifile should be used by `spam-split'."
 237   :type 'boolean
 238   :group 'spam)
 239
 240 (defcustom spam-use-stat nil
 241   "Whether `spam-stat' should be used by `spam-split'."
 242   :type 'boolean
 243   :group 'spam)
 244
 245 (defcustom spam-use-spamoracle nil
 246   "Whether spamoracle should be used by `spam-split'."
 247   :type 'boolean
 248   :group 'spam)
 249
 250 (defcustom spam-install-hooks (or
 251                                spam-use-dig
 252                                spam-use-blacklist
 253                                spam-use-whitelist
 254                                spam-use-whitelist-exclusive
 255                                spam-use-blackholes
 256                                spam-use-hashcash
 257                                spam-use-regex-headers
 258                                spam-use-regex-body
 259                                spam-use-bogofilter-headers
 260                                spam-use-bogofilter
 261                                spam-use-BBDB
 262                                spam-use-BBDB-exclusive
 263                                spam-use-ifile
 264                                spam-use-stat
 265                                spam-use-spamoracle)
 266   "Whether the spam hooks should be installed.
 267 Default to t if one of the spam-use-* variables is set."
 268   :group 'spam
 269   :type 'boolean)
 270
 271 (defcustom spam-split-group "spam"
 272   "Group name where incoming spam should be put by `spam-split'."
 273   :type 'string
 274   :group 'spam)
 275
 276 ;;; TODO: deprecate this variable, it's confusing since it's a list of strings,
 277 ;;; not regular expressions
 278 (defcustom spam-junk-mailgroups (cons
 279                                  spam-split-group
 280                                  '("mail.junk" "poste.pourriel"))
 281   "Mailgroups with spam contents.
 282 All unmarked article in such group receive the spam mark on group entry."
 283   :type '(repeat (string :tag "Group"))
 284   :group 'spam)
 285
 286 (defcustom spam-blackhole-servers '("bl.spamcop.net" "relays.ordb.org"
 287                                     "dev.null.dk" "relays.visi.com")
 288   "List of blackhole servers."
 289   :type '(repeat (string :tag "Server"))
 290   :group 'spam)
 291
 292 (defcustom spam-blackhole-good-server-regex nil
 293   "String matching IP addresses that should not be checked in the blackholes."
 294   :type '(radio (const nil)
 295                 (regexp :format "%t: %v\n" :size 0))
 296   :group 'spam)
 297
 298 (defcustom spam-face 'gnus-splash-face
 299   "Face for spam-marked articles."
 300   :type 'face
 301   :group 'spam)
 302
 303 (defcustom spam-regex-headers-spam '("^X-Spam-Flag: YES")
 304   "Regular expression for positive header spam matches."
 305   :type '(repeat (regexp :tag "Regular expression to match spam header"))
 306   :group 'spam)
 307
 308 (defcustom spam-regex-headers-ham '("^X-Spam-Flag: NO")
 309   "Regular expression for positive header ham matches."
 310   :type '(repeat (regexp :tag "Regular expression to match ham header"))
 311   :group 'spam)
 312
 313 (defcustom spam-regex-body-spam '()
 314   "Regular expression for positive body spam matches."
 315   :type '(repeat (regexp :tag "Regular expression to match spam body"))
 316   :group 'spam)
 317
 318 (defcustom spam-regex-body-ham '()
 319   "Regular expression for positive body ham matches."
 320   :type '(repeat (regexp :tag "Regular expression to match ham body"))
 321   :group 'spam)
 322
 323 (defgroup spam-ifile nil
 324   "Spam ifile configuration."
 325   :group 'spam)
 326
 327 (defcustom spam-ifile-path (exec-installed-p "ifile")
 328   "File path of the ifile executable program."
 329   :type '(choice (file :tag "Location of ifile")
 330                  (const :tag "ifile is not installed"))
 331   :group 'spam-ifile)
 332
 333 (defcustom spam-ifile-database-path nil
 334   "File path of the ifile database."
 335   :type '(choice (file :tag "Location of the ifile database")
 336                  (const :tag "Use the default"))
 337   :group 'spam-ifile)
 338
 339 (defcustom spam-ifile-spam-category "spam"
 340   "Name of the spam ifile category."
 341   :type 'string
 342   :group 'spam-ifile)
 343
 344 (defcustom spam-ifile-ham-category nil
 345   "Name of the ham ifile category.
 346 If nil, the current group name will be used."
 347   :type '(choice (string :tag "Use a fixed category")
 348                  (const :tag "Use the current group name"))
 349   :group 'spam-ifile)
 350
 351 (defcustom spam-ifile-all-categories nil
 352   "Whether the ifile check will return all categories, or just spam.
 353 Set this to t if you want to use the `spam-split' invocation of ifile as
 354 your main source of newsgroup names."
 355   :type 'boolean
 356   :group 'spam-ifile)
 357
 358 (defgroup spam-bogofilter nil
 359   "Spam bogofilter configuration."
 360   :group 'spam)
 361
 362 (defcustom spam-bogofilter-path (exec-installed-p "bogofilter")
 363   "File path of the Bogofilter executable program."
 364   :type '(choice (file :tag "Location of bogofilter")
 365                  (const :tag "Bogofilter is not installed"))
 366   :group 'spam-bogofilter)
 367
 368 (defcustom spam-bogofilter-header "X-Bogosity"
 369   "The header that Bogofilter inserts in messages."
 370   :type 'string
 371   :group 'spam-bogofilter)
 372
 373 (defcustom spam-bogofilter-spam-switch "-s"
 374   "The switch that Bogofilter uses to register spam messages."
 375   :type 'string
 376   :group 'spam-bogofilter)
 377
 378 (defcustom spam-bogofilter-ham-switch "-n"
 379   "The switch that Bogofilter uses to register ham messages."
 380   :type 'string
 381   :group 'spam-bogofilter)
 382
 383 (defcustom spam-bogofilter-spam-strong-switch "-S"
 384   "The switch that Bogofilter uses to unregister ham messages."
 385   :type 'string
 386   :group 'spam-bogofilter)
 387
 388 (defcustom spam-bogofilter-ham-strong-switch "-N"
 389   "The switch that Bogofilter uses to unregister spam messages."
 390   :type 'string
 391   :group 'spam-bogofilter)
 392
 393 (defcustom spam-bogofilter-bogosity-positive-spam-header "^\\(Yes\\|Spam\\)"
 394   "The regex on `spam-bogofilter-header' for positive spam identification."
 395   :type 'regexp
 396   :group 'spam-bogofilter)
 397
 398 (defcustom spam-bogofilter-database-directory nil
 399   "Directory path of the Bogofilter databases."
 400   :type '(choice (directory
 401                   :tag "Location of the Bogofilter database directory")
 402                  (const :tag "Use the default"))
 403   :group 'spam-bogofilter)
 404
 405 (defgroup spam-spamoracle nil
 406   "Spam spamoracle configuration."
 407   :group 'spam)
 408
 409 (defcustom spam-spamoracle-database nil
 410   "Location of spamoracle database file. When nil, use the default
 411 spamoracle database."
 412   :type '(choice (directory :tag "Location of spamoracle database file.")
 413                  (const :tag "Use the default"))
 414   :group 'spam-spamoracle)
 415
 416 (defcustom spam-spamoracle-binary (executable-find "spamoracle")
 417   "Location of the spamoracle binary."
 418   :type '(choice (directory :tag "Location of the spamoracle binary")
 419                  (const :tag "Use the default"))
 420   :group 'spam-spamoracle)
 421
 422 ;;; Key bindings for spam control.
 423
 424 (gnus-define-keys gnus-summary-mode-map
 425   "St" spam-bogofilter-score
 426   "Sx" gnus-summary-mark-as-spam
 427   "Mst" spam-bogofilter-score
 428   "Msx" gnus-summary-mark-as-spam
 429   "\M-d" gnus-summary-mark-as-spam)
 430
 431 (defvar spam-cache-lookups t
 432   "Whether spam.el will try to cache lookups using spam-caches.")
 433
 434 (defvar spam-caches (make-hash-table
 435                      :size 10
 436                      :test 'equal)
 437   "Cache of spam detection entries.")
 438
 439 (defvar spam-old-ham-articles nil
 440   "List of old ham articles, generated when a group is entered.")
 441
 442 (defvar spam-old-spam-articles nil
 443   "List of old spam articles, generated when a group is entered.")
 444
 445 (defvar spam-split-disabled nil
 446   "If non-nil, `spam-split' is disabled, and always returns nil.")
 447
 448 (defvar spam-split-last-successful-check nil
 449   "`spam-split' will set this to nil or a spam-use-XYZ check if it
 450   finds ham or spam.")
 451
 452 ;; convenience functions
 453 (defun spam-clear-cache (symbol)
 454   (remhash symbol spam-caches))
 455
 456 (defun spam-xor (a b)
 457   "Logical exclusive `or'."
 458   (and (or a b) (not (and a b))))
 459
 460 (defun spam-group-ham-mark-p (group mark &optional spam)
 461   (when (stringp group)
 462     (let* ((marks (spam-group-ham-marks group spam))
 463            (marks (if (symbolp mark)
 464                       marks
 465                     (mapcar 'symbol-value marks))))
 466       (memq mark marks))))
 467
 468 (defun spam-group-spam-mark-p (group mark)
 469   (spam-group-ham-mark-p group mark t))
 470
 471 (defun spam-group-ham-marks (group &optional spam)
 472   (when (stringp group)
 473     (let* ((marks (if spam
 474                       (gnus-parameter-spam-marks group)
 475                     (gnus-parameter-ham-marks group)))
 476            (marks (car marks))
 477            (marks (if (listp (car marks)) (car marks) marks)))
 478       marks)))
 479
 480 (defun spam-group-spam-marks (group)
 481   (spam-group-ham-marks group t))
 482
 483 (defun spam-group-spam-contents-p (group)
 484   (if (stringp group)
 485       (or (member group spam-junk-mailgroups)
 486           (memq 'gnus-group-spam-classification-spam
 487                 (gnus-parameter-spam-contents group)))
 488     nil))
 489
 490 (defun spam-group-ham-contents-p (group)
 491   (if (stringp group)
 492       (memq 'gnus-group-spam-classification-ham
 493             (gnus-parameter-spam-contents group))
 494     nil))
 495
 496 (defvar spam-list-of-processors
 497   '((gnus-group-spam-exit-processor-report-gmane spam spam-use-gmane)
 498     (gnus-group-spam-exit-processor-bogofilter   spam spam-use-bogofilter)
 499     (gnus-group-spam-exit-processor-blacklist    spam spam-use-blacklist)
 500     (gnus-group-spam-exit-processor-ifile        spam spam-use-ifile)
 501     (gnus-group-spam-exit-processor-stat         spam spam-use-stat)
 502     (gnus-group-spam-exit-processor-spamoracle   spam spam-use-spamoracle)
 503     (gnus-group-ham-exit-processor-ifile         ham spam-use-ifile)
 504     (gnus-group-ham-exit-processor-bogofilter    ham spam-use-bogofilter)
 505     (gnus-group-ham-exit-processor-stat          ham spam-use-stat)
 506     (gnus-group-ham-exit-processor-whitelist     ham spam-use-whitelist)
 507     (gnus-group-ham-exit-processor-BBDB          ham spam-use-BBDB)
 508     (gnus-group-ham-exit-processor-copy          ham spam-use-ham-copy)
 509     (gnus-group-ham-exit-processor-spamoracle    ham spam-use-spamoracle))
 510   "The spam-list-of-processors list contains pairs associating a
 511 ham/spam exit processor variable with a classification and a
 512 spam-use-* variable.")
 513
 514 (defun spam-group-processor-p (group processor)
 515   (if (and (stringp group)
 516            (symbolp processor))
 517       (or (member processor (nth 0 (gnus-parameter-spam-process group)))
 518           (spam-group-processor-multiple-p
 519            group
 520            (cdr-safe (assoc processor spam-list-of-processors))))
 521     nil))
 522
 523 (defun spam-group-processor-multiple-p (group processor-info)
 524   (let* ((classification (nth 0 processor-info))
 525          (check (nth 1 processor-info))
 526          (parameters (nth 0 (gnus-parameter-spam-process group)))
 527          found)
 528     (dolist (parameter parameters)
 529       (when (and (null found)
 530                  (listp parameter)
 531                  (eq classification (nth 0 parameter))
 532                  (eq check (nth 1 parameter)))
 533         (setq found t)))
 534     found))
 535
 536 (defun spam-group-spam-processor-report-gmane-p (group)
 537   (spam-group-processor-p group 'gnus-group-spam-exit-processor-report-gmane))
 538
 539 (defun spam-group-spam-processor-bogofilter-p (group)
 540   (spam-group-processor-p group 'gnus-group-spam-exit-processor-bogofilter))
 541
 542 (defun spam-group-spam-processor-blacklist-p (group)
 543   (spam-group-processor-p group 'gnus-group-spam-exit-processor-blacklist))
 544
 545 (defun spam-group-spam-processor-ifile-p (group)
 546   (spam-group-processor-p group 'gnus-group-spam-exit-processor-ifile))
 547
 548 (defun spam-group-ham-processor-ifile-p (group)
 549   (spam-group-processor-p group 'gnus-group-ham-exit-processor-ifile))
 550
 551 (defun spam-group-spam-processor-spamoracle-p (group)
 552   (spam-group-processor-p group 'gnus-group-spam-exit-processor-spamoracle))
 553
 554 (defun spam-group-ham-processor-bogofilter-p (group)
 555   (spam-group-processor-p group 'gnus-group-ham-exit-processor-bogofilter))
 556
 557 (defun spam-group-spam-processor-stat-p (group)
 558   (spam-group-processor-p group 'gnus-group-spam-exit-processor-stat))
 559
 560 (defun spam-group-ham-processor-stat-p (group)
 561   (spam-group-processor-p group 'gnus-group-ham-exit-processor-stat))
 562
 563 (defun spam-group-ham-processor-whitelist-p (group)
 564   (spam-group-processor-p group 'gnus-group-ham-exit-processor-whitelist))
 565
 566 (defun spam-group-ham-processor-BBDB-p (group)
 567   (spam-group-processor-p group 'gnus-group-ham-exit-processor-BBDB))
 568
 569 (defun spam-group-ham-processor-copy-p (group)
 570   (spam-group-processor-p group 'gnus-group-ham-exit-processor-copy))
 571
 572 (defun spam-group-ham-processor-spamoracle-p (group)
 573   (spam-group-processor-p group 'gnus-group-ham-exit-processor-spamoracle))
 574
 575 (defun spam-report-articles-gmane (n)
 576   "Report the current message as spam.
 577 Respects the process/prefix convention."
 578   (interactive "P")
 579   (dolist (article (gnus-summary-work-articles n))
 580     (gnus-summary-remove-process-mark article)
 581     (spam-report-gmane article)))
 582
 583 ;;; Summary entry and exit processing.
 584
 585 (defun spam-summary-prepare ()
 586   (setq spam-old-ham-articles
 587         (spam-list-articles gnus-newsgroup-articles 'ham))
 588   (setq spam-old-spam-articles
 589         (spam-list-articles gnus-newsgroup-articles 'spam))
 590   (spam-mark-junk-as-spam-routine))
 591
 592 ;; The spam processors are invoked for any group, spam or ham or neither
 593 (defun spam-summary-prepare-exit ()
 594   (unless gnus-group-is-exiting-without-update-p
 595     (gnus-message 6 "Exiting summary buffer and applying spam rules")
 596
 597     ;; first of all, unregister any articles that are no longer ham or spam
 598     ;; we have to iterate over the processors, or else we'll be too slow
 599     (dolist (classification '(spam ham))
 600       (let* ((old-articles (if (eq classification 'spam)
 601                                spam-old-spam-articles
 602                              spam-old-ham-articles))
 603              (new-articles (spam-list-articles
 604                             gnus-newsgroup-articles
 605                             classification))
 606              (changed-articles (gnus-set-difference old-articles new-articles)))
 607         ;; now that we have the changed articles, we go through the processors
 608         (dolist (processor-param spam-list-of-processors)
 609           (let ((processor (nth 0 processor-param))
 610                 (processor-classification (nth 1 processor-param))
 611                 (check (nth 2 processor-param))
 612                 unregister-list)
 613             (dolist (article changed-articles)
 614               (let ((id (spam-fetch-field-message-id-fast article)))
 615                 (when (spam-log-unregistration-needed-p
 616                        id 'process classification check)
 617                   (push article unregister-list))))
 618             ;; call spam-register-routine with specific articles to unregister,
 619             ;; when there are articles to unregister and the check is enabled
 620             (when (and unregister-list (symbol-value check))
 621               (spam-register-routine classification check t unregister-list))))))
 622
 623     ;; find all the spam processors applicable to this group
 624     (dolist (processor-param spam-list-of-processors)
 625       (let ((processor (nth 0 processor-param))
 626             (classification (nth 1 processor-param))
 627             (check (nth 2 processor-param)))
 628         (when (and (eq 'spam classification)
 629                    (spam-group-processor-p gnus-newsgroup-name processor))
 630           (spam-register-routine classification check))))
 631
 632     (if spam-move-spam-nonspam-groups-only
 633         (when (not (spam-group-spam-contents-p gnus-newsgroup-name))
 634           (spam-mark-spam-as-expired-and-move-routine
 635            (gnus-parameter-spam-process-destination gnus-newsgroup-name)))
 636       (gnus-message 5 "Marking spam as expired and moving it to %s"
 637                     gnus-newsgroup-name)
 638       (spam-mark-spam-as-expired-and-move-routine
 639        (gnus-parameter-spam-process-destination gnus-newsgroup-name)))
 640
 641     ;; now we redo spam-mark-spam-as-expired-and-move-routine to only
 642     ;; expire spam, in case the above did not expire them
 643     (gnus-message 5 "Marking spam as expired without moving it")
 644     (spam-mark-spam-as-expired-and-move-routine nil)
 645
 646     (when (or (spam-group-ham-contents-p gnus-newsgroup-name)
 647               (and (spam-group-spam-contents-p gnus-newsgroup-name)
 648                    spam-process-ham-in-spam-groups)
 649               spam-process-ham-in-nonham-groups)
 650       ;; find all the ham processors applicable to this group
 651       (dolist (processor-param spam-list-of-processors)
 652         (let ((processor (nth 0 processor-param))
 653               (classification (nth 1 processor-param))
 654               (check (nth 2 processor-param)))
 655           (when (and (eq 'ham classification)
 656                      (spam-group-processor-p gnus-newsgroup-name processor))
 657             (spam-register-routine classification check)))))
 658
 659     (when (spam-group-ham-processor-copy-p gnus-newsgroup-name)
 660       (gnus-message 5 "Copying ham")
 661       (spam-ham-copy-routine
 662        (gnus-parameter-ham-process-destination gnus-newsgroup-name)))
 663
 664     ;; now move all ham articles out of spam groups
 665     (when (spam-group-spam-contents-p gnus-newsgroup-name)
 666       (gnus-message 5 "Moving ham messages from spam group")
 667       (spam-ham-move-routine
 668        (gnus-parameter-ham-process-destination gnus-newsgroup-name))))
 669
 670   (setq spam-old-ham-articles nil)
 671   (setq spam-old-spam-articles nil))
 672
 673 (defun spam-mark-junk-as-spam-routine ()
 674   ;; check the global list of group names spam-junk-mailgroups and the
 675   ;; group parameters
 676   (when (spam-group-spam-contents-p gnus-newsgroup-name)
 677     (gnus-message 5 "Marking %s articles as spam"
 678                   (if spam-mark-only-unseen-as-spam
 679                       "unseen"
 680                     "unread"))
 681     (let ((articles (if spam-mark-only-unseen-as-spam
 682                         gnus-newsgroup-unseen
 683                       gnus-newsgroup-unreads)))
 684       (dolist (article articles)
 685         (gnus-summary-mark-article article gnus-spam-mark)))))
 686
 687 (defun spam-mark-spam-as-expired-and-move-routine (&rest groups)
 688   (if (and (car-safe groups) (listp (car-safe groups)))
 689       (apply 'spam-mark-spam-as-expired-and-move-routine (car groups))
 690     (gnus-summary-kill-process-mark)
 691     (let ((articles gnus-newsgroup-articles)
 692           (backend-supports-deletions
 693            (gnus-check-backend-function
 694             'request-move-article gnus-newsgroup-name))
 695           article tomove deletep)
 696       (dolist (article articles)
 697         (when (eq (gnus-summary-article-mark article) gnus-spam-mark)
 698           (gnus-summary-mark-article article gnus-expirable-mark)
 699           (push article tomove)))
 700
 701       ;; now do the actual copies
 702       (dolist (group groups)
 703         (when (and tomove
 704                    (stringp group))
 705           (dolist (article tomove)
 706             (gnus-summary-set-process-mark article))
 707           (when tomove
 708             (if (or (not backend-supports-deletions)
 709                     (> (length groups) 1))
 710                 (progn
 711                   (gnus-summary-copy-article nil group)
 712                   (setq deletep t))
 713               (gnus-summary-move-article nil group)))))
 714
 715       ;; now delete the articles, if there was a copy done, and the
 716       ;; backend allows it
 717       (when (and deletep backend-supports-deletions)
 718         (dolist (article tomove)
 719           (gnus-summary-set-process-mark article))
 720         (when tomove
 721           (let ((gnus-novice-user nil)) ; don't ask me if I'm sure
 722             (gnus-summary-delete-article nil))))
 723
 724       (gnus-summary-yank-process-mark))))
 725
 726 (defun spam-ham-copy-or-move-routine (copy groups)
 727   (gnus-summary-kill-process-mark)
 728   (let ((todo (spam-list-articles gnus-newsgroup-articles 'ham))
 729         (backend-supports-deletions
 730          (gnus-check-backend-function
 731           'request-move-article gnus-newsgroup-name))
 732         (respool-method (gnus-find-method-for-group gnus-newsgroup-name))
 733         article mark todo deletep respool)
 734
 735     (when (member 'respool groups)
 736       (setq respool t)                  ; boolean for later
 737       (setq groups '("fake"))) ; when respooling, groups are dynamic so fake it
 738
 739     ;; now do the actual move
 740     (dolist (group groups)
 741       (when (and todo (stringp group))
 742         (dolist (article todo)
 743           (when spam-mark-ham-unread-before-move-from-spam-group
 744             (gnus-summary-mark-article article gnus-unread-mark))
 745           (gnus-summary-set-process-mark article))
 746
 747         (if respool                        ; respooling is with a "fake" group
 748             (let ((spam-split-disabled
 749                    (or spam-split-disabled
 750                        spam-disable-spam-split-during-ham-respool)))
 751               (gnus-summary-respool-article nil respool-method))
 752           (if (or (not backend-supports-deletions) ; else, we are not respooling
 753                   (> (length groups) 1))
 754               (progn                ; if copying, copy and set deletep
 755                 (gnus-summary-copy-article nil group)
 756                 (setq deletep t))
 757             (gnus-summary-move-article nil group))))) ; else move articles
 758
 759     ;; now delete the articles, unless a) copy is t, and there was a copy done
 760     ;;                                 b) a move was done to a single group
 761     ;;                                 c) backend-supports-deletions is nil
 762     (unless copy
 763       (when (and deletep backend-supports-deletions)
 764         (dolist (article todo)
 765           (gnus-summary-set-process-mark article))
 766         (when todo
 767           (let ((gnus-novice-user nil)) ; don't ask me if I'm sure
 768             (gnus-summary-delete-article nil))))))
 769
 770   (gnus-summary-yank-process-mark))
 771
 772 (defun spam-ham-copy-routine (&rest groups)
 773   (if (and (car-safe groups) (listp (car-safe groups)))
 774       (apply 'spam-ham-copy-routine (car groups))
 775     (spam-ham-copy-or-move-routine t groups)))
 776
 777 (defun spam-ham-move-routine (&rest groups)
 778   (if (and (car-safe groups) (listp (car-safe groups)))
 779       (apply 'spam-ham-move-routine (car groups))
 780     (spam-ham-copy-or-move-routine nil groups)))
 781
 782 (defun spam-get-article-as-string (article)
 783   (let ((article-buffer (spam-get-article-as-buffer article))
 784         article-string)
 785     (when article-buffer
 786       (save-window-excursion
 787         (set-buffer article-buffer)
 788         (setq article-string (buffer-string))))
 789     article-string))
 790
 791 (defun spam-get-article-as-buffer (article)
 792   (let ((article-buffer))
 793     (when (numberp article)
 794       (save-window-excursion
 795         (gnus-summary-goto-subject article)
 796         (gnus-summary-show-article t)
 797         (setq article-buffer (get-buffer gnus-article-buffer))))
 798     article-buffer))
 799
 800 ;; disabled for now
 801 ;; (defun spam-get-article-as-filename (article)
 802 ;;   (let ((article-filename))
 803 ;;     (when (numberp article)
 804 ;;       (nnml-possibly-change-directory
 805 ;;        (gnus-group-real-name gnus-newsgroup-name))
 806 ;;       (setq article-filename (expand-file-name
 807 ;;                              (int-to-string article) nnml-current-directory)))
 808 ;;     (if (file-exists-p article-filename)
 809 ;;      article-filename
 810 ;;       nil)))
 811
 812 (defun spam-fetch-field-from-fast (article)
 813   "Fetch the `from' field quickly, using the internal gnus-data-list function"
 814   (if (and (numberp article)
 815            (assoc article (gnus-data-list nil)))
 816       (mail-header-from
 817        (gnus-data-header (assoc article (gnus-data-list nil))))
 818     nil))
 819
 820 (defun spam-fetch-field-subject-fast (article)
 821   "Fetch the `subject' field quickly, using the internal
 822   gnus-data-list function"
 823   (if (and (numberp article)
 824            (assoc article (gnus-data-list nil)))
 825       (mail-header-subject
 826        (gnus-data-header (assoc article (gnus-data-list nil))))
 827     nil))
 828
 829 (defun spam-fetch-field-message-id-fast (article)
 830   "Fetch the `Message-ID' field quickly, using the internal
 831   gnus-data-list function"
 832   (if (and (numberp article)
 833            (assoc article (gnus-data-list nil)))
 834       (mail-header-message-id
 835        (gnus-data-header (assoc article (gnus-data-list nil))))
 836     nil))
 837
 838 \f
 839 ;;;; Spam determination.
 840
 841 (defvar spam-list-of-checks
 842   '((spam-use-blacklist          . spam-check-blacklist)
 843     (spam-use-regex-headers      . spam-check-regex-headers)
 844     (spam-use-regex-body         . spam-check-regex-body)
 845     (spam-use-whitelist          . spam-check-whitelist)
 846     (spam-use-BBDB               . spam-check-BBDB)
 847     (spam-use-ifile              . spam-check-ifile)
 848     (spam-use-spamoracle         . spam-check-spamoracle)
 849     (spam-use-stat               . spam-check-stat)
 850     (spam-use-blackholes         . spam-check-blackholes)
 851     (spam-use-hashcash           . spam-check-hashcash)
 852     (spam-use-bogofilter-headers . spam-check-bogofilter-headers)
 853     (spam-use-bogofilter         . spam-check-bogofilter))
 854   "The spam-list-of-checks list contains pairs associating a
 855 parameter variable with a spam checking function.  If the
 856 parameter variable is true, then the checking function is called,
 857 and its value decides what happens.  Each individual check may
 858 return nil, t, or a mailgroup name.  The value nil means that the
 859 check does not yield a decision, and so, that further checks are
 860 needed.  The value t means that the message is definitely not
 861 spam, and that further spam checks should be inhibited.
 862 Otherwise, a mailgroup name or the symbol 'spam (depending on
 863 spam-split-symbolic-return) is returned where the mail should go,
 864 and further checks are also inhibited.  The usual mailgroup name
 865 is the value of `spam-split-group', meaning that the message is
 866 definitely a spam.")
 867
 868 (defvar spam-list-of-statistical-checks
 869   '(spam-use-ifile
 870     spam-use-regex-body
 871     spam-use-stat
 872     spam-use-bogofilter
 873     spam-use-spamoracle)
 874   "The spam-list-of-statistical-checks list contains all the mail
 875 splitters that need to have the full message body available.")
 876
 877 ;;;TODO: modify to invoke self with each check if invoked without specifics
 878 (defun spam-split (&rest specific-checks)
 879   "Split this message into the `spam' group if it is spam.
 880 This function can be used as an entry in the variable `nnmail-split-fancy',
 881 for example like this: (: spam-split).  It can take checks as
 882 parameters.  A string as a parameter will set the
 883 spam-split-group to that string.
 884
 885 See the Info node `(gnus)Fancy Mail Splitting' for more details."
 886   (interactive)
 887   (setq spam-split-last-successful-check nil)
 888   (unless spam-split-disabled
 889     (let ((spam-split-group-choice spam-split-group))
 890       (dolist (check specific-checks)
 891         (when (stringp check)
 892           (setq spam-split-group-choice check)
 893           (setq specific-checks (delq check specific-checks))))
 894
 895       (let ((spam-split-group spam-split-group-choice))
 896         (save-excursion
 897           (save-restriction
 898             (dolist (check spam-list-of-statistical-checks)
 899               (when (and (symbolp check) (symbol-value check))
 900                 (widen)
 901                 (gnus-message 8 "spam-split: widening the buffer (%s requires it)"
 902                               (symbol-name check))
 903                 (return)))
 904             ;;   (progn (widen) (debug (buffer-string)))
 905             (let ((list-of-checks spam-list-of-checks)
 906                   decision)
 907               (while (and list-of-checks (not decision))
 908                 (let ((pair (pop list-of-checks)))
 909                   (when (or
 910                          ;; either, given specific checks, this is one of them
 911                          (and specific-checks (memq (car pair) specific-checks))
 912                          ;; or, given no specific checks, spam-use-CHECK is set
 913                          (and (null specific-checks) (symbol-value (car pair))))
 914                     (gnus-message 5 "spam-split: calling the %s function"
 915                                   (symbol-name (cdr pair)))
 916                     (setq decision (funcall (cdr pair)))
 917                     ;; if we got a decision at all, save the current check
 918                     (when decision
 919                       (setq spam-split-last-successful-check (car pair)))
 920
 921                     (when (eq decision 'spam)
 922                       (unless spam-split-symbolic-return
 923                         (gnus-error
 924                          5
 925                          (format "spam-split got %s but %s is nil"
 926                                  (symbol-name decision)
 927                                  (symbol-name spam-split-symbolic-return))))))))
 928               (if (eq decision t)
 929                   (if spam-split-symbolic-return-positive 'ham nil)
 930                 decision))))))))
 931
 932 (defun spam-find-spam ()
 933   "This function will detect spam in the current newsgroup using spam-split."
 934   (interactive)
 935
 936   (let* ((group gnus-newsgroup-name)
 937          (autodetect (gnus-parameter-spam-autodetect group))
 938          (methods (gnus-parameter-spam-autodetect-methods group))
 939          (first-method (nth 0 methods))
 940          (articles (if spam-autodetect-recheck-messages
 941                        gnus-newsgroup-articles
 942                      gnus-newsgroup-unseen)))
 943
 944     (when (and autodetect
 945                (not (equal first-method 'none)))
 946     (mapcar
 947      (lambda (article)
 948        (let ((id (spam-fetch-field-message-id-fast article))
 949              (subject (spam-fetch-field-subject-fast article))
 950              (sender (spam-fetch-field-from-fast article))
 951              registry-lookup)
 952
 953          (unless id
 954            (gnus-error 5 "Article %d has no message ID!" article))
 955
 956          (when (and id spam-log-to-registry)
 957            (setq registry-lookup (spam-log-registration-type id 'incoming))
 958            (when registry-lookup
 959              (gnus-message
 960               9
 961               "spam-find-spam: message %s was already registered incoming"
 962               id)))
 963
 964          (let* ((spam-split-symbolic-return t)
 965                 (spam-split-symbolic-return-positive t)
 966                 (split-return
 967                  (or registry-lookup
 968                      (with-temp-buffer
 969                        (gnus-request-article-this-buffer
 970                         article
 971                         group)
 972                        (if (or (null first-method)
 973                                (equal first-method 'default))
 974                            (spam-split)
 975                          (apply 'spam-split methods))))))
 976            (if (equal split-return 'spam)
 977                (gnus-summary-mark-article article gnus-spam-mark))
 978
 979            (when (and id split-return spam-log-to-registry)
 980              (when (zerop (gnus-registry-group-count id))
 981                (gnus-registry-add-group
 982                 id group subject sender))
 983
 984              (unless registry-lookup
 985                (spam-log-processing-to-registry
 986                 id
 987                 'incoming
 988                 split-return
 989                 spam-split-last-successful-check
 990                 group))))))
 991     articles))))
 992
 993 (defvar spam-registration-functions
 994   ;; first the ham register, second the spam register function
 995   ;; third the ham unregister, fourth the spam unregister function
 996   '((spam-use-blacklist  nil
 997                          spam-blacklist-register-routine
 998                          nil
 999                          spam-blacklist-unregister-routine)
1000     (spam-use-whitelist  spam-whitelist-register-routine
1001                          nil
1002                          spam-whitelist-unregister-routine
1003                          nil)
1004     (spam-use-BBDB       spam-BBDB-register-routine
1005                          nil
1006                          spam-BBDB-unregister-routine
1007                          nil)
1008     (spam-use-ifile      spam-ifile-register-ham-routine
1009                          spam-ifile-register-spam-routine
1010                          spam-ifile-unregister-ham-routine
1011                          spam-ifile-unregister-spam-routine)
1012     (spam-use-spamoracle spam-spamoracle-learn-ham
1013                          spam-spamoracle-learn-spam
1014                          spam-spamoracle-unlearn-ham
1015                          spam-spamoracle-unlearn-spam)
1016     (spam-use-stat       spam-stat-register-ham-routine
1017                          spam-stat-register-spam-routine
1018                          spam-stat-unregister-ham-routine
1019                          spam-stat-unregister-spam-routine)
1020     ;; note that spam-use-gmane is not a legitimate check
1021     (spam-use-gmane      nil
1022                          spam-report-gmane-register-routine
1023                          ;; does Gmane support unregistration?
1024                          nil
1025                          nil)
1026     (spam-use-bogofilter spam-bogofilter-register-ham-routine
1027                          spam-bogofilter-register-spam-routine
1028                          spam-bogofilter-unregister-ham-routine
1029                          spam-bogofilter-unregister-spam-routine))
1030   "The spam-registration-functions list contains pairs
1031 associating a parameter variable with the ham and spam
1032 registration functions, and the ham and spam unregistration
1033 functions")
1034
1035 (defun spam-classification-valid-p (classification)
1036   (or  (eq classification 'spam)
1037        (eq classification 'ham)))
1038
1039 (defun spam-process-type-valid-p (process-type)
1040   (or  (eq process-type 'incoming)
1041        (eq process-type 'process)))
1042
1043 (defun spam-registration-check-valid-p (check)
1044   (assoc check spam-registration-functions))
1045
1046 (defun spam-unregistration-check-valid-p (check)
1047   (assoc check spam-registration-functions))
1048
1049 (defun spam-registration-function (classification check)
1050   (let ((flist (cdr-safe (assoc check spam-registration-functions))))
1051     (if (eq classification 'spam)
1052         (nth 1 flist)
1053       (nth 0 flist))))
1054
1055 (defun spam-unregistration-function (classification check)
1056   (let ((flist (cdr-safe (assoc check spam-registration-functions))))
1057     (if (eq classification 'spam)
1058         (nth 3 flist)
1059       (nth 2 flist))))
1060
1061 (defun spam-list-articles (articles classification)
1062   (let ((mark-check (if (eq classification 'spam)
1063                         'spam-group-spam-mark-p
1064                       'spam-group-ham-mark-p))
1065         list mark-cache-yes mark-cache-no)
1066     (dolist (article articles)
1067       (let ((mark (gnus-summary-article-mark article)))
1068         (unless (memq mark mark-cache-no)
1069           (if (memq mark mark-cache-yes)
1070               (push article list)
1071             ;; else, we have to actually check the mark
1072             (if (funcall mark-check
1073                          gnus-newsgroup-name
1074                          mark)
1075                 (progn
1076                   (push article list)
1077                   (push mark mark-cache-yes))
1078               (push mark mark-cache-no))))))
1079     list))
1080
1081 (defun spam-register-routine (classification
1082                               check
1083                               &optional unregister
1084                               specific-articles)
1085   (when (and (spam-classification-valid-p classification)
1086              (spam-registration-check-valid-p check))
1087     (let* ((register-function
1088             (spam-registration-function classification check))
1089            (unregister-function
1090             (spam-unregistration-function classification check))
1091            (run-function (if unregister
1092                              unregister-function
1093                            register-function))
1094            (log-function (if unregister
1095                              'spam-log-undo-registration
1096                            'spam-log-processing-to-registry))
1097            article articles)
1098
1099       (when run-function
1100         ;; make list of articles, using specific-articles if given
1101         (setq articles (or specific-articles
1102                            (spam-list-articles
1103                             gnus-newsgroup-articles
1104                             classification)))
1105         ;; process them
1106         (gnus-message 5 "%s %d %s articles with classification %s, check %s"
1107                       (if unregister "Unregistering" "Registering")
1108                       (length articles)
1109                       (if specific-articles "specific" "")
1110                       (symbol-name classification)
1111                       (symbol-name check))
1112         (funcall run-function articles)
1113         ;; now log all the registrations (or undo them, depending on unregister)
1114         (dolist (article articles)
1115           (funcall log-function
1116                    (spam-fetch-field-message-id-fast article)
1117                    'process
1118                    classification
1119                    check
1120                    gnus-newsgroup-name))))))
1121
1122 ;;; log a ham- or spam-processor invocation to the registry
1123 (defun spam-log-processing-to-registry (id type classification check group)
1124   (when spam-log-to-registry
1125     (if (and (stringp id)
1126              (stringp group)
1127              (spam-process-type-valid-p type)
1128              (spam-classification-valid-p classification)
1129              (spam-registration-check-valid-p check))
1130         (let ((cell-list (cdr-safe (gnus-registry-fetch-extra id type)))
1131               (cell (list classification check group)))
1132           (push cell cell-list)
1133           (gnus-registry-store-extra-entry
1134            id
1135            type
1136            cell-list))
1137
1138       (gnus-error 5 (format "%s called with bad ID, type, classification, check, or group"
1139                             "spam-log-processing-to-registry")))))
1140
1141 ;;; check if a ham- or spam-processor registration has been done
1142 (defun spam-log-registered-p (id type)
1143   (when spam-log-to-registry
1144     (if (and (stringp id)
1145              (spam-process-type-valid-p type))
1146         (cdr-safe (gnus-registry-fetch-extra id type))
1147       (progn
1148         (gnus-error 5 (format "%s called with bad ID, type, classification, or check"
1149                               "spam-log-registered-p"))
1150         nil))))
1151
1152 ;;; check what a ham- or spam-processor registration says
1153 ;;; returns nil if conflicting registrations are found
1154 (defun spam-log-registration-type (id type)
1155   (let ((count 0)
1156         decision)
1157     (dolist (reg (spam-log-registered-p id type))
1158       (let ((classification (nth 0 reg)))
1159         (when (spam-classification-valid-p classification)
1160           (when (and decision
1161                      (not (eq classification decision)))
1162             (setq count (+ 1 count)))
1163           (setq decision classification))))
1164     (if (< 0 count)
1165         nil
1166       decision)))
1167
1168 ;;; check if a ham- or spam-processor registration needs to be undone
1169 (defun spam-log-unregistration-needed-p (id type classification check)
1170   (when spam-log-to-registry
1171     (if (and (stringp id)
1172              (spam-process-type-valid-p type)
1173              (spam-classification-valid-p classification)
1174              (spam-registration-check-valid-p check))
1175         (let ((cell-list (cdr-safe (gnus-registry-fetch-extra id type)))
1176               found)
1177           (dolist (cell cell-list)
1178             (unless found
1179               (when (and (eq classification (nth 0 cell))
1180                          (eq check (nth 1 cell)))
1181                 (setq found t))))
1182           found)
1183       (progn
1184         (gnus-error 5 (format "%s called with bad ID, type, classification, or check"
1185                               "spam-log-unregistration-needed-p"))
1186         nil))))
1187
1188
1189 ;;; undo a ham- or spam-processor registration (the group is not used)
1190 (defun spam-log-undo-registration (id type classification check &optional group)
1191   (when (and spam-log-to-registry
1192              (spam-log-unregistration-needed-p id type classification check))
1193     (if (and (stringp id)
1194              (spam-process-type-valid-p type)
1195              (spam-classification-valid-p classification)
1196              (spam-registration-check-valid-p check))
1197         (let ((cell-list (cdr-safe (gnus-registry-fetch-extra id type)))
1198               new-cell-list found)
1199           (dolist (cell cell-list)
1200             (unless (and (eq classification (nth 0 cell))
1201                          (eq check (nth 1 cell)))
1202               (push cell new-cell-list)))
1203           (gnus-registry-store-extra-entry
1204            id
1205            type
1206            new-cell-list))
1207       (progn
1208         (gnus-error 5 (format "%s called with bad ID, type, check, or group"
1209                               "spam-log-undo-registration"))
1210         nil))))
1211
1212 ;;; set up IMAP widening if it's necessary
1213 (defun spam-setup-widening ()
1214   (dolist (check spam-list-of-statistical-checks)
1215     (when (symbol-value check)
1216       (setq nnimap-split-download-body-default t))))
1217
1218 \f
1219 ;;;; Regex body
1220
1221 (defun spam-check-regex-body ()
1222   (let ((spam-regex-headers-ham spam-regex-body-ham)
1223         (spam-regex-headers-spam spam-regex-body-spam))
1224     (spam-check-regex-headers t)))
1225
1226 \f
1227 ;;;; Regex headers
1228
1229 (defun spam-check-regex-headers (&optional body)
1230   (let ((type (if body "body" "header"))
1231         (spam-split-group (if spam-split-symbolic-return
1232                               'spam
1233                             spam-split-group))
1234         ret found)
1235     (dolist (h-regex spam-regex-headers-ham)
1236       (unless found
1237         (goto-char (point-min))
1238         (when (re-search-forward h-regex nil t)
1239           (message "Ham regex %s search positive." type)
1240           (setq found t))))
1241     (dolist (s-regex spam-regex-headers-spam)
1242       (unless found
1243         (goto-char (point-min))
1244         (when (re-search-forward s-regex nil t)
1245           (message "Spam regex %s search positive." type)
1246           (setq found t)
1247           (setq ret spam-split-group))))
1248     ret))
1249
1250 \f
1251 ;;;; Blackholes.
1252
1253 (defun spam-reverse-ip-string (ip)
1254   (when (stringp ip)
1255     (mapconcat 'identity
1256                (nreverse (split-string ip "\\."))
1257                ".")))
1258
1259 (defun spam-check-blackholes ()
1260   "Check the Received headers for blackholed relays."
1261   (let ((headers (nnmail-fetch-field "received"))
1262         (spam-split-group (if spam-split-symbolic-return
1263                               'spam
1264                             spam-split-group))
1265         ips matches)
1266     (when headers
1267       (with-temp-buffer
1268         (insert headers)
1269         (goto-char (point-min))
1270         (gnus-message 5 "Checking headers for relay addresses")
1271         (while (re-search-forward
1272                 "\\([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\\)" nil t)
1273           (gnus-message 9 "Blackhole search found host IP %s." (match-string 1))
1274           (push (spam-reverse-ip-string (match-string 1))
1275                 ips)))
1276       (dolist (server spam-blackhole-servers)
1277         (dolist (ip ips)
1278           (unless (and spam-blackhole-good-server-regex
1279                        ;; match the good-server-regex against the reversed (again) IP string
1280                        (string-match
1281                         spam-blackhole-good-server-regex
1282                         (spam-reverse-ip-string ip)))
1283             (unless matches
1284               (let ((query-string (concat ip "." server)))
1285                 (if spam-use-dig
1286                     (let ((query-result (query-dig query-string)))
1287                       (when query-result
1288                         (gnus-message 5 "(DIG): positive blackhole check '%s'"
1289                                       query-result)
1290                         (push (list ip server query-result)
1291                               matches)))
1292                   ;; else, if not using dig.el
1293                   (when (query-dns query-string)
1294                     (gnus-message 5 "positive blackhole check")
1295                     (push (list ip server (query-dns query-string 'TXT))
1296                           matches)))))))))
1297     (when matches
1298       spam-split-group)))
1299 \f
1300 ;;;; Hashcash.
1301
1302 (condition-case nil
1303     (progn
1304       (require 'hashcash)
1305
1306       (defun spam-check-hashcash ()
1307         "Check the headers for hashcash payments."
1308         (mail-check-payment)))   ;mail-check-payment returns a boolean
1309
1310   (file-error (progn
1311                 (defalias 'mail-check-payment 'ignore)
1312                 (defalias 'spam-check-hashcash 'ignore))))
1313 \f
1314 ;;;; BBDB
1315
1316 ;;; original idea for spam-check-BBDB from Alexander Kotelnikov
1317 ;;; <sacha@giotto.sj.ru>
1318
1319 ;; all this is done inside a condition-case to trap errors
1320
1321 (condition-case nil
1322     (progn
1323       (require 'bbdb)
1324       (require 'bbdb-com)
1325
1326       ;; when the BBDB changes, we want to clear out our cache
1327       (defun spam-clear-cache-BBDB (&rest immaterial)
1328         (spam-clear-cache 'spam-use-BBDB))
1329
1330       (add-hook 'bbdb-change-hook 'spam-clear-cache-BBDB)
1331
1332       (defun spam-enter-ham-BBDB (addresses &optional remove)
1333         "Enter an address into the BBDB; implies ham (non-spam) sender"
1334         (dolist (from addresses)
1335           (when (stringp from)
1336             (let* ((parsed-address (gnus-extract-address-components from))
1337                    (name (or (nth 0 parsed-address) "Ham Sender"))
1338                    (remove-function (if remove
1339                                         'bbdb-delete-record-internal
1340                                       'ignore))
1341                    (net-address (nth 1 parsed-address))
1342                    (record (and net-address
1343                                 (bbdb-search-simple nil net-address))))
1344               (when net-address
1345                 (gnus-message 5 "%s address %s %s BBDB"
1346                               (if remove "Deleting" "Adding")
1347                               from
1348                               (if remove "from" "to"))
1349                 (if record
1350                     (funcall remove-function record)
1351                   (bbdb-create-internal name nil net-address nil nil
1352                                         "ham sender added by spam.el")))))))
1353
1354       (defun spam-BBDB-register-routine (articles &optional unregister)
1355         (let (addresses)
1356           (dolist (article articles)
1357             (when (stringp (spam-fetch-field-from-fast article))
1358               (push (spam-fetch-field-from-fast article) addresses)))
1359           ;; now do the register/unregister action
1360           (spam-enter-ham-BBDB addresses unregister)))
1361
1362       (defun spam-BBDB-unregister-routine (articles)
1363         (spam-BBDB-register-routine articles t))
1364
1365       (defun spam-check-BBDB ()
1366         "Mail from people in the BBDB is classified as ham or non-spam"
1367         (let ((who (nnmail-fetch-field "from"))
1368               (spam-split-group (if spam-split-symbolic-return
1369                                     'spam
1370                                   spam-split-group))
1371               bbdb-cache bbdb-hashtable)
1372
1373           (when spam-cache-lookups
1374             (setq bbdb-cache (gethash 'spam-use-BBDB spam-caches))
1375             (unless bbdb-cache
1376               (setq bbdb-cache
1377                     ;; this is the expanded (bbdb-hashtable) macro
1378                     ;; without the debugging support
1379                     (with-current-buffer (bbdb-buffer)
1380                       (save-excursion
1381                         (save-window-excursion
1382                           (bbdb-records nil t)
1383                           bbdb-hashtable))))
1384               (puthash 'spam-use-BBDB bbdb-cache spam-caches)))
1385           (when who
1386             (setq who (nth 1 (gnus-extract-address-components who)))
1387             (if
1388                 (if spam-cache-lookups
1389                     (symbol-value
1390                      (intern-soft who bbdb-cache))
1391                   (bbdb-search-simple nil who))
1392                 t
1393               (if spam-use-BBDB-exclusive
1394                   spam-split-group
1395                 nil))))))
1396
1397   (file-error (progn
1398                 (defalias 'bbdb-search-simple 'ignore)
1399                 (defalias 'bbdb-records 'ignore)
1400                 (defalias 'bbdb-buffer 'ignore)
1401                 (defalias 'spam-check-BBDB 'ignore)
1402                 (defalias 'spam-BBDB-register-routine 'ignore)
1403                 (defalias 'spam-enter-ham-BBDB 'ignore)
1404                 (defalias 'bbdb-create-internal 'ignore)
1405                 (defalias 'bbdb-delete-record-internal 'ignore)
1406                 (defalias 'bbdb-records 'ignore))))
1407
1408 \f
1409 ;;;; ifile
1410
1411 ;;; check the ifile backend; return nil if the mail was NOT classified
1412 ;;; as spam
1413
1414 (defun spam-get-ifile-database-parameter ()
1415   "Get the command-line parameter for ifile's database from
1416   spam-ifile-database-path."
1417   (if spam-ifile-database-path
1418       (format "--db-file=%s" spam-ifile-database-path)
1419     nil))
1420
1421 (defun spam-check-ifile ()
1422   "Check the ifile backend for the classification of this message."
1423   (let ((article-buffer-name (buffer-name))
1424         (spam-split-group (if spam-split-symbolic-return
1425                               'spam
1426                             spam-split-group))
1427         category return)
1428     (with-temp-buffer
1429       (let ((temp-buffer-name (buffer-name))
1430             (db-param (spam-get-ifile-database-parameter)))
1431         (save-excursion
1432           (set-buffer article-buffer-name)
1433           (apply 'call-process-region
1434                  (point-min) (point-max) spam-ifile-path
1435                  nil temp-buffer-name nil "-c"
1436                  (if db-param `(,db-param "-q") `("-q"))))
1437         ;; check the return now (we're back in the temp buffer)
1438         (goto-char (point-min))
1439         (if (not (eobp))
1440             (setq category (buffer-substring (point) (point-at-eol))))
1441         (when (not (zerop (length category))) ; we need a category here
1442           (if spam-ifile-all-categories
1443               (setq return category)
1444             ;; else, if spam-ifile-all-categories is not set...
1445             (when (string-equal spam-ifile-spam-category category)
1446               (setq return spam-split-group)))))) ; note return is nil otherwise
1447     return))
1448
1449 (defun spam-ifile-register-with-ifile (articles category &optional unregister)
1450   "Register an article, given as a string, with a category.
1451 Uses `gnus-newsgroup-name' if category is nil (for ham registration)."
1452   (let ((category (or category gnus-newsgroup-name))
1453         (add-or-delete-option (if unregister "-d" "-i"))
1454         (db (spam-get-ifile-database-parameter))
1455         parameters)
1456     (with-temp-buffer
1457       (dolist (article articles)
1458         (let ((article-string (spam-get-article-as-string article)))
1459           (when (stringp article-string)
1460             (insert article-string))))
1461       (apply 'call-process-region
1462              (point-min) (point-max) spam-ifile-path
1463              nil nil nil
1464              add-or-delete-option category
1465              (if db `(,db "-h") `("-h"))))))
1466
1467 (defun spam-ifile-register-spam-routine (articles &optional unregister)
1468   (spam-ifile-register-with-ifile articles spam-ifile-spam-category unregister))
1469
1470 (defun spam-ifile-unregister-spam-routine (articles)
1471   (spam-ifile-register-spam-routine articles t))
1472
1473 (defun spam-ifile-register-ham-routine (articles &optional unregister)
1474   (spam-ifile-register-with-ifile articles spam-ifile-ham-category unregister))
1475
1476 (defun spam-ifile-unregister-ham-routine (articles)
1477   (spam-ifile-register-ham-routine articles t))
1478
1479 \f
1480 ;;;; spam-stat
1481
1482 (condition-case nil
1483     (progn
1484       (let ((spam-stat-install-hooks nil))
1485         (require 'spam-stat))
1486
1487       (defun spam-check-stat ()
1488         "Check the spam-stat backend for the classification of this message"
1489         (let ((spam-split-group (if spam-split-symbolic-return
1490                                     'spam
1491                                   spam-split-group))
1492               (spam-stat-split-fancy-spam-group spam-split-group) ; override
1493               (spam-stat-buffer (buffer-name)) ; stat the current buffer
1494               category return)
1495           (spam-stat-split-fancy)))
1496
1497       (defun spam-stat-register-spam-routine (articles &optional unregister)
1498         (dolist (article articles)
1499           (let ((article-string (spam-get-article-as-string article)))
1500             (with-temp-buffer
1501               (insert article-string)
1502               (if unregister
1503                   (spam-stat-buffer-change-to-non-spam)
1504               (spam-stat-buffer-is-spam))))))
1505
1506       (defun spam-stat-unregister-spam-routine (articles)
1507         (spam-stat-register-spam-routine articles t))
1508
1509       (defun spam-stat-register-ham-routine (articles &optional unregister)
1510         (dolist (article articles)
1511           (let ((article-string (spam-get-article-as-string article)))
1512             (with-temp-buffer
1513               (insert article-string)
1514               (if unregister
1515                   (spam-stat-buffer-change-to-spam)
1516               (spam-stat-buffer-is-non-spam))))))
1517
1518       (defun spam-stat-unregister-ham-routine (articles)
1519         (spam-stat-register-ham-routine articles t))
1520
1521       (defun spam-maybe-spam-stat-load ()
1522         (when spam-use-stat (spam-stat-load)))
1523
1524       (defun spam-maybe-spam-stat-save ()
1525         (when spam-use-stat (spam-stat-save))))
1526
1527   (file-error (progn
1528                 (defalias 'spam-stat-load 'ignore)
1529                 (defalias 'spam-stat-save 'ignore)
1530                 (defalias 'spam-maybe-spam-stat-load 'ignore)
1531                 (defalias 'spam-maybe-spam-stat-save 'ignore)
1532                 (defalias 'spam-stat-register-ham-routine 'ignore)
1533                 (defalias 'spam-stat-unregister-ham-routine 'ignore)
1534                 (defalias 'spam-stat-register-spam-routine 'ignore)
1535                 (defalias 'spam-stat-unregister-spam-routine 'ignore)
1536                 (defalias 'spam-stat-buffer-is-spam 'ignore)
1537                 (defalias 'spam-stat-buffer-change-to-spam 'ignore)
1538                 (defalias 'spam-stat-buffer-is-non-spam 'ignore)
1539                 (defalias 'spam-stat-buffer-change-to-non-spam 'ignore)
1540                 (defalias 'spam-stat-split-fancy 'ignore)
1541                 (defalias 'spam-check-stat 'ignore))))
1542
1543 \f
1544
1545 ;;;; Blacklists and whitelists.
1546
1547 (defvar spam-whitelist-cache nil)
1548 (defvar spam-blacklist-cache nil)
1549
1550 (defun spam-kill-whole-line ()
1551   (beginning-of-line)
1552   (let ((kill-whole-line t))
1553     (kill-line)))
1554
1555 ;;; address can be a list, too
1556 (defun spam-enter-whitelist (address &optional remove)
1557   "Enter ADDRESS (list or single) into the whitelist.
1558 With a non-nil REMOVE, remove them."
1559   (interactive "sAddress: ")
1560   (spam-enter-list address spam-whitelist remove)
1561   (setq spam-whitelist-cache nil)
1562   (spam-clear-cache 'spam-use-whitelist))
1563
1564 ;;; address can be a list, too
1565 (defun spam-enter-blacklist (address &optional remove)
1566   "Enter ADDRESS (list or single) into the blacklist.
1567 With a non-nil REMOVE, remove them."
1568   (interactive "sAddress: ")
1569   (spam-enter-list address spam-blacklist remove)
1570   (setq spam-blacklist-cache nil)
1571   (spam-clear-cache 'spam-use-whitelist))
1572
1573 (defun spam-enter-list (addresses file &optional remove)
1574   "Enter ADDRESSES into the given FILE.
1575 Either the whitelist or the blacklist files can be used.  With
1576 REMOVE not nil, remove the ADDRESSES."
1577   (if (stringp addresses)
1578       (spam-enter-list (list addresses) file remove)
1579     ;; else, we have a list of addresses here
1580     (unless (file-exists-p (file-name-directory file))
1581       (make-directory (file-name-directory file) t))
1582     (save-excursion
1583       (set-buffer
1584        (find-file-noselect file))
1585       (dolist (a addresses)
1586         (when (stringp a)
1587           (goto-char (point-min))
1588           (if (re-search-forward (regexp-quote a) nil t)
1589               ;; found the address
1590               (when remove
1591                 (spam-kill-whole-line))
1592             ;; else, the address was not found
1593             (unless remove
1594               (goto-char (point-max))
1595               (unless (bobp)
1596                 (insert "\n"))
1597               (insert a "\n")))))
1598       (save-buffer))))
1599
1600 (defun spam-filelist-build-cache (type)
1601   (let ((cache (if (eq type 'spam-use-blacklist)
1602                    spam-blacklist-cache
1603                  spam-whitelist-cache))
1604         parsed-cache)
1605     (unless (gethash type spam-caches)
1606       (while cache
1607         (let ((address (pop cache)))
1608           (unless (zerop (length address)) ; 0 for a nil address too
1609             (setq address (regexp-quote address))
1610             ;; fix regexp-quote's treatment of user-intended regexes
1611             (while (string-match "\\\\\\*" address)
1612               (setq address (replace-match ".*" t t address))))
1613           (push address parsed-cache)))
1614       (puthash type parsed-cache spam-caches))))
1615
1616 (defun spam-filelist-check-cache (type from)
1617   (when (stringp from)
1618     (spam-filelist-build-cache type)
1619     (let (found)
1620       (dolist (address (gethash type spam-caches))
1621         (when (and address (string-match address from))
1622           (setq found t)
1623           (return)))
1624       found)))
1625
1626 ;;; returns t if the sender is in the whitelist, nil or
1627 ;;; spam-split-group otherwise
1628 (defun spam-check-whitelist ()
1629   ;; FIXME!  Should it detect when file timestamps change?
1630   (let ((spam-split-group (if spam-split-symbolic-return
1631                               'spam
1632                             spam-split-group)))
1633     (unless spam-whitelist-cache
1634       (setq spam-whitelist-cache (spam-parse-list spam-whitelist)))
1635     (if (spam-from-listed-p 'spam-use-whitelist)
1636         t
1637       (if spam-use-whitelist-exclusive
1638           spam-split-group
1639         nil))))
1640
1641 (defun spam-check-blacklist ()
1642   ;; FIXME!  Should it detect when file timestamps change?
1643   (let ((spam-split-group (if spam-split-symbolic-return
1644                               'spam
1645                             spam-split-group)))
1646     (unless spam-blacklist-cache
1647       (setq spam-blacklist-cache (spam-parse-list spam-blacklist)))
1648     (and (spam-from-listed-p 'spam-use-blacklist) spam-split-group)))
1649
1650 (defun spam-parse-list (file)
1651   (when (file-readable-p file)
1652     (let (contents address)
1653       (with-temp-buffer
1654         (insert-file-contents file)
1655         (while (not (eobp))
1656           (setq address (buffer-substring (point) (point-at-eol)))
1657           (forward-line 1)
1658           ;; insert the e-mail address if detected, otherwise the raw data
1659           (unless (zerop (length address))
1660             (let ((pure-address (nth 1 (gnus-extract-address-components address))))
1661               (push (or pure-address address) contents)))))
1662       (nreverse contents))))
1663
1664 (defun spam-from-listed-p (type)
1665   (let ((from (nnmail-fetch-field "from"))
1666         found)
1667     (spam-filelist-check-cache type from)))
1668
1669 (defun spam-filelist-register-routine (articles blacklist &optional unregister)
1670   (let ((de-symbol (if blacklist 'spam-use-whitelist 'spam-use-blacklist))
1671         (declassification (if blacklist 'ham 'spam))
1672         (enter-function
1673          (if blacklist 'spam-enter-blacklist 'spam-enter-whitelist))
1674         (remove-function
1675          (if blacklist 'spam-enter-whitelist 'spam-enter-blacklist))
1676         from addresses unregister-list)
1677     (dolist (article articles)
1678       (let ((from (spam-fetch-field-from-fast article))
1679             (id (spam-fetch-field-message-id-fast article))
1680             sender-ignored)
1681         (when (stringp from)
1682           (dolist (ignore-regex spam-blacklist-ignored-regexes)
1683             (when (and (not sender-ignored)
1684                        (stringp ignore-regex)
1685                        (string-match ignore-regex from))
1686               (setq sender-ignored t)))
1687           ;; remember the messages we need to unregister, unless remove is set
1688           (when (and
1689                  (null unregister)
1690                  (spam-log-unregistration-needed-p
1691                   id 'process declassification de-symbol))
1692             (push from unregister-list))
1693           (unless sender-ignored
1694             (push from addresses)))))
1695
1696     (if unregister
1697         (funcall enter-function addresses t) ; unregister all these addresses
1698       ;; else, register normally and unregister what we need to
1699       (funcall remove-function unregister-list t)
1700       (dolist (article unregister-list)
1701         (spam-log-undo-registration
1702          (spam-fetch-field-message-id-fast article)
1703          'process
1704          declassification
1705          de-symbol))
1706       (funcall enter-function addresses nil))))
1707
1708 (defun spam-blacklist-unregister-routine (articles)
1709   (spam-blacklist-register-routine articles t))
1710
1711 (defun spam-blacklist-register-routine (articles &optional unregister)
1712   (spam-filelist-register-routine articles t unregister))
1713
1714 (defun spam-whitelist-unregister-routine (articles)
1715   (spam-whitelist-register-routine articles t))
1716
1717 (defun spam-whitelist-register-routine (articles &optional unregister)
1718   (spam-filelist-register-routine articles nil unregister))
1719
1720 \f
1721 ;;;; Spam-report glue
1722 (defun spam-report-gmane-register-routine (articles)
1723   (when articles
1724     (apply 'spam-report-gmane articles)))
1725
1726 \f
1727 ;;;; Bogofilter
1728 (defun spam-check-bogofilter-headers (&optional score)
1729   (let ((header (nnmail-fetch-field spam-bogofilter-header))
1730         (spam-split-group (if spam-split-symbolic-return
1731                               'spam
1732                             spam-split-group)))
1733     (when header                        ; return nil when no header
1734       (if score                         ; scoring mode
1735           (if (string-match "spamicity=\\([0-9.]+\\)" header)
1736               (match-string 1 header)
1737             "0")
1738         ;; spam detection mode
1739         (when (string-match spam-bogofilter-bogosity-positive-spam-header
1740                             header)
1741           spam-split-group)))))
1742
1743 ;; return something sensible if the score can't be determined
1744 (defun spam-bogofilter-score ()
1745   "Get the Bogofilter spamicity score"
1746   (interactive)
1747   (save-window-excursion
1748     (gnus-summary-show-article t)
1749     (set-buffer gnus-article-buffer)
1750     (let ((score (or (spam-check-bogofilter-headers t)
1751                      (spam-check-bogofilter t))))
1752       (message "Spamicity score %s" score)
1753       (or score "0"))
1754     (gnus-summary-show-article)))
1755
1756 (defun spam-check-bogofilter (&optional score)
1757   "Check the Bogofilter backend for the classification of this message"
1758   (let ((article-buffer-name (buffer-name))
1759         (db spam-bogofilter-database-directory)
1760         return)
1761     (with-temp-buffer
1762       (let ((temp-buffer-name (buffer-name)))
1763         (save-excursion
1764           (set-buffer article-buffer-name)
1765           (apply 'call-process-region
1766                  (point-min) (point-max)
1767                  spam-bogofilter-path
1768                  nil temp-buffer-name nil
1769                  (if db `("-d" ,db "-v") `("-v"))))
1770         (setq return (spam-check-bogofilter-headers score))))
1771     return))
1772
1773 (defun spam-bogofilter-register-with-bogofilter (articles
1774                                                  spam
1775                                                  &optional unregister)
1776   "Register an article, given as a string, as spam or non-spam."
1777   (dolist (article articles)
1778     (let ((article-string (spam-get-article-as-string article))
1779           (db spam-bogofilter-database-directory)
1780           (switch (if unregister
1781                       (if spam
1782                           spam-bogofilter-spam-strong-switch
1783                         spam-bogofilter-ham-strong-switch)
1784                     (if spam
1785                         spam-bogofilter-spam-switch
1786                       spam-bogofilter-ham-switch))))
1787       (when (stringp article-string)
1788         (with-temp-buffer
1789           (insert article-string)
1790
1791           (apply 'call-process-region
1792                  (point-min) (point-max)
1793                  spam-bogofilter-path
1794                  nil nil nil switch
1795                  (if db `("-d" ,db "-v") `("-v"))))))))
1796
1797 (defun spam-bogofilter-register-spam-routine (articles &optional unregister)
1798   (spam-bogofilter-register-with-bogofilter articles t unregister))
1799
1800 (defun spam-bogofilter-unregister-spam-routine (articles)
1801   (spam-bogofilter-register-spam-routine articles t))
1802
1803 (defun spam-bogofilter-register-ham-routine (articles &optional unregister)
1804   (spam-bogofilter-register-with-bogofilter articles nil unregister))
1805
1806 (defun spam-bogofilter-unregister-ham-routine (articles)
1807   (spam-bogofilter-register-ham-routine articles t))
1808
1809
1810 \f
1811 ;;;; spamoracle
1812 (defun spam-check-spamoracle ()
1813   "Run spamoracle on an article to determine whether it's spam."
1814   (let ((article-buffer-name (buffer-name))
1815         (spam-split-group (if spam-split-symbolic-return
1816                               'spam
1817                             spam-split-group)))
1818     (with-temp-buffer
1819       (let ((temp-buffer-name (buffer-name)))
1820         (save-excursion
1821           (set-buffer article-buffer-name)
1822           (let ((status
1823                  (apply 'call-process-region
1824                         (point-min) (point-max)
1825                         spam-spamoracle-binary
1826                         nil temp-buffer-name nil
1827                         (if spam-spamoracle-database
1828                             `("-f" ,spam-spamoracle-database "mark")
1829                           '("mark")))))
1830             (if (eq 0 status)
1831                 (progn
1832                   (set-buffer temp-buffer-name)
1833                   (goto-char (point-min))
1834                   (when (re-search-forward "^X-Spam: yes;" nil t)
1835                     spam-split-group))
1836               (error "Error running spamoracle" status))))))))
1837
1838 (defun spam-spamoracle-learn (articles article-is-spam-p &optional unregister)
1839   "Run spamoracle in training mode."
1840   (with-temp-buffer
1841     (let ((temp-buffer-name (buffer-name)))
1842       (save-excursion
1843         (goto-char (point-min))
1844         (dolist (article articles)
1845           (insert (spam-get-article-as-string article)))
1846         (let* ((arg (if (spam-xor unregister article-is-spam-p)
1847                         "-spam"
1848                       "-good"))
1849                (status
1850                 (apply 'call-process-region
1851                        (point-min) (point-max)
1852                        spam-spamoracle-binary
1853                        nil temp-buffer-name nil
1854                        (if spam-spamoracle-database
1855                            `("-f" ,spam-spamoracle-database
1856                              "add" ,arg)
1857                          `("add" ,arg)))))
1858           (when (not (eq 0 status))
1859             (error "Error running spamoracle" status)))))))
1860
1861 (defun spam-spamoracle-learn-ham (articles &optional unregister)
1862   (spam-spamoracle-learn articles nil unregister))
1863
1864 (defun spam-spamoracle-unlearn-ham (articles &optional unregister)
1865   (spam-spamoracle-learn-ham articles t))
1866
1867 (defun spam-spamoracle-learn-spam (articles &optional unregister)
1868   (spam-spamoracle-learn articles t unregister))
1869
1870 (defun spam-spamoracle-unlearn-spam (articles &optional unregister)
1871   (spam-spamoracle-learn-spam articles t))
1872
1873 \f
1874 ;;;; Hooks
1875
1876 ;;;###autoload
1877 (defun spam-initialize ()
1878   "Install the spam.el hooks and do other initialization"
1879   (interactive)
1880   (setq spam-install-hooks t)
1881   ;; TODO: How do we redo this every time spam-face is customized?
1882   (push '((eq mark gnus-spam-mark) . spam-face)
1883         gnus-summary-highlight)
1884   ;; Add hooks for loading and saving the spam stats
1885   (add-hook 'gnus-save-newsrc-hook 'spam-maybe-spam-stat-save)
1886   (add-hook 'gnus-get-top-new-news-hook 'spam-maybe-spam-stat-load)
1887   (add-hook 'gnus-startup-hook 'spam-maybe-spam-stat-load)
1888   (add-hook 'gnus-summary-prepare-exit-hook 'spam-summary-prepare-exit)
1889   (add-hook 'gnus-summary-prepare-hook 'spam-summary-prepare)
1890   (add-hook 'gnus-get-new-news-hook 'spam-setup-widening)
1891   (add-hook 'gnus-summary-prepare-hook 'spam-find-spam))
1892
1893 (defun spam-unload-hook ()
1894   "Uninstall the spam.el hooks"
1895   (interactive)
1896   (remove-hook 'gnus-save-newsrc-hook 'spam-maybe-spam-stat-save)
1897   (remove-hook 'gnus-get-top-new-news-hook 'spam-maybe-spam-stat-load)
1898   (remove-hook 'gnus-startup-hook 'spam-maybe-spam-stat-load)
1899   (remove-hook 'gnus-summary-prepare-exit-hook 'spam-summary-prepare-exit)
1900   (remove-hook 'gnus-summary-prepare-hook 'spam-summary-prepare)
1901   (remove-hook 'gnus-get-new-news-hook 'spam-setup-widening)
1902   (remove-hook 'gnus-summary-prepare-hook 'spam-find-spam))
1903
1904 (when spam-install-hooks
1905   (spam-initialize))
1906
1907 (provide 'spam)
1908
1909 ;;; spam.el ends here.