git.chise.org Git - elisp/gnus.git-/blob - lisp/spam.el

   1 ;;; spam.el --- Identifying spam
   2 ;; Copyright (C) 2002, 2003, 2004 Free Software Foundation, Inc.
   3
   4 ;; Author: Lars Magne Ingebrigtsen <larsi@gnus.org>
   5 ;; Keywords: network
   6
   7 ;; This file is part of GNU Emacs.
   8
   9 ;; GNU Emacs is free software; you can redistribute it and/or modify
  10 ;; it under the terms of the GNU General Public License as published by
  11 ;; the Free Software Foundation; either version 2, or (at your option)
  12 ;; any later version.
  13
  14 ;; GNU Emacs is distributed in the hope that it will be useful,
  15 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 ;; GNU General Public License for more details.
  18
  19 ;; You should have received a copy of the GNU General Public License
  20 ;; along with GNU Emacs; see the file COPYING.  If not, write to the
  21 ;; Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  22 ;; Boston, MA 02111-1307, USA.
  23
  24 ;;; Commentary:
  25
  26 ;;; This module addresses a few aspects of spam control under Gnus.  Page
  27 ;;; breaks are used for grouping declarations and documentation relating to
  28 ;;; each particular aspect.
  29
  30 ;;; The integration with Gnus is not yet complete.  See various `FIXME'
  31 ;;; comments, below, for supplementary explanations or discussions.
  32
  33 ;;; Several TODO items are marked as such
  34
  35 ;; TODO: spam scores, detection of spam in newsgroups, cross-server splitting,
  36 ;; remote processing, training through files
  37
  38 ;;; Code:
  39
  40 (require 'path-util)
  41
  42 (eval-when-compile (require 'cl))
  43
  44 (require 'gnus-sum)
  45
  46 (require 'gnus-uu)                      ; because of key prefix issues
  47 ;;; for the definitions of group content classification and spam processors
  48 (require 'gnus)
  49 (require 'message)              ;for the message-fetch-field functions
  50
  51 ;; for nnimap-split-download-body-default
  52 (eval-when-compile (require 'nnimap))
  53
  54 ;; autoload query-dig
  55 (eval-and-compile
  56   (autoload 'query-dig "dig"))
  57
  58 ;; autoload spam-report
  59 (eval-and-compile
  60   (autoload 'spam-report-gmane "spam-report"))
  61
  62 ;; autoload gnus-registry
  63 (eval-and-compile
  64   (autoload 'gnus-registry-group-count "gnus-registry")
  65   (autoload 'gnus-registry-add-group "gnus-registry")
  66   (autoload 'gnus-registry-store-extra-entry "gnus-registry")
  67   (autoload 'gnus-registry-fetch-extra "gnus-registry"))
  68
  69 ;; autoload query-dns
  70 (eval-and-compile
  71   (autoload 'query-dns "dns"))
  72
  73 ;;; Main parameters.
  74
  75 (defgroup spam nil
  76   "Spam configuration.")
  77
  78 (defcustom spam-directory "~/News/spam/"
  79   "Directory for spam whitelists and blacklists."
  80   :type 'directory
  81   :group 'spam)
  82
  83 (defcustom spam-move-spam-nonspam-groups-only t
  84   "Whether spam should be moved in non-spam groups only.
  85 When t, only ham and unclassified groups will have their spam moved
  86 to the spam-process-destination.  When nil, spam will also be moved from
  87 spam groups."
  88   :type 'boolean
  89   :group 'spam)
  90
  91 (defcustom spam-process-ham-in-nonham-groups nil
  92   "Whether ham should be processed in non-ham groups."
  93   :type 'boolean
  94   :group 'spam)
  95
  96 (defcustom spam-log-to-registry nil
  97   "Whether spam/ham processing should be logged in the registry."
  98   :type 'boolean
  99   :group 'spam)
 100
 101 (defcustom spam-split-symbolic-return nil
 102   "Whether `spam-split' should work with symbols or group names."
 103   :type 'boolean
 104   :group 'spam)
 105
 106 (defcustom spam-split-symbolic-return-positive nil
 107   "Whether `spam-split' should ALWAYS work with symbols or group names.
 108 Do not set this if you use `spam-split' in a fancy split
 109   method."
 110   :type 'boolean
 111   :group 'spam)
 112
 113 (defcustom spam-process-ham-in-spam-groups nil
 114   "Whether ham should be processed in spam groups."
 115   :type 'boolean
 116   :group 'spam)
 117
 118 (defcustom spam-mark-only-unseen-as-spam t
 119   "Whether only unseen articles should be marked as spam in spam groups.
 120 When nil, all unread articles in a spam group are marked as
 121 spam.  Set this if you want to leave an article unread in a spam group
 122 without losing it to the automatic spam-marking process."
 123   :type 'boolean
 124   :group 'spam)
 125
 126 (defcustom spam-mark-ham-unread-before-move-from-spam-group nil
 127   "Whether ham should be marked unread before it's moved.
 128 The article is moved out of a spam group according to ham-process-destination.
 129 This variable is an official entry in the international Longest Variable Name
 130 Competition."
 131   :type 'boolean
 132   :group 'spam)
 133
 134 (defcustom spam-disable-spam-split-during-ham-respool nil
 135   "Whether `spam-split' should be ignored while resplitting ham.
 136 This is useful to prevent ham from ending up in the same spam
 137 group after the resplit.  Don't set this to t if you have `spam-split' as the
 138 last rule in your split configuration."
 139   :type 'boolean
 140   :group 'spam)
 141
 142 (defcustom spam-autodetect-recheck-messages nil
 143   "Should spam.el recheck all meessages when autodetecting?
 144 Normally this is nil, so only unseen messages will be checked."
 145   :type 'boolean
 146   :group 'spam)
 147
 148 (defcustom spam-whitelist (expand-file-name "whitelist" spam-directory)
 149   "The location of the whitelist.
 150 The file format is one regular expression per line.
 151 The regular expression is matched against the address."
 152   :type 'file
 153   :group 'spam)
 154
 155 (defcustom spam-blacklist (expand-file-name "blacklist" spam-directory)
 156   "The location of the blacklist.
 157 The file format is one regular expression per line.
 158 The regular expression is matched against the address."
 159   :type 'file
 160   :group 'spam)
 161
 162 (defcustom spam-use-dig t
 163   "Whether `query-dig' should be used instead of `query-dns'."
 164   :type 'boolean
 165   :group 'spam)
 166
 167 (defcustom spam-use-blacklist nil
 168   "Whether the blacklist should be used by `spam-split'."
 169   :type 'boolean
 170   :group 'spam)
 171
 172 (defcustom spam-blacklist-ignored-regexes nil
 173   "Regular expressions that the blacklist should ignore."
 174   :type '(repeat (regexp :tag "Regular expression to ignore when blacklisting"))
 175   :group 'spam)
 176
 177 (defcustom spam-use-whitelist nil
 178   "Whether the whitelist should be used by `spam-split'."
 179   :type 'boolean
 180   :group 'spam)
 181
 182 (defcustom spam-use-whitelist-exclusive nil
 183   "Whether whitelist-exclusive should be used by `spam-split'.
 184 Exclusive whitelisting means that all messages from senders not in the whitelist
 185 are considered spam."
 186   :type 'boolean
 187   :group 'spam)
 188
 189 (defcustom spam-use-blackholes nil
 190   "Whether blackholes should be used by `spam-split'."
 191   :type 'boolean
 192   :group 'spam)
 193
 194 (defcustom spam-use-hashcash nil
 195   "Whether hashcash payments should be detected by `spam-split'."
 196   :type 'boolean
 197   :group 'spam)
 198
 199 (defcustom spam-use-regex-headers nil
 200   "Whether a header regular expression match should be used by `spam-split'.
 201 Also see the variables `spam-regex-headers-spam' and `spam-regex-headers-ham'."
 202   :type 'boolean
 203   :group 'spam)
 204
 205 (defcustom spam-use-regex-body nil
 206   "Whether a body regular expression match should be used by `spam-split'.
 207 Also see the variables `spam-regex-body-spam' and `spam-regex-body-ham'."
 208   :type 'boolean
 209   :group 'spam)
 210
 211 (defcustom spam-use-bogofilter-headers nil
 212   "Whether bogofilter headers should be used by `spam-split'.
 213 Enable this if you pre-process messages with Bogofilter BEFORE Gnus sees them."
 214   :type 'boolean
 215   :group 'spam)
 216
 217 (defcustom spam-use-bogofilter nil
 218   "Whether bogofilter should be invoked by `spam-split'.
 219 Enable this if you want Gnus to invoke Bogofilter on new messages."
 220   :type 'boolean
 221   :group 'spam)
 222
 223 (defcustom spam-use-BBDB nil
 224   "Whether BBDB should be used by `spam-split'."
 225   :type 'boolean
 226   :group 'spam)
 227
 228 (defcustom spam-use-BBDB-exclusive nil
 229   "Whether BBDB-exclusive should be used by `spam-split'.
 230 Exclusive BBDB means that all messages from senders not in the BBDB are
 231 considered spam."
 232   :type 'boolean
 233   :group 'spam)
 234
 235 (defcustom spam-use-ifile nil
 236   "Whether ifile should be used by `spam-split'."
 237   :type 'boolean
 238   :group 'spam)
 239
 240 (defcustom spam-use-stat nil
 241   "Whether `spam-stat' should be used by `spam-split'."
 242   :type 'boolean
 243   :group 'spam)
 244
 245 (defcustom spam-use-spamoracle nil
 246   "Whether spamoracle should be used by `spam-split'."
 247   :type 'boolean
 248   :group 'spam)
 249
 250 (defcustom spam-install-hooks (or
 251                                spam-use-dig
 252                                spam-use-blacklist
 253                                spam-use-whitelist
 254                                spam-use-whitelist-exclusive
 255                                spam-use-blackholes
 256                                spam-use-hashcash
 257                                spam-use-regex-headers
 258                                spam-use-regex-body
 259                                spam-use-bogofilter-headers
 260                                spam-use-bogofilter
 261                                spam-use-BBDB
 262                                spam-use-BBDB-exclusive
 263                                spam-use-ifile
 264                                spam-use-stat
 265                                spam-use-spamoracle)
 266   "Whether the spam hooks should be installed.
 267 Default to t if one of the spam-use-* variables is set."
 268   :group 'spam
 269   :type 'boolean)
 270
 271 (defcustom spam-split-group "spam"
 272   "Group name where incoming spam should be put by `spam-split'."
 273   :type 'string
 274   :group 'spam)
 275
 276 ;;; TODO: deprecate this variable, it's confusing since it's a list of strings,
 277 ;;; not regular expressions
 278 (defcustom spam-junk-mailgroups (cons
 279                                  spam-split-group
 280                                  '("mail.junk" "poste.pourriel"))
 281   "Mailgroups with spam contents.
 282 All unmarked article in such group receive the spam mark on group entry."
 283   :type '(repeat (string :tag "Group"))
 284   :group 'spam)
 285
 286 (defcustom spam-blackhole-servers '("bl.spamcop.net" "relays.ordb.org"
 287                                     "dev.null.dk" "relays.visi.com")
 288   "List of blackhole servers."
 289   :type '(repeat (string :tag "Server"))
 290   :group 'spam)
 291
 292 (defcustom spam-blackhole-good-server-regex nil
 293   "String matching IP addresses that should not be checked in the blackholes."
 294   :type '(radio (const nil)
 295                 (regexp :format "%t: %v\n" :size 0))
 296   :group 'spam)
 297
 298 (defcustom spam-face 'gnus-splash-face
 299   "Face for spam-marked articles."
 300   :type 'face
 301   :group 'spam)
 302
 303 (defcustom spam-regex-headers-spam '("^X-Spam-Flag: YES")
 304   "Regular expression for positive header spam matches."
 305   :type '(repeat (regexp :tag "Regular expression to match spam header"))
 306   :group 'spam)
 307
 308 (defcustom spam-regex-headers-ham '("^X-Spam-Flag: NO")
 309   "Regular expression for positive header ham matches."
 310   :type '(repeat (regexp :tag "Regular expression to match ham header"))
 311   :group 'spam)
 312
 313 (defcustom spam-regex-body-spam '()
 314   "Regular expression for positive body spam matches."
 315   :type '(repeat (regexp :tag "Regular expression to match spam body"))
 316   :group 'spam)
 317
 318 (defcustom spam-regex-body-ham '()
 319   "Regular expression for positive body ham matches."
 320   :type '(repeat (regexp :tag "Regular expression to match ham body"))
 321   :group 'spam)
 322
 323 (defgroup spam-ifile nil
 324   "Spam ifile configuration."
 325   :group 'spam)
 326
 327 (defcustom spam-ifile-path (exec-installed-p "ifile")
 328   "File path of the ifile executable program."
 329   :type '(choice (file :tag "Location of ifile")
 330                  (const :tag "ifile is not installed"))
 331   :group 'spam-ifile)
 332
 333 (defcustom spam-ifile-database-path nil
 334   "File path of the ifile database."
 335   :type '(choice (file :tag "Location of the ifile database")
 336                  (const :tag "Use the default"))
 337   :group 'spam-ifile)
 338
 339 (defcustom spam-ifile-spam-category "spam"
 340   "Name of the spam ifile category."
 341   :type 'string
 342   :group 'spam-ifile)
 343
 344 (defcustom spam-ifile-ham-category nil
 345   "Name of the ham ifile category.
 346 If nil, the current group name will be used."
 347   :type '(choice (string :tag "Use a fixed category")
 348                  (const :tag "Use the current group name"))
 349   :group 'spam-ifile)
 350
 351 (defcustom spam-ifile-all-categories nil
 352   "Whether the ifile check will return all categories, or just spam.
 353 Set this to t if you want to use the `spam-split' invocation of ifile as
 354 your main source of newsgroup names."
 355   :type 'boolean
 356   :group 'spam-ifile)
 357
 358 (defgroup spam-bogofilter nil
 359   "Spam bogofilter configuration."
 360   :group 'spam)
 361
 362 (defcustom spam-bogofilter-path (exec-installed-p "bogofilter")
 363   "File path of the Bogofilter executable program."
 364   :type '(choice (file :tag "Location of bogofilter")
 365                  (const :tag "Bogofilter is not installed"))
 366   :group 'spam-bogofilter)
 367
 368 (defcustom spam-bogofilter-header "X-Bogosity"
 369   "The header that Bogofilter inserts in messages."
 370   :type 'string
 371   :group 'spam-bogofilter)
 372
 373 (defcustom spam-bogofilter-spam-switch "-s"
 374   "The switch that Bogofilter uses to register spam messages."
 375   :type 'string
 376   :group 'spam-bogofilter)
 377
 378 (defcustom spam-bogofilter-ham-switch "-n"
 379   "The switch that Bogofilter uses to register ham messages."
 380   :type 'string
 381   :group 'spam-bogofilter)
 382
 383 (defcustom spam-bogofilter-spam-strong-switch "-S"
 384   "The switch that Bogofilter uses to unregister ham messages."
 385   :type 'string
 386   :group 'spam-bogofilter)
 387
 388 (defcustom spam-bogofilter-ham-strong-switch "-N"
 389   "The switch that Bogofilter uses to unregister spam messages."
 390   :type 'string
 391   :group 'spam-bogofilter)
 392
 393 (defcustom spam-bogofilter-bogosity-positive-spam-header "^\\(Yes\\|Spam\\)"
 394   "The regex on `spam-bogofilter-header' for positive spam identification."
 395   :type 'regexp
 396   :group 'spam-bogofilter)
 397
 398 (defcustom spam-bogofilter-database-directory nil
 399   "Directory path of the Bogofilter databases."
 400   :type '(choice (directory
 401                   :tag "Location of the Bogofilter database directory")
 402                  (const :tag "Use the default"))
 403   :group 'spam-bogofilter)
 404
 405 (defgroup spam-spamoracle nil
 406   "Spam spamoracle configuration."
 407   :group 'spam)
 408
 409 (defcustom spam-spamoracle-database nil
 410   "Location of spamoracle database file.
 411 When nil, use the default spamoracle database."
 412   :type '(choice (directory :tag "Location of spamoracle database file.")
 413                  (const :tag "Use the default"))
 414   :group 'spam-spamoracle)
 415
 416 (defcustom spam-spamoracle-binary (executable-find "spamoracle")
 417   "Location of the spamoracle binary."
 418   :type '(choice (directory :tag "Location of the spamoracle binary")
 419                  (const :tag "Use the default"))
 420   :group 'spam-spamoracle)
 421
 422 ;;; Key bindings for spam control.
 423
 424 (gnus-define-keys gnus-summary-mode-map
 425   "St" spam-bogofilter-score
 426   "Sx" gnus-summary-mark-as-spam
 427   "Mst" spam-bogofilter-score
 428   "Msx" gnus-summary-mark-as-spam
 429   "\M-d" gnus-summary-mark-as-spam)
 430
 431 (defvar spam-cache-lookups t
 432   "Whether spam.el will try to cache lookups using `spam-caches'.")
 433
 434 (defvar spam-caches (make-hash-table
 435                      :size 10
 436                      :test 'equal)
 437   "Cache of spam detection entries.")
 438
 439 (defvar spam-old-ham-articles nil
 440   "List of old ham articles, generated when a group is entered.")
 441
 442 (defvar spam-old-spam-articles nil
 443   "List of old spam articles, generated when a group is entered.")
 444
 445 (defvar spam-split-disabled nil
 446   "If non-nil, `spam-split' is disabled, and always returns nil.")
 447
 448 (defvar spam-split-last-successful-check nil
 449   "Internal variable.
 450 `spam-split' will set this to nil or a spam-use-XYZ check if it
 451 finds ham or spam.")
 452
 453 ;; convenience functions
 454 (defun spam-clear-cache (symbol)
 455   "Clear the spam-caches entry for a check."
 456   (remhash symbol spam-caches))
 457
 458 (defun spam-xor (a b)
 459   "Logical A xor B."
 460   (and (or a b) (not (and a b))))
 461
 462 (defun spam-group-ham-mark-p (group mark &optional spam)
 463   "Checks if MARK is considered a ham mark in GROUP."
 464   (when (stringp group)
 465     (let* ((marks (spam-group-ham-marks group spam))
 466            (marks (if (symbolp mark)
 467                       marks
 468                     (mapcar 'symbol-value marks))))
 469       (memq mark marks))))
 470
 471 (defun spam-group-spam-mark-p (group mark)
 472   "Checks if MARK is considered a spam mark in GROUP."
 473   (spam-group-ham-mark-p group mark t))
 474
 475 (defun spam-group-ham-marks (group &optional spam)
 476   "In GROUP, get all the ham marks."
 477   (when (stringp group)
 478     (let* ((marks (if spam
 479                       (gnus-parameter-spam-marks group)
 480                     (gnus-parameter-ham-marks group)))
 481            (marks (car marks))
 482            (marks (if (listp (car marks)) (car marks) marks)))
 483       marks)))
 484
 485 (defun spam-group-spam-marks (group)
 486   "In GROUP, get all the spam marks."
 487   (spam-group-ham-marks group t))
 488
 489 (defun spam-group-spam-contents-p (group)
 490   "Is GROUP a spam group?"
 491   (if (stringp group)
 492       (or (member group spam-junk-mailgroups)
 493           (memq 'gnus-group-spam-classification-spam
 494                 (gnus-parameter-spam-contents group)))
 495     nil))
 496
 497 (defun spam-group-ham-contents-p (group)
 498   "Is GROUP a ham group?"
 499   (if (stringp group)
 500       (memq 'gnus-group-spam-classification-ham
 501             (gnus-parameter-spam-contents group))
 502     nil))
 503
 504 (defvar spam-list-of-processors
 505   '((gnus-group-spam-exit-processor-report-gmane spam spam-use-gmane)
 506     (gnus-group-spam-exit-processor-bogofilter   spam spam-use-bogofilter)
 507     (gnus-group-spam-exit-processor-blacklist    spam spam-use-blacklist)
 508     (gnus-group-spam-exit-processor-ifile        spam spam-use-ifile)
 509     (gnus-group-spam-exit-processor-stat         spam spam-use-stat)
 510     (gnus-group-spam-exit-processor-spamoracle   spam spam-use-spamoracle)
 511     (gnus-group-ham-exit-processor-ifile         ham spam-use-ifile)
 512     (gnus-group-ham-exit-processor-bogofilter    ham spam-use-bogofilter)
 513     (gnus-group-ham-exit-processor-stat          ham spam-use-stat)
 514     (gnus-group-ham-exit-processor-whitelist     ham spam-use-whitelist)
 515     (gnus-group-ham-exit-processor-BBDB          ham spam-use-BBDB)
 516     (gnus-group-ham-exit-processor-copy          ham spam-use-ham-copy)
 517     (gnus-group-ham-exit-processor-spamoracle    ham spam-use-spamoracle))
 518   "The `spam-list-of-processors' list.
 519 This list contains pairs associating a ham/spam exit processor
 520 variable with a classification and a spam-use-* variable.")
 521
 522 (defun spam-group-processor-p (group processor)
 523   (if (and (stringp group)
 524            (symbolp processor))
 525       (or (member processor (nth 0 (gnus-parameter-spam-process group)))
 526           (spam-group-processor-multiple-p
 527            group
 528            (cdr-safe (assoc processor spam-list-of-processors))))
 529     nil))
 530
 531 (defun spam-group-processor-multiple-p (group processor-info)
 532   (let* ((classification (nth 0 processor-info))
 533          (check (nth 1 processor-info))
 534          (parameters (nth 0 (gnus-parameter-spam-process group)))
 535          found)
 536     (dolist (parameter parameters)
 537       (when (and (null found)
 538                  (listp parameter)
 539                  (eq classification (nth 0 parameter))
 540                  (eq check (nth 1 parameter)))
 541         (setq found t)))
 542     found))
 543
 544 (defun spam-group-spam-processor-report-gmane-p (group)
 545   (spam-group-processor-p group 'gnus-group-spam-exit-processor-report-gmane))
 546
 547 (defun spam-group-spam-processor-bogofilter-p (group)
 548   (spam-group-processor-p group 'gnus-group-spam-exit-processor-bogofilter))
 549
 550 (defun spam-group-spam-processor-blacklist-p (group)
 551   (spam-group-processor-p group 'gnus-group-spam-exit-processor-blacklist))
 552
 553 (defun spam-group-spam-processor-ifile-p (group)
 554   (spam-group-processor-p group 'gnus-group-spam-exit-processor-ifile))
 555
 556 (defun spam-group-ham-processor-ifile-p (group)
 557   (spam-group-processor-p group 'gnus-group-ham-exit-processor-ifile))
 558
 559 (defun spam-group-spam-processor-spamoracle-p (group)
 560   (spam-group-processor-p group 'gnus-group-spam-exit-processor-spamoracle))
 561
 562 (defun spam-group-ham-processor-bogofilter-p (group)
 563   (spam-group-processor-p group 'gnus-group-ham-exit-processor-bogofilter))
 564
 565 (defun spam-group-spam-processor-stat-p (group)
 566   (spam-group-processor-p group 'gnus-group-spam-exit-processor-stat))
 567
 568 (defun spam-group-ham-processor-stat-p (group)
 569   (spam-group-processor-p group 'gnus-group-ham-exit-processor-stat))
 570
 571 (defun spam-group-ham-processor-whitelist-p (group)
 572   (spam-group-processor-p group 'gnus-group-ham-exit-processor-whitelist))
 573
 574 (defun spam-group-ham-processor-BBDB-p (group)
 575   (spam-group-processor-p group 'gnus-group-ham-exit-processor-BBDB))
 576
 577 (defun spam-group-ham-processor-copy-p (group)
 578   (spam-group-processor-p group 'gnus-group-ham-exit-processor-copy))
 579
 580 (defun spam-group-ham-processor-spamoracle-p (group)
 581   (spam-group-processor-p group 'gnus-group-ham-exit-processor-spamoracle))
 582
 583 (defun spam-report-articles-gmane (n)
 584   "Report the current message as spam.
 585 Respects the process/prefix convention."
 586   (interactive "P")
 587   (dolist (article (gnus-summary-work-articles n))
 588     (gnus-summary-remove-process-mark article)
 589     (spam-report-gmane article)))
 590
 591 ;;; Summary entry and exit processing.
 592
 593 (defun spam-summary-prepare ()
 594   (setq spam-old-ham-articles
 595         (spam-list-articles gnus-newsgroup-articles 'ham))
 596   (setq spam-old-spam-articles
 597         (spam-list-articles gnus-newsgroup-articles 'spam))
 598   (spam-mark-junk-as-spam-routine))
 599
 600 ;; The spam processors are invoked for any group, spam or ham or neither
 601 (defun spam-summary-prepare-exit ()
 602   (unless gnus-group-is-exiting-without-update-p
 603     (gnus-message 6 "Exiting summary buffer and applying spam rules")
 604
 605     ;; first of all, unregister any articles that are no longer ham or spam
 606     ;; we have to iterate over the processors, or else we'll be too slow
 607     (dolist (classification '(spam ham))
 608       (let* ((old-articles (if (eq classification 'spam)
 609                                spam-old-spam-articles
 610                              spam-old-ham-articles))
 611              (new-articles (spam-list-articles
 612                             gnus-newsgroup-articles
 613                             classification))
 614              (changed-articles (gnus-set-difference old-articles new-articles)))
 615         ;; now that we have the changed articles, we go through the processors
 616         (dolist (processor-param spam-list-of-processors)
 617           (let ((processor (nth 0 processor-param))
 618                 (processor-classification (nth 1 processor-param))
 619                 (check (nth 2 processor-param))
 620                 unregister-list)
 621             (dolist (article changed-articles)
 622               (let ((id (spam-fetch-field-message-id-fast article)))
 623                 (when (spam-log-unregistration-needed-p
 624                        id 'process classification check)
 625                   (push article unregister-list))))
 626             ;; call spam-register-routine with specific articles to unregister,
 627             ;; when there are articles to unregister and the check is enabled
 628             (when (and unregister-list (symbol-value check))
 629               (spam-register-routine classification check t unregister-list))))))
 630
 631     ;; find all the spam processors applicable to this group
 632     (dolist (processor-param spam-list-of-processors)
 633       (let ((processor (nth 0 processor-param))
 634             (classification (nth 1 processor-param))
 635             (check (nth 2 processor-param)))
 636         (when (and (eq 'spam classification)
 637                    (spam-group-processor-p gnus-newsgroup-name processor))
 638           (spam-register-routine classification check))))
 639
 640     (if spam-move-spam-nonspam-groups-only
 641         (when (not (spam-group-spam-contents-p gnus-newsgroup-name))
 642           (spam-mark-spam-as-expired-and-move-routine
 643            (gnus-parameter-spam-process-destination gnus-newsgroup-name)))
 644       (gnus-message 5 "Marking spam as expired and moving it to %s"
 645                     gnus-newsgroup-name)
 646       (spam-mark-spam-as-expired-and-move-routine
 647        (gnus-parameter-spam-process-destination gnus-newsgroup-name)))
 648
 649     ;; now we redo spam-mark-spam-as-expired-and-move-routine to only
 650     ;; expire spam, in case the above did not expire them
 651     (gnus-message 5 "Marking spam as expired without moving it")
 652     (spam-mark-spam-as-expired-and-move-routine nil)
 653
 654     (when (or (spam-group-ham-contents-p gnus-newsgroup-name)
 655               (and (spam-group-spam-contents-p gnus-newsgroup-name)
 656                    spam-process-ham-in-spam-groups)
 657               spam-process-ham-in-nonham-groups)
 658       ;; find all the ham processors applicable to this group
 659       (dolist (processor-param spam-list-of-processors)
 660         (let ((processor (nth 0 processor-param))
 661               (classification (nth 1 processor-param))
 662               (check (nth 2 processor-param)))
 663           (when (and (eq 'ham classification)
 664                      (spam-group-processor-p gnus-newsgroup-name processor))
 665             (spam-register-routine classification check)))))
 666
 667     (when (spam-group-ham-processor-copy-p gnus-newsgroup-name)
 668       (gnus-message 5 "Copying ham")
 669       (spam-ham-copy-routine
 670        (gnus-parameter-ham-process-destination gnus-newsgroup-name)))
 671
 672     ;; now move all ham articles out of spam groups
 673     (when (spam-group-spam-contents-p gnus-newsgroup-name)
 674       (gnus-message 5 "Moving ham messages from spam group")
 675       (spam-ham-move-routine
 676        (gnus-parameter-ham-process-destination gnus-newsgroup-name))))
 677
 678   (setq spam-old-ham-articles nil)
 679   (setq spam-old-spam-articles nil))
 680
 681 (defun spam-mark-junk-as-spam-routine ()
 682   ;; check the global list of group names spam-junk-mailgroups and the
 683   ;; group parameters
 684   (when (spam-group-spam-contents-p gnus-newsgroup-name)
 685     (gnus-message 5 "Marking %s articles as spam"
 686                   (if spam-mark-only-unseen-as-spam
 687                       "unseen"
 688                     "unread"))
 689     (let ((articles (if spam-mark-only-unseen-as-spam
 690                         gnus-newsgroup-unseen
 691                       gnus-newsgroup-unreads)))
 692       (dolist (article articles)
 693         (gnus-summary-mark-article article gnus-spam-mark)))))
 694
 695 (defun spam-mark-spam-as-expired-and-move-routine (&rest groups)
 696   (if (and (car-safe groups) (listp (car-safe groups)))
 697       (apply 'spam-mark-spam-as-expired-and-move-routine (car groups))
 698     (gnus-summary-kill-process-mark)
 699     (let ((articles gnus-newsgroup-articles)
 700           (backend-supports-deletions
 701            (gnus-check-backend-function
 702             'request-move-article gnus-newsgroup-name))
 703           article tomove deletep)
 704       (dolist (article articles)
 705         (when (eq (gnus-summary-article-mark article) gnus-spam-mark)
 706           (gnus-summary-mark-article article gnus-expirable-mark)
 707           (push article tomove)))
 708
 709       ;; now do the actual copies
 710       (dolist (group groups)
 711         (when (and tomove
 712                    (stringp group))
 713           (dolist (article tomove)
 714             (gnus-summary-set-process-mark article))
 715           (when tomove
 716             (if (or (not backend-supports-deletions)
 717                     (> (length groups) 1))
 718                 (progn
 719                   (gnus-summary-copy-article nil group)
 720                   (setq deletep t))
 721               (gnus-summary-move-article nil group)))))
 722
 723       ;; now delete the articles, if there was a copy done, and the
 724       ;; backend allows it
 725       (when (and deletep backend-supports-deletions)
 726         (dolist (article tomove)
 727           (gnus-summary-set-process-mark article))
 728         (when tomove
 729           (let ((gnus-novice-user nil)) ; don't ask me if I'm sure
 730             (gnus-summary-delete-article nil))))
 731
 732       (gnus-summary-yank-process-mark))))
 733
 734 (defun spam-ham-copy-or-move-routine (copy groups)
 735   (gnus-summary-kill-process-mark)
 736   (let ((todo (spam-list-articles gnus-newsgroup-articles 'ham))
 737         (backend-supports-deletions
 738          (gnus-check-backend-function
 739           'request-move-article gnus-newsgroup-name))
 740         (respool-method (gnus-find-method-for-group gnus-newsgroup-name))
 741         article mark todo deletep respool)
 742
 743     (when (member 'respool groups)
 744       (setq respool t)                  ; boolean for later
 745       (setq groups '("fake"))) ; when respooling, groups are dynamic so fake it
 746
 747     ;; now do the actual move
 748     (dolist (group groups)
 749       (when (and todo (stringp group))
 750         (dolist (article todo)
 751           (when spam-mark-ham-unread-before-move-from-spam-group
 752             (gnus-summary-mark-article article gnus-unread-mark))
 753           (gnus-summary-set-process-mark article))
 754
 755         (if respool                        ; respooling is with a "fake" group
 756             (let ((spam-split-disabled
 757                    (or spam-split-disabled
 758                        spam-disable-spam-split-during-ham-respool)))
 759               (gnus-summary-respool-article nil respool-method))
 760           (if (or (not backend-supports-deletions) ; else, we are not respooling
 761                   (> (length groups) 1))
 762               (progn                ; if copying, copy and set deletep
 763                 (gnus-summary-copy-article nil group)
 764                 (setq deletep t))
 765             (gnus-summary-move-article nil group))))) ; else move articles
 766
 767     ;; now delete the articles, unless a) copy is t, and there was a copy done
 768     ;;                                 b) a move was done to a single group
 769     ;;                                 c) backend-supports-deletions is nil
 770     (unless copy
 771       (when (and deletep backend-supports-deletions)
 772         (dolist (article todo)
 773           (gnus-summary-set-process-mark article))
 774         (when todo
 775           (let ((gnus-novice-user nil)) ; don't ask me if I'm sure
 776             (gnus-summary-delete-article nil))))))
 777
 778   (gnus-summary-yank-process-mark))
 779
 780 (defun spam-ham-copy-routine (&rest groups)
 781   (if (and (car-safe groups) (listp (car-safe groups)))
 782       (apply 'spam-ham-copy-routine (car groups))
 783     (spam-ham-copy-or-move-routine t groups)))
 784
 785 (defun spam-ham-move-routine (&rest groups)
 786   (if (and (car-safe groups) (listp (car-safe groups)))
 787       (apply 'spam-ham-move-routine (car groups))
 788     (spam-ham-copy-or-move-routine nil groups)))
 789
 790 (defun spam-get-article-as-string (article)
 791   (let ((article-buffer (spam-get-article-as-buffer article))
 792         article-string)
 793     (when article-buffer
 794       (save-window-excursion
 795         (set-buffer article-buffer)
 796         (setq article-string (buffer-string))))
 797     article-string))
 798
 799 (defun spam-get-article-as-buffer (article)
 800   (let ((article-buffer))
 801     (when (numberp article)
 802       (save-window-excursion
 803         (gnus-summary-goto-subject article)
 804         (gnus-summary-show-article t)
 805         (setq article-buffer (get-buffer gnus-article-buffer))))
 806     article-buffer))
 807
 808 ;; disabled for now
 809 ;; (defun spam-get-article-as-filename (article)
 810 ;;   (let ((article-filename))
 811 ;;     (when (numberp article)
 812 ;;       (nnml-possibly-change-directory
 813 ;;        (gnus-group-real-name gnus-newsgroup-name))
 814 ;;       (setq article-filename (expand-file-name
 815 ;;                              (int-to-string article) nnml-current-directory)))
 816 ;;     (if (file-exists-p article-filename)
 817 ;;      article-filename
 818 ;;       nil)))
 819
 820 (defun spam-fetch-field-fast (article field &optional prepared-data-header)
 821   "Fetch a field quickly, using the internal gnus-data-list function"
 822   (when (numberp article)
 823     (let* ((data-header (or prepared-data-header
 824                             (spam-fetch-article-header article))))
 825       (if (arrayp data-header)
 826         (cond
 827          ((equal field 'from)
 828           (mail-header-from data-header))
 829          ((equal field 'message-id)
 830           (mail-header-message-id data-header))
 831          ((equal field 'subject)
 832           (mail-header-subject data-header))
 833          ((equal field 'references)
 834           (mail-header-references data-header))
 835          ((equal field 'date)
 836           (mail-header-date data-header))
 837          ((equal field 'xref)
 838           (mail-header-xref data-header))
 839          ((equal field 'extra)
 840           (mail-header-extra data-header))
 841          (t
 842           nil))
 843         (gnus-error 5 "Article %d has a nil data header" article)))))
 844
 845 (defun spam-fetch-field-from-fast (article &optional prepared-data-header)
 846   (spam-fetch-field-fast article 'from prepared-data-header))
 847
 848 (defun spam-fetch-field-subject-fast (article &optional prepared-data-header)
 849   (spam-fetch-field-fast article 'subject prepared-data-header))
 850
 851 (defun spam-fetch-field-message-id-fast (article &optional prepared-data-header)
 852   (spam-fetch-field-fast article 'message-id prepared-data-header))
 853
 854 (defun spam-generate-fake-headers (article)
 855   (let ((dh (spam-fetch-article-header article)))
 856     (if dh
 857         (concat
 858          (format
 859           (concat "From: %s\nSubject: %s\nMessage-ID: %s\n"
 860                   "Date: %s\nReferences: %s\nXref: %s\n")
 861           (spam-fetch-field-fast article 'from dh)
 862           (spam-fetch-field-fast article 'subject dh)
 863           (spam-fetch-field-fast article 'message-id dh)
 864           (spam-fetch-field-fast article 'date dh)
 865           (spam-fetch-field-fast article 'references dh)
 866           (spam-fetch-field-fast article 'xref dh))
 867          (when (spam-fetch-field-fast article 'extra dh)
 868            (format "%s\n" (spam-fetch-field-fast article 'extra dh))))
 869       (gnus-error
 870        5
 871        "spam-generate-fake-headers: article %d didn't have a valid header"
 872        article))))
 873
 874 (defun spam-fetch-article-header (article)
 875   (save-excursion
 876     (set-buffer gnus-summary-buffer)
 877     (nth 3 (assq article gnus-newsgroup-data))))
 878
 879 \f
 880 ;;;; Spam determination.
 881
 882 (defvar spam-list-of-checks
 883   '((spam-use-blacklist          . spam-check-blacklist)
 884     (spam-use-regex-headers      . spam-check-regex-headers)
 885     (spam-use-regex-body         . spam-check-regex-body)
 886     (spam-use-whitelist          . spam-check-whitelist)
 887     (spam-use-BBDB               . spam-check-BBDB)
 888     (spam-use-ifile              . spam-check-ifile)
 889     (spam-use-spamoracle         . spam-check-spamoracle)
 890     (spam-use-stat               . spam-check-stat)
 891     (spam-use-blackholes         . spam-check-blackholes)
 892     (spam-use-hashcash           . spam-check-hashcash)
 893     (spam-use-bogofilter-headers . spam-check-bogofilter-headers)
 894     (spam-use-bogofilter         . spam-check-bogofilter))
 895   "The spam-list-of-checks list contains pairs associating a
 896 parameter variable with a spam checking function.  If the
 897 parameter variable is true, then the checking function is called,
 898 and its value decides what happens.  Each individual check may
 899 return nil, t, or a mailgroup name.  The value nil means that the
 900 check does not yield a decision, and so, that further checks are
 901 needed.  The value t means that the message is definitely not
 902 spam, and that further spam checks should be inhibited.
 903 Otherwise, a mailgroup name or the symbol 'spam (depending on
 904 spam-split-symbolic-return) is returned where the mail should go,
 905 and further checks are also inhibited.  The usual mailgroup name
 906 is the value of `spam-split-group', meaning that the message is
 907 definitely a spam.")
 908
 909 (defvar spam-list-of-statistical-checks
 910   '(spam-use-ifile
 911     spam-use-regex-body
 912     spam-use-stat
 913     spam-use-bogofilter
 914     spam-use-blackholes
 915     spam-use-spamoracle)
 916   "The spam-list-of-statistical-checks list contains all the mail
 917 splitters that need to have the full message body available.
 918 Note that you should fetch extra headers if you don't like this,
 919 e.g. fetch the 'Received' header for spam-use-blackholes.")
 920
 921 (defun spam-split (&rest specific-checks)
 922   "Split this message into the `spam' group if it is spam.
 923 This function can be used as an entry in the variable `nnmail-split-fancy',
 924 for example like this: (: spam-split).  It can take checks as
 925 parameters.  A string as a parameter will set the
 926 spam-split-group to that string.
 927
 928 See the Info node `(gnus)Fancy Mail Splitting' for more details."
 929   (interactive)
 930   (setq spam-split-last-successful-check nil)
 931   (unless spam-split-disabled
 932     (let ((spam-split-group-choice spam-split-group))
 933       (dolist (check specific-checks)
 934         (when (stringp check)
 935           (setq spam-split-group-choice check)
 936           (setq specific-checks (delq check specific-checks))))
 937
 938       (let ((spam-split-group spam-split-group-choice))
 939         (save-excursion
 940           (save-restriction
 941             (dolist (check spam-list-of-statistical-checks)
 942               (when (and (symbolp check)
 943                          (or (symbol-value check)
 944                              (memq check specific-checks)))
 945                 (widen)
 946                 (gnus-message 8 "spam-split: widening the buffer (%s requires it)"
 947                               (symbol-name check))
 948                 (return)))
 949             ;;   (progn (widen) (debug (buffer-string)))
 950             (let ((list-of-checks spam-list-of-checks)
 951                   decision)
 952               (while (and list-of-checks (not decision))
 953                 (let ((pair (pop list-of-checks)))
 954                   (when (or
 955                          ;; either, given specific checks, this is one of them
 956                          (and specific-checks (memq (car pair) specific-checks))
 957                          ;; or, given no specific checks, spam-use-CHECK is set
 958                          (and (null specific-checks) (symbol-value (car pair))))
 959                     (gnus-message 5 "spam-split: calling the %s function"
 960                                   (symbol-name (cdr pair)))
 961                     (setq decision (funcall (cdr pair)))
 962                     ;; if we got a decision at all, save the current check
 963                     (when decision
 964                       (setq spam-split-last-successful-check (car pair)))
 965
 966                     (when (eq decision 'spam)
 967                       (unless spam-split-symbolic-return
 968                         (gnus-error
 969                          5
 970                          (format "spam-split got %s but %s is nil"
 971                                  (symbol-name decision)
 972                                  (symbol-name spam-split-symbolic-return))))))))
 973               (if (eq decision t)
 974                   (if spam-split-symbolic-return-positive 'ham nil)
 975                 decision))))))))
 976
 977 (defun spam-find-spam ()
 978   "This function will detect spam in the current newsgroup using spam-split."
 979   (interactive)
 980
 981   (let* ((group gnus-newsgroup-name)
 982          (autodetect (gnus-parameter-spam-autodetect group))
 983          (methods (gnus-parameter-spam-autodetect-methods group))
 984          (first-method (nth 0 methods))
 985          (articles (if spam-autodetect-recheck-messages
 986                        gnus-newsgroup-articles
 987                      gnus-newsgroup-unseen))
 988          article-cannot-be-faked)
 989
 990     (dolist (check spam-list-of-statistical-checks)
 991       (when (and (symbolp check)
 992                  (memq check methods))
 993         (setq article-cannot-be-faked t)
 994         (return)))
 995
 996     (when (memq 'default methods)
 997       (setq article-cannot-be-faked t))
 998
 999     (when (and autodetect
1000                (not (equal first-method 'none)))
1001     (mapcar
1002      (lambda (article)
1003        (let ((id (spam-fetch-field-message-id-fast article))
1004              (subject (spam-fetch-field-subject-fast article))
1005              (sender (spam-fetch-field-from-fast article))
1006              registry-lookup)
1007
1008          (unless id
1009            (gnus-error 5 "Article %d has no message ID!" article))
1010
1011          (when (and id spam-log-to-registry)
1012            (setq registry-lookup (spam-log-registration-type id 'incoming))
1013            (when registry-lookup
1014              (gnus-message
1015               9
1016               "spam-find-spam: message %s was already registered incoming"
1017               id)))
1018
1019          (let* ((spam-split-symbolic-return t)
1020                 (spam-split-symbolic-return-positive t)
1021                 (fake-headers (spam-generate-fake-headers article))
1022                 (split-return
1023                  (or registry-lookup
1024                      (with-temp-buffer
1025                        (if article-cannot-be-faked
1026                            (gnus-request-article-this-buffer
1027                             article
1028                             group)
1029                          ;; else, we fake the article
1030                          (when fake-headers (insert fake-headers)))
1031                        (if (or (null first-method)
1032                                (equal first-method 'default))
1033                            (spam-split)
1034                          (apply 'spam-split methods))))))
1035            (if (equal split-return 'spam)
1036                (gnus-summary-mark-article article gnus-spam-mark))
1037
1038            (when (and id split-return spam-log-to-registry)
1039              (when (zerop (gnus-registry-group-count id))
1040                (gnus-registry-add-group
1041                 id group subject sender))
1042
1043              (unless registry-lookup
1044                (spam-log-processing-to-registry
1045                 id
1046                 'incoming
1047                 split-return
1048                 spam-split-last-successful-check
1049                 group))))))
1050     articles))))
1051
1052 (defvar spam-registration-functions
1053   ;; first the ham register, second the spam register function
1054   ;; third the ham unregister, fourth the spam unregister function
1055   '((spam-use-blacklist  nil
1056                          spam-blacklist-register-routine
1057                          nil
1058                          spam-blacklist-unregister-routine)
1059     (spam-use-whitelist  spam-whitelist-register-routine
1060                          nil
1061                          spam-whitelist-unregister-routine
1062                          nil)
1063     (spam-use-BBDB       spam-BBDB-register-routine
1064                          nil
1065                          spam-BBDB-unregister-routine
1066                          nil)
1067     (spam-use-ifile      spam-ifile-register-ham-routine
1068                          spam-ifile-register-spam-routine
1069                          spam-ifile-unregister-ham-routine
1070                          spam-ifile-unregister-spam-routine)
1071     (spam-use-spamoracle spam-spamoracle-learn-ham
1072                          spam-spamoracle-learn-spam
1073                          spam-spamoracle-unlearn-ham
1074                          spam-spamoracle-unlearn-spam)
1075     (spam-use-stat       spam-stat-register-ham-routine
1076                          spam-stat-register-spam-routine
1077                          spam-stat-unregister-ham-routine
1078                          spam-stat-unregister-spam-routine)
1079     ;; note that spam-use-gmane is not a legitimate check
1080     (spam-use-gmane      nil
1081                          spam-report-gmane-register-routine
1082                          ;; does Gmane support unregistration?
1083                          nil
1084                          nil)
1085     (spam-use-bogofilter spam-bogofilter-register-ham-routine
1086                          spam-bogofilter-register-spam-routine
1087                          spam-bogofilter-unregister-ham-routine
1088                          spam-bogofilter-unregister-spam-routine))
1089   "The spam-registration-functions list contains pairs
1090 associating a parameter variable with the ham and spam
1091 registration functions, and the ham and spam unregistration
1092 functions")
1093
1094 (defun spam-classification-valid-p (classification)
1095   (or  (eq classification 'spam)
1096        (eq classification 'ham)))
1097
1098 (defun spam-process-type-valid-p (process-type)
1099   (or  (eq process-type 'incoming)
1100        (eq process-type 'process)))
1101
1102 (defun spam-registration-check-valid-p (check)
1103   (assoc check spam-registration-functions))
1104
1105 (defun spam-unregistration-check-valid-p (check)
1106   (assoc check spam-registration-functions))
1107
1108 (defun spam-registration-function (classification check)
1109   (let ((flist (cdr-safe (assoc check spam-registration-functions))))
1110     (if (eq classification 'spam)
1111         (nth 1 flist)
1112       (nth 0 flist))))
1113
1114 (defun spam-unregistration-function (classification check)
1115   (let ((flist (cdr-safe (assoc check spam-registration-functions))))
1116     (if (eq classification 'spam)
1117         (nth 3 flist)
1118       (nth 2 flist))))
1119
1120 (defun spam-list-articles (articles classification)
1121   (let ((mark-check (if (eq classification 'spam)
1122                         'spam-group-spam-mark-p
1123                       'spam-group-ham-mark-p))
1124         list mark-cache-yes mark-cache-no)
1125     (dolist (article articles)
1126       (let ((mark (gnus-summary-article-mark article)))
1127         (unless (memq mark mark-cache-no)
1128           (if (memq mark mark-cache-yes)
1129               (push article list)
1130             ;; else, we have to actually check the mark
1131             (if (funcall mark-check
1132                          gnus-newsgroup-name
1133                          mark)
1134                 (progn
1135                   (push article list)
1136                   (push mark mark-cache-yes))
1137               (push mark mark-cache-no))))))
1138     list))
1139
1140 (defun spam-register-routine (classification
1141                               check
1142                               &optional unregister
1143                               specific-articles)
1144   (when (and (spam-classification-valid-p classification)
1145              (spam-registration-check-valid-p check))
1146     (let* ((register-function
1147             (spam-registration-function classification check))
1148            (unregister-function
1149             (spam-unregistration-function classification check))
1150            (run-function (if unregister
1151                              unregister-function
1152                            register-function))
1153            (log-function (if unregister
1154                              'spam-log-undo-registration
1155                            'spam-log-processing-to-registry))
1156            article articles)
1157
1158       (when run-function
1159         ;; make list of articles, using specific-articles if given
1160         (setq articles (or specific-articles
1161                            (spam-list-articles
1162                             gnus-newsgroup-articles
1163                             classification)))
1164         ;; process them
1165         (gnus-message 5 "%s %d %s articles with classification %s, check %s"
1166                       (if unregister "Unregistering" "Registering")
1167                       (length articles)
1168                       (if specific-articles "specific" "")
1169                       (symbol-name classification)
1170                       (symbol-name check))
1171         (funcall run-function articles)
1172         ;; now log all the registrations (or undo them, depending on unregister)
1173         (dolist (article articles)
1174           (funcall log-function
1175                    (spam-fetch-field-message-id-fast article)
1176                    'process
1177                    classification
1178                    check
1179                    gnus-newsgroup-name))))))
1180
1181 ;;; log a ham- or spam-processor invocation to the registry
1182 (defun spam-log-processing-to-registry (id type classification check group)
1183   (when spam-log-to-registry
1184     (if (and (stringp id)
1185              (stringp group)
1186              (spam-process-type-valid-p type)
1187              (spam-classification-valid-p classification)
1188              (spam-registration-check-valid-p check))
1189         (let ((cell-list (cdr-safe (gnus-registry-fetch-extra id type)))
1190               (cell (list classification check group)))
1191           (push cell cell-list)
1192           (gnus-registry-store-extra-entry
1193            id
1194            type
1195            cell-list))
1196
1197       (gnus-error 5 (format "%s called with bad ID, type, classification, check, or group"
1198                             "spam-log-processing-to-registry")))))
1199
1200 ;;; check if a ham- or spam-processor registration has been done
1201 (defun spam-log-registered-p (id type)
1202   (when spam-log-to-registry
1203     (if (and (stringp id)
1204              (spam-process-type-valid-p type))
1205         (cdr-safe (gnus-registry-fetch-extra id type))
1206       (progn
1207         (gnus-error 5 (format "%s called with bad ID, type, classification, or check"
1208                               "spam-log-registered-p"))
1209         nil))))
1210
1211 ;;; check what a ham- or spam-processor registration says
1212 ;;; returns nil if conflicting registrations are found
1213 (defun spam-log-registration-type (id type)
1214   (let ((count 0)
1215         decision)
1216     (dolist (reg (spam-log-registered-p id type))
1217       (let ((classification (nth 0 reg)))
1218         (when (spam-classification-valid-p classification)
1219           (when (and decision
1220                      (not (eq classification decision)))
1221             (setq count (+ 1 count)))
1222           (setq decision classification))))
1223     (if (< 0 count)
1224         nil
1225       decision)))
1226
1227 ;;; check if a ham- or spam-processor registration needs to be undone
1228 (defun spam-log-unregistration-needed-p (id type classification check)
1229   (when spam-log-to-registry
1230     (if (and (stringp id)
1231              (spam-process-type-valid-p type)
1232              (spam-classification-valid-p classification)
1233              (spam-registration-check-valid-p check))
1234         (let ((cell-list (cdr-safe (gnus-registry-fetch-extra id type)))
1235               found)
1236           (dolist (cell cell-list)
1237             (unless found
1238               (when (and (eq classification (nth 0 cell))
1239                          (eq check (nth 1 cell)))
1240                 (setq found t))))
1241           found)
1242       (progn
1243         (gnus-error 5 (format "%s called with bad ID, type, classification, or check"
1244                               "spam-log-unregistration-needed-p"))
1245         nil))))
1246
1247
1248 ;;; undo a ham- or spam-processor registration (the group is not used)
1249 (defun spam-log-undo-registration (id type classification check &optional group)
1250   (when (and spam-log-to-registry
1251              (spam-log-unregistration-needed-p id type classification check))
1252     (if (and (stringp id)
1253              (spam-process-type-valid-p type)
1254              (spam-classification-valid-p classification)
1255              (spam-registration-check-valid-p check))
1256         (let ((cell-list (cdr-safe (gnus-registry-fetch-extra id type)))
1257               new-cell-list found)
1258           (dolist (cell cell-list)
1259             (unless (and (eq classification (nth 0 cell))
1260                          (eq check (nth 1 cell)))
1261               (push cell new-cell-list)))
1262           (gnus-registry-store-extra-entry
1263            id
1264            type
1265            new-cell-list))
1266       (progn
1267         (gnus-error 5 (format "%s called with bad ID, type, check, or group"
1268                               "spam-log-undo-registration"))
1269         nil))))
1270
1271 ;;; set up IMAP widening if it's necessary
1272 (defun spam-setup-widening ()
1273   (dolist (check spam-list-of-statistical-checks)
1274     (when (symbol-value check)
1275       (setq nnimap-split-download-body-default t))))
1276
1277 \f
1278 ;;;; Regex body
1279
1280 (defun spam-check-regex-body ()
1281   (let ((spam-regex-headers-ham spam-regex-body-ham)
1282         (spam-regex-headers-spam spam-regex-body-spam))
1283     (spam-check-regex-headers t)))
1284
1285 \f
1286 ;;;; Regex headers
1287
1288 (defun spam-check-regex-headers (&optional body)
1289   (let ((type (if body "body" "header"))
1290         (spam-split-group (if spam-split-symbolic-return
1291                               'spam
1292                             spam-split-group))
1293         ret found)
1294     (dolist (h-regex spam-regex-headers-ham)
1295       (unless found
1296         (goto-char (point-min))
1297         (when (re-search-forward h-regex nil t)
1298           (message "Ham regex %s search positive." type)
1299           (setq found t))))
1300     (dolist (s-regex spam-regex-headers-spam)
1301       (unless found
1302         (goto-char (point-min))
1303         (when (re-search-forward s-regex nil t)
1304           (message "Spam regex %s search positive." type)
1305           (setq found t)
1306           (setq ret spam-split-group))))
1307     ret))
1308
1309 \f
1310 ;;;; Blackholes.
1311
1312 (defun spam-reverse-ip-string (ip)
1313   (when (stringp ip)
1314     (mapconcat 'identity
1315                (nreverse (split-string ip "\\."))
1316                ".")))
1317
1318 (defun spam-check-blackholes ()
1319   "Check the Received headers for blackholed relays."
1320   (let ((headers (message-fetch-field "received"))
1321         (spam-split-group (if spam-split-symbolic-return
1322                               'spam
1323                             spam-split-group))
1324         ips matches)
1325     (when headers
1326       (with-temp-buffer
1327         (insert headers)
1328         (goto-char (point-min))
1329         (gnus-message 5 "Checking headers for relay addresses")
1330         (while (re-search-forward
1331                 "\\([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\\)" nil t)
1332           (gnus-message 9 "Blackhole search found host IP %s." (match-string 1))
1333           (push (spam-reverse-ip-string (match-string 1))
1334                 ips)))
1335       (dolist (server spam-blackhole-servers)
1336         (dolist (ip ips)
1337           (unless (and spam-blackhole-good-server-regex
1338                        ;; match the good-server-regex against the reversed (again) IP string
1339                        (string-match
1340                         spam-blackhole-good-server-regex
1341                         (spam-reverse-ip-string ip)))
1342             (unless matches
1343               (let ((query-string (concat ip "." server)))
1344                 (if spam-use-dig
1345                     (let ((query-result (query-dig query-string)))
1346                       (when query-result
1347                         (gnus-message 5 "(DIG): positive blackhole check '%s'"
1348                                       query-result)
1349                         (push (list ip server query-result)
1350                               matches)))
1351                   ;; else, if not using dig.el
1352                   (when (query-dns query-string)
1353                     (gnus-message 5 "positive blackhole check")
1354                     (push (list ip server (query-dns query-string 'TXT))
1355                           matches)))))))))
1356     (when matches
1357       spam-split-group)))
1358 \f
1359 ;;;; Hashcash.
1360
1361 (condition-case nil
1362     (progn
1363       (require 'hashcash)
1364
1365       (defun spam-check-hashcash ()
1366         "Check the headers for hashcash payments."
1367         (mail-check-payment)))   ;mail-check-payment returns a boolean
1368
1369   (file-error (progn
1370                 (defalias 'mail-check-payment 'ignore)
1371                 (defalias 'spam-check-hashcash 'ignore))))
1372 \f
1373 ;;;; BBDB
1374
1375 ;;; original idea for spam-check-BBDB from Alexander Kotelnikov
1376 ;;; <sacha@giotto.sj.ru>
1377
1378 ;; all this is done inside a condition-case to trap errors
1379
1380 (condition-case nil
1381     (progn
1382       (require 'bbdb)
1383       (require 'bbdb-com)
1384
1385       ;; when the BBDB changes, we want to clear out our cache
1386       (defun spam-clear-cache-BBDB (&rest immaterial)
1387         (spam-clear-cache 'spam-use-BBDB))
1388
1389       (add-hook 'bbdb-change-hook 'spam-clear-cache-BBDB)
1390
1391       (defun spam-enter-ham-BBDB (addresses &optional remove)
1392         "Enter an address into the BBDB; implies ham (non-spam) sender"
1393         (dolist (from addresses)
1394           (when (stringp from)
1395             (let* ((parsed-address (gnus-extract-address-components from))
1396                    (name (or (nth 0 parsed-address) "Ham Sender"))
1397                    (remove-function (if remove
1398                                         'bbdb-delete-record-internal
1399                                       'ignore))
1400                    (net-address (nth 1 parsed-address))
1401                    (record (and net-address
1402                                 (bbdb-search-simple nil net-address))))
1403               (when net-address
1404                 (gnus-message 5 "%s address %s %s BBDB"
1405                               (if remove "Deleting" "Adding")
1406                               from
1407                               (if remove "from" "to"))
1408                 (if record
1409                     (funcall remove-function record)
1410                   (bbdb-create-internal name nil net-address nil nil
1411                                         "ham sender added by spam.el")))))))
1412
1413       (defun spam-BBDB-register-routine (articles &optional unregister)
1414         (let (addresses)
1415           (dolist (article articles)
1416             (when (stringp (spam-fetch-field-from-fast article))
1417               (push (spam-fetch-field-from-fast article) addresses)))
1418           ;; now do the register/unregister action
1419           (spam-enter-ham-BBDB addresses unregister)))
1420
1421       (defun spam-BBDB-unregister-routine (articles)
1422         (spam-BBDB-register-routine articles t))
1423
1424       (defun spam-check-BBDB ()
1425         "Mail from people in the BBDB is classified as ham or non-spam"
1426         (let ((who (message-fetch-field "from"))
1427               (spam-split-group (if spam-split-symbolic-return
1428                                     'spam
1429                                   spam-split-group))
1430               bbdb-cache bbdb-hashtable)
1431           (when spam-cache-lookups
1432             (setq bbdb-cache (gethash 'spam-use-BBDB spam-caches))
1433             (unless bbdb-cache
1434               (setq bbdb-cache
1435                     ;; this is the expanded (bbdb-hashtable) macro
1436                     ;; without the debugging support
1437                     (with-current-buffer (bbdb-buffer)
1438                       (save-excursion
1439                         (save-window-excursion
1440                           (bbdb-records nil t)
1441                           bbdb-hashtable))))
1442               (puthash 'spam-use-BBDB bbdb-cache spam-caches)))
1443           (when who
1444             (setq who (nth 1 (gnus-extract-address-components who)))
1445             (if
1446                 (if spam-cache-lookups
1447                     (symbol-value
1448                      (intern-soft who bbdb-cache))
1449                   (bbdb-search-simple nil who))
1450                 t
1451               (if spam-use-BBDB-exclusive
1452                   spam-split-group
1453                 nil))))))
1454
1455   (file-error (progn
1456                 (defalias 'bbdb-search-simple 'ignore)
1457                 (defalias 'bbdb-records 'ignore)
1458                 (defalias 'bbdb-buffer 'ignore)
1459                 (defalias 'spam-check-BBDB 'ignore)
1460                 (defalias 'spam-BBDB-register-routine 'ignore)
1461                 (defalias 'spam-enter-ham-BBDB 'ignore)
1462                 (defalias 'bbdb-create-internal 'ignore)
1463                 (defalias 'bbdb-delete-record-internal 'ignore)
1464                 (defalias 'bbdb-records 'ignore))))
1465
1466 \f
1467 ;;;; ifile
1468
1469 ;;; check the ifile backend; return nil if the mail was NOT classified
1470 ;;; as spam
1471
1472 (defun spam-get-ifile-database-parameter ()
1473   "Get the command-line parameter for ifile's database from
1474   spam-ifile-database-path."
1475   (if spam-ifile-database-path
1476       (format "--db-file=%s" spam-ifile-database-path)
1477     nil))
1478
1479 (defun spam-check-ifile ()
1480   "Check the ifile backend for the classification of this message."
1481   (let ((article-buffer-name (buffer-name))
1482         (spam-split-group (if spam-split-symbolic-return
1483                               'spam
1484                             spam-split-group))
1485         category return)
1486     (with-temp-buffer
1487       (let ((temp-buffer-name (buffer-name))
1488             (db-param (spam-get-ifile-database-parameter)))
1489         (save-excursion
1490           (set-buffer article-buffer-name)
1491           (apply 'call-process-region
1492                  (point-min) (point-max) spam-ifile-path
1493                  nil temp-buffer-name nil "-c"
1494                  (if db-param `(,db-param "-q") `("-q"))))
1495         ;; check the return now (we're back in the temp buffer)
1496         (goto-char (point-min))
1497         (if (not (eobp))
1498             (setq category (buffer-substring (point) (point-at-eol))))
1499         (when (not (zerop (length category))) ; we need a category here
1500           (if spam-ifile-all-categories
1501               (setq return category)
1502             ;; else, if spam-ifile-all-categories is not set...
1503             (when (string-equal spam-ifile-spam-category category)
1504               (setq return spam-split-group)))))) ; note return is nil otherwise
1505     return))
1506
1507 (defun spam-ifile-register-with-ifile (articles category &optional unregister)
1508   "Register an article, given as a string, with a category.
1509 Uses `gnus-newsgroup-name' if category is nil (for ham registration)."
1510   (let ((category (or category gnus-newsgroup-name))
1511         (add-or-delete-option (if unregister "-d" "-i"))
1512         (db (spam-get-ifile-database-parameter))
1513         parameters)
1514     (with-temp-buffer
1515       (dolist (article articles)
1516         (let ((article-string (spam-get-article-as-string article)))
1517           (when (stringp article-string)
1518             (insert article-string))))
1519       (apply 'call-process-region
1520              (point-min) (point-max) spam-ifile-path
1521              nil nil nil
1522              add-or-delete-option category
1523              (if db `(,db "-h") `("-h"))))))
1524
1525 (defun spam-ifile-register-spam-routine (articles &optional unregister)
1526   (spam-ifile-register-with-ifile articles spam-ifile-spam-category unregister))
1527
1528 (defun spam-ifile-unregister-spam-routine (articles)
1529   (spam-ifile-register-spam-routine articles t))
1530
1531 (defun spam-ifile-register-ham-routine (articles &optional unregister)
1532   (spam-ifile-register-with-ifile articles spam-ifile-ham-category unregister))
1533
1534 (defun spam-ifile-unregister-ham-routine (articles)
1535   (spam-ifile-register-ham-routine articles t))
1536
1537 \f
1538 ;;;; spam-stat
1539
1540 (condition-case nil
1541     (progn
1542       (let ((spam-stat-install-hooks nil))
1543         (require 'spam-stat))
1544
1545       (defun spam-check-stat ()
1546         "Check the spam-stat backend for the classification of this message"
1547         (let ((spam-split-group (if spam-split-symbolic-return
1548                                     'spam
1549                                   spam-split-group))
1550               (spam-stat-split-fancy-spam-group spam-split-group) ; override
1551               (spam-stat-buffer (buffer-name)) ; stat the current buffer
1552               category return)
1553           (spam-stat-split-fancy)))
1554
1555       (defun spam-stat-register-spam-routine (articles &optional unregister)
1556         (dolist (article articles)
1557           (let ((article-string (spam-get-article-as-string article)))
1558             (with-temp-buffer
1559               (insert article-string)
1560               (if unregister
1561                   (spam-stat-buffer-change-to-non-spam)
1562               (spam-stat-buffer-is-spam))))))
1563
1564       (defun spam-stat-unregister-spam-routine (articles)
1565         (spam-stat-register-spam-routine articles t))
1566
1567       (defun spam-stat-register-ham-routine (articles &optional unregister)
1568         (dolist (article articles)
1569           (let ((article-string (spam-get-article-as-string article)))
1570             (with-temp-buffer
1571               (insert article-string)
1572               (if unregister
1573                   (spam-stat-buffer-change-to-spam)
1574               (spam-stat-buffer-is-non-spam))))))
1575
1576       (defun spam-stat-unregister-ham-routine (articles)
1577         (spam-stat-register-ham-routine articles t))
1578
1579       (defun spam-maybe-spam-stat-load ()
1580         (when spam-use-stat (spam-stat-load)))
1581
1582       (defun spam-maybe-spam-stat-save ()
1583         (when spam-use-stat (spam-stat-save))))
1584
1585   (file-error (progn
1586                 (defalias 'spam-stat-load 'ignore)
1587                 (defalias 'spam-stat-save 'ignore)
1588                 (defalias 'spam-maybe-spam-stat-load 'ignore)
1589                 (defalias 'spam-maybe-spam-stat-save 'ignore)
1590                 (defalias 'spam-stat-register-ham-routine 'ignore)
1591                 (defalias 'spam-stat-unregister-ham-routine 'ignore)
1592                 (defalias 'spam-stat-register-spam-routine 'ignore)
1593                 (defalias 'spam-stat-unregister-spam-routine 'ignore)
1594                 (defalias 'spam-stat-buffer-is-spam 'ignore)
1595                 (defalias 'spam-stat-buffer-change-to-spam 'ignore)
1596                 (defalias 'spam-stat-buffer-is-non-spam 'ignore)
1597                 (defalias 'spam-stat-buffer-change-to-non-spam 'ignore)
1598                 (defalias 'spam-stat-split-fancy 'ignore)
1599                 (defalias 'spam-check-stat 'ignore))))
1600
1601 \f
1602
1603 ;;;; Blacklists and whitelists.
1604
1605 (defvar spam-whitelist-cache nil)
1606 (defvar spam-blacklist-cache nil)
1607
1608 (defun spam-kill-whole-line ()
1609   (beginning-of-line)
1610   (let ((kill-whole-line t))
1611     (kill-line)))
1612
1613 ;;; address can be a list, too
1614 (defun spam-enter-whitelist (address &optional remove)
1615   "Enter ADDRESS (list or single) into the whitelist.
1616 With a non-nil REMOVE, remove them."
1617   (interactive "sAddress: ")
1618   (spam-enter-list address spam-whitelist remove)
1619   (setq spam-whitelist-cache nil)
1620   (spam-clear-cache 'spam-use-whitelist))
1621
1622 ;;; address can be a list, too
1623 (defun spam-enter-blacklist (address &optional remove)
1624   "Enter ADDRESS (list or single) into the blacklist.
1625 With a non-nil REMOVE, remove them."
1626   (interactive "sAddress: ")
1627   (spam-enter-list address spam-blacklist remove)
1628   (setq spam-blacklist-cache nil)
1629   (spam-clear-cache 'spam-use-whitelist))
1630
1631 (defun spam-enter-list (addresses file &optional remove)
1632   "Enter ADDRESSES into the given FILE.
1633 Either the whitelist or the blacklist files can be used.  With
1634 REMOVE not nil, remove the ADDRESSES."
1635   (if (stringp addresses)
1636       (spam-enter-list (list addresses) file remove)
1637     ;; else, we have a list of addresses here
1638     (unless (file-exists-p (file-name-directory file))
1639       (make-directory (file-name-directory file) t))
1640     (save-excursion
1641       (set-buffer
1642        (find-file-noselect file))
1643       (dolist (a addresses)
1644         (when (stringp a)
1645           (goto-char (point-min))
1646           (if (re-search-forward (regexp-quote a) nil t)
1647               ;; found the address
1648               (when remove
1649                 (spam-kill-whole-line))
1650             ;; else, the address was not found
1651             (unless remove
1652               (goto-char (point-max))
1653               (unless (bobp)
1654                 (insert "\n"))
1655               (insert a "\n")))))
1656       (save-buffer))))
1657
1658 (defun spam-filelist-build-cache (type)
1659   (let ((cache (if (eq type 'spam-use-blacklist)
1660                    spam-blacklist-cache
1661                  spam-whitelist-cache))
1662         parsed-cache)
1663     (unless (gethash type spam-caches)
1664       (while cache
1665         (let ((address (pop cache)))
1666           (unless (zerop (length address)) ; 0 for a nil address too
1667             (setq address (regexp-quote address))
1668             ;; fix regexp-quote's treatment of user-intended regexes
1669             (while (string-match "\\\\\\*" address)
1670               (setq address (replace-match ".*" t t address))))
1671           (push address parsed-cache)))
1672       (puthash type parsed-cache spam-caches))))
1673
1674 (defun spam-filelist-check-cache (type from)
1675   (when (stringp from)
1676     (spam-filelist-build-cache type)
1677     (let (found)
1678       (dolist (address (gethash type spam-caches))
1679         (when (and address (string-match address from))
1680           (setq found t)
1681           (return)))
1682       found)))
1683
1684 ;;; returns t if the sender is in the whitelist, nil or
1685 ;;; spam-split-group otherwise
1686 (defun spam-check-whitelist ()
1687   ;; FIXME!  Should it detect when file timestamps change?
1688   (let ((spam-split-group (if spam-split-symbolic-return
1689                               'spam
1690                             spam-split-group)))
1691     (unless spam-whitelist-cache
1692       (setq spam-whitelist-cache (spam-parse-list spam-whitelist)))
1693     (if (spam-from-listed-p 'spam-use-whitelist)
1694         t
1695       (if spam-use-whitelist-exclusive
1696           spam-split-group
1697         nil))))
1698
1699 (defun spam-check-blacklist ()
1700   ;; FIXME!  Should it detect when file timestamps change?
1701   (let ((spam-split-group (if spam-split-symbolic-return
1702                               'spam
1703                             spam-split-group)))
1704     (unless spam-blacklist-cache
1705       (setq spam-blacklist-cache (spam-parse-list spam-blacklist)))
1706     (and (spam-from-listed-p 'spam-use-blacklist) spam-split-group)))
1707
1708 (defun spam-parse-list (file)
1709   (when (file-readable-p file)
1710     (let (contents address)
1711       (with-temp-buffer
1712         (insert-file-contents file)
1713         (while (not (eobp))
1714           (setq address (buffer-substring (point) (point-at-eol)))
1715           (forward-line 1)
1716           ;; insert the e-mail address if detected, otherwise the raw data
1717           (unless (zerop (length address))
1718             (let ((pure-address (nth 1 (gnus-extract-address-components address))))
1719               (push (or pure-address address) contents)))))
1720       (nreverse contents))))
1721
1722 (defun spam-from-listed-p (type)
1723   (let ((from (message-fetch-field "from"))
1724         found)
1725     (spam-filelist-check-cache type from)))
1726
1727 (defun spam-filelist-register-routine (articles blacklist &optional unregister)
1728   (let ((de-symbol (if blacklist 'spam-use-whitelist 'spam-use-blacklist))
1729         (declassification (if blacklist 'ham 'spam))
1730         (enter-function
1731          (if blacklist 'spam-enter-blacklist 'spam-enter-whitelist))
1732         (remove-function
1733          (if blacklist 'spam-enter-whitelist 'spam-enter-blacklist))
1734         from addresses unregister-list)
1735     (dolist (article articles)
1736       (let ((from (spam-fetch-field-from-fast article))
1737             (id (spam-fetch-field-message-id-fast article))
1738             sender-ignored)
1739         (when (stringp from)
1740           (dolist (ignore-regex spam-blacklist-ignored-regexes)
1741             (when (and (not sender-ignored)
1742                        (stringp ignore-regex)
1743                        (string-match ignore-regex from))
1744               (setq sender-ignored t)))
1745           ;; remember the messages we need to unregister, unless remove is set
1746           (when (and
1747                  (null unregister)
1748                  (spam-log-unregistration-needed-p
1749                   id 'process declassification de-symbol))
1750             (push from unregister-list))
1751           (unless sender-ignored
1752             (push from addresses)))))
1753
1754     (if unregister
1755         (funcall enter-function addresses t) ; unregister all these addresses
1756       ;; else, register normally and unregister what we need to
1757       (funcall remove-function unregister-list t)
1758       (dolist (article unregister-list)
1759         (spam-log-undo-registration
1760          (spam-fetch-field-message-id-fast article)
1761          'process
1762          declassification
1763          de-symbol))
1764       (funcall enter-function addresses nil))))
1765
1766 (defun spam-blacklist-unregister-routine (articles)
1767   (spam-blacklist-register-routine articles t))
1768
1769 (defun spam-blacklist-register-routine (articles &optional unregister)
1770   (spam-filelist-register-routine articles t unregister))
1771
1772 (defun spam-whitelist-unregister-routine (articles)
1773   (spam-whitelist-register-routine articles t))
1774
1775 (defun spam-whitelist-register-routine (articles &optional unregister)
1776   (spam-filelist-register-routine articles nil unregister))
1777
1778 \f
1779 ;;;; Spam-report glue
1780 (defun spam-report-gmane-register-routine (articles)
1781   (when articles
1782     (apply 'spam-report-gmane articles)))
1783
1784 \f
1785 ;;;; Bogofilter
1786 (defun spam-check-bogofilter-headers (&optional score)
1787   (let ((header (message-fetch-field spam-bogofilter-header))
1788         (spam-split-group (if spam-split-symbolic-return
1789                               'spam
1790                             spam-split-group)))
1791     (when header                        ; return nil when no header
1792       (if score                         ; scoring mode
1793           (if (string-match "spamicity=\\([0-9.]+\\)" header)
1794               (match-string 1 header)
1795             "0")
1796         ;; spam detection mode
1797         (when (string-match spam-bogofilter-bogosity-positive-spam-header
1798                             header)
1799           spam-split-group)))))
1800
1801 ;; return something sensible if the score can't be determined
1802 (defun spam-bogofilter-score ()
1803   "Get the Bogofilter spamicity score"
1804   (interactive)
1805   (save-window-excursion
1806     (gnus-summary-show-article t)
1807     (set-buffer gnus-article-buffer)
1808     (let ((score (or (spam-check-bogofilter-headers t)
1809                      (spam-check-bogofilter t))))
1810       (message "Spamicity score %s" score)
1811       (or score "0"))
1812     (gnus-summary-show-article)))
1813
1814 (defun spam-check-bogofilter (&optional score)
1815   "Check the Bogofilter backend for the classification of this message"
1816   (let ((article-buffer-name (buffer-name))
1817         (db spam-bogofilter-database-directory)
1818         return)
1819     (with-temp-buffer
1820       (let ((temp-buffer-name (buffer-name)))
1821         (save-excursion
1822           (set-buffer article-buffer-name)
1823           (apply 'call-process-region
1824                  (point-min) (point-max)
1825                  spam-bogofilter-path
1826                  nil temp-buffer-name nil
1827                  (if db `("-d" ,db "-v") `("-v"))))
1828         (setq return (spam-check-bogofilter-headers score))))
1829     return))
1830
1831 (defun spam-bogofilter-register-with-bogofilter (articles
1832                                                  spam
1833                                                  &optional unregister)
1834   "Register an article, given as a string, as spam or non-spam."
1835   (dolist (article articles)
1836     (let ((article-string (spam-get-article-as-string article))
1837           (db spam-bogofilter-database-directory)
1838           (switch (if unregister
1839                       (if spam
1840                           spam-bogofilter-spam-strong-switch
1841                         spam-bogofilter-ham-strong-switch)
1842                     (if spam
1843                         spam-bogofilter-spam-switch
1844                       spam-bogofilter-ham-switch))))
1845       (when (stringp article-string)
1846         (with-temp-buffer
1847           (insert article-string)
1848
1849           (apply 'call-process-region
1850                  (point-min) (point-max)
1851                  spam-bogofilter-path
1852                  nil nil nil switch
1853                  (if db `("-d" ,db "-v") `("-v"))))))))
1854
1855 (defun spam-bogofilter-register-spam-routine (articles &optional unregister)
1856   (spam-bogofilter-register-with-bogofilter articles t unregister))
1857
1858 (defun spam-bogofilter-unregister-spam-routine (articles)
1859   (spam-bogofilter-register-spam-routine articles t))
1860
1861 (defun spam-bogofilter-register-ham-routine (articles &optional unregister)
1862   (spam-bogofilter-register-with-bogofilter articles nil unregister))
1863
1864 (defun spam-bogofilter-unregister-ham-routine (articles)
1865   (spam-bogofilter-register-ham-routine articles t))
1866
1867
1868 \f
1869 ;;;; spamoracle
1870 (defun spam-check-spamoracle ()
1871   "Run spamoracle on an article to determine whether it's spam."
1872   (let ((article-buffer-name (buffer-name))
1873         (spam-split-group (if spam-split-symbolic-return
1874                               'spam
1875                             spam-split-group)))
1876     (with-temp-buffer
1877       (let ((temp-buffer-name (buffer-name)))
1878         (save-excursion
1879           (set-buffer article-buffer-name)
1880           (let ((status
1881                  (apply 'call-process-region
1882                         (point-min) (point-max)
1883                         spam-spamoracle-binary
1884                         nil temp-buffer-name nil
1885                         (if spam-spamoracle-database
1886                             `("-f" ,spam-spamoracle-database "mark")
1887                           '("mark")))))
1888             (if (eq 0 status)
1889                 (progn
1890                   (set-buffer temp-buffer-name)
1891                   (goto-char (point-min))
1892                   (when (re-search-forward "^X-Spam: yes;" nil t)
1893                     spam-split-group))
1894               (error "Error running spamoracle" status))))))))
1895
1896 (defun spam-spamoracle-learn (articles article-is-spam-p &optional unregister)
1897   "Run spamoracle in training mode."
1898   (with-temp-buffer
1899     (let ((temp-buffer-name (buffer-name)))
1900       (save-excursion
1901         (goto-char (point-min))
1902         (dolist (article articles)
1903           (insert (spam-get-article-as-string article)))
1904         (let* ((arg (if (spam-xor unregister article-is-spam-p)
1905                         "-spam"
1906                       "-good"))
1907                (status
1908                 (apply 'call-process-region
1909                        (point-min) (point-max)
1910                        spam-spamoracle-binary
1911                        nil temp-buffer-name nil
1912                        (if spam-spamoracle-database
1913                            `("-f" ,spam-spamoracle-database
1914                              "add" ,arg)
1915                          `("add" ,arg)))))
1916           (when (not (eq 0 status))
1917             (error "Error running spamoracle" status)))))))
1918
1919 (defun spam-spamoracle-learn-ham (articles &optional unregister)
1920   (spam-spamoracle-learn articles nil unregister))
1921
1922 (defun spam-spamoracle-unlearn-ham (articles &optional unregister)
1923   (spam-spamoracle-learn-ham articles t))
1924
1925 (defun spam-spamoracle-learn-spam (articles &optional unregister)
1926   (spam-spamoracle-learn articles t unregister))
1927
1928 (defun spam-spamoracle-unlearn-spam (articles &optional unregister)
1929   (spam-spamoracle-learn-spam articles t))
1930
1931 \f
1932 ;;;; Hooks
1933
1934 ;;;###autoload
1935 (defun spam-initialize ()
1936   "Install the spam.el hooks and do other initialization"
1937   (interactive)
1938   (setq spam-install-hooks t)
1939   ;; TODO: How do we redo this every time spam-face is customized?
1940   (push '((eq mark gnus-spam-mark) . spam-face)
1941         gnus-summary-highlight)
1942   ;; Add hooks for loading and saving the spam stats
1943   (add-hook 'gnus-save-newsrc-hook 'spam-maybe-spam-stat-save)
1944   (add-hook 'gnus-get-top-new-news-hook 'spam-maybe-spam-stat-load)
1945   (add-hook 'gnus-startup-hook 'spam-maybe-spam-stat-load)
1946   (add-hook 'gnus-summary-prepare-exit-hook 'spam-summary-prepare-exit)
1947   (add-hook 'gnus-summary-prepare-hook 'spam-summary-prepare)
1948   (add-hook 'gnus-get-new-news-hook 'spam-setup-widening)
1949   (add-hook 'gnus-summary-prepared-hook 'spam-find-spam))
1950
1951 (defun spam-unload-hook ()
1952   "Uninstall the spam.el hooks"
1953   (interactive)
1954   (remove-hook 'gnus-save-newsrc-hook 'spam-maybe-spam-stat-save)
1955   (remove-hook 'gnus-get-top-new-news-hook 'spam-maybe-spam-stat-load)
1956   (remove-hook 'gnus-startup-hook 'spam-maybe-spam-stat-load)
1957   (remove-hook 'gnus-summary-prepare-exit-hook 'spam-summary-prepare-exit)
1958   (remove-hook 'gnus-summary-prepare-hook 'spam-summary-prepare)
1959   (remove-hook 'gnus-get-new-news-hook 'spam-setup-widening)
1960   (remove-hook 'gnus-summary-prepare-hook 'spam-find-spam))
1961
1962 (when spam-install-hooks
1963   (spam-initialize))
1964
1965 (provide 'spam)
1966
1967 ;;; spam.el ends here