Import Oort Gnus v0.14.
[elisp/gnus.git-] / lisp / gnus-agent.el
index bfcfa28..a40e48a 100644 (file)
@@ -234,12 +234,12 @@ node `(gnus)Server Buffer'.")
            (cadr gnus-command-method))))
 
 (defsubst gnus-agent-directory ()
-  "Path of the Gnus agent directory."
+  "The name of the Gnus agent directory."
   (nnheader-concat gnus-agent-directory
                   (nnheader-translate-file-chars (gnus-agent-method)) "/"))
 
 (defun gnus-agent-lib-file (file)
-  "The full path of the Gnus agent library FILE."
+  "The full name of the Gnus agent library FILE."
   (expand-file-name file
                    (file-name-as-directory
                     (expand-file-name "agent.lib" (gnus-agent-directory)))))
@@ -333,6 +333,7 @@ node `(gnus)Server Buffer'.")
 (gnus-define-keys gnus-agent-summary-mode-map
   "Jj" gnus-agent-toggle-plugged
   "Ju" gnus-agent-summary-fetch-group
+  "JS" gnus-agent-fetch-group
   "Js" gnus-agent-summary-fetch-series
   "J#" gnus-agent-mark-article
   "J\M-#" gnus-agent-unmark-article
@@ -532,12 +533,13 @@ be a select method."
     (error "Groups can't be fetched when Gnus is unplugged"))
   (gnus-group-iterate n 'gnus-agent-fetch-group))
 
-(defun gnus-agent-fetch-group (group)
+(defun gnus-agent-fetch-group (&optional group)
   "Put all new articles in GROUP into the Agent."
   (interactive (list (gnus-group-group-name)))
   (let ((state gnus-plugged))
     (unwind-protect
        (progn
+          (setq group (or group gnus-newsgroup-name))
          (unless group
            (error "No group on the current line"))
          (unless state
@@ -746,49 +748,73 @@ article's mark is toggled."
   (let ((gnus-command-method (gnus-find-method-for-group gnus-newsgroup-name)))
     (when (set (make-local-variable 'gnus-newsgroup-agentized) (gnus-agent-method-p gnus-command-method))
       (let* ((alist (gnus-agent-load-alist gnus-newsgroup-name))
-           (headers gnus-newsgroup-headers)
-           (undownloaded (list nil))
-           (tail undownloaded))
+             (headers gnus-newsgroup-headers)
+             (undownloaded (list nil))
+             (tail-undownloaded undownloaded)
+             (unfetched (list nil))
+             (tail-unfetched unfetched))
        (while (and alist headers)
          (let ((a (caar alist))
                (h (mail-header-number (car headers))))
            (cond ((< a h)
                   (pop alist)) ; ignore IDs in the alist that are not being displayed in the summary
                  ((> a h)
-                  (pop headers)) ; ignore headers that are not in the alist as these should be fictious (see nnagent-retrieve-headers).
+                   ;; headers that are not in the alist should be
+                   ;; fictious (see nnagent-retrieve-headers); they
+                   ;; imply that this article isn't in the agent.
+                  (gnus-agent-append-to-list tail-undownloaded h)
+                  (gnus-agent-append-to-list tail-unfetched    h)
+                   (pop headers)) 
                  ((cdar alist)
                   (pop alist)
                   (pop headers)
-                  nil; ignore already downloaded
+                  nil                  ; ignore already downloaded
                   )
                  (t
                   (pop alist)
                   (pop headers)
-                  (gnus-agent-append-to-list tail a)))))
-       (setq gnus-newsgroup-undownloaded (cdr undownloaded))))))
+                  (gnus-agent-append-to-list tail-undownloaded a)))))
+
+       (while headers
+          (let ((num (mail-header-number (pop headers))))
+            (gnus-agent-append-to-list tail-undownloaded num)
+            (gnus-agent-append-to-list tail-unfetched    num)))
+
+       (setq gnus-newsgroup-undownloaded (cdr undownloaded)
+              gnus-newsgroup-unfetched    (cdr unfetched))))))
 
 (defun gnus-agent-catchup ()
-  "Mark all undownloaded articles as read."
+  "Mark all articles as read that are neither cached, downloaded, nor downloadable."
   (interactive)
   (save-excursion
-    (while gnus-newsgroup-undownloaded
-      (gnus-summary-mark-article
-       (pop gnus-newsgroup-undownloaded) gnus-catchup-mark)))
-  (gnus-summary-position-point))
+    (let ((articles gnus-newsgroup-undownloaded))
+      (when (or gnus-newsgroup-downloadable
+                gnus-newsgroup-cached)
+        (setq articles (gnus-sorted-ndifference (gnus-sorted-ndifference (copy-sequence articles) gnus-newsgroup-downloadable) gnus-newsgroup-cached)))
+
+      (while articles
+        (gnus-summary-mark-article
+         (pop articles) gnus-catchup-mark)))
+    (gnus-summary-position-point)))
 
 (defun gnus-agent-summary-fetch-series ()
   (interactive)
-  (let ((dl gnus-newsgroup-downloadable))
-    (while gnus-newsgroup-processable
-      (let* ((art (car (last gnus-newsgroup-processable)))
-             (gnus-newsgroup-downloadable (list art)))
-        (gnus-summary-goto-subject art)
-        (sit-for 0)
-        (gnus-agent-summary-fetch-group)
-        (setq dl (delq art dl))
-        (gnus-summary-remove-process-mark art)
-        (sit-for 0)))
-    (setq gnus-newsgroup-downloadable dl)))
+  (when gnus-newsgroup-processable
+    (setq gnus-newsgroup-downloadable
+          (let* ((dl gnus-newsgroup-downloadable)
+                 (gnus-newsgroup-downloadable (sort (copy-sequence gnus-newsgroup-processable) '<))
+                 (fetched-articles (gnus-agent-summary-fetch-group)))
+            ;; The preceeding call to (gnus-agent-summary-fetch-group)
+            ;; updated gnus-newsgroup-downloadable to remove each
+            ;; article successfully fetched.
+
+            ;; For each article that I processed, remove its
+            ;; processable mark IF the article is no longer
+            ;; downloadable (i.e. it's already downloaded)
+            (dolist (article gnus-newsgroup-processable)
+              (unless (memq article gnus-newsgroup-downloadable)
+                (gnus-summary-remove-process-mark article)))
+            (gnus-sorted-ndifference dl fetched-articles)))))
 
 (defun gnus-agent-summary-fetch-group (&optional all)
   "Fetch the downloadable articles in the group.
@@ -798,7 +824,8 @@ Optional arg ALL, if non-nil, means to fetch all articles."
         (if all gnus-newsgroup-articles
           gnus-newsgroup-downloadable))
        (gnus-command-method (gnus-find-method-for-group gnus-newsgroup-name))
-       (state gnus-plugged))
+       (state gnus-plugged)
+        fetched-articles)
     (unwind-protect
        (progn
          (unless state
@@ -806,19 +833,22 @@ Optional arg ALL, if non-nil, means to fetch all articles."
          (unless articles
            (error "No articles to download"))
          (gnus-agent-with-fetch
-             (setq gnus-newsgroup-undownloaded
-                   (gnus-sorted-ndifference gnus-newsgroup-undownloaded
-                                            (gnus-agent-fetch-articles gnus-newsgroup-name articles))))
+            (setq gnus-newsgroup-undownloaded
+                  (gnus-sorted-ndifference gnus-newsgroup-undownloaded
+                                           (setq fetched-articles (gnus-agent-fetch-articles gnus-newsgroup-name articles)))))
          (save-excursion
-           (dolist (article articles)
+
+            (dolist (article articles)
              (setq gnus-newsgroup-downloadable
                    (delq article gnus-newsgroup-downloadable))
-             (if gnus-agent-mark-unread-after-downloaded
-                 (gnus-summary-mark-article article gnus-unread-mark))
-             (gnus-summary-update-download-mark article))))
+              (if gnus-agent-mark-unread-after-downloaded
+                  (gnus-summary-mark-article article gnus-unread-mark))
+              (when (gnus-summary-goto-subject article nil t)
+                (gnus-summary-update-download-mark article)))))
       (when (and (not state)
                 gnus-plugged)
-       (gnus-agent-toggle-plugged nil)))))
+       (gnus-agent-toggle-plugged nil)))
+    fetched-articles))
 
 (defun gnus-agent-fetch-selected-article ()
   "Fetch the current article as it is selected.
@@ -829,8 +859,11 @@ This can be added to `gnus-select-article-hook' or
       (when (gnus-agent-fetch-articles
              gnus-newsgroup-name
             (list gnus-current-article))
-       (setq gnus-newsgroup-undownloaded (delq gnus-current-article gnus-newsgroup-undownloaded))
-       (gnus-summary-update-article gnus-current-article)))))
+       (setq gnus-newsgroup-undownloaded
+             (delq gnus-current-article gnus-newsgroup-undownloaded))
+       (gnus-summary-update-article-line
+        gnus-current-article
+        (gnus-summary-article-header gnus-current-article))))))
 
 ;;;
 ;;; Internal functions
@@ -907,7 +940,7 @@ This can be added to `gnus-select-article-hook' or
          (delete-char 1))))))
 
 (defun gnus-agent-group-path (group)
-  "Translate GROUP into a path."
+  "Translate GROUP into a file name."
   (if nnmail-use-long-file-names
       (gnus-group-real-name group)
     (nnheader-translate-file-chars
@@ -958,84 +991,120 @@ This can be added to `gnus-select-article-hook' or
 
 (defun gnus-agent-fetch-articles (group articles)
   "Fetch ARTICLES from GROUP and put them into the Agent."
-  (gnus-agent-load-alist group)
   (when articles
-    ;; Prune off articles that we have already fetched.
-    (while (and articles
-               (cdr (assq (car articles) gnus-agent-article-alist)))
-      (pop articles))
-    (let ((arts articles))
-      (while (cdr arts)
-       (if (cdr (assq (cadr arts) gnus-agent-article-alist))
-           (setcdr arts (cddr arts))
-         (setq arts (cdr arts)))))
-    (when articles
-      (let* ((fetched-articles (list nil))
-            (tail-fetched-articles fetched-articles)
-            (dir (concat
-                 (gnus-agent-directory)
-                 (gnus-agent-group-path group) "/"))
-           (date (time-to-days (current-time)))
-           (case-fold-search t)
-            pos crosses id)
-       (gnus-make-directory dir)
-       (gnus-message 7 "Fetching articles for %s..." group)
-       ;; Fetch the articles from the backend.
-       (if (gnus-check-backend-function 'retrieve-articles group)
-           (setq pos (gnus-retrieve-articles articles group))
-         (with-temp-buffer
-           (let (article)
-             (while (setq article (pop articles))
-               (gnus-message 10 "Fetching article %s for %s..."
-                             article group)
-               (when (or
-                      (gnus-backlog-request-article group article
-                                                    nntp-server-buffer)
-                      (gnus-request-article article group))
-                 (goto-char (point-max))
-                 (push (cons article (point)) pos)
-                 (insert-buffer-substring nntp-server-buffer)))
-             (copy-to-buffer nntp-server-buffer (point-min) (point-max))
-             (setq pos (nreverse pos)))))
-       ;; Then save these articles into the Agent.
-       (save-excursion
-         (set-buffer nntp-server-buffer)
-         (while pos
-           (narrow-to-region (cdar pos) (or (cdadr pos) (point-max)))
-           (goto-char (point-min))
-           (unless (eobp)  ;; Don't save empty articles.
-             (when (search-forward "\n\n" nil t)
-               (when (search-backward "\nXrefs: " nil t)
-                 ;; Handle cross posting.
-                  (goto-char (match-end 0)) ; move to end of header name
-                 (skip-chars-forward "^ ") ; skip server name
-                 (skip-chars-forward " ")
-                 (setq crosses nil)
-                 (while (looking-at "\\([^: \n]+\\):\\([0-9]+\\) *")
-                   (push (cons (buffer-substring (match-beginning 1)
-                                                 (match-end 1))
-                               (string-to-int (buffer-substring (match-beginning 2)
-                                                                 (match-end 2))))
-                         crosses)
-                   (goto-char (match-end 0)))
-                 (gnus-agent-crosspost crosses (caar pos) date)))
-             (goto-char (point-min))
-             (if (not (re-search-forward
-                       "^Message-ID: *<\\([^>\n]+\\)>" nil t))
-                 (setq id "No-Message-ID-in-article")
-               (setq id (buffer-substring (match-beginning 1) (match-end 1))))
-             (let ((coding-system-for-write
-                    gnus-agent-file-coding-system))
-               (write-region (point-min) (point-max)
-                             (concat dir (number-to-string (caar pos)))
-                             nil 'silent))
-
-             (gnus-agent-append-to-list tail-fetched-articles (caar pos)))
-           (widen)
-           (pop pos)))
-
-       (gnus-agent-save-alist group (cdr fetched-articles) date)
-       (cdr fetched-articles)))))
+    (gnus-agent-load-alist group)
+    (let* ((alist   gnus-agent-article-alist)
+           (headers (if (< (length articles) 2) nil gnus-newsgroup-headers))
+           (selected-sets (list nil))
+           (current-set-size 0)
+           article
+           header-number)
+      ;; Check each article
+      (while (setq article (pop articles))
+        ;; Skip alist entries preceeding this article
+        (while (> article (or (caar alist) (1+ article)))
+          (setq alist (cdr alist)))
+
+        ;; Prune off articles that we have already fetched.
+        (unless (and (eq article (caar alist))
+                     (cdar alist))
+          ;; Skip headers preceeding this article
+          (while (> article 
+                    (setq header-number
+                          (let* ((header (car headers)))
+                            (if header
+                                (mail-header-number header)
+                              (1+ article)))))
+            (setq headers (cdr headers)))
+
+          ;; Add this article to the current set
+          (setcar selected-sets (cons article (car selected-sets)))
+
+          ;; Update the set size, when the set is too large start a
+          ;; new one.  I do this after adding the article as I want at
+          ;; least one article in each set.
+          (when (< gnus-agent-max-fetch-size
+                   (setq current-set-size (+ current-set-size (if (= header-number article)
+                                                                  (mail-header-chars (car headers))
+                                                                0))))
+            (setcar selected-sets (nreverse (car selected-sets)))
+            (setq selected-sets (cons nil selected-sets)
+                  current-set-size 0))))
+
+      (when (or (cdr selected-sets) (car selected-sets))
+        (let* ((fetched-articles (list nil))
+               (tail-fetched-articles fetched-articles)
+               (dir (concat
+                     (gnus-agent-directory)
+                     (gnus-agent-group-path group) "/"))
+               (date (time-to-days (current-time)))
+               (case-fold-search t)
+               pos crosses id)
+
+          (setcar selected-sets (nreverse (car selected-sets)))
+          (setq selected-sets (nreverse selected-sets))
+
+          (gnus-make-directory dir)
+          (gnus-message 7 "Fetching articles for %s..." group)
+          
+          (unwind-protect
+              (while (setq articles (pop selected-sets))
+                ;; Fetch the articles from the backend.
+                (if (gnus-check-backend-function 'retrieve-articles group)
+                    (setq pos (gnus-retrieve-articles articles group))
+                  (with-temp-buffer
+                    (let (article)
+                      (while (setq article (pop articles))
+                        (gnus-message 10 "Fetching article %s for %s..."
+                                      article group)
+                        (when (or
+                               (gnus-backlog-request-article group article
+                                                             nntp-server-buffer)
+                               (gnus-request-article article group))
+                          (goto-char (point-max))
+                          (push (cons article (point)) pos)
+                          (insert-buffer-substring nntp-server-buffer)))
+                      (copy-to-buffer nntp-server-buffer (point-min) (point-max))
+                      (setq pos (nreverse pos)))))
+                ;; Then save these articles into the Agent.
+                (save-excursion
+                  (set-buffer nntp-server-buffer)
+                  (while pos
+                    (narrow-to-region (cdar pos) (or (cdadr pos) (point-max)))
+                    (goto-char (point-min))
+                    (unless (eobp) ;; Don't save empty articles.
+                      (when (search-forward "\n\n" nil t)
+                        (when (search-backward "\nXrefs: " nil t)
+                          ;; Handle cross posting.
+                          (goto-char (match-end 0)) ; move to end of header name
+                          (skip-chars-forward "^ ") ; skip server name
+                          (skip-chars-forward " ")
+                          (setq crosses nil)
+                          (while (looking-at "\\([^: \n]+\\):\\([0-9]+\\) *")
+                            (push (cons (buffer-substring (match-beginning 1)
+                                                          (match-end 1))
+                                        (string-to-int (buffer-substring (match-beginning 2)
+                                                                         (match-end 2))))
+                                  crosses)
+                            (goto-char (match-end 0)))
+                          (gnus-agent-crosspost crosses (caar pos) date)))
+                      (goto-char (point-min))
+                      (if (not (re-search-forward
+                                "^Message-ID: *<\\([^>\n]+\\)>" nil t))
+                          (setq id "No-Message-ID-in-article")
+                        (setq id (buffer-substring (match-beginning 1) (match-end 1))))
+                      (let ((coding-system-for-write
+                             gnus-agent-file-coding-system))
+                        (write-region (point-min) (point-max)
+                                      (concat dir (number-to-string (caar pos)))
+                                      nil 'silent))
+
+                      (gnus-agent-append-to-list tail-fetched-articles (caar pos)))
+                    (widen)
+                    (pop pos))))
+
+            (gnus-agent-save-alist group (cdr fetched-articles) date))
+          (cdr fetched-articles))))))
 
 (defun gnus-agent-crosspost (crosses article &optional date)
   (setq date (or date t))
@@ -1067,11 +1136,24 @@ This can be added to `gnus-select-article-hook' or
         (gnus-agent-check-overview-buffer))
       (pop crosses))))
 
+(defun gnus-agent-backup-overview-buffer ()
+  (when gnus-newsgroup-name
+    (let ((root (gnus-agent-article-name ".overview" gnus-newsgroup-name))
+          (cnt 0)
+          name)
+      (while (file-exists-p (setq name (concat root "~" (int-to-string (setq cnt (1+ cnt))) "~"))))
+      (write-region (point-min) (point-max) name nil 'no-msg)
+      (gnus-message 1 "Created backup copy of overview in %s." name)
+      )
+    )
+  t)
+
 (defun gnus-agent-check-overview-buffer (&optional buffer)
   "Check the overview file given for sanity.
 In particular, checks that the file is sorted by article number
 and that there are no duplicates."
-  (let ((prev-num -1))
+  (let ((prev-num -1)
+        (backed-up nil))
     (save-excursion
       (when buffer
        (set-buffer buffer))
@@ -1087,22 +1169,25 @@ and that there are no duplicates."
            (cond
             ((or (not (integerp cur))
                  (not (eq (char-after) ?\t)))
+              (or backed-up
+                  (setq backed-up (gnus-agent-backup-overview-buffer)))
              (gnus-message 1
                            "Overview buffer contains garbage '%s'."
                            (buffer-substring
                             p (gnus-point-at-eol))))
             ((= cur prev-num)
-             (gnus-message 1
+             (or backed-up
+                  (setq backed-up (gnus-agent-backup-overview-buffer)))
+              (gnus-message 1
                            "Duplicate overview line for %d" cur)
              (delete-region (point) (progn (forward-line 1) (point))))
-            ((< cur 0)
-             (gnus-message 1 "Junk article number %d" cur)
-             (delete-region (point) (progn (forward-line 1) (point))))
             ((< cur prev-num)
+             (or backed-up
+                  (setq backed-up (gnus-agent-backup-overview-buffer)))
+              (gnus-message 1 "Overview buffer not sorted!")
              (sort-numeric-fields 1 (point-min) (point-max))
              (goto-char (point-min))
-             (setq prev-num -1)
-             (gnus-message 1 "Overview buffer not sorted!"))
+             (setq prev-num -1))
             (t
              (setq prev-num cur)))
            (forward-line 1)))))))
@@ -1151,7 +1236,7 @@ article numbers will be returned."
       ;; interesting marks.  (We have to fetch articles with boring marks
       ;; because otherwise the agent will remove their marks.)
       (dolist (arts (gnus-info-marks (gnus-get-info group)))
-        (unless (memq (car arts) '(seen recent))
+        (unless (memq (car arts) '(seen recent killed cache))
           (setq articles (gnus-range-add articles (cdr arts)))))
       (setq articles (sort (gnus-uncompress-sequence articles) '<)))
 
@@ -1181,6 +1266,9 @@ article numbers will be returned."
             ;; that no headers need to be fetched. -- Kevin
             (setq articles (gnus-list-range-intersection
                             articles (list (cons low high)))))))
+
+      (gnus-message 10 "gnus-agent-fetch-headers: undownloaded articles are '%s'" (gnus-compress-sequence articles t))
+
       (save-excursion
         (set-buffer nntp-server-buffer)
 
@@ -1449,31 +1537,42 @@ of FILE placing the combined headers in nntp-server-buffer."
   "Fetch GROUP."
   (let ((gnus-command-method method)
        (gnus-newsgroup-name group)
-       gnus-newsgroup-dependencies gnus-newsgroup-headers
-       gnus-newsgroup-scored gnus-headers gnus-score
-       gnus-use-cache articles arts
-       category predicate info marks score-param
+       (gnus-newsgroup-dependencies gnus-newsgroup-dependencies)
+        (gnus-newsgroup-headers gnus-newsgroup-headers)
+       (gnus-newsgroup-scored gnus-newsgroup-scored)
+       (gnus-use-cache gnus-use-cache)
        (gnus-summary-expunge-below gnus-summary-expunge-below)
        (gnus-summary-mark-below gnus-summary-mark-below)
        (gnus-orphan-score gnus-orphan-score)
        ;; Maybe some other gnus-summary local variables should also
        ;; be put here.
+
+        gnus-headers
+        gnus-score
+        articles arts
+       category predicate info marks score-param
        )
     (unless (gnus-check-group group)
       (error "Can't open server for %s" group))
 
     ;; Fetch headers.
-    (when (or (gnus-active group)
+    (when (or gnus-newsgroup-active
+              (gnus-active group)
               (gnus-activate-group group))
-      (let ((marked-articles nil))
+      (let ((marked-articles gnus-newsgroup-downloadable))
         ;; Identify the articles marked for download
-        (dolist (mark gnus-agent-download-marks)
-          (let ((arts (cdr (assq mark (gnus-info-marks
-                                       (setq info (gnus-get-info group)))))))
-            (when arts
-              (setq marked-articles (nconc (gnus-uncompress-range arts)
-                                           marked-articles))
-              )))
+        (unless gnus-newsgroup-active ;; This needs to be a
+          ;; gnus-summary local variable
+          ;; that is NOT bound to any
+          ;; value above (It's global
+          ;; value should default to nil).
+          (dolist (mark gnus-agent-download-marks)
+            (let ((arts (cdr (assq mark (gnus-info-marks
+                                         (setq info (gnus-get-info group)))))))
+              (when arts
+                (setq marked-articles (nconc (gnus-uncompress-range arts)
+                                             marked-articles))
+                ))))
         (setq marked-articles (sort marked-articles '<))
 
         ;; Fetch any new articles from the server
@@ -1485,11 +1584,12 @@ of FILE placing the combined headers in nntp-server-buffer."
         (when articles
           ;; Parse them and see which articles we want to fetch.
           (setq gnus-newsgroup-dependencies
-                (make-vector (length articles) 0))
-
+                (or gnus-newsgroup-dependencies
+                    (make-vector (length articles) 0)))
           (setq gnus-newsgroup-headers
-                (gnus-get-newsgroup-headers-xover articles nil nil
-                                                  group))
+                (or gnus-newsgroup-headers
+                    (gnus-get-newsgroup-headers-xover articles nil nil
+                                                      group)))
           ;; `gnus-agent-overview-buffer' may be killed for
           ;; timeout reason.  If so, recreate it.
           (gnus-agent-create-buffer)
@@ -1520,66 +1620,81 @@ of FILE placing the combined headers in nntp-server-buffer."
 
           (unless (and (eq predicate 'gnus-agent-false)
                        (not marked-articles))
-            (let* ((arts (list nil))
-                   (arts-tail arts)
-                   (chunk-size 0)
-                   (marked-articles marked-articles)
-                   is-marked)
-              (while (setq gnus-headers (pop gnus-newsgroup-headers))
-                (let ((num (mail-header-number gnus-headers)))
-                  ;; Determine if this article was marked for download.
-                  (while (and marked-articles
-                              (cond ((< num (car marked-articles))
-                                     nil)
-                                    ((= num (car marked-articles))
-                                     (setq is-marked t)
-                                     nil)
-                                    (t
-                                     (setq marked-articles
-                                           (cdr marked-articles))))))
-
-                  ;; When this article is marked, or selected by the
-                  ;; predicate, add it to the download list
-                  (when (or is-marked
-                            (let ((gnus-score
-                                   (or (cdr (assq num gnus-newsgroup-scored))
-                                       gnus-summary-default-score)))
-                              (funcall predicate)))
-                    (gnus-agent-append-to-list arts-tail num)
-
-                    ;; When the expected size of the fetched articles
-                    ;; exceeds gnus-agent-max-fetch-size, perform the
-                    ;; fetch.
-                    (when (< gnus-agent-max-fetch-size
-                             (setq chunk-size
-                                   (+ chunk-size
-                                      (mail-header-chars gnus-headers))))
-                      (gnus-agent-fetch-articles group (cdr arts))
-                      (setcdr arts nil)
-                      (setq arts-tail arts)
-                      (setq chunk-size 0)))))
-
-              ;; Fetch all remaining articles
-              (when (cdr arts)
-                (gnus-agent-fetch-articles group (cdr arts)))))
-
-          ;; When some, or all, of the marked articles came
-          ;; from the download mark.  Remove that mark.  I
-          ;; didn't do this earlier as I only want to remove
-          ;; the marks after the fetch is completed.
-
-          (when marked-articles
-            (dolist (mark gnus-agent-download-marks)
-              (when (eq mark 'download)
-                (setq arts (assq mark (gnus-info-marks
-                                       (setq info (gnus-get-info group)))))
-                (when (cdr arts)
-                  (setq marks (delq arts (gnus-info-marks info)))
-                  (gnus-info-set-marks info marks)
-                  (gnus-dribble-enter
-                   (concat "(gnus-group-set-info '"
-                           (gnus-prin1-to-string info)
-                           ")")))))))))))
+            (let ((arts (list nil)))
+              (let ((arts-tail arts)
+                    (alist (gnus-agent-load-alist group))
+                    (marked-articles marked-articles)
+                    (gnus-newsgroup-headers gnus-newsgroup-headers))
+                (while (setq gnus-headers (pop gnus-newsgroup-headers))
+                  (let ((num (mail-header-number gnus-headers)))
+                    ;; Determine if this article is already in the cache
+                    (while (and alist
+                                (> num (caar alist)))
+                      (setq alist (cdr alist)))
+
+                    (unless (and (eq num (caar alist))
+                                 (cdar alist))
+
+                      ;; Determine if this article was marked for download.
+                      (while (and marked-articles
+                                  (> num (car marked-articles)))
+                        (setq marked-articles
+                              (cdr marked-articles)))
+
+                      ;; When this article is marked, or selected by the
+                      ;; predicate, add it to the download list
+                      (when (or (eq num (car marked-articles))
+                                (let ((gnus-score
+                                       (or (cdr (assq num gnus-newsgroup-scored))
+                                           gnus-summary-default-score)))
+                                  (funcall predicate)))
+                        (gnus-agent-append-to-list arts-tail num))))))
+
+              (let (fetched-articles)
+                ;; Fetch all selected articles
+                (setq gnus-newsgroup-undownloaded
+                      (gnus-sorted-ndifference gnus-newsgroup-undownloaded
+                                               (setq fetched-articles (if (cdr arts) (gnus-agent-fetch-articles group (cdr arts)) nil))))
+
+                (let ((unfetched-articles (gnus-sorted-ndifference (cdr arts) fetched-articles)))
+                  (if gnus-newsgroup-active
+                      ;; Update the summary buffer
+                      (progn
+                        (dolist (article marked-articles)
+                          (when (gnus-summary-goto-subject article nil t)
+                            (gnus-summary-set-agent-mark article t)))
+                        (dolist (article fetched-articles)
+                          (if gnus-agent-mark-unread-after-downloaded
+                              (gnus-summary-mark-article article gnus-unread-mark))
+                          (when (gnus-summary-goto-subject article nil t)
+                            (gnus-summary-update-download-mark article)))
+                        (dolist (article unfetched-articles)
+                          (gnus-summary-mark-article article gnus-canceled-mark)))
+
+                    ;; Update the group buffer.
+
+                    ;; When some, or all, of the marked articles came
+                    ;; from the download mark.  Remove that mark.  I
+                    ;; didn't do this earlier as I only want to remove
+                    ;; the marks after the fetch is completed.
+
+                    (dolist (mark gnus-agent-download-marks)
+                      (when (eq mark 'download)
+                        (let ((marked-arts (assq mark (gnus-info-marks
+                                                       (setq info (gnus-get-info group))))))
+                          (when (cdr marked-arts)
+                            (setq marks (delq marked-arts (gnus-info-marks info)))
+                            (gnus-info-set-marks info marks)))))
+                    (let ((read (gnus-info-read (or info (setq info (gnus-get-info group))))))
+                      (gnus-info-set-read info (gnus-add-to-range read unfetched-articles)))
+
+                    (gnus-group-update-group group t)
+                    (sit-for 0)
+
+                    (gnus-dribble-enter
+                     (concat "(gnus-group-set-info '"
+                             (gnus-prin1-to-string info)
+                             ")"))))))))))))
 
 ;;;
 ;;; Agent Category Mode
@@ -1978,7 +2093,20 @@ FORCE is equivalent to setting gnus-agent-expire-days to zero(0)."
                                     (gnus-agent-directory)
                                     (gnus-agent-group-path expiring-group) "/"))
                               (active
-                               (gnus-gethash-safe expiring-group orig)))
+                               (gnus-gethash-safe expiring-group orig))
+                               (day (if (numberp day)
+                                        day
+                                      (let (found
+                                            (days gnus-agent-expire-days))
+                                        (catch 'found
+                                          (while (and (not found)
+                                                      days)
+                                            (when (eq 0 (string-match (caar days) expiring-group))
+                                              (throw 'found (- (time-to-days (current-time)) (cadar days))))
+                                            (pop days))
+                                          ;; No regexp matched so set a limit that will block expiration in this group
+                                          0)))))
+                                        
                          (when active
                            (gnus-agent-load-alist expiring-group)
                            (gnus-message 5 "Expiring articles in %s" expiring-group)
@@ -2099,15 +2227,15 @@ FORCE is equivalent to setting gnus-agent-expire-days to zero(0)."
                                     (alist (list nil))
                                     (tail-alist alist))
                                (while dlist
-                                 (let ((new-completed (* 100.0 (/ (setq cnt (1+ cnt)) len))))
+                                 (let ((new-completed (truncate (* 100.0 (/ (setq cnt (1+ cnt)) len)))))
                                    (when (> new-completed completed)
                                      (setq completed new-completed)
                                      (gnus-message 9 "%3d%% completed..."  completed)))
-                                 (let* ((entry (car dlist))
+                                 (let* ((entry          (car dlist))
                                         (article-number (nth 0 entry))
-                                        (fetch-date (nth 1 entry))
-                                        (keep (nth 2 entry))
-                                        (marker (nth 3 entry)))
+                                        (fetch-date     (nth 1 entry))
+                                        (keep           (nth 2 entry))
+                                        (marker         (nth 3 entry)))
 
                                    (cond
                                     ;; Kept articles are unread, marked, or special.
@@ -2132,17 +2260,7 @@ FORCE is equivalent to setting gnus-agent-expire-days to zero(0)."
 
                                             ;; We now have the arrival day, so we see
                                             ;; whether it's old enough to be expired.
-                                            ((< fetch-date
-                                                (if (numberp day)
-                                                    day
-                                                  (let (found
-                                                        (days gnus-agent-expire-days))
-                                                    (while (and (not found)
-                                                                days)
-                                                      (when (eq 0 (string-match (caar days) expiring-group))
-                                                        (setq found (cadar days)))
-                                                      (pop days))
-                                                    found)))
+                                            ((< fetch-date day)
                                              'expired)
                                             (force
                                              'forced)))
@@ -2158,7 +2276,7 @@ FORCE is equivalent to setting gnus-agent-expire-days to zero(0)."
                                          )
 
                                        (when marker
-                                         (push "NOV entry removed" article)
+                                         (push "NOV entry removed" actions)
                                          (goto-char marker)
                                          (gnus-delete-line))
 
@@ -2170,6 +2288,8 @@ FORCE is equivalent to setting gnus-agent-expire-days to zero(0)."
                                          (push (format "Removed %s article number from article alist" type) actions))
 
                                        (gnus-message 7 "gnus-agent-expire: Article %d: %s" article-number (mapconcat 'identity actions ", "))))
+                                     (t
+                                      (gnus-agent-append-to-list tail-alist (cons article-number fetch-date)))
                                     )
 
                                    ;; Clean up markers as I want to recycle this buffer over several groups.
@@ -2309,9 +2429,10 @@ FORCE is equivalent to setting gnus-agent-expire-days to zero(0)."
                        
                        ;; Get the list of articles that were fetched
                        (goto-char (point-min))
-                       (ignore-errors 
-                         (while t
-                          (gnus-agent-append-to-list tail-fetched-articles (read (current-buffer)))
+                       (let ((pm (point-max)))
+                         (while (< (point) pm)
+                           (when (looking-at "[0-9]+\t")
+                             (gnus-agent-append-to-list tail-fetched-articles (read (current-buffer))))
                            (forward-line 1)))
                        
                        ;; Clip this list to the headers that will actually be returned
@@ -2390,6 +2511,15 @@ FORCE is equivalent to setting gnus-agent-expire-days to zero(0)."
 
 (defun gnus-agent-regenerate-group (group &optional reread)
   "Regenerate GROUP.  If REREAD is t, all articles in the .overview are marked as unread.  If REREAD is not nil, downloaded articles are marked as unread."
+  (interactive (list (let ((def (or (gnus-group-group-name)
+                                    gnus-newsgroup-name)))
+                       (let ((select (read-string (if def (concat "Group Name (" def "): ")
+                                          "Group Name: "))))
+                         (if (and (equal "" select)
+                                  def)
+                             def
+                           select)))
+                     (intern-soft (read-string "Reread (nil)? (t=>all, nil=>none, some=>all downloaded): "))))
   (gnus-message 5 "Regenerating in %s" group)
   (let* ((gnus-command-method (or gnus-command-method
                                   (gnus-find-method-for-group group)))