Function: ert-summarize-tests-batch-and-exit

ert-summarize-tests-batch-and-exit is a byte-compiled function defined in ert.el.gz.

Signature

(ert-summarize-tests-batch-and-exit &optional HIGH)

Documentation

Summarize the results of testing.

Expects to be called in batch mode, with logfiles as command-line arguments. The logfiles should have the ert-run-tests-batch format. When finished, this exits Emacs, with status as per ert-run-tests-batch-and-exit.

If HIGH is a natural number, the HIGH long lasting tests are summarized.

Probably introduced at or before Emacs version 25.1.

Source Code

;; Defined in /usr/src/emacs/lisp/emacs-lisp/ert.el.gz
(defun ert-summarize-tests-batch-and-exit (&optional high)
  "Summarize the results of testing.
Expects to be called in batch mode, with logfiles as command-line arguments.
The logfiles should have the `ert-run-tests-batch' format.  When finished,
this exits Emacs, with status as per `ert-run-tests-batch-and-exit'.

If HIGH is a natural number, the HIGH long lasting tests are summarized."
  (or noninteractive
      (user-error "This function is only for use in batch mode"))
  (or (natnump high) (setq high 0))
  ;; Better crash loudly than attempting to recover from undefined
  ;; behavior.
  (setq attempt-stack-overflow-recovery nil
        attempt-orderly-shutdown-on-fatal-signal nil)
  (when (getenv "EMACS_TEST_JUNIT_REPORT")
    (apply #'ert-write-junit-test-summary-report command-line-args-left))
  (let ((nlogs (length command-line-args-left))
        (ntests 0) (nrun 0) (nexpected 0) (nunexpected 0) (nskipped 0)
        nnotrun logfile notests badtests unexpected skipped tests)
    (with-temp-buffer
      (while (setq logfile (pop command-line-args-left))
        (erase-buffer)
        (when (file-readable-p logfile) (insert-file-contents logfile))
        (if (not (re-search-forward "^Running \\([0-9]+\\) tests" nil t))
            (push logfile notests)
          (setq ntests (+ ntests (string-to-number (match-string 1))))
          (if (not (re-search-forward "^\\(Aborted: \\)?\
Ran \\([0-9]+\\) tests, \\([0-9]+\\) results as expected\
\\(?:, \\([0-9]+\\) unexpected\\)?\
\\(?:, \\([0-9]+\\) skipped\\)?" nil t))
              (push logfile badtests)
            (if (match-string 1) (push logfile badtests))
            (setq nrun (+ nrun (string-to-number (match-string 2)))
                  nexpected (+ nexpected (string-to-number (match-string 3))))
            (when (match-string 4)
	      (let ((n (string-to-number (match-string 4))))
		(unless (zerop n)
		  (push logfile unexpected)
		  (setq nunexpected (+ nunexpected n)))))
            (when (match-string 5)
              (push logfile skipped)
              (setq nskipped (+ nskipped
                                (string-to-number (match-string 5)))))
            (unless (zerop high)
              (goto-char (point-min))
              (while (< (point) (point-max))
                (if (looking-at "^\\s-+\\w+\\s-+[[:digit:]]+/[[:digit:]]+\\s-+\\S-+\\s-+(\\([.[:digit:]]+\\)\\s-+sec)$")
                    (push (cons (string-to-number (match-string 1))
                                (match-string 0))
                          tests))
                (forward-line)))))))
    (setq nnotrun (- ntests nrun))
    (message "\nSUMMARY OF TEST RESULTS")
    (message "-----------------------")
    (message "Files examined: %d" nlogs)
    (message "Ran %d tests%s, %d results as expected, %d unexpected, %d skipped"
             nrun
             (if (zerop nnotrun) "" (format ", %d failed to run" nnotrun))
             nexpected nunexpected nskipped)
    (when notests
      (message "%d files did not contain any tests:" (length notests))
      (mapc (lambda (l) (message "  %s" l)) notests))
    (when badtests
      (message "%d files did not finish:" (length badtests))
      (mapc (lambda (l) (message "  %s" l)) badtests)
      (if (or (getenv "EMACS_HYDRA_CI") (getenv "EMACS_EMBA_CI"))
          (with-temp-buffer
            (dolist (f badtests)
              (erase-buffer)
              (insert-file-contents f)
              (message "Contents of unfinished file %s:" f)
              (message "-----\n%s\n-----" (buffer-string))))))
    (when unexpected
      (message "%d files contained unexpected results:" (length unexpected))
      (mapc (lambda (l) (message "  %s" l)) unexpected))
    (unless (or (null tests) (zerop high))
      (message "\nLONG-RUNNING TESTS")
      (message "------------------")
      (setq tests (ntake high (sort tests (lambda (x y) (> (car x) (car y))))))
      (message "%s" (mapconcat #'cdr tests "\n")))
    ;; More details on hydra and emba, where the logs are harder to get to.
    (when (and (or (getenv "EMACS_HYDRA_CI") (getenv "EMACS_EMBA_CI"))
               (not (zerop (+ nunexpected nskipped))))
      (message "\nDETAILS")
      (message "-------")
      (with-temp-buffer
        (dolist (x (list (list skipped "skipped" "SKIPPED")
                         (list unexpected "unexpected"
                               "\\(?:FAILED\\|PASSED\\)")))
          (mapc (lambda (l)
                  (erase-buffer)
                  (insert-file-contents l)
                  (message "%s:" l)
                  (when (re-search-forward (format "^[ \t]*[0-9]+ %s results:"
                                                   (nth 1 x))
                                           nil t)
                    (while (and (zerop (forward-line 1))
                                (looking-at (format "^[ \t]*%s" (nth 2 x))))
                      (message "%s" (buffer-substring (line-beginning-position)
                                                      (line-end-position))))))
                (car x)))))
    (kill-emacs (cond ((or notests badtests (not (zerop nnotrun))) 2)
                      (unexpected 1)
                      (t 0)))))