Expanded testing of MH-E with multiple MH variants

* test/lisp/mh-e/mh-utils-tests.el: Environment variable TEST_MH_PATH
controls which installed MH variant to test with.  Moved the commentary
about testing with different MH variants from above 'with-mh-test-env'
definition to "Commentary" section at the top of the file.
* test/lisp/mh-e/test-all-mh-variants.sh: New script to test all
installed MH variants.

test/lisp/mh-e/mh-utils-tests.el

@@ -17,6 +17,34 @@
 ;; You should have received a copy of the GNU General Public License
 ;; along with GNU Emacs.  If not, see <https://www.gnu.org/licenses/>.
 
+;;; Commentary:
+
+;; This test suite runs tests that use and depend on MH programs
+;; installed on the system.
+
+;; When running such tests, MH-E can use a particular MH variant
+;; installed on the system, or it can use the mocks provided here.
+;; (Setup is done by the `with-mh-test-env' macro.)
+
+;; By setting environment variable TEST_MH_PATH, you can select which of
+;; the installed MH variants to use, or ignore them all and use mocks.
+;; See also the script test-all-mh-variants.sh in this directory.
+
+;; 1.  To run these tests against the default MH variant installed on
+;;     this system:
+;; cd ../.. && make lisp/mh-e/mh-utils-tests
+
+;; 2.  To run these tests against an MH variant installed in a
+;;     specific directory, set TEST_MH_PATH, as in this example:
+;; cd ../.. && make lisp/mh-e/mh-utils-tests TEST_MH_PATH=/usr/local/nmh/bin
+
+;; 3.  To search for and run these tests against all MH variants
+;;     installed on this system:
+;; ./test-all-mh-variants.sh
+
+;; Setting the environment variable TEST_MH_DEBUG or the Lisp variable
+;; mh-test-utils-debug-mocks logs access to the file system during the test.
+
 ;;; Code:
 
 (require 'ert)
@@ -56,34 +84,32 @@
 ;; Folder names that are used by the following tests.
 (defvar mh-test-rel-folder "rela-folder")
 (defvar mh-test-abs-folder "/abso-folder")
-(defvar mh-test-no-such-folder "/testdir/none"
+(defvar mh-test-no-such-folder "/testdir/none" "A folder that does not exist.")
-  "Name of a folder that the user does not have.")
+
+(defvar mh-test-utils-variants nil
+  "The value of `mh-variants' used for these tests.
+This variable allows setting `mh-variants' to a limited set for targeted
+testing.  Its value can be different from the normal value when
+environment variable TEST_MH_PATH is set.  By remembering the value, we
+can log the choice only once, which makes the batch log easier to read.")
 
 (defvar mh-test-variant-logged-already nil
   "Whether `with-mh-test-env' has written the MH variant to the log.")
-(setq mh-test-variant-logged-already nil) ;reset if buffer is re-evaluated
 
-(defvar mh-test-utils-debug-mocks nil
+(defvar mh-test-utils-debug-mocks (> (length (getenv "TEST_MH_DEBUG")) 0)
   "Whether to log detailed behavior of mock functions.")
 
 (defvar mh-test-call-process-real (symbol-function 'call-process))
 (defvar mh-test-file-directory-p-real (symbol-function 'file-directory-p))
 
-
+;;; The macro with-mh-test-env wraps tests that touch the file system
-;;; This macro wraps tests that touch the file system and/or run programs.
+;;; and/or run programs.
-;;; When running such tests, MH-E can use a particular MH variant
-;;; installed on the system, or it can use the mocks provided below.
-
-;;; By setting PATH and mh-sys-path, you can select which of the
-;;; installed MH variants to use or ignore them all and use mocks.
 
 (defmacro with-mh-test-env (&rest body)
   "Evaluate BODY with a test mail environment.
 Functions that touch the file system or run MH programs are either
-mocked out or pointed at a test tree.  When called from Emacs's batch
+mocked out or pointed at a test tree.  Uses `mh-test-utils-setup' to
-testing infrastructure, this will use mocks and thus run on systems
+select which."
-that do not have any MH variant installed.  MH-E developers can
-install an MH variant and test it interactively."
   (declare (indent defun))
   `(cl-letf ((temp-home-dir nil)
              ;; make local bindings for things we will modify for test env
@@ -93,26 +119,56 @@ install an MH variant and test it interactively."
              ((symbol-function 'file-directory-p))
              ;; the test always gets its own sub-folders cache
              (mh-sub-folders-cache (make-hash-table :test #'equal))
+             ;; Allow envvar TEST_MH_PATH to control mh-variants.
+             (mh-variants mh-test-utils-variants)
              ;; remember the original value
+             (original-mh-test-variant-logged mh-test-variant-logged-already)
+             (original-mh-path mh-path)
+             (original-mh-sys-path mh-sys-path)
+             (original-exec-path exec-path)
+             (original-mh-variant-in-use mh-variant-in-use)
+             (original-mh-progs mh-progs)
+             (original-mh-lib mh-lib)
+             (original-mh-lib-progs mh-lib-progs)
              (original-mh-envvar (getenv "MH")))
      (unwind-protect
          (progn
            (setq temp-home-dir (mh-test-utils-setup))
            ,@body)
+       (unless noninteractive
+         ;; If interactive, forget that we logged the variant and
+         ;; restore any changes TEST_MH_PATH made.
+         (setq mh-test-variant-logged-already original-mh-test-variant-logged
+               mh-path original-mh-path
+               mh-sys-path original-mh-sys-path
+               exec-path original-exec-path
+               mh-variant-in-use original-mh-variant-in-use
+               mh-progs original-mh-progs
+               mh-lib original-mh-lib
+               mh-lib-progs original-mh-lib-progs))
        (if temp-home-dir (delete-directory temp-home-dir t))
        (setenv "MH" original-mh-envvar))))
 
 (defun mh-test-utils-setup ()
   "Set dynamically bound variables needed by mock and/or variants.
+Call `mh-variant-set' to look through the directories named by
+envionment variable `TEST_MH_PATH' (default: `mh-path' and `mh-sys-path')
+to find the MH variant to use, if any.
 Return the name of the root of the created directory tree, if any."
+  (when (getenv "TEST_MH_PATH")
+    ;; force mh-variants to use only TEST_MH_PATH
+    (setq mh-path (split-string (getenv "TEST_MH_PATH") path-separator t)
+          mh-sys-path nil
+          exec-path '("/bin" "/usr/bin")))
   (unless mh-test-variant-logged-already
     (mh-variant-set mh-variant)
+    (setq mh-test-utils-variants mh-variants)
     (setq mh-test-variant-logged-already t))
-  ;; As `call-process'' and `file-directory-p' will be redefined, the
-  ;; native compiler will invoke `call-process' to compile the
-  ;; respective trampolines.  To avoid interference with the
-  ;; `call-process' mocking, we build these ahead of time.
   (when (native-comp-available-p)
+    ;; As `call-process'' and `file-directory-p' will be redefined, the
+    ;; native compiler will invoke `call-process' to compile the
+    ;; respective trampolines.  To avoid interference with the
+    ;; `call-process' mocking, we build these ahead of time.
     (mapc #'comp-subr-trampoline-install '(call-process file-directory-p)))
   (if mh-variant-in-use
       (mh-test-utils-setup-with-variant)

test/lisp/mh-e/test-all-mh-variants.sh

@@ -0,0 +1,104 @@
+#! /bin/bash
+# Run the mh-utils-tests against all MH variants found on this system.
+
+# Copyright (C) 2021 Free Software Foundation, Inc.
+
+# This file is part of GNU Emacs.
+
+# GNU Emacs is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# GNU Emacs is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with GNU Emacs.  If not, see <https://www.gnu.org/licenses/>.
+
+# Commentary:
+
+# By default runs all tests; test names or Emacs-style regexps may be
+# given on the command line to run just those tests.
+#
+# Option -d turns on Emacs variable mh-test-utils-debug-mocks, which
+# causes the tests to output all interactions with the file system.
+
+# If you want to run the tests for only one MH variant, you don't need
+# to use this script, because "make" can do it.  See the commentary at
+# the top of ./mh-utils-tests.el for the recipe.
+
+debug=
+if [[ "$1" = -* ]]; then
+    if [[ "$1" != -d ]]; then
+        echo "Usage: $(basename "$0") [-d] [test ...]" >&2
+        exit 2
+    fi
+    debug=t
+    shift
+fi
+
+shopt -s extglob
+ert_test_list=()
+for tst; do
+    # Guess the type the test spec
+    case $tst in
+        *[\[\].*+\\]*)  # Regexp: put in string quotes
+            ert_test_list+=("\"$tst\"")
+            ;;
+        *)  # Lisp expression, keyword, or symbol: use as is
+            ert_test_list+=("$tst")
+            ;;
+    esac
+done
+if [[ ${#ert_test_list[@]} -eq 0 ]]; then
+    # t means true for all tests, runs everything
+    ert_test_list=(t)
+fi
+
+# This script is 3 directories down in the Emacs source tree.
+cd "$(dirname "$0")"
+cd ../../..
+emacs=(src/emacs --batch -Q)
+
+# MH-E has a good list of directories where an MH variant might be installed,
+# so we look in each of those.
+read -r -a mh_sys_path \
+    < <("${emacs[@]}" -l mh-e --eval "(princ mh-sys-path)" | sed 's/[()]//g')
+
+have_done_mocked_variant=false
+declare -i tests_total=0 tests_passed=0
+
+for path in "${mh_sys_path[@]}"; do
+    if [[ ! -x "$path/mhparam" ]]; then
+        if [[ "$have_done_mocked_variant" = false ]]; then
+            have_done_mocked_variant=true
+        else
+            continue
+        fi
+    fi
+    echo "Testing with PATH $path"
+    ((++tests_total))
+    # The LD_LIBRARY_PATH setting is needed
+    # to run locally installed Mailutils.
+    TEST_MH_PATH=$path TEST_MH_DEBUG=$debug \
+    LD_LIBRARY_PATH=/usr/local/lib HOME=/nonexistent \
+    "${emacs[@]}" -l ert \
+        --eval "(setq load-prefer-newer t)" \
+        --eval "(load \"$PWD/test/lisp/mh-e/mh-utils-tests\" nil t)" \
+        --eval "(ert-run-tests-batch-and-exit '(or ${ert_test_list[*]}))" \
+        && ((++tests_passed))
+done
+
+if (( tests_total == 0 )); then
+    echo "NO tests run"
+    exit 1
+elif (( tests_total == tests_passed )); then
+    echo "All tested variants pass: $tests_passed/$tests_total"
+else
+    echo "Tested variants passing: $tests_passed/$tests_total," \
+         "FAILING: $((tests_total - tests_passed))/$tests_total"
+    exit 1
+fi