Run admin/cus-tests.el tests from test suite

* test/Makefile.in (SUBDIRS): Run tests in new directory "misc",
intended for tests not belonging to any one file.
* test/misc/test-custom-deps.el:
* test/misc/test-custom-libs.el:
* test/misc/test-custom-noloads.el:
* test/misc/test-custom-opts.el: New files.
* test/lisp/custom-tests.el (custom--test-local-option): Move test to
above new file test-custom-opts.el.

* admin/cus-test.el: Document running tests from regular test suite.
* test/file-organization.org (Test Files): Document new test directory
"misc" for tests not belonging to any one file.

admin/cus-test.el

@@ -37,6 +37,13 @@
 ;;
 ;;   src/emacs -batch -l admin/cus-test.el -f cus-test-noloads
 ;;
+;; or as a part of the test suite with
+;;
+;;   make -C test test-custom-opts
+;;   make -C test test-custom-deps
+;;   make -C test test-custom-libs
+;;   make -C test test-custom-noloads
+;;
 ;; in the emacs source directory.
 ;;
 ;; For interactive use: Load this file.  Then

test/Makefile.in

@@ -257,7 +257,7 @@ endef
 $(foreach test,${TESTS},$(eval $(call test_template,${test})))
 
 ## Get the tests for only a specific directory.
-SUBDIRS = $(sort $(shell find lib-src lisp src -type d ! -path "*resources*" -print))
+SUBDIRS = $(sort $(shell find lib-src lisp misc src -type d ! -path "*resources*" -print))
 
 define subdir_template
   .PHONY: check-$(subst /,-,$(1))

test/file-organization.org

@@ -43,6 +43,10 @@ Similarly, tests of features implemented in C should reside in
 ~-tests.el~ added to the base-name of the tested source file.  Thus,
 tests for ~src/fileio.c~ should be in ~test/src/fileio-tests.el~.
 
+Some tests do not belong to any one particular file.  Such tests
+should be put in the ~misc~ directory and be given a descriptive name
+that does /not/ end with ~-tests.el~.
+
 There are also some test materials that cannot be run automatically
 (i.e. via ert).  These should be placed in ~/test/manual~; they are
 not run by the "make check" command and its derivatives.

test/lisp/custom-tests.el

@@ -145,17 +145,6 @@
                                (widget-apply field :value-to-internal origvalue)
                                "bar"))))))
 
-(defconst custom-test-admin-cus-test
-  (expand-file-name "admin/cus-test.el" source-directory))
-
-(declare-function cus-test-opts custom-test-admin-cus-test)
-
-(ert-deftest check-for-wrong-custom-types ()
-  :tags '(:expensive-test)
-  (skip-unless (file-readable-p custom-test-admin-cus-test))
-  (load custom-test-admin-cus-test)
-  (should (null (cus-test-opts t))))
-
 (ert-deftest custom-test-enable-theme-keeps-settings ()
   "Test that enabling a theme doesn't change its settings."
   (let* ((custom-theme-load-path `(,(ert-resource-directory)))

test/misc/test-custom-deps.el

@@ -0,0 +1,42 @@
+;;; test-custom-deps.el --- Test custom deps  -*- lexical-binding:t -*-
+
+;; Copyright (C) 2021 Free Software Foundation, Inc.
+
+;; This file is part of GNU Emacs.
+
+;; GNU Emacs is free software: you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation, either version 3 of the License, or
+;; (at your option) any later version.
+
+;; GNU Emacs is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;; GNU General Public License for more details.
+
+;; You should have received a copy of the GNU General Public License
+;; along with GNU Emacs.  If not, see <https://www.gnu.org/licenses/>.
+
+;;; Commentary:
+
+;; The command `cus-test-deps' loads all (!) custom dependencies and
+;; reports about load errors.
+
+;;; Code:
+
+(require 'ert)
+
+(defconst custom-test-admin-cus-test
+  (expand-file-name "admin/cus-test.el" source-directory))
+
+(declare-function cus-test-deps custom-test-admin-cus-test)
+(defvar cus-test-deps-errors)  ; from admin/cus-tests.el
+
+(ert-deftest test-custom-deps ()
+  :tags '(:expensive-test)
+  (skip-unless (file-readable-p custom-test-admin-cus-test))
+  (load custom-test-admin-cus-test)
+  (cus-test-deps)
+  (should-not cus-test-deps-errors))
+
+;;; test-custom-deps.el ends here

test/misc/test-custom-libs.el

@@ -0,0 +1,46 @@
+;;; test-custom-libs.el --- Test custom loads  -*- lexical-binding:t -*-
+
+;; Copyright (C) 2021 Free Software Foundation, Inc.
+
+;; This file is part of GNU Emacs.
+
+;; GNU Emacs is free software: you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation, either version 3 of the License, or
+;; (at your option) any later version.
+
+;; GNU Emacs is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;; GNU General Public License for more details.
+
+;; You should have received a copy of the GNU General Public License
+;; along with GNU Emacs.  If not, see <https://www.gnu.org/licenses/>.
+
+;;; Commentary:
+
+;; This file runs for all libraries with autoloads separate emacs
+;; processes of the form "emacs -batch -l LIB".
+
+;;; Code:
+
+(require 'ert)
+
+(defconst custom-test-admin-cus-test
+  (expand-file-name "admin/cus-test.el" source-directory))
+
+(declare-function cus-test-libs custom-test-admin-cus-test)
+(defvar cus-test-libs-errors)  ; from admin/cus-tests.el
+
+;; FIXME: Currently fails for:
+;;        - lisp/term/ns-win.el
+;;        - lisp/org/org-num.el
+(ert-deftest test-custom-libs ()
+  :tags '(:expensive-test)
+  :expected-result :failed ; FIXME: See above.
+  (skip-unless (file-readable-p custom-test-admin-cus-test))
+  (load custom-test-admin-cus-test)
+  (cus-test-libs t)
+  (should-not cus-test-libs-errors))
+
+;;; test-custom-deps.el ends here

test/misc/test-custom-noloads.el

@@ -0,0 +1,45 @@
+;;; test-custom-deps.el --- Test custom noloads  -*- lexical-binding:t -*-
+
+;; Copyright (C) 2021 Free Software Foundation, Inc.
+
+;; This file is part of GNU Emacs.
+
+;; GNU Emacs is free software: you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation, either version 3 of the License, or
+;; (at your option) any later version.
+
+;; GNU Emacs is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;; GNU General Public License for more details.
+
+;; You should have received a copy of the GNU General Public License
+;; along with GNU Emacs.  If not, see <https://www.gnu.org/licenses/>.
+
+;;; Commentary:
+
+;; The command `cus-test-noloads' returns a list of variables which
+;; are somewhere declared as custom options, but not loaded by
+;; `custom-load-symbol'.
+
+;;; Code:
+
+(require 'ert)
+
+(defconst custom-test-admin-cus-test
+  (expand-file-name "admin/cus-test.el" source-directory))
+
+(declare-function cus-test-noloads custom-test-admin-cus-test)
+(defvar cus-test-vars-not-cus-loaded)  ; from admin/cus-tests.el
+
+;; FIXME: Multiple failures here.
+(ert-deftest custom-test-load ()
+  :tags '(:expensive-test)
+  :expected-result :failed ; FIXME: See above.
+  (skip-unless (file-readable-p custom-test-admin-cus-test))
+  (load custom-test-admin-cus-test)
+  (cus-test-noloads)
+  (should-not cus-test-vars-not-cus-loaded))
+
+;;; test-custom-deps.el ends here

test/misc/test-custom-opts.el

@@ -0,0 +1,39 @@
+;;; test-custom-opts.el --- Test custom opts  -*- lexical-binding:t -*-
+
+;; Copyright (C) 2021 Free Software Foundation, Inc.
+
+;; This file is part of GNU Emacs.
+
+;; GNU Emacs is free software: you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation, either version 3 of the License, or
+;; (at your option) any later version.
+
+;; GNU Emacs is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;; GNU General Public License for more details.
+
+;; You should have received a copy of the GNU General Public License
+;; along with GNU Emacs.  If not, see <https://www.gnu.org/licenses/>.
+
+;;; Commentary:
+
+;; The command `cus-test-opts' tests many (all?) custom options.
+
+;;; Code:
+
+(require 'ert)
+
+(defconst custom-test-admin-cus-test
+  (expand-file-name "admin/cus-test.el" source-directory))
+
+(declare-function cus-test-opts custom-test-admin-cus-test)
+
+(ert-deftest check-for-wrong-custom-opts ()
+  :tags '(:expensive-test)
+  (skip-unless (file-readable-p custom-test-admin-cus-test))
+  (load custom-test-admin-cus-test)
+  (should (null (cus-test-opts t))))
+
+;;; test-custom-opts.el ends here