Merge branch 'feature/standard-test-location'

.gitignore

@@ -42,10 +42,10 @@ InfoPlist.strings
 Makefile
 makefile
 !etc/refcards/Makefile
-!test/automated/data/flymake/Makefile
-!test/indent/Makefile
-!test/etags/Makefile
-!test/etags/make-src/Makefile
+!test/lisp/progmodes/flymake-resources/Makefile
+!test/manual/indent/Makefile
+!test/manual/etags/Makefile
+!test/manual/etags/make-src/Makefile
 /*.cache
 /confdefs.h
 /config.status

CONTRIBUTE

@@ -251,9 +251,9 @@ for more information on writing and running tests.
 
 To run tests on the entire Emacs tree, run "make check" from the
 top-level directory.  Most tests are in the directory
-"test/automated".  From the "test/automated" directory, run "make
+"test/".  From the "test/" directory, run "make
 <filename>" to run the tests for <filename>.el(c).  See
-"test/automated/Makefile" for more information.
+"test/Makefile" for more information.
 
 ** Understanding Emacs Internals.
 

Makefile.in

@@ -922,11 +922,11 @@ TAGS tags: lib lib-src src
 	$(MAKE) -C src tags
 
 check: all
-	@if test ! -d test/automated; then \
+	@if test ! -d test; then \
 	  echo "You do not seem to have the test/ directory."; \
 	  echo "Maybe you are using a release tarfile, rather than a repository checkout."; \
 	else \
-	  $(MAKE) -C test/automated check; \
+	  $(MAKE) -C test check; \
 	fi
 
 dist:

configure.ac

@@ -4167,7 +4167,7 @@ else
   SEPCHAR=':'
 fi
 AC_DEFINE_UNQUOTED(SEPCHAR, ['$SEPCHAR'], [Character that separates PATH elements.])
-dnl This is for MinGW, and is used in test/automated/Makefile.in.
+dnl This is for MinGW, and is used in test/Makefile.in.
 dnl The MSYS Bash has heuristics for replacing ':' with ';' when it
 dnl decides that a command-line argument to be passed to a MinGW program
 dnl is a PATH-style list of directories.  But that heuristics plays it
@@ -5318,13 +5318,13 @@ AC_CONFIG_FILES([Makefile lib/Makefile lib-src/Makefile oldXMenu/Makefile \
        leim/Makefile nextstep/Makefile nt/Makefile])
 
 dnl test/ is not present in release tarfiles.
-opt_makefile=test/automated/Makefile
+opt_makefile=test/Makefile
 
 if test -f "$srcdir/$opt_makefile.in"; then
   SUBDIR_MAKEFILES="$SUBDIR_MAKEFILES $opt_makefile"
   dnl Again, it's best not to use a variable.  Though you can add
   dnl ", [], [opt_makefile='$opt_makefile']" and it should work.
-  AC_CONFIG_FILES([test/automated/Makefile])
+  AC_CONFIG_FILES([test/Makefile])
 fi
 
 

test/Makefile.in

@@ -37,7 +37,7 @@ SEPCHAR = @SEPCHAR@
 # We never change directory before running Emacs, so a relative file
 # name is fine, and makes life easier.  If we need to change
 # directory, we can use emacs --chdir.
-EMACS = ../../src/emacs
+EMACS = ../src/emacs
 
 EMACS_EXTRAOPT=
 
@@ -99,9 +99,11 @@ WRITE_LOG = > $@ 2>&1 || { stat=ERROR; cat $@; }; echo $$stat: $@
 	$(emacs) -l ert -l $$loadfile \
 	  -f ert-run-tests-batch-and-exit ${WRITE_LOG}
 
-ELFILES = $(sort $(wildcard ${srcdir}/*.el))
-LOGFILES = $(patsubst %.el,%.log,$(notdir ${ELFILES}))
-TESTS = ${LOGFILES:.log=}
+ELFILES = $(filter-out ./manual%,\
+	$(sort $(wildcard ${srcdir}/*.el) $(wildcard ${srcdir}/*/*.el) \
+	$(wildcard ${srcdir}/*/*/*.el) $(wildcard ${srcdir}/*/*/*/*.el)))
+LOGFILES = $(patsubst %.el,%.log,${ELFILES})
+TESTS = $(subst ${srcdir}/,,$(LOGFILES:.log=))
 
 ## If we have to interrupt a hanging test, preserve the log so we can
 ## see what the problem was.
@@ -112,8 +114,8 @@ TESTS = ${LOGFILES:.log=}
 ## The short aliases that always re-run the tests, with no logging.
 define test_template
 $(1):
-	@test ! -f $(1).log || mv $(1).log $(1).log~
-	@${MAKE} $(1).log WRITE_LOG=
+	@test ! -f ${srcdir}/$(1).log || mv ${srcdir}/$(1).log ${srcdir}/$(1).log~
+	@${MAKE} ${srcdir}/$(1).log WRITE_LOG=
 endef
 
 $(foreach test,${TESTS},$(eval $(call test_template,${test})))
@@ -132,10 +134,12 @@ check-maybe: ${LOGFILES}
 .PHONY: mostlyclean clean bootstrap-clean distclean maintainer-clean
 
 clean mostlyclean:
-	-rm -f *.log *.log~
+	-rm -f ${srcdir}/*.log ${srcdir}/*.log~ ${srcdir}/*/*.log \
+		${srcdir}/*/*.log~ ${srcdir}/*/*/*.log \
+		${srcdir}/*/*/*.log~
 
 bootstrap-clean: clean
-	-rm -f ${srcdir}/*.elc
+	-rm -f ${srcdir}/*.elc ${srcdir}/*/*.elc ${srcdir}/*/*/*.elc
 
 distclean: clean
 	rm -f Makefile

test/file-organisation.org

@@ -0,0 +1,54 @@
+#+TITLE: The Location of Emacs-Lisp Tests
+
+
+
+* The Main Emacs Repository
+
+The Emacs repository contains a very large number of Emacs-Lisp files, many of
+which pre-date both formal package support for Emacs and automated unit
+testing.
+
+All paths are relative to the Emacs root directory.
+
+** Source
+
+Lisp files are stored in the ~lisp~ directory or its sub-directories.
+Sub-directories are in many cases themed after packages (~gnus~, ~org~,
+~calc~), related functionality (~net~, ~emacs-lisp~, ~progmodes~) or status
+(~obsolete~).
+
+C source is stored in the ~src~ directory, which is flat.
+
+** Test Files
+
+Automated tests should be stored in the ~test/automated/lisp~ directory. Tests
+should reflect the directory structure of the source tree; so tests for files
+in the ~emacs-lisp~ source directory should reside in the
+~test/lisp/emacs-lisp~ directory.
+
+Tests should normally reside in a file with ~-tests~ added to the name of
+the tested source file; hence ~ert.el~ is tested in ~ert-tests.el~, or
+~pcase.el~ is tested in ~pcase-tests.el~. Exceptionally, tests for a
+single feature may be placed into multiple files of any name which are
+themselves placed in a directory named after the feature with ~-tests~
+appended, such as ~/test/lisp/emacs-lisp/eieio-tests~
+
+Where features of the C source are tested using Emacs-Lisp test files, these
+should reside in ~/test/src~ and be named after the C file.
+
+A few test suites which predate this scheme and do not fit cleanly
+into it are placed in ~/test/lisp/legacy~.
+
+There are also some test materials that cannot be run automatically
+(i.e. via ert). These should be placed in ~/test/manual~
+
+** Resource Files
+
+Resource files for tests (containing test data) should reside in a directory
+named after the feature with a ~-resources~ suffix, and located in the same
+directory as the feature. Hence, the lisp file ~flymake.el~ should have test
+files in ~/test/automated/lisp/progmodes/flymake-tests.el~ should reside in a
+directory called ~/test/automated/lisp/progmodes/flymake-resources~.
+
+No guidance is given for the organisation of resource files inside the
+~-resource~ directory; files can be organised at the author's discretion.

test/lisp/emacs-lisp/package-tests.el

@@ -97,7 +97,7 @@
                                (multi-file (0 1))))
   "`package-desc' used for testing dependencies.")
 
-(defvar package-test-data-dir (expand-file-name "data/package" package-test-file-dir)
+(defvar package-test-data-dir (expand-file-name "package-resources" package-test-file-dir)
   "Base directory of package test files.")
 
 (defvar package-test-fake-contents-file
@@ -190,18 +190,18 @@ Must called from within a `tar-mode' buffer."
 
 (ert-deftest package-test-desc-from-buffer ()
   "Parse an elisp buffer to get a `package-desc' object."
-  (with-package-test (:basedir "data/package" :file "simple-single-1.3.el")
+  (with-package-test (:basedir "package-resources" :file "simple-single-1.3.el")
     (should (equal (package-buffer-info) simple-single-desc)))
-  (with-package-test (:basedir "data/package" :file "simple-depend-1.0.el")
+  (with-package-test (:basedir "package-resources" :file "simple-depend-1.0.el")
     (should (equal (package-buffer-info) simple-depend-desc)))
-  (with-package-test (:basedir "data/package"
+  (with-package-test (:basedir "package-resources"
                                :file "multi-file-0.2.3.tar")
     (tar-mode)
     (should (equal (package-tar-file-info) multi-file-desc))))
 
 (ert-deftest package-test-install-single ()
   "Install a single file without using an archive."
-  (with-package-test (:basedir "data/package" :file "simple-single-1.3.el")
+  (with-package-test (:basedir "package-resources" :file "simple-single-1.3.el")
     (should (package-install-from-buffer))
     (package-initialize)
     (should (package-installed-p 'simple-single))
@@ -269,7 +269,7 @@ Must called from within a `tar-mode' buffer."
 (ert-deftest package-test-install-prioritized ()
   "Install a lower version from a higher-prioritized archive."
   (with-package-test ()
-    (let* ((newer-version (expand-file-name "data/package/newer-versions"
+    (let* ((newer-version (expand-file-name "package-resources/newer-versions"
                                             package-test-file-dir))
            (package-archives `(("older" . ,package-test-data-dir)
                                ("newer" . ,newer-version)))
@@ -285,7 +285,7 @@ Must called from within a `tar-mode' buffer."
 
 (ert-deftest package-test-install-multifile ()
   "Check properties of the installed multi-file package."
-  (with-package-test (:basedir "data/package" :install '(multi-file))
+  (with-package-test (:basedir "package-resources" :install '(multi-file))
     (let ((autoload-file
            (expand-file-name "multi-file-autoloads.el"
                              (expand-file-name
@@ -336,7 +336,7 @@ Must called from within a `tar-mode' buffer."
       (package-menu-execute)
       (should (package-installed-p 'simple-single))
       (let ((package-test-data-dir
-             (expand-file-name "data/package/newer-versions" package-test-file-dir)))
+             (expand-file-name "package-resources/newer-versions" package-test-file-dir)))
         (setq package-archives `(("gnu" . ,package-test-data-dir)))
         (package-menu-refresh)
 
@@ -456,7 +456,7 @@ Must called from within a `tar-mode' buffer."
 		     (delete-directory homedir t)))))
   (let* ((keyring (expand-file-name "key.pub" package-test-data-dir))
 	 (package-test-data-dir
-	   (expand-file-name "data/package/signed" package-test-file-dir)))
+	   (expand-file-name "package-resources/signed" package-test-file-dir)))
     (with-package-test ()
       (package-initialize)
       (package-import-keyring keyring)
@@ -508,7 +508,7 @@ Must called from within a `tar-mode' buffer."
 
 (ert-deftest package-x-test-upload-buffer ()
   "Test creating an \"archive-contents\" file"
-  (with-package-test (:basedir "data/package"
+  (with-package-test (:basedir "package-resources"
                                :file "simple-single-1.3.el"
                                :upload-base t)
     (package-upload-buffer)
@@ -532,7 +532,7 @@ Must called from within a `tar-mode' buffer."
 
 (ert-deftest package-x-test-upload-new-version ()
   "Test uploading a new version of a package"
-  (with-package-test (:basedir "data/package"
+  (with-package-test (:basedir "package-resources"
                                :file "simple-single-1.3.el"
                                :upload-base t)
     (package-upload-buffer)
@@ -599,6 +599,7 @@ Must called from within a `tar-mode' buffer."
                simple-depend-desc-2)))
     (should
      (equal (package--sort-by-dependence delete-list)
+
             (list simple-depend-desc-2 simple-depend-desc-1 new-pkg-desc
                   multi-file-desc simple-depend-desc simple-single-desc)))
     (should

test/lisp/progmodes/elisp-mode-tests.el

@@ -244,7 +244,7 @@ to (xref-elisp-test-descr-to-target xref)."
     (xref-make "(cl-defstruct (xref-elisp-location (:constructor xref-make-elisp-location)))"
                (xref-make-elisp-location
                 'xref-elisp-location 'define-type
-                (expand-file-name "../../lisp/progmodes/elisp-mode.el" emacs-test-dir)))
+                (expand-file-name "../../../lisp/progmodes/elisp-mode.el" emacs-test-dir)))
     ;; It's not worth adding another special case to `xref-elisp-test-descr-to-target' for this
     "(cl-defstruct (xref-elisp-location")
    ))
@@ -255,11 +255,11 @@ to (xref-elisp-test-descr-to-target xref)."
    (xref-make "(defalias Buffer-menu-sort)"
 	      (xref-make-elisp-location
 	       'Buffer-menu-sort 'defalias
-	       (expand-file-name "../../lisp/buff-menu.elc" emacs-test-dir)))
+	       (expand-file-name "../../../lisp/buff-menu.elc" emacs-test-dir)))
    (xref-make "(defun tabulated-list-sort)"
 	      (xref-make-elisp-location
 	       'tabulated-list-sort nil
-	       (expand-file-name "../../lisp/emacs-lisp/tabulated-list.el" emacs-test-dir)))
+	       (expand-file-name "../../../lisp/emacs-lisp/tabulated-list.el" emacs-test-dir)))
    ))
 
 ;; FIXME: defconst
@@ -406,27 +406,27 @@ to (xref-elisp-test-descr-to-target xref)."
    (xref-make "(cl-defgeneric xref-location-marker)"
 	      (xref-make-elisp-location
 	       'xref-location-marker 'cl-defgeneric
-	       (expand-file-name "../../lisp/progmodes/xref.el" emacs-test-dir)))
+	       (expand-file-name "../../../lisp/progmodes/xref.el" emacs-test-dir)))
    (xref-make "(cl-defmethod xref-location-marker ((l xref-elisp-location)))"
 	      (xref-make-elisp-location
 	       '(xref-location-marker xref-elisp-location) 'cl-defmethod
-	       (expand-file-name "../../lisp/progmodes/elisp-mode.el" emacs-test-dir)))
+	       (expand-file-name "../../../lisp/progmodes/elisp-mode.el" emacs-test-dir)))
    (xref-make "(cl-defmethod xref-location-marker ((l xref-file-location)))"
 	      (xref-make-elisp-location
 	       '(xref-location-marker xref-file-location) 'cl-defmethod
-	       (expand-file-name "../../lisp/progmodes/xref.el" emacs-test-dir)))
+	       (expand-file-name "../../../lisp/progmodes/xref.el" emacs-test-dir)))
    (xref-make "(cl-defmethod xref-location-marker ((l xref-buffer-location)))"
 	      (xref-make-elisp-location
 	       '(xref-location-marker xref-buffer-location) 'cl-defmethod
-	       (expand-file-name "../../lisp/progmodes/xref.el" emacs-test-dir)))
+	       (expand-file-name "../../../lisp/progmodes/xref.el" emacs-test-dir)))
    (xref-make "(cl-defmethod xref-location-marker ((l xref-bogus-location)))"
 	      (xref-make-elisp-location
 	       '(xref-location-marker xref-bogus-location) 'cl-defmethod
-	       (expand-file-name "../../lisp/progmodes/xref.el" emacs-test-dir)))
+	       (expand-file-name "../../../lisp/progmodes/xref.el" emacs-test-dir)))
    (xref-make "(cl-defmethod xref-location-marker ((l xref-etags-location)))"
               (xref-make-elisp-location
                '(xref-location-marker xref-etags-location) 'cl-defmethod
-               (expand-file-name "../../lisp/progmodes/etags.el" emacs-test-dir)))
+               (expand-file-name "../../../lisp/progmodes/etags.el" emacs-test-dir)))
    ))
 
 (xref-elisp-deftest find-defs-defgeneric-eval
@@ -528,7 +528,7 @@ to (xref-elisp-test-descr-to-target xref)."
    (xref-make "(defun xref-find-definitions)"
 	      (xref-make-elisp-location
 	       'xref-find-definitions nil
-	       (expand-file-name "../../lisp/progmodes/xref.el" emacs-test-dir)))))
+	       (expand-file-name "../../../lisp/progmodes/xref.el" emacs-test-dir)))))
 
 (xref-elisp-deftest find-defs-defun-eval
   (elisp--xref-find-definitions (eval '(defun stephe-leake-defun ())))
@@ -561,7 +561,7 @@ to (xref-elisp-test-descr-to-target xref)."
     (xref-make "(defun abbrev-mode)"
                (xref-make-elisp-location
                 'abbrev-mode nil
-                (expand-file-name "../../lisp/abbrev.el" emacs-test-dir)))
+                (expand-file-name "../../../lisp/abbrev.el" emacs-test-dir)))
     "(define-minor-mode abbrev-mode"))
   )
 
@@ -582,7 +582,7 @@ to (xref-elisp-test-descr-to-target xref)."
     (xref-make "(defun compilation-minor-mode)"
                (xref-make-elisp-location
                 'compilation-minor-mode nil
-                (expand-file-name "../../lisp/progmodes/compile.el" emacs-test-dir)))
+                (expand-file-name "../../../lisp/progmodes/compile.el" emacs-test-dir)))
     "(define-minor-mode compilation-minor-mode")
    ))
 
@@ -592,7 +592,7 @@ to (xref-elisp-test-descr-to-target xref)."
    (xref-make "(defvar xref--marker-ring)"
 	      (xref-make-elisp-location
 	       'xref--marker-ring 'defvar
-	       (expand-file-name "../../lisp/progmodes/xref.el" emacs-test-dir)))
+	       (expand-file-name "../../../lisp/progmodes/xref.el" emacs-test-dir)))
     ))
 
 (xref-elisp-deftest find-defs-defvar-c
@@ -615,11 +615,11 @@ to (xref-elisp-test-descr-to-target xref)."
    (xref-make "(defvar font-lock-keyword-face)"
 	      (xref-make-elisp-location
 	       'font-lock-keyword-face 'defvar
-	       (expand-file-name "../../lisp/font-lock.el" emacs-test-dir)))
+	       (expand-file-name "../../../lisp/font-lock.el" emacs-test-dir)))
    (xref-make "(defface font-lock-keyword-face)"
 	      (xref-make-elisp-location
 	       'font-lock-keyword-face 'defface
-	       (expand-file-name "../../lisp/font-lock.el" emacs-test-dir)))
+	       (expand-file-name "../../../lisp/font-lock.el" emacs-test-dir)))
    ))
 
 (xref-elisp-deftest find-defs-face-eval
@@ -633,7 +633,7 @@ to (xref-elisp-test-descr-to-target xref)."
     (xref-make "(feature xref)"
 	      (xref-make-elisp-location
 	       'xref 'feature
-	       (expand-file-name "../../lisp/progmodes/xref.el" emacs-test-dir)))
+	       (expand-file-name "../../../lisp/progmodes/xref.el" emacs-test-dir)))
     ";;; Code:")
    ))
 

test/lisp/progmodes/flymake-tests.el

@@ -26,7 +26,7 @@
 (require 'flymake)
 
 (defvar flymake-tests-data-directory
-  (expand-file-name "data/flymake" (getenv "EMACS_TEST_DIRECTORY"))
+  (expand-file-name "lisp/progmodes/flymake-resources" (getenv "EMACS_TEST_DIRECTORY"))
   "Directory containing flymake test data.")