diff --git a/.gitignore b/.gitignore index f1abb2ab687..7baee47b0a1 100644 --- a/.gitignore +++ b/.gitignore @@ -159,6 +159,7 @@ test/manual/etags/CTAGS test/manual/indent/*.new test/lisp/gnus/mml-sec-resources/random_seed test/lisp/play/fortune-resources/fortunes.dat +test/**/*.xml # ctags, etags. TAGS diff --git a/admin/notes/emba b/admin/notes/emba index f1b52b2cde0..2135c7a97cc 100644 --- a/admin/notes/emba +++ b/admin/notes/emba @@ -63,6 +63,10 @@ They can be downloaded from the server, visiting the URL , and selecting the job in question. +Every pipeline generates a JUnit test report for the respective test +jobs, which can be inspected on the pipeline web page. This test +report counts completed ERT tests, aborted tests are not counted. + * Emba configuration The emba configuration files are hosted on diff --git a/etc/NEWS b/etc/NEWS index b55b30665be..8d83b2a7e36 100644 --- a/etc/NEWS +++ b/etc/NEWS @@ -163,6 +163,12 @@ the previous definition to be discarded, which was probably not intended when this occurs in batch mode. To remedy the error, rename tests so that they all have unique names. ++++ +*** ERT can generate JUnit test reports. +When environment variable 'EMACS_TEST_JUNIT_REPORT' is set, ERT +generates a JUnit test report under this file name. This is useful +for Emacs integration into CI/CD test environments. + ** Emoji +++ @@ -1143,6 +1149,7 @@ This variable is bound to t during the preparation of a "*Help*" buffer. ** 'date-to-time' now assumes earliest values if its argument lacks month, day, or time. For example, (date-to-time "2021-12-04") now assumes a time of 00:00 instead of signaling an error. + * Changes in Emacs 29.1 on Non-Free Operating Systems diff --git a/lisp/emacs-lisp/ert.el b/lisp/emacs-lisp/ert.el index 946193e40dc..981e23931c2 100644 --- a/lisp/emacs-lisp/ert.el +++ b/lisp/emacs-lisp/ert.el @@ -65,6 +65,8 @@ (require 'pp) (require 'map) +(autoload 'xml-escape-string "xml.el") + ;;; UI customization options. (defgroup ert () @@ -247,7 +249,6 @@ in batch mode, an error is signalled. "%s\\(\\s-\\|$\\)") "The regexp the `find-function' mechanisms use for finding test definitions.") - (define-error 'ert-test-failed "Test failed") (define-error 'ert-test-skipped "Test skipped") @@ -677,7 +678,6 @@ and is displayed in front of the value of MESSAGE-FORM." ,@body)) - ;;; Facilities for running a single test. (defvar ert-debug-on-error nil @@ -1437,7 +1437,9 @@ Returns the stats object." (if (getenv "EMACS_TEST_VERBOSE") (ert-reason-for-test-result result) "")))) - (message "%s" ""))))) + (message "%s" "")) + (when (getenv "EMACS_TEST_JUNIT_REPORT") + (ert-write-junit-test-report stats))))) (test-started) (test-ended (cl-destructuring-bind (stats test result) event-args @@ -1525,6 +1527,128 @@ the tests)." (backtrace)) (kill-emacs 2)))) +(defun ert-write-junit-test-report (stats) + "Write a JUnit test report, generated from STATS." + ;; https://www.ibm.com/docs/de/developer-for-zos/14.1.0?topic=formats-junit-xml-format + ;; https://llg.cubic.org/docs/junit/ + (unless (zerop (length (ert--stats-tests stats))) + (when-let ((test-file + (symbol-file + (ert-test-name (aref (ert--stats-tests stats) 0)) 'ert--test))) + (with-temp-file (file-name-with-extension test-file "xml") + (insert "\n") + (insert (format "\n" + (file-name-nondirectory test-file) + (ert-stats-total stats) + (ert-stats-completed-unexpected stats) + (ert-stats-skipped stats) + (float-time + (time-subtract + (ert--stats-end-time stats) + (ert--stats-start-time stats))))) + (insert (format " \n" + (file-name-nondirectory test-file) + (ert-stats-total stats) + (ert-stats-completed-unexpected stats) + (ert-stats-skipped stats) + (float-time + (time-subtract + (ert--stats-end-time stats) + (ert--stats-start-time stats))) + (ert--format-time-iso8601 (ert--stats-end-time stats)))) + (insert " \n" + (format " \n" + (ert--stats-selector stats)) + " \n") + (cl-loop for test across (ert--stats-tests stats) + for result = (ert-test-most-recent-result test) do + (insert (format " \n") + (insert ">\n") + (if (ert-test-skipped-p result) + (insert (format " \n" + (xml-escape-string + (string-trim + (ert-reason-for-test-result result))) + (ert-string-for-test-result + result + (ert-test-result-expected-p + test result))) + (xml-escape-string + (string-trim + (ert-reason-for-test-result result))) + "\n" + " \n") + (unless + (ert-test-result-type-p + result (ert-test-expected-result-type test)) + (insert (format " \n" + (xml-escape-string + (string-trim + (ert-reason-for-test-result result))) + (ert-string-for-test-result + result + (ert-test-result-expected-p + test result))) + (xml-escape-string + (string-trim + (ert-reason-for-test-result result))) + "\n" + " \n"))) + (unless (zerop (length (ert-test-result-messages result))) + (insert " \n" + (xml-escape-string + (ert-test-result-messages result)) + " \n")) + (insert " \n"))) + (insert " \n") + (insert "\n"))))) + +(defun ert-write-junit-test-summary-report (&rest logfiles) + "Write a JUnit summary test report, generated from LOGFILES." + (let ((report (file-name-with-extension + (getenv "EMACS_TEST_JUNIT_REPORT") "xml")) + (tests 0) (failures 0) (skipped 0) (time 0) (id 0)) + (with-temp-file report + (dolist (logfile logfiles) + (let ((test-file (file-name-with-extension logfile "xml"))) + (when (file-readable-p test-file) + (insert-file-contents-literally test-file) + (when (looking-at-p + (regexp-quote "")) + (delete-region (point) (line-beginning-position 2))) + (when (looking-at + "") + (cl-incf tests (string-to-number (match-string 1))) + (cl-incf failures (string-to-number (match-string 2))) + (cl-incf skipped (string-to-number (match-string 3))) + (cl-incf time (string-to-number (match-string 4))) + (delete-region (point) (line-beginning-position 2))) + (when (looking-at " ") + (delete-region (point) (line-beginning-position 2))) + (narrow-to-region (point-max) (point-max))))) + + (insert "\n") + (widen) + (goto-char (point-min)) + (insert "\n") + (insert (format "\n" + (file-name-nondirectory report) + tests failures skipped time))))) (defun ert-summarize-tests-batch-and-exit (&optional high) "Summarize the results of testing. @@ -1540,6 +1664,8 @@ If HIGH is a natural number, the HIGH long lasting tests are summarized." ;; behavior. (setq attempt-stack-overflow-recovery nil attempt-orderly-shutdown-on-fatal-signal nil) + (when (getenv "EMACS_TEST_JUNIT_REPORT") + (apply #'ert-write-junit-test-summary-report command-line-args-left)) (let ((nlogs (length command-line-args-left)) (ntests 0) (nrun 0) (nexpected 0) (nunexpected 0) (nskipped 0) nnotrun logfile notests badtests unexpected skipped tests) @@ -1855,7 +1981,6 @@ Also sets `ert--results-progress-bar-button-begin'." ;; should test it again.) "\n"))) - (defvar ert-test-run-redisplay-interval-secs .1 "How many seconds ERT should wait between redisplays while running tests. @@ -2037,7 +2162,6 @@ STATS is the stats object; LISTENER is the results listener." (goto-char (1- (point-max))) buffer))))) - (defvar ert--selector-history nil "List of recent test selectors read from terminal.") diff --git a/test/Makefile.in b/test/Makefile.in index f2c49584e7f..eeda2918fa3 100644 --- a/test/Makefile.in +++ b/test/Makefile.in @@ -353,6 +353,7 @@ mostlyclean: clean: find . '(' -name '*.log' -o -name '*.log~' ')' $(FIND_DELETE) + find . '(' -name '*.xml' -a ! -path '*resources*' ')' $(FIND_DELETE) rm -f ${srcdir}/lisp/gnus/mml-sec-resources/random_seed rm -f $(test_module_dir)/*.o $(test_module_dir)/*.so \ $(test_module_dir)/*.dll diff --git a/test/README b/test/README index 4d447c9bf15..2bd84b5f9b3 100644 --- a/test/README +++ b/test/README @@ -114,6 +114,9 @@ mode--only the names of the failed tests are listed. If the $EMACS_TEST_VERBOSE environment variable is set, the failure summaries will also include the data from the failing test. +If the $EMACS_TEST_JUNIT_REPORT environment variable is set to a file +name, a JUnit test report is generated under this name. + Some of the tests require a remote temporary directory (autorevert-tests.el, filenotify-tests.el, shadowfile-tests.el and tramp-tests.el). Per default, a mock-up connection method is used diff --git a/test/infra/Makefile.in b/test/infra/Makefile.in index fd11d367983..e4f99743e09 100644 --- a/test/infra/Makefile.in +++ b/test/infra/Makefile.in @@ -93,10 +93,8 @@ all: generate-test-jobs .PHONY: generate-test-jobs $(FILE) $(SUBDIR_TARGETS) -generate-test-jobs: clean $(FILE) $(SUBDIR_TARGETS) +generate-test-jobs: $(FILE) $(SUBDIR_TARGETS) $(FILE): $(AM_V_GEN) - -clean: - @rm -f $(FILE) + @echo "# Generated by \"make generate-test-jobs\", don't edit." >$(FILE) diff --git a/test/infra/gitlab-ci.yml b/test/infra/gitlab-ci.yml index b0ea6813b30..3903642c792 100644 --- a/test/infra/gitlab-ci.yml +++ b/test/infra/gitlab-ci.yml @@ -44,6 +44,7 @@ workflow: variables: GIT_STRATEGY: fetch EMACS_EMBA_CI: 1 + EMACS_TEST_JUNIT_REPORT: junit-test-report.xml EMACS_TEST_TIMEOUT: 3600 EMACS_TEST_VERBOSE: 1 # Use TLS https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#tls-enabled @@ -85,7 +86,7 @@ default: # TODO: with make -j4 several of the tests were failing, for # example shadowfile-tests, but passed without it. - 'export PWD=$(pwd)' - - 'docker run -i -e EMACS_EMBA_CI=${EMACS_EMBA_CI} -e EMACS_TEST_TIMEOUT=${EMACS_TEST_TIMEOUT} -e EMACS_TEST_VERBOSE=${EMACS_TEST_VERBOSE} --volumes-from $(docker ps -q -f "label=com.gitlab.gitlab-runner.job.id=${CI_JOB_ID}"):ro --name ${test_name} ${CI_REGISTRY_IMAGE}:${target}-${BUILD_TAG} /bin/bash -c "git fetch ${PWD} HEAD && echo checking out these updated files && git diff --name-only FETCH_HEAD && ( git diff --name-only FETCH_HEAD | xargs git checkout -f FETCH_HEAD ) && make -j4 && make ${make_params}"' + - 'docker run -i -e EMACS_EMBA_CI=${EMACS_EMBA_CI} -e EMACS_TEST_JUNIT_REPORT=${EMACS_TEST_JUNIT_REPORT} -e EMACS_TEST_TIMEOUT=${EMACS_TEST_TIMEOUT} -e EMACS_TEST_VERBOSE=${EMACS_TEST_VERBOSE} --volumes-from $(docker ps -q -f "label=com.gitlab.gitlab-runner.job.id=${CI_JOB_ID}"):ro --name ${test_name} ${CI_REGISTRY_IMAGE}:${target}-${BUILD_TAG} /bin/bash -c "git fetch ${PWD} HEAD && echo checking out these updated files && git diff --name-only FETCH_HEAD && ( git diff --name-only FETCH_HEAD | xargs git checkout -f FETCH_HEAD ) && make -j4 && make ${make_params}"' after_script: # - docker ps -a # - printenv @@ -93,7 +94,8 @@ default: # Prepare test artifacts. - test -n "$(docker ps -aq -f name=${test_name})" && docker cp ${test_name}:checkout/test ${test_name} - test -n "$(docker ps -aq -f name=${test_name})" && docker rm ${test_name} - - find ${test_name} ! -name "*.log" -type f -delete + - find ${test_name} ! \( -name "*.log" -o -name ${EMACS_TEST_JUNIT_REPORT} \) -type f -delete + # BusyBox find does not know -empty. - find ${test_name} -type d -depth -exec rmdir {} + 2>/dev/null .build-template: @@ -103,7 +105,6 @@ default: when: always - changes: - "**.in" - - "**.yml" - GNUmakefile - aclocal.m4 - autogen.sh @@ -112,7 +113,7 @@ default: - lib/malloc/*.{h,c} - lisp/emacs-lisp/*.el - src/*.{h,c} - - test/infra/Dockerfile.emba + - test/infra/* - changes: # gfilemonitor, kqueue - src/gfilenotify.c @@ -133,9 +134,11 @@ default: name: ${test_name} public: true expire_in: 1 week + when: always paths: - ${test_name}/ - when: always + reports: + junit: ${test_name}/${EMACS_TEST_JUNIT_REPORT} .gnustep-template: rules: @@ -143,12 +146,11 @@ default: - if: '$CI_PIPELINE_SOURCE == "schedule"' changes: - "**.in" - - "**.yml" - src/ns*.{h,m} - src/macfont.{h,m} - lisp/term/ns-win.el - nextstep/** - - test/infra/Dockerfile.emba + - test/infra/* .filenotify-gio-template: rules: @@ -156,12 +158,11 @@ default: - if: '$CI_PIPELINE_SOURCE == "schedule"' changes: - "**.in" - - "**.yml" - lisp/autorevert.el - lisp/filenotify.el - lisp/net/tramp-sh.el - src/gfilenotify.c - - test/infra/Dockerfile.emba + - test/infra/* - test/lisp/autorevert-tests.el - test/lisp/filenotify-tests.el @@ -171,11 +172,10 @@ default: - if: '$CI_PIPELINE_SOURCE == "schedule"' changes: - "**.in" - - "**.yml" - lisp/emacs-lisp/comp.el - lisp/emacs-lisp/comp-cstr.el - src/comp.{h,m} - - test/infra/Dockerfile.emba + - test/infra/* - test/src/comp-resources/*.el - test/src/comp-tests.el timeout: 8 hours diff --git a/test/infra/test-jobs.yml b/test/infra/test-jobs.yml index bad8575b5c5..63b052bf8c7 100644 --- a/test/infra/test-jobs.yml +++ b/test/infra/test-jobs.yml @@ -1,3 +1,4 @@ +# Generated by "make generate-test-jobs", don't edit. test-lib-src-inotify: stage: normal