procyberian/emacs
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

ERT can generate JUnit test reports

Browse source 
* .gitignore: Add test/**/*.xml.

* admin/notes/emba: Mention JUnit test report.

* etc/NEWS: ERT can generate JUnit test reports.

* lisp/emacs-lisp/ert.el (xml-escape-string): Autoload.
(ert-write-junit-test-report)
(ert-write-junit-test-summary-report): New defuns.
(ert-run-tests-batch, ert-summarize-tests-batch-and-exit): Call them.

* test/Makefile.in (clean): Remove *.xml.

* test/README: Mention $EMACS_TEST_JUNIT_REPORT environment variable.

* test/infra/Makefile.in ($(FILE)): Generate header commentary.
(clean): Remove.

* test/infra/gitlab-ci.yml (variables): Set EMACS_TEST_JUNIT_REPORT.
(.job-template): Use it in script and after_script.
(.build-template, .gnustep-template, .filenotify-gio-template)
(.native-comp-template): Adapt rules.
(.test-template): Trigger JUnit test report.

* test/infra/test-jobs.yml: Regenerate.
This commit is contained in:
Michael Albinus 2021-12-13 16:09:56 +01:00
parent c1476afb99
commit b30b33ed9b
9 changed files with 159 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -159,6 +159,7 @@ test/manual/etags/CTAGS
test/manual/indent/*.new
test/lisp/gnus/mml-sec-resources/random_seed
test/lisp/play/fortune-resources/fortunes.dat
test/**/*.xml
# ctags, etags.
TAGS

4
admin/notes/emba
View file

@ -63,6 +63,10 @@ They can be downloaded from the server, visiting the URL
<https://emba.gnu.org/emacs/emacs/-/pipelines>, and selecting the job
in question.
Every pipeline generates a JUnit test report for the respective test
jobs, which can be inspected on the pipeline web page. This test
report counts completed ERT tests, aborted tests are not counted.
* Emba configuration
The emba configuration files are hosted on

7
etc/NEWS
View file

@ -163,6 +163,12 @@ the previous definition to be discarded, which was probably not
intended when this occurs in batch mode. To remedy the error, rename
tests so that they all have unique names.
+++
*** ERT can generate JUnit test reports.
When environment variable 'EMACS_TEST_JUNIT_REPORT' is set, ERT
generates a JUnit test report under this file name. This is useful
for Emacs integration into CI/CD test environments.
** Emoji
+++
@ -1143,6 +1149,7 @@ This variable is bound to t during the preparation of a "*Help*" buffer.
** 'date-to-time' now assumes earliest values if its argument lacks
month, day, or time. For example, (date-to-time "2021-12-04") now
assumes a time of 00:00 instead of signaling an error.
* Changes in Emacs 29.1 on Non-Free Operating Systems

134
lisp/emacs-lisp/ert.el
View file

@ -65,6 +65,8 @@
(require 'pp)
(require 'map)
(autoload 'xml-escape-string "xml.el")
;;; UI customization options.
(defgroup ert ()
@ -247,7 +249,6 @@ in batch mode, an error is signalled.
"%s\\(\\s-\\|$\\)")
"The regexp the `find-function' mechanisms use for finding test definitions.")
(define-error 'ert-test-failed "Test failed")
(define-error 'ert-test-skipped "Test skipped")
@ -677,7 +678,6 @@ and is displayed in front of the value of MESSAGE-FORM."
,@body))
;;; Facilities for running a single test.
(defvar ert-debug-on-error nil
@ -1437,7 +1437,9 @@ Returns the stats object."
(if (getenv "EMACS_TEST_VERBOSE")
(ert-reason-for-test-result result)
""))))
(message "%s" "")))))
(message "%s" ""))
(when (getenv "EMACS_TEST_JUNIT_REPORT")
(ert-write-junit-test-report stats)))))
(test-started)
(test-ended
(cl-destructuring-bind (stats test result) event-args
@ -1525,6 +1527,128 @@ the tests)."
(backtrace))
(kill-emacs 2))))
(defun ert-write-junit-test-report (stats)
"Write a JUnit test report, generated from STATS."
;; https://www.ibm.com/docs/de/developer-for-zos/14.1.0?topic=formats-junit-xml-format
;; https://llg.cubic.org/docs/junit/
(unless (zerop (length (ert--stats-tests stats)))
(when-let ((test-file
(symbol-file
(ert-test-name (aref (ert--stats-tests stats) 0)) 'ert--test)))
(with-temp-file (file-name-with-extension test-file "xml")
(insert "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n")
(insert (format "<testsuites name=\"%s\" tests=\"%s\" failures=\"%s\" skipped=\"%s\" time=\"%s\">\n"
(file-name-nondirectory test-file)
(ert-stats-total stats)
(ert-stats-completed-unexpected stats)
(ert-stats-skipped stats)
(float-time
(time-subtract
(ert--stats-end-time stats)
(ert--stats-start-time stats)))))
(insert (format " <testsuite id=\"0\" name=\"%s\" tests=\"%s\" failures=\"%s\" skipped=\"%s\" time=\"%s\" timestamp=\"%s\">\n"
(file-name-nondirectory test-file)
(ert-stats-total stats)
(ert-stats-completed-unexpected stats)
(ert-stats-skipped stats)
(float-time
(time-subtract
(ert--stats-end-time stats)
(ert--stats-start-time stats)))
(ert--format-time-iso8601 (ert--stats-end-time stats))))
(insert " <properties>\n"
(format " <property name=\"selector\" value=\"%s\"/>\n"
(ert--stats-selector stats))
" </properties>\n")
(cl-loop for test across (ert--stats-tests stats)
for result = (ert-test-most-recent-result test) do
(insert (format " <testcase name=\"%s\" status=\"%s\" time=\"%s\""
(xml-escape-string
(symbol-name (ert-test-name test)))
(ert-string-for-test-result
result
(ert-test-result-expected-p test result))
(ert-test-result-duration result)))
(if (and (ert-test-result-expected-p test result)
(not (ert-test-skipped-p result))
(zerop (length (ert-test-result-messages result))))
(insert "/>\n")
(insert ">\n")
(if (ert-test-skipped-p result)
(insert (format " <skipped message=\"%s\" type=\"%s\">\n"
(xml-escape-string
(string-trim
(ert-reason-for-test-result result)))
(ert-string-for-test-result
result
(ert-test-result-expected-p
test result)))
(xml-escape-string
(string-trim
(ert-reason-for-test-result result)))
"\n"
" </skipped>\n")
(unless
(ert-test-result-type-p
result (ert-test-expected-result-type test))
(insert (format " <failure message=\"%s\" type=\"%s\">\n"
(xml-escape-string
(string-trim
(ert-reason-for-test-result result)))
(ert-string-for-test-result
result
(ert-test-result-expected-p
test result)))
(xml-escape-string
(string-trim
(ert-reason-for-test-result result)))
"\n"
" </failure>\n")))
(unless (zerop (length (ert-test-result-messages result)))
(insert " <system-out>\n"
(xml-escape-string
(ert-test-result-messages result))
" </system-out>\n"))
(insert " </testcase>\n")))
(insert " </testsuite>\n")
(insert "</testsuites>\n")))))
(defun ert-write-junit-test-summary-report (&rest logfiles)
"Write a JUnit summary test report, generated from LOGFILES."
(let ((report (file-name-with-extension
(getenv "EMACS_TEST_JUNIT_REPORT") "xml"))
(tests 0) (failures 0) (skipped 0) (time 0) (id 0))
(with-temp-file report
(dolist (logfile logfiles)
(let ((test-file (file-name-with-extension logfile "xml")))
(when (file-readable-p test-file)
(insert-file-contents-literally test-file)
(when (looking-at-p
(regexp-quote "<?xml version=\"1.0\" encoding=\"utf-8\"?>"))
(delete-region (point) (line-beginning-position 2)))
(when (looking-at
"<testsuites name=\".+\" tests=\"\\(.+\\)\" failures=\"\\(.+\\)\" skipped=\"\\(.+\\)\" time=\"\\(.+\\)\">")
(cl-incf tests (string-to-number (match-string 1)))
(cl-incf failures (string-to-number (match-string 2)))
(cl-incf skipped (string-to-number (match-string 3)))
(cl-incf time (string-to-number (match-string 4)))
(delete-region (point) (line-beginning-position 2)))
(when (looking-at " <testsuite id=\"\\(0\\)\"")
(replace-match (number-to-string id) nil nil nil 1)
(cl-incf id 1))
(goto-char (point-max))
(beginning-of-line 0)
(when (looking-at-p "</testsuites>")
(delete-region (point) (line-beginning-position 2)))
(narrow-to-region (point-max) (point-max)))))
(insert "</testsuites>\n")
(widen)
(goto-char (point-min))
(insert "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n")
(insert (format "<testsuites name=\"%s\" tests=\"%s\" failures=\"%s\" skipped=\"%s\" time=\"%s\">\n"
(file-name-nondirectory report)
tests failures skipped time)))))
(defun ert-summarize-tests-batch-and-exit (&optional high)
"Summarize the results of testing.
@ -1540,6 +1664,8 @@ If HIGH is a natural number, the HIGH long lasting tests are summarized."
;; behavior.
(setq attempt-stack-overflow-recovery nil
attempt-orderly-shutdown-on-fatal-signal nil)
(when (getenv "EMACS_TEST_JUNIT_REPORT")
(apply #'ert-write-junit-test-summary-report command-line-args-left))
(let ((nlogs (length command-line-args-left))
(ntests 0) (nrun 0) (nexpected 0) (nunexpected 0) (nskipped 0)
nnotrun logfile notests badtests unexpected skipped tests)
@ -1855,7 +1981,6 @@ Also sets `ert--results-progress-bar-button-begin'."
;; should test it again.)
"\n")))
(defvar ert-test-run-redisplay-interval-secs .1
"How many seconds ERT should wait between redisplays while running tests.
@ -2037,7 +2162,6 @@ STATS is the stats object; LISTENER is the results listener."
(goto-char (1- (point-max)))
buffer)))))
(defvar ert--selector-history nil
"List of recent test selectors read from terminal.")

1
test/Makefile.in
View file

@ -353,6 +353,7 @@ mostlyclean:
clean:
find . '(' -name '*.log' -o -name '*.log~' ')' $(FIND_DELETE)
find . '(' -name '*.xml' -a ! -path '*resources*' ')' $(FIND_DELETE)
rm -f ${srcdir}/lisp/gnus/mml-sec-resources/random_seed
rm -f $(test_module_dir)/*.o $(test_module_dir)/*.so \
$(test_module_dir)/*.dll

3
test/README
View file

@ -114,6 +114,9 @@ mode--only the names of the failed tests are listed. If the
$EMACS_TEST_VERBOSE environment variable is set, the failure summaries
will also include the data from the failing test.
If the $EMACS_TEST_JUNIT_REPORT environment variable is set to a file
name, a JUnit test report is generated under this name.
Some of the tests require a remote temporary directory
(autorevert-tests.el, filenotify-tests.el, shadowfile-tests.el and
tramp-tests.el). Per default, a mock-up connection method is used

6
test/infra/Makefile.in
View file

@ -93,10 +93,8 @@ all: generate-test-jobs
.PHONY: generate-test-jobs $(FILE) $(SUBDIR_TARGETS)
generate-test-jobs: clean $(FILE) $(SUBDIR_TARGETS)
generate-test-jobs: $(FILE) $(SUBDIR_TARGETS)
$(FILE):
$(AM_V_GEN)
clean:
@rm -f $(FILE)
@echo "# Generated by \"make generate-test-jobs\", don't edit." >$(FILE)

22
test/infra/gitlab-ci.yml
View file

@ -44,6 +44,7 @@ workflow:
variables:
GIT_STRATEGY: fetch
EMACS_EMBA_CI: 1
EMACS_TEST_JUNIT_REPORT: junit-test-report.xml
EMACS_TEST_TIMEOUT: 3600
EMACS_TEST_VERBOSE: 1
# Use TLS https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#tls-enabled
@ -85,7 +86,7 @@ default:
# TODO: with make -j4 several of the tests were failing, for
# example shadowfile-tests, but passed without it.
- 'export PWD=$(pwd)'
- 'docker run -i -e EMACS_EMBA_CI=${EMACS_EMBA_CI} -e EMACS_TEST_TIMEOUT=${EMACS_TEST_TIMEOUT} -e EMACS_TEST_VERBOSE=${EMACS_TEST_VERBOSE} --volumes-from $(docker ps -q -f "label=com.gitlab.gitlab-runner.job.id=${CI_JOB_ID}"):ro --name ${test_name} ${CI_REGISTRY_IMAGE}:${target}-${BUILD_TAG} /bin/bash -c "git fetch ${PWD} HEAD && echo checking out these updated files && git diff --name-only FETCH_HEAD && ( git diff --name-only FETCH_HEAD | xargs git checkout -f FETCH_HEAD ) && make -j4 && make ${make_params}"'
- 'docker run -i -e EMACS_EMBA_CI=${EMACS_EMBA_CI} -e EMACS_TEST_JUNIT_REPORT=${EMACS_TEST_JUNIT_REPORT} -e EMACS_TEST_TIMEOUT=${EMACS_TEST_TIMEOUT} -e EMACS_TEST_VERBOSE=${EMACS_TEST_VERBOSE} --volumes-from $(docker ps -q -f "label=com.gitlab.gitlab-runner.job.id=${CI_JOB_ID}"):ro --name ${test_name} ${CI_REGISTRY_IMAGE}:${target}-${BUILD_TAG} /bin/bash -c "git fetch ${PWD} HEAD && echo checking out these updated files && git diff --name-only FETCH_HEAD && ( git diff --name-only FETCH_HEAD | xargs git checkout -f FETCH_HEAD ) && make -j4 && make ${make_params}"'
after_script:
# - docker ps -a
# - printenv
@ -93,7 +94,8 @@ default:
# Prepare test artifacts.
- test -n "$(docker ps -aq -f name=${test_name})" && docker cp ${test_name}:checkout/test ${test_name}
- test -n "$(docker ps -aq -f name=${test_name})" && docker rm ${test_name}
- find ${test_name} ! -name "*.log" -type f -delete
- find ${test_name} ! \( -name "*.log" -o -name ${EMACS_TEST_JUNIT_REPORT} \) -type f -delete
# BusyBox find does not know -empty.
- find ${test_name} -type d -depth -exec rmdir {} + 2>/dev/null
.build-template:
@ -103,7 +105,6 @@ default:
when: always
- changes:
- "**.in"
- "**.yml"
- GNUmakefile
- aclocal.m4
- autogen.sh
@ -112,7 +113,7 @@ default:
- lib/malloc/*.{h,c}
- lisp/emacs-lisp/*.el
- src/*.{h,c}
- test/infra/Dockerfile.emba
- test/infra/*
- changes:
# gfilemonitor, kqueue
- src/gfilenotify.c
@ -133,9 +134,11 @@ default:
name: ${test_name}
public: true
expire_in: 1 week
when: always
paths:
- ${test_name}/
when: always
reports:
junit: ${test_name}/${EMACS_TEST_JUNIT_REPORT}
.gnustep-template:
rules:
@ -143,12 +146,11 @@ default:
- if: '$CI_PIPELINE_SOURCE == "schedule"'
changes:
- "**.in"
- "**.yml"
- src/ns*.{h,m}
- src/macfont.{h,m}
- lisp/term/ns-win.el
- nextstep/**
- test/infra/Dockerfile.emba
- test/infra/*
.filenotify-gio-template:
rules:
@ -156,12 +158,11 @@ default:
- if: '$CI_PIPELINE_SOURCE == "schedule"'
changes:
- "**.in"
- "**.yml"
- lisp/autorevert.el
- lisp/filenotify.el
- lisp/net/tramp-sh.el
- src/gfilenotify.c
- test/infra/Dockerfile.emba
- test/infra/*
- test/lisp/autorevert-tests.el
- test/lisp/filenotify-tests.el
@ -171,11 +172,10 @@ default:
- if: '$CI_PIPELINE_SOURCE == "schedule"'
changes:
- "**.in"
- "**.yml"
- lisp/emacs-lisp/comp.el
- lisp/emacs-lisp/comp-cstr.el
- src/comp.{h,m}
- test/infra/Dockerfile.emba
- test/infra/*
- test/src/comp-resources/*.el
- test/src/comp-tests.el
timeout: 8 hours

1
test/infra/test-jobs.yml
View file

@ -1,3 +1,4 @@
# Generated by "make generate-test-jobs", don't edit.
test-lib-src-inotify:
stage: normal