From 84e0749b8b180bb94a5c32ebda11b5f22942dc22 Mon Sep 17 00:00:00 2001 From: Ted Zlatanov Date: Sat, 16 Jan 2021 13:03:58 +0000 Subject: [PATCH] EMBA container build improvements for Emacs build testing. * test/infra/gitlab-ci.yml: Moved from .gitlab-ci.yml. Use the EMBA container registry with a different login token storage file for each commit. Split test stages into prep, build, fast tests, normal tests, platform tests, and slow (everything) and use templates where possible. * .gitlab-ci.yml: Include test/infra/gitlab-ci.yml and move all content there. --- .gitlab-ci.yml | 142 +------------------------ test/infra/Dockerfile.emba | 2 +- test/infra/gitlab-ci.yml | 208 +++++++++++++++++++++++++++++++++++++ 3 files changed, 212 insertions(+), 140 deletions(-) create mode 100644 test/infra/gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eb884767c95..3138f4184e6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -# Copyright (C) 2017-2021 Free Software Foundation, Inc. +# Copyright (C) 2021 Free Software Foundation, Inc. # # This file is part of GNU Emacs. # @@ -24,141 +24,5 @@ # Maintainer: Ted Zlatanov # URL: https://emba.gnu.org/emacs/emacs -# Never run merge request pipelines, they usually duplicate push pipelines -# see https://docs.gitlab.com/ee/ci/yaml/README.html#common-if-clauses-for-rules -workflow: - rules: - - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - when: never - - when: always - -variables: - GIT_STRATEGY: fetch - EMACS_EMBA_CI: 1 - -default: - image: docker:19.03.12 - timeout: 3 hours - before_script: - - docker info - -.job-template: - # these will be cached across builds - cache: - key: ${CI_COMMIT_REF_SLUG} - paths: [] - policy: pull-push - # these will be saved for followup builds - artifacts: - expire_in: 24 hrs - paths: [] - # - "test/**/*.log" - # - "**/*.log" - -.test-template: - rules: - - changes: - - "**/Makefile.in" - - .gitlab-ci.yml - - aclocal.m4 - - autogen.sh - - configure.ac - - lib/*.{h,c} - - lisp/**/*.el - - src/*.{h,c} - - test/infra/* - - test/lisp/**/*.el - - test/src/*.el - - changes: - # gfilemonitor, kqueue - - src/gfilenotify.c - - src/kqueue.c - # MS Windows - - "**/w32*" - # GNUstep - - lisp/term/ns-win.el - - src/ns*.{h,m} - - src/macfont.{h,m} - when: never - - # using the variables for each job - script: - - docker build --target ${target} -t ${target}:${CI_COMMIT_REF_SLUG} -t ${target}:${CI_COMMIT_SHA} -f test/infra/Dockerfile.emba . - # TODO: with make -j4 several of the tests were failing, for example shadowfile-tests, but passed without it - - docker run -i --rm -e EMACS_EMBA_CI=${EMACS_EMBA_CI} ${target}:${CI_COMMIT_SHA} make ${make_params} - -stages: - - fast - - normal - - slow - -test-fast: - stage: fast - extends: [.job-template, .test-template] - variables: - target: emacs-inotify - make_params: "-C test check" - -test-lisp: - stage: normal - extends: [.job-template, .test-template] - variables: - target: emacs-inotify - make_params: "-C test check-lisp" - -test-net: - stage: normal - extends: [.job-template, .test-template] - variables: - target: emacs-inotify - make_params: "-C test check-net" - -test-filenotify-gio: - # This tests file monitor libraries gfilemonitor and gio. - stage: normal - extends: [.job-template, .test-template] - rules: - - if: '$CI_PIPELINE_SOURCE == "schedule"' - changes: - - "**/Makefile.in" - - .gitlab-ci.yml - - lisp/autorevert.el - - lisp/filenotify.el - - lisp/net/tramp-sh.el - - src/gfilenotify.c - - test/infra/* - - test/lisp/autorevert-tests.el - - test/lisp/filenotify-tests.el - variables: - target: emacs-filenotify-gio - make_params: "-k -C test autorevert-tests filenotify-tests" - -test-gnustep: - # This tests the GNUstep build process - stage: normal - extends: [.job-template, .test-template] - rules: - - if: '$CI_PIPELINE_SOURCE == "schedule"' - changes: - - "**/Makefile.in" - - .gitlab-ci.yml - - configure.ac - - src/ns*.{h,m} - - src/macfont.{h,m} - - lisp/term/ns-win.el - - nextstep/**/* - - test/infra/* - variables: - target: emacs-gnustep - make_params: install - -test-all: - # This tests also file monitor libraries inotify and inotifywatch. - stage: slow - extends: [.job-template, .test-template] - rules: - # note there's no "changes" section, so this always runs on a schedule - - if: '$CI_PIPELINE_SOURCE == "schedule"' - variables: - target: emacs-inotify - make_params: check-expensive +# Just load from test/infra, to keep build automation files there. +include: '/test/infra/gitlab-ci.yml' diff --git a/test/infra/Dockerfile.emba b/test/infra/Dockerfile.emba index dd41982ad59..421264db9c9 100644 --- a/test/infra/Dockerfile.emba +++ b/test/infra/Dockerfile.emba @@ -41,7 +41,7 @@ COPY . /checkout WORKDIR /checkout RUN ./autogen.sh autoconf RUN ./configure --without-makeinfo -RUN make bootstrap +RUN make -j4 bootstrap RUN make -j4 FROM emacs-base as emacs-filenotify-gio diff --git a/test/infra/gitlab-ci.yml b/test/infra/gitlab-ci.yml new file mode 100644 index 00000000000..d8934551b00 --- /dev/null +++ b/test/infra/gitlab-ci.yml @@ -0,0 +1,208 @@ +# Copyright (C) 2017-2021 Free Software Foundation, Inc. +# +# This file is part of GNU Emacs. +# +# GNU Emacs is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# GNU Emacs is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with GNU Emacs. If not, see . + +# GNU Emacs support for the GitLab protocol for CI + +# The presence of this file does not imply any FSF/GNU endorsement of +# any particular service that uses that protocol. Also, it is intended for +# evaluation purposes, thus possibly temporary. + +# Maintainer: Ted Zlatanov +# URL: https://emba.gnu.org/emacs/emacs + +# Never run merge request pipelines, they usually duplicate push pipelines +# see https://docs.gitlab.com/ee/ci/yaml/README.html#common-if-clauses-for-rules +workflow: + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: never + - when: always + +variables: + GIT_STRATEGY: fetch + EMACS_EMBA_CI: 1 + # # Use TLS https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#tls-enabled + # DOCKER_HOST: tcp://docker:2376 + # DOCKER_TLS_CERTDIR: "/certs" + # Put the configuration for each run in a separate directory to avoid conflicts + DOCKER_CONFIG: "/.docker-config-${CI_COMMIT_SHA}" + +default: + image: docker:19.03.12 + timeout: 3 hours + before_script: + - docker info + - echo "docker registry is ${CI_REGISTRY}" + - docker login -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY} + +.job-template: + # these will be cached across builds + cache: + key: ${CI_COMMIT_SHA} + paths: [] + policy: pull-push + # these will be saved for followup builds + artifacts: + expire_in: 24 hrs + paths: [] + # - "test/**/*.log" + # - "**/*.log" + +.build-template: + script: + - docker build --pull --target ${target} -t ${CI_REGISTRY_IMAGE}:${target}-${CI_COMMIT_SHA} -f test/infra/Dockerfile.emba . + - docker push ${CI_REGISTRY_IMAGE}:${target}-${CI_COMMIT_SHA} + +.gnustep-template: + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule"' + changes: + - "**/Makefile.in" + - .gitlab-ci.yml + - configure.ac + - src/ns*.{h,m} + - src/macfont.{h,m} + - lisp/term/ns-win.el + - nextstep/**/* + - test/infra/* + +.filenotify-gio-template: + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule"' + changes: + - "**/Makefile.in" + - .gitlab-ci.yml + - lisp/autorevert.el + - lisp/filenotify.el + - lisp/net/tramp-sh.el + - src/gfilenotify.c + - test/infra/* + - test/lisp/autorevert-tests.el + - test/lisp/filenotify-tests.el + +.test-template: + rules: + - changes: + - "**/Makefile.in" + - .gitlab-ci.yml + - aclocal.m4 + - autogen.sh + - configure.ac + - lib/*.{h,c} + - lisp/**/*.el + - src/*.{h,c} + - test/infra/* + - test/lisp/**/*.el + - test/src/*.el + - changes: + # gfilemonitor, kqueue + - src/gfilenotify.c + - src/kqueue.c + # MS Windows + - "**/w32*" + # GNUstep + - lisp/term/ns-win.el + - src/ns*.{h,m} + - src/macfont.{h,m} + when: never + + # using the variables for each job + script: + - docker pull ${CI_REGISTRY_IMAGE}:${target}-${CI_COMMIT_SHA} + # TODO: with make -j4 several of the tests were failing, for example shadowfile-tests, but passed without it + - docker run -i --rm -e EMACS_EMBA_CI=${EMACS_EMBA_CI} ${CI_REGISTRY_IMAGE}:${target}-${CI_COMMIT_SHA} make ${make_params} + +stages: + - prep-images + - build-images + - fast + - normal + - platform-images + - platforms + - slow + +prep-image-base: + stage: prep-images + extends: [.job-template, .build-template] + variables: + target: emacs-base + +build-image-inotify: + stage: build-images + extends: [.job-template, .build-template] + variables: + target: emacs-inotify + +test-fast-inotify: + stage: fast + extends: [.job-template, .test-template] + variables: + target: emacs-inotify + make_params: "-C test check" + +build-image-filenotify-gio: + stage: platform-images + extends: [.job-template, .build-template, .filenotify-gio-template] + variables: + target: emacs-filenotify-gio + +build-image-gnustep: + stage: platform-images + extends: [.job-template, .build-template, .gnustep-template] + variables: + target: emacs-gnustep + +test-lisp-inotify: + stage: normal + extends: [.job-template, .test-template] + variables: + target: emacs-inotify + make_params: "-C test check-lisp" + +test-net-inotify: + stage: normal + extends: [.job-template, .test-template] + variables: + target: emacs-inotify + make_params: "-C test check-net" + +test-filenotify-gio: + # This tests file monitor libraries gfilemonitor and gio. + stage: platforms + extends: [.job-template, .test-template, .filenotify-gio-template] + variables: + target: emacs-filenotify-gio + make_params: "-k -C test autorevert-tests filenotify-tests" + +test-gnustep: + # This tests the GNUstep build process + stage: platforms + extends: [.job-template, .test-template, .gnustep-template] + variables: + target: emacs-gnustep + make_params: install + +test-all-inotify: + # This tests also file monitor libraries inotify and inotifywatch. + stage: slow + extends: [.job-template, .test-template] + rules: + # note there's no "changes" section, so this always runs on a schedule + - if: '$CI_PIPELINE_SOURCE == "schedule"' + variables: + target: emacs-inotify + make_params: check-expensive