[libvirt PATCH 3/5] ci: Use GitLab container registry
Daniel P. Berrangé
berrange at redhat.com
Tue Jun 2 10:33:23 UTC 2020
On Fri, May 29, 2020 at 03:00:42PM +0200, Andrea Bolognani wrote:
> Instead of using pre-built containers hosted on Quay, build
> containers as part of the GitLab CI pipeline and upload them to the
> GitLab container registry for later use.
>
> This will not significantly slow down builds, because containers are
> only rebuilt when the corresponding Dockerfile has been modified.
>
> Signed-off-by: Andrea Bolognani <abologna at redhat.com>
> ---
> .gitlab-ci.yml | 234 +++++++++++++++++-
> ci/containers/README.rst | 14 ++
> ci/containers/ci-centos-7.Dockerfile | 137 ++++++++++
> ci/containers/ci-centos-8.Dockerfile | 108 ++++++++
> .../ci-debian-10-cross-aarch64.Dockerfile | 122 +++++++++
> .../ci-debian-10-cross-armv6l.Dockerfile | 120 +++++++++
> .../ci-debian-10-cross-armv7l.Dockerfile | 121 +++++++++
> .../ci-debian-10-cross-i686.Dockerfile | 121 +++++++++
> .../ci-debian-10-cross-mips.Dockerfile | 121 +++++++++
> .../ci-debian-10-cross-mips64el.Dockerfile | 121 +++++++++
> .../ci-debian-10-cross-mipsel.Dockerfile | 121 +++++++++
> .../ci-debian-10-cross-ppc64le.Dockerfile | 121 +++++++++
> .../ci-debian-10-cross-s390x.Dockerfile | 121 +++++++++
> ci/containers/ci-debian-10.Dockerfile | 112 +++++++++
> .../ci-debian-9-cross-aarch64.Dockerfile | 126 ++++++++++
> .../ci-debian-9-cross-armv6l.Dockerfile | 124 ++++++++++
> .../ci-debian-9-cross-armv7l.Dockerfile | 125 ++++++++++
> .../ci-debian-9-cross-mips.Dockerfile | 125 ++++++++++
> .../ci-debian-9-cross-mips64el.Dockerfile | 125 ++++++++++
> .../ci-debian-9-cross-mipsel.Dockerfile | 125 ++++++++++
> .../ci-debian-9-cross-ppc64le.Dockerfile | 125 ++++++++++
> .../ci-debian-9-cross-s390x.Dockerfile | 125 ++++++++++
> ci/containers/ci-debian-9.Dockerfile | 116 +++++++++
> .../ci-debian-sid-cross-aarch64.Dockerfile | 122 +++++++++
> .../ci-debian-sid-cross-armv6l.Dockerfile | 120 +++++++++
> .../ci-debian-sid-cross-armv7l.Dockerfile | 121 +++++++++
> .../ci-debian-sid-cross-i686.Dockerfile | 121 +++++++++
> .../ci-debian-sid-cross-mips.Dockerfile | 121 +++++++++
> .../ci-debian-sid-cross-mips64el.Dockerfile | 121 +++++++++
> .../ci-debian-sid-cross-mipsel.Dockerfile | 120 +++++++++
> .../ci-debian-sid-cross-ppc64le.Dockerfile | 121 +++++++++
> .../ci-debian-sid-cross-s390x.Dockerfile | 121 +++++++++
> ci/containers/ci-debian-sid.Dockerfile | 112 +++++++++
> ci/containers/ci-fedora-31.Dockerfile | 109 ++++++++
> ci/containers/ci-fedora-32.Dockerfile | 109 ++++++++
> ...ci-fedora-rawhide-cross-mingw32.Dockerfile | 129 ++++++++++
> ...ci-fedora-rawhide-cross-mingw64.Dockerfile | 129 ++++++++++
> ci/containers/ci-fedora-rawhide.Dockerfile | 110 ++++++++
> ci/containers/ci-opensuse-151.Dockerfile | 109 ++++++++
> ci/containers/ci-ubuntu-1804.Dockerfile | 117 +++++++++
> ci/containers/ci-ubuntu-2004.Dockerfile | 113 +++++++++
> ci/containers/refresh | 43 ++++
> 42 files changed, 4973 insertions(+), 5 deletions(-)
> create mode 100644 ci/containers/README.rst
> create mode 100644 ci/containers/ci-centos-7.Dockerfile
> create mode 100644 ci/containers/ci-centos-8.Dockerfile
> create mode 100644 ci/containers/ci-debian-10-cross-aarch64.Dockerfile
> create mode 100644 ci/containers/ci-debian-10-cross-armv6l.Dockerfile
> create mode 100644 ci/containers/ci-debian-10-cross-armv7l.Dockerfile
> create mode 100644 ci/containers/ci-debian-10-cross-i686.Dockerfile
> create mode 100644 ci/containers/ci-debian-10-cross-mips.Dockerfile
> create mode 100644 ci/containers/ci-debian-10-cross-mips64el.Dockerfile
> create mode 100644 ci/containers/ci-debian-10-cross-mipsel.Dockerfile
> create mode 100644 ci/containers/ci-debian-10-cross-ppc64le.Dockerfile
> create mode 100644 ci/containers/ci-debian-10-cross-s390x.Dockerfile
> create mode 100644 ci/containers/ci-debian-10.Dockerfile
> create mode 100644 ci/containers/ci-debian-9-cross-aarch64.Dockerfile
> create mode 100644 ci/containers/ci-debian-9-cross-armv6l.Dockerfile
> create mode 100644 ci/containers/ci-debian-9-cross-armv7l.Dockerfile
> create mode 100644 ci/containers/ci-debian-9-cross-mips.Dockerfile
> create mode 100644 ci/containers/ci-debian-9-cross-mips64el.Dockerfile
> create mode 100644 ci/containers/ci-debian-9-cross-mipsel.Dockerfile
> create mode 100644 ci/containers/ci-debian-9-cross-ppc64le.Dockerfile
> create mode 100644 ci/containers/ci-debian-9-cross-s390x.Dockerfile
> create mode 100644 ci/containers/ci-debian-9.Dockerfile
> create mode 100644 ci/containers/ci-debian-sid-cross-aarch64.Dockerfile
> create mode 100644 ci/containers/ci-debian-sid-cross-armv6l.Dockerfile
> create mode 100644 ci/containers/ci-debian-sid-cross-armv7l.Dockerfile
> create mode 100644 ci/containers/ci-debian-sid-cross-i686.Dockerfile
> create mode 100644 ci/containers/ci-debian-sid-cross-mips.Dockerfile
> create mode 100644 ci/containers/ci-debian-sid-cross-mips64el.Dockerfile
> create mode 100644 ci/containers/ci-debian-sid-cross-mipsel.Dockerfile
> create mode 100644 ci/containers/ci-debian-sid-cross-ppc64le.Dockerfile
> create mode 100644 ci/containers/ci-debian-sid-cross-s390x.Dockerfile
> create mode 100644 ci/containers/ci-debian-sid.Dockerfile
> create mode 100644 ci/containers/ci-fedora-31.Dockerfile
> create mode 100644 ci/containers/ci-fedora-32.Dockerfile
> create mode 100644 ci/containers/ci-fedora-rawhide-cross-mingw32.Dockerfile
> create mode 100644 ci/containers/ci-fedora-rawhide-cross-mingw64.Dockerfile
> create mode 100644 ci/containers/ci-fedora-rawhide.Dockerfile
> create mode 100644 ci/containers/ci-opensuse-151.Dockerfile
> create mode 100644 ci/containers/ci-ubuntu-1804.Dockerfile
> create mode 100644 ci/containers/ci-ubuntu-2004.Dockerfile
> create mode 100755 ci/containers/refresh
>
> diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
> index 8a5b3372de..0e7917d6cd 100644
> --- a/.gitlab-ci.yml
> +++ b/.gitlab-ci.yml
> @@ -3,6 +3,7 @@ variables:
>
> stages:
> - prebuild
> + - containers
> - native_build
> - cross_build
> - other
> @@ -16,10 +17,35 @@ stages:
>
> # Common templates
>
> +.container_job_template: &container_job_definition
> + image: docker:stable
> + stage: containers
> + services:
> + - docker:dind
> + before_script:
> + - export TAG="$CI_REGISTRY_IMAGE/ci-$NAME:latest"
> + - export COMMON_TAG="$CI_REGISTRY/libvirt/libvirt/ci-$NAME:latest"
> + - docker info
> + - docker login registry.gitlab.com -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD"
> + script:
> + - docker pull "$TAG" || docker pull "$COMMON_TAG" || true
> + - docker build --cache-from "$TAG" --cache-from "$COMMON_TAG" --tag "$TAG" -f "ci/containers/ci-$NAME.Dockerfile" ci/containers
> + - docker push "$TAG"
> + after_script:
> + - docker logout
> +
> +# We build many containers which can be useful to debug problems but are not
> +# needed for the pipeline itself to complete: those sometimes fail, and when
> +# that happens it's mostly because of temporary issues with Debian sid. We
> +# don't want those failures to affect the overall pipeline status
> +.container_optional_job_template: &container_optional_job_definition
> + <<: *container_job_definition
> + allow_failure: true
I don't think we should be building container images that we're not going
to be using in any of the jobs, as it can only ever slow down the build
overall.
> +
> # Default native build jobs that are always run
> .native_build_default_job_template: &native_build_default_job_definition
> stage: native_build
> - image: quay.io/libvirt/buildenv-libvirt-$NAME:latest
> + image: $CI_REGISTRY_IMAGE/ci-$NAME:latest
> cache:
> paths:
> - ccache/
> @@ -44,7 +70,7 @@ stages:
> # Default cross build jobs that are always run
> .cross_build_default_job_template: &cross_build_default_job_definition
> stage: cross_build
> - image: quay.io/libvirt/buildenv-libvirt-$NAME-cross-$CROSS:latest
> + image: $CI_REGISTRY_IMAGE/ci-$NAME-cross-$CROSS:latest
> cache:
> paths:
> - ccache/
> @@ -66,6 +92,204 @@ stages:
> - /^ci-full-.*$/
>
>
> +# Container build jobs
> +
> +centos-7-container:
IMHO we should name these to match the build job. eg arch, then distro
x64-centos-7-container
> + <<: *container_job_definition
> + variables:
> + NAME: centos-7
> +
> +centos-8-container:
> + <<: *container_job_definition
> + variables:
> + NAME: centos-8
> +
> +debian-9-container:
> + <<: *container_job_definition
> + variables:
> + NAME: debian-9
> +
> +debian-9-cross-aarch64-container:
> + <<: *container_optional_job_definition
> + variables:
> + NAME: debian-9-cross-aarch64
> +
> +debian-9-cross-armv6l-container:
> + <<: *container_job_definition
> + variables:
> + NAME: debian-9-cross-armv6l
This container, and many others are only used by the "extra" build
jobs, so should be subject to the same filtering.
> +
> +debian-9-cross-armv7l-container:
> + <<: *container_optional_job_definition
> + variables:
> + NAME: debian-9-cross-armv7l
> +
> +debian-9-cross-mips-container:
> + <<: *container_job_definition
> + variables:
> + NAME: debian-9-cross-mips
> +
> +debian-9-cross-mips64el-container:
> + <<: *container_job_definition
> + variables:
> + NAME: debian-9-cross-mips64el
> +
> +debian-9-cross-mipsel-container:
> + <<: *container_optional_job_definition
> + variables:
> + NAME: debian-9-cross-mipsel
> +
> +debian-9-cross-ppc64le-container:
> + <<: *container_optional_job_definition
> + variables:
> + NAME: debian-9-cross-ppc64le
> +
> +debian-9-cross-s390x-container:
> + <<: *container_optional_job_definition
> + variables:
> + NAME: debian-9-cross-s390x
> +
> +debian-10-container:
> + <<: *container_job_definition
> + variables:
> + NAME: debian-10
> +
> +debian-10-cross-aarch64-container:
> + <<: *container_job_definition
> + variables:
> + NAME: debian-10-cross-aarch64
> +
> +debian-10-cross-armv6l-container:
> + <<: *container_optional_job_definition
> + variables:
> + NAME: debian-10-cross-armv6l
> +
> +debian-10-cross-armv7l-container:
> + <<: *container_optional_job_definition
> + variables:
> + NAME: debian-10-cross-armv7l
> +
> +debian-10-cross-i686-container:
> + <<: *container_optional_job_definition
> + variables:
> + NAME: debian-10-cross-i686
> +
> +debian-10-cross-mips-container:
> + <<: *container_optional_job_definition
> + variables:
> + NAME: debian-10-cross-mips
> +
> +debian-10-cross-mips64el-container:
> + <<: *container_optional_job_definition
> + variables:
> + NAME: debian-10-cross-mips64el
> +
> +debian-10-cross-mipsel-container:
> + <<: *container_optional_job_definition
> + variables:
> + NAME: debian-10-cross-mipsel
> +
> +debian-10-cross-ppc64le-container:
> + <<: *container_job_definition
> + variables:
> + NAME: debian-10-cross-ppc64le
> +
> +debian-10-cross-s390x-container:
> + <<: *container_job_definition
> + variables:
> + NAME: debian-10-cross-s390x
> +
> +debian-sid-container:
> + <<: *container_job_definition
> + variables:
> + NAME: debian-sid
> +
> +debian-sid-cross-aarch64-container:
> + <<: *container_optional_job_definition
> + variables:
> + NAME: debian-sid-cross-aarch64
> +
> +debian-sid-cross-armv6l-container:
> + <<: *container_optional_job_definition
> + variables:
> + NAME: debian-sid-cross-armv6l
> +
> +debian-sid-cross-armv7l-container:
> + <<: *container_job_definition
> + variables:
> + NAME: debian-sid-cross-armv7l
> +
> +debian-sid-cross-i686-container:
> + <<: *container_job_definition
> + variables:
> + NAME: debian-sid-cross-i686
> +
> +debian-sid-cross-mips-container:
> + <<: *container_optional_job_definition
> + variables:
> + NAME: debian-sid-cross-mips
> +
> +debian-sid-cross-mips64el-container:
> + <<: *container_optional_job_definition
> + variables:
> + NAME: debian-sid-cross-mips64el
> +
> +debian-sid-cross-mipsel-container:
> + <<: *container_job_definition
> + variables:
> + NAME: debian-sid-cross-mipsel
> +
> +debian-sid-cross-ppc64le-container:
> + <<: *container_optional_job_definition
> + variables:
> + NAME: debian-sid-cross-ppc64le
> +
> +debian-sid-cross-s390x-container:
> + <<: *container_optional_job_definition
> + variables:
> + NAME: debian-sid-cross-s390x
> +
> +fedora-31-container:
> + <<: *container_job_definition
> + variables:
> + NAME: fedora-31
> +
> +fedora-32-container:
> + <<: *container_job_definition
> + variables:
> + NAME: fedora-32
> +
> +fedora-rawhide-container:
> + <<: *container_job_definition
> + variables:
> + NAME: fedora-rawhide
> +
> +fedora-rawhide-cross-mingw32-container:
> + <<: *container_job_definition
> + variables:
> + NAME: fedora-rawhide-cross-mingw32
> +
> +fedora-rawhide-cross-mingw64-container:
> + <<: *container_job_definition
> + variables:
> + NAME: fedora-rawhide-cross-mingw64
> +
> +opensuse-151-container:
> + <<: *container_job_definition
> + variables:
> + NAME: opensuse-151
> + RPM: skip
> +
> +ubuntu-1804-container:
> + <<: *container_job_definition
> + variables:
> + NAME: ubuntu-1804
> +
> +ubuntu-2004-container:
> + <<: *container_job_definition
> + variables:
> + NAME: ubuntu-2004
> +
> # Native architecture build + test jobs
>
> x64-debian-9:
> @@ -198,6 +422,7 @@ mingw64-fedora-rawhide:
> # https://gitlab.com/libvirt/libvirt/-/jobs/artifacts/master/download?job=website
> website:
> stage: other
> + image: $CI_REGISTRY_IMAGE/ci-centos-8:latest
> before_script:
> - *script_variables
> script:
> @@ -208,7 +433,6 @@ website:
> - $MAKE -C docs install
> - cd ..
> - mv vroot/share/doc/libvirt/html/ website
> - image: quay.io/libvirt/buildenv-libvirt-centos-8:latest
> artifacts:
> expose_as: 'Website'
> name: 'website'
> @@ -220,6 +444,7 @@ website:
>
> codestyle:
> stage: other
> + image: $CI_REGISTRY_IMAGE/ci-centos-8:latest
> before_script:
> - *script_variables
> script:
> @@ -227,7 +452,6 @@ codestyle:
> - cd build
> - ../autogen.sh || (cat config.log && exit 1)
> - $MAKE syntax-check
> - image: quay.io/libvirt/buildenv-libvirt-centos-8:latest
>
>
> # This artifact published by this job is downloaded to push to Weblate
> @@ -235,6 +459,7 @@ codestyle:
> # https://gitlab.com/libvirt/libvirt/-/jobs/artifacts/master/download?job=potfile
> potfile:
> stage: prebuild
> + image: $CI_REGISTRY_IMAGE/ci-centos-8:latest
> only:
> - master
> before_script:
> @@ -247,7 +472,6 @@ potfile:
> - $MAKE -C po libvirt.pot
> - cd ..
> - mv build/po/libvirt.pot libvirt.pot
> - image: quay.io/libvirt/buildenv-libvirt-centos-8:latest
> artifacts:
> expose_as: 'Potfile'
> name: 'potfile'
Regards,
Daniel
--
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org -o- https://fstop138.berrange.com :|
|: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
More information about the libvir-list
mailing list