.gitlab-ci.yml

# This file configures automatic builds of Sage on [GitLab](https://gitlab.com).
# To make the build time not too excessive, we seed the build cache with
# sagemath/sagemath-dev:develop. When basic SPKGs changed, this does not help
# much and the full build might exceed the set time limit in GitLab.  You can
# increase that limit in Settings → CI/CD.
# You can also provision your own private more powerful runner in the same
# place
# https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#use-docker-in-docker-executor;
# or set up your favourite cloud service to provide an on-demand autoscale
# runner. More details below.

# As of early 2018 a run on GitLab CI takes about 45 minutes. We could probably
# save 10 minutes by not building/pushing/testing dev images for branches other
# than master/develop.

# Note that most of the time during CI is spent with pulling and pushing of
# docker images and copying files locally as part of the docker build.  At the
# moment there is no reliable way of passing the docker images to the following
# stages without explicit pushing/pulling or similar:
# https://gitlab.com/gitlab-org/gitlab-runner/issues/1107

# The timings mentioned above are typical values. The shared runners provided
# on gitlab.com are sometimes much slower depending on the runner you are
# scheduled on. Sometimes it's slower for no apparent reason, probably just an
# overcommittment of virtual machines on hardware.

# GitLab provides several flavours of shared runners (as of early 2018):
# * runners tagged as "do" (digitalocean.com) provide about 60GB of HDD, two
#   cores, but only 2GB of RAM. The RAM is sometimes not sufficient to build
#   the documentation.
# * runners tagged as "gce" (Google Compute Engine) provide about 22GB of HDD,
#   a single core, 4GB of RAM. Since we are relying on OverlayFS, the disk
#   space is not sufficient to build sage from scratch.
# The shared runners are terminated after three hours. Currently, this is often
# insufficient to build sage from scratch.

# If you want to provide your own runners, make sure to tag them as follows:
# * "big" (60GB of disk space are available) to make build-from-clean pass.

image: docker:stable

stages:
  - build
  - test
  - release

variables:
  DOCKER_TAG: $CI_COMMIT_REF_NAME
  # Builds are very I/O intensive; make sure we have a fast file system.
  DOCKER_DRIVER: overlay2
  DEFAULT_ARTIFACT_BASE: sagemath/sagemath-dev:develop

before_script:
  # GitLab has no mechanism yet to hide secret variables: https://gitlab.com/gitlab-org/gitlab-ce/issues/13784
  # So we roll our own which protects all variables that start with SECRET_
  - . .ci/protect-secrets.sh
  # Collect debug infos about the system we are running on
  - .ci/describe-system.sh
  # Set DOCKER_TAG according to the current branch/tag
  - . .ci/update-env.sh
  # Set MAKEFLAGS and SAGE_NUM_THREADS according to the machine we are running on
  - . .ci/setup-make-parallelity.sh

# We use docker-in-docker to build our docker images, i.e., we run a
# docker:dind "service" container and link to it from the container running the
# actual scripts below.
# Our scripts automatically connect to this service (unless you override it by
# setting DOCKER_HOST.) For example, each RUN statement in the Dockerfile
# spawns a docker container inside the docker:dind container to perform the RUN
# command there.
# It can be faster to expose your outer docker daemon by mounting
# /var/run/docker.sock to /var/run/docker.sock and setting DOCKER_HOST in
# Settings -> CI/CD -> Secret variable to unix:///var/run/docker.sock. (The
# speedup is mostly due to sharing layers of intermediate images.) However,
# this is only possible if you provision your own runners. Shared gitlab
# runners, do not bind mount /var/run/docker.sock. Also, docker:dind provides
# better isolation. If you expect many builds to run simultaneously on a host,
# conflicting tags can cause issues with a mounted DOCKER_HOST.
services:
- docker:stable-dind

# Build Sage and its documentation.
# The build starts from the build artifacts of DEFAULT_ARTIFACT_BASE which is
# usually much faster than building from a clean checkout of Sage.
build-from-latest:
  stage: build
  artifacts:
    when: always
    paths:
    - gitlab-build-docker.log
    - html
    expire_in: 1 month
  script:
    - apk --update add coreutils rsync
    # The output of the build can get larger than gitlab.com's limit; only
    # print the first 1MB (and the last 80 lines.) GitLab's limit is 4MB,
    # however, the list of all branches and tags that shows up in the initial
    # checkout takes already 1.5 MB:
    # https://gitlab.com/gitlab-org/gitlab-runner/issues/4142
    - .ci/build-docker.sh | tee gitlab-build-docker.log | .ci/head-tail.sh 1048576
    - .ci/push-gitlab.sh sagemath-dev
    - .ci/push-gitlab.sh sagemath
    - DOCKER_TAG=$CI_COMMIT_SHA .ci/push-gitlab.sh sagemath
  except:
    - master
    - develop
    - tags
    - web

# Build Sage and its documentation from a clean checkout of Sage.
# Note that this takes several hours. You probably want to run this on your own
# gitlab-runner and increase the standard GitLab time limit for CI runs.
# Some of the shared runners provided by GitLab for free do not have enough
# disk space for this to work. If a build fails with "no space left on device",
# you could just retry it and hope to be scheduled on a machine with more disk
# space, or provision your own runner.
build-from-clean:
  extends:
    - build-from-latest
  artifacts:
    when: always
    paths:
    - gitlab-build-docker.log
    - html
    expire_in: 99 years
  variables:
    ARTIFACT_BASE: "source-clean"
  only:
    - master
    - develop
    - tags
    # Run build-from-clean for a pipeline that has been explicitly created
    # through GitLab's web interface.
    - web
  except: []
  tags:
    # 60 GB of HDD are available
    - big
  # This build takes several CPU hours. It is very unlikely that there are any
  # actual build errors for a tagged release but the (discounted) cloud
  # machines this is running on might be preempted during the long build time.
  # So let's try three times before we give up.
  retry: 2

test-dev:
  stage: test
  dependencies: []
  script:
    - . .ci/pull-gitlab.sh sagemath-dev
    - sh .ci/test-dev.sh "$DOCKER_IMAGE"

test-cli:
  stage: test
  dependencies: []
  script:
    - . .ci/pull-gitlab.sh sagemath
    - sh .ci/test-cli.sh "$DOCKER_IMAGE"

test-jupyter:
  stage: test
  dependencies: []
  script:
    - . .ci/pull-gitlab.sh sagemath
    - sh .ci/test-jupyter.sh "$DOCKER_IMAGE" docker

# Pushes the built images to Docker Hub if the Settings -> CI/CD -> Secret
# variables DOCKER_USER and SECRET_DOCKER_PASS have been set up.
push-dockerhub:
  stage: release
  dependencies: []
  only:
    refs:
      - branches
      - tags
    variables:
      - $SECRET_DOCKER_PASS
  script:
    - . .ci/pull-gitlab.sh sagemath
    - sh .ci/push-dockerhub.sh sagemath

# Pushes the built dev images to Docker Hub if the Settings -> CI/CD -> Secret
# variables DOCKER_USER and SECRET_DOCKER_PASS have been set up.
push-dockerhub-dev:
  stage: release
  dependencies: []
  only:
    refs:
      - master
      - develop
      - tags
    variables:
      - $SECRET_DOCKER_PASS
  script:
    - . .ci/pull-gitlab.sh sagemath-dev
    - sh .ci/push-dockerhub.sh sagemath-dev