.gitlab-ci.yml

---

image: registry.gitlab.com/python-devs/ci-images:main

variables:
  LC_ALL: "C.UTF-8"
  LANG: "C.UTF-8"

  # We store ssh known hosts in the cached directory, and we accept the
  # fingerprint if we don't have a known_host file yet.
  # So humans don't have to cope with fingerprints (file is created as needed).
  # The security is not that bad (while the cache is here, no mitm can occur).
  GIT_SSH_COMMAND: "ssh -o UserKnownHostsFile=$CI_PROJECT_DIR/.cache/known_hosts -o StrictHostKeyChecking=accept-new"

  # Change pip's cache directory to be inside the project directory since we can
  # only cache local items.
  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"

cache:
  paths:
    - .cache/

stages:
  - test
  - refresh

check:
  stage: test
  rules:
    - if: $CI_PIPELINE_SOURCE != "schedule"
  script:
    - python3 -m venv .venv
    - .venv/bin/python -m pip install validators
    - .venv/bin/python scripts/check.py

refresh:
  # This pipeline is triggered by a schedule job at:
  # https://gitlab.adullact.net/dinum/noms-de-domaine-organismes-secteur-public/-/pipeline_schedules
  #
  # It can be started manually using the `Run pipeline` button from:
  # https://gitlab.adullact.net/dinum/noms-de-domaine-organismes-secteur-public/-/pipelines
  # by manually entering those variables:
  # - CI_PIPELINE_SOURCE to schedule
  # - SSH_PRIVATE_KEY to an accepted private key
  stage: refresh
  # We're cloning ourself for multiple reasons:
  # - To avoid permission issues (gitlab would clone as root)
  # - To be on a branch (gitlab would land us in detached head)
  # - To simplify configuration (pushurl would be HTTP, we need SSH)
  variables:
    GIT_STRATEGY: none
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
  before_script:
    # If we need to commit, we'll need a username and an email:
    - git config --global user.name "${GITLAB_USER_NAME}"
    - git config --global user.email "${GITLAB_USER_EMAIL}"

    # Setup of private ssh key in case we need to push:
    - mkdir -p ~/.ssh/
    - chmod 700 ~/.ssh
    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' > ~/.ssh/id_ed25519
    - chmod 600 ~/.ssh/id_ed25519

    # Cloning the repo ourself
    - git clone --depth 1 git@gitlab.adullact.net:dinum/noms-de-domaine-organismes-secteur-public.git
    - cd noms-de-domaine-organismes-secteur-public

    # And while we're at showing debug information:
    - python3 --version

  script:
    - python3 -m venv .venv
    - .venv/bin/python -m pip install aiohttp tqdm
    # Each day we're checking 1/28 of the domains we know.
    # This ensures us that all domains are tested every month.
    # For simplicity, we just don't test on days that may not exist
    # in some months (29, 30, 31).

    - if [[ $(date +%d) -le 28 ]];
      then
          echo "Check some domains over HTTP...";
          .venv/bin/python scripts/http_checker.py --silent --partial $(date +%d)/28;
          echo "What changed:";
          git diff --stat urls.txt;
          CHANGED="$(git diff urls.txt | grep ^-http | cut -d/ -f3)";
          if [[ $CHANGED ]]; then
              echo "Re-check domains that we just considered down";
              echo "just in case they're up now...";
              .venv/bin/python scripts/http_checker.py
                  --silent
                  --slow
                  --partial $(date +%d)/28
                  --grep $CHANGED;
              echo "What changed now:";
              git diff --stat urls.txt;
          fi
      fi
    # Here, if some files has changed, we're committing and pushing
    # back to the repo.
    # For this to work we need a ssh key pair:
    # - The public part is configured as a "deploy key"
    #   with write permission on the repo.
    # - The private part is configured in the CI schedule configuration
    #   so it is **not** available from pull requests.
    - if [[ $(git status --porcelain --untracked-files=no) ]];
      then
          echo "Some files has changed, let's commit and push them.";
          git commit -am "Refreshing 1k domains.";
          git push origin;
      fi