.gitlab-ci.yml

# default installed image for docker executor is: python:3.6
# using an image that can do git, docker, docker-compose
image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/ci-runner:latest

# build directory inside
# https://gitlab.mpcdf.mpg.de/help/ci/runners/configure_runners.md#custom-build-directories

# https://docs.gitlab.com/ee/ci/yaml/workflow.html
# https://docs.gitlab.com/ee/ci/variables/predefined_variables.html
# if: CI_COMMIT_BRANCH && CI_COMMIT_BEFORE_SHA == "0000000000000000000000000000000000000000"
# A branch pipeline, but it is the first commit for that branch
# if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push"
# For an existing workflow section to switch from branch pipelines to merge request pipelines when a merge request is created.
# if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
# A branch pipeline, but a merge request is open for that branch, do not run the branch pipeline.
# if: $CI_PIPELINE_SOURCE == "merge_request_event"
# A merge request pipeline, start the pipeline.
# if: $CI_COMMIT_BRANCH
# A branch pipeline, but there is no merge request open for the branch, run the branch pipeline.

default:
  tags:
    # Necessary to select the right CI runner
    - cloud

variables:
  DOCKER_TAG: ${CI_COMMIT_REF_SLUG}
  DOCKER_HOST: tcp://docker-dind:2375
  DOCKER_TLS_CERTDIR: ""

workflow:
  rules:
    - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BEFORE_SHA == "0000000000000000000000000000000000000000"
      when: never
    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push"
      when: never
    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
      when: never
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_BRANCH
    - if: $CI_COMMIT_TAG
      variables:
        DOCKER_TAG: ${CI_COMMIT_REF_NAME}
    - when: never

stages:
  - build
  - test
  - deploy
  - release

update changelog:
  stage: build
  script:
    - curl -X POST "https://gitlab.mpcdf.mpg.de/api/v4/projects/$CI_PROJECT_ID/repository/changelog?version=${CI_COMMIT_TAG:1}&access_token=$CI_ACCESS_TOKEN&message=Add%20changelog%20for%20version%20$CI_COMMIT_TAG%20%5Bskip-ci%5D"
  rules:
    - if: $CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+$/

build:
  stage: build
  variables:
    GIT_SUBMODULE_STRATEGY: recursive
    GIT_SUBMODULE_UPDATE_FLAGS: --jobs 4
    DOCKER_BUILDKIT: 1
  before_script:
    - docker --version
    - echo "${CI_REGISTRY_PASSWORD}" | docker login ${CI_REGISTRY} --username ${CI_REGISTRY_USER} --password-stdin
    - pip install --upgrade setuptools_scm
    - export SETUPTOOLS_SCM_PRETEND_VERSION=$(python -m setuptools_scm)
    - echo "SETUPTOOLS_SCM_PRETEND_VERSION=$SETUPTOOLS_SCM_PRETEND_VERSION"
  script:
    - docker build --build-arg SETUPTOOLS_SCM_PRETEND_VERSION -t ${CI_REGISTRY_IMAGE}:${DOCKER_TAG} .
    - docker build --build-arg SETUPTOOLS_SCM_PRETEND_VERSION --target dev_node -t ${CI_REGISTRY_IMAGE}/dev_node:${DOCKER_TAG}  .
    - docker build --build-arg SETUPTOOLS_SCM_PRETEND_VERSION --target dev_python -t ${CI_REGISTRY_IMAGE}/dev_python:${DOCKER_TAG}  .
    - docker push ${CI_REGISTRY_IMAGE}:${DOCKER_TAG}
    - docker push ${CI_REGISTRY_IMAGE}/dev_node:${DOCKER_TAG}
    - docker push ${CI_REGISTRY_IMAGE}/dev_python:${DOCKER_TAG}
  after_script:
    - docker run --rm ${CI_REGISTRY_IMAGE}/dev_python:${DOCKER_TAG} bash -c 'cat dist/`ls dist`' > nomad-lab.tar.gz
    - docker run --rm ${CI_REGISTRY_IMAGE}/dev_python:${DOCKER_TAG} cat tests/data/parsers/vasp/vasp.xml > vasp.xml
  artifacts:
    paths:
      - nomad-lab.tar.gz
      - vasp.xml
    expire_in: 1 hour
  rules:
    - when: on_success

build helm chart:
  stage: build
  script:
    - helm package -u ops/kubernetes/nomad -d ops/kubernetes
    - 'curl --request POST --user gitlab-ci-token:$CI_JOB_TOKEN --form "chart=@ops/kubernetes/nomad-1.0.0.tgz" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/helm/api/latest/charts"'

python linting:
  stage: test
  image: ${CI_REGISTRY_IMAGE}/dev_python:${DOCKER_TAG}
  variables:
    GIT_STRATEGY: none
  before_script:
    - cd /app
  script:
    - ruff nomad tests
    - ruff format nomad tests --check
    - ruff nomad tests --output-format gitlab > $CI_PROJECT_DIR/gl-code-quality-report.json
    - mypy nomad tests
  rules:
    - if: $CI_COMMIT_TAG
      when: never
    - when: on_success
  artifacts:
    name: "nomad_code_quality"
    when: always
    reports:
      codequality: gl-code-quality-report.json

    expire_in: never

gui linting:
  stage: test
  image: ${CI_REGISTRY_IMAGE}/dev_node:${DOCKER_TAG}
  variables:
    GIT_STRATEGY: none
  before_script:
    - cd /app/gui
  script:
    - yarn run lint
  rules:
    - if: $CI_COMMIT_TAG
      when: never
    - when: on_success

check python dependencies:
  stage: test
  image: python:3.9
  variables:
    GIT_SUBMODULE_STRATEGY: recursive
    GIT_SUBMODULE_UPDATE_FLAGS: --jobs 4
  before_script:
    - pip install --upgrade pip-tools
  script:
    - scripts/check_python_dependencies.sh
  rules:
    - if: $CI_COMMIT_TAG
      when: never
    - when: manual
      allow_failure: true

check gui artifacts:
  stage: test
  image: ${CI_REGISTRY_IMAGE}/dev_python:${DOCKER_TAG}
  variables:
    GIT_STRATEGY: none
  before_script:
    - cd /app
  script:
    - scripts/check_gui_artifacts.sh
  after_script:
    - >
      if [ $CI_JOB_STATUS == 'failed' ]; then
        echo 'Make sure that the right GUI artifacts have been regenerated and committed by running'
        echo './scripts/generate_gui_artifacts.sh'
      fi
  rules:
    - if: $CI_COMMIT_TAG
      when: never
    - when: on_success

python tests:
  stage: test
  image: ${CI_REGISTRY_IMAGE}/dev_python:${DOCKER_TAG}
  services:
    - name: rabbitmq:3.11.5
      alias: rabbitmq
    - name: docker.elastic.co/elasticsearch/elasticsearch:7.17.1
      alias: elastic
      command:
        - bash
        - "-c"
        - ES_JAVA_OPTS="-Xms512m -Xmx512m" docker-entrypoint.sh elasticsearch -Ediscovery.type=single-node -Expack.security.enabled=false
    - name: mongo:5.0.6
      alias: mongo
  variables:
    GIT_STRATEGY: none
    RABBITMQ_ERLANG_COOKIE: SWQOKODSQALRPCLNMEQG
    RABBITMQ_DEFAULT_USER: rabbitmq
    RABBITMQ_DEFAULT_PASS: rabbitmq
    RABBITMQ_DEFAULT_VHOST: /
    NOMAD_RABBITMQ_HOST: rabbitmq
    NOMAD_ELASTIC_HOST: elastic
    NOMAD_MONGO_HOST: mongo
    NOMAD_KEYCLOAK_PASSWORD: ${CI_KEYCLOAK_ADMIN_PASSWORD}
    NOMAD_NORMALIZE_SPRINGER_DB_PATH: /nomad/fairdi/db/data/springer.msg
  before_script:
    - cd /app
    - scripts/check_elastic.sh
  script:
    - python -m pytest --cov=nomad --cov-report term --cov-report xml:coverage.xml --cov-config=.coveragerc -sv tests
  after_script:
    - cd /app
    - cp coverage.xml $CI_PROJECT_DIR
  coverage: /(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/
  artifacts:
    when: always
    reports:
      coverage_report:
        coverage_format: cobertura
        path: coverage.xml
  rules:
    - if: $CI_COMMIT_TAG
      when: never
    - when: on_success

install tests:
  stage: test
  image: python:3.9
  script:
    - pip install --upgrade pip
    - pip install nomad-lab.tar.gz
    - python -c 'import nomad.cli'
    - python -c 'from nomad.client import ArchiveQuery'
    - python -m nomad.cli parse vasp.xml
    - pip install git+https://github.com/nomad-coe/nomad-parser-example.git
    - python -m exampleparser tests/data/examples/example.out

gui tests:
  stage: test
  image: ${CI_REGISTRY_IMAGE}/dev_node:${DOCKER_TAG}
  variables:
    GIT_STRATEGY: none
  before_script:
    - cd /app/gui
  script:
    - yarn test --ci --collectCoverage --reporters=default --reporters=jest-junit --silent
      --testPathIgnorePatterns=src/components/entry/ArchiveEntryView.spec.js
      --testPathIgnorePatterns=src/components/archive/FileBrowser.spec.js
      --testPathIgnorePatterns=src/components/archive/MetainfoBrowser.spec.js

  after_script:
    - cd /app/gui
    - cp junit.xml $CI_PROJECT_DIR
    - cp coverage/cobertura-coverage.xml $CI_PROJECT_DIR
  timeout: 1h 30m
  coverage: /Lines\s*:\s*(\d+.?\d*)%/
  artifacts:
    when: always
    reports:
      junit:
        - junit.xml
      coverage_report:
        coverage_format: cobertura
        path: cobertura-coverage.xml
  rules:
    - if: $CI_COMMIT_TAG
      when: never
    - when: on_success

deploy prod:
  stage: deploy
  environment:
    name: staging
    deployment_tier: production
    url: https://nomad-lab.eu/prod/v1
  before_script:
    - mkdir ~/.kube/
    - echo ${CI_K8S_CLOUD_CONFIG} | base64 -d > ~/.kube/config
    - echo "${CI_REGISTRY_PASSWORD}" | docker login ${CI_REGISTRY} --username ${CI_REGISTRY_USER} --password-stdin
  script:
    - docker pull ${CI_REGISTRY_IMAGE}:${DOCKER_TAG}
    - docker tag ${CI_REGISTRY_IMAGE}:${DOCKER_TAG} ${CI_REGISTRY_IMAGE}:prod
    - docker push ${CI_REGISTRY_IMAGE}:prod
    - helm dependency update ops/kubernetes/nomad
    - helm upgrade nomad-prod ops/kubernetes/nomad
      --install
      --namespace nomad-prod
      --values ops/kubernetes/values.yaml
      --values ops/kubernetes/nomad-prod.yaml
      --set nomad.image.tag=prod
      --set roll=true
      --timeout=15m
      --wait
    - docker pull ${CI_REGISTRY_IMAGE}:${DOCKER_TAG}
    - docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod -e NOMAD_KEYCLOAK_SERVER_URL=https://nomad-lab.eu/fairdi/keycloak/auth/ ${CI_REGISTRY_IMAGE}/dev_python:${DOCKER_TAG}
      nomad client -n https://nomad-lab.eu/prod/v1/api -u test -w $CI_NOMAD_TEST_PASSWORD integrationtests --skip-publish --skip-doi
  rules:
    - when: manual
      allow_failure: true

deploy prod staging:
  stage: deploy
  environment:
    name: staging
    deployment_tier: production
    url: https://nomad-lab.eu/prod/v1/staging
  before_script:
    - mkdir ~/.kube/
    - echo ${CI_K8S_CLOUD_CONFIG} | base64 -d > ~/.kube/config
    - echo "${CI_REGISTRY_PASSWORD}" | docker login ${CI_REGISTRY} --username ${CI_REGISTRY_USER} --password-stdin
  script:
    - docker pull ${CI_REGISTRY_IMAGE}:${DOCKER_TAG}
    - docker tag ${CI_REGISTRY_IMAGE}:${DOCKER_TAG} ${CI_REGISTRY_IMAGE}:staging
    - docker push ${CI_REGISTRY_IMAGE}:staging
    - helm dependency update ops/kubernetes/nomad
    - helm upgrade nomad-prod-staging ops/kubernetes/nomad
      --install
      --namespace nomad-prod-staging
      --values ops/kubernetes/values.yaml
      --values ops/kubernetes/nomad-prod-staging.yaml
      --set nomad.image.tag=staging
      --set roll=true
      --timeout=15m
      --wait
    - docker pull ${CI_REGISTRY_IMAGE}:${DOCKER_TAG}
    - docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod -e NOMAD_KEYCLOAK_SERVER_URL=https://nomad-lab.eu/fairdi/keycloak/auth/ ${CI_REGISTRY_IMAGE}/dev_python:${DOCKER_TAG}
      nomad client -n https://nomad-lab.eu/prod/v1/staging/api -u test -w $CI_NOMAD_TEST_PASSWORD integrationtests --skip-publish --skip-doi
  rules:
    - when: manual
      allow_failure: true

deploy prod develop:
  stage: deploy
  environment:
    name: develop
    deployment_tier: production
    url: https://nomad-lab.eu/prod/v1/develop
  before_script:
    - mkdir ~/.kube/
    - echo ${CI_K8S_CLOUD_CONFIG} | base64 -d > ~/.kube/config
    - echo "${CI_REGISTRY_PASSWORD}" | docker login ${CI_REGISTRY} --username ${CI_REGISTRY_USER} --password-stdin
  script:
    - docker pull ${CI_REGISTRY_IMAGE}:${DOCKER_TAG}
    - docker tag ${CI_REGISTRY_IMAGE}:${DOCKER_TAG} ${CI_REGISTRY_IMAGE}:develop
    - docker push ${CI_REGISTRY_IMAGE}:develop
    - helm dependency update ops/kubernetes/nomad
    - helm upgrade nomad-prod-develop ops/kubernetes/nomad
      --install
      --namespace nomad-prod-develop
      --values ops/kubernetes/values.yaml
      --values ops/kubernetes/nomad-prod-develop.yaml
      --set nomad.image.tag=develop
      --set roll=true
      --timeout=15m
      --wait
    - docker pull ${CI_REGISTRY_IMAGE}:${DOCKER_TAG}
    - docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod -e NOMAD_KEYCLOAK_SERVER_URL=https://nomad-lab.eu/fairdi/keycloak/auth/ ${CI_REGISTRY_IMAGE}/dev_python:${DOCKER_TAG}
      nomad client -n https://nomad-lab.eu/prod/v1/develop/api -u test -w $CI_NOMAD_TEST_PASSWORD integrationtests --skip-publish --skip-doi
  rules:
    - if: $CI_COMMIT_BRANCH == "develop" && $NIGHTLY
    - when: manual
      allow_failure: true

deploy prod test:
  stage: deploy
  environment:
    name: test
    deployment_tier: production
    url: https://nomad-lab.eu/prod/v1/test
  before_script:
    - mkdir ~/.kube/
    - echo ${CI_K8S_CLOUD_CONFIG} | base64 -d > ~/.kube/config
    - echo "${CI_REGISTRY_PASSWORD}" | docker login ${CI_REGISTRY} --username ${CI_REGISTRY_USER} --password-stdin
  script:
    - docker pull ${CI_REGISTRY_IMAGE}:${DOCKER_TAG}
    - docker tag ${CI_REGISTRY_IMAGE}:${DOCKER_TAG} ${CI_REGISTRY_IMAGE}:test
    - docker push ${CI_REGISTRY_IMAGE}:test
    - helm dependency update ops/kubernetes/nomad
    - helm upgrade nomad-prod-test ops/kubernetes/nomad
      --install
      --namespace nomad-prod-test
      --values ops/kubernetes/values.yaml
      --values ops/kubernetes/nomad-prod-test.yaml
      --set nomad.image.tag=test
      --set roll=true
      --timeout=15m
      --wait
    - docker pull ${CI_REGISTRY_IMAGE}:${DOCKER_TAG}
    - docker run -t -e NOMAD_KEYCLOAK_REALM_NAME=fairdi_nomad_prod -e NOMAD_KEYCLOAK_SERVER_URL=https://nomad-lab.eu/fairdi/keycloak/auth/ ${CI_REGISTRY_IMAGE}/dev_python:${DOCKER_TAG}
      nomad client -n https://nomad-lab.eu/prod/v1/test/api -u test -w $CI_NOMAD_TEST_PASSWORD integrationtests --skip-publish --skip-doi
  rules:
    - when: manual
      allow_failure: true

release latest image:
  stage: release
  variables:
    GIT_STRATEGY: none
  before_script:
    - echo "${CI_REGISTRY_PASSWORD}" | docker login ${CI_REGISTRY} --username ${CI_REGISTRY_USER} --password-stdin
    - docker pull ${CI_REGISTRY_IMAGE}:${DOCKER_TAG}
  script:
    - docker tag ${CI_REGISTRY_IMAGE}:${DOCKER_TAG} ${CI_REGISTRY_IMAGE}:latest
    - docker push ${CI_REGISTRY_IMAGE}:latest
  rules:
    - when: manual
      allow_failure: true

release stable image:
  stage: release
  variables:
    GIT_STRATEGY: none
  before_script:
    - echo "${CI_REGISTRY_PASSWORD}" | docker login ${CI_REGISTRY} --username ${CI_REGISTRY_USER} --password-stdin
    - docker pull ${CI_REGISTRY_IMAGE}:${DOCKER_TAG}
  script:
    - docker tag ${CI_REGISTRY_IMAGE}:${DOCKER_TAG} ${CI_REGISTRY_IMAGE}:stable
    - docker push ${CI_REGISTRY_IMAGE}:stable
  rules:
    - if: $CI_COMMIT_TAG
      when: manual
      allow_failure: true
    - when: never

python package:
  stage: release
  variables:
    GIT_STRATEGY: none
  before_script:
    - echo "${CI_REGISTRY_PASSWORD}" | docker login ${CI_REGISTRY} --username ${CI_REGISTRY_USER} --password-stdin
    - docker pull ${CI_REGISTRY_IMAGE}/dev_python:${DOCKER_TAG}
  script:
    - docker run --rm ${CI_REGISTRY_IMAGE}/dev_python:${DOCKER_TAG}
      twine upload -u gitlab-ci-token -p ${CI_JOB_TOKEN} --repository-url https://gitlab.mpcdf.mpg.de/api/v4/projects/${CI_PROJECT_ID}/packages/pypi dist/nomad-lab-*.tar.gz
  rules:
    - when: manual
      allow_failure: true

pypi package:
  stage: release
  variables:
    GIT_STRATEGY: none
  before_script:
    - echo "${CI_REGISTRY_PASSWORD}" | docker login ${CI_REGISTRY} --username ${CI_REGISTRY_USER} --password-stdin
    - docker pull ${CI_REGISTRY_IMAGE}/dev_python:${DOCKER_TAG}
  script:
    - docker run --rm ${CI_REGISTRY_IMAGE}/dev_python:${DOCKER_TAG}
      twine upload -u $CI_TWINE_USER -p $CI_TWINE_PASSWORD dist/nomad-lab-*.tar.gz
  rules:
    - if: $CI_COMMIT_TAG
      when: manual
      allow_failure: true
    - when: never

push to github:
  stage: release
  script:
    - git checkout ${CI_COMMIT_REF_NAME}
    - git push "https://${CI_GITHUB_ACCESS_TOKEN}@github.com/nomad-coe/nomad.git" ${CI_COMMIT_REF_NAME}
  rules:
    - if: $CI_COMMIT_BRANCH == "develop" || $CI_COMMIT_TAG