conduit/.gitlab-ci.yml

stages:
  - build
  - build docker image
  - test
  - upload artifacts

variables:
  GIT_SUBMODULE_STRATEGY: recursive
  FF_USE_FASTZIP: 1
  CACHE_COMPRESSION_LEVEL: fastest
  # Docker in Docker
  DOCKER_HOST: tcp://docker:2375/
  DOCKER_TLS_CERTDIR: ""
  DOCKER_DRIVER: overlay2

# --------------------------------------------------------------------- #
#  Cargo: Compiling for different architectures                         #
# --------------------------------------------------------------------- #

.build-cargo-shared-settings:
  stage: "build"
  needs: []
  rules:
    - if: '$CI_COMMIT_BRANCH == "master"'
    - if: '$CI_COMMIT_BRANCH == "next"'
    - if: "$CI_COMMIT_TAG"
    - if: '($CI_MERGE_REQUEST_APPROVED == "true") || $BUILD_EVERYTHING' # Once MR is approved, test all builds. Or if BUILD_EVERYTHING is set.
  interruptible: true
  image: "registry.gitlab.com/jfowl/conduit-containers/rust-with-tools:latest"
  tags: ["docker"]
  services: ["docker:dind"]
  variables:
    SHARED_PATH: $CI_PROJECT_DIR/shared
    CARGO_PROFILE_RELEASE_LTO: "true"
    CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "1"
    CARGO_INCREMENTAL: "false" # https://matklad.github.io/2021/09/04/fast-rust-builds.html#ci-workflow
  before_script:
    - 'echo "Building for target $TARGET"'
    - "rustup show && rustc --version && cargo --version" # Print version info for debugging
    # fix cargo and rustup mounts from this container (https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41227)
    - "mkdir -p $SHARED_PATH/cargo"
    - "cp -r $CARGO_HOME/bin $SHARED_PATH/cargo"
    - "cp -r $RUSTUP_HOME $SHARED_PATH"
    - "export CARGO_HOME=$SHARED_PATH/cargo RUSTUP_HOME=$SHARED_PATH/rustup"
    # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results.
    - if [ -n "${SCCACHE_ENDPOINT}" ]; then export RUSTC_WRAPPER=/sccache; fi
  script:
    # cross-compile conduit for target
    - 'time cross build --target="$TARGET" --locked --release'
    - 'mv "target/$TARGET/release/conduit" "conduit-$TARGET"'
    # print information about linking for debugging
    - "file conduit-$TARGET" # print file information
    - 'readelf --dynamic conduit-$TARGET | sed -e "/NEEDED/q1"' # ensure statically linked
  cache:
    # https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci
    key: "cargo-cache-$TARGET"
    paths:
      - $SHARED_PATH/cargo/registry/index
      - $SHARED_PATH/cargo/registry/cache
      - $SHARED_PATH/cargo/git/db
  artifacts:
    expire_in: never

build:release:cargo:x86_64-unknown-linux-musl-with-debug:
  extends: .build-cargo-shared-settings
  variables:
    CARGO_PROFILE_RELEASE_DEBUG: 2 # Enable debug info for flamegraph profiling
    TARGET: "x86_64-unknown-linux-musl"
  after_script:
    - "mv ./conduit-x86_64-unknown-linux-musl ./conduit-x86_64-unknown-linux-musl-with-debug"
  artifacts:
    name: "conduit-x86_64-unknown-linux-musl-with-debug"
    paths:
      - "conduit-x86_64-unknown-linux-musl-with-debug"
    expose_as: "Conduit for x86_64-unknown-linux-musl-with-debug"

build:release:cargo:x86_64-unknown-linux-musl:
  extends: .build-cargo-shared-settings
  variables:
    TARGET: "x86_64-unknown-linux-musl"
  artifacts:
    name: "conduit-x86_64-unknown-linux-musl"
    paths:
      - "conduit-x86_64-unknown-linux-musl"
    expose_as: "Conduit for x86_64-unknown-linux-musl"

build:release:cargo:arm-unknown-linux-musleabihf:
  extends: .build-cargo-shared-settings
  variables:
    TARGET: "arm-unknown-linux-musleabihf"
  artifacts:
    name: "conduit-arm-unknown-linux-musleabihf"
    paths:
      - "conduit-arm-unknown-linux-musleabihf"
    expose_as: "Conduit for arm-unknown-linux-musleabihf"

build:release:cargo:armv7-unknown-linux-musleabihf:
  extends: .build-cargo-shared-settings
  variables:
    TARGET: "armv7-unknown-linux-musleabihf"
  artifacts:
    name: "conduit-armv7-unknown-linux-musleabihf"
    paths:
      - "conduit-armv7-unknown-linux-musleabihf"
    expose_as: "Conduit for armv7-unknown-linux-musleabihf"

build:release:cargo:aarch64-unknown-linux-musl:
  extends: .build-cargo-shared-settings
  variables:
    TARGET: "aarch64-unknown-linux-musl"
  artifacts:
    name: "conduit-aarch64-unknown-linux-musl"
    paths:
      - "conduit-aarch64-unknown-linux-musl"
    expose_as: "Conduit for aarch64-unknown-linux-musl"

.cargo-debug-shared-settings:
  extends: ".build-cargo-shared-settings"
  rules:
    - when: "always"
  cache:
    key: "build_cache--$TARGET--$CI_COMMIT_BRANCH--debug"
  script:
    # cross-compile conduit for target
    - 'time time cross build --target="$TARGET" --locked'
    - 'mv "target/$TARGET/debug/conduit" "conduit-debug-$TARGET"'
    # print information about linking for debugging
    - "file conduit-debug-$TARGET" # print file information
    - 'readelf --dynamic conduit-debug-$TARGET | sed -e "/NEEDED/q1"' # ensure statically linked
  artifacts:
    expire_in: 4 weeks

build:debug:cargo:x86_64-unknown-linux-musl:
  extends: ".cargo-debug-shared-settings"
  variables:
    TARGET: "x86_64-unknown-linux-musl"
  artifacts:
    name: "conduit-debug-x86_64-unknown-linux-musl"
    paths:
      - "conduit-debug-x86_64-unknown-linux-musl"
    expose_as: "Conduit DEBUG for x86_64-unknown-linux-musl"

# --------------------------------------------------------------------- #
#  Create and publish docker image                                      #
# --------------------------------------------------------------------- #

.docker-shared-settings:
  stage: "build docker image"
  image: jdrouet/docker-with-buildx:stable
  tags: ["docker"]
  services:
    - docker:dind
  needs:
    - "build:release:cargo:x86_64-unknown-linux-musl"
    - "build:release:cargo:arm-unknown-linux-musleabihf"
    - "build:release:cargo:armv7-unknown-linux-musleabihf"
    - "build:release:cargo:aarch64-unknown-linux-musl"
  variables:
    PLATFORMS: "linux/arm/v6,linux/arm/v7,linux/arm64,linux/amd64"
    DOCKER_FILE: "docker/ci-binaries-packaging.Dockerfile"
  cache:
    paths:
      - docker_cache
    key: "$CI_JOB_NAME"
  before_script:
    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
    # Only log in to Dockerhub if the credentials are given:
    - if [ -n "${DOCKER_HUB}" ]; then docker login -u "$DOCKER_HUB_USER" -p "$DOCKER_HUB_PASSWORD" "$DOCKER_HUB"; fi
  script:
    # Prepare buildx to build multiarch stuff:
    - docker context create 'ci-context'
    - docker buildx create --name 'multiarch-builder' --use 'ci-context'
    # Copy binaries to their docker arch path
    - mkdir -p linux/ && mv ./conduit-x86_64-unknown-linux-musl linux/amd64
    - mkdir -p linux/arm/ && mv ./conduit-arm-unknown-linux-musleabihf linux/arm/v6
    - mkdir -p linux/arm/ && mv ./conduit-armv7-unknown-linux-musleabihf linux/arm/v7
    - mv ./conduit-aarch64-unknown-linux-musl linux/arm64
    - 'export CREATED=$(date -u +''%Y-%m-%dT%H:%M:%SZ'') && echo "Docker image creation date: $CREATED"'
    # Build and push image:
    - >
      docker buildx build
      --pull
      --push
      --cache-from=type=local,src=$CI_PROJECT_DIR/docker_cache
      --cache-to=type=local,dest=$CI_PROJECT_DIR/docker_cache
      --build-arg CREATED=$CREATED
      --build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml)
      --build-arg "GIT_REF=$CI_COMMIT_SHORT_SHA"
      --platform "$PLATFORMS"
      --tag "$TAG"
      --tag "$TAG-alpine"
      --tag "$TAG-commit-$CI_COMMIT_SHORT_SHA"
      --file "$DOCKER_FILE" .

docker:next:gitlab:
  extends: .docker-shared-settings
  rules:
    - if: '$CI_COMMIT_BRANCH == "next"'
  variables:
    TAG: "$CI_REGISTRY_IMAGE/matrix-conduit:next"

docker:next:dockerhub:
  extends: .docker-shared-settings
  rules:
    - if: '$CI_COMMIT_BRANCH == "next" && $DOCKER_HUB'
  variables:
    TAG: "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:next"

docker:master:gitlab:
  extends: .docker-shared-settings
  rules:
    - if: '$CI_COMMIT_BRANCH == "master"'
  variables:
    TAG: "$CI_REGISTRY_IMAGE/matrix-conduit:latest"

docker:master:dockerhub:
  extends: .docker-shared-settings
  rules:
    - if: '$CI_COMMIT_BRANCH == "master" && $DOCKER_HUB'
  variables:
    TAG: "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:latest"

docker:tags:gitlab:
  extends: .docker-shared-settings
  rules:
    - if: "$CI_COMMIT_TAG"
  variables:
    TAG: "$CI_REGISTRY_IMAGE/matrix-conduit:$CI_COMMIT_TAG"

docker:tags:dockerhub:
  extends: .docker-shared-settings
  rules:
    - if: "$CI_COMMIT_TAG && $DOCKER_HUB"
  variables:
    TAG: "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:$CI_COMMIT_TAG"

# --------------------------------------------------------------------- #
#  Run tests                                                            #
# --------------------------------------------------------------------- #

.test-shared-settings:
  stage: "test"
  needs: []
  image: "registry.gitlab.com/jfowl/conduit-containers/rust-with-tools:latest"
  tags: ["docker"]
  variables:
    CARGO_INCREMENTAL: "false" # https://matklad.github.io/2021/09/04/fast-rust-builds.html#ci-workflow
  interruptible: true

test:cargo:
  extends: .test-shared-settings
  before_script:
    - rustup component add clippy
    # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results:
    - if [ -n "${SCCACHE_ENDPOINT}" ]; then export RUSTC_WRAPPER=/usr/local/cargo/bin/sccache; fi
  script:
    - rustc --version && cargo --version # Print version info for debugging
    - "cargo test --color always --workspace --verbose --locked --no-fail-fast -- -Z unstable-options --format json | gitlab-report -p test > $CI_PROJECT_DIR/report.xml"
    - "cargo clippy --color always --verbose --message-format=json | gitlab-report -p clippy > $CI_PROJECT_DIR/gl-code-quality-report.json"
  artifacts:
    when: always
    reports:
      junit: report.xml
      codequality: gl-code-quality-report.json

test:format:
  extends: .test-shared-settings
  before_script:
    - rustup component add rustfmt
  script:
    - cargo fmt --all -- --check

test:audit:
  extends: .test-shared-settings
  allow_failure: true
  script:
    - cargo audit --color always || true
    - cargo audit --stale --json | gitlab-report -p audit > gl-sast-report.json
  artifacts:
    when: always
    reports:
      sast: gl-sast-report.json

test:sytest:
  stage: "test"
  allow_failure: true
  needs:
    - "build:debug:cargo:x86_64-unknown-linux-musl"
  image:
    name: "valkum/sytest-conduit:latest"
    entrypoint: [""]
  tags: ["docker"]
  variables:
    PLUGINS: "https://github.com/valkum/sytest_conduit/archive/master.tar.gz"
  interruptible: true
  before_script:
    - "mkdir -p /app"
    - "cp ./conduit-debug-x86_64-unknown-linux-musl /app/conduit"
    - "chmod +x /app/conduit"
    - "rm -rf /src && ln -s $CI_PROJECT_DIR/ /src"
    - "mkdir -p /work/server-0/database/ && mkdir -p /work/server-1/database/ && mkdir -p /work/server-2/database/"
    - "cd /"
  script:
    - "SYTEST_EXIT_CODE=0"
    - "/bootstrap.sh conduit || SYTEST_EXIT_CODE=1"
    - 'perl /sytest/tap-to-junit-xml.pl --puretap --input /logs/results.tap --output $CI_PROJECT_DIR/sytest.xml "Sytest" && cp /logs/results.tap $CI_PROJECT_DIR/results.tap'
    - "exit $SYTEST_EXIT_CODE"
  artifacts:
    when: always
    paths:
      - "$CI_PROJECT_DIR/sytest.xml"
      - "$CI_PROJECT_DIR/results.tap"
    reports:
      junit: "$CI_PROJECT_DIR/sytest.xml"

test:dockerlint:
  stage: "test"
  needs: []
  image: "ghcr.io/hadolint/hadolint:latest-alpine"
  interruptible: true
  script:
    # First pass: Print for CI log:
    - >
      hadolint
      --no-fail --verbose
      ./Dockerfile
      ./docker/ci-binaries-packaging.Dockerfile
    # Then output the results into a json for GitLab to pretty-print this in the MR:
    - >
      hadolint
      --format gitlab_codeclimate
      --failure-threshold error
      ./Dockerfile
      ./docker/ci-binaries-packaging.Dockerfile > dockerlint.json
  artifacts:
    when: always
    reports:
      codequality: dockerlint.json
    paths:
      - dockerlint.json
  rules:
    - if: '$CI_COMMIT_REF_NAME != "master"'
      changes:
        - docker/*Dockerfile
        - Dockerfile
        - .gitlab-ci.yml
    - if: '$CI_COMMIT_REF_NAME == "master"'
    - if: '$CI_COMMIT_REF_NAME == "next"'

# --------------------------------------------------------------------- #
#  Store binaries as package so they have download urls                 #
# --------------------------------------------------------------------- #

publish:package:
  stage: "upload artifacts"
  needs:
    - "build:release:cargo:x86_64-unknown-linux-musl"
    - "build:release:cargo:arm-unknown-linux-musleabihf"
    - "build:release:cargo:armv7-unknown-linux-musleabihf"
    - "build:release:cargo:aarch64-unknown-linux-musl"
  #    - "build:cargo-deb:x86_64-unknown-linux-gnu"
  rules:
    - if: '$CI_COMMIT_BRANCH == "master"'
    - if: '$CI_COMMIT_BRANCH == "next"'
    - if: "$CI_COMMIT_TAG"
  image: curlimages/curl:latest
  tags: ["docker"]
  variables:
    GIT_STRATEGY: "none" # Don't need a clean copy of the code, we just operate on artifacts
  script:
    - 'BASE_URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/conduit-${CI_COMMIT_REF_SLUG}/build-${CI_PIPELINE_ID}"'
    - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-x86_64-unknown-linux-musl "${BASE_URL}/conduit-x86_64-unknown-linux-musl"'
    - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-arm-unknown-linux-musleabihf "${BASE_URL}/conduit-arm-unknown-linux-musleabihf"'
    - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-armv7-unknown-linux-musleabihf "${BASE_URL}/conduit-armv7-unknown-linux-musleabihf"'
    - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-aarch64-unknown-linux-musl "${BASE_URL}/conduit-aarch64-unknown-linux-musl"'

# Avoid duplicate pipelines
# See: https://docs.gitlab.com/ee/ci/yaml/workflow.html#switch-between-branch-pipelines-and-merge-request-pipelines
workflow:
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    - if: "$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS"
      when: never
    - if: "$CI_COMMIT_BRANCH"
    - if: "$CI_COMMIT_TAG"