Materialize/ci/test/pipeline.template.yml

# Copyright Materialize, Inc. and contributors. All rights reserved.
#
# Use of this software is governed by the Business Source License
# included in the LICENSE file at the root of this repository.
#
# As of the Change Date specified in that file, in accordance with
# the Business Source License, use of this software will be governed
# by the Apache License, Version 2.0.

# This file is processed by mkpipeline.py to trim unnecessary steps in PR
# builds. The inputs for steps using the `mzcompose` plugin are computed
# automatically. Inputs for other steps need to be manually listed in the
# `inputs` key.

dag: true

env:
  CI_BAZEL_BUILD: 0
  CI_BAZEL_REMOTE_CACHE: $BAZEL_REMOTE_CACHE
  CI_BAZEL_LTO: 0
  CARGO_BUILD_JOBS: "default"

# When resources are constrained, run early since this might be blocking a PR from merging
priority: 20

steps:
  - group: Builds
    key: builds
    steps:
      - id: build-x86_64
        label: ":rust: Build x86_64"
        env:
        command: bin/ci-builder run stable bin/pyactivate -m ci.test.build
        inputs:
          - "*"
        artifact_paths: bazel-explain.log
        depends_on: []
        timeout_in_minutes: 60
        agents:
          queue: l-builder-linux-x86_64

      - id: build-aarch64
        label: ":rust: Build aarch64"
        env:
        command: bin/ci-builder run stable bin/pyactivate -m ci.test.build
        inputs:
          - "*"
        artifact_paths: bazel-explain.log
        depends_on: []
        timeout_in_minutes: 60
        agents:
          queue: l-builder-linux-aarch64

      - id: upload-debug-symbols-x86_64
        label: "Upload debug symbols for x86_64"
        env:
          CI_BAZEL_BUILD: 0
        command: bin/ci-builder run stable bin/pyactivate -m materialize.ci_util.upload_debug_symbols_to_polarsignals
        inputs:
          - "*"
        depends_on: [build-x86_64]
        timeout_in_minutes: 40
        priority: 50
        agents:
          queue: linux-x86_64
        coverage: skip
        sanitizer: skip

      - id: upload-debug-symbols-aarch64
        label: "Upload debug symbols for aarch64"
        env:
          CI_BAZEL_BUILD: 0
        command: bin/ci-builder run stable bin/pyactivate -m materialize.ci_util.upload_debug_symbols_to_polarsignals
        inputs:
          - "*"
        depends_on: [build-aarch64]
        priority: 50
        timeout_in_minutes: 40
        agents:
          queue: linux-aarch64
        coverage: skip
        sanitizer: skip

      - id: build-wasm
        label: ":rust: Build WASM"
        command: bin/ci-builder run stable bin/pyactivate -m ci.deploy.npm --no-release
        inputs:
          - "ci/deploy/npm.py"
          - "bin/wasm-build"
          - "misc/wasm"
        depends_on: []
        timeout_in_minutes: 20
        agents:
          # Error: no prebuilt wasm-opt binaries are available for this platform: Unrecognized target!
          queue: hetzner-x86-64-4cpu-8gb
        coverage: skip
        sanitizer: skip

      - id: check-merge-with-target
        label: Merge skew cargo check
        command: ci/test/check-merge-with-target.sh
        inputs:
          - Cargo.lock
          - Cargo.toml
          - "**/Cargo.toml"
          - "**/*.rs"
        depends_on: []
        timeout_in_minutes: 45
        agents:
          queue: hetzner-x86-64-16cpu-32gb-merge-skew
        if: "build.pull_request.id != null"
        coverage: skip
        sanitizer: skip

      - id: devel-docker-tags
        label: Tag development docker images
        command: bin/ci-builder run min bin/pyactivate -m ci.test.dev_tag
        inputs:
          - "*"
        depends_on:
          - build-x86_64
          - build-aarch64
        timeout_in_minutes: 20
        agents:
          queue: linux-aarch64-small
        coverage: skip
        # Fortify against intermittent DockerHub issues
        retry:
          automatic:
            - exit_status: 1
              limit: 2

  - group: Lints
    key: lints
    steps:
      - id: lint-fast
        label: Lint and rustfmt
        command: bin/ci-builder run stable ci/test/lint-fast.sh
        inputs:
          - "*"
        depends_on: []
        timeout_in_minutes: 20
        agents:
          # TODO: Revert to aarch64 when https://github.com/trufflesecurity/trufflehog/issues/4229 is solved
          queue: hetzner-x86-64-16cpu-32gb-lint-rustfmt
        coverage: skip
        sanitizer: skip

      - id: lint-clippy
        label: Clippy
        command: bin/ci-builder run stable ci/test/lint-clippy.sh
        inputs:
          - Cargo.lock
          - Cargo.toml
          - "**/Cargo.toml"
          - "**/*.rs"
        depends_on: []
        timeout_in_minutes: 40
        agents:
          queue: hetzner-x86-64-16cpu-32gb-clippy
        coverage: skip
        sanitizer: skip

      - id: lint-doctests
        label: Doctests
        command: bin/ci-builder run stable ci/test/lint-doc.sh
        inputs:
          - Cargo.lock
          - Cargo.toml
          - "**/Cargo.toml"
          - "**/*.rs"
          - misc/helm-charts
        depends_on: []
        timeout_in_minutes: 40
        agents:
          queue: hetzner-x86-64-16cpu-32gb-doctests
        coverage: skip
        sanitizer: skip

      - id: lint-macos
        label: ":rust: macOS Clippy"
        command: cargo clippy --all-targets -- -D warnings
        env:
          CARGO_INCREMENTAL: "0"
          RUSTUP_TOOLCHAIN: $RUST_VERSION
        inputs:
          - Cargo.lock
          - Cargo.toml
          - "**/Cargo.toml"
          - "**/*.rs"
        depends_on: []
        timeout_in_minutes: 30
        agents:
          queue: mac
        coverage: skip
        sanitizer: skip

      - id: lint-deps
        label: Lint dependencies
        command: bin/ci-builder run stable ci/test/lint-deps.sh
        inputs:
          - Cargo.lock
          - Cargo.toml
          - "**/Cargo.toml"
          - "**/*.rs"
        depends_on: []
        timeout_in_minutes: 30
        agents:
          queue: hetzner-aarch64-4cpu-8gb
        coverage: skip
        sanitizer: skip

  - id: cargo-test
    label: ":rust: Cargo test"
    timeout_in_minutes: 60
    inputs:
      - Cargo.lock
      - Cargo.toml
      - ".config/nextest.toml"
      - "**/Cargo.toml"
      - "**/*.rs"
      - "**/*.pt"
      - "**/*.proto"
      - "**/testdata/**"
    depends_on: []
    parallelism: 2
    env:
      AWS_DEFAULT_REGION: "us-east-1"
      # cargo-test's coverage is handled separately by cargo-llvm-cov
      BUILDKITE_MZCOMPOSE_PLUGIN_SKIP_COVERAGE: "true"
      # some tests run into stack overflows
      RUST_MIN_STACK: "4194304"
    plugins:
      - ./ci/plugins/mzcompose:
          composition: cargo-test
          ci-builder: stable
    agents:
      queue: hetzner-x86-64-dedi-32cpu-128gb-cargo-test

  - id: testdrive
    label: "Testdrive"
    depends_on: build-aarch64
    timeout_in_minutes: 40
    inputs: [test/testdrive]
    parallelism: 20
    plugins:
      - ./ci/plugins/mzcompose:
          composition: testdrive
    agents:
      queue: hetzner-aarch64-8cpu-16gb

  - id: cluster-tests
    label: "Cluster tests"
    depends_on: build-aarch64
    timeout_in_minutes: 30
    inputs: [test/cluster]
    parallelism: 16
    plugins:
      - ./ci/plugins/mzcompose:
          composition: cluster
    agents:
      queue: hetzner-aarch64-16cpu-32gb

  - id: sqllogictest-fast
    label: "Fast SQL logic tests"
    depends_on: build-aarch64
    timeout_in_minutes: 30
    inputs: [test/sqllogictest]
    parallelism: 3
    plugins:
      - ./ci/plugins/mzcompose:
          composition: sqllogictest
          run: fast-tests
    agents:
      queue: hetzner-aarch64-16cpu-32gb

  - id: restarts
    label: "Restart test"
    depends_on: build-aarch64
    timeout_in_minutes: 30
    parallelism: 4
    plugins:
      - ./ci/plugins/mzcompose:
          composition: restart
    agents:
      queue: hetzner-aarch64-8cpu-16gb

  - group: "MySQL tests"
    key: mysql-tests
    steps:
      - id: mysql-cdc
        label: "MySQL CDC tests"
        parallelism: 8
        depends_on: build-aarch64
        timeout_in_minutes: 30
        inputs: [test/mysql-cdc]
        plugins:
          - ./ci/plugins/mzcompose:
              composition: mysql-cdc
        agents:
          queue: hetzner-aarch64-4cpu-8gb

      - id: mysql-rtr
        label: "MySQL RTR tests"
        depends_on: build-aarch64
        timeout_in_minutes: 30
        inputs: [test/mysql-rtr]
        plugins:
          - ./ci/plugins/mzcompose:
              composition: mysql-rtr
        agents:
          queue: hetzner-aarch64-4cpu-8gb

  - group: "Postgres tests"
    key: postgres-tests
    steps:
      - id: pg-cdc
        label: "Postgres CDC tests"
        parallelism: 8
        depends_on: build-aarch64
        timeout_in_minutes: 30
        inputs: [test/pg-cdc]
        plugins:
          - ./ci/plugins/mzcompose:
              composition: pg-cdc
        agents:
          queue: hetzner-aarch64-16cpu-32gb
        # the mzbuild postgres version will be used, which depends on the Dockerfile specification

      - id: pg-rtr
        label: "Postgres RTR tests"
        depends_on: build-aarch64
        timeout_in_minutes: 30
        inputs: [test/pg-rtr]
        plugins:
          - ./ci/plugins/mzcompose:
              composition: pg-rtr
        agents:
          queue: hetzner-aarch64-4cpu-8gb

  - id: yugabyte-cdc
    label: "Yugabyte CDC tests"
    depends_on: build-x86_64
    timeout_in_minutes: 30
    inputs: [test/yugabyte-cdc]
    plugins:
      - ./ci/plugins/mzcompose:
          composition: yugabyte-cdc
    agents:
      # Too slow on aarch64
      queue: hetzner-x86-64-4cpu-8gb

  - id: sql-server-cdc
    label: "SQL Server CDC tests"
    depends_on: build-x86_64
    timeout_in_minutes: 30
    inputs: [test/sql-server-cdc]
    plugins:
      - ./ci/plugins/mzcompose:
          composition: sql-server-cdc
    agents:
      # The SQL Server Docker image isn't available on ARM.
      #
      # See: <https://github.com/microsoft/mssql-docker/issues/864>
      queue: hetzner-x86-64-4cpu-8gb
    skip: "database-issues#9519 and database-issues#9514"

  - group: "Connection tests"
    key: connection-tests
    steps:
      - id: ssh-connection
        label: SSH connection tests
        depends_on: build-aarch64
        timeout_in_minutes: 40
        inputs: [test/ssh-connection]
        parallelism: 2
        plugins:
          - ./ci/plugins/mzcompose:
              composition: ssh-connection
        agents:
          queue: hetzner-aarch64-8cpu-16gb

      - id: fivetran-destination-tests
        label: Fivetran Destination tests
        depends_on: build-aarch64
        timeout_in_minutes: 30
        inputs: [test/fivetran-destination]
        plugins:
          - ./ci/plugins/mzcompose:
              composition: fivetran-destination
        agents:
          queue: hetzner-aarch64-4cpu-8gb

  - group: "Kafka tests"
    key: kafka-tests
    steps:
      - id: kafka-auth
        label: Kafka auth test
        depends_on: build-aarch64
        timeout_in_minutes: 30
        inputs: [test/kafka-auth]
        parallelism: 2
        plugins:
          - ./ci/plugins/mzcompose:
              composition: kafka-auth
        agents:
          queue: hetzner-aarch64-8cpu-16gb

      - id: kafka-exactly-once
        label: Kafka exactly-once tests
        depends_on: build-aarch64
        timeout_in_minutes: 30
        plugins:
          - ./ci/plugins/mzcompose:
              composition: kafka-exactly-once
        agents:
          queue: hetzner-aarch64-4cpu-8gb

      - id: kafka-rtr
        label: "Kafka RTR tests"
        depends_on: build-aarch64
        timeout_in_minutes: 30
        artifact_paths: junit_*.xml
        plugins:
          - ./ci/plugins/mzcompose:
              composition: kafka-rtr
        agents:
          queue: hetzner-aarch64-4cpu-8gb

  - id: zippy-kafka-sources-short
    label: "Short Zippy"
    depends_on: build-aarch64
    inputs: [misc/python/materialize/zippy]
    timeout_in_minutes: 30
    agents:
      queue: hetzner-aarch64-4cpu-8gb
    plugins:
      - ./ci/plugins/mzcompose:
          composition: zippy
          args: [--scenario=KafkaSources, --actions=80]

  - id: checks-no-restart-no-upgrade
    label: "Checks without restart or upgrade"
    depends_on: build-aarch64
    inputs: [misc/python/materialize/checks]
    timeout_in_minutes: 45
    parallelism: 16
    agents:
      queue: hetzner-aarch64-8cpu-16gb
    plugins:
      - ./ci/plugins/mzcompose:
          composition: platform-checks
          args: [
              --scenario=NoRestartNoUpgrade,
              --default-replication-factor=1, # faster
              "--seed=$BUILDKITE_JOB_ID"
          ]

  - id: source-sink-errors
    label: "Source/Sink Error Reporting"
    depends_on: build-aarch64
    parallelism: 3
    timeout_in_minutes: 30
    agents:
      queue: hetzner-aarch64-4cpu-8gb
    plugins:
      - ./ci/plugins/mzcompose:
          composition: source-sink-errors

  # Fast tests closer to the end, doesn't matter as much if they have to wait
  # for an agent
  - id: persistence
    label: Persistence tests
    depends_on: build-aarch64
    timeout_in_minutes: 30
    plugins:
      - ./ci/plugins/mzcompose:
          composition: persistence
    agents:
      queue: hetzner-aarch64-8cpu-16gb

  - id: cluster-isolation
    label: Cluster isolation test
    depends_on: build-aarch64
    timeout_in_minutes: 20
    inputs: [test/cluster-isolation]
    plugins:
      - ./ci/plugins/mzcompose:
          composition: cluster-isolation
    agents:
      queue: hetzner-aarch64-4cpu-8gb

  - id: dbt-materialize
    label: dbt-materialize tests
    depends_on: build-aarch64
    timeout_in_minutes: 30
    plugins:
      - ./ci/plugins/mzcompose:
          composition: dbt-materialize
    agents:
      queue: hetzner-aarch64-4cpu-8gb

  - group: "Debezium tests"
    key: debezium-tests
    steps:
      - id: debezium-postgres
        label: "Debezium Postgres tests"
        depends_on: build-aarch64
        timeout_in_minutes: 30
        inputs: [test/debezium]
        plugins:
          - ./ci/plugins/mzcompose:
              composition: debezium
              run: postgres
        agents:
          queue: hetzner-aarch64-8cpu-16gb

      - id: debezium-sql-server
        label: "Debezium SQL Server tests"
        depends_on: build-x86_64
        timeout_in_minutes: 30
        inputs: [test/debezium]
        plugins:
          - ./ci/plugins/mzcompose:
              composition: debezium
              run: sql-server
        agents:
          # too slow to run emulated on aarch64, SQL Server's docker image is not yet available for aarch64 natively yet: https://github.com/microsoft/mssql-docker/issues/802
          queue: hetzner-x86-64-4cpu-8gb

      - id: debezium-mysql
        label: "Debezium MySQL tests"
        depends_on: build-aarch64
        timeout_in_minutes: 30
        inputs: [test/debezium]
        plugins:
          - ./ci/plugins/mzcompose:
              composition: debezium
              run: mysql
        agents:
          queue: hetzner-aarch64-4cpu-8gb

  - id: storage-usage
    label: "Storage Usage Table Test"
    depends_on: build-aarch64
    timeout_in_minutes: 30
    agents:
      queue: hetzner-aarch64-4cpu-8gb
    plugins:
      - ./ci/plugins/mzcompose:
          composition: storage-usage

  - id: tracing
    label: "Tracing Fast Path"
    depends_on: build-aarch64
    timeout_in_minutes: 30
    inputs: [test/tracing]
    plugins:
      - ./ci/plugins/mzcompose:
          composition: tracing
    agents:
      # Requires BUILDKITE_SENTRY_DSN
      queue: linux-aarch64-small

  - id: rtr-combined
    label: RTR with all sources
    depends_on: build-aarch64
    timeout_in_minutes: 30
    inputs: [test/rtr-combined]
    plugins:
      - ./ci/plugins/mzcompose:
          composition: rtr-combined
    agents:
      queue: hetzner-aarch64-16cpu-32gb

  - id: skip-version-upgrade
    label: "Skip Version Upgrade"
    depends_on: build-aarch64
    timeout_in_minutes: 30
    inputs: [doc/user/content/releases]
    plugins:
      - ./ci/plugins/mzcompose:
          composition: skip-version-upgrade
    agents:
      queue: hetzner-aarch64-4cpu-8gb
    skip: "Version upgrade skips are allowed for Self-Managed releases now"

  - id: mz-debug
    label: "mz-debug tool"
    depends_on: build-aarch64
    timeout_in_minutes: 45
    inputs: [test/mz-debug]
    plugins:
      - ./ci/plugins/mzcompose:
          composition: mz-debug
    agents:
      queue: hetzner-aarch64-4cpu-8gb

  - id: secrets-logging
    label: "Secrets Logging"
    depends_on: build-aarch64
    timeout_in_minutes: 45
    plugins:
      - ./ci/plugins/mzcompose:
          composition: secrets-logging
    agents:
      queue: hetzner-aarch64-4cpu-8gb

  - id: copy-to-s3
    label: Copy to S3
    depends_on: build-aarch64
    timeout_in_minutes: 30
    inputs: [test/copy-to-s3]
    plugins:
      - ./ci/plugins/mzcompose:
          composition: copy-to-s3
          run: ci
    agents:
      queue: hetzner-aarch64-4cpu-8gb

  - id: mcp-materialize
    label: mcp-materialize tests
    depends_on: build-aarch64
    timeout_in_minutes: 30
    plugins:
      - ./ci/plugins/mzcompose:
          composition: mcp-materialize
    agents:
      queue: hetzner-aarch64-4cpu-8gb

  - id: chbench-demo
    label: chbench smoke test
    depends_on: build-aarch64
    plugins:
      - ./ci/plugins/mzcompose:
          composition: chbench
          run: no-load
          args: [--run-seconds=10, --wait]
    timeout_in_minutes: 30
    agents:
      queue: hetzner-aarch64-4cpu-8gb

  - id: metabase-demo
    label: Metabase smoke test
    depends_on: build-x86_64
    timeout_in_minutes: 30
    plugins:
      - ./ci/plugins/mzcompose:
          composition: metabase
    agents:
      # too slow to run emulated on aarch64, Metabase'ss docker image is not yet available for aarch64 natively yet: https://github.com/metabase/metabase/issues/13119
      queue: hetzner-x86-64-4cpu-8gb

  - group: Docs tests
    key: docs-tests
    label: ":rust: Docs tests"
    steps:
      - id: lint-docs
        label: Lint docs
        command: bin/ci-builder run stable ci/test/lint-docs.sh
        inputs:
          - doc/user
          - src/adapter/src/catalog
          - test/sqllogictest/autogenerated
        depends_on: []
        timeout_in_minutes: 30
        agents:
          # hugo: command not found
          queue: hetzner-x86-64-4cpu-8gb
        coverage: skip
        sanitizer: skip

      - id: preview-docs
        label: Preview docs
        command: bin/ci-builder run stable ci/test/preview-docs.sh
        inputs: [doc/user]
        depends_on: []
        timeout_in_minutes: 30
        agents:
          # hugo: command not found
          queue: linux-x86_64-small
        coverage: skip
        sanitizer: skip

      - id: docs-widgets-test
        label: Run Docs JS Widgets Tests
        command: bin/ci-builder run stable ci/test/docs-widgets/docs-widgets.sh
        inputs:
          - ci/test/docs-widgets/**
          - doc/user
        depends_on: []
        timeout_in_minutes: 15
        agents:
          queue: hetzner-aarch64-4cpu-8gb
        coverage: skip
        sanitizer: skip

  - id: deploy-website
    label: Deploy website
    depends_on: lint-docs
    trigger: deploy-website
    async: true
    branches: "main self-managed-docs/*"
    build:
      commit: "$BUILDKITE_COMMIT"
      branch: "$BUILDKITE_BRANCH"
      env:
        BUILDKITE_TAG: "$BUILDKITE_TAG"
    coverage: skip
    sanitizer: skip

  - id: release-qualification
    label: Release qualification
    depends_on: devel-docker-tags
    trigger: release-qualification
    async: true
    build:
      commit: "$BUILDKITE_COMMIT"
      branch: "$BUILDKITE_BRANCH"
      env:
        BUILDKITE_TAG: "$BUILDKITE_TAG"
    if: build.tag != "" && build.branch =~ /^v.*\..*\$/
    coverage: skip
    sanitizer: skip

  - id: nightly-if-release
    label: Nightly for releases
    depends_on: devel-docker-tags
    trigger: nightly
    async: true
    build:
      commit: "$BUILDKITE_COMMIT"
      branch: "$BUILDKITE_BRANCH"
      env:
        BUILDKITE_TAG: "$BUILDKITE_TAG"
    if: build.tag != "" && build.branch =~ /^v.*\..*\$/
    coverage: skip
    sanitizer: skip

  - wait: ~
    continue_on_failure: true

  - id: coverage-pr-analyze
    label: Analyze code coverage for PR
    timeout_in_minutes: 20
    command: bin/ci-builder run stable ci/test/coverage_report.sh
    inputs: ["*"]
    priority: 1
    agents:
      queue: hetzner-aarch64-8cpu-16gb
    coverage: only

  - wait: ~
    continue_on_failure: true

  - id: deploy
    label: Deploy
    trigger: deploy
    async: true
    branches: "main v*.*"
    build:
      commit: "$BUILDKITE_COMMIT"
      branch: "$BUILDKITE_BRANCH"
      env:
        BUILDKITE_TAG: "$BUILDKITE_TAG"
    coverage: skip
    sanitizer: skip