name: System Tests

on:
  workflow_call: # allows to reuse this workflow
    inputs:
      branch_ref:
        description: 'The branch to run the workflow on'
        required: true
        type: string
  push:
    branches:
      - release-v*
      - mq-working-branch-**
    tags-ignore:
      - 'contrib/**'
      - 'instrumentation/**'
      - 'internal/**'
      - 'orchestrion/**'
      - 'scripts/**'
  pull_request:
  workflow_dispatch:
    inputs:
        ref:
            description: 'System Tests ref/tag/branch'
            required: true
            default: main
            type: string
  schedule:
    - cron:  '00 04 * * 2-6'

permissions:
  contents: read
  packages: write

jobs:
  warm-repo-cache:
    runs-on: ubuntu-latest
    outputs:
      sha: ${{ steps.pin.outputs.sha }}
    steps:
      - name: Checkout system-tests
        uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
        with:
          repository: 'DataDog/system-tests'
          ref: ${{ inputs.ref }}
      - name: Pin exact commit SHA for system-tests
        id: pin
        run: |
          echo "sha=$(git rev-parse FETCH_HEAD)" >> "$GITHUB_OUTPUT"
      - name: Cache
        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
        with:
          path: .git
          key: gitdb-system-tests-${{ steps.pin.outputs.sha }}
  system-tests:
    if: github.event_name != 'pull_request' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'DataDog/dd-trace-go')
    # Note: Not using large runners because the jobs spawned by this pipeline
    # don't seem to get a noticable speedup from using larger runners.
    runs-on: ubuntu-latest
    needs:
      - warm-repo-cache
    strategy:
      matrix:
        weblog-variant:
          - net-http
          - echo
          - chi
          - gin
          - uds-echo
        scenario:
          - DEFAULT
          - INTEGRATIONS
          - CROSSED_TRACING_LIBRARIES
          - APPSEC_DISABLED
          - APPSEC_BLOCKING
          - APPSEC_BLOCKING_FULL_DENYLIST
          - APPSEC_API_SECURITY
          - APPSEC_RASP
          - APPSEC_RUNTIME_ACTIVATION
          - APM_TRACING_E2E_SINGLE_SPAN
          - APM_TRACING_E2E_OTEL
          - TRACING_CONFIG_SCENARIOS
        include:
          - weblog-variant: net-http
            scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES
          - weblog-variant: net-http
            scenario: REMOTE_CONFIG_MOCKED_BACKEND_LIVE_DEBUGGING
          - weblog-variant: net-http
            scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD
          # AppSec scenarios that don't depend on the integrations, so we just run on the net/http variant
          - weblog-variant: net-http
            scenario: APPSEC_RATE_LIMITER
          - weblog-variant: net-http
            scenario: APPSEC_CUSTOM_RULES
          - weblog-variant: net-http
            scenario: APPSEC_MISSING_RULES
          - weblog-variant: net-http
            scenario: APPSEC_CORRUPTED_RULES
          - weblog-variant: net-http
            scenario: APPSEC_LOW_WAF_TIMEOUT
          - weblog-variant: net-http
            scenario: APPSEC_STANDALONE
          - weblog-variant: net-http
            scenario: APPSEC_STANDALONE_V2
          - weblog-variant: net-http
            scenario: APPSEC_META_STRUCT_DISABLED
          - weblog-variant: net-http
            scenario: APPSEC_CUSTOM_OBFUSCATION
          - weblog-variant: net-http
            scenario: APPSEC_WAF_TELEMETRY
          # APM scenarios requiring specific environment settings
          - scenario: APM_TRACING_E2E_SINGLE_SPAN
            env:
              DD_API_KEY=$SYSTEM_TESTS_E2E_DD_API_KEY
              DD_APPLICATION_KEY=$SYSTEM_TESTS_E2E_DD_APP_KEY
              DD_SITE="datadoghq.com"
          - scenario: APM_TRACING_E2E_OTEL
            env:
              DD_API_KEY=$SYSTEM_TESTS_E2E_DD_API_KEY
              DD_APPLICATION_KEY=$SYSTEM_TESTS_E2E_DD_APP_KEY
              DD_SITE="datadoghq.com"
          # GraphQL System Tests
          - scenario: GRAPHQL_APPSEC
            weblog-variant: graph-gophers
          - scenario: GRAPHQL_APPSEC
            weblog-variant: graphql-go
          - scenario: GRAPHQL_APPSEC
            weblog-variant: gqlgen
          # Telemetry
          - weblog-variant: net-http
            scenario: TELEMETRY_APP_STARTED_PRODUCTS_DISABLED
          - weblog-variant: net-http
            scenario: TELEMETRY_LOG_GENERATION_DISABLED
          - weblog-variant: net-http
            # TELEMETRY_METRIC_GENERATION_ENABLED scenario merged into AGENT_SUPPORTING_SPAN_EVENTS
            scenario: AGENT_SUPPORTING_SPAN_EVENTS
          - weblog-variant: net-http
            scenario: TELEMETRY_METRIC_GENERATION_DISABLED
          - weblog-variant: net-http
            scenario: APM_TRACING_EFFICIENT_PAYLOAD
          - weblog-variant: net-http
            scenario: FEATURE_FLAGGING_AND_EXPERIMENTATION
      fail-fast: false
    env:
      TEST_LIBRARY: golang
      WEBLOG_VARIANT: ${{ matrix.weblog-variant }}
      DD_API_KEY: ${{ secrets.DD_API_KEY }}
      SYSTEM_TESTS_E2E_DD_API_KEY: ${{ secrets.SYSTEM_TESTS_E2E_DD_API_KEY }}
      SYSTEM_TESTS_E2E_DD_APP_KEY: ${{ secrets.SYSTEM_TESTS_E2E_DD_APP_KEY }}
    name: Test (${{ matrix.weblog-variant }}, ${{ matrix.scenario }})
    steps:
      - name: Restore repo cache
        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
        with:
          path: .git
          key: gitdb-system-tests-${{ needs.warm-repo-cache.outputs.sha }}

      - name: Checkout system tests
        shell: bash
        run: |
          git config safe.directory "$GITHUB_WORKSPACE"
          git checkout -f ${{ needs.warm-repo-cache.outputs.sha }}

      - name: Checkout dd-trace-go
        uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
        with:
          ref: ${{ inputs.branch_ref || github.ref }}
          path: 'binaries/dd-trace-go'

      - name: Build weblog
        run: ./build.sh -i weblog

      - name: Build runner
        uses: ./.github/actions/install_runner

      - name: Build agent
        run: ./build.sh -i agent

      - name: Run
        run: env ${{ matrix.env }} ./run.sh ${{ matrix.scenario }}

      - name: Compress artifact
        if: ${{ always() }}
        run: |
          if compgen -G "logs*/" > /dev/null; then
            tar -czvf artifact.tar.gz logs*/
          else
            echo "No logs*/ found; skipping tar."
          fi

      - name: Upload artifact
        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
        if: ${{ always() }}
        with:
          name: logs_${{ matrix.weblog-variant }}_${{ matrix.scenario }}
          path: artifact.tar.gz

  # Pushing an image tagged with "dev" only on commit to main,
  # otherwise build and use the image from the artifact.
  build-service-extensions-callout:
    if: github.event_name != 'pull_request' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'DataDog/dd-trace-go')
    uses: ./.github/workflows/docker-build-and-push.yml
    with:
      image: ghcr.io/datadog/dd-trace-go/service-extensions-callout
      dockerfile: ./contrib/envoyproxy/go-control-plane/cmd/serviceextensions/Dockerfile
      artifact_prefix: service-extensions-image
      commit_sha: ${{ github.sha }}
      tags: >-
        dev
      push: ${{ github.ref == 'refs/heads/main' }}
      platforms: '["linux/amd64"]'

  # Pushing an image tagged with "dev" only on commit to main,
  # otherwise build and use the image from the artifact.
  build-haproxy:
    if: github.event_name != 'pull_request' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'DataDog/dd-trace-go')
    uses: ./.github/workflows/docker-build-and-push.yml
    with:
      image: ghcr.io/datadog/dd-trace-go/haproxy-spoa
      dockerfile: ./contrib/haproxy/stream-processing-offload/cmd/spoa/Dockerfile
      artifact_prefix: haproxy-spoa-image
      commit_sha: ${{ github.sha }}
      tags: >-
        dev
      push: ${{ github.ref == 'refs/heads/main' }}
      platforms: '["linux/amd64"]'

  system-tests-docker-images:
    if: github.event_name != 'pull_request' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'DataDog/dd-trace-go')
    runs-on: ubuntu-latest
    needs:
      - build-service-extensions-callout
      - build-haproxy
    strategy:
      fail-fast: false
      matrix:
        include:
          - target: external_processing
            scenario: EXTERNAL_PROCESSING
            image_name: service-extensions-callout
            image_artifact: service-extensions-image-linux-amd64
            system_tests_name: external_processing
          - target: external_processing
            scenario: EXTERNAL_PROCESSING_BLOCKING
            image_name: service-extensions-callout
            image_artifact: service-extensions-image-linux-amd64
            system_tests_name: external_processing
          - target: haproxy
            scenario: STREAM_PROCESSING_OFFLOAD
            image_name: haproxy-spoa
            image_artifact: haproxy-spoa-image-linux-amd64
            system_tests_name: stream_processing_offload
          - target: haproxy
            scenario: STREAM_PROCESSING_OFFLOAD_BLOCKING
            image_name: haproxy-spoa
            image_artifact: haproxy-spoa-image-linux-amd64
            system_tests_name: stream_processing_offload

    name: Test (${{ matrix.system_tests_name }}, ${{ matrix.scenario }})
    steps:
      - name: Download ${{ matrix.target }}) artifacts
        if: github.ref != 'refs/heads/main'
        uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0
        with:
          name: ${{ matrix.image_artifact }}

      - name: Load ${{ matrix.target }} image
        if: github.ref != 'refs/heads/main'
        run: docker load -i ${{ matrix.image_artifact }}.tar
      - name: Checkout system tests
        uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
        with:
          repository: 'DataDog/system-tests'
          ref: ${{ inputs.ref }}

      - name: Set ${{ matrix.image_name }} image name
        run: echo "ghcr.io/datadog/dd-trace-go/${{ matrix.image_name }}:dev" > binaries/golang-${{ matrix.image_name }}-image

      - name: Checkout dd-trace-go
        uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
        with:
          ref: ${{ inputs.branch_ref || github.ref }}
          path: 'binaries/dd-trace-go'

      - name: Build runner
        uses: ./.github/actions/install_runner

      - name: Build agent
        run: ./build.sh -i agent

      - name: Run
        run: ./run.sh ${{ matrix.scenario }}

      - name: Compress artifact
        if: ${{ always() }}
        run: |
          if compgen -G "logs*/" > /dev/null; then
            tar -czvf artifact.tar.gz logs*/
          else
            echo "No logs*/ found; skipping tar."
          fi

      - name: Upload artifact
        uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
        if: ${{ always() }}
        with:
          name: logs_${{ matrix.system_tests_name }}_${{ matrix.scenario }}
          path: artifact.tar.gz

  system-tests-done:
    name: System Tests
    needs:
      - system-tests
      - system-tests-docker-images
    runs-on: ubuntu-latest
    if: success() || failure()
    steps:
      - name: Success
        if: needs.system-tests.result == 'success' && needs.system-tests-docker-images.result == 'success'
        run: echo "Success!"
      - name: Failure
        if: needs.system-tests.result != 'success' || needs.system-tests-docker-images.result != 'success'
        run: echo "Failure!" && exit 1
