1# This file defines our primary CI workflow that runs on pull requests2# and also on pushes to special branches (auto, try).3#4# The actual definition of the executed jobs is calculated by the5# `src/ci/citool` crate, which6# uses job definition data from src/ci/github-actions/jobs.yml.7# You should primarily modify the `jobs.yml` file if you want to modify8# what jobs are executed in CI.910name: CI11on:12  push:13    branches:14      - automation/bors/auto15      - automation/bors/try16      - try-perf17  pull_request:18    branches:19      - "**"2021permissions:22  contents: read23  packages: write2425defaults:26  run:27    # On Linux, macOS, and Windows, use the system-provided bash as the default28    # shell. (This should only make a difference on Windows, where the default29    # shell is PowerShell.)30    shell: bash3132concurrency:33  # For a given workflow, if we push to the same branch, cancel all previous builds on that branch.34  # We add an exception for try builds (automation/bors/try branch) and unrolled rollup builds35  # (try-perf), which are all triggered on the same branch, but which should be able to run36  # concurrently.37  group: ${{ github.workflow }}-${{ ((github.ref == 'refs/heads/try-perf' || github.ref == 'refs/heads/automation/bors/try') && github.sha) || github.ref }}38  cancel-in-progress: true39env:40  TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate"41  # This will be empty in PR jobs.42  TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}43jobs:44  # The job matrix for `calculate_matrix` is defined in src/ci/github-actions/jobs.yml.45  # It calculates which jobs should be executed, based on the data of the ${{ github }} context.46  # If you want to modify CI jobs, take a look at src/ci/github-actions/jobs.yml.47  calculate_matrix:48    name: Calculate job matrix49    runs-on: ubuntu-24.04-arm50    outputs:51      jobs: ${{ steps.jobs.outputs.jobs }}52      run_type: ${{ steps.jobs.outputs.run_type }}53    steps:54      - name: Checkout the source code55        uses: actions/checkout@v556      - name: Test citool57        # Only test citool on the auto branch, to reduce latency of the calculate matrix job58        # on PR/try builds.59        if: ${{ github.ref == 'refs/heads/automation/bors/auto' }}60        run: |61          cd src/ci/citool62          CARGO_INCREMENTAL=0 cargo test63      - name: Calculate the CI job matrix64        env:65          COMMIT_MESSAGE: ${{ github.event.head_commit.message }}66        run: |67          cd src/ci/citool68          CARGO_INCREMENTAL=0 cargo run calculate-job-matrix >> $GITHUB_OUTPUT69        id: jobs70  job:71    name: ${{ matrix.full_name }}72    needs: [ calculate_matrix ]73    runs-on: "${{ matrix.os }}"74    timeout-minutes: 36075    # The bors environment contains secrets required for elevated workflows (try and auto builds),76    # which need to access e.g. S3 and upload artifacts. We want to provide access to that77    # environment only on the try/auto branches, which are only accessible to bors.78    # This also ensures that PR CI (which doesn't get write access to S3) works, as it cannot79    # access the environment.80    #81    # We only enable the environment for the rust-lang/rust repository, so that CI works on forks.82    environment: ${{ ((github.repository == 'rust-lang/rust' && (github.ref == 'refs/heads/try-perf' || github.ref == 'refs/heads/automation/bors/try' || github.ref == 'refs/heads/automation/bors/auto')) && 'bors') || '' }}83    env:84      CI_JOB_NAME: ${{ matrix.name }}85      CI_JOB_DOC_URL: ${{ matrix.doc_url }}86      GITHUB_WORKFLOW_RUN_ID: ${{ github.run_id }}87      GITHUB_REPOSITORY: ${{ github.repository }}88      CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse89      # commit of PR sha or commit sha. `GITHUB_SHA` is not accurate for PRs.90      HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}91      DOCKER_TOKEN: ${{ secrets.GITHUB_TOKEN }}92      SCCACHE_BUCKET: rust-lang-ci-sccache293      SCCACHE_REGION: us-west-194      CACHE_DOMAIN: ci-caches.rust-lang.org95    continue-on-error: ${{ matrix.continue_on_error || false }}96    strategy:97      # If the user starts multiple jobs in a try build, let them all finish.98      # Try builds are sometimes used to test several jobs at once, and it is useful to know which99      # of them would succeed or not.100      fail-fast: ${{ needs.calculate_matrix.outputs.run_type != 'try' }}101      matrix:102        # Check the `calculate_matrix` job to see how is the matrix defined.103        include: ${{ fromJSON(needs.calculate_matrix.outputs.jobs) }}104    steps:105      - name: Install cargo in AWS CodeBuild106        if: matrix.codebuild107        run: |108          # Check if cargo is installed109          if ! command -v cargo &> /dev/null; then110            echo "Cargo not found, installing Rust..."111            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile=minimal112            # Make cargo available in PATH113            echo "$HOME/.cargo/bin" >> $GITHUB_PATH114          fi115116      - name: disable git crlf conversion117        run: git config --global core.autocrlf false118119      - name: checkout the source code120        uses: actions/checkout@v5121        with:122          fetch-depth: 2123124      # Free up disk space on Linux by removing preinstalled components that125      # we do not need. We do this to enable some of the less resource126      # intensive jobs to run on free runners, which however also have127      # less disk space.128      - name: free up disk space129        run: src/ci/scripts/free-disk-space-linux.sh130        if: matrix.free_disk131132      # If we don't need to free up disk space then just report how much space we have133      - name: print disk usage134        run: |135          echo "disk usage:"136          df -h137        if: matrix.free_disk == false138139      # Rust Log Analyzer can't currently detect the PR number of a GitHub140      # Actions build on its own, so a hint in the log message is needed to141      # point it in the right direction.142      - name: configure the PR in which the error message will be posted143        run: echo "[CI_PR_NUMBER=$num]"144        env:145          num: ${{ github.event.number }}146        if: needs.calculate_matrix.outputs.run_type == 'pr'147148      - name: add extra environment variables149        run: src/ci/scripts/setup-environment.sh150        env:151          # Since it's not possible to merge `${{ matrix.env }}` with the other152          # variables in `job.<name>.env`, the variables defined in the matrix153          # are passed to the `setup-environment.sh` script encoded in JSON,154          # which then uses log commands to actually set them.155          EXTRA_VARIABLES: ${{ toJson(matrix.env) }}156157      - name: ensure the channel matches the target branch158        run: src/ci/scripts/verify-channel.sh159160      - name: collect CPU statistics161        run: src/ci/scripts/collect-cpu-stats.sh162163      - name: show the current environment164        run: src/ci/scripts/dump-environment.sh165166      - name: install awscli167        run: src/ci/scripts/install-awscli.sh168169      - name: install sccache170        run: src/ci/scripts/install-sccache.sh171172      - name: install clang173        run: src/ci/scripts/install-clang.sh174175      - name: install tidy176        run: src/ci/scripts/install-tidy.sh177178      - name: install WIX179        run: src/ci/scripts/install-wix.sh180181      - name: disable git crlf conversion182        run: src/ci/scripts/disable-git-crlf-conversion.sh183184      - name: checkout submodules185        run: src/ci/scripts/checkout-submodules.sh186187      - name: install MinGW188        run: src/ci/scripts/install-mingw.sh189190      - name: install ninja191        run: src/ci/scripts/install-ninja.sh192193      - name: enable ipv6 on Docker194        # Don't run on codebuild because systemctl is not available195        if: ${{ !matrix.codebuild }}196        run: src/ci/scripts/enable-docker-ipv6.sh197198      # Disable automatic line ending conversion (again). On Windows, when we're199      # installing dependencies, something switches the git configuration directory or200      # re-enables autocrlf. We've not tracked down the exact cause -- and there may201      # be multiple -- but this should ensure submodules are checked out with the202      # appropriate line endings.203      - name: disable git crlf conversion204        run: src/ci/scripts/disable-git-crlf-conversion.sh205206      - name: ensure line endings are correct207        run: src/ci/scripts/verify-line-endings.sh208209      - name: ensure backported commits are in upstream branches210        run: src/ci/scripts/verify-backported-commits.sh211212      - name: ensure the stable version number is correct213        run: src/ci/scripts/verify-stable-version-number.sh214215      # Show the environment just before we run the build216      # This makes it easier to diagnose problems with the above install scripts.217      - name: show the current environment218        run: src/ci/scripts/dump-environment.sh219220      # Pre-build citool before the following step uninstalls rustup221      # Build it into the build directory, to avoid modifying sources222      - name: build citool223        run: |224          cd src/ci/citool225          CARGO_INCREMENTAL=0 CARGO_TARGET_DIR=../../../build/citool cargo build226227      - name: run the build228        run: |229          set +e230          # Redirect stderr to stdout to avoid reordering the two streams in the GHA logs.231          src/ci/scripts/run-build-from-ci.sh 2>&1232          STATUS=$?233          set -e234235          if [[ "$STATUS" -ne 0 && -n "$CI_JOB_DOC_URL" ]]; then236            echo "****************************************************************************"237            echo "To find more information about this job, visit the following URL:"238            echo "$CI_JOB_DOC_URL"239            echo "****************************************************************************"240          fi241          exit ${STATUS}242        env:243          AWS_ACCESS_KEY_ID: ${{ secrets.CACHES_AWS_ACCESS_KEY_ID }}244          AWS_SECRET_ACCESS_KEY: ${{ secrets.CACHES_AWS_SECRET_ACCESS_KEY }}245246      - name: create github artifacts247        run: src/ci/scripts/create-doc-artifacts.sh248249      - name: print disk usage250        # We also want to know the disk usage when the job fails.251        if: always()252        run: |253          echo "disk usage:"254          df -h255256      - name: upload artifacts to github257        uses: actions/upload-artifact@v7258        with:259          # name is set in previous step260          name: ${{ env.DOC_ARTIFACT_NAME }}261          path: obj/artifacts/doc262          if-no-files-found: ignore263          retention-days: 5264265      - name: upload artifacts to S3266        run: src/ci/scripts/upload-artifacts.sh267        env:268          AWS_ACCESS_KEY_ID: ${{ secrets.ARTIFACTS_AWS_ACCESS_KEY_ID }}269          AWS_SECRET_ACCESS_KEY: ${{ secrets.ARTIFACTS_AWS_SECRET_ACCESS_KEY }}270        # Adding a condition on DEPLOY=1 or DEPLOY_ALT=1 is not needed as all deploy271        # builders *should* have the AWS credentials available. Still, explicitly272        # adding the condition is helpful as this way CI will not silently skip273        # deploying artifacts from a dist builder if the variables are misconfigured,274        # erroring about invalid credentials instead.275        if: github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1'276277      - name: postprocess metrics into the summary278        # This step is not critical, and if some I/O problem happens, we don't want279        # to cancel the build.280        continue-on-error: true281        run: |282          if [ -f build/metrics.json ]; then283            METRICS=build/metrics.json284          elif [ -f obj/build/metrics.json ]; then285            METRICS=obj/build/metrics.json286          else287            echo "No metrics.json found"288            exit 0289          fi290291          # Get closest bors merge commit292          PARENT_COMMIT=`git rev-list --author='bors@rust-lang.org' --author='122020455+rust-bors\[bot\]@users.noreply.github.com' -n1 --first-parent HEAD^1`293294          ./build/citool/debug/citool postprocess-metrics \295              --job-name ${CI_JOB_NAME} \296              --parent ${PARENT_COMMIT} \297              ${METRICS} >> ${GITHUB_STEP_SUMMARY}298299      - name: upload job metrics to DataDog300        # This step is not critical, and if some I/O problem happens, we don't want301        # to cancel the build.302        continue-on-error: true303        if: needs.calculate_matrix.outputs.run_type != 'pr'304        env:305          DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }}306          DD_GITHUB_JOB_NAME: ${{ matrix.full_name }}307        run: ./build/citool/debug/citool upload-build-metrics build/cpu-usage.csv308309  # This job is used to publish toolstate for successful auto builds.310  outcome:311    name: publish toolstate312    runs-on: ubuntu-24.04313    needs: [ calculate_matrix, job ]314    if: ${{ needs.calculate_matrix.outputs.run_type == 'auto' }}315    environment: ${{ (github.repository == 'rust-lang/rust' && 'bors') || '' }}316    steps:317      - name: checkout the source code318        uses: actions/checkout@v5319        with:320          fetch-depth: 2321      # Publish the toolstate if an auto build succeeds (just before push to the default branch)322      - name: publish toolstate323        run: src/ci/publish_toolstate.sh324        shell: bash325        env:326          TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/rust-lang/rust/issues327          TOOLSTATE_PUBLISH: 1