coreutils/.github/workflows/CICD.yml at main · uutils/coreutils

905 lines (871 loc) · 40.8 KB
# spell-checker:ignore (abbrev/names) CACHEDIR CICD CodeCOV MacOS MinGW MSVC musl taiki
# spell-checker:ignore (env/flags) Awarnings Ccodegen Coverflow Cpanic Dwarnings RUSTDOCFLAGS RUSTFLAGS Zpanic CARGOFLAGS CLEVEL nodocs
# spell-checker:ignore (jargon) SHAs deps dequote softprops subshell toolchain fuzzers dedupe devel profdata
# spell-checker:ignore (people) Peltoche rivy Anson dawidd
# spell-checker:ignore (shell/tools) binutils choco clippy dmake esac fakeroot fdesc fdescfs gmake grcov halium lcov libclang libfuse libssl limactl nextest nocross pacman popd printf pushd redoxer rsync rustc rustfmt rustup shopt sccache utmpdump xargs zstd
# spell-checker:ignore (misc) aarch alnum armhf bindir busytest coreutils defconfig DESTDIR gecos getenforce gnueabihf issuecomment maint manpages msys multisize noconfirm nofeatures nullglob onexitbegin onexitend pell runtest tempfile testsuite toybox uutils libsystemd codspeed wasip libexecinfo
  PROJECT_NAME: coreutils
  PROJECT_DESC: "Core universal (cross-platform) utilities"
  PROJECT_AUTH: "uutils"
  RUST_MIN_SRV: "1.88.0"
  CARGO_INCREMENTAL: "0"
  # * style job configuration
  STYLE_FAIL_ON_FAULT: true ## (bool) fail the build if a style job contains a fault (error or warning); may be overridden on a per-job basis
  pull_request:
      - '*'
    branches:
      - '*'
permissions:
  contents: read # to fetch code (actions/checkout)
# End the current execution if there is a new changeset in the PR.
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
  cargo-deny:
    name: Style/cargo-deny
    runs-on: ubuntu-latest
    - uses: actions/checkout@v6
      with:
        persist-credentials: false
    - uses: EmbarkStudios/cargo-deny-action@v2
  style_deps:
    ## ToDO: [2021-11-10; rivy] 'Style/deps' needs more informative output and better integration of results into the GHA dashboard
    name: Style/deps
    runs-on: ${{ matrix.job.os }}
    strategy:
      fail-fast: false
      matrix:
        job:
          # note: `cargo-udeps` panics when processing stdbuf/libstdbuf ("uu_stdbuf_libstdbuf"); either b/c of the 'cpp' crate or 'libstdbuf' itself
          #   ... b/c of the panic, a more limited feature set is tested (though only excluding `stdbuf`)
          - { os: ubuntu-latest  , features: "feat_Tier1,feat_require_unix,feat_require_unix_utmpx" }
          - { os: macos-latest   , features: "feat_Tier1,feat_require_unix,feat_require_unix_utmpx" }
          - { os: windows-latest , features: feat_os_windows }
    - uses: actions/checkout@v6
      with:
        persist-credentials: false
    - uses: taiki-e/install-action@cargo-udeps
    - uses: Swatinem/rust-cache@v2
    - name: Initialize workflow variables
      id: vars
      shell: bash
      run: |
        echo "RUSTC_BOOTSTRAP=1" >> "${GITHUB_ENV}" # Use -Z
        ## VARs setup
        outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; }
        # failure mode
        unset FAIL_ON_FAULT ; case '${{ env.STYLE_FAIL_ON_FAULT }}' in
          ''|0|f|false|n|no|off) FAULT_TYPE=warning ;;
          *) FAIL_ON_FAULT=true ; FAULT_TYPE=error ;;
        esac;
        outputs FAIL_ON_FAULT FAULT_TYPE
        # target-specific options
        # * CARGO_FEATURES_OPTION
        CARGO_FEATURES_OPTION='' ;
        if [ -n "${{ matrix.job.features }}" ]; then CARGO_FEATURES_OPTION='--features "${{ matrix.job.features }}"' ; fi
        outputs CARGO_FEATURES_OPTION
    - name: Detect unused dependencies
      shell: bash
      run: |
        ## Detect unused dependencies
        unset fault
        fault_type="${{ steps.vars.outputs.FAULT_TYPE }}"
        fault_prefix=$(echo "$fault_type" | tr '[:lower:]' '[:upper:]')
        cargo udeps ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} --all-targets &> udeps.log || cat udeps.log
        grep --ignore-case "all deps seem to have been used" udeps.log || { printf "%s\n" "::${fault_type} ::${fault_prefix}: \`cargo udeps\`: style violation (unused dependency found)" ; fault=true ; }
        if [ -n "${{ steps.vars.outputs.FAIL_ON_FAULT }}" ] && [ -n "$fault" ]; then exit 1 ; fi
  doc_warnings:
    name: Documentation/warnings
    runs-on: ${{ matrix.job.os }}
    strategy:
      fail-fast: false
      matrix:
        job:
          - { os: ubuntu-latest  , features: feat_os_unix }
# for now, don't build it on mac & windows because the doc is only published from linux
# + it needs a bunch of duplication for build
# and I don't want to add a doc step in the regular build to avoid long builds
#          - { os: macos-latest   , features: feat_os_unix }
#          - { os: windows-latest , features: feat_os_windows }
    - uses: actions/checkout@v6
      with:
        persist-credentials: false
    - uses: dtolnay/rust-toolchain@master
      with:
        toolchain: stable
        components: clippy
    - uses: Swatinem/rust-cache@v2
    - name: Run sccache-cache
      id: sccache-setup
      uses: mozilla-actions/sccache-action@v0.0.9
      continue-on-error: true
    - name: Export sccache
      if: steps.sccache-setup.outcome == 'success'
      run: |
        echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
        echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV
    - name: Install/setup prerequisites
      shell: bash
      run: |
        sudo apt-get -y update ; sudo apt-get -y install gcc-aarch64-linux-gnu libselinux1-dev
    - name: Initialize workflow variables
      id: vars
      shell: bash
      run: |
        ## VARs setup
        outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; }
        # failure mode
        unset FAIL_ON_FAULT ; case '${{ env.STYLE_FAIL_ON_FAULT }}' in
          ''|0|f|false|n|no|off) FAULT_TYPE=warning ;;
          *) FAIL_ON_FAULT=true ; FAULT_TYPE=error ;;
        esac;
        outputs FAIL_ON_FAULT FAULT_TYPE
        # target-specific options
        # * CARGO_FEATURES_OPTION
        CARGO_FEATURES_OPTION='--all-features' ;
        if [ -n "${{ matrix.job.features }}" ]; then CARGO_FEATURES_OPTION='--features ${{ matrix.job.features }}' ; fi
        outputs CARGO_FEATURES_OPTION
        # * determine sub-crate utility list
        UTILITY_LIST="$(./util/show-utils.sh ${CARGO_FEATURES_OPTION})"
        echo UTILITY_LIST=${UTILITY_LIST}
        CARGO_UTILITY_LIST_OPTIONS="$(for u in ${UTILITY_LIST}; do echo -n "-puu_${u} "; done;)"
        outputs CARGO_UTILITY_LIST_OPTIONS
    - name: "`cargo doc` with warnings"
      shell: bash
      run: |
        RUSTDOCFLAGS="-Dwarnings" cargo doc  ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} --no-deps --workspace --document-private-items
    - uses: DavidAnson/markdownlint-cli2-action@v23
      with:
        fix: "true"
        globs: |
          *.md
          docs/src/*.md
          src/uu/*/*.md
  min_version:
    name: MinRustV # Minimum supported rust version (aka, MinSRV or MSRV)
    runs-on: ${{ matrix.job.os }}
    strategy:
      matrix:
        job:
          - { os: ubuntu-latest , features: feat_os_unix }
    - uses: actions/checkout@v6
      with:
        persist-credentials: false
    - uses: dtolnay/rust-toolchain@master
      with:
        toolchain: ${{ env.RUST_MIN_SRV }}
        components: rustfmt
    - uses: taiki-e/install-action@nextest
    - uses: Swatinem/rust-cache@v2
    - name: Run sccache-cache
      id: sccache-setup
      uses: mozilla-actions/sccache-action@v0.0.9
      continue-on-error: true
    - name: Export sccache
      if: steps.sccache-setup.outcome == 'success'
      run: |
        echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
        echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV
    - name: Initialize workflow variables
      id: vars
      shell: bash
      run: |
        ## VARs setup
        outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; }
        # target-specific options
        # * CARGO_FEATURES_OPTION
        unset CARGO_FEATURES_OPTION
        if [ -n "${{ matrix.job.features }}" ]; then CARGO_FEATURES_OPTION='--features "${{ matrix.job.features }}"' ; fi
        outputs CARGO_FEATURES_OPTION
    - name: Confirm MinSRV compatible '*/Cargo.lock'
      shell: bash
      run: |
        ## Confirm MinSRV compatible '*/Cargo.lock'
        # * '*/Cargo.lock' is required to be in a format that `cargo` of MinSRV can interpret (eg, v1-format for MinSRV < v1.38)
        for dir in "." "fuzz"; do
          ( cd "$dir" && cargo fetch --locked --quiet --target $(rustc --print host-tuple)) || { echo "::error file=$dir/Cargo.lock::Incompatible (or out-of-date) '$dir/Cargo.lock' file; update using \`cd '$dir' && cargo +${{ env.RUST_MIN_SRV }} update\`" ; exit 1 ; }
        done
    - name: Install/setup prerequisites
      shell: bash
      run: |
        # Install a package for one of the tests
        sudo apt-get -y update ; sudo apt-get -y install attr
    - name: Info
      shell: bash
      run: |
        ## Info
        # environment
        echo "## environment"
        echo "CI='${CI}'"
        # tooling info display
        echo "## tooling"
        which gcc >/dev/null 2>&1 && (gcc --version | head -1) || true
        rustup -V 2>/dev/null
        rustup show active-toolchain
        cargo -V
        rustc -V
        cargo tree -V
        # dependencies
        echo "## dependency list"
        ## * using the 'stable' toolchain is necessary to avoid "unexpected '--filter-platform'" errors
        cargo +stable fetch --locked --quiet --target $(rustc --print host-tuple)
        cargo +stable tree --no-dedupe --locked -e=no-dev --prefix=none ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} | grep -vE "$PWD" | sort --unique
    - name: Test
      run: cargo nextest run --hide-progress-bar --profile ci ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} -p uucore -p coreutils
        RUSTFLAGS: "-Awarnings"
        RUST_BACKTRACE: "1"
    - name: Upload test results to Codecov
      if: ${{ !cancelled() }}
      uses: codecov/codecov-action@v6
      with:
        token: ${{ secrets.CODECOV_TOKEN }}
        report_type: test_results
        files: target/nextest/ci/junit.xml
        disable_search: true
        flags: msrv,${{ matrix.job.os }}
        fail_ci_if_error: false
    name: Dependencies
    runs-on: ${{ matrix.job.os }}
    strategy:
      fail-fast: false
      matrix:
        job:
          - { os: ubuntu-latest , features: feat_os_unix }
    - uses: actions/checkout@v6
      with:
        persist-credentials: false
    - name: "`cargo update` testing"
      shell: bash
      run: |
        ## `cargo update` testing
        # * convert any errors/warnings to GHA UI annotations; ref: <https://help.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-a-warning-message>
        for dir in "." "fuzz"; do
          ( cd "$dir" && cargo fetch --locked --quiet --target $(rustc --print host-tuple)) || { echo "::error file=$dir/Cargo.lock::'$dir/Cargo.lock' file requires update (use \`cd '$dir' && cargo +${{ env.RUST_MIN_SRV }} update\`)" ; exit 1 ; }
        done
  build_rust_stable:
    name: Build/stable
    needs: [ min_version, deps ]
    runs-on: ${{ matrix.job.os }}
    timeout-minutes: 90
    strategy:
      fail-fast: false
      matrix:
        job:
          - { os: ubuntu-latest  , features: feat_os_unix }
          - { os: macos-latest   , features: feat_os_unix }
          - { os: windows-latest , features: feat_os_windows }
    - uses: actions/checkout@v6
      with:
        persist-credentials: false
    - uses: dtolnay/rust-toolchain@stable
    - uses: taiki-e/install-action@nextest
    - uses: Swatinem/rust-cache@v2
    - name: Run sccache-cache
      id: sccache-setup
      uses: mozilla-actions/sccache-action@v0.0.9
      continue-on-error: true
    - name: Export sccache
      if: steps.sccache-setup.outcome == 'success'
      run: |
        echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
        echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV
    - name: Test
      run: cargo nextest run --hide-progress-bar --profile ci --features ${{ matrix.job.features }}
        RUST_BACKTRACE: "1"
    - name: Upload test results to Codecov
      if: ${{ !cancelled() }}
      uses: codecov/codecov-action@v6
      with:
        token: ${{ secrets.CODECOV_TOKEN }}
        report_type: test_results
        files: target/nextest/ci/junit.xml
        disable_search: true
        flags: stable,${{ matrix.job.os }}
        fail_ci_if_error: false
  build_rust_nightly:
    name: Build/nightly
    needs: [ min_version, deps ]
    runs-on: ${{ matrix.job.os }}
    timeout-minutes: 90
    strategy:
      fail-fast: false
      matrix:
        job:
          - { os: ubuntu-latest  , features: feat_os_unix }
          - { os: macos-latest   , features: feat_os_unix }
          - { os: windows-latest , features: feat_os_windows }
    - uses: actions/checkout@v6
      with:
        persist-credentials: false
    - uses: dtolnay/rust-toolchain@nightly
    - uses: taiki-e/install-action@nextest
    - uses: Swatinem/rust-cache@v2
    - name: Run sccache-cache
      id: sccache-setup
      uses: mozilla-actions/sccache-action@v0.0.9
      continue-on-error: true
    - name: Export sccache
      if: steps.sccache-setup.outcome == 'success'
      run: |
        echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
        echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV
    - name: Test
      run: cargo nextest run --hide-progress-bar --profile ci --features ${{ matrix.job.features }}
        RUST_BACKTRACE: "1"
    - name: Upload test results to Codecov
      if: ${{ !cancelled() }}
      uses: codecov/codecov-action@v6
      with:
        token: ${{ secrets.CODECOV_TOKEN }}
        report_type: test_results
        files: target/nextest/ci/junit.xml
        disable_search: true
        flags: nightly,${{ matrix.job.os }}
        fail_ci_if_error: false
    permissions:
      contents: write # to create GitHub release (softprops/action-gh-release)
    name: Build
    needs: [ min_version, deps ]
    runs-on: ${{ matrix.job.os }}
    timeout-minutes: 90
      DOCKER_OPTS: '--volume /etc/passwd:/etc/passwd --volume /etc/group:/etc/group'
    strategy:
      fail-fast: false
      matrix:
        job:
          # - { os , target , cargo-options , default-features,  features , use-cross , toolchain, skip-tests, workspace-tests, skip-package, skip-publish }
          - { os: ubuntu-latest  , target: arm-unknown-linux-gnueabihf , features: feat_os_unix_gnueabihf , use-cross: use-cross , skip-tests: true }
          - { os: ubuntu-24.04-arm  , target: aarch64-unknown-linux-gnu   , features: feat_os_unix_gnueabihf }
          - { os: ubuntu-latest  , target: aarch64-unknown-linux-musl  , features: feat_os_unix_musl , use-cross: use-cross , skip-tests: true }
          - { os: ubuntu-latest  , target: riscv64gc-unknown-linux-musl  , features: feat_os_unix_musl , use-cross: use-cross , skip-tests: true }
          # - { os: ubuntu-latest  , target: x86_64-unknown-linux-gnu    , features: feat_selinux           , use-cross: use-cross }
          - { os: ubuntu-latest  , target: i686-unknown-linux-gnu      , features: "feat_os_unix,test_risky_names", use-cross: use-cross }
          - { os: ubuntu-latest  , target: i686-unknown-linux-musl     , features: feat_os_unix_musl , use-cross: use-cross }
          - { os: ubuntu-latest  , target: x86_64-unknown-linux-gnu    , features: "feat_os_unix,test_risky_names", use-cross: use-cross, skip-publish: true }
          - { os: ubuntu-latest  , target: x86_64-unknown-linux-gnu    , features: "feat_os_unix,uudoc"   , use-cross: no, workspace-tests: true }
          - { os: ubuntu-latest  , target: x86_64-unknown-linux-musl   , features: feat_os_unix_musl , use-cross: use-cross }
          - { os: ubuntu-latest  , target: x86_64-unknown-netbsd, features: "feat_os_unix", use-cross: use-cross , skip-tests: true , check-only: true }
          - { os: ubuntu-latest  , target: x86_64-unknown-redox        , features: feat_os_unix_redox     , use-cross: redoxer , skip-tests: true , check-only: true }
          - { os: ubuntu-latest  , target: wasm32-wasip1, default-features: false, features: feat_wasm, skip-tests: true }
          - { os: macos-latest   , target: aarch64-apple-darwin        , features: feat_os_unix, workspace-tests: true } # M1 CPU
          # PR #7964: chcon should not break build without the feature. cargo check is enough to detect it.
          - { os: macos-latest   , target: aarch64-apple-darwin        , workspace-tests: true, check-only: true } # M1 CPU
          - { os: macos-latest   , target: x86_64-apple-darwin         , features: feat_os_unix, workspace-tests: true }
          - { os: windows-latest , target: i686-pc-windows-msvc        , features: feat_os_windows }
          - { os: windows-latest , target: x86_64-pc-windows-gnu       , features: feat_os_windows }
          - { os: windows-latest , target: x86_64-pc-windows-msvc      , features: feat_os_windows }
          - { os: windows-latest , target: aarch64-pc-windows-msvc     , features: feat_os_windows, use-cross: use-cross , skip-tests: true }
    - uses: actions/checkout@v6
      with:
        persist-credentials: false
    - name: Avoid no space left on device
      run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android &
    - uses: dtolnay/rust-toolchain@master
      with:
        toolchain: ${{ env.RUST_MIN_SRV }}
        targets: ${{ matrix.job.target }}
    - uses: Swatinem/rust-cache@v2
      with:
        key: "${{ matrix.job.os }}_${{ matrix.job.target }}"
    - name: Run sccache-cache
      id: sccache-setup
      uses: mozilla-actions/sccache-action@v0.0.9
      continue-on-error: true
    - name: Export sccache
      if: steps.sccache-setup.outcome == 'success'
      run: |
        echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
        echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV
    - name: Initialize workflow variables
      id: vars
      shell: bash
      run: |
        ## VARs setup
        outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; }
        # toolchain
        TOOLCHAIN="stable" ## default to "stable" toolchain
        # * specify alternate/non-default TOOLCHAIN for *-pc-windows-gnu targets; gnu targets on Windows are broken for the standard *-pc-windows-msvc toolchain (refs: GH:rust-lang/rust#47048, GH:rust-lang/rust#53454, GH:rust-lang/cargo#6754)
        case ${{ matrix.job.target }} in *-pc-windows-gnu) TOOLCHAIN="stable-${{ matrix.job.target }}" ;; esac;
        # * use requested TOOLCHAIN if specified
        if [ -n "${{ matrix.job.toolchain }}" ]; then TOOLCHAIN="${{ matrix.job.toolchain }}" ; fi
        outputs TOOLCHAIN
        # staging directory
        STAGING='_staging'
        outputs STAGING
        # parse commit reference info
        echo GITHUB_REF=${GITHUB_REF}
        echo GITHUB_SHA=${GITHUB_SHA}
        REF_NAME=${GITHUB_REF#refs/*/}
        unset REF_BRANCH ; case "${GITHUB_REF}" in refs/heads/*) REF_BRANCH=${GITHUB_REF#refs/heads/} ;; esac;
        unset REF_TAG ; case "${GITHUB_REF}" in refs/tags/*) REF_TAG=${GITHUB_REF#refs/tags/} ;; esac;
        REF_SHAS=${GITHUB_SHA:0:10}
        outputs REF_NAME REF_BRANCH REF_TAG REF_SHAS
        # parse target
        unset TARGET_ARCH
        case '${{ matrix.job.target }}' in
          aarch64-*) TARGET_ARCH=arm64 ;;
          riscv64gc-*) TARGET_ARCH=riscv64 ;;
          arm-*-*hf) TARGET_ARCH=armhf ;;
          i686-*) TARGET_ARCH=i686 ;;
          x86_64-*) TARGET_ARCH=x86_64 ;;
        esac;
        unset TARGET_OS
        case '${{ matrix.job.target }}' in
          *-linux-*) TARGET_OS=linux ;;
          *-apple-*) TARGET_OS=macos ;;
          *-windows-*) TARGET_OS=windows ;;
          *-redox*) TARGET_OS=redox ;;
        esac
        outputs TARGET_ARCH TARGET_OS
        # package name
        PKG_suffix=".tar.gz" ; case '${{ matrix.job.target }}' in *-pc-windows-*) PKG_suffix=".zip" ;; esac;
        # Some 3rd party utils need version at file names
        # But we remove it from tag/latest-commit
        test ${REF_TAG} \
          && PKG_BASENAME=${PROJECT_NAME}-${REF_TAG}-${{ matrix.job.target }} \
          || PKG_BASENAME=${PROJECT_NAME}-${{ matrix.job.target }}
        PKG_NAME=${PKG_BASENAME}${PKG_suffix}
        outputs PKG_suffix PKG_BASENAME PKG_NAME
        # deployable tag? (ie, leading "vM" or "M"; M == version number)
        unset DEPLOY ; if [[ $REF_TAG =~ ^[vV]?[0-9].* ]]; then DEPLOY='true' ; fi
        outputs DEPLOY
        # target-specific options
        # * CARGO_FEATURES_OPTION
        CARGO_FEATURES_OPTION='' ;
        if [ -n "${{ matrix.job.features }}" ]; then CARGO_FEATURES_OPTION='--features=${{ matrix.job.features }}' ; fi
        outputs CARGO_FEATURES_OPTION
        # * CARGO_DEFAULT_FEATURES_OPTION
        CARGO_DEFAULT_FEATURES_OPTION='' ;
        if [ "${{ matrix.job.default-features }}" == "false" ]; then CARGO_DEFAULT_FEATURES_OPTION='--no-default-features' ; fi
        outputs CARGO_DEFAULT_FEATURES_OPTION
        # * CARGO_CMD
        CARGO_CMD='cross'
        CARGO_CMD_OPTIONS='+${{ env.RUST_MIN_SRV }}'
        # Added suffix for artifacts, needed when multiple jobs use the same target.
        ARTIFACTS_SUFFIX=''
        case '${{ matrix.job.use-cross }}' in
          ''|0|f|false|n|no)
            CARGO_CMD='cargo'
            ARTIFACTS_SUFFIX='-nocross'
          redoxer)
            CARGO_CMD='redoxer'
            CARGO_CMD_OPTIONS=''
        esac
        # needed for target "aarch64-apple-darwin". There are two jobs, and the difference between them is whether "features" is set
        if [ -z "${{ matrix.job.features }}" ]; then ARTIFACTS_SUFFIX='-nofeatures' ; fi
        outputs CARGO_CMD
        outputs CARGO_CMD_OPTIONS
        outputs ARTIFACTS_SUFFIX
        CARGO_TEST_OPTIONS=''
        case '${{ matrix.job.workspace-tests }}' in
          1|t|true|y|yes)
            # This also runs tests in other packages in the source directory (e.g. uucore).
            # We cannot enable this everywhere as some platforms are currently broken, and
            # we cannot use `cross` as its Docker image is ancient (Ubuntu 16.04) and is
            # missing required system dependencies (e.g. recent libclang-dev).
            CARGO_TEST_OPTIONS='--workspace'
        esac
    - uses: taiki-e/install-action@v2
      # `cross` v0.2.5 lacks `libexecinfo` on NetBSD. However, this has been added in main.
      # See https://github.com/cross-rs/cross/blob/main/docker/netbsd.sh
      # We are pulling cross from a specific commit hash rather than HEAD to avoid any potential breakage that may be introduced.
      # Once `cross` v0.3 is out, these conditionals can be removed.
      if: steps.vars.outputs.CARGO_CMD == 'cross' && matrix.job.target != 'x86_64-unknown-netbsd'
      with:
        tool: cross@0.2.5
    - name: Install cross from git (NetBSD)
      if: steps.vars.outputs.CARGO_CMD == 'cross' && matrix.job.target == 'x86_64-unknown-netbsd'
      run: cargo install cross --git https://github.com/cross-rs/cross --rev f86fd03bb70b4c6802847c18087e21391498b0b4
    - name: Create all needed build/work directories
      shell: bash
      run: |
        ## Create build/work space
        mkdir -p '${{ steps.vars.outputs.STAGING }}/${{ steps.vars.outputs.PKG_BASENAME }}'
    - name: Install/setup prerequisites
      shell: bash
      run: |
        ## Install/setup prerequisites
        case '${{ matrix.job.target }}' in
          arm-unknown-linux-gnueabihf)
            sudo apt-get -y update
            sudo apt-get -y install gcc-arm-linux-gnueabihf
          aarch64-unknown-linux-*)
            sudo apt-get -y update
            sudo apt-get -y install gcc-aarch64-linux-gnu
          riscv64gc-unknown-linux-*)
            sudo apt-get -y update
            sudo apt-get -y install gcc-riscv64-linux-gnu
          *-redox*)
            sudo apt-get -y update
            sudo apt-get -y install fuse3 libfuse-dev
        esac
        case '${{ matrix.job.os }}' in
          macos-latest) brew install coreutils ;; # needed for testing
        esac
        case '${{ matrix.job.os }}' in
          ubuntu-*)
            # selinux and systemd headers needed to build tests
            sudo apt-get -y update
            sudo apt-get -y install libselinux1-dev libsystemd-dev
            # pinky is a tool to show logged-in users from utmp, and gecos fields from /etc/passwd.
            # In GitHub Action *nix VMs, no accounts log in, even the "runner" account that runs the commands, and "system boot" entry is missing.
            # The account also has empty gecos fields.
            # To work around these issues for pinky (and who) tests, we create a fake utmp file with a
            # system boot entry and a login entry for the GH runner account.
            FAKE_UTMP_2='[2] [00000] [~~  ] [reboot] [~   ] [6.0.0-test] [0.0.0.0] [2022-02-22T22:11:22,222222+00:00]'
            FAKE_UTMP_7='[7] [999999] [tty2] [runner] [tty2] [          ] [0.0.0.0] [2022-02-22T22:22:22,222222+00:00]'
            (echo "$FAKE_UTMP_2" ; echo "$FAKE_UTMP_7") | sudo utmpdump -r -o /var/run/utmp
            # ... and add a full name to each account with a gecos field but no full name.
            sudo sed -i 's/:,/:runner name,/' /etc/passwd
            # We also create a couple optional files pinky looks for
            touch /home/runner/.project
            echo "foo" > /home/runner/.plan
            # add user with digital username for testing with issue #7787
            echo 200:x:2000:2000::/home/200:/bin/bash | sudo tee -a /etc/passwd
            echo 200:x:2000: | sudo tee -a /etc/group
        esac
    - uses: taiki-e/install-action@v2
      if: steps.vars.outputs.CARGO_CMD == 'redoxer'
      with:
        tool: redoxer@0.2.56
    - name: Initialize toolchain-dependent workflow variables
      id: dep_vars
      shell: bash
      run: |
        ## Dependent VARs setup
        outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; }
        # * determine sub-crate utility list
        UTILITY_LIST="$(./util/show-utils.sh ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }})"
        echo UTILITY_LIST=${UTILITY_LIST}
        CARGO_UTILITY_LIST_OPTIONS="$(for u in ${UTILITY_LIST}; do echo -n "-puu_${u} "; done;)"
        outputs CARGO_UTILITY_LIST_OPTIONS
    - name: Info
      shell: bash
      run: |
        ## Info
        # commit info
        echo "## commit"
        echo GITHUB_REF=${GITHUB_REF}
        echo GITHUB_SHA=${GITHUB_SHA}
        # environment
        echo "## environment"
        echo "CI='${CI}'"
        # tooling info display
        echo "## tooling"
        which gcc >/dev/null 2>&1 && (gcc --version | head -1) || true
        rustup -V 2>/dev/null
        rustup show active-toolchain
        cargo -V
        rustc -V
        cargo tree -V
        # dependencies
        echo "## dependency list"
        cargo fetch --locked --quiet --target $(rustc --print host-tuple)
        cargo tree --locked --target=${{ matrix.job.target }} ${{ matrix.job.cargo-options }} ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} ${{ steps.vars.outputs.CARGO_DEFAULT_FEATURES_OPTION }} --no-dedupe -e=no-dev --prefix=none | grep -vE "$PWD" | sort --unique
    - name: Check
      shell: bash
      if: matrix.job.skip-publish != true && matrix.job.check-only == true
      run: |
        # expr breaks redox
        if [[ "${{ matrix.job.target }}" == *"redox"* ]]; then sed -i.b '/"expr",/d' Cargo.toml; fi
        ${{ steps.vars.outputs.CARGO_CMD }} ${{ steps.vars.outputs.CARGO_CMD_OPTIONS }} check \
        --target=${{ matrix.job.target }} ${{ matrix.job.cargo-options }} ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} ${{ steps.vars.outputs.CARGO_DEFAULT_FEATURES_OPTION }}
    - name: Test
      if: matrix.job.skip-tests != true && matrix.job.check-only != true
      shell: bash
      run: |
        ## Test
        ${{ steps.vars.outputs.CARGO_CMD }} ${{ steps.vars.outputs.CARGO_CMD_OPTIONS }} test --target=${{ matrix.job.target }} \
        ${{ steps.vars.outputs.CARGO_TEST_OPTIONS}} ${{ matrix.job.cargo-options }} ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} ${{ steps.vars.outputs.CARGO_DEFAULT_FEATURES_OPTION }} \
        ${{ steps.dep_vars.outputs.CARGO_UTILITY_LIST_OPTIONS }} -p coreutils
        RUST_BACKTRACE: "1"
    - name: Build coreutils
      shell: bash
      if: matrix.job.skip-publish != true && matrix.job.check-only != true && matrix.job.target != 'x86_64-pc-windows-msvc'
      run: |
        ## Build
        ${{ steps.vars.outputs.CARGO_CMD }} ${{ steps.vars.outputs.CARGO_CMD_OPTIONS }} build --release --config=profile.release.strip=true \
        --target=${{ matrix.job.target }} ${{ matrix.job.cargo-options }} ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} ${{ steps.vars.outputs.CARGO_DEFAULT_FEATURES_OPTION }}
    # We don't want to have many duplicated long jobs at here
    # So we build individual binaries for few platforms until we deduplicate many release build for Linux
    - name: Build individual binaries
      if: matrix.job.skip-tests != true && matrix.job.target == 'x86_64-pc-windows-msvc'
      shell: bash
      run: |
        ${{ steps.vars.outputs.CARGO_CMD }} ${{ steps.vars.outputs.CARGO_CMD_OPTIONS }} build --release --config=profile.release.strip=true \
        --target=${{ matrix.job.target }} ${{ steps.dep_vars.outputs.CARGO_UTILITY_LIST_OPTIONS }}
    - name: Package
      if: matrix.job.skip-publish != true && matrix.job.check-only != true
      shell: bash
      run: |
        ## Package artifact(s)
        find target/${{ matrix.job.target }}/release -maxdepth 1 -type f \
          \( -name "*.wasm" -o -name "*.exe" -o -perm -u+x \) \
          -exec ln -v {} "${{ steps.vars.outputs.STAGING }}/${{ steps.vars.outputs.PKG_BASENAME }}/" \;
        # README and LICENSE
        # * spell-checker:ignore EADME ICENSE
        (shopt -s nullglob; for f in [R]"EADME"{,.*}; do cp $f '${{ steps.vars.outputs.STAGING }}/${{ steps.vars.outputs.PKG_BASENAME }}/' ; done)
        (shopt -s nullglob; for f in [L]"ICENSE"{-*,}{,.*}; do cp $f '${{ steps.vars.outputs.STAGING }}/${{ steps.vars.outputs.PKG_BASENAME }}/' ; done)
        # core compressed package
        pushd '${{ steps.vars.outputs.STAGING }}/' >/dev/null
        case '${{ matrix.job.target }}' in
          *-pc-windows-*) 7z -y a '${{ steps.vars.outputs.PKG_NAME }}' '${{ steps.vars.outputs.PKG_BASENAME }}'/* | tail -2 ;;
          *) tar czf '${{ steps.vars.outputs.PKG_NAME }}' '${{ steps.vars.outputs.PKG_BASENAME }}'/* ;;
        esac
        popd >/dev/null
    - name: Package manpages and completions
      if: matrix.job.target == 'x86_64-unknown-linux-gnu' && matrix.job.features == 'feat_os_unix,uudoc'
      run: |
        mkdir -p share/{man/man1,bash-completion/completions,fish/vendor_completions.d,zsh/site-functions,elvish/lib}
        _uudoc=target/${{ matrix.job.target }}/release/uudoc
        for bin in $('target/${{ matrix.job.target }}/release/coreutils' --list) coreutils;do
          ${_uudoc} manpage ${bin} > share/man/man1/${bin}.1
          ${_uudoc} completion ${bin} bash > share/bash-completion/completions/${bin}.bash
          ${_uudoc} completion ${bin} fish > share/fish/vendor_completions.d/${bin}.fish
          ${_uudoc} completion ${bin} zsh > share/zsh/site-functions/_${bin}
          ${_uudoc} completion ${bin} elvish > share/elvish/lib/${bin}.elv
        done
        rm share/zsh/site-functions/_[ # not supported
        tar --zstd -cf docs.tar.zst share
    - name: Publish
      uses: softprops/action-gh-release@v3
      if: steps.vars.outputs.DEPLOY && matrix.job.skip-publish != true && matrix.job.check-only != true
      with:
        draft: true
        files: |
          ${{ steps.vars.outputs.STAGING }}/${{ steps.vars.outputs.PKG_NAME }}
          docs.tar.zst
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
    - name: Publish latest commit
      uses: softprops/action-gh-release@v3
      if: github.event_name == 'push' && github.ref == 'refs/heads/main' && matrix.job.skip-publish != true && matrix.job.check-only != true
      with:
        tag_name: latest-commit
        body: |
          commit: ${{ github.sha }}
        draft: false
        prerelease: true
        files: |
          ${{ steps.vars.outputs.STAGING }}/${{ steps.vars.outputs.PKG_NAME }}
          docs.tar.zst
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  coverage:
    name: Code Coverage
    runs-on: ${{ matrix.job.os }}
    timeout-minutes: 90
      RUSTC_BOOTSTRAP: 1
    strategy:
      fail-fast: false
      matrix:
        job:
          - { os: ubuntu-latest  , features: unix }
          # FIXME: Re-enable macos code coverage
          # - { os: macos-latest   , features: macos }
          # FIXME: Re-enable Code Coverage on windows, which currently fails due to "profiler_builtins". See #6686.
          # - { os: windows-latest , features: windows, toolchain: stable-x86_64-pc-windows-gnu }
    - uses: actions/checkout@v6
    - uses: taiki-e/install-action@v2
      with:
        tool: nextest,grcov@0.8.24
    - uses: Swatinem/rust-cache@v2
    - name: Run sccache-cache
      id: sccache-setup
      uses: mozilla-actions/sccache-action@v0.0.9
      continue-on-error: true
    - name: Export sccache
      if: steps.sccache-setup.outcome == 'success'
      run: |
        echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
        echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV
    # - name: Reattach HEAD ## may be needed for accurate code coverage info
    #   run: git checkout ${{ github.head_ref }}
    - name: Initialize workflow variables
      id: vars
      shell: bash
      run: |
        ## VARs setup
        outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; }
        # * specify gnu-type TOOLCHAIN for windows; `grcov` requires gnu-style code coverage data files
        case ${{ matrix.job.os }} in windows-*) TOOLCHAIN="$TOOLCHAIN-x86_64-pc-windows-gnu" ;; esac;
        # * use requested TOOLCHAIN if specified
        if [ -n "${{ matrix.job.toolchain }}" ]; then TOOLCHAIN="${{ matrix.job.toolchain }}" ; fi
        outputs TOOLCHAIN
        # target-specific options
        # * CARGO_FEATURES_OPTION
        CARGO_FEATURES_OPTION='--all-features' ;  ## default to '--all-features' for code coverage
        if [ -n "${{ matrix.job.features }}" ]; then CARGO_FEATURES_OPTION='--features=${{ matrix.job.features }}' ; fi
        outputs CARGO_FEATURES_OPTION
        # * CODECOV_FLAGS
        CODECOV_FLAGS=$( echo "${{ matrix.job.os }}" | sed 's/[^[:alnum:]]/_/g' )
        outputs CODECOV_FLAGS
    - name: Install/setup prerequisites
      shell: bash
      run: |
        ## Install/setup prerequisites
        case '${{ matrix.job.os }}' in
          macos-latest) brew install coreutils ;; # needed for testing
        esac
        case '${{ matrix.job.os }}' in
          ubuntu-latest)
            # selinux and systemd headers needed to build tests
            sudo apt-get -y update
            sudo apt-get -y install libselinux1-dev libsystemd-dev
            # pinky is a tool to show logged-in users from utmp, and gecos fields from /etc/passwd.
            # In GitHub Action *nix VMs, no accounts log in, even the "runner" account that runs the commands, and "system boot" entry is missing.
            # The account also has empty gecos fields.
            # To work around these issues for pinky (and who) tests, we create a fake utmp file with a
            # system boot entry and a login entry for the GH runner account.
            FAKE_UTMP_2='[2] [00000] [~~  ] [reboot] [~   ] [6.0.0-test] [0.0.0.0] [2022-02-22T22:11:22,222222+00:00]'
            FAKE_UTMP_7='[7] [999999] [tty2] [runner] [tty2] [          ] [0.0.0.0] [2022-02-22T22:22:22,222222+00:00]'
            (echo "$FAKE_UTMP_2" ; echo "$FAKE_UTMP_7") | sudo utmpdump -r -o /var/run/utmp
            # ... and add a full name to each account with a gecos field but no full name.
            sudo sed -i 's/:,/:runner name,/' /etc/passwd
            # We also create a couple optional files pinky looks for
            touch /home/runner/.project
            echo "foo" > /home/runner/.plan
            # add user with digital username for testing with issue #7787
            echo 200:x:2000:2000::/home/200:/bin/bash | sudo tee -a /etc/passwd
            echo 200:x:2000: | sudo tee -a /etc/group
        esac
        ## Install the llvm-tools component to get access to `llvm-profdata`
        rustup component add llvm-tools
    - name: Run test and coverage
      id: run_test_cov
      run: |
        outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; }
        # Run the coverage script
        ./util/build-run-test-coverage-linux.sh
        outputs REPORT_FILE
        COVERAGE_DIR: ${{ github.workspace }}/coverage
        FEATURES_OPTION: ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }}
        # RUSTUP_TOOLCHAIN: ${{ steps.vars.outputs.TOOLCHAIN }}
    - name: Upload coverage results (to Codecov.io)
      uses: codecov/codecov-action@v6
      with:
        token: ${{ secrets.CODECOV_TOKEN }}
        files: ${{ steps.run_test_cov.outputs.report }}
        ## flags: IntegrationTests, UnitTests, ${{ steps.vars.outputs.CODECOV_FLAGS }}
        flags: ${{ steps.vars.outputs.CODECOV_FLAGS }}
        name: codecov-umbrella
        fail_ci_if_error: false
    - name: Upload test results to Codecov
      if: ${{ !cancelled() }}
      uses: codecov/codecov-action@v6
      with:
        token: ${{ secrets.CODECOV_TOKEN }}
        report_type: test_results
        files: target/nextest/coverage/junit.xml
        disable_search: true
        flags: coverage,${{ matrix.job.os }}
        fail_ci_if_error: false
  test_selinux:
    name: Build/SELinux
    needs: [ min_version, deps ]
    runs-on: ubuntu-latest
    - uses: actions/checkout@v6
      with:
        persist-credentials: false
    - name: Setup Lima
      uses: lima-vm/lima-actions/setup@v1
      id: lima-actions-setup
    - name: Cache ~/.cache/lima
      uses: actions/cache@v5
      with:
        path: ~/.cache/lima
        key: lima-${{ steps.lima-actions-setup.outputs.version }}
    - name: Start Fedora VM with SELinux
      run: limactl start --plain --name=default --cpus=4 --disk=30 --memory=4 --network=lima:user-v2 template:fedora
    - run: rsync -v -a -e ssh . lima-default:~/work/
    - name: Setup Rust and other build deps in VM
      run: |
        lima sudo dnf install --nodocs gcc g++ git rustup libselinux-devel clang-devel attr -y
        lima rustup-init -y --default-toolchain stable --profile minimal -c clippy
    - name: Verify SELinux Status
      run: |
        lima getenforce
        lima ls -laZ /etc/selinux
    - name: Build and Test with SELinux
      run: |
        lima ls
        lima bash -c "cd work && cargo test --features 'feat_selinux' --no-default-features"
    - name: Lint with SELinux
      run: lima bash -c "cd work && cargo clippy --all-targets --features 'feat_selinux' --no-default-features -- -D warnings"
  test_selinux_stubs:
    name: Build/SELinux-Stubs (Non-Linux)
    needs: [ min_version, deps ]
    runs-on: ${{ matrix.job.os }}
    strategy:
      fail-fast: false
      matrix:
        job:
          - { os: macos-latest   , features: feat_os_unix }
          - { os: windows-latest , features: feat_os_windows }
    - uses: actions/checkout@v6
      with:
        persist-credentials: false
    - uses: dtolnay/rust-toolchain@stable
    - uses: Swatinem/rust-cache@v2
    - name: Build SELinux utilities as stubs
      run: cargo build -p uu_chcon -p uu_runcon
    - name: Verify stub binaries exist
      shell: bash
      run: |
        test -f target/debug/chcon || test -f target/debug/chcon.exe
        test -f target/debug/runcon || test -f target/debug/runcon.exe
    # check is enough to detect workspace breakage by chcon
    - name: Verify workspace builds with stubs
      run: cargo check --features ${{ matrix.job.features }}
  test_safe_traversal:
    name: Safe Traversal Security Check
    runs-on: ubuntu-latest
    needs: [ min_version, deps ]
    - uses: actions/checkout@v6
      with:
        persist-credentials: false
    - uses: dtolnay/rust-toolchain@stable
    - uses: Swatinem/rust-cache@v2
    - name: Install strace
      run: sudo apt-get update && sudo apt-get install -y strace
    - name: Build utilities with safe traversal
      run: cargo build --profile=release-small -p uu_rm -p uu_chmod -p uu_chown -p uu_chgrp -p uu_mv -p uu_du
    - name: Run safe traversal verification
      run: ./util/check-safe-traversal.sh
Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

Uh oh!

FilesExpand file tree

CICD.yml

Latest commit

History

CICD.yml

File metadata and controls
