From ff49d85c38395f7d545917c1d9c8be25720dd5b5 Mon Sep 17 00:00:00 2001 From: Avi Deitcher Date: Fri, 3 Oct 2025 10:31:50 +0300 Subject: [PATCH] restructure actions to reusable build flow with parameters Signed-off-by: Avi Deitcher --- .github/workflows/build-reusable.yaml | 174 ++++++++++++++++++ .github/workflows/build.yaml | 236 ++----------------------- .github/workflows/nightly-release.yaml | 218 +++++------------------ 3 files changed, 239 insertions(+), 389 deletions(-) create mode 100644 .github/workflows/build-reusable.yaml diff --git a/.github/workflows/build-reusable.yaml b/.github/workflows/build-reusable.yaml new file mode 100644 index 00000000..071520b6 --- /dev/null +++ b/.github/workflows/build-reusable.yaml @@ -0,0 +1,174 @@ +name: Build + +on: + workflow_call: + inputs: + artifact-name: + description: "Named identifier for the artifact" + required: true + type: string + os: + description: "OS list" + type: string + default: '["ubuntu-22.04","ubuntu-24.04"]' + mode: + description: "Mode list" + type: string + default: '["newlib","linux","musl","uclibc"]' + target: + description: "Target list" + type: string + default: '["rv32gc-ilp32d","rv64gc-lp64d"]' + compiler: + description: "Compiler list" + type: string + default: '["gcc","llvm"]' + sim: + description: "Simulator" + type: string + default: '[""]' + +env: + submodule_paths: | + binutils + dejagnu + gcc + gdb + glibc + llvm + musl + newlib + pk + qemu + spike + uclibc-ng + .git/modules + +jobs: + report: + runs-on: ubuntu-latest + steps: + - name: Report inputs + run: | + echo "Artifact name: ${{ inputs.artifact-name }}" + echo "OS list: ${{ inputs.os }}" + echo "Mode list: ${{ inputs.mode }}" + echo "Target list: ${{ inputs.target }}" + echo "Compiler list: ${{ inputs.compiler }}" + echo "Simulator: ${{ inputs.sim }}" + + submodule_cache: + name: Initialize submodule cache + runs-on: ubuntu-latest + outputs: + key: ${{ steps.keygen.outputs.smcache_key }} + steps: + - uses: actions/checkout@v4 + + - name: Remove unneeded frameworks to recover disk space + run: sudo ./.github/cleanup-rootfs.sh + + - name: Generate submodule cache key + id: keygen + run: echo "smcache_key=smcache-$(printf $(git submodule | sha1sum))" >> $GITHUB_OUTPUT + + - name: Setup submodule cache + id: smcache + uses: actions/cache@v4 + with: + path: ${{ env.submodule_paths }} + key: ${{ steps.keygen.outputs.smcache_key }} + + - name: Checkout required submodules + if: steps.smcache.outputs.cache-hit != 'true' + run: git submodule update --init -j $(nproc) --depth 1 $(echo ${submodule_paths} | sed '$d' | tr '\n' ' ') + + - name: Storage size optimization + if: steps.smcache.outputs.cache-hit != 'true' + run: | + git submodule foreach 'git maintenance run' + + build: + runs-on: ${{ matrix.os }} + needs: [submodule_cache] + env: + smcache_key: ${{ needs.submodule_cache.outputs.key }} + strategy: + matrix: + os: ${{ fromJSON(inputs.os) }} + mode: ${{ fromJSON(inputs.mode) }} + target: ${{ fromJSON(inputs.target) }} + compiler: ${{ fromJSON(inputs.compiler) }} + sim: ${{ fromJSON(inputs.sim) }} + exclude: + - mode: musl + compiler: llvm + - mode: uclibc + compiler: llvm + outputs: + toolchain-name: ${{ steps.toolchain-name-generator.outputs.TOOLCHAIN_NAME }} + steps: + - uses: actions/checkout@v4 + + - name: Remove unneeded frameworks to recover disk space + run: sudo ./.github/cleanup-rootfs.sh + + - name: install dependencies + run: sudo ./.github/setup-apt.sh + + - name: Load submodule cache + uses: actions/cache/restore@v4 + with: + path: ${{ env.submodule_paths }} + key: ${{ env.smcache_key }} + + - name: build toolchain + run: | + TARGET_TUPLE=($(echo ${{ matrix.target }} | tr "-" "\n")) + BUILD_TOOLCHAIN="./configure --prefix=/mnt/riscv --with-arch=${TARGET_TUPLE[0]} --with-abi=${TARGET_TUPLE[1]}" + ARGS="" + if [ "${{ matrix.compiler }}" == "llvm" ]; then # build toolchain with llvm + ARGS="$ARGS --enable-llvm" + fi + if [ -n "${{ matrix.sim }}" ]; then + ARGS="$ARGS --with-sim=${{ matrix.sim }}" + fi + $BUILD_TOOLCHAIN $ARGS + sudo mkdir /mnt/riscv + sudo chown runner:runner /mnt/riscv + make -j $(nproc) ${{ matrix.mode }} + + - name: tarball build + run: | + du -s -h /mnt/riscv + ./.github/dedup-dir.sh /mnt/riscv/ + XZ_OPT="-e -T0" tar cJvf riscv.tar.xz -C /mnt/ riscv/ + + - name: make report + if: | + matrix.os == 'ubuntu-24.04' + && (matrix.mode == 'linux' || matrix.mode == 'newlib') + && matrix.compiler == 'gcc' + run: | + make report-${{ matrix.mode }} -j $(nproc) + + - name: generate prebuilt toolchain name + id: toolchain-name-generator + run: | + if [[ "${{ matrix.target }}" == *"32"* ]]; then BITS=32; else BITS=64; fi + case "${{ matrix.mode }}" in + "linux") + MODE="glibc";; + "musl") + MODE="musl";; + "uclibc") + MODE="uclibc-ng";; + *) + MODE="elf";; + esac + echo "TOOLCHAIN_NAME=riscv$BITS-$MODE-${{ matrix.os }}-${{ matrix.compiler }}-${{ inputs.artifact-name }}" >> $GITHUB_OUTPUT + + - uses: actions/upload-artifact@v4 + with: + name: ${{ steps.toolchain-name-generator.outputs.TOOLCHAIN_NAME }} + path: riscv.tar.xz diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 7c06fb7b..3a31d34a 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -9,231 +9,27 @@ on: branches: - master -env: - submodule_paths: | - binutils - dejagnu - gcc - gdb - glibc - llvm - musl - newlib - pk - qemu - spike - uclibc-ng - .git/modules - jobs: - submodule_cache: - name: Initialize submodule cache - runs-on: ubuntu-latest - outputs: - key: ${{ steps.keygen.outputs.smcache_key }} - steps: - - uses: actions/checkout@v4 - - - name: Remove unneeded frameworks to recover disk space - run: sudo ./.github/cleanup-rootfs.sh - - - name: Generate submodule cache key - id: keygen - run: echo "smcache_key=smcache-$(printf $(git submodule | sha1sum))" >> $GITHUB_OUTPUT - - - name: Setup submodule cache - id: smcache - uses: actions/cache@v4 - with: - path: ${{ env.submodule_paths }} - key: ${{ steps.keygen.outputs.smcache_key }} - - - name: Checkout required submodules - if: steps.smcache.outputs.cache-hit != 'true' - run: git submodule update --init -j $(nproc) --depth 1 $(echo ${submodule_paths} | sed '$d' | tr '\n' ' ') - - - name: Storage size optimization - if: steps.smcache.outputs.cache-hit != 'true' - run: | - git submodule foreach 'git maintenance run' - build: - runs-on: ${{ matrix.os }} - needs: [submodule_cache] - env: - smcache_key: ${{ needs.submodule_cache.outputs.key }} - strategy: - matrix: - os: [ubuntu-22.04, ubuntu-24.04] - mode: [newlib, linux, musl, uclibc] - target: [rv32gc-ilp32d, rv64gc-lp64d] - compiler: [gcc, llvm] - exclude: - - mode: musl - compiler: llvm - - mode: uclibc - compiler: llvm - steps: - - uses: actions/checkout@v4 - - - name: Remove unneeded frameworks to recover disk space - run: sudo ./.github/cleanup-rootfs.sh - - - name: install dependencies - run: sudo ./.github/setup-apt.sh - - - name: Load submodule cache - uses: actions/cache/restore@v4 - with: - path: ${{ env.submodule_paths }} - key: ${{ env.smcache_key }} - - - name: build toolchain - run: | - TARGET_TUPLE=($(echo ${{ matrix.target }} | tr "-" "\n")) - BUILD_TOOLCHAIN="./configure --prefix=/mnt/riscv --with-arch=${TARGET_TUPLE[0]} --with-abi=${TARGET_TUPLE[1]}" - if [ "${{ matrix.compiler }}" == "llvm" ]; then # build toolchain with llvm - $BUILD_TOOLCHAIN --enable-llvm - else - $BUILD_TOOLCHAIN - fi - sudo mkdir /mnt/riscv - sudo chown runner:runner /mnt/riscv - make -j $(nproc) ${{ matrix.mode }} - - - name: tarball build - run: | - du -s -h /mnt/riscv - ./.github/dedup-dir.sh /mnt/riscv/ - XZ_OPT="-e -T0" tar cJvf riscv.tar.xz -C /mnt/ riscv/ - - - name: make report - if: | - matrix.os == 'ubuntu-24.04' - && (matrix.mode == 'linux' || matrix.mode == 'newlib') - && matrix.compiler == 'gcc' - run: | - make report-${{ matrix.mode }} -j $(nproc) - - - name: generate prebuilt toolchain name - id: toolchain-name-generator - run: | - if [[ "${{ matrix.target }}" == *"32"* ]]; then BITS=32; else BITS=64; fi - case "${{ matrix.mode }}" in - "linux") - MODE="glibc";; - "musl") - MODE="musl";; - "uclibc") - MODE="uclibc-ng";; - *) - MODE="elf";; - esac - echo "TOOLCHAIN_NAME=riscv$BITS-$MODE-${{ matrix.os }}-${{ matrix.compiler }}-nightly" >> $GITHUB_OUTPUT - - - uses: actions/upload-artifact@v4 - with: - name: ${{ steps.toolchain-name-generator.outputs.TOOLCHAIN_NAME }} - path: riscv.tar.xz + uses: ./.github/workflows/build-reusable.yaml + with: + artifact-name: build test-sim: - runs-on: ${{ matrix.os }} - needs: [submodule_cache] - env: - smcache_key: ${{ needs.submodule_cache.outputs.key }} - strategy: - matrix: - os: [ubuntu-24.04] - mode: [newlib] - target: [rv64gc-lp64d] - sim: [spike] - steps: - - uses: actions/checkout@v4 - - - name: Remove unneeded frameworks to recover disk space - run: sudo ./.github/cleanup-rootfs.sh - - - name: install dependencies - run: sudo ./.github/setup-apt.sh - - - name: Load submodule cache - uses: actions/cache/restore@v4 - with: - path: ${{ env.submodule_paths }} - key: ${{ env.smcache_key }} - - - name: build toolchain - run: | - TARGET_TUPLE=($(echo ${{ matrix.target }} | tr "-" "\n")) - ./configure --prefix=/mnt/riscv --with-arch=${TARGET_TUPLE[0]} --with-abi=${TARGET_TUPLE[1]} --with-sim=${{ matrix.sim }} - sudo mkdir /mnt/riscv - sudo chown runner:runner /mnt/riscv - make -j $(nproc) ${{ matrix.mode }} - - - name: make report - run: make report-${{ matrix.mode }} -j $(nproc) + uses: ./.github/workflows/build-reusable.yaml + with: + artifact-name: sim + os: '["ubuntu-24.04"]' + mode: '["newlib"]' + target: '["rv64gc-lp64d"]' + sim: '["spike"]' build-multilib: if: ${{ false }} # Disable until multilib errors are triaged - runs-on: ${{ matrix.os }} - needs: [submodule_cache] - env: - smcache_key: ${{ needs.submodule_cache.outputs.key }} - strategy: - matrix: - os: [ubuntu-24.04] - mode: [newlib, linux] - target: [rv64gc-lp64d] - steps: - - uses: actions/checkout@v4 + uses: ./.github/workflows/build-reusable.yaml + with: + artifact-name: multilib + os: '["ubuntu-24.04"]' + mode: '["newlib","linux"]' + target: '["rv64gc-lp64d"]' - - name: Remove unneeded frameworks to recover disk space - run: sudo ./.github/cleanup-rootfs.sh - - - name: install dependencies - run: sudo ./.github/setup-apt.sh - - - name: Load submodule cache - uses: actions/cache/restore@v4 - with: - path: ${{ env.submodule_paths }} - key: ${{ env.smcache_key }} - - - name: build toolchain - run: | - TARGET_TUPLE=($(echo ${{ matrix.target }} | tr "-" "\n")) - ./configure --prefix=/mnt/riscv --with-arch=${TARGET_TUPLE[0]} --with-abi=${TARGET_TUPLE[1]} --enable-multilib - sudo mkdir /mnt/riscv - sudo chown runner:runner /mnt/riscv - make -j $(nproc) ${{ matrix.mode }} - - - name: tarball build - run: | - du -s -h /mnt/riscv - ./.github/dedup-dir.sh /mnt/riscv/ - XZ_OPT="-e -T0" tar cJvf riscv.tar.xz -C /mnt/ riscv/ - - - name: make report - run: | - make report-${{ matrix.mode }} -j $(nproc) - - - name: generate prebuilt toolchain name - id: toolchain-name-generator - run: | - if [[ "${{ matrix.target }}" == *"32"* ]]; then BITS=32; else BITS=64; fi - case "${{ matrix.mode }}" in - "linux") - MODE="glibc";; - "musl") - MODE="musl";; - "uclibc") - MODE="uclibc-ng";; - *) - MODE="elf";; - esac - echo "TOOLCHAIN_NAME=riscv$BITS-$MODE-${{ matrix.os }}-multilib-nightly" >> $GITHUB_OUTPUT - - - uses: actions/upload-artifact@v4 - with: - name: ${{ steps.toolchain-name-generator.outputs.TOOLCHAIN_NAME }} - path: riscv.tar.xz diff --git a/.github/workflows/nightly-release.yaml b/.github/workflows/nightly-release.yaml index 4c0f6b52..d72dacb3 100644 --- a/.github/workflows/nightly-release.yaml +++ b/.github/workflows/nightly-release.yaml @@ -2,7 +2,7 @@ name: Nightly Release on: schedule: - - cron: '0 0 * * ?' + - cron: '0 0 * * *' jobs: @@ -12,28 +12,37 @@ jobs: outputs: stale: ${{ steps.activity_check.outputs.stale }} steps: + - uses: actions/checkout@v4 - name: Activity check id: activity_check run: | - curl -sL https://api.github.com/repos/$GITHUB_REPOSITORY/commits | jq -r '[.[]][0]' > $HOME/commit.json - date="$(jq -r '.commit.committer.date' $HOME/commit.json)" - timestamp=$(date --utc -d "$date" +%s) - author="$(jq -r '.commit.author.name' $HOME/commit.json)" - url="$(jq -r '.html_url' $HOME/commit.json)" - hours=$(( ( $(date --utc +%s) - $timestamp ) / 3600 )) - rm -f $HOME/commit.json - echo "Latest Repository activity : $timestamp $author $url" + # get the author and timestamp of the latest commit + AUTHOR=$(git log -1 --pretty=format:'%an') + # Get the commit date in ISO 8601 format (UTC) + COMMIT_DATE=$(git log -1 --pretty=format:'%cI') + echo "Commit date: $COMMIT_DATE" + # Convert both dates to epoch seconds + COMMIT_EPOCH=$(date -d "$COMMIT_DATE" +%s) + NOW_EPOCH=$(date -u +%s) + + # Calculate difference in seconds + AGE_SECONDS=$(( NOW_EPOCH - COMMIT_EPOCH )) + + # 86400 seconds = 24 hours + echo "Latest Repository activity : $COMMIT_DATE $AUTHOR, $AGE_SECONDS seconds ago" STALE=false if [ "${{ github.event_name }}" == "repository_dispatch" ]; then echo "[WARNING] Ignoring activity limits : workflow triggered manually" + STALE=false + elif [ "$AGE_SECONDS" -gt 86400 ]; then + echo "[ERROR] Repository not updated : event<${{ github.event_name }}> not allowed to modify stale repository" + echo "Commit is stale (older than 24 hours)" + STALE=true else - echo Repository active last $hours hours ago - if [ $hours -ge 24 ]; then - echo "[ERROR] Repository not updated : event<${{ github.event_name }}> not allowed to modify stale repository" - STALE=true - fi - fi + echo "Commit is fresh (within last 24 hours)" + STALE=false + fi echo "stale=$STALE" >> $GITHUB_OUTPUT if [ "$STALE" == "true" ]; then @@ -41,180 +50,51 @@ jobs: fi shell: bash - + build: - needs: activity-check + needs: [activity-check] if: needs.activity-check.outputs.stale != 'true' - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-22.04, ubuntu-24.04] - mode: [newlib, linux, musl, uclibc] - target: [rv32gc-ilp32d, rv64gc-lp64d] - compiler: [gcc, llvm] - exclude: - - mode: musl - compiler: llvm - - - mode: uclibc - compiler: llvm - steps: - - uses: actions/checkout@v4 - - - name: Remove unneeded frameworks to recover disk space - run: sudo ./.github/cleanup-rootfs.sh - - - name: install apt dependencies - run: sudo ./.github/setup-apt.sh - - - name: build toolchain - run: | - TARGET_TUPLE=($(echo ${{ matrix.target }} | tr "-" "\n")) - BUILD_TOOLCHAIN="./configure --prefix=/mnt/riscv --with-arch=${TARGET_TUPLE[0]} --with-abi=${TARGET_TUPLE[1]}" - if [ "${{ matrix.compiler }}" == "llvm" ]; then # build toolchain with llvm - $BUILD_TOOLCHAIN --enable-llvm - else - $BUILD_TOOLCHAIN - fi - sudo mkdir /mnt/riscv - sudo chown runner:runner /mnt/riscv - make -j $(nproc) ${{ matrix.mode }} - - - name: tarball build - run: | - du -s -h /mnt/riscv - ./.github/dedup-dir.sh /mnt/riscv/ - XZ_OPT="-e -T0" tar cJvf riscv.tar.xz -C /mnt/ riscv/ - - - name: generate prebuilt toolchain name - id: toolchain-name-generator - run: | - if [[ "${{ matrix.target }}" == *"32"* ]]; then BITS=32; else BITS=64; fi - case "${{ matrix.mode }}" in - "linux") - MODE="glibc";; - "musl") - MODE="musl";; - "uclibc") - MODE="uclibc-ng";; - *) - MODE="elf";; - esac - echo "TOOLCHAIN_NAME=riscv$BITS-$MODE-${{ matrix.os }}-${{ matrix.compiler }}-nightly" >> $GITHUB_OUTPUT - - - uses: actions/upload-artifact@v4 - with: - name: ${{ steps.toolchain-name-generator.outputs.TOOLCHAIN_NAME }} - path: riscv.tar.xz - + uses: ./.github/workflows/build-reusable.yaml + with: + artifact-name: nightly create-release: - needs: build + needs: [build] runs-on: ubuntu-latest outputs: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_matrix: ${{ steps.asset_names.outputs.asset_matrix }} datestamp: ${{ env.DATESTAMP }} + env: + ARTIFACTS_DIR: /mnt/artifacts/ steps: - - name: Remove unneeded frameworks to recover disk space - run: | - echo "-- Before --" - df -h - sudo rm -rf /usr/share/dotnet - sudo rm -rf /usr/local/lib/android - echo "-- After --" - df -h - name: Run Configuration Commands + id: metadata run: | DATESTAMP="$(date --utc '+%Y.%m.%d')" echo "Version: ${DATESTAMP}-nightly" - # Setup Artifacts Directory - ARTIFACTS_DIR="/mnt/artifacts/" - sudo mkdir -p $ARTIFACTS_DIR - sudo chown runner:runner $ARTIFACTS_DIR - # Setup environment variables - echo "DATESTAMP=${DATESTAMP}" >> $GITHUB_ENV - echo "DATEWORD=$(date --utc '+%B %d, %Y')" >> $GITHUB_ENV - echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> $GITHUB_ENV + echo "datestamp=${DATESTAMP}" >> $GITHUB_OUTPUT + echo "dateword=$(date --utc '+%B %d, %Y')" >> $GITHUB_OUTPUT shell: bash - - - name: Create Release - id: create_release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ env.DATESTAMP }} - release_name: "Nightly: ${{ env.DATEWORD }}" - body: | - **Automated Nightly Release** - ${{ env.DATESTAMP }}-nightly - draft: false - prerelease: true - - name: Download Built Artifacts uses: actions/download-artifact@v4 with: path: ${{ env.ARTIFACTS_DIR }} - - # IMPORTANT: Each artifact must only have one file - - name: Designate Asset Names - id: asset_names + - name: List downloaded artifacts run: | - ASSET_MATRIX=$( - find ${ARTIFACTS_DIR} -mindepth 2 -maxdepth 2 -type f | - awk '{ - fs_n=split($0, fs, "/") # Split file paths - art_name=fs[fs_n-1] # Get artifact name - fname=fs[fs_n] # Get file name from the artifact - ext = substr(fs[fs_n], index(fs[fs_n],".")) # File Extension + echo "Contents of ${{ env.ARTIFACTS_DIR }}" + ls -R ${{ env.ARTIFACTS_DIR }} - print art_name ":" fname ":" ext # format - }' | - jq -R -s -c 'split("\n") | .[:-1] | { # Split by newlines (remove last entry) - include: [ - .[] | split(":") | { # Put it in JSON format - artifact: .[0], - file: .[1], - extension: .[2] - } - ] - }' - ) - - echo "asset_matrix=${ASSET_MATRIX}" >> $GITHUB_OUTPUT - shell: bash - - - upload-assets: - needs: create-release - runs-on: ubuntu-latest - strategy: - matrix: ${{ fromJson( needs.create-release.outputs.asset_matrix ) }} - name: upload ${{ matrix.artifact }} - steps: - - - name: Remove unneeded frameworks to recover disk space - run: | - echo "-- Before --" - df -h - sudo rm -rf /usr/share/dotnet - sudo rm -rf /usr/local/lib/android - echo "-- After --" - df -h - - - uses: actions/download-artifact@v4 + - name: Create Release + id: create_release + uses: softprops/action-gh-release@v2 with: - name: ${{ matrix.artifact }} - - - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ needs.create-release.outputs.upload_url }} - asset_path: ${{ matrix.file }} - asset_name: ${{ matrix.artifact }}-${{ needs.create-release.outputs.datestamp }}-nightly${{ matrix.extension }} - asset_content_type: application/x-xz + tag_name: ${{ steps.metadata.outputs.datestamp }} + name: "Nightly: ${{ steps.metadata.outputs.dateword }}" + body: | + **Automated Nightly Release** + ${{ steps.metadata.outputs.datestamp }}-nightly + draft: false + files: | + ${{ env.ARTIFACTS_DIR }}/*