mirror of
https://github.com/OrcaSlicer/OrcaSlicer.git
synced 2026-10-11 01:41:03 +00:00
ci: wait for the base branch's macOS deps instead of rebuilding them
This commit is contained in:
raistlin7447
@@ -20,19 +20,37 @@ on:
|
||||
required: false
|
||||
type: boolean
|
||||
|
||||
# Read by scripts/ci_deps_cache.sh.
|
||||
env:
|
||||
REPO: ${{ github.repository }}
|
||||
WORKFLOW_REF: ${{ github.workflow_ref }}
|
||||
BASE_REF: ${{ github.base_ref }}
|
||||
|
||||
jobs:
|
||||
check_cache: # determines if there is a cache and outputs variables used in caching process
|
||||
name: Check Cache
|
||||
runs-on: ${{ inputs.os }}
|
||||
# Hosted macOS runners are scarce, so their lookup runs on Linux.
|
||||
runs-on: ${{ startsWith(inputs.os, 'macos-') && 'ubuntu-24.04' || inputs.os }}
|
||||
outputs:
|
||||
cache-key: ${{ steps.set_outputs.outputs.cache-key }}
|
||||
cache-path: ${{ steps.set_outputs.outputs.cache-path }}
|
||||
valid-cache: ${{ steps.cache_deps.outputs.cache-hit }}
|
||||
macos-path: ${{ steps.macos_path.outputs.path }}
|
||||
key-inputs: ${{ steps.wait_check.outputs.key-inputs }}
|
||||
wait-for: ${{ steps.wait_check.outputs.wait-for }}
|
||||
wait-since: ${{ steps.wait_check.outputs.wait-since }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
lfs: 'false'
|
||||
# The wait check compares the merge commit with its first parent.
|
||||
fetch-depth: ${{ startsWith(inputs.os, 'macos-') && 2 || 1 }}
|
||||
# Fetches only deps/, which the key hashes, and the wait script.
|
||||
sparse-checkout: |
|
||||
/deps/
|
||||
/scripts/ci_deps_cache.sh
|
||||
sparse-checkout-cone-mode: false
|
||||
|
||||
- name: set outputs
|
||||
id: set_outputs
|
||||
@@ -40,31 +58,89 @@ jobs:
|
||||
# Anything that changes how the tree is built belongs in the key, or a job
|
||||
# restores one it cannot use. Linux amd64 passes no arch deliberately, so
|
||||
# 'linux-clang' keeps the cache it already has.
|
||||
cache-os: ${{ runner.os == 'macOS' && format('{0}-{1}', inputs.os, inputs.arch) || (runner.os == 'Windows' && format('windows-{0}-{1}', inputs.arch, inputs.compiler) || format('linux-clang{0}', inputs.arch && format('-{0}', inputs.arch) || '')) }}
|
||||
cache-os: ${{ (runner.os == 'macOS' || startsWith(inputs.os, 'macos-')) && format('{0}-{1}', inputs.os, inputs.arch) || (runner.os == 'Windows' && format('windows-{0}-{1}', inputs.arch, inputs.compiler) || format('linux-clang{0}', inputs.arch && format('-{0}', inputs.arch) || '')) }}
|
||||
# The Windows ARM64 deps build in build-arm64, all others under build;
|
||||
# build_deps.yml and build_orca.yml pass the Windows directory to build_win.bat.
|
||||
dep-folder-name: ${{ runner.os == 'macOS' && format('/{0}', inputs.arch) || (runner.os == 'Windows' && inputs.arch == 'arm64') && '-arm64/OrcaSlicer_dep' || '/OrcaSlicer_dep' }}
|
||||
dep-folder-name: ${{ (runner.os == 'macOS' || startsWith(inputs.os, 'macos-')) && format('/{0}', inputs.arch) || (runner.os == 'Windows' && inputs.arch == 'arm64') && '-arm64/OrcaSlicer_dep' || '/OrcaSlicer_dep' }}
|
||||
output-cmd: ${{ runner.os == 'Windows' && '$env:GITHUB_OUTPUT' || '"$GITHUB_OUTPUT"'}}
|
||||
run: |
|
||||
echo cache-key=${{ env.cache-os }}-cache-orcaslicer_deps-build-${{ hashFiles('deps/**') }} >> ${{ env.output-cmd }}
|
||||
echo cache-path=${{ github.workspace }}/deps/build${{ env.dep-folder-name }} >> ${{ env.output-cmd }}
|
||||
# Relative to the workspace, which differs from the build's when this job runs on Linux.
|
||||
echo cache-path=deps/build${{ env.dep-folder-name }} >> ${{ env.output-cmd }}
|
||||
|
||||
# actions/cache matches the path too, so a Linux lookup for macOS uses the
|
||||
# workspace of a hosted macOS job.
|
||||
- name: macOS cache path
|
||||
id: macos_path
|
||||
if: ${{ startsWith(inputs.os, 'macos-') }}
|
||||
run: echo "path=/Users/runner/work/${GITHUB_REPOSITORY#*/}/${GITHUB_REPOSITORY#*/}/${{ steps.set_outputs.outputs.cache-path }}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: load cache
|
||||
id: cache_deps
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ${{ steps.set_outputs.outputs.cache-path }}
|
||||
path: ${{ steps.macos_path.outputs.path || format('{0}/{1}', github.workspace, steps.set_outputs.outputs.cache-path) }}
|
||||
key: ${{ steps.set_outputs.outputs.cache-key }}
|
||||
lookup-only: true
|
||||
|
||||
# A pull request that misses the cache can wait for a base branch build of the same deps.
|
||||
- name: check for a base branch build to wait for
|
||||
id: wait_check
|
||||
if: ${{ startsWith(inputs.os, 'macos-') && github.event_name == 'pull_request' && steps.cache_deps.outputs.cache-hit != 'true' }}
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: bash scripts/ci_deps_cache.sh check
|
||||
|
||||
wait_for_deps:
|
||||
name: Wait for base branch deps
|
||||
needs: [check_cache]
|
||||
if: ${{ needs.check_cache.outputs.wait-for != '' }}
|
||||
runs-on: ubuntu-24.04
|
||||
env:
|
||||
WAIT_MINUTES: 120
|
||||
# Must exceed WAIT_MINUTES.
|
||||
timeout-minutes: 130
|
||||
# Waiting runs queue here, and only the first one holds a runner.
|
||||
concurrency:
|
||||
group: deps-wait-${{ github.base_ref }}-${{ needs.check_cache.outputs.cache-key }}
|
||||
queue: max
|
||||
outputs:
|
||||
hit: ${{ steps.lookup.outputs.cache-hit }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
sparse-checkout: scripts/ci_deps_cache.sh
|
||||
sparse-checkout-cone-mode: false
|
||||
|
||||
- name: wait for the deps cache
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
KEY: ${{ needs.check_cache.outputs.cache-key }}
|
||||
KEY_INPUTS: ${{ needs.check_cache.outputs.key-inputs }}
|
||||
WAIT_FOR: ${{ needs.check_cache.outputs.wait-for }}
|
||||
WAIT_SINCE: ${{ needs.check_cache.outputs.wait-since }}
|
||||
ARCH: ${{ inputs.arch }}
|
||||
run: bash scripts/ci_deps_cache.sh wait
|
||||
|
||||
- name: load cache
|
||||
id: lookup
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ${{ needs.check_cache.outputs.macos-path }}
|
||||
key: ${{ needs.check_cache.outputs.cache-key }}
|
||||
lookup-only: true
|
||||
|
||||
build_deps: # call next step
|
||||
name: Build Deps
|
||||
needs: [check_cache]
|
||||
needs: [check_cache, wait_for_deps]
|
||||
# A failed wait counts as no cache, so the deps are built here.
|
||||
if: ${{ !cancelled() && needs.check_cache.result == 'success' }}
|
||||
uses: ./.github/workflows/build_deps.yml
|
||||
with:
|
||||
cache-key: ${{ needs.check_cache.outputs.cache-key }}
|
||||
cache-path: ${{ needs.check_cache.outputs.cache-path }}
|
||||
valid-cache: ${{ needs.check_cache.outputs.valid-cache == 'true' }}
|
||||
valid-cache: ${{ needs.check_cache.outputs.valid-cache == 'true' || needs.wait_for_deps.outputs.hit == 'true' }}
|
||||
os: ${{ inputs.os }}
|
||||
arch: ${{ inputs.arch }}
|
||||
compiler: ${{ inputs.compiler }}
|
||||
|
||||
@@ -45,7 +45,7 @@ jobs:
|
||||
- name: load cached deps
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ${{ inputs.cache-path }}
|
||||
path: ${{ github.workspace }}/${{ inputs.cache-path }}
|
||||
key: ${{ inputs.cache-key }}
|
||||
|
||||
- uses: lukka/get-cmake@latest
|
||||
@@ -122,7 +122,7 @@ jobs:
|
||||
choco install strawberryperl
|
||||
}
|
||||
# cache-path is the install directory inside the deps build directory.
|
||||
$deps = (Split-Path "${{ inputs.cache-path }}").Replace('\', '/')
|
||||
$deps = (Split-Path "${{ github.workspace }}/${{ inputs.cache-path }}").Replace('\', '/')
|
||||
# -l compiles with Visual Studio's clang-cl and -x builds with Ninja; --msvc --msbuild is cl under the Visual Studio generator.
|
||||
$flags = if ("${{ inputs.compiler }}" -eq "clang") { "-l", "-x" } else { "--msvc", "--msbuild" }
|
||||
.\build_win.bat -d --arch ${{ inputs.arch }} --deps-dir $deps @flags
|
||||
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
if: ${{ !(runner.os == 'macOS' && inputs.macos-combine-only) }}
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ${{ inputs.cache-path }}
|
||||
path: ${{ github.workspace }}/${{ inputs.cache-path }}
|
||||
key: ${{ inputs.cache-key }}
|
||||
fail-on-cache-miss: true
|
||||
|
||||
@@ -460,7 +460,7 @@ jobs:
|
||||
# --tests builds the unit tests too; the unit_tests_windows_* jobs run them.
|
||||
run: |
|
||||
# cache-path is the install directory inside the deps build directory.
|
||||
$deps = (Split-Path "${{ inputs.cache-path }}").Replace('\', '/')
|
||||
$deps = (Split-Path "${{ github.workspace }}/${{ inputs.cache-path }}").Replace('\', '/')
|
||||
# -l compiles with Visual Studio's clang-cl and -x builds with Ninja; --msvc --msbuild is cl under the Visual Studio generator.
|
||||
$flags = if ("${{ inputs.compiler }}" -eq "clang") { "-l", "-x" } else { "--msvc", "--msbuild" }
|
||||
.\build_win.bat -s --tests -i --arch ${{ inputs.arch }} --build-dir $env:BUILD_DIR --deps-dir $deps @flags
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env bash
|
||||
# Lets a pull request's macOS deps build wait for a push build of the base
|
||||
# branch that builds the same deps, from a Linux job. `check` runs after a cache
|
||||
# miss and names a queued or running base build with the same deps/ and .github/
|
||||
# to wait for, unless the pull request changes either. `wait` waits until that
|
||||
# build saves KEY. actions/cache then confirms the entry can be restored.
|
||||
#
|
||||
# Environment: GH_TOKEN, REPO, WORKFLOW_REF and BASE_REF, plus KEY_INPUTS,
|
||||
# WAIT_FOR, WAIT_SINCE, WAIT_MINUTES, KEY and ARCH for `wait`.
|
||||
# Writes key-inputs, wait-for and wait-since to $GITHUB_OUTPUT.
|
||||
set -uo pipefail
|
||||
|
||||
workflow=${WORKFLOW_REF%%@*}
|
||||
workflow=${workflow##*/}
|
||||
|
||||
# Must match the job names in build_all.yml, build_check_cache.yml and build_deps.yml.
|
||||
check_job_name="build_macos_arch (${ARCH:-}) / Check Cache"
|
||||
deps_job_name="build_macos_arch (${ARCH:-}) / Build Deps / Build Deps"
|
||||
|
||||
out() { echo "$1" >> "${GITHUB_OUTPUT:-/dev/stdout}"; }
|
||||
|
||||
# The cache key depends on deps/ and on the workflows under .github/.
|
||||
local_key_inputs() { echo "$(git rev-parse "$1:.github") $(git rev-parse "$1:deps")"; }
|
||||
remote_key_inputs() {
|
||||
gh api "repos/$REPO/git/trees/$1" < /dev/null \
|
||||
--jq '[.tree[] | select(.path == ".github" or .path == "deps") | {key: .path, value: .sha}] | from_entries | "\(.[".github"]) \(.deps)"'
|
||||
}
|
||||
|
||||
# Prints the id of a push build of the base branch that has not finished, is not
|
||||
# one of the ids in $1, and has KEY_INPUTS.
|
||||
matching_base_build() {
|
||||
local status runs id sha
|
||||
runs=$(for status in in_progress queued pending waiting requested; do
|
||||
gh api -X GET "repos/$REPO/actions/workflows/$workflow/runs" \
|
||||
-f branch="$BASE_REF" -f event=push -f status="$status" -f per_page=10 < /dev/null \
|
||||
| jq -r '.workflow_runs[] | "\(.id) \(.head_sha)"'
|
||||
done)
|
||||
while read -r id sha; do
|
||||
[ -n "$id" ] || continue
|
||||
[[ " ${1:-} " == *" $id "* ]] && continue
|
||||
if [ "$(remote_key_inputs "$sha")" = "$KEY_INPUTS" ]; then
|
||||
echo "$id"
|
||||
return
|
||||
fi
|
||||
done <<< "$runs"
|
||||
}
|
||||
|
||||
check() {
|
||||
KEY_INPUTS=$(local_key_inputs HEAD)
|
||||
out "key-inputs=$KEY_INPUTS"
|
||||
if [ "$KEY_INPUTS" != "$(local_key_inputs HEAD^1)" ]; then
|
||||
echo "This pull request changes deps/ or .github/, so it builds its own deps."
|
||||
return
|
||||
fi
|
||||
local id
|
||||
id=$(matching_base_build)
|
||||
if [ -n "$id" ]; then
|
||||
echo "Build $id of $BASE_REF has the same deps/ and .github/. Waiting for its cache."
|
||||
out "wait-for=$id"
|
||||
out "wait-since=$(date +%s)"
|
||||
else
|
||||
echo "No build of $BASE_REF is building these deps."
|
||||
fi
|
||||
}
|
||||
|
||||
key_saved() {
|
||||
gh cache list -R "$REPO" --ref "refs/heads/$BASE_REF" --key "$KEY" --json key < /dev/null \
|
||||
| jq -e --arg key "$KEY" 'any(.[]; .key == $key)' > /dev/null
|
||||
}
|
||||
|
||||
wait_for() {
|
||||
# Counts the limit from the check, including time queued behind other waiting
|
||||
# pull requests.
|
||||
local id=$WAIT_FOR seen=$WAIT_FOR deadline=$((${WAIT_SINCE:-$(date +%s)} + WAIT_MINUTES * 60)) jobs job check warned=
|
||||
while [ "$(date +%s)" -lt "$deadline" ]; do
|
||||
if key_saved; then
|
||||
echo "Found $KEY."
|
||||
return
|
||||
fi
|
||||
jobs=$(gh api -X GET "repos/$REPO/actions/runs/$id/jobs" -f filter=latest -f per_page=100 < /dev/null)
|
||||
job=$(jq -r --arg name "$deps_job_name" 'first(.jobs[] | select(.name == $name) | "\(.status) \(.conclusion)") // ""' <<< "$jobs")
|
||||
case "$job" in
|
||||
"completed success" | "completed skipped")
|
||||
# The new entry can take a moment to be listed, and a skipped job means
|
||||
# the build found the cache, possibly under the default branch.
|
||||
for _ in 1 2 3; do
|
||||
key_saved && { echo "Found $KEY."; return; }
|
||||
sleep 20
|
||||
done
|
||||
echo "Build $id of $BASE_REF finished its deps without saving $KEY."
|
||||
return ;;
|
||||
"completed cancelled") ;;
|
||||
completed*)
|
||||
echo "The deps job of build $id of $BASE_REF did not succeed."
|
||||
return ;;
|
||||
# The deps job exists only once the build's cache check has succeeded.
|
||||
"")
|
||||
case "$(gh api "repos/$REPO/actions/runs/$id" --jq '"\(.status) \(.conclusion)"' < /dev/null)" in
|
||||
"completed cancelled") ;;
|
||||
completed*)
|
||||
check=$(jq -r --arg name "$check_job_name" 'first(.jobs[] | select(.name == $name) | .conclusion) // ""' <<< "$jobs")
|
||||
if [ -z "$warned" ] && { [ -z "$check" ] || [ "$check" = success ]; }; then
|
||||
echo "::warning title=Deps wait::Build $id of $BASE_REF has no job named \"$deps_job_name\". If the build jobs were renamed, update scripts/ci_deps_cache.sh. This does not affect this pull request."
|
||||
warned=1
|
||||
fi ;;
|
||||
*)
|
||||
sleep 120
|
||||
continue ;;
|
||||
esac ;;
|
||||
*)
|
||||
sleep 120
|
||||
continue ;;
|
||||
esac
|
||||
# Follows another unfinished build with the same key inputs, such as the
|
||||
# push that replaced a cancelled one.
|
||||
id=$(matching_base_build "$seen")
|
||||
[ -n "$id" ] || break
|
||||
seen="$seen $id"
|
||||
echo "Following build $id of $BASE_REF."
|
||||
done
|
||||
echo "No build of $BASE_REF saved $KEY."
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
check) check ;;
|
||||
wait) wait_for ;;
|
||||
*) echo "usage: $0 check|wait" >&2; exit 2 ;;
|
||||
esac
|
||||
Reference in New Issue
Block a user