diff --git a/.github/workflows/build_check_cache.yml b/.github/workflows/build_check_cache.yml index 31fdd6cb28..0da6c2b7b4 100644 --- a/.github/workflows/build_check_cache.yml +++ b/.github/workflows/build_check_cache.yml @@ -1,73 +1,198 @@ -name: Check Cache - -on: - workflow_call: - inputs: - os: - required: true - type: string - arch: - required: false - type: string - compiler: - required: false - type: string - default: msvc - build-deps-only: - required: false - type: boolean - force-build: - required: false - type: boolean - -jobs: - check_cache: # determines if there is a cache and outputs variables used in caching process - name: Check Cache - runs-on: ${{ inputs.os }} - outputs: - cache-key: ${{ steps.set_outputs.outputs.cache-key }} - cache-path: ${{ steps.set_outputs.outputs.cache-path }} - valid-cache: ${{ steps.cache_deps.outputs.cache-hit }} - steps: - - name: Checkout - uses: actions/checkout@v7 - with: - lfs: 'false' - - - name: set outputs - id: set_outputs - env: - # Anything that changes how the tree is built belongs in the key, or a job - # restores one it cannot use. Linux amd64 passes no arch deliberately, so - # 'linux-clang' keeps the cache it already has. - cache-os: ${{ runner.os == 'macOS' && format('{0}-{1}', inputs.os, inputs.arch) || (runner.os == 'Windows' && format('windows-{0}-{1}', inputs.arch, inputs.compiler) || format('linux-clang{0}', inputs.arch && format('-{0}', inputs.arch) || '')) }} - # The Windows ARM64 deps build in build-arm64, all others under build; - # build_deps.yml and build_orca.yml pass the Windows directory to build_win.bat. - dep-folder-name: ${{ runner.os == 'macOS' && format('/{0}', inputs.arch) || (runner.os == 'Windows' && inputs.arch == 'arm64') && '-arm64/OrcaSlicer_dep' || '/OrcaSlicer_dep' }} - output-cmd: ${{ runner.os == 'Windows' && '$env:GITHUB_OUTPUT' || '"$GITHUB_OUTPUT"'}} - run: | - echo cache-key=${{ env.cache-os }}-cache-orcaslicer_deps-build-${{ hashFiles('deps/**') }} >> ${{ env.output-cmd }} - echo cache-path=${{ github.workspace }}/deps/build${{ env.dep-folder-name }} >> ${{ env.output-cmd }} - - - name: load cache - id: cache_deps - uses: actions/cache@v6 - with: - path: ${{ steps.set_outputs.outputs.cache-path }} - key: ${{ steps.set_outputs.outputs.cache-key }} - lookup-only: true - - build_deps: # call next step - name: Build Deps - needs: [check_cache] - uses: ./.github/workflows/build_deps.yml - with: - cache-key: ${{ needs.check_cache.outputs.cache-key }} - cache-path: ${{ needs.check_cache.outputs.cache-path }} - valid-cache: ${{ needs.check_cache.outputs.valid-cache == 'true' }} - os: ${{ inputs.os }} - arch: ${{ inputs.arch }} - compiler: ${{ inputs.compiler }} - build-deps-only: ${{ inputs.build-deps-only }} - force-build: ${{ inputs.force-build }} - secrets: inherit +name: Check Cache + +on: + workflow_call: + inputs: + os: + required: true + type: string + arch: + required: false + type: string + compiler: + required: false + type: string + default: msvc + build-deps-only: + required: false + type: boolean + force-build: + required: false + type: boolean + +# Read by scripts/ci_deps_cache.sh. +env: + REPO: ${{ github.repository }} + WORKFLOW_REF: ${{ github.workflow_ref }} + BASE_REF: ${{ github.base_ref }} + +jobs: + check_cache: # determines if there is a cache and outputs variables used in caching process + name: Check Cache + # Hosted macOS runners are scarce, so their lookup runs on Linux. + runs-on: ${{ startsWith(inputs.os, 'macos-') && 'ubuntu-24.04' || inputs.os }} + outputs: + cache-key: ${{ steps.set_outputs.outputs.cache-key }} + cache-path: ${{ steps.set_outputs.outputs.cache-path }} + valid-cache: ${{ steps.cache_deps.outputs.cache-hit }} + macos-path: ${{ steps.macos_path.outputs.path }} + key-inputs: ${{ steps.wait_check.outputs.key-inputs }} + wait-for: ${{ steps.wait_check.outputs.wait-for }} + wait-since: ${{ steps.wait_check.outputs.wait-since }} + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + lfs: 'false' + # The wait check compares the merge commit with its first parent. + fetch-depth: ${{ startsWith(inputs.os, 'macos-') && 2 || 1 }} + # Fetches only deps/, which the key hashes, and the wait script. + sparse-checkout: | + /deps/ + /scripts/ci_deps_cache.sh + sparse-checkout-cone-mode: false + + - name: set outputs + id: set_outputs + env: + # Anything that changes how the tree is built belongs in the key, or a job + # restores one it cannot use. Linux amd64 passes no arch deliberately, so + # 'linux-clang' keeps the cache it already has. + cache-os: ${{ (runner.os == 'macOS' || startsWith(inputs.os, 'macos-')) && format('{0}-{1}', inputs.os, inputs.arch) || (runner.os == 'Windows' && format('windows-{0}-{1}', inputs.arch, inputs.compiler) || format('linux-clang{0}', inputs.arch && format('-{0}', inputs.arch) || '')) }} + # The Windows ARM64 deps build in build-arm64, all others under build; + # build_deps.yml and build_orca.yml pass the Windows directory to build_win.bat. + dep-folder-name: ${{ (runner.os == 'macOS' || startsWith(inputs.os, 'macos-')) && format('/{0}', inputs.arch) || (runner.os == 'Windows' && inputs.arch == 'arm64') && '-arm64/OrcaSlicer_dep' || '/OrcaSlicer_dep' }} + output-cmd: ${{ runner.os == 'Windows' && '$env:GITHUB_OUTPUT' || '"$GITHUB_OUTPUT"'}} + run: | + echo cache-key=${{ env.cache-os }}-cache-orcaslicer_deps-build-${{ hashFiles('deps/**') }} >> ${{ env.output-cmd }} + # Relative to the workspace, which differs from the build's when this job runs on Linux. + echo cache-path=deps/build${{ env.dep-folder-name }} >> ${{ env.output-cmd }} + + # actions/cache matches the path too, so a Linux lookup for macOS uses the + # workspace of a hosted macOS job. + - name: macOS cache path + id: macos_path + if: ${{ startsWith(inputs.os, 'macos-') }} + run: echo "path=/Users/runner/work/${GITHUB_REPOSITORY#*/}/${GITHUB_REPOSITORY#*/}/${{ steps.set_outputs.outputs.cache-path }}" >> "$GITHUB_OUTPUT" + + - name: load cache + id: cache_deps + uses: actions/cache@v6 + with: + path: ${{ steps.macos_path.outputs.path || format('{0}/{1}', github.workspace, steps.set_outputs.outputs.cache-path) }} + key: ${{ steps.set_outputs.outputs.cache-key }} + lookup-only: true + + # A pull request that misses the cache can wait for a base branch build of the same deps. + - name: check for a base branch build to wait for + id: wait_check + if: ${{ startsWith(inputs.os, 'macos-') && github.event_name == 'pull_request' && steps.cache_deps.outputs.cache-hit != 'true' }} + env: + GH_TOKEN: ${{ github.token }} + run: bash scripts/ci_deps_cache.sh check + + wait_for_deps: + name: Wait for base branch deps + needs: [check_cache] + if: ${{ needs.check_cache.outputs.wait-for != '' }} + runs-on: ubuntu-24.04 + env: + WAIT_MINUTES: 120 + # Must exceed WAIT_MINUTES. + timeout-minutes: 130 + # Waiting runs queue here, and only the first one holds a runner. + concurrency: + group: deps-wait-${{ github.base_ref }}-${{ needs.check_cache.outputs.cache-key }} + queue: max + outputs: + hit: ${{ steps.lookup.outputs.cache-hit }} + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + sparse-checkout: scripts/ci_deps_cache.sh + sparse-checkout-cone-mode: false + + - name: wait for the deps cache + env: + GH_TOKEN: ${{ github.token }} + KEY: ${{ needs.check_cache.outputs.cache-key }} + KEY_INPUTS: ${{ needs.check_cache.outputs.key-inputs }} + WAIT_FOR: ${{ needs.check_cache.outputs.wait-for }} + WAIT_SINCE: ${{ needs.check_cache.outputs.wait-since }} + ARCH: ${{ inputs.arch }} + run: bash scripts/ci_deps_cache.sh wait + + - name: load cache + id: lookup + uses: actions/cache@v6 + with: + path: ${{ needs.check_cache.outputs.macos-path }} + key: ${{ needs.check_cache.outputs.cache-key }} + lookup-only: true + + # Hosted macOS runners are scarce, so a pull request's macOS build waits until + # one is free that push and nightly builds do not need. See the script. + macos_admission: + name: Wait for a macOS runner + needs: [check_cache, wait_for_deps] + if: ${{ !cancelled() && needs.check_cache.result == 'success' && startsWith(inputs.os, 'macos-') && github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'macos-priority') }} + runs-on: ubuntu-24.04 + env: + WAIT_MINUTES: 240 + PRIORITY: false + # Must exceed WAIT_MINUTES. + timeout-minutes: 250 + # One line for every arch of every pull request, and only its first job polls. + concurrency: + group: macos-admission + queue: max + steps: &macos_admission_steps + - name: Checkout + uses: actions/checkout@v7 + with: + sparse-checkout: scripts/ci_macos_admission.py + sparse-checkout-cone-mode: false + + - name: wait for a macOS runner + env: + GH_TOKEN: ${{ github.token }} + MACOS_RUNNER_LIMIT: ${{ vars.MACOS_RUNNER_LIMIT }} + # -u, or the log shows nothing until the wait ends. + run: python3 -u scripts/ci_macos_admission.py + + # The macos-priority label, when the run starts, puts a pull request in its own + # line, which the normal one yields to. The names are fixed, because a skipped + # job is listed under its unevaluated name, and the script matches both. + macos_admission_priority: + name: Wait for a macOS runner (priority) + needs: [check_cache, wait_for_deps] + if: ${{ !cancelled() && needs.check_cache.result == 'success' && startsWith(inputs.os, 'macos-') && github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'macos-priority') }} + runs-on: ubuntu-24.04 + env: + WAIT_MINUTES: 240 + PRIORITY: true + # Must exceed WAIT_MINUTES. + timeout-minutes: 250 + concurrency: + group: macos-admission-priority + queue: max + steps: *macos_admission_steps + + build_deps: # call next step + name: Build Deps + needs: [check_cache, wait_for_deps, macos_admission, macos_admission_priority] + # A failed wait counts as no cache, so the deps are built here. A failed + # admission still builds rather than skip macOS. + if: ${{ !cancelled() && needs.check_cache.result == 'success' }} + uses: ./.github/workflows/build_deps.yml + with: + cache-key: ${{ needs.check_cache.outputs.cache-key }} + cache-path: ${{ needs.check_cache.outputs.cache-path }} + valid-cache: ${{ needs.check_cache.outputs.valid-cache == 'true' || needs.wait_for_deps.outputs.hit == 'true' }} + os: ${{ inputs.os }} + arch: ${{ inputs.arch }} + compiler: ${{ inputs.compiler }} + build-deps-only: ${{ inputs.build-deps-only }} + force-build: ${{ inputs.force-build }} + secrets: inherit diff --git a/.github/workflows/build_deps.yml b/.github/workflows/build_deps.yml index f1cc9a9723..3c778d03e6 100644 --- a/.github/workflows/build_deps.yml +++ b/.github/workflows/build_deps.yml @@ -45,7 +45,7 @@ jobs: - name: load cached deps uses: actions/cache@v6 with: - path: ${{ inputs.cache-path }} + path: ${{ github.workspace }}/${{ inputs.cache-path }} key: ${{ inputs.cache-key }} - uses: lukka/get-cmake@latest @@ -122,7 +122,7 @@ jobs: choco install strawberryperl } # cache-path is the install directory inside the deps build directory. - $deps = (Split-Path "${{ inputs.cache-path }}").Replace('\', '/') + $deps = (Split-Path "${{ github.workspace }}/${{ inputs.cache-path }}").Replace('\', '/') # -l compiles with Visual Studio's clang-cl and -x builds with Ninja; --msvc --msbuild is cl under the Visual Studio generator. $flags = if ("${{ inputs.compiler }}" -eq "clang") { "-l", "-x" } else { "--msvc", "--msbuild" } .\build_win.bat -d --arch ${{ inputs.arch }} --deps-dir $deps @flags diff --git a/.github/workflows/build_orca.yml b/.github/workflows/build_orca.yml index 9ebc319bf9..f29d7f0cc6 100644 --- a/.github/workflows/build_orca.yml +++ b/.github/workflows/build_orca.yml @@ -44,7 +44,7 @@ jobs: if: ${{ !(runner.os == 'macOS' && inputs.macos-combine-only) }} uses: actions/cache@v6 with: - path: ${{ inputs.cache-path }} + path: ${{ github.workspace }}/${{ inputs.cache-path }} key: ${{ inputs.cache-key }} fail-on-cache-miss: true @@ -467,7 +467,7 @@ jobs: # --tests builds the unit tests too; the unit_tests_windows_* jobs run them. run: | # cache-path is the install directory inside the deps build directory. - $deps = (Split-Path "${{ inputs.cache-path }}").Replace('\', '/') + $deps = (Split-Path "${{ github.workspace }}/${{ inputs.cache-path }}").Replace('\', '/') # -l compiles with Visual Studio's clang-cl and -x builds with Ninja; --msvc --msbuild is cl under the Visual Studio generator. $flags = if ("${{ inputs.compiler }}" -eq "clang") { "-l", "-x" } else { "--msvc", "--msbuild" } .\build_win.bat -s --tests -i --arch ${{ inputs.arch }} --build-dir $env:BUILD_DIR --deps-dir $deps @flags diff --git a/scripts/ci_deps_cache.sh b/scripts/ci_deps_cache.sh new file mode 100644 index 0000000000..a27b7e2522 --- /dev/null +++ b/scripts/ci_deps_cache.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash +# Lets a pull request's macOS deps build wait for a push build of the base +# branch that builds the same deps, from a Linux job. `check` runs after a cache +# miss and names a queued or running base build with the same deps/ and .github/ +# to wait for, unless the pull request changes either. `wait` waits until that +# build saves KEY. actions/cache then confirms the entry can be restored. +# +# Environment: GH_TOKEN, REPO, WORKFLOW_REF and BASE_REF, plus KEY_INPUTS, +# WAIT_FOR, WAIT_SINCE, WAIT_MINUTES, KEY and ARCH for `wait`. +# Writes key-inputs, wait-for and wait-since to $GITHUB_OUTPUT. +set -uo pipefail + +workflow=${WORKFLOW_REF%%@*} +workflow=${workflow##*/} + +# Must match the job names in build_all.yml, build_check_cache.yml and build_deps.yml. +check_job_name="build_macos_arch (${ARCH:-}) / Check Cache" +deps_job_name="build_macos_arch (${ARCH:-}) / Build Deps / Build Deps" + +out() { echo "$1" >> "${GITHUB_OUTPUT:-/dev/stdout}"; } + +# The cache key depends on deps/ and on the workflows under .github/. +local_key_inputs() { echo "$(git rev-parse "$1:.github") $(git rev-parse "$1:deps")"; } +remote_key_inputs() { + gh api "repos/$REPO/git/trees/$1" < /dev/null \ + --jq '[.tree[] | select(.path == ".github" or .path == "deps") | {key: .path, value: .sha}] | from_entries | "\(.[".github"]) \(.deps)"' +} + +# Prints the id of a push build of the base branch that has not finished, is not +# one of the ids in $1, and has KEY_INPUTS. +matching_base_build() { + local status runs id sha + runs=$(for status in in_progress queued pending waiting requested; do + gh api -X GET "repos/$REPO/actions/workflows/$workflow/runs" \ + -f branch="$BASE_REF" -f event=push -f status="$status" -f per_page=10 < /dev/null \ + | jq -r '.workflow_runs[] | "\(.id) \(.head_sha)"' + done) + while read -r id sha; do + [ -n "$id" ] || continue + [[ " ${1:-} " == *" $id "* ]] && continue + if [ "$(remote_key_inputs "$sha")" = "$KEY_INPUTS" ]; then + echo "$id" + return + fi + done <<< "$runs" +} + +check() { + KEY_INPUTS=$(local_key_inputs HEAD) + out "key-inputs=$KEY_INPUTS" + if [ "$KEY_INPUTS" != "$(local_key_inputs HEAD^1)" ]; then + echo "This pull request changes deps/ or .github/, so it builds its own deps." + return + fi + local id + id=$(matching_base_build) + if [ -n "$id" ]; then + echo "Build $id of $BASE_REF has the same deps/ and .github/. Waiting for its cache." + out "wait-for=$id" + out "wait-since=$(date +%s)" + else + echo "No build of $BASE_REF is building these deps." + fi +} + +key_saved() { + gh cache list -R "$REPO" --ref "refs/heads/$BASE_REF" --key "$KEY" --json key < /dev/null \ + | jq -e --arg key "$KEY" 'any(.[]; .key == $key)' > /dev/null +} + +wait_for() { + # Counts the limit from the check, including time queued behind other waiting + # pull requests. + local id=$WAIT_FOR seen=$WAIT_FOR deadline=$((${WAIT_SINCE:-$(date +%s)} + WAIT_MINUTES * 60)) jobs job check warned= + while [ "$(date +%s)" -lt "$deadline" ]; do + if key_saved; then + echo "Found $KEY." + return + fi + jobs=$(gh api -X GET "repos/$REPO/actions/runs/$id/jobs" -f filter=latest -f per_page=100 < /dev/null) + job=$(jq -r --arg name "$deps_job_name" 'first(.jobs[] | select(.name == $name) | "\(.status) \(.conclusion)") // ""' <<< "$jobs") + case "$job" in + "completed success" | "completed skipped") + # The new entry can take a moment to be listed, and a skipped job means + # the build found the cache, possibly under the default branch. + for _ in 1 2 3; do + key_saved && { echo "Found $KEY."; return; } + sleep 20 + done + echo "Build $id of $BASE_REF finished its deps without saving $KEY." + return ;; + "completed cancelled") ;; + completed*) + echo "The deps job of build $id of $BASE_REF did not succeed." + return ;; + # The deps job exists only once the build's cache check has succeeded. + "") + case "$(gh api "repos/$REPO/actions/runs/$id" --jq '"\(.status) \(.conclusion)"' < /dev/null)" in + "completed cancelled") ;; + completed*) + check=$(jq -r --arg name "$check_job_name" 'first(.jobs[] | select(.name == $name) | .conclusion) // ""' <<< "$jobs") + if [ -z "$warned" ] && { [ -z "$check" ] || [ "$check" = success ]; }; then + echo "::warning title=Deps wait::Build $id of $BASE_REF has no job named \"$deps_job_name\". If the build jobs were renamed, update scripts/ci_deps_cache.sh. This does not affect this pull request." + warned=1 + fi ;; + *) + sleep 120 + continue ;; + esac ;; + *) + sleep 120 + continue ;; + esac + # Follows another unfinished build with the same key inputs, such as the + # push that replaced a cancelled one. + id=$(matching_base_build "$seen") + [ -n "$id" ] || break + seen="$seen $id" + echo "Following build $id of $BASE_REF." + done + echo "No build of $BASE_REF saved $KEY." +} + +case "${1:-}" in + check) check ;; + wait) wait_for ;; + *) echo "usage: $0 check|wait" >&2; exit 2 ;; +esac diff --git a/scripts/ci_macos_admission.py b/scripts/ci_macos_admission.py new file mode 100644 index 0000000000..0b5089ac2b --- /dev/null +++ b/scripts/ci_macos_admission.py @@ -0,0 +1,269 @@ +#!/usr/bin/env python3 +"""Holds a pull request's macOS build until a hosted macOS runner is free for it. + +Runs from a Linux job in build_check_cache.yml, once per macOS arch, in a single +repo-wide line (a concurrency group with queue: max), so only the job at the +front of the line polls. It lets its arch in when the runners every active Build +all run holds or still needs, plus one for this arch, fit in MACOS_RUNNER_LIMIT: + +- A push, nightly or manual run holds RESERVE runners until its macOS work is + done. The jobs API lists only the jobs a run has reached, so what it still + needs cannot be counted and is reserved instead. +- A pull request run holds one runner for each arch let in until that arch's + app build is done, and none after. A run holding a runner while it waits for + its other arch would deadlock: the other arch can sit in the line behind a + pull request that waits for that runner. +- A run holds at least the macOS jobs it has queued or running. + +A pull request's universal build and tests skip the line and are not held for +in advance. They take minutes, and holding two runners for them through the +slower arch's build left runners idle while other builds waited. +- In the minutes around the nightly's cron time, RESERVE runners are held for it + until its run appears. + +A pull request labelled macos-priority when its run starts waits in a separate +line under the same rule, and the normal line counts every priority arch still +waiting as holding a runner, so the next free runners go to it. + +Any API error repeated FAILURES_BEFORE_ADMIT times, and the WAIT_MINUTES limit, +let the arch in, so a fault here never blocks pull requests. + +Environment: GH_TOKEN, REPO, WORKFLOW_REF, GITHUB_RUN_ID, and optionally PRIORITY, +MACOS_RUNNER_LIMIT, WAIT_MINUTES, POLL_SECONDS and GITHUB_API_URL. +""" + +import datetime +import json +import os +import sys +import time +import urllib.error +import urllib.parse +import urllib.request + +# A push, nightly or manual run builds arm64 and x86_64 at the same time, then +# the universal build and the macOS tests at the same time. +RESERVE = 2 + +# Must match the job names in build_all.yml and build_check_cache.yml. +ARCH_PREFIX = "build_macos_arch (" +GATE = "Wait for a macOS runner" +PRIORITY_GATE = GATE + " (priority)" +FINAL_JOBS = {"Build macOS Universal", "macOS arm64"} +# Must match the cron in build_all.yml. +NIGHTLY_UTC = datetime.time(2, 15) +NIGHTLY_REPO = "OrcaSlicer/OrcaSlicer" +NIGHTLY_LEAD = datetime.timedelta(minutes=10) +NIGHTLY_GRACE = datetime.timedelta(minutes=45) + +ACTIVE = {"queued", "in_progress", "waiting", "pending", "requested"} +# Runs created this recently are listed whatever their status. +RECENT = datetime.timedelta(hours=24) +FAILURES_BEFORE_ADMIT = 3 + + +def is_macos(job): + return any(label.startswith("macos-") for label in job.get("labels") or []) + + +def macos_done(jobs): + """True once the universal build and the macOS tests have finished or been skipped. + Both start together when the arch builds finish.""" + # A skipped caller job is listed under its own name, a started one as " / ". + final = [job for job in jobs if job["name"].split(" / ")[0] in FINAL_JOBS] + return bool(final) and all(job["status"] == "completed" for job in final) + + +def gates(jobs, names=(GATE, PRIORITY_GATE)): + return [job for job in jobs if job["name"].split(" / ")[-1] in names] + + +def arch_of(job): + name = job["name"] + return name[len(ARCH_PREFIX):name.find(")")] if name.startswith(ARCH_PREFIX) else None + + +def admitted_arches(jobs): + return {arch_of(job) for job in gates(jobs) if job["conclusion"] == "success"} + + +def arch_built(jobs, arch): + """True once the arch's app build finished, or one of its jobs failed or was cancelled.""" + own = [job for job in jobs if arch_of(job) == arch] + return any(job["conclusion"] in ("failure", "cancelled") for job in own) or \ + any(" / Build OrcaSlicer" in job["name"] and job["status"] == "completed" for job in own) + + +def priority_waiting(jobs): + """How many of the run's arches wait in the priority line.""" + return sum(1 for job in gates(jobs, (PRIORITY_GATE,)) if job["status"] != "completed") + + +def run_demand(run, jobs, yield_to_priority=False): + """macOS runners a run holds or still needs. With yield_to_priority, each + priority arch still waiting counts as holding one.""" + # A run with no jobs that is pending waits behind another run of its + # concurrency group, which holds the runners for both. + if not jobs and run["status"] in ("pending", "waiting"): + return 0 + active = sum(1 for job in jobs if is_macos(job) and job["status"] in ACTIVE) + if macos_done(jobs): + return active + if run["event"] != "pull_request": + return max(active, RESERVE) + held = sum(1 for arch in admitted_arches(jobs) if not arch_built(jobs, arch)) + if yield_to_priority: + held += priority_waiting(jobs) + return max(active, held) + + +def nightly_window(now): + """The window around today's nightly cron time, as (start, end) in UTC.""" + cron = datetime.datetime.combine(now.date(), NIGHTLY_UTC, tzinfo=datetime.timezone.utc) + return cron - NIGHTLY_LEAD, cron + NIGHTLY_GRACE + + +def parse_time(value): + return datetime.datetime.fromisoformat(value.replace("Z", "+00:00")) + + +class Api: + def __init__(self, token, url="https://api.github.com"): + self.token = token + self.url = url.rstrip("/") + # The token's hourly limit is shared by every workflow in the repository, + # so what the last response says is left shows their use too. + self.calls = 0 + self.remaining = self.limit = "?" + + def get(self, path, **params): + query = urllib.parse.urlencode(params) + request = urllib.request.Request(f"{self.url}/{path}?{query}", headers={ + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {self.token}", + "X-GitHub-Api-Version": "2022-11-28", + }) + self.calls += 1 + try: + with urllib.request.urlopen(request, timeout=30) as response: + self.note_rate_limit(response.headers) + return json.load(response) + except urllib.error.HTTPError as error: + # A rate-limited request fails, and its headers say so. + self.note_rate_limit(error.headers) + raise + + def note_rate_limit(self, headers): + self.remaining = headers.get("X-RateLimit-Remaining", "?") + self.limit = headers.get("X-RateLimit-Limit", "?") + + def usage(self, calls_before): + return f" API calls this check: {self.calls - calls_before}, rate limit left: {self.remaining} of {self.limit}" + + +def list_runs(api, repo, workflow, **params): + runs, page = [], 1 + while True: + body = api.get(f"repos/{repo}/actions/workflows/{workflow}/runs", + per_page=100, page=page, **params) + runs += body["workflow_runs"] + if len(runs) >= body["total_count"] or not body["workflow_runs"]: + return runs + page += 1 + + +def latest_run(api, repo, workflow, **params): + runs = api.get(f"repos/{repo}/actions/workflows/{workflow}/runs", + per_page=1, **params)["workflow_runs"] + return runs[0] if runs else None + + +def measure(api, repo, workflow, run_id, now, priority=False): + """Total macOS runners held or needed, and one line per run that holds any. + The priority line does not count the priority runs waiting behind it.""" + total, lines = 0, [] + # A run is listed as queued whenever one of its jobs waits for a runner, and + # as pending whenever one waits in a concurrency group, so runs in any of these + # can hold runners. A run that moves between the lists between the calls is + # in neither, as one is whenever the job ahead of this one leaves the line, so + # runs created recently are listed by creation time as well, and this run is + # read directly. + since = (now - RECENT).strftime("%Y-%m-%dT%H:%M:%SZ") + runs = {run["id"]: run for status in ("in_progress", "queued", "pending", "waiting") + for run in list_runs(api, repo, workflow, status=status)} + runs.update((run["id"], run) for run in list_runs(api, repo, workflow, created=f">={since}") + if run["status"] != "completed") + runs[run_id] = api.get(f"repos/{repo}/actions/runs/{run_id}") + for run in runs.values(): + jobs = api.get(f"repos/{repo}/actions/runs/{run['id']}/jobs", + filter="latest", per_page=100)["jobs"] + own = run["id"] == run_id + demand = run_demand(run, jobs, yield_to_priority=not priority and not own) + if demand: + total += demand + waiting = ", priority, waiting" if priority_waiting(jobs) and not own else "" + lines.append(f" {demand} run {run['id']} ({run['event']}, {run['head_branch']}{waiting})") + + # build_all.yml runs the nightly only in the main repository. + start, end = nightly_window(now) + if repo == NIGHTLY_REPO and start <= now < end: + last = latest_run(api, repo, workflow, event="schedule") + if not last or parse_time(last["created_at"]) < start: + total += RESERVE + lines.append(f" {RESERVE} the nightly, due at {NIGHTLY_UTC:%H:%M} UTC") + return total, lines + + +def wait(measure_now, limit, wait_minutes, poll_seconds, + clock=time.monotonic, sleep=time.sleep, log=print): + """Polls until this arch fits. Returns the reason it was let in.""" + deadline = clock() + wait_minutes * 60 + failures = 0 + while True: + try: + total, lines = measure_now() + except (urllib.error.URLError, OSError, ValueError, KeyError, TypeError) as error: + failures += 1 + log(f"::warning title=macOS admission::Could not read the queue ({error}).") + if failures >= FAILURES_BEFORE_ADMIT: + return "the queue could not be read" + else: + failures = 0 + log(f"{total} of {limit} macOS runners held or needed:") + for line in lines: + log(line) + if total + 1 <= limit: + return "a runner is free" + if clock() + poll_seconds >= deadline: + return f"it waited {wait_minutes} minutes" + sleep(poll_seconds) + + +def main(): + repo = os.environ["REPO"] + workflow = os.environ["WORKFLOW_REF"].split("@")[0].rsplit("/", 1)[-1] + api = Api(os.environ["GH_TOKEN"], os.environ.get("GITHUB_API_URL", "https://api.github.com")) + run_id = int(os.environ["GITHUB_RUN_ID"]) + limit = int(os.environ.get("MACOS_RUNNER_LIMIT") or 5) + + def measure_now(): + calls_before = api.calls + try: + total, lines = measure(api, repo, workflow, run_id, datetime.datetime.now(datetime.timezone.utc), + priority=os.environ.get("PRIORITY") == "true") + except Exception: + print(api.usage(calls_before).strip()) + raise + return total, lines + [api.usage(calls_before)] + + reason = wait( + measure_now, + limit, + wait_minutes=int(os.environ.get("WAIT_MINUTES") or 240), + poll_seconds=int(os.environ.get("POLL_SECONDS") or 180), + ) + print(f"Letting this macOS build in: {reason}.") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tests/test_ci_macos_admission.py b/scripts/tests/test_ci_macos_admission.py new file mode 100644 index 0000000000..b872545d3f --- /dev/null +++ b/scripts/tests/test_ci_macos_admission.py @@ -0,0 +1,370 @@ +#!/usr/bin/env python3 +"""Tests for scripts/ci_macos_admission.py (stdlib unittest, no external deps). + +Run from the repo root: python -m unittest discover -s scripts/tests -v +""" + +import datetime +import os +import sys +import unittest +import urllib.error +from unittest import mock + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) + +import ci_macos_admission as admission # noqa: E402 + +UTC = datetime.timezone.utc +NOON = datetime.datetime(2026, 10, 10, 12, 0, tzinfo=UTC) + + +def job(name, status="completed", conclusion="success", labels=("macos-15",)): + return {"name": name, "status": status, "conclusion": conclusion, "labels": list(labels)} + + +def gate(arch, status="completed", conclusion="success", priority=False): + suffix = " (priority)" if priority else "" + return job(f"build_macos_arch ({arch}) / Wait for a macOS runner{suffix}", status, conclusion, + labels=("ubuntu-24.04",)) + + +def build(arch, status="in_progress"): + return job(f"build_macos_arch ({arch}) / Build Deps / Build OrcaSlicer / Build OrcaSlicer", + status, None if status != "completed" else "success") + + +CHECK_CACHE = job("build_macos_arch (arm64) / Check Cache", labels=("ubuntu-24.04",)) +UNIVERSAL_DONE = job("Build macOS Universal / Build OrcaSlicer") +TESTS_DONE = job("macOS arm64 / Unit Tests") +PUSH = {"id": 1, "event": "push", "head_branch": "main", "status": "in_progress"} +PR = {"id": 2, "event": "pull_request", "head_branch": "topic", "status": "in_progress"} + + +class MacosDoneTest(unittest.TestCase): + def test_not_done_before_the_final_jobs_are_listed(self): + self.assertFalse(admission.macos_done([CHECK_CACHE, build("arm64", "completed")])) + + def test_not_done_while_a_final_job_runs(self): + running = job("macOS arm64 / Unit Tests", "in_progress", None) + self.assertFalse(admission.macos_done([UNIVERSAL_DONE, running])) + + def test_done_when_the_final_jobs_finished(self): + self.assertTrue(admission.macos_done([build("arm64", "completed"), UNIVERSAL_DONE, TESTS_DONE])) + + def test_done_when_the_final_jobs_were_skipped(self): + # A skipped caller job is listed under its own name, without labels. + skipped = [job("Build macOS Universal", conclusion="skipped", labels=()), + job("macOS arm64", conclusion="skipped", labels=())] + self.assertTrue(admission.macos_done(skipped)) + + +class RunDemandTest(unittest.TestCase): + def test_push_reserves_before_its_macos_jobs_are_listed(self): + self.assertEqual(admission.run_demand(PUSH, []), admission.RESERVE) + self.assertEqual(admission.run_demand(PUSH, [CHECK_CACHE]), admission.RESERVE) + + def test_push_keeps_its_reserve_between_stages(self): + jobs = [build("arm64", "completed"), build("x86_64", "completed")] + self.assertEqual(admission.run_demand(PUSH, jobs), admission.RESERVE) + + def test_push_releases_when_macos_is_done(self): + jobs = [build("arm64", "completed"), UNIVERSAL_DONE, TESTS_DONE] + self.assertEqual(admission.run_demand(PUSH, jobs), 0) + + def test_queued_jobs_count_as_busy(self): + jobs = [build("arm64", "queued")] + self.assertEqual(admission.run_demand(PR, jobs), 1) + + def test_pull_request_holds_nothing_before_it_is_let_in(self): + self.assertEqual(admission.run_demand(PR, [CHECK_CACHE, gate("arm64", "in_progress", None)]), 0) + + def test_one_arch_let_in_holds_a_runner_until_it_is_built(self): + # Including before its first macOS job is listed, and between its stages. + self.assertEqual(admission.run_demand(PR, [gate("arm64")]), 1) + self.assertEqual(admission.run_demand(PR, [gate("arm64"), build("arm64")]), 1) + + def test_one_arch_let_in_holds_nothing_once_it_is_built(self): + # Its other arch may be in the line behind a pull request waiting for this + # runner, so holding it would deadlock. + jobs = [gate("arm64"), build("arm64", "completed"), gate("x86_64", "pending", None)] + self.assertEqual(admission.run_demand(PR, jobs), 0) + + def test_one_arch_let_in_holds_nothing_once_it_failed(self): + failed = job("build_macos_arch (arm64) / Build Deps / Build Deps", conclusion="failure") + self.assertEqual(admission.run_demand(PR, [gate("arm64"), failed]), 0) + + def test_both_arches_let_in_hold_only_what_still_builds(self): + # The universal build and tests are not held for in advance. + jobs = [gate("arm64"), gate("x86_64"), build("arm64", "completed"), build("x86_64")] + self.assertEqual(admission.run_demand(PR, jobs), 1) + + def test_universal_build_and_tests_count_once_queued(self): + built = [gate("arm64"), gate("x86_64"), build("arm64", "completed"), build("x86_64", "completed")] + self.assertEqual(admission.run_demand(PR, built), 0) + queued = [job("Build macOS Universal / Build OrcaSlicer", "queued", None), + job("macOS arm64 / Unit Tests", "in_progress", None)] + self.assertEqual(admission.run_demand(PR, built + queued), 2) + + def test_pull_request_releases_when_macos_is_done(self): + jobs = [gate("arm64"), gate("x86_64"), UNIVERSAL_DONE, TESTS_DONE] + self.assertEqual(admission.run_demand(PR, jobs), 0) + + def test_pull_request_without_the_line_holds_what_it_runs(self): + # A run started from a workflow without the line. + self.assertEqual(admission.run_demand(PR, [build("arm64"), build("x86_64", "queued")]), 2) + + def test_finished_and_non_macos_jobs_are_not_counted(self): + jobs = [build("arm64", "completed"), CHECK_CACHE, + job("build_linux (ubuntu-24.04) / Check Cache", "in_progress", None, ("ubuntu-24.04",))] + self.assertEqual(admission.run_demand(PR, jobs), 0) + + +class PriorityTest(unittest.TestCase): + WAITING = [gate("arm64", "in_progress", None, priority=True), gate("x86_64", "pending", None, priority=True)] + + def test_normal_line_yields_to_a_waiting_priority_run(self): + self.assertEqual(admission.run_demand(PR, self.WAITING, yield_to_priority=True), admission.RESERVE) + + def test_priority_line_does_not_count_priority_runs_behind_it(self): + self.assertEqual(admission.run_demand(PR, self.WAITING), 0) + + def test_a_waiting_normal_run_is_not_yielded_to(self): + waiting = [gate("arm64", "in_progress", None), gate("x86_64", "pending", None)] + self.assertEqual(admission.run_demand(PR, waiting, yield_to_priority=True), 0) + + def test_a_priority_run_half_let_in_still_counts_its_waiting_arch(self): + jobs = [gate("arm64", priority=True), gate("x86_64", "pending", None, priority=True)] + self.assertEqual(admission.run_demand(PR, jobs), 1) + self.assertEqual(admission.run_demand(PR, jobs, yield_to_priority=True), 2) + + def test_skipped_priority_jobs_of_other_platforms_are_not_waiting(self): + jobs = [job("build_linux (ubuntu-24.04) / Wait for a macOS runner (priority)", + conclusion="skipped", labels=("ubuntu-24.04",))] + self.assertEqual(admission.run_demand(PR, jobs, yield_to_priority=True), 0) + + def test_measure_from_each_line(self): + api = FakeApi({("runs", "pending"): [dict(PR, status="pending")], jobs_path(2): self.WAITING}) + total, lines = admission.measure(api, "o/r", "build_all.yml", 99, NOON) + self.assertEqual(total, admission.RESERVE) + self.assertIn("priority, waiting", lines[0]) + total, _ = admission.measure(api, "o/r", "build_all.yml", 99, NOON, priority=True) + self.assertEqual(total, 0) + + def test_own_waiting_run_is_not_counted(self): + api = FakeApi({("runs", "pending"): [dict(PR, status="pending")], jobs_path(2): self.WAITING}) + total, _ = admission.measure(api, "o/r", "build_all.yml", 2, NOON) + self.assertEqual(total, 0) + + +class FakeApi: + """Answers GET requests from a dict of path -> list of pages (or one body).""" + + def __init__(self, responses): + self.responses = responses + self.calls = [] + + def get(self, path, **params): + self.calls.append((path, params)) + if path.endswith("/runs") and ("status" in params or "created" in params): + runs = self.responses.get(("runs", params.get("status", "recent")), []) + start = (params["page"] - 1) * params["per_page"] + return {"total_count": len(runs), "workflow_runs": runs[start:start + params["per_page"]]} + if path.endswith("/runs"): + return {"workflow_runs": self.responses.get(("runs", params.get("event")), [])[:1]} + if path.endswith("/jobs"): + return {"jobs": self.responses.get(path, [])} + # A run read by its id, by default a pull request with no jobs listed. + return self.responses.get(path, dict(PR, id=int(path.rsplit("/", 1)[-1]))) + + +def jobs_path(run_id): + return f"repos/o/r/actions/runs/{run_id}/jobs" + + +class MeasureTest(unittest.TestCase): + def test_sums_in_progress_and_queued_runs(self): + queued_pr = dict(PR, id=3, status="queued") + api = FakeApi({ + ("runs", "in_progress"): [PUSH, PR], + ("runs", "queued"): [queued_pr], + jobs_path(1): [CHECK_CACHE], + jobs_path(2): [gate("arm64"), build("arm64")], + jobs_path(3): [gate("arm64"), gate("x86_64"), build("arm64", "queued")], + }) + total, lines = admission.measure(api, "o/r", "build_all.yml", 99, NOON) + self.assertEqual(total, admission.RESERVE + 1 + admission.RESERVE) + self.assertEqual(len(lines), 3) + + def test_counts_a_pending_run_that_holds_runners(self): + # One of its jobs waits in a concurrency group while its macOS builds run. + pending_pr = dict(PR, status="pending") + api = FakeApi({("runs", "pending"): [pending_pr], jobs_path(2): [gate("arm64"), build("arm64")]}) + total, _ = admission.measure(api, "o/r", "build_all.yml", 99, NOON) + self.assertEqual(total, 1) + + def test_a_push_waiting_behind_another_holds_nothing(self): + api = FakeApi({("runs", "pending"): [dict(PUSH, status="pending")], jobs_path(1): []}) + total, _ = admission.measure(api, "o/r", "build_all.yml", 99, NOON) + self.assertEqual(total, 0) + + def test_reads_every_page_of_runs(self): + runs = [dict(PR, id=i) for i in range(150)] + responses = {("runs", "in_progress"): runs} + responses.update({jobs_path(i): [gate("arm64")] for i in range(150)}) + total, _ = admission.measure(FakeApi(responses), "o/r", "build_all.yml", 999, NOON) + self.assertEqual(total, 150) + + def test_reserves_for_the_nightly_until_its_run_appears(self): + due = datetime.datetime(2026, 10, 10, 2, 10, tzinfo=UTC) + yesterday = {"created_at": "2026-10-09T02:20:00Z"} + total, lines = admission.measure( + FakeApi({("runs", "schedule"): [yesterday]}), "OrcaSlicer/OrcaSlicer", "build_all.yml", 99, due) + self.assertEqual(total, admission.RESERVE) + self.assertIn("nightly", lines[0]) + + def test_no_nightly_reserve_once_its_run_exists(self): + due = datetime.datetime(2026, 10, 10, 2, 30, tzinfo=UTC) + today = {"created_at": "2026-10-10T02:21:00Z"} + total, _ = admission.measure( + FakeApi({("runs", "schedule"): [today]}), "OrcaSlicer/OrcaSlicer", "build_all.yml", 99, due) + self.assertEqual(total, 0) + + def test_no_nightly_reserve_in_a_fork(self): + due = datetime.datetime(2026, 10, 10, 2, 10, tzinfo=UTC) + total, _ = admission.measure(FakeApi({}), "fork/OrcaSlicer", "build_all.yml", 99, due) + self.assertEqual(total, 0) + + def test_counts_a_run_in_both_lists_once(self): + api = FakeApi({ + ("runs", "in_progress"): [PUSH], + ("runs", "queued"): [dict(PUSH, status="queued")], + jobs_path(1): [], + }) + total, _ = admission.measure(api, "o/r", "build_all.yml", 99, NOON) + self.assertEqual(total, admission.RESERVE) + + def test_counts_its_own_run(self): + jobs = [gate("arm64"), build("arm64"), gate("x86_64", "in_progress", None)] + api = FakeApi({("runs", "in_progress"): [PR], jobs_path(2): jobs}) + self.assertEqual(admission.measure(api, "o/r", "build_all.yml", 2, NOON)[0], 1) + + def test_a_run_whose_other_arch_waits_behind_this_one_does_not_block_it(self): + # The line is A-arm64, B-arm64, A-x86_64. A's arm64 is built and its + # x86_64 is pending behind B, so A must not hold a runner B waits for. + other = dict(PR, id=1, status="pending") + api = FakeApi({ + ("runs", "pending"): [other, dict(PR, status="pending")], + jobs_path(1): [gate("arm64"), build("arm64", "completed"), gate("x86_64", "pending", None)], + jobs_path(2): [gate("arm64", "in_progress", None)], + }) + total, _ = admission.measure(api, "o/r", "build_all.yml", 2, NOON) + self.assertEqual(total, 0) + + def test_counts_its_own_run_when_no_status_list_has_it(self): + # The run moves from pending to in_progress as its last gate leaves the + # line, and is in neither list while that gate checks. + jobs = [gate("x86_64"), build("x86_64", "queued"), gate("arm64", "in_progress", None)] + api = FakeApi({jobs_path(2): jobs}) + self.assertEqual(admission.measure(api, "o/r", "build_all.yml", 2, NOON)[0], 1) + + def test_counts_a_recent_run_that_no_status_list_has(self): + api = FakeApi({("runs", "recent"): [PR], jobs_path(2): [gate("arm64"), build("arm64")]}) + self.assertEqual(admission.measure(api, "o/r", "build_all.yml", 99, NOON)[0], 1) + + def test_lists_recent_runs_from_a_day_before(self): + api = FakeApi({}) + admission.measure(api, "o/r", "build_all.yml", 99, NOON) + self.assertIn(">=2026-10-09T12:00:00Z", [params.get("created") for _, params in api.calls]) + + def test_finished_recent_runs_are_not_read(self): + done = dict(PUSH, status="completed") + api = FakeApi({("runs", "recent"): [done]}) + total, _ = admission.measure(api, "o/r", "build_all.yml", 99, NOON) + self.assertEqual(total, 0) + self.assertNotIn(jobs_path(1), [path for path, _ in api.calls]) + + def test_no_nightly_lookup_outside_its_window(self): + api = FakeApi({}) + admission.measure(api, "o/r", "build_all.yml", 99, NOON) + self.assertFalse(any("event" in params for _, params in api.calls)) + + +class ApiTest(unittest.TestCase): + class Response: + def __init__(self, headers): + self.headers = headers + + def __enter__(self): + return self + + def __exit__(self, *_): + return False + + def read(self): + return b"{}" + + def test_reports_calls_and_the_rate_limit_left(self): + api = admission.Api("token") + responses = [self.Response({}), + self.Response({"X-RateLimit-Remaining": "812", "X-RateLimit-Limit": "1000"})] + with mock.patch.object(admission.urllib.request, "urlopen", side_effect=responses): + api.get("first") + calls_before = api.calls + api.get("second") + self.assertEqual(api.usage(0), " API calls this check: 2, rate limit left: 812 of 1000") + self.assertEqual(api.usage(calls_before), " API calls this check: 1, rate limit left: 812 of 1000") + + def test_reads_the_rate_limit_from_a_failed_request(self): + api = admission.Api("token") + headers = {"X-RateLimit-Remaining": "0", "X-RateLimit-Limit": "1000"} + error = urllib.error.HTTPError("url", 403, "Forbidden", headers, None) + with mock.patch.object(admission.urllib.request, "urlopen", side_effect=error), \ + self.assertRaises(urllib.error.HTTPError): + api.get("runs") + self.assertEqual(api.usage(0), " API calls this check: 1, rate limit left: 0 of 1000") + + +class WaitTest(unittest.TestCase): + def run_wait(self, results, limit=5, wait_minutes=60, poll_seconds=120): + results = iter(results) + now = [0.0] + + def measure_now(): + result = next(results) + if isinstance(result, Exception): + raise result + return result, [] + + def sleep(seconds): + now[0] += seconds + + reason = admission.wait(measure_now, limit, wait_minutes, poll_seconds, + clock=lambda: now[0], sleep=sleep, log=lambda _: None) + return reason, now[0] + + def test_lets_in_when_a_runner_fits(self): + self.assertEqual(self.run_wait([4]), ("a runner is free", 0)) + + def test_waits_while_full(self): + self.assertEqual(self.run_wait([5, 6, 4]), ("a runner is free", 240)) + + + def test_lets_in_after_repeated_api_errors(self): + error = urllib.error.URLError("rate limited") + reason, _ = self.run_wait([error, error, error]) + self.assertEqual(reason, "the queue could not be read") + + def test_a_good_read_resets_the_error_count(self): + error = urllib.error.URLError("rate limited") + reason, _ = self.run_wait([error, error, 5, error, error, 3]) + self.assertEqual(reason, "a runner is free") + + def test_lets_in_at_the_time_limit(self): + reason, elapsed = self.run_wait([5] * 100, wait_minutes=10, poll_seconds=120) + self.assertEqual(reason, "it waited 10 minutes") + self.assertLess(elapsed, 600) + + +if __name__ == "__main__": + unittest.main()