diff --git a/.github/workflows/build_check_cache.yml b/.github/workflows/build_check_cache.yml index a01a9f3a8e..f9ba63a3e6 100644 --- a/.github/workflows/build_check_cache.yml +++ b/.github/workflows/build_check_cache.yml @@ -1,179 +1,182 @@ -name: Check Cache - -on: - workflow_call: - inputs: - os: - required: true - type: string - arch: - required: false - type: string - compiler: - required: false - type: string - default: msvc - build-deps-only: - required: false - type: boolean - force-build: - required: false - type: boolean - -# Read by scripts/ci_deps_cache.sh. -env: - REPO: ${{ github.repository }} - WORKFLOW_REF: ${{ github.workflow_ref }} - BASE_REF: ${{ github.base_ref }} - -jobs: - check_cache: # determines if there is a cache and outputs variables used in caching process - name: Check Cache - # Hosted macOS runners are scarce, so their lookup runs on Linux. - runs-on: ${{ startsWith(inputs.os, 'macos-') && 'ubuntu-24.04' || inputs.os }} - outputs: - cache-key: ${{ steps.set_outputs.outputs.cache-key }} - cache-path: ${{ steps.set_outputs.outputs.cache-path }} - valid-cache: ${{ steps.cache_deps.outputs.cache-hit }} - macos-path: ${{ steps.macos_path.outputs.path }} - key-inputs: ${{ steps.wait_check.outputs.key-inputs }} - wait-for: ${{ steps.wait_check.outputs.wait-for }} - wait-since: ${{ steps.wait_check.outputs.wait-since }} - steps: - - name: Checkout - uses: actions/checkout@v7 - with: - lfs: 'false' - # The wait check compares the merge commit with its first parent. - fetch-depth: ${{ startsWith(inputs.os, 'macos-') && 2 || 1 }} - # Fetches only deps/, which the key hashes, and the wait script. - sparse-checkout: | - /deps/ - /scripts/ci_deps_cache.sh - sparse-checkout-cone-mode: false - - - name: set outputs - id: set_outputs - env: - # Anything that changes how the tree is built belongs in the key, or a job - # restores one it cannot use. Linux amd64 passes no arch deliberately, so - # 'linux-clang' keeps the cache it already has. - cache-os: ${{ (runner.os == 'macOS' || startsWith(inputs.os, 'macos-')) && format('{0}-{1}', inputs.os, inputs.arch) || (runner.os == 'Windows' && format('windows-{0}-{1}', inputs.arch, inputs.compiler) || format('linux-clang{0}', inputs.arch && format('-{0}', inputs.arch) || '')) }} - # The Windows ARM64 deps build in build-arm64, all others under build; - # build_deps.yml and build_orca.yml pass the Windows directory to build_win.bat. - dep-folder-name: ${{ (runner.os == 'macOS' || startsWith(inputs.os, 'macos-')) && format('/{0}', inputs.arch) || (runner.os == 'Windows' && inputs.arch == 'arm64') && '-arm64/OrcaSlicer_dep' || '/OrcaSlicer_dep' }} - output-cmd: ${{ runner.os == 'Windows' && '$env:GITHUB_OUTPUT' || '"$GITHUB_OUTPUT"'}} - run: | - echo cache-key=${{ env.cache-os }}-cache-orcaslicer_deps-build-${{ hashFiles('deps/**') }} >> ${{ env.output-cmd }} - # Relative to the workspace, which differs from the build's when this job runs on Linux. - echo cache-path=deps/build${{ env.dep-folder-name }} >> ${{ env.output-cmd }} - - # actions/cache matches the path too, so a Linux lookup for macOS uses the - # workspace of a hosted macOS job. - - name: macOS cache path - id: macos_path - if: ${{ startsWith(inputs.os, 'macos-') }} - run: echo "path=/Users/runner/work/${GITHUB_REPOSITORY#*/}/${GITHUB_REPOSITORY#*/}/${{ steps.set_outputs.outputs.cache-path }}" >> "$GITHUB_OUTPUT" - - - name: load cache - id: cache_deps - uses: actions/cache@v6 - with: - path: ${{ steps.macos_path.outputs.path || format('{0}/{1}', github.workspace, steps.set_outputs.outputs.cache-path) }} - key: ${{ steps.set_outputs.outputs.cache-key }} - lookup-only: true - - # A pull request that misses the cache can wait for a base branch build of the same deps. - - name: check for a base branch build to wait for - id: wait_check - if: ${{ startsWith(inputs.os, 'macos-') && github.event_name == 'pull_request' && steps.cache_deps.outputs.cache-hit != 'true' }} - env: - GH_TOKEN: ${{ github.token }} - run: bash scripts/ci_deps_cache.sh check - - wait_for_deps: - name: Wait for base branch deps - needs: [check_cache] - if: ${{ needs.check_cache.outputs.wait-for != '' }} - runs-on: ubuntu-24.04 - env: - WAIT_MINUTES: 120 - # Must exceed WAIT_MINUTES. - timeout-minutes: 130 - # Waiting runs queue here, and only the first one holds a runner. - concurrency: - group: deps-wait-${{ github.base_ref }}-${{ needs.check_cache.outputs.cache-key }} - queue: max - outputs: - hit: ${{ steps.lookup.outputs.cache-hit }} - steps: - - name: Checkout - uses: actions/checkout@v7 - with: - sparse-checkout: scripts/ci_deps_cache.sh - sparse-checkout-cone-mode: false - - - name: wait for the deps cache - env: - GH_TOKEN: ${{ github.token }} - KEY: ${{ needs.check_cache.outputs.cache-key }} - KEY_INPUTS: ${{ needs.check_cache.outputs.key-inputs }} - WAIT_FOR: ${{ needs.check_cache.outputs.wait-for }} - WAIT_SINCE: ${{ needs.check_cache.outputs.wait-since }} - ARCH: ${{ inputs.arch }} - run: bash scripts/ci_deps_cache.sh wait - - - name: load cache - id: lookup - uses: actions/cache@v6 - with: - path: ${{ needs.check_cache.outputs.macos-path }} - key: ${{ needs.check_cache.outputs.cache-key }} - lookup-only: true - - # Hosted macOS runners are scarce, so a pull request's macOS build waits until - # one is free that push and nightly builds do not need. See the script. - macos_admission: - name: Wait for a macOS runner - needs: [check_cache, wait_for_deps] - if: ${{ !cancelled() && needs.check_cache.result == 'success' && startsWith(inputs.os, 'macos-') && github.event_name == 'pull_request' }} - runs-on: ubuntu-24.04 - env: - WAIT_MINUTES: 240 - # Must exceed WAIT_MINUTES. - timeout-minutes: 250 - # One line for every arch of every pull request, and only its first job polls. - concurrency: - group: macos-admission - queue: max - steps: - - name: Checkout - uses: actions/checkout@v7 - with: - sparse-checkout: scripts/ci_macos_admission.py - sparse-checkout-cone-mode: false - - - name: wait for a macOS runner - env: - GH_TOKEN: ${{ github.token }} - MACOS_RUNNER_LIMIT: ${{ vars.MACOS_RUNNER_LIMIT }} +name: Check Cache + +on: + workflow_call: + inputs: + os: + required: true + type: string + arch: + required: false + type: string + compiler: + required: false + type: string + default: msvc + build-deps-only: + required: false + type: boolean + force-build: + required: false + type: boolean + +# Read by scripts/ci_deps_cache.sh. +env: + REPO: ${{ github.repository }} + WORKFLOW_REF: ${{ github.workflow_ref }} + BASE_REF: ${{ github.base_ref }} + +jobs: + check_cache: # determines if there is a cache and outputs variables used in caching process + name: Check Cache + # Hosted macOS runners are scarce, so their lookup runs on Linux. + runs-on: ${{ startsWith(inputs.os, 'macos-') && 'ubuntu-24.04' || inputs.os }} + outputs: + cache-key: ${{ steps.set_outputs.outputs.cache-key }} + cache-path: ${{ steps.set_outputs.outputs.cache-path }} + valid-cache: ${{ steps.cache_deps.outputs.cache-hit }} + macos-path: ${{ steps.macos_path.outputs.path }} + key-inputs: ${{ steps.wait_check.outputs.key-inputs }} + wait-for: ${{ steps.wait_check.outputs.wait-for }} + wait-since: ${{ steps.wait_check.outputs.wait-since }} + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + lfs: 'false' + # The wait check compares the merge commit with its first parent. + fetch-depth: ${{ startsWith(inputs.os, 'macos-') && 2 || 1 }} + # Fetches only deps/, which the key hashes, and the wait script. + sparse-checkout: | + /deps/ + /scripts/ci_deps_cache.sh + sparse-checkout-cone-mode: false + + - name: set outputs + id: set_outputs + env: + # Anything that changes how the tree is built belongs in the key, or a job + # restores one it cannot use. Linux amd64 passes no arch deliberately, so + # 'linux-clang' keeps the cache it already has. + cache-os: ${{ (runner.os == 'macOS' || startsWith(inputs.os, 'macos-')) && format('{0}-{1}', inputs.os, inputs.arch) || (runner.os == 'Windows' && format('windows-{0}-{1}', inputs.arch, inputs.compiler) || format('linux-clang{0}', inputs.arch && format('-{0}', inputs.arch) || '')) }} + # The Windows ARM64 deps build in build-arm64, all others under build; + # build_deps.yml and build_orca.yml pass the Windows directory to build_win.bat. + dep-folder-name: ${{ (runner.os == 'macOS' || startsWith(inputs.os, 'macos-')) && format('/{0}', inputs.arch) || (runner.os == 'Windows' && inputs.arch == 'arm64') && '-arm64/OrcaSlicer_dep' || '/OrcaSlicer_dep' }} + output-cmd: ${{ runner.os == 'Windows' && '$env:GITHUB_OUTPUT' || '"$GITHUB_OUTPUT"'}} + run: | + echo cache-key=${{ env.cache-os }}-cache-orcaslicer_deps-build-${{ hashFiles('deps/**') }} >> ${{ env.output-cmd }} + # Relative to the workspace, which differs from the build's when this job runs on Linux. + echo cache-path=deps/build${{ env.dep-folder-name }} >> ${{ env.output-cmd }} + + # actions/cache matches the path too, so a Linux lookup for macOS uses the + # workspace of a hosted macOS job. + - name: macOS cache path + id: macos_path + if: ${{ startsWith(inputs.os, 'macos-') }} + run: echo "path=/Users/runner/work/${GITHUB_REPOSITORY#*/}/${GITHUB_REPOSITORY#*/}/${{ steps.set_outputs.outputs.cache-path }}" >> "$GITHUB_OUTPUT" + + - name: load cache + id: cache_deps + uses: actions/cache@v6 + with: + path: ${{ steps.macos_path.outputs.path || format('{0}/{1}', github.workspace, steps.set_outputs.outputs.cache-path) }} + key: ${{ steps.set_outputs.outputs.cache-key }} + lookup-only: true + + # A pull request that misses the cache can wait for a base branch build of the same deps. + - name: check for a base branch build to wait for + id: wait_check + if: ${{ startsWith(inputs.os, 'macos-') && github.event_name == 'pull_request' && steps.cache_deps.outputs.cache-hit != 'true' }} + env: + GH_TOKEN: ${{ github.token }} + run: bash scripts/ci_deps_cache.sh check + + wait_for_deps: + name: Wait for base branch deps + needs: [check_cache] + if: ${{ needs.check_cache.outputs.wait-for != '' }} + runs-on: ubuntu-24.04 + env: + WAIT_MINUTES: 120 + # Must exceed WAIT_MINUTES. + timeout-minutes: 130 + # Waiting runs queue here, and only the first one holds a runner. + concurrency: + group: deps-wait-${{ github.base_ref }}-${{ needs.check_cache.outputs.cache-key }} + queue: max + outputs: + hit: ${{ steps.lookup.outputs.cache-hit }} + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + sparse-checkout: scripts/ci_deps_cache.sh + sparse-checkout-cone-mode: false + + - name: wait for the deps cache + env: + GH_TOKEN: ${{ github.token }} + KEY: ${{ needs.check_cache.outputs.cache-key }} + KEY_INPUTS: ${{ needs.check_cache.outputs.key-inputs }} + WAIT_FOR: ${{ needs.check_cache.outputs.wait-for }} + WAIT_SINCE: ${{ needs.check_cache.outputs.wait-since }} + ARCH: ${{ inputs.arch }} + run: bash scripts/ci_deps_cache.sh wait + + - name: load cache + id: lookup + uses: actions/cache@v6 + with: + path: ${{ needs.check_cache.outputs.macos-path }} + key: ${{ needs.check_cache.outputs.cache-key }} + lookup-only: true + + # Hosted macOS runners are scarce, so a pull request's macOS build waits until + # one is free that push and nightly builds do not need. See the script. + macos_admission: + # The macos-priority label, when the run starts, puts a pull request in its own + # line, which the normal one yields to. The script matches both names. + name: ${{ contains(github.event.pull_request.labels.*.name, 'macos-priority') && 'Wait for a macOS runner (priority)' || 'Wait for a macOS runner' }} + needs: [check_cache, wait_for_deps] + if: ${{ !cancelled() && needs.check_cache.result == 'success' && startsWith(inputs.os, 'macos-') && github.event_name == 'pull_request' }} + runs-on: ubuntu-24.04 + env: + WAIT_MINUTES: 240 + PRIORITY: ${{ contains(github.event.pull_request.labels.*.name, 'macos-priority') }} + # Must exceed WAIT_MINUTES. + timeout-minutes: 250 + # One line for every arch of every pull request, and only its first job polls. + concurrency: + group: ${{ contains(github.event.pull_request.labels.*.name, 'macos-priority') && 'macos-admission-priority' || 'macos-admission' }} + queue: max + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + sparse-checkout: scripts/ci_macos_admission.py + sparse-checkout-cone-mode: false + + - name: wait for a macOS runner + env: + GH_TOKEN: ${{ github.token }} + MACOS_RUNNER_LIMIT: ${{ vars.MACOS_RUNNER_LIMIT }} # -u, or the log shows nothing until the wait ends. - run: python3 -u scripts/ci_macos_admission.py - - build_deps: # call next step - name: Build Deps - needs: [check_cache, wait_for_deps, macos_admission] - # A failed wait counts as no cache, so the deps are built here. A failed - # admission still builds rather than skip macOS. - if: ${{ !cancelled() && needs.check_cache.result == 'success' }} - uses: ./.github/workflows/build_deps.yml - with: - cache-key: ${{ needs.check_cache.outputs.cache-key }} - cache-path: ${{ needs.check_cache.outputs.cache-path }} - valid-cache: ${{ needs.check_cache.outputs.valid-cache == 'true' || needs.wait_for_deps.outputs.hit == 'true' }} - os: ${{ inputs.os }} - arch: ${{ inputs.arch }} - compiler: ${{ inputs.compiler }} - build-deps-only: ${{ inputs.build-deps-only }} - force-build: ${{ inputs.force-build }} - secrets: inherit + run: python3 -u scripts/ci_macos_admission.py + + build_deps: # call next step + name: Build Deps + needs: [check_cache, wait_for_deps, macos_admission] + # A failed wait counts as no cache, so the deps are built here. A failed + # admission still builds rather than skip macOS. + if: ${{ !cancelled() && needs.check_cache.result == 'success' }} + uses: ./.github/workflows/build_deps.yml + with: + cache-key: ${{ needs.check_cache.outputs.cache-key }} + cache-path: ${{ needs.check_cache.outputs.cache-path }} + valid-cache: ${{ needs.check_cache.outputs.valid-cache == 'true' || needs.wait_for_deps.outputs.hit == 'true' }} + os: ${{ inputs.os }} + arch: ${{ inputs.arch }} + compiler: ${{ inputs.compiler }} + build-deps-only: ${{ inputs.build-deps-only }} + force-build: ${{ inputs.force-build }} + secrets: inherit diff --git a/scripts/ci_macos_admission.py b/scripts/ci_macos_admission.py index d13395e281..9ec4f49bd4 100644 --- a/scripts/ci_macos_admission.py +++ b/scripts/ci_macos_admission.py @@ -19,11 +19,15 @@ by a run waiting for a second one for its other arch. - In the minutes around the nightly's cron time, RESERVE runners are held for it until its run appears. +A pull request labelled macos-priority when its run starts waits in a separate +line under the same rule, and the normal line counts every priority run still +waiting as holding RESERVE, so the next free runners go to it. + Any API error repeated FAILURES_BEFORE_ADMIT times, and the WAIT_MINUTES limit, let the arch in, so a fault here never blocks pull requests. -Environment: GH_TOKEN, REPO, WORKFLOW_REF, GITHUB_RUN_ID, and optionally MACOS_RUNNER_LIMIT, -WAIT_MINUTES, POLL_SECONDS and GITHUB_API_URL. +Environment: GH_TOKEN, REPO, WORKFLOW_REF, GITHUB_RUN_ID, and optionally PRIORITY, +MACOS_RUNNER_LIMIT, WAIT_MINUTES, POLL_SECONDS and GITHUB_API_URL. """ import datetime @@ -40,7 +44,8 @@ import urllib.request RESERVE = 2 # Must match the job names in build_all.yml and build_check_cache.yml. -GATE_SUFFIX = " / Wait for a macOS runner" +GATE = "Wait for a macOS runner" +PRIORITY_GATE = GATE + " (priority)" FINAL_JOBS = {"Build macOS Universal", "macOS arm64"} # Must match the cron in build_all.yml. NIGHTLY_UTC = datetime.time(2, 15) @@ -64,13 +69,22 @@ def macos_done(jobs): return bool(final) and all(job["status"] == "completed" for job in final) +def gates(jobs, names=(GATE, PRIORITY_GATE)): + return [job for job in jobs if job["name"].split(" / ")[-1] in names] + + def admitted(jobs): """True once one of the run's arches was let in.""" - return any(job["name"].endswith(GATE_SUFFIX) and job["conclusion"] == "success" for job in jobs) + return any(job["conclusion"] == "success" for job in gates(jobs)) -def run_demand(run, jobs): - """macOS runners a run holds or still needs.""" +def priority_waiting(jobs): + return not admitted(jobs) and any(job["status"] != "completed" for job in gates(jobs, (PRIORITY_GATE,))) + + +def run_demand(run, jobs, yield_to_priority=False): + """macOS runners a run holds or still needs. With yield_to_priority, a priority + run still waiting counts as holding RESERVE.""" # A run with no jobs that is pending waits behind another run of its # concurrency group, which holds the runners for both. if not jobs and run["status"] in ("pending", "waiting"): @@ -79,7 +93,7 @@ def run_demand(run, jobs): if macos_done(jobs): return active if run["event"] == "pull_request" and not admitted(jobs): - return active + return max(active, RESERVE) if yield_to_priority and priority_waiting(jobs) else active return max(active, RESERVE) @@ -127,9 +141,10 @@ def latest_run(api, repo, workflow, **params): return runs[0] if runs else None -def measure(api, repo, workflow, run_id, now): +def measure(api, repo, workflow, run_id, now, priority=False): """Total macOS runners held or needed, one line per run that holds any, and - whether run_id was already let in.""" + whether run_id was already let in. The priority line does not count the + priority runs waiting behind it.""" total, lines, here = 0, [], False # A run is listed as queued whenever one of its jobs waits for a runner, and # as pending whenever one waits in a concurrency group, so runs in any of these @@ -139,11 +154,12 @@ def measure(api, repo, workflow, run_id, now): for run in runs.values(): jobs = api.get(f"repos/{repo}/actions/runs/{run['id']}/jobs", filter="latest", per_page=100)["jobs"] - demand = run_demand(run, jobs) + demand = run_demand(run, jobs, yield_to_priority=not priority and run["id"] != run_id) here = here or (run["id"] == run_id and admitted(jobs)) if demand: total += demand - lines.append(f" {demand} run {run['id']} ({run['event']}, {run['head_branch']})") + waiting = ", priority, waiting" if priority_waiting(jobs) else "" + lines.append(f" {demand} run {run['id']} ({run['event']}, {run['head_branch']}{waiting})") # build_all.yml runs the nightly only in the main repository. start, end = nightly_window(now) @@ -189,7 +205,8 @@ def main(): run_id = int(os.environ["GITHUB_RUN_ID"]) limit = int(os.environ.get("MACOS_RUNNER_LIMIT") or 5) reason = wait( - lambda: measure(api, repo, workflow, run_id, datetime.datetime.now(datetime.timezone.utc)), + lambda: measure(api, repo, workflow, run_id, datetime.datetime.now(datetime.timezone.utc), + priority=os.environ.get("PRIORITY") == "true"), limit, wait_minutes=int(os.environ.get("WAIT_MINUTES") or 240), poll_seconds=int(os.environ.get("POLL_SECONDS") or 180), diff --git a/scripts/tests/test_ci_macos_admission.py b/scripts/tests/test_ci_macos_admission.py index a427536fd7..424d7f8199 100644 --- a/scripts/tests/test_ci_macos_admission.py +++ b/scripts/tests/test_ci_macos_admission.py @@ -22,8 +22,9 @@ def job(name, status="completed", conclusion="success", labels=("macos-15",)): return {"name": name, "status": status, "conclusion": conclusion, "labels": list(labels)} -def gate(arch, status="completed", conclusion="success"): - return job(f"build_macos_arch ({arch}) / Wait for a macOS runner", status, conclusion, +def gate(arch, status="completed", conclusion="success", priority=False): + suffix = " (priority)" if priority else "" + return job(f"build_macos_arch ({arch}) / Wait for a macOS runner{suffix}", status, conclusion, labels=("ubuntu-24.04",)) @@ -102,6 +103,43 @@ class RunDemandTest(unittest.TestCase): self.assertEqual(admission.run_demand(PR, jobs), 0) +class PriorityTest(unittest.TestCase): + WAITING = [gate("arm64", "in_progress", None, priority=True), gate("x86_64", "pending", None, priority=True)] + + def test_normal_line_yields_to_a_waiting_priority_run(self): + self.assertEqual(admission.run_demand(PR, self.WAITING, yield_to_priority=True), admission.RESERVE) + + def test_priority_line_does_not_count_priority_runs_behind_it(self): + self.assertEqual(admission.run_demand(PR, self.WAITING), 0) + + def test_a_waiting_normal_run_is_not_yielded_to(self): + waiting = [gate("arm64", "in_progress", None), gate("x86_64", "pending", None)] + self.assertEqual(admission.run_demand(PR, waiting, yield_to_priority=True), 0) + + def test_a_priority_run_let_in_holds_its_runners(self): + jobs = [gate("arm64", priority=True), gate("x86_64", "pending", None, priority=True)] + self.assertEqual(admission.run_demand(PR, jobs), admission.RESERVE) + self.assertEqual(admission.run_demand(PR, jobs, yield_to_priority=True), admission.RESERVE) + + def test_skipped_priority_jobs_of_other_platforms_are_not_waiting(self): + jobs = [job("build_linux (ubuntu-24.04) / Wait for a macOS runner (priority)", + conclusion="skipped", labels=("ubuntu-24.04",))] + self.assertEqual(admission.run_demand(PR, jobs, yield_to_priority=True), 0) + + def test_measure_from_each_line(self): + api = FakeApi({("runs", "pending"): [dict(PR, status="pending")], jobs_path(2): self.WAITING}) + total, lines, _ = admission.measure(api, "o/r", "build_all.yml", 99, NOON) + self.assertEqual(total, admission.RESERVE) + self.assertIn("priority, waiting", lines[0]) + total, _, _ = admission.measure(api, "o/r", "build_all.yml", 99, NOON, priority=True) + self.assertEqual(total, 0) + + def test_own_waiting_run_is_not_counted(self): + api = FakeApi({("runs", "pending"): [dict(PR, status="pending")], jobs_path(2): self.WAITING}) + total, _, here = admission.measure(api, "o/r", "build_all.yml", 2, NOON) + self.assertEqual((total, here), (0, False)) + + class FakeApi: """Answers GET requests from a dict of path -> list of pages (or one body)."""