mirror of
https://github.com/OrcaSlicer/OrcaSlicer.git
synced 2026-09-23 08:53:05 +00:00
HanifKoh
ef956b995a
The nightly found its build with a filtered run listing (branch=main, status=success) and trusted the first result. GitHub serves filtered listings from a run search index that has intermittently returned weeks-old results, so some nights tested a build from weeks earlier and reported its differences as regressions. The same filter also matched fork PR builds whose branch is named main. The build is now picked from the unfiltered listing, which stays current, and filtered here: a successful build_all run of this repository on the requested branch. Fork PR builds are excluded by repository. A feature branch is normally built only for its PR, so this repository's own PR builds stay eligible, but a PR build compiles the PR merged into its base rather than the head commit the later jobs check out, so a push or dispatch build of the branch is preferred when the same page of the listing has one. A scheduled run fails instead of testing a build more than 48 hours old, and every run names the build it tested in the job summary. Manual runs scan further back, so a branch that last built weeks ago can still be tested, and a new build_run_id input pins one build_all run, read directly rather than through a search.
264 lines
11 KiB
YAML
264 lines
11 KiB
YAML
# Nightly parity checks from OrcaSlicer/orca-test-repo, kept out of the
|
|
# per-build "Run external slicer regression tests" step because they take far
|
|
# longer than that step's budget:
|
|
# effect - the CLI override sweep's full effect stage: every landed option
|
|
# re-sliced on its own to see whether it changes the G-code
|
|
# harness - the GUI-vs-CLI parity harness (metrics only, never fails)
|
|
# Both test the latest successful build_all.yml Linux AppImage from main, with
|
|
# sources checked out at the commit that build was made from; a manual run can
|
|
# name another branch, or pin one build by its run id. Nothing here gates a
|
|
# build or a PR.
|
|
name: Parity Nightly
|
|
|
|
on:
|
|
schedule:
|
|
# build_all.yml starts at 17:00 UTC and has finished by ~20:00
|
|
- cron: "0 21 * * *"
|
|
workflow_dispatch:
|
|
inputs:
|
|
test_repo_ref:
|
|
description: "orca-test-repo ref to run"
|
|
required: false
|
|
default: "main"
|
|
build_branch:
|
|
description: "branch whose newest successful build_all artifact to test (a PR build is the PR merged into its base; sources are checked out at the PR head)"
|
|
required: false
|
|
default: "main"
|
|
build_run_id:
|
|
description: "build_all run id to test instead of build_branch's newest (same PR caveat)"
|
|
required: false
|
|
default: ""
|
|
fixtures:
|
|
description: "harness fixture ids, space-separated (empty = all)"
|
|
required: false
|
|
default: ""
|
|
cli_presets:
|
|
description: "harness lane C presets: flat = flatten inherits first, raw = leaf profile as-is"
|
|
required: false
|
|
default: "flat"
|
|
|
|
permissions:
|
|
contents: read
|
|
actions: read
|
|
|
|
jobs:
|
|
build:
|
|
name: Find the build to test
|
|
# Don't run scheduled checks on forks
|
|
if: github.event_name != 'schedule' || github.repository == 'OrcaSlicer/OrcaSlicer'
|
|
runs-on: ubuntu-24.04
|
|
outputs:
|
|
run_id: ${{ steps.find.outputs.run_id }}
|
|
head_sha: ${{ steps.find.outputs.head_sha }}
|
|
steps:
|
|
- id: find
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
GH_REPO: ${{ github.repository }}
|
|
BRANCH: ${{ inputs.build_branch || 'main' }}
|
|
RUN_ID: ${{ inputs.build_run_id }}
|
|
SCHEDULED: ${{ github.event_name == 'schedule' }}
|
|
run: |
|
|
set -euo pipefail
|
|
if [ -n "$RUN_ID" ]; then
|
|
[[ $RUN_ID =~ ^[0-9]+$ ]] || { echo "build_run_id must be a numeric run id, got '$RUN_ID'" >&2; exit 1; }
|
|
# a pinned build is read directly, not through a search; it must come
|
|
# from this repository, because the later jobs check out its commit here
|
|
found=$(gh api "repos/$GH_REPO/actions/runs/$RUN_ID" --jq \
|
|
'select(.path == ".github/workflows/build_all.yml" and .conclusion == "success"
|
|
and .head_repository.full_name == env.GH_REPO)
|
|
| "\(.id) \(.head_sha) \(.created_at)"')
|
|
[ -n "$found" ] || { echo "run $RUN_ID is not a successful build_all run of $GH_REPO" >&2; exit 1; }
|
|
else
|
|
# GitHub serves filtered run listings (branch=, status=, head_sha=, ...)
|
|
# from a search index that has returned weeks-old results, while the
|
|
# unfiltered listing stays current, so list unfiltered and filter here.
|
|
# The repository check keeps out fork PRs whose branch has the same
|
|
# name. A feature branch is normally built only for its PR, and a PR
|
|
# build compiles the PR merged into its base rather than head_sha, so
|
|
# a build of the branch itself (push or dispatch) is preferred when
|
|
# the same page has one.
|
|
pick='([.workflow_runs[] | select(.head_branch == env.BRANCH and .conclusion == "success"
|
|
and .head_repository.full_name == env.GH_REPO)]
|
|
| map(select(.event != "pull_request"))[0] // .[0])
|
|
| select(.) | "\(.id) \(.head_sha) \(.created_at)"'
|
|
# a page of 100 runs spans about a day and a half; a manual run may
|
|
# target a branch that last built weeks ago
|
|
pages=3
|
|
if [ "$SCHEDULED" != true ]; then pages=20; fi
|
|
found=""
|
|
for page in $(seq "$pages"); do
|
|
found=$(gh api "repos/$GH_REPO/actions/workflows/build_all.yml/runs?per_page=100&page=$page" --jq "$pick")
|
|
if [ -n "$found" ]; then break; fi
|
|
done
|
|
[ -n "$found" ] || { echo "no successful $BRANCH build among the last $((pages * 100)) build_all runs; pass build_run_id to test an older one" >&2; exit 1; }
|
|
fi
|
|
read -r run_id head_sha created <<< "$found"
|
|
# the nightly fails rather than report on a stale build
|
|
if [ "$SCHEDULED" = true ] && [ $(( $(date +%s) - $(date -d "$created" +%s) )) -gt 172800 ]; then
|
|
echo "newest $BRANCH build $run_id is from $created, over 48 hours old" >&2
|
|
exit 1
|
|
fi
|
|
printf 'run_id=%s\nhead_sha=%s\n' "$run_id" "$head_sha" >> "$GITHUB_OUTPUT"
|
|
cat "$GITHUB_OUTPUT"
|
|
echo "Testing build [$run_id](https://github.com/$GH_REPO/actions/runs/$run_id) of \`$head_sha\`, built $created" >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
effect:
|
|
name: Override sweep effect stage (shard ${{ matrix.shard }})
|
|
needs: build
|
|
runs-on: ubuntu-24.04
|
|
timeout-minutes: 60
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
# orca-test-repo's parity/effect_routing.json holds a 2-way split,
|
|
# ~12.5 min a shard on this runner
|
|
shard: [0, 1]
|
|
steps:
|
|
- &checkout-suite
|
|
name: Check out the test suite
|
|
uses: actions/checkout@v7
|
|
with:
|
|
repository: OrcaSlicer/orca-test-repo
|
|
ref: ${{ inputs.test_repo_ref || 'main' }}
|
|
path: orca-test-repo
|
|
|
|
# The AppImage ships only packed preset caches, so profiles and the CLI
|
|
# option surface come from the sources the build was made from
|
|
- &checkout-slicer
|
|
name: Check out OrcaSlicer at the build's commit
|
|
uses: actions/checkout@v7
|
|
with:
|
|
ref: ${{ needs.build.outputs.head_sha }}
|
|
path: slicer
|
|
lfs: 'false'
|
|
|
|
- &extract-appimage
|
|
name: Download and extract the Linux AppImage
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
GH_REPO: ${{ github.repository }}
|
|
run: |
|
|
set -euo pipefail
|
|
gh run download "${{ needs.build.outputs.run_id }}" --dir appimage \
|
|
--pattern "OrcaSlicer_Linux_ubuntu_2404*"
|
|
appimage=$(find appimage -name "*.AppImage" ! -name "*aarch64*" | head -1)
|
|
[ -n "$appimage" ] || { echo "no x86_64 AppImage in run ${{ needs.build.outputs.run_id }}"; exit 1; }
|
|
chmod +x "$appimage"
|
|
"$appimage" --appimage-extract > /dev/null
|
|
# The bare binary cannot find the AppImage's bundled libraries; AppRun
|
|
# sets them up and execs it, so exit codes and signals pass through
|
|
[ -x squashfs-root/AppRun ] || { echo "no AppRun in the AppImage"; exit 1; }
|
|
echo "ORCA_BIN=$PWD/squashfs-root/AppRun" >> "$GITHUB_ENV"
|
|
echo "ORCA_SOURCE=$PWD/slicer" >> "$GITHUB_ENV"
|
|
|
|
- name: Install the AppImage's host runtime dependencies
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y --no-install-recommends \
|
|
libopengl0 libglu1-mesa libgl1 libegl1 libwebkit2gtk-4.1-0
|
|
|
|
- uses: actions/setup-python@v6
|
|
with:
|
|
python-version: "3.12"
|
|
|
|
- name: Install suite dependencies
|
|
run: pip install -r orca-test-repo/requirements.txt
|
|
|
|
- name: Run the override sweep with the full effect stage
|
|
id: run
|
|
continue-on-error: true
|
|
working-directory: orca-test-repo
|
|
run: |
|
|
set -o pipefail
|
|
# -rA keeps the per-stage summaries, which pytest otherwise swallows
|
|
# for passing tests
|
|
python -m pytest test_cli_overrides.py -c pytest.ini -v -rA \
|
|
--effect-full --effect-shard ${{ matrix.shard }}/2 \
|
|
--orca-bin "$ORCA_BIN" --orca-source "$ORCA_SOURCE" \
|
|
2>&1 | tee ../sweep.log
|
|
|
|
- name: Publish job summary
|
|
if: always()
|
|
run: |
|
|
{
|
|
echo "## Override sweep effect stage, shard ${{ matrix.shard }}/2"
|
|
echo "Build ${{ needs.build.outputs.head_sha }} (run ${{ needs.build.outputs.run_id }})"
|
|
echo '```'
|
|
grep -E "\[override sweep" sweep.log || echo "no stage summaries, see the log"
|
|
grep -E "^=+ .*(passed|failed)" sweep.log | tail -1 || true
|
|
echo '```'
|
|
} >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
- name: Upload the override report
|
|
if: always()
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: override-report-shard${{ matrix.shard }}
|
|
path: |
|
|
orca-test-repo/.pytest_cache/override_report.json
|
|
sweep.log
|
|
if-no-files-found: warn
|
|
retention-days: 30
|
|
|
|
# The sweep step continues on error so the summary and report still get
|
|
# published; this puts the failure back on the job
|
|
- name: Fail the job if the sweep failed
|
|
if: steps.run.outcome == 'failure'
|
|
run: |
|
|
echo "the override sweep failed, see the job summary and the uploaded report" >&2
|
|
exit 1
|
|
|
|
harness:
|
|
name: GUI-vs-CLI parity harness
|
|
needs: build
|
|
runs-on: ubuntu-24.04
|
|
timeout-minutes: 180
|
|
steps:
|
|
- *checkout-suite
|
|
- *checkout-slicer
|
|
- *extract-appimage
|
|
|
|
- name: Install display tooling and the AppImage's host runtime
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y --no-install-recommends \
|
|
xvfb xdotool imagemagick openbox mesa-utils \
|
|
libopengl0 libglu1-mesa libgl1 libegl1 libwebkit2gtk-4.1-0
|
|
|
|
- name: Run the parity harness
|
|
run: |
|
|
set -euo pipefail
|
|
fixtures=()
|
|
for f in ${{ inputs.fixtures || '' }}; do
|
|
fixtures+=(--fixture "$f")
|
|
done
|
|
# 2 GUI displays: ~1.5 cores peak / ~1.9 GB on this 4-vCPU runner,
|
|
# and each fixture is fully isolated, so results match a serial run
|
|
python3 orca-test-repo/parity/run_parity.py \
|
|
--slicer-root "$ORCA_SOURCE" --bin "$ORCA_BIN" \
|
|
--cli-presets "${{ inputs.cli_presets || 'flat' }}" \
|
|
--gui-workers 2 --out "$PWD/parity-out" "${fixtures[@]}"
|
|
|
|
- name: Publish job summary
|
|
if: always()
|
|
run: |
|
|
if [ -f parity-out/report.md ]; then
|
|
cat parity-out/report.md >> "$GITHUB_STEP_SUMMARY"
|
|
else
|
|
echo "the harness produced no report, see the log" >> "$GITHUB_STEP_SUMMARY"
|
|
fi
|
|
|
|
- name: Drop per-lane datadirs before upload
|
|
if: always()
|
|
run: rm -rf parity-out/*/seed parity-out/*/datadir-* || true
|
|
|
|
- name: Upload the scorecard and evidence
|
|
if: always()
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: parity-scorecard
|
|
path: parity-out/
|
|
if-no-files-found: warn
|
|
retention-days: 30
|