From e6501bb1cecea4459df5481bfe76216cffaae2fa Mon Sep 17 00:00:00 2001 From: Kris Austin Date: Wed, 2 Sep 2026 19:45:16 -0500 Subject: [PATCH] build: stop rebuilding the Flatpak dependencies on every run (#15501) --- .github/workflows/build_all.yml | 14 ++++- build_flatpak.sh | 3 + scripts/build_flatpak_with_docker.sh | 3 + scripts/flatpak/.gitignore | 1 + scripts/flatpak/check_manifest_cacheable.sh | 52 ++++++++++++++++++ scripts/flatpak/com.orcaslicer.OrcaSlicer.yml | 9 +-- scripts/flatpak/make_deps_tar.sh | 55 +++++++++++++++++++ scripts/flatpak/setup_env_ubuntu24.04.sh | 7 ++- 8 files changed, 135 insertions(+), 9 deletions(-) create mode 100755 scripts/flatpak/check_manifest_cacheable.sh create mode 100755 scripts/flatpak/make_deps_tar.sh diff --git a/.github/workflows/build_all.yml b/.github/workflows/build_all.yml index d991bedca6..0ab9cfe41d 100644 --- a/.github/workflows/build_all.yml +++ b/.github/workflows/build_all.yml @@ -279,20 +279,24 @@ jobs: echo "date=$(date +'%Y%m%d')" >> $GITHUB_ENV echo "git_commit_hash=$git_commit_hash" >> $GITHUB_ENV shell: bash + - name: Compute the flatpak-builder cache key + id: fp_cache_key + run: echo "key=flatpak-builder-${{ matrix.variant.arch }}-${{ hashFiles('deps/**', 'scripts/flatpak/com.orcaslicer.OrcaSlicer.yml', 'scripts/flatpak/make_deps_tar.sh') }}" >> "$GITHUB_OUTPUT" + shell: bash # Manage flatpak-builder cache externally so PRs restore but never upload - name: Restore flatpak-builder cache if: github.event_name == 'pull_request' uses: actions/cache/restore@v6 with: path: .flatpak-builder - key: flatpak-builder-${{ matrix.variant.arch }}-${{ github.event.pull_request.base.sha }} + key: ${{ steps.fp_cache_key.outputs.key }} restore-keys: flatpak-builder-${{ matrix.variant.arch }}- - name: Save/restore flatpak-builder cache if: github.event_name != 'pull_request' uses: actions/cache@v6 with: path: .flatpak-builder - key: flatpak-builder-${{ matrix.variant.arch }}-${{ github.sha }} + key: ${{ steps.fp_cache_key.outputs.key }} restore-keys: flatpak-builder-${{ matrix.variant.arch }}- - name: Disable debug info for faster CI builds run: | @@ -304,6 +308,12 @@ jobs: sed -i "/name: OrcaSlicer/{n;s|buildsystem: simple|buildsystem: simple\n build-options:\n env:\n git_commit_hash: \"$git_commit_hash\"|}" \ scripts/flatpak/com.orcaslicer.OrcaSlicer.yml shell: bash + - name: Check the manifest keeps orca_deps cacheable + run: ./scripts/flatpak/check_manifest_cacheable.sh + shell: bash + - name: Pack deps/ for the Flatpak manifest + run: ./scripts/flatpak/make_deps_tar.sh + shell: bash - uses: flatpak/flatpak-github-actions/flatpak-builder@master with: bundle: OrcaSlicer-Linux-flatpak_${{ env.ver }}_${{ matrix.variant.arch }}.flatpak diff --git a/build_flatpak.sh b/build_flatpak.sh index b805bb3422..12ccac372b 100755 --- a/build_flatpak.sh +++ b/build_flatpak.sh @@ -260,6 +260,9 @@ if [[ ! -f "./scripts/flatpak/com.orcaslicer.OrcaSlicer.yml" ]]; then exit 1 fi +echo -e "${YELLOW}Packing deps/ for the manifest...${NC}" +./scripts/flatpak/make_deps_tar.sh + # Build the Flatpak echo -e "${YELLOW}Building Flatpak package...${NC}" echo -e "This may take a while (30+ minutes depending on your system)..." diff --git a/scripts/build_flatpak_with_docker.sh b/scripts/build_flatpak_with_docker.sh index 9e8805af2f..eedfe53692 100755 --- a/scripts/build_flatpak_with_docker.sh +++ b/scripts/build_flatpak_with_docker.sh @@ -122,6 +122,9 @@ sed "/name: OrcaSlicer/{ \1 git_commit_hash: \"$GIT_COMMIT_HASH\"| }" > "$MANIFEST_DOCKER" +# ---------- pack deps/ ---------- +./scripts/flatpak/make_deps_tar.sh + # ---------- run build in Docker ---------- DOCKER="${DOCKER:-docker}" diff --git a/scripts/flatpak/.gitignore b/scripts/flatpak/.gitignore index 3b01f15d01..21abf7a876 100644 --- a/scripts/flatpak/.gitignore +++ b/scripts/flatpak/.gitignore @@ -1,3 +1,4 @@ builddir .flatpak-builder *.docker.yml +deps.tar* diff --git a/scripts/flatpak/check_manifest_cacheable.sh b/scripts/flatpak/check_manifest_cacheable.sh new file mode 100755 index 0000000000..e82190ca81 --- /dev/null +++ b/scripts/flatpak/check_manifest_cacheable.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# Check a Flatpak manifest for `type: dir` sources at or before the orca_deps +# module. +# +# Usage: check_manifest_cacheable.sh [manifest] +# Defaults to com.orcaslicer.OrcaSlicer.yml next to this script. +# +# Exits 0 when none are found, 1 when any are, listing them as file:line, and +# 2 when the manifest is missing or has no orca_deps module. + +set -euo pipefail + +anchor=orca_deps +manifest=${1:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/com.orcaslicer.OrcaSlicer.yml} + +module_re='^ - name: ' +dir_re='(^|[-{,[:space:]])type:[[:space:]]*dir([,}[:space:]]|$)' + +if [ ! -f "$manifest" ]; then + echo "$manifest: no such file" >&2 + exit 2 +fi + +anchor_start=$(grep -n "${module_re}${anchor}[[:space:]]*$" "$manifest" | cut -d: -f1 || true) +if [ -z "$anchor_start" ]; then + echo "$manifest: no module named '$anchor'; this check needs updating" >&2 + exit 2 +fi + +# First module header after the anchor, or EOF if the anchor is last. +anchor_end=$(grep -n "$module_re" "$manifest" | cut -d: -f1 | + awk -v s="$anchor_start" '$1 > s { print $1; exit }') +[ -n "$anchor_end" ] || anchor_end=$(awk 'END { print NR + 1 }' "$manifest") + +violations=$(awk -v e="$anchor_end" -v f="$manifest" -v mre="$module_re" -v dre="$dir_re" ' + { sub(/\r$/, "") } + /^[[:space:]]*#/ { next } + $0 ~ mre { module = $3 } + NR < e && $0 ~ dre { + printf "%s:%d: %s (module %s)\n", f, NR, $0, module + }' "$manifest") + +if [ -n "$violations" ]; then + printf '%s\n' "$violations" >&2 + echo >&2 + echo "flatpak-builder cannot checksum a directory, so each of these makes" >&2 + echo "$anchor and every module after it rebuild from scratch on every run." >&2 + echo "See the $anchor sources in this manifest for the tarball used instead." >&2 + exit 1 +fi + +echo "manifest OK: no 'type: dir' source at or before $anchor" diff --git a/scripts/flatpak/com.orcaslicer.OrcaSlicer.yml b/scripts/flatpak/com.orcaslicer.OrcaSlicer.yml index a3efaced8c..00aa430f84 100644 --- a/scripts/flatpak/com.orcaslicer.OrcaSlicer.yml +++ b/scripts/flatpak/com.orcaslicer.OrcaSlicer.yml @@ -144,6 +144,7 @@ modules: env: BUILD_DIR: deps/build_flatpak build-commands: + - tar xf deps.tar - | cmake -S deps -B $BUILD_DIR \ -DFLATPAK=ON \ @@ -169,10 +170,10 @@ modules: - /libpython/include sources: - # OrcaSlicer deps/ directory (avoids copying .git from worktree) - - type: dir - path: ../../deps - dest: deps + # flatpak-builder cannot checksum a directory, so a `type: dir` here would + # rebuild every dependency on every run. Generated by make_deps_tar.sh. + - type: file + path: deps.tar # --------------------------------------------------------------- # Pre-downloaded dependency archives diff --git a/scripts/flatpak/make_deps_tar.sh b/scripts/flatpak/make_deps_tar.sh new file mode 100755 index 0000000000..e561da2e7b --- /dev/null +++ b/scripts/flatpak/make_deps_tar.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# Pack deps/ into scripts/flatpak/deps.tar for the Flatpak manifest's orca_deps +# module. +# +# Usage: make_deps_tar.sh +# Requires GNU tar. On macOS, brew install gnu-tar. +# +# The archive is byte-reproducible. Member order, mtimes, ownership and the +# group/other write bits are pinned, so identical deps/ contents always produce +# an identical file. deps/build* and deps/DL_CACHE are excluded. +# +# Prints the output path, size and sha256. + +set -euo pipefail + +if [ "$#" -ne 0 ]; then + echo "usage: ${0##*/}" >&2 + exit 2 +fi + +tar_bin=$(command -v gtar || command -v tar || true) +tar_version=$([ -n "$tar_bin" ] && "$tar_bin" --version 2>/dev/null || true) +case $tar_version in + *"GNU tar"*) ;; + *) echo "${0##*/}: needs GNU tar; on macOS run 'brew install gnu-tar'" >&2 + exit 2 ;; +esac + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +repo_root=$(cd "$script_dir/../.." && pwd) +out=$script_dir/deps.tar +tmp=$out.tmp +trap 'rm -f "$tmp"' EXIT + +"$tar_bin" --format=gnu \ + --sort=name \ + --mtime=@0 \ + --owner=0 --group=0 --numeric-owner \ + --mode=go-w \ + --exclude='deps/build*' \ + --exclude='deps/DL_CACHE' \ + -cf "$tmp" \ + -C "$repo_root" deps + +mv -f "$tmp" "$out" + +if command -v sha256sum >/dev/null 2>&1; then + sum=$(sha256sum "$out" | cut -d' ' -f1) +elif command -v shasum >/dev/null 2>&1; then + sum=$(shasum -a 256 "$out" | cut -d' ' -f1) +else + sum=unavailable +fi + +echo "Wrote $out ($(du -h "$out" | cut -f1), sha256 $sum)" diff --git a/scripts/flatpak/setup_env_ubuntu24.04.sh b/scripts/flatpak/setup_env_ubuntu24.04.sh index a0ccbb9b32..74b7629f60 100755 --- a/scripts/flatpak/setup_env_ubuntu24.04.sh +++ b/scripts/flatpak/setup_env_ubuntu24.04.sh @@ -8,8 +8,9 @@ flatpak install flathub org.gnome.Platform//50 org.gnome.Sdk//50 org.freedesktop ## # in OrcaSlicer folder, run following command to build Orca -# # First time build -# flatpak-builder --state-dir=.flatpak-builder --keep-build-dirs --user --force-clean build-dir scripts/flatpak/com.orcaslicer.OrcaSlicer.yml -# # Subsequent builds (only rebuilding OrcaSlicer) +# # First time build +# ./scripts/flatpak/make_deps_tar.sh && flatpak-builder --state-dir=.flatpak-builder --keep-build-dirs --user --force-clean build-dir scripts/flatpak/com.orcaslicer.OrcaSlicer.yml + +# # Subsequent builds (only rebuilding OrcaSlicer; run make_deps_tar.sh first if deps/ changed) # flatpak-builder --state-dir=.flatpak-builder --keep-build-dirs --user build-dir scripts/flatpak/com.orcaslicer.OrcaSlicer.yml --build-only=OrcaSlicer \ No newline at end of file