test: benchmark gha job summaries (#484)

* build: docker client timeout doesn't do anything anymore

this has been deprecated for years

* Revert "test: correct benchmark workflow permissions (#480)"

This reverts commit d3b14c29ae.

* docs: comment on why py + pipx are needed for java

* test: increase ci timeouts

* test: attempt to add benchmark summary to gha job

* docs: note about rerunning a release

.github/workflows/benchmark.yml

@@ -1,61 +0,0 @@
-name: Docker Image Benchmark
-
-# Runs in the base repo context (full write permissions) after "Run Tests"
-# completes. This is necessary because fork PRs restrict GITHUB_TOKEN to
-# read-only, which prevents posting benchmark comparison comments.
-on:
-  workflow_run:
-    workflows: ["Run Tests"]
-    types: [completed]
-
-jobs:
-  docker-image-benchmark:
-    # skip if the triggering workflow did not succeed
-    if: github.event.workflow_run.conclusion == 'success'
-    runs-on: ubuntu-latest
-    permissions:
-      # required for benchmark-action to push to gh-pages on main
-      contents: write
-      # required to post comparison comments on PRs
-      pull-requests: write
-
-    steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
-
-      # actions/download-artifact cannot cross workflow boundaries.
-      # dawidd6/action-download-artifact works around this limitation and fetches artifacts from a specific run ID.
-      - name: Download all size artifacts
-        uses: dawidd6/action-download-artifact@2536c51d3d126276eb39f74d6bc9c72ac6ef30d3
-        with:
-          run_id: ${{ github.event.workflow_run.id }}
-          name: size-*
-          name_is_regexp: true
-
-      - name: Merge size.json files into benchmark format
-        # customSmallerIsBetter expects [{name, unit, value}]; also emit a simple size.json for manual inspection
-        run: |
-          # create a master size.json for manual debugging
-          jq -s '[.[] | .] | sort_by(.name)' size-*/size.json > size.json
-          # benchmark payload compared against the history stored in gh-pages
-          jq -s '[.[] | {name: .name, unit: "bytes", value: .size}]' size-*/size.json > benchmark.json
-
-      - name: Upload merged size.json
-        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f
-        with:
-          name: sizes
-          path: size.json
-
-      - name: Store benchmark results
-        uses: benchmark-action/github-action-benchmark@a7bc2366eda11037936ea57d811a43b3418d3073
-        with:
-          tool: customSmallerIsBetter
-          output-file-path: benchmark.json
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          # only persist history when the triggering push landed on main
-          auto-push: ${{ github.event.workflow_run.head_branch == 'main' && github.event.workflow_run.event == 'push' }}
-          # post size comparison comment on every PR
-          comment-always: true
-          benchmark-data-dir-path: dev/bench
-          # alert when an image grows more than 20%
-          alert-threshold: '120%'
-          comment-on-alert: true

.github/workflows/run_tests.yml

@@ -159,3 +159,54 @@ jobs:
       - name: Stop BuildKit
         if: always()
         run: docker stop buildkit
+
+  # Collects size.json artifacts from all matrix jobs, merges them into a single
+  # benchmark payload. On main, pushes results to gh-pages to track history.
+  # On PRs, compares against stored history and posts a comment with the delta.
+  docker-image-benchmark:
+    needs: test
+    runs-on: ubuntu-latest
+    permissions:
+      # required for benchmark-action to push to gh-pages on main
+      contents: write
+      # required to post comparison comments on PRs
+      pull-requests: write
+
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+
+      - name: Download all size artifacts
+        uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3
+        # each artifact lands in its own size-<example>/ directory
+        with:
+          pattern: size-*
+          merge-multiple: false
+
+      - name: Merge size.json files into benchmark format
+        # customSmallerIsBetter expects [{name, unit, value}]; also emit a simple size.json for manual inspection
+        run: |
+          # create a master size.json for manual debugging
+          jq -s '[.[] | .] | sort_by(.name)' size-*/size.json > size.json
+          # this debugging data is compared against the "master" copy in gh-pages
+          jq -s '[.[] | {name: .name, unit: "bytes", value: .size}]' size-*/size.json > benchmark.json
+
+      - name: Upload merged size.json
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f
+        with:
+          name: sizes
+          path: size.json
+
+      - name: Store benchmark results
+        uses: benchmark-action/github-action-benchmark@a7bc2366eda11037936ea57d811a43b3418d3073
+        with:
+          tool: customSmallerIsBetter
+          output-file-path: benchmark.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          # only persist history on main
+          auto-push: ${{ github.ref == 'refs/heads/main' }}
+          benchmark-data-dir-path: dev/bench
+          # alert when an image grows more than 20%
+          alert-threshold: '120%'
+          # Enable Job Summary for PRs
+          summary-always: true
+          # no comment-on-alert, comment-always; just on PR

RELEASE.md

@@ -26,6 +26,14 @@ This document outlines the process for creating new releases of Railpack.
    - Create a GitHub release with changelog
    - Build and attach binaries for multiple platforms
 
+### Rerunning a Failed Release
+
+This should *rarely* occur. If you need to rerun a release workflow, on a newer commit:
+
+```
+git tag -d v0.18.0 && git push --delete origin v0.18.0 && git tag v0.18.0 origin/main && git push origin v0.18.0
+```
+
 ## Release Artifacts
 
 ### Frontend Docker Image

core/providers/cpp/meson.go

@@ -15,10 +15,11 @@ func (_ *CppProvider) DetectMeson(ctx *generate.GenerateContext) (buildSystem, b
 }
 
 func (_ *meson) Install(ctx *generate.GenerateContext, mise *generate.MiseStepBuilder) {
-	mise.Default("python", "latest")
-	mise.Default("pipx", "latest")
 	mise.Default("meson", "latest")
 	mise.Default("ninja", "latest")
+	// python + pipx are needed because of an installation issue with meson; this could be fixed in the future
+	mise.Default("python", "latest")
+	mise.Default("pipx", "latest")
 	mise.UseMiseVersions(ctx, []string{"meson", "ninja"})
 }
 

mise.ci.toml

@@ -1,9 +1,6 @@
 # define all required variables for CI, so we can limit the configuration and logic within GHA to test more of the CI process locally
 
 [env]
-# in rare cases, docker hub can timeout, and we want our CI runs to be as resilient as possible
-DOCKER_CLIENT_TIMEOUT = 300
-
 PRODUCTION_CONTAINER_REGISTRY = "ghcr.io"
 PRODUCTION_GITHUB_REPOSITORY = "railwayapp/railpack"
 PRODUCTION_MISE_TAG = "mise-${RAILPACK_MISE_VERSION}"
@@ -17,5 +14,8 @@ PRODUCTION_RUNTIME_IMAGE_REF = "${PRODUCTION_RUNTIME_IMAGE_NAME}:${PRODUCTION_MI
 PRODUCTION_FRONTEND_IMAGE_NAME = "${PRODUCTION_CONTAINER_REGISTRY}/${PRODUCTION_GITHUB_REPOSITORY}-frontend"
 
 [settings]
+# intermittent failures on CI are a big time suck, let's be generous with our installation retries
+http_retries = 5
+
 # require URLs to be present in the mise.lockfile to ensure there is consistent versions used across CI runs
 locked = true