hanzoai/pack
2
0
Fork 0
mirror of https://github.com/hanzoai/pack.git synced 2026-09-30 23:43:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: respect cache backend type for registry cache import/export, add cache support to build subcommand (#595)

Browse Source 
* fix: respect cache backend type for registry cache import/export

* docs: update local development build instructions

- Update local image building to use the provided project task.
- Document the workflow for building with a local registry.
- Add examples for configuring buildctl with caching and debugging flags.

Generated-by: aiautocommit

* refactor(buildkit): clean up code comments and formatting

- Remove redundant comments and improve internal documentation.
- Fix minor formatting inconsistencies in code blocks and comment descriptions.

Generated-by: aiautocommit

* feat: add ParseBool utility to handle common boolean string variants

Generated-by: aiautocommit

* feat: integrate local registry and buildkit configuration

- Add buildkitd configuration to enable HTTP/insecure access to the local registry.
- Introduce a local registry service container on port 7890.
- Update the buildkit setup to mount the new registry configuration.
- Add tasks for building the frontend image and running project builds using the local registry and buildctl.

Generated-by: aiautocommit

* docs: update CLAUDE.md with architecture and style guidance

Generated-by: aiautocommit

* feat: add support for multiple cache-from and cache-to build flags

- Allow users to configure external cache sources and destinations via the `build` subcommand.
- Update cache parsing logic to support multiple entries matching the Docker buildx CLI format.
- Remove hardcoded default cache type to provide better control over BuildKit cache configurations.

Generated-by: aiautocommit

* test: add integration tests for cache-import parsing

Generated-by: aiautocommit

* refactor(buildkit): extract cache entry parsing logic

- Centralize cache configuration logic by moving loop-based parsing into a reusable `cacheEntriesFromFlags` helper.
- Improve code maintainability and readability by reducing duplication in `BuildWithBuildkitClient`.
- Add comprehensive unit tests for cache entry processing, including edge cases for flag parsing and empty inputs.

Generated-by: aiautocommit

* fix(test): remove unused buildkit cache flags from integration tests

* Remove unused flag variables that were meant for remote BuildKit caching.
* Ensure integration tests avoid remote cache (e.g., type=gha) to prevent exceeding GitHub Actions cache limits and excessive eviction.

Generated-by: aiautocommit

* test(buildkit): consolidate cache import tests

- Move TestParseCacheImports from its own file to frontend_test.go to consolidate related buildkit frontend tests.

Generated-by: aiautocommit

* style: fix indentation in build test helper

Generated-by: aiautocommit

* docs: document support for BuildKit cache backends

Generated-by: aiautocommit

* docs(developing-locally): add information on Docker, BuildKit, and caching

- Add missing setup prerequisites.
- Include debugging and configuration guidance for BuildKit/Docker.
- Clarify cache export and import implementation expectations in Railpack.

Generated-by: aiautocommit

* docs: clarify cache flag parsing and pnpm version logic

Generated-by: aiautocommit

---------

Co-authored-by: Matt Van Horn <[email protected]>

Co-authored-by: Michael Bianco <[email protected]>

Co-authored-by: Hanzo Dev <[email protected]>
This commit is contained in:
Matt Van Horn
2026-07-13 11:15:16 -06:00
committed by GitHub
co-authored by Hanzo Dev
parent 0d3e0a6b4f
commit 3e53630854
18 changed files with 584 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.mise/tasks/buildkitd.toml.template
+9
View File
@@ -0,0 +1,9 @@
# Allow HTTP pulls/pushes to the local registry:2 used for frontend images and cache.
# See .mise/tasks/images.toml (image-run-registry / image-frontend-build).
[registry."host.docker.internal:7890"]
http = true
insecure = true
[registry."localhost:7890"]
http = true
insecure = true
.mise/tasks/images.toml
+70
View File
@@ -0,0 +1,70 @@
[image-builder-build]
run = "docker build --build-arg MISE_VERSION=$RAILPACK_MISE_VERSION -f images/debian/build/Dockerfile -t railpack-builder:local ."
[mise-builder-shell]
description = "Shell into the railpack-builder image"
usage = 'flag "--local" help="Build and use railpack-builder:local instead of the ghcr image"'
run = '''
if [ "${usage_local:-false}" = "true" ]; then
mise run image-builder-build
image="railpack-builder:local"
else
image="ghcr.io/railwayapp/railpack-builder:mise-$RAILPACK_MISE_VERSION"
fi
exec docker run -it --rm "$image" bash
'''
[image-runtime-build]
run = "docker build -f images/debian/runtime/Dockerfile -t railpack-runtime:local ."
[image-frontend-build]
description = "Build the railpack frontend image and push to local registry"
run = [
"docker build -f images/alpine/frontend/Dockerfile -t railpack-frontend:local .",
# in order for builtctl to use this image, it needs to exist on a registry
"docker tag railpack-frontend:local localhost:7890/railpack-frontend:local",
"docker push localhost:7890/railpack-frontend:local",
]
[image-run-registry]
description = "Local image registry for frontend images and BuildKit cache (HTTP on :7890)"
run = '''
docker rm -f railpack-registry 2>/dev/null || true
docker run -d --rm \
--name railpack-registry \
-p 7890:5000 \
registry:2
'''
[image-frontend-build-project]
description = "Build a project dir with the local railpack frontend via buildctl with debug logging enabled and a local cache registry"
usage = 'arg "<dir>" help="Project directory to build (e.g. examples/node-bun)"'
run = '''
set -euxo pipefail
dir="${usage_dir:?dir is required}"
if [ ! -d "$dir" ]; then
echo "directory not found: $dir" >&2
exit 1
fi
project_name="$(basename "$dir")"
plan_dir="tmp/frontend-plan"
mkdir -p "$plan_dir"
mise run cli plan "$dir" --out "$plan_dir/railpack-plan.json"
# host.docker.internal is required since the frontend container is running inside a buildx container
buildctl --debug build \
--frontend=gateway.v0 \
--opt source=host.docker.internal:7890/railpack-frontend:local \
--opt build-arg:verbose=true \
--opt image-resolve-mode=pull \
--local "context=$dir" \
--local "dockerfile=$plan_dir" \
--progress=plain \
--trace="$plan_dir/build.trace" \
--debug-json-cache-metrics stdout \
--export-cache type=registry,ref=host.docker.internal:7890/"$project_name":cache,mode=max \
--import-cache type=registry,ref=host.docker.internal:7890/"$project_name":cache
'''
CLAUDE.md
+6 -6
View File
@@ -5,18 +5,18 @@ it into a container image. It's built on BuildKit with support for Node, Python,
# Architecture
- **Core**: Analyzes apps and generates JSON build plans using language
providers
- **CLI**: Main entry point that coordinates core analysis and BuildKit execution
- **Core**: Analyzes apps and generates JSON build plans using language providers
- **BuildKit**: Converts build plans to BuildKit LLB (Low-Level Builder) format
for efficient image construction
- **CLI**: Main entry point that coordinates core analysis and BuildKit
execution
for image construction.
- **Providers**: Language-specific modules that detect project types (e.g. Node
detects package.json) and generate appropriate build steps
detects package.json) and generate appropriate build steps.
- **Runtime**: The built images are based on @images/debian/runtime/Dockerfile
# Code style
Follow these instructions carefully when writing code:
- When writing a comment describing a function, do not start the comment with the name of the function
- Assume the person reading this code is an expert software engineer, but is not familiar with the internals of every system. Include concise one-line comments explaining key hooks, API usage, blocks of logic, etc., to help the reader quickly understand the code you've written.
- Follow Go conventions and existing patterns in the codebase
buildkit/build.go
+55 -31
View File
@@ -1,3 +1,6 @@
// called by the build CLI entrypoint and runs the build using the buildkit client
// also used by the integration tests to run builds in a test environment
package buildkit
import (
@@ -5,6 +8,7 @@ import (
"errors"
"fmt"
"io"
"maps"
"os"
"os/exec"
"strings"
@@ -39,8 +43,7 @@ Most likely the $BUILDKIT_HOST is not running. Here's an example of how to start
docker run --rm --privileged -d --name buildkit moby/buildkit
Use 'railpack --verbose' to view more error details.
`
Use 'railpack --verbose' to view more error details`
)
type BuildWithBuildkitClientOptions struct {
@@ -51,8 +54,8 @@ type BuildWithBuildkitClientOptions struct {
SecretsHash string
Secrets map[string]string
Platform string
ImportCache string
ExportCache string
ImportCache []string
ExportCache []string
CacheKey string
GitHubToken string
}
@@ -204,21 +207,11 @@ func BuildWithBuildkitClient(appDir string, plan *plan.BuildPlan, opts BuildWith
},
}
// Add cache import if specified
if opts.ImportCache != "" {
solveOpts.CacheImports = append(solveOpts.CacheImports, client.CacheOptionsEntry{
Type: "gha",
Attrs: parseKeyValue(opts.ImportCache),
})
}
solveOpts.CacheImports = cacheEntriesFromFlags(opts.ImportCache)
solveOpts.CacheExports = cacheEntriesFromFlags(opts.ExportCache)
// Add cache export if specified
if opts.ExportCache != "" {
solveOpts.CacheExports = append(solveOpts.CacheExports, client.CacheOptionsEntry{
Type: "gha",
Attrs: parseKeyValue(opts.ExportCache),
})
}
log.Infof("cache imports: %v", solveOpts.CacheImports)
log.Infof("cache exports: %v", solveOpts.CacheExports)
// Save the resulting filesystem to a directory
if opts.OutputDir != "" {
@@ -227,15 +220,10 @@ func BuildWithBuildkitClient(appDir string, plan *plan.BuildPlan, opts BuildWith
return fmt.Errorf("error creating output directory: %w", err)
}
solveOpts = client.SolveOpt{
LocalMounts: map[string]fsutil.FS{
"context": appFS,
},
Exports: []client.ExportEntry{
{
Type: client.ExporterLocal,
OutputDir: opts.OutputDir,
},
solveOpts.Exports = []client.ExportEntry{
{
Type: client.ExporterLocal,
OutputDir: opts.OutputDir,
},
}
}
@@ -273,25 +261,61 @@ func BuildWithBuildkitClient(appDir string, plan *plan.BuildPlan, opts BuildWith
return nil
}
// determine the image name from the app dir path
func getImageName(appDir string) string {
parts := strings.Split(appDir, string(os.PathSeparator))
name := parts[len(parts)-1]
// TODO how could this happen in practice?
if name == "" {
name = "railpack-app" // Fallback if path ends in separator
}
// Docker requires image names to be lowercase
return strings.ToLower(name)
}
// Helper function to parse key=value strings into a map
// Converts docker buildx-style cache flag values (e.g. type=registry,ref=...)
// into BuildKit CacheOptionsEntry values. Empty strings are skipped.
//
// Intentionally hand-rolled instead of using github.com/docker/buildx/util/buildflags
// (or pulling buildx solely for ParseCacheEntry). BuildKit has no public parser for
// these strings; buildx does, but the dependency cost outweighs the small amount of
// logic we need for the type=... form we document.
func cacheEntriesFromFlags(entries []string) []client.CacheOptionsEntry {
var out []client.CacheOptionsEntry
for _, entry := range entries {
if entry == "" {
continue
}
cacheType, attrs := extractCacheType(parseKeyValue(entry))
out = append(out, client.CacheOptionsEntry{
Type: cacheType,
Attrs: attrs,
})
}
return out
}
// parse comma-separated key=value strings into a map, ignoring entries without an "="
func parseKeyValue(s string) map[string]string {
attrs := make(map[string]string)
parts := strings.SplitSeq(s, ",")
for part := range parts {
kv := strings.SplitN(part, "=", 2)
if len(kv) == 2 {
attrs[kv[0]] = kv[1]
key, value, found := strings.Cut(part, "=")
if !found {
continue
}
attrs[strings.TrimSpace(key)] = strings.TrimSpace(value)
}
return attrs
}
func extractCacheType(attrs map[string]string) (string, map[string]string) {
cacheType := attrs["type"]
cleanedAttrs := maps.Clone(attrs)
delete(cleanedAttrs, "type")
return cacheType, cleanedAttrs
}
buildkit/build_test.go
+141
View File
@@ -1,8 +1,12 @@
package buildkit
import (
"maps"
"path/filepath"
"testing"
"github.com/moby/buildkit/client"
"github.com/stretchr/testify/require"
)
func TestGetImageName(t *testing.T) {
@@ -52,3 +56,140 @@ func TestGetImageName(t *testing.T) {
})
}
}
func TestExtractCacheType(t *testing.T) {
tests := []struct {
name string
input string
expectedType string
expectedAttrs map[string]string
}{
{
name: "registry cache type is extracted",
input: "type=registry,ref=localhost:5555/cache",
expectedType: "registry",
expectedAttrs: map[string]string{
"ref": "localhost:5555/cache",
},
},
{
name: "mode max kept in attrs",
input: "type=registry,ref=host.docker.internal:7890/node-bun:cache,mode=max",
expectedType: "registry",
expectedAttrs: map[string]string{
"ref": "host.docker.internal:7890/node-bun:cache",
"mode": "max",
},
},
{
name: "gha shape",
input: "type=gha,scope=my-cache",
expectedType: "gha",
expectedAttrs: map[string]string{
"scope": "my-cache",
},
},
{
name: "type only",
input: "type=registry",
expectedType: "registry",
expectedAttrs: map[string]string{},
},
{
name: "spaces around keys and values",
input: " type = registry , ref = myapp:cache ",
expectedType: "registry",
expectedAttrs: map[string]string{
"ref": "myapp:cache",
},
},
{
name: "missing type is not defaulted",
input: "scope=my-cache,mode=max",
expectedType: "",
expectedAttrs: map[string]string{
"scope": "my-cache",
"mode": "max",
},
},
{
name: "empty attrs",
input: "",
expectedType: "",
expectedAttrs: map[string]string{},
},
{
name: "skips segments without equals",
input: "type=registry,notavalue,ref=x",
expectedType: "registry",
expectedAttrs: map[string]string{
"ref": "x",
},
},
{
name: "value may contain equals",
input: "type=registry,ref=image:tag=latest",
expectedType: "registry",
expectedAttrs: map[string]string{
"ref": "image:tag=latest",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotType, gotAttrs := extractCacheType(parseKeyValue(tt.input))
if gotType != tt.expectedType {
t.Errorf("extractCacheType(%q) type = %q, want %q", tt.input, gotType, tt.expectedType)
}
if !maps.Equal(gotAttrs, tt.expectedAttrs) {
t.Errorf("extractCacheType(%q) attrs = %v, want %v", tt.input, gotAttrs, tt.expectedAttrs)
}
})
}
}
func TestCacheEntriesFromFlags(t *testing.T) {
t.Run("nil slice", func(t *testing.T) {
require.Nil(t, cacheEntriesFromFlags(nil))
})
t.Run("empty slice", func(t *testing.T) {
require.Nil(t, cacheEntriesFromFlags([]string{}))
})
t.Run("skips empty strings", func(t *testing.T) {
got := cacheEntriesFromFlags([]string{"", "type=registry,ref=x", ""})
require.Equal(t, []client.CacheOptionsEntry{
{Type: "registry", Attrs: map[string]string{"ref": "x"}},
}, got)
})
t.Run("multiple registry entries", func(t *testing.T) {
got := cacheEntriesFromFlags([]string{
"type=registry,ref=myapp:cache-main",
"type=registry,ref=myapp:cache-branch",
})
require.Equal(t, []client.CacheOptionsEntry{
{Type: "registry", Attrs: map[string]string{"ref": "myapp:cache-main"}},
{Type: "registry", Attrs: map[string]string{"ref": "myapp:cache-branch"}},
}, got)
})
t.Run("mixed registry and gha with mode", func(t *testing.T) {
got := cacheEntriesFromFlags([]string{
"type=registry,ref=host.docker.internal:7890/node-bun:cache,mode=max",
"type=gha,scope=ci",
})
require.Equal(t, []client.CacheOptionsEntry{
{
Type: "registry",
Attrs: map[string]string{
"ref": "host.docker.internal:7890/node-bun:cache",
"mode": "max",
},
},
{Type: "gha", Attrs: map[string]string{"scope": "ci"}},
}, got)
})
}
buildkit/frontend.go
+54 -8
View File
@@ -1,4 +1,7 @@
// for platforms: used by `ghcr.io/railwayapp/railpack-frontend` to accept railpack plans via BuildKit
// for platforms: used by `ghcr.io/railwayapp/railpack-frontend` as a buildkit frontend
// the buildkit library consumes buildkit input and exposes it to us via the client.Client interface
// note that `frontend` and `build` are completely separate paths
package buildkit
import (
@@ -27,14 +30,18 @@ const (
// default filename for the serialized Railpack plan
defaultRailpackPlan = "railpack-plan.json"
// Build arg keys
// railpack build args
secretsHash = "secrets-hash"
cacheKey = "cache-key"
githubToken = "github-token"
// buildctl --import-cache is serialized into this frontend opt by the BuildKit client
// `docker buildx` uses a different arg name, but the buildkit frontend normalizes the opt name the frontend receives
keyCacheImports = "cache-imports"
)
func StartFrontend() {
log.Info("Starting frontend")
log.Info("starting frontend")
ctx := appcontext.Context()
if err := gw.RunFromEnvironment(ctx, Build); err != nil {
@@ -43,6 +50,7 @@ func StartFrontend() {
}
}
// handler for the buildkit gateway to let us read the railplan plan and generate a buildkit solve
func Build(ctx context.Context, c client.Client) (*client.Result, error) {
opts := c.BuildOpts().Opts
buildArgs := parseBuildArgs(opts)
@@ -88,8 +96,18 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) {
return nil, fmt.Errorf("error marshalling image: %w", err)
}
// buildkit does not auto-apply --import-cache to this solve, we need to parse the frontend opt and set the CacheImports explicitly
// cache exports are applied automatically for us since they do not impact the solve
cacheImports, err := parseCacheImports(opts)
if err != nil {
return nil, err
}
// NOTE logs are swallowed and outputted to the buildkit container logs, not the buildctl logs
log.Infof("frontend cache imports: %v", cacheImports)
res, err := c.Solve(ctx, client.SolveRequest{
Definition: def.ToPB(),
Definition: def.ToPB(),
CacheImports: cacheImports,
})
if err != nil {
return nil, err
@@ -121,7 +139,7 @@ func readRailpackPlan(ctx context.Context, c client.Client) (*plan.BuildPlan, er
return plan, nil
}
// validatePlatform checks if the platform is supported and returns the corresponding specs.Platform
// checks if the platform is supported and returns the corresponding platform specs
func validatePlatform(opts map[string]string) (specs.Platform, error) {
platformStr := opts["platform"]
@@ -130,7 +148,6 @@ func validatePlatform(opts map[string]string) (specs.Platform, error) {
return specs.Platform{}, fmt.Errorf("multiple platforms are not supported, got: %s", platformStr)
}
// Parse the platform using our helper function
platform, err := ParsePlatformWithDefaults(platformStr)
if err != nil {
return specs.Platform{}, fmt.Errorf("invalid platform format: %s. Must be one of: linux/amd64, linux/arm64, etc", platformStr)
@@ -139,7 +156,8 @@ func validatePlatform(opts map[string]string) (specs.Platform, error) {
return platform, nil
}
// Read a file from the build context
// Read a file from the build context. The frontend does not have a full `App` struct, which is why we have this helper
// to read railpack-plan.json.
func readFile(ctx context.Context, c client.Client, filename string) (string, error) {
// Create a Local source for the dockerfile
src := llb.Local(configMountName,
@@ -177,6 +195,12 @@ func readFile(ctx context.Context, c client.Client, filename string) (string, er
return fileContents, nil
}
// Extracts Docker/buildx --build-arg values from frontend opts.
//
// Docker/buildx always namespaces those as "build-arg:<name>" (e.g.
// --build-arg cache-key=x → opts["build-arg:cache-key"]). bare buildctl
// --opt cache-key=x is opts["cache-key"] and is not returned here; use
// --opt build-arg:cache-key=x for the same shape as Docker.
func parseBuildArgs(opts map[string]string) map[string]string {
buildArgs := make(map[string]string)
@@ -185,10 +209,32 @@ func parseBuildArgs(opts map[string]string) map[string]string {
continue
}
// Remove the "build-arg:" prefix
name := strings.TrimPrefix(key, "build-arg:")
buildArgs[name] = arg
}
return buildArgs
}
// reads the "cache-imports" frontend opt set by the BuildKit
func parseCacheImports(opts map[string]string) ([]client.CacheOptionsEntry, error) {
cacheImportsStr := opts[keyCacheImports]
if cacheImportsStr == "" {
return nil, nil
}
// Same JSON shape as control API CacheOptionsEntry (Type / Attrs).
var entries []struct {
Type string `json:"Type"`
Attrs map[string]string `json:"Attrs"`
}
if err := json.Unmarshal([]byte(cacheImportsStr), &entries); err != nil {
return nil, errors.Wrapf(err, "failed to unmarshal %s (%q)", keyCacheImports, cacheImportsStr)
}
cacheImports := make([]client.CacheOptionsEntry, 0, len(entries))
for _, e := range entries {
cacheImports = append(cacheImports, client.CacheOptionsEntry{Type: e.Type, Attrs: e.Attrs})
}
return cacheImports, nil
}
buildkit/frontend_test.go
+42
View File
@@ -2,6 +2,8 @@ package buildkit
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestParseBuildArgs(t *testing.T) {
@@ -29,3 +31,43 @@ func TestParseBuildArgs(t *testing.T) {
}
}
}
func TestParseCacheImports(t *testing.T) {
t.Parallel()
t.Run("empty opts", func(t *testing.T) {
imports, err := parseCacheImports(nil)
require.NoError(t, err)
require.Empty(t, imports)
})
t.Run("cache-imports JSON", func(t *testing.T) {
imports, err := parseCacheImports(map[string]string{
keyCacheImports: `[{"Type":"registry","Attrs":{"ref":"host.docker.internal:7890/node-bun:cache"}}]`,
})
require.NoError(t, err)
require.Len(t, imports, 1)
require.Equal(t, "registry", imports[0].Type)
require.Equal(t, "host.docker.internal:7890/node-bun:cache", imports[0].Attrs["ref"])
})
t.Run("multiple cache-imports", func(t *testing.T) {
imports, err := parseCacheImports(map[string]string{
keyCacheImports: `[{"Type":"registry","Attrs":{"ref":"a:cache"}},{"Type":"gha","Attrs":{"scope":"my-scope"}}]`,
})
require.NoError(t, err)
require.Len(t, imports, 2)
require.Equal(t, "registry", imports[0].Type)
require.Equal(t, "a:cache", imports[0].Attrs["ref"])
require.Equal(t, "gha", imports[1].Type)
require.Equal(t, "my-scope", imports[1].Attrs["scope"])
})
t.Run("invalid cache-imports JSON", func(t *testing.T) {
_, err := parseCacheImports(map[string]string{
keyCacheImports: `not-json`,
})
require.Error(t, err)
require.Contains(t, err.Error(), keyCacheImports)
})
}
cli/build.go
+17 -5
View File
@@ -51,6 +51,14 @@ var BuildCommand = &cli.Command{
Name: "cache-key",
Usage: "Unique id to prefix to cache keys",
},
&cli.StringSliceFlag{
Name: "cache-from",
Usage: "External cache sources",
},
&cli.StringSliceFlag{
Name: "cache-to",
Usage: "Cache export destinations",
},
&cli.BoolFlag{
Name: "dump-llb",
Hidden: true,
@@ -99,10 +107,13 @@ var BuildCommand = &cli.Command{
OutputDir: cmd.String("output"),
ProgressMode: cmd.String("progress"),
CacheKey: cmd.String("cache-key"),
SecretsHash: secretsHash,
Secrets: env.Variables,
Platform: platformStr,
GitHubToken: os.Getenv("GITHUB_TOKEN"),
// StringSlice to support multiple cache-from / cache-to entries, same shape as docker buildx
ImportCache: cmd.StringSlice("cache-from"),
ExportCache: cmd.StringSlice("cache-to"),
SecretsHash: secretsHash,
Secrets: env.Variables,
Platform: platformStr,
GitHubToken: os.Getenv("GITHUB_TOKEN"),
})
if err != nil {
return cli.Exit(err, 1)
@@ -112,10 +123,11 @@ var BuildCommand = &cli.Command{
},
}
// make sure all secrets referenced in the build plan are present in the environment
func validateSecrets(plan *plan.BuildPlan, env *app.Environment) error {
for _, secret := range plan.Secrets {
if _, ok := env.Variables[secret]; !ok {
return fmt.Errorf("missing environment variable: %s. Please set the envvar with --env %s=%s", secret, secret, "...")
return fmt.Errorf("missing environment variable: %s. Please set using --env %s=%s", secret, secret, "...")
}
}
return nil
core/providers/node/package_manager.go
+1
View File
@@ -184,6 +184,7 @@ func usesPnpmBinSubdir(version string) bool {
return false
}
// `latest` is definitely >= 11
if version == "latest" {
return true
}
docs/src/content/docs/architecture/caching.md
+8 -1
View File
@@ -8,7 +8,7 @@ builds.
## Layer Cache
Railpack takes advantage of BuildKit's layer cache and avoids busting the cache
Railpack uses BuildKit's layer cache and avoids busting the cache
when possible. Cache busting events are defined in a granular way as part of the
[steps commands list](/architecture/overview/#build-step). These include:
@@ -17,6 +17,13 @@ when possible. Cache busting events are defined in a granular way as part of the
- Adding new generated files to the build context
- Executing shell commands in the build context
### Cache Backends
Railpack supports all [BuildKit cache backends](https://docs.docker.com/build/cache/backends/).
* `railpack build` supports the same CLI arguments as `docker buildx` for cache import/export. See the [CLI reference](../reference/cli/#build) for more information.
* Cache import/export references are supported when using the Railpack frontend directly with `docker buildx` or `buildctl`. See the [frontend reference](../reference/frontend/#configuration) for more information.
## Mount Cache
The [BuildKit mount
docs/src/content/docs/guides/developing-locally.md
+84 -14
View File
@@ -3,14 +3,17 @@ title: Developing Locally
description: Learn how to develop Railpack locally
---
Once you've [checked out the repo](https://github.com/railwayapp/railpack), you
can follow this to start developing locally.
We love contributions to Railpack! This guide is to help Railpack developers understand the system quickly.
Some pre-requisites:
* [Check out the repo](https://github.com/railwayapp/railpack)
* [Install Mise](https://mise.jdx.dev/installing-mise.html)
## Getting Setup
We use [Mise](https://mise.jdx.dev/) for managing language dependencies and
tasks for building and testing Railpack. You don't have to use Mise, but it's
recommended.
[Mise](https://mise.jdx.dev/) is used to manage language dependencies and
tasks for building and testing Railpack.
Install and use all versions of tools needed for Railpack
@@ -71,11 +74,10 @@ Multiple Docker images are used in Railpack:
You can build with a [custom BuildKit frontend](/guides/custom-frontend), but
this is a bit tedious for local iteration.
The frontend needs to be built into an image and accessible to the BuildKit
instance. You can build this image locally using standard Docker commands from the root of the repository:
The frontend needs to be built into an image and accessible to the BuildKit instance:
```bash
docker build -f images/alpine/frontend/Dockerfile -t railpack-frontend:local .
mise run image-runtime-build
```
Then, generate a build plan for an app:
@@ -93,22 +95,71 @@ docker buildx \
examples/node-bun
```
By default, `ghcr.io/railwayapp/railpack:railpack-frontend` is used when running `railpack build`.
You can also use the `buildctl` command to run BuildKit directly. This is helpful as it's a lower level command which
exposes helpful debugging flags. However, you can't reference the locally built image without loading it into a registry first.
You can also use the `buildctl` command to run BuildKit directly:
This is done automatically for you with `image-runtime-build` but you must start the registry first with `image-run-registry`.
Then, you can run the build with the locally-build frontend:
```bash
buildctl build \
--frontend=gateway.v0 \
--opt source=railpack-frontend:local \
--local context=examples/node-bun \
--local dockerfile=test \
--frontend=gateway.v0 \
--opt source=ghcr.io/railwayapp/railpack:railpack-frontend \
--output type=docker,name=test | docker load
```
*Note the `docker load` here to load the image into Docker. However, you can
The `dockerfile=` param instructs railpack to use that directory to look for the `railpack-plan.json` file. The `context=` param is the path to the app to build. More specifically, `--local` 'uploads' the referenced directories
to the buildkit daemon.
Note the `docker load` here to load the image into Docker. However, you can
change the [output](https://github.com/moby/buildkit?tab=readme-ov-file#output)
or push to a registry instead.*
or push to a registry instead.
You can also provide additional configuration to buildctl, like registry
cache import/export (use top-level flags, not `--opt`):
```bash
buildctl build \
--frontend=gateway.v0 \
--opt source=host.docker.internal:7890/railpack-frontend:local \
--local context=examples/node-bun \
--local dockerfile=tmp/frontend-plan \
--export-cache type=registry,ref=host.docker.internal:7890/node-bun:cache,mode=max \
--import-cache type=registry,ref=host.docker.internal:7890/node-bun:cache
```
Note that the cache arguments are different than what `docker buildx`. The equivalent `docker buildx` command would be:
```bash
docker buildx build \
--build-arg BUILDKIT_SYNTAX="host.docker.internal:7890/railpack-frontend:local" \
--cache-to=type=registry,ref=host.docker.internal:7890/node-bun:cache,mode=max \
--cache-from=type=registry,ref=host.docker.internal:7890/node-bun:cache \
-f tmp/frontend-plan/railpack-plan.json \
examples/node-bun
```
Debugging a buildkit related problem? Enable debug logging:
```bash
buildctl --debug build \
--frontend=gateway.v0 \
--opt source=railpack-frontend:local \
--local context=examples/node-bun \
--progress=plain \
--trace=tmp/builtctl-build-trace.log \
--debug-json-cache-metrics stdout
```
Quick note about `builtctl` vs `docker buildx`. These two ways of invoking the railpack frontend handle arguments differently:
* `--build-arg` prefixes the argument with `build-arg:`.
* `--opt` does not prefix the build arg at all. You must prefix args with `build-arg:` if they are
arguments handled by the railpack frontend.
## Integration Tests
@@ -252,6 +303,10 @@ mise exec pipx:httpie -- http google.com
Here's some helpful debugging tricks:
* `URFAVE_CLI_TRACING=on` for debugging CLI argument parsing
* `RAILPACK_DEBUG=1` for debugging Railpack debug logging
* `--build-arg verbose=true` for debugging the frontend (or `--opt build-arg:verbose=true` with `buildctl`)
* `docker logs -f buildkit` to see the BuildKit daemon logs, which includes railpack logs when it's used as a frontend
* `docker logs -f railpack-registry` to inspect local registry logs. Helpful for debugging cache import/export issues.
* `mise run cli -- --verbose build --show-plan --progress plain examples/node-bun`
* `mise run build`, add `./bin/` to your `$PATH`, and then run `railpack` in a separate local directory
* `docker exec buildkit buildctl prune` to clean the builder cache
@@ -305,6 +360,21 @@ continue
The commands you probably want: `ls`, `print build.Commands`, `continue`, `next`, `locals`,
## Docker / BuildKit
### Frontend
* When using Railpack as a frontend, all logs go to the buildkit container logs, and are not outputted to the build progress logs.
* `buildctl` is the lower level interface to BuildKit compared to `docker buildx`. There are more options available for debugging.
* `builtctl` and `docker buildx` handle arguments differently. `--build-arg` prefixes the argument with `build-arg:`. `--opt` does not prefix the build arg at all. You must prefix args with `build-arg:` if they are arguments handled by the railpack frontend.
### Cache
* Cache export does not require any logic within railpack. This is given "for free" since we are using BuildKit LLB.
* However, all import cache support must be implemented in Railpack. BuildKit is careful not to be too opinionated about defaults.
* If you use a registry cache you can tail the logs to inspect what is actually being pulled/pushed when building an image.
* There's no util methods for parsing the cache kv comma-separated strings in the buildkit module.
## Node
The node provider is the most complex.
docs/src/content/docs/reference/cli.md
+10 -8
View File
@@ -32,14 +32,16 @@ railpack build [options] DIRECTORY
**Options:**
| Flag | Description | Default |
| ------------- | ----------------------------------------------------- | ------- |
| `--name` | Name of the image to build | |
| `--output` | Output the final filesystem to a local directory | |
| `--platform` | Platform to build for (e.g. linux/amd64, linux/arm64) | |
| `--progress` | BuildKit progress output mode (auto, plain, tty) | `auto` |
| `--show-plan` | Show the build plan before building | `false` |
| `--cache-key` | Unique id to prefix to cache keys | |
| Flag | Description | Default |
| -------------- | --------------------------------------------------------------------------- | ------- |
| `--name` | Name of the image to build | |
| `--output` | Output the final filesystem to a local directory | |
| `--platform` | Platform to build for (e.g. linux/amd64, linux/arm64) | |
| `--progress` | BuildKit progress output mode (auto, plain, tty) | `auto` |
| `--show-plan` | Show the build plan before building | `false` |
| `--cache-key` | Unique id to prefix to cache keys | |
| `--cache-from` | External cache sources (same as docker buildx). e.g. type=registry,ref=... | |
| `--cache-to` | Cache export destinations (same as docker buildx). e.g. type=registry,ref=... | |
### prepare
docs/src/content/docs/reference/frontend.md
+25 -17
View File
@@ -3,9 +3,12 @@ title: BuildKit Frontend Reference
description: Complete reference for the Railpack BuildKit frontend
---
Complete reference documentation for the Railpack BuildKit frontend. For a full example of how to use the frontend for a production build, see the [running Railpack in production](../guides/running-railpack-in-production) guide.
If you are a platform, you'll want to use the Railpack BuildKit frontend to build projects with Railpack. If you aren't a platform and you want to build a single container, you should use the
other `railpack` subcommands and ignore this documentation.
## Expected
For a full example of how to use the frontend for a production build, see the [running Railpack in production](../guides/running-railpack-in-production) guide.
## Using Railpack as a BuildKit Frontend
To use the Railpack BuildKit frontend, you must provide a build plan file (e.g.
`railpack-plan.json`) generated by the `railpack prepare` command. This file
@@ -23,6 +26,10 @@ docker buildx build \
/path/to/app/to/build
```
The build plan file does not need to be in the same directory as your app, but
you must reference it correctly with the `-f` flag (Docker) or `--local
dockerfile` (BuildKit).
For **BuildKit** directly:
```sh
@@ -34,9 +41,7 @@ buildctl build \
--output type=docker,name=test
```
The build plan file does not need to be in the same directory as your app, but
you must reference it correctly with the `-f` flag (Docker) or `--local
dockerfile` (BuildKit).
`buildctl` is a lower-level interface to BuildKit. It's arguments and semantics are similar to `docker buildx`, but not identical.
## Configuration
@@ -44,9 +49,9 @@ You can pass advanced options to the frontend using the `--opt` flag (for BuildK
| Flag | Description | Default |
| ---------------- | -------------------------------------------------------------------------------------- | ------- |
| `--cache-key` | Unique ID to prefix to cache keys for cache invalidation. | |
| `--secrets-hash` | Hash of all secret values, used to invalidate cache when secrets change. | |
| `--github-token` | GitHub token to increase API rate limits for private repositories or package installs. | |
| `cache-key` | Unique ID to prefix to cache keys for cache invalidation. | |
| `secrets-hash` | Hash of all secret values, used to invalidate cache when secrets change. | |
| `github-token` | GitHub token to increase API rate limits for private repositories or package installs. | |
### Example
@@ -58,7 +63,6 @@ docker buildx build \
--build-arg cache-key=my-key \
--build-arg secrets-hash=abc123 \
--build-arg github-token=ghp_xxx \
--build-arg FOO=bar \
-f /path/to/railpack-plan.json \
/path/to/app/to/build
```
@@ -67,13 +71,15 @@ docker buildx build \
```sh
buildctl build \
--frontend=railwayapp/railpack \
--opt cache-key=my-key \
--opt secrets-hash=abc123 \
--opt github-token=ghp_xxx \
--opt build-arg:FOO=bar
--frontend=gateway.v0 \
--opt source=ghcr.io/railwayapp/railpack-frontend:latest \
--opt build-arg:cache-key=my-key \
--opt build-arg:secrets-hash=abc123 \
--opt build-arg:github-token=ghp_xxx
```
Note that the `build-arg:` prefix is required only when using `buildctl` to ensure the arg structure matches `docker buildx`.
## Secrets
To use secrets in your build, you must:
@@ -90,7 +96,8 @@ railpack prepare /dir/to/build --env STRIPE_LIVE_KEY=sk_live_asdf
**Docker:**
```sh
STRIPE_LIVE_KEY=asdfasdf docker buildx build \
STRIPE_LIVE_KEY=sk_live_123 \
docker buildx build \
--build-arg BUILDKIT_SYNTAX="ghcr.io/railwayapp/railpack-frontend" \
-f /path/to/railpack-plan.json \
--secret id=STRIPE_LIVE_KEY,env=STRIPE_LIVE_KEY \
@@ -100,11 +107,12 @@ STRIPE_LIVE_KEY=asdfasdf docker buildx build \
**BuildKit:**
```sh
STRIPE_LIVE_KEY=sk_live_123 \
buildctl build \
--local context=/path/to/app/to/build \
--local dockerfile=/path/to/dir/containing/railpack-plan.json \
--frontend=gateway.v0 \
--opt source=ghcr.io/railwayapp/railpack-frontend:latest \
--local context=/path/to/app/to/build \
--local dockerfile=/path/to/dir/containing/railpack-plan.json \
--secret id=STRIPE_LIVE_KEY,env=STRIPE_LIVE_KEY \
--output type=docker,name=test
```
images/alpine/frontend/Dockerfile
+1
View File
@@ -7,6 +7,7 @@ COPY go.mod go.sum ./
RUN go mod download
COPY . .
# TODO we should really pull from github releases here instead of recompiling, but allow for a local development build for builtctl testing
RUN CGO_ENABLED=0 go build -ldflags="-w -s" -o /usr/bin/railpack cmd/cli/main.go
FROM alpine
integration_tests/run_test.go
+7 -9
View File
@@ -4,7 +4,6 @@ import (
"bufio"
"context"
"encoding/json"
"flag"
"fmt"
"os"
"os/exec"
@@ -24,8 +23,9 @@ import (
"github.com/stretchr/testify/require"
)
var buildkitCacheImport = flag.String("buildkit-cache-import", "", "BuildKit cache import configuration")
var buildkitCacheExport = flag.String("buildkit-cache-export", "", "BuildKit cache export configuration")
// Integration builds intentionally do not use BuildKit remote cache (e.g. type=gha).
// Example images would overrun GitHub Actions' 10GB cache limit and thrash eviction:
// https://docs.github.com/en/actions/reference/workflows-and-actions/dependency-caching#usage-limits-and-eviction-policy
type StringOrArray []string
@@ -161,12 +161,10 @@ func TestExamplesIntegration(t *testing.T) {
strings.ToLower(uuid.New().String()))
if err := buildkit.BuildWithBuildkitClient(examplePath, buildResult.Plan, buildkit.BuildWithBuildkitClientOptions{
ImageName: imageName,
Platform: testCase.Platform,
ImportCache: *buildkitCacheImport,
ExportCache: *buildkitCacheExport,
Secrets: testCase.Envs,
CacheKey: imageName,
ImageName: imageName,
Platform: testCase.Platform,
Secrets: testCase.Envs,
CacheKey: imageName,
// Pass through GITHUB_TOKEN if it exists, this avoids mise timeouts during build
// this can easily occur since we run all integration tests in parallel via GHA
GitHubToken: os.Getenv("GITHUB_TOKEN"),
internal/utils/utils.go
+13
View File
@@ -22,6 +22,19 @@ func RemoveDuplicates[T comparable](sliceList []T) []T {
return list
}
// Parses common true/false string values (true/1/yes/on and false/0/no/off).
// The second return is false when the string is empty or unrecognized.
func ParseBool(s string) (value bool, ok bool) {
switch strings.ToLower(strings.TrimSpace(s)) {
case "true", "1", "yes", "on":
return true, true
case "false", "0", "no", "off":
return false, true
default:
return false, false
}
}
// MergeStringSlicePointers combines multiple string slice pointers, deduplicates values, and sorts them
func MergeStringSlicePointers(slices ...*[]string) *[]string {
if len(slices) == 0 {
internal/utils/utils_test.go
+30
View File
@@ -6,6 +6,36 @@ import (
"testing"
)
func TestParseBool(t *testing.T) {
tests := []struct {
name string
input string
wantValue bool
wantOK bool
}{
{name: "true", input: "true", wantValue: true, wantOK: true},
{name: "TRUE spaced", input: " TRUE ", wantValue: true, wantOK: true},
{name: "1", input: "1", wantValue: true, wantOK: true},
{name: "yes", input: "yes", wantValue: true, wantOK: true},
{name: "on", input: "on", wantValue: true, wantOK: true},
{name: "false", input: "false", wantValue: false, wantOK: true},
{name: "0", input: "0", wantValue: false, wantOK: true},
{name: "no", input: "no", wantValue: false, wantOK: true},
{name: "off", input: "off", wantValue: false, wantOK: true},
{name: "empty", input: "", wantValue: false, wantOK: false},
{name: "unknown", input: "maybe", wantValue: false, wantOK: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotValue, gotOK := ParseBool(tt.input)
if gotValue != tt.wantValue || gotOK != tt.wantOK {
t.Errorf("ParseBool(%q) = (%v, %v), want (%v, %v)", tt.input, gotValue, gotOK, tt.wantValue, tt.wantOK)
}
})
}
}
func TestParseSemver(t *testing.T) {
tests := []struct {
name string
mise.toml
+11 -20
View File
@@ -20,11 +20,20 @@ env_shell_expand = true
[tasks.setup]
run = [
"mise install",
"mise run run-buildkit-container || true",
"mise run run-buildkit-container",
]
[tasks.run-buildkit-container]
run = "docker run --rm --privileged -d --name buildkit -e BUILDKIT_DEBUG=1 moby/buildkit:latest"
# Mount buildkitd.toml so host.docker.internal:7890 (local registry) is reachable over HTTP
# `.template` ensures it isn't picked up by the mise task management
run = '''
docker rm -f buildkit 2>/dev/null || true
docker run --rm --privileged -d --name buildkit \
-e BUILDKIT_DEBUG=1 \
-v "$PWD/.mise/tasks/buildkitd.toml.template:/etc/buildkit/buildkitd.toml:ro" \
moby/buildkit:latest \
--config /etc/buildkit/buildkitd.toml
'''
[tasks.cli]
run = "go run cmd/cli/main.go"
@@ -140,21 +149,3 @@ run = ["bun install --frozen-lockfile", "bun run build"]
dir = "docs"
run = ["bun install", "bun run dev"]
[tasks.image-builder-build]
run = "docker build --build-arg MISE_VERSION=$RAILPACK_MISE_VERSION -f images/debian/build/Dockerfile -t railpack-builder:local ."
[tasks.mise-builder-shell]
description = "Shell into the railpack-builder image"
usage = 'flag "--local" help="Build and use railpack-builder:local instead of the ghcr image"'
run = '''
if [ "${usage_local:-false}" = "true" ]; then
mise run image-builder-build
image="railpack-builder:local"
else
image="ghcr.io/railwayapp/railpack-builder:mise-$RAILPACK_MISE_VERSION"
fi
exec docker run -it --rm "$image" bash
'''
[tasks.image-runtime-build]
run = "docker build -f images/debian/runtime/Dockerfile -t railpack-runtime:local ."