test: http checks on example projects (#250)

* test: add HTTP checks

* test: http checks on docker containers

* test: removing justBuild if we have an http check

* test: missing bin/rails

* fix: warning if bin/rails dne

* test: allow json5 in test.json

* test: add justBuild for svelte

core/providers/ruby/ruby.go

@@ -122,6 +122,10 @@ func (p *RubyProvider) GetStartCommand(ctx *generate.GenerateContext) string {
 		if app.HasMatch("rails") {
 			return "bundle exec rails server -b 0.0.0.0 -p ${PORT:-3000}"
 		} else {
+			if !app.HasMatch("bin/rails") {
+				ctx.Logger.LogWarn("bin/rails not found, run `bundle binstubs railties` to avoid potential startup problems")
+			}
+
 			return "bundle exec bin/rails server -b 0.0.0.0 -p ${PORT:-3000} -e $RAILS_ENV"
 		}
 	} else if app.HasMatch("config/environment.rb") && app.HasMatch("script") {

examples/elixir-phoenix/test.json

@@ -1,5 +1,13 @@
 [
   {
-    "justBuild": true
+    "httpCheck": {
+      "path": "/",
+      "expected": 200,
+      "internalPort": 4000
+    },
+    "envs": {
+      "PHX_SERVER": "true",
+      "SECRET_KEY_BASE": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+    }
   }
 ]

examples/java-maven/test.json

@@ -1,5 +1,5 @@
 [
-    {
-        "expectedOutput": "Hello Railpack!"
-    }
-]
+  {
+    "expectedOutput": "Hello Railpack!"
+  }
+]

examples/node-angular/test.json

@@ -0,0 +1,9 @@
+[
+  {
+    "httpCheck": {
+      "path": "/",
+      "expected": 200,
+      "internalPort": 80
+    }
+  }
+]

examples/node-astro-server/test.json

@@ -1,5 +1,12 @@
 [
   {
     "expectedOutput": "Server listening on"
+  },
+  {
+    "httpCheck": {
+        "path": "/",
+        "expected": 200,
+        "internalPort": 4321
+    }
   }
 ]

examples/node-astro/test.json

@@ -1,5 +1,12 @@
 [
   {
     "expectedOutput": "using config from file"
+  },
+  {
+    "httpCheck": {
+      "path": "/",
+      "expected": 200,
+      "internalPort": 80
+    }
   }
-]
+]

examples/node-cra/test.json

@@ -0,0 +1,9 @@
+[
+  {
+    "httpCheck": {
+      "path": "/",
+      "expected": 200,
+      "internalPort": 80
+    }
+  }
+]

examples/node-next/test.json

@@ -0,0 +1,9 @@
+[
+  {
+    "httpCheck": {
+      "path": "/",
+      "expected": 200,
+      "internalPort": 3000
+    }
+  }
+]

examples/node-nuxt/test.json

@@ -0,0 +1,9 @@
+[
+  {
+    "httpCheck": {
+      "path": "/",
+      "expected": 200,
+      "internalPort": 3000
+    }
+  }
+]

examples/node-remix/test.json

@@ -1,5 +1,12 @@
 [
   {
     "expectedOutput": "[remix-serve]"
+  },
+  {
+    "httpCheck": {
+      "path": "/",
+      "expected": 200,
+      "internalPort": 3000
+    }
   }
-]
+]

examples/node-svelte-kit/test.json

@@ -0,0 +1,13 @@
+[
+  {
+    "justBuild": true
+  }
+  // TODO right now caddy is not automatically added to a svelte project
+  // {
+  //   "httpCheck": {
+  //     "path": "/",
+  //     "expected": 200,
+  //     "internalPort": 3000
+  //   }
+  // }
+]

examples/node-tanstack-start/test.json

@@ -1,5 +1,12 @@
 [
   {
     "expectedOutput": "Listening on"
+  },
+  {
+    "httpCheck": {
+      "path": "/",
+      "expected": 200,
+      "internalPort": 3000
+    }
   }
-]
+]

examples/node-vite-react-router-spa/test.json

@@ -1,5 +1,9 @@
 [
   {
-    "justBuild": true
+    "httpCheck": {
+      "path": "/",
+      "expected": 200,
+      "internalPort": 80
+    }
   }
 ]

examples/node-vite-react-router-ssr/test.json

@@ -1,5 +1,9 @@
 [
   {
-    "justBuild": true
+    "httpCheck": {
+      "path": "/",
+      "expected": 200,
+      "internalPort": 3000
+    }
   }
 ]

examples/node-vite-react/test.json

@@ -0,0 +1,12 @@
+[
+  {
+    "expectedOutput": "using config from file"
+  },
+  {
+    "httpCheck": {
+      "path": "/",
+      "expected": 200,
+      "internalPort": 80
+    }
+  }
+]

examples/php-laravel-12-react/test.json

@@ -1,5 +1,15 @@
 [
     {
         "expectedOutput": "starting server loop"
+    },
+    {
+        "httpCheck": {
+            "path": "/",
+            "expected": 200,
+            "internalPort": 80
+        },
+        "envs": {
+            "APP_KEY": "base64:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa="
+        }
     }
 ]

examples/python-django/test.json

@@ -2,4 +2,11 @@
   {
     "justBuild": true
   }
+  // {
+  //   "httpCheck": {
+  //       "path": "/",
+  //       "expected": 200,
+  //       "internalPort": 8000
+  //   }
+  // }
 ]

examples/python-fasthtml/test.json

@@ -0,0 +1,9 @@
+[
+  {
+    "httpCheck": {
+      "path": "/",
+      "expected": 200,
+      "internalPort": 8000
+    }
+  }
+]

examples/python-flask/test.json

@@ -0,0 +1,9 @@
+[
+  {
+    "httpCheck": {
+      "path": "/",
+      "expected": 200,
+      "internalPort": 8000
+    }
+  }
+]

examples/ruby-rails-api-app/bin/rails

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'rails' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../Gemfile", __dir__)
+
+bundle_binstub = File.expand_path("bundle", __dir__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 300).include?("This file was generated by Bundler")
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("railties", "rails")

examples/ruby-rails-api-app/test.json

@@ -1,5 +1,9 @@
 [
   {
-    "justBuild": true
+    "httpCheck": {
+      "path": "/",
+      "expected": 200,
+      "internalPort": 3000
+    }
   }
 ]

examples/ruby-rails-postgres/test.json

@@ -2,4 +2,12 @@
   {
     "justBuild": true
   }
+  // needs a postgres server
+  // {
+  //   "httpCheck": {
+  //       "path": "/",
+  //       "expected": 200,
+  //       "internalPort": 3000
+  //   }
+  // }
 ]

examples/ruby-sinatra/test.json

@@ -1,5 +1,12 @@
 [
   {
     "expectedOutput": "Hello from Sinatra"
+  },
+  {
+    "httpCheck": {
+        "path": "/",
+        "expected": 200,
+        "internalPort": 4567
+    }
   }
 ]

examples/rust-rocket/test.json

@@ -1,5 +1,12 @@
 [
   {
     "expectedOutput": "Rocket has launched"
+  },
+  {
+    "httpCheck": {
+        "path": "/",
+        "expected": 200,
+        "internalPort": 8000
+    }
   }
 ]

examples/staticfile-config/test.json

@@ -0,0 +1,9 @@
+[
+  {
+    "httpCheck": {
+      "internalPort": 80,
+      "path": "/",
+      "expected": 200
+    }
+  }
+]

examples/staticfile-index/test.json

@@ -1,5 +1,12 @@
 [
   {
     "expectedOutput": "using config from file"
+  },
+  {
+    "httpCheck": {
+      "path": "/",
+      "expected": 200,
+      "internalPort": 80
+    }
   }
 ]

integration_tests/http_check.go

@@ -0,0 +1,130 @@
+package integration_tests
+
+import (
+	"fmt"
+	"net"
+	"net/http"
+	"os/exec"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+)
+
+const (
+	// Grace period for servers to bind after initial log lines.
+	containerStartGracePeriod = 500 * time.Millisecond
+	// Total timeout for the HTTP check to succeed. Instead of specifying the number of retries, we use a total timeout.
+	httpCheckTimeout = 35 * time.Second
+	// Time between HTTP check retries.
+	httpCheckInterval = 300 * time.Millisecond
+	// Timeout for a single HTTP GET request.
+	httpCheckClientTimeout = 3 * time.Second
+)
+
+// HTTPCheck defines an HTTP endpoint check for a container
+// Path defaults to /, Expected defaults to 200.
+type HTTPCheck struct {
+	InternalPort int    `json:"internalPort"`
+	Path         string `json:"path"`
+	Expected     int    `json:"expected"`
+}
+
+// runContainerWithHTTPCheck starts a container from the given image,
+// maps the internalPort to a random host port, and checks the given path returns the expected status code.
+func runContainerWithHTTPCheck(t *testing.T, imageName string, envs map[string]string, hc *HTTPCheck) error {
+	if hc.Path == "" {
+		hc.Path = "/"
+	}
+	if hc.Expected == 0 {
+		hc.Expected = 200
+	}
+	if hc.InternalPort == 0 {
+		return fmt.Errorf("httpCheck.internalPort must be specified")
+	}
+
+	hostPort, err := pickFreePort()
+	if err != nil {
+		return fmt.Errorf("allocate host port: %w", err)
+	}
+
+	// at this point, the container has already been built, so GITHUB_TOKEN should not be needed
+	containerName := uniqueContainerName(uuid.New().String())
+	// Run detached without --rm so we can fetch logs even if it exits.
+	// Cleanup is handled explicitly in the deferred stopContainer call below.
+	args := []string{"run", "-d", "--name", containerName, "-p", fmt.Sprintf("127.0.0.1:%d:%d", hostPort, hc.InternalPort)}
+	for k, v := range envs { // test-provided envs
+		args = append(args, "-e", fmt.Sprintf("%s=%s", k, v))
+	}
+	args = append(args, imageName)
+
+	if out, runErr := exec.Command("docker", args...).CombinedOutput(); runErr != nil {
+		return fmt.Errorf("docker run failed: %v: %s", runErr, string(out))
+	}
+	t.Logf("Started container %s from image %s (mapped host %d -> %d)", containerName, imageName, hostPort, hc.InternalPort)
+
+	defer func() {
+		logs := fetchContainerLogs(containerName)
+		t.Logf("Container logs:\n%s", logs)
+		stopContainer(containerName)
+	}()
+
+	// Small grace period: some servers bind after initial log lines
+	time.Sleep(containerStartGracePeriod)
+	url := fmt.Sprintf("http://127.0.0.1:%d%s", hostPort, hc.Path)
+	if err := waitForHTTPStatus(url, hc.Expected); err != nil {
+		return fmt.Errorf("http check failed: %w", err)
+	}
+
+	t.Logf("HTTP check passed (%s => %d)", hc.Path, hc.Expected)
+	return nil
+}
+
+func uniqueContainerName(parts ...string) string {
+	return fmt.Sprintf("railpack-test-%s", strings.Join(parts, "-"))
+}
+
+func pickFreePort() (int, error) {
+	l, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		return 0, err
+	}
+	defer l.Close()
+	return l.Addr().(*net.TCPAddr).Port, nil
+}
+
+func waitForHTTPStatus(url string, expected int) error {
+	deadline := time.Now().Add(httpCheckTimeout)
+	client := &http.Client{Timeout: httpCheckClientTimeout}
+
+	var lastErr error
+	for time.Now().Before(deadline) {
+		resp, err := client.Get(url)
+		if err == nil {
+			_ = resp.Body.Close()
+			if resp.StatusCode == expected {
+				return nil
+			}
+			lastErr = fmt.Errorf("http status %d != %d", resp.StatusCode, expected)
+		} else {
+			lastErr = err
+		}
+
+		time.Sleep(httpCheckInterval)
+	}
+
+	if lastErr == nil {
+		// if we haven't returned, and the HTTP client failed, then we've timed out
+		lastErr = fmt.Errorf("timeout waiting for %s", url)
+	}
+
+	return lastErr
+}
+
+func fetchContainerLogs(name string) string {
+	out, _ := exec.Command("docker", "logs", name).CombinedOutput()
+	return string(out)
+}
+
+func stopContainer(name string) { _ = exec.Command("docker", "rm", "-f", name).Run() }

integration_tests/run_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/railwayapp/railpack/buildkit"
 	"github.com/railwayapp/railpack/core"
 	"github.com/railwayapp/railpack/core/app"
+	"github.com/railwayapp/railpack/internal/utils"
 	"github.com/stretchr/testify/require"
 )
 
@@ -44,12 +45,14 @@ func (s *StringOrArray) UnmarshalJSON(data []byte) error {
 
 type TestCase struct {
 	Platform string `json:"platform"`
+	// can be a single string, or an array of strings for multiple expected outputs
 	// matches against the entire output of the container if it cannot be found in a single line
 	ExpectedOutput StringOrArray     `json:"expectedOutput"`
 	Envs           map[string]string `json:"envs"`
 	ConfigFilePath string            `json:"configFile"`
 	JustBuild      bool              `json:"justBuild"`
 	ShouldFail     bool              `json:"shouldFail"`
+	HTTPCheck      *HTTPCheck        `json:"httpCheck"`
 }
 
 func TestExamplesIntegration(t *testing.T) {
@@ -79,10 +82,15 @@ func TestExamplesIntegration(t *testing.T) {
 		testConfigBytes, err := os.ReadFile(testConfigPath)
 		require.NoError(t, err)
 
+		// allow json5/hujson in the test.json file
+		testConfigBytes, err = utils.StandardizeJSON(testConfigBytes)
+		require.NoError(t, err)
+
 		var testCases []TestCase
 		err = json.Unmarshal(testConfigBytes, &testCases)
 		require.NoError(t, err)
 
+		// each entry in the tests.json array is an individual test case which runs it's own container
 		for i, testCase := range testCases {
 			testCase := testCase // capture for parallel execution
 			i := i
@@ -118,7 +126,7 @@ func TestExamplesIntegration(t *testing.T) {
 					t.Fatal("build result is nil")
 				}
 
-				imageName := fmt.Sprintf("railpack-test-%s-%s",
+				imageName := uniqueContainerName(
 					strings.ToLower(strings.ReplaceAll(testName, "/", "-")),
 					strings.ToLower(uuid.New().String()))
 
@@ -140,6 +148,13 @@ func TestExamplesIntegration(t *testing.T) {
 					return
 				}
 
+				if testCase.HTTPCheck != nil {
+					if err := runContainerWithHTTPCheck(t, imageName, testCase.Envs, testCase.HTTPCheck); err != nil {
+						t.Fatal(err)
+					}
+					return
+				}
+
 				if err := runContainerWithTimeout(t, imageName, testCase.ExpectedOutput, testCase.Envs, testCase.Platform); err != nil {
 					t.Fatal(err)
 				}
@@ -161,7 +176,7 @@ func runContainerWithTimeout(t *testing.T, imageName string, expectedOutputs []s
 	defer cancel()
 
 	// Generate a unique container name so we can reference it later for cleanup
-	containerName := fmt.Sprintf("railpack-test-%s", uuid.New().String())
+	containerName := uniqueContainerName(uuid.New().String())
 
 	// Build docker run command with environment variables
 	args := []string{"run", "--rm", "--name", containerName}