ingress/pkg/middlewares/staticfiles/static_files.go

package staticfiles

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"html/template"
	"io"
	"io/fs"
	"mime"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/hanzoai/ingress/pkg/config/dynamic"
	"github.com/hanzoai/ingress/pkg/middlewares"
)

const typeName = "StaticFiles"

// Extensions Go's built-in table does not carry. It falls back to the system
// mime.types, which a distroless image does not ship, so on a developer's box
// these resolve and in production they come back empty — and this plane sets
// `nosniff`, so an empty answer means the browser is told the type is unknown
// and forbidden to guess. Markdown served that way is offered as a download
// instead of a page; hanzoskills.com publishes a .md twin of every document.
func init() {
	for ext, typ := range map[string]string{
		".md":          "text/markdown; charset=utf-8",
		".markdown":    "text/markdown; charset=utf-8",
		".woff":        "font/woff",
		".woff2":       "font/woff2",
		".ttf":         "font/ttf",
		".otf":         "font/otf",
		".map":         "application/json",
		".webmanifest": "application/manifest+json",
	} {
		// AddExtensionType only returns an error on a malformed extension.
		_ = mime.AddExtensionType(ext, typ)
	}
}

type dirEntry struct {
	Name    string
	Size    int64
	Mode    os.FileMode
	ModTime time.Time
	IsDir   bool
}

type staticFiles struct {
	root             http.FileSystem
	enableDirListing bool
	indexFiles       []string
	spaMode          bool
	spaIndex         string
	errorPage404     string
	event            []byte
	eventSum         string
	cacheControl     map[string]string
	cacheTag         string
	name             string
	next             http.Handler
}

// siteTag is the edge cache-tag of the Sites plane site an object root serves:
// `site-<org>-<slug>` for a root `s3://<bucket>/<org>/<slug>`, the tag cloud
// purges when that site publishes (hanzo-inc/cloud apps/sites CacheTag), and ""
// for any other root.
//
// Without it the purge flushes nothing: a page the edge holds carries no tag,
// so a publish waited out the page's s-maxage at the edge after every object
// was already in the store.
func siteTag(prefix string) string {
	org, slug, ok := strings.Cut(prefix, "/")
	if !ok || org == "" || slug == "" || strings.Contains(slug, "/") {
		return ""
	}
	return "site-" + org + "-" + slug
}

// New creates a new static files middleware.
func New(ctx context.Context, next http.Handler, config dynamic.StaticFiles, name string) (http.Handler, error) {
	logger := middlewares.GetLogger(ctx, name, typeName)
	logger.Debug().Msg("Creating middleware")

	root := config.Root
	if root == "" {
		root = "."
	}

	// The origin is a union: a local directory or an object store, selected by
	// the Root form. An "s3://bucket/prefix" Root serves from the object store;
	// anything else is a local path. Both are served through the same handler
	// via http.FileSystem, so only the origin differs.
	var rootFS http.FileSystem
	var tag string
	if _, prefix, ok := parseObjectRoot(root); ok {
		tag = siteTag(prefix)
		fsys, err := newObjectFS(root)
		if err != nil {
			return nil, err
		}
		rootFS = fsys
	} else {
		absRoot, err := filepath.Abs(root)
		if err != nil {
			return nil, fmt.Errorf("invalid root path: %w", err)
		}
		if _, err := os.Stat(absRoot); os.IsNotExist(err) {
			if err := os.MkdirAll(absRoot, 0755); err != nil {
				return nil, fmt.Errorf("failed to create root directory: %w", err)
			}
		}
		rootFS = http.Dir(absRoot)
	}

	indexFiles := config.IndexFiles
	if len(indexFiles) == 0 {
		indexFiles = []string{"index.html", "index.htm"}
	}

	spaIndex := config.SPAIndex
	if spaIndex == "" {
		spaIndex = "index.html"
	}

	// A bad key is an operator's mistake on one site, and refusing the build would
	// take the whole host down with it; the pages go out without the tag.
	event, eventSum := eventOf(config.Tag)
	if config.Tag != "" && event == nil {
		logger.Error().Msg("tag is not a publishable key (pk-...); serving pages without the event tag")
	}

	return &staticFiles{
		root:             rootFS,
		enableDirListing: config.EnableDirectoryListing,
		indexFiles:       indexFiles,
		spaMode:          config.SPAMode,
		spaIndex:         spaIndex,
		errorPage404:     config.ErrorPage404,
		event:            event,
		eventSum:         eventSum,
		cacheControl:     config.CacheControl,
		cacheTag:         tag,
		name:             name,
		next:             next,
	}, nil
}

func (h *staticFiles) GetTracingInformation() (string, string) {
	return h.name, typeName
}

// contextFS is an http.FileSystem whose opens can be bound to a request context.
type contextFS interface {
	openCtx(ctx context.Context, name string) (http.File, error)
}

// open resolves a path against the root, binding object-store reads to ctx when
// the origin supports it so a client disconnect cancels the upstream read. The
// local origin (http.Dir) has no context to bind and is opened directly.
func (h *staticFiles) open(ctx context.Context, name string) (http.File, error) {
	if cf, ok := h.root.(contextFS); ok {
		return cf.openCtx(ctx, name)
	}
	return h.root.Open(name)
}

// kind is what a request names, read from the media type of its path: the same
// table that types a served file, and the one cloud's Sites plane reads
// (apps/sites contentType). A dot in a route segment names no type, so
// "/models/glm-5.2" is a page, not a file with the extension ".2".
type kind int

const (
	page  kind = iota // HTML, or a path no media type claims
	data              // JSON
	asset             // any other typed file: script, style, image, font
)

func kindOf(p string) kind {
	t, _, _ := strings.Cut(mime.TypeByExtension(path.Ext(p)), ";")
	switch t = strings.TrimSpace(t); {
	case t == "" || t == "text/html":
		return page
	case t == "application/json" || t == "text/json" || strings.HasSuffix(t, "+json"):
		return data
	default:
		return asset
	}
}

// pageCandidates returns the files a static export may have written for an
// extensionless request, in the order a static host should try them.
//
// A prerendered site has no router: "/pricing" is a FILE, and which file depends
// on a build flag. Next.js `output: export` writes "pricing.html" by default and
// "pricing/index.html" under `trailingSlash: true`; Hugo, Astro and Eleventy all
// emit one of the same two shapes. Serving only the literal key 404s every route
// of every such site but the root — the whole site, from an origin that holds it
// complete.
//
// Both shapes are tried because both are ordinary output of the same generators,
// and this is the same ladder cloud's own site plane resolves (apps/sites
// streamSite: the key, then .html, then /index.html), so a bundle served here and
// the same bundle served there answer identically.
//
// The root is excluded: it is a directory, and the indexFiles loop owns it.
func pageCandidates(upath string) []string {
	trimmed := strings.TrimSuffix(upath, "/")
	if trimmed == "" {
		return nil
	}
	return []string{trimmed + ".html", path.Join(trimmed, "index.html")}
}

// openRegular opens name and hands it back only if it resolved to a regular
// file, so a caller probing candidates can never mistake a directory for a page.
func (h *staticFiles) openRegular(ctx context.Context, name string) (http.File, fs.FileInfo, bool) {
	f, err := h.open(ctx, name)
	if err != nil {
		return nil, nil, false
	}

	d, err := f.Stat()
	if err != nil || d.IsDir() {
		f.Close()
		return nil, nil, false
	}

	return f, d, true
}

// serveOpen writes an already-resolved file: cache headers, content type from
// the extension, then the body through ServeContent so Range and conditional
// requests keep working.
func (h *staticFiles) serveOpen(w http.ResponseWriter, r *http.Request, f http.File, d fs.FileInfo) {
	spliced, err := h.tagged(f, d)
	if err != nil {
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}

	h.setCacheHeaders(w, r.URL.Path, d)
	// Pages only: they are what a publish replaces in place. A content-hashed
	// asset never changes under its name, so purging it would only send every
	// reader's next request for it back to the origin.
	if h.cacheTag != "" && kindOf(d.Name()) == page {
		w.Header().Set("Cache-Tag", h.cacheTag)
	}

	if contentType := mime.TypeByExtension(filepath.Ext(d.Name())); contentType != "" {
		w.Header().Set("Content-Type", contentType)
	}

	if spliced != nil {
		// The page is not the file, so neither is its validator. Its bytes are the
		// file's version and the tag's, so an ETag naming both stays strong, and
		// If-None-Match, Range and If-Range answer from the page without reading
		// it. The file's time cannot say when the tag changed, so no date is sent
		// or compared. The encoded twin holds the file's bytes; the page goes as
		// identity.
		if tag := w.Header().Get("ETag"); tag != "" {
			w.Header().Set("ETag", strings.TrimSuffix(tag, `"`)+"-"+h.eventSum+`"`)
		}
		w.Header().Del("Last-Modified")
		http.ServeContent(w, r, d.Name(), time.Time{}, spliced)
		return
	}

	if e, ok := f.(encoded); ok {
		if br := e.brotli(); br != nil {
			// Both representations name what they vary on, so no cache between
			// here and the reader hands one to a client that asked for the other.
			w.Header().Add("Vary", "Accept-Encoding")
			// A range names bytes of the identity representation; it is answered
			// from that one.
			if r.Header.Get("Range") == "" && acceptsBrotli(r.Header.Get("Accept-Encoding")) {
				// A strong validator names one representation (RFC 9110 8.8.3).
				if tag := w.Header().Get("ETag"); tag != "" {
					w.Header().Set("ETag", strings.TrimSuffix(tag, `"`)+`-br"`)
				}
				http.ServeContent(brotliWriter{w}, r, d.Name(), d.ModTime(), bytes.NewReader(br))
				return
			}
		}
	}

	http.ServeContent(w, r, d.Name(), d.ModTime(), f.(io.ReadSeeker))
}

// encoded is a file that also holds its bytes brotli-encoded, as the object
// cache's files do.
type encoded interface{ brotli() []byte }

// brotliWriter labels a 200 as the brotli representation. ServeContent is left
// to set Content-Length (it does only while Content-Encoding is unset) and to
// answer 304 and 412 as itself, so the label goes on at the status line, on
// success only. Ranges are answered from the identity bytes, so this one offers
// none.
type brotliWriter struct{ http.ResponseWriter }

func (w brotliWriter) WriteHeader(code int) {
	if code == http.StatusOK {
		w.Header().Set("Content-Encoding", "br")
		w.Header().Del("Accept-Ranges")
	}
	w.ResponseWriter.WriteHeader(code)
}

// acceptsBrotli reports whether an Accept-Encoding value names br with a weight
// above zero.
func acceptsBrotli(header string) bool {
	for _, part := range strings.Split(header, ",") {
		coding, params, _ := strings.Cut(part, ";")
		if !strings.EqualFold(strings.TrimSpace(coding), "br") {
			continue
		}
		if q, ok := strings.CutPrefix(strings.ToLower(strings.TrimSpace(params)), "q="); ok {
			w, err := strconv.ParseFloat(strings.TrimSpace(q), 64)
			return err == nil && w > 0
		}
		return true
	}
	return false
}

func (h *staticFiles) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	upath := r.URL.Path
	if !strings.HasPrefix(upath, "/") {
		upath = "/" + upath
	}

	f, err := h.open(r.Context(), upath)
	if err != nil {
		if os.IsNotExist(err) {
			// A prerendered route before any fallback policy: the file exists,
			// it is just named "pricing.html" rather than "pricing".
			//
			// Not gated on kind. The ladder asks for ONE exact file per
			// candidate and never falls back to a shell, so a missing chunk looks
			// for "nope.js.html" and "nope.js/index.html", finds neither, and
			// reaches miss below.
			for _, candidate := range pageCandidates(upath) {
				if cf, cd, ok := h.openRegular(r.Context(), candidate); ok {
					defer cf.Close()
					h.serveOpen(w, r, cf, cd)
					return
				}
			}

			if h.spaMode && kindOf(upath) == page {
				h.serveFile(w, r, h.spaIndex)
				return
			}
			h.miss(w, r, upath)
			return
		}
		// An object-store outage is an availability failure, not authorization:
		// answer 502, and keep access-denied / local errors as 403.
		if errors.Is(err, errObjectStoreUnavailable) {
			http.Error(w, "Bad Gateway", http.StatusBadGateway)
			return
		}
		http.Error(w, "Forbidden", http.StatusForbidden)
		return
	}
	defer f.Close()

	d, err := f.Stat()
	if err != nil {
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}

	if d.IsDir() {
		url := r.URL.Path
		if len(url) == 0 || url[len(url)-1] != '/' {
			localRedirect(w, r, url+"/")
			return
		}

		// SERVE the index, do not redirect to it. Redirecting made "/" answer
		// 301 -> "/index.html", which moves every site's canonical URL to a
		// path no one links to, costs a round trip on the first request, and is
		// not what any static host does. The bytes are the index's either way.
		for _, index := range h.indexFiles {
			indexPath := path.Join(upath, index)
			if indexFile, indexInfo, ok := h.openRegular(r.Context(), indexPath); ok {
				defer indexFile.Close()
				h.serveOpen(w, r, indexFile, indexInfo)
				return
			}
		}

		// A directory with no index may still have a page of its own one level
		// up ("/blog/" served by "blog.html"), which is what an export writes
		// for a route that also has children.
		for _, candidate := range pageCandidates(upath) {
			if cf, cd, ok := h.openRegular(r.Context(), candidate); ok {
				defer cf.Close()
				h.serveOpen(w, r, cf, cd)
				return
			}
		}

		if !h.enableDirListing {
			h.miss(w, r, upath)
			return
		}

		h.serveDirectoryListing(w, r, f)
		return
	}

	h.serveOpen(w, r, f, d)
}

func (h *staticFiles) serveDirectoryListing(w http.ResponseWriter, r *http.Request, f http.File) {
	dirs, err := f.Readdir(-1)
	if err != nil {
		http.Error(w, "Error reading directory", http.StatusInternalServerError)
		return
	}

	sort.Slice(dirs, func(i, j int) bool {
		if dirs[i].IsDir() && !dirs[j].IsDir() {
			return true
		}
		if !dirs[i].IsDir() && dirs[j].IsDir() {
			return false
		}
		return dirs[i].Name() < dirs[j].Name()
	})

	entries := make([]dirEntry, len(dirs))
	for i, entry := range dirs {
		entries[i] = dirEntry{
			Name:    entry.Name(),
			Size:    entry.Size(),
			Mode:    entry.Mode(),
			ModTime: entry.ModTime(),
			IsDir:   entry.IsDir(),
		}
	}

	w.Header().Set("Content-Type", "text/html; charset=utf-8")

	tmpl := template.Must(template.New("dirlist").Parse(dirListingTemplate))

	data := struct {
		Path  string
		Files []dirEntry
	}{
		Path:  r.URL.Path,
		Files: entries,
	}

	if err = tmpl.Execute(w, data); err != nil {
		http.Error(w, "Error rendering directory listing", http.StatusInternalServerError)
	}
}

func (h *staticFiles) setCacheHeaders(w http.ResponseWriter, name string, d fs.FileInfo) {
	ext := filepath.Ext(d.Name())

	switch {
	case cacheControlHas(h.cacheControl, ext):
		w.Header().Set("Cache-Control", h.cacheControl[ext])
	case strings.Contains(name, "/_next/static/") && kindOf(name) != page:
		// Next names every file it writes here by its content, so a new build is
		// a new URL and this one never changes, whatever its extension.
		w.Header().Set("Cache-Control", "public, max-age=31536000, immutable")
	case cacheControlHas(h.cacheControl, "*"):
		w.Header().Set("Cache-Control", h.cacheControl["*"])
	case ext == ".html" || ext == ".htm":
		// The shell references content-hashed assets; served stale it would pin
		// clients to an old asset graph, so a browser revalidates every time
		// (cheap: the ETag / Last-Modified below answers 304 when unchanged). A
		// shared cache may hold it for a minute. A Sites plane page carries its
		// site's cache-tag (serveOpen), which cloud purges when the site
		// publishes; the minute bounds a page whose purge did not arrive.
		w.Header().Set("Cache-Control", htmlCacheControl)
	default:
		w.Header().Set("Cache-Control", "max-age=86400")
	}

	// This plane serves third-party bundles; never let the browser MIME-sniff.
	w.Header().Set("X-Content-Type-Options", "nosniff")

	// A strong validator lets http.ServeContent answer conditional requests
	// (If-None-Match). Only the object-store origin carries one; the local
	// origin is unchanged.
	if e, ok := d.(etagger); ok {
		if tag := e.ETag(); tag != "" {
			if !strings.HasPrefix(tag, `"`) {
				tag = `"` + tag + `"`
			}
			w.Header().Set("ETag", tag)
		}
	}

	w.Header().Set("Last-Modified", d.ModTime().UTC().Format(http.TimeFormat))
}

// htmlCacheControl is a page's policy: revalidated by browsers, held for a
// minute by a shared cache that is told to keep pages (Cloudflare holds none
// unless a cache rule names the host).
const htmlCacheControl = "public, max-age=0, s-maxage=60"

func cacheControlHas(m map[string]string, key string) bool {
	_, ok := m[key]
	return ok
}

// miss answers a path the root holds nothing for, by what the path names: a page
// gets the site's 404 page, JSON gets a JSON body, anything else a bare 404. Every
// answer is status 404, so a missing chunk is never markup and a missing page is
// never a success. Cloud's Sites plane answers the same way (apps/sites notFound).
//
// Every 404 is no-store. A 404 that says nothing is given a browser lifetime by
// the CDN, and a chunk asked for while a publish is still uploading would then
// stay missing in that browser for hours after it lands.
func (h *staticFiles) miss(w http.ResponseWriter, r *http.Request, upath string) {
	w.Header().Set("Cache-Control", "no-store")
	switch kindOf(upath) {
	case page:
		if h.errorPage404 != "" && h.notFoundPage(w, r) {
			return
		}
	case data:
		w.Header().Set("Content-Type", "application/json; charset=utf-8")
		w.Header().Set("X-Content-Type-Options", "nosniff")
		w.WriteHeader(http.StatusNotFound)
		_ = json.NewEncoder(w).Encode(map[string]string{"error": "not found", "path": upath})
		return
	}
	http.NotFound(w, r)
}

// notFoundPage writes the configured 404 page with status 404, or reports false
// when the root holds no such file. No validators: a 404 is not revalidated.
func (h *staticFiles) notFoundPage(w http.ResponseWriter, r *http.Request) bool {
	f, d, ok := h.openRegular(r.Context(), "/"+strings.TrimPrefix(h.errorPage404, "/"))
	if !ok {
		return false
	}
	defer f.Close()

	spliced, err := h.tagged(f, d)
	if err != nil {
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return true
	}
	body, size := io.Reader(f), d.Size()
	if spliced != nil {
		body, size = spliced, spliced.len()
	}

	if t := mime.TypeByExtension(filepath.Ext(d.Name())); t != "" {
		w.Header().Set("Content-Type", t)
	}
	w.Header().Set("Cache-Control", "no-store")
	w.Header().Set("X-Content-Type-Options", "nosniff")
	w.Header().Set("Content-Length", strconv.FormatInt(size, 10))
	w.WriteHeader(http.StatusNotFound)
	if r.Method != http.MethodHead {
		_, _ = io.Copy(w, body)
	}
	return true
}

// serveFile serves the SPA index named relative to the root, through the same
// origin as everything else — local disk or object store.
func (h *staticFiles) serveFile(w http.ResponseWriter, r *http.Request, name string) {
	f, err := h.open(r.Context(), "/"+strings.TrimPrefix(name, "/"))
	if err != nil {
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}
	defer f.Close()

	d, err := f.Stat()
	if err != nil {
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}

	h.serveOpen(w, r, f, d)
}

func localRedirect(w http.ResponseWriter, r *http.Request, newPath string) {
	if q := r.URL.RawQuery; q != "" {
		newPath += "?" + q
	}
	w.Header().Set("Location", newPath)
	w.WriteHeader(http.StatusMovedPermanently)
}

const dirListingTemplate = `<!DOCTYPE html>
<html>
<head>
    <meta charset="utf-8">
    <title>Index of {{.Path}}</title>
    <style>
        body { font-family: sans-serif; margin: 2em; }
        table { border-collapse: collapse; width: 100%; }
        th, td { text-align: left; padding: 8px; }
        tr:nth-child(even) { background-color: #f2f2f2; }
        th { background-color: #333; color: white; }
        a { text-decoration: none; }
        a:hover { text-decoration: underline; }
    </style>
</head>
<body>
    <h1>Index of {{.Path}}</h1>
    <table>
        <tr>
            <th>Name</th>
            <th>Size</th>
            <th>Modified</th>
        </tr>
        {{if ne .Path "/"}}
        <tr>
            <td><a href="../">../</a></td>
            <td>-</td>
            <td>-</td>
        </tr>
        {{end}}
        {{range .Files}}
        <tr>
            <td><a href="{{.Name}}{{if .IsDir}}/{{end}}">{{.Name}}{{if .IsDir}}/{{end}}</a></td>
            <td>{{if .IsDir}}-{{else}}{{.Size}} bytes{{end}}</td>
            <td>{{.ModTime.Format "2006-01-02 15:04:05"}}</td>
        </tr>
        {{end}}
    </table>
</body>
</html>`