mirror of
https://github.com/hanzoai/insights.git
synced 2026-10-30 16:15:31 +00:00
The rename runs on the way in and nothing checked it afterwards, so paths and content drifted apart in three places, and each one broke something quietly. The script language had three names at once. Files on disk carried `.iql`/`.iqle`, the source matchers said `.script`, and the bytecode matchers said `.hoge`, so the test harness globbed zero files and the STL compiler regenerated an empty table. It is `script` now, in one place: `.script` source, `.scriptc` bytecode, `bin/script` to run one and `bin/scriptc` to compile one. 190 TypeScript specifiers named files that had been renamed away, which is the whole CDP consumer graph -- the event worker, not the web boot. `--check` only ever read Python imports; it reads both module systems now, and repoints both. `-hog-` became `-mascot-` in paths while content said `script`, so the code review product's skill directories stopped matching the prefix that discovers them and it ran with no perspectives at all. PATH_REGEX is CONTENT_REGEX -- the same list, not a copy -- so a rule can no longer land in one and miss the other, and the four disagreeing `hog` path rules collapse into the one word rule content already had. `HOGLI_*` becomes `INSIGHTSCLI_*` at every reader and writer, with no alias: an env var read under two names is two names. The seven casings of `hogql` and five of `hogvm` collapse into one rule per prefix casing, which leaves the suffix alone because `QL` is SQL's and `VM` is a virtual machine's. `--check` now refuses every retired spelling, in content and in paths, and draws the line the rules draw: `insights_hogfunction` is a Postgres table so it is left alone, and a migration filename is its identity in `django_migrations` so it is frozen rather than failed. Third-party names we cannot rename are reported as pending, never exempted.
708 lines
31 KiB
Python
708 lines
31 KiB
Python
"""Generate process manager configuration from resolved environment.
|
|
|
|
This module provides an abstract generator interface and an mprocs implementation.
|
|
The system is designed to be process-manager agnostic - mprocs is just one output format.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import re
|
|
import sys
|
|
import shlex
|
|
import shutil
|
|
from abc import ABC, abstractmethod
|
|
from collections.abc import Callable
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import TYPE_CHECKING, Any
|
|
|
|
import yaml
|
|
from pydantic import BaseModel
|
|
|
|
from .registry import create_mprocs_registry, find_repo_file
|
|
|
|
if TYPE_CHECKING:
|
|
from .registry import ProcessRegistry
|
|
from .resolver import ResolvedEnvironment
|
|
|
|
|
|
def _dev_sandbox_enabled() -> bool:
|
|
"""Dev filesystem sandbox is on by default; opt out with INSIGHTS_DEV_SANDBOX=0 (strict)."""
|
|
return os.getenv("INSIGHTS_DEV_SANDBOX") != "0"
|
|
|
|
|
|
class DevenvConfig(BaseModel):
|
|
"""Source configuration for dev environment (what the user selected).
|
|
|
|
This gets embedded in the generated mprocs.yaml under _insights key,
|
|
allowing re-resolution when intent-map changes.
|
|
"""
|
|
|
|
intents: list[str] = []
|
|
include_units: list[str] = []
|
|
exclude_units: list[str] = []
|
|
skip_autostart: list[str] = []
|
|
enable_autostart: list[str] = []
|
|
log_to_files: bool = False
|
|
|
|
|
|
# Docker compose command building
|
|
|
|
|
|
def _get_docker_compose_base() -> str:
|
|
# Pin the project name so worktrees share one dev stack instead of each
|
|
# defaulting to its own directory name.
|
|
project_name = os.environ.get("COMPOSE_PROJECT_NAME") or "insights"
|
|
return f"docker compose -p {shlex.quote(project_name)} -f docker-compose.dev.yml -f docker-compose.profiles.yml"
|
|
|
|
|
|
def build_docker_compose_command(profiles: list[str], action: str = "up -d") -> str:
|
|
"""Build docker compose command with profile flags.
|
|
|
|
Args:
|
|
profiles: List of docker compose profiles to activate
|
|
action: The docker compose action (e.g., "up -d", "down", "down -v")
|
|
|
|
Returns:
|
|
Complete docker compose command string
|
|
"""
|
|
base = _get_docker_compose_base()
|
|
if profiles:
|
|
profile_flags = " ".join(f"--profile {p}" for p in profiles)
|
|
return f"{base} {profile_flags} {action}"
|
|
return f"{base} {action}"
|
|
|
|
|
|
# The proc's ready_pattern in bin/mprocs.yaml and bin/mprocs-e2e.yaml must match
|
|
# this literal, or mprocs waits forever for a signal that never comes.
|
|
_READY_MARKER = "docker-compose ready"
|
|
|
|
|
|
def _build_docker_compose_shell(profiles: list[str]) -> str:
|
|
"""Sandboxes have no docker CLI, so the shell short-circuits instead of failing on `docker compose`."""
|
|
up_cmd = build_docker_compose_command(profiles, "up --pull always -d")
|
|
logs_cmd = build_docker_compose_command(profiles, "logs --tail=100 -f")
|
|
sandbox_cmd = (
|
|
f'echo "Sandbox: infra managed by docker compose externally" && echo "{_READY_MARKER}" && sleep infinity'
|
|
)
|
|
# The `exit 1` is load-bearing: without it the `{ ...; }` group exits 0, the `||`
|
|
# is satisfied, and the ready marker fires against a stack that never came up.
|
|
fail_cmd = (
|
|
'echo "docker compose up failed (see the error above). A common cause: another stack or process '
|
|
'holding a required port (see the pre-flight output from bin/start). Inspect with: docker ps --all"; '
|
|
"exit 1"
|
|
)
|
|
dev_cmd = f'{up_cmd} || {{ {fail_cmd}; }} && echo "{_READY_MARKER}" && {logs_cmd}'
|
|
return f'if [ "${{INSIGHTS_SANDBOX:-}}" = "1" ]; then {sandbox_cmd}; else {dev_cmd}; fi'
|
|
|
|
|
|
# bin/mprocs.yaml has no intent resolution: every proc in it autostarts, so it
|
|
# needs every profile those procs depend on (temporal-worker and
|
|
# recording-rasterizer wait on `temporal`; personinsights-etcd-init docker-execs
|
|
# into `etcd`; session replay needs `replay`) — this reproduces master's
|
|
# unprofiled `docker-compose.dev.yml` service set exactly.
|
|
_STATIC_MPROCS_PROFILES = ["dev_tools", "etcd", "localstack", "observability", "replay", "temporal"]
|
|
|
|
|
|
def build_static_docker_compose_shell() -> str:
|
|
"""bin/mprocs.yaml's docker-compose shell — every proc autostarts, so every legacy profile is active."""
|
|
return _build_docker_compose_shell(_STATIC_MPROCS_PROFILES)
|
|
|
|
|
|
def build_e2e_docker_compose_shell() -> str:
|
|
"""bin/mprocs-e2e.yaml's docker-compose shell: never sandboxed, plus an explicit
|
|
`up` for personinsights-replica/router under the ingestion profile."""
|
|
up_cmd = build_docker_compose_command([], "up --pull always -d")
|
|
personinsights_cmd = build_docker_compose_command(
|
|
["ingestion"], "up -d personinsights-replica personinsights-router"
|
|
)
|
|
logs_cmd = build_docker_compose_command([], "logs --tail=100 -f")
|
|
return f'{up_cmd} && {personinsights_cmd} && echo "{_READY_MARKER}" && {logs_cmd}'
|
|
|
|
|
|
# Matches the docker-compose proc's `shell:` scalar in a checked-in mprocs YAML
|
|
# file (bin/mprocs.yaml or bin/mprocs-e2e.yaml). Anchored on the exact
|
|
# indentation/key ordering of that block rather than parsed as YAML, so
|
|
# regeneration can rewrite just this one line and leave the rest of the
|
|
# hand-maintained file (comments, formatting, other procs) untouched.
|
|
_DOCKER_COMPOSE_SHELL_LINE_RE = re.compile(r"(?P<prefix>^ docker-compose:\n shell: )'[^\n]*'$", re.M)
|
|
|
|
|
|
def is_mprocs_shell_up_to_date(mprocs_path: Path, expected: str) -> bool:
|
|
"""Whether mprocs_path's docker-compose proc shell already matches expected."""
|
|
current_shell = create_mprocs_registry(mprocs_path).get_processes()["docker-compose"].shell
|
|
return current_shell == expected
|
|
|
|
|
|
def regenerate_mprocs_shell(mprocs_path: Path, expected: str) -> bool:
|
|
"""Sync mprocs_path's docker-compose proc shell with expected.
|
|
|
|
Returns True if the file was rewritten, False if it was already up to date.
|
|
"""
|
|
if is_mprocs_shell_up_to_date(mprocs_path, expected):
|
|
return False
|
|
# The entry is single-quoted YAML; a literal `'` would need escaping this
|
|
# substitution doesn't do.
|
|
if "'" in expected:
|
|
raise ValueError("Generated docker-compose shell contains a single quote; update the YAML quoting logic")
|
|
text = mprocs_path.read_text()
|
|
new_text, replaced = _DOCKER_COMPOSE_SHELL_LINE_RE.subn(lambda m: f"{m.group('prefix')}'{expected}'", text, count=1)
|
|
if not replaced:
|
|
raise ValueError(
|
|
f"{mprocs_path}: no docker-compose `shell:` line matched _DOCKER_COMPOSE_SHELL_LINE_RE. "
|
|
"The proc's key order, indentation, or quoting changed; update the pattern."
|
|
)
|
|
mprocs_path.write_text(new_text)
|
|
return True
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class TrackedMprocsFile:
|
|
"""A checked-in mprocs YAML file whose docker-compose proc shell is kept in
|
|
sync with a Python-built string, by `insightscli dev:regenerate-mprocs` and the
|
|
ci_preflight "mprocs" check."""
|
|
|
|
name: str # repo-relative path, for display and as a preflight trigger glob
|
|
path: Path
|
|
build_shell: Callable[[], str]
|
|
|
|
|
|
TRACKED_MPROCS_FILES: list[TrackedMprocsFile] = [
|
|
TrackedMprocsFile("bin/mprocs.yaml", find_repo_file("bin/mprocs.yaml"), build_static_docker_compose_shell),
|
|
TrackedMprocsFile("bin/mprocs-e2e.yaml", find_repo_file("bin/mprocs-e2e.yaml"), build_e2e_docker_compose_shell),
|
|
]
|
|
|
|
|
|
class ConfigGenerator(ABC):
|
|
"""Abstract generator for process manager configurations."""
|
|
|
|
@abstractmethod
|
|
def generate(self, resolved: ResolvedEnvironment, source_config: DevenvConfig | None = None) -> Any:
|
|
"""Generate configuration for resolved environment."""
|
|
...
|
|
|
|
@abstractmethod
|
|
def save(self, config: Any, output_path: Path) -> Path:
|
|
"""Save configuration to file."""
|
|
...
|
|
|
|
def generate_and_save(
|
|
self, resolved: ResolvedEnvironment, output_path: Path, source_config: DevenvConfig | None = None
|
|
) -> Path:
|
|
"""Generate and save configuration in one step."""
|
|
config = self.generate(resolved, source_config)
|
|
return self.save(config, output_path)
|
|
|
|
|
|
class MprocsConfig(BaseModel):
|
|
"""Represents an mprocs.yaml configuration."""
|
|
|
|
procs: dict[str, dict[str, Any]]
|
|
group_order: dict[str, list[str]] = {} # display order per grouping dimension
|
|
default_group: str = "" # grouping dimension the sidebar starts grouped by ("" = ungrouped)
|
|
mouse_scroll_speed: int = 1
|
|
scrollback: int = 10000
|
|
insights_config: DevenvConfig | None = None # embedded source config
|
|
|
|
def to_yaml_dict(self) -> dict[str, Any]:
|
|
"""Serialize to dict with _insights first for YAML output."""
|
|
result: dict[str, Any] = {}
|
|
if self.insights_config:
|
|
result["_insights"] = self.insights_config.model_dump(exclude_defaults=True)
|
|
result["procs"] = self.procs
|
|
if self.group_order:
|
|
result["group_order"] = self.group_order
|
|
if self.default_group:
|
|
result["default_group"] = self.default_group
|
|
result["mouse_scroll_speed"] = self.mouse_scroll_speed
|
|
result["scrollback"] = self.scrollback
|
|
return result
|
|
|
|
|
|
class MprocsGenerator(ConfigGenerator):
|
|
"""Generates mprocs.yaml configuration from resolved environment."""
|
|
|
|
def __init__(self, registry: ProcessRegistry):
|
|
"""Initialize generator.
|
|
|
|
Args:
|
|
registry: ProcessRegistry to get process configs from
|
|
"""
|
|
self.registry = registry
|
|
|
|
def generate(self, resolved: ResolvedEnvironment, source_config: DevenvConfig | None = None) -> MprocsConfig:
|
|
"""Generate mprocs configuration for resolved environment.
|
|
|
|
Args:
|
|
resolved: The resolved environment with required units
|
|
source_config: Optional source config to embed for re-resolution
|
|
|
|
Returns:
|
|
MprocsConfig with only the required processes
|
|
"""
|
|
procs: dict[str, dict[str, Any]] = {}
|
|
|
|
# Info process is always first
|
|
info_config = self.registry.get_process_config("info") or {}
|
|
procs["info"] = self._build_info_process(info_config, resolved)
|
|
|
|
# Iterate in original mprocs.yaml order to preserve ordering
|
|
for name in self.registry.get_processes():
|
|
proc_config = self.registry.get_process_config(name)
|
|
if not proc_config:
|
|
continue
|
|
|
|
# Include if: in resolved units, or autostart: false (manual start)
|
|
is_resolved = name in resolved.units
|
|
is_manual_start = proc_config.get("autostart") is False
|
|
|
|
if not is_resolved and not is_manual_start:
|
|
continue
|
|
|
|
# Copy config to avoid mutating registry's internal state
|
|
proc_config = proc_config.copy()
|
|
|
|
# Remove metadata fields - not mprocs config.
|
|
# (note that capability is kept, because it's used in groups)
|
|
proc_config.pop("ask_skip", None)
|
|
|
|
# Give procs without an explicit capability a synthetic one so they
|
|
# don't all fall into "Ungrouped" when the user groups by capability,
|
|
# as many important ones are in that category (e.g. backend, frontend).
|
|
if not proc_config.get("capability"):
|
|
if name in resolved.always_required:
|
|
proc_config["capability"] = "always_required"
|
|
elif is_manual_start:
|
|
proc_config["capability"] = "tools"
|
|
|
|
# Set autostart: false for skipped processes
|
|
if name in resolved.skip_autostart:
|
|
proc_config["autostart"] = False
|
|
|
|
# Enable autostart for processes user opted into (overrides source autostart: false)
|
|
if name in resolved.enable_autostart:
|
|
proc_config.pop("autostart", None)
|
|
|
|
# Add startup message for resolved units (not manual-start ones)
|
|
if is_resolved and name not in resolved.skip_autostart:
|
|
reason = resolved.get_unit_reason(name)
|
|
proc_config = self._add_startup_message(proc_config, name, reason)
|
|
|
|
# Special handling for docker-compose
|
|
if name == "docker-compose":
|
|
proc_config = self._generate_docker_compose_config(proc_config, resolved.get_docker_profiles_list())
|
|
|
|
# Special handling for nodejs - set capability groups based on resolved nodejs_* capabilities
|
|
if name == "nodejs":
|
|
proc_config = self._add_nodejs_capability_groups(proc_config, resolved)
|
|
|
|
# Special handling for backend/nodejs - wire up personinsights env vars when capability is active
|
|
if name == "backend":
|
|
proc_config = self._add_personinsights_env(proc_config, resolved)
|
|
if name == "nodejs":
|
|
proc_config = self._add_personinsights_env(proc_config, resolved)
|
|
|
|
# Special handling for temporal-worker - install uv groups when capabilities require them
|
|
if name == "temporal-worker":
|
|
proc_config = self._add_uv_groups(proc_config, resolved)
|
|
|
|
# Wrap Python/Node service commands in the dev sandbox when opted in
|
|
proc_config = self._add_sandbox_wrapper(proc_config, name)
|
|
|
|
# Add logging wrapper if enabled
|
|
if source_config and source_config.log_to_files:
|
|
proc_config = self._add_logging(proc_config, name)
|
|
|
|
procs[name] = proc_config
|
|
|
|
# Get global settings from registry
|
|
global_settings = self.registry.get_global_settings()
|
|
|
|
return MprocsConfig(
|
|
procs=procs,
|
|
group_order=global_settings.get("group_order", {}),
|
|
default_group=global_settings.get("default_group", ""),
|
|
mouse_scroll_speed=global_settings.get("mouse_scroll_speed", 1),
|
|
scrollback=global_settings.get("scrollback", 10000),
|
|
insights_config=source_config,
|
|
)
|
|
|
|
def _build_info_process(self, proc_config: dict[str, Any], resolved: ResolvedEnvironment) -> dict[str, Any]:
|
|
"""Update the info process config with a generated shell command.
|
|
|
|
News is fetched live from master so developers on older branches still
|
|
see current announcements. Falls back to the local copy when offline.
|
|
"""
|
|
process_count = len(resolved.units)
|
|
products = sorted(resolved.intents) if resolved.intents else ["(none)"]
|
|
|
|
# The banner runs on whatever terminal theme the developer picked, and a
|
|
# fixed grey that reads on one ground is invisible on the other. So
|
|
# emphasis is weight, not color: bold for the things you act on, the
|
|
# terminal's own foreground for body text, one mid grey for asides.
|
|
gray = r"\033[38;5;245m"
|
|
green = r"\033[32m"
|
|
bold = r"\033[1m"
|
|
reset = r"\033[0m"
|
|
|
|
# Reflect the *effective* dependency-sandbox state: on by default (opt out
|
|
# with INSIGHTS_DEV_SANDBOX=0) AND on macOS with sandbox-exec (the wrapper
|
|
# no-ops elsewhere). Mirrors the gate in _add_sandbox_wrapper, so the banner
|
|
# can't claim isolation the platform won't deliver.
|
|
sandbox_enabled = _dev_sandbox_enabled()
|
|
sandbox_supported = sys.platform == "darwin" and shutil.which("sandbox-exec") is not None
|
|
if sandbox_enabled and sandbox_supported:
|
|
sandbox_status = f"{green}on{reset}"
|
|
elif sandbox_enabled:
|
|
sandbox_status = f"{gray}off — unsupported on this platform{reset}"
|
|
else:
|
|
sandbox_status = f"{gray}off{reset} {gray}(INSIGHTS_DEV_SANDBOX=0; unset to re-enable){reset}"
|
|
|
|
news_url = "https://raw.githubusercontent.com/insights/insights/master/devenv/news.txt"
|
|
news_local = "devenv/news.txt"
|
|
|
|
shell = f"""\
|
|
echo ''
|
|
printf '{bold} Insights Dev Environment{reset}\\n'
|
|
printf '{gray} ─────────────────────────────────────{reset}\\n'
|
|
echo ''
|
|
_news=$(curl -sf --max-time 2 '{news_url}' 2>/dev/null || cat {news_local} 2>/dev/null || true)
|
|
if [ -n "$_news" ]; then
|
|
printf ' {bold}News:{reset}\\n'
|
|
printf '%s\\n' "$_news" | sed 's/^/ /'
|
|
echo ''
|
|
fi
|
|
printf ' {bold}Commands:{reset}\\n'
|
|
printf ' {bold}insightscli dev:setup{reset} Configure which services run\\n'
|
|
printf ' {bold}insightscli dev:explain{reset} Show why each service is running\\n'
|
|
echo ''
|
|
printf '{gray} ─────────────────────────────────────{reset}\\n'
|
|
printf ' {bold}Path:{reset} {gray}%s{reset}\\n' "${{REPOSITORY_ROOT:-$PWD}}"
|
|
printf ' {bold}Products:{reset} {", ".join(products)}\\n'
|
|
printf ' {bold}Processes:{reset} {process_count} active\\n'
|
|
if [ -n "${{_INSIGHTS_OP_RESOLVED:-}}" ]; then
|
|
printf ' {bold}Secrets:{reset} 1Password {gray}(op run){reset}\\n'
|
|
else
|
|
printf ' {bold}Secrets:{reset} {gray}local .env files{reset}\\n'
|
|
fi
|
|
printf ' {bold}Sandbox:{reset} {sandbox_status}\\n'
|
|
echo ''
|
|
printf ' {bold}Log in with:{reset} [email protected] - {bold}12345678{reset}\\n'
|
|
printf ' {gray}Run {reset}{bold}insightscli dev:setup{reset}{gray} to tailor this to your workflow.{reset}\\n'
|
|
"""
|
|
proc_config["shell"] = shell
|
|
return proc_config
|
|
|
|
def _add_startup_message(self, proc_config: dict[str, Any], process_name: str, reason: str) -> dict[str, Any]:
|
|
"""Add a startup message to a process config.
|
|
|
|
Args:
|
|
proc_config: The process configuration dict
|
|
process_name: Name of the process
|
|
reason: Why this process is starting (capability name or "always required")
|
|
|
|
Returns:
|
|
Modified process configuration with startup message
|
|
"""
|
|
original_shell = proc_config.get("shell", "")
|
|
if not original_shell:
|
|
return proc_config
|
|
|
|
# Create a friendly message with config hint
|
|
message = f"echo '▶ {process_name}: {reason} (configure via: insightscli dev:setup)' && "
|
|
|
|
proc_config["shell"] = message + original_shell
|
|
return proc_config
|
|
|
|
def _generate_docker_compose_config(self, proc_config: dict[str, Any], profiles: list[str]) -> dict[str, Any]:
|
|
"""Update docker-compose process config with profile flags.
|
|
|
|
Args:
|
|
proc_config: The existing process configuration dict
|
|
profiles: List of docker compose profiles to activate
|
|
|
|
Returns:
|
|
Modified process configuration dict
|
|
"""
|
|
profiles = sorted(profiles)
|
|
|
|
# Build the profile flags (may be empty for minimal stack)
|
|
if profiles:
|
|
message = (
|
|
f"echo '▶ docker-compose: profiles: {', '.join(profiles)} (configure via: insightscli dev:setup)' && "
|
|
)
|
|
else:
|
|
message = "echo '▶ docker-compose: core services only (configure via: insightscli dev:setup)' && "
|
|
|
|
proc_config["shell"] = message + _build_docker_compose_shell(profiles)
|
|
proc_config["ready_pattern"] = _READY_MARKER
|
|
return proc_config
|
|
|
|
def _add_nodejs_capability_groups(
|
|
self, proc_config: dict[str, Any], resolved: ResolvedEnvironment
|
|
) -> dict[str, Any]:
|
|
"""Add NODEJS_CAPABILITY_GROUPS env var based on resolved nodejs_* capabilities.
|
|
|
|
Strips 'nodejs_' prefix from capability names to get the group name.
|
|
e.g. nodejs_cdp -> cdp, nodejs_session_replay -> session_replay
|
|
"""
|
|
prefix = "nodejs_"
|
|
enabled_groups = [cap.removeprefix(prefix) for cap in resolved.capabilities if cap.startswith(prefix)]
|
|
|
|
# If no specific groups are enabled, don't set the env var (use default behavior)
|
|
if not enabled_groups:
|
|
return proc_config
|
|
|
|
# Build the env var value
|
|
groups_value = ",".join(enabled_groups)
|
|
|
|
# Prepend the env var export to the shell command
|
|
original_shell = proc_config.get("shell", "")
|
|
if original_shell:
|
|
proc_config["shell"] = f"export NODEJS_CAPABILITY_GROUPS='{groups_value}' && {original_shell}"
|
|
|
|
return proc_config
|
|
|
|
def _add_personinsights_env(self, proc_config: dict[str, Any], resolved: ResolvedEnvironment) -> dict[str, Any]:
|
|
"""Add PERSONFN_* env vars to backend when personinsights capability is active."""
|
|
if "personinsights" not in resolved.capabilities:
|
|
return proc_config
|
|
|
|
original_shell = proc_config.get("shell", "")
|
|
if original_shell:
|
|
env_exports = "export PERSONFN_ADDR='127.0.0.1:50052'"
|
|
proc_config["shell"] = f"{env_exports} && {original_shell}"
|
|
|
|
return proc_config
|
|
|
|
def _add_uv_groups(self, proc_config: dict[str, Any], resolved: ResolvedEnvironment) -> dict[str, Any]:
|
|
"""Prepend uv sync --group and model download commands when uv_groups are resolved.
|
|
|
|
Currently handles the sentiment group (installs torch/optimum and downloads the ONNX model).
|
|
"""
|
|
if not resolved.uv_groups:
|
|
return proc_config
|
|
|
|
original_shell = proc_config.get("shell", "")
|
|
if not original_shell:
|
|
return proc_config
|
|
|
|
# Build uv sync command with all resolved groups
|
|
group_flags = " ".join(f"--group {g}" for g in sorted(resolved.uv_groups))
|
|
prefix = f"uv sync {group_flags} --inexact"
|
|
|
|
# Add model download for sentiment group
|
|
if "sentiment" in resolved.uv_groups:
|
|
prefix += " && uv run --group sentiment bin/download-sentiment-model"
|
|
|
|
proc_config["shell"] = f"{prefix} && {original_shell}"
|
|
return proc_config
|
|
|
|
# Trusted infra-wait scripts that need the docker daemon socket. They run
|
|
# OUTSIDE the sandbox (the socket is denied inside it — reaching the daemon is
|
|
# a full escape via `docker run -v $HOME:/host`). They contain no dependency
|
|
# code, so running them unsandboxed doesn't widen the untrusted attack surface.
|
|
_SANDBOX_DOCKER_GATES = ("bin/wait-for-docker", "bin/wait-for-postgres-tables")
|
|
|
|
# Trusted helpers peeled out to run OUTSIDE the sandbox for a reason other than
|
|
# the docker socket. bin/dev-open-when-ready backgrounds a poller and opens the
|
|
# browser once a server is up — the OS browser-open path (open → LaunchServices)
|
|
# reads $HOME, which the sandbox denies. It self-backgrounds and returns 0
|
|
# immediately, so hoisting it to the front is safe. It takes a fixed URL from the
|
|
# proc def (never dependency input) and lives in $PROJECT/bin, which the profile
|
|
# write-denies, so a compromised dep can't tamper with it.
|
|
_SANDBOX_UNSANDBOXED_PREFIXES = ("bin/dev-open-when-ready",)
|
|
|
|
# Procs excluded from the sandbox by default: they need the docker socket the
|
|
# profile denies (e.g. temporal-worker running Insights Desktop tasks with
|
|
# SANDBOX_PROVIDER=docker). INSIGHTS_DEV_SANDBOX_EXCLUDE adds more on top.
|
|
_SANDBOX_DEFAULT_EXCLUDES = frozenset({"temporal-worker"})
|
|
|
|
def _add_sandbox_wrapper(self, proc_config: dict[str, Any], name: str = "") -> dict[str, Any]:
|
|
"""Wrap a service command in bin/dev-sandbox (macOS Seatbelt) by default.
|
|
|
|
On by default; opt out globally with INSIGHTS_DEV_SANDBOX=0 (e.g. in
|
|
.env.local). A proc is wrapped only if it declares ``sandbox: true`` in the
|
|
registry — an explicit, reviewable per-proc boundary rather than one inferred
|
|
from the cosmetic ``groups.tech`` field (which is optional, so a forgotten
|
|
annotation silently dropped a service out of the sandbox). Rust/Docker/migration
|
|
procs simply omit the flag. The sandbox restricts reads to the repo + toolchain
|
|
caches so a malicious dependency can't read credentials (~/.ssh, ~/.aws, ...)
|
|
and denies the docker/agent sockets so it can't escape via a container mount.
|
|
|
|
Some procs are excluded by default (``_SANDBOX_DEFAULT_EXCLUDES``) because
|
|
they need the docker socket the profile denies, e.g. temporal-worker running
|
|
Insights Desktop tasks with SANDBOX_PROVIDER=docker. INSIGHTS_DEV_SANDBOX_EXCLUDE
|
|
(comma-separated proc names) opts additional procs back out. Excluded procs
|
|
run fully unsandboxed, so the dependency-isolation guarantee no longer covers
|
|
them.
|
|
|
|
The docker-gate preamble (bin/wait-for-docker) is peeled out to run
|
|
unsandboxed, since it needs the very socket the sandbox denies; the actual
|
|
server (which reaches infra over TCP) is what gets sandboxed. bin/dev-sandbox
|
|
is a no-op passthrough on non-macOS, so the generated config stays portable.
|
|
|
|
bin/dev-open-when-ready (``_SANDBOX_UNSANDBOXED_PREFIXES``) is peeled out the
|
|
same way so a proc can auto-open its browser once it's listening — the OS
|
|
browser-open path the sandbox denies runs outside it, not by widening the
|
|
profile.
|
|
"""
|
|
# `sandbox` is a registry-only selector; pop it so it never leaks into the
|
|
# emitted proc config (which phrocs/mprocs would otherwise see).
|
|
should_sandbox = bool(proc_config.pop("sandbox", False))
|
|
excluded = set(self._SANDBOX_DEFAULT_EXCLUDES)
|
|
excluded |= {p.strip() for p in os.getenv("INSIGHTS_DEV_SANDBOX_EXCLUDE", "").split(",") if p.strip()}
|
|
if not _dev_sandbox_enabled() or not should_sandbox or (name and name in excluded):
|
|
return proc_config
|
|
|
|
shell = proc_config.get("shell", "")
|
|
if not shell:
|
|
return proc_config
|
|
|
|
# Split into docker-gate segments (run unsandboxed) and the rest (sandboxed).
|
|
# Collapse `\`-newline continuations first; splitting on && is safe here
|
|
# because the generated commands keep branching inside if/;-blocks, not &&.
|
|
#
|
|
# Gates are matched anywhere in the chain and hoisted to the front, not kept
|
|
# in place: _add_startup_message and _add_uv_groups prepend echo/uv-sync
|
|
# preambles, so a gate is never the leading segment (a leading-run scan would
|
|
# wrongly sandbox it). Hoisting reorders segments, which is safe because the
|
|
# peeled-out helpers (wait-for-docker / wait-for-postgres-tables block until
|
|
# infra is ready; dev-open-when-ready self-backgrounds and returns at once)
|
|
# are all order-independent w.r.t. install/echo/server steps. A future
|
|
# order-sensitive helper would need interleaving instead of hoisting.
|
|
unsandboxed_prefixes = self._SANDBOX_DOCKER_GATES + self._SANDBOX_UNSANDBOXED_PREFIXES
|
|
normalized = re.sub(r"\\\n\s*", " ", shell)
|
|
gates: list[str] = []
|
|
rest: list[str] = []
|
|
for segment in normalized.split("&&"):
|
|
stripped = segment.strip()
|
|
if not stripped:
|
|
continue
|
|
first_token = stripped.split(maxsplit=1)[0]
|
|
(gates if first_token in unsandboxed_prefixes else rest).append(stripped)
|
|
|
|
rest_cmd = " && ".join(rest)
|
|
if not rest_cmd:
|
|
# Command is only gate scripts (trusted, need the docker socket) — there
|
|
# is nothing to sandbox, so run them as-is. Avoids `bin/dev-sandbox ''`,
|
|
# which the wrapper rejects (its ${1:?} fires on an empty argument).
|
|
proc_config["shell"] = " && ".join(gates)
|
|
elif not gates:
|
|
proc_config["shell"] = f"bin/dev-sandbox {shlex.quote(rest_cmd)}"
|
|
else:
|
|
proc_config["shell"] = f"{' && '.join(gates)} && bin/dev-sandbox {shlex.quote(rest_cmd)}"
|
|
return proc_config
|
|
|
|
def _add_logging(self, proc_config: dict[str, Any], process_name: str) -> dict[str, Any]:
|
|
"""Wrap shell command to log output to /tmp/insights-{name}.log.
|
|
|
|
Args:
|
|
proc_config: The process configuration dict
|
|
process_name: Name of the process (used in log filename)
|
|
|
|
Returns:
|
|
Modified process configuration with tee logging
|
|
"""
|
|
shell = proc_config.get("shell", "")
|
|
if not shell:
|
|
return proc_config
|
|
|
|
log_file = f"/tmp/insights-{process_name}.log"
|
|
proc_config["shell"] = f"{shell} 2>&1 | tee {log_file}"
|
|
return proc_config
|
|
|
|
def save(self, config: MprocsConfig, output_path: Path) -> Path:
|
|
"""Save mprocs configuration to YAML file."""
|
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
with open(output_path, "w") as f:
|
|
# Add header comment for log mode
|
|
if config.insights_config and config.insights_config.log_to_files:
|
|
f.write("# Log mode: Output logged to /tmp/insights-*.log\n")
|
|
|
|
yaml.dump(config.to_yaml_dict(), f, default_flow_style=False, sort_keys=False)
|
|
return output_path
|
|
|
|
|
|
def load_devenv_config(mprocs_path: Path) -> DevenvConfig | None:
|
|
"""Load DevenvConfig from a generated mprocs.yaml file.
|
|
|
|
Args:
|
|
mprocs_path: Path to generated mprocs.yaml
|
|
|
|
Returns:
|
|
DevenvConfig if _insights section exists, None otherwise
|
|
"""
|
|
if not mprocs_path.exists():
|
|
return None
|
|
|
|
try:
|
|
with open(mprocs_path) as f:
|
|
data = yaml.safe_load(f) or {}
|
|
except (yaml.YAMLError, OSError):
|
|
return None
|
|
|
|
insights_data = data.get("_insights")
|
|
if not insights_data:
|
|
return None
|
|
|
|
return DevenvConfig.model_validate(insights_data)
|
|
|
|
|
|
def get_main_repo_from_worktree() -> Path | None:
|
|
"""If in a worktree, return the main repo root. Otherwise None."""
|
|
current = Path.cwd().resolve()
|
|
for parent in [current, *current.parents]:
|
|
git_path = parent / ".git"
|
|
if git_path.is_file():
|
|
# Worktree: .git file contains "gitdir: /path/to/main/.git/worktrees/<name>"
|
|
try:
|
|
content = git_path.read_text().strip()
|
|
except OSError:
|
|
return None
|
|
if content.startswith("gitdir: ") and "worktrees" in content:
|
|
gitdir = Path(content.removeprefix("gitdir: ").strip())
|
|
# Resolve relative paths against .git file's directory
|
|
if not gitdir.is_absolute():
|
|
gitdir = (git_path.parent / gitdir).resolve()
|
|
return gitdir.parent.parent.parent
|
|
elif git_path.is_dir():
|
|
break # Regular repo, not a worktree
|
|
return None
|
|
|
|
|
|
def get_generated_mprocs_path() -> Path:
|
|
"""Get the default path for generated mprocs config.
|
|
|
|
Checks local path first, then main repo if in a worktree.
|
|
"""
|
|
override_path = os.getenv("INSIGHTSCLI_MPROCS_PATH")
|
|
if override_path:
|
|
return Path(override_path)
|
|
|
|
current = Path.cwd().resolve()
|
|
local_path = current / ".insights" / ".generated" / "mprocs.yaml"
|
|
for parent in [current, *current.parents]:
|
|
if (parent / ".git").exists():
|
|
local_path = parent / ".insights" / ".generated" / "mprocs.yaml"
|
|
break
|
|
|
|
# If local config exists (or is symlink), use it
|
|
if local_path.exists():
|
|
return local_path
|
|
|
|
# Check main repo if in a worktree
|
|
main_repo = get_main_repo_from_worktree()
|
|
if main_repo:
|
|
main_path = main_repo / ".insights" / ".generated" / "mprocs.yaml"
|
|
if main_path.exists():
|
|
# Create local symlink so bin/start (which uses $REPOSITORY_ROOT) finds it
|
|
local_path.parent.mkdir(parents=True, exist_ok=True)
|
|
if local_path.is_symlink():
|
|
local_path.unlink()
|
|
local_path.symlink_to(main_path)
|
|
return main_path
|
|
|
|
return local_path
|