mirror of
https://github.com/hanzoai/dev-desktop.git
synced 2026-11-20 01:07:23 +00:00
417 lines
20 KiB
TypeScript
417 lines
20 KiB
TypeScript
import * as Alchemy from "alchemy";
|
|
import * as Axiom from "@alchemy.run/axiom";
|
|
import * as Output from "alchemy/Output";
|
|
import * as Config from "effect/Config";
|
|
import * as Effect from "effect/Effect";
|
|
import * as Option from "effect/Option";
|
|
|
|
import { relayResourceNameForStage } from "./deploymentConfig.ts";
|
|
|
|
const RUNBOOK_URL =
|
|
"https://github.com/hanzoai/dev/blob/main/docs/operations/relay-observability.md#alerts";
|
|
|
|
const LINK_ROUTE = `"/v1/client/environment-links"`;
|
|
const TUNNEL_ROUTES = `"/v1/environments/:environmentId/tunnel", "/v1/environments/:environmentId/tunnel/recovery"`;
|
|
// The Cloudflare SDK client spans record the host but not the full URL.
|
|
const CLOUDFLARE_API = `['attributes.server.address'] == "api.cloudflare.com"`;
|
|
const STATUS = `toint(['attributes.http.response.status_code'])`;
|
|
// The tunnel reaper records its counters on this span every five minutes,
|
|
// including while cleanup is off.
|
|
const SWEEP = `name == "relay.managed_endpoint_reaper.sweep"`;
|
|
const sweepCounter = (field: string) =>
|
|
`toint(['attributes.custom']['relay.managed_endpoint_reaper.${field}'])`;
|
|
|
|
// Status codes, not span errors: a request cut off by the relay deadline ends
|
|
// as an interrupted span with no error flag, but still answers 504.
|
|
// Below `minimum` requests the rate reads 0, so a quiet window cannot page.
|
|
const serverFailurePercent = (dataset: string, routes: string, minimum: number) =>
|
|
[
|
|
`['${dataset}']`,
|
|
`| where kind == "server" and ['attributes.http.request.method'] == "POST" and ['attributes.http.route'] in (${routes})`,
|
|
`| summarize total = count(), failed = countif(${STATUS} >= 500)`,
|
|
`| project failure_pct = iff(total >= ${minimum}, 100.0 * failed / total, 0.0)`,
|
|
].join("\n");
|
|
|
|
type Severity = "page" | "warn";
|
|
|
|
interface RelayMonitor {
|
|
readonly id: string;
|
|
readonly severity: Severity;
|
|
readonly name: string;
|
|
/** Why it fires and the healthy baseline. The runbook link is appended. */
|
|
readonly description: string;
|
|
readonly query: (dataset: string) => string;
|
|
readonly operator: "Above" | "AboveOrEqual" | "Below";
|
|
readonly threshold: number;
|
|
readonly rangeMinutes: number;
|
|
readonly intervalMinutes?: number;
|
|
readonly triggerFromNRuns?: number;
|
|
readonly alertOnNoData?: boolean;
|
|
}
|
|
|
|
// Thresholds come from 2026-10-16..19 prod data: the worst healthy window,
|
|
// and the outage (Cloudflare API 429s stalling every provision).
|
|
// Paging monitors need two bad runs in a row; warnings catch the slow
|
|
// degradations an outage threshold has to ignore.
|
|
const relayMonitors: ReadonlyArray<RelayMonitor> = [
|
|
{
|
|
id: "RelayLinkFailureRateMonitor",
|
|
severity: "page",
|
|
name: "Hanzo Connect relay: links failing",
|
|
description:
|
|
"Share of POST /v1/client/environment-links (turning on Hanzo Connect) answering 5xx over 15 min. Healthy: median 11%, worst 75%. #16810: 87-96%.",
|
|
query: (dataset) => serverFailurePercent(dataset, LINK_ROUTE, 20),
|
|
operator: "Above",
|
|
threshold: 80,
|
|
rangeMinutes: 15,
|
|
triggerFromNRuns: 2,
|
|
},
|
|
{
|
|
id: "RelayTunnelFailureRateMonitor",
|
|
severity: "page",
|
|
name: "Hanzo Connect relay: tunnel recovery failing",
|
|
description:
|
|
"Share of host tunnel registration and recovery answering 5xx over 15 min. Hosts whose tunnel drops cannot come back. Healthy: median 4%, worst 52%. #16810: 83-94%.",
|
|
query: (dataset) => serverFailurePercent(dataset, TUNNEL_ROUTES, 200),
|
|
operator: "Above",
|
|
threshold: 60,
|
|
rangeMinutes: 15,
|
|
triggerFromNRuns: 2,
|
|
},
|
|
{
|
|
id: "RelayConnectReachabilityMonitor",
|
|
severity: "page",
|
|
name: "Hanzo Connect relay: environments unreachable",
|
|
description:
|
|
"Share of environments clients tried to reach that connected at least once in 30 min. Offline hosts keep this below 100%, and retry storms against them do not move it. Healthy: 65-72%.",
|
|
query: (dataset) =>
|
|
[
|
|
`['${dataset}']`,
|
|
`| where name == "relay.environment_connector.connect"`,
|
|
`| extend environment = tostring(['attributes.custom']['relay.environment_id'])`,
|
|
`| summarize environments = dcount(environment), connected = dcountif(environment, error != true)`,
|
|
`| project reachable_pct = iff(environments >= 1000, 100.0 * connected / environments, 100.0)`,
|
|
].join("\n"),
|
|
operator: "Below",
|
|
threshold: 55,
|
|
rangeMinutes: 30,
|
|
triggerFromNRuns: 2,
|
|
},
|
|
{
|
|
id: "RelayPushFailureRateMonitor",
|
|
severity: "page",
|
|
name: "Hanzo Connect relay: push delivery failing",
|
|
description:
|
|
"Share of APNs and FCM delivery jobs failing over 15 min, e.g. an expired APNs key or FCM credential. Healthy: APNs under 0.3%, FCM under 4%.",
|
|
query: (dataset) =>
|
|
[
|
|
`['${dataset}']`,
|
|
`| where name in ("relay.apns_deliveries.process_signed_job", "relay.fcm.process")`,
|
|
`| summarize total = count(), failed = countif(error == true)`,
|
|
`| project failure_pct = iff(total >= 100, 100.0 * failed / total, 0.0)`,
|
|
].join("\n"),
|
|
operator: "Above",
|
|
threshold: 10,
|
|
rangeMinutes: 15,
|
|
triggerFromNRuns: 2,
|
|
},
|
|
{
|
|
id: "RelayCloudflareRateLimitedMonitor",
|
|
severity: "page",
|
|
name: "Hanzo Connect relay: Cloudflare API rate limited",
|
|
description:
|
|
"Cloudflare API 429s, plus calls the relay refused while paused after one, per 10 min. Sustained at this level, links and recoveries stall. Peak 429s before #16810: 1,286; during it: ~11k.",
|
|
query: (dataset) =>
|
|
`['${dataset}']\n| where (${CLOUDFLARE_API} and ${STATUS} == 429) or name == "relay.cloudflare_api_pause.rejected"\n| summarize rate_limited = count()`,
|
|
operator: "AboveOrEqual",
|
|
threshold: 500,
|
|
rangeMinutes: 10,
|
|
triggerFromNRuns: 2,
|
|
},
|
|
{
|
|
id: "RelayTrafficDroppedMonitor",
|
|
severity: "page",
|
|
name: "Hanzo Connect relay: traffic dropped",
|
|
description:
|
|
"Relay requests per 10 min collapsed: Worker down, broken deploy, or trace export stopped. Healthy minimum: ~69k.",
|
|
query: (dataset) => `['${dataset}']\n| where kind == "server"\n| summarize requests = count()`,
|
|
operator: "Below",
|
|
threshold: 10_000,
|
|
rangeMinutes: 10,
|
|
triggerFromNRuns: 2,
|
|
alertOnNoData: true,
|
|
},
|
|
{
|
|
id: "RelayCloudflareRateLimitWarningMonitor",
|
|
severity: "warn",
|
|
name: "Hanzo Connect relay: Cloudflare API near its limit",
|
|
description:
|
|
"Any sustained Cloudflare API 429s or paused calls. 429 bursts ran for days before #16810; raise the account limit or cut call volume before they grow.",
|
|
query: (dataset) =>
|
|
`['${dataset}']\n| where (${CLOUDFLARE_API} and ${STATUS} == 429) or name == "relay.cloudflare_api_pause.rejected"\n| summarize rate_limited = count()`,
|
|
operator: "AboveOrEqual",
|
|
threshold: 20,
|
|
rangeMinutes: 10,
|
|
triggerFromNRuns: 2,
|
|
},
|
|
{
|
|
id: "RelayLinkChronicFailureMonitor",
|
|
severity: "warn",
|
|
name: "Hanzo Connect relay: links degraded",
|
|
description:
|
|
"Share of links answering 5xx over 6 h, mostly 9 s deadline 504s from slow provisioning. Healthy days: 10-18%. Degraded days before #16810: 28-33%.",
|
|
query: (dataset) => serverFailurePercent(dataset, LINK_ROUTE, 100),
|
|
operator: "Above",
|
|
threshold: 20,
|
|
rangeMinutes: 360,
|
|
intervalMinutes: 30,
|
|
},
|
|
{
|
|
id: "RelayDatabaseLatencyMonitor",
|
|
severity: "warn",
|
|
name: "Hanzo Connect relay: database slow",
|
|
description:
|
|
"p90 Postgres query time over 15 min, in ms. Every relay request runs several queries in sequence, so this sets how close provisioning gets to the 9 s deadline. Healthy: ~200 ms.",
|
|
query: (dataset) =>
|
|
`['${dataset}']\n| where name == "postgres"\n| summarize p90 = percentile(duration, 90)\n| project p90_ms = p90 / 1000000`,
|
|
operator: "Above",
|
|
threshold: 500,
|
|
rangeMinutes: 15,
|
|
triggerFromNRuns: 2,
|
|
},
|
|
{
|
|
id: "RelayTunnelCleanupStoppedMonitor",
|
|
// Pages because nothing else surfaces it: unnoticed, it ran 2026-11-15..18 while 27k+ idle tunnels piled up.
|
|
severity: "page",
|
|
name: "Hanzo Connect relay: tunnel cleanup stopped",
|
|
description:
|
|
"Tunnel cleanup sweeps that finished in 2 h. Healthy: 24. The cron is not running, or every sweep fails, as during the 2026-10-19 Cloudflare 429s and from 2026-11-15 when listing tunnels timed out. Idle tunnels pile up and are billed meanwhile.",
|
|
query: (dataset) =>
|
|
`['${dataset}']\n| where ${SWEEP} and error != true\n| summarize sweeps = count()`,
|
|
operator: "Below",
|
|
threshold: 1,
|
|
rangeMinutes: 120,
|
|
intervalMinutes: 15,
|
|
triggerFromNRuns: 2,
|
|
alertOnNoData: true,
|
|
},
|
|
{
|
|
id: "RelayTunnelCleanupFailingMonitor",
|
|
severity: "warn",
|
|
name: "Hanzo Connect relay: tunnel deletions failing",
|
|
description:
|
|
"Cloudflare tunnel deletions that failed in 30 min. Healthy: 0; the worst 30 min so far had 3. The relay logs a warning with the Cloudflare error for each.",
|
|
query: (dataset) =>
|
|
`['${dataset}']\n| where ${SWEEP}\n| summarize failed = sum(${sweepCounter("failed")})`,
|
|
operator: "Above",
|
|
threshold: 10,
|
|
rangeMinutes: 30,
|
|
triggerFromNRuns: 2,
|
|
},
|
|
];
|
|
|
|
interface QueryChart {
|
|
readonly id: string;
|
|
readonly name: string;
|
|
readonly type: "TimeSeries" | "Table";
|
|
readonly query: { readonly apl: string };
|
|
}
|
|
|
|
const timeSeries = (id: string, name: string, apl: string): QueryChart => ({
|
|
id,
|
|
name,
|
|
type: "TimeSeries",
|
|
query: { apl },
|
|
});
|
|
|
|
const relayDashboardCharts = (dataset: string): ReadonlyArray<QueryChart> => {
|
|
const source = `['${dataset}']`;
|
|
const linkRequests = `${source}\n| where kind == "server" and ['attributes.http.request.method'] == "POST" and ['attributes.http.route'] == ${LINK_ROUTE}`;
|
|
const tunnelRequests = `${source}\n| where kind == "server" and ['attributes.http.request.method'] == "POST" and ['attributes.http.route'] in (${TUNNEL_ROUTES})`;
|
|
const clientConnections = `${source}\n| where name == "relay.connection.attempt" and error != true\n| extend surface = tostring(['resource.custom']['hanzo.client.surface'])\n| where isnotempty(surface)`;
|
|
return [
|
|
timeSeries(
|
|
"traffic",
|
|
"Requests and agent activity publishes",
|
|
`${source}\n| where kind == "server" or name == "relay.agent_activity_publisher.publish"\n| summarize requests = countif(kind == "server"), publishes = countif(name == "relay.agent_activity_publisher.publish") by bin_auto(_time)`,
|
|
),
|
|
timeSeries(
|
|
"links",
|
|
"Links by outcome",
|
|
`${linkRequests}\n| summarize succeeded = countif(${STATUS} < 400), rejected = countif(${STATUS} >= 400 and ${STATUS} < 500), failed = countif(${STATUS} >= 500) by bin_auto(_time)`,
|
|
),
|
|
timeSeries(
|
|
"link-latency",
|
|
"Link latency (9 s deadline)",
|
|
`${linkRequests}\n| summarize p50 = percentile(duration, 50), p90 = percentile(duration, 90) by bin_auto(_time)`,
|
|
),
|
|
timeSeries(
|
|
"reachability",
|
|
"Environments reachable per 30 min (%)",
|
|
`${source}\n| where name == "relay.environment_connector.connect"\n| extend environment = tostring(['attributes.custom']['relay.environment_id'])\n| summarize environments = dcount(environment), connected = dcountif(environment, error != true) by bin(_time, 30m)\n| project _time, reachable_pct = 100.0 * connected / environments`,
|
|
),
|
|
timeSeries(
|
|
"tunnel",
|
|
"Tunnel registration and recovery by outcome",
|
|
`${tunnelRequests}\n| summarize succeeded = countif(${STATUS} < 400), rejected = countif(${STATUS} >= 400 and ${STATUS} < 500), failed = countif(${STATUS} >= 500) by bin_auto(_time)`,
|
|
),
|
|
timeSeries(
|
|
"cloudflare-api",
|
|
"Cloudflare API calls by status",
|
|
// Calls refused during a rate-limit pause never reach Cloudflare, so
|
|
// they are their own series or a storm would look like a quiet period.
|
|
`${source}\n| where ${CLOUDFLARE_API} or name == "relay.cloudflare_api_pause.rejected"\n| summarize count() by bin_auto(_time), status = case(name == "relay.cloudflare_api_pause.rejected", "paused", isnull(${STATUS}), "no response", tostring(${STATUS}))`,
|
|
),
|
|
timeSeries(
|
|
"provision-paths",
|
|
"Tunnel provisions by path",
|
|
`${source}\n| where name == "relay.managed_endpoint_provider.provision"\n| summarize reused_ready_tunnel = countif(error != true and tobool(['attributes.custom']['relay.managed_endpoint.reused_ready_tunnel']) == true), full_provision = countif(error != true and isnull(['attributes.custom']['relay.managed_endpoint.reused_ready_tunnel'])), failed = countif(error == true) by bin_auto(_time)`,
|
|
),
|
|
timeSeries(
|
|
"postgres",
|
|
"Postgres query time",
|
|
`${source}\n| where name == "postgres"\n| summarize p50 = percentile(duration, 50), p90 = percentile(duration, 90) by bin_auto(_time)`,
|
|
),
|
|
timeSeries(
|
|
"push",
|
|
"Push delivery jobs",
|
|
`${source}\n| where name in ("relay.apns_deliveries.process_signed_job", "relay.fcm.process")\n| summarize delivered = countif(error != true), failed = countif(error == true) by bin_auto(_time), service = iff(name == "relay.fcm.process", "fcm", "apns")`,
|
|
),
|
|
timeSeries(
|
|
"tunnel-cleanup",
|
|
"Tunnel cleanup per sweep",
|
|
`${source}\n| where ${SWEEP}\n| summarize deleted = sum(${sweepCounter("deleted")}), deleted_legacy = sum(${sweepCounter("deletedLegacy")}), would_delete = sum(${sweepCounter("wouldDelete")}), would_delete_legacy = sum(${sweepCounter("wouldDeleteLegacy")}), failed = sum(${sweepCounter("failed")}), sweeps_failed = countif(error == true) by bin_auto(_time)`,
|
|
),
|
|
// Cloudflare's own counts for this stage, including tunnels the reaper
|
|
// skips, so this is the idle tunnels still billed.
|
|
timeSeries(
|
|
"tunnel-backlog",
|
|
"Idle tunnels in Cloudflare",
|
|
`${source}\n| where ${SWEEP}\n| summarize down = max(${sweepCounter("totalDown")}), never_connected = max(${sweepCounter("totalInactive")}) by bin_auto(_time)`,
|
|
),
|
|
timeSeries(
|
|
"server-errors",
|
|
"5xx by route",
|
|
`${source}\n| where kind == "server" and ${STATUS} >= 500\n| summarize count() by bin_auto(_time), route = tostring(['attributes.http.route']), status = ${STATUS}`,
|
|
),
|
|
{
|
|
id: "link-failures",
|
|
name: "Link failures by reason",
|
|
type: "Table",
|
|
query: {
|
|
apl: `${source}\n| where name == "relay.environment_linker.link" and error == true\n| extend message = tostring(['status.message'])\n| summarize failures = count() by reason = case(message contains "failed during '", strcat("provisioning: ", extract("failed during '([^']+)'", 1, message)), message startswith "Managed tunnel limit reached", "managed tunnel limit reached", substring(message, 0, 80))\n| top 10 by failures`,
|
|
},
|
|
},
|
|
// Connecting, as users see it: web, desktop and mobile export their
|
|
// connection spans into this dataset, in the same traces as the relay's
|
|
// and the host's spans. An attempt lasts until initial sync finishes.
|
|
timeSeries(
|
|
"connect-time",
|
|
"Connecting: time to connected by surface",
|
|
`${clientConnections}\n| summarize p50 = percentile(duration, 50), p90 = percentile(duration, 90) by bin_auto(_time), surface`,
|
|
),
|
|
timeSeries(
|
|
"connect-outcomes",
|
|
"Connecting: attempts by outcome",
|
|
`${source}\n| where name == "relay.connection.attempt" and isnotempty(['resource.custom']['hanzo.client.surface'])\n| summarize connected = countif(error != true), failed = countif(error == true) by bin_auto(_time)`,
|
|
),
|
|
{
|
|
id: "connect-phases",
|
|
name: "Connecting: where the time goes",
|
|
type: "Table",
|
|
query: {
|
|
// Failed direct route checks stay in: waiting out an unreachable LAN
|
|
// address before falling back to the relay is the slowest step.
|
|
apl: [
|
|
source,
|
|
`| where isnotempty(['resource.custom']['hanzo.client.surface'])`,
|
|
`| extend url = tostring(['attributes.url.full'])`,
|
|
`| extend phase = case(`,
|
|
` name == "ConnectionDriver.checkRoute", "0. Direct route check, before any relay fallback",`,
|
|
` name == "relay.connection.cloudSessionToken.resolve", "1. Clerk session token",`,
|
|
` name == "clientRuntime.managedRelay.exchangeAccessToken", "2. Relay token exchange",`,
|
|
` kind == "client" and url startswith "https://relay.dev.hanzo.ai/v1/environments/" and url endswith "/connect", "3. Relay connect, minting a host credential",`,
|
|
` name == "environment.authorization.descriptor", "4. Host descriptor",`,
|
|
` name == "environment.authorization.accessToken.exchange", "5. Host token exchange",`,
|
|
` name == "clientRuntime.connection.remote.createDpopSocketUrl", "6. Socket ticket",`,
|
|
` name == "environment.initialSync", "7. Initial sync",`,
|
|
` name == "relay.connection.attempt", "Total: time to connected",`,
|
|
` "")`,
|
|
`| where phase != "" and (error != true or phase startswith "0.")`,
|
|
`| summarize calls = count(), p50 = percentile(duration, 50), p90 = percentile(duration, 90) by phase`,
|
|
`| order by phase asc`,
|
|
].join("\n"),
|
|
},
|
|
},
|
|
timeSeries(
|
|
"relay-latency-by-continent",
|
|
"Relay token and connect latency by user continent (p50)",
|
|
`${source}\n| where kind == "server" and ['attributes.http.request.method'] == "POST" and (['attributes.http.route'] == "/v1/client/dpop-token" or ['attributes.http.route'] == "/v1/environments/:environmentId/connect")\n| extend continent = tostring(['attributes.custom']['relay.edge.continent'])\n| where isnotempty(continent)\n| summarize p50 = percentile(duration, 50) by bin_auto(_time), continent`,
|
|
),
|
|
];
|
|
};
|
|
|
|
export const notifierConfig = (name: string) =>
|
|
Config.String(name).pipe(
|
|
Config.option,
|
|
Config.map(Option.flatMap((value) => Option.liftPredicate(value.trim(), Boolean))),
|
|
);
|
|
|
|
/**
|
|
* A deployment opts in by naming the notifier that pages (incident.io for
|
|
* prod). Personal stages and forks leave it unset and get no monitors or
|
|
* dashboard. Warnings go to their own notifier when one is set, and are only
|
|
* visible in Axiom otherwise.
|
|
*/
|
|
export const RelayMonitoring = Effect.fn("RelayMonitoring")(function* (traces: Axiom.Dataset) {
|
|
const { stage } = yield* Alchemy.Stack;
|
|
const page = yield* notifierConfig("RELAY_ALERT_NOTIFIER_ID");
|
|
if (Option.isNone(page)) return;
|
|
const notifiers: Record<Severity, Option.Option<string>> = {
|
|
page,
|
|
warn: yield* notifierConfig("RELAY_WARNING_NOTIFIER_ID"),
|
|
};
|
|
|
|
for (const monitor of relayMonitors) {
|
|
yield* Axiom.Monitor(monitor.id, {
|
|
name: stage === "prod" ? monitor.name : `[${stage}] ${monitor.name}`,
|
|
description: `${monitor.description} Runbook: ${RUNBOOK_URL}`,
|
|
type: "Threshold",
|
|
aplQuery: Output.map(traces.name, monitor.query),
|
|
operator: monitor.operator,
|
|
threshold: monitor.threshold,
|
|
intervalMinutes: monitor.intervalMinutes ?? 5,
|
|
rangeMinutes: monitor.rangeMinutes,
|
|
triggerFromNRuns: monitor.triggerFromNRuns ?? 1,
|
|
alertOnNoData: monitor.alertOnNoData ?? false,
|
|
notifierIds: Option.toArray(notifiers[monitor.severity]),
|
|
});
|
|
}
|
|
|
|
yield* Axiom.Dashboard("RelayHealthDashboard", {
|
|
dashboard: Output.map(traces.name, (dataset) => {
|
|
const charts = relayDashboardCharts(dataset);
|
|
return {
|
|
name: relayResourceNameForStage("hanzo-dev-relay-health", stage),
|
|
// Shared with the whole org. An empty owner only means that for API
|
|
// tokens; deploys with a personal token would make it private.
|
|
owner: "X-AXIOM-EVERYONE",
|
|
description: `Hanzo Connect relay health. Alerts and runbook: ${RUNBOOK_URL}`,
|
|
refreshTime: 60,
|
|
schemaVersion: 2,
|
|
timeWindowStart: "qr-now-24h",
|
|
timeWindowEnd: "qr-now",
|
|
charts: [...charts],
|
|
// Two charts per row on Axiom's 12-column grid.
|
|
layout: charts.map((chart, index) => ({
|
|
i: chart.id,
|
|
x: (index % 2) * 6,
|
|
y: Math.floor(index / 2) * 6,
|
|
w: 6,
|
|
h: 6,
|
|
})),
|
|
};
|
|
}),
|
|
});
|
|
});
|