mirror of
https://github.com/hanzoai/insights.git
synced 2026-10-30 16:15:31 +00:00
zeekay
1b367e2b62
The tree was a debranded snapshot of upstream taken around 2026-02-20, and upstream has moved about five months since. This replaces it with upstream master, mechanically re-derived: bin/debrand does the rename, and the pipeline and squashed migrations are restored from the shipped tree. Not a merge. The previous import replaced the tree wholesale in a single commit, so there is no shared content history to merge against. Reproducing the rename on the new tree is what actually converges. What survives: .hanzo/workflows, the Dockerfiles and bin/docker* entrypoints, compose files, LLM.md, and insights/migrations. The squashed initial migration is load-bearing — the live database records it by name, so upstream's ~918 individual migrations cannot come back. Nine products are dropped as unreferenced. Twenty-one more that we do not run are kept anyway because the core app still imports them; unpicking those is a refactor and does not belong in a rebase. This base does not build yet, and the reasons are not in the rename: upstream now wants Python 3.13.13 against our 3.12.12 and Django 5.2 against our 4.2, and the dependency manifests still name upstream packages that we republish under our own names. Those are decisions, not renames.
132 lines
4.5 KiB
Python
132 lines
4.5 KiB
Python
from datetime import timedelta
|
|
from typing import Any
|
|
|
|
from django.utils import timezone
|
|
|
|
from celery import shared_task
|
|
from structlog import get_logger
|
|
|
|
from insights.celery_queues import CeleryQueue
|
|
from insights.scoping_audit import skip_team_scope_audit
|
|
|
|
from products.endpoints.backend.logic.ducklake_shadow import run_ducklake_shadow_comparison
|
|
from products.endpoints.backend.metrics import ENDPOINT_MATERIALIZATION_EVENT_TOTAL
|
|
from products.endpoints.backend.models import EndpointVersion
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
STALE_THRESHOLD_DAYS = 30
|
|
|
|
|
|
@shared_task(
|
|
ignore_result=True,
|
|
queue=CeleryQueue.LONG_RUNNING.value,
|
|
name="products.endpoints.backend.tasks.shadow_compare_ducklake_execution",
|
|
)
|
|
@skip_team_scope_audit
|
|
def shadow_compare_ducklake_execution(
|
|
team_id: int,
|
|
endpoint_id: str,
|
|
version_id: str,
|
|
variables: dict[str, Any] | None,
|
|
execution_type: str,
|
|
datastore_cached: bool,
|
|
datastore_ms: float,
|
|
datastore_row_count: int | None,
|
|
limit: int | None,
|
|
offset: int | None,
|
|
) -> None:
|
|
run_ducklake_shadow_comparison(
|
|
team_id=team_id,
|
|
endpoint_id=endpoint_id,
|
|
version_id=version_id,
|
|
variables=variables,
|
|
execution_type=execution_type,
|
|
datastore_cached=datastore_cached,
|
|
datastore_ms=datastore_ms,
|
|
datastore_row_count=datastore_row_count,
|
|
limit=limit,
|
|
offset=offset,
|
|
)
|
|
|
|
|
|
@shared_task(ignore_result=True, name="products.endpoints.backend.tasks.deactivate_stale_materializations")
|
|
@skip_team_scope_audit
|
|
def deactivate_stale_materializations() -> None:
|
|
"""
|
|
Deactivate materializations for endpoint versions that haven't been executed in over 30 days.
|
|
|
|
This task finds endpoint versions where:
|
|
1. The version has an active materialization (saved_query.is_materialized = True)
|
|
2. The materialization has run in the past 24h (saved_query.last_run_at within 24h)
|
|
3. The materialization was enabled at least 30 days ago (saved_query.created_at)
|
|
4. The endpoint was last executed over 30 days ago (via API key)
|
|
|
|
For matching versions, the materialization is reverted to save resources.
|
|
"""
|
|
now = timezone.now()
|
|
twenty_four_hours_ago = now - timedelta(hours=24)
|
|
stale_threshold = now - timedelta(days=STALE_THRESHOLD_DAYS)
|
|
|
|
stale_versions = EndpointVersion.objects.filter(
|
|
saved_query__isnull=False,
|
|
saved_query__is_materialized=True,
|
|
saved_query__last_run_at__gte=twenty_four_hours_ago,
|
|
saved_query__deleted=False,
|
|
saved_query__created_at__lte=stale_threshold,
|
|
endpoint__last_executed_at__lt=stale_threshold,
|
|
endpoint__deleted=False,
|
|
).select_related("saved_query", "endpoint")
|
|
|
|
if not stale_versions.exists():
|
|
logger.info("deactivate_stale_materializations_no_candidates")
|
|
return
|
|
|
|
deactivated_count = 0
|
|
|
|
for version in stale_versions:
|
|
try:
|
|
_deactivate_version_materialization(version)
|
|
except Exception as e:
|
|
logger.exception(
|
|
"deactivate_stale_materialization_failed",
|
|
endpoint_id=str(version.endpoint.id),
|
|
endpoint_name=version.endpoint.name,
|
|
version=version.version,
|
|
team_id=version.endpoint.team_id,
|
|
error=str(e),
|
|
)
|
|
ENDPOINT_MATERIALIZATION_EVENT_TOTAL.labels(action="deactivate_stale", status="error").inc()
|
|
continue
|
|
deactivated_count += 1
|
|
ENDPOINT_MATERIALIZATION_EVENT_TOTAL.labels(action="deactivate_stale", status="success").inc()
|
|
|
|
logger.info(
|
|
"deactivate_stale_materializations_completed",
|
|
deactivated_count=deactivated_count,
|
|
)
|
|
|
|
|
|
def _deactivate_version_materialization(version: EndpointVersion) -> None:
|
|
"""
|
|
Deactivate materialization for an endpoint version.
|
|
|
|
This reverts the materialization (removes Temporal schedule, cleans up S3 tables)
|
|
and soft-deletes the saved_query.
|
|
"""
|
|
saved_query = version.saved_query
|
|
if not saved_query:
|
|
return
|
|
|
|
logger.info(
|
|
"deactivating_stale_materialization",
|
|
endpoint_id=str(version.endpoint.id),
|
|
endpoint_name=version.endpoint.name,
|
|
version=version.version,
|
|
team_id=version.endpoint.team_id,
|
|
last_executed_at=str(version.endpoint.last_executed_at) if version.endpoint.last_executed_at else None,
|
|
last_run_at=str(saved_query.last_run_at) if saved_query.last_run_at else None,
|
|
)
|
|
|
|
version.disable_materialization()
|