Files
zeekay 40d120dbf7 refactor(api): say /v1/ in the Python callers
Every first-party Python caller and test now names the mount the API actually
answers on. Mechanical: URL literals only, and only where the /api/ was ours --
a preceding word character means it was a module path like insights/api/ or a
third-party host like slack.com/api/, and those are untouched.

Left saying /api/, each for a reason rather than for safety:
- products/warehouse_sources/.../sources/** is a catalogue of vendor endpoints
- services/llm-gateway deploys separately and is under a code freeze, so moving
  its two calls would break it during the skew
- the mcp_store and Slack/Teams redirect_uris are registered with third parties
  and must byte-match what was registered
- the Wizard and Raycast client-metadata paths are OAuth client_ids
- /api/surveys and /api/feature_flag/local_evaluation are SDK aliases the
  ingress routes to the Rust fleets, held by released SDK versions

Test URLconfs moved with the requests that call them, and the Datastore
user-selection fixtures now carry the leading slash that request.path has.
2026-08-08 18:38:44 -07:00

651 lines
24 KiB
TypeScript

import {
MakeLogicType,
actions,
afterMount,
beforeUnmount,
kea,
listeners,
path,
props,
reducers,
selectors,
} from 'kea'
import { forms } from 'kea-forms'
import type { DeepPartial, DeepPartialMap, FieldName, ValidationErrorType } from 'kea-forms'
import { loaders } from 'kea-loaders'
import { router, urlToAction } from 'kea-router'
import api, { ApiConfig } from 'lib/api'
import { dayjs } from 'lib/dayjs'
import { Dialog } from 'lib/elements/Dialog'
import { toast } from 'lib/elements/Toast/Toast'
import { urls } from 'scenes/urls'
import { FileSystemIconType } from '~/queries/schema/schema-general'
import { Breadcrumb, ProjectTreeRef } from '~/types'
import {
managedMigrationsAwsIamSetupRetrieve,
managedMigrationsPromoteCreate,
managedMigrationsTrialRecordsRetrieve,
} from './generated/api'
import type { BatchImportAWSIAMSetupApi, TrialRecordsResponseApi } from './generated/api.schemas'
import { ManagedMigration } from './types'
export const S3_ROLE_ARN_REGEX = /^arn:aws:iam::\d{12}:role\/.+$/
export interface ManagedMigrationForm {
source_type: 's3' | 's3_gzip' | 'mixpanel' | 'amplitude'
access_key?: string
secret_key?: string
content_type: 'captured' | 'mixpanel' | 'amplitude'
// s3 specific fields
s3_auth_method?: 'iam_role' | 'access_keys'
role_arn?: string
s3_region?: string
s3_bucket?: string
s3_prefix?: string
endpoint_url?: string
// date range specific fields
start_date?: string
end_date?: string
// EU region support for amplitude/mixpanel
is_eu_region?: boolean
// Amplitude-specific options
import_events?: boolean
generate_identify_events?: boolean
generate_group_identify_events?: boolean
// Trial run options
is_trial?: boolean
trial_record_limit?: number
}
export const TRIAL_RECORD_LIMIT_DEFAULT = 1000
export const TRIAL_RECORD_LIMIT_MAX = 50000
const NEW_MANAGED_MIGRATION: ManagedMigrationForm = {
source_type: 's3',
access_key: '',
secret_key: '',
// Promoted to 'iam_role' once aws_iam_setup confirms role auth is available, so the
// validator (which only sees form values) can never demand a role_arn the UI is hiding
s3_auth_method: 'access_keys',
role_arn: '',
s3_region: '',
s3_bucket: '',
s3_prefix: '',
endpoint_url: '',
content_type: 'captured',
start_date: '',
end_date: '',
is_eu_region: false,
import_events: true,
generate_identify_events: true,
generate_group_identify_events: true,
is_trial: false,
trial_record_limit: TRIAL_RECORD_LIMIT_DEFAULT,
}
// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
export interface managedMigrationLogicValues {
awsIamSetup: BatchImportAWSIAMSetupApi | null
awsIamSetupLoading: boolean
breadcrumbs: Breadcrumb[]
isManagedMigrationSubmitting: boolean
isManagedMigrationValid: boolean
isPolling: boolean
managedMigration: ManagedMigrationForm
managedMigrationAllErrors: Record<string, any>
managedMigrationChanged: boolean
managedMigrationErrors: DeepPartialMap<ManagedMigrationForm, ValidationErrorType>
managedMigrationHasErrors: boolean
managedMigrationId: string | null
managedMigrationManualErrors: Record<string, any>
managedMigrationTouched: boolean
managedMigrationTouches: Record<string, boolean>
managedMigrationValidationErrors: DeepPartialMap<ManagedMigrationForm, ValidationErrorType>
migrations: ManagedMigration[]
migrationsLoading: boolean
projectTreeRef: ProjectTreeRef
promotingMigrationId: string | null
showManagedMigrationErrors: boolean
trialRecords: TrialRecordsResponseApi | null
trialRecordsLoading: boolean
trialResultsId: string | null
}
// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
export interface managedMigrationLogicActions {
closeTrialResults: () => {
value: true
}
closeTrialResultsFailure: (
error: string,
errorObject?: any
) => {
error: string
errorObject?: any
}
closeTrialResultsSuccess: (
trialRecords: null,
payload?: {
value: true
}
) => {
trialRecords: null
payload?: {
value: true
}
}
confirmPromoteTrial: (id: string) => {
id: string
}
editManagedMigration: (id: string | null) => {
id: string | null
}
loadAwsIamSetup: () => any
loadAwsIamSetupFailure: (
error: string,
errorObject?: any
) => {
error: string
errorObject?: any
}
loadAwsIamSetupSuccess: (
awsIamSetup: BatchImportAWSIAMSetupApi,
payload?: any
) => {
awsIamSetup: BatchImportAWSIAMSetupApi
payload?: any
}
loadMigrations: () => any
loadMigrationsFailure: (
error: string,
errorObject?: any
) => {
error: string
errorObject?: any
}
loadMigrationsSuccess: (
migrations: ManagedMigration[],
payload?: any
) => {
migrations: ManagedMigration[]
payload?: any
}
loadTrialRecordsPage: ({ page }: { page: number }) => {
page: number
}
loadTrialRecordsPageFailure: (
error: string,
errorObject?: any
) => {
error: string
errorObject?: any
}
loadTrialRecordsPageSuccess: (
trialRecords: TrialRecordsResponseApi | null,
payload?: {
page: number
}
) => {
trialRecords: TrialRecordsResponseApi | null
payload?: {
page: number
}
}
pauseMigration: (id: string) => {
id: string
}
promoteTrial: (id: string) => {
id: string
}
promoteTrialFinished: () => {
value: true
}
resetManagedMigration: (values?: ManagedMigrationForm) => {
values?: ManagedMigrationForm
}
resumeMigration: (id: string) => {
id: string
}
setManagedMigrationManualErrors: (errors: Record<string, any>) => {
errors: Record<string, any>
}
setManagedMigrationValue: (
key: FieldName,
value: any
) => {
name: FieldName
value: any
}
setManagedMigrationValues: (values: DeepPartial<ManagedMigrationForm>) => {
values: DeepPartial<ManagedMigrationForm>
}
startPolling: () => {
value: true
}
stopPolling: () => {
value: true
}
submitManagedMigration: () => {
value: boolean
}
submitManagedMigrationFailure: (
error: Error,
errors: Record<string, any>
) => {
error: Error
errors: Record<string, any>
}
submitManagedMigrationRequest: (managedMigration: ManagedMigrationForm) => {
managedMigration: ManagedMigrationForm
}
submitManagedMigrationSuccess: (managedMigration: ManagedMigrationForm) => {
managedMigration: ManagedMigrationForm
}
touchManagedMigrationField: (key: string) => {
key: string
}
viewTrialResults: (id: string) => {
id: string
}
}
// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
export interface managedMigrationLogicProps {
managedMigrationId: null
}
// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
export interface managedMigrationLogicMeta {
__keaTypeGenInternalSelectorTypes: {
breadcrumbs: (managedMigrationId: null) => Breadcrumb[]
projectTreeRef: (managedMigrationId: null) => ProjectTreeRef
}
}
export type managedMigrationLogicType = MakeLogicType<
managedMigrationLogicValues,
managedMigrationLogicActions,
managedMigrationLogicProps,
managedMigrationLogicMeta
>
export const managedMigrationLogic = kea<managedMigrationLogicType>([
path(['products', 'managed_migrations', 'frontend', 'managedMigrationLogic']),
props({
managedMigrationId: null,
}),
actions({
editManagedMigration: (id: string | null) => ({ id }),
pauseMigration: (id: string) => ({ id }),
resumeMigration: (id: string) => ({ id }),
promoteTrial: (id: string) => ({ id }),
confirmPromoteTrial: (id: string) => ({ id }),
promoteTrialFinished: true,
viewTrialResults: (id: string) => ({ id }),
closeTrialResults: true,
startPolling: true,
stopPolling: true,
}),
reducers({
managedMigrationId: [null as string | null, { editManagedMigration: (_, { id }) => id }],
isPolling: [false, { startPolling: () => true, stopPolling: () => false }],
trialResultsId: [null as string | null, { viewTrialResults: (_, { id }) => id, closeTrialResults: () => null }],
promotingMigrationId: [
null as string | null,
{ confirmPromoteTrial: (_, { id }) => id, promoteTrialFinished: () => null },
],
}),
loaders(({ values }) => ({
migrations: [
[] as ManagedMigration[],
{
loadMigrations: async () => {
const projectId = ApiConfig.getCurrentProjectId()
// nosemgrep: prefer-codegen-api
const response = await api.get(`v1/projects/${projectId}/managed_migrations`)
return response.results
},
},
],
trialRecords: [
null as TrialRecordsResponseApi | null,
{
loadTrialRecordsPage: async ({ page }: { page: number }, breakpoint) => {
if (!values.trialResultsId) {
return null
}
await breakpoint(100)
const projectId = String(ApiConfig.getCurrentProjectId())
const response = await managedMigrationsTrialRecordsRetrieve(projectId, values.trialResultsId, {
page,
})
// Drop out-of-order responses so stale pages can't overwrite newer state.
breakpoint()
return response
},
closeTrialResults: () => null,
},
],
awsIamSetup: [
null as BatchImportAWSIAMSetupApi | null,
{
loadAwsIamSetup: async () => {
const projectId = ApiConfig.getCurrentProjectId()
return await managedMigrationsAwsIamSetupRetrieve(String(projectId))
},
},
],
})),
forms({
managedMigration: {
defaults: NEW_MANAGED_MIGRATION,
errors: ({
source_type,
content_type,
access_key,
secret_key,
s3_auth_method,
role_arn,
s3_region,
s3_bucket,
start_date,
end_date,
import_events,
generate_identify_events,
generate_group_identify_events,
is_trial,
trial_record_limit,
}: ManagedMigrationForm) => {
const usesIamRole = (source_type === 's3' || source_type === 's3_gzip') && s3_auth_method === 'iam_role'
const errors: Record<string, string | null> = {
secret_key:
!secret_key && !usesIamRole
? source_type === 'mixpanel'
? 'Project secret is required'
: 'Secret key is required'
: null,
}
if (is_trial) {
errors.trial_record_limit =
!trial_record_limit || trial_record_limit < 1 || trial_record_limit > TRIAL_RECORD_LIMIT_MAX
? `Number of records must be between 1 and ${TRIAL_RECORD_LIMIT_MAX}`
: null
}
// Mixpanel authenticates with the project secret alone — no access key.
if (source_type !== 'mixpanel' && !usesIamRole) {
errors.access_key = !access_key ? 'Access key is required' : null
}
if (usesIamRole) {
errors.role_arn = !role_arn
? 'IAM role ARN is required'
: !S3_ROLE_ARN_REGEX.test(role_arn)
? 'Enter a valid IAM role ARN, e.g. arn:aws:iam::123456789012:role/insights-import'
: null
}
if (source_type === 's3' || source_type === 's3_gzip') {
errors.s3_region = !s3_region ? 'S3 region is required' : null
errors.s3_bucket = !s3_bucket ? 'S3 bucket is required' : null
if (content_type === 'amplitude') {
if (!import_events && !generate_identify_events && !generate_group_identify_events) {
errors.import_events =
'At least one of "Import events", "Generate identify events", or "Generate group identify events" must be enabled'
}
}
} else if (source_type === 'mixpanel' || source_type === 'amplitude') {
errors.start_date = !start_date ? 'Start date is required' : null
errors.end_date = !end_date ? 'End date is required' : null
if (start_date && end_date) {
const startDateParsed = dayjs(start_date)
const endDateParsed = dayjs(end_date)
if (endDateParsed.isBefore(startDateParsed)) {
errors.end_date = 'End date must be after start date'
} else if (endDateParsed.diff(startDateParsed, 'year', true) > 1) {
errors.end_date =
'Date range cannot exceed 1 year. Please create multiple migration jobs for longer periods.'
} else if (
source_type === 'amplitude' &&
endDateParsed.diff(startDateParsed, 'hour', true) < 1
) {
errors.end_date = 'Date range must be at least 1 hour for Amplitude migrations.'
}
}
// For Amplitude, ensure at least one of import_events, generate_identify_events, or generate_group_identify_events is enabled
if (source_type === 'amplitude') {
if (!import_events && !generate_identify_events && !generate_group_identify_events) {
errors.import_events =
'At least one of "Import events", "Generate identify events", or "Generate group identify events" must be enabled'
}
}
}
return errors
},
submit: async (values: ManagedMigrationForm) => {
const projectId = ApiConfig.getCurrentProjectId()
const usesIamRole =
(values.source_type === 's3' || values.source_type === 's3_gzip') &&
values.s3_auth_method === 'iam_role'
let payload: ManagedMigrationForm = {
source_type: values.source_type,
content_type: values.content_type,
...(values.is_trial ? { is_trial: true, trial_record_limit: values.trial_record_limit } : {}),
...(usesIamRole
? { role_arn: values.role_arn }
: { access_key: values.access_key, secret_key: values.secret_key }),
}
if (values.source_type === 's3' || values.source_type === 's3_gzip') {
payload = {
...payload,
s3_region: values.s3_region,
s3_bucket: values.s3_bucket,
s3_prefix: values.s3_prefix,
...(values.endpoint_url && !usesIamRole ? { endpoint_url: values.endpoint_url } : {}),
}
if (values.content_type === 'amplitude') {
payload.import_events = values.import_events
payload.generate_identify_events = values.generate_identify_events
payload.generate_group_identify_events = values.generate_group_identify_events
}
} else if (values.source_type === 'mixpanel' || values.source_type === 'amplitude') {
payload = {
...payload,
start_date: values.start_date,
end_date: values.end_date,
is_eu_region: values.is_eu_region,
}
// Only include Amplitude-specific options for Amplitude migrations
if (values.source_type === 'amplitude') {
payload.import_events = values.import_events
payload.generate_identify_events = values.generate_identify_events
payload.generate_group_identify_events = values.generate_group_identify_events
}
}
try {
// nosemgrep: prefer-codegen-api
const response = await api.create(`v1/projects/${projectId}/managed_migrations`, payload)
return response
} catch (error: any) {
if (error.status === 400 && error.data?.error) {
throw new Error(error.data.error)
}
throw error
}
},
},
}),
listeners(({ actions, values, cache }) => ({
loadAwsIamSetupSuccess: ({ awsIamSetup }) => {
// Default to the recommended role auth once we know it's available, unless the
// user already started entering access keys while the setup call was in flight
if (awsIamSetup?.available && !values.managedMigration.access_key && !values.managedMigration.secret_key) {
actions.setManagedMigrationValue('s3_auth_method', 'iam_role')
}
},
submitManagedMigrationSuccess: async ({ managedMigration }) => {
if (managedMigration?.is_trial) {
toast.success('Trial run started — results will appear here when it completes')
}
actions.loadMigrations()
router.actions.push(urls.managedMigration())
},
submitManagedMigrationFailure: async ({ error }) => {
if (error?.message) {
toast.error(error.message)
} else {
toast.error('Failed to create migration. Please try again.')
}
},
pauseMigration: async ({ id }) => {
try {
const projectId = ApiConfig.getCurrentProjectId()
// nosemgrep: prefer-codegen-api
await api.create(`v1/projects/${projectId}/managed_migrations/${id}/pause/`)
toast.success('Migration paused successfully')
actions.loadMigrations()
} catch (error: any) {
toast.error(error?.message || 'Failed to pause migration')
}
},
resumeMigration: async ({ id }) => {
try {
const projectId = ApiConfig.getCurrentProjectId()
// nosemgrep: prefer-codegen-api
await api.create(`v1/projects/${projectId}/managed_migrations/${id}/resume/`)
toast.success('Migration resumed successfully')
actions.loadMigrations()
} catch (error: any) {
toast.error(error?.message || 'Failed to resume migration')
}
},
viewTrialResults: () => {
actions.loadTrialRecordsPage({ page: 0 })
},
loadTrialRecordsPageFailure: async ({ errorObject }) => {
if (errorObject?.status === 410) {
toast.error('Trial results have expired and are no longer available')
} else {
toast.error('Failed to load trial results')
}
actions.closeTrialResults()
},
promoteTrial: ({ id }) => {
if (values.promotingMigrationId) {
return
}
const trial = values.migrations.find((migration: ManagedMigration) => migration.id === id)
const failedRecords = trial?.state?.trial?.summary?.dropped_records ?? 0
if (failedRecords === 0) {
actions.confirmPromoteTrial(id)
return
}
Dialog.open({
title: 'Start the full import?',
description: `This trial found ${failedRecords} record${
failedRecords === 1 ? '' : 's'
} that can't be imported. These errors are not retriable: the import will pause when it reaches the first failing record, and stay paused until the source data is fixed.`,
primaryButton: {
children: 'Start import anyway',
type: 'primary',
onClick: () => actions.confirmPromoteTrial(id),
},
secondaryButton: { children: 'Cancel' },
})
},
confirmPromoteTrial: async ({ id }) => {
try {
const projectId = String(ApiConfig.getCurrentProjectId())
await managedMigrationsPromoteCreate(projectId, id)
toast.success('Import started from trial run')
actions.loadMigrations()
} catch (error: any) {
toast.error(error?.data?.error || error?.message || 'Failed to start import from trial')
} finally {
actions.promoteTrialFinished()
}
},
loadMigrationsSuccess: () => {
const hasActiveMigrations = values.migrations.some(
(migration: ManagedMigration) =>
migration.display_status === 'running' || migration.display_status === 'waiting_to_start'
)
if (hasActiveMigrations && !values.isPolling) {
actions.startPolling()
} else if (!hasActiveMigrations && values.isPolling) {
actions.stopPolling()
}
},
startPolling: () => {
cache.disposables.add(() => {
const intervalId = setInterval(() => {
actions.loadMigrations()
}, 5000)
return () => clearInterval(intervalId)
}, 'pollMigrations')
},
stopPolling: () => {
cache.disposables.dispose('pollMigrations')
},
})),
selectors({
breadcrumbs: [
(_, p) => [p.managedMigrationId],
(managedMigrationId: string | null): Breadcrumb[] => [
{
key: 'managed-migrations',
name: 'Managed Migrations',
path: urls.managedMigration(),
iconType: 'data_pipeline_metadata',
},
...(managedMigrationId
? [
{
key: 'edit-migration',
name: 'Manage migration',
path: urls.managedMigration(),
iconType: 'data_pipeline_metadata' as FileSystemIconType,
},
]
: []),
],
],
projectTreeRef: [
(_, p) => [p.managedMigrationId],
(managedMigrationId: string | null): ProjectTreeRef => ({
type: 'managed-migration',
ref: managedMigrationId,
}),
],
}),
urlToAction(({ actions }) => ({
[urls.managedMigrationNew()]: () => {
actions.editManagedMigration('new')
},
[`${urls.managedMigration()}/:id`]: ({ id }) => {
actions.editManagedMigration(id ?? null)
},
[urls.managedMigration()]: () => {
actions.editManagedMigration(null)
},
})),
afterMount(({ actions }) => {
actions.loadMigrations()
actions.loadAwsIamSetup()
}),
beforeUnmount(({ actions }) => {
actions.stopPolling()
}),
])