diff --git a/web-common/src/features/dashboards/DashboardAssets.svelte b/web-common/src/features/dashboards/DashboardAssets.svelte index 9d496c078fe..88e0ed85a19 100644 --- a/web-common/src/features/dashboards/DashboardAssets.svelte +++ b/web-common/src/features/dashboards/DashboardAssets.svelte @@ -8,8 +8,8 @@ import { MenuItem } from "@rilldata/web-common/components/menu"; import { Divider } from "@rilldata/web-common/components/menu/index.js"; import { useDashboardNames } from "@rilldata/web-common/features/dashboards/selectors"; - import { deleteFileArtifact } from "@rilldata/web-common/features/entity-management/actions"; import { getFilePathFromNameAndType } from "@rilldata/web-common/features/entity-management/entity-mappers"; + import { deleteFile } from "@rilldata/web-common/features/entity-management/file-actions"; import { FileArtifactsData, fileArtifactsStore, @@ -142,13 +142,10 @@ $fileArtifactsStore.entities, dashboardName ); - await deleteFileArtifact( - queryClient, + await deleteFile( instanceId, dashboardName, EntityType.MetricsDefinition, - $deleteDashboard, - $appScreen, $dashboardNames.data ); diff --git a/web-common/src/features/entity-management/RenameAssetModal.svelte b/web-common/src/features/entity-management/RenameAssetModal.svelte index 464e83303ba..071b0f3f1f6 100644 --- a/web-common/src/features/entity-management/RenameAssetModal.svelte +++ b/web-common/src/features/entity-management/RenameAssetModal.svelte @@ -3,16 +3,12 @@ import Input from "@rilldata/web-common/components/forms/Input.svelte"; import SubmissionError from "@rilldata/web-common/components/forms/SubmissionError.svelte"; import { Dialog } from "@rilldata/web-common/components/modal/index"; + import { renameFile } from "@rilldata/web-common/features/entity-management/rename-entity"; import type { EntityType } from "@rilldata/web-common/features/entity-management/types"; - import { useQueryClient } from "@tanstack/svelte-query"; import { createForm } from "svelte-forms-lib"; import * as yup from "yup"; - import { - createRuntimeServiceGetCatalogEntry, - createRuntimeServiceRenameFileAndReconcile, - } from "../../runtime-client"; + import { createRuntimeServiceRenameFile } from "../../runtime-client"; import { runtime } from "../../runtime-client/runtime-store"; - import { renameFileArtifact } from "./actions"; import { getLabel, getRouteFromName } from "./entity-mappers"; import { isDuplicateName } from "./name-utils"; import { useAllNames } from "./selectors"; @@ -21,18 +17,12 @@ export let entityType: EntityType; export let currentAssetName: string; - const queryClient = useQueryClient(); - let error: string; $: runtimeInstanceId = $runtime.instanceId; - $: getCatalog = createRuntimeServiceGetCatalogEntry( - runtimeInstanceId, - currentAssetName - ); $: allNamesQuery = useAllNames(runtimeInstanceId); - const renameAsset = createRuntimeServiceRenameFileAndReconcile(); + const renameAsset = createRuntimeServiceRenameFile(); const { form, errors, handleSubmit } = createForm({ initialValues: { @@ -56,13 +46,12 @@ return; } try { - await renameFileArtifact( - queryClient, + await renameFile( runtimeInstanceId, currentAssetName, values.newName, entityType, - $renameAsset + renameAsset ); goto(getRouteFromName(values.newName, entityType), { replaceState: true, diff --git a/web-common/src/features/entity-management/actions.ts b/web-common/src/features/entity-management/actions.ts deleted file mode 100644 index 7949cf417b0..00000000000 --- a/web-common/src/features/entity-management/actions.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { goto } from "$app/navigation"; -import { notifications } from "@rilldata/web-common/components/notifications"; -import type { ActiveEntity } from "@rilldata/web-common/layout/app-store"; -import { currentHref } from "@rilldata/web-common/layout/navigation/stores"; -import type { - V1DeleteFileAndReconcileResponse, - V1RenameFileAndReconcileResponse, -} from "@rilldata/web-common/runtime-client"; -import { httpRequestQueue } from "@rilldata/web-common/runtime-client/http-client"; -import { - invalidateAfterReconcile, - removeEntityQueries, -} from "@rilldata/web-common/runtime-client/invalidation"; -import type { - CreateBaseMutationResult, - QueryClient, -} from "@tanstack/svelte-query"; -import { - getFilePathFromNameAndType, - getLabel, - getRouteFromName, -} from "./entity-mappers"; -import { fileArtifactsStore } from "./file-artifacts-store"; -import { getNextEntityName } from "./name-utils"; -import type { EntityType } from "./types"; - -export async function renameFileArtifact( - queryClient: QueryClient, - instanceId: string, - fromName: string, - toName: string, - type: EntityType, - renameMutation: CreateBaseMutationResult -) { - const resp = await renameMutation.mutateAsync({ - data: { - instanceId, - fromPath: getFilePathFromNameAndType(fromName, type), - toPath: getFilePathFromNameAndType(toName, type), - }, - }); - fileArtifactsStore.setErrors(resp.affectedPaths, resp.errors); - - httpRequestQueue.removeByName(fromName); - notifications.send({ - message: `Renamed ${getLabel(type)} ${fromName} to ${toName}`, - }); - - removeEntityQueries( - queryClient, - instanceId, - getFilePathFromNameAndType(fromName, type) - ); - invalidateAfterReconcile(queryClient, instanceId, resp); -} - -export async function deleteFileArtifact( - queryClient: QueryClient, - instanceId: string, - name: string, - type: EntityType, - deleteMutation: CreateBaseMutationResult, - activeEntity: ActiveEntity, - names: Array, - showNotification = true -) { - const path = getFilePathFromNameAndType(name, type); - try { - const resp = await deleteMutation.mutateAsync({ - data: { - instanceId, - path, - }, - }); - fileArtifactsStore.setErrors(resp.affectedPaths, resp.errors); - - httpRequestQueue.removeByName(name); - if (showNotification) { - notifications.send({ message: `Deleted ${getLabel(type)} ${name}` }); - } - - removeEntityQueries(queryClient, instanceId, path); - - invalidateAfterReconcile(queryClient, instanceId, resp); - if (activeEntity?.name === name) { - const route = getRouteFromName(getNextEntityName(names, name), type); - /** set the href store so the menu selection has an immediate visual update. */ - currentHref.set(route); - goto(route); - } - } catch (err) { - console.error(err); - } -} diff --git a/web-common/src/features/entity-management/entity-action-queue.ts b/web-common/src/features/entity-management/entity-action-queue.ts new file mode 100644 index 00000000000..d0992a3d4f0 --- /dev/null +++ b/web-common/src/features/entity-management/entity-action-queue.ts @@ -0,0 +1,107 @@ +import { ResourceKind } from "@rilldata/web-common/features/entity-management/resource-selectors"; +import { sourceIngestionTelemetry } from "@rilldata/web-common/features/sources/source-ingestion-telemetry"; +import type { TelemetryParams } from "@rilldata/web-common/metrics/service/metrics-helpers"; +import type { V1ResourceEvent } from "@rilldata/web-common/runtime-client"; +import type { V1Resource } from "@rilldata/web-common/runtime-client"; +import { Readable, writable } from "svelte/store"; + +export enum EntityAction { + Create, + Update, + Rename, + Delete, +} + +export type EntityCreateFunction = ( + resource: V1Resource, + sourceName: string, + pathPrefix?: string +) => Promise; + +export type ChainParams = { + chainFunction: EntityCreateFunction; + sourceName: string; + pathPrefix?: string; +}; + +/** + * A global queue for entity actions. + * This is used to emit telemetry for async response from reconcile. + */ +export type EntityActionQueueState = { + entities: Record>; +}; +export type EntityActionInstance = { + action: EntityAction; + + // telemetry + telemetryParams: TelemetryParams; + + chainParams?: ChainParams; +}; +type EntityActionQueueReducers = { + add: ( + name: string, + action: EntityAction, + telemetryParams: TelemetryParams, + chainParams?: ChainParams + ) => void; + resolved: (resource: V1Resource, event: V1ResourceEvent) => void; +}; +export type EntityActionQueueStore = Readable & + EntityActionQueueReducers; + +// TODO: how does reconcile handle cases like, create source => rename soon after before ingestion is completed +const { update, subscribe } = writable({ + entities: {}, +}); + +export const entityActionQueueStore: EntityActionQueueStore = { + subscribe, + + add( + name: string, + action: EntityAction, + telemetryParams: TelemetryParams, + chainParams?: ChainParams + ) { + update((state) => { + state.entities[name] ??= []; + state.entities[name].push({ + action, + telemetryParams, + chainParams, + }); + return state; + }); + }, + + resolved(resource: V1Resource, _: V1ResourceEvent) { + update((state) => { + if (!state.entities[resource.meta.name.name]?.length) return state; + + if (resource.meta.renamedFrom) { + // TODO: rename telemetry + return state; + } + + const action = state.entities[resource.meta.name.name].shift(); + + switch (resource.meta.name.kind) { + case ResourceKind.Source: + sourceIngestionTelemetry(resource, action); + break; + } + + if (action.chainParams) { + action.chainParams.chainFunction( + resource, + action.chainParams.sourceName, + action.chainParams.pathPrefix + ); + } + + return state; + }); + }, +}; diff --git a/web-common/src/features/entity-management/file-actions.ts b/web-common/src/features/entity-management/file-actions.ts new file mode 100644 index 00000000000..1f96ad33c03 --- /dev/null +++ b/web-common/src/features/entity-management/file-actions.ts @@ -0,0 +1,54 @@ +import { goto } from "$app/navigation"; +import { notifications } from "@rilldata/web-common/components/notifications"; +import { + getFilePathFromNameAndType, + getLabel, + getRouteFromName, +} from "@rilldata/web-common/features/entity-management/entity-mappers"; +import { getNextEntityName } from "@rilldata/web-common/features/entity-management/name-utils"; +import type { EntityType } from "@rilldata/web-common/features/entity-management/types"; +import { appScreen } from "@rilldata/web-common/layout/app-store"; +import { currentHref } from "@rilldata/web-common/layout/navigation/stores"; +import { runtimeServiceDeleteFile } from "@rilldata/web-common/runtime-client"; +import type { createRuntimeServicePutFile } from "@rilldata/web-common/runtime-client"; +import { get } from "svelte/store"; + +export async function saveFile( + instanceId: string, + name: string, + type: EntityType, + blob: string, + saveMutation: ReturnType +) { + const filePath = getFilePathFromNameAndType(name, type); + + await get(saveMutation).mutateAsync({ + instanceId, + data: { + blob, + create: false, + createOnly: false, + }, + path: filePath, + }); +} + +export async function deleteFile( + instanceId: string, + name: string, + type: EntityType, + names: Array +) { + const path = getFilePathFromNameAndType(name, type); + await runtimeServiceDeleteFile(instanceId, path); + + notifications.send({ message: `Deleted ${getLabel(type)} ${name}` }); + + // only redirect if the deleted entity is in focus + if (get(appScreen)?.name !== name) return; + + const route = getRouteFromName(getNextEntityName(names, name), type); + /** set the href store so the menu selection has an immediate visual update. */ + currentHref.set(route); + goto(route); +} diff --git a/web-common/src/features/entity-management/rename-entity.ts b/web-common/src/features/entity-management/rename-entity.ts new file mode 100644 index 00000000000..fb335ec6a61 --- /dev/null +++ b/web-common/src/features/entity-management/rename-entity.ts @@ -0,0 +1,72 @@ +import { goto } from "$app/navigation"; +import { notifications } from "@rilldata/web-common/components/notifications"; +import { + getFilePathFromNameAndType, + getLabel, + getRouteFromName, +} from "@rilldata/web-common/features/entity-management/entity-mappers"; +import { isDuplicateName } from "@rilldata/web-common/features/entity-management/name-utils"; +import { EntityType } from "@rilldata/web-common/features/entity-management/types"; +import type { createRuntimeServiceRenameFile } from "@rilldata/web-common/runtime-client"; +import { httpRequestQueue } from "@rilldata/web-common/runtime-client/http-client"; +import { get } from "svelte/types/runtime/store"; + +export async function validateAndRenameEntity( + instanceId: string, + fromName: string, + toName: string, + allNames: Array, + entityType: EntityType, + renameMutation: ReturnType +): Promise { + if (!toName.match(/^[a-zA-Z_][a-zA-Z0-9_]*$/)) { + notifications.send({ + message: `${getLabel( + entityType + )} name must start with a letter or underscore and contain only letters, numbers, and underscores`, + }); + return false; + } + + if (isDuplicateName(toName, fromName, allNames)) { + notifications.send({ + message: `Name ${toName} is already in use`, + }); + return false; + } + + try { + await renameFile(instanceId, fromName, toName, entityType, renameMutation); + } catch (err) { + console.error(err.response.data.message); + } + + return true; +} + +export async function renameFile( + instanceId: string, + fromName: string, + toName: string, + entityType: EntityType, + renameMutation: ReturnType +) { + await get(renameMutation).mutateAsync({ + instanceId, + data: { + fromPath: getFilePathFromNameAndType(fromName, entityType), + toPath: getFilePathFromNameAndType(toName, entityType), + }, + }); + + httpRequestQueue.removeByName(fromName); + notifications.send({ + message: `Renamed ${getLabel(entityType)} ${fromName} to ${toName}`, + }); + + const route = getRouteFromName(toName, entityType); + goto(entityType === EntityType.MetricsDefinition ? `${route}/edit` : route, { + replaceState: true, + }); + // TODO: no telemetry for rename? +} diff --git a/web-common/src/features/entity-management/resource-selectors.ts b/web-common/src/features/entity-management/resource-selectors.ts new file mode 100644 index 00000000000..fe4916c70e5 --- /dev/null +++ b/web-common/src/features/entity-management/resource-selectors.ts @@ -0,0 +1,53 @@ +import { + createRuntimeServiceGetResource, + createRuntimeServiceListResources, +} from "@rilldata/web-common/runtime-client"; + +export enum ResourceKind { + Source = "source", + Model = "model", + MetricsView = "metricsview", + // TODO: do a correct map based on backend code +} + +export function useSource(instanceId: string, name: string) { + return createRuntimeServiceGetResource( + instanceId, + { + "name.kind": ResourceKind.Source, + "name.name": name, + }, + { + query: { + select: (data) => data?.resource?.source, + }, + } + ); +} + +export function useFilteredEntityNames(instanceId: string, kind: ResourceKind) { + return createRuntimeServiceListResources( + instanceId, + { + kind, + }, + { + query: { + select: (data) => data.resources.map((res) => res.meta.name.name), + }, + } + ); +} + +// TODO: replace usage of this with appropriate ones for de-duping names +export function useAllEntityNames(instanceId: string) { + return createRuntimeServiceListResources( + instanceId, + {}, + { + query: { + select: (data) => data.resources.map((res) => res.meta.name.name), + }, + } + ); +} diff --git a/web-common/src/features/entity-management/watch-files-client.ts b/web-common/src/features/entity-management/watch-files-client.ts index de42e0006c0..4876e030546 100644 --- a/web-common/src/features/entity-management/watch-files-client.ts +++ b/web-common/src/features/entity-management/watch-files-client.ts @@ -2,6 +2,7 @@ import { WatchRequestClient } from "@rilldata/web-common/runtime-client/watch-re import { getRuntimeServiceGetFileQueryKey, getRuntimeServiceListFilesQueryKey, + V1FileEvent, V1WatchFilesResponse, } from "@rilldata/web-common/runtime-client"; import { runtime } from "@rilldata/web-common/runtime-client/runtime-store"; @@ -33,13 +34,13 @@ function handleWatchFileResponse( // invalidations will wait until the re-fetched query is completed // so, we should not `await` here on `refetchQueries` switch (res.event) { - case "FILE_EVENT_WRITE": + case V1FileEvent.FILE_EVENT_WRITE: queryClient.refetchQueries( getRuntimeServiceGetFileQueryKey(instanceId, res.path) ); break; - case "FILE_EVENT_DELETE": + case V1FileEvent.FILE_EVENT_DELETE: queryClient.removeQueries( getRuntimeServiceGetFileQueryKey(instanceId, res.path) ); diff --git a/web-common/src/features/entity-management/watch-resources-client.ts b/web-common/src/features/entity-management/watch-resources-client.ts index a09de797819..c5de68fff5f 100644 --- a/web-common/src/features/entity-management/watch-resources-client.ts +++ b/web-common/src/features/entity-management/watch-resources-client.ts @@ -1,8 +1,10 @@ +import { entityActionQueueStore } from "@rilldata/web-common/features/entity-management/entity-action-queue"; import { WatchRequestClient } from "@rilldata/web-common/runtime-client/watch-request-client"; import { getRuntimeServiceGetResourceQueryKey, getRuntimeServiceListResourcesQueryKey, V1Resource, + V1ResourceEvent, V1WatchResourcesResponse, } from "@rilldata/web-common/runtime-client"; import { runtime } from "@rilldata/web-common/runtime-client/runtime-store"; @@ -24,15 +26,17 @@ function handleWatchResourceResponse( ) { if (!res.resource) return; + entityActionQueueStore.resolved(res.resource, res.event); + const instanceId = get(runtime).instanceId; // invalidations will wait until the re-fetched query is completed // so, we should not `await` here switch (res.event) { - case "RESOURCE_EVENT_WRITE": + case V1ResourceEvent.RESOURCE_EVENT_WRITE: invalidateResource(queryClient, instanceId, res.resource); break; - case "RESOURCE_EVENT_DELETE": + case V1ResourceEvent.RESOURCE_EVENT_DELETE: invalidateRemovedResource(queryClient, instanceId, res.resource); break; } diff --git a/web-common/src/features/metrics-views/column-selectors.ts b/web-common/src/features/metrics-views/column-selectors.ts index 1e462eb840e..5b62682a54c 100644 --- a/web-common/src/features/metrics-views/column-selectors.ts +++ b/web-common/src/features/metrics-views/column-selectors.ts @@ -1,6 +1,7 @@ import { TIMESTAMPS } from "@rilldata/web-common/lib/duckdb-data-types"; import type { StructTypeField, + V1ProfileColumn, V1StructType, } from "@rilldata/web-common/runtime-client"; @@ -11,3 +12,9 @@ const isFieldColumnATimestamp = (field: StructTypeField) => export const selectTimestampColumnFromSchema = (schema: V1StructType) => (schema?.fields?.filter(isFieldColumnATimestamp) ?? []).map((f) => f.name); + +const isFieldColumnATimestampV2 = (column: V1ProfileColumn) => + TIMESTAMPS.has(column.type); +export const selectTimestampColumnFromSchemaV2 = ( + columns: Array +) => (columns.filter(isFieldColumnATimestampV2) ?? []).map((c) => c.name); diff --git a/web-common/src/features/metrics-views/metrics-internal-store.ts b/web-common/src/features/metrics-views/metrics-internal-store.ts index 87cbccdb528..2b3c6daae0f 100644 --- a/web-common/src/features/metrics-views/metrics-internal-store.ts +++ b/web-common/src/features/metrics-views/metrics-internal-store.ts @@ -1,8 +1,15 @@ import { CATEGORICALS } from "@rilldata/web-common/lib/duckdb-data-types"; import { DEFAULT_TIMEZONES } from "@rilldata/web-common/lib/time/config"; -import type { V1Model } from "@rilldata/web-common/runtime-client"; +import type { + V1Model, + V1ProfileColumn, + V1TableColumnsResponse, +} from "@rilldata/web-common/runtime-client"; import { Document, parseDocument } from "yaml"; -import { selectTimestampColumnFromSchema } from "./column-selectors"; +import { + selectTimestampColumnFromSchema, + selectTimestampColumnFromSchemaV2, +} from "./column-selectors"; export interface MetricsConfig extends MetricsParams { measures: MeasureEntity[]; @@ -114,3 +121,55 @@ export function generateDashboardYAMLForModel( return doc.toString({ collectionStyle: "block" }); } + +export function generateDashboardYAMLForModelV2( + modelName: string, + columns: Array, + dashboardTitle = "" +) { + const doc = new Document(); + + doc.commentBefore = ` Visit https://docs.rilldata.com/reference/project-files to learn more about Rill project files.`; + + if (dashboardTitle) { + doc.set("title", dashboardTitle); + } + doc.set("model", modelName); + + const timestampColumns = selectTimestampColumnFromSchemaV2(columns); + if (timestampColumns?.length) { + doc.set("timeseries", timestampColumns[0]); + } else { + doc.set("timeseries", ""); + } + + const measureNode = doc.createNode({ + label: "Total records", + expression: "count(*)", + name: "total_records", + description: "Total number of records present", + format_preset: "humanize", + valid_percent_of_total: true, + }); + doc.set("measures", [measureNode]); + + const dimensionSeq = columns + .filter((c) => { + return CATEGORICALS.has(c.type); + }) + .map((c) => { + return { + name: c.name, + label: capitalize(c.name), + column: c.name, + description: "", + }; + }); + + const dimensionNode = doc.createNode(dimensionSeq); + doc.set("dimensions", dimensionNode); + + doc.set("available_time_zones", DEFAULT_TIMEZONES); + + return doc.toString({ collectionStyle: "block" }); +} diff --git a/web-common/src/features/metrics-views/workspace/MetricsWorkspaceHeader.svelte b/web-common/src/features/metrics-views/workspace/MetricsWorkspaceHeader.svelte index 7851d34f3fc..0aaab52dfe7 100644 --- a/web-common/src/features/metrics-views/workspace/MetricsWorkspaceHeader.svelte +++ b/web-common/src/features/metrics-views/workspace/MetricsWorkspaceHeader.svelte @@ -1,13 +1,9 @@ diff --git a/web-common/src/features/sources/createModel.ts b/web-common/src/features/sources/createModel.ts deleted file mode 100644 index b9c088e0637..00000000000 --- a/web-common/src/features/sources/createModel.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { getName } from "@rilldata/web-common/features/entity-management/name-utils"; -import { createModel } from "@rilldata/web-common/features/models/createModel"; -import type { - CreateBaseMutationResult, - QueryClient, -} from "@tanstack/svelte-query"; -import { get } from "svelte/store"; -import { notifications } from "../../components/notifications"; -import { - runtimeServicePutFileAndReconcile, - type V1PutFileAndReconcileResponse, -} from "../../runtime-client"; -import { invalidateAfterReconcile } from "../../runtime-client/invalidation"; -import { runtime } from "../../runtime-client/runtime-store"; -import { getFilePathFromNameAndType } from "../entity-management/entity-mappers"; -import { fileArtifactsStore } from "../entity-management/file-artifacts-store"; -import { EntityType } from "../entity-management/types"; -import { getModelNames } from "../models/selectors"; - -export async function createModelFromSource( - queryClient: QueryClient, - instanceId: string, - modelNames: Array, - sourceName: string, - sourceNameInQuery: string, - createModelMutation: CreateBaseMutationResult, // TODO: type - setAsActive = true -): Promise { - const newModelName = getName(`${sourceName}_model`, modelNames); - await createModel( - queryClient, - instanceId, - newModelName, - createModelMutation, - `select * from ${sourceNameInQuery}`, - setAsActive - ); - notifications.send({ - message: `Queried ${sourceNameInQuery} in workspace`, - }); - return newModelName; -} - -export async function createModelFromSourceV2( - queryClient: QueryClient, - sourceName: string -): Promise { - const instanceId = get(runtime).instanceId; - - // Get new model name - const modelNames = await getModelNames(queryClient, instanceId); - const newModelName = getName(`${sourceName}_model`, modelNames); - - // Create model - const resp = await runtimeServicePutFileAndReconcile({ - instanceId, - path: getFilePathFromNameAndType(newModelName, EntityType.Model), - blob: `select * from ${sourceName}`, - createOnly: true, - strict: true, - }); - - // Handle errors - fileArtifactsStore.setErrors(resp.affectedPaths, resp.errors); - - // Invalidate relevant queries - invalidateAfterReconcile(queryClient, instanceId, resp); - - // Done - return newModelName; -} diff --git a/web-common/src/features/sources/createModelFromSource.ts b/web-common/src/features/sources/createModelFromSource.ts new file mode 100644 index 00000000000..04754a00ea8 --- /dev/null +++ b/web-common/src/features/sources/createModelFromSource.ts @@ -0,0 +1,58 @@ +import type { EntityCreateFunction } from "@rilldata/web-common/features/entity-management/entity-action-queue"; +import { + EntityAction, + entityActionQueueStore, +} from "@rilldata/web-common/features/entity-management/entity-action-queue"; +import { getName } from "@rilldata/web-common/features/entity-management/name-utils"; +import { createModelCreator } from "@rilldata/web-common/features/models/createModel"; +import type { TelemetryParams } from "@rilldata/web-common/metrics/service/metrics-helpers"; +import type { V1Resource } from "@rilldata/web-common/runtime-client"; +import type { CreateQueryResult } from "@tanstack/svelte-query"; +import { get } from "svelte/store"; +import { notifications } from "../../components/notifications"; + +export function createModelFromSourceCreator( + allNamesQuery: CreateQueryResult>, + telemetryParams?: TelemetryParams, + chainFunction?: EntityCreateFunction +) { + const modelCreator = createModelCreator(telemetryParams); + + // getting the pathPrefix from the argument makes it easy to add folders + return async ( + source: V1Resource, + sourceName: string, + pathPrefix?: string + ) => { + const modelPathPrefix = pathPrefix ?? "/models/"; + + const newModelName = getName( + `${sourceName}_model`, + get(allNamesQuery).data + ); + + if (chainFunction) { + // add the chain with telemetry params + entityActionQueueStore.add( + newModelName, + EntityAction.Create, + telemetryParams, + { + chainFunction, + sourceName, + // pass in the original path prefix and not the defaulted one from the beginning of the function + pathPrefix, + } + ); + } + + await modelCreator( + newModelName, + modelPathPrefix, + `select * from ${sourceName}` + ); + notifications.send({ + message: `Queried ${sourceName} in workspace`, + }); + }; +} diff --git a/web-common/src/features/sources/inspector/SourceInspector.svelte b/web-common/src/features/sources/inspector/SourceInspector.svelte index 11a380418a2..6fec4995369 100644 --- a/web-common/src/features/sources/inspector/SourceInspector.svelte +++ b/web-common/src/features/sources/inspector/SourceInspector.svelte @@ -6,6 +6,7 @@ } from "@rilldata/web-common/components/column-profile/queries"; import Tooltip from "@rilldata/web-common/components/tooltip/Tooltip.svelte"; import TooltipContent from "@rilldata/web-common/components/tooltip/TooltipContent.svelte"; + import { useSource } from "@rilldata/web-common/features/entity-management/resource-selectors"; import CollapsibleSectionTitle from "@rilldata/web-common/layout/CollapsibleSectionTitle.svelte"; import { formatBigNumberPercentage, @@ -14,9 +15,7 @@ import { createQueryServiceTableCardinality, createQueryServiceTableColumns, - createRuntimeServiceGetCatalogEntry, - V1CatalogEntry, - V1Source, + V1SourceV2, } from "@rilldata/web-common/runtime-client"; import type { Readable } from "svelte/store"; import { slide } from "svelte/transition"; @@ -30,12 +29,9 @@ $: runtimeInstanceId = $runtime.instanceId; - $: getSource = createRuntimeServiceGetCatalogEntry( - runtimeInstanceId, - sourceName - ); - let sourceCatalog: V1CatalogEntry; - $: sourceCatalog = $getSource?.data?.entry; + $: sourceQuery = useSource(runtimeInstanceId, sourceName); + let source: V1SourceV2; + $: source = $sourceQuery.data; let showColumns = true; @@ -63,8 +59,8 @@ } } - function getFileExtension(source: V1Source): string { - const path = source?.properties?.path?.toLowerCase(); + function getFileExtension(source: V1SourceV2): string { + const path = source?.spec?.properties?.path?.toLowerCase(); if (path?.includes(".csv")) return "CSV"; if (path?.includes(".parquet")) return "Parquet"; if (path?.includes(".json")) return "JSON"; @@ -72,8 +68,8 @@ return ""; } - $: connectorType = formatConnectorType(sourceCatalog?.source?.connector); - $: fileExtension = getFileExtension(sourceCatalog); + $: connectorType = formatConnectorType(source?.spec?.sourceConnector); + $: fileExtension = getFileExtension(source); $: cardinalityQuery = createQueryServiceTableCardinality( $runtime.instanceId, @@ -90,13 +86,6 @@ }`; } - /** get the current column count */ - $: { - columnCount = `${formatInteger( - sourceCatalog?.source?.schema?.fields?.length - )} columns`; - } - /** total % null cells */ $: profileColumns = createQueryServiceTableColumns( @@ -108,6 +97,11 @@ let summaries: Readable>; $: if ($profileColumns?.data?.profileColumns) { + /** get the current column count */ + columnCount = `${formatInteger( + $profileColumns.data.profileColumns.length + )} columns`; + summaries = getSummaries(sourceName, $runtime?.instanceId, $profileColumns); } @@ -121,7 +115,7 @@ } $: { const totalCells = - sourceCatalog?.source?.schema?.fields?.length * cardinality; + $profileColumns?.data?.profileColumns?.length * cardinality; nullPercentage = formatBigNumberPercentage(totalNulls / totalCells); } @@ -138,7 +132,7 @@
- {#if sourceCatalog && !$getSource.isError} + {#if source && !$sourceQuery.isError}
diff --git a/web-common/src/features/sources/modal/FileDrop.svelte b/web-common/src/features/sources/modal/FileDrop.svelte index 4b42058b4ba..dc73a78fdca 100644 --- a/web-common/src/features/sources/modal/FileDrop.svelte +++ b/web-common/src/features/sources/modal/FileDrop.svelte @@ -1,43 +1,25 @@ diff --git a/web-common/src/features/sources/modal/LocalSourceUpload.svelte b/web-common/src/features/sources/modal/LocalSourceUpload.svelte index 57fdefea0bb..768735f6e7e 100644 --- a/web-common/src/features/sources/modal/LocalSourceUpload.svelte +++ b/web-common/src/features/sources/modal/LocalSourceUpload.svelte @@ -6,44 +6,27 @@ uploadTableFiles, } from "@rilldata/web-common/features/sources/modal/file-upload"; import { useSourceNames } from "@rilldata/web-common/features/sources/selectors"; - import { appScreen } from "@rilldata/web-common/layout/app-store"; import { overlay } from "@rilldata/web-common/layout/overlay-store"; - import { - createRuntimeServicePutFileAndReconcile, - createRuntimeServiceUnpackEmpty, - } from "@rilldata/web-common/runtime-client"; - import { useQueryClient } from "@tanstack/svelte-query"; + import { BehaviourEventMedium } from "@rilldata/web-common/metrics/service/BehaviourEventTypes"; + import { createRuntimeServiceUnpackEmpty } from "@rilldata/web-common/runtime-client"; import { createEventDispatcher } from "svelte"; - import { BehaviourEventMedium } from "../../../metrics/service/BehaviourEventTypes"; - import { MetricsEventSpace } from "../../../metrics/service/MetricsTypes"; - import { SourceConnectionType } from "../../../metrics/service/SourceEventTypes"; import { runtime } from "../../../runtime-client/runtime-store"; import { useModelNames } from "../../models/selectors"; import { EMPTY_PROJECT_TITLE } from "../../welcome/constants"; import { useIsProjectInitialized } from "../../welcome/is-project-initialized"; - import { - compileCreateSourceYAML, - emitSourceErrorTelemetry, - emitSourceSuccessTelemetry, - getSourceError, - } from "../sourceUtils"; - import { createSource } from "./createSource"; + import { compileCreateSourceYAML } from "../sourceUtils"; + import { createSourceCreator } from "./createSource"; const dispatch = createEventDispatcher(); - const queryClient = useQueryClient(); - $: runtimeInstanceId = $runtime.instanceId; $: sourceNames = useSourceNames(runtimeInstanceId); $: modelNames = useModelNames(runtimeInstanceId); $: isProjectInitialized = useIsProjectInitialized(runtimeInstanceId); - const createSourceMutation = createRuntimeServicePutFileAndReconcile(); const unpackEmptyProject = createRuntimeServiceUnpackEmpty(); - - $: createSourceMutationError = ($createSourceMutation?.error as any)?.response - ?.data; + const sourceCreator = createSourceCreator(BehaviourEventMedium.Button); async function handleOpenFileDialog() { return handleUpload(await openFileUploadDialog()); @@ -57,8 +40,6 @@ false ); for await (const { tableName, filePath } of uploadedFiles) { - let errors; - try { // If project is uninitialized, initialize an empty project if (!$isProjectInitialized.data) { @@ -78,43 +59,14 @@ "local_file" ); - // TODO: errors - errors = await createSource( - queryClient, - runtimeInstanceId, - tableName, - yaml, - $createSourceMutation - ); + await sourceCreator(tableName, yaml); } catch (err) { - // no-op + // TODO: file write errors } overlay.set(null); dispatch("close"); - // Emit telemetry - const sourceError = getSourceError(errors, tableName); - if ($createSourceMutation.isError || sourceError) { - // Error - emitSourceErrorTelemetry( - MetricsEventSpace.Modal, - $appScreen, - createSourceMutationError?.message ?? sourceError?.message, - SourceConnectionType.Local, - filePath - ); - } else { - // Success - emitSourceSuccessTelemetry( - MetricsEventSpace.Modal, - $appScreen, - BehaviourEventMedium.Button, - SourceConnectionType.Local, - filePath - ); - } - // Navigate to source page goto(`/source/${tableName}`); } diff --git a/web-common/src/features/sources/modal/RemoteSourceForm.svelte b/web-common/src/features/sources/modal/RemoteSourceForm.svelte index c7af138ea73..03f8827c67f 100644 --- a/web-common/src/features/sources/modal/RemoteSourceForm.svelte +++ b/web-common/src/features/sources/modal/RemoteSourceForm.svelte @@ -1,5 +1,4 @@