From 07ccff8ef814a33680f71423f65c4239bc897378 Mon Sep 17 00:00:00 2001 From: Aditya Hegde Date: Mon, 11 Sep 2023 21:20:06 +0530 Subject: [PATCH 1/4] Initial refactor for new reconcile --- .../entity-management/RenameAssetModal.svelte | 21 +--- .../entity-management/file-actions.ts | 24 ++++ .../entity-management/rename-entity.ts | 72 ++++++++++++ .../entity-management/resource-selectors.ts | 22 ++++ .../workspace/MetricsWorkspaceHeader.svelte | 48 ++------ .../workspace/editor/update-metrics.ts | 22 ++-- .../workspace/ModelWorkspaceHeader.svelte | 61 +++------- .../sources/inspector/SourceInspector.svelte | 38 +++---- .../src/features/sources/saveAndRefresh.ts | 2 + .../workspace/SourceWorkspaceHeader.svelte | 107 +++++++----------- 10 files changed, 218 insertions(+), 199 deletions(-) create mode 100644 web-common/src/features/entity-management/file-actions.ts create mode 100644 web-common/src/features/entity-management/rename-entity.ts create mode 100644 web-common/src/features/entity-management/resource-selectors.ts diff --git a/web-common/src/features/entity-management/RenameAssetModal.svelte b/web-common/src/features/entity-management/RenameAssetModal.svelte index 464e83303ba..071b0f3f1f6 100644 --- a/web-common/src/features/entity-management/RenameAssetModal.svelte +++ b/web-common/src/features/entity-management/RenameAssetModal.svelte @@ -3,16 +3,12 @@ import Input from "@rilldata/web-common/components/forms/Input.svelte"; import SubmissionError from "@rilldata/web-common/components/forms/SubmissionError.svelte"; import { Dialog } from "@rilldata/web-common/components/modal/index"; + import { renameFile } from "@rilldata/web-common/features/entity-management/rename-entity"; import type { EntityType } from "@rilldata/web-common/features/entity-management/types"; - import { useQueryClient } from "@tanstack/svelte-query"; import { createForm } from "svelte-forms-lib"; import * as yup from "yup"; - import { - createRuntimeServiceGetCatalogEntry, - createRuntimeServiceRenameFileAndReconcile, - } from "../../runtime-client"; + import { createRuntimeServiceRenameFile } from "../../runtime-client"; import { runtime } from "../../runtime-client/runtime-store"; - import { renameFileArtifact } from "./actions"; import { getLabel, getRouteFromName } from "./entity-mappers"; import { isDuplicateName } from "./name-utils"; import { useAllNames } from "./selectors"; @@ -21,18 +17,12 @@ export let entityType: EntityType; export let currentAssetName: string; - const queryClient = useQueryClient(); - let error: string; $: runtimeInstanceId = $runtime.instanceId; - $: getCatalog = createRuntimeServiceGetCatalogEntry( - runtimeInstanceId, - currentAssetName - ); $: allNamesQuery = useAllNames(runtimeInstanceId); - const renameAsset = createRuntimeServiceRenameFileAndReconcile(); + const renameAsset = createRuntimeServiceRenameFile(); const { form, errors, handleSubmit } = createForm({ initialValues: { @@ -56,13 +46,12 @@ return; } try { - await renameFileArtifact( - queryClient, + await renameFile( runtimeInstanceId, currentAssetName, values.newName, entityType, - $renameAsset + renameAsset ); goto(getRouteFromName(values.newName, entityType), { replaceState: true, diff --git a/web-common/src/features/entity-management/file-actions.ts b/web-common/src/features/entity-management/file-actions.ts new file mode 100644 index 00000000000..199480b82ed --- /dev/null +++ b/web-common/src/features/entity-management/file-actions.ts @@ -0,0 +1,24 @@ +import { getFilePathFromNameAndType } from "@rilldata/web-common/features/entity-management/entity-mappers"; +import type { EntityType } from "@rilldata/web-common/features/entity-management/types"; +import type { createRuntimeServicePutFile } from "@rilldata/web-common/runtime-client"; +import { get } from "svelte/store"; + +export async function saveFile( + instanceId: string, + name: string, + type: EntityType, + blob: string, + saveMutation: ReturnType +) { + const filePath = getFilePathFromNameAndType(name, type); + + await get(saveMutation).mutateAsync({ + instanceId, + data: { + blob, + create: false, + createOnly: false, + }, + path: filePath, + }); +} diff --git a/web-common/src/features/entity-management/rename-entity.ts b/web-common/src/features/entity-management/rename-entity.ts new file mode 100644 index 00000000000..fb335ec6a61 --- /dev/null +++ b/web-common/src/features/entity-management/rename-entity.ts @@ -0,0 +1,72 @@ +import { goto } from "$app/navigation"; +import { notifications } from "@rilldata/web-common/components/notifications"; +import { + getFilePathFromNameAndType, + getLabel, + getRouteFromName, +} from "@rilldata/web-common/features/entity-management/entity-mappers"; +import { isDuplicateName } from "@rilldata/web-common/features/entity-management/name-utils"; +import { EntityType } from "@rilldata/web-common/features/entity-management/types"; +import type { createRuntimeServiceRenameFile } from "@rilldata/web-common/runtime-client"; +import { httpRequestQueue } from "@rilldata/web-common/runtime-client/http-client"; +import { get } from "svelte/types/runtime/store"; + +export async function validateAndRenameEntity( + instanceId: string, + fromName: string, + toName: string, + allNames: Array, + entityType: EntityType, + renameMutation: ReturnType +): Promise { + if (!toName.match(/^[a-zA-Z_][a-zA-Z0-9_]*$/)) { + notifications.send({ + message: `${getLabel( + entityType + )} name must start with a letter or underscore and contain only letters, numbers, and underscores`, + }); + return false; + } + + if (isDuplicateName(toName, fromName, allNames)) { + notifications.send({ + message: `Name ${toName} is already in use`, + }); + return false; + } + + try { + await renameFile(instanceId, fromName, toName, entityType, renameMutation); + } catch (err) { + console.error(err.response.data.message); + } + + return true; +} + +export async function renameFile( + instanceId: string, + fromName: string, + toName: string, + entityType: EntityType, + renameMutation: ReturnType +) { + await get(renameMutation).mutateAsync({ + instanceId, + data: { + fromPath: getFilePathFromNameAndType(fromName, entityType), + toPath: getFilePathFromNameAndType(toName, entityType), + }, + }); + + httpRequestQueue.removeByName(fromName); + notifications.send({ + message: `Renamed ${getLabel(entityType)} ${fromName} to ${toName}`, + }); + + const route = getRouteFromName(toName, entityType); + goto(entityType === EntityType.MetricsDefinition ? `${route}/edit` : route, { + replaceState: true, + }); + // TODO: no telemetry for rename? +} diff --git a/web-common/src/features/entity-management/resource-selectors.ts b/web-common/src/features/entity-management/resource-selectors.ts new file mode 100644 index 00000000000..9d5797e19af --- /dev/null +++ b/web-common/src/features/entity-management/resource-selectors.ts @@ -0,0 +1,22 @@ +import { createRuntimeServiceGetResource } from "@rilldata/web-common/runtime-client"; + +export enum ResourceKind { + Source = "source", + Model = "model", + MetricsView = "metricsview", +} + +export function useSource(instanceId: string, name: string) { + return createRuntimeServiceGetResource( + instanceId, + { + "name.kind": ResourceKind.Source, + "name.name": name, + }, + { + query: { + select: (data) => data?.resource?.source, + }, + } + ); +} diff --git a/web-common/src/features/metrics-views/workspace/MetricsWorkspaceHeader.svelte b/web-common/src/features/metrics-views/workspace/MetricsWorkspaceHeader.svelte index 7851d34f3fc..0aaab52dfe7 100644 --- a/web-common/src/features/metrics-views/workspace/MetricsWorkspaceHeader.svelte +++ b/web-common/src/features/metrics-views/workspace/MetricsWorkspaceHeader.svelte @@ -1,13 +1,9 @@ diff --git a/web-common/src/features/sources/inspector/SourceInspector.svelte b/web-common/src/features/sources/inspector/SourceInspector.svelte index 11a380418a2..6fec4995369 100644 --- a/web-common/src/features/sources/inspector/SourceInspector.svelte +++ b/web-common/src/features/sources/inspector/SourceInspector.svelte @@ -6,6 +6,7 @@ } from "@rilldata/web-common/components/column-profile/queries"; import Tooltip from "@rilldata/web-common/components/tooltip/Tooltip.svelte"; import TooltipContent from "@rilldata/web-common/components/tooltip/TooltipContent.svelte"; + import { useSource } from "@rilldata/web-common/features/entity-management/resource-selectors"; import CollapsibleSectionTitle from "@rilldata/web-common/layout/CollapsibleSectionTitle.svelte"; import { formatBigNumberPercentage, @@ -14,9 +15,7 @@ import { createQueryServiceTableCardinality, createQueryServiceTableColumns, - createRuntimeServiceGetCatalogEntry, - V1CatalogEntry, - V1Source, + V1SourceV2, } from "@rilldata/web-common/runtime-client"; import type { Readable } from "svelte/store"; import { slide } from "svelte/transition"; @@ -30,12 +29,9 @@ $: runtimeInstanceId = $runtime.instanceId; - $: getSource = createRuntimeServiceGetCatalogEntry( - runtimeInstanceId, - sourceName - ); - let sourceCatalog: V1CatalogEntry; - $: sourceCatalog = $getSource?.data?.entry; + $: sourceQuery = useSource(runtimeInstanceId, sourceName); + let source: V1SourceV2; + $: source = $sourceQuery.data; let showColumns = true; @@ -63,8 +59,8 @@ } } - function getFileExtension(source: V1Source): string { - const path = source?.properties?.path?.toLowerCase(); + function getFileExtension(source: V1SourceV2): string { + const path = source?.spec?.properties?.path?.toLowerCase(); if (path?.includes(".csv")) return "CSV"; if (path?.includes(".parquet")) return "Parquet"; if (path?.includes(".json")) return "JSON"; @@ -72,8 +68,8 @@ return ""; } - $: connectorType = formatConnectorType(sourceCatalog?.source?.connector); - $: fileExtension = getFileExtension(sourceCatalog); + $: connectorType = formatConnectorType(source?.spec?.sourceConnector); + $: fileExtension = getFileExtension(source); $: cardinalityQuery = createQueryServiceTableCardinality( $runtime.instanceId, @@ -90,13 +86,6 @@ }`; } - /** get the current column count */ - $: { - columnCount = `${formatInteger( - sourceCatalog?.source?.schema?.fields?.length - )} columns`; - } - /** total % null cells */ $: profileColumns = createQueryServiceTableColumns( @@ -108,6 +97,11 @@ let summaries: Readable>; $: if ($profileColumns?.data?.profileColumns) { + /** get the current column count */ + columnCount = `${formatInteger( + $profileColumns.data.profileColumns.length + )} columns`; + summaries = getSummaries(sourceName, $runtime?.instanceId, $profileColumns); } @@ -121,7 +115,7 @@ } $: { const totalCells = - sourceCatalog?.source?.schema?.fields?.length * cardinality; + $profileColumns?.data?.profileColumns?.length * cardinality; nullPercentage = formatBigNumberPercentage(totalNulls / totalCells); } @@ -138,7 +132,7 @@
- {#if sourceCatalog && !$getSource.isError} + {#if source && !$sourceQuery.isError}
diff --git a/web-common/src/features/sources/saveAndRefresh.ts b/web-common/src/features/sources/saveAndRefresh.ts index e7d38277081..4790ccd50d5 100644 --- a/web-common/src/features/sources/saveAndRefresh.ts +++ b/web-common/src/features/sources/saveAndRefresh.ts @@ -56,6 +56,8 @@ export async function saveAndRefresh( } catch (e) { // ignore } + // TODO: should this emit after a save? + // should this emit only when a source is modified from UI? or perhaps change screen and others based on where it is emitted from? if (resp.errors.length > 0) { emitSourceErrorTelemetry( MetricsEventSpace.RightPanel, diff --git a/web-common/src/features/sources/workspace/SourceWorkspaceHeader.svelte b/web-common/src/features/sources/workspace/SourceWorkspaceHeader.svelte index 5766b78ff98..0c582fd74c2 100644 --- a/web-common/src/features/sources/workspace/SourceWorkspaceHeader.svelte +++ b/web-common/src/features/sources/workspace/SourceWorkspaceHeader.svelte @@ -5,21 +5,21 @@ IconSpaceFixer, } from "@rilldata/web-common/components/button"; import RefreshIcon from "@rilldata/web-common/components/icons/RefreshIcon.svelte"; - import { notifications } from "@rilldata/web-common/components/notifications"; import PanelCTA from "@rilldata/web-common/components/panel/PanelCTA.svelte"; import ResponsiveButtonText from "@rilldata/web-common/components/panel/ResponsiveButtonText.svelte"; + import { saveFile } from "@rilldata/web-common/features/entity-management/file-actions"; + import { validateAndRenameEntity } from "@rilldata/web-common/features/entity-management/rename-entity"; + import { useSource } from "@rilldata/web-common/features/entity-management/resource-selectors"; import { EntityType } from "@rilldata/web-common/features/entity-management/types"; import { overlay } from "@rilldata/web-common/layout/overlay-store"; import { slideRight } from "@rilldata/web-common/lib/transitions"; import { - createRuntimeServiceGetCatalogEntry, createRuntimeServiceGetFile, - createRuntimeServicePutFileAndReconcile, + createRuntimeServicePutFile, createRuntimeServiceRefreshAndReconcile, - createRuntimeServiceRenameFileAndReconcile, + createRuntimeServiceRenameFile, getRuntimeServiceGetCatalogEntryQueryKey, - V1CatalogEntry, - V1Source, + V1SourceV2, } from "@rilldata/web-common/runtime-client"; import { appQueryStatusStore } from "@rilldata/web-common/runtime-client/application-store"; import { useQueryClient } from "@tanstack/svelte-query"; @@ -34,20 +34,14 @@ MetricsEventSpace, } from "../../../metrics/service/MetricsTypes"; import { runtime } from "../../../runtime-client/runtime-store"; - import { renameFileArtifact } from "../../entity-management/actions"; - import { - getFilePathFromNameAndType, - getRouteFromName, - } from "../../entity-management/entity-mappers"; + import { getFilePathFromNameAndType } from "../../entity-management/entity-mappers"; import { fileArtifactsStore, getFileArtifactReconciliationErrors, } from "../../entity-management/file-artifacts-store"; - import { isDuplicateName } from "../../entity-management/name-utils"; import { useAllNames } from "../../entity-management/selectors"; import { createModelFromSourceV2 } from "../createModel"; import { refreshSource } from "../refreshSource"; - import { saveAndRefresh } from "../saveAndRefresh"; import { useIsSourceUnsaved } from "../selectors"; import { useSourceStore } from "../sources-store"; @@ -55,63 +49,38 @@ const queryClient = useQueryClient(); - const renameSource = createRuntimeServiceRenameFileAndReconcile(); const refreshSourceMutation = createRuntimeServiceRefreshAndReconcile(); - const createSource = createRuntimeServicePutFileAndReconcile(); + const renameSource = createRuntimeServiceRenameFile(); + const saveSource = createRuntimeServicePutFile(); $: runtimeInstanceId = $runtime.instanceId; - $: getSource = createRuntimeServiceGetCatalogEntry( - runtimeInstanceId, - sourceName - ); + + $: sourceQuery = useSource(runtimeInstanceId, sourceName); + let source: V1SourceV2; + $: source = $sourceQuery.data; + $: file = createRuntimeServiceGetFile( runtimeInstanceId, getFilePathFromNameAndType(sourceName, EntityType.Table) ); - let entry: V1CatalogEntry; - let source: V1Source; - $: entry = $getSource?.data?.entry; - $: source = entry?.source; - let connector: string; - $: connector = $getSource.data?.entry?.source?.connector as string; + $: connector = source?.spec?.sourceConnector; $: allNamesQuery = useAllNames(runtimeInstanceId); const onChangeCallback = async (e) => { - if (!e.target.value.match(/^[a-zA-Z_][a-zA-Z0-9_]*$/)) { - notifications.send({ - message: - "Source name must start with a letter or underscore and contain only letters, numbers, and underscores", - }); - e.target.value = sourceName; // resets the input - return; - } - if (isDuplicateName(e.target.value, sourceName, $allNamesQuery.data)) { - notifications.send({ - message: `Name ${e.target.value} is already in use`, - }); - e.target.value = sourceName; // resets the input - return; - } - - try { - const toName = e.target.value; - const entityType = EntityType.Table; - await renameFileArtifact( - queryClient, + if ( + !(await validateAndRenameEntity( runtimeInstanceId, + e.target.value, sourceName, - toName, - entityType, - $renameSource - ); - goto(getRouteFromName(toName, entityType), { - replaceState: true, - }); - } catch (err) { - console.error(err.response.data.message); + $allNamesQuery.data, + EntityType.Table, + renameSource + )) + ) { + e.target.value = sourceName; // resets the input } }; @@ -121,23 +90,31 @@ const onSaveAndRefreshClick = async (tableName: string) => { overlay.set({ title: `Importing ${tableName}.yaml` }); - await saveAndRefresh(queryClient, tableName, $sourceStore.clientYAML); + await saveFile( + runtimeInstanceId, + tableName, + EntityType.Table, + $sourceStore.clientYAML, + saveSource + ); + // TODO: emit telemetry + // should it emit only when a source is modified from UI? + // or perhaps change screen and others based on where it is emitted from and always fire? overlay.set(null); }; const onRefreshClick = async (tableName: string) => { try { + // TODO: what is the replacement for refresh in new reconcile? await refreshSource( connector, tableName, runtimeInstanceId, $refreshSourceMutation, - $createSource, + $saveSource, queryClient, - source?.connector === "s3" || - source?.connector === "gcs" || - source?.connector === "https" - ? source?.properties?.path + connector === "s3" || connector === "gcs" || connector === "https" + ? source?.spec?.properties?.path : sourceName ); // invalidate the "refreshed_on" time @@ -209,14 +186,14 @@ Refreshing... {:else}
- {#if $getSource.isSuccess && $getSource.data?.entry?.refreshedOn} + {#if $sourceQuery.isSuccess && $sourceQuery.data?.state?.refreshedOn}
Imported on {formatRefreshedOn( - $getSource.data?.entry?.refreshedOn + $sourceQuery.data?.state?.refreshedOn )}
{/if} @@ -226,9 +203,9 @@
{/key}
- {#if entry && entry?.model?.sql?.trim()?.length} + {#if entry && entry?.model?.spec?.sql?.trim()?.length}
- +
{/if} {/if} diff --git a/web-common/src/features/models/createModel.ts b/web-common/src/features/models/createModel.ts index e847dd64f12..e45de670c4a 100644 --- a/web-common/src/features/models/createModel.ts +++ b/web-common/src/features/models/createModel.ts @@ -1,41 +1,18 @@ import { goto } from "$app/navigation"; -import { fileArtifactsStore } from "@rilldata/web-common/features/entity-management/file-artifacts-store"; import { EntityType } from "@rilldata/web-common/features/entity-management/types"; -import type { V1PutFileAndReconcileResponse } from "@rilldata/web-common/runtime-client"; -import { invalidateAfterReconcile } from "@rilldata/web-common/runtime-client/invalidation"; -import type { - CreateBaseMutationResult, - QueryClient, -} from "@tanstack/svelte-query"; +import { runtimeServicePutFile } from "@rilldata/web-common/runtime-client"; +import { runtime } from "@rilldata/web-common/runtime-client/runtime-store"; +import { get } from "svelte/store"; import { getFilePathFromNameAndType } from "../entity-management/entity-mappers"; -export async function createModel( - queryClient: QueryClient, - instanceId: string, - newModelName: string, - createModelMutation: CreateBaseMutationResult, // TODO: type - sql = "", - setAsActive = true -) { - const resp = await createModelMutation.mutateAsync({ - data: { - instanceId, - path: getFilePathFromNameAndType(newModelName, EntityType.Model), +export async function createModel(newModelName: string, sql = "") { + await runtimeServicePutFile( + get(runtime).instanceId, + getFilePathFromNameAndType(newModelName, EntityType.Model), + { blob: sql, createOnly: true, - strict: true, - }, - }); - fileArtifactsStore.setErrors(resp.affectedPaths, resp.errors); + } + ); goto(`/model/${newModelName}?focus`); - - invalidateAfterReconcile(queryClient, instanceId, resp); - if (resp.errors?.length && sql !== "") { - resp.errors.forEach((error) => { - console.error(error); - }); - throw new Error(resp.errors[0].filePath); - } - - if (!setAsActive) return; } diff --git a/web-common/src/features/models/navigation/ModelAssets.svelte b/web-common/src/features/models/navigation/ModelAssets.svelte index 97bc96ce883..4bf61346d31 100644 --- a/web-common/src/features/models/navigation/ModelAssets.svelte +++ b/web-common/src/features/models/navigation/ModelAssets.svelte @@ -2,9 +2,12 @@ import { page } from "$app/stores"; import ColumnProfile from "@rilldata/web-common/components/column-profile/ColumnProfile.svelte"; import RenameAssetModal from "@rilldata/web-common/features/entity-management/RenameAssetModal.svelte"; + import { + ResourceKind, + useAllEntityNames, + useFilteredEntityNames, + } from "@rilldata/web-common/features/entity-management/resource-selectors"; import { EntityType } from "@rilldata/web-common/features/entity-management/types"; - import { createRuntimeServicePutFileAndReconcile } from "@rilldata/web-common/runtime-client"; - import { useQueryClient } from "@tanstack/svelte-query"; import { slide } from "svelte/transition"; import { LIST_SLIDE_DURATION } from "../../../layout/config"; import NavigationEntry from "../../../layout/navigation/NavigationEntry.svelte"; @@ -12,27 +15,26 @@ import { runtime } from "../../../runtime-client/runtime-store"; import AddAssetButton from "../../entity-management/AddAssetButton.svelte"; import { getName } from "../../entity-management/name-utils"; - import { useSourceNames } from "../../sources/selectors"; import { createModel } from "../createModel"; - import { useModelNames } from "../selectors"; import ModelMenuItems from "./ModelMenuItems.svelte"; import ModelTooltip from "./ModelTooltip.svelte"; - $: sourceNames = useSourceNames($runtime.instanceId); - $: modelNames = useModelNames($runtime.instanceId); - - const queryClient = useQueryClient(); - - const createModelMutation = createRuntimeServicePutFileAndReconcile(); + $: sourceNames = useFilteredEntityNames( + $runtime.instanceId, + ResourceKind.Source + ); + $: modelNames = useFilteredEntityNames( + $runtime.instanceId, + ResourceKind.Model + ); + $: allEntityNames = useAllEntityNames($runtime.instanceId); let showModels = true; async function handleAddModel() { await createModel( - queryClient, $runtime.instanceId, - getName("model", $modelNames.data), - $createModelMutation + getName("model", $allEntityNames.data) ); // if the models are not visible in the assets list, show them. if (!showModels) { diff --git a/web-common/src/features/models/navigation/ModelMenuItems.svelte b/web-common/src/features/models/navigation/ModelMenuItems.svelte index 2efec030cf5..ae9f592f900 100644 --- a/web-common/src/features/models/navigation/ModelMenuItems.svelte +++ b/web-common/src/features/models/navigation/ModelMenuItems.svelte @@ -6,6 +6,7 @@ import { Divider, MenuItem } from "@rilldata/web-common/components/menu"; import { useDashboardNames } from "@rilldata/web-common/features/dashboards/selectors"; import { getFilePathFromNameAndType } from "@rilldata/web-common/features/entity-management/entity-mappers"; + import { deleteFile } from "@rilldata/web-common/features/entity-management/file-actions"; import { fileArtifactsStore } from "@rilldata/web-common/features/entity-management/file-artifacts-store"; import { EntityType } from "@rilldata/web-common/features/entity-management/types"; import { appScreen } from "@rilldata/web-common/layout/app-store"; @@ -17,7 +18,6 @@ MetricsEventSpace, } from "@rilldata/web-common/metrics/service/MetricsTypes"; import { - createRuntimeServiceDeleteFileAndReconcile, createRuntimeServiceGetCatalogEntry, createRuntimeServicePutFileAndReconcile, V1Model, @@ -27,7 +27,6 @@ import { useQueryClient } from "@tanstack/svelte-query"; import { createEventDispatcher } from "svelte"; import { runtime } from "../../../runtime-client/runtime-store"; - import { deleteFileArtifact } from "../../entity-management/actions"; import { getName } from "../../entity-management/name-utils"; import { generateDashboardYAMLForModel } from "../../metrics-views/metrics-internal-store"; import { useModelNames } from "../selectors"; @@ -40,7 +39,6 @@ const queryClient = useQueryClient(); - const deleteModel = createRuntimeServiceDeleteFileAndReconcile(); const createFileMutation = createRuntimeServicePutFileAndReconcile(); $: modelNames = useModelNames($runtime.instanceId); @@ -109,13 +107,10 @@ }; const handleDeleteModel = async (modelName: string) => { - await deleteFileArtifact( - queryClient, + await deleteFile( $runtime.instanceId, modelName, EntityType.Model, - $deleteModel, - $appScreen, $modelNames.data ); toggleMenu(); diff --git a/web-common/src/features/sources/createModel.ts b/web-common/src/features/sources/createModel.ts index b9c088e0637..407b55a8695 100644 --- a/web-common/src/features/sources/createModel.ts +++ b/web-common/src/features/sources/createModel.ts @@ -1,15 +1,9 @@ import { getName } from "@rilldata/web-common/features/entity-management/name-utils"; import { createModel } from "@rilldata/web-common/features/models/createModel"; -import type { - CreateBaseMutationResult, - QueryClient, -} from "@tanstack/svelte-query"; +import type { QueryClient } from "@tanstack/svelte-query"; import { get } from "svelte/store"; import { notifications } from "../../components/notifications"; -import { - runtimeServicePutFileAndReconcile, - type V1PutFileAndReconcileResponse, -} from "../../runtime-client"; +import { runtimeServicePutFileAndReconcile } from "../../runtime-client"; import { invalidateAfterReconcile } from "../../runtime-client/invalidation"; import { runtime } from "../../runtime-client/runtime-store"; import { getFilePathFromNameAndType } from "../entity-management/entity-mappers"; @@ -18,23 +12,12 @@ import { EntityType } from "../entity-management/types"; import { getModelNames } from "../models/selectors"; export async function createModelFromSource( - queryClient: QueryClient, - instanceId: string, - modelNames: Array, sourceName: string, - sourceNameInQuery: string, - createModelMutation: CreateBaseMutationResult, // TODO: type - setAsActive = true + allNames: Array, + sourceNameInQuery = sourceName ): Promise { - const newModelName = getName(`${sourceName}_model`, modelNames); - await createModel( - queryClient, - instanceId, - newModelName, - createModelMutation, - `select * from ${sourceNameInQuery}`, - setAsActive - ); + const newModelName = getName(`${sourceName}_model`, allNames); + await createModel(newModelName, `select * from ${sourceNameInQuery}`); notifications.send({ message: `Queried ${sourceNameInQuery} in workspace`, }); diff --git a/web-common/src/features/sources/modal/FileDrop.svelte b/web-common/src/features/sources/modal/FileDrop.svelte index 4b42058b4ba..7207ac16c6c 100644 --- a/web-common/src/features/sources/modal/FileDrop.svelte +++ b/web-common/src/features/sources/modal/FileDrop.svelte @@ -1,44 +1,26 @@ diff --git a/web-common/src/features/sources/modal/LocalSourceUpload.svelte b/web-common/src/features/sources/modal/LocalSourceUpload.svelte index 57fdefea0bb..f2078299dd3 100644 --- a/web-common/src/features/sources/modal/LocalSourceUpload.svelte +++ b/web-common/src/features/sources/modal/LocalSourceUpload.svelte @@ -6,45 +6,26 @@ uploadTableFiles, } from "@rilldata/web-common/features/sources/modal/file-upload"; import { useSourceNames } from "@rilldata/web-common/features/sources/selectors"; - import { appScreen } from "@rilldata/web-common/layout/app-store"; import { overlay } from "@rilldata/web-common/layout/overlay-store"; - import { - createRuntimeServicePutFileAndReconcile, - createRuntimeServiceUnpackEmpty, - } from "@rilldata/web-common/runtime-client"; - import { useQueryClient } from "@tanstack/svelte-query"; + import { createRuntimeServiceUnpackEmpty } from "@rilldata/web-common/runtime-client"; import { createEventDispatcher } from "svelte"; - import { BehaviourEventMedium } from "../../../metrics/service/BehaviourEventTypes"; - import { MetricsEventSpace } from "../../../metrics/service/MetricsTypes"; - import { SourceConnectionType } from "../../../metrics/service/SourceEventTypes"; import { runtime } from "../../../runtime-client/runtime-store"; import { useModelNames } from "../../models/selectors"; import { EMPTY_PROJECT_TITLE } from "../../welcome/constants"; import { useIsProjectInitialized } from "../../welcome/is-project-initialized"; - import { - compileCreateSourceYAML, - emitSourceErrorTelemetry, - emitSourceSuccessTelemetry, - getSourceError, - } from "../sourceUtils"; + import { compileCreateSourceYAML } from "../sourceUtils"; import { createSource } from "./createSource"; const dispatch = createEventDispatcher(); - const queryClient = useQueryClient(); - $: runtimeInstanceId = $runtime.instanceId; $: sourceNames = useSourceNames(runtimeInstanceId); $: modelNames = useModelNames(runtimeInstanceId); $: isProjectInitialized = useIsProjectInitialized(runtimeInstanceId); - const createSourceMutation = createRuntimeServicePutFileAndReconcile(); const unpackEmptyProject = createRuntimeServiceUnpackEmpty(); - $: createSourceMutationError = ($createSourceMutation?.error as any)?.response - ?.data; - async function handleOpenFileDialog() { return handleUpload(await openFileUploadDialog()); } @@ -57,8 +38,6 @@ false ); for await (const { tableName, filePath } of uploadedFiles) { - let errors; - try { // If project is uninitialized, initialize an empty project if (!$isProjectInitialized.data) { @@ -78,43 +57,14 @@ "local_file" ); - // TODO: errors - errors = await createSource( - queryClient, - runtimeInstanceId, - tableName, - yaml, - $createSourceMutation - ); + await createSource(runtimeInstanceId, tableName, yaml); } catch (err) { - // no-op + // TODO: file write errors } overlay.set(null); dispatch("close"); - // Emit telemetry - const sourceError = getSourceError(errors, tableName); - if ($createSourceMutation.isError || sourceError) { - // Error - emitSourceErrorTelemetry( - MetricsEventSpace.Modal, - $appScreen, - createSourceMutationError?.message ?? sourceError?.message, - SourceConnectionType.Local, - filePath - ); - } else { - // Success - emitSourceSuccessTelemetry( - MetricsEventSpace.Modal, - $appScreen, - BehaviourEventMedium.Button, - SourceConnectionType.Local, - filePath - ); - } - // Navigate to source page goto(`/source/${tableName}`); } diff --git a/web-common/src/features/sources/modal/RemoteSourceForm.svelte b/web-common/src/features/sources/modal/RemoteSourceForm.svelte index c7af138ea73..03f8827c67f 100644 --- a/web-common/src/features/sources/modal/RemoteSourceForm.svelte +++ b/web-common/src/features/sources/modal/RemoteSourceForm.svelte @@ -1,5 +1,4 @@ From eaa62f7a19c00529aa4329060da9d8771b699ac1 Mon Sep 17 00:00:00 2001 From: Aditya Hegde Date: Wed, 13 Sep 2023 21:57:00 +0530 Subject: [PATCH 4/4] Adding source to dashboard chain --- .../entity-management/entity-action-queue.ts | 45 +++++++----- .../models/navigation/ModelAssets.svelte | 4 +- .../features/sources/createModelFromSource.ts | 32 ++++++++- .../features/sources/modal/FileDrop.svelte | 11 +-- .../sources/modal/LocalSourceUpload.svelte | 6 +- .../features/sources/modal/createSource.ts | 35 ++++++++- .../sources/modal/submitRemoteSourceForm.ts | 3 +- .../sources/navigation/SourceMenuItems.svelte | 72 +++++-------------- .../sources/navigation/TableAssets.svelte | 4 +- .../sources/source-ingestion-telemetry.ts | 10 +-- .../workspace/SourceWorkspaceHeader.svelte | 4 +- .../src/metrics/service/metrics-helpers.ts | 4 +- 12 files changed, 127 insertions(+), 103 deletions(-) diff --git a/web-common/src/features/entity-management/entity-action-queue.ts b/web-common/src/features/entity-management/entity-action-queue.ts index c6cbb8c911a..d0992a3d4f0 100644 --- a/web-common/src/features/entity-management/entity-action-queue.ts +++ b/web-common/src/features/entity-management/entity-action-queue.ts @@ -12,10 +12,17 @@ export enum EntityAction { Delete, } -export enum ChainAction { - ModelFromSource, - DashboardFromModel, -} +export type EntityCreateFunction = ( + resource: V1Resource, + sourceName: string, + pathPrefix?: string +) => Promise; + +export type ChainParams = { + chainFunction: EntityCreateFunction; + sourceName: string; + pathPrefix?: string; +}; /** * A global queue for entity actions. @@ -26,17 +33,18 @@ export type EntityActionQueueState = { }; export type EntityActionInstance = { action: EntityAction; - chain?: Array; // telemetry - params: TelemetryParams; + telemetryParams: TelemetryParams; + + chainParams?: ChainParams; }; type EntityActionQueueReducers = { add: ( name: string, action: EntityAction, - params: TelemetryParams, - chain?: Array + telemetryParams: TelemetryParams, + chainParams?: ChainParams ) => void; resolved: (resource: V1Resource, event: V1ResourceEvent) => void; }; @@ -54,15 +62,15 @@ export const entityActionQueueStore: EntityActionQueueStore = { add( name: string, action: EntityAction, - params: TelemetryParams, - chain?: Array + telemetryParams: TelemetryParams, + chainParams?: ChainParams ) { update((state) => { state.entities[name] ??= []; state.entities[name].push({ action, - params, - chain, + telemetryParams, + chainParams, }); return state; }); @@ -85,13 +93,12 @@ export const entityActionQueueStore: EntityActionQueueStore = { break; } - if (action.chain?.length) { - switch (action.chain[0]) { - case ChainAction.DashboardFromModel: - this.add(); - break; - // TODO: others - } + if (action.chainParams) { + action.chainParams.chainFunction( + resource, + action.chainParams.sourceName, + action.chainParams.pathPrefix + ); } return state; diff --git a/web-common/src/features/models/navigation/ModelAssets.svelte b/web-common/src/features/models/navigation/ModelAssets.svelte index 9290503fa93..f1df30d68f8 100644 --- a/web-common/src/features/models/navigation/ModelAssets.svelte +++ b/web-common/src/features/models/navigation/ModelAssets.svelte @@ -8,7 +8,7 @@ useFilteredEntityNames, } from "@rilldata/web-common/features/entity-management/resource-selectors"; import { EntityType } from "@rilldata/web-common/features/entity-management/types"; - import { getLeftPanelModelParams } from "@rilldata/web-common/metrics/service/metrics-helpers"; + import { getLeftPanelParams } from "@rilldata/web-common/metrics/service/metrics-helpers"; import { slide } from "svelte/transition"; import { LIST_SLIDE_DURATION } from "../../../layout/config"; import NavigationEntry from "../../../layout/navigation/NavigationEntry.svelte"; @@ -30,7 +30,7 @@ ); $: allEntityNames = useAllEntityNames($runtime.instanceId); - const modelCreator = createModelCreator(getLeftPanelModelParams()); + const modelCreator = createModelCreator(getLeftPanelParams()); let showModels = true; diff --git a/web-common/src/features/sources/createModelFromSource.ts b/web-common/src/features/sources/createModelFromSource.ts index 190ece607be..04754a00ea8 100644 --- a/web-common/src/features/sources/createModelFromSource.ts +++ b/web-common/src/features/sources/createModelFromSource.ts @@ -1,3 +1,8 @@ +import type { EntityCreateFunction } from "@rilldata/web-common/features/entity-management/entity-action-queue"; +import { + EntityAction, + entityActionQueueStore, +} from "@rilldata/web-common/features/entity-management/entity-action-queue"; import { getName } from "@rilldata/web-common/features/entity-management/name-utils"; import { createModelCreator } from "@rilldata/web-common/features/models/createModel"; import type { TelemetryParams } from "@rilldata/web-common/metrics/service/metrics-helpers"; @@ -8,7 +13,8 @@ import { notifications } from "../../components/notifications"; export function createModelFromSourceCreator( allNamesQuery: CreateQueryResult>, - telemetryParams?: TelemetryParams + telemetryParams?: TelemetryParams, + chainFunction?: EntityCreateFunction ) { const modelCreator = createModelCreator(telemetryParams); @@ -18,13 +24,33 @@ export function createModelFromSourceCreator( sourceName: string, pathPrefix?: string ) => { - pathPrefix ??= "/models/"; + const modelPathPrefix = pathPrefix ?? "/models/"; const newModelName = getName( `${sourceName}_model`, get(allNamesQuery).data ); - await modelCreator(newModelName, pathPrefix, `select * from ${sourceName}`); + + if (chainFunction) { + // add the chain with telemetry params + entityActionQueueStore.add( + newModelName, + EntityAction.Create, + telemetryParams, + { + chainFunction, + sourceName, + // pass in the original path prefix and not the defaulted one from the beginning of the function + pathPrefix, + } + ); + } + + await modelCreator( + newModelName, + modelPathPrefix, + `select * from ${sourceName}` + ); notifications.send({ message: `Queried ${sourceName} in workspace`, }); diff --git a/web-common/src/features/sources/modal/FileDrop.svelte b/web-common/src/features/sources/modal/FileDrop.svelte index 7207ac16c6c..dc73a78fdca 100644 --- a/web-common/src/features/sources/modal/FileDrop.svelte +++ b/web-common/src/features/sources/modal/FileDrop.svelte @@ -2,14 +2,13 @@ import Overlay from "@rilldata/web-common/components/overlay/Overlay.svelte"; import { useSourceNames } from "@rilldata/web-common/features/sources/selectors"; import { createRuntimeServiceUnpackEmpty } from "@rilldata/web-common/runtime-client"; - import { appScreen } from "../../../layout/app-store"; import { BehaviourEventMedium } from "../../../metrics/service/BehaviourEventTypes"; import { runtime } from "../../../runtime-client/runtime-store"; import { useModelNames } from "../../models/selectors"; import { EMPTY_PROJECT_TITLE } from "../../welcome/constants"; import { useIsProjectInitialized } from "../../welcome/is-project-initialized"; import { compileCreateSourceYAML } from "../sourceUtils"; - import { createSource } from "./createSource"; + import { createSourceCreator } from "./createSource"; import { uploadTableFiles } from "./file-upload"; export let showDropOverlay: boolean; @@ -20,6 +19,7 @@ $: isProjectInitialized = useIsProjectInitialized(runtimeInstanceId); const unpackEmptyProject = createRuntimeServiceUnpackEmpty(); + const sourceCreator = createSourceCreator(BehaviourEventMedium.Drag); const handleSourceDrop = async (e: DragEvent) => { showDropOverlay = false; @@ -48,12 +48,7 @@ }, "local_file" ); - await createSource( - runtimeInstanceId, - tableName, - yaml, - BehaviourEventMedium.Drag - ); + await sourceCreator(tableName, yaml); } catch (err) { // TODO: file write errors console.error(err); diff --git a/web-common/src/features/sources/modal/LocalSourceUpload.svelte b/web-common/src/features/sources/modal/LocalSourceUpload.svelte index f2078299dd3..768735f6e7e 100644 --- a/web-common/src/features/sources/modal/LocalSourceUpload.svelte +++ b/web-common/src/features/sources/modal/LocalSourceUpload.svelte @@ -7,6 +7,7 @@ } from "@rilldata/web-common/features/sources/modal/file-upload"; import { useSourceNames } from "@rilldata/web-common/features/sources/selectors"; import { overlay } from "@rilldata/web-common/layout/overlay-store"; + import { BehaviourEventMedium } from "@rilldata/web-common/metrics/service/BehaviourEventTypes"; import { createRuntimeServiceUnpackEmpty } from "@rilldata/web-common/runtime-client"; import { createEventDispatcher } from "svelte"; import { runtime } from "../../../runtime-client/runtime-store"; @@ -14,7 +15,7 @@ import { EMPTY_PROJECT_TITLE } from "../../welcome/constants"; import { useIsProjectInitialized } from "../../welcome/is-project-initialized"; import { compileCreateSourceYAML } from "../sourceUtils"; - import { createSource } from "./createSource"; + import { createSourceCreator } from "./createSource"; const dispatch = createEventDispatcher(); @@ -25,6 +26,7 @@ $: isProjectInitialized = useIsProjectInitialized(runtimeInstanceId); const unpackEmptyProject = createRuntimeServiceUnpackEmpty(); + const sourceCreator = createSourceCreator(BehaviourEventMedium.Button); async function handleOpenFileDialog() { return handleUpload(await openFileUploadDialog()); @@ -57,7 +59,7 @@ "local_file" ); - await createSource(runtimeInstanceId, tableName, yaml); + await sourceCreator(tableName, yaml); } catch (err) { // TODO: file write errors } diff --git a/web-common/src/features/sources/modal/createSource.ts b/web-common/src/features/sources/modal/createSource.ts index d7892c67e17..6611ad7fdbf 100644 --- a/web-common/src/features/sources/modal/createSource.ts +++ b/web-common/src/features/sources/modal/createSource.ts @@ -8,9 +8,42 @@ import { EntityType } from "@rilldata/web-common/features/entity-management/type import { appScreen } from "@rilldata/web-common/layout/app-store"; import { BehaviourEventMedium } from "@rilldata/web-common/metrics/service/BehaviourEventTypes"; import { MetricsEventSpace } from "@rilldata/web-common/metrics/service/MetricsTypes"; -import { runtimeServicePutFile } from "@rilldata/web-common/runtime-client"; +import { + createRuntimeServicePutFile, + runtimeServicePutFile, +} from "@rilldata/web-common/runtime-client"; +import { runtime } from "@rilldata/web-common/runtime-client/runtime-store"; import { get } from "svelte/store"; +export function createSourceCreator( + behaviourEventMedium: BehaviourEventMedium +) { + const putFileMutation = createRuntimeServicePutFile(); + + return async (tableName: string, yaml: string, pathPrefix?: string) => { + pathPrefix ??= "/sources"; + + entityActionQueueStore.add(tableName, EntityAction.Create, { + space: MetricsEventSpace.Modal, + screenName: get(appScreen).type, + medium: behaviourEventMedium, + }); + + await get(putFileMutation).mutateAsync({ + instanceId: get(runtime).instanceId, + path: `${pathPrefix}${tableName}.yaml`, + data: { + blob: yaml, + create: true, + createOnly: false, // The modal might be opened from a YAML file with placeholder text, so the file might already exist + }, + }); + + // Navigate to source page + goto(`/source/${tableName}`); + }; +} + export async function createSource( instanceId: string, tableName: string, diff --git a/web-common/src/features/sources/modal/submitRemoteSourceForm.ts b/web-common/src/features/sources/modal/submitRemoteSourceForm.ts index aeeb38ee77a..9d89e13d928 100644 --- a/web-common/src/features/sources/modal/submitRemoteSourceForm.ts +++ b/web-common/src/features/sources/modal/submitRemoteSourceForm.ts @@ -21,6 +21,7 @@ export interface RemoteSourceFormValues { [key: string]: any; } +// TODO: move this to new reconcile export async function submitRemoteSourceForm( queryClient: QueryClient, connectorName: string, @@ -32,7 +33,7 @@ export async function submitRemoteSourceForm( behaviourEvent?.fireSourceTriggerEvent( BehaviourEventAction.SourceAdd, BehaviourEventMedium.Button, - get(appScreen), + get(appScreen).type, MetricsEventSpace.Modal ); diff --git a/web-common/src/features/sources/navigation/SourceMenuItems.svelte b/web-common/src/features/sources/navigation/SourceMenuItems.svelte index fd384d0ee17..691f33d103f 100644 --- a/web-common/src/features/sources/navigation/SourceMenuItems.svelte +++ b/web-common/src/features/sources/navigation/SourceMenuItems.svelte @@ -1,5 +1,4 @@