From 10d7b94c710e0d44d4a016a1ec76ce6a8dee0c86 Mon Sep 17 00:00:00 2001 From: Thomas Draier Date: Wed, 8 Jan 2025 16:19:39 +0100 Subject: [PATCH] [front] Add analytics tab (#9823) * Add analytics tab * lint * remove label * remove log * block analytics for non-builder/admin * disable feedbacks for global * Update front/pages/api/w/[wId]/assistant/agent_configurations/[aId]/analytics.ts Co-authored-by: Lucas Massemin * Update front/pages/api/w/[wId]/assistant/agent_configurations/[aId]/analytics.ts Co-authored-by: Lucas Massemin * review --------- Co-authored-by: Lucas Massemin --- .../components/assistant/AssistantDetails.tsx | 279 ++++++++++++++++-- front/lib/api/assistant/agent_usage.ts | 111 ++++++- front/lib/swr/assistants.ts | 28 ++ .../agent_configurations/[aId]/analytics.ts | 151 ++++++++++ .../w/[wId]/builder/assistants/index.tsx | 3 +- 5 files changed, 532 insertions(+), 40 deletions(-) create mode 100644 front/pages/api/w/[wId]/assistant/agent_configurations/[aId]/analytics.ts diff --git a/front/components/assistant/AssistantDetails.tsx b/front/components/assistant/AssistantDetails.tsx index e9e0deb58b55..3cc4e996df88 100644 --- a/front/components/assistant/AssistantDetails.tsx +++ b/front/components/assistant/AssistantDetails.tsx @@ -1,6 +1,18 @@ import { Avatar, + BarChartIcon, + Button, + Card, + CardGrid, + ChatBubbleLeftRightIcon, + ChatBubbleThoughtIcon, ContentMessage, + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, + HandThumbDownIcon, + HandThumbUpIcon, InformationCircleIcon, Page, Sheet, @@ -8,8 +20,17 @@ import { SheetContent, SheetHeader, SheetTitle, + Tabs, + TabsList, + TabsTrigger, + Tooltip, } from "@dust-tt/sparkle"; -import type { AgentConfigurationScope, WorkspaceType } from "@dust-tt/types"; +import type { + AgentConfigurationScope, + AgentConfigurationType, + WorkspaceType, +} from "@dust-tt/types"; +import { removeNulls } from "@dust-tt/types"; import { useCallback, useState } from "react"; import { AssistantDetailsButtonBar } from "@app/components/assistant/AssistantDetailsButtonBar"; @@ -18,24 +39,222 @@ import { AssistantUsageSection } from "@app/components/assistant/details/Assista import { ReadOnlyTextArea } from "@app/components/assistant/ReadOnlyTextArea"; import { SharingDropdown } from "@app/components/assistant_builder/Sharing"; import { + useAgentAnalytics, useAgentConfiguration, useUpdateAgentScope, } from "@app/lib/swr/assistants"; import { classNames } from "@app/lib/utils"; +const PERIODS = [ + { value: 7, label: "Last 7 days" }, + { value: 15, label: "Last 15 days" }, + { value: 30, label: "Last 30 days" }, +]; + type AssistantDetailsProps = { owner: WorkspaceType; onClose: () => void; assistantId: string | null; + showPerformanceTab?: boolean; }; +function AssistantDetailsInfo({ + agentConfiguration, + owner, +}: { + agentConfiguration: AgentConfigurationType; + owner: WorkspaceType; +}) { + return ( + <> +
+ {agentConfiguration?.description} +
+ {agentConfiguration && ( + + )} + + + + + {agentConfiguration?.instructions ? ( +
+
Instructions
+ +
+ ) : ( + "This assistant has no instructions." + )} + + ); +} + +function AssistantDetailsPerformance({ + agentConfiguration, + owner, +}: { + agentConfiguration: AgentConfigurationType; + owner: WorkspaceType; +}) { + const [period, setPeriod] = useState(30); + const { agentAnalytics } = useAgentAnalytics({ + workspaceId: owner.sId, + agentConfigurationId: agentConfiguration.sId, + period, + }); + + return ( + <> +
+ Analytics +
+ + +
+
+ +
+ + +
+
+
Active Users
+
+
+ {agentAnalytics?.users ? ( + <> +
+ {agentAnalytics.users.length} +
+ + + {removeNulls(agentAnalytics.users.map((top) => top.user)) + .slice(0, 5) + .map((user) => ( + + } + label={user.fullName} + /> + ))} + + + ) : ( + "-" + )} +
+
+ + + +
+
+
Reactions
+
+
+ {agentConfiguration.scope !== "global" && + agentAnalytics?.feedbacks ? ( + <> +
+
+ +
+
{agentAnalytics.feedbacks.positiveFeedbacks}
+
+
+
+ +
+
{agentAnalytics.feedbacks.negativeFeedbacks}
+
+ + ) : ( + "-" + )} +
+
+ + +
+
+
Conversations
+
+
+
+
+ +
+
+ {agentAnalytics?.mentions + ? `${agentAnalytics.mentions.conversationCount}` + : "-"} +
+
+
+
+ + +
+
+
Messages
+
+
+
+
+ +
+
+ {agentAnalytics?.mentions + ? `${agentAnalytics.mentions.messageCount}` + : "-"} +
+
+
+
+ + + + ); +} + export function AssistantDetails({ assistantId, onClose, owner, + showPerformanceTab = false, }: AssistantDetailsProps) { const [isUpdatingScope, setIsUpdatingScope] = useState(false); - + const [selectedTab, setSelectedTab] = useState("info"); const { agentConfiguration, isAgentConfigurationValidating } = useAgentConfiguration({ workspaceId: owner.sId, @@ -103,30 +322,9 @@ export function AssistantDetails({ It is no longer active and cannot be used. )} - -
- {agentConfiguration?.description} -
- {agentConfiguration && ( - - )} - ); - const InstructionsSection = () => - agentConfiguration?.instructions ? ( -
-
Instructions
- -
- ) : ( - "This assistant has no instructions." - ); - return ( @@ -137,11 +335,36 @@ export function AssistantDetails({ {agentConfiguration && (
- - + {showPerformanceTab && ( + + + setSelectedTab("info")} + /> + setSelectedTab("performance")} + /> + + + )} + {selectedTab === "info" && ( + + )} + {selectedTab === "performance" && ( + + )}
)} diff --git a/front/lib/api/assistant/agent_usage.ts b/front/lib/api/assistant/agent_usage.ts index 88ce62f00516..a7710a486317 100644 --- a/front/lib/api/assistant/agent_usage.ts +++ b/front/lib/api/assistant/agent_usage.ts @@ -1,4 +1,8 @@ -import type { AgentConfigurationType } from "@dust-tt/types"; +import type { + AgentConfigurationType, + LightAgentConfigurationType, + LightWorkspaceType, +} from "@dust-tt/types"; import _ from "lodash"; import type { RedisClientType } from "redis"; import { literal, Op, Sequelize } from "sequelize"; @@ -9,14 +13,15 @@ import { Conversation, Mention, Message, + UserMessage, } from "@app/lib/models/assistant/conversation"; import { Workspace } from "@app/lib/models/workspace"; import { getAssistantUsageData } from "@app/lib/workspace_usage"; import { launchMentionsCountWorkflow } from "@app/temporal/mentions_count_queue/client"; // Ranking of agents is done over a 30 days period. -const rankingUsageDays = 30; -const rankingTimeframeSec = 60 * 60 * 24 * rankingUsageDays; +const RANKING_USAGE_DAYS = 30; +const RANKING_TIMEFRAME_SEC = 60 * 60 * 24 * RANKING_USAGE_DAYS; const MENTION_COUNT_TTL = 60 * 60 * 24 * 7; // 7 days @@ -32,9 +37,10 @@ type AgentUsageCount = { timePeriodSec: number; }; -type mentionCount = { +type MentionCount = { agentId: string; count: number; + conversationCount: number; timePeriodSec: number; }; @@ -86,7 +92,7 @@ export async function getAgentsUsage({ .map(([agentId, count]) => ({ agentId, messageCount: parseInt(count), - timePeriodSec: rankingTimeframeSec, + timePeriodSec: RANKING_TIMEFRAME_SEC, })) .sort((a, b) => b.messageCount - a.messageCount) .slice(0, limit); @@ -97,10 +103,12 @@ export async function getAgentUsage( { workspaceId, agentConfiguration, + rankingUsageDays = RANKING_USAGE_DAYS, }: { workspaceId: string; agentConfiguration: AgentConfigurationType; providedRedis?: RedisClientType; + rankingUsageDays?: number; } ): Promise { const owner = auth.workspace(); @@ -126,14 +134,16 @@ export async function getAgentUsage( ? { agentId: agentConfiguration.sId, messageCount: parseInt(agentUsage, 10), - timePeriodSec: rankingTimeframeSec, + timePeriodSec: RANKING_TIMEFRAME_SEC, } : null; } export async function agentMentionsCount( - workspaceId: number -): Promise { + workspaceId: number, + agentConfiguration?: LightAgentConfigurationType, + rankingUsageDays: number = RANKING_USAGE_DAYS +): Promise { // We retrieve mentions from conversations in order to optimize the query // Since we need to filter out by workspace id, retrieving mentions first // would lead to retrieve every single messages @@ -147,6 +157,10 @@ export async function agentMentionsCount( Sequelize.fn("COUNT", Sequelize.literal('"messages->mentions"."id"')), "count", ], + [ + Sequelize.fn("COUNT", Sequelize.literal("DISTINCT conversation.id")), + "conversationCount", + ], ], where: { workspaceId, @@ -163,6 +177,9 @@ export async function agentMentionsCount( required: true, attributes: [], where: { + ...(agentConfiguration + ? { agentConfigurationId: agentConfiguration.sId } + : {}), createdAt: { [Op.gt]: literal(`NOW() - INTERVAL '${rankingUsageDays} days'`), }, @@ -180,18 +197,20 @@ export async function agentMentionsCount( const castMention = mention as unknown as { agentConfigurationId: string; count: number; + conversationCount: number; }; return { agentId: castMention.agentConfigurationId, count: castMention.count, - timePeriodSec: rankingTimeframeSec, + conversationCount: castMention.conversationCount, + timePeriodSec: rankingUsageDays * 24 * 60 * 60, }; }); } export async function storeCountsInRedis( workspaceId: string, - agentMessageCounts: mentionCount[], + agentMessageCounts: MentionCount[], redis: RedisClientType ) { const agentMessageCountKey = _getUsageKey(workspaceId); @@ -207,7 +226,8 @@ export async function storeCountsInRedis( amcByAgentId[agentId] = { agentId, count: 0, - timePeriodSec: rankingTimeframeSec, + conversationCount: 0, + timePeriodSec: RANKING_TIMEFRAME_SEC, }; } } @@ -244,3 +264,72 @@ export async function signalAgentUsage({ await redis.hIncrBy(agentMessageCountKey, agentConfigurationId, 1); } } + +type UsersUsageCount = { + userId: number; + messageCount: number; + timePeriodSec: number; +}; + +export async function getAgentUsers( + owner: LightWorkspaceType, + agentConfiguration?: LightAgentConfigurationType, + rankingUsageDays: number = RANKING_USAGE_DAYS +): Promise { + const mentions = await Conversation.findAll({ + attributes: [ + [Sequelize.literal('"messages->userMessage"."userId"'), "userId"], + [ + Sequelize.fn("COUNT", Sequelize.literal('"messages->mentions"."id"')), + "count", + ], + ], + where: { + workspaceId: owner.id, + }, + include: [ + { + model: Message, + required: true, + attributes: [], + include: [ + { + model: Mention, + as: "mentions", + required: true, + attributes: [], + where: { + ...(agentConfiguration + ? { agentConfigurationId: agentConfiguration.sId } + : {}), + createdAt: { + [Op.gt]: literal(`NOW() - INTERVAL '${rankingUsageDays} days'`), + }, + }, + }, + { + model: UserMessage, + as: "userMessage", + required: true, + attributes: [], + }, + ], + }, + ], + order: [["count", "DESC"]], + group: ['"messages->userMessage"."userId"'], + raw: true, + }); + + return mentions.map((mention) => { + const castMention = mention as unknown as { + userId: number; + count: number; + }; + return { + userId: castMention.userId, + messageCount: castMention.count, + timePeriodSec: rankingUsageDays * 24 * 60 * 60, + }; + }); +} diff --git a/front/lib/swr/assistants.ts b/front/lib/swr/assistants.ts index 89fd81548d55..2709bdc45a71 100644 --- a/front/lib/swr/assistants.ts +++ b/front/lib/swr/assistants.ts @@ -22,6 +22,7 @@ import { useSWRWithDefaults, } from "@app/lib/swr/swr"; import type { GetAgentConfigurationsResponseBody } from "@app/pages/api/w/[wId]/assistant/agent_configurations"; +import type { GetAgentConfigurationAnalyticsResponseBody } from "@app/pages/api/w/[wId]/assistant/agent_configurations/[aId]/analytics"; import type { PostAgentScopeRequestBody } from "@app/pages/api/w/[wId]/assistant/agent_configurations/[aId]/scope"; import type { GetAgentUsageResponseBody } from "@app/pages/api/w/[wId]/assistant/agent_configurations/[aId]/usage"; import type { GetSlackChannelsLinkedWithAgentResponseBody } from "@app/pages/api/w/[wId]/assistant/builder/slack/channels_linked_with_agent"; @@ -387,6 +388,33 @@ export function useAgentUsage({ }; } +export function useAgentAnalytics({ + workspaceId, + agentConfigurationId, + period, + disabled, +}: { + workspaceId: string; + agentConfigurationId: string | null; + period: number; + disabled?: boolean; +}) { + const agentAnalyticsFetcher: Fetcher = + fetcher; + const fetchUrl = agentConfigurationId + ? `/api/w/${workspaceId}/assistant/agent_configurations/${agentConfigurationId}/analytics?period=${period}` + : null; + const { data, error } = useSWRWithDefaults(fetchUrl, agentAnalyticsFetcher, { + disabled, + }); + + return { + agentAnalytics: data ? data : null, + isAgentAnayticsLoading: !error && !data && !disabled, + isAgentAnayticsError: error, + }; +} + export function useSlackChannelsLinkedWithAgent({ workspaceId, disabled, diff --git a/front/pages/api/w/[wId]/assistant/agent_configurations/[aId]/analytics.ts b/front/pages/api/w/[wId]/assistant/agent_configurations/[aId]/analytics.ts new file mode 100644 index 000000000000..55e685bda05a --- /dev/null +++ b/front/pages/api/w/[wId]/assistant/agent_configurations/[aId]/analytics.ts @@ -0,0 +1,151 @@ +import type { + AgentConfigurationType, + UserType, + WithAPIErrorResponse, +} from "@dust-tt/types"; +import { isLeft } from "fp-ts/lib/Either"; +import * as t from "io-ts"; +import * as reporter from "io-ts-reporters"; +import type { NextApiRequest, NextApiResponse } from "next"; + +import { + agentMentionsCount, + getAgentUsers, +} from "@app/lib/api/assistant/agent_usage"; +import { getAgentConfiguration } from "@app/lib/api/assistant/configuration"; +import { withSessionAuthenticationForWorkspace } from "@app/lib/api/auth_wrappers"; +import type { Authenticator } from "@app/lib/auth"; +import { AgentMessageFeedbackResource } from "@app/lib/resources/agent_message_feedback_resource"; +import { UserResource } from "@app/lib/resources/user_resource"; +import { apiError } from "@app/logger/withlogging"; +export type GetAgentConfigurationResponseBody = { + agentConfiguration: AgentConfigurationType; +}; +export type DeleteAgentConfigurationResponseBody = { + success: boolean; +}; + +const GetAgentConfigurationsAnalyticsQuerySchema = t.type({ + period: t.string, +}); + +export type GetAgentConfigurationAnalyticsResponseBody = { + users: { + user: UserType | undefined; + count: number; + timePeriodSec: number; + }[]; + mentions: { + messageCount: number; + conversationCount: number; + timePeriodSec: number; + }; + feedbacks: { + positiveFeedbacks: number; + negativeFeedbacks: number; + timePeriodSec: number; + }; +}; + +async function handler( + req: NextApiRequest, + res: NextApiResponse< + WithAPIErrorResponse + >, + auth: Authenticator +): Promise { + const assistant = await getAgentConfiguration(auth, req.query.aId as string); + + if ( + !assistant || + (assistant.scope === "private" && + assistant.versionAuthorId !== auth.user()?.id) + ) { + return apiError(req, res, { + status_code: 404, + api_error: { + type: "agent_configuration_not_found", + message: "The assistant you're trying to access was not found.", + }, + }); + } + + if (assistant.scope === "workspace" && !auth.isBuilder()) { + return apiError(req, res, { + status_code: 404, + api_error: { + type: "app_auth_error", + message: "Only builders can get assistant analytics.", + }, + }); + } + + switch (req.method) { + case "GET": + const queryValidation = GetAgentConfigurationsAnalyticsQuerySchema.decode( + req.query + ); + if (isLeft(queryValidation)) { + const pathError = reporter.formatValidationErrors(queryValidation.left); + return apiError(req, res, { + status_code: 400, + api_error: { + type: "invalid_request_error", + message: `Invalid query parameters: ${pathError}`, + }, + }); + } + const period = parseInt(queryValidation.right.period); + + const owner = auth.getNonNullableWorkspace(); + const agentUsers = await getAgentUsers(owner, assistant, period); + const users = await UserResource.fetchByModelIds( + agentUsers.map((r) => r.userId) + ); + + const feedbacks = + await AgentMessageFeedbackResource.getFeedbackCountForAssistants( + auth, + [assistant.sId], + period + ); + const positiveFeedbacks = + feedbacks.find((f) => f.thumbDirection === "up")?.count ?? 0; + const negativeFeedbacks = + feedbacks.find((f) => f.thumbDirection === "down")?.count ?? 0; + + const mentionCounts = ( + await agentMentionsCount(owner.id, assistant, period) + )[0]; + + return res.status(200).json({ + users: agentUsers + .map((r) => ({ + user: users.find((u) => u.id === r.userId)?.toJSON(), + count: r.messageCount, + timePeriodSec: r.timePeriodSec, + })) + .filter((r) => r.user), + mentions: { + messageCount: mentionCounts?.count ?? 0, + conversationCount: mentionCounts?.conversationCount ?? 0, + timePeriodSec: mentionCounts?.timePeriodSec ?? period * 60 * 60 * 24, + }, + feedbacks: { + positiveFeedbacks, + negativeFeedbacks, + timePeriodSec: period * 60 * 60 * 24, + }, + }); + default: + return apiError(req, res, { + status_code: 405, + api_error: { + type: "method_not_supported_error", + message: "The method passed is not supported, GET is expected.", + }, + }); + } +} + +export default withSessionAuthenticationForWorkspace(handler); diff --git a/front/pages/w/[wId]/builder/assistants/index.tsx b/front/pages/w/[wId]/builder/assistants/index.tsx index b31ec22f2bb8..a841a077141a 100644 --- a/front/pages/w/[wId]/builder/assistants/index.tsx +++ b/front/pages/w/[wId]/builder/assistants/index.tsx @@ -110,7 +110,7 @@ export default function WorkspaceAssistants({ return a.scope === activeTab; } }); - console.log(activeTab, filteredAgents.length); + const [showDetails, setShowDetails] = useState(null); @@ -187,6 +187,7 @@ export default function WorkspaceAssistants({ navChildren={} > setShowDetails(null)}