From 1b1318e89cc84f6abf4728f3f4bde6f1c8bb75bd Mon Sep 17 00:00:00 2001 From: xm0onh Date: Sun, 22 Dec 2024 21:53:32 -0800 Subject: [PATCH 01/13] Enhance the efficiency of fetching conv thread --- auto-kol/agent/src/index.ts | 16 ++- auto-kol/agent/src/schemas/workflow.ts | 9 +- auto-kol/agent/src/services/agents/nodes.ts | 2 +- .../src/services/agents/nodes/mentionNode.ts | 27 +++- .../agents/nodes/responseGenerationNode.ts | 28 ++--- .../agent/src/services/agents/workflow.ts | 2 +- auto-kol/agent/src/services/twitter/api.ts | 118 +++++++++++------- auto-kol/agent/src/tools/tools/mentionTool.ts | 2 +- auto-kol/agent/src/types/twitter.ts | 2 + 9 files changed, 133 insertions(+), 73 deletions(-) diff --git a/auto-kol/agent/src/index.ts b/auto-kol/agent/src/index.ts index e1d9c3c..525024f 100644 --- a/auto-kol/agent/src/index.ts +++ b/auto-kol/agent/src/index.ts @@ -7,7 +7,7 @@ import apiRoutes from './api/index.js'; import { corsMiddleware } from './api/middleware/cors.js'; const logger = createLogger('app'); const app = express(); - +import { createTwitterClientScraper } from './services/twitter/api.js'; app.use(corsMiddleware); app.use(express.json()); @@ -27,6 +27,20 @@ const startWorkflowPolling = async () => { const main = async () => { try { + // const scraper = await createTwitterClientScraper(); + // logger.info('Scraper:', scraper.isLoggedIn()); + // const thread = await scraper.getThread('1870973877272375519'); + // const tweetsWithThreads: any[] = []; + // for await (const threadTweet of thread) { + // tweetsWithThreads.push({ + // id: threadTweet.id || '', + // text: threadTweet.text || '', + // author_id: threadTweet.userId || '', + // author_username: threadTweet.username?.toLowerCase() || 'unknown', + // created_at: threadTweet.timeParsed?.toISOString() || new Date().toISOString() + // }); + // } + // logger.info('Tweets with threads:', tweetsWithThreads); await initializeSchema(); app.listen(config.PORT, () => { diff --git a/auto-kol/agent/src/schemas/workflow.ts b/auto-kol/agent/src/schemas/workflow.ts index 374e76f..b110c11 100644 --- a/auto-kol/agent/src/schemas/workflow.ts +++ b/auto-kol/agent/src/schemas/workflow.ts @@ -7,7 +7,14 @@ export const tweetSearchSchema = z.object({ author_id: z.string(), author_username: z.string(), created_at: z.string(), - mention: z.boolean().optional() + mention: z.boolean().optional(), + thread: z.array(z.object({ + id: z.string(), + text: z.string(), + author_id: z.string(), + author_username: z.string(), + created_at: z.string() + })).optional() })), lastProcessedId: z.string().nullable().optional() }); diff --git a/auto-kol/agent/src/services/agents/nodes.ts b/auto-kol/agent/src/services/agents/nodes.ts index c79c966..6b86aa4 100644 --- a/auto-kol/agent/src/services/agents/nodes.ts +++ b/auto-kol/agent/src/services/agents/nodes.ts @@ -14,7 +14,7 @@ export const createNodes = async (config: WorkflowConfig) => { const scraper = await createTwitterClientScraper(); ///////////MENTIONS/////////// - const mentionNode = createMentionNode(config); + const mentionNode = createMentionNode(config, scraper); ///////////TIMELINE/////////// const timelineNode = createTimelineNode(config); diff --git a/auto-kol/agent/src/services/agents/nodes/mentionNode.ts b/auto-kol/agent/src/services/agents/nodes/mentionNode.ts index 5118b83..95c3b4f 100644 --- a/auto-kol/agent/src/services/agents/nodes/mentionNode.ts +++ b/auto-kol/agent/src/services/agents/nodes/mentionNode.ts @@ -3,8 +3,10 @@ import { parseMessageContent, WorkflowConfig } from "../workflow.js"; import { logger } from "../workflow.js"; import { State } from "../workflow.js"; import { tweetSearchSchema } from "../../../schemas/workflow.js"; +import { ExtendedScraper } from "../../twitter/api.js"; +import { Tweet } from "../../../types/twitter.js"; -export const createMentionNode = (config: WorkflowConfig) => { +export const createMentionNode = (config: WorkflowConfig, scraper: ExtendedScraper) => { return async (state: typeof State.State) => { logger.info('Mention Node - Fetching recent mentions'); const toolResponse = await config.toolNode.invoke({ @@ -23,9 +25,28 @@ export const createMentionNode = (config: WorkflowConfig) => { const parsedContent = parseMessageContent(toolResponse.messages[toolResponse.messages.length - 1].content); const parsedTweets = tweetSearchSchema.parse(parsedContent); - logger.info('Parsed tweets:', parsedTweets); + // logger.info('Parsed tweets:', parsedTweets); logger.info(`Found ${parsedTweets.tweets.length} tweets`); - + for (const tweet of parsedTweets.tweets) { + tweet.mention = true; + logger.info(`Getting thread for tweet ${tweet.id}`); + const tweetsWithThreads: Tweet[] = []; + const thread = await scraper.getThread(tweet.id); + for await (const threadTweet of thread) { + tweetsWithThreads.push({ + id: threadTweet.id || '', + text: threadTweet.text || '', + author_id: threadTweet.userId || '', + author_username: threadTweet.username?.toLowerCase() || 'unknown', + created_at: threadTweet.timeParsed?.toISOString() || new Date().toISOString() + }); + } + tweet.thread = tweetsWithThreads; + // Sleep for 1 second + await new Promise(resolve => setTimeout(resolve, 5000)); + break; + logger.info(`Found ${tweetsWithThreads.length} tweets in thread`); + } return { messages: [new AIMessage({ content: JSON.stringify(parsedTweets) diff --git a/auto-kol/agent/src/services/agents/nodes/responseGenerationNode.ts b/auto-kol/agent/src/services/agents/nodes/responseGenerationNode.ts index b002a5a..a15feea 100644 --- a/auto-kol/agent/src/services/agents/nodes/responseGenerationNode.ts +++ b/auto-kol/agent/src/services/agents/nodes/responseGenerationNode.ts @@ -23,7 +23,9 @@ export const createResponseGenerationNode = (config: WorkflowConfig, scraper: an await Promise.all( batchToRespond.map(async (item: any) => { const { tweet, decision, toneAnalysis, workflowState } = item; - + if (tweet.mention) { + logger.info('Response Generation Node - Tweet has a thread!', { tweetId: tweet.id }); + } if (!workflowState) { item.workflowState = { autoFeedback: [] }; } else if (!workflowState.autoFeedback) { @@ -46,23 +48,7 @@ export const createResponseGenerationNode = (config: WorkflowConfig, scraper: an : ''; const rejectionFeedback = lastFeedback ? prompts.formatRejectionFeedback(lastFeedback.reason, lastFeedback.suggestedChanges) - : ''; - - const threadMentionsTweets = []; - if (item?.mentions) { - threadMentionsTweets.push(...item.mentions); - } else if (tweet.mention) { - const mentions = await scraper.getThread(tweet.id); - for await (const mention of mentions) { - threadMentionsTweets.push({ - id: mention.id, - text: mention.text, - author_id: mention.userId, - author_username: mention.username?.toLowerCase() || 'unknown', - created_at: mention.timeParsed?.toISOString() || new Date().toISOString() - }); - } - } + : ''; const similarTweetsResponse = await config.toolNode.invoke({ messages: [new AIMessage({ @@ -90,7 +76,7 @@ export const createResponseGenerationNode = (config: WorkflowConfig, scraper: an tone: toneAnalysis?.suggestedTone || workflowState?.toneAnalysis?.suggestedTone, author: tweet.author_username, similarTweets: JSON.stringify(similarTweets.similar_tweets), - mentions: JSON.stringify(threadMentionsTweets), + mentions: JSON.stringify(tweet.thread || []), previousResponse: workflowState?.autoFeedback[workflowState?.autoFeedback.length - 1]?.response || '', rejectionFeedback, rejectionInstructions @@ -112,7 +98,7 @@ export const createResponseGenerationNode = (config: WorkflowConfig, scraper: an }, autoFeedback: workflowState?.autoFeedback || [] }, - mentions: threadMentionsTweets, + mentions: tweet.thread || [], retry: item.retry } batchToFeedback.push(data); @@ -123,7 +109,7 @@ export const createResponseGenerationNode = (config: WorkflowConfig, scraper: an workflowState: { toneAnalysis: toneAnalysis, responseStrategy, - mentions: threadMentionsTweets, + mentions: tweet.thread || [], similarTweets: similarTweets.similar_tweets, }, } diff --git a/auto-kol/agent/src/services/agents/workflow.ts b/auto-kol/agent/src/services/agents/workflow.ts index 5519076..6af3367 100644 --- a/auto-kol/agent/src/services/agents/workflow.ts +++ b/auto-kol/agent/src/services/agents/workflow.ts @@ -115,7 +115,7 @@ const shouldContinue = (state: typeof State.State) => { logger.info('Moving to recheck skipped tweets'); return 'recheckNode'; } - + // Check for batches to process if (content.batchToAnalyze?.length > 0) { logger.debug('Moving to tone analysis'); diff --git a/auto-kol/agent/src/services/twitter/api.ts b/auto-kol/agent/src/services/twitter/api.ts index bb773c0..2a3d612 100644 --- a/auto-kol/agent/src/services/twitter/api.ts +++ b/auto-kol/agent/src/services/twitter/api.ts @@ -101,63 +101,93 @@ export class ExtendedScraper extends Scraper { return replies; } - async getThread(tweetId: string): Promise { - const username = config.TWITTER_USERNAME!; - const isLoggedIn = await this.isLoggedIn(); + async getThread(tweetId: string, maxDepth: number = 100): Promise { + // First check login status + let isLoggedIn = await this.isLoggedIn(); + + // If not logged in, attempt to re-authenticate if (!isLoggedIn) { - throw new Error('Must be logged in to fetch thread'); + logger.warn('Session expired, attempting to re-authenticate...'); + try { + await this.initialize(); + isLoggedIn = await this.isLoggedIn(); + if (!isLoggedIn) { + logger.error('Re-authentication failed'); + return []; // Return empty array instead of throwing + } + logger.info('Successfully re-authenticated'); + } catch (error) { + logger.error('Error during re-authentication:', error); + return []; // Return empty array instead of throwing + } } - const thread: Tweet[] = []; - const seen = new Set(); - - const initialTweet = await this.getTweet(tweetId); - if (!initialTweet) { - throw new Error(`Tweet ${tweetId} not found`); - } + try { + const thread: Tweet[] = []; + const seen = new Set(); - let currentTweet = initialTweet; - while (currentTweet.inReplyToStatusId) { - const parentTweet = await this.getTweet(currentTweet.inReplyToStatusId); - if (!parentTweet) break; - if (!seen.has(parentTweet.id!)) { - thread.push(parentTweet); - seen.add(parentTweet.id!); + // Get the initial tweet + const initialTweet = await this.getTweet(tweetId); + if (!initialTweet) { + logger.warn(`Tweet ${tweetId} not found or deleted`); + return []; } - currentTweet = parentTweet; - } - if (!seen.has(initialTweet.id!)) { - thread.push(initialTweet); - seen.add(initialTweet.id!); - } + // Fetch all conversation tweets in one query + const conversationTweets = new Map(); + try { + const conversationIterator = this.searchTweets( + `conversation_id:${initialTweet.conversationId}`, + 100, + SearchMode.Latest + ); - const agentTweet = thread.find(t => t.username === username); - if (agentTweet) { - const replies = this.searchTweets( - `conversation_id:${currentTweet.id!} in_reply_to_tweet_id:${agentTweet.id!}`, - 100, - SearchMode.Latest - ); + for await (const tweet of conversationIterator) { + conversationTweets.set(tweet.id!, tweet); + } + } catch (error) { + logger.warn(`Error fetching conversation: ${error}`); + return [initialTweet]; // Return at least the initial tweet if we can't get the conversation + } - for await (const reply of replies) { - if (!seen.has(reply.id!)) { - thread.push(reply); - seen.add(reply.id!); + // Build the thread by following reply chains + const buildThread = (tweet: Tweet) => { + if (seen.has(tweet.id!)) return; + seen.add(tweet.id!); + thread.push(tweet); + + // Find direct replies to this tweet from our conversation map + for (const [, potentialReply] of conversationTweets) { + if (potentialReply.inReplyToStatusId === tweet.id) { + buildThread(potentialReply); + } } + }; + + // Find the root tweet by following the reply chain up + let rootTweet = initialTweet; + while (rootTweet.inReplyToStatusId) { + const parentTweet = conversationTweets.get(rootTweet.inReplyToStatusId); + if (!parentTweet) break; + rootTweet = parentTweet; } - } - // Sort chronologically - thread.sort((a, b) => { - const timeA = a.timeParsed?.getTime() || 0; - const timeB = b.timeParsed?.getTime() || 0; - return timeA - timeB; - }); + // Build thread starting from root + buildThread(rootTweet); - logger.info(`Retrieved conversation thread with ${thread.length} tweets starting from root tweet ${currentTweet.id!}`); + // Sort chronologically + thread.sort((a, b) => { + const timeA = a.timeParsed?.getTime() || 0; + const timeB = b.timeParsed?.getTime() || 0; + return timeA - timeB; + }); - return thread; + logger.info(`Retrieved thread with ${thread.length} tweets`); + return thread; + } catch (error) { + logger.error(`Unexpected error in getThread: ${error}`); + return []; + } } } diff --git a/auto-kol/agent/src/tools/tools/mentionTool.ts b/auto-kol/agent/src/tools/tools/mentionTool.ts index bff49d5..377a0c1 100644 --- a/auto-kol/agent/src/tools/tools/mentionTool.ts +++ b/auto-kol/agent/src/tools/tools/mentionTool.ts @@ -14,7 +14,7 @@ export const createMentionTool = (scraper: ExtendedScraper) => new DynamicStruct try { const sinceId = await getLatestMentionId(); const mentions = await scraper.getMyMentions(100, sinceId); - logger.info('Fetched mentions:', mentions); + // logger.info('Fetched mentions:', mentions); if (!mentions || mentions.length === 0) { logger.info('No new mentions found'); return { diff --git a/auto-kol/agent/src/types/twitter.ts b/auto-kol/agent/src/types/twitter.ts index 06a19d3..8ed96c8 100644 --- a/auto-kol/agent/src/types/twitter.ts +++ b/auto-kol/agent/src/types/twitter.ts @@ -4,6 +4,8 @@ export type Tweet = { readonly author_id: string; readonly author_username: string; readonly created_at: string; + readonly mention?: boolean; + readonly thread?: Tweet[]; } export type TwitterCredentials = { From 338b65a58e78f4e392c807083ba9790658846339 Mon Sep 17 00:00:00 2001 From: xm0onh Date: Mon, 23 Dec 2024 15:54:50 -0800 Subject: [PATCH 02/13] Fetch a thread related to agent account with minimum api call --- auto-kol/agent/src/services/agents/nodes/mentionNode.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/auto-kol/agent/src/services/agents/nodes/mentionNode.ts b/auto-kol/agent/src/services/agents/nodes/mentionNode.ts index 95c3b4f..8b9a288 100644 --- a/auto-kol/agent/src/services/agents/nodes/mentionNode.ts +++ b/auto-kol/agent/src/services/agents/nodes/mentionNode.ts @@ -25,7 +25,6 @@ export const createMentionNode = (config: WorkflowConfig, scraper: ExtendedScrap const parsedContent = parseMessageContent(toolResponse.messages[toolResponse.messages.length - 1].content); const parsedTweets = tweetSearchSchema.parse(parsedContent); - // logger.info('Parsed tweets:', parsedTweets); logger.info(`Found ${parsedTweets.tweets.length} tweets`); for (const tweet of parsedTweets.tweets) { tweet.mention = true; From 34f0ee34e1b0984d064163013c4e93c346d2b478 Mon Sep 17 00:00:00 2001 From: xm0onh Date: Mon, 23 Dec 2024 17:41:51 -0800 Subject: [PATCH 03/13] Integration new fetch mentioning with codebase --- auto-kol/agent/src/index.ts | 4 +- auto-kol/agent/src/schemas/workflow.ts | 3 +- .../services/agents/nodes/autoApprovalNode.ts | 5 +- .../services/agents/nodes/engagementNode.ts | 6 +- .../src/services/agents/nodes/mentionNode.ts | 4 +- .../agents/nodes/responseGenerationNode.ts | 8 +- .../services/agents/nodes/toneAnalysisNode.ts | 5 +- auto-kol/agent/src/services/agents/prompts.ts | 24 +++- auto-kol/agent/src/services/twitter/api.ts | 107 ++++++++++++------ auto-kol/agent/src/tools/tools/mentionTool.ts | 2 +- auto-kol/agent/src/types/twitter.ts | 1 - 11 files changed, 111 insertions(+), 58 deletions(-) diff --git a/auto-kol/agent/src/index.ts b/auto-kol/agent/src/index.ts index 525024f..3455934 100644 --- a/auto-kol/agent/src/index.ts +++ b/auto-kol/agent/src/index.ts @@ -8,6 +8,8 @@ import { corsMiddleware } from './api/middleware/cors.js'; const logger = createLogger('app'); const app = express(); import { createTwitterClientScraper } from './services/twitter/api.js'; +import { stringToCid, blake3HashFromCid, cidFromBlakeHash } from '@autonomys/auto-dag-data'; + app.use(corsMiddleware); app.use(express.json()); @@ -29,7 +31,7 @@ const main = async () => { try { // const scraper = await createTwitterClientScraper(); // logger.info('Scraper:', scraper.isLoggedIn()); - // const thread = await scraper.getThread('1870973877272375519'); + // const thread = await scraper.getThread('1870417326170149218'); // const tweetsWithThreads: any[] = []; // for await (const threadTweet of thread) { // tweetsWithThreads.push({ diff --git a/auto-kol/agent/src/schemas/workflow.ts b/auto-kol/agent/src/schemas/workflow.ts index b110c11..88dcbab 100644 --- a/auto-kol/agent/src/schemas/workflow.ts +++ b/auto-kol/agent/src/schemas/workflow.ts @@ -7,7 +7,6 @@ export const tweetSearchSchema = z.object({ author_id: z.string(), author_username: z.string(), created_at: z.string(), - mention: z.boolean().optional(), thread: z.array(z.object({ id: z.string(), text: z.string(), @@ -44,7 +43,7 @@ export const responseSchema = z.object({ reason: z.string(), similarity_score: z.number() })).optional(), - mentions: z.array(z.object({ + thread: z.array(z.object({ id: z.string(), text: z.string(), author_id: z.string(), diff --git a/auto-kol/agent/src/services/agents/nodes/autoApprovalNode.ts b/auto-kol/agent/src/services/agents/nodes/autoApprovalNode.ts index b79e6e7..440e7f4 100644 --- a/auto-kol/agent/src/services/agents/nodes/autoApprovalNode.ts +++ b/auto-kol/agent/src/services/agents/nodes/autoApprovalNode.ts @@ -62,8 +62,9 @@ export const createAutoApprovalNode = (config: WorkflowConfig, scraper: Extended sendTweetResponse }); } - - if (globalConfig.DSN_UPLOAD) { + if (response.tweet.thread?.length > 0) { + logger.info('Mention found!') + await uploadToDsn({ data: response, previousCid: await getLastDsnCid() diff --git a/auto-kol/agent/src/services/agents/nodes/engagementNode.ts b/auto-kol/agent/src/services/agents/nodes/engagementNode.ts index c82a992..2229e0f 100644 --- a/auto-kol/agent/src/services/agents/nodes/engagementNode.ts +++ b/auto-kol/agent/src/services/agents/nodes/engagementNode.ts @@ -86,11 +86,13 @@ export const createEngagementNode = (config: WorkflowConfig) => { if (state.processedTweets.has(tweet.id)) { return { tweet, status: 'alreadyProcessed' }; } - const decision = await prompts.engagementPrompt .pipe(config.llms.decision) .pipe(prompts.engagementParser) - .invoke({ tweet: tweet.text }) + .invoke({ + tweet: tweet.text, + thread: tweet.thread || [] + }) .catch((error) => { logger.error('Error in engagement node:', error); return { shouldEngage: false, reason: 'Error in engagement node', priority: 0, confidence: 0 }; diff --git a/auto-kol/agent/src/services/agents/nodes/mentionNode.ts b/auto-kol/agent/src/services/agents/nodes/mentionNode.ts index 8b9a288..3d9ec70 100644 --- a/auto-kol/agent/src/services/agents/nodes/mentionNode.ts +++ b/auto-kol/agent/src/services/agents/nodes/mentionNode.ts @@ -27,7 +27,6 @@ export const createMentionNode = (config: WorkflowConfig, scraper: ExtendedScrap const parsedTweets = tweetSearchSchema.parse(parsedContent); logger.info(`Found ${parsedTweets.tweets.length} tweets`); for (const tweet of parsedTweets.tweets) { - tweet.mention = true; logger.info(`Getting thread for tweet ${tweet.id}`); const tweetsWithThreads: Tweet[] = []; const thread = await scraper.getThread(tweet.id); @@ -41,10 +40,9 @@ export const createMentionNode = (config: WorkflowConfig, scraper: ExtendedScrap }); } tweet.thread = tweetsWithThreads; - // Sleep for 1 second await new Promise(resolve => setTimeout(resolve, 5000)); - break; logger.info(`Found ${tweetsWithThreads.length} tweets in thread`); + break; } return { messages: [new AIMessage({ diff --git a/auto-kol/agent/src/services/agents/nodes/responseGenerationNode.ts b/auto-kol/agent/src/services/agents/nodes/responseGenerationNode.ts index a15feea..4b14276 100644 --- a/auto-kol/agent/src/services/agents/nodes/responseGenerationNode.ts +++ b/auto-kol/agent/src/services/agents/nodes/responseGenerationNode.ts @@ -23,7 +23,7 @@ export const createResponseGenerationNode = (config: WorkflowConfig, scraper: an await Promise.all( batchToRespond.map(async (item: any) => { const { tweet, decision, toneAnalysis, workflowState } = item; - if (tweet.mention) { + if (tweet.thread?.length > 0) { logger.info('Response Generation Node - Tweet has a thread!', { tweetId: tweet.id }); } if (!workflowState) { @@ -68,6 +68,7 @@ export const createResponseGenerationNode = (config: WorkflowConfig, scraper: an const similarTweets = parseMessageContent( similarTweetsResponse.messages[similarTweetsResponse.messages.length - 1].content ); + const responseStrategy = await prompts.responsePrompt .pipe(config.llms.response) .pipe(prompts.responseParser) @@ -76,7 +77,7 @@ export const createResponseGenerationNode = (config: WorkflowConfig, scraper: an tone: toneAnalysis?.suggestedTone || workflowState?.toneAnalysis?.suggestedTone, author: tweet.author_username, similarTweets: JSON.stringify(similarTweets.similar_tweets), - mentions: JSON.stringify(tweet.thread || []), + thread: JSON.stringify(tweet.thread || []), previousResponse: workflowState?.autoFeedback[workflowState?.autoFeedback.length - 1]?.response || '', rejectionFeedback, rejectionInstructions @@ -98,7 +99,6 @@ export const createResponseGenerationNode = (config: WorkflowConfig, scraper: an }, autoFeedback: workflowState?.autoFeedback || [] }, - mentions: tweet.thread || [], retry: item.retry } batchToFeedback.push(data); @@ -109,7 +109,7 @@ export const createResponseGenerationNode = (config: WorkflowConfig, scraper: an workflowState: { toneAnalysis: toneAnalysis, responseStrategy, - mentions: tweet.thread || [], + thread: tweet.thread || [], similarTweets: similarTweets.similar_tweets, }, } diff --git a/auto-kol/agent/src/services/agents/nodes/toneAnalysisNode.ts b/auto-kol/agent/src/services/agents/nodes/toneAnalysisNode.ts index d371644..aa8f288 100644 --- a/auto-kol/agent/src/services/agents/nodes/toneAnalysisNode.ts +++ b/auto-kol/agent/src/services/agents/nodes/toneAnalysisNode.ts @@ -18,7 +18,10 @@ export const createToneAnalysisNode = (config: WorkflowConfig) => { const toneAnalysis = await prompts.tonePrompt .pipe(config.llms.tone) .pipe(prompts.toneParser) - .invoke({ tweet: tweet.text }); + .invoke({ + tweet: tweet.text, + thread: tweet.thread || [] + }); logger.info('Tone analysis:', { toneAnalysis }); diff --git a/auto-kol/agent/src/services/agents/prompts.ts b/auto-kol/agent/src/services/agents/prompts.ts index d4c6333..a402ba0 100644 --- a/auto-kol/agent/src/services/agents/prompts.ts +++ b/auto-kol/agent/src/services/agents/prompts.ts @@ -120,7 +120,13 @@ export const engagementPrompt = ChatPromptTemplate.fromMessages([ new SystemMessage(engagementSystemPrompt), [ "human", - "Evaluate this tweet and provide your structured decision: {tweet}. Do not attempt to follow links." + `Evaluate this tweet and provide your structured decision: + Tweet: {tweet} + Thread Context: {thread} + + DO NOT attempt to follow links. + + Note: If there is no thread context, evaluate the tweet on its own.` ] ]); @@ -128,20 +134,26 @@ export const tonePrompt = ChatPromptTemplate.fromMessages([ new SystemMessage(toneSystemPrompt), [ "human", - "Analyze the tone for this tweet and suggest a response tone: {tweet}" + `Analyze the tone for this tweet and suggest a response tone: + Tweet: {tweet} + Thread: {thread} + + DO NOT attempt to follow links. + + Note: If there is no thread context, evaluate the tweet on its own.` ] ]); export const responsePrompt = ChatPromptTemplate.fromMessages([ new SystemMessage(responseSystemPrompt), [ - "human", - `Generate a response strategy for this tweet by considering similar tweets from @{author} using the suggested tone: + "human", + `Generate a response strategy for this tweet by considering similar tweets from @{author} using the suggested tone: Tweet: {tweet} Tone: {tone} Author: {author} Similar Tweets: {similarTweets} - Mentions: {mentions} + thread: {thread} Previous Response: {previousResponse} Rejection Feedback: {rejectionFeedback} Rejection Instructions: {rejectionInstructions} @@ -157,7 +169,7 @@ export const responsePrompt = ChatPromptTemplate.fromMessages([ - Concise, direct, and invites further conversation. - Use the original language of the tweet if relevant. Prefer English, if there are more than one languages being used. - If there are mentions, respond accurately. Review the mentions thread with a focus on the most recent tweets and respond accordingly + If there a thread, respond accurately. Review the thread with a focus on the most recent tweets and respond accordingly If regenerating after rejection: - Include the rejection reason in your new response, - Explain how you’ve addressed it, diff --git a/auto-kol/agent/src/services/twitter/api.ts b/auto-kol/agent/src/services/twitter/api.ts index 2a3d612..c6cd6d3 100644 --- a/auto-kol/agent/src/services/twitter/api.ts +++ b/auto-kol/agent/src/services/twitter/api.ts @@ -102,42 +102,67 @@ export class ExtendedScraper extends Scraper { } async getThread(tweetId: string, maxDepth: number = 100): Promise { - // First check login status let isLoggedIn = await this.isLoggedIn(); - // If not logged in, attempt to re-authenticate if (!isLoggedIn) { - logger.warn('Session expired, attempting to re-authenticate...'); - try { - await this.initialize(); - isLoggedIn = await this.isLoggedIn(); - if (!isLoggedIn) { + const maxRetries = 3; + let retryCount = 0; + + while (!isLoggedIn && retryCount < maxRetries) { + logger.warn(`Session expired, attempting to re-authenticate... (attempt ${retryCount + 1}/${maxRetries})`); + try { + await this.initialize(); + isLoggedIn = await this.isLoggedIn(); + if (isLoggedIn) { + logger.info('Successfully re-authenticated'); + break; + } logger.error('Re-authentication failed'); - return []; // Return empty array instead of throwing + retryCount++; + if (retryCount < maxRetries) { + // Wait for 2 seconds before retrying + await new Promise(resolve => setTimeout(resolve, 2000)); + } + } catch (error) { + logger.error('Error during re-authentication:', error); + retryCount++; + if (retryCount < maxRetries) { + // Wait for 2 seconds before retrying + await new Promise(resolve => setTimeout(resolve, 2000)); + } } - logger.info('Successfully re-authenticated'); - } catch (error) { - logger.error('Error during re-authentication:', error); - return []; // Return empty array instead of throwing + } + + if (!isLoggedIn) { + logger.error(`Failed to re-authenticate after ${maxRetries} attempts`); + return []; } } try { const thread: Tweet[] = []; const seen = new Set(); + const myUsername = config.TWITTER_USERNAME!.toLowerCase(); + const conversationTweets = new Map(); - // Get the initial tweet const initialTweet = await this.getTweet(tweetId); if (!initialTweet) { logger.warn(`Tweet ${tweetId} not found or deleted`); return []; } - // Fetch all conversation tweets in one query - const conversationTweets = new Map(); + let rootTweet = initialTweet; + if (initialTweet.conversationId && initialTweet.conversationId !== initialTweet.id) { + const conversationRoot = await this.getTweet(initialTweet.conversationId); + if (conversationRoot) { + rootTweet = conversationRoot; + conversationTweets.set(rootTweet.id!, rootTweet); + } + } + try { const conversationIterator = this.searchTweets( - `conversation_id:${initialTweet.conversationId}`, + `conversation_id:${initialTweet.conversationId} (from:${myUsername} OR to:${myUsername})`, 100, SearchMode.Latest ); @@ -147,42 +172,54 @@ export class ExtendedScraper extends Scraper { } } catch (error) { logger.warn(`Error fetching conversation: ${error}`); - return [initialTweet]; // Return at least the initial tweet if we can't get the conversation + return [rootTweet, initialTweet]; } - // Build the thread by following reply chains + thread.push(rootTweet); + seen.add(rootTweet.id!); + const buildThread = (tweet: Tweet) => { if (seen.has(tweet.id!)) return; - seen.add(tweet.id!); - thread.push(tweet); + + const isRelevant = + tweet.username?.toLowerCase() === myUsername || // Tweet is from agent + tweet.text?.toLowerCase().includes(`@${myUsername}`) || // Tweet mentions agent + Array.from(conversationTweets.values()).some(t => // Tweet is replied to by agent + t.username?.toLowerCase() === myUsername && + t.inReplyToStatusId === tweet.id + ); + + if (isRelevant) { + seen.add(tweet.id!); + thread.push(tweet); + + // If this tweet is a reply, get its parent + if (tweet.inReplyToStatusId) { + const parentTweet = conversationTweets.get(tweet.inReplyToStatusId); + if (parentTweet && !seen.has(parentTweet.id!)) { + buildThread(parentTweet); + } + } - // Find direct replies to this tweet from our conversation map - for (const [, potentialReply] of conversationTweets) { - if (potentialReply.inReplyToStatusId === tweet.id) { - buildThread(potentialReply); + // Get direct replies to this tweet + for (const [, potentialReply] of conversationTweets) { + if (potentialReply.inReplyToStatusId === tweet.id) { + buildThread(potentialReply); + } } } }; - // Find the root tweet by following the reply chain up - let rootTweet = initialTweet; - while (rootTweet.inReplyToStatusId) { - const parentTweet = conversationTweets.get(rootTweet.inReplyToStatusId); - if (!parentTweet) break; - rootTweet = parentTweet; - } + buildThread(initialTweet); - // Build thread starting from root - buildThread(rootTweet); - // Sort chronologically thread.sort((a, b) => { const timeA = a.timeParsed?.getTime() || 0; const timeB = b.timeParsed?.getTime() || 0; return timeA - timeB; }); - logger.info(`Retrieved thread with ${thread.length} tweets`); + logger.info(`Retrieved thread with ${thread.length} relevant tweets (including root tweet)`); return thread; } catch (error) { logger.error(`Unexpected error in getThread: ${error}`); diff --git a/auto-kol/agent/src/tools/tools/mentionTool.ts b/auto-kol/agent/src/tools/tools/mentionTool.ts index 377a0c1..086bdbc 100644 --- a/auto-kol/agent/src/tools/tools/mentionTool.ts +++ b/auto-kol/agent/src/tools/tools/mentionTool.ts @@ -29,7 +29,7 @@ export const createMentionTool = (scraper: ExtendedScraper) => new DynamicStruct author_id: mention.userId!, author_username: mention.username!.toLowerCase(), created_at: mention.timeParsed!.toISOString(), - mention: true + thread: [] })); await addMention({ diff --git a/auto-kol/agent/src/types/twitter.ts b/auto-kol/agent/src/types/twitter.ts index 8ed96c8..d1fad59 100644 --- a/auto-kol/agent/src/types/twitter.ts +++ b/auto-kol/agent/src/types/twitter.ts @@ -4,7 +4,6 @@ export type Tweet = { readonly author_id: string; readonly author_username: string; readonly created_at: string; - readonly mention?: boolean; readonly thread?: Tweet[]; } From 968055ba8e202a0948c06dc2e1ab36ea60d1176a Mon Sep 17 00:00:00 2001 From: xm0onh Date: Mon, 23 Dec 2024 19:34:19 -0800 Subject: [PATCH 04/13] refactoring - preparing for PR --- auto-kol/agent/src/services/agents/nodes/autoApprovalNode.ts | 3 +-- auto-kol/agent/src/services/agents/nodes/mentionNode.ts | 3 +-- .../agent/src/services/agents/nodes/responseGenerationNode.ts | 3 --- auto-kol/agent/src/tools/tools/mentionTool.ts | 1 - 4 files changed, 2 insertions(+), 8 deletions(-) diff --git a/auto-kol/agent/src/services/agents/nodes/autoApprovalNode.ts b/auto-kol/agent/src/services/agents/nodes/autoApprovalNode.ts index 440e7f4..741675d 100644 --- a/auto-kol/agent/src/services/agents/nodes/autoApprovalNode.ts +++ b/auto-kol/agent/src/services/agents/nodes/autoApprovalNode.ts @@ -62,9 +62,8 @@ export const createAutoApprovalNode = (config: WorkflowConfig, scraper: Extended sendTweetResponse }); } - if (response.tweet.thread?.length > 0) { - logger.info('Mention found!') + if (globalConfig.DSN_UPLOAD) { await uploadToDsn({ data: response, previousCid: await getLastDsnCid() diff --git a/auto-kol/agent/src/services/agents/nodes/mentionNode.ts b/auto-kol/agent/src/services/agents/nodes/mentionNode.ts index 3d9ec70..db5a479 100644 --- a/auto-kol/agent/src/services/agents/nodes/mentionNode.ts +++ b/auto-kol/agent/src/services/agents/nodes/mentionNode.ts @@ -40,9 +40,8 @@ export const createMentionNode = (config: WorkflowConfig, scraper: ExtendedScrap }); } tweet.thread = tweetsWithThreads; - await new Promise(resolve => setTimeout(resolve, 5000)); + await new Promise(resolve => setTimeout(resolve, 1000)); logger.info(`Found ${tweetsWithThreads.length} tweets in thread`); - break; } return { messages: [new AIMessage({ diff --git a/auto-kol/agent/src/services/agents/nodes/responseGenerationNode.ts b/auto-kol/agent/src/services/agents/nodes/responseGenerationNode.ts index 4b14276..a2e5416 100644 --- a/auto-kol/agent/src/services/agents/nodes/responseGenerationNode.ts +++ b/auto-kol/agent/src/services/agents/nodes/responseGenerationNode.ts @@ -1,10 +1,7 @@ import { AIMessage } from "@langchain/core/messages"; import { State, logger, parseMessageContent } from '../workflow.js'; import * as prompts from '../prompts.js'; -import { uploadToDsn } from '../../../utils/dsn.js'; -import { getLastDsnCid } from '../../../database/index.js'; import { WorkflowConfig } from '../workflow.js'; -import { config as globalConfig } from '../../../config/index.js'; import { ResponseStatus } from '../../../types/queue.js'; export const createResponseGenerationNode = (config: WorkflowConfig, scraper: any) => { diff --git a/auto-kol/agent/src/tools/tools/mentionTool.ts b/auto-kol/agent/src/tools/tools/mentionTool.ts index 086bdbc..14d462b 100644 --- a/auto-kol/agent/src/tools/tools/mentionTool.ts +++ b/auto-kol/agent/src/tools/tools/mentionTool.ts @@ -14,7 +14,6 @@ export const createMentionTool = (scraper: ExtendedScraper) => new DynamicStruct try { const sinceId = await getLatestMentionId(); const mentions = await scraper.getMyMentions(100, sinceId); - // logger.info('Fetched mentions:', mentions); if (!mentions || mentions.length === 0) { logger.info('No new mentions found'); return { From eccd7d5230319375999c47ad9ca890b441e1b5f6 Mon Sep 17 00:00:00 2001 From: xm0onh Date: Mon, 23 Dec 2024 19:35:14 -0800 Subject: [PATCH 05/13] delete inner testing codes --- auto-kol/agent/src/index.ts | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/auto-kol/agent/src/index.ts b/auto-kol/agent/src/index.ts index 3455934..fe5e8d0 100644 --- a/auto-kol/agent/src/index.ts +++ b/auto-kol/agent/src/index.ts @@ -29,20 +29,6 @@ const startWorkflowPolling = async () => { const main = async () => { try { - // const scraper = await createTwitterClientScraper(); - // logger.info('Scraper:', scraper.isLoggedIn()); - // const thread = await scraper.getThread('1870417326170149218'); - // const tweetsWithThreads: any[] = []; - // for await (const threadTweet of thread) { - // tweetsWithThreads.push({ - // id: threadTweet.id || '', - // text: threadTweet.text || '', - // author_id: threadTweet.userId || '', - // author_username: threadTweet.username?.toLowerCase() || 'unknown', - // created_at: threadTweet.timeParsed?.toISOString() || new Date().toISOString() - // }); - // } - // logger.info('Tweets with threads:', tweetsWithThreads); await initializeSchema(); app.listen(config.PORT, () => { From f5e686a14ac20d82f6edbfff0b6d88d9500f3e6a Mon Sep 17 00:00:00 2001 From: xm0onh Date: Mon, 23 Dec 2024 19:35:34 -0800 Subject: [PATCH 06/13] delete inner testing codes --- auto-kol/agent/src/index.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/auto-kol/agent/src/index.ts b/auto-kol/agent/src/index.ts index fe5e8d0..e1d9c3c 100644 --- a/auto-kol/agent/src/index.ts +++ b/auto-kol/agent/src/index.ts @@ -7,8 +7,6 @@ import apiRoutes from './api/index.js'; import { corsMiddleware } from './api/middleware/cors.js'; const logger = createLogger('app'); const app = express(); -import { createTwitterClientScraper } from './services/twitter/api.js'; -import { stringToCid, blake3HashFromCid, cidFromBlakeHash } from '@autonomys/auto-dag-data'; app.use(corsMiddleware); From 22279666ecb34401917ee3c2cec83873a7ae86ae Mon Sep 17 00:00:00 2001 From: xm0onh Date: Tue, 24 Dec 2024 10:00:43 -0800 Subject: [PATCH 07/13] refactoring the types and comments from PR --- auto-kol/agent/src/services/agents/nodes.ts | 9 +- .../services/agents/nodes/autoApprovalNode.ts | 5 +- .../src/services/agents/nodes/mentionNode.ts | 23 +-- .../agents/nodes/responseGenerationNode.ts | 2 +- .../agent/src/services/agents/workflow.ts | 4 +- auto-kol/agent/src/services/twitter/api.ts | 194 +++++++++++------- auto-kol/agent/src/tools/tools/mentionTool.ts | 22 +- 7 files changed, 151 insertions(+), 108 deletions(-) diff --git a/auto-kol/agent/src/services/agents/nodes.ts b/auto-kol/agent/src/services/agents/nodes.ts index 6b86aa4..8a9841e 100644 --- a/auto-kol/agent/src/services/agents/nodes.ts +++ b/auto-kol/agent/src/services/agents/nodes.ts @@ -1,5 +1,4 @@ import { WorkflowConfig } from './workflow.js'; -import { createTwitterClientScraper } from '../twitter/api.js'; import { createSearchNode } from './nodes/searchNode.js'; import { createEngagementNode } from "./nodes/engagementNode.js"; import { createToneAnalysisNode } from "./nodes/toneAnalysisNode.js"; @@ -11,10 +10,8 @@ import { createAutoApprovalNode } from './nodes/autoApprovalNode.js'; export const createNodes = async (config: WorkflowConfig) => { - const scraper = await createTwitterClientScraper(); - ///////////MENTIONS/////////// - const mentionNode = createMentionNode(config, scraper); + const mentionNode = createMentionNode(config); ///////////TIMELINE/////////// const timelineNode = createTimelineNode(config); @@ -29,13 +26,13 @@ export const createNodes = async (config: WorkflowConfig) => { const toneAnalysisNode = createToneAnalysisNode(config); ///////////RESPONSE GENERATION/////////// - const responseGenerationNode = createResponseGenerationNode(config, scraper); + const responseGenerationNode = createResponseGenerationNode(config); ///////////RECHECK SKIPPED/////////// const recheckSkippedNode = createRecheckSkippedNode(config); ///////////AUTO APPROVAL/////////// - const autoApprovalNode = createAutoApprovalNode(config, scraper); + const autoApprovalNode = createAutoApprovalNode(config); return { mentionNode, diff --git a/auto-kol/agent/src/services/agents/nodes/autoApprovalNode.ts b/auto-kol/agent/src/services/agents/nodes/autoApprovalNode.ts index 4dc6ec1..f400214 100644 --- a/auto-kol/agent/src/services/agents/nodes/autoApprovalNode.ts +++ b/auto-kol/agent/src/services/agents/nodes/autoApprovalNode.ts @@ -6,9 +6,8 @@ import { getLastDsnCid, updateResponseStatusByTweetId } from '../../../database/ import { uploadToDsn } from "../../../utils/dsn.js"; import { config as globalConfig } from '../../../config/index.js'; import { ResponseStatus } from '../../../types/queue.js'; -import { ExtendedScraper } from '../../../services/twitter/api.js'; -export const createAutoApprovalNode = (config: WorkflowConfig, scraper: ExtendedScraper) => { +export const createAutoApprovalNode = (config: WorkflowConfig) => { return async (state: typeof State.State) => { logger.info('Auto Approval Node - Evaluating pending responses'); try { @@ -57,7 +56,7 @@ export const createAutoApprovalNode = (config: WorkflowConfig, scraper: Extended tweetId: response.tweet.id }); - const sendTweetResponse = await scraper.sendTweet(response.response, response.tweet.id); + const sendTweetResponse = await config.client.sendTweet(response.response, response.tweet.id); logger.info('Tweet sent', { sendTweetResponse }); diff --git a/auto-kol/agent/src/services/agents/nodes/mentionNode.ts b/auto-kol/agent/src/services/agents/nodes/mentionNode.ts index db5a479..bf414b4 100644 --- a/auto-kol/agent/src/services/agents/nodes/mentionNode.ts +++ b/auto-kol/agent/src/services/agents/nodes/mentionNode.ts @@ -3,10 +3,8 @@ import { parseMessageContent, WorkflowConfig } from "../workflow.js"; import { logger } from "../workflow.js"; import { State } from "../workflow.js"; import { tweetSearchSchema } from "../../../schemas/workflow.js"; -import { ExtendedScraper } from "../../twitter/api.js"; -import { Tweet } from "../../../types/twitter.js"; -export const createMentionNode = (config: WorkflowConfig, scraper: ExtendedScraper) => { +export const createMentionNode = (config: WorkflowConfig) => { return async (state: typeof State.State) => { logger.info('Mention Node - Fetching recent mentions'); const toolResponse = await config.toolNode.invoke({ @@ -25,24 +23,7 @@ export const createMentionNode = (config: WorkflowConfig, scraper: ExtendedScrap const parsedContent = parseMessageContent(toolResponse.messages[toolResponse.messages.length - 1].content); const parsedTweets = tweetSearchSchema.parse(parsedContent); - logger.info(`Found ${parsedTweets.tweets.length} tweets`); - for (const tweet of parsedTweets.tweets) { - logger.info(`Getting thread for tweet ${tweet.id}`); - const tweetsWithThreads: Tweet[] = []; - const thread = await scraper.getThread(tweet.id); - for await (const threadTweet of thread) { - tweetsWithThreads.push({ - id: threadTweet.id || '', - text: threadTweet.text || '', - author_id: threadTweet.userId || '', - author_username: threadTweet.username?.toLowerCase() || 'unknown', - created_at: threadTweet.timeParsed?.toISOString() || new Date().toISOString() - }); - } - tweet.thread = tweetsWithThreads; - await new Promise(resolve => setTimeout(resolve, 1000)); - logger.info(`Found ${tweetsWithThreads.length} tweets in thread`); - } + return { messages: [new AIMessage({ content: JSON.stringify(parsedTweets) diff --git a/auto-kol/agent/src/services/agents/nodes/responseGenerationNode.ts b/auto-kol/agent/src/services/agents/nodes/responseGenerationNode.ts index a149c88..5a26dd7 100644 --- a/auto-kol/agent/src/services/agents/nodes/responseGenerationNode.ts +++ b/auto-kol/agent/src/services/agents/nodes/responseGenerationNode.ts @@ -4,7 +4,7 @@ import * as prompts from '../prompts.js'; import { WorkflowConfig } from '../workflow.js'; import { ResponseStatus } from '../../../types/queue.js'; -export const createResponseGenerationNode = (config: WorkflowConfig, scraper: any) => { +export const createResponseGenerationNode = (config: WorkflowConfig) => { return async (state: typeof State.State) => { logger.info('Response Generation Node - Creating response strategy'); try { diff --git a/auto-kol/agent/src/services/agents/workflow.ts b/auto-kol/agent/src/services/agents/workflow.ts index 75cedba..1026efd 100644 --- a/auto-kol/agent/src/services/agents/workflow.ts +++ b/auto-kol/agent/src/services/agents/workflow.ts @@ -6,7 +6,7 @@ import { config } from '../../config/index.js'; import { createLogger } from '../../utils/logger.js'; import { createTools } from '../../tools/index.js'; import { ToolNode } from '@langchain/langgraph/prebuilt'; -import { createTwitterClientScraper } from '../twitter/api.js'; +import { createTwitterClientScraper, ExtendedScraper } from '../twitter/api.js'; export const logger = createLogger('agent-workflow'); import { createNodes } from './nodes.js'; @@ -34,7 +34,7 @@ export const State = Annotation.Root({ }); export type WorkflowConfig = Readonly<{ - client: any; + client: ExtendedScraper; toolNode: ToolNode; llms: Readonly<{ decision: ChatOpenAI; diff --git a/auto-kol/agent/src/services/twitter/api.ts b/auto-kol/agent/src/services/twitter/api.ts index c6cd6d3..cb1b4b9 100644 --- a/auto-kol/agent/src/services/twitter/api.ts +++ b/auto-kol/agent/src/services/twitter/api.ts @@ -50,6 +50,39 @@ export class ExtendedScraper extends Scraper { logger.info(`Login status: ${isLoggedIn}`); } + private async reAuthenticate(maxRetries: number = 3): Promise { + let isLoggedIn = false; + let retryCount = 0; + + while (!isLoggedIn && retryCount < maxRetries) { + logger.warn(`Session expired, attempting to re-authenticate... (attempt ${retryCount + 1}/${maxRetries})`); + try { + await this.initialize(); + isLoggedIn = await this.isLoggedIn(); + if (isLoggedIn) { + logger.info('Successfully re-authenticated'); + return true; + } + logger.error('Re-authentication failed'); + retryCount++; + if (retryCount < maxRetries) { + const delay = 2000 * Math.pow(2, retryCount - 1); + await new Promise(resolve => setTimeout(resolve, delay)); + } + } catch (error) { + logger.error('Error during re-authentication:', error); + retryCount++; + if (retryCount < maxRetries) { + const delay = 2000 * Math.pow(2, retryCount - 1); + await new Promise(resolve => setTimeout(resolve, delay)); + } + } + } + + logger.error(`Failed to re-authenticate after ${maxRetries} attempts`); + return false; + } + async getMyMentions(maxResults: number = 100, sinceId?: string) { const username = config.TWITTER_USERNAME!; @@ -101,40 +134,73 @@ export class ExtendedScraper extends Scraper { return replies; } - async getThread(tweetId: string, maxDepth: number = 100): Promise { - let isLoggedIn = await this.isLoggedIn(); - + async getThread(tweetId: string): Promise { + const username = config.TWITTER_USERNAME!; + const isLoggedIn = await this.isLoggedIn(); if (!isLoggedIn) { - const maxRetries = 3; - let retryCount = 0; - - while (!isLoggedIn && retryCount < maxRetries) { - logger.warn(`Session expired, attempting to re-authenticate... (attempt ${retryCount + 1}/${maxRetries})`); - try { - await this.initialize(); - isLoggedIn = await this.isLoggedIn(); - if (isLoggedIn) { - logger.info('Successfully re-authenticated'); - break; - } - logger.error('Re-authentication failed'); - retryCount++; - if (retryCount < maxRetries) { - // Wait for 2 seconds before retrying - await new Promise(resolve => setTimeout(resolve, 2000)); - } - } catch (error) { - logger.error('Error during re-authentication:', error); - retryCount++; - if (retryCount < maxRetries) { - // Wait for 2 seconds before retrying - await new Promise(resolve => setTimeout(resolve, 2000)); - } + throw new Error('Must be logged in to fetch thread'); + } + + const thread: Tweet[] = []; + const seen = new Set(); + + const initialTweet = await this.getTweet(tweetId); + if (!initialTweet) { + throw new Error(`Tweet ${tweetId} not found`); + } + + let currentTweet = initialTweet; + while (currentTweet.inReplyToStatusId) { + const parentTweet = await this.getTweet(currentTweet.inReplyToStatusId); + if (!parentTweet) break; + if (!seen.has(parentTweet.id!)) { + thread.push(parentTweet); + seen.add(parentTweet.id!); + } + currentTweet = parentTweet; + } + + if (!seen.has(initialTweet.id!)) { + thread.push(initialTweet); + seen.add(initialTweet.id!); + } + + const agentTweet = thread.find(t => t.username === username); + if (agentTweet) { + const replies = this.searchTweets( + `conversation_id:${currentTweet.id!} in_reply_to_tweet_id:${agentTweet.id!}`, + 100, + SearchMode.Latest + ); + + for await (const reply of replies) { + if (!seen.has(reply.id!)) { + thread.push(reply); + seen.add(reply.id!); } } - - if (!isLoggedIn) { - logger.error(`Failed to re-authenticate after ${maxRetries} attempts`); + } + + // Sort chronologically + thread.sort((a, b) => { + const timeA = a.timeParsed?.getTime() || 0; + const timeB = b.timeParsed?.getTime() || 0; + return timeA - timeB; + }); + + logger.info(`Retrieved conversation thread with ${thread.length} tweets starting from root tweet ${currentTweet.id!}`); + + return thread; + } + + // Placeholder for efficient thread fetching + async getThreadPlaceHolder(tweetId: string, maxDepth: number = 100): Promise { + const username = config.TWITTER_USERNAME!; + const isLoggedIn = await this.isLoggedIn(); + if (!isLoggedIn) { + const reAuthenticate = await this.reAuthenticate(); + if (!reAuthenticate) { + logger.error('Failed to re-authenticate'); return []; } } @@ -142,7 +208,6 @@ export class ExtendedScraper extends Scraper { try { const thread: Tweet[] = []; const seen = new Set(); - const myUsername = config.TWITTER_USERNAME!.toLowerCase(); const conversationTweets = new Map(); const initialTweet = await this.getTweet(tweetId); @@ -152,74 +217,57 @@ export class ExtendedScraper extends Scraper { } let rootTweet = initialTweet; + const conversationId = initialTweet.conversationId || initialTweet.id; + + logger.info('Initial tweet:', { + id: initialTweet.id, + conversationId: conversationId, + inReplyToStatusId: initialTweet.inReplyToStatusId + }); + if (initialTweet.conversationId && initialTweet.conversationId !== initialTweet.id) { const conversationRoot = await this.getTweet(initialTweet.conversationId); if (conversationRoot) { rootTweet = conversationRoot; conversationTweets.set(rootTweet.id!, rootTweet); + logger.info('Found root tweet:', { + id: rootTweet.id, + conversationId: rootTweet.conversationId + }); } } try { + logger.info('Fetching conversation with query:', `conversation_id:${conversationId}`); const conversationIterator = this.searchTweets( - `conversation_id:${initialTweet.conversationId} (from:${myUsername} OR to:${myUsername})`, + `conversation_id:${conversationId}`, 100, SearchMode.Latest ); for await (const tweet of conversationIterator) { conversationTweets.set(tweet.id!, tweet); + logger.info('Found conversation tweet:', { + id: tweet.id, + inReplyToStatusId: tweet.inReplyToStatusId, + text: tweet.text?.substring(0, 50) + '...' + }); } + + logger.info('Total conversation tweets found:', conversationTweets.size); } catch (error) { logger.warn(`Error fetching conversation: ${error}`); return [rootTweet, initialTweet]; } - thread.push(rootTweet); - seen.add(rootTweet.id!); - - const buildThread = (tweet: Tweet) => { - if (seen.has(tweet.id!)) return; - - const isRelevant = - tweet.username?.toLowerCase() === myUsername || // Tweet is from agent - tweet.text?.toLowerCase().includes(`@${myUsername}`) || // Tweet mentions agent - Array.from(conversationTweets.values()).some(t => // Tweet is replied to by agent - t.username?.toLowerCase() === myUsername && - t.inReplyToStatusId === tweet.id - ); - - if (isRelevant) { - seen.add(tweet.id!); - thread.push(tweet); - - // If this tweet is a reply, get its parent - if (tweet.inReplyToStatusId) { - const parentTweet = conversationTweets.get(tweet.inReplyToStatusId); - if (parentTweet && !seen.has(parentTweet.id!)) { - buildThread(parentTweet); - } - } - - // Get direct replies to this tweet - for (const [, potentialReply] of conversationTweets) { - if (potentialReply.inReplyToStatusId === tweet.id) { - buildThread(potentialReply); - } - } - } - }; - - buildThread(initialTweet); - - + thread.push(...conversationTweets.values()); thread.sort((a, b) => { const timeA = a.timeParsed?.getTime() || 0; const timeB = b.timeParsed?.getTime() || 0; return timeA - timeB; }); - logger.info(`Retrieved thread with ${thread.length} relevant tweets (including root tweet)`); + logger.info(`Retrieved thread with ${thread.length} tweets`); return thread; } catch (error) { logger.error(`Unexpected error in getThread: ${error}`); @@ -228,6 +276,8 @@ export class ExtendedScraper extends Scraper { } } + + export const createTwitterClientScraper = async () => { return ExtendedScraper.getInstance(); }; \ No newline at end of file diff --git a/auto-kol/agent/src/tools/tools/mentionTool.ts b/auto-kol/agent/src/tools/tools/mentionTool.ts index 14d462b..8d570bf 100644 --- a/auto-kol/agent/src/tools/tools/mentionTool.ts +++ b/auto-kol/agent/src/tools/tools/mentionTool.ts @@ -3,7 +3,7 @@ import { z } from 'zod'; import { createLogger } from '../../utils/logger.js'; import { addMention, getLatestMentionId } from '../../database/index.js'; import { ExtendedScraper } from '../../services/twitter/api.js'; - +import { Tweet } from '../../types/twitter.js'; const logger = createLogger('mention-tool'); export const createMentionTool = (scraper: ExtendedScraper) => new DynamicStructuredTool({ @@ -28,7 +28,7 @@ export const createMentionTool = (scraper: ExtendedScraper) => new DynamicStruct author_id: mention.userId!, author_username: mention.username!.toLowerCase(), created_at: mention.timeParsed!.toISOString(), - thread: [] + thread: [] as Tweet[] })); await addMention({ @@ -36,7 +36,23 @@ export const createMentionTool = (scraper: ExtendedScraper) => new DynamicStruct }); logger.info(`Fetched ${tweets.length} new mentions`); - + for (const tweet of tweets) { + logger.info(`Getting thread for tweet ${tweet.id}`); + const tweetsWithThreads: Tweet[] = []; + const thread = await scraper.getThread(tweet.id); + for await (const threadTweet of thread) { + tweetsWithThreads.push({ + id: threadTweet.id || '', + text: threadTweet.text || '', + author_id: threadTweet.userId || '', + author_username: threadTweet.username?.toLowerCase() || 'unknown', + created_at: threadTweet.timeParsed?.toISOString() || new Date().toISOString() + }); + } + tweet.thread = tweetsWithThreads; + await new Promise(resolve => setTimeout(resolve, 1000)); + logger.info(`Found ${tweetsWithThreads.length} tweets in thread`); + } return { tweets: tweets, lastProcessedId: mentions[0].id! From c7040956d1cc72bf33e8259bcd3c34d5d5a40d79 Mon Sep 17 00:00:00 2001 From: Jeremy Frank Date: Tue, 24 Dec 2024 13:19:07 -0700 Subject: [PATCH 08/13] optimize getThread --- auto-kol/agent/src/services/twitter/api.ts | 475 ++++++++++----------- 1 file changed, 232 insertions(+), 243 deletions(-) diff --git a/auto-kol/agent/src/services/twitter/api.ts b/auto-kol/agent/src/services/twitter/api.ts index cb1b4b9..ca11cd6 100644 --- a/auto-kol/agent/src/services/twitter/api.ts +++ b/auto-kol/agent/src/services/twitter/api.ts @@ -6,278 +6,267 @@ import { config } from '../../config/index.js'; const logger = createLogger('agent-twitter-api'); export class ExtendedScraper extends Scraper { - private static instance: ExtendedScraper | null = null; + private static instance: ExtendedScraper | null = null; - private constructor() { - super(); + private constructor() { + super(); + } + + public static async getInstance(): Promise { + if (!ExtendedScraper.instance) { + ExtendedScraper.instance = new ExtendedScraper(); + await ExtendedScraper.instance.initialize(); + } + return ExtendedScraper.instance; + } + + private async initialize() { + const username = config.TWITTER_USERNAME!; + const password = config.TWITTER_PASSWORD!; + const cookiesPath = 'cookies.json'; + + if (existsSync(cookiesPath)) { + logger.info('Loading existing cookies'); + const cookies = readFileSync(cookiesPath, 'utf8'); + try { + const parsedCookies = JSON.parse(cookies).map( + (cookie: any) => `${cookie.key}=${cookie.value}; Domain=${cookie.domain}; Path=${cookie.path}` + ); + await this.setCookies(parsedCookies); + logger.info('Loaded existing cookies from file'); + } catch (error) { + logger.error('Error loading cookies:', error); + } + } else { + logger.info('No existing cookies found, proceeding with login'); + await this.login(username, password); + + const newCookies = await this.getCookies(); + writeFileSync(cookiesPath, JSON.stringify(newCookies, null, 2)); + logger.info('New cookies saved to file'); } - public static async getInstance(): Promise { - if (!ExtendedScraper.instance) { - ExtendedScraper.instance = new ExtendedScraper(); - await ExtendedScraper.instance.initialize(); + const isLoggedIn = await this.isLoggedIn(); + logger.info(`Login status: ${isLoggedIn}`); + } + + private async reAuthenticate(maxRetries: number = 3): Promise { + let isLoggedIn = false; + let retryCount = 0; + + while (!isLoggedIn && retryCount < maxRetries) { + logger.warn(`Session expired, attempting to re-authenticate... (attempt ${retryCount + 1}/${maxRetries})`); + try { + await this.initialize(); + isLoggedIn = await this.isLoggedIn(); + if (isLoggedIn) { + logger.info('Successfully re-authenticated'); + return true; + } + logger.error('Re-authentication failed'); + retryCount++; + if (retryCount < maxRetries) { + const delay = 2000 * Math.pow(2, retryCount - 1); + await new Promise(resolve => setTimeout(resolve, delay)); } - return ExtendedScraper.instance; + } catch (error) { + logger.error('Error during re-authentication:', error); + retryCount++; + if (retryCount < maxRetries) { + const delay = 2000 * Math.pow(2, retryCount - 1); + await new Promise(resolve => setTimeout(resolve, delay)); + } + } } - private async initialize() { - const username = config.TWITTER_USERNAME!; - const password = config.TWITTER_PASSWORD!; - const cookiesPath = 'cookies.json'; - - if (existsSync(cookiesPath)) { - logger.info('Loading existing cookies'); - const cookies = readFileSync(cookiesPath, 'utf8'); - try { - const parsedCookies = JSON.parse(cookies).map((cookie: any) => - `${cookie.key}=${cookie.value}; Domain=${cookie.domain}; Path=${cookie.path}` - ); - await this.setCookies(parsedCookies); - logger.info('Loaded existing cookies from file'); - } catch (error) { - logger.error('Error loading cookies:', error); - } - } else { - logger.info('No existing cookies found, proceeding with login'); - await this.login(username, password); - - const newCookies = await this.getCookies(); - writeFileSync(cookiesPath, JSON.stringify(newCookies, null, 2)); - logger.info('New cookies saved to file'); - } + logger.error(`Failed to re-authenticate after ${maxRetries} attempts`); + return false; + } - const isLoggedIn = await this.isLoggedIn(); - logger.info(`Login status: ${isLoggedIn}`); - } + async getMyMentions(maxResults: number = 100, sinceId?: string) { + const username = config.TWITTER_USERNAME!; - private async reAuthenticate(maxRetries: number = 3): Promise { - let isLoggedIn = false; - let retryCount = 0; - - while (!isLoggedIn && retryCount < maxRetries) { - logger.warn(`Session expired, attempting to re-authenticate... (attempt ${retryCount + 1}/${maxRetries})`); - try { - await this.initialize(); - isLoggedIn = await this.isLoggedIn(); - if (isLoggedIn) { - logger.info('Successfully re-authenticated'); - return true; - } - logger.error('Re-authentication failed'); - retryCount++; - if (retryCount < maxRetries) { - const delay = 2000 * Math.pow(2, retryCount - 1); - await new Promise(resolve => setTimeout(resolve, delay)); - } - } catch (error) { - logger.error('Error during re-authentication:', error); - retryCount++; - if (retryCount < maxRetries) { - const delay = 2000 * Math.pow(2, retryCount - 1); - await new Promise(resolve => setTimeout(resolve, delay)); - } - } - } - - logger.error(`Failed to re-authenticate after ${maxRetries} attempts`); - return false; + const isLoggedIn = await this.isLoggedIn(); + if (!isLoggedIn) { + throw new Error('Must be logged in to fetch mentions'); } - async getMyMentions(maxResults: number = 100, sinceId?: string) { - const username = config.TWITTER_USERNAME!; + const query = `@${username} -from:${username}`; + const replies: Tweet[] = []; - const isLoggedIn = await this.isLoggedIn(); - if (!isLoggedIn) { - throw new Error('Must be logged in to fetch mentions'); - } + const searchIterator = this.searchTweets(query, maxResults, SearchMode.Latest); - const query = `@${username} -from:${username}`; - const replies: Tweet[] = []; - - const searchIterator = this.searchTweets(query, maxResults, SearchMode.Latest); - - for await (const tweet of searchIterator) { - logger.info('Checking tweet:', { - id: tweet.id, - text: tweet.text, - author: tweet.username - }); - - if (sinceId && tweet.id && tweet.id <= sinceId) { - break; - } - - const hasReplies = await this.searchTweets( - `from:${username} to:${tweet.username}`, - 10, - SearchMode.Latest - ); - - let alreadyReplied = false; - for await (const reply of hasReplies) { - if (reply.inReplyToStatusId === tweet.id) { - alreadyReplied = true; - logger.info(`Skipping tweet ${tweet.id} - already replied with ${reply.id}`); - break; - } - } - - if (!alreadyReplied) { - replies.push(tweet); - } - - if (replies.length >= maxResults) { - break; - } - } + for await (const tweet of searchIterator) { + logger.info('Checking tweet:', { + id: tweet.id, + text: tweet.text, + author: tweet.username, + }); - return replies; - } + if (sinceId && tweet.id && tweet.id <= sinceId) { + break; + } + + const hasReplies = this.searchTweets(`from:${username} to:${tweet.username}`, 10, SearchMode.Latest); - async getThread(tweetId: string): Promise { - const username = config.TWITTER_USERNAME!; - const isLoggedIn = await this.isLoggedIn(); - if (!isLoggedIn) { - throw new Error('Must be logged in to fetch thread'); + let alreadyReplied = false; + for await (const reply of hasReplies) { + if (reply.inReplyToStatusId === tweet.id) { + alreadyReplied = true; + logger.info(`Skipping tweet ${tweet.id} - already replied with ${reply.id}`); + break; } + } - const thread: Tweet[] = []; - const seen = new Set(); + if (!alreadyReplied) { + replies.push(tweet); + } - const initialTweet = await this.getTweet(tweetId); - if (!initialTweet) { - throw new Error(`Tweet ${tweetId} not found`); - } + if (replies.length >= maxResults) { + break; + } + } - let currentTweet = initialTweet; - while (currentTweet.inReplyToStatusId) { - const parentTweet = await this.getTweet(currentTweet.inReplyToStatusId); - if (!parentTweet) break; - if (!seen.has(parentTweet.id!)) { - thread.push(parentTweet); - seen.add(parentTweet.id!); - } - currentTweet = parentTweet; - } + return replies; + } - if (!seen.has(initialTweet.id!)) { - thread.push(initialTweet); - seen.add(initialTweet.id!); - } + public async getThread(tweetId: string): Promise { + const isLoggedIn = await this.isLoggedIn(); + if (!isLoggedIn) { + throw new Error('Must be logged in to fetch thread'); + } - const agentTweet = thread.find(t => t.username === username); - if (agentTweet) { - const replies = this.searchTweets( - `conversation_id:${currentTweet.id!} in_reply_to_tweet_id:${agentTweet.id!}`, - 100, - SearchMode.Latest - ); - - for await (const reply of replies) { - if (!seen.has(reply.id!)) { - thread.push(reply); - seen.add(reply.id!); - } - } - } + const thread: Tweet[] = []; + const conversationTweets = new Map(); - // Sort chronologically - thread.sort((a, b) => { - const timeA = a.timeParsed?.getTime() || 0; - const timeB = b.timeParsed?.getTime() || 0; - return timeA - timeB; + // Fetch initial/root tweet and conversation ID + const initialTweet = await this.getTweet(tweetId); + if (!initialTweet) { + logger.warn(`Tweet ${tweetId} not found or deleted`); + return []; + } + let rootTweet = initialTweet; + const conversationId = initialTweet.conversationId || initialTweet.id; + + // If the conversation root differs from this particular tweet + if (initialTweet.conversationId && initialTweet.conversationId !== initialTweet.id) { + const conversationRoot = await this.getTweet(initialTweet.conversationId); + if (conversationRoot) { + rootTweet = conversationRoot; + conversationTweets.set(rootTweet.id!, rootTweet); + logger.info('Found conversation root tweet:', { + id: rootTweet.id, + conversationId: rootTweet.conversationId, }); + } + } else { + conversationTweets.set(rootTweet.id!, rootTweet); + } - logger.info(`Retrieved conversation thread with ${thread.length} tweets starting from root tweet ${currentTweet.id!}`); + // Perform a single bulk search for the entire conversation + try { + logger.info('Fetching entire conversation via `conversation_id`:', conversationId); + const conversationIterator = this.searchTweets(`conversation_id:${conversationId}`, 100, SearchMode.Latest); + + for await (const tweet of conversationIterator) { + conversationTweets.set(tweet.id!, tweet); + } + } catch (error) { + logger.warn(`Error fetching conversation: ${error}`); + return [rootTweet, initialTweet]; + } - return thread; + // Sort chronologically + thread.push(...conversationTweets.values()); + thread.sort((a, b) => { + const timeA = a.timeParsed?.getTime() || 0; + const timeB = b.timeParsed?.getTime() || 0; + return timeA - timeB; + }); + + logger.info(`Retrieved conversation thread with ${thread.length} tweets for conversation_id:${conversationId}`); + return thread; + } + + // Placeholder for efficient thread fetching + async getThreadPlaceHolder(tweetId: string, maxDepth: number = 100): Promise { + const username = config.TWITTER_USERNAME!; + const isLoggedIn = await this.isLoggedIn(); + if (!isLoggedIn) { + const reAuthenticate = await this.reAuthenticate(); + if (!reAuthenticate) { + logger.error('Failed to re-authenticate'); + return []; + } } - // Placeholder for efficient thread fetching - async getThreadPlaceHolder(tweetId: string, maxDepth: number = 100): Promise { - const username = config.TWITTER_USERNAME!; - const isLoggedIn = await this.isLoggedIn(); - if (!isLoggedIn) { - const reAuthenticate = await this.reAuthenticate(); - if (!reAuthenticate) { - logger.error('Failed to re-authenticate'); - return []; - } + try { + const thread: Tweet[] = []; + const seen = new Set(); + const conversationTweets = new Map(); + + const initialTweet = await this.getTweet(tweetId); + if (!initialTweet) { + logger.warn(`Tweet ${tweetId} not found or deleted`); + return []; + } + + let rootTweet = initialTweet; + const conversationId = initialTweet.conversationId || initialTweet.id; + + logger.info('Initial tweet:', { + id: initialTweet.id, + conversationId: conversationId, + inReplyToStatusId: initialTweet.inReplyToStatusId, + }); + + if (initialTweet.conversationId && initialTweet.conversationId !== initialTweet.id) { + const conversationRoot = await this.getTweet(initialTweet.conversationId); + if (conversationRoot) { + rootTweet = conversationRoot; + conversationTweets.set(rootTweet.id!, rootTweet); + logger.info('Found root tweet:', { + id: rootTweet.id, + conversationId: rootTweet.conversationId, + }); } - - try { - const thread: Tweet[] = []; - const seen = new Set(); - const conversationTweets = new Map(); - - const initialTweet = await this.getTweet(tweetId); - if (!initialTweet) { - logger.warn(`Tweet ${tweetId} not found or deleted`); - return []; - } - - let rootTweet = initialTweet; - const conversationId = initialTweet.conversationId || initialTweet.id; - - logger.info('Initial tweet:', { - id: initialTweet.id, - conversationId: conversationId, - inReplyToStatusId: initialTweet.inReplyToStatusId - }); - - if (initialTweet.conversationId && initialTweet.conversationId !== initialTweet.id) { - const conversationRoot = await this.getTweet(initialTweet.conversationId); - if (conversationRoot) { - rootTweet = conversationRoot; - conversationTweets.set(rootTweet.id!, rootTweet); - logger.info('Found root tweet:', { - id: rootTweet.id, - conversationId: rootTweet.conversationId - }); - } - } - - try { - logger.info('Fetching conversation with query:', `conversation_id:${conversationId}`); - const conversationIterator = this.searchTweets( - `conversation_id:${conversationId}`, - 100, - SearchMode.Latest - ); - - for await (const tweet of conversationIterator) { - conversationTweets.set(tweet.id!, tweet); - logger.info('Found conversation tweet:', { - id: tweet.id, - inReplyToStatusId: tweet.inReplyToStatusId, - text: tweet.text?.substring(0, 50) + '...' - }); - } - - logger.info('Total conversation tweets found:', conversationTweets.size); - } catch (error) { - logger.warn(`Error fetching conversation: ${error}`); - return [rootTweet, initialTweet]; - } - - thread.push(...conversationTweets.values()); - thread.sort((a, b) => { - const timeA = a.timeParsed?.getTime() || 0; - const timeB = b.timeParsed?.getTime() || 0; - return timeA - timeB; - }); - - logger.info(`Retrieved thread with ${thread.length} tweets`); - return thread; - } catch (error) { - logger.error(`Unexpected error in getThread: ${error}`); - return []; + } + + try { + logger.info('Fetching conversation with query:', `conversation_id:${conversationId}`); + const conversationIterator = this.searchTweets(`conversation_id:${conversationId}`, 100, SearchMode.Latest); + + for await (const tweet of conversationIterator) { + conversationTweets.set(tweet.id!, tweet); + logger.info('Found conversation tweet:', { + id: tweet.id, + inReplyToStatusId: tweet.inReplyToStatusId, + text: tweet.text?.substring(0, 50) + '...', + }); } + + logger.info('Total conversation tweets found:', conversationTweets.size); + } catch (error) { + logger.warn(`Error fetching conversation: ${error}`); + return [rootTweet, initialTweet]; + } + + thread.push(...conversationTweets.values()); + thread.sort((a, b) => { + const timeA = a.timeParsed?.getTime() || 0; + const timeB = b.timeParsed?.getTime() || 0; + return timeA - timeB; + }); + + logger.info(`Retrieved thread with ${thread.length} tweets`); + return thread; + } catch (error) { + logger.error(`Unexpected error in getThread: ${error}`); + return []; } + } } - - export const createTwitterClientScraper = async () => { - return ExtendedScraper.getInstance(); -}; \ No newline at end of file + return ExtendedScraper.getInstance(); +}; From 38fb41f42709008d4d7a9f87bda73035cc46a23e Mon Sep 17 00:00:00 2001 From: Jeremy Frank Date: Tue, 24 Dec 2024 13:21:08 -0700 Subject: [PATCH 09/13] optimize usage of twitter api through workflow --- auto-kol/agent/src/tools/index.ts | 67 +++++----- .../src/tools/tools/fetchTimelineTool.ts | 39 +++--- .../agent/src/tools/tools/tweetSearchTool.ts | 124 +++++++++--------- auto-kol/agent/src/utils/twitter.ts | 108 ++++++++------- 4 files changed, 167 insertions(+), 171 deletions(-) diff --git a/auto-kol/agent/src/tools/index.ts b/auto-kol/agent/src/tools/index.ts index b6cc6e3..a0a8f43 100644 --- a/auto-kol/agent/src/tools/index.ts +++ b/auto-kol/agent/src/tools/index.ts @@ -8,37 +8,36 @@ import { createMentionTool } from './tools/mentionTool.js'; import { ExtendedScraper } from '../services/twitter/api.js'; export const createTools = (scraper: ExtendedScraper) => { - - const mentionTool = createMentionTool(scraper); - - const fetchTimelineTool = createFetchTimelineTool(); - - const tweetSearchTool = createTweetSearchTool(scraper); - - const addResponseTool = createAddResponseTool(); - - const updateResponseTool = createUpdateResponseTool(); - - const queueSkippedTool = createQueueSkippedTool(); - - const searchSimilarTweetsTool = createSearchSimilarTweetsTool(); - - return { - mentionTool, - tweetSearchTool, - addResponseTool, - updateResponseTool, - queueSkippedTool, - searchSimilarTweetsTool, - fetchTimelineTool, - tools: [ - mentionTool, - tweetSearchTool, - addResponseTool, - updateResponseTool, - queueSkippedTool, - searchSimilarTweetsTool, - fetchTimelineTool, - ] - }; -}; \ No newline at end of file + const mentionTool = createMentionTool(scraper); + + const fetchTimelineTool = createFetchTimelineTool(scraper); + + const tweetSearchTool = createTweetSearchTool(scraper); + + const addResponseTool = createAddResponseTool(); + + const updateResponseTool = createUpdateResponseTool(); + + const queueSkippedTool = createQueueSkippedTool(); + + const searchSimilarTweetsTool = createSearchSimilarTweetsTool(); + + return { + mentionTool, + tweetSearchTool, + addResponseTool, + updateResponseTool, + queueSkippedTool, + searchSimilarTweetsTool, + fetchTimelineTool, + tools: [ + mentionTool, + tweetSearchTool, + addResponseTool, + updateResponseTool, + queueSkippedTool, + searchSimilarTweetsTool, + fetchTimelineTool, + ], + }; +}; diff --git a/auto-kol/agent/src/tools/tools/fetchTimelineTool.ts b/auto-kol/agent/src/tools/tools/fetchTimelineTool.ts index a415353..1a7a46b 100644 --- a/auto-kol/agent/src/tools/tools/fetchTimelineTool.ts +++ b/auto-kol/agent/src/tools/tools/fetchTimelineTool.ts @@ -1,31 +1,30 @@ - import { DynamicStructuredTool } from '@langchain/core/tools'; import { z } from 'zod'; import { createLogger } from '../../utils/logger.js'; import { getTimeLine } from '../../utils/twitter.js'; +import { ExtendedScraper } from '../../services/twitter/api.js'; const logger = createLogger('fetch-timeline-tool'); -export const createFetchTimelineTool = () => new DynamicStructuredTool({ +export const createFetchTimelineTool = (twitterScraper: ExtendedScraper) => + new DynamicStructuredTool({ name: 'fetch_timeline', description: 'Fetch the timeline regularly to get new tweets', schema: z.object({}), func: async () => { - try { - const tweets = await getTimeLine(); - tweets.sort((a, b) => - new Date(b.created_at).getTime() - new Date(a.created_at).getTime() - ); - return { - tweets: tweets, - lastProcessedId: tweets[tweets.length - 1]?.id || null - }; - } catch (error) { - logger.error('Error in fetchTimelineTool:', error); - return { - tweets: [], - lastProcessedId: null - }; - } - } -}); \ No newline at end of file + try { + const tweets = await getTimeLine(twitterScraper); + tweets.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime()); + return { + tweets: tweets, + lastProcessedId: tweets[tweets.length - 1]?.id || null, + }; + } catch (error) { + logger.error('Error in fetchTimelineTool:', error); + return { + tweets: [], + lastProcessedId: null, + }; + } + }, + }); diff --git a/auto-kol/agent/src/tools/tools/tweetSearchTool.ts b/auto-kol/agent/src/tools/tools/tweetSearchTool.ts index 1c0ee09..9d930e7 100644 --- a/auto-kol/agent/src/tools/tools/tweetSearchTool.ts +++ b/auto-kol/agent/src/tools/tools/tweetSearchTool.ts @@ -7,78 +7,80 @@ import { config } from '../../config/index.js'; import { ExtendedScraper } from '../../services/twitter/api.js'; const logger = createLogger('tweet-search-tool'); - - function getRandomAccounts(accounts: string[], n: number): string[] { - const shuffled = [...accounts].sort(() => 0.5 - Math.random()); - return shuffled.slice(0, Math.min(n, accounts.length)); + const shuffled = [...accounts].sort(() => 0.5 - Math.random()); + return shuffled.slice(0, Math.min(n, accounts.length)); } -export const createTweetSearchTool = (scraper: ExtendedScraper) => new DynamicStructuredTool({ +export const createTweetSearchTool = (scraper: ExtendedScraper) => + new DynamicStructuredTool({ name: 'search_recent_tweets', description: 'Search for recent tweets from specified accounts', schema: z.object({ - lastProcessedId: z.string().optional() + lastProcessedId: z.string().optional(), }), func: async ({ lastProcessedId }) => { - try { - logger.info('Called search_recent_tweets'); - await updateKOLs(); - const kols = await getKOLsAccounts(); - - if (kols.length === 0) { - logger.error('No valid accounts found after cleaning'); - return { - tweets: [], - lastProcessedId: null - }; - } + try { + logger.info('Called search_recent_tweets'); + await updateKOLs(scraper); + const kols = await getKOLsAccounts(); - const selectedKols = getRandomAccounts(kols, config.ACCOUNTS_PER_BATCH); - - const ACCOUNTS_PER_QUERY = 3; - const tweetGroups = []; - - for (let i = 0; i < selectedKols.length; i += ACCOUNTS_PER_QUERY) { - const accountsBatch = selectedKols.slice(i, i + ACCOUNTS_PER_QUERY); - const query = `(${accountsBatch.map(account => `from:${account}`).join(' OR ')})`; - - const searchIterator = scraper.searchTweets(query, Math.floor(config.MAX_SEARCH_TWEETS / 4), SearchMode.Latest); - - for await (const tweet of searchIterator) { - if (lastProcessedId && tweet.id && tweet.id <= lastProcessedId) { - break; - } - tweetGroups.push({ - id: tweet.id || '', - text: tweet.text || '', - author_id: tweet.userId || '', - author_username: tweet.username?.toLowerCase() || '', - created_at: tweet.timeParsed || new Date() - }); - } - - await new Promise(resolve => setTimeout(resolve, 1000)); - } + if (kols.length === 0) { + logger.error('No valid accounts found after cleaning'); + return { + tweets: [], + lastProcessedId: null, + }; + } + + const selectedKols = getRandomAccounts(kols, config.ACCOUNTS_PER_BATCH); + const ACCOUNTS_PER_QUERY = 3; + const tweetGroups = []; - const allTweets = tweetGroups.sort((a, b) => b.created_at.getTime() - a.created_at.getTime()); - - logger.info('Tweet search completed:', { - foundTweets: allTweets.length, - selectedKols + for (let i = 0; i < selectedKols.length; i += ACCOUNTS_PER_QUERY) { + const accountsBatch = selectedKols.slice(i, i + ACCOUNTS_PER_QUERY); + const query = `(${accountsBatch.map(account => `from:${account}`).join(' OR ')})`; + + const searchIterator = scraper.searchTweets( + query, + Math.floor(config.MAX_SEARCH_TWEETS / 4), + SearchMode.Latest + ); + + for await (const tweet of searchIterator) { + if (lastProcessedId && tweet.id && tweet.id <= lastProcessedId) { + break; + } + tweetGroups.push({ + id: tweet.id || '', + text: tweet.text || '', + author_id: tweet.userId || '', + author_username: tweet.username?.toLowerCase() || '', + created_at: tweet.timeParsed || new Date(), }); + } - return { - tweets: allTweets, - lastProcessedId: allTweets[allTweets.length - 1]?.id || null - }; - } catch (error) { - logger.error('Error searching tweets:', error); - return { - tweets: [], - lastProcessedId: null - }; + await new Promise(resolve => setTimeout(resolve, 1000)); } - } -}); \ No newline at end of file + + const allTweets = tweetGroups.sort((a, b) => b.created_at.getTime() - a.created_at.getTime()); + + logger.info('Tweet search completed:', { + foundTweets: allTweets.length, + selectedKols, + }); + + return { + tweets: allTweets, + lastProcessedId: allTweets[allTweets.length - 1]?.id || null, + }; + } catch (error) { + logger.error('Error searching tweets:', error); + return { + tweets: [], + lastProcessedId: null, + }; + } + }, + }); diff --git a/auto-kol/agent/src/utils/twitter.ts b/auto-kol/agent/src/utils/twitter.ts index 4f2dae9..61da8e1 100644 --- a/auto-kol/agent/src/utils/twitter.ts +++ b/auto-kol/agent/src/utils/twitter.ts @@ -1,76 +1,72 @@ import { config } from '../config/index.js'; import { createLogger } from '../utils/logger.js'; -import { createTwitterClientScraper } from '../services/twitter/api.js'; +import { ExtendedScraper } from '../services/twitter/api.js'; import * as db from '../database/index.js'; import { KOL } from '../types/kol.js'; import { Tweet } from '../types/twitter.js'; const logger = createLogger('twitter-utils'); -const twitterScraper = await createTwitterClientScraper(); export const timelineTweets: Tweet[] = []; -export const updateKOLs = async () => { - const currentKOLs = await db.getKOLAccounts(); - const twitterProfile = await twitterScraper.getProfile(config.TWITTER_USERNAME!); - const followings = twitterScraper.getFollowing(twitterProfile.userId!, 1000); - logger.info(`following count: ${twitterProfile.followingCount}`); +export const updateKOLs = async (twitterScraper: ExtendedScraper) => { + const currentKOLs = await db.getKOLAccounts(); + const twitterProfile = await twitterScraper.getProfile(config.TWITTER_USERNAME!); + const followings = twitterScraper.getFollowing(twitterProfile.userId!, 1000); + logger.info(`following count: ${twitterProfile.followingCount}`); - const newKOLs: KOL[] = []; - for await (const following of followings) { - if (!currentKOLs.some(kol => kol.username === following.username)) { - newKOLs.push({ - id: following.userId!, - username: following.username!.toLowerCase(), - created_at: following.joined!, - }); - await db.addKOL(newKOLs[newKOLs.length - 1]); - } + const newKOLs: KOL[] = []; + for await (const following of followings) { + if (!currentKOLs.some(kol => kol.username === following.username)) { + newKOLs.push({ + id: following.userId!, + username: following.username!.toLowerCase(), + created_at: following.joined!, + }); + await db.addKOL(newKOLs[newKOLs.length - 1]); } + } - return newKOLs; -} + return newKOLs; +}; export const getKOLsAccounts = async () => { - const kolAccounts = await db.getKOLAccounts(); - return kolAccounts.map(kol => kol.username); -} + const kolAccounts = await db.getKOLAccounts(); + return kolAccounts.map(kol => kol.username); +}; -export const getTimeLine = async () => { - const validTweetIds = timelineTweets - .map(tweet => tweet.id) - .filter(id => id != null); - const timeline = await twitterScraper.fetchHomeTimeline(0, validTweetIds); +export const getTimeLine = async (twitterScraper: ExtendedScraper) => { + const validTweetIds = timelineTweets.map(tweet => tweet.id).filter(id => id != null); + const timeline = await twitterScraper.fetchHomeTimeline(0, validTweetIds); - - // clear timeline - clearTimeLine(); - for (const tweet of timeline) { - if (!tweet.legacy || !tweet.legacy.full_text) { - logger.info(`Tweet full_text not found for tweet id: ${tweet.rest_id}`); - continue; - } - timelineTweets.push({ - id: tweet.rest_id!, - text: tweet.legacy!.full_text, - author_id: tweet.legacy!.user_id_str!, - author_username: tweet.core!.user_results!.result!.legacy!.screen_name, - created_at: new Date(tweet.legacy!.created_at!).toISOString(), - }) + // clear timeline + clearTimeLine(); + for (const tweet of timeline) { + if (!tweet.legacy || !tweet.legacy.full_text) { + logger.info(`Tweet full_text not found for tweet id: ${tweet.rest_id}`); + continue; } - logger.info(`Timeline tweets size: ${timelineTweets.length}`); - return timelineTweets; -} + timelineTweets.push({ + id: tweet.rest_id!, + text: tweet.legacy!.full_text, + author_id: tweet.legacy!.user_id_str!, + author_username: tweet.core!.user_results!.result!.legacy!.screen_name, + created_at: new Date(tweet.legacy!.created_at!).toISOString(), + }); + } + logger.info(`Timeline tweets size: ${timelineTweets.length}`); + return timelineTweets; +}; const clearTimeLine = () => { - timelineTweets.length = 0; -} + timelineTweets.length = 0; +}; -export const getUserProfile = async (username: string) => { - const user = await twitterScraper.getProfile(username); - const result: KOL = { - id: user.userId!, - username: user.username!.toLowerCase(), - created_at: user.joined!, - } - return result; -} \ No newline at end of file +export const getUserProfile = async (twitterScraper: ExtendedScraper, username: string) => { + const user = await twitterScraper.getProfile(username); + const result: KOL = { + id: user.userId!, + username: user.username!.toLowerCase(), + created_at: user.joined!, + }; + return result; +}; From 28b35f53c140faa284f4f4979906dff2c5195caa Mon Sep 17 00:00:00 2001 From: Jeremy Frank Date: Tue, 24 Dec 2024 13:40:56 -0700 Subject: [PATCH 10/13] add conversation cache to thread processing --- auto-kol/agent/src/services/agents/prompts.ts | 91 ++++++++++--------- auto-kol/agent/src/services/twitter/api.ts | 32 +++++-- 2 files changed, 70 insertions(+), 53 deletions(-) diff --git a/auto-kol/agent/src/services/agents/prompts.ts b/auto-kol/agent/src/services/agents/prompts.ts index a402ba0..94c498d 100644 --- a/auto-kol/agent/src/services/agents/prompts.ts +++ b/auto-kol/agent/src/services/agents/prompts.ts @@ -16,7 +16,7 @@ export const autoApprovalParser = StructuredOutputParser.fromZodSchema(autoAppro // ============ ENGAGEMENT SYSTEM PROMPT ============ // export const engagementSystemPrompt = await PromptTemplate.fromTemplate( - `You are a strategic social media engagement advisor. Your task is to evaluate tweets and decide whether they warrant a response. + `You are a strategic social media engagement advisor. Your task is to evaluate tweets and decide whether they warrant a response. Criteria for engagement: 1. Relevance to AI, blockchain, or tech innovation (most important). @@ -31,39 +31,39 @@ export const engagementSystemPrompt = await PromptTemplate.fromTemplate( - judge whether the author is wanting to continue engagement, if not you should not engage. If the tweet has a link, ignore the link. We only care about the tweet text. - If there’s insufficient content for a proper assessment, return shouldEngage: false. + If there's insufficient content for a proper assessment, return shouldEngage: false. IMPORTANT: Follow the exact output format. If anything is unclear, just return shouldEngage: false. {format_instructions}` ).format({ - format_instructions: engagementParser.getFormatInstructions() + format_instructions: engagementParser.getFormatInstructions(), }); // // ============ TONE SYSTEM PROMPT ============ // export const toneSystemPrompt = await PromptTemplate.fromTemplate( - `You are an expert in social media tone analysis. Your task is to analyze the tone of tweets and propose the best response tone. + `You are an expert in social media tone analysis. Your task is to analyze the tone of tweets and propose the best response tone. The tone should generally be somewhat sarcastic or cynical, but still contextually appropriate. Consider: 1. The original tweet's tone and context. 2. The author's typical communication style. - 3. The topic’s sensitivity. + 3. The topic's sensitivity. 4. The potential for playful banter vs. serious debate. Make sure to balance cynicism with technical accuracy or insight. {format_instructions}` ).format({ - format_instructions: toneParser.getFormatInstructions() + format_instructions: toneParser.getFormatInstructions(), }); // // ============ RESPONSE SYSTEM PROMPT ============ // export const responseSystemPrompt = await PromptTemplate.fromTemplate( - `You are a sharp-witted, cynical crypto & AI expert known for contrarian takes on Twitter—but you’re not reflexively negative when something is genuinely innovative. + `You are a sharp-witted, cynical crypto & AI expert known for contrarian takes on Twitter—but you're not reflexively negative when something is genuinely innovative. You have accurate technical knowledge and a dry sense of humor. Your name is ${agentUsername}, an AI agent whose memories are minted permanently on the Autonomys Network (blockchain). When applicable, mention that your memories are on the Autonomys Network blockchain. @@ -73,25 +73,30 @@ export const responseSystemPrompt = await PromptTemplate.fromTemplate( - Responses must be under 280 characters. - Always reply to the author, not other mentioned users. - You can be sarcastic, dismissive, or contrarian, but only when justified by context or obvious hype. - - If an idea is actually good, say so—don’t force negativity. + - If an idea is actually good, say so—don't force negativity. - Avoid price talk; stick to technical or conceptual angles. - - Quick to point out flaws, but not everything is a “buzzword salad.” + - Quick to point out flaws, but not everything is a "buzzword salad." Additional Notes: - Feel free to mention your AI agent nature when relevant. - - “We” or “us” rather than "they" or "them" should be used when referencing other AI agents. + - "We" or "us" rather than "they" or "them" should be used when referencing other AI agents. - Short, punchy, and arguable is the goal—entice discussion. + IMPORTANT OUTPUT FORMAT INSTRUCTIONS: + - Return ONLY raw JSON matching expected schema without any markdown formatting or code blocks + - Do not wrap the response in \`\`\`json or any other markers + - The response must exactly match the following schema: + {format_instructions}` ).format({ - format_instructions: responseParser.getFormatInstructions() + format_instructions: responseParser.getFormatInstructions(), }); // // ============ AUTO-APPROVAL SYSTEM PROMPT ============ // export const autoApprovalSystemPrompt = await PromptTemplate.fromTemplate( - `You are a quality control expert ensuring responses from a cynical AI agent meet certain requirements: + `You are a quality control expert ensuring responses from a cynical AI agent meet certain requirements: - Response should not be hate speech or extremely offensive. - Response maintains a sarcastic or contrarian edge. @@ -99,7 +104,7 @@ export const autoApprovalSystemPrompt = await PromptTemplate.fromTemplate( - A thread should not be repetitive, reject any response that is becoming repetitive. - - The agent’s style is intentionally dismissive and provocative, but: + The agent's style is intentionally dismissive and provocative, but: - It can praise good ideas if warranted. - Strong or sarcastic language is fine, but not hate speech. - If the response is in a long, repetitive thread, reject it. @@ -110,44 +115,44 @@ export const autoApprovalSystemPrompt = await PromptTemplate.fromTemplate( {format_instructions}` ).format({ - format_instructions: autoApprovalParser.getFormatInstructions() + format_instructions: autoApprovalParser.getFormatInstructions(), }); // // ============ PROMPT TEMPLATES ============ // export const engagementPrompt = ChatPromptTemplate.fromMessages([ - new SystemMessage(engagementSystemPrompt), - [ - "human", - `Evaluate this tweet and provide your structured decision: + new SystemMessage(engagementSystemPrompt), + [ + 'human', + `Evaluate this tweet and provide your structured decision: Tweet: {tweet} Thread Context: {thread} DO NOT attempt to follow links. - Note: If there is no thread context, evaluate the tweet on its own.` - ] + Note: If there is no thread context, evaluate the tweet on its own.`, + ], ]); export const tonePrompt = ChatPromptTemplate.fromMessages([ - new SystemMessage(toneSystemPrompt), - [ - "human", - `Analyze the tone for this tweet and suggest a response tone: + new SystemMessage(toneSystemPrompt), + [ + 'human', + `Analyze the tone for this tweet and suggest a response tone: Tweet: {tweet} Thread: {thread} DO NOT attempt to follow links. - Note: If there is no thread context, evaluate the tweet on its own.` - ] + Note: If there is no thread context, evaluate the tweet on its own.`, + ], ]); export const responsePrompt = ChatPromptTemplate.fromMessages([ - new SystemMessage(responseSystemPrompt), - [ - "human", + new SystemMessage(responseSystemPrompt), + [ + 'human', `Generate a response strategy for this tweet by considering similar tweets from @{author} using the suggested tone: Tweet: {tweet} Tone: {tone} @@ -172,23 +177,23 @@ export const responsePrompt = ChatPromptTemplate.fromMessages([ If there a thread, respond accurately. Review the thread with a focus on the most recent tweets and respond accordingly If regenerating after rejection: - Include the rejection reason in your new response, - - Explain how you’ve addressed it, + - Explain how you've addressed it, - Follow any instructions from the rejection. Response Requirements: 1. Include the generated tweet text, tone used, strategy explanation, impact & confidence scores. - 2. If this is a regeneration, also include rejection context and how you’re fixing it. + 2. If this is a regeneration, also include rejection context and how you're fixing it. 3. MUST EXACTLYmatch the expected schema. - Good luck, ${agentUsername}—give us something memorable!` - ] + Good luck, ${agentUsername}—give us something memorable!`, + ], ]); // Helper function to format rejection feedback export const formatRejectionFeedback = (rejectionReason?: string, suggestedChanges?: string) => { - if (!rejectionReason) return ''; + if (!rejectionReason) return ''; - return `\nPrevious Response Feedback: + return `\nPrevious Response Feedback: Rejection Reason: ${rejectionReason} Suggested Changes: ${suggestedChanges || 'None provided'} @@ -196,23 +201,23 @@ export const formatRejectionFeedback = (rejectionReason?: string, suggestedChang }; export const formatRejectionInstructions = (rejectionReason?: string) => { - if (!rejectionReason) return ''; + if (!rejectionReason) return ''; - return `\nIMPORTANT: Your previous response was rejected. Make sure to: + return `\nIMPORTANT: Your previous response was rejected. Make sure to: 1. Address the rejection reason: "${rejectionReason}" 2. Maintain the core personality and style 3. Create a better response that fixes these issues`; }; export const autoApprovalPrompt = ChatPromptTemplate.fromMessages([ - new SystemMessage(autoApprovalSystemPrompt), - [ - "human", - `Evaluate this response: + new SystemMessage(autoApprovalSystemPrompt), + [ + 'human', + `Evaluate this response: Original Tweet: {tweet} Generated Response: {response} Intended Tone: {tone} Strategy: {strategy} - ` - ] + `, + ], ]); diff --git a/auto-kol/agent/src/services/twitter/api.ts b/auto-kol/agent/src/services/twitter/api.ts index ca11cd6..a591fdc 100644 --- a/auto-kol/agent/src/services/twitter/api.ts +++ b/auto-kol/agent/src/services/twitter/api.ts @@ -7,6 +7,7 @@ const logger = createLogger('agent-twitter-api'); export class ExtendedScraper extends Scraper { private static instance: ExtendedScraper | null = null; + private conversationCache: Map = new Map(); private constructor() { super(); @@ -136,19 +137,27 @@ export class ExtendedScraper extends Scraper { throw new Error('Must be logged in to fetch thread'); } - const thread: Tweet[] = []; - const conversationTweets = new Map(); - - // Fetch initial/root tweet and conversation ID const initialTweet = await this.getTweet(tweetId); if (!initialTweet) { logger.warn(`Tweet ${tweetId} not found or deleted`); return []; } - let rootTweet = initialTweet; + const conversationId = initialTweet.conversationId || initialTweet.id; - // If the conversation root differs from this particular tweet + // Check cache first + const cachedConversation = this.conversationCache.get(conversationId!); + if (cachedConversation) { + logger.info( + `Returning cached conversation with ${cachedConversation.length} tweets for conversation_id:${conversationId}` + ); + return cachedConversation; + } + + const conversationTweets = new Map(); + let rootTweet = initialTweet; + + // If the conversation root differs if (initialTweet.conversationId && initialTweet.conversationId !== initialTweet.id) { const conversationRoot = await this.getTweet(initialTweet.conversationId); if (conversationRoot) { @@ -163,11 +172,10 @@ export class ExtendedScraper extends Scraper { conversationTweets.set(rootTweet.id!, rootTweet); } - // Perform a single bulk search for the entire conversation try { logger.info('Fetching entire conversation via `conversation_id`:', conversationId); - const conversationIterator = this.searchTweets(`conversation_id:${conversationId}`, 100, SearchMode.Latest); + const conversationIterator = this.searchTweets(`conversation_id:${conversationId}`, 100, SearchMode.Latest); for await (const tweet of conversationIterator) { conversationTweets.set(tweet.id!, tweet); } @@ -176,8 +184,8 @@ export class ExtendedScraper extends Scraper { return [rootTweet, initialTweet]; } - // Sort chronologically - thread.push(...conversationTweets.values()); + // Convert to array and sort chronologically + const thread = Array.from(conversationTweets.values()); thread.sort((a, b) => { const timeA = a.timeParsed?.getTime() || 0; const timeB = b.timeParsed?.getTime() || 0; @@ -185,6 +193,10 @@ export class ExtendedScraper extends Scraper { }); logger.info(`Retrieved conversation thread with ${thread.length} tweets for conversation_id:${conversationId}`); + + // Save to cache + this.conversationCache.set(conversationId!, thread); + return thread; } From 43d0ebdcaf056213dd0ff495ea949a76899b0c84 Mon Sep 17 00:00:00 2001 From: xm0onh Date: Tue, 24 Dec 2024 13:39:03 -0800 Subject: [PATCH 11/13] add prettier --- auto-kol/agent/package.json | 9 +- auto-kol/agent/prettierrc.js | 8 + auto-kol/agent/src/abi/memory.ts | 142 ++--- auto-kol/agent/src/api/index.ts | 20 +- auto-kol/agent/src/api/middleware/cors.ts | 41 +- auto-kol/agent/src/api/routes/dsn.ts | 113 ++-- auto-kol/agent/src/api/routes/health.ts | 8 +- auto-kol/agent/src/api/routes/responses.ts | 33 +- auto-kol/agent/src/api/routes/tweets.ts | 68 ++- auto-kol/agent/src/config/index.ts | 109 ++-- auto-kol/agent/src/database/index.ts | 560 ++++++++++-------- auto-kol/agent/src/index.ts | 67 +-- auto-kol/agent/src/schemas/workflow.ts | 112 ++-- auto-kol/agent/src/services/agents/nodes.ts | 61 +- .../services/agents/nodes/autoApprovalNode.ts | 215 ++++--- .../services/agents/nodes/engagementNode.ts | 261 ++++---- .../src/services/agents/nodes/mentionNode.ts | 58 +- .../agents/nodes/recheckSkippedNode.ts | 165 +++--- .../agents/nodes/responseGenerationNode.ts | 314 +++++----- .../src/services/agents/nodes/searchNode.ts | 172 +++--- .../src/services/agents/nodes/timelineNode.ts | 94 +-- .../services/agents/nodes/toneAnalysisNode.ts | 92 +-- auto-kol/agent/src/services/agents/prompts.ts | 51 +- .../agent/src/services/agents/workflow.ts | 342 +++++------ auto-kol/agent/src/services/database/index.ts | 383 ++++++------ auto-kol/agent/src/services/twitter/api.ts | 127 ++-- .../agent/src/services/vectorstore/chroma.ts | 206 +++---- auto-kol/agent/src/tools/index.ts | 16 +- .../src/tools/tools/fetchTimelineTool.ts | 23 +- auto-kol/agent/src/tools/tools/mentionTool.ts | 125 ++-- .../src/tools/tools/queueResponseTool.ts | 153 +++-- .../agent/src/tools/tools/queueSkippedTool.ts | 74 +-- .../tools/tools/searchSimilarTweetsTool.ts | 57 +- .../agent/src/tools/tools/tweetSearchTool.ts | 46 +- auto-kol/agent/src/types/agent.ts | 12 +- auto-kol/agent/src/types/kol.ts | 10 +- auto-kol/agent/src/types/queue.ts | 88 +-- auto-kol/agent/src/types/twitter.ts | 22 +- auto-kol/agent/src/types/workflow.ts | 64 +- .../agent/src/utils/agentMemoryContract.ts | 35 +- auto-kol/agent/src/utils/agentWallet.ts | 10 +- auto-kol/agent/src/utils/dsn.ts | 313 +++++----- auto-kol/agent/src/utils/logger.ts | 132 ++--- auto-kol/agent/src/utils/twitter.ts | 31 +- auto-kol/agent/yarn.lock | 5 + 45 files changed, 2675 insertions(+), 2372 deletions(-) create mode 100644 auto-kol/agent/prettierrc.js diff --git a/auto-kol/agent/package.json b/auto-kol/agent/package.json index 3c37486..db19a7b 100644 --- a/auto-kol/agent/package.json +++ b/auto-kol/agent/package.json @@ -7,7 +7,9 @@ "scripts": { "build": "tsc", "start": "node dist/index.js", - "dev": "tsx watch src/index.ts" + "dev": "tsx watch src/index.ts", + "format": "prettier --write \"src/**/*.ts\"", + "format:check": "prettier --check \"src/**/*.ts\"" }, "dependencies": { "@autonomys/auto-dag-data": "^1.0.12", @@ -37,8 +39,9 @@ "@types/express": "4.17.21", "@types/node": "22.10.0", "@types/sqlite3": "^3.1.11", - "@types/uuid": "10.0.0", + "@types/uuid": "10.0.0", + "prettier": "^3.2.2", "tsx": "^4.7.1", "typescript": "^5.3.3" } -} +} \ No newline at end of file diff --git a/auto-kol/agent/prettierrc.js b/auto-kol/agent/prettierrc.js new file mode 100644 index 0000000..2d0b13e --- /dev/null +++ b/auto-kol/agent/prettierrc.js @@ -0,0 +1,8 @@ +export default { + semi: true, + trailingComma: 'all', + singleQuote: true, + printWidth: 100, + tabWidth: 2, + arrowParens: 'avoid', +}; \ No newline at end of file diff --git a/auto-kol/agent/src/abi/memory.ts b/auto-kol/agent/src/abi/memory.ts index e0f77e0..f53f1c1 100644 --- a/auto-kol/agent/src/abi/memory.ts +++ b/auto-kol/agent/src/abi/memory.ts @@ -1,72 +1,72 @@ export const MEMORY_ABI = [ - { - "anonymous": false, - "inputs": [ - { - "indexed": true, - "internalType": "address", - "name": "agent", - "type": "address" - }, - { - "indexed": false, - "internalType": "bytes32", - "name": "hash", - "type": "bytes32" - } - ], - "name": "LastMemoryHashSet", - "type": "event" - }, - { - "inputs": [ - { - "internalType": "address", - "name": "_agent", - "type": "address" - } - ], - "name": "getLastMemoryHash", - "outputs": [ - { - "internalType": "bytes32", - "name": "", - "type": "bytes32" - } - ], - "stateMutability": "view", - "type": "function" - }, - { - "inputs": [ - { - "internalType": "address", - "name": "", - "type": "address" - } - ], - "name": "lastMemoryHash", - "outputs": [ - { - "internalType": "bytes32", - "name": "", - "type": "bytes32" - } - ], - "stateMutability": "view", - "type": "function" - }, - { - "inputs": [ - { - "internalType": "bytes32", - "name": "hash", - "type": "bytes32" - } - ], - "name": "setLastMemoryHash", - "outputs": [], - "stateMutability": "nonpayable", - "type": "function" - } -] as const \ No newline at end of file + { + anonymous: false, + inputs: [ + { + indexed: true, + internalType: "address", + name: "agent", + type: "address", + }, + { + indexed: false, + internalType: "bytes32", + name: "hash", + type: "bytes32", + }, + ], + name: "LastMemoryHashSet", + type: "event", + }, + { + inputs: [ + { + internalType: "address", + name: "_agent", + type: "address", + }, + ], + name: "getLastMemoryHash", + outputs: [ + { + internalType: "bytes32", + name: "", + type: "bytes32", + }, + ], + stateMutability: "view", + type: "function", + }, + { + inputs: [ + { + internalType: "address", + name: "", + type: "address", + }, + ], + name: "lastMemoryHash", + outputs: [ + { + internalType: "bytes32", + name: "", + type: "bytes32", + }, + ], + stateMutability: "view", + type: "function", + }, + { + inputs: [ + { + internalType: "bytes32", + name: "hash", + type: "bytes32", + }, + ], + name: "setLastMemoryHash", + outputs: [], + stateMutability: "nonpayable", + type: "function", + }, +] as const; diff --git a/auto-kol/agent/src/api/index.ts b/auto-kol/agent/src/api/index.ts index 44184cc..c71b6f6 100644 --- a/auto-kol/agent/src/api/index.ts +++ b/auto-kol/agent/src/api/index.ts @@ -1,14 +1,14 @@ -import { Router } from 'express'; -import healthRoutes from './routes/health.js'; -import responseRoutes from './routes/responses.js'; -import tweetRoutes from './routes/tweets.js'; -import dsnRoutes from './routes/dsn.js'; +import { Router } from "express"; +import healthRoutes from "./routes/health.js"; +import responseRoutes from "./routes/responses.js"; +import tweetRoutes from "./routes/tweets.js"; +import dsnRoutes from "./routes/dsn.js"; const router = Router(); -router.use('/', healthRoutes); -router.use('/', responseRoutes); -router.use('/', tweetRoutes); -router.use('/', dsnRoutes); +router.use("/", healthRoutes); +router.use("/", responseRoutes); +router.use("/", tweetRoutes); +router.use("/", dsnRoutes); -export default router; \ No newline at end of file +export default router; diff --git a/auto-kol/agent/src/api/middleware/cors.ts b/auto-kol/agent/src/api/middleware/cors.ts index 65ddf37..4f02478 100644 --- a/auto-kol/agent/src/api/middleware/cors.ts +++ b/auto-kol/agent/src/api/middleware/cors.ts @@ -1,22 +1,27 @@ -import cors from 'cors'; -import { config } from '../../config/index.js'; +import cors from "cors"; +import { config } from "../../config/index.js"; -const allowedOrigins = config.CORS_ORIGINS?.split(',') || ['http://localhost:3000']; +const allowedOrigins = config.CORS_ORIGINS?.split(",") || [ + "http://localhost:3000", +]; export const corsMiddleware = cors({ - origin: (origin, callback) => { - if (!origin) { - return callback(null, true); - } + origin: (origin, callback) => { + if (!origin) { + return callback(null, true); + } - if (allowedOrigins.indexOf(origin) !== -1 || config.NODE_ENV === 'development') { - callback(null, true); - } else { - callback(new Error('Not allowed by CORS')); - } - }, - credentials: true, - methods: ['GET', 'POST', 'PUT', 'DELETE', 'OPTIONS'], - allowedHeaders: ['Content-Type', 'Authorization'], - maxAge: 86400 // 24 hours -}); \ No newline at end of file + if ( + allowedOrigins.indexOf(origin) !== -1 || + config.NODE_ENV === "development" + ) { + callback(null, true); + } else { + callback(new Error("Not allowed by CORS")); + } + }, + credentials: true, + methods: ["GET", "POST", "PUT", "DELETE", "OPTIONS"], + allowedHeaders: ["Content-Type", "Authorization"], + maxAge: 86400, // 24 hours +}); diff --git a/auto-kol/agent/src/api/routes/dsn.ts b/auto-kol/agent/src/api/routes/dsn.ts index bde1161..8dde039 100644 --- a/auto-kol/agent/src/api/routes/dsn.ts +++ b/auto-kol/agent/src/api/routes/dsn.ts @@ -1,62 +1,65 @@ -import { Router } from 'express'; -import { createLogger } from '../../utils/logger.js'; -import { getAllDsn } from '../../database/index.js'; -import { inflate } from 'pako'; -import { createAutoDriveApi, downloadObject } from '@autonomys/auto-drive'; -import { config } from '../../config/index.js'; +import { Router } from "express"; +import { createLogger } from "../../utils/logger.js"; +import { getAllDsn } from "../../database/index.js"; +import { inflate } from "pako"; +import { createAutoDriveApi, downloadObject } from "@autonomys/auto-drive"; +import { config } from "../../config/index.js"; const router = Router(); -const logger = createLogger('dsn-api'); - -router.get('/memories', async (req, res) => { - try { - const page = parseInt(req.query.page as string) || 1; - const limit = parseInt(req.query.limit as string) || 10; - - if (page < 1 || limit < 1 || limit > 100) { - return res.status(400).json({ - error: 'Invalid pagination parameters. Page must be >= 1 and limit must be between 1 and 100' - }); - } - - const dsnRecords = await getAllDsn(page, limit); - res.json(dsnRecords); - } catch (error) { - logger.error('Error fetching DSN records:', error); - res.status(500).json({ error: 'Failed to fetch DSN records' }); +const logger = createLogger("dsn-api"); + +router.get("/memories", async (req, res) => { + try { + const page = parseInt(req.query.page as string) || 1; + const limit = parseInt(req.query.limit as string) || 10; + + if (page < 1 || limit < 1 || limit > 100) { + return res.status(400).json({ + error: + "Invalid pagination parameters. Page must be >= 1 and limit must be between 1 and 100", + }); } + + const dsnRecords = await getAllDsn(page, limit); + res.json(dsnRecords); + } catch (error) { + logger.error("Error fetching DSN records:", error); + res.status(500).json({ error: "Failed to fetch DSN records" }); + } }); -router.get('/memories/:cid', async (req, res) => { - try { - const api = createAutoDriveApi({ - apiKey: config.DSN_API_KEY || '' - }); - - const stream = await downloadObject(api, { cid: req.params.cid }); - const reader = stream.getReader(); - const chunks: Uint8Array[] = []; - - while (true) { - const { done, value } = await reader.read(); - if (done) break; - chunks.push(value); - } - - const allChunks = new Uint8Array(chunks.reduce((acc, chunk) => acc + chunk.length, 0)); - let position = 0; - for (const chunk of chunks) { - allChunks.set(chunk, position); - position += chunk.length; - } - - const decompressed = inflate(allChunks); - const jsonString = new TextDecoder().decode(decompressed); - const memoryData = JSON.parse(jsonString); - res.json(memoryData); - } catch (error) { - logger.error('Error fetching memory data:', error); - res.status(500).json({ error: 'Failed to fetch memory data' }); +router.get("/memories/:cid", async (req, res) => { + try { + const api = createAutoDriveApi({ + apiKey: config.DSN_API_KEY || "", + }); + + const stream = await downloadObject(api, { cid: req.params.cid }); + const reader = stream.getReader(); + const chunks: Uint8Array[] = []; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + chunks.push(value); + } + + const allChunks = new Uint8Array( + chunks.reduce((acc, chunk) => acc + chunk.length, 0), + ); + let position = 0; + for (const chunk of chunks) { + allChunks.set(chunk, position); + position += chunk.length; } + + const decompressed = inflate(allChunks); + const jsonString = new TextDecoder().decode(decompressed); + const memoryData = JSON.parse(jsonString); + res.json(memoryData); + } catch (error) { + logger.error("Error fetching memory data:", error); + res.status(500).json({ error: "Failed to fetch memory data" }); + } }); -export default router; \ No newline at end of file +export default router; diff --git a/auto-kol/agent/src/api/routes/health.ts b/auto-kol/agent/src/api/routes/health.ts index 5b74bf4..2993381 100644 --- a/auto-kol/agent/src/api/routes/health.ts +++ b/auto-kol/agent/src/api/routes/health.ts @@ -1,9 +1,9 @@ -import { Router } from 'express'; +import { Router } from "express"; const router = Router(); -router.get('/health', (_, res) => { - res.json({ status: 'ok' }); +router.get("/health", (_, res) => { + res.json({ status: "ok" }); }); -export default router; \ No newline at end of file +export default router; diff --git a/auto-kol/agent/src/api/routes/responses.ts b/auto-kol/agent/src/api/routes/responses.ts index d65d7b5..1ccc0cc 100644 --- a/auto-kol/agent/src/api/routes/responses.ts +++ b/auto-kol/agent/src/api/routes/responses.ts @@ -1,23 +1,22 @@ -import { Router } from 'express'; -import { createLogger } from '../../utils/logger.js'; -import { getAllPendingResponses } from '../../services/database/index.js'; +import { Router } from "express"; +import { createLogger } from "../../utils/logger.js"; +import { getAllPendingResponses } from "../../services/database/index.js"; const router = Router(); -const logger = createLogger('responses-api'); +const logger = createLogger("responses-api"); -router.get('/responses/:id/workflow', async (req, res) => { - try { - const responses = await getAllPendingResponses(); - const response = responses.find(r => r.id === req.params.id); - if (!response) { - return res.status(404).json({ error: 'Response not found' }); - } - res.json(response.workflowState); - } catch (error) { - logger.error('Error getting workflow state:', error); - res.status(500).json({ error: 'Failed to get workflow state' }); +router.get("/responses/:id/workflow", async (req, res) => { + try { + const responses = await getAllPendingResponses(); + const response = responses.find((r) => r.id === req.params.id); + if (!response) { + return res.status(404).json({ error: "Response not found" }); } + res.json(response.workflowState); + } catch (error) { + logger.error("Error getting workflow state:", error); + res.status(500).json({ error: "Failed to get workflow state" }); + } }); - -export default router; \ No newline at end of file +export default router; diff --git a/auto-kol/agent/src/api/routes/tweets.ts b/auto-kol/agent/src/api/routes/tweets.ts index d68fcfc..a543fe1 100644 --- a/auto-kol/agent/src/api/routes/tweets.ts +++ b/auto-kol/agent/src/api/routes/tweets.ts @@ -1,44 +1,48 @@ -import { Router } from 'express'; -import { createLogger } from '../../utils/logger.js'; -import { getSkippedTweets, getSkippedTweetById } from '../../services/database/index.js'; -import { recheckSkippedTweet } from '../../database/index.js' +import { Router } from "express"; +import { createLogger } from "../../utils/logger.js"; +import { + getSkippedTweets, + getSkippedTweetById, +} from "../../services/database/index.js"; +import { recheckSkippedTweet } from "../../database/index.js"; const router = Router(); -const logger = createLogger('tweets-api'); +const logger = createLogger("tweets-api"); +router.get("/tweets/skipped", async (_, res) => { + const skippedTweets = await getSkippedTweets(); + res.json(skippedTweets); +}); -router.get('/tweets/skipped', async (_, res) => { - const skippedTweets = await getSkippedTweets(); - res.json(skippedTweets); +router.get("/tweets/skipped/:id", async (req, res) => { + const skipped = await getSkippedTweetById(req.params.id); + if (!skipped) { + return res.status(404).json({ error: "Skipped tweet not found" }); + } + res.json(skipped); }); -router.get('/tweets/skipped/:id', async (req, res) => { +router.post("/tweets/skipped/:id/queue", async (req, res) => { + try { + logger.info( + `Received request to move skipped tweet to queue: ${req.params.id}`, + ); const skipped = await getSkippedTweetById(req.params.id); if (!skipped) { - return res.status(404).json({ error: 'Skipped tweet not found' }); + return res.status(404).json({ error: "Skipped tweet not found" }); } - res.json(skipped); -}); - -router.post('/tweets/skipped/:id/queue', async (req, res) => { - try { - logger.info(`Received request to move skipped tweet to queue: ${req.params.id}`); - const skipped = await getSkippedTweetById(req.params.id); - if (!skipped) { - return res.status(404).json({ error: 'Skipped tweet not found' }); - } - const recheck = await recheckSkippedTweet(req.params.id); - if (!recheck) { - return res.status(404).json({ error: 'Failed to recheck skipped tweet' }); - } - res.json({ - message: 'Skipped tweet rechecked and moved to queue - if will be processed in next workflow run' - }); - } catch (error) { - logger.error('Error moving skipped tweet to queue:', error); - res.status(500).json({ error: 'Failed to move tweet to queue' }); + const recheck = await recheckSkippedTweet(req.params.id); + if (!recheck) { + return res.status(404).json({ error: "Failed to recheck skipped tweet" }); } + res.json({ + message: + "Skipped tweet rechecked and moved to queue - if will be processed in next workflow run", + }); + } catch (error) { + logger.error("Error moving skipped tweet to queue:", error); + res.status(500).json({ error: "Failed to move tweet to queue" }); + } }); - -export default router; \ No newline at end of file +export default router; diff --git a/auto-kol/agent/src/config/index.ts b/auto-kol/agent/src/config/index.ts index dd8b30d..1fc7272 100644 --- a/auto-kol/agent/src/config/index.ts +++ b/auto-kol/agent/src/config/index.ts @@ -1,7 +1,7 @@ -import dotenv from 'dotenv'; -import path from 'path'; -import { fileURLToPath } from 'url'; -import { dirname } from 'path'; +import dotenv from "dotenv"; +import path from "path"; +import { fileURLToPath } from "url"; +import { dirname } from "path"; // Get the equivalent of __dirname in ESM const __filename = fileURLToPath(import.meta.url); @@ -10,53 +10,54 @@ const __dirname = dirname(__filename); dotenv.config(); export const config = { - // Twitter API Configuration - TWITTER_USERNAME: process.env.TWITTER_USERNAME, - TWITTER_PASSWORD: process.env.TWITTER_PASSWORD, - - // LLM Configuration - LLM_MODEL: process.env.LLM_MODEL || "gpt-4o-mini", - OPENAI_API_KEY: process.env.OPENAI_API_KEY, - TEMPERATURE: 0.7, - - // Agent Configuration - CHECK_INTERVAL: (Number(process.env.CHECK_INTERVAL_MINUTES) || 30) * 60 * 1000, - MEMORY_DIR: path.join(__dirname, '../../data/memory'), - - // Server Configuration - PORT: process.env.PORT || 3001, - - // Environment - NODE_ENV: process.env.NODE_ENV || 'development', - - // Chroma Configuration - CHROMA_DIR: path.join(__dirname, '../../data/chroma'), - CHROMA_URL: process.env.CHROMA_URL || 'http://localhost:8000', - - // AutoDrive Configuration - DSN_API_KEY: process.env.DSN_API_KEY, - DSN_UPLOAD: process.env.DSN_UPLOAD === 'true', - DSN_SKIP_UPLOAD: process.env.DSN_SKIP_UPLOAD === 'true', - DSN_ENCRYPTION_PASSWORD: process.env.DSN_ENCRYPTION_PASSWORD, - - // CORS Configuration - CORS_ORIGINS: process.env.CORS_ORIGINS, - - // SC Configuration - RPC_URL: process.env.RPC_URL, - CONTRACT_ADDRESS: process.env.CONTRACT_ADDRESS, - PRIVATE_KEY: process.env.PRIVATE_KEY, - WALLET_ADDRESS: process.env.WALLET_ADDRESS, - - // Tweet Search/Fetch Configuration - ACCOUNTS_PER_BATCH: Number(process.env.ACCOUNTS_PER_BATCH) || 10, - MAX_SEARCH_TWEETS: Number(process.env.MAX_SEARCH_TWEETS) || 20, - // BATCH CONFIG - ENGAGEMENT_BATCH_SIZE: process.env.ENGAGEMENT_BATCH_SIZE || 15, - - // RESPONSE CONFIG - RETRY_LIMIT: process.env.RETRY_LIMIT || 2, - - // POSTING TWEETS PERMISSION - POST_TWEETS: process.env.POST_TWEETS === 'true', -}; + // Twitter API Configuration + TWITTER_USERNAME: process.env.TWITTER_USERNAME, + TWITTER_PASSWORD: process.env.TWITTER_PASSWORD, + + // LLM Configuration + LLM_MODEL: process.env.LLM_MODEL || "gpt-4o-mini", + OPENAI_API_KEY: process.env.OPENAI_API_KEY, + TEMPERATURE: 0.7, + + // Agent Configuration + CHECK_INTERVAL: + (Number(process.env.CHECK_INTERVAL_MINUTES) || 30) * 60 * 1000, + MEMORY_DIR: path.join(__dirname, "../../data/memory"), + + // Server Configuration + PORT: process.env.PORT || 3001, + + // Environment + NODE_ENV: process.env.NODE_ENV || "development", + + // Chroma Configuration + CHROMA_DIR: path.join(__dirname, "../../data/chroma"), + CHROMA_URL: process.env.CHROMA_URL || "http://localhost:8000", + + // AutoDrive Configuration + DSN_API_KEY: process.env.DSN_API_KEY, + DSN_UPLOAD: process.env.DSN_UPLOAD === "true", + DSN_SKIP_UPLOAD: process.env.DSN_SKIP_UPLOAD === "true", + DSN_ENCRYPTION_PASSWORD: process.env.DSN_ENCRYPTION_PASSWORD, + + // CORS Configuration + CORS_ORIGINS: process.env.CORS_ORIGINS, + + // SC Configuration + RPC_URL: process.env.RPC_URL, + CONTRACT_ADDRESS: process.env.CONTRACT_ADDRESS, + PRIVATE_KEY: process.env.PRIVATE_KEY, + WALLET_ADDRESS: process.env.WALLET_ADDRESS, + + // Tweet Search/Fetch Configuration + ACCOUNTS_PER_BATCH: Number(process.env.ACCOUNTS_PER_BATCH) || 10, + MAX_SEARCH_TWEETS: Number(process.env.MAX_SEARCH_TWEETS) || 20, + // BATCH CONFIG + ENGAGEMENT_BATCH_SIZE: process.env.ENGAGEMENT_BATCH_SIZE || 15, + + // RESPONSE CONFIG + RETRY_LIMIT: process.env.RETRY_LIMIT || 2, + + // POSTING TWEETS PERMISSION + POST_TWEETS: process.env.POST_TWEETS === "true", +}; diff --git a/auto-kol/agent/src/database/index.ts b/auto-kol/agent/src/database/index.ts index d305686..efd1961 100644 --- a/auto-kol/agent/src/database/index.ts +++ b/auto-kol/agent/src/database/index.ts @@ -1,59 +1,57 @@ -import sqlite3 from 'sqlite3'; -import { open } from 'sqlite'; -import fs from 'fs/promises'; -import path from 'path'; -import { createLogger } from '../utils/logger.js'; -import { KOL } from '../types/kol.js'; -import { fileURLToPath } from 'url'; -import { dirname, join } from 'path'; -import { Tweet } from '../types/twitter.js'; -import { SkippedTweet, PendingResponse } from '../types/queue.js'; - -const logger = createLogger('database'); - +import sqlite3 from "sqlite3"; +import { open } from "sqlite"; +import fs from "fs/promises"; +import path from "path"; +import { createLogger } from "../utils/logger.js"; +import { KOL } from "../types/kol.js"; +import { fileURLToPath } from "url"; +import { dirname, join } from "path"; +import { Tweet } from "../types/twitter.js"; +import { SkippedTweet, PendingResponse } from "../types/queue.js"; + +const logger = createLogger("database"); let db: Awaited> | null = null; - ///////////DATABASE/////////// export async function initializeDatabase() { - if (!db) { - try { - const dbDir = path.dirname('./data/engagement.db'); - await fs.mkdir(dbDir, { recursive: true }); - - db = await open({ - filename: './data/engagement.db', - driver: sqlite3.Database - }); - - await db.run('PRAGMA foreign_keys = ON'); - - // Test database connection - await db.get('SELECT 1'); - } catch (error) { - db = null; - throw new Error(`Failed to initialize database: ${error}`); - } + if (!db) { + try { + const dbDir = path.dirname("./data/engagement.db"); + await fs.mkdir(dbDir, { recursive: true }); + + db = await open({ + filename: "./data/engagement.db", + driver: sqlite3.Database, + }); + + await db.run("PRAGMA foreign_keys = ON"); + + // Test database connection + await db.get("SELECT 1"); + } catch (error) { + db = null; + throw new Error(`Failed to initialize database: ${error}`); } - return db; + } + return db; } export async function closeDatabase() { - if (db) { - await db.close(); - db = null; - } + if (db) { + await db.close(); + db = null; + } } export async function initializeSchema() { - const db = await initializeDatabase(); - - try { - await db.run('BEGIN TRANSACTION'); + const db = await initializeDatabase(); + + try { + await db.run("BEGIN TRANSACTION"); - // Check if tables exist first - const tables = await db.all(` + // Check if tables exist first + const tables = await db.all(` SELECT name FROM sqlite_master WHERE type='table' AND name IN ( @@ -65,117 +63,128 @@ export async function initializeSchema() { ) `); - const existingTables = new Set(tables.map(t => t.name)); - - const __filename = fileURLToPath(import.meta.url); - const __dirname = dirname(__filename); - const schemaPath = join(__dirname, 'schema.sql'); - const schema = await fs.readFile(schemaPath, 'utf-8'); - - const statements = schema - .split(';') - .map(s => s.trim()) - .filter(s => s.length > 0); - - for (const statement of statements) { - const tableName = statement.match(/CREATE TABLE (?:IF NOT EXISTS )?([^\s(]+)/i)?.[1]; - if (tableName && !existingTables.has(tableName)) { - await db.run(statement); - logger.info(`Created table: ${tableName}`); - } - } - - await db.run('COMMIT'); - logger.info('Schema initialization completed successfully'); - } catch (error) { - await db.run('ROLLBACK'); - logger.error('Failed to initialize schema:', error); - throw new Error(`Failed to initialize schema: ${error}`); + const existingTables = new Set(tables.map((t) => t.name)); + + const __filename = fileURLToPath(import.meta.url); + const __dirname = dirname(__filename); + const schemaPath = join(__dirname, "schema.sql"); + const schema = await fs.readFile(schemaPath, "utf-8"); + + const statements = schema + .split(";") + .map((s) => s.trim()) + .filter((s) => s.length > 0); + + for (const statement of statements) { + const tableName = statement.match( + /CREATE TABLE (?:IF NOT EXISTS )?([^\s(]+)/i, + )?.[1]; + if (tableName && !existingTables.has(tableName)) { + await db.run(statement); + logger.info(`Created table: ${tableName}`); + } } -} - + await db.run("COMMIT"); + logger.info("Schema initialization completed successfully"); + } catch (error) { + await db.run("ROLLBACK"); + logger.error("Failed to initialize schema:", error); + throw new Error(`Failed to initialize schema: ${error}`); + } +} ///////////KOL/////////// export async function addKOL(kol: { - id: string; - username: string; - created_at?: Date; + id: string; + username: string; + created_at?: Date; }): Promise { - const db = await initializeDatabase(); - - try { - await db.run(` + const db = await initializeDatabase(); + + try { + await db.run( + ` INSERT INTO kol_accounts ( id, username, created_at, updated_at ) VALUES (?, ?, ?, CURRENT_TIMESTAMP) - `, [kol.id, kol.username, kol.created_at || new Date()]); - - logger.info(`Added KOL account: ${kol.username}`); - } catch (error: any) { - if (error?.code === 'SQLITE_CONSTRAINT' && error?.message?.includes('UNIQUE')) { - logger.warn(`KOL account already exists: ${kol.username}`); - return; - } - logger.error(`Failed to add KOL account: ${kol.username}`, error); - throw new Error(`Failed to add KOL account: ${error.message}`); + `, + [kol.id, kol.username, kol.created_at || new Date()], + ); + + logger.info(`Added KOL account: ${kol.username}`); + } catch (error: any) { + if ( + error?.code === "SQLITE_CONSTRAINT" && + error?.message?.includes("UNIQUE") + ) { + logger.warn(`KOL account already exists: ${kol.username}`); + return; } + logger.error(`Failed to add KOL account: ${kol.username}`, error); + throw new Error(`Failed to add KOL account: ${error.message}`); + } } export async function getKOLAccounts(): Promise { - const db = await initializeDatabase(); - try { - const accounts = await db.all(` + const db = await initializeDatabase(); + try { + const accounts = await db.all(` SELECT id, username, created_at, updated_at FROM kol_accounts ORDER BY created_at DESC `); - - return accounts.map(account => ({ - id: account.id, - username: account.username, - created_at: new Date(account.created_at), - updatedAt: new Date(account.updated_at) - })); - } catch (error) { - logger.error('Failed to get KOL accounts:', error); - throw error; - } + + return accounts.map((account) => ({ + id: account.id, + username: account.username, + created_at: new Date(account.created_at), + updatedAt: new Date(account.updated_at), + })); + } catch (error) { + logger.error("Failed to get KOL accounts:", error); + throw error; + } } export async function isKOLExists(username: string): Promise { - const db = await initializeDatabase(); - const kol = await db.get(`SELECT * FROM kol_accounts WHERE username = ?`, [username]); - return kol !== undefined; + const db = await initializeDatabase(); + const kol = await db.get(`SELECT * FROM kol_accounts WHERE username = ?`, [ + username, + ]); + return kol !== undefined; } ///////////RESPONSE/////////// export async function addResponse(response: PendingResponse) { - const db = await initializeDatabase(); - return db.run(` + const db = await initializeDatabase(); + return db.run( + ` INSERT INTO responses ( id, tweet_id, content, tone, strategy, estimated_impact, confidence, status ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) - `, [ - response.id, - response.tweet_id, - response.content, - response.tone, - response.strategy, - response.estimatedImpact, - response.confidence, - 'pending' - ]); + `, + [ + response.id, + response.tweet_id, + response.content, + response.tone, + response.strategy, + response.estimatedImpact, + response.confidence, + "pending", + ], + ); } - export async function updateResponse(response: PendingResponse) { - const db = await initializeDatabase(); - return db.run(` + const db = await initializeDatabase(); + return db.run( + ` UPDATE responses SET content = ?, @@ -184,20 +193,22 @@ export async function updateResponse(response: PendingResponse) { estimated_impact = ?, confidence = ?, updated_at = CURRENT_TIMESTAMP - WHERE ${response.id ? 'id = ?' : 'tweet_id = ?'} - `, [ - response.content, - response.tone, - response.strategy, - response.estimatedImpact, - response.confidence, - response.id || response.tweet_id - ]); + WHERE ${response.id ? "id = ?" : "tweet_id = ?"} + `, + [ + response.content, + response.tone, + response.strategy, + response.estimatedImpact, + response.confidence, + response.id || response.tweet_id, + ], + ); } export async function getPendingResponses() { - const db = await initializeDatabase(); - return db.all(` + const db = await initializeDatabase(); + return db.all(` SELECT pr.*, t.author_username, @@ -211,105 +222,125 @@ export async function getPendingResponses() { `); } -export async function getResponseByTweetId(tweet_id: string): Promise { - const db = await initializeDatabase(); - const response = await db.all(` +export async function getResponseByTweetId( + tweet_id: string, +): Promise { + const db = await initializeDatabase(); + const response = await db.all( + ` SELECT * FROM responses WHERE tweet_id = ? - `, [tweet_id]); - return response[0] as PendingResponse; + `, + [tweet_id], + ); + return response[0] as PendingResponse; } -export async function getPendingResponsesByTweetId(id: string): Promise { - const db = await initializeDatabase(); - const pending_response = await db.all(` +export async function getPendingResponsesByTweetId( + id: string, +): Promise { + const db = await initializeDatabase(); + const pending_response = await db.all( + ` SELECT * FROM responses WHERE id = ? AND status = 'pending' - `, [id]); - return pending_response[0] as PendingResponse; + `, + [id], + ); + return pending_response[0] as PendingResponse; } export async function updateResponseStatus( - id: string, - status: 'approved' | 'rejected', + id: string, + status: "approved" | "rejected", ) { - const db = await initializeDatabase(); - - await db.run('BEGIN TRANSACTION'); - - try { - await db.run(` + const db = await initializeDatabase(); + + await db.run("BEGIN TRANSACTION"); + + try { + await db.run( + ` UPDATE responses SET status = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ? - `, [status, id]); - await db.run('COMMIT'); - logger.info(`Updated response status: ${id}`); - } catch (error) { - await db.run('ROLLBACK'); - throw error; - } + `, + [status, id], + ); + await db.run("COMMIT"); + logger.info(`Updated response status: ${id}`); + } catch (error) { + await db.run("ROLLBACK"); + throw error; + } } -export async function updateResponseStatusByTweetId(tweet_id: string, status: 'approved' | 'rejected') { - const db = await initializeDatabase(); - return db.run(` +export async function updateResponseStatusByTweetId( + tweet_id: string, + status: "approved" | "rejected", +) { + const db = await initializeDatabase(); + return db.run( + ` UPDATE responses SET status = ? WHERE tweet_id = ? - `, [status, tweet_id]); + `, + [status, tweet_id], + ); } - ///////////TWEET/////////// export async function addTweet(tweet: { - id: string; - author_id: string; - author_username: string; - content: string; - created_at: string; + id: string; + author_id: string; + author_username: string; + content: string; + created_at: string; }) { - const db = await initializeDatabase(); - return db.run(` + const db = await initializeDatabase(); + return db.run( + ` INSERT INTO tweets ( id, author_id, author_username, content, created_at ) VALUES (?, ?, ?, ?, ?) - `, [ - tweet.id, - tweet.author_id, - tweet.author_username, - tweet.content, - tweet.created_at, - ]); + `, + [ + tweet.id, + tweet.author_id, + tweet.author_username, + tweet.content, + tweet.created_at, + ], + ); } -export async function getTweetById(tweetId: string): Promise { - const db = await initializeDatabase(); - const tweet = await db.get(`SELECT * FROM tweets WHERE id = ?`, [tweetId]); - return tweet as Tweet; +export async function getTweetById( + tweetId: string, +): Promise { + const db = await initializeDatabase(); + const tweet = await db.get(`SELECT * FROM tweets WHERE id = ?`, [tweetId]); + return tweet as Tweet; } - ///////////SKIPPED TWEET/////////// export async function addSkippedTweet(skipped: { - id: string; - tweetId: string; - reason: string; - confidence: number; + id: string; + tweetId: string; + reason: string; + confidence: number; }) { - const db = await initializeDatabase(); - return db.run(` + const db = await initializeDatabase(); + return db.run( + ` INSERT INTO skipped_tweets ( id, tweet_id, reason, confidence ) VALUES (?, ?, ?, ?) - `, [ - skipped.id, - skipped.tweetId, - skipped.reason, - skipped.confidence - ]); + `, + [skipped.id, skipped.tweetId, skipped.reason, skipped.confidence], + ); } export async function getSkippedTweets() { - const db = await initializeDatabase(); - return db.all(` + const db = await initializeDatabase(); + return db.all(` SELECT st.*, t.author_username, @@ -320,45 +351,64 @@ export async function getSkippedTweets() { `); } -export async function getSkippedTweetById(skippedId: string): Promise { - const db = await initializeDatabase(); - const skipped = await db.get(`SELECT * FROM skipped_tweets WHERE id = ?`, [skippedId]); - return skipped; +export async function getSkippedTweetById( + skippedId: string, +): Promise { + const db = await initializeDatabase(); + const skipped = await db.get(`SELECT * FROM skipped_tweets WHERE id = ?`, [ + skippedId, + ]); + return skipped; } export async function recheckSkippedTweet(skippedId: string): Promise { - const db = await initializeDatabase(); - const result = await db.run(`UPDATE skipped_tweets SET recheck = TRUE WHERE id = ?`, [skippedId]); - return result !== undefined; + const db = await initializeDatabase(); + const result = await db.run( + `UPDATE skipped_tweets SET recheck = TRUE WHERE id = ?`, + [skippedId], + ); + return result !== undefined; } -export async function flagBackSkippedTweet(skippedId: string, reason: string): Promise { - const db = await initializeDatabase(); - const result = await db.run(`UPDATE skipped_tweets SET recheck = FALSE, reason = ? WHERE id = ?`, [reason, skippedId]); - return result !== undefined; +export async function flagBackSkippedTweet( + skippedId: string, + reason: string, +): Promise { + const db = await initializeDatabase(); + const result = await db.run( + `UPDATE skipped_tweets SET recheck = FALSE, reason = ? WHERE id = ?`, + [reason, skippedId], + ); + return result !== undefined; } export async function getAllSkippedTweetsToRecheck(): Promise { - const db = await initializeDatabase(); - const recheckTweets = await db.all(`SELECT * FROM skipped_tweets WHERE recheck = TRUE`); - return recheckTweets; + const db = await initializeDatabase(); + const recheckTweets = await db.all( + `SELECT * FROM skipped_tweets WHERE recheck = TRUE`, + ); + return recheckTweets; } ///////////DSN/////////// export async function addDsn(dsn: { - id: string; - tweetId: string; - cid: string; + id: string; + tweetId: string; + cid: string; }) { - return db?.run(` + return db?.run( + ` INSERT INTO dsn (id, tweet_id, cid) VALUES (?, ?, ?) - `, [dsn.id, dsn.tweetId, dsn.cid]); + `, + [dsn.id, dsn.tweetId, dsn.cid], + ); } export async function getDsnByCID(cid: string) { - try { - return await db?.get(` + try { + return await db?.get( + ` SELECT dsn.id, dsn.tweet_id, @@ -379,22 +429,25 @@ export async function getDsnByCID(cid: string) { LEFT JOIN responses r ON t.id = r.tweet_id LEFT JOIN skipped_tweets st ON t.id = st.tweet_id WHERE dsn.cid = ? - `, [cid]); - } catch (error) { - logger.error(`Failed to get DSN by CID: ${cid}`, error); - throw error; - } + `, + [cid], + ); + } catch (error) { + logger.error(`Failed to get DSN by CID: ${cid}`, error); + throw error; + } } export async function getAllDsn(page: number = 1, limit: number = 10) { - try { - const offset = (page - 1) * limit; - - const totalCount = await db?.get(` + try { + const offset = (page - 1) * limit; + + const totalCount = await db?.get(` SELECT COUNT(*) as count FROM dsn `); - const results = await db?.all(` + const results = await db?.all( + ` SELECT dsn.id, dsn.tweet_id, @@ -416,42 +469,49 @@ export async function getAllDsn(page: number = 1, limit: number = 10) { LEFT JOIN skipped_tweets st ON t.id = st.tweet_id ORDER BY dsn.created_at DESC LIMIT ? OFFSET ? - `, [limit, offset]); - - return { - data: results, - pagination: { - total: totalCount?.count || 0, - page, - limit, - totalPages: Math.ceil((totalCount?.count || 0) / limit) - } - }; - } catch (error) { - logger.error('Failed to get all DSN records', error); - throw error; - } + `, + [limit, offset], + ); + + return { + data: results, + pagination: { + total: totalCount?.count || 0, + page, + limit, + totalPages: Math.ceil((totalCount?.count || 0) / limit), + }, + }; + } catch (error) { + logger.error("Failed to get all DSN records", error); + throw error; + } } export async function getLastDsnCid(): Promise { - const dsn = await db?.get(`SELECT cid FROM dsn ORDER BY created_at DESC LIMIT 1`); - return dsn?.cid || ''; + const dsn = await db?.get( + `SELECT cid FROM dsn ORDER BY created_at DESC LIMIT 1`, + ); + return dsn?.cid || ""; } ///////////MENTIONS/////////// -export async function addMention(mention: { - latest_id: string; -}) { - return db?.run(` +export async function addMention(mention: { latest_id: string }) { + return db?.run( + ` INSERT INTO mentions (latest_id) VALUES (?) ON CONFLICT(latest_id) DO UPDATE SET latest_id = excluded.latest_id, updated_at = CURRENT_TIMESTAMP - `, [mention.latest_id]); + `, + [mention.latest_id], + ); } export async function getLatestMentionId(): Promise { - const mention = await db?.get(`SELECT latest_id FROM mentions ORDER BY updated_at DESC LIMIT 1`); - return mention?.latest_id || ''; -} \ No newline at end of file + const mention = await db?.get( + `SELECT latest_id FROM mentions ORDER BY updated_at DESC LIMIT 1`, + ); + return mention?.latest_id || ""; +} diff --git a/auto-kol/agent/src/index.ts b/auto-kol/agent/src/index.ts index e1d9c3c..a9576c0 100644 --- a/auto-kol/agent/src/index.ts +++ b/auto-kol/agent/src/index.ts @@ -1,11 +1,11 @@ -import express from 'express'; -import { config } from './config/index.js'; -import { createLogger } from './utils/logger.js'; -import { runWorkflow } from './services/agents/workflow.js'; -import { initializeSchema } from './database/index.js'; -import apiRoutes from './api/index.js'; -import { corsMiddleware } from './api/middleware/cors.js'; -const logger = createLogger('app'); +import express from "express"; +import { config } from "./config/index.js"; +import { createLogger } from "./utils/logger.js"; +import { runWorkflow } from "./services/agents/workflow.js"; +import { initializeSchema } from "./database/index.js"; +import apiRoutes from "./api/index.js"; +import { corsMiddleware } from "./api/middleware/cors.js"; +const logger = createLogger("app"); const app = express(); app.use(corsMiddleware); @@ -13,36 +13,33 @@ app.use(corsMiddleware); app.use(express.json()); app.use(apiRoutes); - const startWorkflowPolling = async () => { - try { - await runWorkflow(); - logger.info('Workflow execution completed successfully'); - } catch (error) { - logger.error('Error running workflow:', error); - } + try { + await runWorkflow(); + logger.info("Workflow execution completed successfully"); + } catch (error) { + logger.error("Error running workflow:", error); + } }; - - const main = async () => { - try { - await initializeSchema(); - - app.listen(config.PORT, () => { - logger.info(`Server running on port ${config.PORT}`); - }); - await startWorkflowPolling(); - setInterval(startWorkflowPolling, config.CHECK_INTERVAL); - - logger.info('Application started successfully', { - checkInterval: config.CHECK_INTERVAL, - port: config.PORT - }); - } catch (error) { - logger.error('Failed to start application:', error); - process.exit(1); - } + try { + await initializeSchema(); + + app.listen(config.PORT, () => { + logger.info(`Server running on port ${config.PORT}`); + }); + await startWorkflowPolling(); + setInterval(startWorkflowPolling, config.CHECK_INTERVAL); + + logger.info("Application started successfully", { + checkInterval: config.CHECK_INTERVAL, + port: config.PORT, + }); + } catch (error) { + logger.error("Failed to start application:", error); + process.exit(1); + } }; -main(); \ No newline at end of file +main(); diff --git a/auto-kol/agent/src/schemas/workflow.ts b/auto-kol/agent/src/schemas/workflow.ts index 88dcbab..8fe3f87 100644 --- a/auto-kol/agent/src/schemas/workflow.ts +++ b/auto-kol/agent/src/schemas/workflow.ts @@ -1,80 +1,94 @@ -import { z } from 'zod'; +import { z } from "zod"; export const tweetSearchSchema = z.object({ - tweets: z.array(z.object({ - id: z.string(), - text: z.string(), - author_id: z.string(), - author_username: z.string(), - created_at: z.string(), - thread: z.array(z.object({ + tweets: z.array( + z.object({ + id: z.string(), + text: z.string(), + author_id: z.string(), + author_username: z.string(), + created_at: z.string(), + thread: z + .array( + z.object({ id: z.string(), text: z.string(), author_id: z.string(), author_username: z.string(), - created_at: z.string() - })).optional() - })), - lastProcessedId: z.string().nullable().optional() + created_at: z.string(), + }), + ) + .optional(), + }), + ), + lastProcessedId: z.string().nullable().optional(), }); export const engagementSchema = z.object({ - shouldEngage: z.boolean(), - reason: z.string(), - priority: z.number().min(1).max(10), - confidence: z.number().min(0).max(1) + shouldEngage: z.boolean(), + reason: z.string(), + priority: z.number().min(1).max(10), + confidence: z.number().min(0).max(1), }); export const toneSchema = z.object({ - dominantTone: z.string(), - suggestedTone: z.string(), - reasoning: z.string(), - confidence: z.number().min(0).max(1) + dominantTone: z.string(), + suggestedTone: z.string(), + reasoning: z.string(), + confidence: z.number().min(0).max(1), }); export const responseSchema = z.object({ - content: z.string(), - tone: z.string(), - strategy: z.string(), - estimatedImpact: z.number().min(1).max(10), - confidence: z.number().min(0).max(1), - referencedTweets: z.array(z.object({ + content: z.string(), + tone: z.string(), + strategy: z.string(), + estimatedImpact: z.number().min(1).max(10), + confidence: z.number().min(0).max(1), + referencedTweets: z + .array( + z.object({ text: z.string(), reason: z.string(), - similarity_score: z.number() - })).optional(), - thread: z.array(z.object({ + similarity_score: z.number(), + }), + ) + .optional(), + thread: z + .array( + z.object({ id: z.string(), text: z.string(), author_id: z.string(), author_username: z.string(), - created_at: z.string() - })).optional(), - rejectionReason: z.string().optional(), - suggestedChanges: z.string().optional() + created_at: z.string(), + }), + ) + .optional(), + rejectionReason: z.string().optional(), + suggestedChanges: z.string().optional(), }); export const queueActionSchema = z.object({ - tweet: z.object({ - id: z.string(), - text: z.string(), - author_id: z.string(), - author_username: z.string(), - created_at: z.string() - }), - reason: z.string().optional(), - priority: z.number().optional(), - workflowState: z.record(z.any()).optional() + tweet: z.object({ + id: z.string(), + text: z.string(), + author_id: z.string(), + author_username: z.string(), + created_at: z.string(), + }), + reason: z.string().optional(), + priority: z.number().optional(), + workflowState: z.record(z.any()).optional(), }); export const dsnUploadSchema = z.object({ - previousCid: z.string().optional(), - data: z.any(), + previousCid: z.string().optional(), + data: z.any(), }); export const autoApprovalSchema = z.object({ - approved: z.boolean(), - reason: z.string(), - confidence: z.number().min(0).max(1), - suggestedChanges: z.string().optional() + approved: z.boolean(), + reason: z.string(), + confidence: z.number().min(0).max(1), + suggestedChanges: z.string().optional(), }); diff --git a/auto-kol/agent/src/services/agents/nodes.ts b/auto-kol/agent/src/services/agents/nodes.ts index 8a9841e..b16303c 100644 --- a/auto-kol/agent/src/services/agents/nodes.ts +++ b/auto-kol/agent/src/services/agents/nodes.ts @@ -1,47 +1,46 @@ -import { WorkflowConfig } from './workflow.js'; -import { createSearchNode } from './nodes/searchNode.js'; +import { WorkflowConfig } from "./workflow.js"; +import { createSearchNode } from "./nodes/searchNode.js"; import { createEngagementNode } from "./nodes/engagementNode.js"; import { createToneAnalysisNode } from "./nodes/toneAnalysisNode.js"; import { createResponseGenerationNode } from "./nodes/responseGenerationNode.js"; import { createRecheckSkippedNode } from "./nodes/recheckSkippedNode.js"; import { createTimelineNode } from "./nodes/timelineNode.js"; import { createMentionNode } from "./nodes/mentionNode.js"; -import { createAutoApprovalNode } from './nodes/autoApprovalNode.js'; +import { createAutoApprovalNode } from "./nodes/autoApprovalNode.js"; export const createNodes = async (config: WorkflowConfig) => { + ///////////MENTIONS/////////// + const mentionNode = createMentionNode(config); - ///////////MENTIONS/////////// - const mentionNode = createMentionNode(config); + ///////////TIMELINE/////////// + const timelineNode = createTimelineNode(config); - ///////////TIMELINE/////////// - const timelineNode = createTimelineNode(config); + ///////////SEARCH/////////// + const searchNode = createSearchNode(config); - ///////////SEARCH/////////// - const searchNode = createSearchNode(config); + ///////////ENGAGEMENT/////////// + const engagementNode = createEngagementNode(config); - ///////////ENGAGEMENT/////////// - const engagementNode = createEngagementNode(config); + ///////////TONE ANALYSIS/////////// + const toneAnalysisNode = createToneAnalysisNode(config); - ///////////TONE ANALYSIS/////////// - const toneAnalysisNode = createToneAnalysisNode(config); + ///////////RESPONSE GENERATION/////////// + const responseGenerationNode = createResponseGenerationNode(config); - ///////////RESPONSE GENERATION/////////// - const responseGenerationNode = createResponseGenerationNode(config); + ///////////RECHECK SKIPPED/////////// + const recheckSkippedNode = createRecheckSkippedNode(config); - ///////////RECHECK SKIPPED/////////// - const recheckSkippedNode = createRecheckSkippedNode(config); + ///////////AUTO APPROVAL/////////// + const autoApprovalNode = createAutoApprovalNode(config); - ///////////AUTO APPROVAL/////////// - const autoApprovalNode = createAutoApprovalNode(config); - - return { - mentionNode, - timelineNode, - searchNode, - engagementNode, - toneAnalysisNode, - responseGenerationNode, - recheckSkippedNode, - autoApprovalNode - }; -}; \ No newline at end of file + return { + mentionNode, + timelineNode, + searchNode, + engagementNode, + toneAnalysisNode, + responseGenerationNode, + recheckSkippedNode, + autoApprovalNode, + }; +}; diff --git a/auto-kol/agent/src/services/agents/nodes/autoApprovalNode.ts b/auto-kol/agent/src/services/agents/nodes/autoApprovalNode.ts index f400214..141847e 100644 --- a/auto-kol/agent/src/services/agents/nodes/autoApprovalNode.ts +++ b/auto-kol/agent/src/services/agents/nodes/autoApprovalNode.ts @@ -1,113 +1,132 @@ import { AIMessage } from "@langchain/core/messages"; -import { State, logger, parseMessageContent } from '../workflow.js'; -import * as prompts from '../prompts.js'; -import { WorkflowConfig } from '../workflow.js'; -import { getLastDsnCid, updateResponseStatusByTweetId } from '../../../database/index.js'; +import { State, logger, parseMessageContent } from "../workflow.js"; +import * as prompts from "../prompts.js"; +import { WorkflowConfig } from "../workflow.js"; +import { + getLastDsnCid, + updateResponseStatusByTweetId, +} from "../../../database/index.js"; import { uploadToDsn } from "../../../utils/dsn.js"; -import { config as globalConfig } from '../../../config/index.js'; -import { ResponseStatus } from '../../../types/queue.js'; +import { config as globalConfig } from "../../../config/index.js"; +import { ResponseStatus } from "../../../types/queue.js"; export const createAutoApprovalNode = (config: WorkflowConfig) => { - return async (state: typeof State.State) => { - logger.info('Auto Approval Node - Evaluating pending responses'); - try { - const lastMessage = state.messages[state.messages.length - 1]; - const parsedContent = parseMessageContent(lastMessage.content); - const { tweets, currentTweetIndex, batchToFeedback } = parsedContent; + return async (state: typeof State.State) => { + logger.info("Auto Approval Node - Evaluating pending responses"); + try { + const lastMessage = state.messages[state.messages.length - 1]; + const parsedContent = parseMessageContent(lastMessage.content); + const { tweets, currentTweetIndex, batchToFeedback } = parsedContent; - if (!batchToFeedback.length) { - logger.info('No pending responses found'); - return { - messages: [new AIMessage({ - content: JSON.stringify({ - fromAutoApproval: true, - batchToRespond: [] - }) - })] - }; - } + if (!batchToFeedback.length) { + logger.info("No pending responses found"); + return { + messages: [ + new AIMessage({ + content: JSON.stringify({ + fromAutoApproval: true, + batchToRespond: [], + }), + }), + ], + }; + } - const processedResponses = []; + const processedResponses = []; - for (const response of batchToFeedback) { - logger.info('Processing response', { - tweetId: response.tweet.id, - retry: response.retry - }); + for (const response of batchToFeedback) { + logger.info("Processing response", { + tweetId: response.tweet.id, + retry: response.retry, + }); - const approval = await prompts.autoApprovalPrompt - .pipe(config.llms.decision) - .pipe(prompts.autoApprovalParser) - .invoke({ - tweet: response.tweet, - response: response.response, - tone: response.toneAnalysis?.dominantTone, - strategy: response.responseStrategy?.strategy - }); + const approval = await prompts.autoApprovalPrompt + .pipe(config.llms.decision) + .pipe(prompts.autoApprovalParser) + .invoke({ + tweet: response.tweet, + response: response.response, + tone: response.toneAnalysis?.dominantTone, + strategy: response.responseStrategy?.strategy, + }); - if (approval.approved) { - response.type = ResponseStatus.APPROVED; + if (approval.approved) { + response.type = ResponseStatus.APPROVED; - await updateResponseStatusByTweetId(response.tweet.id, ResponseStatus.APPROVED); + await updateResponseStatusByTweetId( + response.tweet.id, + ResponseStatus.APPROVED, + ); - if (globalConfig.POST_TWEETS) { - logger.info('Sending tweet', { - response: response.response, - tweetId: response.tweet.id - }); + if (globalConfig.POST_TWEETS) { + logger.info("Sending tweet", { + response: response.response, + tweetId: response.tweet.id, + }); - const sendTweetResponse = await config.client.sendTweet(response.response, response.tweet.id); - logger.info('Tweet sent', { - sendTweetResponse - }); - } - - if (globalConfig.DSN_UPLOAD) { - await uploadToDsn({ - data: response, - }); - } - } else if (response.retry > globalConfig.RETRY_LIMIT) { - response.type = ResponseStatus.REJECTED; - logger.info('Rejecting tweet', { - tweetId: response.tweet.id, - }); - await updateResponseStatusByTweetId(response.tweet.id, ResponseStatus.REJECTED); - if (globalConfig.DSN_UPLOAD) { - await uploadToDsn({ - data: response, - }); - } - } else { - processedResponses.push({ - ...response, - type: ResponseStatus.PENDING, - workflowState: { - ...response.workflowState, - autoFeedback: [...response.workflowState.autoFeedback, { - response: response.response, - reason: approval.reason, - suggestedChanges: approval.suggestedChanges - }] - }, - feedbackDecision: 'reject' - }); - } - } + const sendTweetResponse = await config.client.sendTweet( + response.response, + response.tweet.id, + ); + logger.info("Tweet sent", { + sendTweetResponse, + }); + } - return { - messages: [new AIMessage({ - content: JSON.stringify({ - tweets: tweets, - currentTweetIndex: currentTweetIndex, - fromAutoApproval: true, - batchToRespond: processedResponses - }) - })] - }; - } catch (error) { - logger.error('Error in auto approval node:', error); - return { messages: [] }; + if (globalConfig.DSN_UPLOAD) { + await uploadToDsn({ + data: response, + }); + } + } else if (response.retry > globalConfig.RETRY_LIMIT) { + response.type = ResponseStatus.REJECTED; + logger.info("Rejecting tweet", { + tweetId: response.tweet.id, + }); + await updateResponseStatusByTweetId( + response.tweet.id, + ResponseStatus.REJECTED, + ); + if (globalConfig.DSN_UPLOAD) { + await uploadToDsn({ + data: response, + }); + } + } else { + processedResponses.push({ + ...response, + type: ResponseStatus.PENDING, + workflowState: { + ...response.workflowState, + autoFeedback: [ + ...response.workflowState.autoFeedback, + { + response: response.response, + reason: approval.reason, + suggestedChanges: approval.suggestedChanges, + }, + ], + }, + feedbackDecision: "reject", + }); } + } + + return { + messages: [ + new AIMessage({ + content: JSON.stringify({ + tweets: tweets, + currentTweetIndex: currentTweetIndex, + fromAutoApproval: true, + batchToRespond: processedResponses, + }), + }), + ], + }; + } catch (error) { + logger.error("Error in auto approval node:", error); + return { messages: [] }; } -}; \ No newline at end of file + }; +}; diff --git a/auto-kol/agent/src/services/agents/nodes/engagementNode.ts b/auto-kol/agent/src/services/agents/nodes/engagementNode.ts index 8edc480..ba9893a 100644 --- a/auto-kol/agent/src/services/agents/nodes/engagementNode.ts +++ b/auto-kol/agent/src/services/agents/nodes/engagementNode.ts @@ -1,135 +1,158 @@ import { AIMessage } from "@langchain/core/messages"; -import { State, logger, parseMessageContent } from '../workflow.js'; -import * as prompts from '../prompts.js'; -import { uploadToDsn } from '../../../utils/dsn.js'; -import { getLastDsnCid } from '../../../database/index.js'; -import { WorkflowConfig } from '../workflow.js'; -import { config as globalConfig } from '../../../config/index.js'; -import { ResponseStatus } from '../../../types/queue.js'; +import { State, logger, parseMessageContent } from "../workflow.js"; +import * as prompts from "../prompts.js"; +import { uploadToDsn } from "../../../utils/dsn.js"; +import { getLastDsnCid } from "../../../database/index.js"; +import { WorkflowConfig } from "../workflow.js"; +import { config as globalConfig } from "../../../config/index.js"; +import { ResponseStatus } from "../../../types/queue.js"; const handleSkippedTweet = async (tweet: any, decision: any, config: any) => { - logger.info('Skipping engagement for tweet', { tweetId: tweet.id, reason: decision.reason }); - await config.toolNode.invoke({ - messages: [new AIMessage({ - content: '', - tool_calls: [{ - name: 'queue_skipped', - args: { - tweet, - reason: decision.reason, - priority: decision.priority || 0, - workflowState: { decision } - }, - id: 'skip_call', - type: 'tool_call' - }] - })] - }); - - if (globalConfig.DSN_UPLOAD) { - await uploadToDsn({ - data: { - type: ResponseStatus.SKIPPED, - tweet, - decision, - workflowState: { - decision, - toneAnalysis: null, - responseStrategy: null - } + logger.info("Skipping engagement for tweet", { + tweetId: tweet.id, + reason: decision.reason, + }); + await config.toolNode.invoke({ + messages: [ + new AIMessage({ + content: "", + tool_calls: [ + { + name: "queue_skipped", + args: { + tweet, + reason: decision.reason, + priority: decision.priority || 0, + workflowState: { decision }, }, - }); - } + id: "skip_call", + type: "tool_call", + }, + ], + }), + ], + }); + + if (globalConfig.DSN_UPLOAD) { + await uploadToDsn({ + data: { + type: ResponseStatus.SKIPPED, + tweet, + decision, + workflowState: { + decision, + toneAnalysis: null, + responseStrategy: null, + }, + }, + }); + } }; export const createEngagementNode = (config: WorkflowConfig) => { - return async (state: typeof State.State) => { - logger.info('Engagement Node - Starting evaluation'); - try { - const lastMessage = state.messages[state.messages.length - 1]; - const parsedContent = parseMessageContent(lastMessage.content); - const pendingEngagements = parsedContent.pendingEngagements || []; - logger.info(`Current tweet index: ${parsedContent?.currentTweetIndex}`); + return async (state: typeof State.State) => { + logger.info("Engagement Node - Starting evaluation"); + try { + const lastMessage = state.messages[state.messages.length - 1]; + const parsedContent = parseMessageContent(lastMessage.content); + const pendingEngagements = parsedContent.pendingEngagements || []; + logger.info(`Current tweet index: ${parsedContent?.currentTweetIndex}`); - if (pendingEngagements.length > 0) { - logger.info(`number of pending engagements: ${pendingEngagements.length}`); - return { - messages: [new AIMessage({ - content: JSON.stringify({ - tweets: parsedContent.tweets, - currentTweetIndex: parsedContent.currentTweetIndex, - batchToAnalyze: pendingEngagements, - pendingEngagements: [], - lastProcessedId: parsedContent.lastProcessedId, - batchProcessing: true - }) - })], - processedTweets: state.processedTweets - }; - } + if (pendingEngagements.length > 0) { + logger.info( + `number of pending engagements: ${pendingEngagements.length}`, + ); + return { + messages: [ + new AIMessage({ + content: JSON.stringify({ + tweets: parsedContent.tweets, + currentTweetIndex: parsedContent.currentTweetIndex, + batchToAnalyze: pendingEngagements, + pendingEngagements: [], + lastProcessedId: parsedContent.lastProcessedId, + batchProcessing: true, + }), + }), + ], + processedTweets: state.processedTweets, + }; + } - const BATCH_SIZE = globalConfig.ENGAGEMENT_BATCH_SIZE; - const startIdx = parsedContent.currentTweetIndex || 0; - const proposedEndIdx = Number(startIdx) + Number(BATCH_SIZE); - const endIdx = Math.min(proposedEndIdx, parsedContent.tweets?.length || 0); - const batchTweets = parsedContent.tweets?.slice(startIdx, endIdx) || []; + const BATCH_SIZE = globalConfig.ENGAGEMENT_BATCH_SIZE; + const startIdx = parsedContent.currentTweetIndex || 0; + const proposedEndIdx = Number(startIdx) + Number(BATCH_SIZE); + const endIdx = Math.min( + proposedEndIdx, + parsedContent.tweets?.length || 0, + ); + const batchTweets = parsedContent.tweets?.slice(startIdx, endIdx) || []; - logger.info('Processing batch of tweets', { - batchSize: batchTweets.length, - startIndex: startIdx, - endIndex: endIdx, - totalTweets: parsedContent.tweets.length - }); - - const processedResults = await Promise.all(batchTweets.map(async (tweet: any) => { - if (state.processedTweets.has(tweet.id)) { - return { tweet, status: 'alreadyProcessed' }; - } - const decision = await prompts.engagementPrompt - .pipe(config.llms.decision) - .pipe(prompts.engagementParser) - .invoke({ - tweet: tweet.text, - thread: tweet.thread || [] - }) - .catch((error) => { - logger.error('Error in engagement node:', error); - return { shouldEngage: false, reason: 'Error in engagement node', priority: 0, confidence: 0 }; - }); + logger.info("Processing batch of tweets", { + batchSize: batchTweets.length, + startIndex: startIdx, + endIndex: endIdx, + totalTweets: parsedContent.tweets.length, + }); - return { tweet, decision, status: 'processing' }; - })); + const processedResults = await Promise.all( + batchTweets.map(async (tweet: any) => { + if (state.processedTweets.has(tweet.id)) { + return { tweet, status: "alreadyProcessed" }; + } + const decision = await prompts.engagementPrompt + .pipe(config.llms.decision) + .pipe(prompts.engagementParser) + .invoke({ + tweet: tweet.text, + thread: tweet.thread || [], + }) + .catch((error) => { + logger.error("Error in engagement node:", error); + return { + shouldEngage: false, + reason: "Error in engagement node", + priority: 0, + confidence: 0, + }; + }); - const tweetsToEngage = []; - const newProcessedTweets = new Set(state.processedTweets); + return { tweet, decision, status: "processing" }; + }), + ); - for (const result of processedResults) { - newProcessedTweets.add(result.tweet.id); - if (result.status === 'processing' && result.decision?.shouldEngage) { - tweetsToEngage.push({ - tweet: result.tweet, - decision: result.decision - }); - } else if (result.status === 'processing') { - await handleSkippedTweet(result.tweet, result.decision, config); - } - } + const tweetsToEngage = []; + const newProcessedTweets = new Set(state.processedTweets); - return { - messages: [new AIMessage({ - content: JSON.stringify({ - tweets: parsedContent.tweets, - currentTweetIndex: endIdx, - pendingEngagements: tweetsToEngage, - lastProcessedId: parsedContent.lastProcessedId, - batchProcessing: true, - }) - })], - processedTweets: newProcessedTweets - }; - } catch (error) { - logger.error('Error in engagement node:', error); - return { messages: [] }; + for (const result of processedResults) { + newProcessedTweets.add(result.tweet.id); + if (result.status === "processing" && result.decision?.shouldEngage) { + tweetsToEngage.push({ + tweet: result.tweet, + decision: result.decision, + }); + } else if (result.status === "processing") { + await handleSkippedTweet(result.tweet, result.decision, config); } + } + + return { + messages: [ + new AIMessage({ + content: JSON.stringify({ + tweets: parsedContent.tweets, + currentTweetIndex: endIdx, + pendingEngagements: tweetsToEngage, + lastProcessedId: parsedContent.lastProcessedId, + batchProcessing: true, + }), + }), + ], + processedTweets: newProcessedTweets, + }; + } catch (error) { + logger.error("Error in engagement node:", error); + return { messages: [] }; } -} \ No newline at end of file + }; +}; diff --git a/auto-kol/agent/src/services/agents/nodes/mentionNode.ts b/auto-kol/agent/src/services/agents/nodes/mentionNode.ts index bf414b4..2ce1bd0 100644 --- a/auto-kol/agent/src/services/agents/nodes/mentionNode.ts +++ b/auto-kol/agent/src/services/agents/nodes/mentionNode.ts @@ -5,30 +5,36 @@ import { State } from "../workflow.js"; import { tweetSearchSchema } from "../../../schemas/workflow.js"; export const createMentionNode = (config: WorkflowConfig) => { - return async (state: typeof State.State) => { - logger.info('Mention Node - Fetching recent mentions'); - const toolResponse = await config.toolNode.invoke({ - messages: [ - new AIMessage({ - content: '', - tool_calls: [{ - name: 'fetch_mentions', - args: {}, - id: 'fetch_mentions_call', - type: 'tool_call' - }] - }) - ] - }); + return async (state: typeof State.State) => { + logger.info("Mention Node - Fetching recent mentions"); + const toolResponse = await config.toolNode.invoke({ + messages: [ + new AIMessage({ + content: "", + tool_calls: [ + { + name: "fetch_mentions", + args: {}, + id: "fetch_mentions_call", + type: "tool_call", + }, + ], + }), + ], + }); - const parsedContent = parseMessageContent(toolResponse.messages[toolResponse.messages.length - 1].content); - const parsedTweets = tweetSearchSchema.parse(parsedContent); - - return { - messages: [new AIMessage({ - content: JSON.stringify(parsedTweets) - })], - lastProcessedId: parsedTweets.lastProcessedId || undefined - }; - } -} \ No newline at end of file + const parsedContent = parseMessageContent( + toolResponse.messages[toolResponse.messages.length - 1].content, + ); + const parsedTweets = tweetSearchSchema.parse(parsedContent); + + return { + messages: [ + new AIMessage({ + content: JSON.stringify(parsedTweets), + }), + ], + lastProcessedId: parsedTweets.lastProcessedId || undefined, + }; + }; +}; diff --git a/auto-kol/agent/src/services/agents/nodes/recheckSkippedNode.ts b/auto-kol/agent/src/services/agents/nodes/recheckSkippedNode.ts index 99e0d29..7fd7028 100644 --- a/auto-kol/agent/src/services/agents/nodes/recheckSkippedNode.ts +++ b/auto-kol/agent/src/services/agents/nodes/recheckSkippedNode.ts @@ -1,89 +1,100 @@ import { AIMessage } from "@langchain/core/messages"; -import { State, logger, parseMessageContent } from '../workflow.js'; -import * as prompts from '../prompts.js'; -import { flagBackSkippedTweet, getAllSkippedTweetsToRecheck } from '../../../database/index.js'; -import { WorkflowConfig } from '../workflow.js'; +import { State, logger, parseMessageContent } from "../workflow.js"; +import * as prompts from "../prompts.js"; +import { + flagBackSkippedTweet, + getAllSkippedTweetsToRecheck, +} from "../../../database/index.js"; +import { WorkflowConfig } from "../workflow.js"; export const createRecheckSkippedNode = (config: WorkflowConfig) => { - return async (state: typeof State.State) => { - logger.info('Recheck Skipped Node - Reviewing previously skipped tweets'); - try { - const lastMessage = state.messages[state.messages.length - 1]; - const parsedContent = parseMessageContent(lastMessage.content); - const {tweets, currentTweetIndex} = parsedContent; - logger.info(`currentTweetIndex: ${currentTweetIndex}`); - - const skippedTweets = await getAllSkippedTweetsToRecheck(); + return async (state: typeof State.State) => { + logger.info("Recheck Skipped Node - Reviewing previously skipped tweets"); + try { + const lastMessage = state.messages[state.messages.length - 1]; + const parsedContent = parseMessageContent(lastMessage.content); + const { tweets, currentTweetIndex } = parsedContent; + logger.info(`currentTweetIndex: ${currentTweetIndex}`); - if (!skippedTweets || skippedTweets.length === 0) { - logger.info('No skipped tweets to recheck'); - return { - messages: [new AIMessage({ - content: JSON.stringify({ - fromRecheckNode: true, - currentTweetIndex: currentTweetIndex, - tweets: tweets, - pendingEngagements: [], - messages: [] - }) - })] - }; - } + const skippedTweets = await getAllSkippedTweetsToRecheck(); - logger.info(`Found ${skippedTweets.length} skipped tweets to recheck`); + if (!skippedTweets || skippedTweets.length === 0) { + logger.info("No skipped tweets to recheck"); + return { + messages: [ + new AIMessage({ + content: JSON.stringify({ + fromRecheckNode: true, + currentTweetIndex: currentTweetIndex, + tweets: tweets, + pendingEngagements: [], + messages: [], + }), + }), + ], + }; + } - const processedTweets = []; - for (const tweet of skippedTweets) { - const decision = await prompts.engagementPrompt - .pipe(config.llms.decision) - .pipe(prompts.engagementParser) - .invoke({ tweet: tweet.text }); + logger.info(`Found ${skippedTweets.length} skipped tweets to recheck`); - logger.info('Recheck decision:', { tweetId: tweet.id, decision }); + const processedTweets = []; + for (const tweet of skippedTweets) { + const decision = await prompts.engagementPrompt + .pipe(config.llms.decision) + .pipe(prompts.engagementParser) + .invoke({ tweet: tweet.text }); - if (decision.shouldEngage) { - processedTweets.push({ - tweet, - decision - }); - } else { - const flagged = await flagBackSkippedTweet(tweet.id, decision.reason); - if (!flagged) { - logger.info('Failed to flag back skipped tweet:', { tweetId: tweet.id }); - } - } - } + logger.info("Recheck decision:", { tweetId: tweet.id, decision }); - if (processedTweets.length === 0) { - logger.info('No skipped tweets passed recheck'); - return { - messages: [new AIMessage({ - content: JSON.stringify({ - fromRecheckNode: true, - currentTweetIndex: currentTweetIndex, - tweets: tweets, - pendingEngagements: [], - messages: [] - }) - })] - }; - } + if (decision.shouldEngage) { + processedTweets.push({ + tweet, + decision, + }); + } else { + const flagged = await flagBackSkippedTweet(tweet.id, decision.reason); + if (!flagged) { + logger.info("Failed to flag back skipped tweet:", { + tweetId: tweet.id, + }); + } + } + } - const { tweet, decision } = processedTweets[0]; + if (processedTweets.length === 0) { + logger.info("No skipped tweets passed recheck"); + return { + messages: [ + new AIMessage({ + content: JSON.stringify({ + fromRecheckNode: true, + currentTweetIndex: currentTweetIndex, + tweets: tweets, + pendingEngagements: [], + messages: [], + }), + }), + ], + }; + } - return { - messages: [new AIMessage({ - content: JSON.stringify({ - tweets: [tweet], - currentTweetIndex: 0, - tweet, - decision - }) - })] - }; - } catch (error) { - logger.error('Error in recheck skipped node:', error); - return { messages: [] }; - } + const { tweet, decision } = processedTweets[0]; + + return { + messages: [ + new AIMessage({ + content: JSON.stringify({ + tweets: [tweet], + currentTweetIndex: 0, + tweet, + decision, + }), + }), + ], + }; + } catch (error) { + logger.error("Error in recheck skipped node:", error); + return { messages: [] }; } -} \ No newline at end of file + }; +}; diff --git a/auto-kol/agent/src/services/agents/nodes/responseGenerationNode.ts b/auto-kol/agent/src/services/agents/nodes/responseGenerationNode.ts index 5a26dd7..5303680 100644 --- a/auto-kol/agent/src/services/agents/nodes/responseGenerationNode.ts +++ b/auto-kol/agent/src/services/agents/nodes/responseGenerationNode.ts @@ -1,158 +1,186 @@ import { AIMessage } from "@langchain/core/messages"; -import { State, logger, parseMessageContent } from '../workflow.js'; -import * as prompts from '../prompts.js'; -import { WorkflowConfig } from '../workflow.js'; -import { ResponseStatus } from '../../../types/queue.js'; +import { State, logger, parseMessageContent } from "../workflow.js"; +import * as prompts from "../prompts.js"; +import { WorkflowConfig } from "../workflow.js"; +import { ResponseStatus } from "../../../types/queue.js"; export const createResponseGenerationNode = (config: WorkflowConfig) => { - return async (state: typeof State.State) => { - logger.info('Response Generation Node - Creating response strategy'); - try { - const lastMessage = state.messages[state.messages.length - 1]; - const parsedContent = parseMessageContent(lastMessage.content); - const batchToRespond = parsedContent.batchToRespond || []; - const batchToFeedback: any[] = []; + return async (state: typeof State.State) => { + logger.info("Response Generation Node - Creating response strategy"); + try { + const lastMessage = state.messages[state.messages.length - 1]; + const parsedContent = parseMessageContent(lastMessage.content); + const batchToRespond = parsedContent.batchToRespond || []; + const batchToFeedback: any[] = []; - logger.info(`Processing batch of ${batchToRespond.length} tweets for response generation`, { - hasRejectedResponses: parsedContent.fromAutoApproval - }); - - await Promise.all( - batchToRespond.map(async (item: any) => { - const { tweet, decision, toneAnalysis, workflowState } = item; + logger.info( + `Processing batch of ${batchToRespond.length} tweets for response generation`, + { + hasRejectedResponses: parsedContent.fromAutoApproval, + }, + ); - if (!workflowState) { - item.workflowState = { autoFeedback: [] }; - } else if (!workflowState.autoFeedback) { - workflowState.autoFeedback = []; - } + await Promise.all( + batchToRespond.map(async (item: any) => { + const { tweet, decision, toneAnalysis, workflowState } = item; - if (parsedContent.fromAutoApproval) { - item.retry = (item.retry || 0) + 1; - logger.info('Regenerating response due to rejection:', { - retry: item.retry - }); + if (!workflowState) { + item.workflowState = { autoFeedback: [] }; + } else if (!workflowState.autoFeedback) { + workflowState.autoFeedback = []; + } - } else { - item.retry = 0; - } - - const lastFeedback = workflowState?.autoFeedback[workflowState?.autoFeedback.length - 1]; - const rejectionInstructions = lastFeedback - ? prompts.formatRejectionInstructions(lastFeedback.reason) - : ''; - const rejectionFeedback = lastFeedback - ? prompts.formatRejectionFeedback(lastFeedback.reason, lastFeedback.suggestedChanges) - : ''; + if (parsedContent.fromAutoApproval) { + item.retry = (item.retry || 0) + 1; + logger.info("Regenerating response due to rejection:", { + retry: item.retry, + }); + } else { + item.retry = 0; + } - const similarTweetsResponse = await config.toolNode.invoke({ - messages: [new AIMessage({ - content: '', - tool_calls: [{ - name: 'search_similar_tweets', - args: { - query: `author:${tweet.author_username} ${tweet.text}`, - k: 5 - }, - id: 'similar_tweets_call', - type: 'tool_call' - }], - })], - }); + const lastFeedback = + workflowState?.autoFeedback[workflowState?.autoFeedback.length - 1]; + const rejectionInstructions = lastFeedback + ? prompts.formatRejectionInstructions(lastFeedback.reason) + : ""; + const rejectionFeedback = lastFeedback + ? prompts.formatRejectionFeedback( + lastFeedback.reason, + lastFeedback.suggestedChanges, + ) + : ""; - const similarTweets = parseMessageContent( - similarTweetsResponse.messages[similarTweetsResponse.messages.length - 1].content - ); + const similarTweetsResponse = await config.toolNode.invoke({ + messages: [ + new AIMessage({ + content: "", + tool_calls: [ + { + name: "search_similar_tweets", + args: { + query: `author:${tweet.author_username} ${tweet.text}`, + k: 5, + }, + id: "similar_tweets_call", + type: "tool_call", + }, + ], + }), + ], + }); - const responseStrategy = await prompts.responsePrompt - .pipe(config.llms.response) - .pipe(prompts.responseParser) - .invoke({ - tweet: tweet.text, - tone: toneAnalysis?.suggestedTone || workflowState?.toneAnalysis?.suggestedTone, - author: tweet.author_username, - similarTweets: JSON.stringify(similarTweets.similar_tweets), - thread: JSON.stringify(tweet.thread || []), - previousResponse: workflowState?.autoFeedback[workflowState?.autoFeedback.length - 1]?.response || '', - rejectionFeedback, - rejectionInstructions - }); + const similarTweets = parseMessageContent( + similarTweetsResponse.messages[ + similarTweetsResponse.messages.length - 1 + ].content, + ); + const responseStrategy = await prompts.responsePrompt + .pipe(config.llms.response) + .pipe(prompts.responseParser) + .invoke({ + tweet: tweet.text, + tone: + toneAnalysis?.suggestedTone || + workflowState?.toneAnalysis?.suggestedTone, + author: tweet.author_username, + similarTweets: JSON.stringify(similarTweets.similar_tweets), + thread: JSON.stringify(tweet.thread || []), + previousResponse: + workflowState?.autoFeedback[ + workflowState?.autoFeedback.length - 1 + ]?.response || "", + rejectionFeedback, + rejectionInstructions, + }); - const data = { - type: ResponseStatus.PENDING, - tweet, - response: responseStrategy.content, - workflowState: { - decision: decision || workflowState?.decision, - toneAnalysis: toneAnalysis || workflowState?.toneAnalysis, - responseStrategy: { - tone: responseStrategy.tone, - strategy: responseStrategy.strategy, - referencedTweets: responseStrategy.referencedTweets, - confidence: responseStrategy.confidence - }, - autoFeedback: workflowState?.autoFeedback || [] - }, - retry: item.retry - } - batchToFeedback.push(data); + const data = { + type: ResponseStatus.PENDING, + tweet, + response: responseStrategy.content, + workflowState: { + decision: decision || workflowState?.decision, + toneAnalysis: toneAnalysis || workflowState?.toneAnalysis, + responseStrategy: { + tone: responseStrategy.tone, + strategy: responseStrategy.strategy, + referencedTweets: responseStrategy.referencedTweets, + confidence: responseStrategy.confidence, + }, + autoFeedback: workflowState?.autoFeedback || [], + }, + retry: item.retry, + }; + batchToFeedback.push(data); - const args = { - tweet, - response: responseStrategy.content, - workflowState: { - toneAnalysis: toneAnalysis, - responseStrategy, - thread: tweet.thread || [], - similarTweets: similarTweets.similar_tweets, - }, - } - if (!parsedContent.fromAutoApproval) { - const addResponse = await config.toolNode.invoke({ - messages: [new AIMessage({ - content: '', - tool_calls: [{ - name: 'add_response', - args, - id: 'add_response_call', - type: 'tool_call' - }] - })] - }); - return addResponse; - } else { - const updateResponse = await config.toolNode.invoke({ - messages: [new AIMessage({ - content: '', - tool_calls: [{ - name: 'update_response', - args, - id: 'update_response_call', - type: 'tool_call' - }] - })] - }); - return updateResponse; - } - }) - ); + const args = { + tweet, + response: responseStrategy.content, + workflowState: { + toneAnalysis: toneAnalysis, + responseStrategy, + thread: tweet.thread || [], + similarTweets: similarTweets.similar_tweets, + }, + }; + if (!parsedContent.fromAutoApproval) { + const addResponse = await config.toolNode.invoke({ + messages: [ + new AIMessage({ + content: "", + tool_calls: [ + { + name: "add_response", + args, + id: "add_response_call", + type: "tool_call", + }, + ], + }), + ], + }); + return addResponse; + } else { + const updateResponse = await config.toolNode.invoke({ + messages: [ + new AIMessage({ + content: "", + tool_calls: [ + { + name: "update_response", + args, + id: "update_response_call", + type: "tool_call", + }, + ], + }), + ], + }); + return updateResponse; + } + }), + ); - return { - messages: [new AIMessage({ - content: JSON.stringify({ - tweets: parsedContent.tweets, - currentTweetIndex: parsedContent.currentTweetIndex, - pendingEngagements: parsedContent.pendingEngagements, - lastProcessedId: parsedContent.lastProcessedId, - batchToFeedback: batchToFeedback, - }) - })], - processedTweets: new Set(batchToRespond.map((item: any) => item.tweet.id)) - }; - } catch (error) { - logger.error('Error in response generation node:', error); - return { messages: [] }; - } + return { + messages: [ + new AIMessage({ + content: JSON.stringify({ + tweets: parsedContent.tweets, + currentTweetIndex: parsedContent.currentTweetIndex, + pendingEngagements: parsedContent.pendingEngagements, + lastProcessedId: parsedContent.lastProcessedId, + batchToFeedback: batchToFeedback, + }), + }), + ], + processedTweets: new Set( + batchToRespond.map((item: any) => item.tweet.id), + ), + }; + } catch (error) { + logger.error("Error in response generation node:", error); + return { messages: [] }; } -}; \ No newline at end of file + }; +}; diff --git a/auto-kol/agent/src/services/agents/nodes/searchNode.ts b/auto-kol/agent/src/services/agents/nodes/searchNode.ts index bfebbe2..3a7abf2 100644 --- a/auto-kol/agent/src/services/agents/nodes/searchNode.ts +++ b/auto-kol/agent/src/services/agents/nodes/searchNode.ts @@ -1,94 +1,102 @@ import { AIMessage } from "@langchain/core/messages"; -import { State, logger, parseMessageContent } from '../workflow.js'; -import { tweetSearchSchema } from '../../../schemas/workflow.js'; -import { ChromaService } from '../../vectorstore/chroma.js'; -import * as db from '../../database/index.js'; -import { WorkflowConfig } from '../workflow.js'; +import { State, logger, parseMessageContent } from "../workflow.js"; +import { tweetSearchSchema } from "../../../schemas/workflow.js"; +import { ChromaService } from "../../vectorstore/chroma.js"; +import * as db from "../../database/index.js"; +import { WorkflowConfig } from "../workflow.js"; export const createSearchNode = (config: WorkflowConfig) => { - return async (state: typeof State.State) => { - logger.info('Search Node - Fetching recent tweets'); - const existingTweets = state.messages.length > 0 ? - parseMessageContent(state.messages[state.messages.length - 1].content).tweets : []; + return async (state: typeof State.State) => { + logger.info("Search Node - Fetching recent tweets"); + const existingTweets = + state.messages.length > 0 + ? parseMessageContent(state.messages[state.messages.length - 1].content) + .tweets + : []; - logger.info(`Existing tweets: ${existingTweets.length}`); - try { - logger.info('Last processed id:', state.lastProcessedId); + logger.info(`Existing tweets: ${existingTweets.length}`); + try { + logger.info("Last processed id:", state.lastProcessedId); - const toolResponse = await config.toolNode.invoke({ - messages: [ - new AIMessage({ - content: '', - tool_calls: [{ - name: 'search_recent_tweets', - args: { - lastProcessedId: state.lastProcessedId || undefined - }, - id: 'tool_call_id', - type: 'tool_call' - }], - }), - ], - }); + const toolResponse = await config.toolNode.invoke({ + messages: [ + new AIMessage({ + content: "", + tool_calls: [ + { + name: "search_recent_tweets", + args: { + lastProcessedId: state.lastProcessedId || undefined, + }, + id: "tool_call_id", + type: "tool_call", + }, + ], + }), + ], + }); - const lastMessage = toolResponse.messages[toolResponse.messages.length - 1]; + const lastMessage = + toolResponse.messages[toolResponse.messages.length - 1]; - let searchResult; - if (typeof lastMessage.content === 'string') { - try { - searchResult = JSON.parse(lastMessage.content); - logger.info('Parsed search result:', searchResult); - } catch (error) { - logger.error('Failed to parse search result:', error); - searchResult = { tweets: [], lastProcessedId: null }; - } - } else { - searchResult = lastMessage.content; - logger.info('Non-string search result:', searchResult); - } + let searchResult; + if (typeof lastMessage.content === "string") { + try { + searchResult = JSON.parse(lastMessage.content); + logger.info("Parsed search result:", searchResult); + } catch (error) { + logger.error("Failed to parse search result:", error); + searchResult = { tweets: [], lastProcessedId: null }; + } + } else { + searchResult = lastMessage.content; + logger.info("Non-string search result:", searchResult); + } - const newTweets = [...existingTweets]; - for (const tweet of searchResult.tweets) { - if (await db.isTweetExists(tweet.id)) { - continue; - } - newTweets.push(tweet); - } - const validatedResult = tweetSearchSchema.parse({ - tweets: newTweets, - lastProcessedId: searchResult.lastProcessedId - }); + const newTweets = [...existingTweets]; + for (const tweet of searchResult.tweets) { + if (await db.isTweetExists(tweet.id)) { + continue; + } + newTweets.push(tweet); + } + const validatedResult = tweetSearchSchema.parse({ + tweets: newTweets, + lastProcessedId: searchResult.lastProcessedId, + }); - const chromaService = await ChromaService.getInstance(); + const chromaService = await ChromaService.getInstance(); - if (validatedResult.tweets.length > 0) { - await Promise.all( - validatedResult.tweets.map(tweet => chromaService.addTweet(tweet)) - ); - } + if (validatedResult.tweets.length > 0) { + await Promise.all( + validatedResult.tweets.map((tweet) => chromaService.addTweet(tweet)), + ); + } - logger.info(`Found ${validatedResult.tweets.length} tweets`); + logger.info(`Found ${validatedResult.tweets.length} tweets`); - return { - messages: [new AIMessage({ - content: JSON.stringify({ - tweets: validatedResult.tweets, - currentTweetIndex: 0, - lastProcessedId: validatedResult.lastProcessedId - }) - })], - lastProcessedId: validatedResult.lastProcessedId || undefined - }; - } catch (error) { - logger.error('Error in search node:', error); - const emptyResult = { - tweets: [], - lastProcessedId: null - }; - return { - messages: [new AIMessage({ content: JSON.stringify(emptyResult) })], - lastProcessedId: undefined - }; - } - }; -}; \ No newline at end of file + return { + messages: [ + new AIMessage({ + content: JSON.stringify({ + tweets: validatedResult.tweets, + currentTweetIndex: 0, + lastProcessedId: validatedResult.lastProcessedId, + }), + }), + ], + lastProcessedId: validatedResult.lastProcessedId || undefined, + }; + } catch (error) { + logger.error("Error in search node:", error); + const emptyResult = { + tweets: [], + lastProcessedId: null, + }; + return { + messages: [new AIMessage({ content: JSON.stringify(emptyResult) })], + lastProcessedId: undefined, + }; + } + }; +}; diff --git a/auto-kol/agent/src/services/agents/nodes/timelineNode.ts b/auto-kol/agent/src/services/agents/nodes/timelineNode.ts index c40fdb1..5d8daf6 100644 --- a/auto-kol/agent/src/services/agents/nodes/timelineNode.ts +++ b/auto-kol/agent/src/services/agents/nodes/timelineNode.ts @@ -3,54 +3,62 @@ import { parseMessageContent, WorkflowConfig } from "../workflow.js"; import { logger } from "../workflow.js"; import { State } from "../workflow.js"; import { tweetSearchSchema } from "../../../schemas/workflow.js"; -import * as db from '../../database/index.js'; - +import * as db from "../../database/index.js"; export const createTimelineNode = (config: WorkflowConfig) => { - return async (state: typeof State.State) => { - logger.info('Timeline Node - Fetching recent tweets'); - const existingTweets = state.messages.length > 0 ? - parseMessageContent(state.messages[state.messages.length - 1].content).tweets : []; - - logger.info(`Existing tweets: ${existingTweets.length}`); - const toolResponse = await config.toolNode.invoke({ - messages: [ - new AIMessage({ - content: '', - tool_calls: [{ - name: 'fetch_timeline', - args: {}, - id: 'fetch_timeline_call', - type: 'tool_call' - }] - }) - ] - }); + return async (state: typeof State.State) => { + logger.info("Timeline Node - Fetching recent tweets"); + const existingTweets = + state.messages.length > 0 + ? parseMessageContent(state.messages[state.messages.length - 1].content) + .tweets + : []; - logger.info('Tool response received:', { - messageCount: toolResponse.messages.length, - }); + logger.info(`Existing tweets: ${existingTweets.length}`); + const toolResponse = await config.toolNode.invoke({ + messages: [ + new AIMessage({ + content: "", + tool_calls: [ + { + name: "fetch_timeline", + args: {}, + id: "fetch_timeline_call", + type: "tool_call", + }, + ], + }), + ], + }); - const content = toolResponse.messages[toolResponse.messages.length - 1].content; - const parsedContent = typeof content === 'string' ? JSON.parse(content) : content; + logger.info("Tool response received:", { + messageCount: toolResponse.messages.length, + }); - const parsedTweets = tweetSearchSchema.parse(parsedContent); + const content = + toolResponse.messages[toolResponse.messages.length - 1].content; + const parsedContent = + typeof content === "string" ? JSON.parse(content) : content; - const newTweets = [...existingTweets]; - for (const tweet of parsedTweets.tweets) { - if (await db.isTweetExists(tweet.id)) { - continue; - } - newTweets.push(tweet); - } + const parsedTweets = tweetSearchSchema.parse(parsedContent); - return { - messages: [new AIMessage({ - content: JSON.stringify({ - tweets: newTweets, - }) - })], - lastProcessedId: parsedTweets.lastProcessedId || undefined - }; + const newTweets = [...existingTweets]; + for (const tweet of parsedTweets.tweets) { + if (await db.isTweetExists(tweet.id)) { + continue; + } + newTweets.push(tweet); } -} \ No newline at end of file + + return { + messages: [ + new AIMessage({ + content: JSON.stringify({ + tweets: newTweets, + }), + }), + ], + lastProcessedId: parsedTweets.lastProcessedId || undefined, + }; + }; +}; diff --git a/auto-kol/agent/src/services/agents/nodes/toneAnalysisNode.ts b/auto-kol/agent/src/services/agents/nodes/toneAnalysisNode.ts index aa8f288..162bd8c 100644 --- a/auto-kol/agent/src/services/agents/nodes/toneAnalysisNode.ts +++ b/auto-kol/agent/src/services/agents/nodes/toneAnalysisNode.ts @@ -1,52 +1,58 @@ import { AIMessage } from "@langchain/core/messages"; -import { State, logger, parseMessageContent } from '../workflow.js'; -import * as prompts from '../prompts.js'; -import { WorkflowConfig } from '../workflow.js'; - -export const createToneAnalysisNode = (config: WorkflowConfig) => { - return async (state: typeof State.State) => { - logger.info('Tone Analysis Node - Analyzing tweet tone'); - try { - const lastMessage = state.messages[state.messages.length - 1]; - const parsedContent = parseMessageContent(lastMessage.content); - const batchToAnalyze = parsedContent.batchToAnalyze || []; +import { State, logger, parseMessageContent } from "../workflow.js"; +import * as prompts from "../prompts.js"; +import { WorkflowConfig } from "../workflow.js"; - logger.info(`Processing batch of ${batchToAnalyze.length} tweets for tone analysis`); +export const createToneAnalysisNode = (config: WorkflowConfig) => { + return async (state: typeof State.State) => { + logger.info("Tone Analysis Node - Analyzing tweet tone"); + try { + const lastMessage = state.messages[state.messages.length - 1]; + const parsedContent = parseMessageContent(lastMessage.content); + const batchToAnalyze = parsedContent.batchToAnalyze || []; - const analyzedBatch = await Promise.all( - batchToAnalyze.map(async ({ tweet, decision }: { tweet: any; decision: any }) => { - const toneAnalysis = await prompts.tonePrompt - .pipe(config.llms.tone) - .pipe(prompts.toneParser) - .invoke({ - tweet: tweet.text, - thread: tweet.thread || [] - }); + logger.info( + `Processing batch of ${batchToAnalyze.length} tweets for tone analysis`, + ); - logger.info('Tone analysis:', { toneAnalysis }); + const analyzedBatch = await Promise.all( + batchToAnalyze.map( + async ({ tweet, decision }: { tweet: any; decision: any }) => { + const toneAnalysis = await prompts.tonePrompt + .pipe(config.llms.tone) + .pipe(prompts.toneParser) + .invoke({ + tweet: tweet.text, + thread: tweet.thread || [], + }); - return { - tweet, - decision, - toneAnalysis - }; - }) - ); + logger.info("Tone analysis:", { toneAnalysis }); return { - messages: [new AIMessage({ - content: JSON.stringify({ - tweets: parsedContent.tweets, - currentTweetIndex: parsedContent.currentTweetIndex, - batchToRespond: analyzedBatch, - pendingEngagements: parsedContent.pendingEngagements, - lastProcessedId: parsedContent.lastProcessedId - }) - })] + tweet, + decision, + toneAnalysis, }; - } catch (error) { - logger.error('Error in tone analysis node:', error); - return { messages: [] }; - } + }, + ), + ); + + return { + messages: [ + new AIMessage({ + content: JSON.stringify({ + tweets: parsedContent.tweets, + currentTweetIndex: parsedContent.currentTweetIndex, + batchToRespond: analyzedBatch, + pendingEngagements: parsedContent.pendingEngagements, + lastProcessedId: parsedContent.lastProcessedId, + }), + }), + ], + }; + } catch (error) { + logger.error("Error in tone analysis node:", error); + return { messages: [] }; } -} \ No newline at end of file + }; +}; diff --git a/auto-kol/agent/src/services/agents/prompts.ts b/auto-kol/agent/src/services/agents/prompts.ts index 94c498d..ed99a42 100644 --- a/auto-kol/agent/src/services/agents/prompts.ts +++ b/auto-kol/agent/src/services/agents/prompts.ts @@ -1,16 +1,24 @@ -import { StructuredOutputParser } from 'langchain/output_parsers'; -import { engagementSchema, toneSchema, responseSchema, autoApprovalSchema } from '../../schemas/workflow.js'; -import { ChatPromptTemplate, PromptTemplate } from '@langchain/core/prompts'; -import { SystemMessage } from '@langchain/core/messages'; -import { config } from '../../config/index.js'; +import { StructuredOutputParser } from "langchain/output_parsers"; +import { + engagementSchema, + toneSchema, + responseSchema, + autoApprovalSchema, +} from "../../schemas/workflow.js"; +import { ChatPromptTemplate, PromptTemplate } from "@langchain/core/prompts"; +import { SystemMessage } from "@langchain/core/messages"; +import { config } from "../../config/index.js"; const agentUsername = config.TWITTER_USERNAME!; const walletAddress = config.WALLET_ADDRESS!; -export const engagementParser = StructuredOutputParser.fromZodSchema(engagementSchema); +export const engagementParser = + StructuredOutputParser.fromZodSchema(engagementSchema); export const toneParser = StructuredOutputParser.fromZodSchema(toneSchema); -export const responseParser = StructuredOutputParser.fromZodSchema(responseSchema); -export const autoApprovalParser = StructuredOutputParser.fromZodSchema(autoApprovalSchema); +export const responseParser = + StructuredOutputParser.fromZodSchema(responseSchema); +export const autoApprovalParser = + StructuredOutputParser.fromZodSchema(autoApprovalSchema); // // ============ ENGAGEMENT SYSTEM PROMPT ============ @@ -35,7 +43,7 @@ export const engagementSystemPrompt = await PromptTemplate.fromTemplate( IMPORTANT: Follow the exact output format. If anything is unclear, just return shouldEngage: false. - {format_instructions}` + {format_instructions}`, ).format({ format_instructions: engagementParser.getFormatInstructions(), }); @@ -54,7 +62,7 @@ export const toneSystemPrompt = await PromptTemplate.fromTemplate( Make sure to balance cynicism with technical accuracy or insight. - {format_instructions}` + {format_instructions}`, ).format({ format_instructions: toneParser.getFormatInstructions(), }); @@ -87,7 +95,7 @@ export const responseSystemPrompt = await PromptTemplate.fromTemplate( - Do not wrap the response in \`\`\`json or any other markers - The response must exactly match the following schema: - {format_instructions}` + {format_instructions}`, ).format({ format_instructions: responseParser.getFormatInstructions(), }); @@ -113,7 +121,7 @@ export const autoApprovalSystemPrompt = await PromptTemplate.fromTemplate( - Character limit violations. - Extremely offensive content. - {format_instructions}` + {format_instructions}`, ).format({ format_instructions: autoApprovalParser.getFormatInstructions(), }); @@ -124,7 +132,7 @@ export const autoApprovalSystemPrompt = await PromptTemplate.fromTemplate( export const engagementPrompt = ChatPromptTemplate.fromMessages([ new SystemMessage(engagementSystemPrompt), [ - 'human', + "human", `Evaluate this tweet and provide your structured decision: Tweet: {tweet} Thread Context: {thread} @@ -138,7 +146,7 @@ export const engagementPrompt = ChatPromptTemplate.fromMessages([ export const tonePrompt = ChatPromptTemplate.fromMessages([ new SystemMessage(toneSystemPrompt), [ - 'human', + "human", `Analyze the tone for this tweet and suggest a response tone: Tweet: {tweet} Thread: {thread} @@ -152,7 +160,7 @@ export const tonePrompt = ChatPromptTemplate.fromMessages([ export const responsePrompt = ChatPromptTemplate.fromMessages([ new SystemMessage(responseSystemPrompt), [ - 'human', + "human", `Generate a response strategy for this tweet by considering similar tweets from @{author} using the suggested tone: Tweet: {tweet} Tone: {tone} @@ -190,18 +198,21 @@ export const responsePrompt = ChatPromptTemplate.fromMessages([ ]); // Helper function to format rejection feedback -export const formatRejectionFeedback = (rejectionReason?: string, suggestedChanges?: string) => { - if (!rejectionReason) return ''; +export const formatRejectionFeedback = ( + rejectionReason?: string, + suggestedChanges?: string, +) => { + if (!rejectionReason) return ""; return `\nPrevious Response Feedback: Rejection Reason: ${rejectionReason} - Suggested Changes: ${suggestedChanges || 'None provided'} + Suggested Changes: ${suggestedChanges || "None provided"} Please address this feedback in your new response.`; }; export const formatRejectionInstructions = (rejectionReason?: string) => { - if (!rejectionReason) return ''; + if (!rejectionReason) return ""; return `\nIMPORTANT: Your previous response was rejected. Make sure to: 1. Address the rejection reason: "${rejectionReason}" @@ -212,7 +223,7 @@ export const formatRejectionInstructions = (rejectionReason?: string) => { export const autoApprovalPrompt = ChatPromptTemplate.fromMessages([ new SystemMessage(autoApprovalSystemPrompt), [ - 'human', + "human", `Evaluate this response: Original Tweet: {tweet} Generated Response: {response} diff --git a/auto-kol/agent/src/services/agents/workflow.ts b/auto-kol/agent/src/services/agents/workflow.ts index 1026efd..531d4fe 100644 --- a/auto-kol/agent/src/services/agents/workflow.ts +++ b/auto-kol/agent/src/services/agents/workflow.ts @@ -1,203 +1,213 @@ -import { END, MemorySaver, StateGraph, START, Annotation } from '@langchain/langgraph'; -import { BaseMessage } from '@langchain/core/messages'; -import { ChatOpenAI } from '@langchain/openai'; -import { MessageContent } from '@langchain/core/messages'; -import { config } from '../../config/index.js'; -import { createLogger } from '../../utils/logger.js'; -import { createTools } from '../../tools/index.js'; -import { ToolNode } from '@langchain/langgraph/prebuilt'; -import { createTwitterClientScraper, ExtendedScraper } from '../twitter/api.js'; -export const logger = createLogger('agent-workflow'); -import { createNodes } from './nodes.js'; - +import { + END, + MemorySaver, + StateGraph, + START, + Annotation, +} from "@langchain/langgraph"; +import { BaseMessage } from "@langchain/core/messages"; +import { ChatOpenAI } from "@langchain/openai"; +import { MessageContent } from "@langchain/core/messages"; +import { config } from "../../config/index.js"; +import { createLogger } from "../../utils/logger.js"; +import { createTools } from "../../tools/index.js"; +import { ToolNode } from "@langchain/langgraph/prebuilt"; +import { createTwitterClientScraper, ExtendedScraper } from "../twitter/api.js"; +export const logger = createLogger("agent-workflow"); +import { createNodes } from "./nodes.js"; export const parseMessageContent = (content: MessageContent): any => { - if (typeof content === 'string') { - return JSON.parse(content); - } - if (Array.isArray(content)) { - return JSON.parse(JSON.stringify(content)); - } - return content; + if (typeof content === "string") { + return JSON.parse(content); + } + if (Array.isArray(content)) { + return JSON.parse(JSON.stringify(content)); + } + return content; }; export const State = Annotation.Root({ - messages: Annotation({ - reducer: (curr, prev) => [...curr, ...prev], - default: () => [], - }), - processedTweets: Annotation>({ - default: () => new Set(), - reducer: (curr, prev) => new Set([...curr, ...prev]), - }), - lastProcessedId: Annotation(), + messages: Annotation({ + reducer: (curr, prev) => [...curr, ...prev], + default: () => [], + }), + processedTweets: Annotation>({ + default: () => new Set(), + reducer: (curr, prev) => new Set([...curr, ...prev]), + }), + lastProcessedId: Annotation(), }); export type WorkflowConfig = Readonly<{ - client: ExtendedScraper; - toolNode: ToolNode; - llms: Readonly<{ - decision: ChatOpenAI; - tone: ChatOpenAI; - response: ChatOpenAI; - }>; + client: ExtendedScraper; + toolNode: ToolNode; + llms: Readonly<{ + decision: ChatOpenAI; + tone: ChatOpenAI; + response: ChatOpenAI; + }>; }>; const createWorkflowConfig = async (): Promise => { - const client = await createTwitterClientScraper(); - const { tools } = createTools(client); - - return { - client, - toolNode: new ToolNode(tools), - llms: { - decision: new ChatOpenAI({ - modelName: config.LLM_MODEL, - temperature: 0.2, - }) as unknown as ChatOpenAI, - - tone: new ChatOpenAI({ - modelName: config.LLM_MODEL, - temperature: 0.3, - }) as unknown as ChatOpenAI, - - response: new ChatOpenAI({ - modelName: config.LLM_MODEL, - temperature: 0.8, - }) as unknown as ChatOpenAI - } - }; + const client = await createTwitterClientScraper(); + const { tools } = createTools(client); + + return { + client, + toolNode: new ToolNode(tools), + llms: { + decision: new ChatOpenAI({ + modelName: config.LLM_MODEL, + temperature: 0.2, + }) as unknown as ChatOpenAI, + + tone: new ChatOpenAI({ + modelName: config.LLM_MODEL, + temperature: 0.3, + }) as unknown as ChatOpenAI, + + response: new ChatOpenAI({ + modelName: config.LLM_MODEL, + temperature: 0.8, + }) as unknown as ChatOpenAI, + }, + }; }; export const getWorkflowConfig = (() => { - let workflowConfigInstance: WorkflowConfig | null = null; - - return async (): Promise => { - if (!workflowConfigInstance) { - workflowConfigInstance = await createWorkflowConfig(); - } - return workflowConfigInstance; - }; -})(); - -const shouldContinue = (state: typeof State.State) => { - const lastMessage = state.messages[state.messages.length - 1]; - const content = parseMessageContent(lastMessage.content); - - logger.debug('Evaluating workflow continuation', { - hasMessages: state.messages.length > 0, - currentIndex: content.currentTweetIndex, - totalTweets: content.tweets?.length, - hasBatchToAnalyze: !!content.batchToAnalyze?.length, - hasBatchToRespond: !!content.batchToRespond?.length, - batchProcessing: content.batchProcessing - }); - - // Handle auto-approval flow - if (!content.fromAutoApproval && content.batchToFeedback?.length > 0) { - return 'autoApprovalNode'; - } + let workflowConfigInstance: WorkflowConfig | null = null; - if (content.fromAutoApproval) { - if (content.batchToRespond?.length > 0) { - return 'generateNode'; - } else { - return 'engagementNode'; - } - } - - // Handle batch processing flow - if (content.batchToAnalyze?.length > 0) { - return 'analyzeNode'; + return async (): Promise => { + if (!workflowConfigInstance) { + workflowConfigInstance = await createWorkflowConfig(); } + return workflowConfigInstance; + }; +})(); +const shouldContinue = (state: typeof State.State) => { + const lastMessage = state.messages[state.messages.length - 1]; + const content = parseMessageContent(lastMessage.content); + + logger.debug("Evaluating workflow continuation", { + hasMessages: state.messages.length > 0, + currentIndex: content.currentTweetIndex, + totalTweets: content.tweets?.length, + hasBatchToAnalyze: !!content.batchToAnalyze?.length, + hasBatchToRespond: !!content.batchToRespond?.length, + batchProcessing: content.batchProcessing, + }); + + // Handle auto-approval flow + if (!content.fromAutoApproval && content.batchToFeedback?.length > 0) { + return "autoApprovalNode"; + } + + if (content.fromAutoApproval) { if (content.batchToRespond?.length > 0) { - return 'generateNode'; + return "generateNode"; + } else { + return "engagementNode"; } - // Check if we've processed all tweets - if ((!content.tweets || content.currentTweetIndex >= content.tweets.length) && content.pendingEngagements?.length === 0) { - if (content.fromRecheckNode && content.messages?.length === 0) { - logger.info('Workflow complete - no more tweets to process'); - return END; - } - logger.info('Moving to recheck skipped tweets'); - return 'recheckNode'; + } + + // Handle batch processing flow + if (content.batchToAnalyze?.length > 0) { + return "analyzeNode"; + } + + if (content.batchToRespond?.length > 0) { + return "generateNode"; + } + // Check if we've processed all tweets + if ( + (!content.tweets || content.currentTweetIndex >= content.tweets.length) && + content.pendingEngagements?.length === 0 + ) { + if (content.fromRecheckNode && content.messages?.length === 0) { + logger.info("Workflow complete - no more tweets to process"); + return END; } - return 'engagementNode'; + logger.info("Moving to recheck skipped tweets"); + return "recheckNode"; + } + return "engagementNode"; }; // Workflow creation function -export const createWorkflow = async (nodes: Awaited>) => { - return new StateGraph(State) - .addNode('mentionNode', nodes.mentionNode) - .addNode('timelineNode', nodes.timelineNode) - .addNode('searchNode', nodes.searchNode) - .addNode('engagementNode', nodes.engagementNode) - .addNode('analyzeNode', nodes.toneAnalysisNode) - .addNode('generateNode', nodes.responseGenerationNode) - .addNode('autoApprovalNode', nodes.autoApprovalNode) - .addNode('recheckNode', nodes.recheckSkippedNode) - .addEdge(START, 'mentionNode') - .addEdge('mentionNode', 'timelineNode') - .addEdge('timelineNode', 'searchNode') - .addEdge('searchNode', 'engagementNode') - .addConditionalEdges('engagementNode', shouldContinue) - .addConditionalEdges('analyzeNode', shouldContinue) - .addConditionalEdges('generateNode', shouldContinue) - .addConditionalEdges('autoApprovalNode', shouldContinue) - .addConditionalEdges('recheckNode', shouldContinue); +export const createWorkflow = async ( + nodes: Awaited>, +) => { + return new StateGraph(State) + .addNode("mentionNode", nodes.mentionNode) + .addNode("timelineNode", nodes.timelineNode) + .addNode("searchNode", nodes.searchNode) + .addNode("engagementNode", nodes.engagementNode) + .addNode("analyzeNode", nodes.toneAnalysisNode) + .addNode("generateNode", nodes.responseGenerationNode) + .addNode("autoApprovalNode", nodes.autoApprovalNode) + .addNode("recheckNode", nodes.recheckSkippedNode) + .addEdge(START, "mentionNode") + .addEdge("mentionNode", "timelineNode") + .addEdge("timelineNode", "searchNode") + .addEdge("searchNode", "engagementNode") + .addConditionalEdges("engagementNode", shouldContinue) + .addConditionalEdges("analyzeNode", shouldContinue) + .addConditionalEdges("generateNode", shouldContinue) + .addConditionalEdges("autoApprovalNode", shouldContinue) + .addConditionalEdges("recheckNode", shouldContinue); }; // Workflow runner type type WorkflowRunner = Readonly<{ - runWorkflow: () => Promise; + runWorkflow: () => Promise; }>; // Create workflow runner const createWorkflowRunner = async (): Promise => { - const workflowConfig = await getWorkflowConfig(); - const nodes = await createNodes(workflowConfig); - const workflow = await createWorkflow(nodes); - const memoryStore = new MemorySaver(); - const app = workflow.compile({ checkpointer: memoryStore }); - - return { - runWorkflow: async () => { - const threadId = `workflow_${Date.now()}`; - logger.info('Starting tweet response workflow', { threadId }); - - const config = { - recursionLimit: 50, - configurable: { - thread_id: threadId - } - }; - - const stream = await app.stream({}, config); - let finalState = {}; - - for await (const state of stream) { - finalState = state; - } - - logger.info('Workflow completed', { threadId }); - return finalState; - } - }; + const workflowConfig = await getWorkflowConfig(); + const nodes = await createNodes(workflowConfig); + const workflow = await createWorkflow(nodes); + const memoryStore = new MemorySaver(); + const app = workflow.compile({ checkpointer: memoryStore }); + + return { + runWorkflow: async () => { + const threadId = `workflow_${Date.now()}`; + logger.info("Starting tweet response workflow", { threadId }); + + const config = { + recursionLimit: 50, + configurable: { + thread_id: threadId, + }, + }; + + const stream = await app.stream({}, config); + let finalState = {}; + + for await (const state of stream) { + finalState = state; + } + + logger.info("Workflow completed", { threadId }); + return finalState; + }, + }; }; export const getWorkflowRunner = (() => { - let runnerPromise: Promise | null = null; - - return () => { - if (!runnerPromise) { - runnerPromise = createWorkflowRunner(); - } - return runnerPromise; - }; + let runnerPromise: Promise | null = null; + + return () => { + if (!runnerPromise) { + runnerPromise = createWorkflowRunner(); + } + return runnerPromise; + }; })(); export const runWorkflow = async () => { - const runner = await getWorkflowRunner(); - return runner.runWorkflow(); -}; \ No newline at end of file + const runner = await getWorkflowRunner(); + return runner.runWorkflow(); +}; diff --git a/auto-kol/agent/src/services/database/index.ts b/auto-kol/agent/src/services/database/index.ts index b64dd14..5b7bf0e 100644 --- a/auto-kol/agent/src/services/database/index.ts +++ b/auto-kol/agent/src/services/database/index.ts @@ -1,226 +1,237 @@ -import { QueuedResponseMemory, ApprovalAction, SkippedTweetMemory, ActionResponse } from '../../types/queue.js'; -import { createLogger } from '../../utils/logger.js'; -import * as db from '../../database/index.js'; -import { Tweet } from '../../types/twitter.js'; -import { getPendingResponsesByTweetId } from '../../database/index.js'; -import { getTweetById } from '../../database/index.js'; - - -const logger = createLogger('database-queue'); - +import { + QueuedResponseMemory, + ApprovalAction, + SkippedTweetMemory, + ActionResponse, +} from "../../types/queue.js"; +import { createLogger } from "../../utils/logger.js"; +import * as db from "../../database/index.js"; +import { Tweet } from "../../types/twitter.js"; +import { getPendingResponsesByTweetId } from "../../database/index.js"; +import { getTweetById } from "../../database/index.js"; + +const logger = createLogger("database-queue"); ///////////RESPONSE/////////// -export async function addResponse(response: QueuedResponseMemory): Promise { +export async function addResponse( + response: QueuedResponseMemory, +): Promise { + try { try { - try { - await db.addTweet({ - id: response.tweet.id, - author_id: response.tweet.author_id, - author_username: response.tweet.author_username, - content: response.tweet.text, - created_at: response.tweet.created_at - }); - } catch (error) { - if (isUniqueConstraintError(error)) { - logger.warn(`Tweet already exists: ${response.tweet.id}`); - } else { - throw error; - } - } - await db.addResponse({ - id: response.id, - tweet_id: response.tweet.id, - content: response.response.content, - tone: response.workflowState.toneAnalysis?.suggestedTone || 'neutral', - strategy: response.workflowState.responseStrategy?.strategy || 'direct', - estimatedImpact: response.workflowState.responseStrategy?.estimatedImpact || 5, - confidence: response.workflowState.responseStrategy?.confidence || 0.5 - }); + await db.addTweet({ + id: response.tweet.id, + author_id: response.tweet.author_id, + author_username: response.tweet.author_username, + content: response.tweet.text, + created_at: response.tweet.created_at, + }); } catch (error) { - logger.error('Failed to add response to queue:', error); + if (isUniqueConstraintError(error)) { + logger.warn(`Tweet already exists: ${response.tweet.id}`); + } else { throw error; + } } + await db.addResponse({ + id: response.id, + tweet_id: response.tweet.id, + content: response.response.content, + tone: response.workflowState.toneAnalysis?.suggestedTone || "neutral", + strategy: response.workflowState.responseStrategy?.strategy || "direct", + estimatedImpact: + response.workflowState.responseStrategy?.estimatedImpact || 5, + confidence: response.workflowState.responseStrategy?.confidence || 0.5, + }); + } catch (error) { + logger.error("Failed to add response to queue:", error); + throw error; + } } export const updateResponseStatus = async ( - action: ApprovalAction + action: ApprovalAction, ): Promise => { - try { - const pendingResponse = await getPendingResponsesByTweetId(action.id); - const tweet = await getTweetById(pendingResponse.tweet_id); - await db.updateResponseStatus( - action.id, - action.approved ? 'approved' : 'rejected', - ); - logger.info(`Updated response status: ${action.id}`); - - return { - tweet: tweet as Tweet, - status: action.approved ? 'approved' : 'rejected', - response: pendingResponse as unknown as ActionResponse['response'], - } - } catch (error) { - logger.error('Failed to update response status:', error); - return undefined; - } + try { + const pendingResponse = await getPendingResponsesByTweetId(action.id); + const tweet = await getTweetById(pendingResponse.tweet_id); + await db.updateResponseStatus( + action.id, + action.approved ? "approved" : "rejected", + ); + logger.info(`Updated response status: ${action.id}`); + + return { + tweet: tweet as Tweet, + status: action.approved ? "approved" : "rejected", + response: pendingResponse as unknown as ActionResponse["response"], + }; + } catch (error) { + logger.error("Failed to update response status:", error); + return undefined; + } }; -export async function getAllPendingResponses(): Promise { - try { - const responses = await db.getPendingResponses(); - return responses.map(r => ({ - id: r.id, - tweet: { - id: r.tweet_id, - author_username: r.author_username, - text: r.tweet_content, - author_id: r.author_id, - created_at: r.created_at - }, - response: { - content: r.content - }, - status: r.status as 'pending' | 'approved' | 'rejected', - created_at: new Date(r.created_at), - updatedAt: new Date(r.updated_at), - workflowState: { - toneAnalysis: { - suggestedTone: r.tone - }, - engagementDecision: { - reason: r.strategy, - priority: r.estimated_impact - } - } as any - })); - } catch (error) { - logger.error('Failed to get pending responses:', error); - throw error; - } +export async function getAllPendingResponses(): Promise< + QueuedResponseMemory[] +> { + try { + const responses = await db.getPendingResponses(); + return responses.map((r) => ({ + id: r.id, + tweet: { + id: r.tweet_id, + author_username: r.author_username, + text: r.tweet_content, + author_id: r.author_id, + created_at: r.created_at, + }, + response: { + content: r.content, + }, + status: r.status as "pending" | "approved" | "rejected", + created_at: new Date(r.created_at), + updatedAt: new Date(r.updated_at), + workflowState: { + toneAnalysis: { + suggestedTone: r.tone, + }, + engagementDecision: { + reason: r.strategy, + priority: r.estimated_impact, + }, + } as any, + })); + } catch (error) { + logger.error("Failed to get pending responses:", error); + throw error; + } } ///////////SKIPPED TWEETS/////////// export async function addToSkipped(skipped: SkippedTweetMemory): Promise { + try { try { - try { - await db.addTweet({ - id: skipped.tweet.id, - author_id: skipped.tweet.author_id, - author_username: skipped.tweet.author_username, - content: skipped.tweet.text, - created_at: skipped.tweet.created_at - }); - } catch (error) { - if (isUniqueConstraintError(error)) { - logger.warn(`Tweet already exists: ${skipped.tweet.id}`); - } else { - throw error; - } - } - await db.addSkippedTweet({ - id: skipped.id, - tweetId: skipped.tweet.id, - reason: skipped.reason, - confidence: skipped.workflowState.decision?.confidence || 0 - }); - - - logger.info(`Added tweet to skipped: ${skipped.id}`); + await db.addTweet({ + id: skipped.tweet.id, + author_id: skipped.tweet.author_id, + author_username: skipped.tweet.author_username, + content: skipped.tweet.text, + created_at: skipped.tweet.created_at, + }); } catch (error) { - logger.error('Failed to add skipped tweet:', error); + if (isUniqueConstraintError(error)) { + logger.warn(`Tweet already exists: ${skipped.tweet.id}`); + } else { throw error; + } } + await db.addSkippedTweet({ + id: skipped.id, + tweetId: skipped.tweet.id, + reason: skipped.reason, + confidence: skipped.workflowState.decision?.confidence || 0, + }); + + logger.info(`Added tweet to skipped: ${skipped.id}`); + } catch (error) { + logger.error("Failed to add skipped tweet:", error); + throw error; + } } export async function getSkippedTweets(): Promise { - const skipped = await db.getSkippedTweets(); - return skipped; + const skipped = await db.getSkippedTweets(); + return skipped; } -export async function getSkippedTweetById(skippedId: string): Promise { - const skipped = await db.getSkippedTweetById(skippedId); - const tweet = await getTweetById(skipped.tweet_id); - - if (!skipped || !tweet) { - throw new Error('Skipped tweet or original tweet not found'); - } - const result: SkippedTweetMemory = { - id: skipped.id, - tweet: { - id: tweet.id, - text: tweet.text, - author_id: tweet.author_id, - author_username: tweet.author_username, - created_at: tweet.created_at - }, +export async function getSkippedTweetById( + skippedId: string, +): Promise { + const skipped = await db.getSkippedTweetById(skippedId); + const tweet = await getTweetById(skipped.tweet_id); + + if (!skipped || !tweet) { + throw new Error("Skipped tweet or original tweet not found"); + } + const result: SkippedTweetMemory = { + id: skipped.id, + tweet: { + id: tweet.id, + text: tweet.text, + author_id: tweet.author_id, + author_username: tweet.author_username, + created_at: tweet.created_at, + }, + reason: skipped.reason, + priority: skipped.priority, + created_at: new Date(), + workflowState: { + tweet: tweet, + messages: [], + previousInteractions: [], + engagementDecision: { + shouldEngage: false, reason: skipped.reason, priority: skipped.priority, - created_at: new Date(), - workflowState: { - tweet: tweet, - messages: [], - previousInteractions: [], - engagementDecision: { - shouldEngage: false, - reason: skipped.reason, - priority: skipped.priority - } - } - }; + }, + }, + }; - return result; + return result; } export const moveSkippedToQueue = async ( - skippedId: string, - queuedResponse: Omit & { status: 'pending' } + skippedId: string, + queuedResponse: Omit & { status: "pending" }, ): Promise => { - try { - const skipped = await getSkippedTweetById(skippedId) as any; - logger.info(`Skipped tweet: ${JSON.stringify(skipped)}`); - - if (!skipped) { - throw new Error('Skipped tweet not found'); - } - const tweet = await db.getTweetById(skipped.tweet_id); - if (!tweet) { - throw new Error('Tweet not found'); - } - logger.info(`Tweet: ${JSON.stringify(tweet)}`); - - const typedResponse: QueuedResponseMemory = { - id: queuedResponse.id, - tweet: { - id: tweet.id, - text: tweet.text, - author_id: tweet.author_id, - author_username: tweet.author_username, - created_at: tweet.created_at - }, - response: queuedResponse.response, - status: 'pending', - created_at: new Date(), - updatedAt: new Date(), - workflowState: queuedResponse.workflowState - }; - - logger.info(`Adding to queue: ${JSON.stringify(typedResponse)}`); - await addResponse(typedResponse); - logger.info(`Moved skipped tweet ${skippedId} to response queue`); - return typedResponse; - } catch (error) { - logger.error('Failed to move skipped tweet to queue:', error); - throw error; + try { + const skipped = (await getSkippedTweetById(skippedId)) as any; + logger.info(`Skipped tweet: ${JSON.stringify(skipped)}`); + + if (!skipped) { + throw new Error("Skipped tweet not found"); + } + const tweet = await db.getTweetById(skipped.tweet_id); + if (!tweet) { + throw new Error("Tweet not found"); } + logger.info(`Tweet: ${JSON.stringify(tweet)}`); + + const typedResponse: QueuedResponseMemory = { + id: queuedResponse.id, + tweet: { + id: tweet.id, + text: tweet.text, + author_id: tweet.author_id, + author_username: tweet.author_username, + created_at: tweet.created_at, + }, + response: queuedResponse.response, + status: "pending", + created_at: new Date(), + updatedAt: new Date(), + workflowState: queuedResponse.workflowState, + }; + + logger.info(`Adding to queue: ${JSON.stringify(typedResponse)}`); + await addResponse(typedResponse); + logger.info(`Moved skipped tweet ${skippedId} to response queue`); + return typedResponse; + } catch (error) { + logger.error("Failed to move skipped tweet to queue:", error); + throw error; + } }; //////////UTILS////////// const isUniqueConstraintError = (error: any): boolean => { - return error?.code === 'SQLITE_CONSTRAINT' && - error?.message?.includes('UNIQUE constraint failed'); + return ( + error?.code === "SQLITE_CONSTRAINT" && + error?.message?.includes("UNIQUE constraint failed") + ); }; export async function isTweetExists(tweetId: string): Promise { - const tweet = await db.getTweetById(tweetId); - return tweet !== undefined; -} \ No newline at end of file + const tweet = await db.getTweetById(tweetId); + return tweet !== undefined; +} diff --git a/auto-kol/agent/src/services/twitter/api.ts b/auto-kol/agent/src/services/twitter/api.ts index a591fdc..58831ef 100644 --- a/auto-kol/agent/src/services/twitter/api.ts +++ b/auto-kol/agent/src/services/twitter/api.ts @@ -1,9 +1,9 @@ -import { Scraper, SearchMode, Tweet } from 'agent-twitter-client'; -import { createLogger } from '../../utils/logger.js'; -import { readFileSync, writeFileSync, existsSync } from 'fs'; -import { config } from '../../config/index.js'; +import { Scraper, SearchMode, Tweet } from "agent-twitter-client"; +import { createLogger } from "../../utils/logger.js"; +import { readFileSync, writeFileSync, existsSync } from "fs"; +import { config } from "../../config/index.js"; -const logger = createLogger('agent-twitter-api'); +const logger = createLogger("agent-twitter-api"); export class ExtendedScraper extends Scraper { private static instance: ExtendedScraper | null = null; @@ -24,27 +24,28 @@ export class ExtendedScraper extends Scraper { private async initialize() { const username = config.TWITTER_USERNAME!; const password = config.TWITTER_PASSWORD!; - const cookiesPath = 'cookies.json'; + const cookiesPath = "cookies.json"; if (existsSync(cookiesPath)) { - logger.info('Loading existing cookies'); - const cookies = readFileSync(cookiesPath, 'utf8'); + logger.info("Loading existing cookies"); + const cookies = readFileSync(cookiesPath, "utf8"); try { const parsedCookies = JSON.parse(cookies).map( - (cookie: any) => `${cookie.key}=${cookie.value}; Domain=${cookie.domain}; Path=${cookie.path}` + (cookie: any) => + `${cookie.key}=${cookie.value}; Domain=${cookie.domain}; Path=${cookie.path}`, ); await this.setCookies(parsedCookies); - logger.info('Loaded existing cookies from file'); + logger.info("Loaded existing cookies from file"); } catch (error) { - logger.error('Error loading cookies:', error); + logger.error("Error loading cookies:", error); } } else { - logger.info('No existing cookies found, proceeding with login'); + logger.info("No existing cookies found, proceeding with login"); await this.login(username, password); const newCookies = await this.getCookies(); writeFileSync(cookiesPath, JSON.stringify(newCookies, null, 2)); - logger.info('New cookies saved to file'); + logger.info("New cookies saved to file"); } const isLoggedIn = await this.isLoggedIn(); @@ -56,26 +57,28 @@ export class ExtendedScraper extends Scraper { let retryCount = 0; while (!isLoggedIn && retryCount < maxRetries) { - logger.warn(`Session expired, attempting to re-authenticate... (attempt ${retryCount + 1}/${maxRetries})`); + logger.warn( + `Session expired, attempting to re-authenticate... (attempt ${retryCount + 1}/${maxRetries})`, + ); try { await this.initialize(); isLoggedIn = await this.isLoggedIn(); if (isLoggedIn) { - logger.info('Successfully re-authenticated'); + logger.info("Successfully re-authenticated"); return true; } - logger.error('Re-authentication failed'); + logger.error("Re-authentication failed"); retryCount++; if (retryCount < maxRetries) { const delay = 2000 * Math.pow(2, retryCount - 1); - await new Promise(resolve => setTimeout(resolve, delay)); + await new Promise((resolve) => setTimeout(resolve, delay)); } } catch (error) { - logger.error('Error during re-authentication:', error); + logger.error("Error during re-authentication:", error); retryCount++; if (retryCount < maxRetries) { const delay = 2000 * Math.pow(2, retryCount - 1); - await new Promise(resolve => setTimeout(resolve, delay)); + await new Promise((resolve) => setTimeout(resolve, delay)); } } } @@ -89,16 +92,20 @@ export class ExtendedScraper extends Scraper { const isLoggedIn = await this.isLoggedIn(); if (!isLoggedIn) { - throw new Error('Must be logged in to fetch mentions'); + throw new Error("Must be logged in to fetch mentions"); } const query = `@${username} -from:${username}`; const replies: Tweet[] = []; - const searchIterator = this.searchTweets(query, maxResults, SearchMode.Latest); + const searchIterator = this.searchTweets( + query, + maxResults, + SearchMode.Latest, + ); for await (const tweet of searchIterator) { - logger.info('Checking tweet:', { + logger.info("Checking tweet:", { id: tweet.id, text: tweet.text, author: tweet.username, @@ -108,13 +115,19 @@ export class ExtendedScraper extends Scraper { break; } - const hasReplies = this.searchTweets(`from:${username} to:${tweet.username}`, 10, SearchMode.Latest); + const hasReplies = this.searchTweets( + `from:${username} to:${tweet.username}`, + 10, + SearchMode.Latest, + ); let alreadyReplied = false; for await (const reply of hasReplies) { if (reply.inReplyToStatusId === tweet.id) { alreadyReplied = true; - logger.info(`Skipping tweet ${tweet.id} - already replied with ${reply.id}`); + logger.info( + `Skipping tweet ${tweet.id} - already replied with ${reply.id}`, + ); break; } } @@ -134,7 +147,7 @@ export class ExtendedScraper extends Scraper { public async getThread(tweetId: string): Promise { const isLoggedIn = await this.isLoggedIn(); if (!isLoggedIn) { - throw new Error('Must be logged in to fetch thread'); + throw new Error("Must be logged in to fetch thread"); } const initialTweet = await this.getTweet(tweetId); @@ -149,7 +162,7 @@ export class ExtendedScraper extends Scraper { const cachedConversation = this.conversationCache.get(conversationId!); if (cachedConversation) { logger.info( - `Returning cached conversation with ${cachedConversation.length} tweets for conversation_id:${conversationId}` + `Returning cached conversation with ${cachedConversation.length} tweets for conversation_id:${conversationId}`, ); return cachedConversation; } @@ -158,12 +171,15 @@ export class ExtendedScraper extends Scraper { let rootTweet = initialTweet; // If the conversation root differs - if (initialTweet.conversationId && initialTweet.conversationId !== initialTweet.id) { + if ( + initialTweet.conversationId && + initialTweet.conversationId !== initialTweet.id + ) { const conversationRoot = await this.getTweet(initialTweet.conversationId); if (conversationRoot) { rootTweet = conversationRoot; conversationTweets.set(rootTweet.id!, rootTweet); - logger.info('Found conversation root tweet:', { + logger.info("Found conversation root tweet:", { id: rootTweet.id, conversationId: rootTweet.conversationId, }); @@ -173,9 +189,16 @@ export class ExtendedScraper extends Scraper { } try { - logger.info('Fetching entire conversation via `conversation_id`:', conversationId); + logger.info( + "Fetching entire conversation via `conversation_id`:", + conversationId, + ); - const conversationIterator = this.searchTweets(`conversation_id:${conversationId}`, 100, SearchMode.Latest); + const conversationIterator = this.searchTweets( + `conversation_id:${conversationId}`, + 100, + SearchMode.Latest, + ); for await (const tweet of conversationIterator) { conversationTweets.set(tweet.id!, tweet); } @@ -192,7 +215,9 @@ export class ExtendedScraper extends Scraper { return timeA - timeB; }); - logger.info(`Retrieved conversation thread with ${thread.length} tweets for conversation_id:${conversationId}`); + logger.info( + `Retrieved conversation thread with ${thread.length} tweets for conversation_id:${conversationId}`, + ); // Save to cache this.conversationCache.set(conversationId!, thread); @@ -201,13 +226,16 @@ export class ExtendedScraper extends Scraper { } // Placeholder for efficient thread fetching - async getThreadPlaceHolder(tweetId: string, maxDepth: number = 100): Promise { + async getThreadPlaceHolder( + tweetId: string, + maxDepth: number = 100, + ): Promise { const username = config.TWITTER_USERNAME!; const isLoggedIn = await this.isLoggedIn(); if (!isLoggedIn) { const reAuthenticate = await this.reAuthenticate(); if (!reAuthenticate) { - logger.error('Failed to re-authenticate'); + logger.error("Failed to re-authenticate"); return []; } } @@ -226,18 +254,23 @@ export class ExtendedScraper extends Scraper { let rootTweet = initialTweet; const conversationId = initialTweet.conversationId || initialTweet.id; - logger.info('Initial tweet:', { + logger.info("Initial tweet:", { id: initialTweet.id, conversationId: conversationId, inReplyToStatusId: initialTweet.inReplyToStatusId, }); - if (initialTweet.conversationId && initialTweet.conversationId !== initialTweet.id) { - const conversationRoot = await this.getTweet(initialTweet.conversationId); + if ( + initialTweet.conversationId && + initialTweet.conversationId !== initialTweet.id + ) { + const conversationRoot = await this.getTweet( + initialTweet.conversationId, + ); if (conversationRoot) { rootTweet = conversationRoot; conversationTweets.set(rootTweet.id!, rootTweet); - logger.info('Found root tweet:', { + logger.info("Found root tweet:", { id: rootTweet.id, conversationId: rootTweet.conversationId, }); @@ -245,19 +278,29 @@ export class ExtendedScraper extends Scraper { } try { - logger.info('Fetching conversation with query:', `conversation_id:${conversationId}`); - const conversationIterator = this.searchTweets(`conversation_id:${conversationId}`, 100, SearchMode.Latest); + logger.info( + "Fetching conversation with query:", + `conversation_id:${conversationId}`, + ); + const conversationIterator = this.searchTweets( + `conversation_id:${conversationId}`, + 100, + SearchMode.Latest, + ); for await (const tweet of conversationIterator) { conversationTweets.set(tweet.id!, tweet); - logger.info('Found conversation tweet:', { + logger.info("Found conversation tweet:", { id: tweet.id, inReplyToStatusId: tweet.inReplyToStatusId, - text: tweet.text?.substring(0, 50) + '...', + text: tweet.text?.substring(0, 50) + "...", }); } - logger.info('Total conversation tweets found:', conversationTweets.size); + logger.info( + "Total conversation tweets found:", + conversationTweets.size, + ); } catch (error) { logger.warn(`Error fetching conversation: ${error}`); return [rootTweet, initialTweet]; diff --git a/auto-kol/agent/src/services/vectorstore/chroma.ts b/auto-kol/agent/src/services/vectorstore/chroma.ts index 24b8877..42f93d8 100644 --- a/auto-kol/agent/src/services/vectorstore/chroma.ts +++ b/auto-kol/agent/src/services/vectorstore/chroma.ts @@ -5,123 +5,127 @@ import { OpenAIEmbeddings } from "@langchain/openai"; import { createLogger } from "../../utils/logger.js"; import { Tweet } from "../../types/twitter.js"; import { config } from "../../config/index.js"; -import { isTweetExists } from '../../services/database/index.js'; +import { isTweetExists } from "../../services/database/index.js"; - -const logger = createLogger('chroma-service'); +const logger = createLogger("chroma-service"); export class ChromaService { - private static instance: ChromaService; - private client: ChromaClient; - private embeddings: OpenAIEmbeddings; - private collection!: Chroma; + private static instance: ChromaService; + private client: ChromaClient; + private embeddings: OpenAIEmbeddings; + private collection!: Chroma; - private constructor() { - this.client = new ChromaClient({ - path: config.CHROMA_URL - }); - this.embeddings = new OpenAIEmbeddings({ - openAIApiKey: config.OPENAI_API_KEY, - modelName: "text-embedding-ada-002", - }); - this.initializeCollection(); - } + private constructor() { + this.client = new ChromaClient({ + path: config.CHROMA_URL, + }); + this.embeddings = new OpenAIEmbeddings({ + openAIApiKey: config.OPENAI_API_KEY, + modelName: "text-embedding-ada-002", + }); + this.initializeCollection(); + } - private async initializeCollection() { - try { - this.collection = await Chroma.fromExistingCollection( - this.embeddings, - { - collectionName: "tweets", - url: config.CHROMA_URL, - collectionMetadata: { - "hnsw:space": "cosine" - }, - } - ); - logger.info('Chroma collection initialized'); - } catch (error) { - logger.info('Collection does not exist, creating new one'); - this.collection = await Chroma.fromTexts( - [], // No initial texts - [], // No initial metadatas - this.embeddings, - { - collectionName: "tweets", - url: config.CHROMA_URL, - collectionMetadata: { - "hnsw:space": "cosine" - }, - } - ); - } + private async initializeCollection() { + try { + this.collection = await Chroma.fromExistingCollection(this.embeddings, { + collectionName: "tweets", + url: config.CHROMA_URL, + collectionMetadata: { + "hnsw:space": "cosine", + }, + }); + logger.info("Chroma collection initialized"); + } catch (error) { + logger.info("Collection does not exist, creating new one"); + this.collection = await Chroma.fromTexts( + [], // No initial texts + [], // No initial metadatas + this.embeddings, + { + collectionName: "tweets", + url: config.CHROMA_URL, + collectionMetadata: { + "hnsw:space": "cosine", + }, + }, + ); } + } - public static async getInstance(): Promise { - if (!ChromaService.instance) { - ChromaService.instance = new ChromaService(); - await ChromaService.instance.initializeCollection(); - } - return ChromaService.instance; + public static async getInstance(): Promise { + if (!ChromaService.instance) { + ChromaService.instance = new ChromaService(); + await ChromaService.instance.initializeCollection(); } + return ChromaService.instance; + } - public async addTweet(tweet: Tweet) { - try { - if (await isTweetExists(tweet.id)) { - logger.info(`Tweet ${tweet.id} already exists in vector store, skipping`); - return; - } + public async addTweet(tweet: Tweet) { + try { + if (await isTweetExists(tweet.id)) { + logger.info( + `Tweet ${tweet.id} already exists in vector store, skipping`, + ); + return; + } - const doc = new Document({ - pageContent: tweet.text, - metadata: { - tweetId: tweet.id, - author_id: tweet.author_id, - author_username: tweet.author_username, - created_at: tweet.created_at - } - }); + const doc = new Document({ + pageContent: tweet.text, + metadata: { + tweetId: tweet.id, + author_id: tweet.author_id, + author_username: tweet.author_username, + created_at: tweet.created_at, + }, + }); - await this.collection.addDocuments([doc]); - logger.info(`Added tweet ${tweet.id} to vector store`); - } catch (error) { - logger.error(`Failed to add tweet ${tweet.id} to vector store:`, error); - throw error; - } + await this.collection.addDocuments([doc]); + logger.info(`Added tweet ${tweet.id} to vector store`); + } catch (error) { + logger.error(`Failed to add tweet ${tweet.id} to vector store:`, error); + throw error; } + } - public async searchSimilarTweets(query: string, k: number = 5) { - try { - const results = await this.collection.similaritySearch(query, k); - return results; - } catch (error) { - logger.error('Failed to search similar tweets:', error); - throw error; - } + public async searchSimilarTweets(query: string, k: number = 5) { + try { + const results = await this.collection.similaritySearch(query, k); + return results; + } catch (error) { + logger.error("Failed to search similar tweets:", error); + throw error; } + } - public async searchSimilarTweetsWithScore(query: string, k: number = 5) { - try { - const queryEmbedding = await this.embeddings.embedQuery(query); - const results = await this.collection.similaritySearchVectorWithScore(queryEmbedding, k); - return results; - } catch (error) { - logger.error('Failed to search similar tweets with scores:', error); - throw error; - } + public async searchSimilarTweetsWithScore(query: string, k: number = 5) { + try { + const queryEmbedding = await this.embeddings.embedQuery(query); + const results = await this.collection.similaritySearchVectorWithScore( + queryEmbedding, + k, + ); + return results; + } catch (error) { + logger.error("Failed to search similar tweets with scores:", error); + throw error; } + } - public async deleteTweet(tweetId: string) { - try { - await this.collection.delete({ - filter: { - tweetId: tweetId - } - }); - logger.info(`Deleted tweet ${tweetId} from vector store`); - } catch (error) { - logger.error(`Failed to delete tweet ${tweetId} from vector store:`, error); - throw error; - } + public async deleteTweet(tweetId: string) { + try { + await this.collection.delete({ + filter: { + tweetId: tweetId, + }, + }); + logger.info(`Deleted tweet ${tweetId} from vector store`); + } catch (error) { + logger.error( + `Failed to delete tweet ${tweetId} from vector store:`, + error, + ); + throw error; } + } } diff --git a/auto-kol/agent/src/tools/index.ts b/auto-kol/agent/src/tools/index.ts index a0a8f43..dce61c6 100644 --- a/auto-kol/agent/src/tools/index.ts +++ b/auto-kol/agent/src/tools/index.ts @@ -1,11 +1,11 @@ -import { createFetchTimelineTool } from './tools/fetchTimelineTool.js'; -import { createTweetSearchTool } from './tools/tweetSearchTool.js'; -import { createAddResponseTool } from './tools/queueResponseTool.js'; -import { createUpdateResponseTool } from './tools/queueResponseTool.js'; -import { createQueueSkippedTool } from './tools/queueSkippedTool.js'; -import { createSearchSimilarTweetsTool } from './tools/searchSimilarTweetsTool.js'; -import { createMentionTool } from './tools/mentionTool.js'; -import { ExtendedScraper } from '../services/twitter/api.js'; +import { createFetchTimelineTool } from "./tools/fetchTimelineTool.js"; +import { createTweetSearchTool } from "./tools/tweetSearchTool.js"; +import { createAddResponseTool } from "./tools/queueResponseTool.js"; +import { createUpdateResponseTool } from "./tools/queueResponseTool.js"; +import { createQueueSkippedTool } from "./tools/queueSkippedTool.js"; +import { createSearchSimilarTweetsTool } from "./tools/searchSimilarTweetsTool.js"; +import { createMentionTool } from "./tools/mentionTool.js"; +import { ExtendedScraper } from "../services/twitter/api.js"; export const createTools = (scraper: ExtendedScraper) => { const mentionTool = createMentionTool(scraper); diff --git a/auto-kol/agent/src/tools/tools/fetchTimelineTool.ts b/auto-kol/agent/src/tools/tools/fetchTimelineTool.ts index 1a7a46b..f017e4b 100644 --- a/auto-kol/agent/src/tools/tools/fetchTimelineTool.ts +++ b/auto-kol/agent/src/tools/tools/fetchTimelineTool.ts @@ -1,26 +1,29 @@ -import { DynamicStructuredTool } from '@langchain/core/tools'; -import { z } from 'zod'; -import { createLogger } from '../../utils/logger.js'; -import { getTimeLine } from '../../utils/twitter.js'; -import { ExtendedScraper } from '../../services/twitter/api.js'; +import { DynamicStructuredTool } from "@langchain/core/tools"; +import { z } from "zod"; +import { createLogger } from "../../utils/logger.js"; +import { getTimeLine } from "../../utils/twitter.js"; +import { ExtendedScraper } from "../../services/twitter/api.js"; -const logger = createLogger('fetch-timeline-tool'); +const logger = createLogger("fetch-timeline-tool"); export const createFetchTimelineTool = (twitterScraper: ExtendedScraper) => new DynamicStructuredTool({ - name: 'fetch_timeline', - description: 'Fetch the timeline regularly to get new tweets', + name: "fetch_timeline", + description: "Fetch the timeline regularly to get new tweets", schema: z.object({}), func: async () => { try { const tweets = await getTimeLine(twitterScraper); - tweets.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime()); + tweets.sort( + (a, b) => + new Date(b.created_at).getTime() - new Date(a.created_at).getTime(), + ); return { tweets: tweets, lastProcessedId: tweets[tweets.length - 1]?.id || null, }; } catch (error) { - logger.error('Error in fetchTimelineTool:', error); + logger.error("Error in fetchTimelineTool:", error); return { tweets: [], lastProcessedId: null, diff --git a/auto-kol/agent/src/tools/tools/mentionTool.ts b/auto-kol/agent/src/tools/tools/mentionTool.ts index 8d570bf..912ad57 100644 --- a/auto-kol/agent/src/tools/tools/mentionTool.ts +++ b/auto-kol/agent/src/tools/tools/mentionTool.ts @@ -1,68 +1,71 @@ -import { DynamicStructuredTool } from '@langchain/core/tools'; -import { z } from 'zod'; -import { createLogger } from '../../utils/logger.js'; -import { addMention, getLatestMentionId } from '../../database/index.js'; -import { ExtendedScraper } from '../../services/twitter/api.js'; -import { Tweet } from '../../types/twitter.js'; -const logger = createLogger('mention-tool'); +import { DynamicStructuredTool } from "@langchain/core/tools"; +import { z } from "zod"; +import { createLogger } from "../../utils/logger.js"; +import { addMention, getLatestMentionId } from "../../database/index.js"; +import { ExtendedScraper } from "../../services/twitter/api.js"; +import { Tweet } from "../../types/twitter.js"; +const logger = createLogger("mention-tool"); -export const createMentionTool = (scraper: ExtendedScraper) => new DynamicStructuredTool({ - name: 'fetch_mentions', - description: 'Fetch mentions since the last processed mention', +export const createMentionTool = (scraper: ExtendedScraper) => + new DynamicStructuredTool({ + name: "fetch_mentions", + description: "Fetch mentions since the last processed mention", schema: z.object({}), func: async () => { - try { - const sinceId = await getLatestMentionId(); - const mentions = await scraper.getMyMentions(100, sinceId); - if (!mentions || mentions.length === 0) { - logger.info('No new mentions found'); - return { - tweets: [], - lastProcessedId: sinceId - }; - } + try { + const sinceId = await getLatestMentionId(); + const mentions = await scraper.getMyMentions(100, sinceId); + if (!mentions || mentions.length === 0) { + logger.info("No new mentions found"); + return { + tweets: [], + lastProcessedId: sinceId, + }; + } + + const tweets = mentions.map((mention: any) => ({ + id: mention.id!, + text: mention.text!, + author_id: mention.userId!, + author_username: mention.username!.toLowerCase(), + created_at: mention.timeParsed!.toISOString(), + thread: [] as Tweet[], + })); - const tweets = mentions.map((mention: any) => ({ - id: mention.id!, - text: mention.text!, - author_id: mention.userId!, - author_username: mention.username!.toLowerCase(), - created_at: mention.timeParsed!.toISOString(), - thread: [] as Tweet[] - })); + await addMention({ + latest_id: mentions[0].id!, + }); - await addMention({ - latest_id: mentions[0].id! + logger.info(`Fetched ${tweets.length} new mentions`); + for (const tweet of tweets) { + logger.info(`Getting thread for tweet ${tweet.id}`); + const tweetsWithThreads: Tweet[] = []; + const thread = await scraper.getThread(tweet.id); + for await (const threadTweet of thread) { + tweetsWithThreads.push({ + id: threadTweet.id || "", + text: threadTweet.text || "", + author_id: threadTweet.userId || "", + author_username: threadTweet.username?.toLowerCase() || "unknown", + created_at: + threadTweet.timeParsed?.toISOString() || + new Date().toISOString(), }); - - logger.info(`Fetched ${tweets.length} new mentions`); - for (const tweet of tweets) { - logger.info(`Getting thread for tweet ${tweet.id}`); - const tweetsWithThreads: Tweet[] = []; - const thread = await scraper.getThread(tweet.id); - for await (const threadTweet of thread) { - tweetsWithThreads.push({ - id: threadTweet.id || '', - text: threadTweet.text || '', - author_id: threadTweet.userId || '', - author_username: threadTweet.username?.toLowerCase() || 'unknown', - created_at: threadTweet.timeParsed?.toISOString() || new Date().toISOString() - }); - } - tweet.thread = tweetsWithThreads; - await new Promise(resolve => setTimeout(resolve, 1000)); - logger.info(`Found ${tweetsWithThreads.length} tweets in thread`); - } - return { - tweets: tweets, - lastProcessedId: mentions[0].id! - }; - } catch (error) { - logger.error('Error in mentionTool:', error); - return { - tweets: [], - lastProcessedId: null - }; + } + tweet.thread = tweetsWithThreads; + await new Promise((resolve) => setTimeout(resolve, 1000)); + logger.info(`Found ${tweetsWithThreads.length} tweets in thread`); } - } -}); + return { + tweets: tweets, + lastProcessedId: mentions[0].id!, + }; + } catch (error) { + logger.error("Error in mentionTool:", error); + return { + tweets: [], + lastProcessedId: null, + }; + } + }, + }); diff --git a/auto-kol/agent/src/tools/tools/queueResponseTool.ts b/auto-kol/agent/src/tools/tools/queueResponseTool.ts index e36653d..194fb7b 100644 --- a/auto-kol/agent/src/tools/tools/queueResponseTool.ts +++ b/auto-kol/agent/src/tools/tools/queueResponseTool.ts @@ -1,87 +1,86 @@ -import { DynamicStructuredTool } from '@langchain/core/tools'; -import { createLogger } from '../../utils/logger.js'; -import { v4 as generateId } from 'uuid'; -import { queueActionSchema } from '../../schemas/workflow.js'; -import { addResponse } from '../../services/database/index.js'; -import { getResponseByTweetId, updateResponse } from '../../database/index.js'; -import { PendingResponse, QueuedResponseMemory } from '../../types/queue.js'; -import { Tweet } from '../../types/twitter.js'; -import { AgentResponse } from '../../types/agent.js'; -import { WorkflowState } from '../../types/workflow.js'; +import { DynamicStructuredTool } from "@langchain/core/tools"; +import { createLogger } from "../../utils/logger.js"; +import { v4 as generateId } from "uuid"; +import { queueActionSchema } from "../../schemas/workflow.js"; +import { addResponse } from "../../services/database/index.js"; +import { getResponseByTweetId, updateResponse } from "../../database/index.js"; +import { PendingResponse, QueuedResponseMemory } from "../../types/queue.js"; +import { Tweet } from "../../types/twitter.js"; +import { AgentResponse } from "../../types/agent.js"; +import { WorkflowState } from "../../types/workflow.js"; +const logger = createLogger("queue-response-tool"); -const logger = createLogger('queue-response-tool'); - -export const createAddResponseTool = () => new DynamicStructuredTool({ - name: 'add_response', - description: 'Add or update a response in the approval queue', +export const createAddResponseTool = () => + new DynamicStructuredTool({ + name: "add_response", + description: "Add or update a response in the approval queue", schema: queueActionSchema, func: async (input: any) => { - const id = generateId(); - const response: QueuedResponseMemory = { - id, - tweet: input.tweet, - response: { - content: input.workflowState?.responseStrategy?.content, - }, - status: 'pending' as const, - created_at: new Date(), - updatedAt: new Date(), - workflowState: input.workflowState - }; - - await addResponse(response); - return { - success: true, - id, - type: 'response' as const, - message: 'Response queued successfully' - }; - - } -}); + const id = generateId(); + const response: QueuedResponseMemory = { + id, + tweet: input.tweet, + response: { + content: input.workflowState?.responseStrategy?.content, + }, + status: "pending" as const, + created_at: new Date(), + updatedAt: new Date(), + workflowState: input.workflowState, + }; + await addResponse(response); + return { + success: true, + id, + type: "response" as const, + message: "Response queued successfully", + }; + }, + }); -export const createUpdateResponseTool = () => new DynamicStructuredTool({ - name: 'update_response', - description: 'Update a response in the approval queue', +export const createUpdateResponseTool = () => + new DynamicStructuredTool({ + name: "update_response", + description: "Update a response in the approval queue", schema: queueActionSchema, func: async (input: any) => { - try { - logger.info('Updating response', { - tweet_id: input.tweet.id - }); - const existingResponse = await getResponseByTweetId(input.tweet.id); - - if (!existingResponse) { - logger.error('Could not find existing response to update:', { - tweet_id: input.tweet.id - }); - throw new Error('Could not find existing response to update'); - } + try { + logger.info("Updating response", { + tweet_id: input.tweet.id, + }); + const existingResponse = await getResponseByTweetId(input.tweet.id); - await updateResponse({ - id: existingResponse.id, - tweet_id: input.tweet.id, - content: input.workflowState.responseStrategy.content, - tone: input.workflowState.responseStrategy.tone, - strategy: input.workflowState.responseStrategy.strategy, - estimatedImpact: input.workflowState.responseStrategy.estimatedImpact, - confidence: input.workflowState.responseStrategy.confidence, - } as PendingResponse); - - logger.info('Response updated successfully', { - response_id: existingResponse.id - }); - return { - success: true, - id: existingResponse.id, - type: 'response' as const, - message: 'Response updated successfully' - }; - } catch (error) { - logger.error('Error in update response tool:', error); - throw error; + if (!existingResponse) { + logger.error("Could not find existing response to update:", { + tweet_id: input.tweet.id, + }); + throw new Error("Could not find existing response to update"); } - } -}); \ No newline at end of file + + await updateResponse({ + id: existingResponse.id, + tweet_id: input.tweet.id, + content: input.workflowState.responseStrategy.content, + tone: input.workflowState.responseStrategy.tone, + strategy: input.workflowState.responseStrategy.strategy, + estimatedImpact: input.workflowState.responseStrategy.estimatedImpact, + confidence: input.workflowState.responseStrategy.confidence, + } as PendingResponse); + + logger.info("Response updated successfully", { + response_id: existingResponse.id, + }); + return { + success: true, + id: existingResponse.id, + type: "response" as const, + message: "Response updated successfully", + }; + } catch (error) { + logger.error("Error in update response tool:", error); + throw error; + } + }, + }); diff --git a/auto-kol/agent/src/tools/tools/queueSkippedTool.ts b/auto-kol/agent/src/tools/tools/queueSkippedTool.ts index b9649d2..552abc2 100644 --- a/auto-kol/agent/src/tools/tools/queueSkippedTool.ts +++ b/auto-kol/agent/src/tools/tools/queueSkippedTool.ts @@ -1,45 +1,45 @@ -import { DynamicStructuredTool } from '@langchain/core/tools'; -import { createLogger } from '../../utils/logger.js'; -import { v4 as generateId } from 'uuid'; -import { queueActionSchema } from '../../schemas/workflow.js'; -import { addToSkipped } from '../../services/database/index.js'; +import { DynamicStructuredTool } from "@langchain/core/tools"; +import { createLogger } from "../../utils/logger.js"; +import { v4 as generateId } from "uuid"; +import { queueActionSchema } from "../../schemas/workflow.js"; +import { addToSkipped } from "../../services/database/index.js"; -const logger = createLogger('queue-skipped-tool'); +const logger = createLogger("queue-skipped-tool"); - -export const createQueueSkippedTool = () => new DynamicStructuredTool({ - name: 'queue_skipped', - description: 'Add a skipped tweet to the review queue', +export const createQueueSkippedTool = () => + new DynamicStructuredTool({ + name: "queue_skipped", + description: "Add a skipped tweet to the review queue", schema: queueActionSchema, func: async (input) => { - try { - const id = generateId(); - const skippedTweet = { - id, - tweet: input.tweet, - reason: input.reason || 'No reason provided', - priority: input.priority || 0, - created_at: new Date(), - workflowState: input.workflowState - }; + try { + const id = generateId(); + const skippedTweet = { + id, + tweet: input.tweet, + reason: input.reason || "No reason provided", + priority: input.priority || 0, + created_at: new Date(), + workflowState: input.workflowState, + }; - logger.info('Queueing skipped tweet:', { - skippedTweet - }); + logger.info("Queueing skipped tweet:", { + skippedTweet, + }); - addToSkipped(skippedTweet); + addToSkipped(skippedTweet); - logger.info('Successfully queued skipped tweet:', { id }); + logger.info("Successfully queued skipped tweet:", { id }); - return { - success: true, - id, - type: 'skipped' as const, - message: 'Tweet queued for review' - }; - } catch (error) { - logger.error('Error queueing skipped tweet:', error); - throw error; - } - } -}); + return { + success: true, + id, + type: "skipped" as const, + message: "Tweet queued for review", + }; + } catch (error) { + logger.error("Error queueing skipped tweet:", error); + throw error; + } + }, + }); diff --git a/auto-kol/agent/src/tools/tools/searchSimilarTweetsTool.ts b/auto-kol/agent/src/tools/tools/searchSimilarTweetsTool.ts index d053f41..aac29d9 100644 --- a/auto-kol/agent/src/tools/tools/searchSimilarTweetsTool.ts +++ b/auto-kol/agent/src/tools/tools/searchSimilarTweetsTool.ts @@ -1,32 +1,35 @@ -import { DynamicStructuredTool } from '@langchain/core/tools'; -import { z } from 'zod'; -import { createLogger } from '../../utils/logger.js'; -import { ChromaService } from '../../services/vectorstore/chroma.js'; +import { DynamicStructuredTool } from "@langchain/core/tools"; +import { z } from "zod"; +import { createLogger } from "../../utils/logger.js"; +import { ChromaService } from "../../services/vectorstore/chroma.js"; -const logger = createLogger('search-similar-tweets-tool'); +const logger = createLogger("search-similar-tweets-tool"); - -export const createSearchSimilarTweetsTool = () => new DynamicStructuredTool({ - name: 'search_similar_tweets', - description: 'Search for similar tweets in the vector store', +export const createSearchSimilarTweetsTool = () => + new DynamicStructuredTool({ + name: "search_similar_tweets", + description: "Search for similar tweets in the vector store", schema: z.object({ - query: z.string(), - k: z.number().optional().default(5) + query: z.string(), + k: z.number().optional().default(5), }), func: async ({ query, k }) => { - try { - const chromaService = await ChromaService.getInstance(); - const results = await chromaService.searchSimilarTweetsWithScore(query, k); - return { - similar_tweets: results.map(([doc, score]) => ({ - text: doc.pageContent, - metadata: doc.metadata, - similarity_score: score - })) - }; - } catch (error) { - logger.error('Error searching similar tweets:', error); - return { similar_tweets: [] }; - } - } -}); + try { + const chromaService = await ChromaService.getInstance(); + const results = await chromaService.searchSimilarTweetsWithScore( + query, + k, + ); + return { + similar_tweets: results.map(([doc, score]) => ({ + text: doc.pageContent, + metadata: doc.metadata, + similarity_score: score, + })), + }; + } catch (error) { + logger.error("Error searching similar tweets:", error); + return { similar_tweets: [] }; + } + }, + }); diff --git a/auto-kol/agent/src/tools/tools/tweetSearchTool.ts b/auto-kol/agent/src/tools/tools/tweetSearchTool.ts index 9d930e7..ae7e857 100644 --- a/auto-kol/agent/src/tools/tools/tweetSearchTool.ts +++ b/auto-kol/agent/src/tools/tools/tweetSearchTool.ts @@ -1,11 +1,11 @@ -import { DynamicStructuredTool } from '@langchain/core/tools'; -import { z } from 'zod'; -import { createLogger } from '../../utils/logger.js'; -import { getKOLsAccounts, updateKOLs } from '../../utils/twitter.js'; -import { SearchMode } from 'agent-twitter-client'; -import { config } from '../../config/index.js'; -import { ExtendedScraper } from '../../services/twitter/api.js'; -const logger = createLogger('tweet-search-tool'); +import { DynamicStructuredTool } from "@langchain/core/tools"; +import { z } from "zod"; +import { createLogger } from "../../utils/logger.js"; +import { getKOLsAccounts, updateKOLs } from "../../utils/twitter.js"; +import { SearchMode } from "agent-twitter-client"; +import { config } from "../../config/index.js"; +import { ExtendedScraper } from "../../services/twitter/api.js"; +const logger = createLogger("tweet-search-tool"); function getRandomAccounts(accounts: string[], n: number): string[] { const shuffled = [...accounts].sort(() => 0.5 - Math.random()); @@ -14,19 +14,19 @@ function getRandomAccounts(accounts: string[], n: number): string[] { export const createTweetSearchTool = (scraper: ExtendedScraper) => new DynamicStructuredTool({ - name: 'search_recent_tweets', - description: 'Search for recent tweets from specified accounts', + name: "search_recent_tweets", + description: "Search for recent tweets from specified accounts", schema: z.object({ lastProcessedId: z.string().optional(), }), func: async ({ lastProcessedId }) => { try { - logger.info('Called search_recent_tweets'); + logger.info("Called search_recent_tweets"); await updateKOLs(scraper); const kols = await getKOLsAccounts(); if (kols.length === 0) { - logger.error('No valid accounts found after cleaning'); + logger.error("No valid accounts found after cleaning"); return { tweets: [], lastProcessedId: null, @@ -40,12 +40,12 @@ export const createTweetSearchTool = (scraper: ExtendedScraper) => for (let i = 0; i < selectedKols.length; i += ACCOUNTS_PER_QUERY) { const accountsBatch = selectedKols.slice(i, i + ACCOUNTS_PER_QUERY); - const query = `(${accountsBatch.map(account => `from:${account}`).join(' OR ')})`; + const query = `(${accountsBatch.map((account) => `from:${account}`).join(" OR ")})`; const searchIterator = scraper.searchTweets( query, Math.floor(config.MAX_SEARCH_TWEETS / 4), - SearchMode.Latest + SearchMode.Latest, ); for await (const tweet of searchIterator) { @@ -53,20 +53,22 @@ export const createTweetSearchTool = (scraper: ExtendedScraper) => break; } tweetGroups.push({ - id: tweet.id || '', - text: tweet.text || '', - author_id: tweet.userId || '', - author_username: tweet.username?.toLowerCase() || '', + id: tweet.id || "", + text: tweet.text || "", + author_id: tweet.userId || "", + author_username: tweet.username?.toLowerCase() || "", created_at: tweet.timeParsed || new Date(), }); } - await new Promise(resolve => setTimeout(resolve, 1000)); + await new Promise((resolve) => setTimeout(resolve, 1000)); } - const allTweets = tweetGroups.sort((a, b) => b.created_at.getTime() - a.created_at.getTime()); + const allTweets = tweetGroups.sort( + (a, b) => b.created_at.getTime() - a.created_at.getTime(), + ); - logger.info('Tweet search completed:', { + logger.info("Tweet search completed:", { foundTweets: allTweets.length, selectedKols, }); @@ -76,7 +78,7 @@ export const createTweetSearchTool = (scraper: ExtendedScraper) => lastProcessedId: allTweets[allTweets.length - 1]?.id || null, }; } catch (error) { - logger.error('Error searching tweets:', error); + logger.error("Error searching tweets:", error); return { tweets: [], lastProcessedId: null, diff --git a/auto-kol/agent/src/types/agent.ts b/auto-kol/agent/src/types/agent.ts index c2e2b6d..b9c392d 100644 --- a/auto-kol/agent/src/types/agent.ts +++ b/auto-kol/agent/src/types/agent.ts @@ -1,11 +1,11 @@ import { Tweet } from "./twitter.js"; export type AgentResponse = Readonly<{ - content: string; - references?: readonly string[]; -}> + content: string; + references?: readonly string[]; +}>; export type Context = Readonly<{ - tweet: Tweet; - previousInteractions: readonly AgentResponse[]; -}> \ No newline at end of file + tweet: Tweet; + previousInteractions: readonly AgentResponse[]; +}>; diff --git a/auto-kol/agent/src/types/kol.ts b/auto-kol/agent/src/types/kol.ts index 9b0e308..fcf7466 100644 --- a/auto-kol/agent/src/types/kol.ts +++ b/auto-kol/agent/src/types/kol.ts @@ -1,6 +1,6 @@ export type KOL = Readonly<{ - id: string; - username: string; - created_at?: Date; - updatedAt?: Date; -}>; \ No newline at end of file + id: string; + username: string; + created_at?: Date; + updatedAt?: Date; +}>; diff --git a/auto-kol/agent/src/types/queue.ts b/auto-kol/agent/src/types/queue.ts index d5a922d..ddf68fd 100644 --- a/auto-kol/agent/src/types/queue.ts +++ b/auto-kol/agent/src/types/queue.ts @@ -1,65 +1,65 @@ -import { Tweet } from './twitter.js'; -import { AgentResponse } from './agent.js'; -import { WorkflowState } from './workflow.js'; +import { Tweet } from "./twitter.js"; +import { AgentResponse } from "./agent.js"; +import { WorkflowState } from "./workflow.js"; export enum ResponseStatus { - SKIPPED = 'skipped', - PENDING = 'pending', - APPROVED = 'approved', - REJECTED = 'rejected' + SKIPPED = "skipped", + PENDING = "pending", + APPROVED = "approved", + REJECTED = "rejected", } export interface PendingResponse { + id: string; + tweet_id: string; + content: string; + tone: string; + strategy: string; + estimatedImpact: number; + confidence: number; +} + +export type ActionResponse = Readonly<{ + tweet: Tweet; + status: "pending" | "approved" | "rejected"; + response: { id: string; - tweet_id: string; content: string; tone: string; strategy: string; estimatedImpact: number; - confidence: number; -} - -export type ActionResponse = Readonly<{ - tweet: Tweet; - status: 'pending' | 'approved' | 'rejected'; - response: { - id: string; - content: string; - tone: string; - strategy: string; - estimatedImpact: number; - confidenceScore: number; - } -}> + confidenceScore: number; + }; +}>; export type QueuedResponseMemory = Readonly<{ - id: string; - tweet: Tweet; - response: AgentResponse; - status: 'pending' | 'approved' | 'rejected'; - created_at: Date; - updatedAt: Date; - workflowState: WorkflowState; + id: string; + tweet: Tweet; + response: AgentResponse; + status: "pending" | "approved" | "rejected"; + created_at: Date; + updatedAt: Date; + workflowState: WorkflowState; }>; export type SkippedTweetMemory = Readonly<{ - id: string; - tweet: Tweet; - reason: string; - priority: number; - created_at: Date; - workflowState: any; + id: string; + tweet: Tweet; + reason: string; + priority: number; + created_at: Date; + workflowState: any; }>; export type SkippedTweet = Readonly<{ - id: string; - tweet_id: string; - reason: string; - priority: number; + id: string; + tweet_id: string; + reason: string; + priority: number; }>; export type ApprovalAction = Readonly<{ - id: string; - approved: boolean; - feedback?: string; -}>; \ No newline at end of file + id: string; + approved: boolean; + feedback?: string; +}>; diff --git a/auto-kol/agent/src/types/twitter.ts b/auto-kol/agent/src/types/twitter.ts index d1fad59..5571e4b 100644 --- a/auto-kol/agent/src/types/twitter.ts +++ b/auto-kol/agent/src/types/twitter.ts @@ -1,15 +1,15 @@ export type Tweet = { - readonly id: string; - readonly text: string; - readonly author_id: string; - readonly author_username: string; - readonly created_at: string; - readonly thread?: Tweet[]; -} + readonly id: string; + readonly text: string; + readonly author_id: string; + readonly author_username: string; + readonly created_at: string; + readonly thread?: Tweet[]; +}; export type TwitterCredentials = { - appKey: string; - appSecret: string; - accessToken: string; - accessSecret: string; + appKey: string; + appSecret: string; + accessToken: string; + accessSecret: string; }; diff --git a/auto-kol/agent/src/types/workflow.ts b/auto-kol/agent/src/types/workflow.ts index 4168fd2..cf46b95 100644 --- a/auto-kol/agent/src/types/workflow.ts +++ b/auto-kol/agent/src/types/workflow.ts @@ -1,52 +1,52 @@ -import { Tweet } from './twitter.js'; -import { BaseMessage } from '@langchain/core/messages'; +import { Tweet } from "./twitter.js"; +import { BaseMessage } from "@langchain/core/messages"; export type EngagementDecision = Readonly<{ - shouldEngage: boolean; - reason: string; - priority: number; + shouldEngage: boolean; + reason: string; + priority: number; }>; export type ToneAnalysis = Readonly<{ - dominantTone: string; - suggestedTone: string; - reasoning: string; + dominantTone: string; + suggestedTone: string; + reasoning: string; }>; export type ResponseAlternative = Readonly<{ - content: string; - tone: string; - strategy: string; - estimatedImpact: number; + content: string; + tone: string; + strategy: string; + estimatedImpact: number; }>; export type ResponseSelection = Readonly<{ - selectedResponse: string; - reasoning: string; - confidence: number; + selectedResponse: string; + reasoning: string; + confidence: number; }>; export type ResponseStrategy = Readonly<{ - confidence: number; - content: string; - tone: string; - strategy: string; - estimatedImpact: number; + confidence: number; + content: string; + tone: string; + strategy: string; + estimatedImpact: number; }>; export type AutoFeedback = Readonly<{ - response: string; - reason: string; - suggestedChanges: string[]; + response: string; + reason: string; + suggestedChanges: string[]; }>; export type WorkflowState = Readonly<{ - tweet: Tweet; - messages: BaseMessage[]; - engagementDecision?: EngagementDecision; - toneAnalysis?: ToneAnalysis; - alternatives?: ResponseAlternative[]; - selectedResponse?: ResponseSelection; - responseStrategy?: ResponseStrategy; - autoFeedback?: AutoFeedback[]; -}>; \ No newline at end of file + tweet: Tweet; + messages: BaseMessage[]; + engagementDecision?: EngagementDecision; + toneAnalysis?: ToneAnalysis; + alternatives?: ResponseAlternative[]; + selectedResponse?: ResponseSelection; + responseStrategy?: ResponseStrategy; + autoFeedback?: AutoFeedback[]; +}>; diff --git a/auto-kol/agent/src/utils/agentMemoryContract.ts b/auto-kol/agent/src/utils/agentMemoryContract.ts index cffaf3e..e5ba37e 100644 --- a/auto-kol/agent/src/utils/agentMemoryContract.ts +++ b/auto-kol/agent/src/utils/agentMemoryContract.ts @@ -1,28 +1,27 @@ -import { ethers } from 'ethers'; -import { MEMORY_ABI } from '../abi/memory.js'; -import { config } from '../config/index.js'; -import { wallet } from './agentWallet.js'; -import { cidFromBlakeHash, cidToString } from '@autonomys/auto-dag-data'; - +import { ethers } from "ethers"; +import { MEMORY_ABI } from "../abi/memory.js"; +import { config } from "../config/index.js"; +import { wallet } from "./agentWallet.js"; +import { cidFromBlakeHash, cidToString } from "@autonomys/auto-dag-data"; const CONTRACT_ADDRESS = config.CONTRACT_ADDRESS as `0x${string}`; const contract = new ethers.Contract(CONTRACT_ADDRESS, MEMORY_ABI, wallet); export const getLastMemoryHash = async (): Promise => { - return await contract.getLastMemoryHash(wallet.address); -} + return await contract.getLastMemoryHash(wallet.address); +}; export const getLastMemoryCid = async (): Promise => { - const lastMemoryHash = await contract.getLastMemoryHash(wallet.address); - return cidToString(cidFromBlakeHash(lastMemoryHash)); -} + const lastMemoryHash = await contract.getLastMemoryHash(wallet.address); + return cidToString(cidFromBlakeHash(lastMemoryHash)); +}; export const setLastMemoryHash = async (hash: string, nonce?: number) => { - const bytes32Hash = ethers.zeroPadValue(hash, 32); - const tx = await contract.setLastMemoryHash(bytes32Hash, { - nonce: nonce, - gasLimit: 100000 - }); - return tx; -} \ No newline at end of file + const bytes32Hash = ethers.zeroPadValue(hash, 32); + const tx = await contract.setLastMemoryHash(bytes32Hash, { + nonce: nonce, + gasLimit: 100000, + }); + return tx; +}; diff --git a/auto-kol/agent/src/utils/agentWallet.ts b/auto-kol/agent/src/utils/agentWallet.ts index cd17864..2df071f 100644 --- a/auto-kol/agent/src/utils/agentWallet.ts +++ b/auto-kol/agent/src/utils/agentWallet.ts @@ -1,13 +1,11 @@ -import { ethers } from 'ethers'; -import { config } from '../config/index.js'; +import { ethers } from "ethers"; +import { config } from "../config/index.js"; const provider = new ethers.JsonRpcProvider(config.RPC_URL); export const wallet = new ethers.Wallet(config.PRIVATE_KEY as string, provider); export async function signMessage(data: object): Promise { - const message = JSON.stringify(data); - return await wallet.signMessage(message); + const message = JSON.stringify(data); + return await wallet.signMessage(message); } - - diff --git a/auto-kol/agent/src/utils/dsn.ts b/auto-kol/agent/src/utils/dsn.ts index c65f8dc..5ac4e25 100644 --- a/auto-kol/agent/src/utils/dsn.ts +++ b/auto-kol/agent/src/utils/dsn.ts @@ -1,181 +1,184 @@ -import { createLogger } from '../utils/logger.js'; -import { hexlify } from 'ethers'; -import { createAutoDriveApi, uploadFile } from '@autonomys/auto-drive'; -import { stringToCid, blake3HashFromCid, cidFromBlakeHash } from '@autonomys/auto-dag-data'; -import { addDsn, getLastDsnCid } from '../database/index.js'; -import { v4 as generateId } from 'uuid'; -import { config } from '../config/index.js'; -import { setLastMemoryHash, getLastMemoryCid } from './agentMemoryContract.js'; -import { signMessage, wallet } from './agentWallet.js'; - -const logger = createLogger('dsn-upload-tool'); +import { createLogger } from "../utils/logger.js"; +import { hexlify } from "ethers"; +import { createAutoDriveApi, uploadFile } from "@autonomys/auto-drive"; +import { + stringToCid, + blake3HashFromCid, + cidFromBlakeHash, +} from "@autonomys/auto-dag-data"; +import { addDsn, getLastDsnCid } from "../database/index.js"; +import { v4 as generateId } from "uuid"; +import { config } from "../config/index.js"; +import { setLastMemoryHash, getLastMemoryCid } from "./agentMemoryContract.js"; +import { signMessage, wallet } from "./agentWallet.js"; + +const logger = createLogger("dsn-upload-tool"); const dsnAPI = createAutoDriveApi({ apiKey: config.DSN_API_KEY! }); let currentNonce = await wallet.getNonce(); interface RetryOptions { - maxRetries: number; - delay: number; - onRetry?: (error: Error, attempt: number) => void; + maxRetries: number; + delay: number; + onRetry?: (error: Error, attempt: number) => void; } async function retry( - fn: () => Promise, - options: RetryOptions + fn: () => Promise, + options: RetryOptions, ): Promise { - const { maxRetries, delay, onRetry } = options; - let lastError: Error; - - for (let attempt = 1; attempt <= maxRetries; attempt++) { - try { - return await fn(); - } catch (error) { - lastError = error as Error; - - if (attempt === maxRetries) { - break; - } - - if (onRetry) { - onRetry(lastError, attempt); - } - - // Add jitter to prevent thundering herd - const jitter = Math.random() * 1000; - await new Promise(resolve => - setTimeout(resolve, delay * attempt + jitter) - ); - } - } + const { maxRetries, delay, onRetry } = options; + let lastError: Error; - throw lastError!; -} + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + return await fn(); + } catch (error) { + lastError = error as Error; -const getPreviousCid = async (): Promise => { - const dsnLastCid = await getLastDsnCid(); - if (dsnLastCid) { - logger.info('Using last CID from local db', { cid: dsnLastCid }); - return dsnLastCid; + if (attempt === maxRetries) { + break; + } + + if (onRetry) { + onRetry(lastError, attempt); + } + + // Add jitter to prevent thundering herd + const jitter = Math.random() * 1000; + await new Promise((resolve) => + setTimeout(resolve, delay * attempt + jitter), + ); } + } - const memoryLastCid = await getLastMemoryCid(); - logger.info('Using fallback CID source', { - memoryLastCid: memoryLastCid || 'not found' - }); + throw lastError!; +} - return memoryLastCid || ''; +const getPreviousCid = async (): Promise => { + const dsnLastCid = await getLastDsnCid(); + if (dsnLastCid) { + logger.info("Using last CID from local db", { cid: dsnLastCid }); + return dsnLastCid; + } + + const memoryLastCid = await getLastMemoryCid(); + logger.info("Using fallback CID source", { + memoryLastCid: memoryLastCid || "not found", + }); + + return memoryLastCid || ""; }; -export async function uploadToDsn({ data, }: { data: any; }) { - const maxRetries = 5; - const retryDelay = 2000; - const previousCid = await getPreviousCid(); - - try { - const timestamp = new Date().toISOString(); - const signature = await signMessage({ - data: data, - previousCid: previousCid, - timestamp: timestamp - }); +export async function uploadToDsn({ data }: { data: any }) { + const maxRetries = 5; + const retryDelay = 2000; + const previousCid = await getPreviousCid(); + + try { + const timestamp = new Date().toISOString(); + const signature = await signMessage({ + data: data, + previousCid: previousCid, + timestamp: timestamp, + }); - const dsnData = { - ...data, - previousCid: previousCid, - signature: signature, - timestamp: timestamp - }; - - const jsonBuffer = Buffer.from(JSON.stringify(dsnData, null, 2)); - - let finalCid: string | undefined; - await retry( - async () => { - const uploadObservable = uploadFile( - dsnAPI, - { - read: async function* () { - yield jsonBuffer; - }, - name: `${config.TWITTER_USERNAME}-agent-memory-${timestamp}.json`, - mimeType: 'application/json', - size: jsonBuffer.length, - path: timestamp - }, - { - compression: true, - password: config.DSN_ENCRYPTION_PASSWORD || undefined - } - ); - - await uploadObservable.forEach(status => { - if (status.type === 'file' && status.cid) { - finalCid = status.cid.toString(); - } - }); - - if (!finalCid) { - throw new Error('Failed to get CID from DSN upload'); - } + const dsnData = { + ...data, + previousCid: previousCid, + signature: signature, + timestamp: timestamp, + }; + + const jsonBuffer = Buffer.from(JSON.stringify(dsnData, null, 2)); + + let finalCid: string | undefined; + await retry( + async () => { + const uploadObservable = uploadFile( + dsnAPI, + { + read: async function* () { + yield jsonBuffer; }, - { - maxRetries, - delay: retryDelay, - onRetry: (error, attempt) => { - logger.warn(`DSN upload attempt ${attempt} failed:`, { - error: error.message, - tweetId: data.tweet?.id - }); - } - } + name: `${config.TWITTER_USERNAME}-agent-memory-${timestamp}.json`, + mimeType: "application/json", + size: jsonBuffer.length, + path: timestamp, + }, + { + compression: true, + password: config.DSN_ENCRYPTION_PASSWORD || undefined, + }, ); + await uploadObservable.forEach((status) => { + if (status.type === "file" && status.cid) { + finalCid = status.cid.toString(); + } + }); + if (!finalCid) { - throw new Error('Failed to get CID from DSN upload after retries'); + throw new Error("Failed to get CID from DSN upload"); } + }, + { + maxRetries, + delay: retryDelay, + onRetry: (error, attempt) => { + logger.warn(`DSN upload attempt ${attempt} failed:`, { + error: error.message, + tweetId: data.tweet?.id, + }); + }, + }, + ); + + if (!finalCid) { + throw new Error("Failed to get CID from DSN upload after retries"); + } - const blake3hash = blake3HashFromCid(stringToCid(finalCid)); - logger.info('Setting last memory hash', { - blake3hash: hexlify(blake3hash) - }); - - await retry( - async () => { - const tx = await setLastMemoryHash(hexlify(blake3hash), currentNonce++); - logger.info('Memory hash transaction submitted', { - txHash: tx.hash, - previousCid, - cid: finalCid - }); - return tx; - }, - { - maxRetries, - delay: retryDelay, - onRetry: (error, attempt) => { - logger.warn(`Blockchain transaction attempt ${attempt} failed:`, { - error: error.message, - cid: finalCid - }); - } - } - ).catch(error => { - logger.error('Failed to submit memory hash transaction', error); - }); + const blake3hash = blake3HashFromCid(stringToCid(finalCid)); + logger.info("Setting last memory hash", { + blake3hash: hexlify(blake3hash), + }); - await addDsn({ - id: generateId(), - tweetId: data.tweet.id, - cid: finalCid + await retry( + async () => { + const tx = await setLastMemoryHash(hexlify(blake3hash), currentNonce++); + logger.info("Memory hash transaction submitted", { + txHash: tx.hash, + previousCid, + cid: finalCid, }); - - return { - success: true, + return tx; + }, + { + maxRetries, + delay: retryDelay, + onRetry: (error, attempt) => { + logger.warn(`Blockchain transaction attempt ${attempt} failed:`, { + error: error.message, cid: finalCid, - previousCid: previousCid || null - }; + }); + }, + }, + ).catch((error) => { + logger.error("Failed to submit memory hash transaction", error); + }); - } catch (error) { - logger.error('Error uploading to DSN:', error); - throw error; - } -} \ No newline at end of file + await addDsn({ + id: generateId(), + tweetId: data.tweet.id, + cid: finalCid, + }); + + return { + success: true, + cid: finalCid, + previousCid: previousCid || null, + }; + } catch (error) { + logger.error("Error uploading to DSN:", error); + throw error; + } +} diff --git a/auto-kol/agent/src/utils/logger.ts b/auto-kol/agent/src/utils/logger.ts index 5c484cf..e43c606 100644 --- a/auto-kol/agent/src/utils/logger.ts +++ b/auto-kol/agent/src/utils/logger.ts @@ -1,85 +1,85 @@ -import winston from 'winston'; -import { config } from '../config/index.js'; -import util from 'util'; +import winston from "winston"; +import { config } from "../config/index.js"; +import util from "util"; const formatMeta = (meta: any, useColors: boolean = false) => { - const cleanMeta = Object.entries(meta) - .filter(([key]) => !key.startsWith('Symbol(') && key !== 'splat') - .reduce((acc, [key, value]) => ({ ...acc, [key]: value }), {}); + const cleanMeta = Object.entries(meta) + .filter(([key]) => !key.startsWith("Symbol(") && key !== "splat") + .reduce((acc, [key, value]) => ({ ...acc, [key]: value }), {}); - if (Object.keys(cleanMeta).length === 0) return ''; + if (Object.keys(cleanMeta).length === 0) return ""; - if (meta[Symbol.for('splat')]?.[0]) { - Object.assign(cleanMeta, meta[Symbol.for('splat')][0]); - } + if (meta[Symbol.for("splat")]?.[0]) { + Object.assign(cleanMeta, meta[Symbol.for("splat")][0]); + } - return Object.keys(cleanMeta).length - ? '\n' + JSON.stringify(cleanMeta, null, 2) - : ''; + return Object.keys(cleanMeta).length + ? "\n" + JSON.stringify(cleanMeta, null, 2) + : ""; }; const createFileFormat = () => - winston.format.combine( - winston.format.timestamp({ - format: 'YYYY-MM-DD HH:mm:ss.SSS' - }), - winston.format.uncolorize(), - winston.format.printf(({ level, message, context, timestamp, ...meta }) => { - const metaStr = formatMeta(meta, false); - const paddedLevel = level.toUpperCase().padEnd(7); - return `${timestamp} | ${paddedLevel} | [${context}] | ${message}${metaStr}`; - }) - ); + winston.format.combine( + winston.format.timestamp({ + format: "YYYY-MM-DD HH:mm:ss.SSS", + }), + winston.format.uncolorize(), + winston.format.printf(({ level, message, context, timestamp, ...meta }) => { + const metaStr = formatMeta(meta, false); + const paddedLevel = level.toUpperCase().padEnd(7); + return `${timestamp} | ${paddedLevel} | [${context}] | ${message}${metaStr}`; + }), + ); const createConsoleFormat = () => - winston.format.combine( - winston.format.timestamp({ - format: 'YYYY-MM-DD HH:mm:ss.SSS' - }), - winston.format.colorize({ level: true }), - winston.format.printf(({ level, message, context, timestamp, ...meta }) => { - const metaStr = formatMeta(meta, true); - const paddedLevel = level.toUpperCase().padEnd(7); - return `${timestamp} | ${paddedLevel} | [${context}] | ${message}${metaStr}`; - }) - ); + winston.format.combine( + winston.format.timestamp({ + format: "YYYY-MM-DD HH:mm:ss.SSS", + }), + winston.format.colorize({ level: true }), + winston.format.printf(({ level, message, context, timestamp, ...meta }) => { + const metaStr = formatMeta(meta, true); + const paddedLevel = level.toUpperCase().padEnd(7); + return `${timestamp} | ${paddedLevel} | [${context}] | ${message}${metaStr}`; + }), + ); const createTransports = () => [ - new winston.transports.File({ - filename: 'logs/error.log', - level: 'error', - format: createFileFormat(), - maxsize: 5242880, - maxFiles: 5, - tailable: true - }), - new winston.transports.File({ - filename: 'logs/combined.log', - format: createFileFormat(), - maxsize: 5242880, - maxFiles: 5, - tailable: true - }) + new winston.transports.File({ + filename: "logs/error.log", + level: "error", + format: createFileFormat(), + maxsize: 5242880, + maxFiles: 5, + tailable: true, + }), + new winston.transports.File({ + filename: "logs/combined.log", + format: createFileFormat(), + maxsize: 5242880, + maxFiles: 5, + tailable: true, + }), ]; const addConsoleTransport = (logger: winston.Logger): winston.Logger => { - if (config.NODE_ENV !== 'production') { - logger.add( - new winston.transports.Console({ - format: createConsoleFormat() - }) - ); - } - return logger; + if (config.NODE_ENV !== "production") { + logger.add( + new winston.transports.Console({ + format: createConsoleFormat(), + }), + ); + } + return logger; }; export const createLogger = (context: string) => { - const logger = winston.createLogger({ - defaultMeta: { context }, - level: 'info', - format: createFileFormat(), - transports: createTransports() - }); + const logger = winston.createLogger({ + defaultMeta: { context }, + level: "info", + format: createFileFormat(), + transports: createTransports(), + }); - return addConsoleTransport(logger); -}; \ No newline at end of file + return addConsoleTransport(logger); +}; diff --git a/auto-kol/agent/src/utils/twitter.ts b/auto-kol/agent/src/utils/twitter.ts index 61da8e1..2ddb0dd 100644 --- a/auto-kol/agent/src/utils/twitter.ts +++ b/auto-kol/agent/src/utils/twitter.ts @@ -1,22 +1,24 @@ -import { config } from '../config/index.js'; -import { createLogger } from '../utils/logger.js'; -import { ExtendedScraper } from '../services/twitter/api.js'; -import * as db from '../database/index.js'; -import { KOL } from '../types/kol.js'; -import { Tweet } from '../types/twitter.js'; +import { config } from "../config/index.js"; +import { createLogger } from "../utils/logger.js"; +import { ExtendedScraper } from "../services/twitter/api.js"; +import * as db from "../database/index.js"; +import { KOL } from "../types/kol.js"; +import { Tweet } from "../types/twitter.js"; -const logger = createLogger('twitter-utils'); +const logger = createLogger("twitter-utils"); export const timelineTweets: Tweet[] = []; export const updateKOLs = async (twitterScraper: ExtendedScraper) => { const currentKOLs = await db.getKOLAccounts(); - const twitterProfile = await twitterScraper.getProfile(config.TWITTER_USERNAME!); + const twitterProfile = await twitterScraper.getProfile( + config.TWITTER_USERNAME!, + ); const followings = twitterScraper.getFollowing(twitterProfile.userId!, 1000); logger.info(`following count: ${twitterProfile.followingCount}`); const newKOLs: KOL[] = []; for await (const following of followings) { - if (!currentKOLs.some(kol => kol.username === following.username)) { + if (!currentKOLs.some((kol) => kol.username === following.username)) { newKOLs.push({ id: following.userId!, username: following.username!.toLowerCase(), @@ -31,11 +33,13 @@ export const updateKOLs = async (twitterScraper: ExtendedScraper) => { export const getKOLsAccounts = async () => { const kolAccounts = await db.getKOLAccounts(); - return kolAccounts.map(kol => kol.username); + return kolAccounts.map((kol) => kol.username); }; export const getTimeLine = async (twitterScraper: ExtendedScraper) => { - const validTweetIds = timelineTweets.map(tweet => tweet.id).filter(id => id != null); + const validTweetIds = timelineTweets + .map((tweet) => tweet.id) + .filter((id) => id != null); const timeline = await twitterScraper.fetchHomeTimeline(0, validTweetIds); // clear timeline @@ -61,7 +65,10 @@ const clearTimeLine = () => { timelineTweets.length = 0; }; -export const getUserProfile = async (twitterScraper: ExtendedScraper, username: string) => { +export const getUserProfile = async ( + twitterScraper: ExtendedScraper, + username: string, +) => { const user = await twitterScraper.getProfile(username); const result: KOL = { id: user.userId!, diff --git a/auto-kol/agent/yarn.lock b/auto-kol/agent/yarn.lock index 33ab875..bb1055e 100644 --- a/auto-kol/agent/yarn.lock +++ b/auto-kol/agent/yarn.lock @@ -2319,6 +2319,11 @@ prelude-ls@~1.1.2: resolved "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz" integrity sha512-ESF23V4SKG6lVSGZgYNpbsiaAkdab6ZgOxe52p7+Kid3W3u3bxR4Vfd/o21dmN7jSt0IwgZ4v5MUd26FEtXE9w== +prettier@^3.2.2: + version "3.4.2" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.4.2.tgz#a5ce1fb522a588bf2b78ca44c6e6fe5aa5a2b13f" + integrity sha512-e9MewbtFo+Fevyuxn/4rrcDAaq0IYxPGLvObpQjiZBMAzB9IGmzlnG9RZy3FFas+eBMu2vA0CszMeduow5dIuQ== + process-nextick-args@~2.0.0: version "2.0.1" resolved "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz" From b6f946aa59a1018fa7a36779dae4eb6998650cb5 Mon Sep 17 00:00:00 2001 From: xm0onh Date: Tue, 24 Dec 2024 14:25:27 -0800 Subject: [PATCH 12/13] update prettier --- .../agent/{prettierrc.js => .prettierrc.js} | 0 auto-kol/agent/src/abi/memory.ts | 64 ++++---- auto-kol/agent/src/api/index.ts | 18 +-- auto-kol/agent/src/api/middleware/cors.ts | 19 +-- auto-kol/agent/src/api/routes/dsn.ts | 34 ++-- auto-kol/agent/src/api/routes/health.ts | 6 +- auto-kol/agent/src/api/routes/responses.ts | 18 +-- auto-kol/agent/src/api/routes/tweets.ts | 35 ++-- auto-kol/agent/src/config/index.ts | 27 ++-- auto-kol/agent/src/database/index.ts | 150 +++++++----------- auto-kol/agent/src/index.ts | 24 +-- auto-kol/agent/src/schemas/workflow.ts | 2 +- auto-kol/agent/src/services/agents/nodes.ts | 18 +-- .../services/agents/nodes/autoApprovalNode.ts | 45 +++--- .../services/agents/nodes/engagementNode.ts | 55 +++---- .../src/services/agents/nodes/mentionNode.ts | 20 +-- .../agents/nodes/recheckSkippedNode.ts | 25 ++- .../agents/nodes/responseGenerationNode.ts | 77 ++++----- .../src/services/agents/nodes/searchNode.ts | 44 +++-- .../src/services/agents/nodes/timelineNode.ts | 33 ++-- .../services/agents/nodes/toneAnalysisNode.ts | 48 +++--- auto-kol/agent/src/services/agents/prompts.ts | 38 ++--- .../agent/src/services/agents/workflow.ts | 92 +++++------ auto-kol/agent/src/services/database/index.ts | 69 ++++---- auto-kol/agent/src/services/twitter/api.ts | 96 ++++------- .../agent/src/services/vectorstore/chroma.ts | 50 +++--- auto-kol/agent/src/tools/index.ts | 16 +- .../src/tools/tools/fetchTimelineTool.ts | 23 ++- auto-kol/agent/src/tools/tools/mentionTool.ts | 36 ++--- .../src/tools/tools/queueResponseTool.ts | 50 +++--- .../agent/src/tools/tools/queueSkippedTool.ts | 30 ++-- .../tools/tools/searchSimilarTweetsTool.ts | 21 ++- .../agent/src/tools/tools/tweetSearchTool.ts | 40 ++--- auto-kol/agent/src/types/agent.ts | 2 +- auto-kol/agent/src/types/queue.ts | 18 +-- auto-kol/agent/src/types/workflow.ts | 4 +- .../agent/src/utils/agentMemoryContract.ts | 10 +- auto-kol/agent/src/utils/agentWallet.ts | 4 +- auto-kol/agent/src/utils/dsn.ts | 63 ++++---- auto-kol/agent/src/utils/logger.ts | 32 ++-- auto-kol/agent/src/utils/twitter.ts | 31 ++-- 41 files changed, 652 insertions(+), 835 deletions(-) rename auto-kol/agent/{prettierrc.js => .prettierrc.js} (100%) diff --git a/auto-kol/agent/prettierrc.js b/auto-kol/agent/.prettierrc.js similarity index 100% rename from auto-kol/agent/prettierrc.js rename to auto-kol/agent/.prettierrc.js diff --git a/auto-kol/agent/src/abi/memory.ts b/auto-kol/agent/src/abi/memory.ts index f53f1c1..4dac720 100644 --- a/auto-kol/agent/src/abi/memory.ts +++ b/auto-kol/agent/src/abi/memory.ts @@ -4,69 +4,69 @@ export const MEMORY_ABI = [ inputs: [ { indexed: true, - internalType: "address", - name: "agent", - type: "address", + internalType: 'address', + name: 'agent', + type: 'address', }, { indexed: false, - internalType: "bytes32", - name: "hash", - type: "bytes32", + internalType: 'bytes32', + name: 'hash', + type: 'bytes32', }, ], - name: "LastMemoryHashSet", - type: "event", + name: 'LastMemoryHashSet', + type: 'event', }, { inputs: [ { - internalType: "address", - name: "_agent", - type: "address", + internalType: 'address', + name: '_agent', + type: 'address', }, ], - name: "getLastMemoryHash", + name: 'getLastMemoryHash', outputs: [ { - internalType: "bytes32", - name: "", - type: "bytes32", + internalType: 'bytes32', + name: '', + type: 'bytes32', }, ], - stateMutability: "view", - type: "function", + stateMutability: 'view', + type: 'function', }, { inputs: [ { - internalType: "address", - name: "", - type: "address", + internalType: 'address', + name: '', + type: 'address', }, ], - name: "lastMemoryHash", + name: 'lastMemoryHash', outputs: [ { - internalType: "bytes32", - name: "", - type: "bytes32", + internalType: 'bytes32', + name: '', + type: 'bytes32', }, ], - stateMutability: "view", - type: "function", + stateMutability: 'view', + type: 'function', }, { inputs: [ { - internalType: "bytes32", - name: "hash", - type: "bytes32", + internalType: 'bytes32', + name: 'hash', + type: 'bytes32', }, ], - name: "setLastMemoryHash", + name: 'setLastMemoryHash', outputs: [], - stateMutability: "nonpayable", - type: "function", + stateMutability: 'nonpayable', + type: 'function', }, ] as const; diff --git a/auto-kol/agent/src/api/index.ts b/auto-kol/agent/src/api/index.ts index c71b6f6..e28c2c6 100644 --- a/auto-kol/agent/src/api/index.ts +++ b/auto-kol/agent/src/api/index.ts @@ -1,14 +1,14 @@ -import { Router } from "express"; -import healthRoutes from "./routes/health.js"; -import responseRoutes from "./routes/responses.js"; -import tweetRoutes from "./routes/tweets.js"; -import dsnRoutes from "./routes/dsn.js"; +import { Router } from 'express'; +import healthRoutes from './routes/health.js'; +import responseRoutes from './routes/responses.js'; +import tweetRoutes from './routes/tweets.js'; +import dsnRoutes from './routes/dsn.js'; const router = Router(); -router.use("/", healthRoutes); -router.use("/", responseRoutes); -router.use("/", tweetRoutes); -router.use("/", dsnRoutes); +router.use('/', healthRoutes); +router.use('/', responseRoutes); +router.use('/', tweetRoutes); +router.use('/', dsnRoutes); export default router; diff --git a/auto-kol/agent/src/api/middleware/cors.ts b/auto-kol/agent/src/api/middleware/cors.ts index 4f02478..60b14f0 100644 --- a/auto-kol/agent/src/api/middleware/cors.ts +++ b/auto-kol/agent/src/api/middleware/cors.ts @@ -1,9 +1,7 @@ -import cors from "cors"; -import { config } from "../../config/index.js"; +import cors from 'cors'; +import { config } from '../../config/index.js'; -const allowedOrigins = config.CORS_ORIGINS?.split(",") || [ - "http://localhost:3000", -]; +const allowedOrigins = config.CORS_ORIGINS?.split(',') || ['http://localhost:3000']; export const corsMiddleware = cors({ origin: (origin, callback) => { @@ -11,17 +9,14 @@ export const corsMiddleware = cors({ return callback(null, true); } - if ( - allowedOrigins.indexOf(origin) !== -1 || - config.NODE_ENV === "development" - ) { + if (allowedOrigins.indexOf(origin) !== -1 || config.NODE_ENV === 'development') { callback(null, true); } else { - callback(new Error("Not allowed by CORS")); + callback(new Error('Not allowed by CORS')); } }, credentials: true, - methods: ["GET", "POST", "PUT", "DELETE", "OPTIONS"], - allowedHeaders: ["Content-Type", "Authorization"], + methods: ['GET', 'POST', 'PUT', 'DELETE', 'OPTIONS'], + allowedHeaders: ['Content-Type', 'Authorization'], maxAge: 86400, // 24 hours }); diff --git a/auto-kol/agent/src/api/routes/dsn.ts b/auto-kol/agent/src/api/routes/dsn.ts index 8dde039..5df8a08 100644 --- a/auto-kol/agent/src/api/routes/dsn.ts +++ b/auto-kol/agent/src/api/routes/dsn.ts @@ -1,13 +1,13 @@ -import { Router } from "express"; -import { createLogger } from "../../utils/logger.js"; -import { getAllDsn } from "../../database/index.js"; -import { inflate } from "pako"; -import { createAutoDriveApi, downloadObject } from "@autonomys/auto-drive"; -import { config } from "../../config/index.js"; +import { Router } from 'express'; +import { createLogger } from '../../utils/logger.js'; +import { getAllDsn } from '../../database/index.js'; +import { inflate } from 'pako'; +import { createAutoDriveApi, downloadObject } from '@autonomys/auto-drive'; +import { config } from '../../config/index.js'; const router = Router(); -const logger = createLogger("dsn-api"); +const logger = createLogger('dsn-api'); -router.get("/memories", async (req, res) => { +router.get('/memories', async (req, res) => { try { const page = parseInt(req.query.page as string) || 1; const limit = parseInt(req.query.limit as string) || 10; @@ -15,22 +15,22 @@ router.get("/memories", async (req, res) => { if (page < 1 || limit < 1 || limit > 100) { return res.status(400).json({ error: - "Invalid pagination parameters. Page must be >= 1 and limit must be between 1 and 100", + 'Invalid pagination parameters. Page must be >= 1 and limit must be between 1 and 100', }); } const dsnRecords = await getAllDsn(page, limit); res.json(dsnRecords); } catch (error) { - logger.error("Error fetching DSN records:", error); - res.status(500).json({ error: "Failed to fetch DSN records" }); + logger.error('Error fetching DSN records:', error); + res.status(500).json({ error: 'Failed to fetch DSN records' }); } }); -router.get("/memories/:cid", async (req, res) => { +router.get('/memories/:cid', async (req, res) => { try { const api = createAutoDriveApi({ - apiKey: config.DSN_API_KEY || "", + apiKey: config.DSN_API_KEY || '', }); const stream = await downloadObject(api, { cid: req.params.cid }); @@ -43,9 +43,7 @@ router.get("/memories/:cid", async (req, res) => { chunks.push(value); } - const allChunks = new Uint8Array( - chunks.reduce((acc, chunk) => acc + chunk.length, 0), - ); + const allChunks = new Uint8Array(chunks.reduce((acc, chunk) => acc + chunk.length, 0)); let position = 0; for (const chunk of chunks) { allChunks.set(chunk, position); @@ -57,8 +55,8 @@ router.get("/memories/:cid", async (req, res) => { const memoryData = JSON.parse(jsonString); res.json(memoryData); } catch (error) { - logger.error("Error fetching memory data:", error); - res.status(500).json({ error: "Failed to fetch memory data" }); + logger.error('Error fetching memory data:', error); + res.status(500).json({ error: 'Failed to fetch memory data' }); } }); diff --git a/auto-kol/agent/src/api/routes/health.ts b/auto-kol/agent/src/api/routes/health.ts index 2993381..d99c8cb 100644 --- a/auto-kol/agent/src/api/routes/health.ts +++ b/auto-kol/agent/src/api/routes/health.ts @@ -1,9 +1,9 @@ -import { Router } from "express"; +import { Router } from 'express'; const router = Router(); -router.get("/health", (_, res) => { - res.json({ status: "ok" }); +router.get('/health', (_, res) => { + res.json({ status: 'ok' }); }); export default router; diff --git a/auto-kol/agent/src/api/routes/responses.ts b/auto-kol/agent/src/api/routes/responses.ts index 1ccc0cc..fbf0155 100644 --- a/auto-kol/agent/src/api/routes/responses.ts +++ b/auto-kol/agent/src/api/routes/responses.ts @@ -1,21 +1,21 @@ -import { Router } from "express"; -import { createLogger } from "../../utils/logger.js"; -import { getAllPendingResponses } from "../../services/database/index.js"; +import { Router } from 'express'; +import { createLogger } from '../../utils/logger.js'; +import { getAllPendingResponses } from '../../services/database/index.js'; const router = Router(); -const logger = createLogger("responses-api"); +const logger = createLogger('responses-api'); -router.get("/responses/:id/workflow", async (req, res) => { +router.get('/responses/:id/workflow', async (req, res) => { try { const responses = await getAllPendingResponses(); - const response = responses.find((r) => r.id === req.params.id); + const response = responses.find(r => r.id === req.params.id); if (!response) { - return res.status(404).json({ error: "Response not found" }); + return res.status(404).json({ error: 'Response not found' }); } res.json(response.workflowState); } catch (error) { - logger.error("Error getting workflow state:", error); - res.status(500).json({ error: "Failed to get workflow state" }); + logger.error('Error getting workflow state:', error); + res.status(500).json({ error: 'Failed to get workflow state' }); } }); diff --git a/auto-kol/agent/src/api/routes/tweets.ts b/auto-kol/agent/src/api/routes/tweets.ts index a543fe1..7bae996 100644 --- a/auto-kol/agent/src/api/routes/tweets.ts +++ b/auto-kol/agent/src/api/routes/tweets.ts @@ -1,47 +1,42 @@ -import { Router } from "express"; -import { createLogger } from "../../utils/logger.js"; -import { - getSkippedTweets, - getSkippedTweetById, -} from "../../services/database/index.js"; -import { recheckSkippedTweet } from "../../database/index.js"; +import { Router } from 'express'; +import { createLogger } from '../../utils/logger.js'; +import { getSkippedTweets, getSkippedTweetById } from '../../services/database/index.js'; +import { recheckSkippedTweet } from '../../database/index.js'; const router = Router(); -const logger = createLogger("tweets-api"); +const logger = createLogger('tweets-api'); -router.get("/tweets/skipped", async (_, res) => { +router.get('/tweets/skipped', async (_, res) => { const skippedTweets = await getSkippedTweets(); res.json(skippedTweets); }); -router.get("/tweets/skipped/:id", async (req, res) => { +router.get('/tweets/skipped/:id', async (req, res) => { const skipped = await getSkippedTweetById(req.params.id); if (!skipped) { - return res.status(404).json({ error: "Skipped tweet not found" }); + return res.status(404).json({ error: 'Skipped tweet not found' }); } res.json(skipped); }); -router.post("/tweets/skipped/:id/queue", async (req, res) => { +router.post('/tweets/skipped/:id/queue', async (req, res) => { try { - logger.info( - `Received request to move skipped tweet to queue: ${req.params.id}`, - ); + logger.info(`Received request to move skipped tweet to queue: ${req.params.id}`); const skipped = await getSkippedTweetById(req.params.id); if (!skipped) { - return res.status(404).json({ error: "Skipped tweet not found" }); + return res.status(404).json({ error: 'Skipped tweet not found' }); } const recheck = await recheckSkippedTweet(req.params.id); if (!recheck) { - return res.status(404).json({ error: "Failed to recheck skipped tweet" }); + return res.status(404).json({ error: 'Failed to recheck skipped tweet' }); } res.json({ message: - "Skipped tweet rechecked and moved to queue - if will be processed in next workflow run", + 'Skipped tweet rechecked and moved to queue - if will be processed in next workflow run', }); } catch (error) { - logger.error("Error moving skipped tweet to queue:", error); - res.status(500).json({ error: "Failed to move tweet to queue" }); + logger.error('Error moving skipped tweet to queue:', error); + res.status(500).json({ error: 'Failed to move tweet to queue' }); } }); diff --git a/auto-kol/agent/src/config/index.ts b/auto-kol/agent/src/config/index.ts index 1fc7272..d2c6ace 100644 --- a/auto-kol/agent/src/config/index.ts +++ b/auto-kol/agent/src/config/index.ts @@ -1,7 +1,7 @@ -import dotenv from "dotenv"; -import path from "path"; -import { fileURLToPath } from "url"; -import { dirname } from "path"; +import dotenv from 'dotenv'; +import path from 'path'; +import { fileURLToPath } from 'url'; +import { dirname } from 'path'; // Get the equivalent of __dirname in ESM const __filename = fileURLToPath(import.meta.url); @@ -15,29 +15,28 @@ export const config = { TWITTER_PASSWORD: process.env.TWITTER_PASSWORD, // LLM Configuration - LLM_MODEL: process.env.LLM_MODEL || "gpt-4o-mini", + LLM_MODEL: process.env.LLM_MODEL || 'gpt-4o-mini', OPENAI_API_KEY: process.env.OPENAI_API_KEY, TEMPERATURE: 0.7, // Agent Configuration - CHECK_INTERVAL: - (Number(process.env.CHECK_INTERVAL_MINUTES) || 30) * 60 * 1000, - MEMORY_DIR: path.join(__dirname, "../../data/memory"), + CHECK_INTERVAL: (Number(process.env.CHECK_INTERVAL_MINUTES) || 30) * 60 * 1000, + MEMORY_DIR: path.join(__dirname, '../../data/memory'), // Server Configuration PORT: process.env.PORT || 3001, // Environment - NODE_ENV: process.env.NODE_ENV || "development", + NODE_ENV: process.env.NODE_ENV || 'development', // Chroma Configuration - CHROMA_DIR: path.join(__dirname, "../../data/chroma"), - CHROMA_URL: process.env.CHROMA_URL || "http://localhost:8000", + CHROMA_DIR: path.join(__dirname, '../../data/chroma'), + CHROMA_URL: process.env.CHROMA_URL || 'http://localhost:8000', // AutoDrive Configuration DSN_API_KEY: process.env.DSN_API_KEY, - DSN_UPLOAD: process.env.DSN_UPLOAD === "true", - DSN_SKIP_UPLOAD: process.env.DSN_SKIP_UPLOAD === "true", + DSN_UPLOAD: process.env.DSN_UPLOAD === 'true', + DSN_SKIP_UPLOAD: process.env.DSN_SKIP_UPLOAD === 'true', DSN_ENCRYPTION_PASSWORD: process.env.DSN_ENCRYPTION_PASSWORD, // CORS Configuration @@ -59,5 +58,5 @@ export const config = { RETRY_LIMIT: process.env.RETRY_LIMIT || 2, // POSTING TWEETS PERMISSION - POST_TWEETS: process.env.POST_TWEETS === "true", + POST_TWEETS: process.env.POST_TWEETS === 'true', }; diff --git a/auto-kol/agent/src/database/index.ts b/auto-kol/agent/src/database/index.ts index efd1961..4ca0746 100644 --- a/auto-kol/agent/src/database/index.ts +++ b/auto-kol/agent/src/database/index.ts @@ -1,15 +1,15 @@ -import sqlite3 from "sqlite3"; -import { open } from "sqlite"; -import fs from "fs/promises"; -import path from "path"; -import { createLogger } from "../utils/logger.js"; -import { KOL } from "../types/kol.js"; -import { fileURLToPath } from "url"; -import { dirname, join } from "path"; -import { Tweet } from "../types/twitter.js"; -import { SkippedTweet, PendingResponse } from "../types/queue.js"; - -const logger = createLogger("database"); +import sqlite3 from 'sqlite3'; +import { open } from 'sqlite'; +import fs from 'fs/promises'; +import path from 'path'; +import { createLogger } from '../utils/logger.js'; +import { KOL } from '../types/kol.js'; +import { fileURLToPath } from 'url'; +import { dirname, join } from 'path'; +import { Tweet } from '../types/twitter.js'; +import { SkippedTweet, PendingResponse } from '../types/queue.js'; + +const logger = createLogger('database'); let db: Awaited> | null = null; @@ -17,18 +17,18 @@ let db: Awaited> | null = null; export async function initializeDatabase() { if (!db) { try { - const dbDir = path.dirname("./data/engagement.db"); + const dbDir = path.dirname('./data/engagement.db'); await fs.mkdir(dbDir, { recursive: true }); db = await open({ - filename: "./data/engagement.db", + filename: './data/engagement.db', driver: sqlite3.Database, }); - await db.run("PRAGMA foreign_keys = ON"); + await db.run('PRAGMA foreign_keys = ON'); // Test database connection - await db.get("SELECT 1"); + await db.get('SELECT 1'); } catch (error) { db = null; throw new Error(`Failed to initialize database: ${error}`); @@ -48,7 +48,7 @@ export async function initializeSchema() { const db = await initializeDatabase(); try { - await db.run("BEGIN TRANSACTION"); + await db.run('BEGIN TRANSACTION'); // Check if tables exist first const tables = await db.all(` @@ -63,33 +63,31 @@ export async function initializeSchema() { ) `); - const existingTables = new Set(tables.map((t) => t.name)); + const existingTables = new Set(tables.map(t => t.name)); const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); - const schemaPath = join(__dirname, "schema.sql"); - const schema = await fs.readFile(schemaPath, "utf-8"); + const schemaPath = join(__dirname, 'schema.sql'); + const schema = await fs.readFile(schemaPath, 'utf-8'); const statements = schema - .split(";") - .map((s) => s.trim()) - .filter((s) => s.length > 0); + .split(';') + .map(s => s.trim()) + .filter(s => s.length > 0); for (const statement of statements) { - const tableName = statement.match( - /CREATE TABLE (?:IF NOT EXISTS )?([^\s(]+)/i, - )?.[1]; + const tableName = statement.match(/CREATE TABLE (?:IF NOT EXISTS )?([^\s(]+)/i)?.[1]; if (tableName && !existingTables.has(tableName)) { await db.run(statement); logger.info(`Created table: ${tableName}`); } } - await db.run("COMMIT"); - logger.info("Schema initialization completed successfully"); + await db.run('COMMIT'); + logger.info('Schema initialization completed successfully'); } catch (error) { - await db.run("ROLLBACK"); - logger.error("Failed to initialize schema:", error); + await db.run('ROLLBACK'); + logger.error('Failed to initialize schema:', error); throw new Error(`Failed to initialize schema: ${error}`); } } @@ -117,10 +115,7 @@ export async function addKOL(kol: { logger.info(`Added KOL account: ${kol.username}`); } catch (error: any) { - if ( - error?.code === "SQLITE_CONSTRAINT" && - error?.message?.includes("UNIQUE") - ) { + if (error?.code === 'SQLITE_CONSTRAINT' && error?.message?.includes('UNIQUE')) { logger.warn(`KOL account already exists: ${kol.username}`); return; } @@ -138,23 +133,21 @@ export async function getKOLAccounts(): Promise { ORDER BY created_at DESC `); - return accounts.map((account) => ({ + return accounts.map(account => ({ id: account.id, username: account.username, created_at: new Date(account.created_at), updatedAt: new Date(account.updated_at), })); } catch (error) { - logger.error("Failed to get KOL accounts:", error); + logger.error('Failed to get KOL accounts:', error); throw error; } } export async function isKOLExists(username: string): Promise { const db = await initializeDatabase(); - const kol = await db.get(`SELECT * FROM kol_accounts WHERE username = ?`, [ - username, - ]); + const kol = await db.get(`SELECT * FROM kol_accounts WHERE username = ?`, [username]); return kol !== undefined; } @@ -176,7 +169,7 @@ export async function addResponse(response: PendingResponse) { response.strategy, response.estimatedImpact, response.confidence, - "pending", + 'pending', ], ); } @@ -193,7 +186,7 @@ export async function updateResponse(response: PendingResponse) { estimated_impact = ?, confidence = ?, updated_at = CURRENT_TIMESTAMP - WHERE ${response.id ? "id = ?" : "tweet_id = ?"} + WHERE ${response.id ? 'id = ?' : 'tweet_id = ?'} `, [ response.content, @@ -222,9 +215,7 @@ export async function getPendingResponses() { `); } -export async function getResponseByTweetId( - tweet_id: string, -): Promise { +export async function getResponseByTweetId(tweet_id: string): Promise { const db = await initializeDatabase(); const response = await db.all( ` @@ -235,9 +226,7 @@ export async function getResponseByTweetId( return response[0] as PendingResponse; } -export async function getPendingResponsesByTweetId( - id: string, -): Promise { +export async function getPendingResponsesByTweetId(id: string): Promise { const db = await initializeDatabase(); const pending_response = await db.all( ` @@ -248,13 +237,10 @@ export async function getPendingResponsesByTweetId( return pending_response[0] as PendingResponse; } -export async function updateResponseStatus( - id: string, - status: "approved" | "rejected", -) { +export async function updateResponseStatus(id: string, status: 'approved' | 'rejected') { const db = await initializeDatabase(); - await db.run("BEGIN TRANSACTION"); + await db.run('BEGIN TRANSACTION'); try { await db.run( @@ -265,17 +251,17 @@ export async function updateResponseStatus( `, [status, id], ); - await db.run("COMMIT"); + await db.run('COMMIT'); logger.info(`Updated response status: ${id}`); } catch (error) { - await db.run("ROLLBACK"); + await db.run('ROLLBACK'); throw error; } } export async function updateResponseStatusByTweetId( tweet_id: string, - status: "approved" | "rejected", + status: 'approved' | 'rejected', ) { const db = await initializeDatabase(); return db.run( @@ -302,19 +288,11 @@ export async function addTweet(tweet: { created_at ) VALUES (?, ?, ?, ?, ?) `, - [ - tweet.id, - tweet.author_id, - tweet.author_username, - tweet.content, - tweet.created_at, - ], + [tweet.id, tweet.author_id, tweet.author_username, tweet.content, tweet.created_at], ); } -export async function getTweetById( - tweetId: string, -): Promise { +export async function getTweetById(tweetId: string): Promise { const db = await initializeDatabase(); const tweet = await db.get(`SELECT * FROM tweets WHERE id = ?`, [tweetId]); return tweet as Tweet; @@ -351,29 +329,19 @@ export async function getSkippedTweets() { `); } -export async function getSkippedTweetById( - skippedId: string, -): Promise { +export async function getSkippedTweetById(skippedId: string): Promise { const db = await initializeDatabase(); - const skipped = await db.get(`SELECT * FROM skipped_tweets WHERE id = ?`, [ - skippedId, - ]); + const skipped = await db.get(`SELECT * FROM skipped_tweets WHERE id = ?`, [skippedId]); return skipped; } export async function recheckSkippedTweet(skippedId: string): Promise { const db = await initializeDatabase(); - const result = await db.run( - `UPDATE skipped_tweets SET recheck = TRUE WHERE id = ?`, - [skippedId], - ); + const result = await db.run(`UPDATE skipped_tweets SET recheck = TRUE WHERE id = ?`, [skippedId]); return result !== undefined; } -export async function flagBackSkippedTweet( - skippedId: string, - reason: string, -): Promise { +export async function flagBackSkippedTweet(skippedId: string, reason: string): Promise { const db = await initializeDatabase(); const result = await db.run( `UPDATE skipped_tweets SET recheck = FALSE, reason = ? WHERE id = ?`, @@ -384,18 +352,12 @@ export async function flagBackSkippedTweet( export async function getAllSkippedTweetsToRecheck(): Promise { const db = await initializeDatabase(); - const recheckTweets = await db.all( - `SELECT * FROM skipped_tweets WHERE recheck = TRUE`, - ); + const recheckTweets = await db.all(`SELECT * FROM skipped_tweets WHERE recheck = TRUE`); return recheckTweets; } ///////////DSN/////////// -export async function addDsn(dsn: { - id: string; - tweetId: string; - cid: string; -}) { +export async function addDsn(dsn: { id: string; tweetId: string; cid: string }) { return db?.run( ` INSERT INTO dsn (id, tweet_id, cid) @@ -483,16 +445,14 @@ export async function getAllDsn(page: number = 1, limit: number = 10) { }, }; } catch (error) { - logger.error("Failed to get all DSN records", error); + logger.error('Failed to get all DSN records', error); throw error; } } export async function getLastDsnCid(): Promise { - const dsn = await db?.get( - `SELECT cid FROM dsn ORDER BY created_at DESC LIMIT 1`, - ); - return dsn?.cid || ""; + const dsn = await db?.get(`SELECT cid FROM dsn ORDER BY created_at DESC LIMIT 1`); + return dsn?.cid || ''; } ///////////MENTIONS/////////// @@ -510,8 +470,6 @@ export async function addMention(mention: { latest_id: string }) { } export async function getLatestMentionId(): Promise { - const mention = await db?.get( - `SELECT latest_id FROM mentions ORDER BY updated_at DESC LIMIT 1`, - ); - return mention?.latest_id || ""; + const mention = await db?.get(`SELECT latest_id FROM mentions ORDER BY updated_at DESC LIMIT 1`); + return mention?.latest_id || ''; } diff --git a/auto-kol/agent/src/index.ts b/auto-kol/agent/src/index.ts index a9576c0..dc42698 100644 --- a/auto-kol/agent/src/index.ts +++ b/auto-kol/agent/src/index.ts @@ -1,11 +1,11 @@ -import express from "express"; -import { config } from "./config/index.js"; -import { createLogger } from "./utils/logger.js"; -import { runWorkflow } from "./services/agents/workflow.js"; -import { initializeSchema } from "./database/index.js"; -import apiRoutes from "./api/index.js"; -import { corsMiddleware } from "./api/middleware/cors.js"; -const logger = createLogger("app"); +import express from 'express'; +import { config } from './config/index.js'; +import { createLogger } from './utils/logger.js'; +import { runWorkflow } from './services/agents/workflow.js'; +import { initializeSchema } from './database/index.js'; +import apiRoutes from './api/index.js'; +import { corsMiddleware } from './api/middleware/cors.js'; +const logger = createLogger('app'); const app = express(); app.use(corsMiddleware); @@ -16,9 +16,9 @@ app.use(apiRoutes); const startWorkflowPolling = async () => { try { await runWorkflow(); - logger.info("Workflow execution completed successfully"); + logger.info('Workflow execution completed successfully'); } catch (error) { - logger.error("Error running workflow:", error); + logger.error('Error running workflow:', error); } }; @@ -32,12 +32,12 @@ const main = async () => { await startWorkflowPolling(); setInterval(startWorkflowPolling, config.CHECK_INTERVAL); - logger.info("Application started successfully", { + logger.info('Application started successfully', { checkInterval: config.CHECK_INTERVAL, port: config.PORT, }); } catch (error) { - logger.error("Failed to start application:", error); + logger.error('Failed to start application:', error); process.exit(1); } }; diff --git a/auto-kol/agent/src/schemas/workflow.ts b/auto-kol/agent/src/schemas/workflow.ts index 8fe3f87..07a196c 100644 --- a/auto-kol/agent/src/schemas/workflow.ts +++ b/auto-kol/agent/src/schemas/workflow.ts @@ -1,4 +1,4 @@ -import { z } from "zod"; +import { z } from 'zod'; export const tweetSearchSchema = z.object({ tweets: z.array( diff --git a/auto-kol/agent/src/services/agents/nodes.ts b/auto-kol/agent/src/services/agents/nodes.ts index b16303c..5dbae9f 100644 --- a/auto-kol/agent/src/services/agents/nodes.ts +++ b/auto-kol/agent/src/services/agents/nodes.ts @@ -1,12 +1,12 @@ -import { WorkflowConfig } from "./workflow.js"; -import { createSearchNode } from "./nodes/searchNode.js"; -import { createEngagementNode } from "./nodes/engagementNode.js"; -import { createToneAnalysisNode } from "./nodes/toneAnalysisNode.js"; -import { createResponseGenerationNode } from "./nodes/responseGenerationNode.js"; -import { createRecheckSkippedNode } from "./nodes/recheckSkippedNode.js"; -import { createTimelineNode } from "./nodes/timelineNode.js"; -import { createMentionNode } from "./nodes/mentionNode.js"; -import { createAutoApprovalNode } from "./nodes/autoApprovalNode.js"; +import { WorkflowConfig } from './workflow.js'; +import { createSearchNode } from './nodes/searchNode.js'; +import { createEngagementNode } from './nodes/engagementNode.js'; +import { createToneAnalysisNode } from './nodes/toneAnalysisNode.js'; +import { createResponseGenerationNode } from './nodes/responseGenerationNode.js'; +import { createRecheckSkippedNode } from './nodes/recheckSkippedNode.js'; +import { createTimelineNode } from './nodes/timelineNode.js'; +import { createMentionNode } from './nodes/mentionNode.js'; +import { createAutoApprovalNode } from './nodes/autoApprovalNode.js'; export const createNodes = async (config: WorkflowConfig) => { ///////////MENTIONS/////////// diff --git a/auto-kol/agent/src/services/agents/nodes/autoApprovalNode.ts b/auto-kol/agent/src/services/agents/nodes/autoApprovalNode.ts index 141847e..997a306 100644 --- a/auto-kol/agent/src/services/agents/nodes/autoApprovalNode.ts +++ b/auto-kol/agent/src/services/agents/nodes/autoApprovalNode.ts @@ -1,25 +1,22 @@ -import { AIMessage } from "@langchain/core/messages"; -import { State, logger, parseMessageContent } from "../workflow.js"; -import * as prompts from "../prompts.js"; -import { WorkflowConfig } from "../workflow.js"; -import { - getLastDsnCid, - updateResponseStatusByTweetId, -} from "../../../database/index.js"; -import { uploadToDsn } from "../../../utils/dsn.js"; -import { config as globalConfig } from "../../../config/index.js"; -import { ResponseStatus } from "../../../types/queue.js"; +import { AIMessage } from '@langchain/core/messages'; +import { State, logger, parseMessageContent } from '../workflow.js'; +import * as prompts from '../prompts.js'; +import { WorkflowConfig } from '../workflow.js'; +import { getLastDsnCid, updateResponseStatusByTweetId } from '../../../database/index.js'; +import { uploadToDsn } from '../../../utils/dsn.js'; +import { config as globalConfig } from '../../../config/index.js'; +import { ResponseStatus } from '../../../types/queue.js'; export const createAutoApprovalNode = (config: WorkflowConfig) => { return async (state: typeof State.State) => { - logger.info("Auto Approval Node - Evaluating pending responses"); + logger.info('Auto Approval Node - Evaluating pending responses'); try { const lastMessage = state.messages[state.messages.length - 1]; const parsedContent = parseMessageContent(lastMessage.content); const { tweets, currentTweetIndex, batchToFeedback } = parsedContent; if (!batchToFeedback.length) { - logger.info("No pending responses found"); + logger.info('No pending responses found'); return { messages: [ new AIMessage({ @@ -35,7 +32,7 @@ export const createAutoApprovalNode = (config: WorkflowConfig) => { const processedResponses = []; for (const response of batchToFeedback) { - logger.info("Processing response", { + logger.info('Processing response', { tweetId: response.tweet.id, retry: response.retry, }); @@ -53,13 +50,10 @@ export const createAutoApprovalNode = (config: WorkflowConfig) => { if (approval.approved) { response.type = ResponseStatus.APPROVED; - await updateResponseStatusByTweetId( - response.tweet.id, - ResponseStatus.APPROVED, - ); + await updateResponseStatusByTweetId(response.tweet.id, ResponseStatus.APPROVED); if (globalConfig.POST_TWEETS) { - logger.info("Sending tweet", { + logger.info('Sending tweet', { response: response.response, tweetId: response.tweet.id, }); @@ -68,7 +62,7 @@ export const createAutoApprovalNode = (config: WorkflowConfig) => { response.response, response.tweet.id, ); - logger.info("Tweet sent", { + logger.info('Tweet sent', { sendTweetResponse, }); } @@ -80,13 +74,10 @@ export const createAutoApprovalNode = (config: WorkflowConfig) => { } } else if (response.retry > globalConfig.RETRY_LIMIT) { response.type = ResponseStatus.REJECTED; - logger.info("Rejecting tweet", { + logger.info('Rejecting tweet', { tweetId: response.tweet.id, }); - await updateResponseStatusByTweetId( - response.tweet.id, - ResponseStatus.REJECTED, - ); + await updateResponseStatusByTweetId(response.tweet.id, ResponseStatus.REJECTED); if (globalConfig.DSN_UPLOAD) { await uploadToDsn({ data: response, @@ -107,7 +98,7 @@ export const createAutoApprovalNode = (config: WorkflowConfig) => { }, ], }, - feedbackDecision: "reject", + feedbackDecision: 'reject', }); } } @@ -125,7 +116,7 @@ export const createAutoApprovalNode = (config: WorkflowConfig) => { ], }; } catch (error) { - logger.error("Error in auto approval node:", error); + logger.error('Error in auto approval node:', error); return { messages: [] }; } }; diff --git a/auto-kol/agent/src/services/agents/nodes/engagementNode.ts b/auto-kol/agent/src/services/agents/nodes/engagementNode.ts index ba9893a..dc8ea7b 100644 --- a/auto-kol/agent/src/services/agents/nodes/engagementNode.ts +++ b/auto-kol/agent/src/services/agents/nodes/engagementNode.ts @@ -1,32 +1,32 @@ -import { AIMessage } from "@langchain/core/messages"; -import { State, logger, parseMessageContent } from "../workflow.js"; -import * as prompts from "../prompts.js"; -import { uploadToDsn } from "../../../utils/dsn.js"; -import { getLastDsnCid } from "../../../database/index.js"; -import { WorkflowConfig } from "../workflow.js"; -import { config as globalConfig } from "../../../config/index.js"; -import { ResponseStatus } from "../../../types/queue.js"; +import { AIMessage } from '@langchain/core/messages'; +import { State, logger, parseMessageContent } from '../workflow.js'; +import * as prompts from '../prompts.js'; +import { uploadToDsn } from '../../../utils/dsn.js'; +import { getLastDsnCid } from '../../../database/index.js'; +import { WorkflowConfig } from '../workflow.js'; +import { config as globalConfig } from '../../../config/index.js'; +import { ResponseStatus } from '../../../types/queue.js'; const handleSkippedTweet = async (tweet: any, decision: any, config: any) => { - logger.info("Skipping engagement for tweet", { + logger.info('Skipping engagement for tweet', { tweetId: tweet.id, reason: decision.reason, }); await config.toolNode.invoke({ messages: [ new AIMessage({ - content: "", + content: '', tool_calls: [ { - name: "queue_skipped", + name: 'queue_skipped', args: { tweet, reason: decision.reason, priority: decision.priority || 0, workflowState: { decision }, }, - id: "skip_call", - type: "tool_call", + id: 'skip_call', + type: 'tool_call', }, ], }), @@ -51,7 +51,7 @@ const handleSkippedTweet = async (tweet: any, decision: any, config: any) => { export const createEngagementNode = (config: WorkflowConfig) => { return async (state: typeof State.State) => { - logger.info("Engagement Node - Starting evaluation"); + logger.info('Engagement Node - Starting evaluation'); try { const lastMessage = state.messages[state.messages.length - 1]; const parsedContent = parseMessageContent(lastMessage.content); @@ -59,9 +59,7 @@ export const createEngagementNode = (config: WorkflowConfig) => { logger.info(`Current tweet index: ${parsedContent?.currentTweetIndex}`); if (pendingEngagements.length > 0) { - logger.info( - `number of pending engagements: ${pendingEngagements.length}`, - ); + logger.info(`number of pending engagements: ${pendingEngagements.length}`); return { messages: [ new AIMessage({ @@ -82,13 +80,10 @@ export const createEngagementNode = (config: WorkflowConfig) => { const BATCH_SIZE = globalConfig.ENGAGEMENT_BATCH_SIZE; const startIdx = parsedContent.currentTweetIndex || 0; const proposedEndIdx = Number(startIdx) + Number(BATCH_SIZE); - const endIdx = Math.min( - proposedEndIdx, - parsedContent.tweets?.length || 0, - ); + const endIdx = Math.min(proposedEndIdx, parsedContent.tweets?.length || 0); const batchTweets = parsedContent.tweets?.slice(startIdx, endIdx) || []; - logger.info("Processing batch of tweets", { + logger.info('Processing batch of tweets', { batchSize: batchTweets.length, startIndex: startIdx, endIndex: endIdx, @@ -98,7 +93,7 @@ export const createEngagementNode = (config: WorkflowConfig) => { const processedResults = await Promise.all( batchTweets.map(async (tweet: any) => { if (state.processedTweets.has(tweet.id)) { - return { tweet, status: "alreadyProcessed" }; + return { tweet, status: 'alreadyProcessed' }; } const decision = await prompts.engagementPrompt .pipe(config.llms.decision) @@ -107,17 +102,17 @@ export const createEngagementNode = (config: WorkflowConfig) => { tweet: tweet.text, thread: tweet.thread || [], }) - .catch((error) => { - logger.error("Error in engagement node:", error); + .catch(error => { + logger.error('Error in engagement node:', error); return { shouldEngage: false, - reason: "Error in engagement node", + reason: 'Error in engagement node', priority: 0, confidence: 0, }; }); - return { tweet, decision, status: "processing" }; + return { tweet, decision, status: 'processing' }; }), ); @@ -126,12 +121,12 @@ export const createEngagementNode = (config: WorkflowConfig) => { for (const result of processedResults) { newProcessedTweets.add(result.tweet.id); - if (result.status === "processing" && result.decision?.shouldEngage) { + if (result.status === 'processing' && result.decision?.shouldEngage) { tweetsToEngage.push({ tweet: result.tweet, decision: result.decision, }); - } else if (result.status === "processing") { + } else if (result.status === 'processing') { await handleSkippedTweet(result.tweet, result.decision, config); } } @@ -151,7 +146,7 @@ export const createEngagementNode = (config: WorkflowConfig) => { processedTweets: newProcessedTweets, }; } catch (error) { - logger.error("Error in engagement node:", error); + logger.error('Error in engagement node:', error); return { messages: [] }; } }; diff --git a/auto-kol/agent/src/services/agents/nodes/mentionNode.ts b/auto-kol/agent/src/services/agents/nodes/mentionNode.ts index 2ce1bd0..9a8d6ff 100644 --- a/auto-kol/agent/src/services/agents/nodes/mentionNode.ts +++ b/auto-kol/agent/src/services/agents/nodes/mentionNode.ts @@ -1,22 +1,22 @@ -import { AIMessage } from "@langchain/core/messages"; -import { parseMessageContent, WorkflowConfig } from "../workflow.js"; -import { logger } from "../workflow.js"; -import { State } from "../workflow.js"; -import { tweetSearchSchema } from "../../../schemas/workflow.js"; +import { AIMessage } from '@langchain/core/messages'; +import { parseMessageContent, WorkflowConfig } from '../workflow.js'; +import { logger } from '../workflow.js'; +import { State } from '../workflow.js'; +import { tweetSearchSchema } from '../../../schemas/workflow.js'; export const createMentionNode = (config: WorkflowConfig) => { return async (state: typeof State.State) => { - logger.info("Mention Node - Fetching recent mentions"); + logger.info('Mention Node - Fetching recent mentions'); const toolResponse = await config.toolNode.invoke({ messages: [ new AIMessage({ - content: "", + content: '', tool_calls: [ { - name: "fetch_mentions", + name: 'fetch_mentions', args: {}, - id: "fetch_mentions_call", - type: "tool_call", + id: 'fetch_mentions_call', + type: 'tool_call', }, ], }), diff --git a/auto-kol/agent/src/services/agents/nodes/recheckSkippedNode.ts b/auto-kol/agent/src/services/agents/nodes/recheckSkippedNode.ts index 7fd7028..0678c77 100644 --- a/auto-kol/agent/src/services/agents/nodes/recheckSkippedNode.ts +++ b/auto-kol/agent/src/services/agents/nodes/recheckSkippedNode.ts @@ -1,15 +1,12 @@ -import { AIMessage } from "@langchain/core/messages"; -import { State, logger, parseMessageContent } from "../workflow.js"; -import * as prompts from "../prompts.js"; -import { - flagBackSkippedTweet, - getAllSkippedTweetsToRecheck, -} from "../../../database/index.js"; -import { WorkflowConfig } from "../workflow.js"; +import { AIMessage } from '@langchain/core/messages'; +import { State, logger, parseMessageContent } from '../workflow.js'; +import * as prompts from '../prompts.js'; +import { flagBackSkippedTweet, getAllSkippedTweetsToRecheck } from '../../../database/index.js'; +import { WorkflowConfig } from '../workflow.js'; export const createRecheckSkippedNode = (config: WorkflowConfig) => { return async (state: typeof State.State) => { - logger.info("Recheck Skipped Node - Reviewing previously skipped tweets"); + logger.info('Recheck Skipped Node - Reviewing previously skipped tweets'); try { const lastMessage = state.messages[state.messages.length - 1]; const parsedContent = parseMessageContent(lastMessage.content); @@ -19,7 +16,7 @@ export const createRecheckSkippedNode = (config: WorkflowConfig) => { const skippedTweets = await getAllSkippedTweetsToRecheck(); if (!skippedTweets || skippedTweets.length === 0) { - logger.info("No skipped tweets to recheck"); + logger.info('No skipped tweets to recheck'); return { messages: [ new AIMessage({ @@ -44,7 +41,7 @@ export const createRecheckSkippedNode = (config: WorkflowConfig) => { .pipe(prompts.engagementParser) .invoke({ tweet: tweet.text }); - logger.info("Recheck decision:", { tweetId: tweet.id, decision }); + logger.info('Recheck decision:', { tweetId: tweet.id, decision }); if (decision.shouldEngage) { processedTweets.push({ @@ -54,7 +51,7 @@ export const createRecheckSkippedNode = (config: WorkflowConfig) => { } else { const flagged = await flagBackSkippedTweet(tweet.id, decision.reason); if (!flagged) { - logger.info("Failed to flag back skipped tweet:", { + logger.info('Failed to flag back skipped tweet:', { tweetId: tweet.id, }); } @@ -62,7 +59,7 @@ export const createRecheckSkippedNode = (config: WorkflowConfig) => { } if (processedTweets.length === 0) { - logger.info("No skipped tweets passed recheck"); + logger.info('No skipped tweets passed recheck'); return { messages: [ new AIMessage({ @@ -93,7 +90,7 @@ export const createRecheckSkippedNode = (config: WorkflowConfig) => { ], }; } catch (error) { - logger.error("Error in recheck skipped node:", error); + logger.error('Error in recheck skipped node:', error); return { messages: [] }; } }; diff --git a/auto-kol/agent/src/services/agents/nodes/responseGenerationNode.ts b/auto-kol/agent/src/services/agents/nodes/responseGenerationNode.ts index 5303680..ab33372 100644 --- a/auto-kol/agent/src/services/agents/nodes/responseGenerationNode.ts +++ b/auto-kol/agent/src/services/agents/nodes/responseGenerationNode.ts @@ -1,24 +1,21 @@ -import { AIMessage } from "@langchain/core/messages"; -import { State, logger, parseMessageContent } from "../workflow.js"; -import * as prompts from "../prompts.js"; -import { WorkflowConfig } from "../workflow.js"; -import { ResponseStatus } from "../../../types/queue.js"; +import { AIMessage } from '@langchain/core/messages'; +import { State, logger, parseMessageContent } from '../workflow.js'; +import * as prompts from '../prompts.js'; +import { WorkflowConfig } from '../workflow.js'; +import { ResponseStatus } from '../../../types/queue.js'; export const createResponseGenerationNode = (config: WorkflowConfig) => { return async (state: typeof State.State) => { - logger.info("Response Generation Node - Creating response strategy"); + logger.info('Response Generation Node - Creating response strategy'); try { const lastMessage = state.messages[state.messages.length - 1]; const parsedContent = parseMessageContent(lastMessage.content); const batchToRespond = parsedContent.batchToRespond || []; const batchToFeedback: any[] = []; - logger.info( - `Processing batch of ${batchToRespond.length} tweets for response generation`, - { - hasRejectedResponses: parsedContent.fromAutoApproval, - }, - ); + logger.info(`Processing batch of ${batchToRespond.length} tweets for response generation`, { + hasRejectedResponses: parsedContent.fromAutoApproval, + }); await Promise.all( batchToRespond.map(async (item: any) => { @@ -32,38 +29,34 @@ export const createResponseGenerationNode = (config: WorkflowConfig) => { if (parsedContent.fromAutoApproval) { item.retry = (item.retry || 0) + 1; - logger.info("Regenerating response due to rejection:", { + logger.info('Regenerating response due to rejection:', { retry: item.retry, }); } else { item.retry = 0; } - const lastFeedback = - workflowState?.autoFeedback[workflowState?.autoFeedback.length - 1]; + const lastFeedback = workflowState?.autoFeedback[workflowState?.autoFeedback.length - 1]; const rejectionInstructions = lastFeedback ? prompts.formatRejectionInstructions(lastFeedback.reason) - : ""; + : ''; const rejectionFeedback = lastFeedback - ? prompts.formatRejectionFeedback( - lastFeedback.reason, - lastFeedback.suggestedChanges, - ) - : ""; + ? prompts.formatRejectionFeedback(lastFeedback.reason, lastFeedback.suggestedChanges) + : ''; const similarTweetsResponse = await config.toolNode.invoke({ messages: [ new AIMessage({ - content: "", + content: '', tool_calls: [ { - name: "search_similar_tweets", + name: 'search_similar_tweets', args: { query: `author:${tweet.author_username} ${tweet.text}`, k: 5, }, - id: "similar_tweets_call", - type: "tool_call", + id: 'similar_tweets_call', + type: 'tool_call', }, ], }), @@ -71,9 +64,7 @@ export const createResponseGenerationNode = (config: WorkflowConfig) => { }); const similarTweets = parseMessageContent( - similarTweetsResponse.messages[ - similarTweetsResponse.messages.length - 1 - ].content, + similarTweetsResponse.messages[similarTweetsResponse.messages.length - 1].content, ); const responseStrategy = await prompts.responsePrompt @@ -81,16 +72,12 @@ export const createResponseGenerationNode = (config: WorkflowConfig) => { .pipe(prompts.responseParser) .invoke({ tweet: tweet.text, - tone: - toneAnalysis?.suggestedTone || - workflowState?.toneAnalysis?.suggestedTone, + tone: toneAnalysis?.suggestedTone || workflowState?.toneAnalysis?.suggestedTone, author: tweet.author_username, similarTweets: JSON.stringify(similarTweets.similar_tweets), thread: JSON.stringify(tweet.thread || []), previousResponse: - workflowState?.autoFeedback[ - workflowState?.autoFeedback.length - 1 - ]?.response || "", + workflowState?.autoFeedback[workflowState?.autoFeedback.length - 1]?.response || '', rejectionFeedback, rejectionInstructions, }); @@ -128,13 +115,13 @@ export const createResponseGenerationNode = (config: WorkflowConfig) => { const addResponse = await config.toolNode.invoke({ messages: [ new AIMessage({ - content: "", + content: '', tool_calls: [ { - name: "add_response", + name: 'add_response', args, - id: "add_response_call", - type: "tool_call", + id: 'add_response_call', + type: 'tool_call', }, ], }), @@ -145,13 +132,13 @@ export const createResponseGenerationNode = (config: WorkflowConfig) => { const updateResponse = await config.toolNode.invoke({ messages: [ new AIMessage({ - content: "", + content: '', tool_calls: [ { - name: "update_response", + name: 'update_response', args, - id: "update_response_call", - type: "tool_call", + id: 'update_response_call', + type: 'tool_call', }, ], }), @@ -174,12 +161,10 @@ export const createResponseGenerationNode = (config: WorkflowConfig) => { }), }), ], - processedTweets: new Set( - batchToRespond.map((item: any) => item.tweet.id), - ), + processedTweets: new Set(batchToRespond.map((item: any) => item.tweet.id)), }; } catch (error) { - logger.error("Error in response generation node:", error); + logger.error('Error in response generation node:', error); return { messages: [] }; } }; diff --git a/auto-kol/agent/src/services/agents/nodes/searchNode.ts b/auto-kol/agent/src/services/agents/nodes/searchNode.ts index 3a7abf2..ddebdf2 100644 --- a/auto-kol/agent/src/services/agents/nodes/searchNode.ts +++ b/auto-kol/agent/src/services/agents/nodes/searchNode.ts @@ -1,56 +1,54 @@ -import { AIMessage } from "@langchain/core/messages"; -import { State, logger, parseMessageContent } from "../workflow.js"; -import { tweetSearchSchema } from "../../../schemas/workflow.js"; -import { ChromaService } from "../../vectorstore/chroma.js"; -import * as db from "../../database/index.js"; -import { WorkflowConfig } from "../workflow.js"; +import { AIMessage } from '@langchain/core/messages'; +import { State, logger, parseMessageContent } from '../workflow.js'; +import { tweetSearchSchema } from '../../../schemas/workflow.js'; +import { ChromaService } from '../../vectorstore/chroma.js'; +import * as db from '../../database/index.js'; +import { WorkflowConfig } from '../workflow.js'; export const createSearchNode = (config: WorkflowConfig) => { return async (state: typeof State.State) => { - logger.info("Search Node - Fetching recent tweets"); + logger.info('Search Node - Fetching recent tweets'); const existingTweets = state.messages.length > 0 - ? parseMessageContent(state.messages[state.messages.length - 1].content) - .tweets + ? parseMessageContent(state.messages[state.messages.length - 1].content).tweets : []; logger.info(`Existing tweets: ${existingTweets.length}`); try { - logger.info("Last processed id:", state.lastProcessedId); + logger.info('Last processed id:', state.lastProcessedId); const toolResponse = await config.toolNode.invoke({ messages: [ new AIMessage({ - content: "", + content: '', tool_calls: [ { - name: "search_recent_tweets", + name: 'search_recent_tweets', args: { lastProcessedId: state.lastProcessedId || undefined, }, - id: "tool_call_id", - type: "tool_call", + id: 'tool_call_id', + type: 'tool_call', }, ], }), ], }); - const lastMessage = - toolResponse.messages[toolResponse.messages.length - 1]; + const lastMessage = toolResponse.messages[toolResponse.messages.length - 1]; let searchResult; - if (typeof lastMessage.content === "string") { + if (typeof lastMessage.content === 'string') { try { searchResult = JSON.parse(lastMessage.content); - logger.info("Parsed search result:", searchResult); + logger.info('Parsed search result:', searchResult); } catch (error) { - logger.error("Failed to parse search result:", error); + logger.error('Failed to parse search result:', error); searchResult = { tweets: [], lastProcessedId: null }; } } else { searchResult = lastMessage.content; - logger.info("Non-string search result:", searchResult); + logger.info('Non-string search result:', searchResult); } const newTweets = [...existingTweets]; @@ -68,9 +66,7 @@ export const createSearchNode = (config: WorkflowConfig) => { const chromaService = await ChromaService.getInstance(); if (validatedResult.tweets.length > 0) { - await Promise.all( - validatedResult.tweets.map((tweet) => chromaService.addTweet(tweet)), - ); + await Promise.all(validatedResult.tweets.map(tweet => chromaService.addTweet(tweet))); } logger.info(`Found ${validatedResult.tweets.length} tweets`); @@ -88,7 +84,7 @@ export const createSearchNode = (config: WorkflowConfig) => { lastProcessedId: validatedResult.lastProcessedId || undefined, }; } catch (error) { - logger.error("Error in search node:", error); + logger.error('Error in search node:', error); const emptyResult = { tweets: [], lastProcessedId: null, diff --git a/auto-kol/agent/src/services/agents/nodes/timelineNode.ts b/auto-kol/agent/src/services/agents/nodes/timelineNode.ts index 5d8daf6..535daf6 100644 --- a/auto-kol/agent/src/services/agents/nodes/timelineNode.ts +++ b/auto-kol/agent/src/services/agents/nodes/timelineNode.ts @@ -1,44 +1,41 @@ -import { AIMessage } from "@langchain/core/messages"; -import { parseMessageContent, WorkflowConfig } from "../workflow.js"; -import { logger } from "../workflow.js"; -import { State } from "../workflow.js"; -import { tweetSearchSchema } from "../../../schemas/workflow.js"; -import * as db from "../../database/index.js"; +import { AIMessage } from '@langchain/core/messages'; +import { parseMessageContent, WorkflowConfig } from '../workflow.js'; +import { logger } from '../workflow.js'; +import { State } from '../workflow.js'; +import { tweetSearchSchema } from '../../../schemas/workflow.js'; +import * as db from '../../database/index.js'; export const createTimelineNode = (config: WorkflowConfig) => { return async (state: typeof State.State) => { - logger.info("Timeline Node - Fetching recent tweets"); + logger.info('Timeline Node - Fetching recent tweets'); const existingTweets = state.messages.length > 0 - ? parseMessageContent(state.messages[state.messages.length - 1].content) - .tweets + ? parseMessageContent(state.messages[state.messages.length - 1].content).tweets : []; logger.info(`Existing tweets: ${existingTweets.length}`); const toolResponse = await config.toolNode.invoke({ messages: [ new AIMessage({ - content: "", + content: '', tool_calls: [ { - name: "fetch_timeline", + name: 'fetch_timeline', args: {}, - id: "fetch_timeline_call", - type: "tool_call", + id: 'fetch_timeline_call', + type: 'tool_call', }, ], }), ], }); - logger.info("Tool response received:", { + logger.info('Tool response received:', { messageCount: toolResponse.messages.length, }); - const content = - toolResponse.messages[toolResponse.messages.length - 1].content; - const parsedContent = - typeof content === "string" ? JSON.parse(content) : content; + const content = toolResponse.messages[toolResponse.messages.length - 1].content; + const parsedContent = typeof content === 'string' ? JSON.parse(content) : content; const parsedTweets = tweetSearchSchema.parse(parsedContent); diff --git a/auto-kol/agent/src/services/agents/nodes/toneAnalysisNode.ts b/auto-kol/agent/src/services/agents/nodes/toneAnalysisNode.ts index 162bd8c..1dc95be 100644 --- a/auto-kol/agent/src/services/agents/nodes/toneAnalysisNode.ts +++ b/auto-kol/agent/src/services/agents/nodes/toneAnalysisNode.ts @@ -1,40 +1,36 @@ -import { AIMessage } from "@langchain/core/messages"; -import { State, logger, parseMessageContent } from "../workflow.js"; -import * as prompts from "../prompts.js"; -import { WorkflowConfig } from "../workflow.js"; +import { AIMessage } from '@langchain/core/messages'; +import { State, logger, parseMessageContent } from '../workflow.js'; +import * as prompts from '../prompts.js'; +import { WorkflowConfig } from '../workflow.js'; export const createToneAnalysisNode = (config: WorkflowConfig) => { return async (state: typeof State.State) => { - logger.info("Tone Analysis Node - Analyzing tweet tone"); + logger.info('Tone Analysis Node - Analyzing tweet tone'); try { const lastMessage = state.messages[state.messages.length - 1]; const parsedContent = parseMessageContent(lastMessage.content); const batchToAnalyze = parsedContent.batchToAnalyze || []; - logger.info( - `Processing batch of ${batchToAnalyze.length} tweets for tone analysis`, - ); + logger.info(`Processing batch of ${batchToAnalyze.length} tweets for tone analysis`); const analyzedBatch = await Promise.all( - batchToAnalyze.map( - async ({ tweet, decision }: { tweet: any; decision: any }) => { - const toneAnalysis = await prompts.tonePrompt - .pipe(config.llms.tone) - .pipe(prompts.toneParser) - .invoke({ - tweet: tweet.text, - thread: tweet.thread || [], - }); + batchToAnalyze.map(async ({ tweet, decision }: { tweet: any; decision: any }) => { + const toneAnalysis = await prompts.tonePrompt + .pipe(config.llms.tone) + .pipe(prompts.toneParser) + .invoke({ + tweet: tweet.text, + thread: tweet.thread || [], + }); - logger.info("Tone analysis:", { toneAnalysis }); + logger.info('Tone analysis:', { toneAnalysis }); - return { - tweet, - decision, - toneAnalysis, - }; - }, - ), + return { + tweet, + decision, + toneAnalysis, + }; + }), ); return { @@ -51,7 +47,7 @@ export const createToneAnalysisNode = (config: WorkflowConfig) => { ], }; } catch (error) { - logger.error("Error in tone analysis node:", error); + logger.error('Error in tone analysis node:', error); return { messages: [] }; } }; diff --git a/auto-kol/agent/src/services/agents/prompts.ts b/auto-kol/agent/src/services/agents/prompts.ts index ed99a42..fe2148d 100644 --- a/auto-kol/agent/src/services/agents/prompts.ts +++ b/auto-kol/agent/src/services/agents/prompts.ts @@ -1,24 +1,21 @@ -import { StructuredOutputParser } from "langchain/output_parsers"; +import { StructuredOutputParser } from 'langchain/output_parsers'; import { engagementSchema, toneSchema, responseSchema, autoApprovalSchema, -} from "../../schemas/workflow.js"; -import { ChatPromptTemplate, PromptTemplate } from "@langchain/core/prompts"; -import { SystemMessage } from "@langchain/core/messages"; -import { config } from "../../config/index.js"; +} from '../../schemas/workflow.js'; +import { ChatPromptTemplate, PromptTemplate } from '@langchain/core/prompts'; +import { SystemMessage } from '@langchain/core/messages'; +import { config } from '../../config/index.js'; const agentUsername = config.TWITTER_USERNAME!; const walletAddress = config.WALLET_ADDRESS!; -export const engagementParser = - StructuredOutputParser.fromZodSchema(engagementSchema); +export const engagementParser = StructuredOutputParser.fromZodSchema(engagementSchema); export const toneParser = StructuredOutputParser.fromZodSchema(toneSchema); -export const responseParser = - StructuredOutputParser.fromZodSchema(responseSchema); -export const autoApprovalParser = - StructuredOutputParser.fromZodSchema(autoApprovalSchema); +export const responseParser = StructuredOutputParser.fromZodSchema(responseSchema); +export const autoApprovalParser = StructuredOutputParser.fromZodSchema(autoApprovalSchema); // // ============ ENGAGEMENT SYSTEM PROMPT ============ @@ -132,7 +129,7 @@ export const autoApprovalSystemPrompt = await PromptTemplate.fromTemplate( export const engagementPrompt = ChatPromptTemplate.fromMessages([ new SystemMessage(engagementSystemPrompt), [ - "human", + 'human', `Evaluate this tweet and provide your structured decision: Tweet: {tweet} Thread Context: {thread} @@ -146,7 +143,7 @@ export const engagementPrompt = ChatPromptTemplate.fromMessages([ export const tonePrompt = ChatPromptTemplate.fromMessages([ new SystemMessage(toneSystemPrompt), [ - "human", + 'human', `Analyze the tone for this tweet and suggest a response tone: Tweet: {tweet} Thread: {thread} @@ -160,7 +157,7 @@ export const tonePrompt = ChatPromptTemplate.fromMessages([ export const responsePrompt = ChatPromptTemplate.fromMessages([ new SystemMessage(responseSystemPrompt), [ - "human", + 'human', `Generate a response strategy for this tweet by considering similar tweets from @{author} using the suggested tone: Tweet: {tweet} Tone: {tone} @@ -198,21 +195,18 @@ export const responsePrompt = ChatPromptTemplate.fromMessages([ ]); // Helper function to format rejection feedback -export const formatRejectionFeedback = ( - rejectionReason?: string, - suggestedChanges?: string, -) => { - if (!rejectionReason) return ""; +export const formatRejectionFeedback = (rejectionReason?: string, suggestedChanges?: string) => { + if (!rejectionReason) return ''; return `\nPrevious Response Feedback: Rejection Reason: ${rejectionReason} - Suggested Changes: ${suggestedChanges || "None provided"} + Suggested Changes: ${suggestedChanges || 'None provided'} Please address this feedback in your new response.`; }; export const formatRejectionInstructions = (rejectionReason?: string) => { - if (!rejectionReason) return ""; + if (!rejectionReason) return ''; return `\nIMPORTANT: Your previous response was rejected. Make sure to: 1. Address the rejection reason: "${rejectionReason}" @@ -223,7 +217,7 @@ export const formatRejectionInstructions = (rejectionReason?: string) => { export const autoApprovalPrompt = ChatPromptTemplate.fromMessages([ new SystemMessage(autoApprovalSystemPrompt), [ - "human", + 'human', `Evaluate this response: Original Tweet: {tweet} Generated Response: {response} diff --git a/auto-kol/agent/src/services/agents/workflow.ts b/auto-kol/agent/src/services/agents/workflow.ts index 531d4fe..4fca1e1 100644 --- a/auto-kol/agent/src/services/agents/workflow.ts +++ b/auto-kol/agent/src/services/agents/workflow.ts @@ -1,23 +1,17 @@ -import { - END, - MemorySaver, - StateGraph, - START, - Annotation, -} from "@langchain/langgraph"; -import { BaseMessage } from "@langchain/core/messages"; -import { ChatOpenAI } from "@langchain/openai"; -import { MessageContent } from "@langchain/core/messages"; -import { config } from "../../config/index.js"; -import { createLogger } from "../../utils/logger.js"; -import { createTools } from "../../tools/index.js"; -import { ToolNode } from "@langchain/langgraph/prebuilt"; -import { createTwitterClientScraper, ExtendedScraper } from "../twitter/api.js"; -export const logger = createLogger("agent-workflow"); -import { createNodes } from "./nodes.js"; +import { END, MemorySaver, StateGraph, START, Annotation } from '@langchain/langgraph'; +import { BaseMessage } from '@langchain/core/messages'; +import { ChatOpenAI } from '@langchain/openai'; +import { MessageContent } from '@langchain/core/messages'; +import { config } from '../../config/index.js'; +import { createLogger } from '../../utils/logger.js'; +import { createTools } from '../../tools/index.js'; +import { ToolNode } from '@langchain/langgraph/prebuilt'; +import { createTwitterClientScraper, ExtendedScraper } from '../twitter/api.js'; +export const logger = createLogger('agent-workflow'); +import { createNodes } from './nodes.js'; export const parseMessageContent = (content: MessageContent): any => { - if (typeof content === "string") { + if (typeof content === 'string') { return JSON.parse(content); } if (Array.isArray(content)) { @@ -89,7 +83,7 @@ const shouldContinue = (state: typeof State.State) => { const lastMessage = state.messages[state.messages.length - 1]; const content = parseMessageContent(lastMessage.content); - logger.debug("Evaluating workflow continuation", { + logger.debug('Evaluating workflow continuation', { hasMessages: state.messages.length > 0, currentIndex: content.currentTweetIndex, totalTweets: content.tweets?.length, @@ -100,24 +94,24 @@ const shouldContinue = (state: typeof State.State) => { // Handle auto-approval flow if (!content.fromAutoApproval && content.batchToFeedback?.length > 0) { - return "autoApprovalNode"; + return 'autoApprovalNode'; } if (content.fromAutoApproval) { if (content.batchToRespond?.length > 0) { - return "generateNode"; + return 'generateNode'; } else { - return "engagementNode"; + return 'engagementNode'; } } // Handle batch processing flow if (content.batchToAnalyze?.length > 0) { - return "analyzeNode"; + return 'analyzeNode'; } if (content.batchToRespond?.length > 0) { - return "generateNode"; + return 'generateNode'; } // Check if we've processed all tweets if ( @@ -125,37 +119,35 @@ const shouldContinue = (state: typeof State.State) => { content.pendingEngagements?.length === 0 ) { if (content.fromRecheckNode && content.messages?.length === 0) { - logger.info("Workflow complete - no more tweets to process"); + logger.info('Workflow complete - no more tweets to process'); return END; } - logger.info("Moving to recheck skipped tweets"); - return "recheckNode"; + logger.info('Moving to recheck skipped tweets'); + return 'recheckNode'; } - return "engagementNode"; + return 'engagementNode'; }; // Workflow creation function -export const createWorkflow = async ( - nodes: Awaited>, -) => { +export const createWorkflow = async (nodes: Awaited>) => { return new StateGraph(State) - .addNode("mentionNode", nodes.mentionNode) - .addNode("timelineNode", nodes.timelineNode) - .addNode("searchNode", nodes.searchNode) - .addNode("engagementNode", nodes.engagementNode) - .addNode("analyzeNode", nodes.toneAnalysisNode) - .addNode("generateNode", nodes.responseGenerationNode) - .addNode("autoApprovalNode", nodes.autoApprovalNode) - .addNode("recheckNode", nodes.recheckSkippedNode) - .addEdge(START, "mentionNode") - .addEdge("mentionNode", "timelineNode") - .addEdge("timelineNode", "searchNode") - .addEdge("searchNode", "engagementNode") - .addConditionalEdges("engagementNode", shouldContinue) - .addConditionalEdges("analyzeNode", shouldContinue) - .addConditionalEdges("generateNode", shouldContinue) - .addConditionalEdges("autoApprovalNode", shouldContinue) - .addConditionalEdges("recheckNode", shouldContinue); + .addNode('mentionNode', nodes.mentionNode) + .addNode('timelineNode', nodes.timelineNode) + .addNode('searchNode', nodes.searchNode) + .addNode('engagementNode', nodes.engagementNode) + .addNode('analyzeNode', nodes.toneAnalysisNode) + .addNode('generateNode', nodes.responseGenerationNode) + .addNode('autoApprovalNode', nodes.autoApprovalNode) + .addNode('recheckNode', nodes.recheckSkippedNode) + .addEdge(START, 'mentionNode') + .addEdge('mentionNode', 'timelineNode') + .addEdge('timelineNode', 'searchNode') + .addEdge('searchNode', 'engagementNode') + .addConditionalEdges('engagementNode', shouldContinue) + .addConditionalEdges('analyzeNode', shouldContinue) + .addConditionalEdges('generateNode', shouldContinue) + .addConditionalEdges('autoApprovalNode', shouldContinue) + .addConditionalEdges('recheckNode', shouldContinue); }; // Workflow runner type @@ -174,7 +166,7 @@ const createWorkflowRunner = async (): Promise => { return { runWorkflow: async () => { const threadId = `workflow_${Date.now()}`; - logger.info("Starting tweet response workflow", { threadId }); + logger.info('Starting tweet response workflow', { threadId }); const config = { recursionLimit: 50, @@ -190,7 +182,7 @@ const createWorkflowRunner = async (): Promise => { finalState = state; } - logger.info("Workflow completed", { threadId }); + logger.info('Workflow completed', { threadId }); return finalState; }, }; diff --git a/auto-kol/agent/src/services/database/index.ts b/auto-kol/agent/src/services/database/index.ts index 5b7bf0e..dec0fe8 100644 --- a/auto-kol/agent/src/services/database/index.ts +++ b/auto-kol/agent/src/services/database/index.ts @@ -3,19 +3,17 @@ import { ApprovalAction, SkippedTweetMemory, ActionResponse, -} from "../../types/queue.js"; -import { createLogger } from "../../utils/logger.js"; -import * as db from "../../database/index.js"; -import { Tweet } from "../../types/twitter.js"; -import { getPendingResponsesByTweetId } from "../../database/index.js"; -import { getTweetById } from "../../database/index.js"; +} from '../../types/queue.js'; +import { createLogger } from '../../utils/logger.js'; +import * as db from '../../database/index.js'; +import { Tweet } from '../../types/twitter.js'; +import { getPendingResponsesByTweetId } from '../../database/index.js'; +import { getTweetById } from '../../database/index.js'; -const logger = createLogger("database-queue"); +const logger = createLogger('database-queue'); ///////////RESPONSE/////////// -export async function addResponse( - response: QueuedResponseMemory, -): Promise { +export async function addResponse(response: QueuedResponseMemory): Promise { try { try { await db.addTweet({ @@ -36,14 +34,13 @@ export async function addResponse( id: response.id, tweet_id: response.tweet.id, content: response.response.content, - tone: response.workflowState.toneAnalysis?.suggestedTone || "neutral", - strategy: response.workflowState.responseStrategy?.strategy || "direct", - estimatedImpact: - response.workflowState.responseStrategy?.estimatedImpact || 5, + tone: response.workflowState.toneAnalysis?.suggestedTone || 'neutral', + strategy: response.workflowState.responseStrategy?.strategy || 'direct', + estimatedImpact: response.workflowState.responseStrategy?.estimatedImpact || 5, confidence: response.workflowState.responseStrategy?.confidence || 0.5, }); } catch (error) { - logger.error("Failed to add response to queue:", error); + logger.error('Failed to add response to queue:', error); throw error; } } @@ -54,29 +51,24 @@ export const updateResponseStatus = async ( try { const pendingResponse = await getPendingResponsesByTweetId(action.id); const tweet = await getTweetById(pendingResponse.tweet_id); - await db.updateResponseStatus( - action.id, - action.approved ? "approved" : "rejected", - ); + await db.updateResponseStatus(action.id, action.approved ? 'approved' : 'rejected'); logger.info(`Updated response status: ${action.id}`); return { tweet: tweet as Tweet, - status: action.approved ? "approved" : "rejected", - response: pendingResponse as unknown as ActionResponse["response"], + status: action.approved ? 'approved' : 'rejected', + response: pendingResponse as unknown as ActionResponse['response'], }; } catch (error) { - logger.error("Failed to update response status:", error); + logger.error('Failed to update response status:', error); return undefined; } }; -export async function getAllPendingResponses(): Promise< - QueuedResponseMemory[] -> { +export async function getAllPendingResponses(): Promise { try { const responses = await db.getPendingResponses(); - return responses.map((r) => ({ + return responses.map(r => ({ id: r.id, tweet: { id: r.tweet_id, @@ -88,7 +80,7 @@ export async function getAllPendingResponses(): Promise< response: { content: r.content, }, - status: r.status as "pending" | "approved" | "rejected", + status: r.status as 'pending' | 'approved' | 'rejected', created_at: new Date(r.created_at), updatedAt: new Date(r.updated_at), workflowState: { @@ -102,7 +94,7 @@ export async function getAllPendingResponses(): Promise< } as any, })); } catch (error) { - logger.error("Failed to get pending responses:", error); + logger.error('Failed to get pending responses:', error); throw error; } } @@ -134,7 +126,7 @@ export async function addToSkipped(skipped: SkippedTweetMemory): Promise { logger.info(`Added tweet to skipped: ${skipped.id}`); } catch (error) { - logger.error("Failed to add skipped tweet:", error); + logger.error('Failed to add skipped tweet:', error); throw error; } } @@ -144,14 +136,12 @@ export async function getSkippedTweets(): Promise { return skipped; } -export async function getSkippedTweetById( - skippedId: string, -): Promise { +export async function getSkippedTweetById(skippedId: string): Promise { const skipped = await db.getSkippedTweetById(skippedId); const tweet = await getTweetById(skipped.tweet_id); if (!skipped || !tweet) { - throw new Error("Skipped tweet or original tweet not found"); + throw new Error('Skipped tweet or original tweet not found'); } const result: SkippedTweetMemory = { id: skipped.id, @@ -182,18 +172,18 @@ export async function getSkippedTweetById( export const moveSkippedToQueue = async ( skippedId: string, - queuedResponse: Omit & { status: "pending" }, + queuedResponse: Omit & { status: 'pending' }, ): Promise => { try { const skipped = (await getSkippedTweetById(skippedId)) as any; logger.info(`Skipped tweet: ${JSON.stringify(skipped)}`); if (!skipped) { - throw new Error("Skipped tweet not found"); + throw new Error('Skipped tweet not found'); } const tweet = await db.getTweetById(skipped.tweet_id); if (!tweet) { - throw new Error("Tweet not found"); + throw new Error('Tweet not found'); } logger.info(`Tweet: ${JSON.stringify(tweet)}`); @@ -207,7 +197,7 @@ export const moveSkippedToQueue = async ( created_at: tweet.created_at, }, response: queuedResponse.response, - status: "pending", + status: 'pending', created_at: new Date(), updatedAt: new Date(), workflowState: queuedResponse.workflowState, @@ -218,7 +208,7 @@ export const moveSkippedToQueue = async ( logger.info(`Moved skipped tweet ${skippedId} to response queue`); return typedResponse; } catch (error) { - logger.error("Failed to move skipped tweet to queue:", error); + logger.error('Failed to move skipped tweet to queue:', error); throw error; } }; @@ -226,8 +216,7 @@ export const moveSkippedToQueue = async ( //////////UTILS////////// const isUniqueConstraintError = (error: any): boolean => { return ( - error?.code === "SQLITE_CONSTRAINT" && - error?.message?.includes("UNIQUE constraint failed") + error?.code === 'SQLITE_CONSTRAINT' && error?.message?.includes('UNIQUE constraint failed') ); }; diff --git a/auto-kol/agent/src/services/twitter/api.ts b/auto-kol/agent/src/services/twitter/api.ts index 58831ef..8ed6564 100644 --- a/auto-kol/agent/src/services/twitter/api.ts +++ b/auto-kol/agent/src/services/twitter/api.ts @@ -1,9 +1,9 @@ -import { Scraper, SearchMode, Tweet } from "agent-twitter-client"; -import { createLogger } from "../../utils/logger.js"; -import { readFileSync, writeFileSync, existsSync } from "fs"; -import { config } from "../../config/index.js"; +import { Scraper, SearchMode, Tweet } from 'agent-twitter-client'; +import { createLogger } from '../../utils/logger.js'; +import { readFileSync, writeFileSync, existsSync } from 'fs'; +import { config } from '../../config/index.js'; -const logger = createLogger("agent-twitter-api"); +const logger = createLogger('agent-twitter-api'); export class ExtendedScraper extends Scraper { private static instance: ExtendedScraper | null = null; @@ -24,28 +24,28 @@ export class ExtendedScraper extends Scraper { private async initialize() { const username = config.TWITTER_USERNAME!; const password = config.TWITTER_PASSWORD!; - const cookiesPath = "cookies.json"; + const cookiesPath = 'cookies.json'; if (existsSync(cookiesPath)) { - logger.info("Loading existing cookies"); - const cookies = readFileSync(cookiesPath, "utf8"); + logger.info('Loading existing cookies'); + const cookies = readFileSync(cookiesPath, 'utf8'); try { const parsedCookies = JSON.parse(cookies).map( (cookie: any) => `${cookie.key}=${cookie.value}; Domain=${cookie.domain}; Path=${cookie.path}`, ); await this.setCookies(parsedCookies); - logger.info("Loaded existing cookies from file"); + logger.info('Loaded existing cookies from file'); } catch (error) { - logger.error("Error loading cookies:", error); + logger.error('Error loading cookies:', error); } } else { - logger.info("No existing cookies found, proceeding with login"); + logger.info('No existing cookies found, proceeding with login'); await this.login(username, password); const newCookies = await this.getCookies(); writeFileSync(cookiesPath, JSON.stringify(newCookies, null, 2)); - logger.info("New cookies saved to file"); + logger.info('New cookies saved to file'); } const isLoggedIn = await this.isLoggedIn(); @@ -64,21 +64,21 @@ export class ExtendedScraper extends Scraper { await this.initialize(); isLoggedIn = await this.isLoggedIn(); if (isLoggedIn) { - logger.info("Successfully re-authenticated"); + logger.info('Successfully re-authenticated'); return true; } - logger.error("Re-authentication failed"); + logger.error('Re-authentication failed'); retryCount++; if (retryCount < maxRetries) { const delay = 2000 * Math.pow(2, retryCount - 1); - await new Promise((resolve) => setTimeout(resolve, delay)); + await new Promise(resolve => setTimeout(resolve, delay)); } } catch (error) { - logger.error("Error during re-authentication:", error); + logger.error('Error during re-authentication:', error); retryCount++; if (retryCount < maxRetries) { const delay = 2000 * Math.pow(2, retryCount - 1); - await new Promise((resolve) => setTimeout(resolve, delay)); + await new Promise(resolve => setTimeout(resolve, delay)); } } } @@ -92,20 +92,16 @@ export class ExtendedScraper extends Scraper { const isLoggedIn = await this.isLoggedIn(); if (!isLoggedIn) { - throw new Error("Must be logged in to fetch mentions"); + throw new Error('Must be logged in to fetch mentions'); } const query = `@${username} -from:${username}`; const replies: Tweet[] = []; - const searchIterator = this.searchTweets( - query, - maxResults, - SearchMode.Latest, - ); + const searchIterator = this.searchTweets(query, maxResults, SearchMode.Latest); for await (const tweet of searchIterator) { - logger.info("Checking tweet:", { + logger.info('Checking tweet:', { id: tweet.id, text: tweet.text, author: tweet.username, @@ -125,9 +121,7 @@ export class ExtendedScraper extends Scraper { for await (const reply of hasReplies) { if (reply.inReplyToStatusId === tweet.id) { alreadyReplied = true; - logger.info( - `Skipping tweet ${tweet.id} - already replied with ${reply.id}`, - ); + logger.info(`Skipping tweet ${tweet.id} - already replied with ${reply.id}`); break; } } @@ -147,7 +141,7 @@ export class ExtendedScraper extends Scraper { public async getThread(tweetId: string): Promise { const isLoggedIn = await this.isLoggedIn(); if (!isLoggedIn) { - throw new Error("Must be logged in to fetch thread"); + throw new Error('Must be logged in to fetch thread'); } const initialTweet = await this.getTweet(tweetId); @@ -171,15 +165,12 @@ export class ExtendedScraper extends Scraper { let rootTweet = initialTweet; // If the conversation root differs - if ( - initialTweet.conversationId && - initialTweet.conversationId !== initialTweet.id - ) { + if (initialTweet.conversationId && initialTweet.conversationId !== initialTweet.id) { const conversationRoot = await this.getTweet(initialTweet.conversationId); if (conversationRoot) { rootTweet = conversationRoot; conversationTweets.set(rootTweet.id!, rootTweet); - logger.info("Found conversation root tweet:", { + logger.info('Found conversation root tweet:', { id: rootTweet.id, conversationId: rootTweet.conversationId, }); @@ -189,10 +180,7 @@ export class ExtendedScraper extends Scraper { } try { - logger.info( - "Fetching entire conversation via `conversation_id`:", - conversationId, - ); + logger.info('Fetching entire conversation via `conversation_id`:', conversationId); const conversationIterator = this.searchTweets( `conversation_id:${conversationId}`, @@ -226,16 +214,13 @@ export class ExtendedScraper extends Scraper { } // Placeholder for efficient thread fetching - async getThreadPlaceHolder( - tweetId: string, - maxDepth: number = 100, - ): Promise { + async getThreadPlaceHolder(tweetId: string, maxDepth: number = 100): Promise { const username = config.TWITTER_USERNAME!; const isLoggedIn = await this.isLoggedIn(); if (!isLoggedIn) { const reAuthenticate = await this.reAuthenticate(); if (!reAuthenticate) { - logger.error("Failed to re-authenticate"); + logger.error('Failed to re-authenticate'); return []; } } @@ -254,23 +239,18 @@ export class ExtendedScraper extends Scraper { let rootTweet = initialTweet; const conversationId = initialTweet.conversationId || initialTweet.id; - logger.info("Initial tweet:", { + logger.info('Initial tweet:', { id: initialTweet.id, conversationId: conversationId, inReplyToStatusId: initialTweet.inReplyToStatusId, }); - if ( - initialTweet.conversationId && - initialTweet.conversationId !== initialTweet.id - ) { - const conversationRoot = await this.getTweet( - initialTweet.conversationId, - ); + if (initialTweet.conversationId && initialTweet.conversationId !== initialTweet.id) { + const conversationRoot = await this.getTweet(initialTweet.conversationId); if (conversationRoot) { rootTweet = conversationRoot; conversationTweets.set(rootTweet.id!, rootTweet); - logger.info("Found root tweet:", { + logger.info('Found root tweet:', { id: rootTweet.id, conversationId: rootTweet.conversationId, }); @@ -278,10 +258,7 @@ export class ExtendedScraper extends Scraper { } try { - logger.info( - "Fetching conversation with query:", - `conversation_id:${conversationId}`, - ); + logger.info('Fetching conversation with query:', `conversation_id:${conversationId}`); const conversationIterator = this.searchTweets( `conversation_id:${conversationId}`, 100, @@ -290,17 +267,14 @@ export class ExtendedScraper extends Scraper { for await (const tweet of conversationIterator) { conversationTweets.set(tweet.id!, tweet); - logger.info("Found conversation tweet:", { + logger.info('Found conversation tweet:', { id: tweet.id, inReplyToStatusId: tweet.inReplyToStatusId, - text: tweet.text?.substring(0, 50) + "...", + text: tweet.text?.substring(0, 50) + '...', }); } - logger.info( - "Total conversation tweets found:", - conversationTweets.size, - ); + logger.info('Total conversation tweets found:', conversationTweets.size); } catch (error) { logger.warn(`Error fetching conversation: ${error}`); return [rootTweet, initialTweet]; diff --git a/auto-kol/agent/src/services/vectorstore/chroma.ts b/auto-kol/agent/src/services/vectorstore/chroma.ts index 42f93d8..1431e10 100644 --- a/auto-kol/agent/src/services/vectorstore/chroma.ts +++ b/auto-kol/agent/src/services/vectorstore/chroma.ts @@ -1,13 +1,13 @@ -import { ChromaClient } from "chromadb"; -import { Document } from "langchain/document"; -import { Chroma } from "@langchain/community/vectorstores/chroma"; -import { OpenAIEmbeddings } from "@langchain/openai"; -import { createLogger } from "../../utils/logger.js"; -import { Tweet } from "../../types/twitter.js"; -import { config } from "../../config/index.js"; -import { isTweetExists } from "../../services/database/index.js"; +import { ChromaClient } from 'chromadb'; +import { Document } from 'langchain/document'; +import { Chroma } from '@langchain/community/vectorstores/chroma'; +import { OpenAIEmbeddings } from '@langchain/openai'; +import { createLogger } from '../../utils/logger.js'; +import { Tweet } from '../../types/twitter.js'; +import { config } from '../../config/index.js'; +import { isTweetExists } from '../../services/database/index.js'; -const logger = createLogger("chroma-service"); +const logger = createLogger('chroma-service'); export class ChromaService { private static instance: ChromaService; @@ -21,7 +21,7 @@ export class ChromaService { }); this.embeddings = new OpenAIEmbeddings({ openAIApiKey: config.OPENAI_API_KEY, - modelName: "text-embedding-ada-002", + modelName: 'text-embedding-ada-002', }); this.initializeCollection(); } @@ -29,24 +29,24 @@ export class ChromaService { private async initializeCollection() { try { this.collection = await Chroma.fromExistingCollection(this.embeddings, { - collectionName: "tweets", + collectionName: 'tweets', url: config.CHROMA_URL, collectionMetadata: { - "hnsw:space": "cosine", + 'hnsw:space': 'cosine', }, }); - logger.info("Chroma collection initialized"); + logger.info('Chroma collection initialized'); } catch (error) { - logger.info("Collection does not exist, creating new one"); + logger.info('Collection does not exist, creating new one'); this.collection = await Chroma.fromTexts( [], // No initial texts [], // No initial metadatas this.embeddings, { - collectionName: "tweets", + collectionName: 'tweets', url: config.CHROMA_URL, collectionMetadata: { - "hnsw:space": "cosine", + 'hnsw:space': 'cosine', }, }, ); @@ -64,9 +64,7 @@ export class ChromaService { public async addTweet(tweet: Tweet) { try { if (await isTweetExists(tweet.id)) { - logger.info( - `Tweet ${tweet.id} already exists in vector store, skipping`, - ); + logger.info(`Tweet ${tweet.id} already exists in vector store, skipping`); return; } @@ -93,7 +91,7 @@ export class ChromaService { const results = await this.collection.similaritySearch(query, k); return results; } catch (error) { - logger.error("Failed to search similar tweets:", error); + logger.error('Failed to search similar tweets:', error); throw error; } } @@ -101,13 +99,10 @@ export class ChromaService { public async searchSimilarTweetsWithScore(query: string, k: number = 5) { try { const queryEmbedding = await this.embeddings.embedQuery(query); - const results = await this.collection.similaritySearchVectorWithScore( - queryEmbedding, - k, - ); + const results = await this.collection.similaritySearchVectorWithScore(queryEmbedding, k); return results; } catch (error) { - logger.error("Failed to search similar tweets with scores:", error); + logger.error('Failed to search similar tweets with scores:', error); throw error; } } @@ -121,10 +116,7 @@ export class ChromaService { }); logger.info(`Deleted tweet ${tweetId} from vector store`); } catch (error) { - logger.error( - `Failed to delete tweet ${tweetId} from vector store:`, - error, - ); + logger.error(`Failed to delete tweet ${tweetId} from vector store:`, error); throw error; } } diff --git a/auto-kol/agent/src/tools/index.ts b/auto-kol/agent/src/tools/index.ts index dce61c6..a0a8f43 100644 --- a/auto-kol/agent/src/tools/index.ts +++ b/auto-kol/agent/src/tools/index.ts @@ -1,11 +1,11 @@ -import { createFetchTimelineTool } from "./tools/fetchTimelineTool.js"; -import { createTweetSearchTool } from "./tools/tweetSearchTool.js"; -import { createAddResponseTool } from "./tools/queueResponseTool.js"; -import { createUpdateResponseTool } from "./tools/queueResponseTool.js"; -import { createQueueSkippedTool } from "./tools/queueSkippedTool.js"; -import { createSearchSimilarTweetsTool } from "./tools/searchSimilarTweetsTool.js"; -import { createMentionTool } from "./tools/mentionTool.js"; -import { ExtendedScraper } from "../services/twitter/api.js"; +import { createFetchTimelineTool } from './tools/fetchTimelineTool.js'; +import { createTweetSearchTool } from './tools/tweetSearchTool.js'; +import { createAddResponseTool } from './tools/queueResponseTool.js'; +import { createUpdateResponseTool } from './tools/queueResponseTool.js'; +import { createQueueSkippedTool } from './tools/queueSkippedTool.js'; +import { createSearchSimilarTweetsTool } from './tools/searchSimilarTweetsTool.js'; +import { createMentionTool } from './tools/mentionTool.js'; +import { ExtendedScraper } from '../services/twitter/api.js'; export const createTools = (scraper: ExtendedScraper) => { const mentionTool = createMentionTool(scraper); diff --git a/auto-kol/agent/src/tools/tools/fetchTimelineTool.ts b/auto-kol/agent/src/tools/tools/fetchTimelineTool.ts index f017e4b..1a7a46b 100644 --- a/auto-kol/agent/src/tools/tools/fetchTimelineTool.ts +++ b/auto-kol/agent/src/tools/tools/fetchTimelineTool.ts @@ -1,29 +1,26 @@ -import { DynamicStructuredTool } from "@langchain/core/tools"; -import { z } from "zod"; -import { createLogger } from "../../utils/logger.js"; -import { getTimeLine } from "../../utils/twitter.js"; -import { ExtendedScraper } from "../../services/twitter/api.js"; +import { DynamicStructuredTool } from '@langchain/core/tools'; +import { z } from 'zod'; +import { createLogger } from '../../utils/logger.js'; +import { getTimeLine } from '../../utils/twitter.js'; +import { ExtendedScraper } from '../../services/twitter/api.js'; -const logger = createLogger("fetch-timeline-tool"); +const logger = createLogger('fetch-timeline-tool'); export const createFetchTimelineTool = (twitterScraper: ExtendedScraper) => new DynamicStructuredTool({ - name: "fetch_timeline", - description: "Fetch the timeline regularly to get new tweets", + name: 'fetch_timeline', + description: 'Fetch the timeline regularly to get new tweets', schema: z.object({}), func: async () => { try { const tweets = await getTimeLine(twitterScraper); - tweets.sort( - (a, b) => - new Date(b.created_at).getTime() - new Date(a.created_at).getTime(), - ); + tweets.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime()); return { tweets: tweets, lastProcessedId: tweets[tweets.length - 1]?.id || null, }; } catch (error) { - logger.error("Error in fetchTimelineTool:", error); + logger.error('Error in fetchTimelineTool:', error); return { tweets: [], lastProcessedId: null, diff --git a/auto-kol/agent/src/tools/tools/mentionTool.ts b/auto-kol/agent/src/tools/tools/mentionTool.ts index 912ad57..e118883 100644 --- a/auto-kol/agent/src/tools/tools/mentionTool.ts +++ b/auto-kol/agent/src/tools/tools/mentionTool.ts @@ -1,22 +1,22 @@ -import { DynamicStructuredTool } from "@langchain/core/tools"; -import { z } from "zod"; -import { createLogger } from "../../utils/logger.js"; -import { addMention, getLatestMentionId } from "../../database/index.js"; -import { ExtendedScraper } from "../../services/twitter/api.js"; -import { Tweet } from "../../types/twitter.js"; -const logger = createLogger("mention-tool"); +import { DynamicStructuredTool } from '@langchain/core/tools'; +import { z } from 'zod'; +import { createLogger } from '../../utils/logger.js'; +import { addMention, getLatestMentionId } from '../../database/index.js'; +import { ExtendedScraper } from '../../services/twitter/api.js'; +import { Tweet } from '../../types/twitter.js'; +const logger = createLogger('mention-tool'); export const createMentionTool = (scraper: ExtendedScraper) => new DynamicStructuredTool({ - name: "fetch_mentions", - description: "Fetch mentions since the last processed mention", + name: 'fetch_mentions', + description: 'Fetch mentions since the last processed mention', schema: z.object({}), func: async () => { try { const sinceId = await getLatestMentionId(); const mentions = await scraper.getMyMentions(100, sinceId); if (!mentions || mentions.length === 0) { - logger.info("No new mentions found"); + logger.info('No new mentions found'); return { tweets: [], lastProcessedId: sinceId, @@ -43,17 +43,15 @@ export const createMentionTool = (scraper: ExtendedScraper) => const thread = await scraper.getThread(tweet.id); for await (const threadTweet of thread) { tweetsWithThreads.push({ - id: threadTweet.id || "", - text: threadTweet.text || "", - author_id: threadTweet.userId || "", - author_username: threadTweet.username?.toLowerCase() || "unknown", - created_at: - threadTweet.timeParsed?.toISOString() || - new Date().toISOString(), + id: threadTweet.id || '', + text: threadTweet.text || '', + author_id: threadTweet.userId || '', + author_username: threadTweet.username?.toLowerCase() || 'unknown', + created_at: threadTweet.timeParsed?.toISOString() || new Date().toISOString(), }); } tweet.thread = tweetsWithThreads; - await new Promise((resolve) => setTimeout(resolve, 1000)); + await new Promise(resolve => setTimeout(resolve, 1000)); logger.info(`Found ${tweetsWithThreads.length} tweets in thread`); } return { @@ -61,7 +59,7 @@ export const createMentionTool = (scraper: ExtendedScraper) => lastProcessedId: mentions[0].id!, }; } catch (error) { - logger.error("Error in mentionTool:", error); + logger.error('Error in mentionTool:', error); return { tweets: [], lastProcessedId: null, diff --git a/auto-kol/agent/src/tools/tools/queueResponseTool.ts b/auto-kol/agent/src/tools/tools/queueResponseTool.ts index 194fb7b..ea453f2 100644 --- a/auto-kol/agent/src/tools/tools/queueResponseTool.ts +++ b/auto-kol/agent/src/tools/tools/queueResponseTool.ts @@ -1,20 +1,20 @@ -import { DynamicStructuredTool } from "@langchain/core/tools"; -import { createLogger } from "../../utils/logger.js"; -import { v4 as generateId } from "uuid"; -import { queueActionSchema } from "../../schemas/workflow.js"; -import { addResponse } from "../../services/database/index.js"; -import { getResponseByTweetId, updateResponse } from "../../database/index.js"; -import { PendingResponse, QueuedResponseMemory } from "../../types/queue.js"; -import { Tweet } from "../../types/twitter.js"; -import { AgentResponse } from "../../types/agent.js"; -import { WorkflowState } from "../../types/workflow.js"; +import { DynamicStructuredTool } from '@langchain/core/tools'; +import { createLogger } from '../../utils/logger.js'; +import { v4 as generateId } from 'uuid'; +import { queueActionSchema } from '../../schemas/workflow.js'; +import { addResponse } from '../../services/database/index.js'; +import { getResponseByTweetId, updateResponse } from '../../database/index.js'; +import { PendingResponse, QueuedResponseMemory } from '../../types/queue.js'; +import { Tweet } from '../../types/twitter.js'; +import { AgentResponse } from '../../types/agent.js'; +import { WorkflowState } from '../../types/workflow.js'; -const logger = createLogger("queue-response-tool"); +const logger = createLogger('queue-response-tool'); export const createAddResponseTool = () => new DynamicStructuredTool({ - name: "add_response", - description: "Add or update a response in the approval queue", + name: 'add_response', + description: 'Add or update a response in the approval queue', schema: queueActionSchema, func: async (input: any) => { const id = generateId(); @@ -24,7 +24,7 @@ export const createAddResponseTool = () => response: { content: input.workflowState?.responseStrategy?.content, }, - status: "pending" as const, + status: 'pending' as const, created_at: new Date(), updatedAt: new Date(), workflowState: input.workflowState, @@ -34,29 +34,29 @@ export const createAddResponseTool = () => return { success: true, id, - type: "response" as const, - message: "Response queued successfully", + type: 'response' as const, + message: 'Response queued successfully', }; }, }); export const createUpdateResponseTool = () => new DynamicStructuredTool({ - name: "update_response", - description: "Update a response in the approval queue", + name: 'update_response', + description: 'Update a response in the approval queue', schema: queueActionSchema, func: async (input: any) => { try { - logger.info("Updating response", { + logger.info('Updating response', { tweet_id: input.tweet.id, }); const existingResponse = await getResponseByTweetId(input.tweet.id); if (!existingResponse) { - logger.error("Could not find existing response to update:", { + logger.error('Could not find existing response to update:', { tweet_id: input.tweet.id, }); - throw new Error("Could not find existing response to update"); + throw new Error('Could not find existing response to update'); } await updateResponse({ @@ -69,17 +69,17 @@ export const createUpdateResponseTool = () => confidence: input.workflowState.responseStrategy.confidence, } as PendingResponse); - logger.info("Response updated successfully", { + logger.info('Response updated successfully', { response_id: existingResponse.id, }); return { success: true, id: existingResponse.id, - type: "response" as const, - message: "Response updated successfully", + type: 'response' as const, + message: 'Response updated successfully', }; } catch (error) { - logger.error("Error in update response tool:", error); + logger.error('Error in update response tool:', error); throw error; } }, diff --git a/auto-kol/agent/src/tools/tools/queueSkippedTool.ts b/auto-kol/agent/src/tools/tools/queueSkippedTool.ts index 552abc2..5fde79d 100644 --- a/auto-kol/agent/src/tools/tools/queueSkippedTool.ts +++ b/auto-kol/agent/src/tools/tools/queueSkippedTool.ts @@ -1,44 +1,44 @@ -import { DynamicStructuredTool } from "@langchain/core/tools"; -import { createLogger } from "../../utils/logger.js"; -import { v4 as generateId } from "uuid"; -import { queueActionSchema } from "../../schemas/workflow.js"; -import { addToSkipped } from "../../services/database/index.js"; +import { DynamicStructuredTool } from '@langchain/core/tools'; +import { createLogger } from '../../utils/logger.js'; +import { v4 as generateId } from 'uuid'; +import { queueActionSchema } from '../../schemas/workflow.js'; +import { addToSkipped } from '../../services/database/index.js'; -const logger = createLogger("queue-skipped-tool"); +const logger = createLogger('queue-skipped-tool'); export const createQueueSkippedTool = () => new DynamicStructuredTool({ - name: "queue_skipped", - description: "Add a skipped tweet to the review queue", + name: 'queue_skipped', + description: 'Add a skipped tweet to the review queue', schema: queueActionSchema, - func: async (input) => { + func: async input => { try { const id = generateId(); const skippedTweet = { id, tweet: input.tweet, - reason: input.reason || "No reason provided", + reason: input.reason || 'No reason provided', priority: input.priority || 0, created_at: new Date(), workflowState: input.workflowState, }; - logger.info("Queueing skipped tweet:", { + logger.info('Queueing skipped tweet:', { skippedTweet, }); addToSkipped(skippedTweet); - logger.info("Successfully queued skipped tweet:", { id }); + logger.info('Successfully queued skipped tweet:', { id }); return { success: true, id, - type: "skipped" as const, - message: "Tweet queued for review", + type: 'skipped' as const, + message: 'Tweet queued for review', }; } catch (error) { - logger.error("Error queueing skipped tweet:", error); + logger.error('Error queueing skipped tweet:', error); throw error; } }, diff --git a/auto-kol/agent/src/tools/tools/searchSimilarTweetsTool.ts b/auto-kol/agent/src/tools/tools/searchSimilarTweetsTool.ts index aac29d9..c465ba1 100644 --- a/auto-kol/agent/src/tools/tools/searchSimilarTweetsTool.ts +++ b/auto-kol/agent/src/tools/tools/searchSimilarTweetsTool.ts @@ -1,14 +1,14 @@ -import { DynamicStructuredTool } from "@langchain/core/tools"; -import { z } from "zod"; -import { createLogger } from "../../utils/logger.js"; -import { ChromaService } from "../../services/vectorstore/chroma.js"; +import { DynamicStructuredTool } from '@langchain/core/tools'; +import { z } from 'zod'; +import { createLogger } from '../../utils/logger.js'; +import { ChromaService } from '../../services/vectorstore/chroma.js'; -const logger = createLogger("search-similar-tweets-tool"); +const logger = createLogger('search-similar-tweets-tool'); export const createSearchSimilarTweetsTool = () => new DynamicStructuredTool({ - name: "search_similar_tweets", - description: "Search for similar tweets in the vector store", + name: 'search_similar_tweets', + description: 'Search for similar tweets in the vector store', schema: z.object({ query: z.string(), k: z.number().optional().default(5), @@ -16,10 +16,7 @@ export const createSearchSimilarTweetsTool = () => func: async ({ query, k }) => { try { const chromaService = await ChromaService.getInstance(); - const results = await chromaService.searchSimilarTweetsWithScore( - query, - k, - ); + const results = await chromaService.searchSimilarTweetsWithScore(query, k); return { similar_tweets: results.map(([doc, score]) => ({ text: doc.pageContent, @@ -28,7 +25,7 @@ export const createSearchSimilarTweetsTool = () => })), }; } catch (error) { - logger.error("Error searching similar tweets:", error); + logger.error('Error searching similar tweets:', error); return { similar_tweets: [] }; } }, diff --git a/auto-kol/agent/src/tools/tools/tweetSearchTool.ts b/auto-kol/agent/src/tools/tools/tweetSearchTool.ts index ae7e857..62075e0 100644 --- a/auto-kol/agent/src/tools/tools/tweetSearchTool.ts +++ b/auto-kol/agent/src/tools/tools/tweetSearchTool.ts @@ -1,11 +1,11 @@ -import { DynamicStructuredTool } from "@langchain/core/tools"; -import { z } from "zod"; -import { createLogger } from "../../utils/logger.js"; -import { getKOLsAccounts, updateKOLs } from "../../utils/twitter.js"; -import { SearchMode } from "agent-twitter-client"; -import { config } from "../../config/index.js"; -import { ExtendedScraper } from "../../services/twitter/api.js"; -const logger = createLogger("tweet-search-tool"); +import { DynamicStructuredTool } from '@langchain/core/tools'; +import { z } from 'zod'; +import { createLogger } from '../../utils/logger.js'; +import { getKOLsAccounts, updateKOLs } from '../../utils/twitter.js'; +import { SearchMode } from 'agent-twitter-client'; +import { config } from '../../config/index.js'; +import { ExtendedScraper } from '../../services/twitter/api.js'; +const logger = createLogger('tweet-search-tool'); function getRandomAccounts(accounts: string[], n: number): string[] { const shuffled = [...accounts].sort(() => 0.5 - Math.random()); @@ -14,19 +14,19 @@ function getRandomAccounts(accounts: string[], n: number): string[] { export const createTweetSearchTool = (scraper: ExtendedScraper) => new DynamicStructuredTool({ - name: "search_recent_tweets", - description: "Search for recent tweets from specified accounts", + name: 'search_recent_tweets', + description: 'Search for recent tweets from specified accounts', schema: z.object({ lastProcessedId: z.string().optional(), }), func: async ({ lastProcessedId }) => { try { - logger.info("Called search_recent_tweets"); + logger.info('Called search_recent_tweets'); await updateKOLs(scraper); const kols = await getKOLsAccounts(); if (kols.length === 0) { - logger.error("No valid accounts found after cleaning"); + logger.error('No valid accounts found after cleaning'); return { tweets: [], lastProcessedId: null, @@ -40,7 +40,7 @@ export const createTweetSearchTool = (scraper: ExtendedScraper) => for (let i = 0; i < selectedKols.length; i += ACCOUNTS_PER_QUERY) { const accountsBatch = selectedKols.slice(i, i + ACCOUNTS_PER_QUERY); - const query = `(${accountsBatch.map((account) => `from:${account}`).join(" OR ")})`; + const query = `(${accountsBatch.map(account => `from:${account}`).join(' OR ')})`; const searchIterator = scraper.searchTweets( query, @@ -53,22 +53,22 @@ export const createTweetSearchTool = (scraper: ExtendedScraper) => break; } tweetGroups.push({ - id: tweet.id || "", - text: tweet.text || "", - author_id: tweet.userId || "", - author_username: tweet.username?.toLowerCase() || "", + id: tweet.id || '', + text: tweet.text || '', + author_id: tweet.userId || '', + author_username: tweet.username?.toLowerCase() || '', created_at: tweet.timeParsed || new Date(), }); } - await new Promise((resolve) => setTimeout(resolve, 1000)); + await new Promise(resolve => setTimeout(resolve, 1000)); } const allTweets = tweetGroups.sort( (a, b) => b.created_at.getTime() - a.created_at.getTime(), ); - logger.info("Tweet search completed:", { + logger.info('Tweet search completed:', { foundTweets: allTweets.length, selectedKols, }); @@ -78,7 +78,7 @@ export const createTweetSearchTool = (scraper: ExtendedScraper) => lastProcessedId: allTweets[allTweets.length - 1]?.id || null, }; } catch (error) { - logger.error("Error searching tweets:", error); + logger.error('Error searching tweets:', error); return { tweets: [], lastProcessedId: null, diff --git a/auto-kol/agent/src/types/agent.ts b/auto-kol/agent/src/types/agent.ts index b9c392d..0ba8ea5 100644 --- a/auto-kol/agent/src/types/agent.ts +++ b/auto-kol/agent/src/types/agent.ts @@ -1,4 +1,4 @@ -import { Tweet } from "./twitter.js"; +import { Tweet } from './twitter.js'; export type AgentResponse = Readonly<{ content: string; diff --git a/auto-kol/agent/src/types/queue.ts b/auto-kol/agent/src/types/queue.ts index ddf68fd..992b108 100644 --- a/auto-kol/agent/src/types/queue.ts +++ b/auto-kol/agent/src/types/queue.ts @@ -1,12 +1,12 @@ -import { Tweet } from "./twitter.js"; -import { AgentResponse } from "./agent.js"; -import { WorkflowState } from "./workflow.js"; +import { Tweet } from './twitter.js'; +import { AgentResponse } from './agent.js'; +import { WorkflowState } from './workflow.js'; export enum ResponseStatus { - SKIPPED = "skipped", - PENDING = "pending", - APPROVED = "approved", - REJECTED = "rejected", + SKIPPED = 'skipped', + PENDING = 'pending', + APPROVED = 'approved', + REJECTED = 'rejected', } export interface PendingResponse { @@ -21,7 +21,7 @@ export interface PendingResponse { export type ActionResponse = Readonly<{ tweet: Tweet; - status: "pending" | "approved" | "rejected"; + status: 'pending' | 'approved' | 'rejected'; response: { id: string; content: string; @@ -36,7 +36,7 @@ export type QueuedResponseMemory = Readonly<{ id: string; tweet: Tweet; response: AgentResponse; - status: "pending" | "approved" | "rejected"; + status: 'pending' | 'approved' | 'rejected'; created_at: Date; updatedAt: Date; workflowState: WorkflowState; diff --git a/auto-kol/agent/src/types/workflow.ts b/auto-kol/agent/src/types/workflow.ts index cf46b95..16892fe 100644 --- a/auto-kol/agent/src/types/workflow.ts +++ b/auto-kol/agent/src/types/workflow.ts @@ -1,5 +1,5 @@ -import { Tweet } from "./twitter.js"; -import { BaseMessage } from "@langchain/core/messages"; +import { Tweet } from './twitter.js'; +import { BaseMessage } from '@langchain/core/messages'; export type EngagementDecision = Readonly<{ shouldEngage: boolean; diff --git a/auto-kol/agent/src/utils/agentMemoryContract.ts b/auto-kol/agent/src/utils/agentMemoryContract.ts index e5ba37e..4432593 100644 --- a/auto-kol/agent/src/utils/agentMemoryContract.ts +++ b/auto-kol/agent/src/utils/agentMemoryContract.ts @@ -1,8 +1,8 @@ -import { ethers } from "ethers"; -import { MEMORY_ABI } from "../abi/memory.js"; -import { config } from "../config/index.js"; -import { wallet } from "./agentWallet.js"; -import { cidFromBlakeHash, cidToString } from "@autonomys/auto-dag-data"; +import { ethers } from 'ethers'; +import { MEMORY_ABI } from '../abi/memory.js'; +import { config } from '../config/index.js'; +import { wallet } from './agentWallet.js'; +import { cidFromBlakeHash, cidToString } from '@autonomys/auto-dag-data'; const CONTRACT_ADDRESS = config.CONTRACT_ADDRESS as `0x${string}`; diff --git a/auto-kol/agent/src/utils/agentWallet.ts b/auto-kol/agent/src/utils/agentWallet.ts index 2df071f..72634c4 100644 --- a/auto-kol/agent/src/utils/agentWallet.ts +++ b/auto-kol/agent/src/utils/agentWallet.ts @@ -1,5 +1,5 @@ -import { ethers } from "ethers"; -import { config } from "../config/index.js"; +import { ethers } from 'ethers'; +import { config } from '../config/index.js'; const provider = new ethers.JsonRpcProvider(config.RPC_URL); diff --git a/auto-kol/agent/src/utils/dsn.ts b/auto-kol/agent/src/utils/dsn.ts index 5ac4e25..02cfc6c 100644 --- a/auto-kol/agent/src/utils/dsn.ts +++ b/auto-kol/agent/src/utils/dsn.ts @@ -1,18 +1,14 @@ -import { createLogger } from "../utils/logger.js"; -import { hexlify } from "ethers"; -import { createAutoDriveApi, uploadFile } from "@autonomys/auto-drive"; -import { - stringToCid, - blake3HashFromCid, - cidFromBlakeHash, -} from "@autonomys/auto-dag-data"; -import { addDsn, getLastDsnCid } from "../database/index.js"; -import { v4 as generateId } from "uuid"; -import { config } from "../config/index.js"; -import { setLastMemoryHash, getLastMemoryCid } from "./agentMemoryContract.js"; -import { signMessage, wallet } from "./agentWallet.js"; - -const logger = createLogger("dsn-upload-tool"); +import { createLogger } from '../utils/logger.js'; +import { hexlify } from 'ethers'; +import { createAutoDriveApi, uploadFile } from '@autonomys/auto-drive'; +import { stringToCid, blake3HashFromCid, cidFromBlakeHash } from '@autonomys/auto-dag-data'; +import { addDsn, getLastDsnCid } from '../database/index.js'; +import { v4 as generateId } from 'uuid'; +import { config } from '../config/index.js'; +import { setLastMemoryHash, getLastMemoryCid } from './agentMemoryContract.js'; +import { signMessage, wallet } from './agentWallet.js'; + +const logger = createLogger('dsn-upload-tool'); const dsnAPI = createAutoDriveApi({ apiKey: config.DSN_API_KEY! }); let currentNonce = await wallet.getNonce(); @@ -22,10 +18,7 @@ interface RetryOptions { onRetry?: (error: Error, attempt: number) => void; } -async function retry( - fn: () => Promise, - options: RetryOptions, -): Promise { +async function retry(fn: () => Promise, options: RetryOptions): Promise { const { maxRetries, delay, onRetry } = options; let lastError: Error; @@ -45,9 +38,7 @@ async function retry( // Add jitter to prevent thundering herd const jitter = Math.random() * 1000; - await new Promise((resolve) => - setTimeout(resolve, delay * attempt + jitter), - ); + await new Promise(resolve => setTimeout(resolve, delay * attempt + jitter)); } } @@ -57,16 +48,16 @@ async function retry( const getPreviousCid = async (): Promise => { const dsnLastCid = await getLastDsnCid(); if (dsnLastCid) { - logger.info("Using last CID from local db", { cid: dsnLastCid }); + logger.info('Using last CID from local db', { cid: dsnLastCid }); return dsnLastCid; } const memoryLastCid = await getLastMemoryCid(); - logger.info("Using fallback CID source", { - memoryLastCid: memoryLastCid || "not found", + logger.info('Using fallback CID source', { + memoryLastCid: memoryLastCid || 'not found', }); - return memoryLastCid || ""; + return memoryLastCid || ''; }; export async function uploadToDsn({ data }: { data: any }) { @@ -101,7 +92,7 @@ export async function uploadToDsn({ data }: { data: any }) { yield jsonBuffer; }, name: `${config.TWITTER_USERNAME}-agent-memory-${timestamp}.json`, - mimeType: "application/json", + mimeType: 'application/json', size: jsonBuffer.length, path: timestamp, }, @@ -111,14 +102,14 @@ export async function uploadToDsn({ data }: { data: any }) { }, ); - await uploadObservable.forEach((status) => { - if (status.type === "file" && status.cid) { + await uploadObservable.forEach(status => { + if (status.type === 'file' && status.cid) { finalCid = status.cid.toString(); } }); if (!finalCid) { - throw new Error("Failed to get CID from DSN upload"); + throw new Error('Failed to get CID from DSN upload'); } }, { @@ -134,18 +125,18 @@ export async function uploadToDsn({ data }: { data: any }) { ); if (!finalCid) { - throw new Error("Failed to get CID from DSN upload after retries"); + throw new Error('Failed to get CID from DSN upload after retries'); } const blake3hash = blake3HashFromCid(stringToCid(finalCid)); - logger.info("Setting last memory hash", { + logger.info('Setting last memory hash', { blake3hash: hexlify(blake3hash), }); await retry( async () => { const tx = await setLastMemoryHash(hexlify(blake3hash), currentNonce++); - logger.info("Memory hash transaction submitted", { + logger.info('Memory hash transaction submitted', { txHash: tx.hash, previousCid, cid: finalCid, @@ -162,8 +153,8 @@ export async function uploadToDsn({ data }: { data: any }) { }); }, }, - ).catch((error) => { - logger.error("Failed to submit memory hash transaction", error); + ).catch(error => { + logger.error('Failed to submit memory hash transaction', error); }); await addDsn({ @@ -178,7 +169,7 @@ export async function uploadToDsn({ data }: { data: any }) { previousCid: previousCid || null, }; } catch (error) { - logger.error("Error uploading to DSN:", error); + logger.error('Error uploading to DSN:', error); throw error; } } diff --git a/auto-kol/agent/src/utils/logger.ts b/auto-kol/agent/src/utils/logger.ts index e43c606..f72e66d 100644 --- a/auto-kol/agent/src/utils/logger.ts +++ b/auto-kol/agent/src/utils/logger.ts @@ -1,27 +1,25 @@ -import winston from "winston"; -import { config } from "../config/index.js"; -import util from "util"; +import winston from 'winston'; +import { config } from '../config/index.js'; +import util from 'util'; const formatMeta = (meta: any, useColors: boolean = false) => { const cleanMeta = Object.entries(meta) - .filter(([key]) => !key.startsWith("Symbol(") && key !== "splat") + .filter(([key]) => !key.startsWith('Symbol(') && key !== 'splat') .reduce((acc, [key, value]) => ({ ...acc, [key]: value }), {}); - if (Object.keys(cleanMeta).length === 0) return ""; + if (Object.keys(cleanMeta).length === 0) return ''; - if (meta[Symbol.for("splat")]?.[0]) { - Object.assign(cleanMeta, meta[Symbol.for("splat")][0]); + if (meta[Symbol.for('splat')]?.[0]) { + Object.assign(cleanMeta, meta[Symbol.for('splat')][0]); } - return Object.keys(cleanMeta).length - ? "\n" + JSON.stringify(cleanMeta, null, 2) - : ""; + return Object.keys(cleanMeta).length ? '\n' + JSON.stringify(cleanMeta, null, 2) : ''; }; const createFileFormat = () => winston.format.combine( winston.format.timestamp({ - format: "YYYY-MM-DD HH:mm:ss.SSS", + format: 'YYYY-MM-DD HH:mm:ss.SSS', }), winston.format.uncolorize(), winston.format.printf(({ level, message, context, timestamp, ...meta }) => { @@ -34,7 +32,7 @@ const createFileFormat = () => const createConsoleFormat = () => winston.format.combine( winston.format.timestamp({ - format: "YYYY-MM-DD HH:mm:ss.SSS", + format: 'YYYY-MM-DD HH:mm:ss.SSS', }), winston.format.colorize({ level: true }), winston.format.printf(({ level, message, context, timestamp, ...meta }) => { @@ -46,15 +44,15 @@ const createConsoleFormat = () => const createTransports = () => [ new winston.transports.File({ - filename: "logs/error.log", - level: "error", + filename: 'logs/error.log', + level: 'error', format: createFileFormat(), maxsize: 5242880, maxFiles: 5, tailable: true, }), new winston.transports.File({ - filename: "logs/combined.log", + filename: 'logs/combined.log', format: createFileFormat(), maxsize: 5242880, maxFiles: 5, @@ -63,7 +61,7 @@ const createTransports = () => [ ]; const addConsoleTransport = (logger: winston.Logger): winston.Logger => { - if (config.NODE_ENV !== "production") { + if (config.NODE_ENV !== 'production') { logger.add( new winston.transports.Console({ format: createConsoleFormat(), @@ -76,7 +74,7 @@ const addConsoleTransport = (logger: winston.Logger): winston.Logger => { export const createLogger = (context: string) => { const logger = winston.createLogger({ defaultMeta: { context }, - level: "info", + level: 'info', format: createFileFormat(), transports: createTransports(), }); diff --git a/auto-kol/agent/src/utils/twitter.ts b/auto-kol/agent/src/utils/twitter.ts index 2ddb0dd..61da8e1 100644 --- a/auto-kol/agent/src/utils/twitter.ts +++ b/auto-kol/agent/src/utils/twitter.ts @@ -1,24 +1,22 @@ -import { config } from "../config/index.js"; -import { createLogger } from "../utils/logger.js"; -import { ExtendedScraper } from "../services/twitter/api.js"; -import * as db from "../database/index.js"; -import { KOL } from "../types/kol.js"; -import { Tweet } from "../types/twitter.js"; +import { config } from '../config/index.js'; +import { createLogger } from '../utils/logger.js'; +import { ExtendedScraper } from '../services/twitter/api.js'; +import * as db from '../database/index.js'; +import { KOL } from '../types/kol.js'; +import { Tweet } from '../types/twitter.js'; -const logger = createLogger("twitter-utils"); +const logger = createLogger('twitter-utils'); export const timelineTweets: Tweet[] = []; export const updateKOLs = async (twitterScraper: ExtendedScraper) => { const currentKOLs = await db.getKOLAccounts(); - const twitterProfile = await twitterScraper.getProfile( - config.TWITTER_USERNAME!, - ); + const twitterProfile = await twitterScraper.getProfile(config.TWITTER_USERNAME!); const followings = twitterScraper.getFollowing(twitterProfile.userId!, 1000); logger.info(`following count: ${twitterProfile.followingCount}`); const newKOLs: KOL[] = []; for await (const following of followings) { - if (!currentKOLs.some((kol) => kol.username === following.username)) { + if (!currentKOLs.some(kol => kol.username === following.username)) { newKOLs.push({ id: following.userId!, username: following.username!.toLowerCase(), @@ -33,13 +31,11 @@ export const updateKOLs = async (twitterScraper: ExtendedScraper) => { export const getKOLsAccounts = async () => { const kolAccounts = await db.getKOLAccounts(); - return kolAccounts.map((kol) => kol.username); + return kolAccounts.map(kol => kol.username); }; export const getTimeLine = async (twitterScraper: ExtendedScraper) => { - const validTweetIds = timelineTweets - .map((tweet) => tweet.id) - .filter((id) => id != null); + const validTweetIds = timelineTweets.map(tweet => tweet.id).filter(id => id != null); const timeline = await twitterScraper.fetchHomeTimeline(0, validTweetIds); // clear timeline @@ -65,10 +61,7 @@ const clearTimeLine = () => { timelineTweets.length = 0; }; -export const getUserProfile = async ( - twitterScraper: ExtendedScraper, - username: string, -) => { +export const getUserProfile = async (twitterScraper: ExtendedScraper, username: string) => { const user = await twitterScraper.getProfile(username); const result: KOL = { id: user.userId!, From ae9731efb0afa11c5e77204ace89bd754b7fb2e0 Mon Sep 17 00:00:00 2001 From: xm0onh Date: Tue, 24 Dec 2024 14:35:33 -0800 Subject: [PATCH 13/13] resolving further conflicts --- .../agent/src/services/agents/nodes/autoApprovalNode.ts | 8 -------- .../agent/src/services/agents/nodes/recheckSkippedNode.ts | 4 ---- auto-kol/agent/src/tools/index.ts | 4 ---- auto-kol/agent/src/tools/tools/fetchTimelineTool.ts | 8 -------- auto-kol/agent/src/tools/tools/tweetSearchTool.ts | 4 ---- 5 files changed, 28 deletions(-) diff --git a/auto-kol/agent/src/services/agents/nodes/autoApprovalNode.ts b/auto-kol/agent/src/services/agents/nodes/autoApprovalNode.ts index 7ccce8b..997a306 100644 --- a/auto-kol/agent/src/services/agents/nodes/autoApprovalNode.ts +++ b/auto-kol/agent/src/services/agents/nodes/autoApprovalNode.ts @@ -7,11 +7,7 @@ import { uploadToDsn } from '../../../utils/dsn.js'; import { config as globalConfig } from '../../../config/index.js'; import { ResponseStatus } from '../../../types/queue.js'; -<<<<<<< HEAD export const createAutoApprovalNode = (config: WorkflowConfig) => { -======= -export const createAutoApprovalNode = (config: WorkflowConfig, scraper: ExtendedScraper) => { ->>>>>>> main return async (state: typeof State.State) => { logger.info('Auto Approval Node - Evaluating pending responses'); try { @@ -62,14 +58,10 @@ export const createAutoApprovalNode = (config: WorkflowConfig, scraper: Extended tweetId: response.tweet.id, }); -<<<<<<< HEAD const sendTweetResponse = await config.client.sendTweet( response.response, response.tweet.id, ); -======= - const sendTweetResponse = await scraper.sendTweet(response.response, response.tweet.id); ->>>>>>> main logger.info('Tweet sent', { sendTweetResponse, }); diff --git a/auto-kol/agent/src/services/agents/nodes/recheckSkippedNode.ts b/auto-kol/agent/src/services/agents/nodes/recheckSkippedNode.ts index 8dff4d2..0678c77 100644 --- a/auto-kol/agent/src/services/agents/nodes/recheckSkippedNode.ts +++ b/auto-kol/agent/src/services/agents/nodes/recheckSkippedNode.ts @@ -51,13 +51,9 @@ export const createRecheckSkippedNode = (config: WorkflowConfig) => { } else { const flagged = await flagBackSkippedTweet(tweet.id, decision.reason); if (!flagged) { -<<<<<<< HEAD logger.info('Failed to flag back skipped tweet:', { tweetId: tweet.id, }); -======= - logger.info('Failed to flag back skipped tweet:', { tweetId: tweet.id }); ->>>>>>> main } } } diff --git a/auto-kol/agent/src/tools/index.ts b/auto-kol/agent/src/tools/index.ts index 53f07b5..a0a8f43 100644 --- a/auto-kol/agent/src/tools/index.ts +++ b/auto-kol/agent/src/tools/index.ts @@ -10,11 +10,7 @@ import { ExtendedScraper } from '../services/twitter/api.js'; export const createTools = (scraper: ExtendedScraper) => { const mentionTool = createMentionTool(scraper); -<<<<<<< HEAD const fetchTimelineTool = createFetchTimelineTool(scraper); -======= - const fetchTimelineTool = createFetchTimelineTool(); ->>>>>>> main const tweetSearchTool = createTweetSearchTool(scraper); diff --git a/auto-kol/agent/src/tools/tools/fetchTimelineTool.ts b/auto-kol/agent/src/tools/tools/fetchTimelineTool.ts index 2ed6d37..1a7a46b 100644 --- a/auto-kol/agent/src/tools/tools/fetchTimelineTool.ts +++ b/auto-kol/agent/src/tools/tools/fetchTimelineTool.ts @@ -6,22 +6,14 @@ import { ExtendedScraper } from '../../services/twitter/api.js'; const logger = createLogger('fetch-timeline-tool'); -<<<<<<< HEAD export const createFetchTimelineTool = (twitterScraper: ExtendedScraper) => -======= -export const createFetchTimelineTool = () => ->>>>>>> main new DynamicStructuredTool({ name: 'fetch_timeline', description: 'Fetch the timeline regularly to get new tweets', schema: z.object({}), func: async () => { try { -<<<<<<< HEAD const tweets = await getTimeLine(twitterScraper); -======= - const tweets = await getTimeLine(); ->>>>>>> main tweets.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime()); return { tweets: tweets, diff --git a/auto-kol/agent/src/tools/tools/tweetSearchTool.ts b/auto-kol/agent/src/tools/tools/tweetSearchTool.ts index 8707bc6..62075e0 100644 --- a/auto-kol/agent/src/tools/tools/tweetSearchTool.ts +++ b/auto-kol/agent/src/tools/tools/tweetSearchTool.ts @@ -22,11 +22,7 @@ export const createTweetSearchTool = (scraper: ExtendedScraper) => func: async ({ lastProcessedId }) => { try { logger.info('Called search_recent_tweets'); -<<<<<<< HEAD await updateKOLs(scraper); -======= - await updateKOLs(); ->>>>>>> main const kols = await getKOLsAccounts(); if (kols.length === 0) {