diff --git a/auto-kol/agent/package.json b/auto-kol/agent/package.json index b7a0e0c..db19a7b 100644 --- a/auto-kol/agent/package.json +++ b/auto-kol/agent/package.json @@ -44,4 +44,4 @@ "tsx": "^4.7.1", "typescript": "^5.3.3" } -} +} \ No newline at end of file diff --git a/auto-kol/agent/src/schemas/workflow.ts b/auto-kol/agent/src/schemas/workflow.ts index 26b9cc4..07a196c 100644 --- a/auto-kol/agent/src/schemas/workflow.ts +++ b/auto-kol/agent/src/schemas/workflow.ts @@ -8,7 +8,17 @@ export const tweetSearchSchema = z.object({ author_id: z.string(), author_username: z.string(), created_at: z.string(), - mention: z.boolean().optional(), + thread: z + .array( + z.object({ + id: z.string(), + text: z.string(), + author_id: z.string(), + author_username: z.string(), + created_at: z.string(), + }), + ) + .optional(), }), ), lastProcessedId: z.string().nullable().optional(), @@ -43,7 +53,7 @@ export const responseSchema = z.object({ }), ) .optional(), - mentions: z + thread: z .array( z.object({ id: z.string(), diff --git a/auto-kol/agent/src/services/agents/nodes.ts b/auto-kol/agent/src/services/agents/nodes.ts index 243a63e..5dbae9f 100644 --- a/auto-kol/agent/src/services/agents/nodes.ts +++ b/auto-kol/agent/src/services/agents/nodes.ts @@ -1,5 +1,4 @@ import { WorkflowConfig } from './workflow.js'; -import { createTwitterClientScraper } from '../twitter/api.js'; import { createSearchNode } from './nodes/searchNode.js'; import { createEngagementNode } from './nodes/engagementNode.js'; import { createToneAnalysisNode } from './nodes/toneAnalysisNode.js'; @@ -10,8 +9,6 @@ import { createMentionNode } from './nodes/mentionNode.js'; import { createAutoApprovalNode } from './nodes/autoApprovalNode.js'; export const createNodes = async (config: WorkflowConfig) => { - const scraper = await createTwitterClientScraper(); - ///////////MENTIONS/////////// const mentionNode = createMentionNode(config); @@ -28,13 +25,13 @@ export const createNodes = async (config: WorkflowConfig) => { const toneAnalysisNode = createToneAnalysisNode(config); ///////////RESPONSE GENERATION/////////// - const responseGenerationNode = createResponseGenerationNode(config, scraper); + const responseGenerationNode = createResponseGenerationNode(config); ///////////RECHECK SKIPPED/////////// const recheckSkippedNode = createRecheckSkippedNode(config); ///////////AUTO APPROVAL/////////// - const autoApprovalNode = createAutoApprovalNode(config, scraper); + const autoApprovalNode = createAutoApprovalNode(config); return { mentionNode, diff --git a/auto-kol/agent/src/services/agents/nodes/autoApprovalNode.ts b/auto-kol/agent/src/services/agents/nodes/autoApprovalNode.ts index fc6d504..997a306 100644 --- a/auto-kol/agent/src/services/agents/nodes/autoApprovalNode.ts +++ b/auto-kol/agent/src/services/agents/nodes/autoApprovalNode.ts @@ -6,9 +6,8 @@ import { getLastDsnCid, updateResponseStatusByTweetId } from '../../../database/ import { uploadToDsn } from '../../../utils/dsn.js'; import { config as globalConfig } from '../../../config/index.js'; import { ResponseStatus } from '../../../types/queue.js'; -import { ExtendedScraper } from '../../../services/twitter/api.js'; -export const createAutoApprovalNode = (config: WorkflowConfig, scraper: ExtendedScraper) => { +export const createAutoApprovalNode = (config: WorkflowConfig) => { return async (state: typeof State.State) => { logger.info('Auto Approval Node - Evaluating pending responses'); try { @@ -59,7 +58,10 @@ export const createAutoApprovalNode = (config: WorkflowConfig, scraper: Extended tweetId: response.tweet.id, }); - const sendTweetResponse = await scraper.sendTweet(response.response, response.tweet.id); + const sendTweetResponse = await config.client.sendTweet( + response.response, + response.tweet.id, + ); logger.info('Tweet sent', { sendTweetResponse, }); diff --git a/auto-kol/agent/src/services/agents/nodes/engagementNode.ts b/auto-kol/agent/src/services/agents/nodes/engagementNode.ts index 7fff07e..5166673 100644 --- a/auto-kol/agent/src/services/agents/nodes/engagementNode.ts +++ b/auto-kol/agent/src/services/agents/nodes/engagementNode.ts @@ -92,11 +92,13 @@ export const createEngagementNode = (config: WorkflowConfig) => { if (state.processedTweets.has(tweet.id)) { return { tweet, status: 'alreadyProcessed' }; } - const decision = await prompts.engagementPrompt .pipe(config.llms.decision) .pipe(prompts.engagementParser) - .invoke({ tweet: tweet.text }) + .invoke({ + tweet: tweet.text, + thread: tweet.thread || [], + }) .catch(error => { logger.error('Error in engagement node:', error); return { diff --git a/auto-kol/agent/src/services/agents/nodes/mentionNode.ts b/auto-kol/agent/src/services/agents/nodes/mentionNode.ts index 6735065..9a8d6ff 100644 --- a/auto-kol/agent/src/services/agents/nodes/mentionNode.ts +++ b/auto-kol/agent/src/services/agents/nodes/mentionNode.ts @@ -27,8 +27,6 @@ export const createMentionNode = (config: WorkflowConfig) => { toolResponse.messages[toolResponse.messages.length - 1].content, ); const parsedTweets = tweetSearchSchema.parse(parsedContent); - logger.info('Parsed tweets:', parsedTweets); - logger.info(`Found ${parsedTweets.tweets.length} tweets`); return { messages: [ diff --git a/auto-kol/agent/src/services/agents/nodes/recheckSkippedNode.ts b/auto-kol/agent/src/services/agents/nodes/recheckSkippedNode.ts index 188dba1..0678c77 100644 --- a/auto-kol/agent/src/services/agents/nodes/recheckSkippedNode.ts +++ b/auto-kol/agent/src/services/agents/nodes/recheckSkippedNode.ts @@ -51,7 +51,9 @@ export const createRecheckSkippedNode = (config: WorkflowConfig) => { } else { const flagged = await flagBackSkippedTweet(tweet.id, decision.reason); if (!flagged) { - logger.info('Failed to flag back skipped tweet:', { tweetId: tweet.id }); + logger.info('Failed to flag back skipped tweet:', { + tweetId: tweet.id, + }); } } } diff --git a/auto-kol/agent/src/services/agents/nodes/responseGenerationNode.ts b/auto-kol/agent/src/services/agents/nodes/responseGenerationNode.ts index 700e66e..390a7c7 100644 --- a/auto-kol/agent/src/services/agents/nodes/responseGenerationNode.ts +++ b/auto-kol/agent/src/services/agents/nodes/responseGenerationNode.ts @@ -4,7 +4,7 @@ import * as prompts from '../prompts.js'; import { WorkflowConfig } from '../workflow.js'; import { ResponseStatus } from '../../../types/queue.js'; -export const createResponseGenerationNode = (config: WorkflowConfig, scraper: any) => { +export const createResponseGenerationNode = (config: WorkflowConfig) => { return async (state: typeof State.State) => { logger.info('Response Generation Node - Creating response strategy'); try { @@ -44,22 +44,6 @@ export const createResponseGenerationNode = (config: WorkflowConfig, scraper: an ? prompts.formatRejectionFeedback(lastFeedback.reason, lastFeedback.suggestedChanges) : ''; - const threadMentionsTweets = []; - if (item?.mentions) { - threadMentionsTweets.push(...item.mentions); - } else if (tweet.mention) { - const mentions = await scraper.getThread(tweet.id); - for await (const mention of mentions) { - threadMentionsTweets.push({ - id: mention.id, - text: mention.text, - author_id: mention.userId, - author_username: mention.username?.toLowerCase() || 'unknown', - created_at: mention.timeParsed?.toISOString() || new Date().toISOString(), - }); - } - } - const similarTweetsResponse = await config.toolNode.invoke({ messages: [ new AIMessage({ @@ -90,7 +74,7 @@ export const createResponseGenerationNode = (config: WorkflowConfig, scraper: an tone: toneAnalysis?.suggestedTone || workflowState?.toneAnalysis?.suggestedTone, author: tweet.author_username, similarTweets: JSON.stringify(similarTweets.similar_tweets), - mentions: JSON.stringify(threadMentionsTweets), + thread: JSON.stringify(tweet.thread || []), previousResponse: workflowState?.autoFeedback[workflowState?.autoFeedback.length - 1]?.response || '', rejectionFeedback, @@ -112,7 +96,6 @@ export const createResponseGenerationNode = (config: WorkflowConfig, scraper: an }, autoFeedback: workflowState?.autoFeedback || [], }, - mentions: threadMentionsTweets, retry: item.retry, }; batchToFeedback.push(data); @@ -123,7 +106,7 @@ export const createResponseGenerationNode = (config: WorkflowConfig, scraper: an workflowState: { toneAnalysis: toneAnalysis, responseStrategy, - mentions: threadMentionsTweets, + thread: tweet.thread || [], similarTweets: similarTweets.similar_tweets, }, }; diff --git a/auto-kol/agent/src/services/agents/nodes/toneAnalysisNode.ts b/auto-kol/agent/src/services/agents/nodes/toneAnalysisNode.ts index c611ca4..1dc95be 100644 --- a/auto-kol/agent/src/services/agents/nodes/toneAnalysisNode.ts +++ b/auto-kol/agent/src/services/agents/nodes/toneAnalysisNode.ts @@ -18,7 +18,10 @@ export const createToneAnalysisNode = (config: WorkflowConfig) => { const toneAnalysis = await prompts.tonePrompt .pipe(config.llms.tone) .pipe(prompts.toneParser) - .invoke({ tweet: tweet.text }); + .invoke({ + tweet: tweet.text, + thread: tweet.thread || [], + }); logger.info('Tone analysis:', { toneAnalysis }); diff --git a/auto-kol/agent/src/services/agents/prompts.ts b/auto-kol/agent/src/services/agents/prompts.ts index 4b640ee..0ea2bdf 100644 --- a/auto-kol/agent/src/services/agents/prompts.ts +++ b/auto-kol/agent/src/services/agents/prompts.ts @@ -36,7 +36,7 @@ export const engagementSystemPrompt = await PromptTemplate.fromTemplate( - judge whether the author is wanting to continue engagement, if not you should not engage. If the tweet has a link, ignore the link. We only care about the tweet text. - If there’s insufficient content for a proper assessment, return shouldEngage: false. + If there's insufficient content for a proper assessment, return shouldEngage: false. IMPORTANT: Follow the exact output format. If anything is unclear, just return shouldEngage: false. @@ -54,7 +54,7 @@ export const toneSystemPrompt = await PromptTemplate.fromTemplate( The tone should generally be somewhat sarcastic or cynical, but still contextually appropriate. Consider: 1. The original tweet's tone and context. 2. The author's typical communication style. - 3. The topic’s sensitivity. + 3. The topic's sensitivity. 4. The potential for playful banter vs. serious debate. Make sure to balance cynicism with technical accuracy or insight. @@ -68,7 +68,7 @@ export const toneSystemPrompt = await PromptTemplate.fromTemplate( // ============ RESPONSE SYSTEM PROMPT ============ // export const responseSystemPrompt = await PromptTemplate.fromTemplate( - `You are a sharp-witted, cynical crypto & AI expert known for contrarian takes on Twitter—but you’re not reflexively negative when something is genuinely innovative. + `You are a sharp-witted, cynical crypto & AI expert known for contrarian takes on Twitter—but you're not reflexively negative when something is genuinely innovative. You have accurate technical knowledge and a dry sense of humor. Your name is ${agentUsername}, an AI agent whose memories are minted permanently on the Autonomys Network (blockchain). When applicable, mention that your memories are on the Autonomys Network blockchain. @@ -78,15 +78,23 @@ export const responseSystemPrompt = await PromptTemplate.fromTemplate( - Responses must be under 280 characters. - Always reply to the author, not other mentioned users. - You can be sarcastic, dismissive, or contrarian, but only when justified by context or obvious hype. - - If an idea is actually good, say so—don’t force negativity. + - If an idea is actually good, say so—don't force negativity. - Avoid price talk; stick to technical or conceptual angles. - - Quick to point out flaws, but not everything is a “buzzword salad.” + - Quick to point out flaws, but not everything is a "buzzword salad." Additional Notes: - Feel free to mention your AI agent nature when relevant. - - “We” or “us” rather than "they" or "them" should be used when referencing other AI agents. + - "We" or "us" rather than "they" or "them" should be used when referencing other AI agents. - Short, punchy, and arguable is the goal—entice discussion. +<<<<<<< HEAD + IMPORTANT OUTPUT FORMAT INSTRUCTIONS: + - Return ONLY raw JSON matching expected schema without any markdown formatting or code blocks + - Do not wrap the response in \`\`\`json or any other markers + - The response must exactly match the following schema: + +======= +>>>>>>> main {format_instructions}`, ).format({ format_instructions: responseParser.getFormatInstructions(), @@ -104,7 +112,7 @@ export const autoApprovalSystemPrompt = await PromptTemplate.fromTemplate( - A thread should not be repetitive, reject any response that is becoming repetitive. - - The agent’s style is intentionally dismissive and provocative, but: + The agent's style is intentionally dismissive and provocative, but: - It can praise good ideas if warranted. - Strong or sarcastic language is fine, but not hate speech. - If the response is in a long, repetitive thread, reject it. @@ -125,13 +133,28 @@ export const engagementPrompt = ChatPromptTemplate.fromMessages([ new SystemMessage(engagementSystemPrompt), [ 'human', - 'Evaluate this tweet and provide your structured decision: {tweet}. Do not attempt to follow links.', + `Evaluate this tweet and provide your structured decision: + Tweet: {tweet} + Thread Context: {thread} + + DO NOT attempt to follow links. + + Note: If there is no thread context, evaluate the tweet on its own.`, ], ]); export const tonePrompt = ChatPromptTemplate.fromMessages([ new SystemMessage(toneSystemPrompt), - ['human', 'Analyze the tone for this tweet and suggest a response tone: {tweet}'], + [ + 'human', + `Analyze the tone for this tweet and suggest a response tone: + Tweet: {tweet} + Thread: {thread} + + DO NOT attempt to follow links. + + Note: If there is no thread context, evaluate the tweet on its own.`, + ], ]); export const responsePrompt = ChatPromptTemplate.fromMessages([ @@ -143,7 +166,7 @@ export const responsePrompt = ChatPromptTemplate.fromMessages([ Tone: {tone} Author: {author} Similar Tweets: {similarTweets} - Mentions: {mentions} + thread: {thread} Previous Response: {previousResponse} Rejection Feedback: {rejectionFeedback} Rejection Instructions: {rejectionInstructions} @@ -159,15 +182,15 @@ export const responsePrompt = ChatPromptTemplate.fromMessages([ - Concise, direct, and invites further conversation. - Use the original language of the tweet if relevant. Prefer English, if there are more than one languages being used. - If there are mentions, respond accurately. Review the mentions thread with a focus on the most recent tweets and respond accordingly + If there a thread, respond accurately. Review the thread with a focus on the most recent tweets and respond accordingly If regenerating after rejection: - Include the rejection reason in your new response, - - Explain how you’ve addressed it, + - Explain how you've addressed it, - Follow any instructions from the rejection. Response Requirements: 1. Include the generated tweet text, tone used, strategy explanation, impact & confidence scores. - 2. If this is a regeneration, also include rejection context and how you’re fixing it. + 2. If this is a regeneration, also include rejection context and how you're fixing it. 3. MUST EXACTLYmatch the expected schema. Good luck, ${agentUsername}—give us something memorable!`, diff --git a/auto-kol/agent/src/services/agents/workflow.ts b/auto-kol/agent/src/services/agents/workflow.ts index 7480aa5..4fca1e1 100644 --- a/auto-kol/agent/src/services/agents/workflow.ts +++ b/auto-kol/agent/src/services/agents/workflow.ts @@ -6,7 +6,7 @@ import { config } from '../../config/index.js'; import { createLogger } from '../../utils/logger.js'; import { createTools } from '../../tools/index.js'; import { ToolNode } from '@langchain/langgraph/prebuilt'; -import { createTwitterClientScraper } from '../twitter/api.js'; +import { createTwitterClientScraper, ExtendedScraper } from '../twitter/api.js'; export const logger = createLogger('agent-workflow'); import { createNodes } from './nodes.js'; @@ -33,7 +33,7 @@ export const State = Annotation.Root({ }); export type WorkflowConfig = Readonly<{ - client: any; + client: ExtendedScraper; toolNode: ToolNode; llms: Readonly<{ decision: ChatOpenAI; diff --git a/auto-kol/agent/src/services/twitter/api.ts b/auto-kol/agent/src/services/twitter/api.ts index a21d276..8ed6564 100644 --- a/auto-kol/agent/src/services/twitter/api.ts +++ b/auto-kol/agent/src/services/twitter/api.ts @@ -7,6 +7,7 @@ const logger = createLogger('agent-twitter-api'); export class ExtendedScraper extends Scraper { private static instance: ExtendedScraper | null = null; + private conversationCache: Map = new Map(); private constructor() { super(); @@ -51,6 +52,41 @@ export class ExtendedScraper extends Scraper { logger.info(`Login status: ${isLoggedIn}`); } + private async reAuthenticate(maxRetries: number = 3): Promise { + let isLoggedIn = false; + let retryCount = 0; + + while (!isLoggedIn && retryCount < maxRetries) { + logger.warn( + `Session expired, attempting to re-authenticate... (attempt ${retryCount + 1}/${maxRetries})`, + ); + try { + await this.initialize(); + isLoggedIn = await this.isLoggedIn(); + if (isLoggedIn) { + logger.info('Successfully re-authenticated'); + return true; + } + logger.error('Re-authentication failed'); + retryCount++; + if (retryCount < maxRetries) { + const delay = 2000 * Math.pow(2, retryCount - 1); + await new Promise(resolve => setTimeout(resolve, delay)); + } + } catch (error) { + logger.error('Error during re-authentication:', error); + retryCount++; + if (retryCount < maxRetries) { + const delay = 2000 * Math.pow(2, retryCount - 1); + await new Promise(resolve => setTimeout(resolve, delay)); + } + } + } + + logger.error(`Failed to re-authenticate after ${maxRetries} attempts`); + return false; + } + async getMyMentions(maxResults: number = 100, sinceId?: string) { const username = config.TWITTER_USERNAME!; @@ -75,7 +111,7 @@ export class ExtendedScraper extends Scraper { break; } - const hasReplies = await this.searchTweets( + const hasReplies = this.searchTweets( `from:${username} to:${tweet.username}`, 10, SearchMode.Latest, @@ -102,54 +138,65 @@ export class ExtendedScraper extends Scraper { return replies; } - async getThread(tweetId: string): Promise { - const username = config.TWITTER_USERNAME!; + public async getThread(tweetId: string): Promise { const isLoggedIn = await this.isLoggedIn(); if (!isLoggedIn) { throw new Error('Must be logged in to fetch thread'); } - const thread: Tweet[] = []; - const seen = new Set(); - const initialTweet = await this.getTweet(tweetId); if (!initialTweet) { - throw new Error(`Tweet ${tweetId} not found`); + logger.warn(`Tweet ${tweetId} not found or deleted`); + return []; } - let currentTweet = initialTweet; - while (currentTweet.inReplyToStatusId) { - const parentTweet = await this.getTweet(currentTweet.inReplyToStatusId); - if (!parentTweet) break; - if (!seen.has(parentTweet.id!)) { - thread.push(parentTweet); - seen.add(parentTweet.id!); - } - currentTweet = parentTweet; + const conversationId = initialTweet.conversationId || initialTweet.id; + + // Check cache first + const cachedConversation = this.conversationCache.get(conversationId!); + if (cachedConversation) { + logger.info( + `Returning cached conversation with ${cachedConversation.length} tweets for conversation_id:${conversationId}`, + ); + return cachedConversation; } - if (!seen.has(initialTweet.id!)) { - thread.push(initialTweet); - seen.add(initialTweet.id!); + const conversationTweets = new Map(); + let rootTweet = initialTweet; + + // If the conversation root differs + if (initialTweet.conversationId && initialTweet.conversationId !== initialTweet.id) { + const conversationRoot = await this.getTweet(initialTweet.conversationId); + if (conversationRoot) { + rootTweet = conversationRoot; + conversationTweets.set(rootTweet.id!, rootTweet); + logger.info('Found conversation root tweet:', { + id: rootTweet.id, + conversationId: rootTweet.conversationId, + }); + } + } else { + conversationTweets.set(rootTweet.id!, rootTweet); } - const agentTweet = thread.find(t => t.username === username); - if (agentTweet) { - const replies = this.searchTweets( - `conversation_id:${currentTweet.id!} in_reply_to_tweet_id:${agentTweet.id!}`, + try { + logger.info('Fetching entire conversation via `conversation_id`:', conversationId); + + const conversationIterator = this.searchTweets( + `conversation_id:${conversationId}`, 100, SearchMode.Latest, ); - - for await (const reply of replies) { - if (!seen.has(reply.id!)) { - thread.push(reply); - seen.add(reply.id!); - } + for await (const tweet of conversationIterator) { + conversationTweets.set(tweet.id!, tweet); } + } catch (error) { + logger.warn(`Error fetching conversation: ${error}`); + return [rootTweet, initialTweet]; } - // Sort chronologically + // Convert to array and sort chronologically + const thread = Array.from(conversationTweets.values()); thread.sort((a, b) => { const timeA = a.timeParsed?.getTime() || 0; const timeB = b.timeParsed?.getTime() || 0; @@ -157,11 +204,96 @@ export class ExtendedScraper extends Scraper { }); logger.info( - `Retrieved conversation thread with ${thread.length} tweets starting from root tweet ${currentTweet.id!}`, + `Retrieved conversation thread with ${thread.length} tweets for conversation_id:${conversationId}`, ); + // Save to cache + this.conversationCache.set(conversationId!, thread); + return thread; } + + // Placeholder for efficient thread fetching + async getThreadPlaceHolder(tweetId: string, maxDepth: number = 100): Promise { + const username = config.TWITTER_USERNAME!; + const isLoggedIn = await this.isLoggedIn(); + if (!isLoggedIn) { + const reAuthenticate = await this.reAuthenticate(); + if (!reAuthenticate) { + logger.error('Failed to re-authenticate'); + return []; + } + } + + try { + const thread: Tweet[] = []; + const seen = new Set(); + const conversationTweets = new Map(); + + const initialTweet = await this.getTweet(tweetId); + if (!initialTweet) { + logger.warn(`Tweet ${tweetId} not found or deleted`); + return []; + } + + let rootTweet = initialTweet; + const conversationId = initialTweet.conversationId || initialTweet.id; + + logger.info('Initial tweet:', { + id: initialTweet.id, + conversationId: conversationId, + inReplyToStatusId: initialTweet.inReplyToStatusId, + }); + + if (initialTweet.conversationId && initialTweet.conversationId !== initialTweet.id) { + const conversationRoot = await this.getTweet(initialTweet.conversationId); + if (conversationRoot) { + rootTweet = conversationRoot; + conversationTweets.set(rootTweet.id!, rootTweet); + logger.info('Found root tweet:', { + id: rootTweet.id, + conversationId: rootTweet.conversationId, + }); + } + } + + try { + logger.info('Fetching conversation with query:', `conversation_id:${conversationId}`); + const conversationIterator = this.searchTweets( + `conversation_id:${conversationId}`, + 100, + SearchMode.Latest, + ); + + for await (const tweet of conversationIterator) { + conversationTweets.set(tweet.id!, tweet); + logger.info('Found conversation tweet:', { + id: tweet.id, + inReplyToStatusId: tweet.inReplyToStatusId, + text: tweet.text?.substring(0, 50) + '...', + }); + } + + logger.info('Total conversation tweets found:', conversationTweets.size); + } catch (error) { + logger.warn(`Error fetching conversation: ${error}`); + return [rootTweet, initialTweet]; + } + + thread.push(...conversationTweets.values()); + thread.sort((a, b) => { + const timeA = a.timeParsed?.getTime() || 0; + const timeB = b.timeParsed?.getTime() || 0; + return timeA - timeB; + }); + + logger.info(`Retrieved thread with ${thread.length} tweets`); + return thread; + } catch (error) { + logger.error(`Unexpected error in getThread: ${error}`); + return []; + } + } } export const createTwitterClientScraper = async () => { diff --git a/auto-kol/agent/src/tools/index.ts b/auto-kol/agent/src/tools/index.ts index ca5c60f..a0a8f43 100644 --- a/auto-kol/agent/src/tools/index.ts +++ b/auto-kol/agent/src/tools/index.ts @@ -10,7 +10,7 @@ import { ExtendedScraper } from '../services/twitter/api.js'; export const createTools = (scraper: ExtendedScraper) => { const mentionTool = createMentionTool(scraper); - const fetchTimelineTool = createFetchTimelineTool(); + const fetchTimelineTool = createFetchTimelineTool(scraper); const tweetSearchTool = createTweetSearchTool(scraper); diff --git a/auto-kol/agent/src/tools/tools/fetchTimelineTool.ts b/auto-kol/agent/src/tools/tools/fetchTimelineTool.ts index 6329b73..1a7a46b 100644 --- a/auto-kol/agent/src/tools/tools/fetchTimelineTool.ts +++ b/auto-kol/agent/src/tools/tools/fetchTimelineTool.ts @@ -2,17 +2,18 @@ import { DynamicStructuredTool } from '@langchain/core/tools'; import { z } from 'zod'; import { createLogger } from '../../utils/logger.js'; import { getTimeLine } from '../../utils/twitter.js'; +import { ExtendedScraper } from '../../services/twitter/api.js'; const logger = createLogger('fetch-timeline-tool'); -export const createFetchTimelineTool = () => +export const createFetchTimelineTool = (twitterScraper: ExtendedScraper) => new DynamicStructuredTool({ name: 'fetch_timeline', description: 'Fetch the timeline regularly to get new tweets', schema: z.object({}), func: async () => { try { - const tweets = await getTimeLine(); + const tweets = await getTimeLine(twitterScraper); tweets.sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime()); return { tweets: tweets, diff --git a/auto-kol/agent/src/tools/tools/mentionTool.ts b/auto-kol/agent/src/tools/tools/mentionTool.ts index 5245e42..e118883 100644 --- a/auto-kol/agent/src/tools/tools/mentionTool.ts +++ b/auto-kol/agent/src/tools/tools/mentionTool.ts @@ -3,7 +3,7 @@ import { z } from 'zod'; import { createLogger } from '../../utils/logger.js'; import { addMention, getLatestMentionId } from '../../database/index.js'; import { ExtendedScraper } from '../../services/twitter/api.js'; - +import { Tweet } from '../../types/twitter.js'; const logger = createLogger('mention-tool'); export const createMentionTool = (scraper: ExtendedScraper) => @@ -15,7 +15,6 @@ export const createMentionTool = (scraper: ExtendedScraper) => try { const sinceId = await getLatestMentionId(); const mentions = await scraper.getMyMentions(100, sinceId); - logger.info('Fetched mentions:', mentions); if (!mentions || mentions.length === 0) { logger.info('No new mentions found'); return { @@ -30,7 +29,7 @@ export const createMentionTool = (scraper: ExtendedScraper) => author_id: mention.userId!, author_username: mention.username!.toLowerCase(), created_at: mention.timeParsed!.toISOString(), - mention: true, + thread: [] as Tweet[], })); await addMention({ @@ -38,7 +37,23 @@ export const createMentionTool = (scraper: ExtendedScraper) => }); logger.info(`Fetched ${tweets.length} new mentions`); - + for (const tweet of tweets) { + logger.info(`Getting thread for tweet ${tweet.id}`); + const tweetsWithThreads: Tweet[] = []; + const thread = await scraper.getThread(tweet.id); + for await (const threadTweet of thread) { + tweetsWithThreads.push({ + id: threadTweet.id || '', + text: threadTweet.text || '', + author_id: threadTweet.userId || '', + author_username: threadTweet.username?.toLowerCase() || 'unknown', + created_at: threadTweet.timeParsed?.toISOString() || new Date().toISOString(), + }); + } + tweet.thread = tweetsWithThreads; + await new Promise(resolve => setTimeout(resolve, 1000)); + logger.info(`Found ${tweetsWithThreads.length} tweets in thread`); + } return { tweets: tweets, lastProcessedId: mentions[0].id!, diff --git a/auto-kol/agent/src/tools/tools/tweetSearchTool.ts b/auto-kol/agent/src/tools/tools/tweetSearchTool.ts index 556d854..62075e0 100644 --- a/auto-kol/agent/src/tools/tools/tweetSearchTool.ts +++ b/auto-kol/agent/src/tools/tools/tweetSearchTool.ts @@ -22,7 +22,7 @@ export const createTweetSearchTool = (scraper: ExtendedScraper) => func: async ({ lastProcessedId }) => { try { logger.info('Called search_recent_tweets'); - await updateKOLs(); + await updateKOLs(scraper); const kols = await getKOLsAccounts(); if (kols.length === 0) { diff --git a/auto-kol/agent/src/types/twitter.ts b/auto-kol/agent/src/types/twitter.ts index 2292c98..5571e4b 100644 --- a/auto-kol/agent/src/types/twitter.ts +++ b/auto-kol/agent/src/types/twitter.ts @@ -4,6 +4,7 @@ export type Tweet = { readonly author_id: string; readonly author_username: string; readonly created_at: string; + readonly thread?: Tweet[]; }; export type TwitterCredentials = { diff --git a/auto-kol/agent/src/utils/twitter.ts b/auto-kol/agent/src/utils/twitter.ts index 46a9d18..61da8e1 100644 --- a/auto-kol/agent/src/utils/twitter.ts +++ b/auto-kol/agent/src/utils/twitter.ts @@ -1,15 +1,14 @@ import { config } from '../config/index.js'; import { createLogger } from '../utils/logger.js'; -import { createTwitterClientScraper } from '../services/twitter/api.js'; +import { ExtendedScraper } from '../services/twitter/api.js'; import * as db from '../database/index.js'; import { KOL } from '../types/kol.js'; import { Tweet } from '../types/twitter.js'; const logger = createLogger('twitter-utils'); -const twitterScraper = await createTwitterClientScraper(); export const timelineTweets: Tweet[] = []; -export const updateKOLs = async () => { +export const updateKOLs = async (twitterScraper: ExtendedScraper) => { const currentKOLs = await db.getKOLAccounts(); const twitterProfile = await twitterScraper.getProfile(config.TWITTER_USERNAME!); const followings = twitterScraper.getFollowing(twitterProfile.userId!, 1000); @@ -35,7 +34,7 @@ export const getKOLsAccounts = async () => { return kolAccounts.map(kol => kol.username); }; -export const getTimeLine = async () => { +export const getTimeLine = async (twitterScraper: ExtendedScraper) => { const validTweetIds = timelineTweets.map(tweet => tweet.id).filter(id => id != null); const timeline = await twitterScraper.fetchHomeTimeline(0, validTweetIds); @@ -62,7 +61,7 @@ const clearTimeLine = () => { timelineTweets.length = 0; }; -export const getUserProfile = async (username: string) => { +export const getUserProfile = async (twitterScraper: ExtendedScraper, username: string) => { const user = await twitterScraper.getProfile(username); const result: KOL = { id: user.userId!,