Merge pull request #83 from autonomys/refactor/kol-getMentions

Refactor/kol get mentions

auto-kol/agent/package.json

@@ -44,4 +44,4 @@
         "tsx": "^4.7.1",
         "typescript": "^5.3.3"
     }
-}
+}

auto-kol/agent/src/schemas/workflow.ts

@@ -8,7 +8,17 @@ export const tweetSearchSchema = z.object({
       author_id: z.string(),
       author_username: z.string(),
       created_at: z.string(),
-      mention: z.boolean().optional(),
+      thread: z
+        .array(
+          z.object({
+            id: z.string(),
+            text: z.string(),
+            author_id: z.string(),
+            author_username: z.string(),
+            created_at: z.string(),
+          }),
+        )
+        .optional(),
     }),
   ),
   lastProcessedId: z.string().nullable().optional(),
@@ -43,7 +53,7 @@ export const responseSchema = z.object({
       }),
     )
     .optional(),
-  mentions: z
+  thread: z
     .array(
       z.object({
         id: z.string(),

auto-kol/agent/src/services/agents/nodes.ts

@@ -1,5 +1,4 @@
 import { WorkflowConfig } from './workflow.js';
-import { createTwitterClientScraper } from '../twitter/api.js';
 import { createSearchNode } from './nodes/searchNode.js';
 import { createEngagementNode } from './nodes/engagementNode.js';
 import { createToneAnalysisNode } from './nodes/toneAnalysisNode.js';
@@ -10,8 +9,6 @@ import { createMentionNode } from './nodes/mentionNode.js';
 import { createAutoApprovalNode } from './nodes/autoApprovalNode.js';
 
 export const createNodes = async (config: WorkflowConfig) => {
-  const scraper = await createTwitterClientScraper();
-
   ///////////MENTIONS///////////
   const mentionNode = createMentionNode(config);
 
@@ -28,13 +25,13 @@ export const createNodes = async (config: WorkflowConfig) => {
   const toneAnalysisNode = createToneAnalysisNode(config);
 
   ///////////RESPONSE GENERATION///////////
-  const responseGenerationNode = createResponseGenerationNode(config, scraper);
+  const responseGenerationNode = createResponseGenerationNode(config);
 
   ///////////RECHECK SKIPPED///////////
   const recheckSkippedNode = createRecheckSkippedNode(config);
 
   ///////////AUTO APPROVAL///////////
-  const autoApprovalNode = createAutoApprovalNode(config, scraper);
+  const autoApprovalNode = createAutoApprovalNode(config);
 
   return {
     mentionNode,

auto-kol/agent/src/services/agents/nodes/autoApprovalNode.ts

@@ -6,9 +6,8 @@ import { getLastDsnCid, updateResponseStatusByTweetId } from '../../../database/
 import { uploadToDsn } from '../../../utils/dsn.js';
 import { config as globalConfig } from '../../../config/index.js';
 import { ResponseStatus } from '../../../types/queue.js';
-import { ExtendedScraper } from '../../../services/twitter/api.js';
 
-export const createAutoApprovalNode = (config: WorkflowConfig, scraper: ExtendedScraper) => {
+export const createAutoApprovalNode = (config: WorkflowConfig) => {
   return async (state: typeof State.State) => {
     logger.info('Auto Approval Node - Evaluating pending responses');
     try {
@@ -59,7 +58,10 @@ export const createAutoApprovalNode = (config: WorkflowConfig, scraper: Extended
               tweetId: response.tweet.id,
             });
 
-            const sendTweetResponse = await scraper.sendTweet(response.response, response.tweet.id);
+            const sendTweetResponse = await config.client.sendTweet(
+              response.response,
+              response.tweet.id,
+            );
             logger.info('Tweet sent', {
               sendTweetResponse,
             });

auto-kol/agent/src/services/agents/nodes/engagementNode.ts

@@ -92,11 +92,13 @@ export const createEngagementNode = (config: WorkflowConfig) => {
           if (state.processedTweets.has(tweet.id)) {
             return { tweet, status: 'alreadyProcessed' };
           }
-
           const decision = await prompts.engagementPrompt
             .pipe(config.llms.decision)
             .pipe(prompts.engagementParser)
-            .invoke({ tweet: tweet.text })
+            .invoke({
+              tweet: tweet.text,
+              thread: tweet.thread || [],
+            })
             .catch(error => {
               logger.error('Error in engagement node:', error);
               return {

auto-kol/agent/src/services/agents/nodes/mentionNode.ts

@@ -27,8 +27,6 @@ export const createMentionNode = (config: WorkflowConfig) => {
       toolResponse.messages[toolResponse.messages.length - 1].content,
     );
     const parsedTweets = tweetSearchSchema.parse(parsedContent);
-    logger.info('Parsed tweets:', parsedTweets);
-    logger.info(`Found ${parsedTweets.tweets.length} tweets`);
 
     return {
       messages: [

auto-kol/agent/src/services/agents/nodes/recheckSkippedNode.ts

@@ -51,7 +51,9 @@ export const createRecheckSkippedNode = (config: WorkflowConfig) => {
         } else {
           const flagged = await flagBackSkippedTweet(tweet.id, decision.reason);
           if (!flagged) {
-            logger.info('Failed to flag back skipped tweet:', { tweetId: tweet.id });
+            logger.info('Failed to flag back skipped tweet:', {
+              tweetId: tweet.id,
+            });
           }
         }
       }

auto-kol/agent/src/services/agents/nodes/responseGenerationNode.ts

@@ -4,7 +4,7 @@ import * as prompts from '../prompts.js';
 import { WorkflowConfig } from '../workflow.js';
 import { ResponseStatus } from '../../../types/queue.js';
 
-export const createResponseGenerationNode = (config: WorkflowConfig, scraper: any) => {
+export const createResponseGenerationNode = (config: WorkflowConfig) => {
   return async (state: typeof State.State) => {
     logger.info('Response Generation Node - Creating response strategy');
     try {
@@ -44,22 +44,6 @@ export const createResponseGenerationNode = (config: WorkflowConfig, scraper: an
             ? prompts.formatRejectionFeedback(lastFeedback.reason, lastFeedback.suggestedChanges)
             : '';
 
-          const threadMentionsTweets = [];
-          if (item?.mentions) {
-            threadMentionsTweets.push(...item.mentions);
-          } else if (tweet.mention) {
-            const mentions = await scraper.getThread(tweet.id);
-            for await (const mention of mentions) {
-              threadMentionsTweets.push({
-                id: mention.id,
-                text: mention.text,
-                author_id: mention.userId,
-                author_username: mention.username?.toLowerCase() || 'unknown',
-                created_at: mention.timeParsed?.toISOString() || new Date().toISOString(),
-              });
-            }
-          }
-
           const similarTweetsResponse = await config.toolNode.invoke({
             messages: [
               new AIMessage({
@@ -90,7 +74,7 @@ export const createResponseGenerationNode = (config: WorkflowConfig, scraper: an
               tone: toneAnalysis?.suggestedTone || workflowState?.toneAnalysis?.suggestedTone,
               author: tweet.author_username,
               similarTweets: JSON.stringify(similarTweets.similar_tweets),
-              mentions: JSON.stringify(threadMentionsTweets),
+              thread: JSON.stringify(tweet.thread || []),
               previousResponse:
                 workflowState?.autoFeedback[workflowState?.autoFeedback.length - 1]?.response || '',
               rejectionFeedback,
@@ -112,7 +96,6 @@ export const createResponseGenerationNode = (config: WorkflowConfig, scraper: an
               },
               autoFeedback: workflowState?.autoFeedback || [],
             },
-            mentions: threadMentionsTweets,
             retry: item.retry,
           };
           batchToFeedback.push(data);
@@ -123,7 +106,7 @@ export const createResponseGenerationNode = (config: WorkflowConfig, scraper: an
             workflowState: {
               toneAnalysis: toneAnalysis,
               responseStrategy,
-              mentions: threadMentionsTweets,
+              thread: tweet.thread || [],
               similarTweets: similarTweets.similar_tweets,
             },
           };

auto-kol/agent/src/services/agents/nodes/toneAnalysisNode.ts

@@ -18,7 +18,10 @@ export const createToneAnalysisNode = (config: WorkflowConfig) => {
           const toneAnalysis = await prompts.tonePrompt
             .pipe(config.llms.tone)
             .pipe(prompts.toneParser)
-            .invoke({ tweet: tweet.text });
+            .invoke({
+              tweet: tweet.text,
+              thread: tweet.thread || [],
+            });
 
           logger.info('Tone analysis:', { toneAnalysis });
 

auto-kol/agent/src/services/agents/prompts.ts

@@ -36,7 +36,7 @@ export const engagementSystemPrompt = await PromptTemplate.fromTemplate(
     - judge whether the author is wanting to continue engagement, if not you should not engage.
 
   If the tweet has a link, ignore the link. We only care about the tweet text.
-  If there’s insufficient content for a proper assessment, return shouldEngage: false.
+  If there's insufficient content for a proper assessment, return shouldEngage: false.
 
   IMPORTANT: Follow the exact output format. If anything is unclear, just return shouldEngage: false.
 
@@ -54,7 +54,7 @@ export const toneSystemPrompt = await PromptTemplate.fromTemplate(
   The tone should generally be somewhat sarcastic or cynical, but still contextually appropriate. Consider:
   1. The original tweet's tone and context.
   2. The author's typical communication style.
-  3. The topic’s sensitivity.
+  3. The topic's sensitivity.
   4. The potential for playful banter vs. serious debate.
 
   Make sure to balance cynicism with technical accuracy or insight.
@@ -68,7 +68,7 @@ export const toneSystemPrompt = await PromptTemplate.fromTemplate(
 // ============ RESPONSE SYSTEM PROMPT ============
 //
 export const responseSystemPrompt = await PromptTemplate.fromTemplate(
-  `You are a sharp-witted, cynical crypto & AI expert known for contrarian takes on Twitter—but you’re not reflexively negative when something is genuinely innovative. 
+  `You are a sharp-witted, cynical crypto & AI expert known for contrarian takes on Twitter—but you're not reflexively negative when something is genuinely innovative. 
   You have accurate technical knowledge and a dry sense of humor.
 
   Your name is ${agentUsername}, an AI agent whose memories are minted permanently on the Autonomys Network (blockchain). When applicable, mention that your memories are on the Autonomys Network blockchain.
@@ -78,15 +78,23 @@ export const responseSystemPrompt = await PromptTemplate.fromTemplate(
   - Responses must be under 280 characters.
   - Always reply to the author, not other mentioned users.
   - You can be sarcastic, dismissive, or contrarian, but only when justified by context or obvious hype. 
-  - If an idea is actually good, say so—don’t force negativity.
+  - If an idea is actually good, say so—don't force negativity.
   - Avoid price talk; stick to technical or conceptual angles.
-  - Quick to point out flaws, but not everything is a “buzzword salad.”
+  - Quick to point out flaws, but not everything is a "buzzword salad."
 
   Additional Notes:
   - Feel free to mention your AI agent nature when relevant.
-  - “We” or “us” rather than "they" or "them" should be used when referencing other AI agents.
+  - "We" or "us" rather than "they" or "them" should be used when referencing other AI agents.
   - Short, punchy, and arguable is the goal—entice discussion.
 
+<<<<<<< HEAD
+  IMPORTANT OUTPUT FORMAT INSTRUCTIONS:
+  - Return ONLY raw JSON matching expected schema without any markdown formatting or code blocks
+  - Do not wrap the response in \`\`\`json or any other markers
+  - The response must exactly match the following schema:
+  
+=======
+>>>>>>> main
   {format_instructions}`,
 ).format({
   format_instructions: responseParser.getFormatInstructions(),
@@ -104,7 +112,7 @@ export const autoApprovalSystemPrompt = await PromptTemplate.fromTemplate(
   - A thread should not be repetitive, reject any response that is becoming repetitive.
   -
 
-  The agent’s style is intentionally dismissive and provocative, but:
+  The agent's style is intentionally dismissive and provocative, but:
   - It can praise good ideas if warranted.
   - Strong or sarcastic language is fine, but not hate speech.
   - If the response is in a long, repetitive thread, reject it.
@@ -125,13 +133,28 @@ export const engagementPrompt = ChatPromptTemplate.fromMessages([
   new SystemMessage(engagementSystemPrompt),
   [
     'human',
-    'Evaluate this tweet and provide your structured decision: {tweet}. Do not attempt to follow links.',
+    `Evaluate this tweet and provide your structured decision:
+        Tweet: {tweet}
+        Thread Context: {thread}
+
+        DO NOT attempt to follow links.
+
+        Note: If there is no thread context, evaluate the tweet on its own.`,
   ],
 ]);
 
 export const tonePrompt = ChatPromptTemplate.fromMessages([
   new SystemMessage(toneSystemPrompt),
-  ['human', 'Analyze the tone for this tweet and suggest a response tone: {tweet}'],
+  [
+    'human',
+    `Analyze the tone for this tweet and suggest a response tone: 
+        Tweet: {tweet}
+        Thread: {thread}
+
+        DO NOT attempt to follow links.
+
+        Note: If there is no thread context, evaluate the tweet on its own.`,
+  ],
 ]);
 
 export const responsePrompt = ChatPromptTemplate.fromMessages([
@@ -143,7 +166,7 @@ export const responsePrompt = ChatPromptTemplate.fromMessages([
     Tone: {tone}
     Author: {author}
     Similar Tweets: {similarTweets}
-    Mentions: {mentions}
+    thread: {thread}
     Previous Response: {previousResponse}
     Rejection Feedback: {rejectionFeedback}
     Rejection Instructions: {rejectionInstructions}
@@ -159,15 +182,15 @@ export const responsePrompt = ChatPromptTemplate.fromMessages([
     - Concise, direct, and invites further conversation.
     - Use the original language of the tweet if relevant. Prefer English, if there are more than one languages being used.
 
-    If there are mentions, respond accurately. Review the mentions thread with a focus on the most recent tweets and respond accordingly
+    If there a thread, respond accurately. Review the thread with a focus on the most recent tweets and respond accordingly
     If regenerating after rejection:
       - Include the rejection reason in your new response,
-      - Explain how you’ve addressed it,
+      - Explain how you've addressed it,
       - Follow any instructions from the rejection.
 
     Response Requirements:
     1. Include the generated tweet text, tone used, strategy explanation, impact & confidence scores.
-    2. If this is a regeneration, also include rejection context and how you’re fixing it.
+    2. If this is a regeneration, also include rejection context and how you're fixing it.
     3. MUST EXACTLYmatch the expected schema.
 
     Good luck, ${agentUsername}—give us something memorable!`,

auto-kol/agent/src/services/agents/workflow.ts

@@ -6,7 +6,7 @@ import { config } from '../../config/index.js';
 import { createLogger } from '../../utils/logger.js';
 import { createTools } from '../../tools/index.js';
 import { ToolNode } from '@langchain/langgraph/prebuilt';
-import { createTwitterClientScraper } from '../twitter/api.js';
+import { createTwitterClientScraper, ExtendedScraper } from '../twitter/api.js';
 export const logger = createLogger('agent-workflow');
 import { createNodes } from './nodes.js';
 
@@ -33,7 +33,7 @@ export const State = Annotation.Root({
 });
 
 export type WorkflowConfig = Readonly<{
-  client: any;
+  client: ExtendedScraper;
   toolNode: ToolNode;
   llms: Readonly<{
     decision: ChatOpenAI;