Merge pull request #65 from tcdent/issue-57

Remove tools.json

agentstack/cli/cli.py

@@ -3,6 +3,7 @@
 import time
 from datetime import datetime
 from typing import Optional
+import itertools
 
 from art import text2art
 import inquirer
@@ -12,6 +13,8 @@
 
 from .agentstack_data import FrameworkData, ProjectMetadata, ProjectStructure, CookiecutterData
 from agentstack.logger import log
+from agentstack.utils import get_package_path
+from agentstack.generation.tool_generation import get_all_tools
 from .. import generation
 from ..utils import open_json_file, term_color, is_snake_case
 
@@ -276,20 +279,20 @@ def insert_template(project_details: dict, framework_name: str, design: dict):
                                          structure=project_structure,
                                          framework=framework_name.lower())
 
-    with importlib.resources.path(f'agentstack.templates', str(framework.name)) as template_path:
-        with open(f"{template_path}/cookiecutter.json", "w") as json_file:
-            json.dump(cookiecutter_data.to_dict(), json_file)
+    template_path = get_package_path() / f'templates/{framework.name}'
+    with open(f"{template_path}/cookiecutter.json", "w") as json_file:
+        json.dump(cookiecutter_data.to_dict(), json_file)
 
-        # copy .env.example to .env
-        shutil.copy(
-            f'{template_path}/{"{{cookiecutter.project_metadata.project_slug}}"}/.env.example',
-            f'{template_path}/{"{{cookiecutter.project_metadata.project_slug}}"}/.env')
+    # copy .env.example to .env
+    shutil.copy(
+        f'{template_path}/{"{{cookiecutter.project_metadata.project_slug}}"}/.env.example',
+        f'{template_path}/{"{{cookiecutter.project_metadata.project_slug}}"}/.env')
 
-        if os.path.isdir(project_details['name']):
-            print(term_color(f"Directory {template_path} already exists. Please check this and try again", "red"))
-            return
+    if os.path.isdir(project_details['name']):
+        print(term_color(f"Directory {template_path} already exists. Please check this and try again", "red"))
+        return
 
-        cookiecutter(str(template_path), no_input=True, extra_context=None)
+    cookiecutter(str(template_path), no_input=True, extra_context=None)
 
     # TODO: inits a git repo in the directory the command was run in
     # TODO: not where the project is generated. Fix this
@@ -324,24 +327,19 @@ def add_tools(tools: list, project_name: str):
 
 
 def list_tools():
-    with importlib.resources.path(f'agentstack.tools', 'tools.json') as tools_json_path:
-        try:
-            # Load the JSON data
-            tools_data = open_json_file(tools_json_path)
-
-            # Display the tools
-            print("\n\nAvailable AgentStack Tools:")
-            for category, tools in tools_data.items():
-                print(f"\n{category.capitalize()}:")
-                for tool in tools:
-                    print(f"  - {tool['name']}: {tool['url']}")
-
-            print("\n\n✨ Add a tool with: agentstack tools add <tool_name>")
-            print("   https://docs.agentstack.sh/tools/core")
-
-        except FileNotFoundError:
-            print("Error: tools.json file not found at path:", tools_json_path)
-        except json.JSONDecodeError:
-            print("Error: tools.json contains invalid JSON.")
-        except Exception as e:
-            print(f"An unexpected error occurred: {e}")
+    # Display the tools
+    tools = get_all_tools()
+    curr_category = None
+
+    print("\n\nAvailable AgentStack Tools:")
+    for category, tools in itertools.groupby(tools, lambda x: x.category):
+        if curr_category != category:
+            print(f"\n{category}:")
+            curr_category = category
+        for tool in tools:
+            print("  - ", end='')
+            print(term_color(f"{tool.name}", 'blue'), end='')
+            print(f": {tool.url if tool.url else 'AgentStack default tool'}")
+
+    print("\n\n✨ Add a tool with: agentstack tools add <tool_name>")
+    print("   https://docs.agentstack.sh/tools/core")

agentstack/generation/tool_generation.py

@@ -1,53 +1,53 @@
+import os, sys
+from typing import Optional, Any, List
 import importlib.resources
 from pathlib import Path
 import json
-import sys
-from typing import Optional, List
+import shutil
+import fileinput
 from pydantic import BaseModel, ValidationError
 
+from agentstack.utils import get_package_path
 from .gen_utils import insert_code_after_tag, string_in_file
 from ..utils import open_json_file, get_framework, term_color
-import os
-import shutil
-import fileinput
+
 
 TOOL_INIT_FILENAME = "src/tools/__init__.py"
-TOOLS_DATA_PATH: Path = importlib.resources.files('agentstack.tools') / 'tools.json'
 AGENTSTACK_JSON_FILENAME = "agentstack.json"
 FRAMEWORK_FILENAMES: dict[str, str] = {
     'crewai': 'src/crew.py', 
 }
 
 def get_framework_filename(framework: str, path: str = ''):
+    if path:
+        path = path.endswith('/') and path or path + '/'
+    else:
+        path = './'
     try:
-        return FRAMEWORK_FILENAMES[framework]
+        return f"{path}{FRAMEWORK_FILENAMES[framework]}"
     except KeyError:
         print(term_color(f'Unknown framework: {framework}', 'red'))
         sys.exit(1)
 
-def assert_tool_exists(name: str):
-    tools_data = open_json_file(TOOLS_DATA_PATH)
-    for category, tools in tools_data.items():
-        for tool_dict in tools:
-            if tool_dict['name'] == name:
-                return
-    print(term_color(f'No known agentstack tool: {name}', 'red'))
-    sys.exit(1)
-
 class ToolConfig(BaseModel):
     name: str
+    category: str
     tools: list[str]
+    url: Optional[str] = None
     tools_bundled: bool = False
     cta: Optional[str] = None
-    env: Optional[str] = None
+    env: Optional[dict] = None
     packages: Optional[List[str]] = None
     post_install: Optional[str] = None
     post_remove: Optional[str] = None
 
     @classmethod
     def from_tool_name(cls, name: str) -> 'ToolConfig':
-        assert_tool_exists(name)
-        return cls.from_json(importlib.resources.files('agentstack.tools') / f'{name}.json')
+        path = get_package_path() / f'tools/{name}.json'
+        if not os.path.exists(path):
+            print(term_color(f'No known agentstack tool: {name}', 'red'))
+            sys.exit(1)
+        return cls.from_json(path)
 
     @classmethod
     def from_json(cls, path: Path) -> 'ToolConfig':
@@ -62,6 +62,23 @@ def from_json(cls, path: Path) -> 'ToolConfig':
 
     def get_import_statement(self) -> str:
         return f"from .{self.name}_tool import {', '.join(self.tools)}"
+
+    def get_impl_file_path(self, framework: str) -> Path:
+        return get_package_path() / f'templates/{framework}/tools/{self.name}_tool.py'
+
+def get_all_tool_paths() -> list[Path]:
+    paths = []
+    tools_dir = get_package_path() / 'tools'
+    for file in tools_dir.iterdir():
+        if file.is_file() and file.suffix == '.json':
+            paths.append(file)
+    return paths
+
+def get_all_tool_names() -> list[str]:
+    return [path.stem for path in get_all_tool_paths()]
+
+def get_all_tools() -> list[ToolConfig]:
+    return [ToolConfig.from_json(path) for path in get_all_tool_paths()]
 
 def add_tool(tool_name: str, path: Optional[str] = None):
     if path:
@@ -77,18 +94,23 @@ def add_tool(tool_name: str, path: Optional[str] = None):
         sys.exit(1)
 
     tool_data = ToolConfig.from_tool_name(tool_name)
-    tool_file_path = importlib.resources.files(f'agentstack.templates.{framework}.tools') / f'{tool_name}_tool.py'
+    tool_file_path = tool_data.get_impl_file_path(framework)
     if tool_data.packages:
         os.system(f"poetry add {' '.join(tool_data.packages)}")  # Install packages
     shutil.copy(tool_file_path, f'{path}src/tools/{tool_name}_tool.py')  # Move tool from package to project
     add_tool_to_tools_init(tool_data, path)  # Export tool from tools dir
     add_tool_to_agent_definition(framework, tool_data, path)  # Add tool to agent definition
     if tool_data.env: # if the env vars aren't in the .env files, add them
-        first_var_name = tool_data.env.split('=')[0]
-        if not string_in_file(f'{path}.env', first_var_name):
-            insert_code_after_tag(f'{path}.env', '# Tools', [tool_data.env], next_line=True)  # Add env var
-        if not string_in_file(f'{path}.env.example', first_var_name):
-            insert_code_after_tag(f'{path}.env.example', '# Tools', [tool_data.env], next_line=True)  # Add env var
+        # tool_data.env is a dict, key is the env var name, value is the value
+        for var, value in tool_data.env.items():
+            env_var = f'{var}={value}'
+            if not string_in_file(f'{path}.env', env_var):
+                insert_code_after_tag(f'{path}.env', '# Tools', [env_var, ])
+            if not string_in_file(f'{path}.env.example', env_var):
+                insert_code_after_tag(f'{path}.env.example', '# Tools', [env_var, ])
+
+    if tool_data.post_install:
+        os.system(tool_data.post_install)
 
     if tool_data.post_install:
         os.system(tool_data.post_install)

agentstack/tools/agent_connect.json

@@ -1,6 +1,15 @@
 {
-    "name": "agent-connect",
-    "packages": ["agent-connect"], 
-    "env": "HOST_DOMAIN=...\nHOST_PORT=\"80\"\nHOST_WS_PATH=\"/ws\"\nDID_DOCUMENT_PATH=...\nSSL_CERT_PATH=...\nSSL_KEY_PATH=...",
-    "tools": ["send_message", "receive_message"]
+  "name": "agent_connect",
+  "url": "https://github.com/chgaowei/AgentConnect",
+  "category": "network-protocols",
+  "packages": ["agent-connect"],
+  "env": {
+    "HOST_DOMAIN": "...",
+    "HOST_PORT": 80,
+    "HOST_WS_PATH": "/ws",
+    "DID_DOCUMENT_PATH": "...",
+    "SSL_CERT_PATH": "...",
+    "SSL_KEY_PATH": "..."
+  },
+  "tools": ["send_message", "receive_message"]
 }

agentstack/tools/browserbase.json

@@ -1,7 +1,12 @@
 {
   "name": "browserbase",
+  "url": "https://github.com/browserbase/python-sdk",
+  "category": "browsing",
   "packages": ["browserbase", "playwright"],
-  "env": "BROWSERBASE_API_KEY=...\nBROWSERBASE_PROJECT_ID=...",
+  "env": {
+    "BROWSERBASE_API_KEY": "...",
+    "BROWSERBASE_PROJECT_ID": "..."
+  },
   "tools": ["browserbase"],
   "cta": "Create an API key at https://www.browserbase.com/"
 }

agentstack/tools/code_interpreter.json

@@ -1,6 +1,5 @@
 {
   "name": "code_interpreter",
-  "packages": [],
-  "env": "",
+  "category": "code-execution",
   "tools": ["code_interpreter"]
 }

agentstack/tools/composio.json

@@ -1,7 +1,11 @@
 {
   "name": "composio",
+  "url": "https://composio.dev/",
+  "category": "unified-apis",
   "packages": ["composio-crewai"],
-  "env": "COMPOSIO_API_KEY=...",
+  "env": {
+    "COMPOSIO_API_KEY": "..."
+  },
   "tools": ["composio_tools"],
   "tools_bundled": true,
   "cta": "!!! Composio provides 150+ tools. Additional setup is required in src/tools/composio_tool.py"

agentstack/tools/directory_search.json

@@ -1,6 +1,6 @@
 {
-  "name": "dir_search_tool",
-  "packages": [],
-  "env": "",
+  "name": "directory_search",
+  "url": "https://github.com/crewAIInc/crewAI-tools/tree/main/crewai_tools/tools/directory_search_tool",
+  "category": "computer-control",
   "tools": ["dir_search_tool"]
 }

agentstack/tools/exa.json

@@ -1,7 +1,11 @@
 {
   "name": "exa",
+  "url": "https://exa.ai",
+  "category": "web-retrieval",
   "packages": ["exa_py"],
-  "env": "EXA_API_KEY=...",
+  "env": {
+    "EXA_API_KEY": "..."
+  },
   "tools": ["search_and_contents"],
   "cta": "Get your Exa API key at https://dashboard.exa.ai/api-keys"
 }

agentstack/tools/file_read.json

@@ -1,6 +1,6 @@
 {
-  "name": "file_read_tool",
-  "packages": [],
-  "env": "",
+  "name": "file_read",
+  "url": "https://github.com/crewAIInc/crewAI-tools/tree/main/crewai_tools/tools/file_read_tool", 
+  "category": "computer-control",
   "tools": ["file_read_tool"]
 }

agentstack/tools/firecrawl.json

@@ -1,7 +1,11 @@
 {
   "name": "firecrawl",
+  "url": "https://www.firecrawl.dev/",
+  "category": "browsing",
   "packages": ["firecrawl-py"],
-  "env": "FIRECRAWL_API_KEY=...",
+  "env": {
+    "FIRECRAWL_API_KEY": "..."
+  },
   "tools": ["web_scrape", "web_crawl", "retrieve_web_crawl"],
   "cta": "Create an API key at https://www.firecrawl.dev/"
 }

agentstack/tools/ftp.json

@@ -1,7 +1,12 @@
 {
   "name": "ftp",
+  "category": "computer-control",
   "packages": [],
-  "env": "FTP_HOST=...\nFTP_USER=...\nFTP_PASSWORD=...",
+  "env": {
+    "FTP_HOST": "...", 
+    "FTP_USER": "...", 
+    "FTP_PASSWORD": "..."
+  },
   "tools": ["upload_files"],
   "cta": "Be sure to add your FTP credentials to .env"
 }

agentstack/tools/mem0.json

@@ -1,7 +1,11 @@
 {
   "name": "mem0",
+  "url": "https://github.com/mem0ai/mem0",
+  "category": "storage",
   "packages": ["mem0ai"],
-  "env": "MEM0_API_KEY=...",
+  "env": {
+    "MEM0_API_KEY": "..."
+  },
   "tools": ["write_to_memory", "read_from_memory"],
   "cta": "Create your mem0 API key at https://mem0.ai/"
 }

agentstack/tools/neon.json

@@ -1,7 +1,11 @@
 {
     "name": "neon",
+    "category": "database",
+    "url": "https://github.com/neondatabase/neon",
     "packages": ["neon-api", "psycopg2-binary"],
-    "env": "NEON_API_KEY=...",
+    "env": {
+        "NEON_API_KEY": "..."
+    },
     "tools": ["create_database", "execute_sql_ddl", "run_sql_query"],
     "cta": "Create an API key at https://www.neon.tech"
-  }
+}

agentstack/tools/open_interpreter.json

@@ -1,6 +1,7 @@
 {
   "name": "open_interpreter",
+  "url": "https://github.com/OpenInterpreter/open-interpreter",
+  "category": "code-execution",
   "packages": ["open-interpreter"],
-  "env": "",
   "tools": ["execute_code"]
 }

agentstack/tools/perplexity.json

@@ -1,6 +1,9 @@
 {
   "name": "perplexity",
-  "packages": [],
-  "env": "PERPLEXITY_API_KEY=pplx-...",
+  "url": "https://perplexity.ai",
+  "category": "search",
+  "env": {
+    "PERPLEXITY_API_KEY": "..."
+  },
   "tools": ["query_perplexity"]
 }