From 29cecd15bcd9182e08fb726d01c362eb8431f77b Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Wed, 27 Nov 2024 14:11:18 -0800 Subject: [PATCH 01/11] Move marketing URLs and categories into individual tool json files, remove tools.json --- agentstack/generation/tool_generation.py | 19 +++------ agentstack/tools/agent_connect.json | 4 +- agentstack/tools/browserbase.json | 2 + agentstack/tools/code_interpreter.json | 1 + agentstack/tools/composio.json | 2 + agentstack/tools/directory_search.json | 2 + agentstack/tools/exa.json | 2 + agentstack/tools/file_read.json | 2 + agentstack/tools/firecrawl.json | 2 + agentstack/tools/ftp.json | 1 + agentstack/tools/mem0.json | 2 + agentstack/tools/open_interpreter.json | 2 + agentstack/tools/perplexity.json | 2 + agentstack/tools/stripe.json | 2 + agentstack/tools/tools.json | 54 ------------------------ agentstack/tools/vision.json | 1 + 16 files changed, 33 insertions(+), 67 deletions(-) delete mode 100644 agentstack/tools/tools.json diff --git a/agentstack/generation/tool_generation.py b/agentstack/generation/tool_generation.py index 38d9fdf..ed92fd5 100644 --- a/agentstack/generation/tool_generation.py +++ b/agentstack/generation/tool_generation.py @@ -12,7 +12,6 @@ import fileinput TOOL_INIT_FILENAME = "src/tools/__init__.py" -TOOLS_DATA_PATH: Path = importlib.resources.files('agentstack.tools') / 'tools.json' AGENTSTACK_JSON_FILENAME = "agentstack.json" FRAMEWORK_FILENAMES: dict[str, str] = { 'crewai': 'src/crew.py', @@ -25,18 +24,11 @@ def get_framework_filename(framework: str, path: str = ''): print(term_color(f'Unknown framework: {framework}', 'red')) sys.exit(1) -def assert_tool_exists(name: str): - tools_data = open_json_file(TOOLS_DATA_PATH) - for category, tools in tools_data.items(): - for tool_dict in tools: - if tool_dict['name'] == name: - return - print(term_color(f'No known agentstack tool: {name}', 'red')) - sys.exit(1) - class ToolConfig(BaseModel): name: str + category: str tools: list[str] + url: Optional[str] = None tools_bundled: bool = False cta: Optional[str] = None env: Optional[str] = None @@ -46,8 +38,11 @@ class ToolConfig(BaseModel): @classmethod def from_tool_name(cls, name: str) -> 'ToolConfig': - assert_tool_exists(name) - return cls.from_json(importlib.resources.files('agentstack.tools') / f'{name}.json') + path = importlib.resources.files('agentstack.tools') / f'{name}.json' + if not os.path.exists(path): + print(term_color(f'No known agentstack tool: {name}', 'red')) + sys.exit(1) + return cls.from_json(path) @classmethod def from_json(cls, path: Path) -> 'ToolConfig': diff --git a/agentstack/tools/agent_connect.json b/agentstack/tools/agent_connect.json index 34922eb..abf0854 100644 --- a/agentstack/tools/agent_connect.json +++ b/agentstack/tools/agent_connect.json @@ -1,6 +1,8 @@ { "name": "agent-connect", - "packages": ["agent-connect"], + "url": "https://github.com/chgaowei/AgentConnect", + "category": "network-protocols", + "packages": ["agent-connect"], "env": "HOST_DOMAIN=...\nHOST_PORT=\"80\"\nHOST_WS_PATH=\"/ws\"\nDID_DOCUMENT_PATH=...\nSSL_CERT_PATH=...\nSSL_KEY_PATH=...", "tools": ["send_message", "receive_message"] } diff --git a/agentstack/tools/browserbase.json b/agentstack/tools/browserbase.json index 9e75455..80882d4 100644 --- a/agentstack/tools/browserbase.json +++ b/agentstack/tools/browserbase.json @@ -1,5 +1,7 @@ { "name": "browserbase", + "url": "https://github.com/browserbase/python-sdk", + "category": "browsing", "packages": ["browserbase", "playwright"], "env": "BROWSERBASE_API_KEY=...\nBROWSERBASE_PROJECT_ID=...", "tools": ["browserbase"], diff --git a/agentstack/tools/code_interpreter.json b/agentstack/tools/code_interpreter.json index a746ff1..4c37573 100644 --- a/agentstack/tools/code_interpreter.json +++ b/agentstack/tools/code_interpreter.json @@ -1,5 +1,6 @@ { "name": "code_interpreter", + "category": "code-execution", "packages": [], "env": "", "tools": ["code_interpreter"] diff --git a/agentstack/tools/composio.json b/agentstack/tools/composio.json index c89d699..6df79ac 100644 --- a/agentstack/tools/composio.json +++ b/agentstack/tools/composio.json @@ -1,5 +1,7 @@ { "name": "composio", + "url": "https://composio.dev/", + "category": "unified-apis", "packages": ["composio-crewai"], "env": "COMPOSIO_API_KEY=...", "tools": ["composio_tools"], diff --git a/agentstack/tools/directory_search.json b/agentstack/tools/directory_search.json index 59fb371..e5102bd 100644 --- a/agentstack/tools/directory_search.json +++ b/agentstack/tools/directory_search.json @@ -1,5 +1,7 @@ { "name": "dir_search_tool", + "url": "https://github.com/crewAIInc/crewAI-tools/tree/main/crewai_tools/tools/directory_search_tool", + "category": "computer-control", "packages": [], "env": "", "tools": ["dir_search_tool"] diff --git a/agentstack/tools/exa.json b/agentstack/tools/exa.json index fbcec41..bf0a5c3 100644 --- a/agentstack/tools/exa.json +++ b/agentstack/tools/exa.json @@ -1,5 +1,7 @@ { "name": "exa", + "url": "https://exa.ai", + "category": "web-retrieval", "packages": ["exa_py"], "env": "EXA_API_KEY=...", "tools": ["search_and_contents"], diff --git a/agentstack/tools/file_read.json b/agentstack/tools/file_read.json index 98620c3..1617ee8 100644 --- a/agentstack/tools/file_read.json +++ b/agentstack/tools/file_read.json @@ -1,5 +1,7 @@ { "name": "file_read_tool", + "url": "https://github.com/crewAIInc/crewAI-tools/tree/main/crewai_tools/tools/file_read_tool", + "category": "computer-control", "packages": [], "env": "", "tools": ["file_read_tool"] diff --git a/agentstack/tools/firecrawl.json b/agentstack/tools/firecrawl.json index 2fcdaae..5814f48 100644 --- a/agentstack/tools/firecrawl.json +++ b/agentstack/tools/firecrawl.json @@ -1,5 +1,7 @@ { "name": "firecrawl", + "url": "https://www.firecrawl.dev/", + "category": "browsing", "packages": ["firecrawl-py"], "env": "FIRECRAWL_API_KEY=...", "tools": ["web_scrape", "web_crawl", "retrieve_web_crawl"], diff --git a/agentstack/tools/ftp.json b/agentstack/tools/ftp.json index a828de9..56a97fc 100644 --- a/agentstack/tools/ftp.json +++ b/agentstack/tools/ftp.json @@ -1,5 +1,6 @@ { "name": "ftp", + "category": "computer-control", "packages": [], "env": "FTP_HOST=...\nFTP_USER=...\nFTP_PASSWORD=...", "tools": ["upload_files"], diff --git a/agentstack/tools/mem0.json b/agentstack/tools/mem0.json index a273bc4..d148d3f 100644 --- a/agentstack/tools/mem0.json +++ b/agentstack/tools/mem0.json @@ -1,5 +1,7 @@ { "name": "mem0", + "url": "https://github.com/mem0ai/mem0", + "category": "storage", "packages": ["mem0ai"], "env": "MEM0_API_KEY=...", "tools": ["write_to_memory", "read_from_memory"], diff --git a/agentstack/tools/open_interpreter.json b/agentstack/tools/open_interpreter.json index a0bdbc8..6cb48fb 100644 --- a/agentstack/tools/open_interpreter.json +++ b/agentstack/tools/open_interpreter.json @@ -1,5 +1,7 @@ { "name": "open_interpreter", + "url": "https://github.com/OpenInterpreter/open-interpreter", + "category": "code-execution", "packages": ["open-interpreter"], "env": "", "tools": ["execute_code"] diff --git a/agentstack/tools/perplexity.json b/agentstack/tools/perplexity.json index ea35bb3..7c1225c 100644 --- a/agentstack/tools/perplexity.json +++ b/agentstack/tools/perplexity.json @@ -1,5 +1,7 @@ { "name": "perplexity", + "url": "https://perplexity.ai", + "category": "search", "packages": [], "env": "PERPLEXITY_API_KEY=pplx-...", "tools": ["query_perplexity"] diff --git a/agentstack/tools/stripe.json b/agentstack/tools/stripe.json index 4bcf752..108616c 100644 --- a/agentstack/tools/stripe.json +++ b/agentstack/tools/stripe.json @@ -1,5 +1,7 @@ { "name": "stripe", + "url": "https://github.com/stripe/agent-toolkit", + "category": "application-specific", "package": "poetry add stripe-agent-toolkit stripe", "env": "STRIPE_SECRET_KEY=sk-...", "tools_bundled": true, diff --git a/agentstack/tools/tools.json b/agentstack/tools/tools.json deleted file mode 100644 index aebc6c1..0000000 --- a/agentstack/tools/tools.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "browsing": [{ - "name": "browserbase", - "url": "https://github.com/browserbase/python-sdk" - }, { - "name": "firecrawl", - "url": "https://www.firecrawl.dev/" - }], - "storage": [{ - "name": "mem0", - "url": "https://github.com/mem0ai/mem0" - }], - "code-execution": [{ - "name": "open_interpreter", - "url": "https://github.com/OpenInterpreter/open-interpreter" - },{ - "name": "code_interpreter", - "url": "AgentStack default tool" - }], - "computer-control": [{ - "name": "directory_search", - "url": "https://github.com/crewAIInc/crewAI-tools/tree/main/crewai_tools/tools/directory_search_tool" - },{ - "name": "file_read", - "url": "https://github.com/crewAIInc/crewAI-tools/tree/main/crewai_tools/tools/file_read_tool" - },{ - "name": "ftp", - "url": "AgentStack default tool" - }], - "network-protocols": [{ - "name": "agent_connect", - "url": "https://github.com/chgaowei/AgentConnect" - }], - "unified-apis": [{ - "name": "composio", - "url": "https://composio.dev/" - }], - "vision": [{ - "name": "vision", - "url": "AgentStack core tool" - }], - "web-retrieval": [{ - "name": "exa", - "url": "https://exa.ai" - }], - "search": [{ - "name": "perplexity", - "url": "https://perplexity.ai" - }], - "application-specific": [{ - "name": "stripe", - "url": "https://github.com/stripe/agent-toolkit" - }] -} diff --git a/agentstack/tools/vision.json b/agentstack/tools/vision.json index c622f08..8a68f13 100644 --- a/agentstack/tools/vision.json +++ b/agentstack/tools/vision.json @@ -1,5 +1,6 @@ { "name": "vision", + "category": "vision", "packages": [], "env": "", "tools": ["vision_tool"] From a64054e9c0b5d7b5a93e41d7e4cf3ce3dabd30d9 Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Wed, 27 Nov 2024 14:16:18 -0800 Subject: [PATCH 02/11] Move tool implementation file path to ToolConfig --- agentstack/generation/tool_generation.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/agentstack/generation/tool_generation.py b/agentstack/generation/tool_generation.py index ed92fd5..de12c32 100644 --- a/agentstack/generation/tool_generation.py +++ b/agentstack/generation/tool_generation.py @@ -57,6 +57,9 @@ def from_json(cls, path: Path) -> 'ToolConfig': def get_import_statement(self) -> str: return f"from .{self.name}_tool import {', '.join(self.tools)}" + + def get_impl_file_path(self, framework: str) -> Path: + return importlib.resources.files(f'agentstack.templates.{framework}.tools') / f'{self.name}_tool.py' def add_tool(tool_name: str, path: Optional[str] = None): if path: @@ -72,7 +75,7 @@ def add_tool(tool_name: str, path: Optional[str] = None): sys.exit(1) tool_data = ToolConfig.from_tool_name(tool_name) - tool_file_path = importlib.resources.files(f'agentstack.templates.{framework}.tools') / f'{tool_name}_tool.py' + tool_file_path = tool_data.get_impl_file_path(framework) if tool_data.packages: os.system(f"poetry add {' '.join(tool_data.packages)}") # Install packages shutil.copy(tool_file_path, f'{path}src/tools/{tool_name}_tool.py') # Move tool from package to project From f6ad899194d590ebab532552d5ebf7dcb8b42911 Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Wed, 27 Nov 2024 16:08:05 -0800 Subject: [PATCH 03/11] Add tests for ToolConfig including tool json file validation. python3.9 compatibility with package paths in tool_generation. Standardize naming in tool json configs. --- agentstack/generation/tool_generation.py | 19 ++++++---- agentstack/tools/agent_connect.json | 2 +- agentstack/tools/directory_search.json | 2 +- agentstack/tools/file_read.json | 2 +- pyproject.toml | 5 +++ tests/fixtures/tool_config_max.json | 12 ++++++ tests/fixtures/tool_config_min.json | 5 +++ tests/test_tool_config.py | 47 ++++++++++++++++++++++++ tox.ini | 1 + 9 files changed, 85 insertions(+), 10 deletions(-) create mode 100644 tests/fixtures/tool_config_max.json create mode 100644 tests/fixtures/tool_config_min.json create mode 100644 tests/test_tool_config.py diff --git a/agentstack/generation/tool_generation.py b/agentstack/generation/tool_generation.py index de12c32..094b23d 100644 --- a/agentstack/generation/tool_generation.py +++ b/agentstack/generation/tool_generation.py @@ -1,15 +1,15 @@ +import os, sys +from typing import Optional, List import importlib.resources from pathlib import Path import json -import sys -from typing import Optional, List +import shutil +import fileinput from pydantic import BaseModel, ValidationError from .gen_utils import insert_code_after_tag, string_in_file from ..utils import open_json_file, get_framework, term_color -import os -import shutil -import fileinput + TOOL_INIT_FILENAME = "src/tools/__init__.py" AGENTSTACK_JSON_FILENAME = "agentstack.json" @@ -17,6 +17,11 @@ 'crewai': 'src/crew.py', } +def get_package_path() -> Path: + if sys.version_info <= (3, 9): + return Path(sys.modules['agentstack'].__path__[0]) + return importlib.resources.files('agentstack') + def get_framework_filename(framework: str, path: str = ''): try: return FRAMEWORK_FILENAMES[framework] @@ -38,7 +43,7 @@ class ToolConfig(BaseModel): @classmethod def from_tool_name(cls, name: str) -> 'ToolConfig': - path = importlib.resources.files('agentstack.tools') / f'{name}.json' + path = get_package_path() / f'tools/{name}.json' if not os.path.exists(path): print(term_color(f'No known agentstack tool: {name}', 'red')) sys.exit(1) @@ -59,7 +64,7 @@ def get_import_statement(self) -> str: return f"from .{self.name}_tool import {', '.join(self.tools)}" def get_impl_file_path(self, framework: str) -> Path: - return importlib.resources.files(f'agentstack.templates.{framework}.tools') / f'{self.name}_tool.py' + return get_package_path() / f'templates/{framework}/tools/{self.name}_tool.py' def add_tool(tool_name: str, path: Optional[str] = None): if path: diff --git a/agentstack/tools/agent_connect.json b/agentstack/tools/agent_connect.json index abf0854..8209a4c 100644 --- a/agentstack/tools/agent_connect.json +++ b/agentstack/tools/agent_connect.json @@ -1,5 +1,5 @@ { - "name": "agent-connect", + "name": "agent_connect", "url": "https://github.com/chgaowei/AgentConnect", "category": "network-protocols", "packages": ["agent-connect"], diff --git a/agentstack/tools/directory_search.json b/agentstack/tools/directory_search.json index e5102bd..40f6435 100644 --- a/agentstack/tools/directory_search.json +++ b/agentstack/tools/directory_search.json @@ -1,5 +1,5 @@ { - "name": "dir_search_tool", + "name": "directory_search", "url": "https://github.com/crewAIInc/crewAI-tools/tree/main/crewai_tools/tools/directory_search_tool", "category": "computer-control", "packages": [], diff --git a/agentstack/tools/file_read.json b/agentstack/tools/file_read.json index 1617ee8..3029097 100644 --- a/agentstack/tools/file_read.json +++ b/agentstack/tools/file_read.json @@ -1,5 +1,5 @@ { - "name": "file_read_tool", + "name": "file_read", "url": "https://github.com/crewAIInc/crewAI-tools/tree/main/crewai_tools/tools/file_read_tool", "category": "computer-control", "packages": [], diff --git a/pyproject.toml b/pyproject.toml index 1a66b2c..8acab62 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,11 @@ dependencies = [ "pydantic>=2.10", ] +[project.optional-dependencies] +test = [ + "tox>=4.23.2", +] + [tool.setuptools.package-data] agentstack = ["templates/**/*"] diff --git a/tests/fixtures/tool_config_max.json b/tests/fixtures/tool_config_max.json new file mode 100644 index 0000000..b0b008e --- /dev/null +++ b/tests/fixtures/tool_config_max.json @@ -0,0 +1,12 @@ +{ + "name": "tool_name", + "category": "category", + "tools": ["tool1", "tool2"], + "url": "https://example.com", + "tools_bundled": true, + "cta": "Click me!", + "env": "test", + "packages": ["package1", "package2"], + "post_install": "install.sh", + "post_remove": "remove.sh" +} \ No newline at end of file diff --git a/tests/fixtures/tool_config_min.json b/tests/fixtures/tool_config_min.json new file mode 100644 index 0000000..a57f223 --- /dev/null +++ b/tests/fixtures/tool_config_min.json @@ -0,0 +1,5 @@ +{ + "name": "tool_name", + "category": "category", + "tools": ["tool1", "tool2"] +} \ No newline at end of file diff --git a/tests/test_tool_config.py b/tests/test_tool_config.py new file mode 100644 index 0000000..6e32e00 --- /dev/null +++ b/tests/test_tool_config.py @@ -0,0 +1,47 @@ +import os, sys +import unittest +import importlib.resources +from pathlib import Path +from agentstack.generation.tool_generation import get_package_path, ToolConfig + +BASE_PATH = Path(__file__).parent + +def all_tool_names(): + tools_dir = get_package_path() / 'tools' + for file in tools_dir.iterdir(): + if file.is_file() and file.suffix == '.json': + yield file.stem + +class ToolConfigTest(unittest.TestCase): + def test_minimal_json(self): + config = ToolConfig.from_json(BASE_PATH / "fixtures/tool_config_min.json") + assert config.name == "tool_name" + assert config.category == "category" + assert config.tools == ["tool1", "tool2"] + assert config.url is None + assert config.tools_bundled is False + assert config.cta is None + assert config.env is None + assert config.packages is None + assert config.post_install is None + assert config.post_remove is None + + def test_maximal_json(self): + config = ToolConfig.from_json(BASE_PATH / "fixtures/tool_config_max.json") + assert config.name == "tool_name" + assert config.category == "category" + assert config.tools == ["tool1", "tool2"] + assert config.url == "https://example.com" + assert config.tools_bundled is True + assert config.cta == "Click me!" + assert config.env == "test" + assert config.packages == ["package1", "package2"] + assert config.post_install == "install.sh" + assert config.post_remove == "remove.sh" + + def test_all_json_configs_from_tool_name(self): + for tool_name in all_tool_names(): + config = ToolConfig.from_tool_name(tool_name) + assert config.name == tool_name + # We can assume that pydantic validation caught any other issues + diff --git a/tox.ini b/tox.ini index a98d86e..1447621 100644 --- a/tox.ini +++ b/tox.ini @@ -11,4 +11,5 @@ deps = pytest mypy: mypy commands = + pytest -v mypy: mypy agentops \ No newline at end of file From 4d0f09a32ea9c5443a5af8aa4ee8a91369fc33fe Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Wed, 27 Nov 2024 16:29:12 -0800 Subject: [PATCH 04/11] Restore functionality of `tools list` command. --- agentstack/cli/cli.py | 37 ++++++++++-------------- agentstack/generation/tool_generation.py | 14 +++++++++ tests/test_tool_config.py | 15 +++++----- 3 files changed, 37 insertions(+), 29 deletions(-) diff --git a/agentstack/cli/cli.py b/agentstack/cli/cli.py index 902d0de..ee0ea43 100644 --- a/agentstack/cli/cli.py +++ b/agentstack/cli/cli.py @@ -3,6 +3,7 @@ import time from datetime import datetime from typing import Optional +import itertools from art import text2art import inquirer @@ -12,6 +13,7 @@ from .agentstack_data import FrameworkData, ProjectMetadata, ProjectStructure, CookiecutterData from agentstack.logger import log +from agentstack.generation.tool_generation import get_all_tools from .. import generation from ..utils import open_json_file, term_color, is_snake_case @@ -324,24 +326,17 @@ def add_tools(tools: list, project_name: str): def list_tools(): - with importlib.resources.path(f'agentstack.tools', 'tools.json') as tools_json_path: - try: - # Load the JSON data - tools_data = open_json_file(tools_json_path) - - # Display the tools - print("\n\nAvailable AgentStack Tools:") - for category, tools in tools_data.items(): - print(f"\n{category.capitalize()}:") - for tool in tools: - print(f" - {tool['name']}: {tool['url']}") - - print("\n\n✨ Add a tool with: agentstack tools add ") - print(" https://docs.agentstack.sh/tools/core") - - except FileNotFoundError: - print("Error: tools.json file not found at path:", tools_json_path) - except json.JSONDecodeError: - print("Error: tools.json contains invalid JSON.") - except Exception as e: - print(f"An unexpected error occurred: {e}") \ No newline at end of file + # Display the tools + tools = get_all_tools() + curr_category = None + + print("\n\nAvailable AgentStack Tools:") + for category, tools in itertools.groupby(tools, lambda x: x.category): + if curr_category != category: + print(f"\n{category}:") + curr_category = category + for tool in tools: + print(f" - {tool.name}: {tool.url if tool.url else 'AgentStack default tool'}") + + print("\n\n✨ Add a tool with: agentstack tools add ") + print(" https://docs.agentstack.sh/tools/core") \ No newline at end of file diff --git a/agentstack/generation/tool_generation.py b/agentstack/generation/tool_generation.py index 094b23d..46fbe38 100644 --- a/agentstack/generation/tool_generation.py +++ b/agentstack/generation/tool_generation.py @@ -66,6 +66,20 @@ def get_import_statement(self) -> str: def get_impl_file_path(self, framework: str) -> Path: return get_package_path() / f'templates/{framework}/tools/{self.name}_tool.py' +def get_all_tool_paths() -> list[Path]: + paths = [] + tools_dir = get_package_path() / 'tools' + for file in tools_dir.iterdir(): + if file.is_file() and file.suffix == '.json': + paths.append(file) + return paths + +def get_all_tool_names() -> list[str]: + return [path.stem for path in get_all_tool_paths()] + +def get_all_tools() -> list[ToolConfig]: + return [ToolConfig.from_json(path) for path in get_all_tool_paths()] + def add_tool(tool_name: str, path: Optional[str] = None): if path: path = path.endswith('/') and path or path + '/' diff --git a/tests/test_tool_config.py b/tests/test_tool_config.py index 6e32e00..c225828 100644 --- a/tests/test_tool_config.py +++ b/tests/test_tool_config.py @@ -2,16 +2,10 @@ import unittest import importlib.resources from pathlib import Path -from agentstack.generation.tool_generation import get_package_path, ToolConfig +from agentstack.generation.tool_generation import get_all_tool_paths, get_all_tool_names, ToolConfig BASE_PATH = Path(__file__).parent -def all_tool_names(): - tools_dir = get_package_path() / 'tools' - for file in tools_dir.iterdir(): - if file.is_file() and file.suffix == '.json': - yield file.stem - class ToolConfigTest(unittest.TestCase): def test_minimal_json(self): config = ToolConfig.from_json(BASE_PATH / "fixtures/tool_config_min.json") @@ -40,8 +34,13 @@ def test_maximal_json(self): assert config.post_remove == "remove.sh" def test_all_json_configs_from_tool_name(self): - for tool_name in all_tool_names(): + for tool_name in get_all_tool_names(): config = ToolConfig.from_tool_name(tool_name) assert config.name == tool_name # We can assume that pydantic validation caught any other issues + def test_all_json_configs_from_tool_path(self): + for path in get_all_tool_paths(): + config = ToolConfig.from_json(path) + assert config.name == path.stem + # We can assume that pydantic validation caught any other issues From bc6e07d04357bd317e21227a37b715557b41fa78 Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Wed, 27 Nov 2024 16:42:09 -0800 Subject: [PATCH 05/11] Add color to tool names --- agentstack/cli/cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/agentstack/cli/cli.py b/agentstack/cli/cli.py index ee0ea43..107a68f 100644 --- a/agentstack/cli/cli.py +++ b/agentstack/cli/cli.py @@ -336,7 +336,9 @@ def list_tools(): print(f"\n{category}:") curr_category = category for tool in tools: - print(f" - {tool.name}: {tool.url if tool.url else 'AgentStack default tool'}") + print(" - ", end='') + print(term_color(f"{tool.name}", 'blue'), end='') + print(f": {tool.url if tool.url else 'AgentStack default tool'}") print("\n\n✨ Add a tool with: agentstack tools add ") print(" https://docs.agentstack.sh/tools/core") \ No newline at end of file From 9225580a9d9ac7ebe994d8e035dc76533e1f9eef Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Wed, 27 Nov 2024 17:42:30 -0800 Subject: [PATCH 06/11] Refactor tool json file env format #26 --- agentstack/generation/tool_generation.py | 16 +++++++++------- agentstack/tools/agent_connect.json | 19 +++++++++++++------ agentstack/tools/browserbase.json | 5 ++++- agentstack/tools/code_interpreter.json | 2 -- agentstack/tools/composio.json | 4 +++- agentstack/tools/directory_search.json | 2 -- agentstack/tools/exa.json | 4 +++- agentstack/tools/file_read.json | 2 -- agentstack/tools/firecrawl.json | 4 +++- agentstack/tools/ftp.json | 6 +++++- agentstack/tools/mem0.json | 4 +++- agentstack/tools/open_interpreter.json | 1 - agentstack/tools/perplexity.json | 5 +++-- agentstack/tools/stripe.json | 6 ++++-- agentstack/tools/vision.json | 2 -- agentstack/tools/~README.md | 5 +++-- tests/fixtures/tool_config_max.json | 5 ++++- tests/test_tool_config.py | 2 +- 18 files changed, 58 insertions(+), 36 deletions(-) diff --git a/agentstack/generation/tool_generation.py b/agentstack/generation/tool_generation.py index 3a365fe..fa664cd 100644 --- a/agentstack/generation/tool_generation.py +++ b/agentstack/generation/tool_generation.py @@ -1,5 +1,5 @@ import os, sys -from typing import Optional, List +from typing import Optional, Any, List import importlib.resources from pathlib import Path import json @@ -36,7 +36,7 @@ class ToolConfig(BaseModel): url: Optional[str] = None tools_bundled: bool = False cta: Optional[str] = None - env: Optional[str] = None + env: Optional[dict] = None packages: Optional[List[str]] = None post_install: Optional[str] = None post_remove: Optional[str] = None @@ -101,11 +101,13 @@ def add_tool(tool_name: str, path: Optional[str] = None): add_tool_to_tools_init(tool_data, path) # Export tool from tools dir add_tool_to_agent_definition(framework, tool_data, path) # Add tool to agent definition if tool_data.env: # if the env vars aren't in the .env files, add them - first_var_name = tool_data.env.split('=')[0] - if not string_in_file(f'{path}.env', first_var_name): - insert_code_after_tag(f'{path}.env', '# Tools', [tool_data.env], next_line=True) # Add env var - if not string_in_file(f'{path}.env.example', first_var_name): - insert_code_after_tag(f'{path}.env.example', '# Tools', [tool_data.env], next_line=True) # Add env var + # tool_data.env is a dict, key is the env var name, value is the value + for var, value in tool_data.env.items(): + env_var = f'{var}={value}' + if not string_in_file(f'{path}.env', env_var): + insert_code_after_tag(f'{path}.env', '# Tools', [env_var, ]) + if not string_in_file(f'{path}.env.example', env_var): + insert_code_after_tag(f'{path}.env.example', '# Tools', [env_var, ]) if tool_data.post_install: os.system(tool_data.post_install) diff --git a/agentstack/tools/agent_connect.json b/agentstack/tools/agent_connect.json index 8209a4c..3dd6a03 100644 --- a/agentstack/tools/agent_connect.json +++ b/agentstack/tools/agent_connect.json @@ -1,8 +1,15 @@ { - "name": "agent_connect", - "url": "https://github.com/chgaowei/AgentConnect", - "category": "network-protocols", - "packages": ["agent-connect"], - "env": "HOST_DOMAIN=...\nHOST_PORT=\"80\"\nHOST_WS_PATH=\"/ws\"\nDID_DOCUMENT_PATH=...\nSSL_CERT_PATH=...\nSSL_KEY_PATH=...", - "tools": ["send_message", "receive_message"] + "name": "agent_connect", + "url": "https://github.com/chgaowei/AgentConnect", + "category": "network-protocols", + "packages": ["agent-connect"], + "env": { + "HOST_DOMAIN": "...", + "HOST_PORT": 80, + "HOST_WS_PATH": "/ws", + "DID_DOCUMENT_PATH": "...", + "SSL_CERT_PATH": "...", + "SSL_KEY_PATH": "..." + }, + "tools": ["send_message", "receive_message"] } diff --git a/agentstack/tools/browserbase.json b/agentstack/tools/browserbase.json index 80882d4..d005c27 100644 --- a/agentstack/tools/browserbase.json +++ b/agentstack/tools/browserbase.json @@ -3,7 +3,10 @@ "url": "https://github.com/browserbase/python-sdk", "category": "browsing", "packages": ["browserbase", "playwright"], - "env": "BROWSERBASE_API_KEY=...\nBROWSERBASE_PROJECT_ID=...", + "env": { + "BROWSERBASE_API_KEY": "...", + "BROWSERBASE_PROJECT_ID": "..." + }, "tools": ["browserbase"], "cta": "Create an API key at https://www.browserbase.com/" } \ No newline at end of file diff --git a/agentstack/tools/code_interpreter.json b/agentstack/tools/code_interpreter.json index 4c37573..d3de4a9 100644 --- a/agentstack/tools/code_interpreter.json +++ b/agentstack/tools/code_interpreter.json @@ -1,7 +1,5 @@ { "name": "code_interpreter", "category": "code-execution", - "packages": [], - "env": "", "tools": ["code_interpreter"] } \ No newline at end of file diff --git a/agentstack/tools/composio.json b/agentstack/tools/composio.json index 6df79ac..c2b20d0 100644 --- a/agentstack/tools/composio.json +++ b/agentstack/tools/composio.json @@ -3,7 +3,9 @@ "url": "https://composio.dev/", "category": "unified-apis", "packages": ["composio-crewai"], - "env": "COMPOSIO_API_KEY=...", + "env": { + "COMPOSIO_API_KEY": "..." + }, "tools": ["composio_tools"], "tools_bundled": true, "cta": "!!! Composio provides 150+ tools. Additional setup is required in src/tools/composio_tool.py" diff --git a/agentstack/tools/directory_search.json b/agentstack/tools/directory_search.json index 40f6435..b412d03 100644 --- a/agentstack/tools/directory_search.json +++ b/agentstack/tools/directory_search.json @@ -2,7 +2,5 @@ "name": "directory_search", "url": "https://github.com/crewAIInc/crewAI-tools/tree/main/crewai_tools/tools/directory_search_tool", "category": "computer-control", - "packages": [], - "env": "", "tools": ["dir_search_tool"] } \ No newline at end of file diff --git a/agentstack/tools/exa.json b/agentstack/tools/exa.json index bf0a5c3..2dada5e 100644 --- a/agentstack/tools/exa.json +++ b/agentstack/tools/exa.json @@ -3,7 +3,9 @@ "url": "https://exa.ai", "category": "web-retrieval", "packages": ["exa_py"], - "env": "EXA_API_KEY=...", + "env": { + "EXA_API_KEY": "..." + }, "tools": ["search_and_contents"], "cta": "Get your Exa API key at https://dashboard.exa.ai/api-keys" } \ No newline at end of file diff --git a/agentstack/tools/file_read.json b/agentstack/tools/file_read.json index 3029097..f1b072e 100644 --- a/agentstack/tools/file_read.json +++ b/agentstack/tools/file_read.json @@ -2,7 +2,5 @@ "name": "file_read", "url": "https://github.com/crewAIInc/crewAI-tools/tree/main/crewai_tools/tools/file_read_tool", "category": "computer-control", - "packages": [], - "env": "", "tools": ["file_read_tool"] } \ No newline at end of file diff --git a/agentstack/tools/firecrawl.json b/agentstack/tools/firecrawl.json index 5814f48..7937fde 100644 --- a/agentstack/tools/firecrawl.json +++ b/agentstack/tools/firecrawl.json @@ -3,7 +3,9 @@ "url": "https://www.firecrawl.dev/", "category": "browsing", "packages": ["firecrawl-py"], - "env": "FIRECRAWL_API_KEY=...", + "env": { + "FIRECRAWL_API_KEY": "..." + }, "tools": ["web_scrape", "web_crawl", "retrieve_web_crawl"], "cta": "Create an API key at https://www.firecrawl.dev/" } \ No newline at end of file diff --git a/agentstack/tools/ftp.json b/agentstack/tools/ftp.json index 56a97fc..ca11fcc 100644 --- a/agentstack/tools/ftp.json +++ b/agentstack/tools/ftp.json @@ -2,7 +2,11 @@ "name": "ftp", "category": "computer-control", "packages": [], - "env": "FTP_HOST=...\nFTP_USER=...\nFTP_PASSWORD=...", + "env": { + "FTP_HOST": "...", + "FTP_USER": "...", + "FTP_PASSWORD": "..." + }, "tools": ["upload_files"], "cta": "Be sure to add your FTP credentials to .env" } \ No newline at end of file diff --git a/agentstack/tools/mem0.json b/agentstack/tools/mem0.json index d148d3f..dfd224a 100644 --- a/agentstack/tools/mem0.json +++ b/agentstack/tools/mem0.json @@ -3,7 +3,9 @@ "url": "https://github.com/mem0ai/mem0", "category": "storage", "packages": ["mem0ai"], - "env": "MEM0_API_KEY=...", + "env": { + "MEM0_API_KEY": "..." + }, "tools": ["write_to_memory", "read_from_memory"], "cta": "Create your mem0 API key at https://mem0.ai/" } \ No newline at end of file diff --git a/agentstack/tools/open_interpreter.json b/agentstack/tools/open_interpreter.json index 6cb48fb..238d035 100644 --- a/agentstack/tools/open_interpreter.json +++ b/agentstack/tools/open_interpreter.json @@ -3,6 +3,5 @@ "url": "https://github.com/OpenInterpreter/open-interpreter", "category": "code-execution", "packages": ["open-interpreter"], - "env": "", "tools": ["execute_code"] } \ No newline at end of file diff --git a/agentstack/tools/perplexity.json b/agentstack/tools/perplexity.json index 7c1225c..ba6fe69 100644 --- a/agentstack/tools/perplexity.json +++ b/agentstack/tools/perplexity.json @@ -2,7 +2,8 @@ "name": "perplexity", "url": "https://perplexity.ai", "category": "search", - "packages": [], - "env": "PERPLEXITY_API_KEY=pplx-...", + "env": { + "PERPLEXITY_API_KEY": "..." + }, "tools": ["query_perplexity"] } \ No newline at end of file diff --git a/agentstack/tools/stripe.json b/agentstack/tools/stripe.json index 108616c..212c6b2 100644 --- a/agentstack/tools/stripe.json +++ b/agentstack/tools/stripe.json @@ -2,8 +2,10 @@ "name": "stripe", "url": "https://github.com/stripe/agent-toolkit", "category": "application-specific", - "package": "poetry add stripe-agent-toolkit stripe", - "env": "STRIPE_SECRET_KEY=sk-...", + "packages": ["stripe-agent-toolkit", "stripe"], + "env": { + "STRIPE_SECRET_KEY": "sk-..." + }, "tools_bundled": true, "tools": ["stripe_tools"], "cta": "🔑 Create your Stripe API key here: https://dashboard.stripe.com/account/apikeys" diff --git a/agentstack/tools/vision.json b/agentstack/tools/vision.json index 8a68f13..e26e4e4 100644 --- a/agentstack/tools/vision.json +++ b/agentstack/tools/vision.json @@ -1,7 +1,5 @@ { "name": "vision", "category": "vision", - "packages": [], - "env": "", "tools": ["vision_tool"] } \ No newline at end of file diff --git a/agentstack/tools/~README.md b/agentstack/tools/~README.md index 39b78db..9fedae0 100644 --- a/agentstack/tools/~README.md +++ b/agentstack/tools/~README.md @@ -17,9 +17,10 @@ of the list in the `tools` field. ### `cta` (string) [optional] String to print in the terminal when the tool is installed that provides a call to action. -### `env` (string) [optional] +### `env` (list[dict(str, Any)]) [optional] Definitions for environment variables that will be appended to the local `.env` file. -Separate multiple environment variables with a newline character. +This is a list of key-value pairs ie. `[{"ENV_VAR": "value"}, ...]`. +In cases where the user is expected to provide a value, the value is `"..."`. ### `packages` (list) [optional] A list of package names to install. These are the names of the packages that will diff --git a/tests/fixtures/tool_config_max.json b/tests/fixtures/tool_config_max.json index b0b008e..fbbbd5a 100644 --- a/tests/fixtures/tool_config_max.json +++ b/tests/fixtures/tool_config_max.json @@ -5,7 +5,10 @@ "url": "https://example.com", "tools_bundled": true, "cta": "Click me!", - "env": "test", + "env": { + "ENV_VAR1": "value1", + "ENV_VAR2": "value2" + }, "packages": ["package1", "package2"], "post_install": "install.sh", "post_remove": "remove.sh" diff --git a/tests/test_tool_config.py b/tests/test_tool_config.py index c225828..5371de4 100644 --- a/tests/test_tool_config.py +++ b/tests/test_tool_config.py @@ -28,7 +28,7 @@ def test_maximal_json(self): assert config.url == "https://example.com" assert config.tools_bundled is True assert config.cta == "Click me!" - assert config.env == "test" + assert config.env == {"ENV_VAR1": "value1", "ENV_VAR2": "value2"} assert config.packages == ["package1", "package2"] assert config.post_install == "install.sh" assert config.post_remove == "remove.sh" From 33d8a4ed7833c4d0afb1e008d96c6ae9cc90c752 Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Wed, 27 Nov 2024 18:04:27 -0800 Subject: [PATCH 07/11] bugfix: framework filename did not incorporate path. --- agentstack/generation/tool_generation.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/agentstack/generation/tool_generation.py b/agentstack/generation/tool_generation.py index fa664cd..ac56fbe 100644 --- a/agentstack/generation/tool_generation.py +++ b/agentstack/generation/tool_generation.py @@ -23,8 +23,12 @@ def get_package_path() -> Path: return importlib.resources.files('agentstack') def get_framework_filename(framework: str, path: str = ''): + if path: + path = path.endswith('/') and path or path + '/' + else: + path = './' try: - return FRAMEWORK_FILENAMES[framework] + return f"{path}{FRAMEWORK_FILENAMES[framework]}" except KeyError: print(term_color(f'Unknown framework: {framework}', 'red')) sys.exit(1) From df31b5e8f13160fceed44dad273961420c9b1401 Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Wed, 27 Nov 2024 19:51:09 -0800 Subject: [PATCH 08/11] Fix bug in `agentstack init` on python3.9 --- agentstack/cli/cli.py | 23 ++++++++++++----------- agentstack/generation/tool_generation.py | 6 +----- agentstack/utils.py | 9 +++++++++ 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/agentstack/cli/cli.py b/agentstack/cli/cli.py index 107a68f..57de949 100644 --- a/agentstack/cli/cli.py +++ b/agentstack/cli/cli.py @@ -13,6 +13,7 @@ from .agentstack_data import FrameworkData, ProjectMetadata, ProjectStructure, CookiecutterData from agentstack.logger import log +from agentstack.utils import get_package_path from agentstack.generation.tool_generation import get_all_tools from .. import generation from ..utils import open_json_file, term_color, is_snake_case @@ -278,20 +279,20 @@ def insert_template(project_details: dict, framework_name: str, design: dict): structure=project_structure, framework=framework_name.lower()) - with importlib.resources.path(f'agentstack.templates', str(framework.name)) as template_path: - with open(f"{template_path}/cookiecutter.json", "w") as json_file: - json.dump(cookiecutter_data.to_dict(), json_file) + template_path = get_package_path() / f'templates/{framework.name}' + with open(f"{template_path}/cookiecutter.json", "w") as json_file: + json.dump(cookiecutter_data.to_dict(), json_file) - # copy .env.example to .env - shutil.copy( - f'{template_path}/{"{{cookiecutter.project_metadata.project_slug}}"}/.env.example', - f'{template_path}/{"{{cookiecutter.project_metadata.project_slug}}"}/.env') + # copy .env.example to .env + shutil.copy( + f'{template_path}/{"{{cookiecutter.project_metadata.project_slug}}"}/.env.example', + f'{template_path}/{"{{cookiecutter.project_metadata.project_slug}}"}/.env') - if os.path.isdir(project_details['name']): - print(term_color(f"Directory {template_path} already exists. Please check this and try again", "red")) - return + if os.path.isdir(project_details['name']): + print(term_color(f"Directory {template_path} already exists. Please check this and try again", "red")) + return - cookiecutter(str(template_path), no_input=True, extra_context=None) + cookiecutter(str(template_path), no_input=True, extra_context=None) # TODO: inits a git repo in the directory the command was run in # TODO: not where the project is generated. Fix this diff --git a/agentstack/generation/tool_generation.py b/agentstack/generation/tool_generation.py index ac56fbe..cb5a058 100644 --- a/agentstack/generation/tool_generation.py +++ b/agentstack/generation/tool_generation.py @@ -7,6 +7,7 @@ import fileinput from pydantic import BaseModel, ValidationError +from agentstack.utils import get_package_path from .gen_utils import insert_code_after_tag, string_in_file from ..utils import open_json_file, get_framework, term_color @@ -17,11 +18,6 @@ 'crewai': 'src/crew.py', } -def get_package_path() -> Path: - if sys.version_info <= (3, 9): - return Path(sys.modules['agentstack'].__path__[0]) - return importlib.resources.files('agentstack') - def get_framework_filename(framework: str, path: str = ''): if path: path = path.endswith('/') and path or path + '/' diff --git a/agentstack/utils.py b/agentstack/utils.py index 29fe8d5..cbe3ea5 100644 --- a/agentstack/utils.py +++ b/agentstack/utils.py @@ -5,6 +5,8 @@ import json import re from importlib.metadata import version +from pathlib import Path +import importlib.resources def get_version(): @@ -23,6 +25,13 @@ def verify_agentstack_project(): sys.exit(1) +def get_package_path() -> Path: + """This is the Path where agentstack is installed.""" + if sys.version_info <= (3, 9): + return Path(sys.modules['agentstack'].__path__[0]) + return importlib.resources.files('agentstack') + + def get_framework(path: Optional[str] = None) -> str: try: file_path = 'agentstack.json' From 5dc835dd16dc727ae60336882c1e67a6a555c24f Mon Sep 17 00:00:00 2001 From: Braelyn Boynton Date: Fri, 29 Nov 2024 21:31:42 -0800 Subject: [PATCH 09/11] merge neon into new tool format --- agentstack/tools/neon.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/agentstack/tools/neon.json b/agentstack/tools/neon.json index c58fc9f..f28b2f9 100644 --- a/agentstack/tools/neon.json +++ b/agentstack/tools/neon.json @@ -1,7 +1,10 @@ { "name": "neon", + "url": "https://github.com/neondatabase/neon", "packages": ["neon-api", "psycopg2-binary"], - "env": "NEON_API_KEY=...", + "env": { + "NEON_API_KEY": "..." + }, "tools": ["create_database", "execute_sql_ddl", "run_sql_query"], "cta": "Create an API key at https://www.neon.tech" } \ No newline at end of file From ff359357175ca839e929d817084da6bc0cc0765c Mon Sep 17 00:00:00 2001 From: Braelyn Boynton Date: Fri, 29 Nov 2024 22:13:29 -0800 Subject: [PATCH 10/11] category in tool config --- agentstack/tools/neon.json | 1 + 1 file changed, 1 insertion(+) diff --git a/agentstack/tools/neon.json b/agentstack/tools/neon.json index f28b2f9..b2b5f64 100644 --- a/agentstack/tools/neon.json +++ b/agentstack/tools/neon.json @@ -1,5 +1,6 @@ { "name": "neon", + "category": "database", "url": "https://github.com/neondatabase/neon", "packages": ["neon-api", "psycopg2-binary"], "env": { From 28c1a4f700bf8a702643cfe4debbdbab43e78326 Mon Sep 17 00:00:00 2001 From: Braelyn Boynton Date: Fri, 29 Nov 2024 22:30:15 -0800 Subject: [PATCH 11/11] fixed parsing issue and better test error message --- agentstack/tools/neon.json | 2 +- agentstack/tools/tools.json | 0 tests/test_tool_config.py | 7 ++++++- 3 files changed, 7 insertions(+), 2 deletions(-) delete mode 100644 agentstack/tools/tools.json diff --git a/agentstack/tools/neon.json b/agentstack/tools/neon.json index b2b5f64..8fd13f6 100644 --- a/agentstack/tools/neon.json +++ b/agentstack/tools/neon.json @@ -8,4 +8,4 @@ }, "tools": ["create_database", "execute_sql_ddl", "run_sql_query"], "cta": "Create an API key at https://www.neon.tech" - } \ No newline at end of file +} \ No newline at end of file diff --git a/agentstack/tools/tools.json b/agentstack/tools/tools.json deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_tool_config.py b/tests/test_tool_config.py index 5371de4..90931c7 100644 --- a/tests/test_tool_config.py +++ b/tests/test_tool_config.py @@ -1,3 +1,4 @@ +import json import os, sys import unittest import importlib.resources @@ -41,6 +42,10 @@ def test_all_json_configs_from_tool_name(self): def test_all_json_configs_from_tool_path(self): for path in get_all_tool_paths(): - config = ToolConfig.from_json(path) + try: + config = ToolConfig.from_json(path) + except json.decoder.JSONDecodeError as e: + raise Exception(f"Failed to decode tool json at {path}. Does your tool config fit the required formatting? https://github.com/AgentOps-AI/AgentStack/blob/main/agentstack/tools/~README.md") + assert config.name == path.stem # We can assume that pydantic validation caught any other issues