Add the global and local prompts/models support

docs/docs/resource-management/model-management.md

@@ -73,3 +73,33 @@ SELECT llm_complete(
 ) AS search_results
 FROM search_data;
 ```
+
+## 4. Global and Local Models
+
+Model creation is database specific if you want it to be available irrespective of the database then make it a GLOBAL mode. Note that previously, the creation was specific to the running database, which is LOCAL by default and the keyword LOCAL is optional.
+
+### Create Models
+
+- Create a global model:
+
+```sql
+CREATE GLOBAL MODEL('model_name', 'model_type', 'provider', {'context_window': 128000, 'max_output_tokens': 8000})
+```
+
+- Create a local model (default if no type is specified):
+
+```sql
+CREATE LOCAL MODEL('model_name', 'model_type', 'provider', {'context_window': 128000, 'max_output_tokens': 8000})
+CREATE MODEL('model_name', 'model_type', 'provider', {'context_window': 128000, 'max_output_tokens': 8000})
+```
+
+### Toggle Model State
+
+- Toggle a model's state between global and local:
+
+```sql
+UPDATE MODEL 'model_name' TO GLOBAL;
+UPDATE MODEL 'model_name' TO LOCAL;
+```
+
+All the other queries remain the same for both global and local prompts.

docs/docs/resource-management/prompt-management.md

@@ -7,7 +7,7 @@ sidebar_position: 2
 
 The **Prompt Management** section provides guidance on how to manage and configure prompts for **analytics and semantic analysis tasks** within FlockMTL. Prompts guide models in generating specific outputs for tasks like content generation, summarization, and ranking. Each database is configured with its own prompt management table during the initial load.
 
-### Prompt Table Structure
+### 1. Prompt Table Structure
 
 | **Column Name** | **Description**                   |
 | --------------- | --------------------------------- |
@@ -75,3 +75,33 @@ SELECT llm_complete(
 ) AS review_summary
 FROM reviews;
 ```
+
+## 4. Global and Local Prompts
+
+Prompt creation is database specific if you want it to be available irrespective of the database then make it a GLOBAL mode. Note that previously, the creation was specific to the running database, which is LOCAL by default and the keyword LOCAL is optional.
+
+### Create Prompts
+
+*   Create a global prompt:
+
+```sql
+CREATE GLOBAL PROMPT('prompt_name', 'prompt');
+```
+
+- Create a local prompt (default if no type is specified):
+
+```sql
+CREATE LOCAL PROMPT('prompt_name', 'prompt');
+CREATE PROMPT('prompt_name', 'prompt');
+```
+
+### Toggle Prompt State
+
+- Toggle a prompt's state between global and local:
+
+```sql
+UPDATE PROMPT 'prompt_name' TO GLOBAL;
+UPDATE PROMPT 'prompt_name' TO LOCAL;
+```
+
+All the other queries remain the same for both global and local prompts.

src/core/config/config.cpp

@@ -1,10 +1,26 @@
 #include "flockmtl/core/config.hpp"
 #include "flockmtl/secret_manager/secret_manager.hpp"
+#include <filesystem>
+#include <fmt/format.h>
 
 namespace flockmtl {
 
 duckdb::DatabaseInstance* Config::db;
 
+std::string Config::get_schema_name() { return "flockmtl_config"; }
+
+std::filesystem::path Config::get_global_storage_path() {
+#ifdef _WIN32
+    const char* homeDir = getenv("USERPROFILE");
+#else
+    const char* homeDir = getenv("HOME");
+#endif
+    if (homeDir == nullptr) {
+        throw std::runtime_error("Could not find home directory");
+    }
+    return std::filesystem::path(homeDir) / ".duckdb" / "flockmtl_storage" / "flockmtl.db";
+}
+
 duckdb::Connection Config::GetConnection(duckdb::DatabaseInstance* db) {
     if (db) {
         Config::db = db;
@@ -13,11 +29,25 @@ duckdb::Connection Config::GetConnection(duckdb::DatabaseInstance* db) {
     return con;
 }
 
-std::string Config::get_schema_name() { return "flockmtl_config"; }
+duckdb::Connection Config::GetGlobalConnection() {
+    const duckdb::DuckDB db(Config::get_global_storage_path().string());
+    duckdb::Connection con(*db.instance);
+    return con;
+}
 
-void Config::ConfigSchema(duckdb::Connection& con, std::string& schema_name) {
+void Config::SetupGlobalStorageLocation() {
+    const auto flockmtl_global_path = get_global_storage_path();
+    const auto flockmtlDir = flockmtl_global_path.parent_path();
+    if (!std::filesystem::exists(flockmtlDir)) {
+        try {
+            std::filesystem::create_directories(flockmtlDir);
+        } catch (const std::filesystem::filesystem_error& e) {
+            std::cerr << "Error creating directories: " << e.what() << std::endl;
+        }
+    }
+}
 
-    // Check if schema exists using fmt
+void Config::ConfigSchema(duckdb::Connection& con, std::string& schema_name) {
     auto result = con.Query(duckdb_fmt::format(" SELECT * "
                                                "   FROM information_schema.schemata "
                                                "  WHERE schema_name = '{}'; ",
@@ -27,19 +57,35 @@ void Config::ConfigSchema(duckdb::Connection& con, std::string& schema_name) {
     }
 }
 
-void Config::Configure(duckdb::DatabaseInstance& db) {
+void Config::ConfigureGlobal() {
+    auto con = Config::GetGlobalConnection();
+    ConfigureTables(con, ConfigType::GLOBAL);
+}
+
+void Config::ConfigureLocal(duckdb::DatabaseInstance& db) {
     auto con = Config::GetConnection(&db);
-    Registry::Register(db);
-    SecretManager::Register(db);
+    ConfigureTables(con, ConfigType::LOCAL);
+}
 
+void Config::ConfigureTables(duckdb::Connection& con, const ConfigType type) {
     con.BeginTransaction();
-
     std::string schema = Config::get_schema_name();
     ConfigSchema(con, schema);
-    ConfigModelTable(con, schema);
-    ConfigPromptTable(con, schema);
-
+    ConfigModelTable(con, schema, type);
+    ConfigPromptTable(con, schema, type);
+    con.Query(
+        duckdb_fmt::format("ATTACH DATABASE '{}' AS flockmtl_storage;", Config::get_global_storage_path().string()));
     con.Commit();
 }
 
+void Config::Configure(duckdb::DatabaseInstance& db) {
+    Registry::Register(db);
+    SecretManager::Register(db);
+    if (const auto db_path = db.config.options.database_path; db_path != get_global_storage_path().string()) {
+        ConfigureLocal(db);
+        SetupGlobalStorageLocation();
+        ConfigureGlobal();
+    }
+}
+
 } // namespace flockmtl

src/core/config/model.cpp

@@ -1,12 +1,13 @@
 #include "flockmtl/core/config.hpp"
+#include <filesystem>
 
 namespace flockmtl {
 
 std::string Config::get_default_models_table_name() { return "FLOCKMTL_MODEL_DEFAULT_INTERNAL_TABLE"; }
 
 std::string Config::get_user_defined_models_table_name() { return "FLOCKMTL_MODEL_USER_DEFINED_INTERNAL_TABLE"; }
 
-void Config::setup_default_models_config(duckdb::Connection& con, std::string& schema_name) {
+void Config::SetupDefaultModelsConfig(duckdb::Connection& con, std::string& schema_name) {
     const std::string table_name = Config::get_default_models_table_name();
     // Ensure schema exists
     auto result = con.Query(duckdb_fmt::format(" SELECT table_name "
@@ -39,7 +40,7 @@ void Config::setup_default_models_config(duckdb::Connection& con, std::string& s
     }
 }
 
-void Config::setup_user_defined_models_config(duckdb::Connection& con, std::string& schema_name) {
+void Config::SetupUserDefinedModelsConfig(duckdb::Connection& con, std::string& schema_name) {
     const std::string table_name = Config::get_user_defined_models_table_name();
     // Ensure schema exists
     auto result = con.Query(duckdb_fmt::format(" SELECT table_name "
@@ -59,9 +60,11 @@ void Config::setup_user_defined_models_config(duckdb::Connection& con, std::stri
     }
 }
 
-void Config::ConfigModelTable(duckdb::Connection& con, std::string& schema_name) {
-    setup_default_models_config(con, schema_name);
-    setup_user_defined_models_config(con, schema_name);
+void Config::ConfigModelTable(duckdb::Connection& con, std::string& schema_name, const ConfigType type) {
+    if (type == ConfigType::GLOBAL) {
+        SetupDefaultModelsConfig(con, schema_name);
+    }
+    SetupUserDefinedModelsConfig(con, schema_name);
 }
 
 } // namespace flockmtl

src/core/config/prompt.cpp

@@ -1,10 +1,11 @@
 #include "flockmtl/core/config.hpp"
+#include <filesystem>
 
 namespace flockmtl {
 
 std::string Config::get_prompts_table_name() { return "FLOCKMTL_PROMPT_INTERNAL_TABLE"; }
 
-void Config::ConfigPromptTable(duckdb::Connection& con, std::string& schema_name) {
+void Config::ConfigPromptTable(duckdb::Connection& con, std::string& schema_name, const ConfigType type) {
     const std::string table_name = "FLOCKMTL_PROMPT_INTERNAL_TABLE";
 
     auto result = con.Query(duckdb_fmt::format(" SELECT table_name "
@@ -16,14 +17,16 @@ void Config::ConfigPromptTable(duckdb::Connection& con, std::string& schema_name
         con.Query(duckdb_fmt::format(" CREATE TABLE {}.{} ( "
                                      " prompt_name VARCHAR NOT NULL, "
                                      " prompt VARCHAR NOT NULL, "
-                                     " update_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, "
+                                     " updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, "
                                      " version INT DEFAULT 1, "
                                      " PRIMARY KEY (prompt_name, version) "
                                      " ); ",
                                      schema_name, table_name));
-        con.Query(duckdb_fmt::format(" INSERT INTO {}.{} (prompt_name, prompt) "
-                                     " VALUES ('hello-world', 'Tell me hello world'); ",
-                                     schema_name, table_name));
+        if (type == ConfigType::GLOBAL) {
+            con.Query(duckdb_fmt::format(" INSERT INTO {}.{} (prompt_name, prompt) "
+                                         " VALUES ('hello-world', 'Tell me hello world'); ",
+                                         schema_name, table_name));
+        }
     }
 }