chore(database-optimizer): added api for analyze slow queries and sug…

…gest the indexes

agent/database_optimizer.py

@@ -130,6 +130,11 @@ def from_frappe_output(cls, data, table) -> DBIndex:
             column=data["column"],
         )
 
+    def to_dict(self, fields: list[str] | None = None) -> dict:
+        if not fields:
+            fields = ["name", "column", "table", "unique", "cardinality", "sequence", "nullable"]
+        return {field: getattr(self, field) for field in fields}
+
 
 @dataclass
 class ColumnStat:
@@ -362,13 +367,13 @@ def can_be_optimized(self) -> bool:
         return False
 
 
-@dataclass
 class OptimizeDatabaseQueries:
-    site: Site
-    database_root_password: str
-    queries: list[str]
-    table_cache: dict[str, DBTable]
-    column_statistics_cache: dict[str, list[ColumnStat]]
+    def __init__(self, site: Site, queries: list[str], database_root_password: str):
+        self.site = site
+        self.database_root_password = database_root_password
+        self.queries = queries
+        self.table_cache: dict[str, DBTable] = {}
+        self.column_statistics_cache: dict[str, list[ColumnStat]] = {}
 
     def analyze(self) -> dict[str, list[DBIndex]] | None:
         # generate explain output for all the queries at once

agent/site.py

@@ -13,6 +13,7 @@
 
 from agent.base import AgentException, Base
 from agent.database import Database
+from agent.database_optimizer import OptimizeDatabaseQueries
 from agent.job import job, step
 from agent.utils import b2mb, compute_file_hash, get_size
 
@@ -853,7 +854,7 @@ def get_database_free_tables(self):
 
     @job("Fetch Database Table Schema")
     def fetch_database_table_schema(self, include_table_size: bool = True, include_index_info: bool = True):
-        database = Database(self.host, 3306, self.user, self.password, self.database)
+        database = self.db_instance()
         tables = {}
         table_schemas = self._fetch_database_table_schema(database, include_index_info=include_index_info)
         for table_name in table_schemas:
@@ -886,6 +887,33 @@ def run_sql_query(self, query: str, commit: bool = False, as_dict: bool = False)
             response["failed_query"] = db.last_executed_query
         return response
 
+    @job("Analyze Slow Queries")
+    def analyze_slow_queries_job(self, queries: list[dict], database_root_password: str):
+        return self.analyze_slow_queries(queries, database_root_password)
+
+    @step("Analyze Slow Queries")
+    def analyze_slow_queries(self, queries: list[dict], database_root_password: str) -> list[dict]:
+        """
+        Args:
+            queries (list[dict]): List of queries to analyze
+                {
+                    "example": "<complete query>",
+                    "normalized": "<normalized query>",
+                }
+        """
+        example_queries = [query["example"] for query in queries]
+        optimizer = OptimizeDatabaseQueries(self, example_queries, database_root_password)
+        analysis = optimizer.analyze()
+        analysis_summary = {}  # map[query -> list[index_info_dict]
+        for query, indexes in analysis.items():
+            analysis_summary[query] = [index.to_dict() for index in indexes]
+
+        result = []  # list[{example, normalized, suggested_indexes}]
+        for query in queries:
+            query["suggested_indexes"] = analysis_summary.get(query["example"], [])
+            result.append(query)
+        return result
+
     def db_instance(self, username: str | None = None, password: str | None = None) -> Database:
         if not username:
             username = self.user

agent/web.py

@@ -583,6 +583,18 @@ def run_sql(bench, site):
     )
 
 
+@application.route(
+    "/benches/<string:bench>/sites/<string:site>/database/analyze_slow_queries", methods=["GET"]
+)
+@validate_bench_and_site
+def analyze_slow_queries(bench: str, site: str):
+    queries = request.json["queries"]
+    mariadb_root_password = request.json["mariadb_root_password"]
+
+    job = Server().benches[bench].sites[site].analyze_slow_queries_job(queries, mariadb_root_password)
+    return {"job": job}
+
+
 @application.route("/benches/<string:bench>/sites/<string:site>/database/users", methods=["POST"])
 @validate_bench_and_site
 def create_database_user(bench, site):
@@ -1007,13 +1019,15 @@ def get_database_deadlocks():
     return jsonify(DatabaseServer().get_deadlocks(**data))
 
 
+# TODO can be removed
 @application.route("/database/column-stats", methods=["POST"])
 def fetch_column_statistics():
     data = request.json
-    job = DatabaseServer().fetch_column_stats(**data)
+    job = DatabaseServer().fetch_column_stats_job(**data)
     return {"job": job}
 
 
+# TODO can be removed
 @application.route("/database/explain", methods=["POST"])
 def explain():
     data = request.json