Merge pull request #148 from MightyGaga/pre-release

Revise the documents of dataflow
microsoft · Dec 16, 2024 · d4a2c8f · d4a2c8f
2 parents 7da39f0 + d95fc3d
commit d4a2c8f
Show file tree

Hide file tree

Showing 11 changed files with 427 additions and 695 deletions.
diff --git a/dataflow/data_flow_controller.py b/dataflow/data_flow_controller.py
@@ -247,8 +247,8 @@ def execute_execution(self, request: str, plan: Dict[str, any]) -> None:
                 self.task_info["time_cost"]["execute_eval"] = execute_flow.eval_time
             else:
                 self.task_info["time_cost"]["execute_eval"] = None
-
-
+            self.app_env.close()
+            
     def instantiation_single_flow(
             self, 
             flow_class: AppAgentProcessor, 

diff --git a/dataflow/env/env_manager.py b/dataflow/env/env_manager.py
@@ -60,12 +60,13 @@ def close(self) -> None:
         """
 
         try:
-            # Attempt to close gracefully
-            if self.app_window:
+            # Gracefully close the application window
+            if self.app_window and self.app_window.process_id():
                 self.app_window.close()
-
-            self._check_and_kill_process()
             sleep(1)
+            # Forcefully close the application window  
+            if self.app_window.element_info.name.lower() != '':            
+                self._check_and_kill_process()
         except Exception as e:
             logging.warning(f"Graceful close failed: {e}. Attempting to forcefully terminate the process.")
             self._check_and_kill_process()
@@ -77,12 +78,10 @@ def _check_and_kill_process(self) -> None:
         """
 
         try:
-            # Ensure the app_window object is still valid and visible
-            if self.app_window and not self.app_window.is_visible():
-                process = psutil.Process(self.app_window.process_id)
-                if process.is_running():
-                    print(f"Killing process: {self.app_window.process_id}")
-                    process.terminate()
+            if self.app_window and self.app_window.process_id():
+                process = psutil.Process(self.app_window.process_id())
+                print(f"Killing process: {self.app_window.process_id}")
+                process.terminate()
         except Exception as e:
             logging.error(f"Error while checking window status: {e}")
             raise e

diff --git a/dataflow/execution/workflow/execute_flow.py b/dataflow/execution/workflow/execute_flow.py
@@ -316,12 +316,11 @@ def execute_action(self) -> None:
         if self.control_text == "":
             control_selected = self.application_window
         else:
-            self._control_label, control_selected = (
-                self._app_env.find_matching_controller(
-                    self.filtered_annotation_dict, self.control_text
+            self._control_label, control_selected = self._app_env.find_matching_controller(
+                self.filtered_annotation_dict, self.control_text
                 )
-            )
-            self._matched_control = control_selected.window_text()
+            if control_selected:
+                self._matched_control = control_selected.window_text()
 
         if not control_selected:
             # If the control is not found, raise an error.

diff --git a/dataflow/schema/execution_schema.json b/dataflow/schema/execution_schema.json
@@ -109,11 +109,11 @@
         "choose_template": { "type": ["number", "null"] },
         "prefill":{ "type": ["number", "null"] },
         "instantiation_evaluation": { "type": ["number", "null"] },
-        "total": { "type": ["number", "null"] },
         "execute": { "type": ["number", "null"] },
-        "execute_eval": { "type": ["number", "null"] }
+        "execute_eval": { "type": ["number", "null"] },
+        "total": { "type": ["number", "null"] }
       },
-      "required": ["choose_template", "prefill", "instantiation_evaluation", "total", "execute", "execute_eval"]
+      "required": ["choose_template", "prefill", "instantiation_evaluation", "execute", "execute_eval", "total"]
     }
   },
   "required": ["unique_id", "app", "original", "execution_result", "instantiation_result", "time_cost"]

diff --git a/documents/docs/dataflow/execution.md b/documents/docs/dataflow/execution.md
@@ -1,75 +1,71 @@
 # Execution
 
-The instantiated plans will be executed by a execute task. After execution, evalution agent will evaluation the quality of the entire execution process.
+The instantiated plans will be executed by a `execute` task. In this phase, given the task-action data, the execution process will match the real controller based on word environment and execute the plan step by step. After execution, `evalution` agent will evaluation the quality of the entire execution process.
 
-In this phase, given the task-action data, the execution process will match the real controller based on word environment and execute the plan step by step.
+<h1 align="center">
+    <img src="../../img/execution.png"/> 
+</h1>
 
 ## ExecuteFlow
 
 The `ExecuteFlow` class is designed to facilitate the execution and evaluation of tasks in a Windows application environment. It provides functionality to interact with the application's UI, execute predefined tasks, capture screenshots, and evaluate the results of the execution. The class also handles logging and error management for the tasks.
 
-
 ### Task Execution
 
-The **task execution** in the `ExecuteFlow` class follows a structured sequence to ensure accurate and traceable task performance:  
+The **task execution** in the `ExecuteFlow` class follows a structured sequence to ensure accurate and traceable task performance:
 
-1. **Initialization**:  
-   - Load configuration settings and log paths.  
-   - Find the application window matching the task.  
-   - Retrieve or create an `ExecuteAgent` for executing the task.  
+1. **Initialization**:
 
-2. **Plan Execution**:  
-   - Loop through each step in the `instantiated_plan`.  
-   - Parse the step to extract information like subtasks, control text, and the required operation.  
+   - Load configuration settings and log paths.
+   - Find the application window matching the task.
+   - Retrieve or create an `ExecuteAgent` for executing the task.
+2. **Plan Execution**:
 
-3. **Action Execution**:  
-   - Find the control in the application window that matches the specified control text.  
-   - If no matching control is found, raise an error.  
-   - Perform the specified action (e.g., click, input text) using the agent's Puppeteer framework.  
-   - Capture screenshots of the application window and selected controls for logging and debugging.  
+   - Loop through each step in the `instantiated_plan`.
+   - Parse the step to extract information like subtasks, control text, and the required operation.
+3. **Action Execution**:
 
-4. **Result Logging**:  
-   - Log details of the step execution, including control information, performed action, and results.  
+   - Find the control in the application window that matches the specified control text.
+   - If no matching control is found, raise an error.
+   - Perform the specified action (e.g., click, input text) using the agent's Puppeteer framework.
+   - Capture screenshots of the application window and selected controls for logging and debugging.
+4. **Result Logging**:
 
-5. **Finalization**:  
-   - Save the final state of the application window.  
-   - Quit the application client gracefully.  
+   - Log details of the step execution, including control information, performed action, and results.
+5. **Finalization**:
 
-Input of `ExecuteAgent`
+   - Save the final state of the application window.
+   - Quit the application client gracefully.
 
-| **Parameter**    | **Type** | **Description**                                                               |
-|-------------------|----------|-------------------------------------------------------------------------------|
-| `name`           | `str`    | The name of the agent. Used for identification and logging purposes.         |
-| `process_name`   | `str`    | The name of the application process that the agent interacts with.           |
-| `app_root_name`  | `str`    | The name of the root application window or main UI component being targeted. |
 ---
 
-### Evaluation  
+## Evaluation
+
+The **evaluation** process in the `ExecuteFlow` class is designed to assess the performance of the executed task based on predefined prompts:
 
-The **evaluation** process in the `ExecuteFlow` class is designed to assess the performance of the executed task based on predefined prompts:  
+1. **Start Evaluation**:
 
-1. **Start Evaluation**:  
-   - Evaluation begins immediately after task execution.  
-   - It uses an `ExecuteEvalAgent` initialized during class construction.  
+   - Evaluation begins immediately after task execution.
+   - It uses an `ExecuteEvalAgent` initialized during class construction.
+2. **Perform Evaluation**:
 
-2. **Perform Evaluation**:  
-   - The `ExecuteEvalAgent` evaluates the task using a combination of input prompts (e.g., main prompt and API prompt) and logs generated during task execution.  
-   - The evaluation process outputs a result summary (e.g., quality flag, comments, and task type).  
+   - The `ExecuteEvalAgent` evaluates the task using a combination of input prompts (e.g., main prompt and API prompt) and logs generated during task execution.
+   - The evaluation process outputs a result summary (e.g., quality flag, comments, and task type).
+3. **Log and Output Results**:
 
-3. **Log and Output Results**:  
-   - Display the evaluation results in the console.  
-   - Return the evaluation summary alongside the executed plan for further analysis or reporting.  
+   - Display the evaluation results in the console.
+   - Return the evaluation summary alongside the executed plan for further analysis or reporting.
 
 # Reference
 
-### ExecuteFlow
+## ExecuteFlow
 
 ::: execution.workflow.execute_flow.ExecuteFlow
 
-### ExecuteAgent
+## ExecuteAgent
 
 ::: execution.agent.execute_agent.ExecuteAgent
 
-### ExecuteEvalAgent
+## ExecuteEvalAgent
 
-::: execution.agent.execute_eval_agent.ExecuteEvalAgent
+::: execution.agent.execute_eval_agent.ExecuteEvalAgent
diff --git a/documents/docs/dataflow/instantiation.md b/documents/docs/dataflow/instantiation.md
@@ -18,36 +18,44 @@ Templates for your app must be defined and described in `dataflow/templates/app`
 
 The `ChooseTemplateFlow` uses semantic matching, where task descriptions are compared with template descriptions using embeddings and FAISS for efficient nearest neighbor search. If semantic matching fails, a random template is chosen from the available files.
 
-#### ChooseTemplateFlow
+## 2. Prefill the Task
 
-::: instantiation.workflow.choose_template_flow.ChooseTemplateFlow
+### PrefillFlow
 
-<br>
+The `PrefillFlow` class orchestrates the refinement of task plans and UI interactions by leveraging `PrefillAgent` for task planning and action generation. It automates UI control updates, captures screenshots, and manages logs for messages and responses during execution.
 
-## 2. Prefill the Task
+### PrefillAgent
 
-The `PrefillFlow` class orchestrates the refinement of task plans and UI interactions by leveraging `PrefillAgent` for task planning and action generation. It automates UI control updates, captures screenshots, and manages logs for messages and responses during execution.
+The `PrefillAgent` class facilitates task instantiation and action sequence generation by constructing tailored prompt messages using the `PrefillPrompter`. It integrates system, user, and dynamic context to generate actionable inputs for down-stream workflows.
 
-#### PrefillFlow
+## 3. Filter Task
 
-::: instantiation.workflow.prefill_flow.PrefillFlow
+### FilterFlow
 
-#### PrefillAgent
+The `FilterFlow` class is designed to process and refine task plans by leveraging a `FilterAgent`. The `FilterFlow` class acts as a bridge between the instantiation of tasks and the execution of a filtering process, aiming to refine task steps and prefill task-related files based on predefined filtering criteria.
 
-The `PrefillAgent` class facilitates task instantiation and action sequence generation by constructing tailored prompt messages using the `PrefillPrompter`. It integrates system, user, and dynamic context to generate actionable inputs for automation workflows.
+### FilterAgent
 
-::: instantiation.agent.prefill_agent.PrefillAgent
+The `FilterAgent` class is a specialized agent used to evaluate whether an instantiated task is correct. It inherits from the BasicAgent class and includes several methods and attributes to handle its functionality.
 
-<br>
+# Reference
 
-### 3. Filter Task
+## ChooseTemplateFlow
 
-The `FilterFlow` class is designed to process and refine task plans by leveraging a `FilterAgent`. 
+::: instantiation.workflow.choose_template_flow.ChooseTemplateFlow
+
+## PrefillFlow
+
+::: instantiation.workflow.prefill_flow.PrefillFlow
+
+## PrefillAgent
+
+::: instantiation.agent.prefill_agent.PrefillAgent
 
-#### FilterFlow
+## FilterFlow
 
 ::: instantiation.workflow.filter_flow.FilterFlow
 
-#### FilterAgent
+## FilterAgent
 
 ::: instantiation.agent.filter_agent.FilterAgent