From 387cf8699114f933517ffe7bb0571e1025a0ffe7 Mon Sep 17 00:00:00 2001
From: Ara Ghukasyan <ara@agnostiq.ai>
Date: Tue, 28 Nov 2023 19:49:02 -0500
Subject: [PATCH 01/33] capture terraform errors and outputs

---
 covalent/cloud_resource_manager/core.py       | 116 ++++++++-------
 covalent_dispatcher/_cli/groups/deploy.py     | 134 +++++++-----------
 .../_cli/groups/deploy_print_callbacks.py     |  99 +++++++++++++
 3 files changed, 217 insertions(+), 132 deletions(-)
 create mode 100644 covalent_dispatcher/_cli/groups/deploy_print_callbacks.py

diff --git a/covalent/cloud_resource_manager/core.py b/covalent/cloud_resource_manager/core.py
index 035efd412..f2a589ded 100644
--- a/covalent/cloud_resource_manager/core.py
+++ b/covalent/cloud_resource_manager/core.py
@@ -121,7 +121,7 @@ def get_plugin_settings(
         for key, value in executor_options.items():
             try:
                 settings_dict[key]["value"] = value
-            except:
+            except Exception:
                 logger.error(f"No such option '{key}'. Use --help for available options")
                 sys.exit()
 
@@ -164,10 +164,9 @@ def __init__(
         self._terraform_log_env_vars = {
             "TF_LOG": "ERROR",
             "TF_LOG_PATH": os.path.join(self.executor_tf_path, "terraform-error.log"),
-            "PATH": "$PATH:/usr/bin",
         }
 
-    def _print_stdout(self, process: subprocess.Popen, print_callback: Callable) -> int:
+    def _poll_process(self, process: subprocess.Popen, print_callback: Callable) -> int:
         """
         Print the stdout from the subprocess to console
 
@@ -179,12 +178,10 @@ def _print_stdout(self, process: subprocess.Popen, print_callback: Callable) ->
             Return code of the process.
 
         """
-        while (retcode := process.poll()) is None:
-            if (proc_stdout := process.stdout.readline()) and print_callback:
-                print_callback(proc_stdout.strip().decode("utf-8"))
-        return retcode
-
-        # TODO: Return the command output along with return code
+        while (returncode := process.poll()) is None:
+            if print_callback:
+                print_callback(process.stdout.readline())
+        return returncode
 
     def _parse_terraform_error_log(self) -> List[str]:
         """Parse the terraform error logs.
@@ -198,7 +195,7 @@ def _parse_terraform_error_log(self) -> List[str]:
         for _, line in enumerate(lines):
             error_index = line.strip().find("error:")
             if error_index != -1:
-                error_message = line.strip()[error_index + len("error:") :]
+                error_message = line.strip()[error_index + len("error:"):]
                 logger.error(error_message)
         return lines
 
@@ -235,16 +232,20 @@ def _get_resource_status(
         Returns:
             status: str - status of plugin
         """
-        _, stderr = proc.communicate()
+
         cmds = cmd.split(" ")
         tfstate_path = cmds[-1].split("=")[-1]
-        if stderr is None:
-            return self._terraform_error_validator(tfstate_path=tfstate_path)
-        else:
-            raise subprocess.CalledProcessError(
-                returncode=1, cmd=cmd, stderr=self._parse_terraform_error_log()
+
+        returncode = self._poll_process(proc, print_callback=None)
+        if returncode != 0:
+            print(
+                "Unable to get resource status due to the following error:\n\n",
+                proc.stderr.read(),
+                file=sys.stderr,
             )
 
+        return self._terraform_error_validator(tfstate_path=tfstate_path)
+
     def _log_error_msg(self, cmd) -> None:
         """
         Log error msg with valid command to terraform-erro.log
@@ -261,7 +262,6 @@ def _log_error_msg(self, cmd) -> None:
     def _run_in_subprocess(
         self,
         cmd: str,
-        workdir: str,
         env_vars: Optional[Dict[str, str]] = None,
         print_callback: Optional[Callable] = None,
     ) -> Union[None, str]:
@@ -270,39 +270,50 @@ def _run_in_subprocess(
 
         Args:
             cmd: Command to execute in the subprocess
-            workdir: Working directory of the subprocess
             env_vars: Dictionary of environment variables to set in the processes execution environment
 
         Returns:
             Union[None, str]
                 - For 'covalent deploy status'
-                    returns status of the deplyment
+                    returns status of the deployment
                 - Others
                     return None
         """
-        if git := shutil.which("git"):
-            proc = subprocess.Popen(
-                args=cmd,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.STDOUT,
-                cwd=workdir,
-                shell=True,
-                env=env_vars,
-            )
-            TERRAFORM_STATE = "state list -state"
-            if TERRAFORM_STATE in cmd:
-                return self._get_resource_status(proc=proc, cmd=cmd)
-            retcode = self._print_stdout(proc, print_callback)
-
-            if retcode != 0:
-                self._log_error_msg(cmd=cmd)
-                raise subprocess.CalledProcessError(
-                    returncode=retcode, cmd=cmd, stderr=self._parse_terraform_error_log()
-                )
-        else:
+        if not shutil.which("git"):
             self._log_error_msg(cmd=cmd)
             logger.error("Git not found on the system.")
-            sys.exit()
+            sys.exit(1)
+
+        env_vars = env_vars or {}
+        env_vars.update({"PATH": os.environ["PATH"]})
+
+        proc = subprocess.Popen(
+            args=cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            cwd=self.executor_tf_path,
+            universal_newlines=True,
+            shell=True,
+            env=env_vars,
+        )
+
+        if "state list -state" in cmd:
+            return self._get_resource_status(proc=proc, cmd=cmd)
+
+        returncode = self._poll_process(proc, print_callback)
+
+        if returncode != 0:
+            self._log_error_msg(cmd=cmd)
+
+            _, stderr = proc.communicate()
+            raise subprocess.CalledProcessError(
+                returncode=returncode,
+                cmd=cmd,
+                stderr=self._parse_terraform_error_log(),
+                output=stderr,
+            )
+
+        return None
 
     def _update_config(self, tf_executor_config_file: str) -> None:
         """
@@ -348,7 +359,11 @@ def _get_tf_path(self) -> str:
         """
         if terraform := shutil.which("terraform"):
             result = subprocess.run(
-                ["terraform --version"], shell=True, capture_output=True, text=True
+                ["terraform --version"],
+                shell=True,
+                capture_output=True,
+                text=True,
+                check=True,
             )
             version = result.stdout.split("v", 1)[1][:3]
             if float(version) < 1.4:
@@ -357,9 +372,9 @@ def _get_tf_path(self) -> str:
                 )
                 sys.exit()
             return terraform
-        else:
-            logger.error("Terraform not found on system")
-            exit()
+
+        logger.error("Terraform not found on system")
+        exit()
 
     def _get_tf_statefile_path(self) -> str:
         """
@@ -401,14 +416,16 @@ def up(self, print_callback: Callable, dry_run: bool = True) -> None:
 
         # Run `terraform init`
         self._run_in_subprocess(
-            cmd=tf_init, workdir=self.executor_tf_path, env_vars=self._terraform_log_env_vars
+            cmd=tf_init,
+            env_vars=self._terraform_log_env_vars,
+            print_callback=print_callback,
         )
 
         # Setup terraform infra variables as passed by the user
         tf_vars_env_dict = os.environ.copy()
 
         if self.executor_options:
-            with open(tfvars_file, "w") as f:
+            with open(tfvars_file, "w", encoding="utf-8") as f:
                 for key, value in self.executor_options.items():
                     tf_vars_env_dict[f"TF_VAR_{key}"] = value
 
@@ -418,7 +435,6 @@ def up(self, print_callback: Callable, dry_run: bool = True) -> None:
         # Run `terraform plan`
         self._run_in_subprocess(
             cmd=tf_plan,
-            workdir=self.executor_tf_path,
             env_vars=self._terraform_log_env_vars,
             print_callback=print_callback,
         )
@@ -426,9 +442,8 @@ def up(self, print_callback: Callable, dry_run: bool = True) -> None:
         # Create infrastructure as per the plan
         # Run `terraform apply`
         if not dry_run:
-            cmd_output = self._run_in_subprocess(
+            self._run_in_subprocess(
                 cmd=tf_apply,
-                workdir=self.executor_tf_path,
                 env_vars=tf_vars_env_dict.update(self._terraform_log_env_vars),
                 print_callback=print_callback,
             )
@@ -472,7 +487,6 @@ def down(self, print_callback: Callable) -> None:
         # Run `terraform destroy`
         self._run_in_subprocess(
             cmd=tf_destroy,
-            workdir=self.executor_tf_path,
             print_callback=print_callback,
             env_vars=self._terraform_log_env_vars,
         )
@@ -511,5 +525,5 @@ def status(self) -> None:
 
         # Run `terraform state list`
         return self._run_in_subprocess(
-            cmd=tf_state, workdir=self.executor_tf_path, env_vars=self._terraform_log_env_vars
+            cmd=tf_state, env_vars=self._terraform_log_env_vars
         )
diff --git a/covalent_dispatcher/_cli/groups/deploy.py b/covalent_dispatcher/_cli/groups/deploy.py
index d6fb85ff1..92582d1c8 100644
--- a/covalent_dispatcher/_cli/groups/deploy.py
+++ b/covalent_dispatcher/_cli/groups/deploy.py
@@ -17,10 +17,10 @@
 
 """Covalent deploy CLI group."""
 
-
 import subprocess
+from functools import partial
 from pathlib import Path
-from typing import Dict, Tuple
+from typing import Callable, Dict, Tuple
 
 import boto3
 import click
@@ -30,9 +30,9 @@
 from covalent.cloud_resource_manager.core import CloudResourceManager
 from covalent.executor import _executor_manager
 
-RESOURCE_ALREADY_EXISTS = "Resources already deployed"
-RESOURCE_ALREADY_DESTROYED = "Resources already destroyed"
-COMPLETED = "Completed"
+from .deploy_print_callbacks import ScrollBufferCallback
+
+_TEMPLATE = "[bold green]{message} [default]\n {text}"
 
 
 def get_crm_object(executor_name: str, options: Dict = None) -> CloudResourceManager:
@@ -49,30 +49,6 @@ def get_crm_object(executor_name: str, options: Dict = None) -> CloudResourceMan
     return CloudResourceManager(executor_name, executor_module_path, options)
 
 
-def get_print_callback(
-    console: Console, console_status: Console.status, prepend_msg: str, verbose: bool
-):
-    """Get print callback method.
-
-    Args:
-        console: Rich console object.
-        console_status: Console status object.
-        prepend_msg: Message to prepend to the output.
-        verbose: Whether to print the output inline or not.
-
-    Returns:
-        Callback method.
-
-    """
-    if verbose:
-        return console.print
-
-    def inline_print_callback(msg):
-        console_status.update(f"{prepend_msg} {msg}")
-
-    return inline_print_callback
-
-
 def get_settings_table(crm: CloudResourceManager) -> Table:
     """Get resource provisioning settings table.
 
@@ -116,6 +92,47 @@ def get_up_help_table(crm: CloudResourceManager) -> Table:
     return table
 
 
+def _run_command_and_show_output(
+    _command: Callable[[Callable], None],
+    _status_message: str,
+    *,
+    verbose: bool
+) -> None:
+    """Run the command and show the output in the console.
+
+    This function handles execution and outputs from the `up` and `down` commands.
+
+    Args:
+        _command: command to run, e.g. `partial(crm.up, dry_run=dry_run)`
+        _status_message: message to show in the console status bar, e.g. "Provisioning resources..."
+        verbose: whether to show the full Terraform output or not.
+    """
+    console = Console(record=True)
+    msg_template = _TEMPLATE.format(message=_status_message, text="{text}")
+
+    with console.status(msg_template.format(text="")) as console_status:
+
+        print_callback = ScrollBufferCallback(
+            console=console,
+            console_status=console_status,
+            msg_template=msg_template,
+            verbose=verbose,
+        )
+
+        try:
+            _command(print_callback=print_callback)
+
+        except subprocess.CalledProcessError as e:
+            console_status.stop()
+            click.echo(e.stdout)  # display error
+            return
+
+        if not verbose:
+            console_status.stop()
+            if (complete_msg := print_callback.complete_msg) is not None:
+                console.print('\n', complete_msg, style="bold green")
+
+
 @click.group(invoke_without_command=True)
 def deploy():
     """
@@ -130,7 +147,6 @@ def deploy():
     4. Show status of all resources via `covalent deploy status`.
 
     """
-    pass
 
 
 @deploy.command(context_settings={"ignore_unknown_options": True})
@@ -172,32 +188,8 @@ def up(executor_name: str, vars: Dict, help: bool, dry_run: bool, verbose: bool)
         click.echo(Console().print(get_up_help_table(crm)))
         return
 
-    console = Console(record=True)
-    prepend_msg = "[bold green] Provisioning resources..."
-
-    with console.status(prepend_msg) as status:
-        try:
-            crm.up(
-                dry_run=dry_run,
-                print_callback=get_print_callback(
-                    console=console,
-                    console_status=status,
-                    prepend_msg=prepend_msg,
-                    verbose=verbose,
-                ),
-            )
-        except subprocess.CalledProcessError as e:
-            click.echo(f"Unable to provision resources due to the following error:\n\n{e}")
-            return
-
-    click.echo(Console().print(get_settings_table(crm)))
-    exists_msg_with_verbose = "Apply complete! Resources: 0 added, 0 changed, 0 destroyed"
-    exists_msg_without_verbose = "found no differences, so no changes are needed"
-    export_data = console.export_text()
-    if exists_msg_with_verbose in export_data or exists_msg_without_verbose in export_data:
-        click.echo(RESOURCE_ALREADY_EXISTS)
-    else:
-        click.echo(COMPLETED)
+    _command = partial(crm.up, dry_run=dry_run)
+    _run_command_and_show_output(_command, "Provisioning resources...", verbose=verbose)
 
 
 @deploy.command()
@@ -223,28 +215,8 @@ def down(executor_name: str, verbose: bool) -> None:
 
     """
     crm = get_crm_object(executor_name)
-
-    console = Console(record=True)
-    prepend_msg = "[bold green] Destroying resources..."
-    with console.status(prepend_msg) as status:
-        try:
-            crm.down(
-                print_callback=get_print_callback(
-                    console=console,
-                    console_status=status,
-                    prepend_msg=prepend_msg,
-                    verbose=verbose,
-                )
-            )
-        except subprocess.CalledProcessError as e:
-            click.echo(f"Unable to destroy resources due to the following error:\n\n{e}")
-            return
-    destroyed_msg = "Destroy complete! Resources: 0 destroyed."
-    export_data = console.export_text()
-    if destroyed_msg in export_data:
-        click.echo(RESOURCE_ALREADY_DESTROYED)
-    else:
-        click.echo(COMPLETED)
+    _command = partial(crm.down)
+    _run_command_and_show_output(_command, "Destroying resources...", verbose=verbose)
 
 
 # TODO - Color code status.
@@ -275,7 +247,7 @@ def status(executor_names: Tuple[str]) -> None:
     if not executor_names:
         executor_names = [
             name
-            for name in _executor_manager.executor_plugins_map.keys()
+            for name in _executor_manager.executor_plugins_map
             if name not in ["dask", "local", "remote_executor"]
         ]
         click.echo(f"Executors: {', '.join(executor_names)}")
@@ -289,8 +261,8 @@ def status(executor_names: Tuple[str]) -> None:
     for executor_name in executor_names:
         try:
             crm = get_crm_object(executor_name)
-            status = crm.status()
-            table.add_row(executor_name, status, description[status])
+            crm_status = crm.status()
+            table.add_row(executor_name, crm_status, description[crm_status])
         except KeyError:
             invalid_executor_names.append(executor_name)
 
diff --git a/covalent_dispatcher/_cli/groups/deploy_print_callbacks.py b/covalent_dispatcher/_cli/groups/deploy_print_callbacks.py
new file mode 100644
index 000000000..0ac453639
--- /dev/null
+++ b/covalent_dispatcher/_cli/groups/deploy_print_callbacks.py
@@ -0,0 +1,99 @@
+# Copyright 2023 Agnostiq Inc.
+#
+# This file is part of Covalent.
+#
+# Licensed under the Apache License 2.0 (the "License"). A copy of the
+# License may be obtained with this software package or at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Use of this file is prohibited except in compliance with the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""Print callbacks for deploy up and deploy down commands."""
+
+import re
+from typing import Union
+
+from rich.console import Console
+from rich.status import Status
+
+
+class ScrollBufferCallback:
+    """Callable print callback that refreshes a buffer of length `max_lines`"""
+
+    _complete_msg_pattern = re.compile(
+        r"^(Apply complete\! Resources: \d+ added, \d+ changed, \d+ destroyed\."
+        "|"
+        r"Destroy complete\! Resources: \d+ destroyed\.)$"
+    )
+
+    def __init__(
+        self,
+        console: Console,
+        console_status: Status,
+        msg_template: str,
+        verbose: bool,
+        max_lines: int = 12,
+    ):
+        """Create a new scroll buffer callback.
+
+        Args:
+            console: Rich console object.
+            console_status: Rich console status object.
+            msg_template: Message template pre-formatted with provision or destroy message.
+            verbose: Whether to print the output inline or not.
+            max_lines: Number of lines in the buffer. Defaults to 5.
+        """
+        self.console = console
+        self.console_status = console_status
+        self.msg_template = msg_template
+        self.verbose = verbose
+        self.max_lines = max_lines
+        self.buffer = []
+
+        self._complete_msg = None
+
+    @property
+    def complete_msg(self) -> Union[str, None]:
+        """Return a complete message matching:
+
+            'Apply complete! Resources: 19 added, 0 changed, 0 destroyed.'
+        or
+            'Destroy complete! Resources: 19 destroyed.'
+
+        Returns:
+            The complete message, if it exists, else None.
+        """
+        return self._complete_msg
+
+    def __call__(self, msg: str):
+        if self.verbose:
+            self._verbose_print(msg)
+        else:
+            self._inline_print(msg)
+
+    def _inline_print(self, msg: str) -> None:
+        """Print into a scrolling buffer of size `self.max_lines`."""
+        if len(self.buffer) > self.max_lines:
+            self.buffer.pop(0)
+        self.buffer.append(msg)
+
+        if self._complete_msg_pattern.match(msg):
+            self._complete_msg = msg
+
+        text = self.render_buffer()
+        self.console_status.update(self.msg_template.format(text=text))
+
+    def _verbose_print(self, msg: str) -> None:
+        """Print normally, line by line."""
+        print(msg.strip() if msg != "\n" else msg)
+
+    def render_buffer(self, sep: str = ' ') -> str:
+        """Render the current buffer as a string."""
+        return sep.join(self.buffer)

From d4e7b3db6ec7e53d09b3d5fc9c7e06e36bffcdc7 Mon Sep 17 00:00:00 2001
From: Ara Ghukasyan <ara@agnostiq.ai>
Date: Tue, 28 Nov 2023 20:17:32 -0500
Subject: [PATCH 02/33] correct leading whitespace

---
 covalent_dispatcher/_cli/groups/deploy_print_callbacks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/covalent_dispatcher/_cli/groups/deploy_print_callbacks.py b/covalent_dispatcher/_cli/groups/deploy_print_callbacks.py
index 0ac453639..ee2e1453b 100644
--- a/covalent_dispatcher/_cli/groups/deploy_print_callbacks.py
+++ b/covalent_dispatcher/_cli/groups/deploy_print_callbacks.py
@@ -92,7 +92,7 @@ def _inline_print(self, msg: str) -> None:
 
     def _verbose_print(self, msg: str) -> None:
         """Print normally, line by line."""
-        print(msg.strip() if msg != "\n" else msg)
+        print(msg.rstrip() if msg != "\n" else msg)
 
     def render_buffer(self, sep: str = ' ') -> str:
         """Render the current buffer as a string."""

From fe620d61fdaabeb652c92f5bdbfaa7e2334592ff Mon Sep 17 00:00:00 2001
From: Ara Ghukasyan <ara@agnostiq.ai>
Date: Tue, 28 Nov 2023 20:18:46 -0500
Subject: [PATCH 03/33] update changelog

---
 CHANGELOG.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ffe62a49f..255dca955 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [UNRELEASED]
 
+### Changed
+
+- Terraform output to use scrolling buffer.
+- Terraform output handling to show errors.
+
 ### Added
 
 - check for `/bin/bash` AND `/bin/sh` (in that order) to execute bash leptons

From 2fc658746e41a22019fe443ef98462353f1fb75f Mon Sep 17 00:00:00 2001
From: Ara Ghukasyan <ara@agnostiq.ai>
Date: Tue, 28 Nov 2023 20:26:40 -0500
Subject: [PATCH 04/33] ignore "No state file .." message

---
 covalent/cloud_resource_manager/core.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/covalent/cloud_resource_manager/core.py b/covalent/cloud_resource_manager/core.py
index f2a589ded..2b2ec8089 100644
--- a/covalent/cloud_resource_manager/core.py
+++ b/covalent/cloud_resource_manager/core.py
@@ -237,10 +237,11 @@ def _get_resource_status(
         tfstate_path = cmds[-1].split("=")[-1]
 
         returncode = self._poll_process(proc, print_callback=None)
-        if returncode != 0:
+        stderr = proc.stderr.read()
+        if returncode != 0 and "No state file was found!" not in stderr:
             print(
                 "Unable to get resource status due to the following error:\n\n",
-                proc.stderr.read(),
+                stderr,
                 file=sys.stderr,
             )
 

From 8e9787144a59dd9b31a9d7c041246e8293619bd4 Mon Sep 17 00:00:00 2001
From: Ara Ghukasyan <ara@agnostiq.ai>
Date: Tue, 28 Nov 2023 20:29:00 -0500
Subject: [PATCH 05/33] revise exit return codes

---
 covalent/cloud_resource_manager/core.py   | 4 ++--
 covalent_dispatcher/_cli/groups/deploy.py | 7 ++++---
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/covalent/cloud_resource_manager/core.py b/covalent/cloud_resource_manager/core.py
index 2b2ec8089..4a22f966f 100644
--- a/covalent/cloud_resource_manager/core.py
+++ b/covalent/cloud_resource_manager/core.py
@@ -371,11 +371,11 @@ def _get_tf_path(self) -> str:
                 logger.error(
                     "Old version of terraform found. Please update it to version greater than 1.3"
                 )
-                sys.exit()
+                sys.exit(1)
             return terraform
 
         logger.error("Terraform not found on system")
-        exit()
+        sys.exit(1)
 
     def _get_tf_statefile_path(self) -> str:
         """
diff --git a/covalent_dispatcher/_cli/groups/deploy.py b/covalent_dispatcher/_cli/groups/deploy.py
index 92582d1c8..f239ee81b 100644
--- a/covalent_dispatcher/_cli/groups/deploy.py
+++ b/covalent_dispatcher/_cli/groups/deploy.py
@@ -18,6 +18,7 @@
 """Covalent deploy CLI group."""
 
 import subprocess
+import sys
 from functools import partial
 from pathlib import Path
 from typing import Callable, Dict, Tuple
@@ -125,7 +126,7 @@ def _run_command_and_show_output(
         except subprocess.CalledProcessError as e:
             console_status.stop()
             click.echo(e.stdout)  # display error
-            return
+            sys.exit(1)
 
         if not verbose:
             console_status.stop()
@@ -182,11 +183,11 @@ def up(executor_name: str, vars: Dict, help: bool, dry_run: bool, verbose: bool)
     cmd_options = {key[2:]: value for key, value in (var.split("=") for var in vars)}
     if msg := validate_args(cmd_options):
         click.echo(msg)
-        return
+        sys.exit(1)
     crm = get_crm_object(executor_name, cmd_options)
     if help:
         click.echo(Console().print(get_up_help_table(crm)))
-        return
+        sys.exit(1)
 
     _command = partial(crm.up, dry_run=dry_run)
     _run_command_and_show_output(_command, "Provisioning resources...", verbose=verbose)

From 872fd5e34fb75f3266e8dce2ec8c290ec427e3bf Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Wed, 29 Nov 2023 01:57:06 +0000
Subject: [PATCH 06/33] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 covalent/cloud_resource_manager/core.py                   | 6 ++----
 covalent_dispatcher/_cli/groups/deploy.py                 | 8 ++------
 covalent_dispatcher/_cli/groups/deploy_print_callbacks.py | 2 +-
 3 files changed, 5 insertions(+), 11 deletions(-)

diff --git a/covalent/cloud_resource_manager/core.py b/covalent/cloud_resource_manager/core.py
index 4a22f966f..1cb71a202 100644
--- a/covalent/cloud_resource_manager/core.py
+++ b/covalent/cloud_resource_manager/core.py
@@ -195,7 +195,7 @@ def _parse_terraform_error_log(self) -> List[str]:
         for _, line in enumerate(lines):
             error_index = line.strip().find("error:")
             if error_index != -1:
-                error_message = line.strip()[error_index + len("error:"):]
+                error_message = line.strip()[error_index + len("error:") :]
                 logger.error(error_message)
         return lines
 
@@ -525,6 +525,4 @@ def status(self) -> None:
         tf_state = " ".join([terraform, "state", "list", f"-state={tf_state_file}"])
 
         # Run `terraform state list`
-        return self._run_in_subprocess(
-            cmd=tf_state, env_vars=self._terraform_log_env_vars
-        )
+        return self._run_in_subprocess(cmd=tf_state, env_vars=self._terraform_log_env_vars)
diff --git a/covalent_dispatcher/_cli/groups/deploy.py b/covalent_dispatcher/_cli/groups/deploy.py
index f239ee81b..e73305e11 100644
--- a/covalent_dispatcher/_cli/groups/deploy.py
+++ b/covalent_dispatcher/_cli/groups/deploy.py
@@ -94,10 +94,7 @@ def get_up_help_table(crm: CloudResourceManager) -> Table:
 
 
 def _run_command_and_show_output(
-    _command: Callable[[Callable], None],
-    _status_message: str,
-    *,
-    verbose: bool
+    _command: Callable[[Callable], None], _status_message: str, *, verbose: bool
 ) -> None:
     """Run the command and show the output in the console.
 
@@ -112,7 +109,6 @@ def _run_command_and_show_output(
     msg_template = _TEMPLATE.format(message=_status_message, text="{text}")
 
     with console.status(msg_template.format(text="")) as console_status:
-
         print_callback = ScrollBufferCallback(
             console=console,
             console_status=console_status,
@@ -131,7 +127,7 @@ def _run_command_and_show_output(
         if not verbose:
             console_status.stop()
             if (complete_msg := print_callback.complete_msg) is not None:
-                console.print('\n', complete_msg, style="bold green")
+                console.print("\n", complete_msg, style="bold green")
 
 
 @click.group(invoke_without_command=True)
diff --git a/covalent_dispatcher/_cli/groups/deploy_print_callbacks.py b/covalent_dispatcher/_cli/groups/deploy_print_callbacks.py
index ee2e1453b..949e5ceb4 100644
--- a/covalent_dispatcher/_cli/groups/deploy_print_callbacks.py
+++ b/covalent_dispatcher/_cli/groups/deploy_print_callbacks.py
@@ -94,6 +94,6 @@ def _verbose_print(self, msg: str) -> None:
         """Print normally, line by line."""
         print(msg.rstrip() if msg != "\n" else msg)
 
-    def render_buffer(self, sep: str = ' ') -> str:
+    def render_buffer(self, sep: str = " ") -> str:
         """Render the current buffer as a string."""
         return sep.join(self.buffer)

From 937e0273260e79391f06179e8e6d7edc87e82787 Mon Sep 17 00:00:00 2001
From: Ara Ghukasyan <ara@agnostiq.ai>
Date: Tue, 28 Nov 2023 21:45:51 -0500
Subject: [PATCH 07/33] update tests

---
 .../cloud_resource_manager/core_test.py       | 56 ++++++++-----------
 1 file changed, 22 insertions(+), 34 deletions(-)

diff --git a/tests/covalent_tests/cloud_resource_manager/core_test.py b/tests/covalent_tests/cloud_resource_manager/core_test.py
index ea2ef058a..fb2a3fd4b 100644
--- a/tests/covalent_tests/cloud_resource_manager/core_test.py
+++ b/tests/covalent_tests/cloud_resource_manager/core_test.py
@@ -163,9 +163,9 @@ def test_cloud_resource_manager_init(mocker, options, executor_name, executor_mo
             )
 
 
-def test_print_stdout(mocker, crm):
+def test_poll_process(mocker, crm):
     """
-    Unit test for CloudResourceManager._print_stdout() method
+    Unit test for CloudResourceManager._poll_process() method
     """
 
     test_stdout = "test_stdout".encode("utf-8")
@@ -177,7 +177,7 @@ def test_print_stdout(mocker, crm):
     mock_process.stdout.readline.side_effect = partial(next, iter([test_stdout, None]))
 
     mock_print = mocker.patch("covalent.cloud_resource_manager.core.print")
-    return_code = crm._print_stdout(
+    return_code = crm._poll_process(
         mock_process,
         print_callback=mock_print(
             test_stdout.decode("utf-8"),
@@ -203,7 +203,6 @@ def test_run_in_subprocess(mocker, test_retcode, crm):
     """
 
     test_cmd = "test_cmd"
-    test_workdir = "test_workdir"
     test_env_vars = {"test_env_key": "test_env_value"}
 
     mock_process = mocker.MagicMock()
@@ -212,8 +211,8 @@ def test_run_in_subprocess(mocker, test_retcode, crm):
         return_value=mock_process,
     )
 
-    mock_print_stdout = mocker.patch(
-        "covalent.cloud_resource_manager.core.CloudResourceManager._print_stdout",
+    mocker.patch(
+        "covalent.cloud_resource_manager.core.CloudResourceManager._poll_process",
         return_value=int(test_retcode),
     )
 
@@ -228,31 +227,29 @@ def test_run_in_subprocess(mocker, test_retcode, crm):
 
     if test_retcode != 0:
         exception = subprocess.CalledProcessError(returncode=test_retcode, cmd=test_cmd)
-        print("sam exception ", exception)
-        with pytest.raises(Exception, match=str(exception)):
+        print("some exception ", exception)
+        with pytest.raises(Exception) as excinfo:
             crm._run_in_subprocess(
                 cmd=test_cmd,
-                workdir=test_workdir,
                 env_vars=test_env_vars,
             )
+            # Errors are contained in the output for printing.
+            assert excinfo.value.output == "some exception "
     else:
         crm._run_in_subprocess(
             cmd=test_cmd,
-            workdir=test_workdir,
             env_vars=test_env_vars,
         )
 
     mock_popen.assert_called_once_with(
         args=test_cmd,
         stdout=subprocess.PIPE,
-        stderr=subprocess.STDOUT,
-        cwd=test_workdir,
+        stderr=subprocess.PIPE,
+        cwd=crm.executor_tf_path,
+        universal_newlines=True,
         shell=True,
         env=test_env_vars,
     )
-    # print("sam mocker process : ", mock_process)
-    # print("sam mocker print   : ", mock_print_stdout)
-    # mock_print_stdout.assert_called_once_with(mock_process)
 
 
 def test_update_config(mocker, crm, executor_name):
@@ -361,14 +358,14 @@ def test_up(mocker, dry_run, executor_options, executor_name, executor_module_pa
     test_tf_path = "test_tf_path"
     test_tf_state_file = "test_tf_state_file"
 
-    mock_get_tf_path = mocker.patch(
+    mocker.patch(
         "covalent.cloud_resource_manager.core.CloudResourceManager._get_tf_path",
         return_value=test_tf_path,
     )
     mocker.patch(
         "covalent.cloud_resource_manager.core.CloudResourceManager._validation_docker",
     )
-    mock_get_tf_statefile_path = mocker.patch(
+    mocker.patch(
         "covalent.cloud_resource_manager.core.CloudResourceManager._get_tf_statefile_path",
         return_value=test_tf_state_file,
     )
@@ -420,18 +417,12 @@ def test_up(mocker, dry_run, executor_options, executor_name, executor_module_pa
         ) as mock_file:
             crm.up(dry_run=dry_run, print_callback=None)
 
-        env_vars = {
-            "PATH": "$PATH:/usr/bin",
-            "TF_LOG": "ERROR",
-            "TF_LOG_PATH": os.path.join(crm.executor_tf_path + "/terraform-error.log"),
-        }
         # mock_get_tf_path.assert_called_once()
         init_cmd = f"{test_tf_path} init"
         mock_run_in_subprocess.assert_any_call(
             cmd=init_cmd,
-            workdir=crm.executor_tf_path,
-            env_vars=env_vars,
-            # print_callback=None,
+            env_vars=crm._terraform_log_env_vars,
+            print_callback=None,
         )
 
         mock_environ_copy.assert_called_once()
@@ -444,18 +435,17 @@ def test_up(mocker, dry_run, executor_options, executor_name, executor_module_pa
 
             key, value = list(executor_options.items())[0]
             mock_file().write.assert_called_once_with(f'{key}="{value}"\n')
+
         mock_run_in_subprocess.assert_any_call(
             cmd=f"{test_tf_path} plan -out tf.plan",  # -state={test_tf_state_file}",
-            workdir=crm.executor_tf_path,
-            env_vars=env_vars,
+            env_vars=crm._terraform_log_env_vars,
             print_callback=None,
         )
 
         if not dry_run:
             mock_run_in_subprocess.assert_any_call(
                 cmd=f"{test_tf_path} apply tf.plan -state={test_tf_state_file}",
-                workdir=crm.executor_tf_path,
-                env_vars=env_vars,
+                env_vars=crm._terraform_log_env_vars,
                 print_callback=None,
             )
 
@@ -471,7 +461,6 @@ def test_down(mocker, crm):
 
     test_tf_path = "test_tf_path"
     test_tf_state_file = "test_tf_state_file"
-    test_tf_log_file = "terraform-error.log"
 
     mock_get_tf_path = mocker.patch(
         "covalent.cloud_resource_manager.core.CloudResourceManager._get_tf_path",
@@ -521,9 +510,9 @@ def test_down(mocker, crm):
             "-auto-approve",
         ]
     )
-    env_vars = {"PATH": "$PATH:/usr/bin", "TF_LOG": "ERROR", "TF_LOG_PATH": log_file_path}
+    env_vars = crm._terraform_log_env_vars
     mock_run_in_subprocess.assert_called_once_with(
-        cmd=cmd, print_callback=None, workdir=crm.executor_tf_path, env_vars=env_vars
+        cmd=cmd, print_callback=None, env_vars=env_vars
     )
 
     assert mock_path_exists.call_count == 5
@@ -562,6 +551,5 @@ def test_status(mocker, crm):
     mock_get_tf_statefile_path.assert_called_once()
     mock_run_in_subprocess.assert_called_once_with(
         cmd=f"{test_tf_path} state list -state={test_tf_state_file}",
-        workdir=crm.executor_tf_path,
-        env_vars={"PATH": "$PATH:/usr/bin", "TF_LOG": "ERROR", "TF_LOG_PATH": log_file_path},
+        env_vars=crm._terraform_log_env_vars,
     )

From edad590f30564934c6b693470f228ff1f18e2d6e Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Wed, 29 Nov 2023 02:46:47 +0000
Subject: [PATCH 08/33] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 tests/covalent_tests/cloud_resource_manager/core_test.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/tests/covalent_tests/cloud_resource_manager/core_test.py b/tests/covalent_tests/cloud_resource_manager/core_test.py
index fb2a3fd4b..d3fd68f4a 100644
--- a/tests/covalent_tests/cloud_resource_manager/core_test.py
+++ b/tests/covalent_tests/cloud_resource_manager/core_test.py
@@ -511,9 +511,7 @@ def test_down(mocker, crm):
         ]
     )
     env_vars = crm._terraform_log_env_vars
-    mock_run_in_subprocess.assert_called_once_with(
-        cmd=cmd, print_callback=None, env_vars=env_vars
-    )
+    mock_run_in_subprocess.assert_called_once_with(cmd=cmd, print_callback=None, env_vars=env_vars)
 
     assert mock_path_exists.call_count == 5
     assert mock_path_unlink.call_count == 4

From 6f3daf79bf0c6d313dd29c2a8bc74aac804f8af0 Mon Sep 17 00:00:00 2001
From: Ara Ghukasyan <ara@agnostiq.ai>
Date: Tue, 28 Nov 2023 23:26:12 -0500
Subject: [PATCH 09/33] add tests for coverage

---
 .../_cli/cli_test.py                          | 52 +++++++++++++
 .../cloud_resource_manager/core_test.py       | 75 ++++++++++++++++++-
 2 files changed, 126 insertions(+), 1 deletion(-)

diff --git a/tests/covalent_dispatcher_tests/_cli/cli_test.py b/tests/covalent_dispatcher_tests/_cli/cli_test.py
index a8b546e90..f1c1eefa3 100644
--- a/tests/covalent_dispatcher_tests/_cli/cli_test.py
+++ b/tests/covalent_dispatcher_tests/_cli/cli_test.py
@@ -18,10 +18,14 @@
 
 """Test for Covalent CLI Tool."""
 
+import subprocess
+
 import click
+import pytest
 from click.testing import CliRunner
 
 from covalent_dispatcher._cli.cli import cli
+from covalent_dispatcher._cli.groups.deploy import _run_command_and_show_output
 
 
 def test_cli(mocker):
@@ -65,3 +69,51 @@ def test_cli_commands():
         "status",
         "stop",
     ]
+
+
+@pytest.mark.parametrize(
+    ("error", "verbose"),
+    [
+        (False, False),
+        (False, True),
+        (True, False),
+        (True, True),
+    ],
+)
+def test_run_command_and_show_output(mocker, error, verbose):
+    """
+    Unit test for `_run_command_and_show_output` function.
+
+    Test that errors are raised and messages printed when expected.
+    """
+
+    mock_console_print = mocker.patch("rich.console.Console.print")
+    mock_click_echo = mocker.patch("click.echo")
+    mocker.patch(
+        "covalent_dispatcher._cli.groups.deploy_print_callbacks.ScrollBufferCallback.complete_msg",
+        return_value="Apply complete! Resources: 19 added, 0 changed, 0 destroyed.",
+    )
+
+    def _command(*args, **kwargs):
+        _command.calls += 1
+        _output = "Testing Error..."
+        _error = subprocess.CalledProcessError(1, "terraform", _output)
+        if error:
+            raise _error
+
+    _command.calls = 0
+
+    if error:
+        msg = "Testing Error..."
+        with pytest.raises(SystemExit):
+            _run_command_and_show_output(_command, msg, verbose=verbose)
+    else:
+        msg = "Testing Success..."
+        _run_command_and_show_output(_command, msg, verbose=verbose)
+
+    if error:
+        mock_click_echo.assert_called_once_with("Testing Error...")
+    elif not verbose:
+        mock_console_print.assert_called_once()
+    else:
+        assert _command.calls == 1
diff --git a/tests/covalent_tests/cloud_resource_manager/core_test.py b/tests/covalent_tests/cloud_resource_manager/core_test.py
index d3fd68f4a..2ac6bfe78 100644
--- a/tests/covalent_tests/cloud_resource_manager/core_test.py
+++ b/tests/covalent_tests/cloud_resource_manager/core_test.py
@@ -524,7 +524,6 @@ def test_status(mocker, crm):
 
     test_tf_path = "test_tf_path"
     test_tf_state_file = "test_tf_state_file"
-    log_file_path = os.path.join(crm.executor_tf_path + "/terraform-error.log")
     mock_get_tf_path = mocker.patch(
         "covalent.cloud_resource_manager.core.CloudResourceManager._get_tf_path",
         return_value=test_tf_path,
@@ -551,3 +550,77 @@ def test_status(mocker, crm):
         cmd=f"{test_tf_path} state list -state={test_tf_state_file}",
         env_vars=crm._terraform_log_env_vars,
     )
+
+
+class _FakeIO:
+    """Mocks process stdout and stderr."""
+
+    def __init__(self, message):
+        self.message = message
+
+    def read(self):
+        return self.message
+
+    def readline(self):
+        return self.read()
+
+
+class _FakeProc:
+    """Mocks process"""
+
+    def __init__(self, returncode):
+        self.returncode = returncode
+        self.args = ()
+        self.stdout = _FakeIO("v99.99")
+        self.stderr = _FakeIO("")
+
+    def poll(self):
+        return self.returncode
+
+
+def test_crm_get_resource_status(mocker, crm):
+    """
+    Unit test for CloudResourceManager._get_resource_status() method.
+
+    Test that errors while getting resource status don't exit, rather print and report status.
+    """
+
+    mock_terraform_error_validator = mocker.patch(
+        "covalent.cloud_resource_manager.core.CloudResourceManager._terraform_error_validator",
+    )
+    mock_print = mocker.patch(
+        "covalent.cloud_resource_manager.core.print",
+    )
+
+    crm._get_resource_status(
+        proc=_FakeProc(1),
+        cmd="fake command"
+    )
+    mock_print.assert_called_once()
+    mock_terraform_error_validator.assert_called_once()
+
+
+def test_no_git(crm, mocker):
+    """
+    Test for exit with status 1 if `git` is not available.
+    """
+
+    mocker.patch("shutil.which", return_value=None)
+    mocker.patch("covalent.cloud_resource_manager.CloudResourceManager._log_error_msg")
+
+    with pytest.raises(SystemExit):
+        crm._run_in_subprocess("fake command")
+
+
+def test_tf_version_error(mocker, crm):
+    """
+    Unit test for CloudResourceManager._get_tf_path() method.
+    """
+    latest_incompatible_version = 1.3
+    mocker.patch(
+        "covalent.cloud_resource_manager.core.float",
+        return_value=latest_incompatible_version
+    )
+
+    with pytest.raises(SystemExit):
+        crm._get_tf_path()

From 6ced594a3982d16fc6bf6aba39e49a9946f665e0 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Wed, 29 Nov 2023 04:27:12 +0000
Subject: [PATCH 10/33] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 tests/covalent_tests/cloud_resource_manager/core_test.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/tests/covalent_tests/cloud_resource_manager/core_test.py b/tests/covalent_tests/cloud_resource_manager/core_test.py
index 2ac6bfe78..12e05c41e 100644
--- a/tests/covalent_tests/cloud_resource_manager/core_test.py
+++ b/tests/covalent_tests/cloud_resource_manager/core_test.py
@@ -592,10 +592,7 @@ def test_crm_get_resource_status(mocker, crm):
         "covalent.cloud_resource_manager.core.print",
     )
 
-    crm._get_resource_status(
-        proc=_FakeProc(1),
-        cmd="fake command"
-    )
+    crm._get_resource_status(proc=_FakeProc(1), cmd="fake command")
     mock_print.assert_called_once()
     mock_terraform_error_validator.assert_called_once()
 
@@ -618,8 +615,7 @@ def test_tf_version_error(mocker, crm):
     """
     latest_incompatible_version = 1.3
     mocker.patch(
-        "covalent.cloud_resource_manager.core.float",
-        return_value=latest_incompatible_version
+        "covalent.cloud_resource_manager.core.float", return_value=latest_incompatible_version
     )
 
     with pytest.raises(SystemExit):

From b36bb69073422553a615df7f483813871b05a40c Mon Sep 17 00:00:00 2001
From: Ara Ghukasyan <ara@agnostiq.ai>
Date: Tue, 28 Nov 2023 23:41:01 -0500
Subject: [PATCH 11/33] check success case in tf version test

---
 .../covalent_tests/cloud_resource_manager/core_test.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/tests/covalent_tests/cloud_resource_manager/core_test.py b/tests/covalent_tests/cloud_resource_manager/core_test.py
index 12e05c41e..952b713dd 100644
--- a/tests/covalent_tests/cloud_resource_manager/core_test.py
+++ b/tests/covalent_tests/cloud_resource_manager/core_test.py
@@ -614,9 +614,19 @@ def test_tf_version_error(mocker, crm):
     Unit test for CloudResourceManager._get_tf_path() method.
     """
     latest_incompatible_version = 1.3
+
+    # fail
     mocker.patch(
         "covalent.cloud_resource_manager.core.float", return_value=latest_incompatible_version
     )
 
     with pytest.raises(SystemExit):
         crm._get_tf_path()
+
+    # succeed
+    mocker.patch(
+        "covalent.cloud_resource_manager.core.float",
+        return_value=latest_incompatible_version + 10_000
+    )
+
+    assert "terraform" in crm._get_tf_path()

From d4ab18eb521935c3d63c0abbc8e1b76be74fadc1 Mon Sep 17 00:00:00 2001
From: Ara Ghukasyan <ara@agnostiq.ai>
Date: Wed, 29 Nov 2023 00:12:12 -0500
Subject: [PATCH 12/33] update tests

---
 covalent/cloud_resource_manager/core.py       |  1 +
 covalent_dispatcher/_cli/groups/deploy.py     |  4 +-
 .../_cli/cli_test.py                          | 53 ++++++++++++++++
 .../cloud_resource_manager/core_test.py       | 62 +++++++++----------
 4 files changed, 88 insertions(+), 32 deletions(-)

diff --git a/covalent/cloud_resource_manager/core.py b/covalent/cloud_resource_manager/core.py
index 1cb71a202..81e04363c 100644
--- a/covalent/cloud_resource_manager/core.py
+++ b/covalent/cloud_resource_manager/core.py
@@ -426,6 +426,7 @@ def up(self, print_callback: Callable, dry_run: bool = True) -> None:
         tf_vars_env_dict = os.environ.copy()
 
         if self.executor_options:
+            # pragma: no cover
             with open(tfvars_file, "w", encoding="utf-8") as f:
                 for key, value in self.executor_options.items():
                     tf_vars_env_dict[f"TF_VAR_{key}"] = value
diff --git a/covalent_dispatcher/_cli/groups/deploy.py b/covalent_dispatcher/_cli/groups/deploy.py
index e73305e11..05a932566 100644
--- a/covalent_dispatcher/_cli/groups/deploy.py
+++ b/covalent_dispatcher/_cli/groups/deploy.py
@@ -176,6 +176,7 @@ def up(executor_name: str, vars: Dict, help: bool, dry_run: bool, verbose: bool)
         $ covalent deploy up awslambda --verbose --region=us-east-1 --instance-type=t2.micro
 
     """
+    # pragma: no cover
     cmd_options = {key[2:]: value for key, value in (var.split("=") for var in vars)}
     if msg := validate_args(cmd_options):
         click.echo(msg)
@@ -211,6 +212,7 @@ def down(executor_name: str, verbose: bool) -> None:
         $ covalent deploy down ecs --verbose
 
     """
+    # pragma: no cover
     crm = get_crm_object(executor_name)
     _command = partial(crm.down)
     _run_command_and_show_output(_command, "Destroying resources...", verbose=verbose)
@@ -240,7 +242,7 @@ def status(executor_names: Tuple[str]) -> None:
         "*up": "Warning: Provisioning error, retry 'up'.",
         "*down": "Warning: Teardown error, retry 'down'.",
     }
-
+    # pragma: no cover
     if not executor_names:
         executor_names = [
             name
diff --git a/tests/covalent_dispatcher_tests/_cli/cli_test.py b/tests/covalent_dispatcher_tests/_cli/cli_test.py
index f1c1eefa3..faf5052e8 100644
--- a/tests/covalent_dispatcher_tests/_cli/cli_test.py
+++ b/tests/covalent_dispatcher_tests/_cli/cli_test.py
@@ -26,6 +26,7 @@
 
 from covalent_dispatcher._cli.cli import cli
 from covalent_dispatcher._cli.groups.deploy import _run_command_and_show_output
+from covalent_dispatcher._cli.groups.deploy_print_callbacks import ScrollBufferCallback
 
 
 def test_cli(mocker):
@@ -117,3 +118,55 @@ def _command(*args, **kwargs):
         mock_console_print.assert_called_once()
     else:
         assert _command.calls == 1
+
+
+@pytest.mark.parametrize(
+    ("verbose", "mode"),
+    [
+        (False, "provision"),
+        (False, "destroy"),
+        (True, "provision"),
+        (True, "destroy"),
+    ]
+)
+def test_scroll_buffer_print_callback(mocker, verbose, mode):
+    """
+    Unit test for the custom buffered print callback.
+    """
+    from rich.console import Console
+    from rich.status import Status
+    from covalent_dispatcher._cli.groups.deploy import _TEMPLATE
+
+    console = Console(record=True)
+    console_status = Status("Testing...", console=console)
+
+    mock_print = mocker.patch("covalent_dispatcher._cli.groups.deploy_print_callbacks.print")
+    mock_console_status_update = mocker.patch("rich.status.Status.update")
+
+    print_callback = ScrollBufferCallback(
+        console=console,
+        console_status=console_status,
+        msg_template=_TEMPLATE.format(message="Testing...", text="{text}"),
+        verbose=verbose,
+    )
+
+    complete_msg = (
+        "Apply complete! Resources: 19 added, 0 changed, 0 destroyed."
+        if mode == "provision" else
+        "Destroy complete! Resources: 19 destroyed."
+    )
+    messages = [
+        "fake", "fake", "fake", complete_msg, "fake", "fake", "fake"
+    ]
+
+    for msg in messages:
+        print_callback(msg)
+        if verbose:
+            mock_print.assert_called_with(msg)
+        else:
+            mock_console_status_update.assert_called_with(
+                print_callback.msg_template.format(text=print_callback.render_buffer())
+            )
+
+    if not verbose:
+        assert print_callback.complete_msg == complete_msg
diff --git a/tests/covalent_tests/cloud_resource_manager/core_test.py b/tests/covalent_tests/cloud_resource_manager/core_test.py
index 952b713dd..f3b10efca 100644
--- a/tests/covalent_tests/cloud_resource_manager/core_test.py
+++ b/tests/covalent_tests/cloud_resource_manager/core_test.py
@@ -56,6 +56,35 @@ def crm(mocker, executor_name, executor_module_path):
     )
 
 
+class _FakeIO:
+    """Mocks process stdout and stderr."""
+
+    def __init__(self, message):
+        self.message = message
+
+    def read(self):
+        return self.message
+
+    def readline(self):
+        return self.read()
+
+
+class _FakeProc:
+    """Mocks process"""
+
+    def __init__(self, returncode, stdout="", stderr=""):
+        self.returncode = returncode
+        self.args = ()
+        self.stdout = _FakeIO(stdout)
+        self.stderr = _FakeIO(stderr)
+
+    def poll(self):
+        return self.returncode
+
+    def communicate(self):
+        return self.stdout.read(), self.stderr.read()
+
+
 def test_get_executor_module(mocker):
     """
     Unit test for get_executor_module method
@@ -205,10 +234,9 @@ def test_run_in_subprocess(mocker, test_retcode, crm):
     test_cmd = "test_cmd"
     test_env_vars = {"test_env_key": "test_env_value"}
 
-    mock_process = mocker.MagicMock()
     mock_popen = mocker.patch(
         "covalent.cloud_resource_manager.core.subprocess.Popen",
-        return_value=mock_process,
+        return_value=_FakeProc(test_retcode),
     )
 
     mocker.patch(
@@ -221,14 +249,12 @@ def test_run_in_subprocess(mocker, test_retcode, crm):
     )
     mocker.patch(
         "covalent.cloud_resource_manager.core.CloudResourceManager._log_error_msg",
-        return_value=None,
-        side_effect=None,
     )
 
     if test_retcode != 0:
         exception = subprocess.CalledProcessError(returncode=test_retcode, cmd=test_cmd)
         print("some exception ", exception)
-        with pytest.raises(Exception) as excinfo:
+        with pytest.raises(subprocess.CalledProcessError) as excinfo:
             crm._run_in_subprocess(
                 cmd=test_cmd,
                 env_vars=test_env_vars,
@@ -552,32 +578,6 @@ def test_status(mocker, crm):
     )
 
 
-class _FakeIO:
-    """Mocks process stdout and stderr."""
-
-    def __init__(self, message):
-        self.message = message
-
-    def read(self):
-        return self.message
-
-    def readline(self):
-        return self.read()
-
-
-class _FakeProc:
-    """Mocks process"""
-
-    def __init__(self, returncode):
-        self.returncode = returncode
-        self.args = ()
-        self.stdout = _FakeIO("v99.99")
-        self.stderr = _FakeIO("")
-
-    def poll(self):
-        return self.returncode
-
-
 def test_crm_get_resource_status(mocker, crm):
     """
     Unit test for CloudResourceManager._get_resource_status() method.

From 6e4ef99592a91393c3bb023382dc072ddc955f52 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Wed, 29 Nov 2023 05:12:41 +0000
Subject: [PATCH 13/33] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 tests/covalent_dispatcher_tests/_cli/cli_test.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/covalent_dispatcher_tests/_cli/cli_test.py b/tests/covalent_dispatcher_tests/_cli/cli_test.py
index faf5052e8..6cbac4026 100644
--- a/tests/covalent_dispatcher_tests/_cli/cli_test.py
+++ b/tests/covalent_dispatcher_tests/_cli/cli_test.py
@@ -135,6 +135,7 @@ def test_scroll_buffer_print_callback(mocker, verbose, mode):
     """
     from rich.console import Console
     from rich.status import Status
+
     from covalent_dispatcher._cli.groups.deploy import _TEMPLATE
 
     console = Console(record=True)

From b1f44f4c19c16cb576dda34b5682a25c907f3d9b Mon Sep 17 00:00:00 2001
From: Ara Ghukasyan <ara@agnostiq.ai>
Date: Wed, 29 Nov 2023 10:08:26 -0500
Subject: [PATCH 14/33] pre-commit fixes

---
 tests/covalent_dispatcher_tests/_cli/cli_test.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/tests/covalent_dispatcher_tests/_cli/cli_test.py b/tests/covalent_dispatcher_tests/_cli/cli_test.py
index 6cbac4026..9a65ff4ab 100644
--- a/tests/covalent_dispatcher_tests/_cli/cli_test.py
+++ b/tests/covalent_dispatcher_tests/_cli/cli_test.py
@@ -127,7 +127,7 @@ def _command(*args, **kwargs):
         (False, "destroy"),
         (True, "provision"),
         (True, "destroy"),
-    ]
+    ],
 )
 def test_scroll_buffer_print_callback(mocker, verbose, mode):
     """
@@ -153,12 +153,10 @@ def test_scroll_buffer_print_callback(mocker, verbose, mode):
 
     complete_msg = (
         "Apply complete! Resources: 19 added, 0 changed, 0 destroyed."
-        if mode == "provision" else
-        "Destroy complete! Resources: 19 destroyed."
+        if mode == "provision"
+        else "Destroy complete! Resources: 19 destroyed."
     )
-    messages = [
-        "fake", "fake", "fake", complete_msg, "fake", "fake", "fake"
-    ]
+    messages = ["fake", "fake", "fake", complete_msg, "fake", "fake", "fake"]
 
     for msg in messages:
         print_callback(msg)

From 8a0e0ed5ca9f18073cbcbd349895d6678b2ff09e Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Wed, 29 Nov 2023 15:09:50 +0000
Subject: [PATCH 15/33] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 tests/covalent_tests/cloud_resource_manager/core_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/covalent_tests/cloud_resource_manager/core_test.py b/tests/covalent_tests/cloud_resource_manager/core_test.py
index f3b10efca..7e017ceab 100644
--- a/tests/covalent_tests/cloud_resource_manager/core_test.py
+++ b/tests/covalent_tests/cloud_resource_manager/core_test.py
@@ -626,7 +626,7 @@ def test_tf_version_error(mocker, crm):
     # succeed
     mocker.patch(
         "covalent.cloud_resource_manager.core.float",
-        return_value=latest_incompatible_version + 10_000
+        return_value=latest_incompatible_version + 10_000,
     )
 
     assert "terraform" in crm._get_tf_path()

From 57c8f74a9f06644820304bbbc56684a68367e519 Mon Sep 17 00:00:00 2001
From: Ara Ghukasyan <ara@agnostiq.ai>
Date: Wed, 29 Nov 2023 10:44:10 -0500
Subject: [PATCH 16/33] modify to pass without `terraform` in CI test env

---
 tests/covalent_tests/cloud_resource_manager/core_test.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/covalent_tests/cloud_resource_manager/core_test.py b/tests/covalent_tests/cloud_resource_manager/core_test.py
index 7e017ceab..fbb5a10ff 100644
--- a/tests/covalent_tests/cloud_resource_manager/core_test.py
+++ b/tests/covalent_tests/cloud_resource_manager/core_test.py
@@ -629,4 +629,7 @@ def test_tf_version_error(mocker, crm):
         return_value=latest_incompatible_version + 10_000,
     )
 
+    # NOTE: Assume terraform does not exist in CI test environment
+    mocker.patch("shutil.which", return_value="/opt/homebrew/bin/terraform")
+
     assert "terraform" in crm._get_tf_path()

From b69479e8a8f9db94f96784c4349026ef6755134e Mon Sep 17 00:00:00 2001
From: Ara Ghukasyan <ara@agnostiq.ai>
Date: Wed, 29 Nov 2023 11:39:08 -0500
Subject: [PATCH 17/33] retry tests fix

---
 tests/covalent_tests/cloud_resource_manager/core_test.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/covalent_tests/cloud_resource_manager/core_test.py b/tests/covalent_tests/cloud_resource_manager/core_test.py
index fbb5a10ff..04f8a0902 100644
--- a/tests/covalent_tests/cloud_resource_manager/core_test.py
+++ b/tests/covalent_tests/cloud_resource_manager/core_test.py
@@ -631,5 +631,6 @@ def test_tf_version_error(mocker, crm):
 
     # NOTE: Assume terraform does not exist in CI test environment
     mocker.patch("shutil.which", return_value="/opt/homebrew/bin/terraform")
+    mocker.patch("covalent.cloud_resource_manager.core.logger.error")
 
     assert "terraform" in crm._get_tf_path()

From e35d4826a10ec341e1e3d1a411ac3c4770cab348 Mon Sep 17 00:00:00 2001
From: Ara Ghukasyan <ara@agnostiq.ai>
Date: Wed, 29 Nov 2023 12:04:45 -0500
Subject: [PATCH 18/33] retry retry fix

---
 tests/covalent_tests/cloud_resource_manager/core_test.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/covalent_tests/cloud_resource_manager/core_test.py b/tests/covalent_tests/cloud_resource_manager/core_test.py
index 04f8a0902..3231f615f 100644
--- a/tests/covalent_tests/cloud_resource_manager/core_test.py
+++ b/tests/covalent_tests/cloud_resource_manager/core_test.py
@@ -632,5 +632,6 @@ def test_tf_version_error(mocker, crm):
     # NOTE: Assume terraform does not exist in CI test environment
     mocker.patch("shutil.which", return_value="/opt/homebrew/bin/terraform")
     mocker.patch("covalent.cloud_resource_manager.core.logger.error")
+    mocker.patch("subprocess.run")
 
     assert "terraform" in crm._get_tf_path()

From c1fcbc8d74c9b91f4ff6df1badf687169705c269 Mon Sep 17 00:00:00 2001
From: Ara Ghukasyan <ara@agnostiq.ai>
Date: Wed, 29 Nov 2023 12:45:42 -0500
Subject: [PATCH 19/33] remove useless pragma tags

---
 covalent/cloud_resource_manager/core.py   | 1 -
 covalent_dispatcher/_cli/groups/deploy.py | 3 ---
 2 files changed, 4 deletions(-)

diff --git a/covalent/cloud_resource_manager/core.py b/covalent/cloud_resource_manager/core.py
index 81e04363c..1cb71a202 100644
--- a/covalent/cloud_resource_manager/core.py
+++ b/covalent/cloud_resource_manager/core.py
@@ -426,7 +426,6 @@ def up(self, print_callback: Callable, dry_run: bool = True) -> None:
         tf_vars_env_dict = os.environ.copy()
 
         if self.executor_options:
-            # pragma: no cover
             with open(tfvars_file, "w", encoding="utf-8") as f:
                 for key, value in self.executor_options.items():
                     tf_vars_env_dict[f"TF_VAR_{key}"] = value
diff --git a/covalent_dispatcher/_cli/groups/deploy.py b/covalent_dispatcher/_cli/groups/deploy.py
index 05a932566..98b15fbab 100644
--- a/covalent_dispatcher/_cli/groups/deploy.py
+++ b/covalent_dispatcher/_cli/groups/deploy.py
@@ -176,7 +176,6 @@ def up(executor_name: str, vars: Dict, help: bool, dry_run: bool, verbose: bool)
         $ covalent deploy up awslambda --verbose --region=us-east-1 --instance-type=t2.micro
 
     """
-    # pragma: no cover
     cmd_options = {key[2:]: value for key, value in (var.split("=") for var in vars)}
     if msg := validate_args(cmd_options):
         click.echo(msg)
@@ -212,7 +211,6 @@ def down(executor_name: str, verbose: bool) -> None:
         $ covalent deploy down ecs --verbose
 
     """
-    # pragma: no cover
     crm = get_crm_object(executor_name)
     _command = partial(crm.down)
     _run_command_and_show_output(_command, "Destroying resources...", verbose=verbose)
@@ -242,7 +240,6 @@ def status(executor_names: Tuple[str]) -> None:
         "*up": "Warning: Provisioning error, retry 'up'.",
         "*down": "Warning: Teardown error, retry 'down'.",
     }
-    # pragma: no cover
     if not executor_names:
         executor_names = [
             name

From 0fea752494d3f3155f7b749f8e202ac086ce2fa0 Mon Sep 17 00:00:00 2001
From: Ara Ghukasyan <ara@agnostiq.ai>
Date: Wed, 29 Nov 2023 12:48:44 -0500
Subject: [PATCH 20/33] use exist status 1 for error

---
 covalent/cloud_resource_manager/core.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/covalent/cloud_resource_manager/core.py b/covalent/cloud_resource_manager/core.py
index 1cb71a202..4721fb70c 100644
--- a/covalent/cloud_resource_manager/core.py
+++ b/covalent/cloud_resource_manager/core.py
@@ -123,7 +123,7 @@ def get_plugin_settings(
                 settings_dict[key]["value"] = value
             except Exception:
                 logger.error(f"No such option '{key}'. Use --help for available options")
-                sys.exit()
+                sys.exit(1)
 
     return settings_dict
 

From b2681863fd82b7504df282857de87cbce9a4a554 Mon Sep 17 00:00:00 2001
From: Ara Ghukasyan <ara@agnostiq.ai>
Date: Wed, 29 Nov 2023 13:07:34 -0500
Subject: [PATCH 21/33] include crm.up test with 'valid' options

---
 .../cloud_resource_manager/core_test.py       | 63 ++++++++++++++-----
 1 file changed, 48 insertions(+), 15 deletions(-)

diff --git a/tests/covalent_tests/cloud_resource_manager/core_test.py b/tests/covalent_tests/cloud_resource_manager/core_test.py
index 3231f615f..0608a0338 100644
--- a/tests/covalent_tests/cloud_resource_manager/core_test.py
+++ b/tests/covalent_tests/cloud_resource_manager/core_test.py
@@ -16,6 +16,7 @@
 
 import os
 import subprocess
+import tempfile
 from configparser import ConfigParser
 from functools import partial
 from pathlib import Path
@@ -430,6 +431,46 @@ def test_up(mocker, dry_run, executor_options, executor_name, executor_module_pa
                 executor_module_path=executor_module_path,
                 options=executor_options,
             )
+
+        # Try again, but force success with patches.
+        # Disable validation.
+        mocker.patch(
+            "covalent.cloud_resource_manager.core.CloudResourceManager.validate_options",
+            return_value=None,
+        )
+        # Disable plugin settings.
+        mocker.patch(
+            "covalent.cloud_resource_manager.core.CloudResourceManager.get_plugin_settings",
+            return_value={},
+        )
+        # Disable path checks so nothing deleted (as it would be, if exists).
+        mocker.patch("covalent.cloud_resource_manager.core.Path.exists", return_value=False)
+        # Disable _run_in_subprocess to avoid side effects.
+        mocker.patch(
+            "covalent.cloud_resource_manager.core.CloudResourceManager._run_in_subprocess",
+        )
+        # Disable _update_config to avoid side effects.
+        mocker.patch(
+            "covalent.cloud_resource_manager.core.CloudResourceManager._update_config",
+        )
+
+        with tempfile.TemporaryDirectory() as d:
+            # Create fake vars file to avoid side effects.
+            fake_tfvars_file = Path(d.name) / "terraform.tfvars"
+            fake_tfvars_file.touch()
+            mocker.patch(
+                "covalent.cloud_resource_manager.core.CloudResourceManager.executor_tf_path",
+                return_value=Path(d.name),
+            )
+
+            crm.up(dry_run=False, print_callback=None)
+
+        crm = CloudResourceManager(
+            executor_name=executor_name,
+            executor_module_path=executor_module_path,
+            options=executor_options,
+        )
+
     else:
         crm = CloudResourceManager(
             executor_name=executor_name,
@@ -453,15 +494,6 @@ def test_up(mocker, dry_run, executor_options, executor_name, executor_module_pa
 
         mock_environ_copy.assert_called_once()
 
-        if executor_options:
-            mock_file.assert_called_once_with(
-                f"{crm.executor_tf_path}/terraform.tfvars",
-                "w",
-            )
-
-            key, value = list(executor_options.items())[0]
-            mock_file().write.assert_called_once_with(f'{key}="{value}"\n')
-
         mock_run_in_subprocess.assert_any_call(
             cmd=f"{test_tf_path} plan -out tf.plan",  # -state={test_tf_state_file}",
             env_vars=crm._terraform_log_env_vars,
@@ -560,22 +592,23 @@ def test_status(mocker, crm):
         return_value=test_tf_state_file,
     )
 
+    mock_terraform_error_validator = mocker.patch(
+        "covalent.cloud_resource_manager.core.CloudResourceManager._terraform_error_validator",
+    )
+
     mocker.patch(
         "covalent.cloud_resource_manager.core.CloudResourceManager._validation_docker",
     )
 
-    mock_run_in_subprocess = mocker.patch(
-        "covalent.cloud_resource_manager.core.CloudResourceManager._run_in_subprocess",
+    mocker.patch(
+        "covalent.cloud_resource_manager.core.subprocess.Popen",
     )
 
     crm.status()
 
     mock_get_tf_path.assert_called_once()
     mock_get_tf_statefile_path.assert_called_once()
-    mock_run_in_subprocess.assert_called_once_with(
-        cmd=f"{test_tf_path} state list -state={test_tf_state_file}",
-        env_vars=crm._terraform_log_env_vars,
-    )
+    mock_terraform_error_validator.assert_called_once_with(tfstate_path=test_tf_state_file)
 
 
 def test_crm_get_resource_status(mocker, crm):

From 0d23504cbc5447cdeb108b0f89fa6ff035442fea Mon Sep 17 00:00:00 2001
From: Ara Ghukasyan <ara@agnostiq.ai>
Date: Wed, 29 Nov 2023 13:12:22 -0500
Subject: [PATCH 22/33] correct exit code

---
 covalent_dispatcher/_cli/groups/deploy.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/covalent_dispatcher/_cli/groups/deploy.py b/covalent_dispatcher/_cli/groups/deploy.py
index 98b15fbab..f40f684b2 100644
--- a/covalent_dispatcher/_cli/groups/deploy.py
+++ b/covalent_dispatcher/_cli/groups/deploy.py
@@ -183,7 +183,7 @@ def up(executor_name: str, vars: Dict, help: bool, dry_run: bool, verbose: bool)
     crm = get_crm_object(executor_name, cmd_options)
     if help:
         click.echo(Console().print(get_up_help_table(crm)))
-        sys.exit(1)
+        sys.exit(0)
 
     _command = partial(crm.up, dry_run=dry_run)
     _run_command_and_show_output(_command, "Provisioning resources...", verbose=verbose)

From 67f222d010e7b54863e966ad912c256532f28707 Mon Sep 17 00:00:00 2001
From: Ara Ghukasyan <ara@agnostiq.ai>
Date: Wed, 29 Nov 2023 14:08:45 -0500
Subject: [PATCH 23/33] update test

---
 tests/covalent_dispatcher_tests/_cli/cli_test.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/covalent_dispatcher_tests/_cli/cli_test.py b/tests/covalent_dispatcher_tests/_cli/cli_test.py
index 9a65ff4ab..b5ac8130e 100644
--- a/tests/covalent_dispatcher_tests/_cli/cli_test.py
+++ b/tests/covalent_dispatcher_tests/_cli/cli_test.py
@@ -149,6 +149,7 @@ def test_scroll_buffer_print_callback(mocker, verbose, mode):
         console_status=console_status,
         msg_template=_TEMPLATE.format(message="Testing...", text="{text}"),
         verbose=verbose,
+        max_lines=3,  # shorten to hit pop inside `._inline_print`
     )
 
     complete_msg = (

From fe392269bd8e91aa5eab06a52140ad6d84cd13bf Mon Sep 17 00:00:00 2001
From: Ara Ghukasyan <ara@agnostiq.ai>
Date: Wed, 29 Nov 2023 14:38:53 -0500
Subject: [PATCH 24/33] new test for up command with 'valid' options

---
 .../cloud_resource_manager/core_test.py       | 97 +++++++++++--------
 1 file changed, 56 insertions(+), 41 deletions(-)

diff --git a/tests/covalent_tests/cloud_resource_manager/core_test.py b/tests/covalent_tests/cloud_resource_manager/core_test.py
index 0608a0338..96e077194 100644
--- a/tests/covalent_tests/cloud_resource_manager/core_test.py
+++ b/tests/covalent_tests/cloud_resource_manager/core_test.py
@@ -426,51 +426,11 @@ def test_up(mocker, dry_run, executor_options, executor_name, executor_module_pa
 
     if executor_options:
         with pytest.raises(SystemExit):
-            crm = CloudResourceManager(
+            CloudResourceManager(
                 executor_name=executor_name,
                 executor_module_path=executor_module_path,
                 options=executor_options,
             )
-
-        # Try again, but force success with patches.
-        # Disable validation.
-        mocker.patch(
-            "covalent.cloud_resource_manager.core.CloudResourceManager.validate_options",
-            return_value=None,
-        )
-        # Disable plugin settings.
-        mocker.patch(
-            "covalent.cloud_resource_manager.core.CloudResourceManager.get_plugin_settings",
-            return_value={},
-        )
-        # Disable path checks so nothing deleted (as it would be, if exists).
-        mocker.patch("covalent.cloud_resource_manager.core.Path.exists", return_value=False)
-        # Disable _run_in_subprocess to avoid side effects.
-        mocker.patch(
-            "covalent.cloud_resource_manager.core.CloudResourceManager._run_in_subprocess",
-        )
-        # Disable _update_config to avoid side effects.
-        mocker.patch(
-            "covalent.cloud_resource_manager.core.CloudResourceManager._update_config",
-        )
-
-        with tempfile.TemporaryDirectory() as d:
-            # Create fake vars file to avoid side effects.
-            fake_tfvars_file = Path(d.name) / "terraform.tfvars"
-            fake_tfvars_file.touch()
-            mocker.patch(
-                "covalent.cloud_resource_manager.core.CloudResourceManager.executor_tf_path",
-                return_value=Path(d.name),
-            )
-
-            crm.up(dry_run=False, print_callback=None)
-
-        crm = CloudResourceManager(
-            executor_name=executor_name,
-            executor_module_path=executor_module_path,
-            options=executor_options,
-        )
-
     else:
         crm = CloudResourceManager(
             executor_name=executor_name,
@@ -512,6 +472,61 @@ def test_up(mocker, dry_run, executor_options, executor_name, executor_module_pa
             )
 
 
+def test_up_executor_options(mocker, executor_name, executor_module_path):
+    """
+    Unit test for CloudResourceManager.up() method with executor options.
+
+    Test expected behavior with 'valid' options. Note that *actual* valid options
+    require executor plugins to be installed, so not suitable for CI tests.
+    """
+    # Disable validation.
+    mocker.patch(
+        "covalent.cloud_resource_manager.core.validate_options",
+        return_value=None,
+    )
+    mocker.patch(
+        "covalent.cloud_resource_manager.core.CloudResourceManager._validation_docker",
+    )
+
+    # Disable actually finding executor.
+    mocker.patch(
+        "covalent.cloud_resource_manager.core.get_executor_module",
+    )
+
+    # Disable plugin settings.
+    mocker.patch(
+        "covalent.cloud_resource_manager.core.get_plugin_settings",
+        return_value={},
+    )
+
+    # Disable path checks so nothing deleted (as it would be, if exists).
+    mocker.patch("covalent.cloud_resource_manager.core.Path.exists", return_value=False)
+
+    # Disable _run_in_subprocess to avoid side effects.
+    mocker.patch(
+        "covalent.cloud_resource_manager.core.CloudResourceManager._run_in_subprocess",
+    )
+
+    # Disable _update_config to avoid side effects.
+    mocker.patch(
+        "covalent.cloud_resource_manager.core.CloudResourceManager._update_config",
+    )
+
+    crm = CloudResourceManager(
+        executor_name=executor_name,
+        executor_module_path=executor_module_path,
+        options={"test_key": "test_value"},
+    )
+
+    with tempfile.TemporaryDirectory() as d:
+        # Create fake vars file to avoid side effects.
+        fake_tfvars_file = Path(d) / "terraform.tfvars"
+        fake_tfvars_file.touch()
+
+        crm.executor_tf_path = d
+        crm.up(dry_run=False, print_callback=None)
+
+
 def test_down(mocker, crm):
     """
     Unit test for CloudResourceManager.down() method.

From a576a670ce1f2302fedf7cb16c9d45d59a20c57d Mon Sep 17 00:00:00 2001
From: Ara Ghukasyan <ara@agnostiq.ai>
Date: Wed, 29 Nov 2023 15:12:38 -0500
Subject: [PATCH 25/33] update to work without terraform install

---
 tests/covalent_tests/cloud_resource_manager/core_test.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/covalent_tests/cloud_resource_manager/core_test.py b/tests/covalent_tests/cloud_resource_manager/core_test.py
index 96e077194..cb48b5f24 100644
--- a/tests/covalent_tests/cloud_resource_manager/core_test.py
+++ b/tests/covalent_tests/cloud_resource_manager/core_test.py
@@ -512,6 +512,9 @@ def test_up_executor_options(mocker, executor_name, executor_module_path):
         "covalent.cloud_resource_manager.core.CloudResourceManager._update_config",
     )
 
+    # For CI tests, pretend homebrew exists.
+    mocker.patch("shutil.which", return_value="/opt/homebrew/bin/terraform")
+
     crm = CloudResourceManager(
         executor_name=executor_name,
         executor_module_path=executor_module_path,

From 7ea1b4b7b4986a128094cec02554b1a1ead3942e Mon Sep 17 00:00:00 2001
From: Ara Ghukasyan <ara@agnostiq.ai>
Date: Wed, 29 Nov 2023 15:42:11 -0500
Subject: [PATCH 26/33] more changes for CI tests

---
 .../covalent_tests/cloud_resource_manager/core_test.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/tests/covalent_tests/cloud_resource_manager/core_test.py b/tests/covalent_tests/cloud_resource_manager/core_test.py
index cb48b5f24..e41097c2a 100644
--- a/tests/covalent_tests/cloud_resource_manager/core_test.py
+++ b/tests/covalent_tests/cloud_resource_manager/core_test.py
@@ -73,11 +73,11 @@ def readline(self):
 class _FakeProc:
     """Mocks process"""
 
-    def __init__(self, returncode, stdout="", stderr=""):
+    def __init__(self, returncode, stdout="", stderr="", fake_stream=True):
         self.returncode = returncode
         self.args = ()
-        self.stdout = _FakeIO(stdout)
-        self.stderr = _FakeIO(stderr)
+        self.stdout = _FakeIO(stdout) if fake_stream else stdout
+        self.stderr = _FakeIO(stderr) if fake_stream else stderr
 
     def poll(self):
         return self.returncode
@@ -514,6 +514,10 @@ def test_up_executor_options(mocker, executor_name, executor_module_path):
 
     # For CI tests, pretend homebrew exists.
     mocker.patch("shutil.which", return_value="/opt/homebrew/bin/terraform")
+    mocker.patch(
+        "covalent.cloud_resource_manager.core.subprocess.run",
+        return_value=_FakeProc(0, stdout="v99.99", fake_stream=False),
+    )
 
     crm = CloudResourceManager(
         executor_name=executor_name,

From 388616eb9b9278060c83f0db14a070bd06608e2e Mon Sep 17 00:00:00 2001
From: Ara Ghukasyan <ara@agnostiq.ai>
Date: Wed, 29 Nov 2023 16:27:20 -0500
Subject: [PATCH 27/33] complete coverage for test_tf_version_error

---
 .../cloud_resource_manager/core_test.py       | 25 ++++++++-----------
 1 file changed, 11 insertions(+), 14 deletions(-)

diff --git a/tests/covalent_tests/cloud_resource_manager/core_test.py b/tests/covalent_tests/cloud_resource_manager/core_test.py
index e41097c2a..7fdca350d 100644
--- a/tests/covalent_tests/cloud_resource_manager/core_test.py
+++ b/tests/covalent_tests/cloud_resource_manager/core_test.py
@@ -668,25 +668,22 @@ def test_tf_version_error(mocker, crm):
     """
     Unit test for CloudResourceManager._get_tf_path() method.
     """
-    latest_incompatible_version = 1.3
-
-    # fail
-    mocker.patch(
-        "covalent.cloud_resource_manager.core.float", return_value=latest_incompatible_version
-    )
 
+    # Fail. Terraform not found on system.
+    mocker.patch("shutil.which", return_value=None)
     with pytest.raises(SystemExit):
         crm._get_tf_path()
 
-    # succeed
-    mocker.patch(
-        "covalent.cloud_resource_manager.core.float",
-        return_value=latest_incompatible_version + 10_000,
-    )
+    fake_proc_1 = _FakeProc(0, stdout="v0.0", fake_stream=False)
+    fake_proc_2 = _FakeProc(0, stdout="v99.99", fake_stream=False)
 
-    # NOTE: Assume terraform does not exist in CI test environment
+    # Fail. Old version of terraform found.
     mocker.patch("shutil.which", return_value="/opt/homebrew/bin/terraform")
-    mocker.patch("covalent.cloud_resource_manager.core.logger.error")
-    mocker.patch("subprocess.run")
+    mocker.patch("subprocess.run", return_value=fake_proc_1)
+    with pytest.raises(SystemExit):
+        crm._get_tf_path()
 
+    # Succeed.
+    mocker.patch("subprocess.run", return_value=fake_proc_2)
+    mocker.patch("covalent.cloud_resource_manager.core.logger.error")
     assert "terraform" in crm._get_tf_path()

From be21e4cd87d2d99f586605370e6e7e734c690ab4 Mon Sep 17 00:00:00 2001
From: Ara Ghukasyan <ara@agnostiq.ai>
Date: Wed, 29 Nov 2023 16:31:34 -0500
Subject: [PATCH 28/33] rename to avoid module vs. function conflict

---
 covalent_dispatcher/_cli/groups/__init__.py                 | 4 ++--
 covalent_dispatcher/_cli/groups/{db.py => db_group.py}      | 0
 .../_cli/groups/{deploy.py => deploy_group.py}              | 2 +-
 ...y_print_callbacks.py => deploy_group_print_callbacks.py} | 0
 tests/covalent_dispatcher_tests/_cli/cli_test.py            | 6 +++---
 tests/covalent_dispatcher_tests/_cli/groups/db_test.py      | 2 +-
 6 files changed, 7 insertions(+), 7 deletions(-)
 rename covalent_dispatcher/_cli/groups/{db.py => db_group.py} (100%)
 rename covalent_dispatcher/_cli/groups/{deploy.py => deploy_group.py} (99%)
 rename covalent_dispatcher/_cli/groups/{deploy_print_callbacks.py => deploy_group_print_callbacks.py} (100%)

diff --git a/covalent_dispatcher/_cli/groups/__init__.py b/covalent_dispatcher/_cli/groups/__init__.py
index 3eee4cd19..3ae5d8d46 100644
--- a/covalent_dispatcher/_cli/groups/__init__.py
+++ b/covalent_dispatcher/_cli/groups/__init__.py
@@ -13,5 +13,5 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from .db import db
-from .deploy import deploy
+from .db_group import db
+from .deploy_group import deploy
diff --git a/covalent_dispatcher/_cli/groups/db.py b/covalent_dispatcher/_cli/groups/db_group.py
similarity index 100%
rename from covalent_dispatcher/_cli/groups/db.py
rename to covalent_dispatcher/_cli/groups/db_group.py
diff --git a/covalent_dispatcher/_cli/groups/deploy.py b/covalent_dispatcher/_cli/groups/deploy_group.py
similarity index 99%
rename from covalent_dispatcher/_cli/groups/deploy.py
rename to covalent_dispatcher/_cli/groups/deploy_group.py
index f40f684b2..ad180754a 100644
--- a/covalent_dispatcher/_cli/groups/deploy.py
+++ b/covalent_dispatcher/_cli/groups/deploy_group.py
@@ -31,7 +31,7 @@
 from covalent.cloud_resource_manager.core import CloudResourceManager
 from covalent.executor import _executor_manager
 
-from .deploy_print_callbacks import ScrollBufferCallback
+from .deploy_group_print_callbacks import ScrollBufferCallback
 
 _TEMPLATE = "[bold green]{message} [default]\n {text}"
 
diff --git a/covalent_dispatcher/_cli/groups/deploy_print_callbacks.py b/covalent_dispatcher/_cli/groups/deploy_group_print_callbacks.py
similarity index 100%
rename from covalent_dispatcher/_cli/groups/deploy_print_callbacks.py
rename to covalent_dispatcher/_cli/groups/deploy_group_print_callbacks.py
diff --git a/tests/covalent_dispatcher_tests/_cli/cli_test.py b/tests/covalent_dispatcher_tests/_cli/cli_test.py
index b5ac8130e..8881fb7b8 100644
--- a/tests/covalent_dispatcher_tests/_cli/cli_test.py
+++ b/tests/covalent_dispatcher_tests/_cli/cli_test.py
@@ -25,8 +25,8 @@
 from click.testing import CliRunner
 
 from covalent_dispatcher._cli.cli import cli
-from covalent_dispatcher._cli.groups.deploy import _run_command_and_show_output
-from covalent_dispatcher._cli.groups.deploy_print_callbacks import ScrollBufferCallback
+from covalent_dispatcher._cli.groups.deploy_group import _run_command_and_show_output
+from covalent_dispatcher._cli.groups.deploy_group_print_callbacks import ScrollBufferCallback
 
 
 def test_cli(mocker):
@@ -136,7 +136,7 @@ def test_scroll_buffer_print_callback(mocker, verbose, mode):
     from rich.console import Console
     from rich.status import Status
 
-    from covalent_dispatcher._cli.groups.deploy import _TEMPLATE
+    from covalent_dispatcher._cli.groups.deploy_group import _TEMPLATE
 
     console = Console(record=True)
     console_status = Status("Testing...", console=console)
diff --git a/tests/covalent_dispatcher_tests/_cli/groups/db_test.py b/tests/covalent_dispatcher_tests/_cli/groups/db_test.py
index 80cd42eec..8830d89a5 100644
--- a/tests/covalent_dispatcher_tests/_cli/groups/db_test.py
+++ b/tests/covalent_dispatcher_tests/_cli/groups/db_test.py
@@ -18,7 +18,7 @@
 
 from click.testing import CliRunner
 
-from covalent_dispatcher._cli.groups.db import MIGRATION_WARNING_MSG, alembic, migrate
+from covalent_dispatcher._cli.groups.db_group import MIGRATION_WARNING_MSG, alembic, migrate
 from covalent_dispatcher._db.datastore import DataStore
 
 

From 2d00ecc446ce047dc1e6b3a56578456ddbf2fcea Mon Sep 17 00:00:00 2001
From: Ara Ghukasyan <ara@agnostiq.ai>
Date: Wed, 29 Nov 2023 17:21:07 -0500
Subject: [PATCH 29/33] add comment

---
 covalent_dispatcher/_cli/groups/deploy_group.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/covalent_dispatcher/_cli/groups/deploy_group.py b/covalent_dispatcher/_cli/groups/deploy_group.py
index ad180754a..55537b4b0 100644
--- a/covalent_dispatcher/_cli/groups/deploy_group.py
+++ b/covalent_dispatcher/_cli/groups/deploy_group.py
@@ -178,6 +178,7 @@ def up(executor_name: str, vars: Dict, help: bool, dry_run: bool, verbose: bool)
     """
     cmd_options = {key[2:]: value for key, value in (var.split("=") for var in vars)}
     if msg := validate_args(cmd_options):
+        # Message is not None, so there was an error.
         click.echo(msg)
         sys.exit(1)
     crm = get_crm_object(executor_name, cmd_options)

From a0e5aeaaf54aaf9ad92dc0206bb5f37e1eb85ce6 Mon Sep 17 00:00:00 2001
From: Ara Ghukasyan <ara@agnostiq.ai>
Date: Wed, 29 Nov 2023 17:23:00 -0500
Subject: [PATCH 30/33] add tests for deploy up, down, status

---
 .../_cli/cli_test.py                          | 108 +++++++++++++++++-
 1 file changed, 104 insertions(+), 4 deletions(-)

diff --git a/tests/covalent_dispatcher_tests/_cli/cli_test.py b/tests/covalent_dispatcher_tests/_cli/cli_test.py
index 8881fb7b8..65001fe4f 100644
--- a/tests/covalent_dispatcher_tests/_cli/cli_test.py
+++ b/tests/covalent_dispatcher_tests/_cli/cli_test.py
@@ -25,8 +25,6 @@
 from click.testing import CliRunner
 
 from covalent_dispatcher._cli.cli import cli
-from covalent_dispatcher._cli.groups.deploy_group import _run_command_and_show_output
-from covalent_dispatcher._cli.groups.deploy_group_print_callbacks import ScrollBufferCallback
 
 
 def test_cli(mocker):
@@ -87,11 +85,12 @@ def test_run_command_and_show_output(mocker, error, verbose):
 
     Test that errors are raised and messages printed when expected.
     """
+    from covalent_dispatcher._cli.groups.deploy_group import _run_command_and_show_output
 
     mock_console_print = mocker.patch("rich.console.Console.print")
     mock_click_echo = mocker.patch("click.echo")
     mocker.patch(
-        "covalent_dispatcher._cli.groups.deploy_print_callbacks.ScrollBufferCallback.complete_msg",
+        "covalent_dispatcher._cli.groups.deploy_group_print_callbacks.ScrollBufferCallback.complete_msg",
         return_value="Apply complete! Resources: 19 added, 0 changed, 0 destroyed.",
     )
 
@@ -137,11 +136,12 @@ def test_scroll_buffer_print_callback(mocker, verbose, mode):
     from rich.status import Status
 
     from covalent_dispatcher._cli.groups.deploy_group import _TEMPLATE
+    from covalent_dispatcher._cli.groups.deploy_group_print_callbacks import ScrollBufferCallback
 
     console = Console(record=True)
     console_status = Status("Testing...", console=console)
 
-    mock_print = mocker.patch("covalent_dispatcher._cli.groups.deploy_print_callbacks.print")
+    mock_print = mocker.patch("covalent_dispatcher._cli.groups.deploy_group_print_callbacks.print")
     mock_console_status_update = mocker.patch("rich.status.Status.update")
 
     print_callback = ScrollBufferCallback(
@@ -170,3 +170,103 @@ def test_scroll_buffer_print_callback(mocker, verbose, mode):
 
     if not verbose:
         assert print_callback.complete_msg == complete_msg
+
+
+def test_deploy_up(mocker):
+    """
+    Unit test for `covalent deploy up [executor_name]` command.
+    """
+
+    from covalent_dispatcher._cli.groups.deploy_group import up
+
+    # Patch function that executes commands.
+    mock_run_command_and_show_output = mocker.patch(
+        "covalent_dispatcher._cli.groups.deploy_group._run_command_and_show_output",
+    )
+
+    # Fail with invalid command options.
+    mocker.patch(
+        "covalent_dispatcher._cli.groups.deploy_group.validate_args",
+        return_value="Non-empty msg",
+    )
+    with pytest.raises(SystemExit) as exc_info:
+        ctx = click.Context(up)
+        ctx.invoke(up)
+
+    assert exc_info.value.code == 1
+
+    # Succeed but exit after help message.
+    mocker.patch(
+        "covalent_dispatcher._cli.groups.deploy_group.validate_args",
+        return_value=None,
+    )
+    mocker.patch(
+        "covalent_dispatcher._cli.groups.deploy_group.get_crm_object",
+    )
+    with pytest.raises(SystemExit) as exc_info:
+        ctx = click.Context(up)
+        ctx.invoke(up, help=True)
+
+    assert exc_info.value.code == 0
+
+    # Succeed with valid command options.
+    ctx = click.Context(up)
+    ctx.invoke(up, verbose=True)
+
+    mock_run_command_and_show_output.assert_called_once()
+
+
+def test_deploy_down(mocker):
+    """
+    Unit test for `covalent deploy down [executor_name]` command.
+    """
+
+    from covalent_dispatcher._cli.groups.deploy_group import down
+
+    # Patch function that executes commands.
+    mock_run_command_and_show_output = mocker.patch(
+        "covalent_dispatcher._cli.groups.deploy_group._run_command_and_show_output",
+    )
+    mocker.patch(
+        "covalent_dispatcher._cli.groups.deploy_group.get_crm_object",
+    )
+
+    ctx = click.Context(down)
+    ctx.invoke(down, verbose=True)
+    mock_run_command_and_show_output.assert_called_once()
+
+
+def test_deploy_status(mocker):
+    """
+    Unit test for `covalent deploy status [executor_name]` command.
+    """
+
+    from covalent_dispatcher._cli.groups.deploy_group import status
+
+    # Succeed with empty `executor_names` argument.
+    ctx = click.Context(status)
+    ctx.invoke(status, executor_names=[])
+
+    # Succeed with invalid `executor_names` argument.
+    mock_click_style = mocker.patch("click.style")
+
+    ctx = click.Context(status)
+    ctx.invoke(status, executor_names=["invalid"])
+
+    mock_click_style.asert_called_once()
+
+    # Succeed with 'valid' `executor_names` argument.
+    mocker.patch(
+        "covalent_dispatcher._cli.groups.deploy_group.get_crm_object",
+    )
+    mocker.patch(
+        "covalent.cloud_resource_manager.core.CloudResourceManager.status",
+        return_value="down",
+    )
+
+    mock_console_print = mocker.patch("rich.console.Console.print")
+
+    ctx = click.Context(status)
+    ctx.invoke(status, executor_names=["awsbatch"])  # OK if not installed
+
+    mock_console_print.assert_called_once()

From ead6555f260553276ec003d5899dcf79213eac46 Mon Sep 17 00:00:00 2001
From: Ara Ghukasyan <ara@agnostiq.ai>
Date: Wed, 29 Nov 2023 17:24:39 -0500
Subject: [PATCH 31/33] fix typo

---
 tests/covalent_dispatcher_tests/_cli/cli_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/covalent_dispatcher_tests/_cli/cli_test.py b/tests/covalent_dispatcher_tests/_cli/cli_test.py
index 65001fe4f..46d056b1f 100644
--- a/tests/covalent_dispatcher_tests/_cli/cli_test.py
+++ b/tests/covalent_dispatcher_tests/_cli/cli_test.py
@@ -253,7 +253,7 @@ def test_deploy_status(mocker):
     ctx = click.Context(status)
     ctx.invoke(status, executor_names=["invalid"])
 
-    mock_click_style.asert_called_once()
+    mock_click_style.assert_called_once()
 
     # Succeed with 'valid' `executor_names` argument.
     mocker.patch(

From 8f694484f7503d4fd48566b92af11d7ab9c08fc7 Mon Sep 17 00:00:00 2001
From: Ara Ghukasyan <ara@agnostiq.ai>
Date: Wed, 29 Nov 2023 18:00:03 -0500
Subject: [PATCH 32/33] fix paths since rename

---
 tests/covalent_dispatcher_tests/_cli/groups/db_test.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tests/covalent_dispatcher_tests/_cli/groups/db_test.py b/tests/covalent_dispatcher_tests/_cli/groups/db_test.py
index 8830d89a5..a5b17bdd4 100644
--- a/tests/covalent_dispatcher_tests/_cli/groups/db_test.py
+++ b/tests/covalent_dispatcher_tests/_cli/groups/db_test.py
@@ -46,7 +46,9 @@ def test_alembic_command_args(mocker):
     MOCK_ALEMBIC_ARGS_INVALID = "some alembic command --with-flags -m 'comment'"
     ALEMBIC_ERROR_STDERR = b"alembic: error: invalid..."
     ALEMBIC_ERROR_STDOUT = b"b60c5 (head)"
-    popen_mock = mocker.patch.object(sys.modules["covalent_dispatcher._cli.groups.db"], "Popen")
+    popen_mock = mocker.patch.object(
+        sys.modules["covalent_dispatcher._cli.groups.db_group"], "Popen"
+    )
     # test valid alembic args
     popen_mock().communicate.return_value = (ALEMBIC_ERROR_STDOUT, b"")
     res = runner.invoke(alembic, MOCK_ALEMBIC_ARGS_VALID, catch_exceptions=False)
@@ -61,7 +63,9 @@ def test_alembic_not_installed_error(mocker):
     runner = CliRunner()
     MOCK_ALEMBIC_ARGS = "current"
     EXCEPTION_MESSAGE = "[Errno 2] No such file or directory: 'alembic'"
-    popen_mock = mocker.patch.object(sys.modules["covalent_dispatcher._cli.groups.db"], "Popen")
+    popen_mock = mocker.patch.object(
+        sys.modules["covalent_dispatcher._cli.groups.db_group"], "Popen"
+    )
     popen_mock.side_effect = FileNotFoundError(EXCEPTION_MESSAGE)
     res = runner.invoke(alembic, MOCK_ALEMBIC_ARGS, catch_exceptions=False)
     assert EXCEPTION_MESSAGE in res.output

From 5b27d5da4150c9a5845818226a6debce55ea6c9b Mon Sep 17 00:00:00 2001
From: Ara Ghukasyan <ara@agnostiq.ai>
Date: Wed, 29 Nov 2023 22:32:49 -0500
Subject: [PATCH 33/33] use importlib for right path of -e install (awsbatch
 issue)

---
 covalent_dispatcher/_cli/groups/deploy_group.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/covalent_dispatcher/_cli/groups/deploy_group.py b/covalent_dispatcher/_cli/groups/deploy_group.py
index 55537b4b0..a9510c09f 100644
--- a/covalent_dispatcher/_cli/groups/deploy_group.py
+++ b/covalent_dispatcher/_cli/groups/deploy_group.py
@@ -20,6 +20,7 @@
 import subprocess
 import sys
 from functools import partial
+from importlib import import_module
 from pathlib import Path
 from typing import Callable, Dict, Tuple
 
@@ -44,9 +45,8 @@ def get_crm_object(executor_name: str, options: Dict = None) -> CloudResourceMan
         CloudResourceManager object.
 
     """
-    executor_module_path = Path(
-        __import__(_executor_manager.executor_plugins_map[executor_name].__module__).__path__[0]
-    )
+    executor_plugin = _executor_manager.executor_plugins_map[executor_name]
+    executor_module_path = Path(import_module(executor_plugin.__module__).__file__).parent
     return CloudResourceManager(executor_name, executor_module_path, options)