Feat/devenv

docs/changelog.md

@@ -1,5 +1,10 @@
 ## Unreleased
 
+- Update Action Server to `2.0.0`.
+- Update RCC to `v18.5.0`.
+- Using RCC directly to build the environment using `package.yaml` (instead of creating a `conda.yaml` first).
+- When dealing with a `package.yaml`, always consider the `dev-environment` when building the environment.
+
 ## New in 2.7.1 (2024-10-23)
 
 - Add `Import Action Package` command to sidebar UI.

sema4ai/bin/develop.bat

@@ -6,7 +6,7 @@ cd /D %scriptPath%
 SET venvDir=venv
 
 :: Get RCC, binary with which we're going to create the master environment.
-SET rccUrl=https://downloads.robocorp.com/rcc/releases/v18.1.5/windows64/rcc.exe
+SET rccUrl=https://downloads.robocorp.com/rcc/releases/v18.5.0/windows64/rcc.exe
 IF NOT EXIST ".\rcc.exe" (
     curl -o rcc.exe %rccUrl% --fail || goto venv_error
 )

sema4ai/bin/develop.sh

@@ -17,8 +17,8 @@ fi
 if [ ! -f rcc ]; then
     system=$(uname -s)
     case ${system} in
-        Linux*)     url=https://downloads.robocorp.com/rcc/releases/v18.1.5/linux64/rcc;;
-        Darwin*)    url=https://downloads.robocorp.com/rcc/releases/v18.1.5/macos64/rcc;;
+        Linux*)     url=https://downloads.robocorp.com/rcc/releases/v18.5.0/linux64/rcc;;
+        Darwin*)    url=https://downloads.robocorp.com/rcc/releases/v18.5.0/macos64/rcc;;
         *)          echo "Invalid platform '$system' detected!"; exit 1;;
     esac
     curl -o rcc $url

sema4ai/dev.py

@@ -58,6 +58,11 @@ def _fix_rcc_contents_version(contents, version):
         r"(RCC_VERSION\s*=\s*)\"v\d+\.\d+\.\d+", rf'\1"{version}', contents
     )
 
+    # Change `rcc/releases/v18.3.0/` to `rcc/releases/v18.5.0/`
+    contents = re.sub(
+        r"rcc/releases/v\d+\.\d+\.\d+/", rf"rcc/releases/{version}/", contents
+    )
+
     return contents
 
 
@@ -93,10 +98,12 @@ def update_version(version, filepath):
             new_contents = _fix_rcc_contents_version(contents, version)
             assert (
                 contents != new_contents
-            ), "Nothing changed after applying new version."
+            ), f"Nothing changed after applying new version (file: {filepath})"
             with open(filepath, "w") as stream:
                 stream.write(new_contents)
 
+        update_version(version, os.path.join(".", "bin", "develop.sh"))
+        update_version(version, os.path.join(".", "bin", "develop.bat"))
         update_version(version, os.path.join(".", "src", "sema4ai_code", "rcc.py"))
         update_version(version, os.path.join(".", "vscode-client", "src", "rcc.ts"))
 

sema4ai/src/sema4ai_code/action_server.py

@@ -28,7 +28,7 @@
 log = get_logger(__name__)
 
 ONE_MINUTE_S = 60
-ACTION_SERVER_VERSION = "1.1.2"
+ACTION_SERVER_VERSION = "2.0.0"
 
 if typing.TYPE_CHECKING:
     from sema4ai_code.vendored_deps.url_callback_server import LastRequestInfoTypedDict

sema4ai/src/sema4ai_code/holetree_manager.py

@@ -36,9 +36,9 @@
                     and should be reclaimed after a timeout.
 """
 
+from collections.abc import Iterable
 from pathlib import Path
 from typing import List, Optional
-from collections.abc import Iterable
 
 from sema4ai_ls_core.core_log import get_logger
 
@@ -140,11 +140,15 @@ def _compute_status(
                         space_info.curr_status = CurrentSpaceStatus.NOT_AVAILABLE
                         return space_info
 
-                    if space_info.conda_contents_match(conda_yaml_contents):
+                    if space_info.conda_contents_match(
+                        self._rcc, conda_yaml_contents, str(conda_yaml_path)
+                    ):
                         env_written = space_info.env_json_path.exists()
                         if (
                             env_written
-                            and space_info.conda_prefix_identity_yaml_still_matches_cached_space()
+                            and space_info.conda_prefix_identity_yaml_still_matches_cached_space(
+                                self._rcc
+                            )
                         ) or not env_written:
                             space_info.update_last_usage()
                             write_text(conda_path, str(conda_yaml_path), "utf-8")
@@ -260,7 +264,7 @@ def compute_valid_space_info(
                 can_reuse,
                 key=lambda status: (
                     not status.damaged_path.exists(),
-                    -status.last_usage,
+                    status.last_usage,
                 ),
             )
             for space_info in can_reuse:
@@ -278,7 +282,7 @@ def compute_valid_space_info(
                 can_reuse + not_available,
                 key=lambda status: (
                     not status.damaged_path.exists(),
-                    -status.last_usage,
+                    status.last_usage,
                 ),
             )
             for space_info in all_envs:

sema4ai/src/sema4ai_code/protocols.py

@@ -1,22 +1,15 @@
-import sys
+import typing
 from enum import Enum
 from pathlib import Path
-from typing import Any, ContextManager, Dict, List, Literal, Optional, Tuple, TypeVar
+from typing import Any, ContextManager, Literal, TypeVar
 
 # Backward-compatibility imports:
 from sema4ai_ls_core.protocols import ActionResult, ActionResultDict  # noqa
 
-# Hack so that we don't break the runtime on versions prior to Python 3.8.
-if sys.version_info[:2] < (3, 8):
+if typing.TYPE_CHECKING:
+    from sema4ai_ls_core.cache import LRUCache
 
-    class Protocol:
-        pass
-
-    class TypedDict:
-        pass
-
-else:
-    from typing import Protocol, TypedDict
+from typing import Protocol, TypedDict
 
 
 class PackageType(Enum):
@@ -392,10 +385,12 @@ def has_timeout_elapsed(self, timeout_to_reuse_space: float) -> bool:
     def acquire_lock(self) -> ContextManager:
         pass
 
-    def conda_contents_match(self, conda_yaml_contents: str) -> bool:
+    def conda_contents_match(
+        self, rcc: "IRcc", conda_yaml_contents: str, conda_yaml_path: str
+    ) -> bool:
         pass
 
-    def matches_conda_identity_yaml(self, conda_id: Path) -> bool:
+    def matches_conda_identity_yaml(self, rcc: "IRcc", conda_id: Path) -> bool:
         pass
 
 
@@ -552,3 +547,10 @@ def holotree_variables(
         self, robot_yaml: Path, space_name: str, no_build: bool
     ) -> ActionResult[str]:
         pass
+
+    def holotree_hash(
+        self,
+        conda_yaml_contents: str,
+        file_path: str,
+    ) -> ActionResult[str]:
+        pass

sema4ai/src/sema4ai_code/rcc.py

@@ -6,9 +6,10 @@
 from dataclasses import dataclass
 from pathlib import Path
 from subprocess import CalledProcessError, TimeoutExpired, list2cmdline
-from typing import Any, Dict, List, Optional, Set
+from typing import Any
 
 from sema4ai_ls_core.basic import as_str, implements
+from sema4ai_ls_core.cache import LRUCache
 from sema4ai_ls_core.constants import NULL
 from sema4ai_ls_core.core_log import get_log_level, get_logger
 from sema4ai_ls_core.process import is_process_alive
@@ -42,6 +43,8 @@
 
 ACCOUNT_NAME = "sema4ai"
 
+_cache_hash: LRUCache[tuple[str, str], ActionResult[str]] = LRUCache(max_size=50)
+
 
 def download_rcc(
     location: str,
@@ -60,7 +63,7 @@ def download_rcc(
     """
     from sema4ai_code.tools import download_tool
 
-    RCC_VERSION = "v18.1.5"
+    RCC_VERSION = "v18.5.0"
     download_tool(
         Tool.RCC,
         location,
@@ -228,6 +231,7 @@ def _run_rcc(
         stderr=Sentinel.SENTINEL,
         show_interactive_output: bool = False,
         hide_in_log: str | None = None,
+        send_to_stdin: str | None = None,
     ) -> LaunchActionResult[str]:
         """
         Returns an ActionResult where the result is the stdout of the executed command.
@@ -293,8 +297,16 @@ def _run_rcc(
                 # interactively while the command is running and another where
                 # we only print if some error happened.
                 if not show_interactive_output:
+                    if send_to_stdin is not None:
+                        kwargs["input"] = send_to_stdin.encode("utf-8")
+
                     boutput = check_output(args, timeout=timeout, **kwargs)
                 else:
+                    if send_to_stdin is not None:
+                        raise ValueError(
+                            "send_to_stdin cannot be provided when show_interactive_output is true (unsupported)"
+                        )
+
                     from sema4ai_ls_core.progress_report import (
                         get_current_progress_reporter,
                     )
@@ -829,11 +841,54 @@ def cloud_create_robot(
 
         return ActionResult(ret.success, None, package_id)
 
+    def holotree_hash(
+        self,
+        conda_yaml_contents: str,
+        file_path: str,
+    ) -> ActionResult[str]:
+        # If the yaml contents / basename are the same, we can reuse the result.
+        key = (conda_yaml_contents, os.path.basename(file_path))
+        if key in _cache_hash:
+            return _cache_hash[key]
+
+        args = [
+            "holotree",
+            "hash",
+            "--json",
+            "--lockless",
+            "--warranty-voided",
+            "--stdin",
+            file_path,
+        ]
+
+        if file_path.endswith("package.yaml"):
+            args.append("--devdeps")
+
+        run_rcc_result = self._run_rcc(
+            args,
+            send_to_stdin=conda_yaml_contents,
+        )
+        if not run_rcc_result.success:
+            _cache_hash[key] = run_rcc_result
+            return run_rcc_result
+
+        try:
+            assert run_rcc_result.result, "Expected result from holotree hash."
+            hash_result: str = json.loads(run_rcc_result.result)["hash"]
+        except Exception as e:
+            return ActionResult.make_failure(
+                f"Unable to load json/get hash from holotree hash contents: {run_rcc_result.result}\nError: {e}"
+            )
+
+        action_result = ActionResult.make_success(hash_result)
+        _cache_hash[key] = action_result
+        return action_result
+
     @implements(IRcc.get_robot_yaml_env_info)
     def get_robot_yaml_env_info(
         self,
-        robot_yaml_path: Path,
-        conda_yaml_path: Path,
+        robot_yaml_path: Path | None,
+        conda_or_package_yaml_path: Path,
         conda_yaml_contents: str,
         env_json_path: Path | None,
         timeout=None,
@@ -855,12 +910,12 @@ def get_robot_yaml_env_info(
         )
 
         if broken_action_result is not None:
-            msg = f"Environment from previously broken {conda_yaml_path.name} requested: {conda_yaml_path}.\n-- VSCode restart required to retry."
+            msg = f"Environment from previously broken {conda_or_package_yaml_path.name} requested: {conda_or_package_yaml_path}.\n-- VSCode restart required to retry."
             log.critical(msg)
             return ActionResult(False, msg, None)
 
         space_info: RCCSpaceInfo = holotree_manager.compute_valid_space_info(
-            conda_yaml_path, conda_yaml_contents
+            conda_or_package_yaml_path, conda_yaml_contents
         )
 
         environ: dict[str, str]
@@ -968,7 +1023,7 @@ def return_failure(msg: str | None) -> ActionResult[IRobotYamlEnvInfo]:
                     "To recreate the environment, please change the related conda yaml\n"
                     "or restart VSCode to retry with the same conda yaml contents."
                 ),
-                conda_yaml_path,
+                conda_or_package_yaml_path,
             )
 
             if not msg:
@@ -996,19 +1051,31 @@ def return_failure(msg: str | None) -> ActionResult[IRobotYamlEnvInfo]:
             "variables",
             "--space",
             space_info.space_name,
-            str(conda_yaml_path),
+            str(conda_or_package_yaml_path),
         ]
         args.append("--json")
         args.append("--no-retry-build")
         args.append("--no-pyc-management")
+
+        if (
+            robot_yaml_path is None
+            and conda_or_package_yaml_path.name == "package.yaml"
+        ):
+            # In VSCode we always want to use a dev environment.
+            args.append("--devdeps")
+
         try:
             sys.stderr.write(
-                f"Collecting environment info for {conda_yaml_path} in space: {space_info.space_name}\n"
+                f"Collecting environment info for {conda_or_package_yaml_path} in space: {space_info.space_name}\n"
             )
             ret = self._run_rcc(
                 args,
                 mutex_name=RCC_CLOUD_ROBOT_MUTEX_NAME,
-                cwd=str(robot_yaml_path.parent),
+                cwd=str(
+                    robot_yaml_path.parent
+                    if robot_yaml_path is not None
+                    else conda_or_package_yaml_path.parent
+                ),
                 timeout=timeout,  # Creating the env may be really slow!
                 show_interactive_output=True,
             )
@@ -1046,7 +1113,9 @@ def return_failure(msg: str | None) -> ActionResult[IRobotYamlEnvInfo]:
                     pass
                 space_info.update_last_usage()
 
-            if not space_info.conda_prefix_identity_yaml_still_matches_cached_space():
+            if not space_info.conda_prefix_identity_yaml_still_matches_cached_space(
+                self
+            ):
                 return return_failure(
                     "Right after creating env, the environment does NOT match the conda yaml!"
                 )
@@ -1063,7 +1132,9 @@ def return_failure(msg: str | None) -> ActionResult[IRobotYamlEnvInfo]:
 
             return ActionResult(True, None, RobotInfoEnv(environ, space_info))
         except Exception as e:
-            log.exception("Error creating environment from: %s", conda_yaml_path)
+            log.exception(
+                "Error creating environment from: %s", conda_or_package_yaml_path
+            )
             return return_failure(str(e))
 
     @implements(IRcc.check_conda_installed)