feat(python): Multiple runtime versions

.github/workflows/docker-image.yml

@@ -453,6 +453,7 @@ jobs:
     runs-on: ubicloud
     if:
       (github.event_name != 'pull_request') && (github.event_name != 'workflow_dispatch')
+
     steps:
       - uses: actions/checkout@v4
         with:

Dockerfile

@@ -95,6 +95,14 @@ ARG WITH_KUBECTL=true
 ARG WITH_HELM=true
 ARG WITH_GIT=true
 
+# To change latest stable version:
+# 1. Change placeholder in instanceSettings.ts
+# 2. Change LATEST_STABLE_PY in dockerfile
+# 3. Change #[default] annotation for PyVersion in backend
+ARG LATEST_STABLE_PY=3.11.10
+ENV UV_PYTHON_INSTALL_DIR=/tmp/windmill/cache/py_runtime
+ENV UV_PYTHON_PREFERENCE=only-managed
+
 RUN pip install --upgrade pip==24.2
 
 RUN apt-get update \
@@ -161,6 +169,10 @@ ENV GO_PATH=/usr/local/go/bin/go
 # Install UV
 RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.5.15/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv
 
+# Preinstall python runtimes
+RUN uv python install 3.11.10
+RUN uv python install $LATEST_STABLE_PY
+
 RUN curl -sL https://deb.nodesource.com/setup_20.x | bash - 
 RUN apt-get -y update && apt-get install -y curl procps nodejs awscli && apt-get clean \
     && rm -rf /var/lib/apt/lists/*

backend/parsers/windmill-parser-py-imports/src/lib.rs

@@ -21,7 +21,7 @@ use rustpython_parser::{
     Parse,
 };
 use sqlx::{Pool, Postgres};
-use windmill_common::error;
+use windmill_common::{error, worker::PythonAnnotations};
 
 const DEF_MAIN: &str = "def main(";
 
@@ -171,14 +171,75 @@ fn parse_code_for_imports(code: &str, path: &str) -> error::Result<Vec<String>>
     return Ok(nimports);
 }
 
-#[async_recursion]
 pub async fn parse_python_imports(
     code: &str,
     w_id: &str,
     path: &str,
     db: &Pool<Postgres>,
     already_visited: &mut Vec<String>,
+    annotated_pyv_numeric: &mut Option<u32>,
+) -> error::Result<Vec<String>> {
+    parse_python_imports_inner(
+        code,
+        w_id,
+        path,
+        db,
+        already_visited,
+        annotated_pyv_numeric,
+        &mut annotated_pyv_numeric.and_then(|_| Some(path.to_owned())),
+    )
+    .await
+}
+
+#[async_recursion]
+async fn parse_python_imports_inner(
+    code: &str,
+    w_id: &str,
+    path: &str,
+    db: &Pool<Postgres>,
+    already_visited: &mut Vec<String>,
+    annotated_pyv_numeric: &mut Option<u32>,
+    path_where_annotated_pyv: &mut Option<String>,
 ) -> error::Result<Vec<String>> {
+    let PythonAnnotations { py310, py311, py312, py313, .. } = PythonAnnotations::parse(&code);
+
+    // we pass only if there is none or only one annotation
+
+    // Naive:
+    // 1. Check if there are multiple annotated version
+    // 2. If no, take one and compare with annotated version
+    // 3. We continue if same or replace none with new one
+
+    // Optimized:
+    // 1. Iterate over all annotations compare each with annotated_pyv and replace on flight
+    // 2. If annotated_pyv is different version, throw and error
+
+    // This way we make sure there is no multiple annotations for same script
+    // and we get detailed span on conflicting versions
+
+    let mut check = |is_py_xyz, numeric| -> error::Result<()> {
+        if is_py_xyz {
+            if let Some(v) = annotated_pyv_numeric {
+                if *v != numeric {
+                    return Err(error::Error::from(anyhow::anyhow!(
+                        "Annotated 2 or more different python versions: \n - py{v} at {}\n - py{numeric} at {path}\nIt is possible to use only one.",
+                        path_where_annotated_pyv.clone().unwrap_or("Unknown".to_owned())
+                    )));
+                }
+            } else {
+                *annotated_pyv_numeric = Some(numeric);
+            }
+
+            *path_where_annotated_pyv = Some(path.to_owned());
+        }
+        Ok(())
+    };
+
+    check(py310, 310)?;
+    check(py311, 311)?;
+    check(py312, 312)?;
+    check(py313, 313)?;
+
     let find_requirements = code
         .lines()
         .find_position(|x| x.starts_with("#requirements:") || x.starts_with("# requirements:"));
@@ -225,11 +286,21 @@ pub async fn parse_python_imports(
                 .fetch_optional(db)
                 .await?
                 .unwrap_or_else(|| "".to_string());
+
                 if already_visited.contains(&rpath) {
                     vec![]
                 } else {
                     already_visited.push(rpath.clone());
-                    parse_python_imports(&code, w_id, &rpath, db, already_visited).await?
+                    parse_python_imports_inner(
+                        &code,
+                        w_id,
+                        &rpath,
+                        db,
+                        already_visited,
+                        annotated_pyv_numeric,
+                        path_where_annotated_pyv,
+                    )
+                    .await?
                 }
             } else {
                 vec![replace_import(n.to_string())]

backend/parsers/windmill-parser-py-imports/tests/tests.rs

@@ -25,6 +25,7 @@ def main():
             "f/foo/bar",
             &db,
             &mut already_visited,
+            &mut None,
         )
         .await?;
         // println!("{}", serde_json::to_string(&r)?);
@@ -57,6 +58,7 @@ def main():
             "f/foo/bar",
             &db,
             &mut already_visited,
+            &mut None,
         )
         .await?;
         println!("{}", serde_json::to_string(&r)?);
@@ -87,6 +89,7 @@ def main():
             "f/foo/bar",
             &db,
             &mut already_visited,
+            &mut None,
         )
         .await?;
         println!("{}", serde_json::to_string(&r)?);

backend/src/main.rs

@@ -9,8 +9,9 @@
 use anyhow::Context;
 use monitor::{
     load_base_url, load_otel, reload_delete_logs_periodically_setting, reload_indexer_config,
-    reload_nuget_config_setting, reload_timeout_wait_result_setting,
-    send_current_log_file_to_object_store, send_logs_to_object_store,
+    reload_instance_python_version_setting, reload_nuget_config_setting,
+    reload_timeout_wait_result_setting, send_current_log_file_to_object_store,
+    send_logs_to_object_store,
 };
 use rand::Rng;
 use sqlx::{postgres::PgListener, Pool, Postgres};
@@ -35,12 +36,12 @@ use windmill_common::{
         CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING,
         DEFAULT_TAGS_WORKSPACES_SETTING, ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING,
         EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING,
-        JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING,
-        LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING,
-        NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING, PIP_INDEX_URL_SETTING,
-        REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING,
-        RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING,
-        TIMEOUT_WAIT_RESULT_SETTING,
+        INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING,
+        KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING,
+        NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING,
+        PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
+        REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
+        SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
     },
     scripts::ScriptLang,
     stats_ee::schedule_stats,
@@ -69,8 +70,9 @@ use windmill_worker::{
     get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR,
     BUN_DEPSTAR_CACHE_DIR, CSHARP_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS,
     DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR, LOCK_CACHE_DIR, PIP_CACHE_DIR,
-    POWERSHELL_CACHE_DIR, PY311_CACHE_DIR, RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TAR_PY311_CACHE_DIR,
-    TMP_LOGS_DIR, UV_CACHE_DIR,
+    POWERSHELL_CACHE_DIR, PY310_CACHE_DIR, PY311_CACHE_DIR, PY312_CACHE_DIR, PY313_CACHE_DIR,
+    RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TAR_PY310_CACHE_DIR, TAR_PY311_CACHE_DIR,
+    TAR_PY312_CACHE_DIR, TAR_PY313_CACHE_DIR, TMP_LOGS_DIR, UV_CACHE_DIR,
 };
 
 use crate::monitor::{
@@ -762,6 +764,9 @@ Windmill Community Edition {GIT_VERSION}
                                                 PIP_INDEX_URL_SETTING => {
                                                     reload_pip_index_url_setting(&db).await
                                                 },
+                                                INSTANCE_PYTHON_VERSION_SETTING => {
+                                                    reload_instance_python_version_setting(&db).await
+                                                },
                                                 NPM_CONFIG_REGISTRY_SETTING => {
                                                     reload_npm_config_registry_setting(&db).await
                                                 },
@@ -1013,12 +1018,18 @@ pub async fn run_workers(
         TMP_LOGS_DIR,
         UV_CACHE_DIR,
         TAR_PIP_CACHE_DIR,
-        TAR_PY311_CACHE_DIR,
         DENO_CACHE_DIR,
         DENO_CACHE_DIR_DEPS,
         DENO_CACHE_DIR_NPM,
         BUN_CACHE_DIR,
+        PY310_CACHE_DIR,
         PY311_CACHE_DIR,
+        PY312_CACHE_DIR,
+        PY313_CACHE_DIR,
+        TAR_PY310_CACHE_DIR,
+        TAR_PY311_CACHE_DIR,
+        TAR_PY312_CACHE_DIR,
+        TAR_PY313_CACHE_DIR,
         PIP_CACHE_DIR,
         BUN_DEPSTAR_CACHE_DIR,
         BUN_BUNDLE_CACHE_DIR,

backend/src/monitor.rs

@@ -41,10 +41,10 @@ use windmill_common::{
         BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING,
         CRITICAL_ERROR_CHANNELS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING,
         DEFAULT_TAGS_WORKSPACES_SETTING, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING,
-        EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING,
-        JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING,
-        MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING,
-        OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
+        EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING,
+        JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING,
+        LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING,
+        NUGET_CONFIG_SETTING, OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
         REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
         SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
     },
@@ -68,8 +68,8 @@ use windmill_common::{
 use windmill_queue::cancel_job;
 use windmill_worker::{
     create_token_for_owner, handle_job_error, AuthedClient, SameWorkerPayload, SameWorkerSender,
-    SendResult, BUNFIG_INSTALL_SCOPES, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, NPM_CONFIG_REGISTRY,
-    NUGET_CONFIG, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, SCRIPT_TOKEN_EXPIRY,
+    SendResult, BUNFIG_INSTALL_SCOPES, INSTANCE_PYTHON_VERSION, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR,
+    NPM_CONFIG_REGISTRY, NUGET_CONFIG, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, SCRIPT_TOKEN_EXPIRY,
 };
 
 #[cfg(feature = "parquet")]
@@ -198,6 +198,7 @@ pub async fn initial_load(
         reload_pip_index_url_setting(&db).await;
         reload_npm_config_registry_setting(&db).await;
         reload_bunfig_install_scopes_setting(&db).await;
+        reload_instance_python_version_setting(&db).await;
         reload_nuget_config_setting(&db).await;
     }
 }
@@ -908,6 +909,16 @@ pub async fn reload_pip_index_url_setting(db: &DB) {
     .await;
 }
 
+pub async fn reload_instance_python_version_setting(db: &DB) {
+    reload_option_setting_with_tracing(
+        db,
+        INSTANCE_PYTHON_VERSION_SETTING,
+        "INSTANCE_PYTHON_VERSION",
+        INSTANCE_PYTHON_VERSION.clone(),
+    )
+    .await;
+}
+
 pub async fn reload_npm_config_registry_setting(db: &DB) {
     reload_option_setting_with_tracing(
         db,

backend/windmill-common/src/error.rs

@@ -66,6 +66,8 @@ pub enum Error {
     AiError(String),
     #[error("{0}")]
     AlreadyCompleted(String),
+    #[error("Find python error: {0}")]
+    FindPythonError(String),
     #[error("{0}")]
     Utf8(#[from] std::string::FromUtf8Error),
     #[error("Encoding/decoding error: {0}")]

backend/windmill-common/src/global_settings.rs

@@ -14,6 +14,7 @@ pub const NUGET_CONFIG_SETTING: &str = "nuget_config";
 
 pub const EXTRA_PIP_INDEX_URL_SETTING: &str = "pip_extra_index_url";
 pub const PIP_INDEX_URL_SETTING: &str = "pip_index_url";
+pub const INSTANCE_PYTHON_VERSION_SETTING: &str = "instance_python_version";
 pub const SCIM_TOKEN_SETTING: &str = "scim_token";
 pub const SAML_METADATA_SETTING: &str = "saml_metadata";
 pub const SMTP_SETTING: &str = "smtp_settings";
@@ -38,7 +39,7 @@ pub const JWT_SECRET_SETTING: &str = "jwt_secret";
 pub const EMAIL_DOMAIN_SETTING: &str = "email_domain";
 pub const OTEL_SETTING: &str = "otel";
 
-pub const ENV_SETTINGS: [&str; 54] = [
+pub const ENV_SETTINGS: [&str; 55] = [
     "DISABLE_NSJAIL",
     "MODE",
     "NUM_WORKERS",
@@ -61,6 +62,7 @@ pub const ENV_SETTINGS: [&str; 54] = [
     "GOPRIVATE",
     "GOPROXY",
     "NETRC",
+    "INSTANCE_PYTHON_VERSION",
     "PIP_INDEX_URL",
     "PIP_EXTRA_INDEX_URL",
     "PIP_TRUSTED_HOST",

backend/windmill-common/src/worker.rs

@@ -333,14 +333,18 @@ fn parse_file<T: FromStr>(path: &str) -> Option<T> {
         .flatten()
 }
 
+#[derive(Copy, Clone)]
 #[annotations("#")]
 pub struct PythonAnnotations {
     pub no_cache: bool,
     pub no_uv: bool,
     pub no_uv_install: bool,
     pub no_uv_compile: bool,
-
     pub no_postinstall: bool,
+    pub py310: bool,
+    pub py311: bool,
+    pub py312: bool,
+    pub py313: bool,
 }
 
 #[annotations("//")]

backend/windmill-worker/nsjail/download.py.config.proto

@@ -20,6 +20,7 @@ clone_newuser: {CLONE_NEWUSER}
 
 keep_caps: true
 keep_env: true
+mount_proc: true
 
 mount {
     src: "/bin"
@@ -79,6 +80,11 @@ mount {
     is_bind: true
     rw: true
 }
+mount {
+    src: "{PY_INSTALL_DIR}"
+    dst: "{PY_INSTALL_DIR}"
+    is_bind: true
+}
 
 mount {
     src: "/dev/urandom"

backend/windmill-worker/nsjail/download_deps.py.sh

@@ -27,6 +27,7 @@ CMD="/usr/local/bin/uv pip install
 --no-color 
 --no-deps 
 --link-mode=copy
+$PY_PATH 
 $INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG
 --index-strategy unsafe-best-match
 --system

backend/windmill-worker/nsjail/run.python3.config.proto

@@ -16,6 +16,7 @@ clone_newuser: {CLONE_NEWUSER}
 
 keep_caps: false
 keep_env: true
+mount_proc: true
 
 mount {
     src: "/bin"
@@ -110,6 +111,12 @@ mount {
     is_bind: true
 }
 
+mount {
+    src: "{PY_INSTALL_DIR}"
+    dst: "{PY_INSTALL_DIR}"
+    is_bind: true
+}
+
 mount {
     src: "/dev/urandom"
     dst: "/dev/urandom"