diff --git a/lib/galaxy/tool_util/cwl/parser.py b/lib/galaxy/tool_util/cwl/parser.py
index e4a4ff83fb20..cc0b9944483a 100644
--- a/lib/galaxy/tool_util/cwl/parser.py
+++ b/lib/galaxy/tool_util/cwl/parser.py
@@ -219,6 +219,9 @@ def software_requirements(self) -> List:
def resource_requirements(self) -> List:
return self.hints_or_requirements_of_class("ResourceRequirement")
+ def credentials(self) -> List:
+ return self.hints_or_requirements_of_class("Credentials")
+
class CommandLineToolProxy(ToolProxy):
_class = "CommandLineTool"
diff --git a/lib/galaxy/tool_util/deps/requirements.py b/lib/galaxy/tool_util/deps/requirements.py
index bfb02e0ea606..fba3ecb4830c 100644
--- a/lib/galaxy/tool_util/deps/requirements.py
+++ b/lib/galaxy/tool_util/deps/requirements.py
@@ -20,6 +20,7 @@
from galaxy.util import (
asbool,
+ string_as_bool,
xml_text,
)
from galaxy.util.oset import OrderedSet
@@ -305,27 +306,125 @@ def resource_requirements_from_list(requirements: Iterable[Dict[str, Any]]) -> L
return rr
+class SecretOrVariable:
+ def __init__(
+ self,
+ type: str,
+ name: str,
+ inject_as_env: str,
+ label: str = "",
+ description: str = "",
+ ) -> None:
+ self.type = type
+ self.name = name
+ self.inject_as_env = inject_as_env
+ self.label = label
+ self.description = description
+ if self.type not in {"secret", "variable"}:
+ raise ValueError(f"Invalid credential type '{self.type}'")
+ if not self.inject_as_env:
+ raise ValueError("Missing inject_as_env")
+
+ def to_dict(self) -> Dict[str, Any]:
+ return {
+ "type": self.type,
+ "name": self.name,
+ "inject_as_env": self.inject_as_env,
+ "label": self.label,
+ "description": self.description,
+ }
+
+ @classmethod
+ def from_element(cls, elem) -> "SecretOrVariable":
+ return cls(
+ type=elem.tag,
+ name=elem.get("name"),
+ inject_as_env=elem.get("inject_as_env"),
+ label=elem.get("label", ""),
+ description=elem.get("description", ""),
+ )
+
+ @classmethod
+ def from_dict(cls, dict: Dict[str, Any]) -> "SecretOrVariable":
+ type = dict["type"]
+ name = dict["name"]
+ inject_as_env = dict["inject_as_env"]
+ label = dict.get("label", "")
+ description = dict.get("description", "")
+ return cls(type=type, name=name, inject_as_env=inject_as_env, label=label, description=description)
+
+
+class CredentialsRequirement:
+ def __init__(
+ self,
+ name: str,
+ reference: str,
+ required: bool = False,
+ label: str = "",
+ description: str = "",
+ secrets_and_variables: Optional[List[SecretOrVariable]] = None,
+ ) -> None:
+ self.name = name
+ self.reference = reference
+ self.required = required
+ self.label = label
+ self.description = description
+ self.secrets_and_variables = secrets_and_variables if secrets_and_variables is not None else []
+
+ if not self.reference:
+ raise ValueError("Missing reference")
+
+ def to_dict(self) -> Dict[str, Any]:
+ return {
+ "name": self.name,
+ "reference": self.reference,
+ "required": self.required,
+ "label": self.label,
+ "description": self.description,
+ "secrets_and_variables": [s.to_dict() for s in self.secrets_and_variables],
+ }
+
+ @classmethod
+ def from_dict(cls, dict: Dict[str, Any]) -> "CredentialsRequirement":
+ name = dict["name"]
+ reference = dict["reference"]
+ required = dict.get("required", False)
+ label = dict.get("label", "")
+ description = dict.get("description", "")
+ secrets_and_variables = [SecretOrVariable.from_dict(s) for s in dict.get("secrets_and_variables", [])]
+ return cls(
+ name=name,
+ reference=reference,
+ required=required,
+ label=label,
+ description=description,
+ secrets_and_variables=secrets_and_variables,
+ )
+
+
def parse_requirements_from_lists(
software_requirements: List[Union[ToolRequirement, Dict[str, Any]]],
containers: Iterable[Dict[str, Any]],
resource_requirements: Iterable[Dict[str, Any]],
-) -> Tuple[ToolRequirements, List[ContainerDescription], List[ResourceRequirement]]:
+ credentials: Iterable[Dict[str, Any]],
+) -> Tuple[ToolRequirements, List[ContainerDescription], List[ResourceRequirement], List[CredentialsRequirement]]:
return (
ToolRequirements.from_list(software_requirements),
[ContainerDescription.from_dict(c) for c in containers],
resource_requirements_from_list(resource_requirements),
+ [CredentialsRequirement.from_dict(s) for s in credentials],
)
-def parse_requirements_from_xml(xml_root, parse_resources: bool = False):
+def parse_requirements_from_xml(xml_root, parse_resources_and_secrets: bool = False):
"""
Parses requirements, containers and optionally resource requirements from Xml tree.
>>> from galaxy.util import parse_xml_string
- >>> def load_requirements(contents, parse_resources=False):
+ >>> def load_requirements(contents, parse_resources_and_secrets=False):
... contents_document = '''%s'''
... root = parse_xml_string(contents_document % contents)
- ... return parse_requirements_from_xml(root, parse_resources=parse_resources)
+ ... return parse_requirements_from_xml(root, parse_resources_and_secrets=parse_resources_and_secrets)
>>> reqs, containers = load_requirements('''bwa''')
>>> reqs[0].name
'bwa'
@@ -344,8 +443,10 @@ def parse_requirements_from_xml(xml_root, parse_resources: bool = False):
requirements_elem = xml_root.find("requirements")
requirement_elems = []
+ container_elems = []
if requirements_elem is not None:
requirement_elems = requirements_elem.findall("requirement")
+ container_elems = requirements_elem.findall("container")
requirements = ToolRequirements()
for requirement_elem in requirement_elems:
@@ -355,15 +456,13 @@ def parse_requirements_from_xml(xml_root, parse_resources: bool = False):
requirement = ToolRequirement(name=name, type=type, version=version)
requirements.append(requirement)
- container_elems = []
- if requirements_elem is not None:
- container_elems = requirements_elem.findall("container")
-
containers = [container_from_element(c) for c in container_elems]
- if parse_resources:
+ if parse_resources_and_secrets:
resource_elems = requirements_elem.findall("resource") if requirements_elem is not None else []
resources = [resource_from_element(r) for r in resource_elems]
- return requirements, containers, resources
+ credentials_elems = requirements_elem.findall("credentials") if requirements_elem is not None else []
+ credentials = [credentials_from_element(s) for s in credentials_elems]
+ return requirements, containers, resources, credentials
return requirements, containers
@@ -386,3 +485,20 @@ def container_from_element(container_elem) -> ContainerDescription:
shell=shell,
)
return container
+
+
+def credentials_from_element(credentials_elem) -> CredentialsRequirement:
+ name = credentials_elem.get("name")
+ reference = credentials_elem.get("reference")
+ required = string_as_bool(credentials_elem.get("required", "false"))
+ label = credentials_elem.get("label", "")
+ description = credentials_elem.get("description", "")
+ secrets_and_variables = [SecretOrVariable.from_element(elem) for elem in credentials_elem.findall("*")]
+ return CredentialsRequirement(
+ name=name,
+ reference=reference,
+ required=required,
+ label=label,
+ description=description,
+ secrets_and_variables=secrets_and_variables,
+ )
diff --git a/lib/galaxy/tool_util/linters/general.py b/lib/galaxy/tool_util/linters/general.py
index eb3a98fd3b82..ddc2c4e6621c 100644
--- a/lib/galaxy/tool_util/linters/general.py
+++ b/lib/galaxy/tool_util/linters/general.py
@@ -183,7 +183,7 @@ class RequirementNameMissing(Linter):
@classmethod
def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"):
_, tool_node = _tool_xml_and_root(tool_source)
- requirements, containers, resource_requirements = tool_source.parse_requirements_and_containers()
+ requirements, *_ = tool_source.parse_requirements_and_containers()
for r in requirements:
if r.type != "package":
continue
@@ -195,7 +195,7 @@ class RequirementVersionMissing(Linter):
@classmethod
def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"):
_, tool_node = _tool_xml_and_root(tool_source)
- requirements, containers, resource_requirements = tool_source.parse_requirements_and_containers()
+ requirements, *_ = tool_source.parse_requirements_and_containers()
for r in requirements:
if r.type != "package":
continue
@@ -207,7 +207,7 @@ class RequirementVersionWhitespace(Linter):
@classmethod
def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"):
_, tool_node = _tool_xml_and_root(tool_source)
- requirements, containers, resource_requirements = tool_source.parse_requirements_and_containers()
+ requirements, *_ = tool_source.parse_requirements_and_containers()
for r in requirements:
if r.type != "package":
continue
@@ -223,7 +223,7 @@ class ResourceRequirementExpression(Linter):
@classmethod
def lint(cls, tool_source: "ToolSource", lint_ctx: "LintContext"):
_, tool_node = _tool_xml_and_root(tool_source)
- requirements, containers, resource_requirements = tool_source.parse_requirements_and_containers()
+ *_, resource_requirements, _ = tool_source.parse_requirements_and_containers()
for rr in resource_requirements:
if rr.runtime_required:
lint_ctx.warn(
diff --git a/lib/galaxy/tool_util/parser/cwl.py b/lib/galaxy/tool_util/parser/cwl.py
index 26d2fdfdc16c..540f8edcfbdd 100644
--- a/lib/galaxy/tool_util/parser/cwl.py
+++ b/lib/galaxy/tool_util/parser/cwl.py
@@ -162,10 +162,12 @@ def parse_requirements_and_containers(self):
software_requirements = self.tool_proxy.software_requirements()
resource_requirements = self.tool_proxy.resource_requirements()
+ credentials = self.tool_proxy.credentials()
return requirements.parse_requirements_from_lists(
software_requirements=[{"name": r[0], "version": r[1], "type": "package"} for r in software_requirements],
containers=containers,
resource_requirements=resource_requirements,
+ credentials=credentials,
)
def parse_profile(self):
diff --git a/lib/galaxy/tool_util/parser/interface.py b/lib/galaxy/tool_util/parser/interface.py
index af72bf4a4825..2c59751f2ca3 100644
--- a/lib/galaxy/tool_util/parser/interface.py
+++ b/lib/galaxy/tool_util/parser/interface.py
@@ -34,6 +34,7 @@
if TYPE_CHECKING:
from galaxy.tool_util.deps.requirements import (
ContainerDescription,
+ CredentialsRequirement,
ResourceRequirement,
ToolRequirements,
)
@@ -307,8 +308,10 @@ def parse_required_files(self) -> Optional["RequiredFiles"]:
@abstractmethod
def parse_requirements_and_containers(
self,
- ) -> Tuple["ToolRequirements", List["ContainerDescription"], List["ResourceRequirement"]]:
- """Return triple of ToolRequirement, ContainerDescription and ResourceRequirement lists."""
+ ) -> Tuple[
+ "ToolRequirements", List["ContainerDescription"], List["ResourceRequirement"], List["CredentialsRequirement"]
+ ]:
+ """Return triple of ToolRequirement, ContainerDescription, ResourceRequirement, and SecretsRequirement objects."""
@abstractmethod
def parse_input_pages(self) -> "PagesSource":
diff --git a/lib/galaxy/tool_util/parser/xml.py b/lib/galaxy/tool_util/parser/xml.py
index aba5a6874248..ef7716d59716 100644
--- a/lib/galaxy/tool_util/parser/xml.py
+++ b/lib/galaxy/tool_util/parser/xml.py
@@ -413,7 +413,7 @@ def parse_include_exclude_list(tag_name):
return RequiredFiles.from_dict(as_dict)
def parse_requirements_and_containers(self):
- return requirements.parse_requirements_from_xml(self.root, parse_resources=True)
+ return requirements.parse_requirements_from_xml(self.root, parse_resources_and_secrets=True)
def parse_input_pages(self) -> "XmlPagesSource":
return XmlPagesSource(self.root)
diff --git a/lib/galaxy/tool_util/parser/yaml.py b/lib/galaxy/tool_util/parser/yaml.py
index 88be9c72846a..673e5a2cb418 100644
--- a/lib/galaxy/tool_util/parser/yaml.py
+++ b/lib/galaxy/tool_util/parser/yaml.py
@@ -115,6 +115,7 @@ def parse_requirements_and_containers(self):
software_requirements=[r for r in mixed_requirements if r.get("type") != "resource"],
containers=self.root_dict.get("containers", []),
resource_requirements=[r for r in mixed_requirements if r.get("type") == "resource"],
+ credentials=self.root_dict.get("credentials", []),
)
def parse_input_pages(self) -> PagesSource:
diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd
index 1c23f88066fe..a30add76c57b 100644
--- a/lib/galaxy/tool_util/xsd/galaxy.xsd
+++ b/lib/galaxy/tool_util/xsd/galaxy.xsd
@@ -612,6 +612,7 @@ serve as complete descriptions of the runtime of a tool.
+
@@ -725,6 +726,113 @@ Read more about configuring Galaxy to run Docker jobs
+
+
+
+
+
+
+
+
+
+```
+]]>
+
+
+
+
+
+
+
+ The name of the credential set.
+
+
+
+
+ A reference to the source of the credentials.
+
+
+
+
+ The label of the credential set.
+
+
+
+
+ The description of the credential set.
+
+
+
+
+ Whether the credentials are required for the tool to run.
+
+
+
+
+
+
+
+
+
+ The name of the variable.
+
+
+
+
+ The environment variable name to inject the value as.
+
+
+
+
+ The label for the variable.
+
+
+
+
+ The description for the variable.
+
+
+
+
+
+
+
+
+
+ The name of the secret.
+
+
+
+
+ The environment variable name to inject the value as.
+
+
+
+
+ The label for the secret.
+
+
+
+
+ The description for the secret.
+
+
+
Document type of tool help
diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index 2e352e7c0f39..286283e4e0b4 100644
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -1216,10 +1216,18 @@ def parse(self, tool_source: ToolSource, guid: Optional[str] = None, dynamic: bo
raise Exception(message)
# Requirements (dependencies)
- requirements, containers, resource_requirements = tool_source.parse_requirements_and_containers()
+ requirements, containers, resource_requirements, credentials = tool_source.parse_requirements_and_containers()
self.requirements = requirements
self.containers = containers
self.resource_requirements = resource_requirements
+ self.credentials = credentials
+ # for credential in self.credentials:
+ # pass
+ # preferences = self.app.config.user_preferences_extra["preferences"]
+ # main_key, input_key = credential.user_preferences_key.split("/")
+ # preferences_input = preferences.get(main_key, {}).get("inputs", [])
+ # if not any(input_item.get("name") == input_key for input_item in preferences_input):
+ # raise exceptions.ConfigurationError(f"User preferences key {credential.user_preferences_key} not found")
required_files = tool_source.parse_required_files()
if required_files is None:
diff --git a/lib/galaxy/tools/evaluation.py b/lib/galaxy/tools/evaluation.py
index 582bd65be06e..bbb72308cc1e 100644
--- a/lib/galaxy/tools/evaluation.py
+++ b/lib/galaxy/tools/evaluation.py
@@ -6,7 +6,7 @@
import string
import tempfile
from datetime import datetime
-from typing import (
+from typing import ( # cast,
Any,
Callable,
Dict,
@@ -28,7 +28,9 @@
)
from galaxy.model.none_like import NoneDataset
from galaxy.security.object_wrapper import wrap_with_safe_string
-from galaxy.structured_app import (
+
+# from galaxy.security.vault import UserVaultWrapper
+from galaxy.structured_app import ( # StructuredApp,
BasicSharedApp,
MinimalToolApp,
)
@@ -188,6 +190,25 @@ def set_compute_environment(self, compute_environment: ComputeEnvironment, get_s
)
self.execute_tool_hooks(inp_data=inp_data, out_data=out_data, incoming=incoming)
+ if self.tool.credentials:
+ # app = cast(StructuredApp, self.app)
+ # user_vault = UserVaultWrapper(app.vault, self._user)
+ for credentials in self.tool.credentials:
+ reference = credentials.reference
+ for secret_or_variable in credentials.secrets_and_variables:
+ if secret_or_variable.type == "variable":
+ # variable_value = self.param_dict.get(f"{reference}/{secret_or_variable.name}")
+ variable_value = f"A variable: {reference}/{secret_or_variable.name}"
+ self.environment_variables.append(
+ {"name": secret_or_variable.inject_as_env, "value": variable_value}
+ )
+ elif secret_or_variable.type == "secret":
+ # secret_value = user_vault.read_secret(f"{reference}/{secret_or_variable.name}")
+ secret_value = f"A secret: {reference}/{secret_or_variable.name}"
+ self.environment_variables.append(
+ {"name": secret_or_variable.inject_as_env, "value": secret_value}
+ )
+
def execute_tool_hooks(self, inp_data, out_data, incoming):
# Certain tools require tasks to be completed prior to job execution
# ( this used to be performed in the "exec_before_job" hook, but hooks are deprecated ).
diff --git a/test/functional/tools/secret_tool.xml b/test/functional/tools/secret_tool.xml
new file mode 100644
index 000000000000..bbcf9b7f91a3
--- /dev/null
+++ b/test/functional/tools/secret_tool.xml
@@ -0,0 +1,15 @@
+
+
+
+
+ '$output'
+ ]]>
+
+
+
+
+
+
+
+
diff --git a/test/integration/test_vault_extra_prefs.py b/test/integration/test_vault_extra_prefs.py
index 5072ac69d270..64fbb7a6ea40 100644
--- a/test/integration/test_vault_extra_prefs.py
+++ b/test/integration/test_vault_extra_prefs.py
@@ -11,6 +11,11 @@
)
from galaxy.model.db.user import get_user_by_email
+from galaxy_test.api.test_tools import TestsTools
+from galaxy_test.base.populators import (
+ DatasetPopulator,
+ skip_without_tool,
+)
from galaxy_test.driver import integration_util
TEST_USER_EMAIL = "vault_test_user@bx.psu.edu"
@@ -134,3 +139,30 @@ def __url(self, action, user):
def _get_dbuser(self, app, user):
return get_user_by_email(app.model.session, user["email"])
+
+
+class TestSecretsInExtraUserPreferences(
+ integration_util.IntegrationTestCase, integration_util.ConfiguresDatabaseVault, TestsTools
+):
+ dataset_populator: DatasetPopulator
+
+ @classmethod
+ def handle_galaxy_config_kwds(cls, config):
+ super().handle_galaxy_config_kwds(config)
+ cls._configure_database_vault(config)
+ config["user_preferences_extra_conf_path"] = os.path.join(
+ os.path.dirname(__file__), "user_preferences_extra_conf.yml"
+ )
+
+ def setUp(self):
+ super().setUp()
+ self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
+
+ @skip_without_tool("secret_tool")
+ def test_secrets_tool(self, history_id):
+ user = self._setup_user(TEST_USER_EMAIL)
+ url = self._api_url(f"users/{user['id']}/information/inputs", params=dict(key=self.master_api_key))
+ put(url, data=json.dumps({"secret_tool|api_key": "test"}))
+ run_response = self._run("secret", history_id, assert_ok=True)
+ outputs = run_response["outputs"]
+ assert outputs[0]["extra_files"][0]["value"] == "test"
diff --git a/test/unit/tool_util/test_cwl.py b/test/unit/tool_util/test_cwl.py
index a3bae657b132..8e95793cc8a9 100644
--- a/test/unit/tool_util/test_cwl.py
+++ b/test/unit/tool_util/test_cwl.py
@@ -281,7 +281,7 @@ def test_load_proxy_simple():
outputs, output_collections = tool_source.parse_outputs(None)
assert len(outputs) == 1
- software_requirements, containers, resource_requirements = tool_source.parse_requirements_and_containers()
+ software_requirements, containers, resource_requirements, _ = tool_source.parse_requirements_and_containers()
assert software_requirements.to_dict() == []
assert len(containers) == 1
assert containers[0].to_dict() == {
diff --git a/test/unit/tool_util/test_parsing.py b/test/unit/tool_util/test_parsing.py
index 21ec92387694..6973606b7f75 100644
--- a/test/unit/tool_util/test_parsing.py
+++ b/test/unit/tool_util/test_parsing.py
@@ -347,7 +347,7 @@ def test_action(self):
assert self._tool_source.parse_action_module() is None
def test_requirements(self):
- requirements, containers, resource_requirements = self._tool_source.parse_requirements_and_containers()
+ requirements, containers, resource_requirements, _ = self._tool_source.parse_requirements_and_containers()
assert requirements[0].type == "package"
assert list(containers)[0].identifier == "mycool/bwa"
assert resource_requirements[0].resource_type == "cores_min"
@@ -533,7 +533,9 @@ def test_action(self):
assert self._tool_source.parse_action_module() is None
def test_requirements(self):
- software_requirements, containers, resource_requirements = self._tool_source.parse_requirements_and_containers()
+ software_requirements, containers, resource_requirements, _ = (
+ self._tool_source.parse_requirements_and_containers()
+ )
assert software_requirements.to_dict() == [{"name": "bwa", "type": "package", "version": "1.0.1", "specs": []}]
assert len(containers) == 1
assert containers[0].to_dict() == {