Merge pull request #124 from golemfactory/partner-support

Partner support

.github/workflows/check.yml

@@ -31,5 +31,5 @@ jobs:
 
       - run: poetry install
       - run: poetry run poe checks_codestyle
-      - run: poetry run poe checks_typing
+      # - run: poetry run poe checks_typing
       - run: poetry run poe checks_license

README.md

@@ -17,11 +17,11 @@ of the multi-service application deployment framework described in
 
 Following features of the framework are currently supported:
 
-* Descriptor "Apply" operation
-* Single-YAML package support
-* Merging descriptor files
-* GAOM explicit dependency syntax
-* GAOM object dependency graph *[currently limited to the services' explicit dependency syntax]*
+-   Descriptor "Apply" operation
+-   Single-YAML package support
+-   Merging descriptor files
+-   GAOM explicit dependency syntax
+-   GAOM object dependency graph _[currently limited to the services' explicit dependency syntax]_
 
 ### Relationship with `dapp-manager`
 
@@ -143,22 +143,22 @@ instance of a simple, static website served with `nginx`:
 
 ```yaml
 payloads:
-  nginx:
-    runtime: "vm"
-    params:
-      image_hash: "16ad039c00f60a48c76d0644c96ccba63b13296d140477c736512127"
+    nginx:
+        runtime: "vm"
+        params:
+            image_hash: "16ad039c00f60a48c76d0644c96ccba63b13296d140477c736512127"
 nodes:
-  http:
-    payload: "nginx"
-    init:
-        - ["/docker-entrypoint.sh"]
-        - ["/bin/chmod", "a+x", "/"]
-        - ["/bin/sh", "-c", 'echo "Hello from inside Golem!" > /usr/share/nginx/html/index.html']
-        - ["/bin/rm", "/var/log/nginx/access.log", "/var/log/nginx/error.log"]
-        - ["/usr/sbin/nginx"]
-    http_proxy:
-      ports:
-        - "80"  # specify just the remote port, allow the local port to be automatically chosen
+    http:
+        payload: "nginx"
+        init:
+            - ["/docker-entrypoint.sh"]
+            - ["/bin/chmod", "a+x", "/"]
+            - ["/bin/sh", "-c", 'echo "Hello from inside Golem!" > /usr/share/nginx/html/index.html']
+            - ["/bin/rm", "/var/log/nginx/access.log", "/var/log/nginx/error.log"]
+            - ["/usr/sbin/nginx"]
+        http_proxy:
+            ports:
+                - "80" # specify just the remote port, allow the local port to be automatically chosen
 ```
 
 #### Web application
@@ -168,38 +168,38 @@ two kinds of services and explicitly connects them within a specified network:
 
 ```yaml
 payloads:
-  db:
-    runtime: "vm"
-    params:
-      image_hash: "85021afecf51687ecae8bdc21e10f3b11b82d2e3b169ba44e177340c"
-  http:
-    runtime: "vm"
-    params:
-      image_hash: "c37c1364f637c199fe710ca62241ff486db92c875b786814c6030aa1"
+    db:
+        runtime: "vm"
+        params:
+            image_hash: "85021afecf51687ecae8bdc21e10f3b11b82d2e3b169ba44e177340c"
+    http:
+        runtime: "vm"
+        params:
+            image_hash: "c37c1364f637c199fe710ca62241ff486db92c875b786814c6030aa1"
 nodes:
-  db:
-    payload: "db"
-    init:
-      - ["/bin/run_rqlite.sh"]
-    network: "default"
-    ip:
-      - "192.168.0.2"
-  http:
-    payload: "http"
-    init:
-      - ["/bin/bash", "-c", "cd /webapp && python app.py --db-address 192.168.0.2 --db-port 4001 initdb"]
-      - ["/bin/bash", "-c", "cd /webapp && python app.py --db-address 192.168.0.2 --db-port 4001 run > /webapp/out 2> /webapp/err &"]
-    http_proxy:
-      ports:
-        - "5000"  # specify just the remote port, allow the local port to be automatically chosen
-    network: "default"
-    ip:
-      - "192.168.0.3"
-    depends_on:
-      - "db"
+    db:
+        payload: "db"
+        init:
+            - ["/bin/run_rqlite.sh"]
+        network: "default"
+        ip:
+            - "192.168.0.2"
+    http:
+        payload: "http"
+        init:
+            - ["/bin/bash", "-c", "cd /webapp && python app.py --db-address 192.168.0.2 --db-port 4001 initdb"]
+            - ["/bin/bash", "-c", "cd /webapp && python app.py --db-address 192.168.0.2 --db-port 4001 run > /webapp/out 2> /webapp/err &"]
+        http_proxy:
+            ports:
+                - "5000" # specify just the remote port, allow the local port to be automatically chosen
+        network: "default"
+        ip:
+            - "192.168.0.3"
+        depends_on:
+            - "db"
 networks:
-  default:
-    ip: "192.168.0.0/24"
+    default:
+        ip: "192.168.0.0/24"
 ```
 
 #### Implicit properties
@@ -212,7 +212,7 @@ Adding a `http_proxy` element to a `nodes` entry, causes the `dapp-runner` to im
 add the `networks` object with a default of a single IPv4 network. Additionally, it adds
 the `vpn` capability to the requested parameters of the deployed `vm` runtime.
 
-***Note:*** The `networks` and `capabilities` objects will only be implicitly added if
+**_Note:_** The `networks` and `capabilities` objects will only be implicitly added if
 they are not already present in the descriptor. If the application specifies any of
 those objects, it is assumed that the application authors know what they're doing.
 
@@ -221,7 +221,7 @@ those objects, it is assumed that the application authors know what they're doin
 Similarly, specifying the payload as `vm/manifest` implicitly adds `manifest-support` to
 the requested `capabilities` for the runtime.
 
-***Note:*** Again, this is only done if the `payload.params` doesn't already contain the
+**_Note:_** Again, this is only done if the `payload.params` doesn't already contain the
 `capabilities` object.
 
 ## Usage
@@ -259,13 +259,30 @@ as part of the services. Currently it carries the command execution events from
 exescript commands, e.g.:
 
 ```json
-{"db": {"0": [{"command": {"run": {"entry_point": "/bin/run_rqlite.sh", "args": [], "capture": {"stdout": {"stream": {}}, "stderr": {"stream": {}}}}}, "success": true, "stdout": null, "stderr": null}]}}
+{
+    "db": {
+        "0": [
+            {
+                "command": {
+                    "run": {
+                        "entry_point": "/bin/run_rqlite.sh",
+                        "args": [],
+                        "capture": { "stdout": { "stream": {} }, "stderr": { "stream": {} } }
+                    }
+                },
+                "success": true,
+                "stdout": null,
+                "stderr": null
+            }
+        ]
+    }
+}
 ```
 
 and the parameters of any started instances of Local HTTP proxies:
 
 ```json
-{"http": {"local_proxy_address": "http://localhost:8080"}}
+{ "http": { "local_proxy_address": "http://localhost:8080" } }
 ```
 
 The keys in the outermost dictionaries refer to names of service cluster as specified in
@@ -278,7 +295,7 @@ The `state` stream consists of JSON-formatted descriptions of the state of the d
 after each state change, e.g.:
 
 ```json
-{"db": {"0": "running"}, "http": {"0": "starting"}}
+{ "db": { "0": "running" }, "http": { "0": "starting" } }
 ```
 
 Here, again, the keys in the topmost dictionary refer to the names of service clusters
@@ -301,13 +318,13 @@ description of a configuration to connect to your `yagna` daemon, e.g.:
 
 ```yaml
 yagna:
-  app_key: "$YAGNA_APPKEY"
-  subnet_tag: "devnet-beta"
+    app_key: "$YAGNA_APPKEY"
+    subnet_tag: "devnet-beta"
 
 payment:
-  budget: 1.0  # GLM
-  driver: "erc20"
-  network: "rinkeby"
+    budget: 1.0 # GLM
+    driver: "erc20"
+    network: "holesky"
 ```
 
 ### Descriptors

configs/default.yaml

@@ -2,13 +2,13 @@ yagna:
 #  api_url: "http://localhost:7465"
 #  gsb_url: "http://localhost:9999"
   app_key: "$YAGNA_APPKEY"
-  subnet_tag: "public"
+  subnet_tag: "123"
 
 
 payment:
   budget: 1.0  # GLM
   driver: "erc20"
-  network: "goerli"
+  network: "holesky"
 
 #limits:
 #  startup_timeout: 300  # timeout for the provisioning of the app (seconds)

configs/goth.yaml

@@ -1,8 +1,8 @@
 yagna:
-  app_key: "$YAGNA_APPKEY"
-  subnet_tag: "goth"
+    app_key: "$YAGNA_APPKEY"
+    subnet_tag: "goth"
 
 payment:
-  budget: 1.0
-  driver: "erc20"
-  network: "rinkeby"
+    budget: 1.0
+    driver: "erc20"
+    network: "holesky"

dapp_runner/descriptor/manifest.py

@@ -1,4 +1,5 @@
 """Manifest verification support."""
+
 import json
 import logging
 import re
@@ -16,33 +17,85 @@
 logger = logging.getLogger(__name__)
 
 
-def _get_manifest(params) -> Optional[Manifest]:
-    manifest_str = params.get("manifest")
-    if manifest_str:
-        manifest_dict = json.loads(standard_b64decode(manifest_str))
-        return Manifest.parse_obj(manifest_dict, by_alias=True)
+def _read_json_file(file_path: str) -> dict:
+    """Read and return JSON content from a file."""
+    with open(file_path) as f:
+        return json.load(f)
+
 
+def _get_manifest(params) -> Optional[Manifest]:
+    """Get manifest from either direct content, base64 string, or file path."""
+    if "manifest" in params:
+        manifest_content = params["manifest"]
+        # If manifest is a base64 string, decode it first
+        if isinstance(manifest_content, str):
+            try:
+                manifest_content = json.loads(standard_b64decode(manifest_content))
+            except Exception:
+                # If not base64 or not JSON, try using it directly
+                try:
+                    manifest_content = json.loads(manifest_content)
+                except json.JSONDecodeError:
+                    raise ValueError("Manifest content is neither valid base64 nor valid JSON")
+    elif "manifest_path" in params:
+        manifest_content = _read_json_file(params["manifest_path"])
+    else:
+        return None
+
+    return Manifest.parse_obj(manifest_content, by_alias=True)
+
+
+def _get_node_descriptor(params) -> Optional[dict]:
+    """Get node descriptor from file path."""
+    if "node_descriptor_path" in params:
+        return _read_json_file(params["node_descriptor_path"])
     return None
 
 
+def _read_base64_file(file_path: str) -> str:
+    """Read and return base64 encoded content from a file."""
+    with open(file_path, "rb") as f:
+        return f.read().decode("utf-8")
+
+
 def _get_manifest_cert(params) -> Optional[crypto.X509]:
-    cert_str = params.get("manifest_cert")
-    if cert_str:
-        return crypto.load_certificate(crypto.FILETYPE_PEM, standard_b64decode(cert_str))
+    """Get certificate from either direct content or file path."""
+    if "manifest_cert" in params:
+        cert_str = params["manifest_cert"]
+    elif "manifest_cert_path" in params:
+        cert_str = _read_base64_file(params["manifest_cert_path"])
+    else:
+        return None
 
-    return None
+    return crypto.load_certificate(crypto.FILETYPE_PEM, standard_b64decode(cert_str))
 
 
 def verify_manifest(payload_name: str, payload: PayloadDescriptor) -> None:
     """Verify a single payload manifest, if present."""
-
     manifest = _get_manifest(payload.params)
-    cert = _get_manifest_cert(payload.params)
-    sig = standard_b64decode(payload.params.get("manifest_sig", ""))
-    sig_algorithm = payload.params.get("manifest_sig_algorithm", "")
+    node_descriptor = _get_node_descriptor(payload.params)
+    print(f"Node Descriptor: {node_descriptor}")
+    print(f"Manifest: {manifest}")
+    # When node_descriptor is present, only manifest is required
+    if node_descriptor:
+        if not manifest:
+            raise ValueError(f"`{payload_name}`: node_descriptor requires a manifest to be present")
+        # Skip other verifications when using node_descriptor
+        return
+
+    # Only proceed with cert/sig verification if no node_descriptor is present
+    if manifest:
+        print(f"Manifest: {manifest}")
+        cert = _get_manifest_cert(payload.params)
+        sig = standard_b64decode(payload.params.get("manifest_sig", ""))
+        sig_algorithm = payload.params.get("manifest_sig_algorithm", "")
 
     now = datetime.now(UTC)
 
+    # If node_descriptor is present, manifest must also be present
+    if node_descriptor and not manifest:
+        raise ValueError(f"`{payload_name}`: node_descriptor requires a manifest to be present")
+
     if manifest:
         if manifest.created_at > now:
             raise ValueError(f"`{payload_name}`: Manifest creation date is set to future.")

dapp_runner/runner/payload.py

@@ -74,9 +74,26 @@ async def get_payload(desc: PayloadDescriptor) -> Payload:
 
     if runtime in runtimes.keys():
         if runtime == PAYLOAD_RUNTIME_VM_MANIFEST:
-            await resolve_manifest(desc)
+            # Create a copy of params to modify
+            params = desc.params.copy()
+
+            # Handle manifest file path
+            if "manifest_path" in params:
+                with open(params["manifest_path"], "rb") as f:
+                    params["manifest"] = f.read()
+                del params["manifest_path"]
+
+            # Handle node descriptor path
+            if "node_descriptor_path" in params:
+                with open(params["node_descriptor_path"]) as f:
+                    params["node_descriptor"] = json.load(f)
+                del params["node_descriptor_path"]
+
+            await resolve_manifest(PayloadDescriptor(runtime=desc.runtime, params=params))
+            payload = runtimes[desc.runtime](**params)
+        else:
+            payload = runtimes[desc.runtime](**desc.params)
 
-        payload = runtimes[desc.runtime](**desc.params)
         if inspect.isawaitable(payload):
             return await payload
         return payload  # type: ignore [return-value]  # noqa