From 3a9dedb73489c66171ff41d86c297d7e7c94245c Mon Sep 17 00:00:00 2001 From: Adam Souzis Date: Wed, 25 Sep 2024 15:49:00 -0700 Subject: [PATCH] docs: update python examples --- docs/configurators.rst | 94 ++------------------- docs/examples/configurators-5.py | 47 +---------- docs/examples/configurators-6.py | 68 +++++---------- docs/examples/configurators-6.yaml | 29 +++++++ docs/examples/configurators-7.py | 49 +---------- docs/examples/configurators-7.yaml | 22 +++++ docs/examples/configurators-8.yaml | 32 +++++++ docs/examples/data-types-1.py | 24 ++++-- docs/examples/data-types-1.yaml | 29 +++++++ docs/examples/tosca-example.py | 4 +- docs/toscaref/spec-data-types.rst | 130 +++++++++++------------------ tests/test_docs.py | 4 + tosca-package/tosca/loader.py | 10 ++- tosca-package/tosca/python2yaml.py | 5 +- unfurl/yamlloader.py | 2 +- 15 files changed, 226 insertions(+), 323 deletions(-) create mode 100644 docs/examples/configurators-6.yaml create mode 100644 docs/examples/configurators-7.yaml create mode 100644 docs/examples/configurators-8.yaml create mode 100644 docs/examples/data-types-1.yaml diff --git a/docs/configurators.rst b/docs/configurators.rst index 4e460778..0135b180 100644 --- a/docs/configurators.rst +++ b/docs/configurators.rst @@ -329,34 +329,8 @@ Example .. tab-set-code:: - .. code-block:: YAML - - apiVersion: unfurl/v1alpha1 - kind: Ensemble - spec: - service_template: - imports: - - repository: unfurl - file: tosca_plugins/artifacts.yaml - topology_template: - node_templates: - - terraform-example: - type: unfurl.nodes.Installer.Terraform - interfaces: - defaults: - inputs: - tfvars: - tag: test - main: | - - variable "tag" { - type = string - } - - output "name" { - value = var.tag - } + .. literalinclude:: ./examples/configurators-6.yaml + :language: yaml .. literalinclude:: ./examples/configurators-6.py :language: python @@ -504,32 +478,8 @@ Example .. tab-set-code:: - .. code-block:: YAML - - apiVersion: unfurl/v1alpha1 - kind: Ensemble - spec: - service_template: - imports: - - repository: unfurl - file: configurators/templates/docker.yaml - topology_template: - node_templates: - hello-world-container: - type: unfurl.nodes.Container.Application.Docker - requirements: - - host: compute - artifacts: - image: - type: tosca.artifacts.Deployment.Image.Container.Docker - file: busybox - interfaces: - Standard: - inputs: - configuration: - command: ["echo", "hello world"] - detach: no - output_logs: yes + .. literalinclude:: ./examples/configurators-7.yaml + :language: yaml .. literalinclude:: ./examples/configurators-7.py :language: python @@ -577,40 +527,8 @@ Example .. tab-set-code:: - .. code-block:: YAML - - apiVersion: unfurl/v1alpha1 - kind: Ensemble - spec: - service_template: - imports: - - repository: unfurl - file: configurators/templates/dns.yaml - topology_template: - node_templates: - example_com_zone: - type: unfurl.nodes.DNSZone - properties: - name: example.com. - provider: - # Amazon Route53 (Note: this provider requires that the zone already exists.) - class: octodns.provider.route53.Route53Provider - - test_app: - type: tosca.nodes.WebServer - requirements: - - host: compute - - dns: - node: example_com_zone - relationship: - type: unfurl.relationships.DNSRecords - properties: - records: - www: - type: A - value: - # get the ip address of the Compute instance that this is hosted on - eval: .source::.requirements::[.name=host]::.target::public_address + .. literalinclude:: ./examples/configurators-8.yaml + :language: yaml .. literalinclude:: ./examples/configurators-8.py :language: python diff --git a/docs/examples/configurators-5.py b/docs/examples/configurators-5.py index ac1999b9..b99808f1 100644 --- a/docs/examples/configurators-5.py +++ b/docs/examples/configurators-5.py @@ -1,6 +1,6 @@ import unfurl import tosca -from unfurl.tosca_plugins.artifacts import * +from unfurl.tosca_plugins.artifacts import artifact_AsdfTool @tosca.operation(name="configure") def terraform_example_configure(**kw): @@ -18,48 +18,3 @@ def terraform_example_configure(**kw): file="ripgrep", ) terraform_example.configure = terraform_example_configure - -configurator_artifacts = unfurl.nodes.LocalRepository( - "configurator-artifacts", - _directives=["default"], -) -configurator_artifacts.terraform = artifact_AsdfTool( - "terraform", - version="1.1.4", - file="terraform", -) -configurator_artifacts.gcloud = artifact_AsdfTool( - "gcloud", - version="398.0.0", - file="gcloud", -) -configurator_artifacts.kompose = artifact_AsdfTool( - "kompose", - version="1.26.1", - file="kompose", -) -configurator_artifacts.google_auth = artifact_PythonPackage( - "google-auth", - file="google-auth", -) -configurator_artifacts.octodns = artifact_PythonPackage( - "octodns", - version="==0.9.14", - file="octodns", -) -configurator_artifacts.kubernetes_core = artifact_AnsibleCollection( - "kubernetes.core", - version="2.4.0", - file="kubernetes.core", -) -configurator_artifacts.community_docker = artifact_AnsibleCollection( - "community.docker", - version="1.10.2", - file="community.docker", -) -configurator_artifacts.ansible_utils = artifact_AnsibleCollection( - "ansible.utils", - version="2.10.3", - file="ansible.utils", -) - diff --git a/docs/examples/configurators-6.py b/docs/examples/configurators-6.py index 49cd8def..48662190 100644 --- a/docs/examples/configurators-6.py +++ b/docs/examples/configurators-6.py @@ -1,50 +1,24 @@ +import tosca import unfurl -from unfurl.tosca_plugins.artifacts import * +from unfurl.tosca_plugins.artifacts import unfurl_nodes_Installer_Terraform +import unfurl.configurators.terraform -terraform_example = unfurl_nodes_Installer_Terraform( - "terraform-example", -) -configurator_artifacts = unfurl.nodes.LocalRepository( - "configurator-artifacts", - _directives=["default"], -) -configurator_artifacts.terraform = artifact_AsdfTool( - "terraform", - version="1.1.4", - file="terraform", -) -configurator_artifacts.gcloud = artifact_AsdfTool( - "gcloud", - version="398.0.0", - file="gcloud", -) -configurator_artifacts.kompose = artifact_AsdfTool( - "kompose", - version="1.26.1", - file="kompose", -) -configurator_artifacts.google_auth = artifact_PythonPackage( - "google-auth", - file="google-auth", -) -configurator_artifacts.octodns = artifact_PythonPackage( - "octodns", - version="==0.9.14", - file="octodns", -) -configurator_artifacts.kubernetes_core = artifact_AnsibleCollection( - "kubernetes.core", - version="2.4.0", - file="kubernetes.core", -) -configurator_artifacts.community_docker = artifact_AnsibleCollection( - "community.docker", - version="1.10.2", - file="community.docker", -) -configurator_artifacts.ansible_utils = artifact_AnsibleCollection( - "ansible.utils", - version="2.10.3", - file="ansible.utils", -) +@tosca.operation( + name="default", apply_to=["Install.check", "Standard.configure", "Standard.delete"] +) +def terraform_example_default(**kw): + return unfurl.configurators.terraform.TerraformConfigurator( + tfvars={"tag": "test"}, + main=""" + variable "tag" { + type = string + } + output "name" { + value = var.tag + }""", + ) + + +terraform_example = unfurl_nodes_Installer_Terraform() +terraform_example.default = terraform_example_default diff --git a/docs/examples/configurators-6.yaml b/docs/examples/configurators-6.yaml new file mode 100644 index 00000000..3ccaef70 --- /dev/null +++ b/docs/examples/configurators-6.yaml @@ -0,0 +1,29 @@ +apiVersion: unfurl/v1alpha1 +kind: Ensemble +spec: + service_template: + imports: + - repository: unfurl + file: tosca_plugins/artifacts.yaml + topology_template: + node_templates: + + terraform-example: + type: unfurl.nodes.Installer.Terraform + interfaces: + defaults: + inputs: + tfvars: + tag: test + main: | + variable "tag" { + type = string + } + + output "name" { + value = var.tag + } + Standard: + operations: + configure: + diff --git a/docs/examples/configurators-7.py b/docs/examples/configurators-7.py index 20944868..50d669ab 100644 --- a/docs/examples/configurators-7.py +++ b/docs/examples/configurators-7.py @@ -1,9 +1,7 @@ -import unfurl -import tosca from unfurl.configurators.templates.docker import ( unfurl_nodes_Container_Application_Docker, ) -from unfurl.tosca_plugins.artifacts import * +import tosca hello_world_container = unfurl_nodes_Container_Application_Docker( "hello-world-container", @@ -12,48 +10,3 @@ file="busybox", ), ) -hello_world_container.host = tosca.find_node("compute") - -configurator_artifacts = unfurl.nodes.LocalRepository( - "configurator-artifacts", - _directives=["default"], -) -configurator_artifacts.terraform = artifact_AsdfTool( - "terraform", - version="1.1.4", - file="terraform", -) -configurator_artifacts.gcloud = artifact_AsdfTool( - "gcloud", - version="398.0.0", - file="gcloud", -) -configurator_artifacts.kompose = artifact_AsdfTool( - "kompose", - version="1.26.1", - file="kompose", -) -configurator_artifacts.google_auth = artifact_PythonPackage( - "google-auth", - file="google-auth", -) -configurator_artifacts.octodns = artifact_PythonPackage( - "octodns", - version="==0.9.14", - file="octodns", -) -configurator_artifacts.kubernetes_core = artifact_AnsibleCollection( - "kubernetes.core", - version="2.4.0", - file="kubernetes.core", -) -configurator_artifacts.community_docker = artifact_AnsibleCollection( - "community.docker", - version="1.10.2", - file="community.docker", -) -configurator_artifacts.ansible_utils = artifact_AnsibleCollection( - "ansible.utils", - version="2.10.3", - file="ansible.utils", -) diff --git a/docs/examples/configurators-7.yaml b/docs/examples/configurators-7.yaml new file mode 100644 index 00000000..92a0bb18 --- /dev/null +++ b/docs/examples/configurators-7.yaml @@ -0,0 +1,22 @@ +apiVersion: unfurl/v1alpha1 +kind: Ensemble +spec: + service_template: + imports: + - repository: unfurl + file: configurators/templates/docker.yaml + topology_template: + node_templates: + hello-world-container: + type: unfurl.nodes.Container.Application.Docker + artifacts: + image: + type: tosca.artifacts.Deployment.Image.Container.Docker + file: busybox + interfaces: + Standard: + inputs: + configuration: + command: ["echo", "hello world"] + detach: no + output_logs: yes diff --git a/docs/examples/configurators-8.yaml b/docs/examples/configurators-8.yaml new file mode 100644 index 00000000..3b6182ce --- /dev/null +++ b/docs/examples/configurators-8.yaml @@ -0,0 +1,32 @@ +apiVersion: unfurl/v1alpha1 +kind: Ensemble +spec: + service_template: + imports: + - repository: unfurl + file: configurators/templates/dns.yaml + topology_template: + node_templates: + example_com_zone: + type: unfurl.nodes.DNSZone + properties: + name: example.com. + provider: + # Amazon Route53 (Note: this provider requires that the zone already exists.) + class: octodns.provider.route53.Route53Provider + + test_app: + type: tosca.nodes.WebServer + requirements: + - host: compute + - dns: + node: example_com_zone + relationship: + type: unfurl.relationships.DNSRecords + properties: + records: + www: + type: A + value: + # get the ip address of the Compute instance that this is hosted on + eval: .source::.requirements::[.name=host]::.target::public_address \ No newline at end of file diff --git a/docs/examples/data-types-1.py b/docs/examples/data-types-1.py index 335bc035..03841be8 100644 --- a/docs/examples/data-types-1.py +++ b/docs/examples/data-types-1.py @@ -1,16 +1,28 @@ -class Endpoint(tosca.datatypes.Root): +import unfurl +import tosca +from tosca import DataType + + +class my_datatypes_Endpoint(DataType): + """Socket endpoint details""" + + _type_name = "my.datatypes.Endpoint" ip: str + """the endpoint IP""" + port: int + """the endpoint port""" + class DatabaseService(tosca.nodes.DBMS): - endpoint: Endpoint + endpoint: "my_datatypes_Endpoint" + my_db_service = DatabaseService( "my_db_service", - endpoint=Endpoint( + endpoint=my_datatypes_Endpoint( ip="192.168.15.85", - port=2233 - ) + port=2233, + ), ) -__all__ = ["Endpoint", "DatabaseService"] diff --git a/docs/examples/data-types-1.yaml b/docs/examples/data-types-1.yaml new file mode 100644 index 00000000..f16f409d --- /dev/null +++ b/docs/examples/data-types-1.yaml @@ -0,0 +1,29 @@ +tosca_definitions_version: tosca_simple_unfurl_1_0_0 + +data_types: + my.datatypes.Endpoint: + description: Socket endpoint details + properties: + ip: + description: the endpoint IP + type: string + port: + description: the endpoint port + type: integer + +node_types: + + DatabaseService: + derived_from: tosca.nodes.DBMS + properties: + endpoint: + type: my.datatypes.Endpoint + +topology_template: + node_templates: + my_db_service: + type: DatabaseService + properties: + endpoint: + ip: 192.168.15.85 + port: 2233 diff --git a/docs/examples/tosca-example.py b/docs/examples/tosca-example.py index 766ede15..99f2939e 100644 --- a/docs/examples/tosca-example.py +++ b/docs/examples/tosca-example.py @@ -2,7 +2,7 @@ import unfurl -from typing import Sequence +from typing import Sequence, Union import tosca from tosca import Attribute, Eval, GB, MB, Property import unfurl.configurators.shell @@ -19,7 +19,7 @@ class myApplication(tosca.nodes.SoftwareComponent): private_address: str = Attribute() host: Sequence[ - tosca.relationships.HostedOn | tosca.nodes.Compute | tosca.capabilities.Compute + Union[tosca.relationships.HostedOn, tosca.nodes.Compute, tosca.capabilities.Compute] ] = () db: "dbconnection" diff --git a/docs/toscaref/spec-data-types.rst b/docs/toscaref/spec-data-types.rst index 8baac947..d8d57d32 100644 --- a/docs/toscaref/spec-data-types.rst +++ b/docs/toscaref/spec-data-types.rst @@ -94,40 +94,10 @@ configures the endpoint properties (i.e. the ``ip`` and ``port``). .. tab-set-code:: - .. code:: yaml - - tosca_definitions_version: tosca_simple_unfurl_1_0_0 - - data_types: - - my.datatypes.Endpoint: - description: Socket endpoint details - properties: - ip: - description: the endpoint IP - type: string - port: - description: the endpoint port - type: integer - - node_types: - - DatabaseService: - derived_from: tosca.nodes.DBMS - properties: - endpoint: - type: my.datatypes.Endpoint - - node_templates: + .. literalinclude:: ../examples/data-types-1.yaml + :language: yaml - my_db_service: - type: DatabaseService - properties: - endpoint: - ip: 192.168.15.85 - port: 2233 - - .. literalinclude:: ./../examples/data-types-1.py + .. literalinclude:: ../examples/data-types-1.py :language: python @@ -202,15 +172,16 @@ that also includes a user name. endpoint: type: my.datatypes.ExtendedEndpoint - node_templates: + topology_template: + node_templates: - my_db_service: - type: DatabaseService - properties: - endpoint: - ip: 192.168.15.85 - port: 2233 - username: jimmy + my_db_service: + type: DatabaseService + properties: + endpoint: + ip: 192.168.15.85 + port: 2233 + username: jimmy .. literalinclude:: ./../examples/data-types-1.py :language: python @@ -229,7 +200,6 @@ information + authentication details. .. code:: yaml tosca_definitions_version: tosca_simple_unfurl_1_0_0 - data_types: my.datatypes.Endpoint: @@ -257,18 +227,19 @@ information + authentication details. connection: type: my.datatypes.Connection - node_templates: + topology_template: + node_templates: - my_db_service: - type: DatabaseService - properties: - connection: - endpoint: - ip: 192.168.15.85 - port: 2233 - auth: - username: jimmy - password: secret + my_db_service: + type: DatabaseService + properties: + connection: + endpoint: + ip: 192.168.15.85 + port: 2233 + auth: + username: jimmy + password: secret .. literalinclude:: ./../examples/data-types-3.py :language: python @@ -286,10 +257,8 @@ will be ``admin``. .. code:: yaml - tosca_definitions_version: tosca_dsl_1_2 - + tosca_definitions_version: tosca_simple_unfurl_1_0_0 data_types: - my.datatypes.Connection: properties: endpoint: @@ -318,21 +287,22 @@ will be ``admin``. node_types: DatabaseService: - derived_from: cloudify.nodes.DBMS + derived_from: nodes.DBMS properties: connection: type: my.datatypes.Connection - node_templates: + topology_template: + node_templates: - my_db_service: - type: DatabaseService - properties: - connection: - endpoint: - ip: 192.168.15.85 - auth: - password: secret + my_db_service: + type: DatabaseService + properties: + connection: + endpoint: + ip: 192.168.15.85 + auth: + password: secret .. literalinclude:: ./../examples/data-types-4.py :language: python @@ -384,10 +354,7 @@ property. .. code:: yaml - tosca_definitions_version: tosca_simple_unfurl_1_0_0 - data_types: - datatypes.Data1: properties: prop1: @@ -406,13 +373,14 @@ property. default: prop2: prop2_override - node_templates: - - my_app: - type: nodes.MyApp - properties: - data1: - prop3: prop3_override + topology_template: + node_templates: + + my_app: + type: nodes.MyApp + properties: + data1: + prop3: prop3_override .. literalinclude:: ./../examples/data-types-5.py :language: python @@ -467,8 +435,7 @@ previously. For example: .. code:: yaml - tosca_definitions_version: tosca_dsl_1_2 - + tosca_definitions_version: tosca_simple_unfurl_1_0_0 data_types: datatypes.Data1: @@ -497,10 +464,11 @@ previously. For example: default: prop3: prop3_override - node_templates: + topology_template + node_templates: - my_app: - type: nodes.DerivedFromMyApp + my_app: + type: nodes.DerivedFromMyApp .. literalinclude:: ./../examples/data-types-6.py :language: python diff --git a/tests/test_docs.py b/tests/test_docs.py index 198d0bab..a665f609 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -6,6 +6,7 @@ from unfurl.localenv import LocalConfig from unfurl.yamlmanifest import YamlManifest, _basepath from unfurl.yamlloader import YamlConfig +from tosca import global_state basedir = os.path.join(os.path.dirname(__file__), "..", "docs", "examples") @@ -33,6 +34,8 @@ def test_python_example(self): # assert ToscaSpec(serviceTemplate.config, path=path) def test_python_snippets(self): + # examples generated like: + # UNFURL_EXPORT_PYTHON_STYLE=concise unfurl -vv export --format python docs/examples/configurators-6.yaml python_files = glob.glob(os.path.join(basedir, "*.py")) required_imports = """ @@ -41,6 +44,7 @@ def test_python_snippets(self): from tosca import Attribute, Eval, Property, operation, GB, MB """ + global_state.mode = "spec" for py_file in python_files: with self.subTest(py_file=py_file): with open(py_file, "r") as f: diff --git a/tosca-package/tosca/loader.py b/tosca-package/tosca/loader.py index 37354b4c..ec6273d4 100644 --- a/tosca-package/tosca/loader.py +++ b/tosca-package/tosca/loader.py @@ -167,7 +167,7 @@ def exec_module(self, module): for i in range(self.full_name.count(".")): path = path.parent restricted_exec( - src, vars(module), path, self.full_name, self.modules, safe_mode + src, vars(module), str(path), self.full_name, self.modules, safe_mode ) @@ -634,7 +634,7 @@ def _call_print(self, *objects, **kwargs): def restricted_exec( python_src: str, namespace, - base_dir, + base_dir: str, full_name="service_template", modules=None, safe_mode=False, @@ -712,8 +712,12 @@ def restricted_exec( namespace["__builtins__"] = tosca_builtins namespace["__name__"] = full_name if base_dir and "__file__" not in namespace: + if full_name.startswith("service_template"): + file_path = full_name[len("service_template"):].lstrip(".").replace(".", "/") + else: + file_path = full_name.replace(".", "/") namespace["__file__"] = ( - os.path.join(base_dir, full_name.replace(".", "/")) + ".py" + os.path.join(base_dir, file_path) + ".py" ) if package and "__package__" not in namespace: namespace["__package__"] = package diff --git a/tosca-package/tosca/python2yaml.py b/tosca-package/tosca/python2yaml.py index f3979c1b..b31e5a04 100644 --- a/tosca-package/tosca/python2yaml.py +++ b/tosca-package/tosca/python2yaml.py @@ -388,7 +388,7 @@ def _import_module( import_path = module_dir / yaml_path self.imports.add(("", import_path)) logger.debug( - f'importing "{module_name}" in "{current_module}": located at "{import_path}", relative to "{module_path}"' + f'"{current_module}" is importing "{module_name}": located at "{import_path}", relative to "{module_path}"' ) except ValueError: # not a subpath of the current module, add a repository @@ -396,6 +396,9 @@ def _import_module( module_name, yaml_path ) if repo_path: + logger.debug( + f'"{current_module}" is importing "{module_name}" in package "{ns}": located at "{repo_path}""' + ) self.imports.add((ns, repo_path)) else: if ns: diff --git a/unfurl/yamlloader.py b/unfurl/yamlloader.py index 9c2ed7eb..fcd8b487 100644 --- a/unfurl/yamlloader.py +++ b/unfurl/yamlloader.py @@ -681,7 +681,7 @@ def resolve_url( url = base else: # url is a local path - assert base + assert base or os.path.isabs(file_name), f"{file_name} isn't absolute and base isn't set" url = os.path.join(base, file_name) repository_root = None # default to checking if its in the project if importsLoader.repository_root: