Merge branch 'master' into fix-missing-controller-yaml

VERSION

@@ -1 +1 @@
-3.2.2.0
+3.2.3.0

docs/changelog.rst

@@ -1,15 +1,47 @@
 Changelog
 ---------
 
+3.2.3.0
+^^^^^^^
+
+## What's Changed
+
+Thursday 26th Oct 2023
+
+* Repository Maintenance Improvements by @cderici in https://github.com/juju/python-libjuju/pull/922
+* Stale bot to not bother feature requests by @cderici in https://github.com/juju/python-libjuju/pull/926
+* Fix linter issues by @cderici in https://github.com/juju/python-libjuju/pull/928
+* Fix docstring typo by @DanielArndt in https://github.com/juju/python-libjuju/pull/927
+* Fix asyncio on README by @marceloneppel in https://github.com/juju/python-libjuju/pull/930
+* Fix integration/test_application.test_action by @cderici in https://github.com/juju/python-libjuju/pull/932
+* Update 3.2 facade clients by @cderici in https://github.com/juju/python-libjuju/pull/931
+* [JUJU-4488] Add licence headers to source files on 3.x by @cderici in https://github.com/juju/python-libjuju/pull/934
+* Update async tests to use builtin python suite by @DanielArndt in https://github.com/juju/python-libjuju/pull/935
+* Pass correct charm url to series selector by @cderici in https://github.com/juju/python-libjuju/pull/942
+* Green CI cleanup for python-libjuju by @cderici in https://github.com/juju/python-libjuju/pull/939
+* Bring forward support for nested assumes expressions on 3x by @cderici in https://github.com/juju/python-libjuju/pull/943
+* Release 3.2.2.0 notes by @cderici in https://github.com/juju/python-libjuju/pull/945
+* Cleanup release process for 3.x by @cderici in https://github.com/juju/python-libjuju/pull/946
+* Use new DeployFromRepository endpoint for deploy by @cderici in https://github.com/juju/python-libjuju/pull/949
+* Handle pending upload resources deployfromrepository by @cderici in https://github.com/juju/python-libjuju/pull/953
+* Optimize connection teardown by @cderici in https://github.com/juju/python-libjuju/pull/952
+* Use `log.warning` instead of the deprecated `warn` by @sed-i in https://github.com/juju/python-libjuju/pull/954
+* Find controller name by endpoint on 3.x track by @cderici in https://github.com/juju/python-libjuju/pull/966
+* Optimize & fix unit removal by @cderici in https://github.com/juju/python-libjuju/pull/967
+* Allow switch kwarg in refresh to switch to local charms by @jack-w-shaw in https://github.com/juju/python-libjuju/pull/971
+* Parse charm URLs consistantly for local charms by @jack-w-shaw in https://github.com/juju/python-libjuju/pull/974
+* Juju config directory location fix on 3.x by @cderici in https://github.com/juju/python-libjuju/pull/976
+* [JUJU-4779] Ensure valid charm origin for local charm switches by @jack-w-shaw in https://github.com/juju/python-libjuju/pull/978
+* Application refresh with resources on 3.x by @cderici in https://github.com/juju/python-libjuju/pull/973
+
+
 3.2.2.0
 ^^^^^^^
 
 Wednesday 6th September 2023
 
 This is a minor release on the 3.x track, works with any Juju 3.x controller.
 
-## What's Changed
-
 * Repository Maintenance Improvements by @cderici in https://github.com/juju/python-libjuju/pull/922
 * Stale bot to not bother feature requests by @cderici in https://github.com/juju/python-libjuju/pull/926
 * Fix linter issues by @cderici in https://github.com/juju/python-libjuju/pull/928

examples/local_refresh.py

@@ -20,7 +20,7 @@ async def main():
 
     try:
         print('Get deployed application')
-        app = model.appplications["ubuntu"]
+        app = model.applications["ubuntu"]
 
         print('Refresh/Upgrade Ubuntu charm with local charm')
         await app.refresh(path="path/to/local/ubuntu.charm")

juju/application.py

@@ -4,15 +4,15 @@
 import hashlib
 import json
 import logging
-import pathlib
+from pathlib import Path
 
 from . import model, tag, utils, jasyncio
 from .url import URL
 from .status import derive_status
 from .annotationhelper import _get_annotations, _set_annotations
 from .client import client
 from .errors import JujuError, JujuApplicationConfigError
-from .bundle import get_charm_series
+from .bundle import get_charm_series, is_local_charm
 from .placement import parse as parse_placement
 from .origin import Channel, Source
 
@@ -661,6 +661,8 @@ async def refresh(
         :param str switch: Crossgrade charm url
 
         """
+        if switch is not None and path is not None:
+            raise ValueError("switch and path are mutually exclusive")
 
         if switch is not None and revision is not None:
             raise ValueError("switch and revision are mutually exclusive")
@@ -677,17 +679,15 @@ async def refresh(
         if charm_url_origin_result.error is not None:
             err = charm_url_origin_result.error
             raise JujuError(f'{err.code} : {err.message}')
-        charm_url = switch or charm_url_origin_result.url
         origin = charm_url_origin_result.charm_origin
 
-        if path is not None:
+        if path is not None or (switch is not None and is_local_charm(switch)):
             await self.local_refresh(origin, force, force_series,
-                                     force_units, path, resources)
+                                     force_units, path or switch, resources)
             return
 
-        if resources is not None:
-            raise NotImplementedError("resources option is not implemented")
-
+        # If switch is not None at this point, that means it's a switch to a store charm
+        charm_url = switch or charm_url_origin_result.url
         parsed_url = URL.parse(charm_url)
         charm_name = parsed_url.name
 
@@ -735,6 +735,20 @@ async def refresh(
 
         # Now take care of the resources:
 
+        # user supplied resources to be used in refresh,
+        # will override the default values if there's any
+        arg_resources = resources or {}
+
+        # need to process the given resources, as they can be
+        # paths or revisions
+        _arg_res_filenames = {}
+        _arg_res_revisions = {}
+        for res, filename_or_rev in arg_resources.items():
+            if isinstance(filename_or_rev, int):
+                _arg_res_revisions[res] = filename_or_rev
+            else:
+                _arg_res_filenames[res] = filename_or_rev
+
         # Already prepped the charm_resources
         # Now get the existing resources from the ResourcesFacade
         request_data = [client.Entity(self.tag)]
@@ -748,23 +762,25 @@ async def refresh(
         # Compute the difference btw resources needed and the existing resources
         resources_to_update = []
         for resource in charm_resources:
-            if utils.should_upgrade_resource(resource, existing_resources):
+            if utils.should_upgrade_resource(resource, existing_resources, arg_resources):
                 resources_to_update.append(resource)
 
         # Update the resources
         if resources_to_update:
             request_data = []
             for resource in resources_to_update:
+                res_name = resource.get('Name', resource.get('name'))
                 request_data.append(client.CharmResource(
                     description=resource.get('Description', resource.get('description')),
-                    fingerprint=resource.get('Fingerprint', resource.get('fingerprint')),
-                    name=resource.get('Name', resource.get('name')),
-                    path=resource.get('Path', resource.get('filename')),
-                    revision=resource.get('Revision', resource.get('revision', -1)),
-                    size=resource.get('Size', resource.get('size')),
+                    name=res_name,
+                    path=_arg_res_filenames.get(res_name,
+                                                resource.get('Path',
+                                                             resource.get('filename', ''))),
+                    revision=_arg_res_revisions.get(res_name, -1),
                     type_=resource.get('Type', resource.get('type')),
                     origin='store',
                 ))
+
             response = await resources_facade.AddPendingResources(
                 application_tag=self.tag,
                 charm_url=charm_url,
@@ -808,22 +824,22 @@ async def local_refresh(
             path=None, resources=None):
         """Refresh the charm for this application with a local charm.
 
-        :param str channel: Channel to use when getting the charm from the
-            charm store, e.g. 'development'
+        :param dict charm_origin: The charm origin of the destination charm
+            we're refreshing to
+        :param bool force: Refresh even if validation checks fail
         :param bool force_series: Refresh even if series of deployed
             application is not supported by the new charm
         :param bool force_units: Refresh all units immediately, even if in
             error state
         :param str path: Refresh to a charm located at path
         :param dict resources: Dictionary of resource name/filepath pairs
-        :param int revision: Explicit refresh revision
-        :param str switch: Crossgrade charm url
 
         """
         app_facade = self._facade()
 
-        if not isinstance(path, pathlib.Path):
-            path = pathlib.Path(path)
+        if isinstance(path, str) and path.startswith("local:"):
+            path = path[6:]
+        path = Path(path)
         charm_dir = path.expanduser().resolve()
         model_config = await self.get_config()
 
@@ -847,6 +863,17 @@ async def local_refresh(
                                                              metadata,
                                                              resources=resources)
 
+        # We know this charm is a local charm, but this charm origin could be
+        # the charm origin of a charmhub charm. Ensure that we update/remove
+        # the appropriate fields.
+        charm_origin.source = "local"
+        charm_origin.track = None
+        charm_origin.risk = None
+        charm_origin.branch = None
+        charm_origin.hash_ = None
+        charm_origin.id_ = None
+        charm_origin.revision = URL.parse(charm_url).revision
+
         set_charm_args = {
             'application': self.entity_id,
             'charm_origin': charm_origin,

juju/bundle.py

@@ -229,7 +229,7 @@ def _resolve_include_file_config(self, bundle_dir):
 
         return self.bundle, self.overlays
 
-    async def fetch_plan(self, bundle_url, origin, overlays=[]):
+    async def fetch_plan(self, bundle, origin, overlays=[]):
         """fetch_plan is called by the model.deploy(). It gathers the information about the
         bundle to be deployed (whether local or CharmHub), straightens it up, applies overlays
         if any overlays are given. Validates the bundle against known issues. Resolves and adds
@@ -245,19 +245,21 @@ async def fetch_plan(self, bundle_url, origin, overlays=[]):
 
         :returns: None
         """
-        entity_id = bundle_url.path()
-        is_local = Schema.LOCAL.matches(bundle_url.schema)
         bundle_dir = None
 
-        if is_local and os.path.isfile(entity_id):
-            bundle_yaml = Path(entity_id).read_text()
-            bundle_dir = Path(entity_id).parent
-        elif is_local and os.path.isdir(entity_id):
-            bundle_yaml = (Path(entity_id) / "bundle.yaml").read_text()
-            bundle_dir = Path(entity_id)
+        if is_local_bundle(str(bundle)):
+            path = str(bundle)
+            if path.startswith("local:"):
+                path = path[6:]
+            bundle_yaml, bundle_dir = read_local_bundle(path)
 
-        if Schema.CHARM_HUB.matches(bundle_url.schema):
-            bundle_yaml = await self._download_bundle(bundle_url, origin)
+        else:
+            if client.CharmsFacade.best_facade_version(self.model.connection()) < 3:
+                url = URL.parse(bundle, default_store=Schema.CHARM_STORE)
+            else:
+                url = URL.parse(bundle)
+            path = url.path()
+            bundle_yaml = await self._download_bundle(bundle, origin)
 
         if not bundle_yaml:
             raise JujuError('empty bundle, nothing to deploy')
@@ -284,7 +286,7 @@ async def fetch_plan(self, bundle_url, origin, overlays=[]):
 
         self.bundle = await self._validate_bundle(self.bundle)
 
-        if is_local:
+        if is_local_bundle(path):
             self.bundle = await self._handle_local_charms(self.bundle, bundle_dir)
 
         self.bundle, self.overlays = self._resolve_include_file_config(bundle_dir)
@@ -295,7 +297,7 @@ async def fetch_plan(self, bundle_url, origin, overlays=[]):
         yaml_data = "---\n".join(_yaml_data)
 
         self.plan = await self.bundle_facade.GetChangesMapArgs(
-            bundleurl=entity_id,
+            bundleurl=path,
             yaml=yaml_data)
 
         if self.plan.errors and any(self.plan.errors):
@@ -389,7 +391,6 @@ async def _resolve_charms(self):
                                             track=track,
                                             base=base,
                                             )
-
                 charm_url, charm_origin = await self.model._resolve_charm(charm_url, origin)
                 spec['charm'] = str(charm_url)
             else:
@@ -443,6 +444,21 @@ def is_local_charm(charm_url):
     return charm_url.startswith('.') or charm_url.startswith('local:') or os.path.isabs(charm_url)
 
 
+is_local_bundle = is_local_charm
+
+
+def read_local_bundle(path):
+    path = Path(path)
+    if os.path.isfile(path):
+        bundle_yaml = path.read_text()
+        bundle_dir = path.parent
+    elif os.path.isdir(path):
+        bundle_yaml = (path / "bundle.yaml").read_text()
+        bundle_dir = path
+
+    return (bundle_yaml, bundle_dir)
+
+
 async def get_charm_series(metadata, model):
     """Inspects the given metadata and returns a default series from its
     metadata.yaml (the first item in the 'series' list).
@@ -676,12 +692,12 @@ async def run(self, context):
 
         # We don't add local charms because they've already been added
         # by self._handle_local_charms
+        if is_local_charm(str(self.charm)):
+            return self.charm
+
         url = URL.parse(str(self.charm))
         ch = None
         identifier = None
-        if Schema.LOCAL.matches(url.schema):
-            return self.charm
-
         if Schema.CHARM_HUB.matches(url.schema):
             ch = Channel('latest', 'stable')
             if self.channel:
@@ -700,7 +716,7 @@ async def run(self, context):
         if identifier is None:
             raise JujuError('unknown charm {}'.format(self.charm))
 
-        await context.model._add_charm(identifier, origin)
+        await context.model._add_charm(str(identifier), origin)
 
         if str(ch) not in context.origins:
             context.origins[str(identifier)] = {}

juju/client/connector.py

@@ -7,9 +7,9 @@
 import macaroonbakery.httpbakery as httpbakery
 from juju.client.connection import Connection
 from juju.client.gocookies import GoCookieJar, go_to_py_cookie
-from juju.client.jujudata import FileJujuData
+from juju.client.jujudata import FileJujuData, API_ENDPOINTS_KEY
 from juju.client.proxy.factory import proxy_from_config
-from juju.errors import JujuConnectionError, JujuError
+from juju.errors import JujuConnectionError, JujuError, PylibjujuProgrammingError
 from juju.client import client
 from juju.version import SUPPORTED_MAJOR_VERSION, TARGET_JUJU_VERSION
 
@@ -83,6 +83,11 @@ async def connect(self, **kwargs):
                 await self._connection.close()
             self._connection = await Connection.connect(**kwargs)
 
+        if not self.controller_name:
+            if 'endpoint' not in kwargs:
+                raise PylibjujuProgrammingError("Please report this error to the maintainers.")
+            self.controller_name = self.jujudata.controller_name_by_endpoint(kwargs['endpoint'])
+
         # Check if we support the target controller
         juju_server_version = self._connection.info['server-version']
         if not juju_server_version.startswith(TARGET_JUJU_VERSION):
@@ -112,7 +117,7 @@ async def connect_controller(self, controller_name=None, specified_facades=None)
             raise JujuConnectionError('No current controller. Is Juju bootstrapped?')
 
         controller = self.jujudata.controllers()[controller_name]
-        endpoints = controller['api-endpoints']
+        endpoints = controller[API_ENDPOINTS_KEY]
         accounts = self.jujudata.accounts().get(controller_name, {})
 
         proxy = proxy_from_config(controller.get('proxy-config', None))
@@ -146,7 +151,7 @@ async def connect_model(self, _model_name=None, **kwargs):
         if controller is None:
             raise JujuConnectionError('Controller {} not found'.format(
                 controller_name))
-        endpoints = controller['api-endpoints']
+        endpoints = controller[API_ENDPOINTS_KEY]
         account = self.jujudata.accounts().get(controller_name, {})
         models = self.jujudata.models().get(controller_name, {}).get('models',
                                                                      {})