Merge pull request #645 from fnordahl/add-charmcraft-integration-test

Fix environment population for build package workaround and add charmcraft integration test

.github/workflows/tests.yml

@@ -0,0 +1,121 @@
+name: Test Suite
+
+on:
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  unit:
+    name: Unit tests
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.8', '3.9', '3.10']
+    steps:
+    - name: Check out code
+      uses: actions/checkout@v2
+    - name: Setup Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install Tox
+      run: pip install tox
+    - name: Run tests
+      run: tox -e py
+
+  build:
+    name: Build snap
+    needs: unit
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Add fake tag to make vergit happy
+      run: git tag v0.0.0
+    - uses: snapcore/action-build@v1
+      id: snap-build
+    - uses: actions/upload-artifact@v1
+      with:
+        name: charm-snap
+        path: ${{ steps.snap-build.outputs.snap }}
+
+  integration:
+    name: Integration test
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+    - name: Init LXD
+      run: |
+        set -euxo pipefail
+        sudo lxd init --auto
+        # This is a throw-away CI environment, do not do this at home
+        sudo chmod 666 /var/snap/lxd/common/lxd/unix.socket
+    - name: Checkout layer-basic
+      uses: actions/checkout@v2
+      with:
+        repository: juju-solutions/layer-basic
+
+    - name: Fixup wheelhouse
+      run: |
+        set -euxo pipefail
+        cat << EOF | tee -a tests/charm-minimal/wheelhouse.txt
+        # https://github.com/pallets/jinja/issues/1496
+        #
+        # We ought to teach charm-tools to seed the virtualenv used to build
+        # wheels with newer versions of pip and setuptools.
+        Jinja2<3
+        EOF
+
+    - name: Download built charm snap
+      uses: actions/download-artifact@v3
+      with:
+        name: charm-snap
+        path: tests/charm-minimal/charm-snap
+
+    - name: Build reactive charm with charmcraft
+      run: |
+        set -euxo pipefail
+        sudo snap install --classic --channel latest/edge charmcraft
+        cat << EOF | tee tests/charm-minimal/charmcraft.yaml
+        type: charm
+        parts:
+          charm-tools:
+            plugin: nil
+            override-build: |
+              snap install --dangerous --classic \$CRAFT_PROJECT_DIR/parts/charm/src/charm-snap/*.snap
+              rm -rf \$CRAFT_PROJECT_DIR/parts/charm/src/charm-snap
+          charm:
+            after: [charm-tools]
+            source: .
+            plugin: reactive
+            reactive-charm-build-arguments:
+              - -v
+              - --binary-wheels-from-source
+            build-packages:
+              - python3-dev
+              - libpq-dev
+        bases:
+          - name: ubuntu
+            channel: "18.04"
+            architectures: [amd64]
+          - name: ubuntu
+            channel: "20.04"
+            architectures: [amd64]
+          - name: ubuntu
+            channel: "22.04"
+            architectures: [amd64]
+        EOF
+        charmcraft pack -p tests/charm-minimal -v
+    - name: Upload charmcraft execution logs
+      if: always()
+      uses: actions/upload-artifact@v3
+      with:
+        name: charmcraft execution logs
+        path: ~/snap/charmcraft/common/cache/charmcraft/log/*.log
+    - name: Upload built charms
+      uses: actions/upload-artifact@v3
+      with:
+        name: Built charms
+        path: |
+          minimal_ubuntu-18.04-amd64.charm
+          minimal_ubuntu-20.04-amd64.charm
+          minimal_ubuntu-22.04-amd64.charm

charmtools/build/builder.py

@@ -786,7 +786,8 @@ def workaround_charmcraft_maybe_ensure_build_packages(self):
                      or os.environ.get('CHARMCRAFT_PART_NAME', None))):
             log.warning('Probably running as root in charmcraft, proactively '
                         'installing the `git` and `virtualenv` packages.')
-            subprocess.run(('apt', '-y', 'install', 'git', 'virtualenv'), check=True, env={})
+            subprocess.run(('apt', '-y', 'install', 'git', 'virtualenv'),
+                           check=True, env=utils.host_env())
 
     def generate(self):
         layers = self.fetch()

charmtools/build/tactics.py

@@ -1064,24 +1064,14 @@ def __str__(self):
         return "Building wheelhouse in {}".format(directory)
 
     def _get_env(self):
-        """Get environment appropriate for executing external commands.
+        """Get environment for executing commands outside snap context.
 
         :returns: Dictionary with environment variables
         :rtype: Dict[str,str]
         """
         if self.use_python_from_snap:
             return os.environ.copy()
-
-        env = os.environ.copy()
-        for key in ('PREFIX', 'PYTHONHOME', 'PYTHONPATH',
-                    'GIT_TEMPLATE_DIR', 'GIT_EXEC_PATH'):
-            if key in env:
-                del(env[key])
-        env['PATH'] = ':'.join([
-            element
-            for element in env['PATH'].split(':')
-            if not element.startswith('/snap/charm/')])
-        return env
+        return utils.host_env()
 
     def combine(self, existing):
         ""  # suppress inherited doc

charmtools/utils.py

@@ -633,3 +633,21 @@ def validate_display_name(entity, linter):
         linter.err('display-name: not in valid format. '
                    'Only letters, numbers, dashes, and hyphens are permitted.')
         return
+
+
+def host_env():
+    """Get environment appropriate for executing commands outside snap context.
+
+    :returns: Dictionary with environment variables
+    :rtype: Dict[str,str]
+    """
+    env = os.environ.copy()
+    for key in ('PREFIX', 'PYTHONHOME', 'PYTHONPATH',
+                'GIT_TEMPLATE_DIR', 'GIT_EXEC_PATH'):
+        if key in env:
+            del(env[key])
+    env['PATH'] = ':'.join([
+        element
+        for element in env['PATH'].split(':')
+        if not element.startswith('/snap/charm/')])
+    return env

tests/test_utils.py

@@ -1,5 +1,6 @@
 from __future__ import print_function
 
+import unittest
 from unittest import TestCase
 from charmtools import utils
 from six import StringIO
@@ -43,3 +44,19 @@ def react(db):
         self.assertIn("Beta", output)
         self.assertIn("@when('db.ready'", output)
         self.assertIn("bar", output)
+
+    @unittest.mock.patch("os.environ")
+    def test_host_env(self, mock_environ):
+        mock_environ.copy.return_value = {
+            'PREFIX': 'fake-prefix',
+            'PYTHONHOME': 'fake-pythonhome',
+            'PYTHONPATH': 'fake-pythonpath',
+            'GIT_TEMPLATE_DIR': 'fake-git-template-dir',
+            'GIT_EXEC_PATH': 'fake-git-exec-path',
+            'SOME_OTHER_KEY': 'fake-some-other-key',
+            'PATH': '/snap/charm/current/bin:/usr/bin:'
+                    '/snap/charm/current/usr/bin:/bin',
+        }
+        self.assertDictEqual(
+            {'SOME_OTHER_KEY': 'fake-some-other-key', 'PATH': '/usr/bin:/bin'},
+            utils.host_env())