Merge pull request #1 from pelson/compose_recipes

Added a tool for flattening directories of recipes.

centrally_managed_conda/compose_recipes.py

@@ -0,0 +1,69 @@
+import argparse
+from collections import OrderedDict
+import itertools
+import os.path
+import shutil
+import sys
+
+import conda_build.config
+from conda_build_all.resolved_distribution import ResolvedDistribution
+from conda_build_all.builder import list_metas
+
+
+def flatten_metas(meta_iterables):
+    """
+    Take a collection of metas, and compose/flatten/project into a single list.
+
+    For example:
+
+        A: pkg1, pkg2a
+        B: pkg2b, pkg3
+
+        Flattened([A, B]) => [pkg1, pkg2a, pkg3]
+        Flattened([B, A]) => [pkg1, pkg2b, pkg3]
+
+    The resulting list of metas will not be ordered in any particular way.
+
+    """
+    visited = {}
+    for metas in meta_iterables:
+        visited_this_depth = {}
+        for meta in metas:
+            if meta.name() not in visited:
+                visited_this_depth.setdefault(meta.name(), []).append(meta)
+        for name, metas in visited_this_depth.items():
+            visited.setdefault(name, []).extend(metas)
+    return itertools.chain.from_iterable(visited.values())
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Removing duplicate recipes which are lower down the pecking order.')
+    parser.add_argument('recipes_dirs', nargs="+",
+                        help=("The directories containing recipes which should be 'flattened'."))
+    parser.add_argument('--output-dir', help='Directory which should be created containing flattened recipes.',
+                        default='flattened_recipes')
+    args = parser.parse_args()
+
+    meta_collections = OrderedDict([(recipes_dir, list_metas(recipes_dir))
+                                    for recipes_dir in args.recipes_dirs])
+
+    flattened = list(flatten_metas(meta_collections.values()))
+
+    flattened_collections = OrderedDict()
+    for recipe_dir, metas in meta_collections.items():
+        for meta in metas:
+            if meta in flattened:
+                flattened_collections.setdefault(recipe_dir, []).append(meta)
+
+    for recipe_dir, metas in flattened_collections.items():
+        recipes_parent_dir = os.path.dirname(os.path.abspath(recipe_dir))
+        for meta in metas:
+            # Figure out where the recipe is, relative to the recipe dir
+            meta_dir = os.path.relpath(os.path.abspath(meta.path), recipes_parent_dir)
+            target_locn = os.path.join(args.output_dir, meta_dir)
+            shutil.copytree(meta.path, target_locn)
+            print('Copying {}'.format(meta_dir))
+
+
+if __name__ == '__main__':
+    main()

centrally_managed_conda/tests/unit/test_compose_recipes.py

@@ -0,0 +1,83 @@
+import os
+import subprocess
+import sys
+import unittest
+
+from conda_build_all.tests.unit.dummy_index import DummyPackage, DummyIndex
+from conda_build_all.tests.integration.test_builder import RecipeCreatingUnit
+
+from centrally_managed_conda.compose_recipes import flatten_metas
+
+
+class Test_flatten_metas(unittest.TestCase):
+    def setUp(self):
+        self.pkgs = {'a': DummyPackage('a', version='1.0'),
+                     'b': DummyPackage('b', version='2.1'),
+                     'c3': DummyPackage('c', version='3.2'),
+                     'c4': DummyPackage('c', version='4.5'),
+                    }
+
+    def test(self):
+        channel1 = [self.pkgs['a'], self.pkgs['c3']]
+        channel2 = [self.pkgs['a'], self.pkgs['b'], self.pkgs['c4']]
+
+        pkgs = flatten_metas([channel1, channel2])
+
+        dists = sorted([pkg.dist() for pkg in pkgs])
+        self.assertEqual(dists, ['a-1.0-0', 'b-2.1-0', 'c-3.2-0'])
+
+        pkgs = flatten_metas([channel2, channel1])
+
+        dists = sorted([pkg.dist() for pkg in pkgs])
+        self.assertEqual(dists, ['a-1.0-0', 'b-2.1-0', 'c-4.5-0'])
+
+
+class Test_cli(RecipeCreatingUnit):
+    def test(self):
+        a1 = self.write_meta(os.path.join('channel1', 'a'),
+                             """
+                             package:
+                                 name: a
+                                 version: 1
+                             """)
+        a2 = self.write_meta(os.path.join('channel2', 'a'),
+                             """
+                             package:
+                                 name: a
+                                 version: 2
+                             """)
+        b1 = self.write_meta(os.path.join('channel1', 'b'),
+                             """
+                             package:
+                                 name: b
+                                 version: 1
+                             """)
+        c1 = self.write_meta(os.path.join('channel2', 'c'),
+                             """
+                             package:
+                                 name: c
+                                 version: 1
+                             """)
+
+        channel1 = os.path.join(self.recipes_root_dir, 'channel1')
+        channel2 = os.path.join(self.recipes_root_dir, 'channel2')
+        output = os.path.join(self.recipes_root_dir, 'flattened_recipes')
+
+        out = subprocess.check_output([sys.executable, '-m', 'centrally_managed_conda.compose_recipes',
+                                     '--output-dir={}'.format(output),
+                                     channel1, channel2])
+        self.assertEqual(out.strip().decode('utf-8'),
+                         '\n'.join(['Copying channel1/a',
+                                    'Copying channel1/b',
+                                    'Copying channel2/c']))
+        if out.strip():
+            print(out.decode('utf-8'))
+        # Check they are all there, except for a2.
+        self.assertTrue(os.path.exists(os.path.join(output, 'channel1', 'a')))
+        self.assertTrue(os.path.exists(os.path.join(output, 'channel1', 'b')))
+        self.assertTrue(os.path.exists(os.path.join(output, 'channel2', 'c')))
+        self.assertFalse(os.path.exists(os.path.join(output, 'channel2', 'a')))
+
+
+if __name__ == '__main__':
+    unittest.main()