Drop unused data from sdist tarball

Hardcode a list of unused source directories and build system blobs to omit from the source tarball. This reduces the size of the development tarball by a factor of five, from 90 MiB to 18 MiB. Signed-off-by: Benjamin Gilbert <[email protected]>
openslide · Sep 28, 2024 · 04af5e6 · 04af5e6
1 parent 2bb7b1e
commit 04af5e6
Show file tree

Hide file tree

Showing 3 changed files with 83 additions and 5 deletions.
diff --git a/artifacts/postprocess-sdist.py b/artifacts/postprocess-sdist.py
@@ -33,6 +33,11 @@
 from common.argparse import TypedArgs  # noqa: E402
 from common.meson import meson_introspect, meson_source_root  # noqa: E402
 from common.python import pyproject_to_message  # noqa: E402
+from common.software import Project  # noqa: E402
+
+
+def walkerr(e: OSError) -> None:
+    raise e
 
 
 class Args(TypedArgs):
@@ -49,10 +54,79 @@ class Args(TypedArgs):
 os.environ['MESONINTROSPECT'] = args.introspect
 src = meson_source_root()
 dest = Path(os.environ['MESON_DIST_ROOT'])
+subprojects = dest / 'subprojects'
 
 # remove those parts of .github not ignored from .gitattributes
 shutil.rmtree(dest / '.github')
 
+# remove some unused subproject directories to reduce tarball size
+REMOVE_SUBDIRS = {
+    'cairo': ['doc', 'perf', 'test'],
+    'gdk-pixbuf': ['tests'],
+    'glib': ['gio/tests', 'glib/tests', 'gobject/tests', 'po'],
+    'libdicom': ['doc/html'],
+    'libffi': ['doc', 'testsuite'],
+    'libjpeg-turbo': ['doc', 'java', 'testimages'],
+    'libopenjp2': ['cmake', 'doc', 'tests', 'thirdparty', 'tools', 'wrapping'],
+    'libpng': ['ci', 'contrib', 'projects'],
+    'libtiff': [
+        'cmake',
+        'config',
+        'contrib',
+        'doc',
+        'test/images',
+        'test/refs',
+        'tools',
+    ],
+    'libxml2': ['fuzz', 'os400', 'python', 'result', 'test'],
+    'openslide': ['doc'],
+    'pcre2': ['doc', 'testdata'],
+    'pixman': ['demos', 'test'],
+    'uthash': ['doc', 'tests'],
+    'zlib': ['contrib', 'doc', 'examples'],
+    'zstd': ['contrib', 'doc', 'programs', 'tests', 'zlibWrapper'],
+}
+for proj_id, subdirs in REMOVE_SUBDIRS.items():
+    proj = Project.get(proj_id)
+    projdir = subprojects / proj.wrap_dir_name
+    # dev deps may be omitted from the tarball
+    if projdir.exists():
+        for subdir in subdirs:
+            shutil.rmtree(projdir / subdir)
+
+# remove blobs from other build systems
+REMOVE_FILENAMES = {
+    'CMakeLists.txt',
+    'configure',
+    'configure.ac',
+    'ltmain.sh',
+    'Makefile',
+    'Makefile.am',
+    'Makefile.in',
+}
+REMOVE_SUFFIXES = {'.cmake', '.m4'}
+ALLOW_FILES = {
+    'glib': [
+        'm4macros/glib-2.0.m4',
+        'm4macros/glib-gettext.m4',
+        'm4macros/gsettings.m4',
+    ],
+    'libjpeg-turbo': ['simd/CMakeLists.txt'],
+    'libxml2': ['libxml.m4'],
+}
+allow_paths = set()
+for proj_id, relpaths in ALLOW_FILES.items():
+    proj = Project.get(proj_id)
+    for relpath in relpaths:
+        allow_paths.add(subprojects / proj.wrap_dir_name / relpath)
+for dirpath, _, filenames in subprojects.walk(on_error=walkerr):
+    for filename in filenames:
+        path = dirpath / filename
+        if path in allow_paths:
+            continue
+        if path.name in REMOVE_FILENAMES or path.suffix in REMOVE_SUFFIXES:
+            path.unlink()
+
 # pin openslide-bin version suffix
 version: str = meson_introspect('projectinfo')['version']
 try:

diff --git a/common/software.py b/common/software.py
@@ -149,14 +149,18 @@ def wrap_path(self) -> Path:
     def override_path(self) -> Path:
         return meson_source_root() / 'override' / self.id
 
+    @cached_property
+    def wrap_dir_name(self) -> str:
+        return self.wrap.get('wrap-file', 'directory')
+
     @cached_property
     def version(self) -> str:
         try:
             # get the wrapdb_version, including the package revision
             ver = self.wrap.get('wrap-file', 'wrapdb_version', fallback=None)
             if not ver:
                 # older or non-wrapdb wrap; parse the directory name
-                ver = self.wrap.get('wrap-file', 'directory').split('-')[-1]
+                ver = self.wrap_dir_name.split('-')[-1]
             return ver
         except FileNotFoundError:
             # overridden source directory
@@ -178,10 +182,10 @@ def version(self) -> str:
                     return version
             raise Exception(f'Missing project info for {self.id}')
 
-    @cached_property
+    @property
     def source_dir(self) -> Path:
         try:
-            dirname = self.wrap.get('wrap-file', 'directory')
+            dirname = self.wrap_dir_name
         except FileNotFoundError:
             # overridden source directory
             dirname = self.id

diff --git a/deps/meson.build b/deps/meson.build
@@ -105,12 +105,12 @@ subproject(
     'builtin_loaders=bmp',
     'introspection=disabled',
     'gio_sniffing=false',
-    'installed_tests=false',
+    'tests=false',
   ],
 )
 subproject(
   'pixman',
-  default_options : ['openmp=disabled', 'tests=disabled'],
+  default_options : ['demos=disabled', 'openmp=disabled', 'tests=disabled'],
 )
 subproject(
   'cairo',