From 53f3f8cb0d0ae03e3f9709639d309c0e5263fb5c Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Fri, 1 Nov 2024 15:47:31 +0100 Subject: [PATCH] builder: defer submodule detection to git --- src/tito/builder/submodule_aware_builder.py | 86 ++++++++------------- 1 file changed, 31 insertions(+), 55 deletions(-) diff --git a/src/tito/builder/submodule_aware_builder.py b/src/tito/builder/submodule_aware_builder.py index 70d96dd6..d49c30fa 100644 --- a/src/tito/builder/submodule_aware_builder.py +++ b/src/tito/builder/submodule_aware_builder.py @@ -27,7 +27,6 @@ find_spec_like_file, get_commit_timestamp, ) -from tito.exception import TitoException from tito.tar import TarFixer @@ -109,54 +108,31 @@ def run_git_archive(self, relative_git_dir, prefix, commit, dest_tar, subdir=Non "%s > /dev/null" % git_archive_cmd, ) - # pylint: disable=too-many-locals, too-many-arguments, consider-using-f-string - def _submodule_archives(self, relative_git_dir, prefix, commit, initial_tar, - submodule_tree="."): - with chdir(submodule_tree): - submodules_cmd = "git config --file .gitmodules --get-regexp path" - submodules_output = run_command(submodules_cmd) - - # split submodules output on newline - # then on tab, and the directory is the last entry - submodules_list = [ - line.split(" ")[-1] for line in submodules_output.split("\n") - ] - - # We ignore the hash in the sub modules list as we'll have to get the correct one - # from the commit id in commit - for submodule in submodules_list: - with chdir(submodule_tree): - submodule_git_dir = os.path.join(submodule, ".git") - if not os.path.exists(submodule_git_dir): - raise TitoException("%r path does not contain '.git'. " - "Have you cloned the repository recursively?\n" - "Try `git submodule update --init --recursive`!" % - os.path.abspath(submodule)) - # to find the submodule shars: - # git rev-parse :./ - rev_parse_cmd = "git rev-parse %s:./%s" % (commit, submodule) - submodule_commit = run_command(rev_parse_cmd) - submodule_tar_file = "%s.%s" % (initial_tar, submodule.replace("/", "_")) - # prefix should be / - submodule_prefix = "%s/%s" % (prefix, submodule) - - self.run_git_archive( - relative_git_dir, - submodule_prefix, - submodule_commit, - submodule_tar_file, - submodule, - ) - yield (submodule_tar_file) - - submodule_dir = os.path.join(submodule_tree, submodule) - gitmodules_path = os.path.join(submodule_dir, ".gitmodules") - if os.path.exists(gitmodules_path): - for archive in self._submodule_archives( - relative_git_dir, submodule_prefix, submodule_commit, - submodule_tar_file, submodule_dir - ): - yield archive + def _submodule_archives(self, relative_git_dir, prefix, initial_tar, source_tree="."): + with chdir(source_tree): + # Let git handle edge cases for .gitmodules (eg: empty files etc) + submodules_status_cmd = "git submodule status --recursive" + submodules_status_output = run_command(submodules_status_cmd) + + for line in submodules_status_output.strip().split("\n"): + row = line.split() + submodule_commit, submodule_relative_dir = row[0], row[1] + + submodule_tar_file_suffix = submodule_relative_dir.replace("/", "_") + submodule_tar_file = f"{initial_tar}.{submodule_tar_file_suffix}" + + # prefix should be / + submodule_prefix = f"{prefix}/{submodule_relative_dir}" + + self.run_git_archive( + relative_git_dir, + submodule_prefix, + submodule_commit, + submodule_tar_file, + submodule_relative_dir, + ) + + yield submodule_tar_file def create_tgz(self, git_root, prefix, commit, relative_dir, dest_tgz): """ @@ -176,12 +152,12 @@ def create_tgz(self, git_root, prefix, commit, relative_dir, dest_tgz): timestamp = get_commit_timestamp(commit) # Accommodate standalone projects with specfile in root of git repo: - relative_git_dir = "%s" % relative_dir + relative_git_dir = str(relative_dir) if relative_git_dir in ["/", "./"]: relative_git_dir = "" basename = os.path.splitext(dest_tgz)[0] - initial_tar = "%s.initial" % basename + initial_tar = f"{basename}.initial" # We need to tar up the following: # 1. the current repo @@ -190,11 +166,11 @@ def create_tgz(self, git_root, prefix, commit, relative_dir, dest_tgz): # 2. all of the submodules # then combine those into a single archive. for submodule_tar_file in self._submodule_archives( - relative_git_dir, prefix, commit, initial_tar + relative_git_dir, prefix, initial_tar ): - run_command("tar -Af %s %s" % (initial_tar, submodule_tar_file)) + run_command(f"tar -Af {initial_tar} {submodule_tar_file}") - fixed_tar = "%s.tar" % basename + fixed_tar = f"{basename}.tar" fixed_tar_fh = open(fixed_tar, "wb") try: tarfixer = TarFixer( @@ -205,4 +181,4 @@ def create_tgz(self, git_root, prefix, commit, relative_dir, dest_tgz): fixed_tar_fh.close() # It's a pity we can't use Python's gzip, but it doesn't offer an equivalent of -n - return run_command("gzip -n -c < %s > %s" % (fixed_tar, dest_tgz)) + return run_command(f"gzip -n -c < {fixed_tar} > {dest_tgz}")