From ea7ee526e331c635c8b05741a925aca6cbe5b5f4 Mon Sep 17 00:00:00 2001 From: Johannes Rudolph Date: Wed, 15 Apr 2020 16:17:37 +0200 Subject: [PATCH 1/2] fix: deepen shallow clones using --deepen partially fixes #315 by preventing moves of the remote HEAD encountered while fetching affecting the final fetch result Signed-off-by: Johannes Rudolph --- README.md | 3 +- ...ow_clone_until_ref_is_found_then_check_out | 13 +++++--- test/get.sh | 32 +++++++++++-------- 3 files changed, 28 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 7895b2fa..46721ffd 100644 --- a/README.md +++ b/README.md @@ -196,7 +196,8 @@ correct key is provided set in `git_crypt_key`. repository using the `--depth` option. To prevent newer commits that do not pass a `paths` filter test from skewing the cloned history away from `version.ref`, this resource will automatically deepen the clone until - `version.ref` is found again. + `version.ref` is found again. It will deepen with exponentially increasing steps + until a maximum of 127 commits and after that resort to unshallow the repository. * `submodules`: *Optional.* If `none`, submodules will not be fetched. If specified as a list of paths, only the given paths will be diff --git a/assets/deepen_shallow_clone_until_ref_is_found_then_check_out b/assets/deepen_shallow_clone_until_ref_is_found_then_check_out index 881ff743..b8d12b3a 100755 --- a/assets/deepen_shallow_clone_until_ref_is_found_then_check_out +++ b/assets/deepen_shallow_clone_until_ref_is_found_then_check_out @@ -3,12 +3,14 @@ set -e -readonly max_depth=128 +readonly max_depth=127 declare depth="$1" readonly ref="$2" readonly tagflag="$3" +declare total_depth="$depth" + # A shallow clone may not contain the Git commit $ref: # 1. The depth of the shallow clone is measured backwards from the latest # commit on the given head (master or branch), and in the meantime there may @@ -32,19 +34,20 @@ while ! git checkout -q "$ref" &>/dev/null; do break fi - echo "Could not find ref ${ref} in a shallow clone of depth ${depth}" + echo "Could not find ref ${ref} in a shallow clone of depth ${total_depth}" (( depth *= 2 )) - if [ "$depth" -gt "$max_depth" ]; then + if [ "$total_depth" -ge "$max_depth" ]; then echo "Reached depth threshold ${max_depth}, falling back to deep clone..." git fetch --unshallow origin $tagflag break fi - echo "Deepening the shallow clone to depth ${depth}..." - git fetch --depth "$depth" origin $tagflag + echo "Deepening the shallow clone by an additional ${depth}..." + git fetch --deepen "$depth" origin $tagflag + (( total_depth += depth )) done git checkout -q "$ref" diff --git a/test/get.sh b/test/get.sh index 7c7ad4b2..ef486744 100755 --- a/test/get.sh +++ b/test/get.sh @@ -199,8 +199,8 @@ it_falls_back_to_deep_clone_if_ref_not_found() { local repo=$(init_repo) local ref1=$(make_commit $repo) - # 128 is the threshold when it starts doing a deep clone - for (( i = 0; i < 128; i++ )); do + # 127 is the threshold when it starts doing a deep clone + for (( i = 0; i < 127; i++ )); do make_commit $repo >/dev/null done @@ -214,16 +214,16 @@ it_falls_back_to_deep_clone_if_ref_not_found() { test "$(git -C $dest rev-parse HEAD)" = $ref1 echo "testing for msg 1" >&2 - for d in 1 2 4 8 16 32 64 128; do + for d in 1 3 7 15 31 63 127; do grep "Could not find ref ${ref1} in a shallow clone of depth ${d}" <$TMPDIR/stderr done echo "test for msg 1 done" >&2 - for d in 2 4 8 16 32 64 128; do - grep "Deepening the shallow clone to depth ${d}..." <$TMPDIR/stderr + for d in 2 4 8 16 32 64; do + grep "Deepening the shallow clone by an additional ${d}..." <$TMPDIR/stderr done - grep "Reached depth threshold 128, falling back to deep clone..." <$TMPDIR/stderr + grep "Reached depth threshold 127, falling back to deep clone..." <$TMPDIR/stderr } it_does_not_enter_an_infinite_loop_if_the_ref_cannot_be_found_and_depth_is_set() { @@ -343,8 +343,8 @@ it_falls_back_to_deep_clone_of_submodule_if_ref_not_found() { local main_repo_last_commit_id=$(git -C $main_repo rev-parse HEAD) local submodule_repo_last_commit_id=$(git -C $submodule_repo rev-parse HEAD) - # 128 is the threshold when it starts doing a deep clone - for (( i = 0; i < 128; i++ )); do + # 127 is the threshold when it starts doing a deep clone + for (( i = 0; i < 127; i++ )); do make_commit $submodule_repo >/dev/null done @@ -360,17 +360,21 @@ it_falls_back_to_deep_clone_of_submodule_if_ref_not_found() { test "$(git -C $main_repo rev-parse HEAD)" = $main_repo_last_commit_id - echo "testing for msg 1" >&2 - for d in 1 2 4 8 16 32 64 128; do + echo "testing for 'could not find' messages'" >&2 + for d in 1 3 7 15 31 63 127; do grep "Could not find ref ${submodule_repo_last_commit_id} in a shallow clone of depth ${d}" <$TMPDIR/stderr done - echo "test for msg 1 done" >&2 + echo "testing for 'could not find' messages done" >&2 - for d in 2 4 8 16 32 64 128; do - grep "Deepening the shallow clone to depth ${d}..." <$TMPDIR/stderr + echo "testing for 'deepening' messages'" >&2 + for d in 2 4 8 16 32 64; do + grep "Deepening the shallow clone by an additional ${d}..." <$TMPDIR/stderr done + echo "testing for 'deepening' messages' done" >&2 - grep "Reached depth threshold 128, falling back to deep clone..." <$TMPDIR/stderr + echo "testing for 'reached threshold' message'" >&2 + grep "Reached depth threshold 127, falling back to deep clone..." <$TMPDIR/stderr + echo "testing for 'reached threshold' message done'" >&2 } it_fails_if_the_ref_cannot_be_found_while_deepening_a_submodule() { From 87b6f9d42fc02835f6e1bea8b0a82c4107dc982a Mon Sep 17 00:00:00 2001 From: Johannes Rudolph Date: Wed, 15 Apr 2020 16:35:13 +0200 Subject: [PATCH 2/2] fix: consider $depth when deepening shallow clones fixes #315 by ensuring we always unshallow to the desired depth below version.ref Signed-off-by: Johannes Rudolph --- README.md | 2 +- ...ow_clone_until_ref_is_found_then_check_out | 5 ++- test/get.sh | 45 ++++++++++++++++++- 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 46721ffd..a93d8828 100644 --- a/README.md +++ b/README.md @@ -197,7 +197,7 @@ correct key is provided set in `git_crypt_key`. not pass a `paths` filter test from skewing the cloned history away from `version.ref`, this resource will automatically deepen the clone until `version.ref` is found again. It will deepen with exponentially increasing steps - until a maximum of 127 commits and after that resort to unshallow the repository. + until a maximum of 127 + `depth` commits or else resort to unshallow the repository. * `submodules`: *Optional.* If `none`, submodules will not be fetched. If specified as a list of paths, only the given paths will be diff --git a/assets/deepen_shallow_clone_until_ref_is_found_then_check_out b/assets/deepen_shallow_clone_until_ref_is_found_then_check_out index b8d12b3a..7f7a3733 100755 --- a/assets/deepen_shallow_clone_until_ref_is_found_then_check_out +++ b/assets/deepen_shallow_clone_until_ref_is_found_then_check_out @@ -10,6 +10,7 @@ readonly ref="$2" readonly tagflag="$3" declare total_depth="$depth" +declare depth_from_ref=$((depth - 1)) # A shallow clone may not contain the Git commit $ref: # 1. The depth of the shallow clone is measured backwards from the latest @@ -26,7 +27,7 @@ declare total_depth="$depth" git_dir="$(git rev-parse --git-dir)" readonly git_dir -while ! git checkout -q "$ref" &>/dev/null; do +while ! git checkout -q "$ref~$depth_from_ref" &>/dev/null; do # once the depth of a shallow clone reaches the max depth of the origin # repo, Git silenty turns it into a deep clone if [ ! -e "$git_dir"/shallow ]; then @@ -34,7 +35,7 @@ while ! git checkout -q "$ref" &>/dev/null; do break fi - echo "Could not find ref ${ref} in a shallow clone of depth ${total_depth}" + echo "Could not find ref ${ref}~${depth_from_ref} in a shallow clone of depth ${total_depth}" (( depth *= 2 )) diff --git a/test/get.sh b/test/get.sh index ef486744..50ce9dbd 100755 --- a/test/get.sh +++ b/test/get.sh @@ -215,7 +215,7 @@ it_falls_back_to_deep_clone_if_ref_not_found() { echo "testing for msg 1" >&2 for d in 1 3 7 15 31 63 127; do - grep "Could not find ref ${ref1} in a shallow clone of depth ${d}" <$TMPDIR/stderr + grep "Could not find ref ${ref1}~0 in a shallow clone of depth ${d}" <$TMPDIR/stderr done echo "test for msg 1 done" >&2 @@ -226,6 +226,46 @@ it_falls_back_to_deep_clone_if_ref_not_found() { grep "Reached depth threshold 127, falling back to deep clone..." <$TMPDIR/stderr } +it_considers_depth_if_ref_not_found() { + local repo=$(init_repo) + local depth=3 + + # make commits we're interested + local ref0=$(make_commit $repo) + local ref1=$(make_commit $repo) + local ref2=$(make_commit $repo) + local ref3=$(make_commit $repo) + + # make a total of 22 commits, so that ref0 is never fetched + for (( i = 0; i < 18; i++ )); do + make_commit $repo >/dev/null + done + + local dest=$TMPDIR/destination + + ( get_uri_at_depth_at_ref "file://$repo" $depth $ref3 $dest 3>&2- 2>&1- 1>&3- 3>&- | tee $TMPDIR/stderr ) 3>&1- 1>&2- 2>&3- 3>&- | jq -e " + .version == {ref: $(echo $ref3 | jq -R .)} + " + + test -e $dest/some-file + test "$(git -C $dest rev-parse HEAD)" = $ref3 + test "$(git -C $dest rev-parse HEAD~1)" = $ref2 + test "$(git -C $dest rev-parse HEAD~2)" = $ref1 + test -e "$dest/.git/shallow" # it's still shallow + + echo "testing for 'could not find' messages'" >&2 + for d in 3 9; do + grep "Could not find ref ${ref3}~2 in a shallow clone of depth ${d}" <$TMPDIR/stderr + done + echo "testing for 'could not find' messages done" >&2 + + echo "testing for 'deepening' messages'" >&2 + for d in 6 12; do + grep "Deepening the shallow clone by an additional ${d}..." <$TMPDIR/stderr + done + echo "testing for 'deepening' messages' done" >&2 +} + it_does_not_enter_an_infinite_loop_if_the_ref_cannot_be_found_and_depth_is_set() { local repo=$(init_repo) local ref1=$(make_commit $repo) @@ -362,7 +402,7 @@ it_falls_back_to_deep_clone_of_submodule_if_ref_not_found() { echo "testing for 'could not find' messages'" >&2 for d in 1 3 7 15 31 63 127; do - grep "Could not find ref ${submodule_repo_last_commit_id} in a shallow clone of depth ${d}" <$TMPDIR/stderr + grep "Could not find ref ${submodule_repo_last_commit_id}~0 in a shallow clone of depth ${d}" <$TMPDIR/stderr done echo "testing for 'could not find' messages done" >&2 @@ -739,6 +779,7 @@ run it_returns_list_of_tags_in_metadata run it_honors_the_depth_flag run it_can_get_from_url_at_depth_at_ref run it_falls_back_to_deep_clone_if_ref_not_found +run it_considers_depth_if_ref_not_found run it_does_not_enter_an_infinite_loop_if_the_ref_cannot_be_found_and_depth_is_set run it_can_get_and_set_git_config run it_returns_same_ref