Refactor alt handling

* Simplify score_file() by using case in instead of nested ifs with regexps.
* Merge record_score() and record_template().
* Alt condition processing no longer stops when a template condition is seen
  but continues processing to verify that all conditions are valid (as the
  documentation says it should). Fixes #478.
* Support alt dirs with deeply nested tracked files (fixes #490).
* Use git ls-files to filter out which tracked files to consider for alt
  processing. Should speed up auto-alt (#505).
* Use nocasematch when comparing distro and distro_family. Fixed #455.

test/test_alt.py

@@ -170,6 +170,21 @@ def test_alt_templates(runner, paths, kind, label):
             assert str(paths.work.join(source_file)) in created
 
 
+@pytest.mark.usefixtures("ds1_copy")
+def test_alt_template_with_condition(runner, paths, tst_arch):
+    """Test template with extra condition"""
+    yadm_dir, yadm_data = setup_standard_yadm_dir(paths)
+
+    suffix = f"##template,arch.not{tst_arch}"
+    utils.create_alt_files(paths, suffix)
+    run = runner([paths.pgm, "-Y", yadm_dir, "--yadm-data", yadm_data, "alt"])
+    assert run.success
+    assert run.err == ""
+
+    created = utils.parse_alt_output(run.out, linked=False)
+    assert len(created) == 0
+
+
 @pytest.mark.usefixtures("ds1_copy")
 @pytest.mark.parametrize("autoalt", [None, "true", "false"])
 def test_auto_alt(runner, yadm_cmd, paths, autoalt):

test/test_alt_copy.py

@@ -40,7 +40,8 @@ def test_alt_copy(runner, yadm_cmd, paths, tst_sys, setting, expect_link, pre_ex
     run = runner(yadm_cmd("alt"))
     assert run.success
     assert run.err == ""
-    assert "Linking" in run.out
+    action = "Copying" if setting is True else "Linking"
+    assert action in run.out
 
     assert alt_path.read() == expected_content
     assert alt_path.islink() == expect_link

test/test_unit_record_score.py

@@ -19,6 +19,7 @@
     echo "SCORES:${alt_scores[@]}"
     echo "TARGETS:${alt_targets[@]}"
     echo "SOURCES:${alt_sources[@]}"
+    echo "TEMPLATE_CMDS:${alt_template_cmds[@]}"
 """
 
 
@@ -38,6 +39,7 @@ def test_dont_record_zeros(runner, yadm):
     assert "SCORES:\n" in run.out
     assert "TARGETS:\n" in run.out
     assert "SOURCES:\n" in run.out
+    assert "TEMPLATE_CMDS:\n" in run.out
 
 
 def test_new_scores(runner, yadm):
@@ -46,9 +48,9 @@ def test_new_scores(runner, yadm):
     script = f"""
         YADM_TEST=1 source {yadm}
         {INIT_VARS}
-        record_score "1" "tgt_one"   "src_one"
-        record_score "2" "tgt_two"   "src_two"
-        record_score "4" "tgt_three" "src_three"
+        record_score "1" "tgt_one"   "src_one"   ""
+        record_score "2" "tgt_two"   "src_two"   ""
+        record_score "4" "tgt_three" "src_three" ""
         {REPORT_RESULTS}
     """
     run = runner(command=["bash"], inp=script)
@@ -58,6 +60,7 @@ def test_new_scores(runner, yadm):
     assert "SCORES:1 2 4\n" in run.out
     assert "TARGETS:tgt_one tgt_two tgt_three\n" in run.out
     assert "SOURCES:src_one src_two src_three\n" in run.out
+    assert "TEMPLATE_CMDS:  \n" in run.out
 
 
 @pytest.mark.parametrize("difference", ["lower", "equal", "higher"])
@@ -81,7 +84,8 @@ def test_existing_scores(runner, yadm, difference):
         alt_scores=(2)
         alt_targets=("testtgt")
         alt_sources=("existing_src")
-        record_score "{score}" "testtgt" "new_src"
+        alt_template_cmds=("")
+        record_score "{score}" "testtgt" "new_src" ""
         {REPORT_RESULTS}
     """
     run = runner(command=["bash"], inp=script)
@@ -91,6 +95,7 @@ def test_existing_scores(runner, yadm, difference):
     assert f"SCORES:{expected_score}\n" in run.out
     assert "TARGETS:testtgt\n" in run.out
     assert f"SOURCES:{expected_src}\n" in run.out
+    assert "TEMPLATE_CMDS:\n" in run.out
 
 
 def test_existing_template(runner, yadm):
@@ -101,9 +106,9 @@ def test_existing_template(runner, yadm):
         {INIT_VARS}
         alt_scores=(1)
         alt_targets=("testtgt")
-        alt_sources=()
+        alt_sources=("src")
         alt_template_cmds=("existing_template")
-        record_score "2" "testtgt" "new_src"
+        record_score "2" "testtgt" "new_src" ""
         {REPORT_RESULTS}
     """
     run = runner(command=["bash"], inp=script)
@@ -112,7 +117,8 @@ def test_existing_template(runner, yadm):
     assert "SIZE:1\n" in run.out
     assert "SCORES:1\n" in run.out
     assert "TARGETS:testtgt\n" in run.out
-    assert "SOURCES:\n" in run.out
+    assert "SOURCES:src\n" in run.out
+    assert "TEMPLATE_CMDS:existing_template\n" in run.out
 
 
 def test_config_first(runner, yadm):
@@ -123,20 +129,61 @@ def test_config_first(runner, yadm):
         YADM_TEST=1 source {yadm}
         {INIT_VARS}
         YADM_CONFIG={config}
-        record_score "1" "tgt_before" "src_before"
-        record_template "tgt_tmp" "cmd_tmp" "src_tmp"
-        record_score "2" "{config}"   "src_config"
-        record_score "3" "tgt_after"  "src_after"
+        record_score "1" "tgt_before" "src_before" ""
+        record_score "1" "tgt_tmp"    "src_tmp"    "cmd_tmp"
+        record_score "2" "{config}"   "src_config" ""
+        record_score "3" "tgt_after"  "src_after"  ""
         {REPORT_RESULTS}
-        echo "CMD_VALUE:${{alt_template_cmds[@]}}"
-        echo "CMD_INDEX:${{!alt_template_cmds[@]}}"
     """
     run = runner(command=["bash"], inp=script)
     assert run.success
     assert run.err == ""
-    assert "SIZE:3\n" in run.out
-    assert "SCORES:2 1 3\n" in run.out
+    assert "SIZE:4\n" in run.out
+    assert "SCORES:2 1 1 3\n" in run.out
     assert f"TARGETS:{config} tgt_before tgt_tmp tgt_after\n" in run.out
     assert "SOURCES:src_config src_before src_tmp src_after\n" in run.out
-    assert "CMD_VALUE:cmd_tmp\n" in run.out
-    assert "CMD_INDEX:2\n" in run.out
+    assert "TEMPLATE_CMDS:  cmd_tmp \n" in run.out
+
+
+def test_new_template(runner, yadm):
+    """Test new template"""
+
+    script = f"""
+        YADM_TEST=1 source {yadm}
+        {INIT_VARS}
+        record_score 0 "tgt_one"   "src_one"   "cmd_one"
+        record_score 0 "tgt_two"   "src_two"   "cmd_two"
+        record_score 0 "tgt_three" "src_three" "cmd_three"
+        {REPORT_RESULTS}
+    """
+    run = runner(command=["bash"], inp=script)
+    assert run.success
+    assert run.err == ""
+    assert "SIZE:3\n" in run.out
+    assert "SCORES:0 0 0\n" in run.out
+    assert "TARGETS:tgt_one tgt_two tgt_three\n" in run.out
+    assert "SOURCES:src_one src_two src_three\n" in run.out
+    assert "TEMPLATE_CMDS:cmd_one cmd_two cmd_three\n" in run.out
+
+
+def test_overwrite_existing_template(runner, yadm):
+    """Overwrite existing templates"""
+
+    script = f"""
+        YADM_TEST=1 source {yadm}
+        {INIT_VARS}
+        alt_scores=(0)
+        alt_targets=("testtgt")
+        alt_template_cmds=("existing_cmd")
+        alt_sources=("existing_src")
+        record_score 0 "testtgt" "new_src" "new_cmd"
+        {REPORT_RESULTS}
+    """
+    run = runner(command=["bash"], inp=script)
+    assert run.success
+    assert run.err == ""
+    assert "SIZE:1\n" in run.out
+    assert "SCORES:0\n" in run.out
+    assert "TARGETS:testtgt\n" in run.out
+    assert "SOURCES:new_src\n" in run.out
+    assert "TEMPLATE_CMDS:new_cmd\n" in run.out

test/test_unit_remove_stale_links.py

@@ -25,7 +25,7 @@ def test_remove_stale_links(runner, yadm, tmpdir, kind, linked):
 
     script = f"""
         YADM_TEST=1 source {yadm}
-        possible_alts=({link})
+        possible_alt_targets=({link})
         alt_linked=({alt_linked})
         function rm() {{ echo rm "$@"; }}
         remove_stale_links

test/test_unit_score_file.py

@@ -89,7 +89,7 @@ def calculate_score(filename):
             else:
                 score = 0
                 break
-        elif label in TEMPLATE_LABELS:
+        elif label not in TEMPLATE_LABELS:
             score = 0
             break
     return score
@@ -190,7 +190,7 @@ def test_score_values(runner, yadm, default, arch, system, distro, cla, host, us
     expected = ""
     for filename, score in filenames.items():
         script += f"""
-            score_file "{filename}"
+            score_file "{filename}" "dest"
             echo "{filename}"
             echo "$score"
         """
@@ -255,7 +255,7 @@ def test_score_values_templates(runner, yadm):
     expected = ""
     for filename, score in filenames.items():
         script += f"""
-            score_file "{filename}"
+            score_file "{filename}" "dest"
             echo "{filename}"
             echo "$score"
         """
@@ -279,7 +279,7 @@ def test_template_recording(runner, yadm, cmd_generated):
 
     script = f"""
         YADM_TEST=1 source {yadm}
-        function record_template() {{ echo "template recorded"; }}
+        function record_score() {{ [ -n "$4" ] && echo "template recorded"; }}
         {mock}
         score_file "testfile##template.kind"
     """
@@ -289,15 +289,15 @@ def test_template_recording(runner, yadm, cmd_generated):
     assert run.out.rstrip() == expected
 
 
-def test_underscores_in_distro_and_family(runner, yadm):
-    """Test replacing spaces in distro / distro_family with underscores"""
+def test_underscores_and_upper_case_in_distro_and_family(runner, yadm):
+    """Test replacing spaces with underscores and lowering case in distro / distro_family"""
     local_distro = "test distro"
     local_distro_family = "test family"
     filenames = {
-        "filename##distro.test distro": 1004,
+        "filename##distro.Test Distro": 1004,
         "filename##distro.test-distro": 0,
         "filename##distro.test_distro": 1004,
-        "filename##distro_family.test family": 1008,
+        "filename##distro_family.test FAMILY": 1008,
         "filename##distro_family.test-family": 0,
         "filename##distro_family.test_family": 1008,
     }

test/utils.py

@@ -12,7 +12,7 @@
 
 # Directory based alternates must have a tracked contained file.
 # This will be the test contained file name
-CONTAINED = "contained_file"
+CONTAINED = "contained_dir/contained_file"
 
 # These variables are used for making include files which will be processed
 # within jinja templates
@@ -84,7 +84,7 @@ def parse_alt_output(output, linked=True):
     """Parse output of 'alt', and return list of linked files"""
     regex = r"Creating (.+) from template (.+)$"
     if linked:
-        regex = r"Linking (.+) to (.+)$"
+        regex = r"(?:Copy|Link)ing (.+) to (.+)$"
     parsed_list = {}
     for line in output.splitlines():
         match = re.match(regex, line)