Skip to content

Commit

Permalink
updates
Browse files Browse the repository at this point in the history
  • Loading branch information
@Atticus1806
Atticus1806 committed Aug 17, 2023
1 parent 69f6c1e commit 03cd77b
Show file tree
Hide file tree
Showing 3 changed files with 22 additions and 42 deletions.
6 changes: 2 additions & 4 deletions tests/job_tests/recognition/files/hyp.ctm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,7 @@ File 1 11.00 01.00 good 1.0000
File 1 13.00 01.00 no 1.0000
File 1 14.00 01.00 clue 1.0000
File 1 15.00 01.00 what 1.0000
File 1 16.00 01.00 was 1.0000
File 1 17.00 01.00 recognized 1.0000
File 1 18.00 01.00 here 1.0000
File 1 16.00 03.00 was 1.0000
;; TestCorpus/File/4 (20.00-22.00)
File 1 20.00 01.00 insertion 1.0000
File 1 21.00 01.00 case 1.0000
File 1 21.00 01.00 case 1.0000
2 changes: 1 addition & 1 deletion tests/job_tests/recognition/files/ref.stm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@ File 1 Test 00.00 04.00 <d0> this one is correct
File 1 Test 06.00 12.00 <d0> this one is a bit longer
File 1 Test 13.00 19.00 <d0> this one is also quite long
File 1 Test 20.00 22.00 <d0> case
;; LABEL "d0" "default0" "all other segments of category 0"
;; LABEL "d0" "default0" "all other segments of category 0"
56 changes: 19 additions & 37 deletions tests/job_tests/recognition/test_scoring.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ def compile_sctk(
branch: Optional[str] = None,
commit: Optional[str] = None,
sctk_git_repository: str = "https://github.com/usnistgov/SCTK.git",
tmpdir=None,
) -> tk.Path:
"""
:param branch: specify a specific branch
Expand All @@ -24,16 +23,13 @@ def compile_sctk(
:return: SCTK binary folder
"""
sctk_job = CloneGitRepositoryJob(url=sctk_git_repository, branch=branch, commit=commit)
sctk_job.out_repository = Path(os.path.join(tmpdir, "sctk_git"))
sctk_job.run()

sctk_make = MakeJob(
folder=sctk_job.out_repository,
make_sequence=["config", "all", "check", "install", "doc"],
link_outputs={"bin": "bin/"},
)
sctk_make.out_repository = Path(os.path.join(tmpdir, "sctk_compiled"))
sctk_make.out_links["bin"] = Path(os.path.join(tmpdir, "sctk_bin"))
sctk_make.run()
# This is needed for the compilation to work in the i6 environment, otherwise still untested
sctk_make._sis_environment.set("CPPFLAGS", "-std=c++11")
Expand All @@ -42,68 +38,54 @@ def compile_sctk(

def test_sclite_job():
with tempfile.TemporaryDirectory() as tmpdir:
sctk_binary = compile_sctk(tmpdir=tmpdir, branch="v2.4.12")
from sisyphus import gs

gs.WORK_DIR = tmpdir
sctk_binary = compile_sctk(branch="v2.4.12")
hyp = Path("files/hyp.ctm")
ref = Path("files/ref.stm")

sclite_job = ScliteJob(ref=ref, hyp=hyp, sctk_binary_path=sctk_binary)

sclite_job.out_report_dir = Path(os.path.join(tmpdir, "reports"))
os.makedirs(os.path.join(tmpdir, "reports"))
sclite_job._sis_output_dirs.add(sclite_job.out_report_dir)
sclite_job.out_wer = tk.Variable(os.path.join(tmpdir, "wer"))
sclite_job.out_num_errors = tk.Variable(os.path.join(tmpdir, "num_errors"))
sclite_job.out_percent_correct = tk.Variable(os.path.join(tmpdir, "percent_correct"))
sclite_job.out_num_correct = tk.Variable(os.path.join(tmpdir, "num_correct"))
sclite_job.out_percent_substitution = tk.Variable(os.path.join(tmpdir, "percent_substitution"))
sclite_job.out_num_substitution = tk.Variable(os.path.join(tmpdir, "num_substitution"))
sclite_job.out_percent_deletions = tk.Variable(os.path.join(tmpdir, "percent_deletions"))
sclite_job.out_num_deletions = tk.Variable(os.path.join(tmpdir, "num_deletions"))
sclite_job.out_percent_insertions = tk.Variable(os.path.join(tmpdir, "percent_insertions"))
sclite_job.out_num_insertions = tk.Variable(os.path.join(tmpdir, "num_insertions"))
sclite_job.out_percent_word_accuracy = tk.Variable(os.path.join(tmpdir, "percent_word_accuracy"))
sclite_job.out_ref_words = tk.Variable(os.path.join(tmpdir, "ref_words"))
sclite_job.out_hyp_words = tk.Variable(os.path.join(tmpdir, "hyp_words"))
sclite_job.out_aligned_words = tk.Variable(os.path.join(tmpdir, "aligned_words"))
sclite_job._sis_setup_directory()
sclite_job.run()

assert sclite_job.out_wer.get() == 68.2, "Wrong WER, %s instead of 68.2" % str(sclite_job.out_wer.get())
assert sclite_job.out_num_errors.get() == 15, "Wrong num errors, %s instead of 15" % str(
assert sclite_job.out_wer.get() == 58.8, "Wrong WER, %s instead of 58.8" % str(sclite_job.out_wer.get())
assert sclite_job.out_num_errors.get() == 10, "Wrong num errors, %s instead of 10" % str(
sclite_job.out_num_errors.get()
)
assert sclite_job.out_percent_correct.get() == 36.4, "Wrong percent correct, %s instead of 36.4" % str(
assert sclite_job.out_percent_correct.get() == 47.1, "Wrong percent correct, %s instead of 47.1" % str(
sclite_job.out_percent_correct.get()
)
assert sclite_job.out_num_correct.get() == 8, "Wrong num correct, %s instead of 8" % str(
sclite_job.out_num_correct.get()
)
assert (
sclite_job.out_percent_substitution.get() == 40.9
), "Wrong percent substitution, %s instead of 40.9" % str(sclite_job.out_percent_substitution.get())
assert sclite_job.out_num_substitution.get() == 9, "Wrong num substitution, %s instead of 9" % str(
sclite_job.out_percent_substitution.get() == 41.2
), "Wrong percent substitution, %s instead of 41.2" % str(sclite_job.out_percent_substitution.get())
assert sclite_job.out_num_substitution.get() == 7, "Wrong num substitution, %s instead of 7" % str(
sclite_job.out_num_substitution.get()
)
assert sclite_job.out_percent_deletions.get() == 22.7, "Wrong percent deletions, %s instead of 22.7" % str(
assert sclite_job.out_percent_deletions.get() == 11.8, "Wrong percent deletions, %s instead of 11.8" % str(
sclite_job.out_percent_deletions.get()
)
assert sclite_job.out_num_deletions.get() == 5, "Wrong num deletions, %s instead of 5" % str(
assert sclite_job.out_num_deletions.get() == 2, "Wrong num deletions, %s instead of 2" % str(
sclite_job.out_num_deletions.get()
)
assert sclite_job.out_percent_insertions.get() == 4.5, "Wrong percent insertions, %s instead of 4.5" % str(
assert sclite_job.out_percent_insertions.get() == 5.9, "Wrong percent insertions, %s instead of 4.5" % str(
sclite_job.out_percent_insertions.get()
)
assert sclite_job.out_num_insertions.get() == 1, "Wrong num insertions, %s instead of 1" % str(
sclite_job.out_num_insertions.get()
)
assert (
sclite_job.out_percent_word_accuracy.get() == 31.8
), "Wrong percent word accuracy, %s instead of 31.8" % str(sclite_job.out_percent_word_accuracy.get())
assert sclite_job.out_ref_words.get() == 22, "Wrong num ref words, %s instead of 22" % str(
sclite_job.out_percent_word_accuracy.get() == 41.2
), "Wrong percent word accuracy, %s instead of 41.2" % str(sclite_job.out_percent_word_accuracy.get())
assert sclite_job.out_ref_words.get() == 17, "Wrong num ref words, %s instead of 17" % str(
sclite_job.out_ref_words.get()
)
assert sclite_job.out_hyp_words.get() == 18, "Wrong num hyp words, %s instead of 18" % str(
assert sclite_job.out_hyp_words.get() == 16, "Wrong num hyp words, %s instead of 16" % str(
sclite_job.out_hyp_words.get()
)
assert sclite_job.out_aligned_words.get() == 23, "Wrong num aligned words, %s instead of 23" % str(
assert sclite_job.out_aligned_words.get() == 18, "Wrong num aligned words, %s instead of 18" % str(
sclite_job.out_aligned_words.get()
)

0 comments on commit 03cd77b

Please sign in to comment.