added shellmetrics

infrastructure/.gitignore

@@ -1 +1,2 @@
 __pycache__/
+target/shellmetrics.sh

infrastructure/Makefile

@@ -1,4 +1,4 @@
-STATIC_OUTPUTS = target/lines_of_code.csv target/nodes_in_scripts.csv target/scripts_to_benchmark.csv
+STATIC_OUTPUTS = target/lines_of_code.csv target/nodes_in_scripts.csv target/scripts_to_benchmark.csv target/shellmetrics.csv
 
 static: $(STATIC_OUTPUTS)
 
@@ -17,6 +17,13 @@ static-test: tests/test_syntax_analysis.py
 clean-static: 
 	rm -f $(STATIC_OUTPUTS)
 
+target/shellmetrics.sh:
+	wget --quiet -O $@ https://raw.githubusercontent.com/shellspec/shellmetrics/b3bfff2af6880443112cdbf2ea449440b30ab9b0/shellmetrics
+	chmod +x $@
+
+target/shellmetrics.csv: get_shellmetrics.py target/shellmetrics.sh
+	python3 get_shellmetrics.py > $@
+
 dynamic:
 
 .PHONY: static dynamic clean-static static-test

infrastructure/get_shellmetrics.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+
+from collections import defaultdict
+from pathlib import Path
+from typing import Optional
+import json
+from subprocess import check_output
+
+from all_scripts import get_all_scripts
+from syntax_analysis import parse_shell_script, count_nodes
+from project_root import get_project_root
+
+root = get_project_root()
+shellmetrics = root / 'infrastructure' / 'target' / 'shellmetrics.sh'
+if not shellmetrics.is_file():
+    raise f'shellmetrics.sh not in `{shellmetrics}`, please put the script at that path'
+
+all_scripts = [s for scripts in get_all_scripts().values() for s in scripts]
+
+output = check_output([shellmetrics, '--csv', '--shell', 'bash', '--no-color', *all_scripts], text=True)
+datas = defaultdict(list)
+for line in output.splitlines()[1:]:
+    file, func, _lineno, lloc, ccn, _lines, _comment, _blank = line.split(',')
+    file = json.loads(file)
+    file = Path(file).relative_to(root)
+    datas[file].append((lloc, ccn))
+
+for file, datas in datas.items():
+    lloc = sum(int(d[0]) for d in datas)
+    ccn = sum(int(d[1]) for d in datas)
+    print(file, lloc, ccn, sep=',')

infrastructure/target/shellmetrics.csv

@@ -0,0 +1,88 @@
+covid-mts/scripts/1.sh,1,1
+covid-mts/scripts/3.sh,1,1
+covid-mts/scripts/4.sh,1,1
+covid-mts/scripts/2.sh,1,1
+covid-mts/scripts/5.sh,1,1
+file-enc/scripts/encrypt_files.sh,7,3
+file-enc/scripts/compress_files.sh,4,2
+log-analysis/scripts/nginx.sh,16,3
+log-analysis/scripts/pcaps.sh,12,3
+max-temp/scripts/temp-analytics.sh,5,1
+media-conv/scripts/to_mp3.sh,7,3
+media-conv/scripts/img_convert.sh,7,3
+nlp/scripts/verses_2om_3om_2instances.sh,9,2
+nlp/scripts/trigram_rec.sh,17,3
+nlp/scripts/uppercase_by_token.sh,7,2
+nlp/scripts/words_no_vowels.sh,7,2
+nlp/scripts/sort_words_by_folding.sh,7,2
+nlp/scripts/syllable_words_2.sh,7,2
+nlp/scripts/count_morphs.sh,7,2
+nlp/scripts/sort_words_by_num_of_syllables.sh,15,3
+nlp/scripts/merge_upper.sh,7,2
+nlp/scripts/syllable_words_1.sh,7,2
+nlp/scripts/sort_words_by_rhyming.sh,7,2
+nlp/scripts/letter_words.sh,8,2
+nlp/scripts/sort.sh,7,2
+nlp/scripts/count_consonant_seq.sh,7,2
+nlp/scripts/count_trigrams.sh,16,3
+nlp/scripts/bigrams.sh,15,3
+nlp/scripts/count_words.sh,7,2
+nlp/scripts/vowel_sequencies_gr_1K.sh,7,2
+nlp/scripts/bigrams_appear_twice.sh,16,3
+nlp/scripts/compare_exodus_genesis.sh,17,3
+nlp/scripts/uppercase_by_type.sh,7,2
+nlp/scripts/find_anagrams.sh,15,3
+nlp/scripts/count_vowel_seq.sh,7,2
+oneliners/scripts/spell.sh,2,1
+oneliners/scripts/wf.sh,1,1
+oneliners/scripts/bi-gram.aux.sh,43,4
+oneliners/scripts/top-n.sh,1,1
+oneliners/scripts/sort-sort.sh,1,1
+oneliners/scripts/diff.sh,3,1
+oneliners/scripts/sort.sh,1,1
+oneliners/scripts/bi-grams.sh,2,1
+oneliners/scripts/set-diff.sh,3,1
+oneliners/scripts/nfa-regex.sh,1,1
+sklearn/scripts/run.sh,49,1
+riker/scripts/lua/build.sh,8,1
+riker/scripts/xz-clang/build.sh,18,1
+riker/scripts/vim/build.sh,22,1
+riker/scripts/xz/build.sh,18,1
+riker/scripts/redis/build.sh,19,1
+riker/scripts/sqlite/build.sh,34,1
+riker/scripts/memcached/build.sh,11,2
+uniq-ips/scripts/run.sh,1,1
+unix50/scripts/1.sh,2,2
+unix50/scripts/10.sh,2,2
+unix50/scripts/16.sh,2,2
+unix50/scripts/21.sh,2,2
+unix50/scripts/19.sh,2,2
+unix50/scripts/6.sh,2,2
+unix50/scripts/36.sh,2,2
+unix50/scripts/28.sh,2,2
+unix50/scripts/12.sh,2,2
+unix50/scripts/13.sh,2,2
+unix50/scripts/26.sh,2,2
+unix50/scripts/31.sh,2,2
+unix50/scripts/24.sh,2,2
+unix50/scripts/11.sh,2,2
+unix50/scripts/23.sh,2,2
+unix50/scripts/14.sh,2,2
+unix50/scripts/25.sh,2,2
+unix50/scripts/7.sh,2,2
+unix50/scripts/29.sh,2,2
+unix50/scripts/30.sh,2,2
+unix50/scripts/3.sh,2,2
+unix50/scripts/20.sh,2,2
+unix50/scripts/4.sh,2,2
+unix50/scripts/34.sh,2,2
+unix50/scripts/2.sh,2,2
+unix50/scripts/18.sh,2,2
+unix50/scripts/33.sh,2,2
+unix50/scripts/35.sh,2,2
+unix50/scripts/32.sh,2,2
+unix50/scripts/15.sh,2,2
+unix50/scripts/17.sh,2,2
+unix50/scripts/9.sh,2,2
+unix50/scripts/8.sh,2,2
+unix50/scripts/5.sh,2,2