sacct logs for all stage2

src/warc2text_runner/two/qualitycontrol/check_linecnt.sh

@@ -3,5 +3,6 @@
 HTMLDIR=$1
 
 module purge; module load LUMI systools parallel
-find $HTMLDIR -name text.zst | parallel --eta  -j250 "echo {//}; zstdcat {//}/metadata.zst|wc;  zstdcat {//}/text.zst|wc; zstdcat {//}/lang.zst|wc" > text_lang_linecnts.log
+find $HTMLDIR -name metadata.zst | parallel --eta  -j250 "echo {//}; zstdcat {}|wc; test -f {//}/text.zst && zstdcat {//}/text.zst|wc || echo 0 0 0; test -f {//}/lang.zst && zstdcat {//}/lang.zst|wc || echo 0 0 0" >text_lang_linecnts.log
 cat text_lang_linecnts.log |tr '\n' ' ' | sed 's!/user!\n/user!g' >text_lang_linecnts.tsv
+python -m warc2text_runner.two.qualitycontrol.check_text_lang_linecnt text_lang_linecnts.tsv

src/warc2text_runner/two/qualitycontrol/check_text_lang_linecnts.py

@@ -1,5 +1,10 @@
 import pandas as pd
+import fire
 
-df=pd.read_csv('text_lang_linecnts.tsv', sep='\s+', header=None, names=['dir']+[f'{f}-{c}' for f in ('metadata','text','lang') for c in ('lines','words','bytes')])
-print('\n'.join(df[df['metadata-lines'] != df['text-lines']]['dir']))
-print('\n'.join(df[df['metadata-lines'] != df['lang-lines']]['dir']))
+def check_tsv(tsv):
+    df=pd.read_csv(tsv, sep='\s+', header=None, names=['dir']+[f'{f}-{c}' for f in ('metadata','text','lang') for c in ('lines','words','bytes')])
+    print('metadata-text lines mismatch:\n', '\n'.join(df[df['metadata-lines'] != df['text-lines']]['dir']))
+    print('metadata-lang lines mismatch:\n','\n'.join(df[df['metadata-lines'] != df['lang-lines']]['dir']))
+
+
+fire.Fire(check_tsv)