stage2 QC: checking the match of the number of lines in input and out…

…put files

src/warc2text_runner/two/qualitycontrol/check_linecnt.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+HTMLDIR=$1
+
+module purge; module load LUMI systools parallel
+find $HTMLDIR -name text.zst | parallel --eta  -j250 "echo {//}; zstdcat {//}/metadata.zst|wc;  zstdcat {//}/text.zst|wc; zstdcat {//}/lang.zst|wc" > text_lang_linecnts.log
+cat text_lang_linecnts.log |tr '\n' ' ' | sed 's!/user!\n/user!g' >text_lang_linecnts.tsv

src/warc2text_runner/two/qualitycontrol/check_text_lang_linecnts.py

@@ -0,0 +1,5 @@
+import pandas as pd
+
+df=pd.read_csv('text_lang_linecnts.tsv', sep='\s+', header=None, names=['dir']+[f'{f}-{c}' for f in ('metadata','text','lang') for c in ('lines','words','bytes')])
+print('\n'.join(df[df['metadata-lines'] != df['text-lines']]['dir']))
+print('\n'.join(df[df['metadata-lines'] != df['lang-lines']]['dir']))

src/warc2text_runner/two/qualitycontrol/traferr_stat.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+# Feed content of text.zst to the stdin
+jq -c .traferr | sort | uniq -c