01.pdf2warc

#!/bin/bash
set -euo pipefail
module load singularity

define(){
	IFS='\n' read -r -d '' ${1} || true
}

random-port() {
	read LOWER UPPER < /proc/sys/net/ipv4/ip_local_port_range
	while :; do
		PORT=$((RANDOM % (UPPER - LOWER) + LOWER))
		# If I can't connect to this port, it must be free?
		if ! (echo "" >/dev/tcp/127.0.0.1/${PORT}) 2>/dev/null; then
			echo $PORT
			break
		fi
	done
}

INPUT=$1

OUTPUT=${1%/*}/pdf-text.warc.gz

IMAGE=/home/cs-vand1/r/jelmervdl/paracrawl-pdf/parsr.sif
SCRIPT=parsr-index.js

PORT=$(random-port)
PARSR_TMPDIR=$(mktemp -d -p $TMPDIR)
trap 'rm -rf $PARSR_TMPDIR' EXIT

singularity exec \
	--contain \
	--env TMPDIR=/tmp \
	-B ${PARSR_TMPDIR:-/tmp}:/tmp:rw \
	-B ${PARSR_TMPDIR:-/tmp}:/opt/app-root/src/api/server/dist/output:rw \
	-B $SCRIPT:/opt/app-root/src/api/server/dist/index.js:ro \
	$IMAGE /bin/bash -c "cd /opt/app-root/src/api/server && exec node dist/index.js $PORT" 2>&1 > /dev/null &
parsr_pid=$!

ATTEMPTS=60
until $(curl --output /dev/null --silent --fail "http://127.0.0.1:$PORT/api/v1/default-config"); do
	ATTEMPTS=$((ATTEMPTS - 1))
	test $ATTEMPTS -gt 0 || exit 1
	echo "Waiting 1s for parsr api to be up... $ATTEMPTS" >&2
	sleep 1
done

$PREFIX/src/pdfwarc2warc/pdfwarc2warc.py \
	--fast \
	--parsr-location "127.0.0.1:$PORT" \
	--timeout 600 \
	$INPUT > $OUTPUT.$$

mv $OUTPUT{.$$,}

# parsr may hang on some pdfs so kill it with force
kill -9 $parsr_pid