adding Makefile, entrypoint, requirements etc

.dockerignore

@@ -0,0 +1,5 @@
+.git
+.gitignore
+Dockerfile
+Makefile
+data/*

Dockerfile

@@ -0,0 +1,25 @@
+# Base Alpine Linux based image with OpenJDK and Maven
+FROM archivesunleashed/docker-aut:latest
+
+# Metadata
+LABEL maintainer="Sebastian Gassner <[email protected]>"
+LABEL description="Docker image for warc2corpus, based on the Archives Unleashed Toolkit."
+LABEL website="https://github.com/sepastian/warc2corpus/"
+
+WORKDIR /w2c
+
+# Copy pyspark starup script to /usr/local/bin/pyspark
+ADD files /
+ADD lib /w2c/lib
+ADD requirements.txt .
+
+# Install pip3 and requirements.txt
+RUN apt-get update && apt-get install -y \
+  python3-pip \
+&& rm -rf /var/lib/apt/lists/* \
+&& pip3 install -r requirements.txt
+
+# Add /w2c/lib to the Python search path
+ENV PYTHONPATH=/w2c/lib
+
+ENTRYPOINT ["/usr/local/bin/aut-spark-shell"]

Makefile

@@ -0,0 +1,5 @@
+all: build
+
+.PHONY: build
+build:
+	docker build -t sepastian/warc2corpus:latest .

files/usr/local/bin/aut-spark-shell

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# Default: run spark-shell (Scala):
+#
+# /spark/bin/spark-shell \
+#   --jars /aut/target/aut-<VERSION>-SNAPSHOT-fatjar.jar
+#
+# If the first argument is "pyspark", run pyspark instead:
+#
+# /spark/bin/pyspark \
+#   --py-files /aut/target/aut.zip \
+#   --jars /aut/target/aut-<VERSION>-SNAPSHOT-fatjar.jar
+#
+# Any additional arguments will be passed to the script to run as-is.
+
+# Select lastest JAR file automatically from /aut/target
+jarfile=$(ls /aut/target/aut-*-SNAPSHOT-fatjar.jar)
+
+# Compose command to run;
+# run spark by default;
+# if first argument is "pyspark", run pyspark instead.
+cmd="/spark/bin/spark-shell"
+opts="--jars ${jarfile}"
+if [[ $1 == "pyspark" ]]; then
+    echo "Starting pyspark..."
+    shift
+    cmd="/spark/bin/pyspark"
+    opts="--py-files /aut/target/aut.zip ${opts}"
+fi
+
+# Run command;
+# pass additional commands to the shell selected above.
+echo "Command: ${cmd} ${opts} $*"
+${cmd} ${opts} $@

requirements.txt

@@ -1 +1,3 @@
 dateparser
+beautifulsoup4
+jsonschema

test.py

@@ -11,3 +11,6 @@
 df2.limit(1).collect()[0]['extract']
 
 #df.select(extract('content',zeit_de))
+
+# UDF closure:
+# https://stackoverflow.com/a/37428126/92049