Just analysis logs

analysis-logs/README.md

@@ -0,0 +1,13 @@
+# Instructions
+
+Ensure that you have obtained `analysis-logs/input/access.log`. 
+
+The script `analysis-logs/input.sh` downloads this file with the Kaggle API. In
+order to use the Kaggle API, you must have the API token: `~/.kaggle/kaggle.json`.
+
+From <https://www.kaggle.com/docs/api>:
+
+> In order to use the Kaggle’s public API, you must first authenticate using an
+> API token. Go to the 'Account' tab of your user profile and select 'Create New
+> Token'. This will trigger the download of kaggle.json, a file containing your
+> API credentials.

analysis-logs/input/cleanup.sh → analysis-logs/cleanup.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 REPO_TOP=$(git rev-parse --show-toplevel)
-results_dir="${REPO_TOP}/covid-mts/results"
+results_dir="${REPO_TOP}/analysis-logs/results"
 
 echo "Cleaning up outputs..."
 rm -rf $results_dir

analysis-logs/hashes/results.full.md5sum

@@ -0,0 +1 @@
+105b333d6b49399cc10763a5f629343a  results.full/out

analysis-logs/hashes/results.small.md5sum

@@ -0,0 +1 @@
+105b333d6b49399cc10763a5f629343a  results.small/out

analysis-logs/input.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+REPO_TOP=$(git rev-parse --show-toplevel)
+DIR=$REPO_TOP/analysis-logs/input
+mkdir -p $DIR
+
+# Set up Kaggle API
+if [[ ! -d ~/.kaggle ]]; then
+    mkdir ~/.kaggle
+    echo "Place your kaggle.json in the ~/.kaggle directory."
+fi
+chmod 600 ~/.kaggle/kaggle.json
+
+cd $DIR
+kaggle datasets download -d eliasdabbas/web-server-access-logs
+unzip web-server-access-logs
+rm -f web-server-access-logs.zip client_hostname.csv

analysis-logs/input/.gitignore

@@ -0,0 +1 @@
+

analysis-logs/run.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+REPO_TOP=$(git rev-parse --show-toplevel)
+
+eval_dir="${REPO_TOP}/analysis-logs"
+results_dir="${eval_dir}/results"
+scripts_dir="${eval_dir}/scripts"
+input_dir="${eval_dir}/input"
+mkdir -p $results_dir
+
+export INPUT=${input_dir}/access.log
+
+suffix=".full"
+if [[ "$@" == *"--small"* ]]; then
+    suffix=".small"
+fi
+
+log_dir="$results_dir/results$suffix"
+mkdir -p $log_dir
+$scripts_dir/nginx.sh > $log_dir/out

analysis-logs/verify.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+REPO_TOP=$(git rev-parse --show-toplevel)
+
+eval_dir="${REPO_TOP}/analysis-logs"
+input_dir="${eval_dir}/input"
+hashes_dir="${eval_dir}/hashes"
+results_dir="${eval_dir}/results"
+mkdir -p $results_dir
+
+suffix=".full"
+if [[ "$@" == *"--small"* ]]; then
+    suffix=".small"
+fi
+
+cd $results_dir # md5sum computes paths relative to cd
+if [[ "$@" == *"--generate"* ]]; then
+    md5sum results$suffix/* > $hashes_dir/results$suffix.md5sum
+fi
+
+okay=0
+if ! md5sum --check --quiet $hashes_dir/results$suffix.md5sum; then
+    okay=1
+    echo "img_convert $suffix failed verification"
+fi
+exit $okay