just log-analysis

log-analysis/cleanup.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 
-cd "$(realpath $(dirname "$0"))"
+REPO_TOP=$(git rev-parse --show-toplevel)
+results_dir="${REPO_TOP}/log-analysis/input"
+results_dir="${REPO_TOP}/log-analysis/results"
 
-rm -rf ./inputs
-rm -rf ./outputs
+rm -rf $input_dir
+rm -rf $results_dir

log-analysis/deps.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+sudo apt update 
+
+sudo apt install -y --no-install-recommends \
+    sudo \
+    tcpdump \
+    curl \
+    wget \
+    unzip \
+    samtools \
+    minimap2 \
+    bcftools \
+    python3-pip \
+    vim \
+    ffmpeg unrtf imagemagick libarchive-tools libncurses5-dev libncursesw5-dev zstd liblzma-dev libbz2-dev zip unzip nodejs tcpdump \
+    git
+

log-analysis/hashes/nginx.full.md5sum

@@ -0,0 +1,8 @@
+e1db42fe14aa938fb2f34bfc5735fd1b  nginx.full/log0
+aaf7960c4ba2c7bf92db9590fdb38542  nginx.full/log1
+27633e1cfb60207d4ac901614f463a9f  nginx.full/log2
+fd419b81767eda4cc9e5aaa1818b487c  nginx.full/log3
+79450f4c4a79cb71ef96d7bc03943a98  nginx.full/log4
+db4185dd14527deb56e09f0099ad43b1  nginx.full/log5
+879fe7329abf549d56706f601f2a120f  nginx.full/log6
+926b41adef30d4695d85cc12ece29334  nginx.full/log7

log-analysis/hashes/nginx.small.md5sum

@@ -0,0 +1,8 @@
+e1db42fe14aa938fb2f34bfc5735fd1b  nginx.small/log0
+aaf7960c4ba2c7bf92db9590fdb38542  nginx.small/log1
+27633e1cfb60207d4ac901614f463a9f  nginx.small/log2
+fd419b81767eda4cc9e5aaa1818b487c  nginx.small/log3
+79450f4c4a79cb71ef96d7bc03943a98  nginx.small/log4
+db4185dd14527deb56e09f0099ad43b1  nginx.small/log5
+879fe7329abf549d56706f601f2a120f  nginx.small/log6
+926b41adef30d4695d85cc12ece29334  nginx.small/log7

log-analysis/hashes/pcaps.full.md5sum

@@ -0,0 +1,69 @@
+4e489b35942d0c45d8b16d1cb1a06210  pcaps.full/4SICS-GeekLounge-151020.pcap.log
+890b35d7af03d13aab446e306ba60782  pcaps.full/4SICS-GeekLounge-151021.pcap.log
+1ebd451920d098aa4b7002054900a0b8  pcaps.full/4SICS-GeekLounge-151022.pcap.log
+d41d8cd98f00b204e9800998ecf8427e  pcaps.full/README.log
+f5b48dd8c8037efc43951efc3e1e16af  pcaps.full/challenge101-0.pcapng.log
+6b2c4e89211bc977ac538ad218845794  pcaps.full/challenge101-1.pcapng.log
+765d385418861c272c081f85abee4448  pcaps.full/challenge101-3.pcapng.log
+6c5777e9a6a4601487f5bb0d637eaef0  pcaps.full/challenge101-4.pcapng.log
+2b632ea272f075b62a1779e6b79c262b  pcaps.full/challenge101-5.pcapng.log
+431993ece40a4f8081cc949661be3b98  pcaps.full/challenge101-6.pcapng.log
+fe1fcee267a72f20de6abd423344b247  pcaps.full/challenge101-7.pcapng.log
+63382ae6dddcc347be2f140853ab57a3  pcaps.full/challenge101-8.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e  pcaps.full/dhcp-serverdiscovery101.pcapng.log
+9267a406fc3cca0cb4d7b755169df6be  pcaps.full/dns-nmap101.pcapng.log
+42164f80e7ff0177cf36922c2caf7917  pcaps.full/f5-honeypot-release.pcap.log
+d41d8cd98f00b204e9800998ecf8427e  pcaps.full/ftp-bounce.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e  pcaps.full/ftp-clientside101.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e  pcaps.full/ftp-crack101.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e  pcaps.full/ftp-download101.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e  pcaps.full/ftp-passwords101.pcapng.log
+ecb77db5b2688d25093117caaa9cf4f6  pcaps.full/gen-startupchatty101.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e  pcaps.full/general101.pcapng.log
+3684f199efa845a7ec8f1053a9944caf  pcaps.full/general101b.pcapng.log
+0ed4ba9fdb8cb940d09f92b7075db46f  pcaps.full/general101c.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e  pcaps.full/general101d.pcapng.log
+0d9772fdd4477af9f2b0523e8ff44e63  pcaps.full/http-au101b.pcapng.log
+4df9f89d8991983f811a4cea8aaf2f12  pcaps.full/http-browse101.pcapng.log
+fdea8971f0510eb2d972385d69b05f2b  pcaps.full/http-browse101b.pcapng.log
+9433ad4a3133a817f8e90f46a94705fd  pcaps.full/http-browse101c.pcapng.log
+9433ad4a3133a817f8e90f46a94705fd  pcaps.full/http-browse101d.pcapng.log
+ef0297df04fd11d88a39669ad3de8452  pcaps.full/http-chappellu101.pcapng.log
+ab0e19847b655e25897de3ad675bbf8f  pcaps.full/http-chappellu101b.pcapng.log
+8cb81097cbef7c810b7dcd57b8a240cf  pcaps.full/http-cheez101.pcapng.log
+542d7c56c0f2cf3c04e72152760392e1  pcaps.full/http-college101.pcapng.log
+ba1d7834a8352ea90b998873a725bd7f  pcaps.full/http-disney101.pcapng.log
+8573185b6cac32ae8468471e51c1a0f6  pcaps.full/http-download-a.pcapng.log
+20b8dc7a31500946202d4d191b7edf16  pcaps.full/http-download101.pcapng.log
+01a2be97c1ab675184e97a71b084e25c  pcaps.full/http-download101c.pcapng.log
+be0103fda2b6751d6abe396c426a1e52  pcaps.full/http-download101d.pcapng.log
+191508af7467c488cd5e6d61535e6c5b  pcaps.full/http-errors101.pcapng.log
+1109ac31b2b0577889340cd98b20eb34  pcaps.full/http-espn101.pcapng.log
+f2543fa2a0bd81ffc279198b7d7a5f62  pcaps.full/http-google101.pcapng.log
+72cab4dcfa839a8275412b36855caeb1  pcaps.full/http-jezebel101.pcapng.log
+77f6419cd8de445503e779b9818a3aa5  pcaps.full/http-misctraffic101.pcapng.log
+28f32e99994ea9cc26df9d2ba3dd12ae  pcaps.full/http-nonstandard101.pcapng.log
+f0356a106329dd181bd7fad31d54d979  pcaps.full/http-openoffice101a.pcapng.log
+7dc37f1efab91a9add8c55adb0009bae  pcaps.full/http-openoffice101b.pcapng.log
+dfa7cf0ac25800ca7affd3ded10addfd  pcaps.full/http-pcaprnet101.pcapng.log
+c9ce2f1e0180a51560f610323f9fa8e7  pcaps.full/http-pictures101.pcapng.log
+6bd2962179641be17333fb5b7f9cb1c6  pcaps.full/http-sfgate101.pcapng.log
+585cc0f172039cc04acb7b5da58f20ed  pcaps.full/http-slow101.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e  pcaps.full/http-winpcap101.cap.log
+4236a958c61e54031d40fc6d8c958291  pcaps.full/http-wiresharkdownload101.pcapng.log
+1b1359e2d6822016dc1a168c41667de4  pcaps.full/mybackground101.pcapng.log
+e95ab82bbf83b0d0af0b9f74fdad88f8  pcaps.full/net-lost-route.pcapng.log
+d293fcc821f7d8b19e1217f55bbb179b  pcaps.full/sec-concern101.pcapng.log
+f47bc44475d79ccb61b1ff5b8562018d  pcaps.full/sec-nessus101.pcapng.log
+85ec363f6ac000d7132b970ec39d78cc  pcaps.full/sec-suspicious101.pcapng.log
+b78b407b7f11fd2e9a980c71bce44b29  pcaps.full/smb-join101.pcapng.log
+0807f74c4cd5ddb7fbd6aab6ff4c8270  pcaps.full/split250_00000_20160704110754.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e  pcaps.full/split250_00001_20160704110759.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e  pcaps.full/split250_00002_20160704110759.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e  pcaps.full/split250_00003_20160704110759.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e  pcaps.full/split250_00004_20160704110759.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e  pcaps.full/split250_00005_20160704110804.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e  pcaps.full/tcp-decodeas.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e  pcaps.full/tr-twohosts.pcapng.log
+5ee264aa76210466a389ba1e9fd5f3aa  pcaps.full/tr-winsize.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e  pcaps.full/wlan-ipadstartstop101.pcapng.log

log-analysis/hashes/pcaps.small.md5sum

@@ -0,0 +1,69 @@
+4e489b35942d0c45d8b16d1cb1a06210  pcaps.small/4SICS-GeekLounge-151020.pcap.log
+890b35d7af03d13aab446e306ba60782  pcaps.small/4SICS-GeekLounge-151021.pcap.log
+1ebd451920d098aa4b7002054900a0b8  pcaps.small/4SICS-GeekLounge-151022.pcap.log
+d41d8cd98f00b204e9800998ecf8427e  pcaps.small/README.log
+f5b48dd8c8037efc43951efc3e1e16af  pcaps.small/challenge101-0.pcapng.log
+6b2c4e89211bc977ac538ad218845794  pcaps.small/challenge101-1.pcapng.log
+765d385418861c272c081f85abee4448  pcaps.small/challenge101-3.pcapng.log
+6c5777e9a6a4601487f5bb0d637eaef0  pcaps.small/challenge101-4.pcapng.log
+2b632ea272f075b62a1779e6b79c262b  pcaps.small/challenge101-5.pcapng.log
+431993ece40a4f8081cc949661be3b98  pcaps.small/challenge101-6.pcapng.log
+fe1fcee267a72f20de6abd423344b247  pcaps.small/challenge101-7.pcapng.log
+63382ae6dddcc347be2f140853ab57a3  pcaps.small/challenge101-8.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e  pcaps.small/dhcp-serverdiscovery101.pcapng.log
+9267a406fc3cca0cb4d7b755169df6be  pcaps.small/dns-nmap101.pcapng.log
+42164f80e7ff0177cf36922c2caf7917  pcaps.small/f5-honeypot-release.pcap.log
+d41d8cd98f00b204e9800998ecf8427e  pcaps.small/ftp-bounce.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e  pcaps.small/ftp-clientside101.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e  pcaps.small/ftp-crack101.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e  pcaps.small/ftp-download101.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e  pcaps.small/ftp-passwords101.pcapng.log
+ecb77db5b2688d25093117caaa9cf4f6  pcaps.small/gen-startupchatty101.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e  pcaps.small/general101.pcapng.log
+3684f199efa845a7ec8f1053a9944caf  pcaps.small/general101b.pcapng.log
+0ed4ba9fdb8cb940d09f92b7075db46f  pcaps.small/general101c.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e  pcaps.small/general101d.pcapng.log
+0d9772fdd4477af9f2b0523e8ff44e63  pcaps.small/http-au101b.pcapng.log
+4df9f89d8991983f811a4cea8aaf2f12  pcaps.small/http-browse101.pcapng.log
+fdea8971f0510eb2d972385d69b05f2b  pcaps.small/http-browse101b.pcapng.log
+9433ad4a3133a817f8e90f46a94705fd  pcaps.small/http-browse101c.pcapng.log
+9433ad4a3133a817f8e90f46a94705fd  pcaps.small/http-browse101d.pcapng.log
+ef0297df04fd11d88a39669ad3de8452  pcaps.small/http-chappellu101.pcapng.log
+ab0e19847b655e25897de3ad675bbf8f  pcaps.small/http-chappellu101b.pcapng.log
+8cb81097cbef7c810b7dcd57b8a240cf  pcaps.small/http-cheez101.pcapng.log
+542d7c56c0f2cf3c04e72152760392e1  pcaps.small/http-college101.pcapng.log
+ba1d7834a8352ea90b998873a725bd7f  pcaps.small/http-disney101.pcapng.log
+8573185b6cac32ae8468471e51c1a0f6  pcaps.small/http-download-a.pcapng.log
+20b8dc7a31500946202d4d191b7edf16  pcaps.small/http-download101.pcapng.log
+01a2be97c1ab675184e97a71b084e25c  pcaps.small/http-download101c.pcapng.log
+be0103fda2b6751d6abe396c426a1e52  pcaps.small/http-download101d.pcapng.log
+191508af7467c488cd5e6d61535e6c5b  pcaps.small/http-errors101.pcapng.log
+1109ac31b2b0577889340cd98b20eb34  pcaps.small/http-espn101.pcapng.log
+f2543fa2a0bd81ffc279198b7d7a5f62  pcaps.small/http-google101.pcapng.log
+72cab4dcfa839a8275412b36855caeb1  pcaps.small/http-jezebel101.pcapng.log
+77f6419cd8de445503e779b9818a3aa5  pcaps.small/http-misctraffic101.pcapng.log
+28f32e99994ea9cc26df9d2ba3dd12ae  pcaps.small/http-nonstandard101.pcapng.log
+f0356a106329dd181bd7fad31d54d979  pcaps.small/http-openoffice101a.pcapng.log
+7dc37f1efab91a9add8c55adb0009bae  pcaps.small/http-openoffice101b.pcapng.log
+dfa7cf0ac25800ca7affd3ded10addfd  pcaps.small/http-pcaprnet101.pcapng.log
+c9ce2f1e0180a51560f610323f9fa8e7  pcaps.small/http-pictures101.pcapng.log
+6bd2962179641be17333fb5b7f9cb1c6  pcaps.small/http-sfgate101.pcapng.log
+585cc0f172039cc04acb7b5da58f20ed  pcaps.small/http-slow101.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e  pcaps.small/http-winpcap101.cap.log
+4236a958c61e54031d40fc6d8c958291  pcaps.small/http-wiresharkdownload101.pcapng.log
+1b1359e2d6822016dc1a168c41667de4  pcaps.small/mybackground101.pcapng.log
+e95ab82bbf83b0d0af0b9f74fdad88f8  pcaps.small/net-lost-route.pcapng.log
+d293fcc821f7d8b19e1217f55bbb179b  pcaps.small/sec-concern101.pcapng.log
+f47bc44475d79ccb61b1ff5b8562018d  pcaps.small/sec-nessus101.pcapng.log
+85ec363f6ac000d7132b970ec39d78cc  pcaps.small/sec-suspicious101.pcapng.log
+b78b407b7f11fd2e9a980c71bce44b29  pcaps.small/smb-join101.pcapng.log
+0807f74c4cd5ddb7fbd6aab6ff4c8270  pcaps.small/split250_00000_20160704110754.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e  pcaps.small/split250_00001_20160704110759.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e  pcaps.small/split250_00002_20160704110759.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e  pcaps.small/split250_00003_20160704110759.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e  pcaps.small/split250_00004_20160704110759.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e  pcaps.small/split250_00005_20160704110804.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e  pcaps.small/tcp-decodeas.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e  pcaps.small/tr-twohosts.pcapng.log
+5ee264aa76210466a389ba1e9fd5f3aa  pcaps.small/tr-winsize.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e  pcaps.small/wlan-ipadstartstop101.pcapng.log

log-analysis/input.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -e
+
+# creates input/pcaps and input/nginx-logs
+
+REPO_TOP=$(git rev-parse --show-toplevel)
+eval_dir="${REPO_TOP}/log-analysis"
+input_dir="${eval_dir}/input"
+
+url=https://atlas-group.cs.brown.edu/data/pcaps.zip
+zip_dst="$input_dir/pcaps.zip"
+wget --no-check-certificate $url -O "$zip_dst"
+unzip "$zip_dst" -d "$input_dir"
+rm "$zip_dst"
+
+url=https://atlas-group.cs.brown.edu/data/nginx.zip
+zip_dst="$input_dir/nginx.zip"
+wget --no-check-certificate $url -O "$zip_dst"
+unzip "$zip_dst" -d "$input_dir"
+rm $zip_dst

log-analysis/input/.gitignore

@@ -0,0 +1 @@
+

log-analysis/run.sh

@@ -1,88 +1,22 @@
 #!/bin/bash
 
-export SUITE_DIR=$(realpath $(dirname "$0"))
-export TIMEFORMAT=%R
-cd $SUITE_DIR
-
-names_scripts=(
-    "LogAnalysis1;nginx"
-    "LogAnalysis2;pcaps"
-  )
-
+REPO_TOP=$(git rev-parse --show-toplevel)
+eval_dir="${REPO_TOP}/log-analysis"
+input_dir="${eval_dir}/input"
+scripts_dir="${eval_dir}/scripts"
+results_dir="${eval_dir}/results"
+mkdir -p $results_dir
+
+nginx_input=$input_dir/nginx-logs
+pcaps_input=$input_dir/pcaps
+suffix=".full"
 if [[ "$@" == *"--small"* ]]; then
-    scripts_inputs=(
-        "nginx;/log_data_small"
-        "pcaps;/pcap_data_small"
-    )
-    scripts_outputs=(
-        "nginx;/log-analysis/nginx_analysis_small"
-        "pcaps;/log-analysis/pcap_analysis_small"
-    )
-else
-    scripts_inputs=(
-        "nginx;/log_data"
-        "pcaps;/pcap_data"
-    )
-    scripts_outputs=(
-        "nginx;/log-analysis/nginx_analysis"
-        "pcaps;/log-analysis/pcap_analysis"
-    )
+    # TODO: vary input size
+    nginx_input=$input_dir/nginx-logs
+    pcaps_input=$input_dir/pcaps
+    suffix=".small"
 fi
 
-parse_directories() {
-    local script_name=$1
-    local scripts_array=("${!2}")
-    for entry in "${scripts_array[@]}"; do
-        IFS=";" read -r -a parsed <<< "${entry}"
-        if [[ "${parsed[0]}" == "${script_name}" ]]; then
-            echo "${parsed[1]}"
-            return
-        fi
-    done
-}
-
-mkdir -p "outputs"
-all_res_file="./outputs/log-analysis.res"
-> $all_res_file
-
-# time_file stores the time taken for each script
-# mode_res_file stores the time taken and the script name for every script in a mode (e.g. bash, pash, dish, fish)
-# all_res_file stores the time taken for each script for every script run, making it easy to copy and paste into the spreadsheet
-log-analysis() {
-    mkdir -p "outputs/$1"
-    mode_res_file="./outputs/$1/log-analysis.res"
-    > $mode_res_file
-
-    echo executing log-analysis $1 $(date) | tee -a $mode_res_file $all_res_file
-    for name_script in ${names_scripts[@]}
-    do
-        IFS=";" read -r -a name_script_parsed <<< "${name_script}"
-        name="${name_script_parsed[0]}"
-        script="${name_script_parsed[1]}"
-        script_file="./scripts/$script.sh"
-        input_dir=$(parse_directories "$script" scripts_inputs[@])
-        output_dir=$(parse_directories "$script" scripts_outputs[@])
-        output_file="./outputs/$1/$script.out"
-        time_file="./outputs/$1/$script.time"
-        log_file="./outputs/$1/$script.log"
-        hash_file="./outputs/$1/$script.hash"
-
-
-        if [[ "$1" == "bash" ]]; then
-            (time $script_file $input_dir $output_dir > $output_file) 2> $time_file
-        fi
-
-        # Generate SHA-256 hash and delete output file
-        shasum -a 256 "$output_file" | awk '{ print $1 }' > "$hash_file"
-        rm "$output_file"
-
-        cat "${time_file}" >> $all_res_file
-        echo "$script_file $(cat "$time_file")" | tee -a $mode_res_file
-    done
-}
-
-
-# adjust the debug flag as required
-d=0
+$scripts_dir/nginx.sh $nginx_input $results_dir/nginx$suffix 
+$scripts_dir/pcaps.sh $pcaps_input $results_dir/pcaps$suffix 
 
-log-analysis "bash"