diff --git a/README.md b/README.md index 4ffd197..2f5c75e 100644 --- a/README.md +++ b/README.md @@ -147,6 +147,13 @@ IP Public: 123.123.123.123 Location: Italy ``` +### `spider` - Launch a spider on a URL + +```shell +$ spider URL [HTTP_AUTH_USER HTTP_AUTH_PASS] +... +``` + ### `spin_container` - Launch a temp docker container ```shell diff --git a/spider b/spider new file mode 100755 index 0000000..0fec9fd --- /dev/null +++ b/spider @@ -0,0 +1,10 @@ +#!/bin/sh +URL=$1 +HTTP_AUTH_USER=$2 +HTTP_AUTH_PASS=$3 +HTTP_AUTH="" +if [ "$HTTP_AUTH_PASS" != "" ]; then + HTTP_AUTH="--http-user=$HTTP_AUTH_USER --http-password=$HTTP_AUTH_PASS" +fi +wget --save-headers --no-clobber --recursive --server-response --no-parent $HTTP_AUTH $URL 2>&1 \ + | awk '/^--/{print $3}/^ HTTP.+ 1.. / {print "\033[34m" $2 "\033[0m" }/^ HTTP.+ 2.. / {print "\033[32m" $2 "\033[0m" }/^ HTTP.+ 3.. / {print "\033[33m" $2 "\033[0m" }/^ HTTP.+ 4.. / {print "\033[35m" $2 "\033[0m" }/^ HTTP.+ 5.. / {print "\033[31m" $2 "\033[0m" }'