From 3e7dfcf733a2a16e57845f1191064a01cd38e6df Mon Sep 17 00:00:00 2001 From: Nhoya Date: Mon, 15 Jan 2018 13:02:31 +0100 Subject: [PATCH 01/26] Added check on go version before building --- build.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/build.sh b/build.sh index 6eedc52..9ed9a15 100755 --- a/build.sh +++ b/build.sh @@ -6,6 +6,22 @@ readonly YELLOW="\033[00;33m" readonly BOLD="\033[01m" readonly END="\033[0m" +version=$(go version 2> /dev/null) +if [[ "$?" != 0 ]]; then + echo "Unable to find go, you need go >= 1.8 to build gOSINT" + exit 1 +fi +go_version_regex="([0-9]).([0-9]).[0-9]" +if [[ "$version" =~ $go_version_regex ]]; then + if [[ ${BASH_REMATCH[1]} -le 1 ]]; then + if [[ ${BASH_REMATCH[2]} -lt 8 ]]; then + echo "This version of go is not supported, you need go >= 1.8" + echo "Current: "$version"" + exit 1 + fi + fi +fi + dependencies=( github.com/deckarep/golang-set github.com/nhoya/goPwned github.com/jessevdk/go-flags gopkg.in/src-d/go-git.v4 github.com/jaytaylor/html2text) From b8e845c815dc0dbdb80a22488cf92d2da9505704 Mon Sep 17 00:00:00 2001 From: Nhoya Date: Mon, 15 Jan 2018 14:38:59 +0100 Subject: [PATCH 02/26] Fixed Image regex, telegram scraper is waaaaay faster --- .gitignore | 3 +++ telegram.go | 8 ++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 28613ee..e9715fa 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,6 @@ # Binary gOSINT + +#Telegram dump +*.dump diff --git a/telegram.go b/telegram.go index 5103ee1..f63a87d 100644 --- a/telegram.go +++ b/telegram.go @@ -53,8 +53,9 @@ func getTelegramGroupHistory(group string, grace int, dumpFlag bool) { } } messageCounter++ - time.Sleep(time.Millisecond * 500) + time.Sleep(time.Millisecond * 100) } + fmt.Println("End of history, if you think there are more messages try to increase the grace perio (--grace [INT])") } func getTelegramMessage(body string) string { @@ -68,7 +69,6 @@ func getTelegramMessage(body string) string { } messageBody = messageBody + getTelegramMedia(body) return messageBody - } func getTelegramMedia(body string) string { @@ -77,10 +77,10 @@ func getTelegramMedia(body string) string { } func getTelegramPhoto(body string) string { - re := regexp.MustCompile(`image:url\('https:\/\/([\w+.\/-]+)'`) + re := regexp.MustCompile(`image:url\('(https:\/\/[\w+.\/-]+)'`) match := re.FindStringSubmatch(body) if len(match) == 2 { - return "Photo: " + match[1] + return "Image: " + match[1] } return "" } From 3bde0a48a148fb9759923f67c9362f1da9af68db Mon Sep 17 00:00:00 2001 From: Nhoya Date: Mon, 15 Jan 2018 14:39:47 +0100 Subject: [PATCH 03/26] Working on 0.4c --- gOSINT.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gOSINT.go b/gOSINT.go index c444c25..074a913 100644 --- a/gOSINT.go +++ b/gOSINT.go @@ -8,7 +8,7 @@ import ( "github.com/jessevdk/go-flags" ) -const ver = "v0.4b" +const ver = "v0.4c" var opts struct { Module string `short:"m" long:"module" description:"Specify module" choice:"pgp" choice:"pwnd" choice:"git" choice:"plainSearch" choice:"telegram"` From eeb43257b9097ac944f0d39227cc03f65c9e3234 Mon Sep 17 00:00:00 2001 From: Nhoya Date: Mon, 15 Jan 2018 15:25:10 +0100 Subject: [PATCH 04/26] Handling service messages right --- telegram.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/telegram.go b/telegram.go index f63a87d..7a68c21 100644 --- a/telegram.go +++ b/telegram.go @@ -72,7 +72,7 @@ func getTelegramMessage(body string) string { } func getTelegramMedia(body string) string { - messageBody := getTelegramVideo(body) + getTelegramPhoto(body) + getTelegramVoice(body) + messageBody := getTelegramVideo(body) + getTelegramPhoto(body) + getTelegramVoice(body) + getTelegramServiceMessage(body) return messageBody } @@ -103,6 +103,13 @@ func getTelegramVideo(body string) string { return "" } +func getTelegramServiceMessage(body string) string { + re := regexp.MustCompile(`Service\smessage<\/div>`) + if re.MatchString(body) { + return "[SERVICE MESSAGE]" + } + return "" +} func getTelegramUsername(body string) (string, string) { re := regexp.MustCompile(`class=\"tgme_widget_message_author_name\"\s?(?:href="https://t\.me/(\w+)")? dir=\"auto\">(.*)<\/(?:span>)?(?:a>)? in  Date: Mon, 15 Jan 2018 19:10:57 +0100 Subject: [PATCH 05/26] Ask to print old dumps, fix problem that should break the resume form file, added author of quoted messages --- telegram.go | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/telegram.go b/telegram.go index 7a68c21..ae96fcd 100644 --- a/telegram.go +++ b/telegram.go @@ -17,9 +17,10 @@ func getTelegramGroupHistory(group string, grace int, dumpFlag bool) { dumpfile := group + ".dump" msgtxt := "" - fmt.Println("==== Dumping messages for " + group + " ====") messageCounter := readFromTelegramDump(dumpfile, dumpFlag) messageCounter++ + startTime := time.Now() + fmt.Println("==== [" + startTime.Format(time.RFC3339) + "] Dumping messages for " + group + " ====") for messageCounter != 0 { messageid := strconv.Itoa(messageCounter) body := retriveRequestBody("https://t.me/" + group + "/" + messageid + "?embed=1") @@ -44,7 +45,7 @@ func getTelegramGroupHistory(group string, grace int, dumpFlag bool) { msg, _ := html2text.FromString(msgtxt) - writeOnFile(dumpfile, "["+messageid+"] "+msg+"\n") + writeOnFile(dumpfile, "["+messageid+"] "+strings.Replace(msg, "\n", " ⏎ ", -1)+"\n") fmt.Println(msg) } else { graceCounter++ @@ -55,7 +56,8 @@ func getTelegramGroupHistory(group string, grace int, dumpFlag bool) { messageCounter++ time.Sleep(time.Millisecond * 100) } - fmt.Println("End of history, if you think there are more messages try to increase the grace perio (--grace [INT])") + fmt.Println("==== [" + time.Now().String() + "(elapsed:" + time.Since(startTime).String() + ")] End of history ==== ") + fmt.Println("If you think there are more messages try to increase the grace perio (--grace [INT])") } func getTelegramMessage(body string) string { @@ -65,7 +67,8 @@ func getTelegramMessage(body string) string { if len(match) == 1 { messageBody = match[0][1] } else if len(match) == 2 { - messageBody = "{" + match[0][1] + "}" + match[1][1] + quotedUser := getMessageRepliedAuthor(body) + messageBody = "{ " + quotedUser + match[0][1] + " } " + match[1][1] } messageBody = messageBody + getTelegramMedia(body) return messageBody @@ -110,6 +113,16 @@ func getTelegramServiceMessage(body string) string { } return "" } + +func getMessageRepliedAuthor(body string) string { + re := regexp.MustCompile(`reply"\shref="https:\/\/t.me\/[\w/]+">[\n\s]+[\n\s]+(.*)`) + match := re.FindStringSubmatch(body) + if len(match) == 2 { + return " " + match[1] + ": " + } + return "" +} + func getTelegramUsername(body string) (string, string) { re := regexp.MustCompile(`class=\"tgme_widget_message_author_name\"\s?(?:href="https://t\.me/(\w+)")? dir=\"auto\">(.*)<\/(?:span>)?(?:a>)? in  Date: Mon, 15 Jan 2018 22:14:56 +0100 Subject: [PATCH 06/26] Added support for channels, now handling rounded videos and documents --- telegram.go | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/telegram.go b/telegram.go index ae96fcd..3135f4a 100644 --- a/telegram.go +++ b/telegram.go @@ -38,7 +38,12 @@ func getTelegramGroupHistory(group string, grace int, dumpFlag bool) { date, time := getTelegramMessageDateTime(body) if username == "" { - msgtxt = "[" + date + " " + time + "] " + nickname + ": " + message + //for channels + if nickname == "" { + msgtxt = "[" + date + " " + time + "] " + message + } else { + msgtxt = "[" + date + " " + time + "] " + nickname + ": " + message + } } else { msgtxt = "[" + date + " " + time + "] " + nickname + "(" + username + "): " + message } @@ -57,7 +62,8 @@ func getTelegramGroupHistory(group string, grace int, dumpFlag bool) { time.Sleep(time.Millisecond * 100) } fmt.Println("==== [" + time.Now().String() + "(elapsed:" + time.Since(startTime).String() + ")] End of history ==== ") - fmt.Println("If you think there are more messages try to increase the grace perio (--grace [INT])") + fmt.Println("[=] If you think there are more messages try to increase the grace perio (--grace [INT])") + } func getTelegramMessage(body string) string { @@ -75,7 +81,7 @@ func getTelegramMessage(body string) string { } func getTelegramMedia(body string) string { - messageBody := getTelegramVideo(body) + getTelegramPhoto(body) + getTelegramVoice(body) + getTelegramServiceMessage(body) + messageBody := getTelegramVideo(body) + getTelegramPhoto(body) + getTelegramVoice(body) + getTelegramServiceMessage(body) + getTelegramDocument(body) return messageBody } @@ -98,7 +104,7 @@ func getTelegramVoice(body string) string { } func getTelegramVideo(body string) string { - re := regexp.MustCompile(`video\ssrc="(https:\/\/[\w.\/-]+)"`) + re := regexp.MustCompile(`video"?\ssrc="(https:\/\/[\w.\/-]+)"`) match := re.FindStringSubmatch(body) if len(match) == 2 { return "Video: " + match[1] @@ -106,6 +112,15 @@ func getTelegramVideo(body string) string { return "" } +func getTelegramDocument(body string) string { + re := regexp.MustCompile(`document_title" dir="auto">(.*)`) + match := re.FindStringSubmatch(body) + if len(match) == 2 { + return "Document: " + match[1] + } + return "" +} + func getTelegramServiceMessage(body string) string { re := regexp.MustCompile(`Service\smessage<\/div>`) if re.MatchString(body) { @@ -126,7 +141,11 @@ func getMessageRepliedAuthor(body string) string { func getTelegramUsername(body string) (string, string) { re := regexp.MustCompile(`class=\"tgme_widget_message_author_name\"\s?(?:href="https://t\.me/(\w+)")? dir=\"auto\">(.*)<\/(?:span>)?(?:a>)? in  Date: Mon, 15 Jan 2018 22:47:22 +0100 Subject: [PATCH 07/26] Formatting end time --- telegram.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telegram.go b/telegram.go index 3135f4a..146227a 100644 --- a/telegram.go +++ b/telegram.go @@ -61,7 +61,7 @@ func getTelegramGroupHistory(group string, grace int, dumpFlag bool) { messageCounter++ time.Sleep(time.Millisecond * 100) } - fmt.Println("==== [" + time.Now().String() + "(elapsed:" + time.Since(startTime).String() + ")] End of history ==== ") + fmt.Println("==== [" + time.Now().Format(time.RFC3339) + " (elapsed:" + time.Since(startTime).String() + ")] End of history ==== ") fmt.Println("[=] If you think there are more messages try to increase the grace perio (--grace [INT])") } From a875a0dbc719dfb4fa84d7970884d7f4dac468d0 Mon Sep 17 00:00:00 2001 From: Nhoya Date: Tue, 16 Jan 2018 00:02:59 +0100 Subject: [PATCH 08/26] --dumpfile is working properly --- telegram.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/telegram.go b/telegram.go index 146227a..fb60fcc 100644 --- a/telegram.go +++ b/telegram.go @@ -29,10 +29,11 @@ func getTelegramGroupHistory(group string, grace int, dumpFlag bool) { if message != "" { for j := 0; j < graceCounter; j++ { msg := "[MESSAGE REMOVED]" - writeOnFile(dumpfile, "["+strconv.Itoa(messageCounter-graceCounter+j)+"] "+msg+"\n") + if dumpFlag { + writeOnFile(dumpfile, "["+strconv.Itoa(messageCounter-graceCounter+j)+"] "+msg+"\n") + } fmt.Println(msg) } - graceCounter = 0 username, nickname := getTelegramUsername(body) date, time := getTelegramMessageDateTime(body) @@ -49,8 +50,9 @@ func getTelegramGroupHistory(group string, grace int, dumpFlag bool) { } msg, _ := html2text.FromString(msgtxt) - - writeOnFile(dumpfile, "["+messageid+"] "+strings.Replace(msg, "\n", " ⏎ ", -1)+"\n") + if dumpFlag { + writeOnFile(dumpfile, "["+messageid+"] "+strings.Replace(msg, "\n", " ⏎ ", -1)+"\n") + } fmt.Println(msg) } else { graceCounter++ From b42dd9941010923b4bb0a164e6d94d67ded1fe82 Mon Sep 17 00:00:00 2001 From: Nhoya Date: Tue, 16 Jan 2018 00:22:13 +0100 Subject: [PATCH 09/26] Fixed Typo --- README.md | 3 +++ telegram.go | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 562101c..498ebc9 100644 --- a/README.md +++ b/README.md @@ -140,3 +140,6 @@ retrive message history for telegram public group `gOSINT -m telegram --target [PublicGroupName] --dumpfile` the output will be stored in a file, if the file is already populated it will resume from the last ID + +## Telegram Crawler Demo +[![asciicast](https://asciinema.org/a/h1DVStNEwlPiqX7pLlNXuRanC.png)](https://asciinema.org/a/h1DVStNEwlPiqX7pLlNXuRanC) diff --git a/telegram.go b/telegram.go index fb60fcc..40728b9 100644 --- a/telegram.go +++ b/telegram.go @@ -64,7 +64,7 @@ func getTelegramGroupHistory(group string, grace int, dumpFlag bool) { time.Sleep(time.Millisecond * 100) } fmt.Println("==== [" + time.Now().Format(time.RFC3339) + " (elapsed:" + time.Since(startTime).String() + ")] End of history ==== ") - fmt.Println("[=] If you think there are more messages try to increase the grace perio (--grace [INT])") + fmt.Println("[=] If you think there are more messages try to increase the grace period (--grace [INT])") } From 2cf2d9212cfcf09bb499fb55d12f3ef527af805f Mon Sep 17 00:00:00 2001 From: Nhoya Date: Tue, 16 Jan 2018 02:22:16 +0100 Subject: [PATCH 10/26] More efficient code --- telegram.go | 16 +++++++++++++--- utils.go | 2 -- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/telegram.go b/telegram.go index 40728b9..b8f8f88 100644 --- a/telegram.go +++ b/telegram.go @@ -13,6 +13,7 @@ import ( ) func getTelegramGroupHistory(group string, grace int, dumpFlag bool) { + checkGroupName(group) graceCounter := 0 dumpfile := group + ".dump" msgtxt := "" @@ -21,7 +22,7 @@ func getTelegramGroupHistory(group string, grace int, dumpFlag bool) { messageCounter++ startTime := time.Now() fmt.Println("==== [" + startTime.Format(time.RFC3339) + "] Dumping messages for " + group + " ====") - for messageCounter != 0 { + for { messageid := strconv.Itoa(messageCounter) body := retriveRequestBody("https://t.me/" + group + "/" + messageid + "?embed=1") message := getTelegramMessage(body) @@ -57,13 +58,14 @@ func getTelegramGroupHistory(group string, grace int, dumpFlag bool) { } else { graceCounter++ if graceCounter == grace { + messageCounter = messageCounter - graceCounter break } } messageCounter++ time.Sleep(time.Millisecond * 100) } - fmt.Println("==== [" + time.Now().Format(time.RFC3339) + " (elapsed:" + time.Since(startTime).String() + ")] End of history ==== ") + fmt.Println("==== [" + time.Now().Format(time.RFC3339) + " (elapsed:" + time.Since(startTime).String() + ")] End of history - " + strconv.Itoa(messageCounter) + " messages scraped ==== ") fmt.Println("[=] If you think there are more messages try to increase the grace period (--grace [INT])") } @@ -156,8 +158,16 @@ func getTelegramMessageDateTime(body string) (string, string) { return match[1], match[2] } +func checkGroupName(group string) { + re := regexp.MustCompile(`^[[:alpha:]](?:\-?[[:alnum:]]){3,31}$`) + if !re.MatchString(group) { + fmt.Println("Invalid Group name, valid chars alphanum, -") + os.Exit(1) + } +} + func readFromTelegramDump(dumpfile string, dumpFlag bool) int { - messageCounter := 0 + messageCounter := 1 if dumpFlag { if fileExists(dumpfile) { fmt.Println("[=] The dump will be saved in " + dumpfile) diff --git a/utils.go b/utils.go index e5fcdc1..bf02dfc 100644 --- a/utils.go +++ b/utils.go @@ -22,8 +22,6 @@ func retriveRequestBody(domain string) string { } func findMailInText(body string, mailSet mapset.Set) { - - //re := regexp.MustCompile(`[\w\-\.]+\@[\w \.\-]+\.[\w]+`) re := regexp.MustCompile(`(?:![\n|\s])*(?:[\w\d\.\w\d]|(?:[\w\d]+[\-]+[\w\d]+))+[\@]+[\w]+[\.]+[\w]+`) mails := re.FindAllString(body, -1) if len(mails) == 0 { From a41dd4b3cc2c78017404b98832d1e4e3f48ec2d0 Mon Sep 17 00:00:00 2001 From: Nhoya Date: Tue, 16 Jan 2018 13:06:23 +0100 Subject: [PATCH 11/26] Added start flag for telegram scraper, renamed -t flag in a more specific -g --- gOSINT.go | 28 +++++++++++++++++----------- telegram.go | 11 +++++------ 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/gOSINT.go b/gOSINT.go index 074a913..6ea7645 100644 --- a/gOSINT.go +++ b/gOSINT.go @@ -11,18 +11,24 @@ import ( const ver = "v0.4c" var opts struct { - Module string `short:"m" long:"module" description:"Specify module" choice:"pgp" choice:"pwnd" choice:"git" choice:"plainSearch" choice:"telegram"` + Module string `short:"m" long:"module" description:"Specify module" choice:"pgp" choice:"pwnd" choice:"git" choice:"plainSearch" choice:"telegram"` + Version bool `short:"v" long:"version" description:"Print version"` + // git module Url string `long:"url" default:"" description:"Specify target URL"` - Target string `short:"t" long:"target" default:"" description:"Specify target"` GitAPIType string `long:"gitAPI" default:"" description:"Specify git website API to use (for git module,optional)" choice:"github" choice:"bitbucket"` - Mail string `long:"mail" default:"" description:"Specify mail target (for pgp and pwnd module)"` - Path string `short:"p" long:"path" description:"Specify target path (for plainSearch module)"` - TgGrace int `long:"grace" default:"15" description:"Specify telegram messages grace period"` - DumpFile bool `long:"dumpfile" description:"Create and resume messages from dumpfile"` - Mode bool `short:"f" long:"full" description:"Make deep search using linked modules"` Clone bool `short:"c" long:"clone" description:"Enable clone function for plainSearch module (need to specify repo URL)"` - Confirm bool `long:"ask-confirmation" description:"Ask confirmation before adding mail to set (for plainSearch module)"` - Version bool `short:"v" long:"version" description:"Print version"` + // pwn and pgp module + Mail string `long:"mail" default:"" description:"Specify mail target (for pgp and pwnd module)"` + // telegram module + TgGrace int `long:"grace" default:"15" description:"Specify telegram messages grace period"` + TgGroup string `short:"g" long:"target" default:"" description:"Specify Telegram group/channel name"` + TgStart int `short:"s" long:"tgstart" default:"1" description:"Specify first message to scrape"` + DumpFile bool `long:"dumpfile" description:"Create and resume messages from dumpfile"` + // plainSearch module + Confirm bool `long:"ask-confirmation" description:"Ask confirmation before adding mail to set (for plainSearch module)"` + Path string `short:"p" long:"path" description:"Specify target path (for plainSearch module)"` + // generic + Mode bool `short:"f" long:"full" description:"Make deep search using linked modules"` } func mailCheck(mailSet mapset.Set) { @@ -88,10 +94,10 @@ func main() { pwnd(mailSet) } case "telegram": - if opts.Target == "" { + if opts.TgGroup == "" { fmt.Println("You must specify target") os.Exit(1) } - getTelegramGroupHistory(opts.Target, opts.TgGrace, opts.DumpFile) + getTelegramGroupHistory(opts.TgGroup, opts.TgGrace, opts.DumpFile, opts.TgStart) } } diff --git a/telegram.go b/telegram.go index b8f8f88..ab37970 100644 --- a/telegram.go +++ b/telegram.go @@ -12,13 +12,13 @@ import ( "github.com/jaytaylor/html2text" ) -func getTelegramGroupHistory(group string, grace int, dumpFlag bool) { +func getTelegramGroupHistory(group string, grace int, dumpFlag bool, messageCounter int) { checkGroupName(group) graceCounter := 0 dumpfile := group + ".dump" msgtxt := "" - messageCounter := readFromTelegramDump(dumpfile, dumpFlag) + readFromTelegramDump(dumpfile, dumpFlag, &messageCounter) messageCounter++ startTime := time.Now() fmt.Println("==== [" + startTime.Format(time.RFC3339) + "] Dumping messages for " + group + " ====") @@ -166,9 +166,9 @@ func checkGroupName(group string) { } } -func readFromTelegramDump(dumpfile string, dumpFlag bool) int { - messageCounter := 1 +func readFromTelegramDump(dumpfile string, dumpFlag bool, messageCounter *int) { if dumpFlag { + fmt.Println("[=] --dumpfile used, ignoring --startpoint") if fileExists(dumpfile) { fmt.Println("[=] The dump will be saved in " + dumpfile) fmt.Println("[?] Print the existing dumb before resuming it? [Y/N]") @@ -186,10 +186,9 @@ func readFromTelegramDump(dumpfile string, dumpFlag bool) int { if resp == "y" || resp == "Y" { fmt.Println(strings.Join(messageSlice[1:], " ")) } - messageCounter, _ = strconv.Atoi(strings.Trim(messageSlice[0], "[]")) + *messageCounter, _ = strconv.Atoi(strings.Trim(messageSlice[0], "[]")) } fmt.Println("[=] Starting from message n.", messageCounter) } } - return messageCounter } From e6de1d1d08832f02d24b60f1414eca50cf980bc2 Mon Sep 17 00:00:00 2001 From: Nhoya Date: Tue, 16 Jan 2018 15:12:25 +0100 Subject: [PATCH 12/26] Added end flag --- gOSINT.go | 5 +++-- telegram.go | 57 +++++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 45 insertions(+), 17 deletions(-) diff --git a/gOSINT.go b/gOSINT.go index 6ea7645..600a817 100644 --- a/gOSINT.go +++ b/gOSINT.go @@ -22,7 +22,8 @@ var opts struct { // telegram module TgGrace int `long:"grace" default:"15" description:"Specify telegram messages grace period"` TgGroup string `short:"g" long:"target" default:"" description:"Specify Telegram group/channel name"` - TgStart int `short:"s" long:"tgstart" default:"1" description:"Specify first message to scrape"` + TgStart int `short:"s" long:"tgstart" default:"1" default-mask:"-" description:"Specify first message to scrape"` + TgEnd int `short:"e" long:"tgend" description:"Specify last message to scrape"` DumpFile bool `long:"dumpfile" description:"Create and resume messages from dumpfile"` // plainSearch module Confirm bool `long:"ask-confirmation" description:"Ask confirmation before adding mail to set (for plainSearch module)"` @@ -98,6 +99,6 @@ func main() { fmt.Println("You must specify target") os.Exit(1) } - getTelegramGroupHistory(opts.TgGroup, opts.TgGrace, opts.DumpFile, opts.TgStart) + getTelegramGroupHistory(opts.TgGroup, opts.TgGrace, opts.DumpFile, (opts.TgStart - 1), opts.TgEnd) } } diff --git a/telegram.go b/telegram.go index ab37970..9ad8e37 100644 --- a/telegram.go +++ b/telegram.go @@ -12,30 +12,43 @@ import ( "github.com/jaytaylor/html2text" ) -func getTelegramGroupHistory(group string, grace int, dumpFlag bool, messageCounter int) { +func getTelegramGroupHistory(group string, grace int, dumpFlag bool, startMessage int, endMessage int) { checkGroupName(group) + if endMessage != 0 { + if endMessage <= startMessage { + fmt.Println("[-] The final message number (-e) must be >= than start message number (-s)") + os.Exit(1) + } + } + graceCounter := 0 + if endMessage != 0 { + fmt.Println("[?] End message set, grace time will be ignored") + } dumpfile := group + ".dump" msgtxt := "" + messageCounter := startMessage readFromTelegramDump(dumpfile, dumpFlag, &messageCounter) messageCounter++ + startTime := time.Now() fmt.Println("==== [" + startTime.Format(time.RFC3339) + "] Dumping messages for " + group + " ====") for { messageid := strconv.Itoa(messageCounter) body := retriveRequestBody("https://t.me/" + group + "/" + messageid + "?embed=1") message := getTelegramMessage(body) - if message != "" { - for j := 0; j < graceCounter; j++ { - msg := "[MESSAGE REMOVED]" - if dumpFlag { - writeOnFile(dumpfile, "["+strconv.Itoa(messageCounter-graceCounter+j)+"] "+msg+"\n") + if endMessage == 0 { + for j := 0; j < graceCounter; j++ { + msg := "[MESSAGE REMOVED]" + if dumpFlag { + writeOnFile(dumpfile, "["+strconv.Itoa(messageCounter)+"] "+msg+"\n") + } + fmt.Println(msg) } - fmt.Println(msg) + graceCounter = 0 } - graceCounter = 0 username, nickname := getTelegramUsername(body) date, time := getTelegramMessageDateTime(body) @@ -56,18 +69,32 @@ func getTelegramGroupHistory(group string, grace int, dumpFlag bool, messageCoun } fmt.Println(msg) } else { - graceCounter++ - if graceCounter == grace { - messageCounter = messageCounter - graceCounter + if endMessage == 0 { + graceCounter++ + if graceCounter == grace { + messageCounter = messageCounter - graceCounter + break + } + } else { + msg := "[DELETED MESSAGE]" + if dumpFlag { + writeOnFile(dumpfile, "["+strconv.Itoa(messageCounter)+"] "+msg+"\n") + } + fmt.Println(msg) + } + } + if endMessage != 0 { + if messageCounter == endMessage { break } } messageCounter++ time.Sleep(time.Millisecond * 100) } - fmt.Println("==== [" + time.Now().Format(time.RFC3339) + " (elapsed:" + time.Since(startTime).String() + ")] End of history - " + strconv.Itoa(messageCounter) + " messages scraped ==== ") - fmt.Println("[=] If you think there are more messages try to increase the grace period (--grace [INT])") - + fmt.Println("==== [" + time.Now().Format(time.RFC3339) + " (elapsed:" + time.Since(startTime).String() + ")] End of history - " + strconv.Itoa(messageCounter-startMessage) + " messages scraped ==== ") + if endMessage == 0 { + fmt.Println("[=] If you think there are more messages try to increase the grace period (--grace [INT])") + } } func getTelegramMessage(body string) string { @@ -188,7 +215,7 @@ func readFromTelegramDump(dumpfile string, dumpFlag bool, messageCounter *int) { } *messageCounter, _ = strconv.Atoi(strings.Trim(messageSlice[0], "[]")) } - fmt.Println("[=] Starting from message n.", messageCounter) + fmt.Println("[=] Starting from message n.", *messageCounter) } } } From f70d1cecf6763625988d2d08d4b76254f6405beb Mon Sep 17 00:00:00 2001 From: Nhoya Date: Tue, 16 Jan 2018 15:15:54 +0100 Subject: [PATCH 13/26] Change --target to -tgroup --- gOSINT.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gOSINT.go b/gOSINT.go index 600a817..7074262 100644 --- a/gOSINT.go +++ b/gOSINT.go @@ -21,7 +21,7 @@ var opts struct { Mail string `long:"mail" default:"" description:"Specify mail target (for pgp and pwnd module)"` // telegram module TgGrace int `long:"grace" default:"15" description:"Specify telegram messages grace period"` - TgGroup string `short:"g" long:"target" default:"" description:"Specify Telegram group/channel name"` + TgGroup string `short:"g" long:"tgroup" default:"" description:"Specify Telegram group/channel name"` TgStart int `short:"s" long:"tgstart" default:"1" default-mask:"-" description:"Specify first message to scrape"` TgEnd int `short:"e" long:"tgend" description:"Specify last message to scrape"` DumpFile bool `long:"dumpfile" description:"Create and resume messages from dumpfile"` From 0a9835e3adc3fd7bafe7e1e7e3ef47c1ae4aca49 Mon Sep 17 00:00:00 2001 From: Nhoya Date: Tue, 16 Jan 2018 15:19:11 +0100 Subject: [PATCH 14/26] more examples --- README.md | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 498ebc9..2755200 100644 --- a/README.md +++ b/README.md @@ -70,18 +70,23 @@ Usage: gOSINT [OPTIONS] Application Options: - -m, --module=[pgp|pwnd|git|plainSearch] Specify module - --url= Specify target URL - --gitAPI=[github|bitbucket] Specify git website API to use (for git module,optional) - --mail= Specify mail target (for pgp and pwnd module) - -p, --path= Specify target path (for plainSearch module) - -f, --full Make deep search using linked modules - -c, --clone Enable clone function for plainSearch module (need to specify repo URL) - --ask-confirmation Ask confirmation before adding mail to set (for plainSearch module) - -v, --version Print version + -m, --module=[pgp|pwnd|git|plainSearch|telegram] Specify module + -v, --version Print version + --url= Specify target URL + --gitAPI=[github|bitbucket] Specify git website API to use (for git module,optional) + -c, --clone Enable clone function for plainSearch module (need to specify repo URL) + --mail= Specify mail target (for pgp and pwnd module) + --grace= Specify telegram messages grace period (default: 15) + -g, --tgroup= Specify Telegram group/channel name + -s, --tgstart= Specify first message to scrape + -e, --tgend= Specify last message to scrape + --dumpfile Create and resume messages from dumpfile + --ask-confirmation Ask confirmation before adding mail to set (for plainSearch module) + -p, --path= Specify target path (for plainSearch module) + -f, --full Make deep search using linked modules Help Options: - -h, --help Show this help message + -h, --help Show this help message ``` ## Examples @@ -133,13 +138,16 @@ pass the resoult to pgp search and haveibeenpwnd modules ask confirmation before adding mail to search results -`gOSINT -m telegram --target [PublicGroupName]` +`gOSINT -m telegram --tgroup | -g [PublicGroupName]` retrive message history for telegram public group -`gOSINT -m telegram --target [PublicGroupName] --dumpfile` +`gOSINT -m telegram --tgroup | -g [PublicGroupName] --dumpfile` the output will be stored in a file, if the file is already populated it will resume from the last ID +`gOSINT -m telegram --tgroup | -g [PublicGroupName] --dumpfile -s [masageID] -e [messageID]` + +Set start and end messages for scraping ## Telegram Crawler Demo [![asciicast](https://asciinema.org/a/h1DVStNEwlPiqX7pLlNXuRanC.png)](https://asciinema.org/a/h1DVStNEwlPiqX7pLlNXuRanC) From 6ef777bbe79b5ef3e3c55ec980948e486ca23d8b Mon Sep 17 00:00:00 2001 From: Nhoya Date: Wed, 17 Jan 2018 14:32:19 +0100 Subject: [PATCH 15/26] removed nested control --- telegram.go | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/telegram.go b/telegram.go index 9ad8e37..c5c9d4f 100644 --- a/telegram.go +++ b/telegram.go @@ -38,17 +38,16 @@ func getTelegramGroupHistory(group string, grace int, dumpFlag bool, startMessag messageid := strconv.Itoa(messageCounter) body := retriveRequestBody("https://t.me/" + group + "/" + messageid + "?embed=1") message := getTelegramMessage(body) - if message != "" { - if endMessage == 0 { - for j := 0; j < graceCounter; j++ { - msg := "[MESSAGE REMOVED]" - if dumpFlag { - writeOnFile(dumpfile, "["+strconv.Itoa(messageCounter)+"] "+msg+"\n") - } - fmt.Println(msg) + if message != "" && graceCounter > 0 { + for j := 0; j < graceCounter; j++ { + msg := "[MESSAGE REMOVED]" + if dumpFlag { + writeOnFile(dumpfile, "["+strconv.Itoa(messageCounter)+"] "+msg+"\n") } - graceCounter = 0 + fmt.Println(msg) } + graceCounter = 0 + } else if message != "" { username, nickname := getTelegramUsername(body) date, time := getTelegramMessageDateTime(body) @@ -69,6 +68,10 @@ func getTelegramGroupHistory(group string, grace int, dumpFlag bool, startMessag } fmt.Println(msg) } else { + if messageCounter == 1 { + fmt.Println("[!!] Invalid group") + break + } if endMessage == 0 { graceCounter++ if graceCounter == grace { @@ -85,6 +88,7 @@ func getTelegramGroupHistory(group string, grace int, dumpFlag bool, startMessag } if endMessage != 0 { if messageCounter == endMessage { + messageCounter-- break } } @@ -92,7 +96,7 @@ func getTelegramGroupHistory(group string, grace int, dumpFlag bool, startMessag time.Sleep(time.Millisecond * 100) } fmt.Println("==== [" + time.Now().Format(time.RFC3339) + " (elapsed:" + time.Since(startTime).String() + ")] End of history - " + strconv.Itoa(messageCounter-startMessage) + " messages scraped ==== ") - if endMessage == 0 { + if endMessage == 0 && messageCounter > 0 { fmt.Println("[=] If you think there are more messages try to increase the grace period (--grace [INT])") } } From 3f7e5e24c894f6557669d87bc188150abc8d9f63 Mon Sep 17 00:00:00 2001 From: Nhoya Date: Wed, 17 Jan 2018 15:54:22 +0100 Subject: [PATCH 16/26] code refactoring --- telegram.go | 76 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 43 insertions(+), 33 deletions(-) diff --git a/telegram.go b/telegram.go index c5c9d4f..dd06bac 100644 --- a/telegram.go +++ b/telegram.go @@ -14,45 +14,50 @@ import ( func getTelegramGroupHistory(group string, grace int, dumpFlag bool, startMessage int, endMessage int) { checkGroupName(group) + // check if -e option is set if endMessage != 0 { + // end can't be less than start if endMessage <= startMessage { fmt.Println("[-] The final message number (-e) must be >= than start message number (-s)") os.Exit(1) } - } - graceCounter := 0 - if endMessage != 0 { fmt.Println("[?] End message set, grace time will be ignored") } - dumpfile := group + ".dump" - msgtxt := "" + //path for dumps + dumpfile := "tgdumps/" + group + ".dump" + //counter for deleted messages + dmCounter := 0 + //set messageCounter as startMessage, is -e is not used the default value of startMessage is 0 (Note: the first message on group is id:1) messageCounter := startMessage readFromTelegramDump(dumpfile, dumpFlag, &messageCounter) + //this is needed because if a file is availabe it will start the next to the last found messageCounter++ startTime := time.Now() fmt.Println("==== [" + startTime.Format(time.RFC3339) + "] Dumping messages for " + group + " ====") + + //we don't know how many first how many messages the group has for { messageid := strconv.Itoa(messageCounter) body := retriveRequestBody("https://t.me/" + group + "/" + messageid + "?embed=1") message := getTelegramMessage(body) - if message != "" && graceCounter > 0 { - for j := 0; j < graceCounter; j++ { + + if message != "" && dmCounter > 0 { + //this is to avoid to write on file the last n empty messages + for j := 0; j < dmCounter; j++ { msg := "[MESSAGE REMOVED]" - if dumpFlag { - writeOnFile(dumpfile, "["+strconv.Itoa(messageCounter)+"] "+msg+"\n") - } - fmt.Println(msg) + writeTelegramLogs(messageCounter, msg, dumpFlag, dumpfile) } - graceCounter = 0 + dmCounter = 0 } else if message != "" { + // message infos username, nickname := getTelegramUsername(body) date, time := getTelegramMessageDateTime(body) - + var msgtxt string if username == "" { - //for channels + //channels have only messages if nickname == "" { msgtxt = "[" + date + " " + time + "] " + message } else { @@ -62,35 +67,33 @@ func getTelegramGroupHistory(group string, grace int, dumpFlag bool, startMessag msgtxt = "[" + date + " " + time + "] " + nickname + "(" + username + "): " + message } + //html format the message before printing it msg, _ := html2text.FromString(msgtxt) - if dumpFlag { - writeOnFile(dumpfile, "["+messageid+"] "+strings.Replace(msg, "\n", " ⏎ ", -1)+"\n") - } - fmt.Println(msg) + writeTelegramLogs(messageCounter, msg, dumpFlag, dumpfile) } else { + //the first message is always a service message, if doesn't exist the groups is not valid if messageCounter == 1 { fmt.Println("[!!] Invalid group") break } - if endMessage == 0 { - graceCounter++ - if graceCounter == grace { - messageCounter = messageCounter - graceCounter - break - } + //if is the last message end -e is not set + if endMessage == 0 && dmCounter == grace { + dmCounter++ + messageCounter = messageCounter - dmCounter + break + } else if endMessage == 0 { + //if -e is not set and is not the last message increase the counter + dmCounter++ } else { + //if -e is set and the message is empty dmCounter is 0 and grace is 0 so print the message msg := "[DELETED MESSAGE]" - if dumpFlag { - writeOnFile(dumpfile, "["+strconv.Itoa(messageCounter)+"] "+msg+"\n") - } - fmt.Println(msg) + writeTelegramLogs(messageCounter, msg, dumpFlag, dumpfile) } } - if endMessage != 0 { - if messageCounter == endMessage { - messageCounter-- - break - } + // if this is the last message quit + if endMessage != 0 && messageCounter == endMessage { + messageCounter-- + break } messageCounter++ time.Sleep(time.Millisecond * 100) @@ -197,6 +200,13 @@ func checkGroupName(group string) { } } +func writeTelegramLogs(messageCounter int, msg string, dumpFlag bool, dumpfile string) { + if dumpFlag { + writeOnFile(dumpfile, "["+strconv.Itoa(messageCounter)+"] "+strings.Replace(msg, "\n", " ⏎ ", -1)+"\n") + } + fmt.Println(msg) +} + func readFromTelegramDump(dumpfile string, dumpFlag bool, messageCounter *int) { if dumpFlag { fmt.Println("[=] --dumpfile used, ignoring --startpoint") From 2bb5d913da7c3ad4636cfc02c719e347a6c2fc2e Mon Sep 17 00:00:00 2001 From: Nhoya Date: Wed, 17 Jan 2018 18:08:46 +0100 Subject: [PATCH 17/26] Refactoring lots of stuff --- README.md | 10 +++++----- gOSINT.go | 43 +++++-------------------------------------- git.go | 22 +++++++++++++++++----- pgp.go | 10 +++++++++- pwnd.go | 5 +++++ sourceSearch.go | 27 +++++++++++++++++++++------ telegram.go | 47 ++++++++++++++++++++--------------------------- 7 files changed, 82 insertions(+), 82 deletions(-) diff --git a/README.md b/README.md index 2755200..99552e2 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # gOSINT [![Build Status](https://travis-ci.org/Nhoya/gOSINT.svg?branch=master)](https://travis-ci.org/Nhoya/gOSINT) [![GitHub stars](https://img.shields.io/github/stars/Nhoya/gOSINT.svg)](https://github.com/Nhoya/gOSINT/stargazers) [![GitHub forks](https://img.shields.io/github/forks/Nhoya/gOSINT.svg)](https://github.com/Nhoya/gOSINT/network) [![Twitter](https://img.shields.io/twitter/url/https/github.com/Nhoya/gOSINT.svg?style=social&style=plastic)](https://twitter.com/intent/tweet?text=Wow:&url=https%3A%2F%2Fgithub.com%2FNhoya%2FgOSINT) -OSINT framework in Go +OSINT framework in Go - Donate BTC: 346dDn4BjTodMkGkb5iTyoJ5Z2wi9g9QiM you probably want to take a look at the develop branch for more updates. @@ -14,9 +14,9 @@ gOSINT is a small OSINT framework in Golang, if you want, feel free to contribut - [x] Find mails from git repository - [x] Find Dumps for mail address - [x] Search for mail address linked to domain/mail address in PGP keyring -- [x] Retrive Info from domain whois (waiting to be implemented) +- [x] Retrieve Info from domain whois (waiting to be implemented) - [x] Search for mail address in source code -- [x] Retrive Telegram Public Groups History +- [x] Retrieve Telegram Public Groups History ## Building @@ -96,7 +96,7 @@ Currently `gOSINT` supports the following actions `gOSINT -m git --url=[RepoURL] --gitAPI [github|bitbucket] (optional)` -retrive mail from git commits +retrieve mail from git commits `gOSINT -m git --url [RepoURL] --gitAPI [github|bitbucket] (optional) -f` @@ -140,7 +140,7 @@ ask confirmation before adding mail to search results `gOSINT -m telegram --tgroup | -g [PublicGroupName]` -retrive message history for telegram public group +retrieve message history for telegram public group `gOSINT -m telegram --tgroup | -g [PublicGroupName] --dumpfile` diff --git a/gOSINT.go b/gOSINT.go index 7074262..eddb261 100644 --- a/gOSINT.go +++ b/gOSINT.go @@ -58,47 +58,14 @@ func main() { switch mod := opts.Module; mod { case "pwnd": - mailCheck(mailSet) - pwnd(mailSet) + initPwnd(mailSet) case "pgp": - mailCheck(mailSet) - mailSet = pgpSearch(mailSet) - if opts.Mode { - pwnd(mailSet) - } + initPGP(mailSet) case "git": - if opts.Url == "" { - fmt.Println("You must specify target URL") - os.Exit(1) - } - mailSet = gitSearch(opts.Url, opts.GitAPIType, mailSet) - if opts.Mode { - mailSet = pgpSearch(mailSet) - pwnd(mailSet) - } + initGit(mailSet) case "plainSearch": - if opts.Clone { - if opts.Url == "" { - fmt.Println("You must specify target URL") - os.Exit(1) - } - mailSet = cloneAndSearch(opts.Url, mailSet, opts.Confirm) - } else { - if opts.Path == "" { - fmt.Println("You must specify Path") - os.Exit(1) - } - mailSet = plainMailSearch(opts.Path, mailSet, opts.Confirm) - } - if opts.Mode { - mailSet = pgpSearch(mailSet) - pwnd(mailSet) - } + initPlainSearch(mailSet) case "telegram": - if opts.TgGroup == "" { - fmt.Println("You must specify target") - os.Exit(1) - } - getTelegramGroupHistory(opts.TgGroup, opts.TgGrace, opts.DumpFile, (opts.TgStart - 1), opts.TgEnd) + initTelegram() } } diff --git a/git.go b/git.go index 5d047d1..930c831 100644 --- a/git.go +++ b/git.go @@ -14,6 +14,18 @@ import ( "gopkg.in/src-d/go-git.v4/storage/memory" ) +func initGit(mailSet mapset.Set) { + if opts.Url == "" { + fmt.Println("You must specify target URL") + os.Exit(1) + } + mailSet = gitSearch(opts.Url, opts.GitAPIType, mailSet) + if opts.Mode { + mailSet = pgpSearch(mailSet) + pwnd(mailSet) + } +} + func gitSearch(target string, WebsiteAPI string, mailSet mapset.Set) mapset.Set { // TODO: add worker for pagination domain := "" @@ -27,11 +39,11 @@ func gitSearch(target string, WebsiteAPI string, mailSet mapset.Set) mapset.Set fmt.Println("[+] Using github API") domain = targetSplit[0] + "//api." + targetSplit[2] + "/repos/" + targetSplit[3] + "/" + targetSplit[4] + "/commits?per_page=100" //GitHub Pagination - lastPage := retriveLastGHPage(domain) + lastPage := retrieveLastGHPage(domain) fmt.Println("[+] Looping through pages.This MAY take a while...") for page := 1; page < lastPage+1; page++ { fmt.Println("[+] Analyzing commits page: " + strconv.Itoa(page)) - commits = retriveRequestBody(domain + "&page=" + strconv.Itoa(page)) + commits = retrieveRequestBody(domain + "&page=" + strconv.Itoa(page)) findMailInText(commits, mailSet) } } else if strings.Contains(target, "https://bitbucket.org") || WebsiteAPI == "bitbucket" { @@ -44,8 +56,8 @@ func gitSearch(target string, WebsiteAPI string, mailSet mapset.Set) mapset.Set for page != 0 { fmt.Println("[+] Analyzing commits page: " + strconv.Itoa(page)) pageDom := domain + "&page=" + strconv.Itoa(page) - //This is needed because we can't unluckily retrive max_page from one single request - pageContent := retriveRequestBody(pageDom) + //This is needed because we can't unluckily retrieve max_page from one single request + pageContent := retrieveRequestBody(pageDom) nextPage := "\"next\": \"" + domain + "&page=" findMailInText(pageContent, mailSet) @@ -70,7 +82,7 @@ func gitSearch(target string, WebsiteAPI string, mailSet mapset.Set) mapset.Set return mailSet } -func retriveLastGHPage(domain string) int { +func retrieveLastGHPage(domain string) int { req, err := http.Get(domain) if err != nil { panic(err) diff --git a/pgp.go b/pgp.go index aa56d61..7ee2e83 100644 --- a/pgp.go +++ b/pgp.go @@ -6,6 +6,14 @@ import ( "github.com/deckarep/golang-set" ) +func initPGP(mailSet mapset.Set) { + mailCheck(mailSet) + mailSet = pgpSearch(mailSet) + if opts.Mode { + pwnd(mailSet) + } +} + func pgpSearch(mailSet mapset.Set) mapset.Set { fmt.Println("==== PGP SEARCH ====") mailIterator := mailSet.Iterator() @@ -13,7 +21,7 @@ func pgpSearch(mailSet mapset.Set) mapset.Set { pgpSet := mapset.NewSet() fmt.Println("[+] pgp search for " + mail.(string)) domain := "http://pgp.mit.edu/pks/lookup?search=" + mail.(string) - body := retriveRequestBody(domain) + body := retrieveRequestBody(domain) findMailInText(body, pgpSet) if pgpSet != nil { pgpIterator := pgpSet.Iterator() diff --git a/pwnd.go b/pwnd.go index 47f4245..21948ce 100644 --- a/pwnd.go +++ b/pwnd.go @@ -8,6 +8,11 @@ import ( "github.com/nhoya/goPwned" ) +func initPwnd(mailSet mapset.Set) { + mailCheck(mailSet) + pwnd(mailSet) +} + func pwnd(mailSet mapset.Set) { fmt.Println("==== HAVEIBEENPWND SEARCH ====") mailIterator := mailSet.Iterator() diff --git a/sourceSearch.go b/sourceSearch.go index e7e070a..5689a7b 100644 --- a/sourceSearch.go +++ b/sourceSearch.go @@ -1,7 +1,6 @@ package main import ( - "bufio" "fmt" "io/ioutil" "os" @@ -11,6 +10,25 @@ import ( "gopkg.in/src-d/go-git.v4" ) +func initPlainSearch(mailSet mapset.Set) { + if opts.Clone { + if opts.Url == "" { + fmt.Println("You must specify target URL") + os.Exit(1) + } + mailSet = cloneAndSearch(opts.Url, mailSet, opts.Confirm) + } else { + if opts.Path == "" { + fmt.Println("You must specify Path") + os.Exit(1) + } + mailSet = plainMailSearch(opts.Path, mailSet, opts.Confirm) + } + if opts.Mode { + mailSet = pgpSearch(mailSet) + pwnd(mailSet) + } +} func checkFile(mailSet mapset.Set) filepath.WalkFunc { return func(path string, info os.FileInfo, err error) error { if !info.IsDir() { @@ -30,12 +48,9 @@ func plainMailSearch(path string, mailSet mapset.Set, confirm bool) mapset.Set { if confirm { fmt.Println("confirm?") tmpIt := tmpSet.Iterator() - scanner := bufio.NewScanner(os.Stdin) for tmpMail := range tmpIt.C { - fmt.Println("[?] Found " + tmpMail.(string) + " remove it?[Y/n]") - scanner.Scan() - text := scanner.Text() - if text == "y" || text == "Y" { + resp := simpleQuestion("Found " + tmpMail.(string) + " remove it?") + if resp { diffSet.Add(tmpMail) } } diff --git a/telegram.go b/telegram.go index dd06bac..26c5071 100644 --- a/telegram.go +++ b/telegram.go @@ -56,13 +56,11 @@ func getTelegramGroupHistory(group string, grace int, dumpFlag bool, startMessag username, nickname := getTelegramUsername(body) date, time := getTelegramMessageDateTime(body) var msgtxt string - if username == "" { - //channels have only messages - if nickname == "" { - msgtxt = "[" + date + " " + time + "] " + message - } else { - msgtxt = "[" + date + " " + time + "] " + nickname + ": " + message - } + //channels don't have username and nickname + if username == "" && nickname == "" { + msgtxt = "[" + date + " " + time + "] " + message + } else if username == "" { + msgtxt = "[" + date + " " + time + "] " + nickname + ": " + message } else { msgtxt = "[" + date + " " + time + "] " + nickname + "(" + username + "): " + message } @@ -70,29 +68,24 @@ func getTelegramGroupHistory(group string, grace int, dumpFlag bool, startMessag //html format the message before printing it msg, _ := html2text.FromString(msgtxt) writeTelegramLogs(messageCounter, msg, dumpFlag, dumpfile) - } else { + } else if messageCounter == 1 { //the first message is always a service message, if doesn't exist the groups is not valid - if messageCounter == 1 { - fmt.Println("[!!] Invalid group") - break - } - //if is the last message end -e is not set - if endMessage == 0 && dmCounter == grace { - dmCounter++ - messageCounter = messageCounter - dmCounter - break - } else if endMessage == 0 { - //if -e is not set and is not the last message increase the counter - dmCounter++ - } else { - //if -e is set and the message is empty dmCounter is 0 and grace is 0 so print the message - msg := "[DELETED MESSAGE]" - writeTelegramLogs(messageCounter, msg, dumpFlag, dumpfile) - } + fmt.Println("[!!] Invalid group") + break + } else if endMessage == 0 && dmCounter == grace { + dmCounter++ + messageCounter = messageCounter - dmCounter + break + } else if endMessage == 0 { + //if -e is not set and is not the last message increase the counter + dmCounter++ + } else { + //if -e is set and the message is empty dmCounter is 0 and grace is 0 so print the message + msg := "[DELETED MESSAGE]" + writeTelegramLogs(messageCounter, msg, dumpFlag, dumpfile) } - // if this is the last message quit + // if this is the last message (defined with -e) quit if endMessage != 0 && messageCounter == endMessage { - messageCounter-- break } messageCounter++ From 631ff1c8f693b50ea041e1970a538f54ac83041a Mon Sep 17 00:00:00 2001 From: Nhoya Date: Wed, 17 Jan 2018 18:10:13 +0100 Subject: [PATCH 18/26] Refactoring pt2 --- telegram.go | 26 ++++++++++++++++---------- utils.go | 30 +++++++++++++++++++++++++----- 2 files changed, 41 insertions(+), 15 deletions(-) diff --git a/telegram.go b/telegram.go index 26c5071..53fbefa 100644 --- a/telegram.go +++ b/telegram.go @@ -12,6 +12,14 @@ import ( "github.com/jaytaylor/html2text" ) +func initTelegram() { + if opts.TgGroup == "" { + fmt.Println("You must specify target") + os.Exit(1) + } + getTelegramGroupHistory(opts.TgGroup, opts.TgGrace, opts.DumpFile, (opts.TgStart - 1), opts.TgEnd) +} + func getTelegramGroupHistory(group string, grace int, dumpFlag bool, startMessage int, endMessage int) { checkGroupName(group) // check if -e option is set @@ -34,14 +42,17 @@ func getTelegramGroupHistory(group string, grace int, dumpFlag bool, startMessag readFromTelegramDump(dumpfile, dumpFlag, &messageCounter) //this is needed because if a file is availabe it will start the next to the last found messageCounter++ - + if dumpFlag && ((endMessage != 0 && messageCounter >= endMessage) || (startMessage != 0 && messageCounter >= startMessage)) { + fmt.Println("[-] You already have this messages") + os.Exit(1) + } startTime := time.Now() fmt.Println("==== [" + startTime.Format(time.RFC3339) + "] Dumping messages for " + group + " ====") //we don't know how many first how many messages the group has for { messageid := strconv.Itoa(messageCounter) - body := retriveRequestBody("https://t.me/" + group + "/" + messageid + "?embed=1") + body := retrieveRequestBody("https://t.me/" + group + "/" + messageid + "?embed=1") message := getTelegramMessage(body) if message != "" && dmCounter > 0 { @@ -202,22 +213,17 @@ func writeTelegramLogs(messageCounter int, msg string, dumpFlag bool, dumpfile s func readFromTelegramDump(dumpfile string, dumpFlag bool, messageCounter *int) { if dumpFlag { + createDirectory("tgdumps") fmt.Println("[=] --dumpfile used, ignoring --startpoint") if fileExists(dumpfile) { fmt.Println("[=] The dump will be saved in " + dumpfile) - fmt.Println("[?] Print the existing dumb before resuming it? [Y/N]") - var resp string - _, err := fmt.Scanln(&resp) - if err != nil { - fmt.Println("[-] Unable to read answer") - os.Exit(1) - } + resp := simpleQuestion("Print the existing dump before resuming it?") fmt.Println("[+] Calculating the last message") file, _ := os.Open(dumpfile) scan := bufio.NewScanner(file) for scan.Scan() { messageSlice := strings.Split(scan.Text(), " ") - if resp == "y" || resp == "Y" { + if resp { fmt.Println(strings.Join(messageSlice[1:], " ")) } *messageCounter, _ = strconv.Atoi(strings.Trim(messageSlice[0], "[]")) diff --git a/utils.go b/utils.go index bf02dfc..670fd19 100644 --- a/utils.go +++ b/utils.go @@ -11,13 +11,13 @@ import ( "github.com/deckarep/golang-set" ) -func retriveRequestBody(domain string) string { - req, err := http.Get(domain) +func retrieveRequestBody(domain string) string { + resp, err := http.Get(domain) if err != nil { panic(err) } - defer req.Body.Close() - body, _ := ioutil.ReadAll(req.Body) + defer resp.Body.Close() + body, _ := ioutil.ReadAll(resp.Body) return string(body) } @@ -32,7 +32,6 @@ func findMailInText(body string, mailSet mapset.Set) { mailSet.Add(mail) } } - } func readFromSet(mailSet mapset.Set) { @@ -70,3 +69,24 @@ func fileExists(file string) bool { } return false } +func createDirectory(dirname string) { + fmt.Println("test") + if !fileExists("tgdumps") { + fmt.Println("[+] Creating directory " + dirname) + os.Mkdir(dirname, os.ModePerm) + } +} + +func simpleQuestion(question string) bool { + fmt.Println("[?] " + question + " [Y/N]") + var resp string + _, err := fmt.Scanln(&resp) + if err != nil { + fmt.Println("[-] Unable to read answer") + os.Exit(1) + } + if resp == "y" || resp == "Y" { + return true + } + return false +} From c91e850f3c05ac1cd9e19bcef88f7fd4deed7907 Mon Sep 17 00:00:00 2001 From: Nhoya Date: Thu, 18 Jan 2018 13:26:45 +0100 Subject: [PATCH 19/26] Refactoring --- telegram.go | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/telegram.go b/telegram.go index 53fbefa..d37accc 100644 --- a/telegram.go +++ b/telegram.go @@ -40,8 +40,9 @@ func getTelegramGroupHistory(group string, grace int, dumpFlag bool, startMessag //set messageCounter as startMessage, is -e is not used the default value of startMessage is 0 (Note: the first message on group is id:1) messageCounter := startMessage readFromTelegramDump(dumpfile, dumpFlag, &messageCounter) - //this is needed because if a file is availabe it will start the next to the last found + //this is needed because if a file is availabe it will start from the next to the last found messageCounter++ + //if -e or - s is set but on the dumpfile the message is already scraped if dumpFlag && ((endMessage != 0 && messageCounter >= endMessage) || (startMessage != 0 && messageCounter >= startMessage)) { fmt.Println("[-] You already have this messages") os.Exit(1) @@ -63,21 +64,8 @@ func getTelegramGroupHistory(group string, grace int, dumpFlag bool, startMessag } dmCounter = 0 } else if message != "" { - // message infos - username, nickname := getTelegramUsername(body) - date, time := getTelegramMessageDateTime(body) - var msgtxt string - //channels don't have username and nickname - if username == "" && nickname == "" { - msgtxt = "[" + date + " " + time + "] " + message - } else if username == "" { - msgtxt = "[" + date + " " + time + "] " + nickname + ": " + message - } else { - msgtxt = "[" + date + " " + time + "] " + nickname + "(" + username + "): " + message - } - - //html format the message before printing it - msg, _ := html2text.FromString(msgtxt) + //retrive the message the message message + msg := createMessage(body, message) writeTelegramLogs(messageCounter, msg, dumpFlag, dumpfile) } else if messageCounter == 1 { //the first message is always a service message, if doesn't exist the groups is not valid @@ -211,6 +199,23 @@ func writeTelegramLogs(messageCounter int, msg string, dumpFlag bool, dumpfile s fmt.Println(msg) } +func createMessage(body string, message string) string { + username, nickname := getTelegramUsername(body) + date, time := getTelegramMessageDateTime(body) + var msgtxt string + //channels don't have username and nickname + if username == "" && nickname == "" { + msgtxt = "[" + date + " " + time + "] " + message + } else if username == "" { + msgtxt = "[" + date + " " + time + "] " + nickname + ": " + message + } else { + msgtxt = "[" + date + " " + time + "] " + nickname + "(" + username + "): " + message + } + //html format the message before printing it + msg, _ := html2text.FromString(msgtxt) + return msg +} + func readFromTelegramDump(dumpfile string, dumpFlag bool, messageCounter *int) { if dumpFlag { createDirectory("tgdumps") From 4af28fe3481d555460d41bd4077adf503b73844c Mon Sep 17 00:00:00 2001 From: Nhoya Date: Thu, 18 Jan 2018 15:21:56 +0100 Subject: [PATCH 20/26] Minor fixes, added goreportcard badge --- README.md | 2 +- git.go | 2 +- sourceSearch.go | 5 +++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 99552e2..ad3ae6c 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# gOSINT [![Build Status](https://travis-ci.org/Nhoya/gOSINT.svg?branch=master)](https://travis-ci.org/Nhoya/gOSINT) [![GitHub stars](https://img.shields.io/github/stars/Nhoya/gOSINT.svg)](https://github.com/Nhoya/gOSINT/stargazers) [![GitHub forks](https://img.shields.io/github/forks/Nhoya/gOSINT.svg)](https://github.com/Nhoya/gOSINT/network) [![Twitter](https://img.shields.io/twitter/url/https/github.com/Nhoya/gOSINT.svg?style=social&style=plastic)](https://twitter.com/intent/tweet?text=Wow:&url=https%3A%2F%2Fgithub.com%2FNhoya%2FgOSINT) +# gOSINT [![Build Status](https://travis-ci.org/Nhoya/gOSINT.svg?branch=master)](https://travis-ci.org/Nhoya/gOSINT) [![GitHub stars](https://img.shields.io/github/stars/Nhoya/gOSINT.svg)](https://github.com/Nhoya/gOSINT/stargazers) [![GitHub forks](https://img.shields.io/github/forks/Nhoya/gOSINT.svg)](https://github.com/Nhoya/gOSINT/network) [![Twitter](https://img.shields.io/twitter/url/https/github.com/Nhoya/gOSINT.svg?style=social&style=plastic)](https://twitter.com/intent/tweet?text=Wow:&url=https%3A%2F%2Fgithub.com%2FNhoya%2FgOSINT) [![Go Report Card](https://goreportcard.com/badge/github.com/Nhoya/gOSINT)](https://goreportcard.com/report/github.com/Nhoya/gOSINT) OSINT framework in Go - Donate BTC: 346dDn4BjTodMkGkb5iTyoJ5Z2wi9g9QiM you probably want to take a look at the develop branch for more updates. diff --git a/git.go b/git.go index 930c831..2a07238 100644 --- a/git.go +++ b/git.go @@ -72,7 +72,7 @@ func gitSearch(target string, WebsiteAPI string, mailSet mapset.Set) mapset.Set findMailInText(commits, mailSet) } - //Check if the mailset has been populated (this avoids problems with mispelled repositories too) + //Check if the mailset has been populated (this avoids problems with misspelled repositories too) if mailSet == nil { fmt.Println("[-] Nothing Found") os.Exit(1) diff --git a/sourceSearch.go b/sourceSearch.go index 5689a7b..56e1be2 100644 --- a/sourceSearch.go +++ b/sourceSearch.go @@ -72,6 +72,11 @@ func cloneRepo(repo string) string { URL: repo, Progress: os.Stdout, }) + + if err != nil { + fmt.Println("[-] Unable to clone clone the repo") + os.Exit(1) + } return tmpdir } From c88f8341bb3a188d8b42706a463ce2930f4b9097 Mon Sep 17 00:00:00 2001 From: Nhoya Date: Thu, 18 Jan 2018 18:13:12 +0100 Subject: [PATCH 21/26] More clear code --- README.md | 2 +- build.sh | 4 ++-- gOSINT.go | 6 +++--- git.go | 8 ++++---- sourceSearch.go | 4 ++-- telegram.go | 3 +-- utils.go | 8 ++++---- 7 files changed, 17 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index ad3ae6c..c32aaba 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# gOSINT [![Build Status](https://travis-ci.org/Nhoya/gOSINT.svg?branch=master)](https://travis-ci.org/Nhoya/gOSINT) [![GitHub stars](https://img.shields.io/github/stars/Nhoya/gOSINT.svg)](https://github.com/Nhoya/gOSINT/stargazers) [![GitHub forks](https://img.shields.io/github/forks/Nhoya/gOSINT.svg)](https://github.com/Nhoya/gOSINT/network) [![Twitter](https://img.shields.io/twitter/url/https/github.com/Nhoya/gOSINT.svg?style=social&style=plastic)](https://twitter.com/intent/tweet?text=Wow:&url=https%3A%2F%2Fgithub.com%2FNhoya%2FgOSINT) [![Go Report Card](https://goreportcard.com/badge/github.com/Nhoya/gOSINT)](https://goreportcard.com/report/github.com/Nhoya/gOSINT) +# gOSINT [![Build Status](https://travis-ci.org/Nhoya/gOSINT.svg?branch=master)](https://travis-ci.org/Nhoya/gOSINT) [![GitHub stars](https://img.shields.io/github/stars/Nhoya/gOSINT.svg)](https://github.com/Nhoya/gOSINT/stargazers) [![GitHub forks](https://img.shields.io/github/forks/Nhoya/gOSINT.svg)](https://github.com/Nhoya/gOSINT/network) [![Twitter](https://img.shields.io/twitter/url/https/github.com/Nhoya/gOSINT.svg?style=social&style=plastic)](https://twitter.com/intent/tweet?text=Wow:&url=https%3A%2F%2Fgithub.com%2FNhoya%2FgOSINT) [![Go Report Card](https://goreportcard.com/badge/github.com/Nhoya/gOSINT)](https://goreportcard.com/report/github.com/Nhoya/gOSINT) [![Codacy Badge](https://api.codacy.com/project/badge/Grade/76673062a30e48bd99d499d32c0c6af0)](https://www.codacy.com/app/Nhoya/gOSINT?utm_source=github.com&utm_medium=referral&utm_content=Nhoya/gOSINT&utm_campaign=Badge_Grade) OSINT framework in Go - Donate BTC: 346dDn4BjTodMkGkb5iTyoJ5Z2wi9g9QiM you probably want to take a look at the develop branch for more updates. diff --git a/build.sh b/build.sh index 9ed9a15..d9c00cd 100755 --- a/build.sh +++ b/build.sh @@ -16,7 +16,7 @@ if [[ "$version" =~ $go_version_regex ]]; then if [[ ${BASH_REMATCH[1]} -le 1 ]]; then if [[ ${BASH_REMATCH[2]} -lt 8 ]]; then echo "This version of go is not supported, you need go >= 1.8" - echo "Current: "$version"" + echo "Current: $version" exit 1 fi fi @@ -40,4 +40,4 @@ done echo -e "${GREEN}[+] Building gOSINT${END}" go build echo -e "${GREEN}[+] Installing gOSINT${END}" -sudo mv gOSINT /usr/local/bin +'sudo mv gOSINT /usr/local/bin diff --git a/gOSINT.go b/gOSINT.go index eddb261..c6fdc25 100644 --- a/gOSINT.go +++ b/gOSINT.go @@ -14,7 +14,7 @@ var opts struct { Module string `short:"m" long:"module" description:"Specify module" choice:"pgp" choice:"pwnd" choice:"git" choice:"plainSearch" choice:"telegram"` Version bool `short:"v" long:"version" description:"Print version"` // git module - Url string `long:"url" default:"" description:"Specify target URL"` + URL string `long:"url" default:"" description:"Specify target URL"` GitAPIType string `long:"gitAPI" default:"" description:"Specify git website API to use (for git module,optional)" choice:"github" choice:"bitbucket"` Clone bool `short:"c" long:"clone" description:"Enable clone function for plainSearch module (need to specify repo URL)"` // pwn and pgp module @@ -52,8 +52,8 @@ func main() { fmt.Println("gOSINT " + ver) os.Exit(0) } - if opts.Url != "" { - isUrl(opts.Url) + if opts.URL != "" { + isURL(opts.URL) } switch mod := opts.Module; mod { diff --git a/git.go b/git.go index 2a07238..76367f3 100644 --- a/git.go +++ b/git.go @@ -15,11 +15,11 @@ import ( ) func initGit(mailSet mapset.Set) { - if opts.Url == "" { + if opts.URL == "" { fmt.Println("You must specify target URL") os.Exit(1) } - mailSet = gitSearch(opts.Url, opts.GitAPIType, mailSet) + mailSet = gitSearch(opts.URL, opts.GitAPIType, mailSet) if opts.Mode { mailSet = pgpSearch(mailSet) pwnd(mailSet) @@ -97,10 +97,10 @@ func retrieveLastGHPage(domain string) int { return 1 } -func cloneAndSearchCommit(Url string) string { +func cloneAndSearchCommit(URL string) string { fmt.Println("[+] Cloning Repo") r, _ := git.Clone(memory.NewStorage(), nil, &git.CloneOptions{ - URL: Url, + URL: URL, }) ref, _ := r.Head() cIter, _ := r.Log(&git.LogOptions{From: ref.Hash()}) diff --git a/sourceSearch.go b/sourceSearch.go index 56e1be2..db31f4e 100644 --- a/sourceSearch.go +++ b/sourceSearch.go @@ -12,11 +12,11 @@ import ( func initPlainSearch(mailSet mapset.Set) { if opts.Clone { - if opts.Url == "" { + if opts.URL == "" { fmt.Println("You must specify target URL") os.Exit(1) } - mailSet = cloneAndSearch(opts.Url, mailSet, opts.Confirm) + mailSet = cloneAndSearch(opts.URL, mailSet, opts.Confirm) } else { if opts.Path == "" { fmt.Println("You must specify Path") diff --git a/telegram.go b/telegram.go index d37accc..0aaadf1 100644 --- a/telegram.go +++ b/telegram.go @@ -173,9 +173,8 @@ func getTelegramUsername(body string) (string, string) { match := re.FindStringSubmatch(body) if len(match) == 3 { return match[1], match[2] - } else { - return "", "" } + return "", "" } func getTelegramMessageDateTime(body string) (string, string) { diff --git a/utils.go b/utils.go index 670fd19..e93ea78 100644 --- a/utils.go +++ b/utils.go @@ -43,10 +43,10 @@ func readFromSet(mailSet mapset.Set) { } } -func isUrl(url string) { - validUrl, _ := regexp.MatchString(`(?i)\b((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s!()\[\]{};:'".,<>?«»“”‘’]))`, url) - if !validUrl { - fmt.Println("[-] " + url + " is not a valid URL") +func isURL(URL string) { + validURL, _ := regexp.MatchString(`(?i)\b((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s!()\[\]{};:'".,<>?«»“”‘’]))`, URL) + if !validURL { + fmt.Println("[-] " + URL + " is not a valid URL") os.Exit(1) } } From 1a01abd3e548eeebdbed721d65e9b84d199d93c1 Mon Sep 17 00:00:00 2001 From: Nhoya Date: Thu, 18 Jan 2018 21:18:07 +0100 Subject: [PATCH 22/26] minor fixes --- telegram.go | 11 ++++++++--- utils.go | 1 - 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/telegram.go b/telegram.go index 0aaadf1..551c804 100644 --- a/telegram.go +++ b/telegram.go @@ -39,7 +39,9 @@ func getTelegramGroupHistory(group string, grace int, dumpFlag bool, startMessag dmCounter := 0 //set messageCounter as startMessage, is -e is not used the default value of startMessage is 0 (Note: the first message on group is id:1) messageCounter := startMessage - readFromTelegramDump(dumpfile, dumpFlag, &messageCounter) + readFromTelegramDump(&startMessage, dumpfile, dumpFlag, &messageCounter) + //add a counter to remember the first message + firstMessageCounter := messageCounter //this is needed because if a file is availabe it will start from the next to the last found messageCounter++ //if -e or - s is set but on the dumpfile the message is already scraped @@ -73,7 +75,9 @@ func getTelegramGroupHistory(group string, grace int, dumpFlag bool, startMessag break } else if endMessage == 0 && dmCounter == grace { dmCounter++ + fmt.Println(messageCounter) messageCounter = messageCounter - dmCounter + fmt.Println(messageCounter) break } else if endMessage == 0 { //if -e is not set and is not the last message increase the counter @@ -90,7 +94,7 @@ func getTelegramGroupHistory(group string, grace int, dumpFlag bool, startMessag messageCounter++ time.Sleep(time.Millisecond * 100) } - fmt.Println("==== [" + time.Now().Format(time.RFC3339) + " (elapsed:" + time.Since(startTime).String() + ")] End of history - " + strconv.Itoa(messageCounter-startMessage) + " messages scraped ==== ") + fmt.Println("==== [" + time.Now().Format(time.RFC3339) + " (elapsed:" + time.Since(startTime).String() + ")] End of history | " + strconv.Itoa(messageCounter-startMessage-firstMessageCounter) + " messages scraped ==== ") if endMessage == 0 && messageCounter > 0 { fmt.Println("[=] If you think there are more messages try to increase the grace period (--grace [INT])") } @@ -215,10 +219,11 @@ func createMessage(body string, message string) string { return msg } -func readFromTelegramDump(dumpfile string, dumpFlag bool, messageCounter *int) { +func readFromTelegramDump(startMessage *int, dumpfile string, dumpFlag bool, messageCounter *int) { if dumpFlag { createDirectory("tgdumps") fmt.Println("[=] --dumpfile used, ignoring --startpoint") + *startMessage = 0 if fileExists(dumpfile) { fmt.Println("[=] The dump will be saved in " + dumpfile) resp := simpleQuestion("Print the existing dump before resuming it?") diff --git a/utils.go b/utils.go index e93ea78..0303566 100644 --- a/utils.go +++ b/utils.go @@ -70,7 +70,6 @@ func fileExists(file string) bool { return false } func createDirectory(dirname string) { - fmt.Println("test") if !fileExists("tgdumps") { fmt.Println("[+] Creating directory " + dirname) os.Mkdir(dirname, os.ModePerm) From 7601ea2261d9dab879312f4b38da615451a52b9b Mon Sep 17 00:00:00 2001 From: Nhoya Date: Fri, 19 Jan 2018 16:07:23 +0100 Subject: [PATCH 23/26] Fixed counter for scraped messages --- README.md | 11 +++++++---- telegram.go | 4 +--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index c32aaba..f39f446 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,16 @@ # gOSINT [![Build Status](https://travis-ci.org/Nhoya/gOSINT.svg?branch=master)](https://travis-ci.org/Nhoya/gOSINT) [![GitHub stars](https://img.shields.io/github/stars/Nhoya/gOSINT.svg)](https://github.com/Nhoya/gOSINT/stargazers) [![GitHub forks](https://img.shields.io/github/forks/Nhoya/gOSINT.svg)](https://github.com/Nhoya/gOSINT/network) [![Twitter](https://img.shields.io/twitter/url/https/github.com/Nhoya/gOSINT.svg?style=social&style=plastic)](https://twitter.com/intent/tweet?text=Wow:&url=https%3A%2F%2Fgithub.com%2FNhoya%2FgOSINT) [![Go Report Card](https://goreportcard.com/badge/github.com/Nhoya/gOSINT)](https://goreportcard.com/report/github.com/Nhoya/gOSINT) [![Codacy Badge](https://api.codacy.com/project/badge/Grade/76673062a30e48bd99d499d32c0c6af0)](https://www.codacy.com/app/Nhoya/gOSINT?utm_source=github.com&utm_medium=referral&utm_content=Nhoya/gOSINT&utm_campaign=Badge_Grade) -OSINT framework in Go - Donate BTC: 346dDn4BjTodMkGkb5iTyoJ5Z2wi9g9QiM +OSINT framework in Go -you probably want to take a look at the develop branch for more updates. +Take a look at the [develop branch](https://github.com/Nhoya/gOSINT/tree/develop) for more updates. ## Introduction +gOSINT is a small OSINT framework in Golang. If you want, feel free to contribute and/or leave a feedback! -gOSINT is a small OSINT framework in Golang, if you want, feel free to contribute! +## Like my project? Consider donation +[![Paypal Badge](https://img.shields.io/badge/Donate-PayPal-yellow.svg)](https://www.paypal.me/Nhoya) +Donate BTC: 346dDn4BjTodMkGkb5iTyoJ5Z2wi9g9QiM [![BTC Badge](https://img.shields.io/badge/Donate-BTC-yellow.svg)](https://pastebin.com/raw/nyDDPwaM) ## What gOSINT can do @@ -53,7 +56,7 @@ go get "github.com/jaytaylor/html2text" Currently `gOSINT` is still an early version and few modules are supported -- [x] git support for mail retriving (using github API, bitbucket API or RAW clone and search) *Now with Pagination*! +- [x] git support for mail retriving (using github API, bitbucket API or RAW clone and search) - [x] Search for mails in PGP Server - [x] [https://haveibeenpwned.com/](http://haveibeenpwned.com/) search for mail in databreach - [x] Retrive Telegram Public Group Messages diff --git a/telegram.go b/telegram.go index 551c804..5209246 100644 --- a/telegram.go +++ b/telegram.go @@ -41,7 +41,7 @@ func getTelegramGroupHistory(group string, grace int, dumpFlag bool, startMessag messageCounter := startMessage readFromTelegramDump(&startMessage, dumpfile, dumpFlag, &messageCounter) //add a counter to remember the first message - firstMessageCounter := messageCounter + firstMessageCounter := messageCounter - startMessage //this is needed because if a file is availabe it will start from the next to the last found messageCounter++ //if -e or - s is set but on the dumpfile the message is already scraped @@ -75,9 +75,7 @@ func getTelegramGroupHistory(group string, grace int, dumpFlag bool, startMessag break } else if endMessage == 0 && dmCounter == grace { dmCounter++ - fmt.Println(messageCounter) messageCounter = messageCounter - dmCounter - fmt.Println(messageCounter) break } else if endMessage == 0 { //if -e is not set and is not the last message increase the counter From e01de7c30597744f878312b26fe2e69885ecb3d1 Mon Sep 17 00:00:00 2001 From: Nhoya Date: Fri, 19 Jan 2018 16:09:35 +0100 Subject: [PATCH 24/26] Fixed donations --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index f39f446..f49f86f 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,7 @@ gOSINT is a small OSINT framework in Golang. If you want, feel free to contribut ## Like my project? Consider donation -[![Paypal Badge](https://img.shields.io/badge/Donate-PayPal-yellow.svg)](https://www.paypal.me/Nhoya) -Donate BTC: 346dDn4BjTodMkGkb5iTyoJ5Z2wi9g9QiM [![BTC Badge](https://img.shields.io/badge/Donate-BTC-yellow.svg)](https://pastebin.com/raw/nyDDPwaM) +[![Paypal Badge](https://img.shields.io/badge/Donate-PayPal-yellow.svg)](https://www.paypal.me/Nhoya) [![BTC Badge](https://img.shields.io/badge/Donate-BTC-yellow.svg)](https://pastebin.com/raw/nyDDPwaM) ## What gOSINT can do From 8d1b4cfbbaadf629a564b9607991e33ce30189e2 Mon Sep 17 00:00:00 2001 From: Nhoya Date: Fri, 19 Jan 2018 18:14:21 +0100 Subject: [PATCH 25/26] More demos --- README.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f49f86f..9b293ae 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,9 @@ Take a look at the [develop branch](https://github.com/Nhoya/gOSINT/tree/develop gOSINT is a small OSINT framework in Golang. If you want, feel free to contribute and/or leave a feedback! -## Like my project? Consider donation +## Like my project? Consider donation :) -[![Paypal Badge](https://img.shields.io/badge/Donate-PayPal-yellow.svg)](https://www.paypal.me/Nhoya) [![BTC Badge](https://img.shields.io/badge/Donate-BTC-yellow.svg)](https://pastebin.com/raw/nyDDPwaM) +[![Paypal Badge](https://img.shields.io/badge/Donate-PayPal-yellow.svg)](https://www.paypal.me/Nhoya) [![BTC Badge](https://img.shields.io/badge/Donate-BTC-yellow.svg)](https://pastebin.com/raw/nyDDPwaM) [![Monero Badge](https://img.shields.io/badge/Donate-XMR-yellow.svg)](https://pastebin.com/raw/dNUFqwuC) ## What gOSINT can do @@ -151,5 +151,12 @@ the output will be stored in a file, if the file is already populated it will re `gOSINT -m telegram --tgroup | -g [PublicGroupName] --dumpfile -s [masageID] -e [messageID]` Set start and end messages for scraping + +## PGP module Demo +[![asciicast](https://asciinema.org/a/21PCpbgFqyHiTbPINexHKEywj.png)](https://asciinema.org/a/21PCpbgFqyHiTbPINexHKEywj) + +## Pwnd module Demo +[![asciicast](https://asciinema.org/a/x9Ap0IRcNNcLfriVujkNUhFSF.png)](https://asciinema.org/a/x9Ap0IRcNNcLfriVujkNUhFSF) + ## Telegram Crawler Demo -[![asciicast](https://asciinema.org/a/h1DVStNEwlPiqX7pLlNXuRanC.png)](https://asciinema.org/a/h1DVStNEwlPiqX7pLlNXuRanC) +[![asciicast](https://asciinema.org/a/nbRO9FNpjiYXAKeI87xn29j9z.png)](https://asciinema.org/a/nbRO9FNpjiYXAKeI87xn29j9z) From 32b6aab34441a898c4b153c4aab2f6c22ba2fea7 Mon Sep 17 00:00:00 2001 From: Nhoya Date: Fri, 19 Jan 2018 18:30:41 +0100 Subject: [PATCH 26/26] Fixed typo --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index d9c00cd..f1e29f7 100755 --- a/build.sh +++ b/build.sh @@ -40,4 +40,4 @@ done echo -e "${GREEN}[+] Building gOSINT${END}" go build echo -e "${GREEN}[+] Installing gOSINT${END}" -'sudo mv gOSINT /usr/local/bin +sudo mv gOSINT /usr/local/bin