forked from 0ceanSlim/crawlr
-
Notifications
You must be signed in to change notification settings - Fork 0
/
main.go
47 lines (37 loc) · 982 Bytes
/
main.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
package main
import (
"fmt"
"os"
"os/signal"
"syscall"
"time"
)
func main() {
exitSignal := make(chan os.Signal, 1)
signal.Notify(exitSignal, os.Interrupt, syscall.SIGTERM)
go func() {
initialRelay := "wss://nos.lol"
concurrency := 50 // Adjust this value based on your needs and system capabilities
for {
err := ReqKind10002(initialRelay)
if err != nil {
fmt.Printf("Initial crawl failed: %v\n", err)
}
crawlClearOnlineRelays(concurrency)
mu.Lock()
fmt.Printf("Discovered relays: %d\n", len(clearOnline))
remainingRelays := len(clearOnline) - len(crawledRelays)
mu.Unlock()
if remainingRelays == 0 {
fmt.Println("No more relays to crawl. Waiting for new relays...")
time.Sleep(30 * time.Second) // Wait before retrying
continue
}
time.Sleep(2 * time.Second)
}
}()
// Wait for an exit signal (Ctrl+C or kill)
<-exitSignal
fmt.Println("\nReceived exit signal, writing logs and exiting...")
finalize()
}