diff --git a/.gitignore b/.gitignore index e3b237e..e3674c9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ ezproxy-log-analyzer logs/* !logs/.keep +ezproxy.json diff --git a/README.md b/README.md index 5f4e51f..a197b57 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Get usage from your EZProxy log files Start an [ezpaarse](https://github.com/ezpaarse-project/ezpaarse) instance ``` -git clone https://github.com/ezpaarse-project/ezpaarse` +git clone https://github.com/ezpaarse-project/ezpaarse cd ezpaarse docker compose pull docker compose up -d @@ -17,12 +17,12 @@ cd .. Aggregate and process ezproxy log files ``` -scp ezproxy.host:/path/to/ezproxy/logs/ . +scp -r ezproxy.host:/path/to/ezproxy/logs . ./process.sh ``` -See usage by platform +Process a clean JSON file you can upload into an analytics tool ``` -jq -r '.[]| "\(.platform_name) \(.publisher_name)"' logs/ezpaarse.json|sort|uniq -c|sort -n +go run main.go ``` diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..fde9a40 --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module github.com/lehigh-university-libraries/ezproxy-log-analyzer + +go 1.22.5 diff --git a/main.go b/main.go new file mode 100644 index 0000000..a53a1ae --- /dev/null +++ b/main.go @@ -0,0 +1,194 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "os" + "regexp" + "strconv" + "strings" + "time" +) + +type EzpaarseLog struct { + Date string `json:"date"` + Datetime string `json:"datetime"` + Domain string `json:"domain"` + City string `json:"geoip-city"` + Coordinates string `json:"geoip-coordinates"` + Country string `json:"geoip-country"` + Latitude string `json:"geoip-latitude"` + Longitude string `json:"geoip-longitude"` + Region string `json:"geoip-region"` + Host string `json:"host"` + Identd string `json:"identd"` + LogID string `json:"log_id"` + Login string `json:"login"` + Middlewares string `json:"middlewares"` + MiddlewaresDate string `json:"middlewares_date"` + MiddlewaresVersion string `json:"middlewares_version"` + OnCampus string `json:"on_campus"` + Platform string `json:"platform"` + PlatformName string `json:"platform_name"` + PlatformsDate string `json:"platforms_date"` + PlatformsVersion string `json:"platforms_version"` + PublisherName string `json:"publisher_name"` + Robot string `json:"robot"` + Size string `json:"size"` + Status string `json:"status"` + Timestamp int `json:"timestamp"` + URL string `json:"url"` +} + +type Quicksight struct { + Date string `json:"date"` + Domain string `json:"domain"` + City string `json:"city"` + Country string `json:"country"` + Latitude float64 `json:"lat"` + Longitude float64 `json:"lng"` + Region string `json:"region"` + Platform string `json:"platform"` + PlatformName string `json:"platform_name"` + PublisherName string `json:"publisher_name"` + Department string `json:"department"` + Role string `json:"role"` +} + +// Function to check if a file exists +func fileExists(filename string) bool { + _, err := os.Stat(filename) + return !os.IsNotExist(err) +} + +// Function to save the response body to a file +func saveToFile(filename string, data io.Reader) error { + file, err := os.Create(filename) + if err != nil { + return err + } + defer file.Close() + _, err = io.Copy(file, data) + return err +} + +// Function to extract description +func extractDescription(filename string) (string, string, error) { + data, err := os.ReadFile(filename) + if err != nil { + return "", "", err + } + // Use regex to capture everything in the second