diff --git a/.gitignore b/.gitignore index 071a894..07ac242 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ bin/* .vscode/* .idea/* *.log -*.conf \ No newline at end of file +*.conf +tmp/* \ No newline at end of file diff --git a/api/v0/API-Accessions.go b/api/v0/apiAccessions.go similarity index 100% rename from api/v0/API-Accessions.go rename to api/v0/apiAccessions.go diff --git a/api/v0/API-Entries.go b/api/v0/apiEntries.go similarity index 100% rename from api/v0/API-Entries.go rename to api/v0/apiEntries.go diff --git a/api/v0/apiReports.go b/api/v0/apiReports.go new file mode 100644 index 0000000..ab10612 --- /dev/null +++ b/api/v0/apiReports.go @@ -0,0 +1,107 @@ +package api + +import ( + "net/http" + "regexp" + "strconv" + + "github.com/gin-gonic/gin" + "github.com/nyudlts/bytemath" + "github.com/nyudlts/go-medialog/database" +) + +type SummaryAndTotals struct { + RepositoryID int `json:"repository_id"` + Repository string `json:"repository"` + TotalCount int `json:"total_count"` + TotalSize int64 `json:"total_Size"` + TotalHuman string `json:"total_human_size"` + Summaries database.Summaries `json:"summaries"` +} + +func SummaryDateRange(c *gin.Context) { + if _, err := checkToken(c); err != nil { + c.JSON(http.StatusUnauthorized, err.Error()) + return + } + + //get the start and end dates + datePattern := regexp.MustCompile("^[0-9]{8}$") + startDate := c.Query("start_date") + endDate := c.Query("end_date") + + //validate start and end dates + if !datePattern.MatchString(startDate) { + c.JSON(http.StatusBadRequest, "invalid start_date") + return + } + + if !datePattern.MatchString(endDate) { + c.JSON(http.StatusBadRequest, "invalid end_date") + return + } + + var isRefreshed bool + //parse is refreshed + if c.Query("is_refreshed") == "true" { + isRefreshed = true + } else { + isRefreshed = false + } + + //create a date range variable + var dr = database.DateRange{} + dr.IsRefreshed = isRefreshed + //parse the repository id + var err error + dr.RepositoryID, err = strconv.Atoi(c.Query("repository_id")) + if err != nil { + dr.RepositoryID = 0 + } + + dr.StartYear, _ = strconv.Atoi(startDate[0:4]) + dr.StartMonth, _ = strconv.Atoi(startDate[4:6]) + dr.StartDay, _ = strconv.Atoi(startDate[6:8]) + dr.EndYear, _ = strconv.Atoi(endDate[0:4]) + dr.EndMonth, _ = strconv.Atoi(endDate[4:6]) + dr.EndDay, _ = strconv.Atoi(endDate[6:8]) + + summaries, err := database.GetSummaryByDateRange(dr) + if err != nil { + c.JSON(http.StatusInternalServerError, err.Error()) + return + } + + var slug string + if dr.RepositoryID == 0 { + slug = "all" + } else { + repository, err := database.FindRepository(uint(dr.RepositoryID)) + if err != nil { + c.JSON(http.StatusInternalServerError, err.Error()) + return + } + slug = repository.Slug + } + + var totalSize int64 = 0 + var totalCount int = 0 + for _, hit := range summaries.GetSlice() { + totalSize = totalSize + int64(hit.Size) + totalCount = totalCount + hit.Count + } + + //convert the total size to a human readable format + f64 := float64(totalSize) + tsize := bytemath.ConvertToBytes(f64, bytemath.B) + humanSize := bytemath.ConvertBytesToHumanReadable(int64(tsize)) + + c.JSON(http.StatusOK, SummaryAndTotals{ + RepositoryID: dr.RepositoryID, + Repository: slug, + TotalCount: totalCount, + TotalSize: totalSize, + TotalHuman: humanSize, + Summaries: summaries, + }) +} diff --git a/api/v0/API-Repositories.go b/api/v0/apiRepositories.go similarity index 100% rename from api/v0/API-Repositories.go rename to api/v0/apiRepositories.go diff --git a/api/v0/API-Resources.go b/api/v0/apiResources.go similarity index 100% rename from api/v0/API-Resources.go rename to api/v0/apiResources.go diff --git a/api/v0/API-V0.go b/api/v0/apiRoot.go similarity index 99% rename from api/v0/API-V0.go rename to api/v0/apiRoot.go index 6061b63..a2e002d 100644 --- a/api/v0/API-V0.go +++ b/api/v0/apiRoot.go @@ -43,7 +43,7 @@ type SummaryTotalsAccession struct { const UNAUTHORIZED = "Please authenticate to access this service" const apiVersion = "v0.1.4" -const medialogVersion = "v1.0.7" +const medialogVersion = "v0.1.8" var ACCESS_DENIED = map[string]string{"error": "access denied"} diff --git a/api_test.go b/api_test.go index dd98691..e42c1c4 100644 --- a/api_test.go +++ b/api_test.go @@ -12,6 +12,7 @@ import ( "testing" "github.com/gin-gonic/gin" + "github.com/nyudlts/go-medialog/api/v0" "github.com/nyudlts/go-medialog/models" router "github.com/nyudlts/go-medialog/router" "github.com/stretchr/testify/assert" @@ -464,6 +465,57 @@ func TestAPI(t *testing.T) { assert.Equal(t, "application/json; charset=utf-8", recorder.Header().Get("content-type")) }) + //report functions + t.Run("test get summary of range", func(t *testing.T) { + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + url := fmt.Sprintf("%s/reports/range?start_date=%s&end_date=%s&repository_id=%d", APIROOT, "20140101", "20241031", repository.ID) + req, err := http.NewRequestWithContext(c, "GET", url, nil) + if err != nil { + t.Error(err) + } + req.Header.Add("X-Medialog-Token", token) + r.ServeHTTP(recorder, req) + assert.Equal(t, 200, recorder.Code) + assert.Equal(t, "application/json; charset=utf-8", recorder.Header().Get("content-type")) + + var summary = api.SummaryAndTotals{} + body, _ := io.ReadAll(recorder.Body) + + //ensure the summary has one entry + if err := json.Unmarshal(body, &summary); err != nil { + c.JSON(http.StatusInternalServerError, err.Error()) + } + + assert.Equal(t, 1, len(summary.Summaries)) + + }) + + t.Run("test get summary of range refreshed only", func(t *testing.T) { + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + url := fmt.Sprintf("%s/reports/range?start_date=%s&end_date=%s&is_refreshed=true", APIROOT, "20140101", "20241031") + req, err := http.NewRequestWithContext(c, "GET", url, nil) + if err != nil { + t.Error(err) + } + req.Header.Add("X-Medialog-Token", token) + r.ServeHTTP(recorder, req) + assert.Equal(t, 200, recorder.Code) + assert.Equal(t, "application/json; charset=utf-8", recorder.Header().Get("content-type")) + + var summary = api.SummaryAndTotals{} + body, _ := io.ReadAll(recorder.Body) + + //ensure the summary has one entry + if err := json.Unmarshal(body, &summary); err != nil { + c.JSON(http.StatusInternalServerError, err.Error()) + } + + assert.Equal(t, 0, len(summary.Summaries)) + + }) + //delete functions t.Run("test delete an entry", func(t *testing.T) { diff --git a/controllers/accessionsController.go b/controllers/accessionsController.go index f16d305..e3f314f 100644 --- a/controllers/accessionsController.go +++ b/controllers/accessionsController.go @@ -570,5 +570,6 @@ func AccessionGenCSV(c *gin.Context) { c.Header("content-type", "text/csv") c.Header("Content-Description", "File Transfer") c.Header("Content-Disposition", "attachment; filename="+csvFileName) + c.Status(http.StatusOK) c.Writer.Write([]byte(csvBuffer.String())) } diff --git a/controllers/reportsController.go b/controllers/reportsController.go index 8be66d3..92eab28 100644 --- a/controllers/reportsController.go +++ b/controllers/reportsController.go @@ -1,10 +1,14 @@ package controllers import ( + "encoding/csv" + "fmt" "net/http" + "strings" "github.com/gin-gonic/gin" "github.com/nyudlts/go-medialog/database" + "github.com/nyudlts/go-medialog/models" ) func ReportsIndex(c *gin.Context) { @@ -45,7 +49,7 @@ func ReportsIndex(c *gin.Context) { }) } -func ReportRange(c *gin.Context) { +func ReportsRange(c *gin.Context) { if err := isLoggedIn(c); err != nil { ThrowError(http.StatusUnauthorized, err.Error(), c, false) @@ -100,6 +104,36 @@ func ReportRange(c *gin.Context) { } +func ReportsCSV(c *gin.Context) { + var dateRange = database.DateRange{} + if err := c.Bind(&dateRange); err != nil { + ThrowError(http.StatusBadRequest, err.Error(), c, true) + return + } + + entries, err := database.GetEntriesByDateRange(dateRange) + if err != nil { + ThrowError(http.StatusInternalServerError, err.Error(), c, true) + return + } + + csvBuffer := new(strings.Builder) + var csvWriter = csv.NewWriter(csvBuffer) + csvWriter.Write(models.CSVHeader) + csvWriter.Flush() + for _, entry := range entries { + csvWriter.Write(entry.ToCSV()) + csvWriter.Flush() + } + + csvFileName := fmt.Sprintf("%s.csv", "report") // make a string formatter for dateRage struct + c.Header("content-type", "text/csv") + c.Header("Content-Description", "File Transfer") + c.Header("Content-Disposition", "attachment; filename="+csvFileName) + c.Status(http.StatusOK) + c.Writer.Write([]byte(csvBuffer.String())) +} + var months = []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12} var years = []int{2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024} //make s range from 2014 to current year var days = []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31} diff --git a/controllers/resourcesController.go b/controllers/resourcesController.go index 404419d..a5062dd 100644 --- a/controllers/resourcesController.go +++ b/controllers/resourcesController.go @@ -397,5 +397,6 @@ func ResourceGenCSV(c *gin.Context) { c.Header("content-type", "text/csv") c.Header("Content-Description", "File Transfer") c.Header("Content-Disposition", "attachment; filename="+csvFileName) + c.Status(http.StatusOK) c.Writer.Write([]byte(csvBuffer.String())) } diff --git a/database/Database-Accessions.go b/database/databaseAccessions.go similarity index 100% rename from database/Database-Accessions.go rename to database/databaseAccessions.go diff --git a/database/Database-Entries.go b/database/databaseEntries.go similarity index 86% rename from database/Database-Entries.go rename to database/databaseEntries.go index 138c0a9..a3ed104 100644 --- a/database/Database-Entries.go +++ b/database/databaseEntries.go @@ -243,36 +243,57 @@ func GetSummaryByYear(year int) (Summaries, error) { return getSummary(entries), nil } -type DateRange struct { - StartYear int `form:"start-year"` - StartMonth int `form:"start-month"` - StartDay int `form:"start-day"` - EndYear int `form:"end-year"` - EndMonth int `form:"end-month"` - EndDay int `form:"end-day"` - RepositoryID int `form:"partner_code"` -} - func (dr DateRange) String() string { return fmt.Sprintf("%d-%d-%d to %d-%d-%d", dr.StartYear, dr.StartMonth, dr.StartDay, dr.EndYear, dr.EndMonth, dr.EndDay) } +func GetEntriesByDateRange(dr DateRange) ([]models.Entry, error) { + startDate := fmt.Sprintf("%d-%d-%dT00:00:00Z", dr.StartYear, dr.StartMonth, dr.StartDay) + endDate := fmt.Sprintf("%d-%d-%dT23:59:59Z", dr.EndYear, dr.EndMonth, dr.EndDay) + entries := []models.Entry{} + if dr.RepositoryID == 0 { + if err := db.Preload(clause.Associations).Where("created_at BETWEEN ? AND ?", startDate, endDate).Find(&entries).Error; err != nil { + return entries, err + } + return entries, nil + } else { + if err := db.Preload(clause.Associations).Where("repository_id = ?", dr.RepositoryID).Where("created_at BETWEEN ? AND ?", startDate, endDate).Find(&entries).Error; err != nil { + return entries, err + } + return entries, nil + } + +} + func GetSummaryByDateRange(dr DateRange) (Summaries, error) { + startDate := fmt.Sprintf("%d-%d-%dT00:00:00Z", dr.StartYear, dr.StartMonth, dr.StartDay) endDate := fmt.Sprintf("%d-%d-%dT23:59:59Z", dr.EndYear, dr.EndMonth, dr.EndDay) entries := []models.Entry{} - if dr.RepositoryID == 0 { - if err := db.Where("created_at BETWEEN ? AND ?", startDate, endDate).Find(&entries).Error; err != nil { - return Summaries{}, err + //this needs to be simplified + if dr.IsRefreshed { + if dr.RepositoryID == 0 { + if err := db.Where("created_at BETWEEN ? AND ?", startDate, endDate).Where("is_refreshed = true").Find(&entries).Error; err != nil { + return Summaries{}, err + } + } else { + if err := db.Where("repository_id = ?", dr.RepositoryID).Where("created_at BETWEEN ? AND ?", startDate, endDate).Where("is_refreshed = true").Find(&entries).Error; err != nil { + return Summaries{}, err + } } - return getSummary(entries), nil } else { - if err := db.Where("repository_id = ?", dr.RepositoryID).Where("created_at BETWEEN ? AND ?", startDate, endDate).Find(&entries).Error; err != nil { - return Summaries{}, err + if dr.RepositoryID == 0 { + if err := db.Where("created_at BETWEEN ? AND ?", startDate, endDate).Find(&entries).Error; err != nil { + return Summaries{}, err + } + } else { + if err := db.Where("repository_id = ?", dr.RepositoryID).Where("created_at BETWEEN ? AND ?", startDate, endDate).Find(&entries).Error; err != nil { + return Summaries{}, err + } } - return getSummary(entries), nil } + return getSummary(entries), nil } func summaryContains(summaries Summaries, mediatype string) bool { diff --git a/database/Database-Migrations.go b/database/databaseMigrations.go similarity index 100% rename from database/Database-Migrations.go rename to database/databaseMigrations.go diff --git a/database/Database-Repositories.go b/database/databaseRepositories.go similarity index 100% rename from database/Database-Repositories.go rename to database/databaseRepositories.go diff --git a/database/Database-Resources.go b/database/databaseResources.go similarity index 100% rename from database/Database-Resources.go rename to database/databaseResources.go diff --git a/database/database.go b/database/databaseRoot.go similarity index 69% rename from database/database.go rename to database/databaseRoot.go index 2cd32e8..f1b3e9e 100644 --- a/database/database.go +++ b/database/databaseRoot.go @@ -16,6 +16,17 @@ type Pagination struct { Sort string `json:"sort"` } +type DateRange struct { + StartYear int `form:"start-year"` + StartMonth int `form:"start-month"` + StartDay int `form:"start-day"` + EndYear int `form:"end-year"` + EndMonth int `form:"end-month"` + EndDay int `form:"end-day"` + RepositoryID int `form:"repository-id"` + IsRefreshed bool +} + func ConnectMySQL(dbconfig models.DatabaseConfig, gormDebug bool) error { dsn := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=utf8mb4&parseTime=True&loc=Local", dbconfig.Username, dbconfig.Password, dbconfig.URL, dbconfig.Port, dbconfig.DatabaseName) var err error diff --git a/database/Database-Sessions.go b/database/databaseSessions.go similarity index 100% rename from database/Database-Sessions.go rename to database/databaseSessions.go diff --git a/database/Database-Tokens.go b/database/databaseTokens.go similarity index 100% rename from database/Database-Tokens.go rename to database/databaseTokens.go diff --git a/database/Database-Users.go b/database/databaseUsers.go similarity index 100% rename from database/Database-Users.go rename to database/databaseUsers.go diff --git a/go.mod b/go.mod index d12211d..19baa5a 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,7 @@ module github.com/nyudlts/go-medialog -go 1.22.3 +go 1.23.1 + require ( github.com/gin-contrib/sessions v1.0.0 diff --git a/medialog.go b/medialog.go index d927885..6e99226 100644 --- a/medialog.go +++ b/medialog.go @@ -25,7 +25,7 @@ var ( createAdmin bool ) -const version = "v1.0.7" +const version = "v1.0.8" func init() { flag.StringVar(&environment, "environment", "", "") diff --git a/models/models.go b/models/models.go index e668703..c92c36a 100644 --- a/models/models.go +++ b/models/models.go @@ -2,7 +2,7 @@ package models import ( "fmt" - "regexp" + "strings" "time" "github.com/google/uuid" @@ -86,8 +86,8 @@ type Entry struct { var CSVHeader = []string{"id", "media_id", "mediatype", "content_type", "label_text", "is_refreshed", "imaging_success", "repository", "resource", "accession", "storage_location"} func (e Entry) ToCSV() []string { - re := regexp.MustCompile(`\r?\n`) - labelText := re.ReplaceAllString(e.LabelText, " ") + + labelText := strings.ReplaceAll(e.LabelText, "\n", " ") csv := []string{ e.ID.String(), diff --git a/public/medialog.js b/public/medialog.js new file mode 100644 index 0000000..629115e --- /dev/null +++ b/public/medialog.js @@ -0,0 +1,8 @@ +function togglePassword() { + var passwd = document.getElementById("password_1"); + if (passwd.type === "password") { + passwd.type = "text"; + } else { + passwd.type = "password"; + } + } \ No newline at end of file diff --git a/router/routes.go b/router/routes.go index 81ea4cf..3eeda35 100644 --- a/router/routes.go +++ b/router/routes.go @@ -83,9 +83,10 @@ func LoadRoutes(router *gin.Engine) { userRoutes.GET(":id/revoke_api", func(c *gin.Context) { controllers.RevokeAPI(c) }) //Report Group - reportRoutes := router.Group("/reports") - reportRoutes.GET("", func(c *gin.Context) { controllers.ReportsIndex(c) }) - reportRoutes.POST("/range", func(c *gin.Context) { controllers.ReportRange(c) }) + reportsRoutes := router.Group("/reports") + reportsRoutes.GET("", func(c *gin.Context) { controllers.ReportsIndex(c) }) + reportsRoutes.POST("/range", func(c *gin.Context) { controllers.ReportsRange(c) }) + reportsRoutes.POST("/csv", func(c *gin.Context) { controllers.ReportsCSV(c) }) //Search Group searchRoutes := router.Group("/search") @@ -149,6 +150,9 @@ func LoadAPI(router *gin.Engine) { //sessions apiV0Routes.DELETE("delete_sessions", func(c *gin.Context) { api.DeleteSessionsV0(c) }) + + //reports + apiV0Routes.GET("reports/range", func(c *gin.Context) { api.SummaryDateRange(c) }) } func Test(c *gin.Context) { diff --git a/templates/global/footer.html b/templates/global/footer.html index a4fac7c..0ef353b 100644 --- a/templates/global/footer.html +++ b/templates/global/footer.html @@ -4,7 +4,7 @@
- go-medialog v1.0.7 + go-medialog v1.0.8
diff --git a/templates/reports/reports-menu.html b/templates/reports/reports-menu.html index 800b086..51ba11b 100644 --- a/templates/reports/reports-menu.html +++ b/templates/reports/reports-menu.html @@ -51,7 +51,8 @@
Reports
repository
- + {{ range $id, $slug := .partner_codes }} {{ end }} diff --git a/templates/reports/reports-range.html b/templates/reports/reports-range.html index 7d0aec7..082bd43 100644 --- a/templates/reports/reports-range.html +++ b/templates/reports/reports-range.html @@ -5,7 +5,28 @@
Range
- {{ .repository }} {{ .dateRange }} +
+
Repository: + {{if eq .repository ""}} + all + {{else}} + {{ .repository }} + {{ end }} +
+
Date range: {{ .dateRange }}
+
+
+ + + + + + + + +
+
+
{{ template "summary-table.html" . }} diff --git a/templates/reports/reports-result.html b/templates/reports/reports-result.html index 2440c55..d1e5015 100644 --- a/templates/reports/reports-result.html +++ b/templates/reports/reports-result.html @@ -76,6 +76,7 @@
Reports
repository
+
+
+
+
@@ -51,5 +55,5 @@

Login

- + {{ template "footer.html" . }} \ No newline at end of file