Merge pull request #10 from nyudlts/report-csv

CSV Exports for reports; API Endpoints for reports

.gitignore

@@ -5,4 +5,5 @@ bin/*
 .vscode/*
 .idea/*
 *.log
-*.conf
+*.conf
+tmp/*

api/v0/apiReports.go

@@ -0,0 +1,107 @@
+package api
+
+import (
+	"net/http"
+	"regexp"
+	"strconv"
+
+	"github.com/gin-gonic/gin"
+	"github.com/nyudlts/bytemath"
+	"github.com/nyudlts/go-medialog/database"
+)
+
+type SummaryAndTotals struct {
+	RepositoryID int                `json:"repository_id"`
+	Repository   string             `json:"repository"`
+	TotalCount   int                `json:"total_count"`
+	TotalSize    int64              `json:"total_Size"`
+	TotalHuman   string             `json:"total_human_size"`
+	Summaries    database.Summaries `json:"summaries"`
+}
+
+func SummaryDateRange(c *gin.Context) {
+	if _, err := checkToken(c); err != nil {
+		c.JSON(http.StatusUnauthorized, err.Error())
+		return
+	}
+
+	//get the start and end dates
+	datePattern := regexp.MustCompile("^[0-9]{8}$")
+	startDate := c.Query("start_date")
+	endDate := c.Query("end_date")
+
+	//validate start and end dates
+	if !datePattern.MatchString(startDate) {
+		c.JSON(http.StatusBadRequest, "invalid start_date")
+		return
+	}
+
+	if !datePattern.MatchString(endDate) {
+		c.JSON(http.StatusBadRequest, "invalid end_date")
+		return
+	}
+
+	var isRefreshed bool
+	//parse is refreshed
+	if c.Query("is_refreshed") == "true" {
+		isRefreshed = true
+	} else {
+		isRefreshed = false
+	}
+
+	//create a date range variable
+	var dr = database.DateRange{}
+	dr.IsRefreshed = isRefreshed
+	//parse the repository id
+	var err error
+	dr.RepositoryID, err = strconv.Atoi(c.Query("repository_id"))
+	if err != nil {
+		dr.RepositoryID = 0
+	}
+
+	dr.StartYear, _ = strconv.Atoi(startDate[0:4])
+	dr.StartMonth, _ = strconv.Atoi(startDate[4:6])
+	dr.StartDay, _ = strconv.Atoi(startDate[6:8])
+	dr.EndYear, _ = strconv.Atoi(endDate[0:4])
+	dr.EndMonth, _ = strconv.Atoi(endDate[4:6])
+	dr.EndDay, _ = strconv.Atoi(endDate[6:8])
+
+	summaries, err := database.GetSummaryByDateRange(dr)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, err.Error())
+		return
+	}
+
+	var slug string
+	if dr.RepositoryID == 0 {
+		slug = "all"
+	} else {
+		repository, err := database.FindRepository(uint(dr.RepositoryID))
+		if err != nil {
+			c.JSON(http.StatusInternalServerError, err.Error())
+			return
+		}
+		slug = repository.Slug
+	}
+
+	var totalSize int64 = 0
+	var totalCount int = 0
+	for _, hit := range summaries.GetSlice() {
+		totalSize = totalSize + int64(hit.Size)
+		totalCount = totalCount + hit.Count
+	}
+
+	//convert the total size to a human readable format
+	f64 := float64(totalSize)
+	tsize := bytemath.ConvertToBytes(f64, bytemath.B)
+	humanSize := bytemath.ConvertBytesToHumanReadable(int64(tsize))
+
+	c.JSON(http.StatusOK, SummaryAndTotals{
+		RepositoryID: dr.RepositoryID,
+		Repository:   slug,
+		TotalCount:   totalCount,
+		TotalSize:    totalSize,
+		TotalHuman:   humanSize,
+		Summaries:    summaries,
+	})
+}

api/v0/API-V0.go → api/v0/apiRoot.go

@@ -43,7 +43,7 @@ type SummaryTotalsAccession struct {
 
 const UNAUTHORIZED = "Please authenticate to access this service"
 const apiVersion = "v0.1.4"
-const medialogVersion = "v1.0.7"
+const medialogVersion = "v0.1.8"
 
 var ACCESS_DENIED = map[string]string{"error": "access denied"}
 

api_test.go

@@ -12,6 +12,7 @@ import (
 	"testing"
 
 	"github.com/gin-gonic/gin"
+	"github.com/nyudlts/go-medialog/api/v0"
 	"github.com/nyudlts/go-medialog/models"
 	router "github.com/nyudlts/go-medialog/router"
 	"github.com/stretchr/testify/assert"
@@ -464,6 +465,57 @@ func TestAPI(t *testing.T) {
 		assert.Equal(t, "application/json; charset=utf-8", recorder.Header().Get("content-type"))
 	})
 
+	//report functions
+	t.Run("test get summary of range", func(t *testing.T) {
+		recorder := httptest.NewRecorder()
+		c, _ := gin.CreateTestContext(recorder)
+		url := fmt.Sprintf("%s/reports/range?start_date=%s&end_date=%s&repository_id=%d", APIROOT, "20140101", "20241031", repository.ID)
+		req, err := http.NewRequestWithContext(c, "GET", url, nil)
+		if err != nil {
+			t.Error(err)
+		}
+		req.Header.Add("X-Medialog-Token", token)
+		r.ServeHTTP(recorder, req)
+		assert.Equal(t, 200, recorder.Code)
+		assert.Equal(t, "application/json; charset=utf-8", recorder.Header().Get("content-type"))
+
+		var summary = api.SummaryAndTotals{}
+		body, _ := io.ReadAll(recorder.Body)
+
+		//ensure the summary has one entry
+		if err := json.Unmarshal(body, &summary); err != nil {
+			c.JSON(http.StatusInternalServerError, err.Error())
+		}
+
+		assert.Equal(t, 1, len(summary.Summaries))
+
+	})
+
+	t.Run("test get summary of range refreshed only", func(t *testing.T) {
+		recorder := httptest.NewRecorder()
+		c, _ := gin.CreateTestContext(recorder)
+		url := fmt.Sprintf("%s/reports/range?start_date=%s&end_date=%s&is_refreshed=true", APIROOT, "20140101", "20241031")
+		req, err := http.NewRequestWithContext(c, "GET", url, nil)
+		if err != nil {
+			t.Error(err)
+		}
+		req.Header.Add("X-Medialog-Token", token)
+		r.ServeHTTP(recorder, req)
+		assert.Equal(t, 200, recorder.Code)
+		assert.Equal(t, "application/json; charset=utf-8", recorder.Header().Get("content-type"))
+
+		var summary = api.SummaryAndTotals{}
+		body, _ := io.ReadAll(recorder.Body)
+
+		//ensure the summary has one entry
+		if err := json.Unmarshal(body, &summary); err != nil {
+			c.JSON(http.StatusInternalServerError, err.Error())
+		}
+
+		assert.Equal(t, 0, len(summary.Summaries))
+
+	})
+
 	//delete functions
 
 	t.Run("test delete an entry", func(t *testing.T) {

controllers/accessionsController.go

@@ -570,5 +570,6 @@ func AccessionGenCSV(c *gin.Context) {
 	c.Header("content-type", "text/csv")
 	c.Header("Content-Description", "File Transfer")
 	c.Header("Content-Disposition", "attachment; filename="+csvFileName)
+	c.Status(http.StatusOK)
 	c.Writer.Write([]byte(csvBuffer.String()))
 }

controllers/reportsController.go

@@ -1,10 +1,14 @@
 package controllers
 
 import (
+	"encoding/csv"
+	"fmt"
 	"net/http"
+	"strings"
 
 	"github.com/gin-gonic/gin"
 	"github.com/nyudlts/go-medialog/database"
+	"github.com/nyudlts/go-medialog/models"
 )
 
 func ReportsIndex(c *gin.Context) {
@@ -45,7 +49,7 @@ func ReportsIndex(c *gin.Context) {
 	})
 }
 
-func ReportRange(c *gin.Context) {
+func ReportsRange(c *gin.Context) {
 
 	if err := isLoggedIn(c); err != nil {
 		ThrowError(http.StatusUnauthorized, err.Error(), c, false)
@@ -100,6 +104,36 @@ func ReportRange(c *gin.Context) {
 
 }
 
+func ReportsCSV(c *gin.Context) {
+	var dateRange = database.DateRange{}
+	if err := c.Bind(&dateRange); err != nil {
+		ThrowError(http.StatusBadRequest, err.Error(), c, true)
+		return
+	}
+
+	entries, err := database.GetEntriesByDateRange(dateRange)
+	if err != nil {
+		ThrowError(http.StatusInternalServerError, err.Error(), c, true)
+		return
+	}
+
+	csvBuffer := new(strings.Builder)
+	var csvWriter = csv.NewWriter(csvBuffer)
+	csvWriter.Write(models.CSVHeader)
+	csvWriter.Flush()
+	for _, entry := range entries {
+		csvWriter.Write(entry.ToCSV())
+		csvWriter.Flush()
+	}
+
+	csvFileName := fmt.Sprintf("%s.csv", "report") // make a string formatter for dateRage struct
+	c.Header("content-type", "text/csv")
+	c.Header("Content-Description", "File Transfer")
+	c.Header("Content-Disposition", "attachment; filename="+csvFileName)
+	c.Status(http.StatusOK)
+	c.Writer.Write([]byte(csvBuffer.String()))
+}
+
 var months = []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}
 var years = []int{2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024} //make s range from 2014 to current year
 var days = []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31}

controllers/resourcesController.go

@@ -397,5 +397,6 @@ func ResourceGenCSV(c *gin.Context) {
 	c.Header("content-type", "text/csv")
 	c.Header("Content-Description", "File Transfer")
 	c.Header("Content-Disposition", "attachment; filename="+csvFileName)
+	c.Status(http.StatusOK)
 	c.Writer.Write([]byte(csvBuffer.String()))
 }

database/Database-Entries.go → database/databaseEntries.go

@@ -243,36 +243,57 @@ func GetSummaryByYear(year int) (Summaries, error) {
 	return getSummary(entries), nil
 }
 
-type DateRange struct {
-	StartYear    int `form:"start-year"`
-	StartMonth   int `form:"start-month"`
-	StartDay     int `form:"start-day"`
-	EndYear      int `form:"end-year"`
-	EndMonth     int `form:"end-month"`
-	EndDay       int `form:"end-day"`
-	RepositoryID int `form:"partner_code"`
-}
-
 func (dr DateRange) String() string {
 	return fmt.Sprintf("%d-%d-%d to %d-%d-%d", dr.StartYear, dr.StartMonth, dr.StartDay, dr.EndYear, dr.EndMonth, dr.EndDay)
 }
 
+func GetEntriesByDateRange(dr DateRange) ([]models.Entry, error) {
+	startDate := fmt.Sprintf("%d-%d-%dT00:00:00Z", dr.StartYear, dr.StartMonth, dr.StartDay)
+	endDate := fmt.Sprintf("%d-%d-%dT23:59:59Z", dr.EndYear, dr.EndMonth, dr.EndDay)
+	entries := []models.Entry{}
+	if dr.RepositoryID == 0 {
+		if err := db.Preload(clause.Associations).Where("created_at BETWEEN ? AND ?", startDate, endDate).Find(&entries).Error; err != nil {
+			return entries, err
+		}
+		return entries, nil
+	} else {
+		if err := db.Preload(clause.Associations).Where("repository_id = ?", dr.RepositoryID).Where("created_at BETWEEN ? AND ?", startDate, endDate).Find(&entries).Error; err != nil {
+			return entries, err
+		}
+		return entries, nil
+	}
+
+}
+
 func GetSummaryByDateRange(dr DateRange) (Summaries, error) {
+
 	startDate := fmt.Sprintf("%d-%d-%dT00:00:00Z", dr.StartYear, dr.StartMonth, dr.StartDay)
 	endDate := fmt.Sprintf("%d-%d-%dT23:59:59Z", dr.EndYear, dr.EndMonth, dr.EndDay)
 
 	entries := []models.Entry{}
-	if dr.RepositoryID == 0 {
-		if err := db.Where("created_at BETWEEN ? AND ?", startDate, endDate).Find(&entries).Error; err != nil {
-			return Summaries{}, err
+	//this needs to be simplified
+	if dr.IsRefreshed {
+		if dr.RepositoryID == 0 {
+			if err := db.Where("created_at BETWEEN ? AND ?", startDate, endDate).Where("is_refreshed = true").Find(&entries).Error; err != nil {
+				return Summaries{}, err
+			}
+		} else {
+			if err := db.Where("repository_id = ?", dr.RepositoryID).Where("created_at BETWEEN ? AND ?", startDate, endDate).Where("is_refreshed = true").Find(&entries).Error; err != nil {
+				return Summaries{}, err
+			}
 		}
-		return getSummary(entries), nil
 	} else {
-		if err := db.Where("repository_id = ?", dr.RepositoryID).Where("created_at BETWEEN ? AND ?", startDate, endDate).Find(&entries).Error; err != nil {
-			return Summaries{}, err
+		if dr.RepositoryID == 0 {
+			if err := db.Where("created_at BETWEEN ? AND ?", startDate, endDate).Find(&entries).Error; err != nil {
+				return Summaries{}, err
+			}
+		} else {
+			if err := db.Where("repository_id = ?", dr.RepositoryID).Where("created_at BETWEEN ? AND ?", startDate, endDate).Find(&entries).Error; err != nil {
+				return Summaries{}, err
+			}
 		}
-		return getSummary(entries), nil
 	}
+	return getSummary(entries), nil
 }
 
 func summaryContains(summaries Summaries, mediatype string) bool {

database/database.go → database/databaseRoot.go

@@ -16,6 +16,17 @@ type Pagination struct {
 	Sort   string `json:"sort"`
 }
 
+type DateRange struct {
+	StartYear    int `form:"start-year"`
+	StartMonth   int `form:"start-month"`
+	StartDay     int `form:"start-day"`
+	EndYear      int `form:"end-year"`
+	EndMonth     int `form:"end-month"`
+	EndDay       int `form:"end-day"`
+	RepositoryID int `form:"repository-id"`
+	IsRefreshed  bool
+}
+
 func ConnectMySQL(dbconfig models.DatabaseConfig, gormDebug bool) error {
 	dsn := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=utf8mb4&parseTime=True&loc=Local", dbconfig.Username, dbconfig.Password, dbconfig.URL, dbconfig.Port, dbconfig.DatabaseName)
 	var err error

go.mod

@@ -1,6 +1,7 @@
 module github.com/nyudlts/go-medialog
 
-go 1.22.3
+go 1.23.1
+
 
 require (
 	github.com/gin-contrib/sessions v1.0.0

medialog.go

@@ -25,7 +25,7 @@ var (
 	createAdmin   bool
 )
 
-const version = "v1.0.7"
+const version = "v1.0.8"
 
 func init() {
 	flag.StringVar(&environment, "environment", "", "")

models/models.go

@@ -2,7 +2,7 @@ package models
 
 import (
 	"fmt"
-	"regexp"
+	"strings"
 	"time"
 
 	"github.com/google/uuid"
@@ -86,8 +86,8 @@ type Entry struct {
 var CSVHeader = []string{"id", "media_id", "mediatype", "content_type", "label_text", "is_refreshed", "imaging_success", "repository", "resource", "accession", "storage_location"}
 
 func (e Entry) ToCSV() []string {
-	re := regexp.MustCompile(`\r?\n`)
-	labelText := re.ReplaceAllString(e.LabelText, " ")
+
+	labelText := strings.ReplaceAll(e.LabelText, "\n", " ")
 
 	csv := []string{
 		e.ID.String(),