Performance improvement : pman ls command is now 4x faster for most…

… use cases (#38)

* add a friendlier error message when a README.md file does not exist

* update README

* fix tests for update implementation of ReadREADME

* record last edit time whenever db changes are made

* perf: make pman ls ~4x faster

this is done by only checking the lastEdited value once a day and then caching those values in the db

* implement ci suggestions

README.md

@@ -43,7 +43,7 @@ Use "pman [command] --help" for more information about a command.
 ## How does it work
 
 ### init
-when you run `pman init .` in any directory, it will look for subdirectories that contain a README.md or a .git folder and consider it as a project directory.
+When you run `pman init .` in any directory, it will look for subdirectories that contain a README.md or a .git folder and consider it as a project directory.
 ![image](https://github.com/theredditbandit/pman/assets/85390033/b9d6fcd3-41ca-4bd2-aa32-9b3e9bff1be8)
 
 ### set, status, info and filter

cmd/alias.go

@@ -3,6 +3,8 @@ package cmd
 import (
 	"errors"
 	"fmt"
+	"log"
+	"time"
 
 	"github.com/spf13/cobra"
 
@@ -42,7 +44,13 @@ avlpn or something smaller and use that to query pman`,
 		if err != nil {
 			return err
 		}
-
+		lastEdit := make(map[string]string)
+		lastEdit["lastWrite"] = fmt.Sprint(time.Now().Format("02 Jan 06 15:04"))
+		err = db.WriteToDB(db.DBName, lastEdit, ConfigBucket)
+		if err != nil {
+			log.Print(err)
+			return err
+		}
 		return nil
 	},
 }

cmd/delete.go

@@ -3,6 +3,8 @@ package cmd
 import (
 	"errors"
 	"fmt"
+	"log"
+	"time"
 
 	"github.com/spf13/cobra"
 
@@ -52,6 +54,13 @@ var delCmd = &cobra.Command{
 		if err != nil {
 			return err
 		}
+		lastEdit := make(map[string]string)
+		lastEdit["lastWrite"] = fmt.Sprint(time.Now().Format("02 Jan 06 15:04"))
+		err = db.WriteToDB(db.DBName, lastEdit, ConfigBucket)
+		if err != nil {
+			log.Print(err)
+			return err
+		}
 		fmt.Printf("Successfully deleted %s from the db \n", projName)
 		return nil
 	},

cmd/ls.go

@@ -20,6 +20,7 @@ var lsCmd = &cobra.Command{
 	RunE: func(cmd *cobra.Command, _ []string) error {
 		filterFlag, _ := cmd.Flags().GetString("f")
 		oldUI, _ := cmd.Flags().GetBool("c")
+		refreshLastEditTime, _ := cmd.Flags().GetBool("r")
 		data, err := db.GetAllRecords(db.DBName, StatusBucket)
 		if err != nil {
 			return err
@@ -29,14 +30,15 @@ var lsCmd = &cobra.Command{
 			data = utils.FilterByStatuses(data, strings.Split(filterFlag, ","))
 		}
 		if oldUI {
-			return ui.RenderTable(data)
+			return ui.RenderTable(data, refreshLastEditTime)
 		}
-		return ui.RenderInteractiveTable(data)
+		return ui.RenderInteractiveTable(data, refreshLastEditTime)
 	},
 }
 
 func init() {
 	rootCmd.AddCommand(lsCmd)
 	lsCmd.Flags().String("f", "", "Filter projects by status. Usage : pman ls --f <status1[,status2]>")
 	lsCmd.Flags().Bool("c", false, "list projects using the colorful table. Usage : pman ls --c")
+	lsCmd.Flags().Bool("r", false, "Refresh Last Edited time: pman ls --r")
 }

cmd/root.go

@@ -10,6 +10,7 @@ const (
 	StatusBucket       = "projects"
 	ProjectPathBucket  = "projectPaths"
 	ProjectAliasBucket = "projectAliases"
+	ConfigBucket       = "config"
 	version            = "1.0"
 )
 

cmd/set.go

@@ -3,6 +3,8 @@ package cmd
 import (
 	"errors"
 	"fmt"
+	"log"
+	"time"
 
 	"github.com/spf13/cobra"
 
@@ -47,6 +49,15 @@ var setCmd = &cobra.Command{
 			fmt.Println("Error updating record : ", err)
 			return err
 		}
+
+		lastEdit := make(map[string]string)
+		lastEdit["lastWrite"] = fmt.Sprint(time.Now().Format("02 Jan 06 15:04"))
+		err = db.WriteToDB(db.DBName, lastEdit, ConfigBucket)
+		if err != nil {
+			log.Print(err)
+			return err
+		}
+
 		fmt.Printf("Project %s set to status %s\n", pname, status)
 		return nil
 	},

pkg/indexer.go

@@ -2,9 +2,11 @@ package pkg
 
 import (
 	"errors"
+	"fmt"
 	"log"
 	"os"
 	"path/filepath"
+	"time"
 
 	"github.com/theredditbandit/pman/pkg/db"
 )
@@ -13,6 +15,8 @@ const (
 	StatusBucket       = "projects"
 	ProjectPaths       = "projectPaths"
 	ProjectAliasBucket = "projectAliases"
+	LastUpdatedBucket  = "lastUpdated"
+	ConfigBucket       = "config"
 )
 
 var (
@@ -60,6 +64,14 @@ func InitDirs(args []string) error {
 		log.Print(err)
 		return err
 	}
+	lastEdit := make(map[string]string)
+	lastEdit["lastWrite"] = fmt.Sprint(time.Now().Format("02 Jan 06 15:04"))
+	err = db.WriteToDB(db.DBName, lastEdit, ConfigBucket)
+	if err != nil {
+		log.Print(err)
+		return err
+	}
+
 	return nil
 }
 

pkg/ui/interactiveTable.go

@@ -57,16 +57,55 @@ func (m tableModel) View() string {
 	return baseStyle.Render(m.table.View()) + "\n"
 }
 
-func RenderInteractiveTable(data map[string]string) error {
+func RenderInteractiveTable(data map[string]string, refreshLastEditedTime bool) error {
+	var rows []table.Row
+	var lastEdited string
+
 	col := []table.Column{
 		{Title: "Status", Width: 20},
 		{Title: "Project", Width: 40},
 		{Title: "Last Edited", Width: 20},
 	}
-	var rows []table.Row
+
+	if refreshLastEditedTime {
+		err := utils.UpdateLastEditedTime()
+		if err != nil {
+			return err
+		}
+	} else {
+		rec, err := db.GetRecord(db.DBName, "lastRefreshTime", pkg.ConfigBucket)
+		if err != nil { // lastRefreshTime key does not exist in db
+			refreshLastEditedTime = true
+			err := utils.UpdateLastEditedTime()
+			if err != nil {
+				return err
+			}
+		}
+		if utils.DayPassed(rec) { // lastEdited values are more than a day old. need to refresh them
+			refreshLastEditedTime = true
+			err := utils.UpdateLastEditedTime()
+			if err != nil {
+				return err
+			}
+		}
+	}
+
 	for proj, status := range data {
 		alias, err := db.GetRecord(db.DBName, proj, pkg.ProjectAliasBucket)
-		lastEdited := utils.GetLastModifiedTime(db.DBName, proj)
+		if refreshLastEditedTime {
+			lastEdited = utils.GetLastModifiedTime(db.DBName, proj)
+			rec := map[string]string{proj: lastEdited}
+			err := db.WriteToDB(db.DBName, rec, pkg.LastUpdatedBucket)
+			if err != nil {
+				return err
+			}
+		} else {
+			lE, err := db.GetRecord(db.DBName, proj, pkg.LastUpdatedBucket)
+			if err != nil {
+				return err
+			}
+			lastEdited = lE
+		}
 		if err == nil {
 			pname := fmt.Sprintf("%s (%s)", proj, alias)
 			row := []string{utils.TitleCase(status), pname, lastEdited} // Status | projectName (alias) | lastEdited

pkg/ui/statusTable.go

@@ -14,13 +14,51 @@ import (
 )
 
 // RenderTable: renders the given data as a table
-func RenderTable(data map[string]string) error {
+func RenderTable(data map[string]string, refreshLastEditedTime bool) error {
 	var tableData [][]string
+	var lastEdited string
+
+	if refreshLastEditedTime {
+		err := utils.UpdateLastEditedTime()
+		if err != nil {
+			return err
+		}
+	} else {
+		rec, err := db.GetRecord(db.DBName, "lastRefreshTime", pkg.ConfigBucket)
+		if err != nil { // lastRefreshTime key does not exist in db
+			refreshLastEditedTime = true
+			err := utils.UpdateLastEditedTime()
+			if err != nil {
+				return err
+			}
+		}
+		if utils.DayPassed(rec) { // lastEdited values are more than a day old. need to refresh them
+			refreshLastEditedTime = true
+			err := utils.UpdateLastEditedTime()
+			if err != nil {
+				return err
+			}
+		}
+	}
+
 	for p, status := range data {
 		alias, err := db.GetRecord(db.DBName, p, pkg.ProjectAliasBucket)
-		lastEdited := utils.GetLastModifiedTime(db.DBName, p)
+		if refreshLastEditedTime {
+			lastEdited = utils.GetLastModifiedTime(db.DBName, p)
+			rec := map[string]string{p: lastEdited}
+			err := db.WriteToDB(db.DBName, rec, pkg.LastUpdatedBucket)
+			if err != nil {
+				return err
+			}
+		} else {
+			lE, err := db.GetRecord(db.DBName, p, pkg.LastUpdatedBucket)
+			if err != nil {
+				return err
+			}
+			lastEdited = lE
+		}
 		if err == nil {
-			pname := fmt.Sprintf("%s (%s)", p, alias)
+			pname := fmt.Sprintf("%s (%s) nil error", p, alias)
 			row := []string{utils.TitleCase(status), pname, lastEdited} // Status | projectName (alias) | lastEdited
 			tableData = append(tableData, row)
 		} else {

pkg/utils/utils.go

@@ -6,6 +6,7 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+	"strconv"
 	"time"
 
 	"github.com/charmbracelet/glamour"
@@ -42,9 +43,7 @@ func FilterByStatuses(data map[string]string, status []string) map[string]string
 
 func GetLastModifiedTime(dbname, pname string) string {
 	var lastModTime time.Time
-	var lastModFile string
 	today := time.Now()
-	_ = lastModFile
 	pPath, err := db.GetRecord(dbname, pname, pkg.ProjectPaths)
 	if err != nil {
 		return "Something went wrong"
@@ -55,7 +54,6 @@ func GetLastModifiedTime(dbname, pname string) string {
 		}
 		if !info.IsDir() && info.ModTime().After(lastModTime) {
 			lastModTime = info.ModTime()
-			lastModFile = info.Name()
 		}
 		return nil
 	})
@@ -101,9 +99,31 @@ func ReadREADME(dbname, projectName string) ([]byte, error) {
 		}
 	}
 	pPath := filepath.Join(path, "README.md")
+	_, err = os.Stat(pPath)
+	if os.IsNotExist(err) {
+		msg := fmt.Sprintf("# README does not exist for %s", projectName)
+		return []byte(msg), nil
+	}
 	data, err := os.ReadFile(pPath)
 	if err != nil {
 		return nil, errors.Join(ErrReadREADME, fmt.Errorf("something went wrong while reading README for %s: %w", projectName, err))
 	}
 	return data, nil
 }
+
+func UpdateLastEditedTime() error {
+	r := fmt.Sprint(time.Now().Unix())
+	rec := map[string]string{"lastRefreshTime": r}
+	err := db.WriteToDB(db.DBName, rec, pkg.ConfigBucket)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func DayPassed(t string) bool {
+	oneDay := 86400
+	now := time.Now().Unix()
+	recTime, _ := strconv.ParseInt(t, 10, 64)
+	return now-recTime > int64(oneDay)
+}

pkg/utils/utils_test.go

@@ -1,6 +1,7 @@
 package utils_test
 
 import (
+	"fmt"
 	"os"
 	"testing"
 	"time"
@@ -181,7 +182,7 @@ func Test_ReadREADME(t *testing.T) {
 	})
 
 	t.Run("Test ReadREADME with invalid README file name", func(t *testing.T) {
-		expected := []byte(nil)
+		expected := []byte(fmt.Sprintf("# README does not exist for %s", projectName))
 
 		t.Cleanup(func() {
 			err := db.DeleteDb(dbname)
@@ -201,7 +202,7 @@ func Test_ReadREADME(t *testing.T) {
 		actual, err := utils.ReadREADME(dbname, projectName)
 
 		assert.Equal(t, expected, actual)
-		require.ErrorIs(t, err, utils.ErrReadREADME)
+		require.NoError(t, err)
 	})
 }