From f8ea6153585ab322db3daedff21d825937fb6273 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20W=C3=A5reus?= Date: Fri, 3 Nov 2023 12:06:51 +0100 Subject: [PATCH 1/3] add 'experience' command to combine callgraphs and blame to OSS expereience calculations Also add ability to calculate blame on a set of files --- .gitignore | 1 + internal/cmd/experience/experience.go | 52 +++++++ internal/cmd/experience/experience_test.go | 1 + internal/cmd/root/root.go | 2 + internal/experience/experience.go | 66 +++++++++ internal/file/exclusion.go | 18 +++ internal/file/exclusion_test.go | 19 +++ internal/git/blame.go | 152 +++++++++++++++++++++ internal/git/blame_test.go | 32 +++++ internal/git/git.go | 33 +++++ internal/wire/cli_container.go | 8 ++ 11 files changed, 384 insertions(+) create mode 100644 internal/cmd/experience/experience.go create mode 100644 internal/cmd/experience/experience_test.go create mode 100644 internal/experience/experience.go create mode 100644 internal/git/blame.go create mode 100644 internal/git/blame_test.go diff --git a/.gitignore b/.gitignore index 3744d443..fd8903d5 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,4 @@ test/resolve/testdata/gradle/*/** **.gradle-init-script.debricked.groovy test/resolve/testdata/gradle/gradle.debricked.lock /mvnproj/target +**debricked.experience.json diff --git a/internal/cmd/experience/experience.go b/internal/cmd/experience/experience.go new file mode 100644 index 00000000..cd101ff5 --- /dev/null +++ b/internal/cmd/experience/experience.go @@ -0,0 +1,52 @@ +package experience + +import ( + "github.com/debricked/cli/internal/experience" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +const ( + ExclusionFlag = "exclusion-experience" +) + +func NewExperienceCmd(experienceCalculator experience.IExperience) *cobra.Command { + + short := "Experience calculator uses git blame and call graphs to calculate who has written code with what open source. [beta feature]" + cmd := &cobra.Command{ + Use: "experience [path]", + Short: short, + Hidden: false, + Long: short, //TODO: Add long description + PreRun: func(cmd *cobra.Command, _ []string) { + _ = viper.BindPFlags(cmd.Flags()) + }, + RunE: RunE(experienceCalculator), + } + + viper.MustBindEnv(ExclusionFlag) + + return cmd +} + +func RunE(e experience.IExperience) func(_ *cobra.Command, args []string) error { + return func(_ *cobra.Command, args []string) error { + path := "" + if len(args) > 0 { + path = args[0] + } + + output, err := e.CalculateExperience(path, viper.GetStringSlice(ExclusionFlag)) + + if err != nil { + return err + } + + err = output.ToFile(experience.OutputFileNameExperience) + if err != nil { + return err + } + + return nil + } +} diff --git a/internal/cmd/experience/experience_test.go b/internal/cmd/experience/experience_test.go new file mode 100644 index 00000000..292782f7 --- /dev/null +++ b/internal/cmd/experience/experience_test.go @@ -0,0 +1 @@ +package experience diff --git a/internal/cmd/root/root.go b/internal/cmd/root/root.go index 5ed6d6ec..a345508d 100644 --- a/internal/cmd/root/root.go +++ b/internal/cmd/root/root.go @@ -2,6 +2,7 @@ package root import ( "github.com/debricked/cli/internal/cmd/callgraph" + "github.com/debricked/cli/internal/cmd/experience" "github.com/debricked/cli/internal/cmd/files" "github.com/debricked/cli/internal/cmd/fingerprint" "github.com/debricked/cli/internal/cmd/report" @@ -47,6 +48,7 @@ Read more: https://portal.debricked.com/administration-47/how-do-i-generate-an-a rootCmd.AddCommand(fingerprint.NewFingerprintCmd(container.Fingerprinter())) rootCmd.AddCommand(resolve.NewResolveCmd(container.Resolver())) rootCmd.AddCommand(callgraph.NewCallgraphCmd(container.CallgraphGenerator())) + rootCmd.AddCommand(experience.NewExperienceCmd(container.Expereince())) rootCmd.CompletionOptions.DisableDefaultCmd = true diff --git a/internal/experience/experience.go b/internal/experience/experience.go new file mode 100644 index 00000000..0e736f12 --- /dev/null +++ b/internal/experience/experience.go @@ -0,0 +1,66 @@ +package experience + +import ( + "encoding/json" + "log" + "os" + + "github.com/debricked/cli/internal/file" + "github.com/debricked/cli/internal/git" +) + +var OutputFileNameExperience = "debricked.experience.json" + +type IExperience interface { + CalculateExperience(rootPath string, exclusions []string) (*Experiences, error) +} + +type ExperienceCalculator struct { + finder file.IFinder +} + +func NewExperience(finder file.IFinder) *ExperienceCalculator { + return &ExperienceCalculator{ + finder: finder, + } +} + +func (e *ExperienceCalculator) CalculateExperience(rootPath string, exclusions []string) (*Experiences, error) { + log.Println("Calculating experience...") + + repo, repoErr := git.FindRepository(rootPath) + if repoErr != nil { + return nil, repoErr + } + + blamer := git.NewBlamer(repo) + + blames, err := blamer.BlamAllFiles() + if err != nil { + return nil, err + } + + log.Println("Blamed files:", len(blames)) + return nil, nil +} + +type Experience struct { + Author string `json:"author"` + Email string `json:"email"` + Count int `json:"count"` + Symbol string `json:"symbol"` +} + +type Experiences struct { + Entries []Experience `json:"experiences"` +} + +func (f *Experiences) ToFile(ouputFile string) error { + file, err := os.Create(ouputFile) + if err != nil { + return err + } + defer file.Close() + + return json.NewEncoder(file).Encode(f) +} diff --git a/internal/file/exclusion.go b/internal/file/exclusion.go index 7523c24b..7dec1add 100644 --- a/internal/file/exclusion.go +++ b/internal/file/exclusion.go @@ -60,3 +60,21 @@ func Excluded(exclusions []string, path string) bool { return false } + +func InclusionsExperience() []string { + return []string{ + "**/*.go", + } +} + +func Included(inclusions []string, path string) bool { + for _, exclusion := range inclusions { + ex := filepath.Clean(exclusion) + matched, _ := doublestar.PathMatch(ex, path) + if matched { + return true + } + } + + return false +} diff --git a/internal/file/exclusion_test.go b/internal/file/exclusion_test.go index 4ed426b8..f18ca32b 100644 --- a/internal/file/exclusion_test.go +++ b/internal/file/exclusion_test.go @@ -165,3 +165,22 @@ func TestExclude(t *testing.T) { }) } } + +func TestIncluded(t *testing.T) { + inclusions := InclusionsExperience() + testCases := []struct { + path string + expected bool + }{ + {"foo/bar/test.go", true}, + {"test.go", true}, + {"foo/bar/test.txt", false}, + } + + for _, tc := range testCases { + result := Included(inclusions, tc.path) + if result != tc.expected { + t.Errorf("Included(%q) = %v; want %v", tc.path, result, tc.expected) + } + } +} diff --git a/internal/git/blame.go b/internal/git/blame.go new file mode 100644 index 00000000..6af964ad --- /dev/null +++ b/internal/git/blame.go @@ -0,0 +1,152 @@ +package git + +import ( + "bufio" + "bytes" + "log" + "os/exec" + "path/filepath" + "strings" + "sync" + + "github.com/debricked/cli/internal/file" + "github.com/go-git/go-git/v5" +) + +type IBlamer interface { + Blame(path string) (*BlameFile, error) +} + +type Blamer struct { + repository *git.Repository + inclusions []string +} + +func NewBlamer(repository *git.Repository) *Blamer { + return &Blamer{ + repository: repository, + inclusions: file.InclusionsExperience(), + } +} + +type BlameFile struct { + Lines []BlameLine + Path string +} + +type BlameLine struct { + Author Author + LineNumber int +} + +type Author struct { + Email string + Name string +} + +// gitBlameFile runs `git blame --line-porcelain` on the given file and parses the output to populate a slice of BlameLine. +func gitBlameFile(filePath string) ([]BlameLine, error) { + cmd := exec.Command("git", "blame", "--line-porcelain", filePath) + + var out bytes.Buffer + cmd.Stdout = &out + + err := cmd.Run() + if err != nil { + return nil, err + } + + scanner := bufio.NewScanner(&out) + var blameLines []BlameLine + var currentBlame BlameLine + + lineNumber := 1 + for scanner.Scan() { + line := scanner.Text() + switch { + case strings.HasPrefix(line, "author "): + currentBlame.Author.Name = strings.TrimPrefix(line, "author ") + case strings.HasPrefix(line, "author-mail "): + currentBlame.Author.Email = strings.Trim(strings.TrimPrefix(line, "author-mail "), "<>") + + case strings.HasPrefix(line, "filename "): + + // End of the current commit block + currentBlame.LineNumber = lineNumber + blameLines = append(blameLines, currentBlame) // Add the populated BlameLine to the slice. + currentBlame = BlameLine{} // Reset for the next block. + lineNumber += 1 + } + } + + if err := scanner.Err(); err != nil { + return nil, err + } + + return blameLines, nil +} + +func (b *Blamer) BlamAllFiles() ([]BlameFile, error) { + files, err := FindAllTrackedFiles(b.repository) + if err != nil { + return nil, err + } + + blameFiles := make([]BlameFile, 0) + + blameFileChan := make(chan BlameFile, len(files)) + errChan := make(chan error, len(files)) + + w, err := b.repository.Worktree() + if err != nil { + log.Fatalf("Could not get workdir: %v", err) + } + + root := w.Filesystem.Root() + + var wg sync.WaitGroup + for _, fileBlame := range files { + + // Add the root path to the file path + fileBlameAbsPath := filepath.Join(root, fileBlame) + + if !file.Included(b.inclusions, fileBlame) { + continue + } + + wg.Add(1) + go func(fileBlame string) { + defer wg.Done() + + blameLines, err := gitBlameFile(fileBlameAbsPath) + + if err != nil { + errChan <- err + return + } + + blameFile := BlameFile{ + Lines: blameLines, + Path: fileBlame, + } + + blameFileChan <- blameFile + }(fileBlame) + } + + wg.Wait() + close(blameFileChan) + close(errChan) + + for bf := range blameFileChan { + blameFiles = append(blameFiles, bf) + } + + // for err := range errChan { + // if err != nil { + // return nil, err + // } + // } + + return blameFiles, nil +} diff --git a/internal/git/blame_test.go b/internal/git/blame_test.go new file mode 100644 index 00000000..74b3c834 --- /dev/null +++ b/internal/git/blame_test.go @@ -0,0 +1,32 @@ +package git + +import ( + "testing" +) + +func TestBlame(t *testing.T) { + + repo, err := FindRepository("../../.") + if err != nil { + t.Fatal("failed to find repo. Error:", err) + } + + blame := NewBlamer(repo) + + blameRes, err := blame.BlamAllFiles() + + if err != nil { + t.Fatal("failed to blame file. Error:", err) + } + + if len(blameRes[0].Lines) == 0 { + t.Fatal("Should be larger than 0 lines, was", len(blameRes[0].Lines)) + } + + if blameRes[0].Lines[0].Author.Name == "" { + t.Fatal("Author should not be empty") + } + if blameRes[0].Lines[0].Author.Email == "" { + t.Fatal("Email should not be empty") + } +} diff --git a/internal/git/git.go b/internal/git/git.go index 1f06b23f..062ebefa 100644 --- a/internal/git/git.go +++ b/internal/git/git.go @@ -151,3 +151,36 @@ func FindCommitHash(repository *git.Repository) (string, error) { return c.Hash.String(), nil } + +func FindAllTrackedFiles(repository *git.Repository) ([]string, error) { + var files []string + + // Get the HEAD reference to start from the latest commit. + ref, err := repository.Head() + if err != nil { + return nil, fmt.Errorf("could not get HEAD reference: %w", err) + } + + // Get the commit object from the HEAD reference. + commit, err := repository.CommitObject(ref.Hash()) + if err != nil { + return nil, fmt.Errorf("could not get commit from HEAD reference: %w", err) + } + + // Get the tree from the commit. + tree, err := commit.Tree() + if err != nil { + return nil, fmt.Errorf("could not get tree from commit: %w", err) + } + + // Walk the tree. + err = tree.Files().ForEach(func(f *object.File) error { + files = append(files, f.Name) + return nil + }) + if err != nil { + return nil, fmt.Errorf("error walking the tree: %w", err) + } + + return files, nil +} diff --git a/internal/wire/cli_container.go b/internal/wire/cli_container.go index 534d3c76..335f1bcb 100644 --- a/internal/wire/cli_container.go +++ b/internal/wire/cli_container.go @@ -9,6 +9,7 @@ import ( callgraphStrategy "github.com/debricked/cli/internal/callgraph/strategy" "github.com/debricked/cli/internal/ci" "github.com/debricked/cli/internal/client" + "github.com/debricked/cli/internal/experience" "github.com/debricked/cli/internal/file" "github.com/debricked/cli/internal/fingerprint" licenseReport "github.com/debricked/cli/internal/report/license" @@ -95,6 +96,8 @@ func (cc *CliContainer) wire() error { cc.licenseReporter = licenseReport.Reporter{DebClient: cc.debClient} cc.vulnerabilityReporter = vulnerabilityReport.Reporter{DebClient: cc.debClient} + cc.expereience = experience.NewExperience(cc.finder) + return nil } @@ -116,6 +119,7 @@ type CliContainer struct { cgFinder finder.IFinder cgScheduler callgraph.IScheduler cgStrategyFactory callgraphStrategy.IFactory + expereience experience.IExperience } func (cc *CliContainer) DebClient() client.IDebClient { @@ -153,3 +157,7 @@ func (cc *CliContainer) Fingerprinter() fingerprint.IFingerprint { func wireErr(err error) error { return fmt.Errorf("failed to wire with cli-container. Error %s", err) } + +func (cc *CliContainer) Expereince() experience.IExperience { + return cc.expereience +} From 771e0c33927dbdba90e876cbd154b3fa7e815edc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20W=C3=A5reus?= Date: Sat, 4 Nov 2023 17:17:30 +0100 Subject: [PATCH 2/3] change to xp instead of experience --- internal/cmd/experience/experience.go | 2 +- internal/experience/experience.go | 10 ++++++---- internal/file/exclusion.go | 10 ++++------ 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/internal/cmd/experience/experience.go b/internal/cmd/experience/experience.go index cd101ff5..4599d1f3 100644 --- a/internal/cmd/experience/experience.go +++ b/internal/cmd/experience/experience.go @@ -14,7 +14,7 @@ func NewExperienceCmd(experienceCalculator experience.IExperience) *cobra.Comman short := "Experience calculator uses git blame and call graphs to calculate who has written code with what open source. [beta feature]" cmd := &cobra.Command{ - Use: "experience [path]", + Use: "xp [path]", Short: short, Hidden: false, Long: short, //TODO: Add long description diff --git a/internal/experience/experience.go b/internal/experience/experience.go index 0e736f12..d6f608ca 100644 --- a/internal/experience/experience.go +++ b/internal/experience/experience.go @@ -7,6 +7,7 @@ import ( "github.com/debricked/cli/internal/file" "github.com/debricked/cli/internal/git" + "github.com/debricked/cli/internal/tui" ) var OutputFileNameExperience = "debricked.experience.json" @@ -16,17 +17,18 @@ type IExperience interface { } type ExperienceCalculator struct { - finder file.IFinder + finder file.IFinder + spinnerManager tui.ISpinnerManager } func NewExperience(finder file.IFinder) *ExperienceCalculator { return &ExperienceCalculator{ - finder: finder, + finder: finder, + spinnerManager: tui.NewSpinnerManager("Calculating OSS-Experience", "0"), } } func (e *ExperienceCalculator) CalculateExperience(rootPath string, exclusions []string) (*Experiences, error) { - log.Println("Calculating experience...") repo, repoErr := git.FindRepository(rootPath) if repoErr != nil { @@ -40,7 +42,7 @@ func (e *ExperienceCalculator) CalculateExperience(rootPath string, exclusions [ return nil, err } - log.Println("Blamed files:", len(blames)) + log.Println("Blamed files: ", len(blames)) return nil, nil } diff --git a/internal/file/exclusion.go b/internal/file/exclusion.go index 7dec1add..fc56f878 100644 --- a/internal/file/exclusion.go +++ b/internal/file/exclusion.go @@ -63,18 +63,16 @@ func Excluded(exclusions []string, path string) bool { func InclusionsExperience() []string { return []string{ - "**/*.go", + ".go", + ".java", } } func Included(inclusions []string, path string) bool { - for _, exclusion := range inclusions { - ex := filepath.Clean(exclusion) - matched, _ := doublestar.PathMatch(ex, path) - if matched { + for _, ex := range inclusions { + if strings.HasSuffix(path, ex) { return true } } - return false } From 1b22b474e7d5d62a23ef5b3c75747def07fdac68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emil=20W=C3=A5reus?= Date: Sat, 4 Nov 2023 17:51:20 +0100 Subject: [PATCH 3/3] add ability to save blame to file --- internal/experience/experience.go | 3 +- internal/git/blame.go | 43 ++++++++++++++++++++----- internal/git/blame_test.go | 52 ++++++++++++++++++++++++++++--- 3 files changed, 86 insertions(+), 12 deletions(-) diff --git a/internal/experience/experience.go b/internal/experience/experience.go index d6f608ca..ac0f949f 100644 --- a/internal/experience/experience.go +++ b/internal/experience/experience.go @@ -42,7 +42,8 @@ func (e *ExperienceCalculator) CalculateExperience(rootPath string, exclusions [ return nil, err } - log.Println("Blamed files: ", len(blames)) + log.Println("Blamed files: ", len(blames.Files)) + blames.ToFile("blames.txt") return nil, nil } diff --git a/internal/git/blame.go b/internal/git/blame.go index 6af964ad..41a559f2 100644 --- a/internal/git/blame.go +++ b/internal/git/blame.go @@ -3,7 +3,9 @@ package git import ( "bufio" "bytes" + "fmt" "log" + "os" "os/exec" "path/filepath" "strings" @@ -29,6 +31,30 @@ func NewBlamer(repository *git.Repository) *Blamer { } } +type BlameFiles struct { + Files []BlameFile +} + +func (b *BlameFiles) ToFile(outputFile string) error { + + file, err := os.Create(outputFile) + if err != nil { + return err + } + defer file.Close() + + for _, blameFile := range b.Files { + for _, line := range blameFile.Lines { + _, err := file.WriteString(fmt.Sprintf("%s,%d,%s,%s\n", blameFile.Path, line.LineNumber, line.Author.Name, line.Author.Email)) + if err != nil { + return err + } + } + } + + return nil +} + type BlameFile struct { Lines []BlameLine Path string @@ -86,7 +112,7 @@ func gitBlameFile(filePath string) ([]BlameLine, error) { return blameLines, nil } -func (b *Blamer) BlamAllFiles() ([]BlameFile, error) { +func (b *Blamer) BlamAllFiles() (*BlameFiles, error) { files, err := FindAllTrackedFiles(b.repository) if err != nil { return nil, err @@ -142,11 +168,14 @@ func (b *Blamer) BlamAllFiles() ([]BlameFile, error) { blameFiles = append(blameFiles, bf) } - // for err := range errChan { - // if err != nil { - // return nil, err - // } - // } + for err := range errChan { + if err != nil { + return nil, err + } + } + + return &BlameFiles{ + Files: blameFiles, + }, nil - return blameFiles, nil } diff --git a/internal/git/blame_test.go b/internal/git/blame_test.go index 74b3c834..26334b2d 100644 --- a/internal/git/blame_test.go +++ b/internal/git/blame_test.go @@ -1,6 +1,8 @@ package git import ( + "io/ioutil" + "os" "testing" ) @@ -19,14 +21,56 @@ func TestBlame(t *testing.T) { t.Fatal("failed to blame file. Error:", err) } - if len(blameRes[0].Lines) == 0 { - t.Fatal("Should be larger than 0 lines, was", len(blameRes[0].Lines)) + if len(blameRes.Files[0].Lines) == 0 { + t.Fatal("Should be larger than 0 lines, was", len(blameRes.Files[0].Lines)) } - if blameRes[0].Lines[0].Author.Name == "" { + if blameRes.Files[0].Lines[0].Author.Name == "" { t.Fatal("Author should not be empty") } - if blameRes[0].Lines[0].Author.Email == "" { + if blameRes.Files[0].Lines[0].Author.Email == "" { t.Fatal("Email should not be empty") } } + +func TestToFile(t *testing.T) { + + tempFile, err := os.CreateTemp("", "test") + if err != nil { + t.Fatal(err) + } + defer os.Remove(tempFile.Name()) // clean up + + blamefiles := BlameFiles{ + Files: []BlameFile{ + { + Lines: []BlameLine{ + { + Author: Author{ + Email: "example@example.com", + Name: "Example", + }, + LineNumber: 1, + }, + }, + Path: "example.txt", + }, + }, + } + + err = blamefiles.ToFile(tempFile.Name()) + if err != nil { + t.Fatal("failed to write to file. Error:", err) + } + + content, err := ioutil.ReadFile(tempFile.Name()) + if err != nil { + t.Fatal(err) + } + + expected := "example.txt,1,Example,example@example.com\n" + if string(content) != expected { + t.Errorf("Expected %s, got %s", expected, content) + } + +}