From a513d19bbb009194dcc55e388d678248fff6c056 Mon Sep 17 00:00:00 2001 From: Austin Ziegler Date: Tue, 8 Aug 2023 22:37:45 -0400 Subject: [PATCH] feat: Add `gitHubReleases` and `gitHubTags` Adds a feature requested in discussion #3161. - This reworks `gitHubLatestTag` to use `gitHubTags` and share the caching between the two functions. - Pagination for both (both size and page number) is not configurable. This will always return the first page of items, where the size of the page is the default specified by GitHub. That is currently thirty items, but is subject to change. --- .../github-functions/gitHubLatestTag.md | 6 +- .../github-functions/gitHubReleases.md | 30 ++++ .../templates/github-functions/gitHubTags.md | 30 ++++ internal/cmd/config.go | 2 + internal/cmd/githubtemplatefuncs.go | 142 ++++++++++++++---- internal/cmd/statecmd.go | 3 +- .../cmd/testdata/scripts/configstate.txtar | 3 +- .../scripts/githubtemplatefuncs.txtar | 11 ++ .../cmd/testdata/scripts/state_unix.txtar | 3 +- .../cmd/testdata/scripts/state_windows.txtar | 3 +- 10 files changed, 194 insertions(+), 39 deletions(-) create mode 100644 assets/chezmoi.io/docs/reference/templates/github-functions/gitHubReleases.md create mode 100644 assets/chezmoi.io/docs/reference/templates/github-functions/gitHubTags.md diff --git a/assets/chezmoi.io/docs/reference/templates/github-functions/gitHubLatestTag.md b/assets/chezmoi.io/docs/reference/templates/github-functions/gitHubLatestTag.md index 1192116dae09..22d8c19c8553 100644 --- a/assets/chezmoi.io/docs/reference/templates/github-functions/gitHubLatestTag.md +++ b/assets/chezmoi.io/docs/reference/templates/github-functions/gitHubLatestTag.md @@ -4,8 +4,10 @@ *owner-repo*, returning structured data as defined by the [GitHub Go API bindings](https://pkg.go.dev/github.com/google/go-github/v53/github#RepositoryTag). -Calls to `gitHubLatestTag` are cached so calling `gitHubLatestTag` with the -same *owner-repo* will only result in one call to the GitHub API. +Calls to `gitHubLatestTag` are cached the same as +[`githubTags`](/reference/templates/functions/gitHubTags.md), so calling +`gitHubLatestTag` with the same *owner-repo* will only result in one call to the +GitHub API. !!! example diff --git a/assets/chezmoi.io/docs/reference/templates/github-functions/gitHubReleases.md b/assets/chezmoi.io/docs/reference/templates/github-functions/gitHubReleases.md new file mode 100644 index 000000000000..f48563200210 --- /dev/null +++ b/assets/chezmoi.io/docs/reference/templates/github-functions/gitHubReleases.md @@ -0,0 +1,30 @@ +# `gitHubReleases` *owner-repo* + +`gitHubReleases` calls the GitHub API to retrieve the first page of releases for +the given *owner-repo*, returning structured data as defined by the [GitHub Go +API +bindings](https://pkg.go.dev/github.com/google/go-github/v53/github#RepositoryRelease). + +Calls to `gitHubReleases` are cached so calling `gitHubReleases` with the same +*owner-repo* will only result in one call to the GitHub API. + +!!! example + + ``` + {{ (index (gitHubReleases "docker/compose") 0).TagName } + ``` + +!!! note + + The maximum number of items returned by `gitHubReleases` is determined by + default page size for the GitHub API. + +!!! warning + + The values returned by `gitHubReleases` are not directly queryable via the + [`jq`](/reference/templates/functions/jq.md) function and must instead be + converted through JSON: + + ``` + {{ gitHubReleases "docker/compose" | toJson | fromJson | jq ".[0].tag_name" }} + ``` diff --git a/assets/chezmoi.io/docs/reference/templates/github-functions/gitHubTags.md b/assets/chezmoi.io/docs/reference/templates/github-functions/gitHubTags.md new file mode 100644 index 000000000000..a259043563cf --- /dev/null +++ b/assets/chezmoi.io/docs/reference/templates/github-functions/gitHubTags.md @@ -0,0 +1,30 @@ +# `gitHubTags` *owner-repo* + +`gitHubTags` calls the GitHub API to retrieve the first page of tags for +the given *owner-repo*, returning structured data as defined by the [GitHub Go +API +bindings](https://pkg.go.dev/github.com/google/go-github/v53/github#RepositoryTag). + +Calls to `gitHubTags` are cached so calling `gitHubTags` with the +same *owner-repo* will only result in one call to the GitHub API. + +!!! example + + ``` + {{ (index (gitHubTags "docker/compose") 0).Name }} + ``` + +!!! note + + The maximum number of items returned by `gitHubReleases` is determined by + default page size for the GitHub API. + +!!! warning + + The values returned by `gitHubTags` are not directly queryable via the + [`jq`](/reference/templates/functions/jq.md) function and must instead be + converted through JSON: + + ``` + {{ gitHubTags "docker/compose" | toJson | fromJson | jq ".[0].name" }} + ``` diff --git a/internal/cmd/config.go b/internal/cmd/config.go index 5d7a8bdb200a..a575651c42ff 100644 --- a/internal/cmd/config.go +++ b/internal/cmd/config.go @@ -410,6 +410,8 @@ func newConfig(options ...configOption) (*Config, error) { "gitHubKeys": c.gitHubKeysTemplateFunc, "gitHubLatestRelease": c.gitHubLatestReleaseTemplateFunc, "gitHubLatestTag": c.gitHubLatestTagTemplateFunc, + "gitHubReleases": c.gitHubReleasesTemplateFunc, + "gitHubTags": c.gitHubTagsTemplateFunc, "glob": c.globTemplateFunc, "gopass": c.gopassTemplateFunc, "gopassRaw": c.gopassRawTemplateFunc, diff --git a/internal/cmd/githubtemplatefuncs.go b/internal/cmd/githubtemplatefuncs.go index 399a0e426ae6..3950f48c8fd6 100644 --- a/internal/cmd/githubtemplatefuncs.go +++ b/internal/cmd/githubtemplatefuncs.go @@ -25,15 +25,21 @@ type gitHubLatestReleaseState struct { Release *github.RepositoryRelease `json:"release" yaml:"release"` } -type gitHubLatestTagState struct { - RequestedAt time.Time `json:"requestedAt" yaml:"requestedAt"` - Tag *github.RepositoryTag `json:"tag" yaml:"tag"` +type gitHubReleasesState struct { + RequestedAt time.Time `json:"requestedAt" yaml:"requestedAt"` + Releases []*github.RepositoryRelease `json:"releases" yaml:"releases"` +} + +type gitHubTagsState struct { + RequestedAt time.Time `json:"requestedAt" yaml:"requestedAt"` + Tags []*github.RepositoryTag `json:"tags" yaml:"tags"` } var ( gitHubKeysStateBucket = []byte("gitHubLatestKeysState") gitHubLatestReleaseStateBucket = []byte("gitHubLatestReleaseState") - gitHubLatestTagStateBucket = []byte("gitHubLatestTagState") + gitHubReleasesStateBucket = []byte("gitHubReleasesState") + gitHubTagsStateBucket = []byte("gitHubTagsState") ) type gitHubData struct { @@ -41,7 +47,8 @@ type gitHubData struct { clientErr error keysCache map[string][]*github.Key latestReleaseCache map[string]map[string]*github.RepositoryRelease - latestTagCache map[string]map[string]*github.RepositoryTag + releasesCache map[string]map[string][]*github.RepositoryRelease + tagsCache map[string]map[string][]*github.RepositoryTag } func (c *Config) gitHubKeysTemplateFunc(user string) []*github.Key { @@ -153,25 +160,38 @@ func (c *Config) gitHubLatestReleaseTemplateFunc(ownerRepo string) *github.Repos return release } -func (c *Config) gitHubLatestTagTemplateFunc(userRepo string) *github.RepositoryTag { - owner, repo, err := gitHubSplitOwnerRepo(userRepo) +func (c *Config) gitHubLatestTagTemplateFunc(ownerRepo string) *github.RepositoryTag { + tags, err := c.getGitHubTags(ownerRepo) + if err != nil { + panic(err) + } + + if len(tags) > 0 { + return tags[0] + } + + return nil +} + +func (c *Config) gitHubReleasesTemplateFunc(ownerRepo string) []*github.RepositoryRelease { + owner, repo, err := gitHubSplitOwnerRepo(ownerRepo) if err != nil { panic(err) } - if tag, ok := c.gitHub.latestTagCache[owner][repo]; ok { - return tag + if releases := c.gitHub.releasesCache[owner][repo]; releases != nil { + return releases } now := time.Now() - gitHubLatestTagKey := []byte(owner + "/" + repo) + gitHubReleasesKey := []byte(owner + "/" + repo) if c.GitHub.RefreshPeriod != 0 { - var gitHubLatestTagValue gitHubLatestTagState - switch ok, err := chezmoi.PersistentStateGet(c.persistentState, gitHubLatestTagStateBucket, gitHubLatestTagKey, &gitHubLatestTagValue); { + var gitHubReleasesStateValue gitHubReleasesState + switch ok, err := chezmoi.PersistentStateGet(c.persistentState, gitHubReleasesStateBucket, gitHubReleasesKey, &gitHubReleasesStateValue); { case err != nil: panic(err) - case ok && now.Before(gitHubLatestTagValue.RequestedAt.Add(c.GitHub.RefreshPeriod)): - return gitHubLatestTagValue.Tag + case ok && now.Before(gitHubReleasesStateValue.RequestedAt.Add(c.GitHub.RefreshPeriod)): + return gitHubReleasesStateValue.Releases } } @@ -183,33 +203,89 @@ func (c *Config) gitHubLatestTagTemplateFunc(userRepo string) *github.Repository panic(err) } - tags, _, err := gitHubClient.Repositories.ListTags(ctx, owner, repo, &github.ListOptions{ - PerPage: 1, - }) + releases, _, err := gitHubClient.Repositories.ListReleases(ctx, owner, repo, nil) if err != nil { panic(err) } - var tag *github.RepositoryTag - if len(tags) > 0 { - tag = tags[0] - } - if err := chezmoi.PersistentStateSet(c.persistentState, gitHubLatestTagStateBucket, gitHubLatestTagKey, &gitHubLatestTagState{ + if err := chezmoi.PersistentStateSet(c.persistentState, gitHubReleasesStateBucket, gitHubReleasesKey, &gitHubReleasesState{ RequestedAt: now, - Tag: tag, + Releases: releases, }); err != nil { panic(err) } - if c.gitHub.latestTagCache == nil { - c.gitHub.latestTagCache = make(map[string]map[string]*github.RepositoryTag) + if c.gitHub.releasesCache == nil { + c.gitHub.releasesCache = make(map[string]map[string][]*github.RepositoryRelease) + } + if c.gitHub.releasesCache[owner] == nil { + c.gitHub.releasesCache[owner] = make(map[string][]*github.RepositoryRelease) + } + c.gitHub.releasesCache[owner][repo] = releases + + return releases +} + +func (c *Config) gitHubTagsTemplateFunc(ownerRepo string) []*github.RepositoryTag { + tags, err := c.getGitHubTags(ownerRepo) + if err != nil { + panic(err) + } + + return tags +} + +func (c *Config) getGitHubTags(ownerRepo string) ([]*github.RepositoryTag, error) { + owner, repo, err := gitHubSplitOwnerRepo(ownerRepo) + if err != nil { + return nil, err + } + + if tags := c.gitHub.tagsCache[owner][repo]; tags != nil { + return tags, nil + } + + now := time.Now() + gitHubTagsKey := []byte(owner + "/" + repo) + if c.GitHub.RefreshPeriod != 0 { + var gitHubTagsStateValue gitHubTagsState + switch ok, err := chezmoi.PersistentStateGet(c.persistentState, gitHubTagsStateBucket, gitHubTagsKey, &gitHubTagsStateValue); { + case err != nil: + return nil, err + case ok && now.Before(gitHubTagsStateValue.RequestedAt.Add(c.GitHub.RefreshPeriod)): + return gitHubTagsStateValue.Tags, nil + } + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + gitHubClient, err := c.getGitHubClient(ctx) + if err != nil { + return nil, err + } + + tags, _, err := gitHubClient.Repositories.ListTags(ctx, owner, repo, nil) + if err != nil { + return nil, err + } + + if err := chezmoi.PersistentStateSet(c.persistentState, gitHubTagsStateBucket, gitHubTagsKey, &gitHubTagsState{ + RequestedAt: now, + Tags: tags, + }); err != nil { + return nil, err + } + + if c.gitHub.tagsCache == nil { + c.gitHub.tagsCache = make(map[string]map[string][]*github.RepositoryTag) } - if c.gitHub.latestTagCache[owner] == nil { - c.gitHub.latestTagCache[owner] = make(map[string]*github.RepositoryTag) + if c.gitHub.tagsCache[owner] == nil { + c.gitHub.tagsCache[owner] = make(map[string][]*github.RepositoryTag) } - c.gitHub.latestTagCache[owner][repo] = tag + c.gitHub.tagsCache[owner][repo] = tags - return tag + return tags, nil } func (c *Config) getGitHubClient(ctx context.Context) (*github.Client, error) { @@ -227,10 +303,10 @@ func (c *Config) getGitHubClient(ctx context.Context) (*github.Client, error) { return c.gitHub.client, nil } -func gitHubSplitOwnerRepo(userRepo string) (string, string, error) { - user, repo, ok := strings.Cut(userRepo, "/") +func gitHubSplitOwnerRepo(ownerRepo string) (string, string, error) { + owner, repo, ok := strings.Cut(ownerRepo, "/") if !ok { - return "", "", fmt.Errorf("%s: not a user/repo", userRepo) + return "", "", fmt.Errorf("%s: not an owner/repo", ownerRepo) } - return user, repo, nil + return owner, repo, nil } diff --git a/internal/cmd/statecmd.go b/internal/cmd/statecmd.go index c3f1c953ee3a..d0a84adce917 100644 --- a/internal/cmd/statecmd.go +++ b/internal/cmd/statecmd.go @@ -206,7 +206,8 @@ func (c *Config) runStateDumpCmd(cmd *cobra.Command, args []string) error { "entryState": chezmoi.EntryStateBucket, "gitHubKeysState": gitHubKeysStateBucket, "gitHubLatestReleaseState": gitHubLatestReleaseStateBucket, - "gitHubLatestTagState": gitHubLatestTagStateBucket, + "gitHubReleasesState": gitHubReleasesStateBucket, + "gitHubTagsState": gitHubTagsStateBucket, "gitRepoExternalState": chezmoi.GitRepoExternalStateBucket, "scriptState": chezmoi.ScriptStateBucket, }) diff --git a/internal/cmd/testdata/scripts/configstate.txtar b/internal/cmd/testdata/scripts/configstate.txtar index 88b24ba1e541..1cc08c5e65b3 100644 --- a/internal/cmd/testdata/scripts/configstate.txtar +++ b/internal/cmd/testdata/scripts/configstate.txtar @@ -69,7 +69,8 @@ configState: entryState: {} gitHubKeysState: {} gitHubLatestReleaseState: {} -gitHubLatestTagState: {} +gitHubReleasesState: {} +gitHubTagsState: {} gitRepoExternalState: {} scriptState: {} -- home/user/.local/share/chezmoi/.chezmoi.toml.tmpl -- diff --git a/internal/cmd/testdata/scripts/githubtemplatefuncs.txtar b/internal/cmd/testdata/scripts/githubtemplatefuncs.txtar index ddbdbb969c08..933a15f73435 100644 --- a/internal/cmd/testdata/scripts/githubtemplatefuncs.txtar +++ b/internal/cmd/testdata/scripts/githubtemplatefuncs.txtar @@ -11,3 +11,14 @@ stdout ^v2\. # test gitHubLatestTag template function exec chezmoi execute-template '{{ (gitHubLatestTag "twpayne/chezmoi").Name }}' stdout ^v2\. + +# test gitHubTags template functions +exec chezmoi execute-template '{{ (index (gitHubTags "twpayne/chezmoi") 0).Name }}' +stdout ^v2\. + +# test gitHubReleases template functions +exec chezmoi execute-template '{{ (index (gitHubReleases "twpayne/chezmoi") 0).TagName }}' +stdout ^v2\. + +# gitHubReleases +# gitHubTags diff --git a/internal/cmd/testdata/scripts/state_unix.txtar b/internal/cmd/testdata/scripts/state_unix.txtar index 15fa59358757..b7130f4500ab 100644 --- a/internal/cmd/testdata/scripts/state_unix.txtar +++ b/internal/cmd/testdata/scripts/state_unix.txtar @@ -42,7 +42,8 @@ configState: {} entryState: {} gitHubKeysState: {} gitHubLatestReleaseState: {} -gitHubLatestTagState: {} +gitHubReleasesState: {} +gitHubTagsState: {} gitRepoExternalState: {} scriptState: {} -- home/user/.local/share/chezmoi/run_once_script.sh -- diff --git a/internal/cmd/testdata/scripts/state_windows.txtar b/internal/cmd/testdata/scripts/state_windows.txtar index 9bc2daf2f07b..17561a3fc59e 100644 --- a/internal/cmd/testdata/scripts/state_windows.txtar +++ b/internal/cmd/testdata/scripts/state_windows.txtar @@ -23,7 +23,8 @@ configState: {} entryState: {} gitHubKeysState: {} gitHubLatestReleaseState: {} -gitHubLatestTagState: {} +gitHubReleasesState: {} +gitHubTagsState: {} gitRepoExternalState: {} scriptState: {} -- home/user/.local/share/chezmoi/run_once_script.cmd --