From 82dd0213b21a76767d50f6c605faa8a2a79139e3 Mon Sep 17 00:00:00 2001 From: Nic Jackson Date: Fri, 8 Dec 2023 11:15:16 +0000 Subject: [PATCH] Fix checks for docs refresh --- .vscode/launch.json | 1 + cmd/dev.go | 5 + examples/docs/docs.hcl | 2 +- pkg/config/resources/docs/provider_book.go | 13 +- pkg/config/resources/docs/provider_chapter.go | 9 +- pkg/config/resources/docs/provider_docs.go | 99 ++++++------ pkg/config/resources/docs/resource_docs.go | 5 + pkg/jumppad/engine.go | 7 +- pkg/utils/{util_test.bak => util_test.go} | 142 ++++++------------ pkg/utils/utils.go | 24 +++ 10 files changed, 146 insertions(+), 161 deletions(-) rename pkg/utils/{util_test.bak => util_test.go} (59%) diff --git a/.vscode/launch.json b/.vscode/launch.json index 3e412160..c368c69d 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -64,6 +64,7 @@ }, "args": [ "dev", + "--disable-tty", "${input:blueprint}", ], }, diff --git a/cmd/dev.go b/cmd/dev.go index ab0e4867..c32c7604 100644 --- a/cmd/dev.go +++ b/cmd/dev.go @@ -160,6 +160,11 @@ func doUpdates(v view.View, e jumppad.Engine, source string, variables map[strin v.Logger().Error(err.Error()) } + v.Logger().Debug("Changes detected", "new", len(new), "changed", len(changed), "removed", len(removed)) + for _, n := range changed { + v.Logger().Debug("Changed", "resource", n.Metadata().ID) + } + if len(new) > 0 || len(changed) > 0 || len(removed) > 0 { v.UpdateStatus( fmt.Sprintf( diff --git a/examples/docs/docs.hcl b/examples/docs/docs.hcl index b429685e..46d235fa 100644 --- a/examples/docs/docs.hcl +++ b/examples/docs/docs.hcl @@ -16,6 +16,6 @@ resource "chapter" "introduction" { title = "Introduction" page "introduction" { - content = "Some content" + content = "Some more content" } } \ No newline at end of file diff --git a/pkg/config/resources/docs/provider_book.go b/pkg/config/resources/docs/provider_book.go index 7bc3b766..e057d399 100644 --- a/pkg/config/resources/docs/provider_book.go +++ b/pkg/config/resources/docs/provider_book.go @@ -2,11 +2,9 @@ package docs import ( "fmt" - "os" htypes "github.com/jumppad-labs/hclconfig/types" "github.com/jumppad-labs/jumppad/pkg/clients/logger" - "github.com/jumppad-labs/jumppad/pkg/utils" ) type BookProvider struct { @@ -27,8 +25,6 @@ func (p *BookProvider) Init(cfg htypes.Resource, l logger.Logger) error { } func (p *BookProvider) Create() error { - p.log.Info(fmt.Sprintf("Creating %s", p.config.Type), "ref", p.config.Name) - index := BookIndex{ Title: p.config.Title, } @@ -48,10 +44,7 @@ func (p *BookProvider) Create() error { } func (p *BookProvider) Destroy() error { - // clean up the library folder - err := os.RemoveAll(utils.LibraryFolder("", os.ModePerm)) - - return err + return nil } func (p *BookProvider) Lookup() ([]string, error) { @@ -59,15 +52,11 @@ func (p *BookProvider) Lookup() ([]string, error) { } func (p *BookProvider) Refresh() error { - p.log.Debug("Refresh Book", "ref", p.config.ID) - - p.Destroy() p.Create() return nil } func (p *BookProvider) Changed() (bool, error) { - p.log.Debug("Checking changes", "ref", p.config.ID) return false, nil } diff --git a/pkg/config/resources/docs/provider_chapter.go b/pkg/config/resources/docs/provider_chapter.go index 56383026..70741aab 100644 --- a/pkg/config/resources/docs/provider_chapter.go +++ b/pkg/config/resources/docs/provider_chapter.go @@ -27,8 +27,6 @@ func (p *ChapterProvider) Init(cfg htypes.Resource, l logger.Logger) error { } func (p *ChapterProvider) Create() error { - p.log.Info(fmt.Sprintf("Creating %s", p.config.Type), "ref", p.config.ID) - index := ChapterIndex{ Title: p.config.Title, } @@ -78,16 +76,11 @@ func (p *ChapterProvider) Lookup() ([]string, error) { } func (p *ChapterProvider) Refresh() error { - p.log.Debug("Refresh Chapter", "ref", p.config.ID) - - p.Destroy() - p.Create() + p.Create() // always generate content return nil } func (p *ChapterProvider) Changed() (bool, error) { - p.log.Debug("Checking changes", "ref", p.config.ID) - return false, nil } diff --git a/pkg/config/resources/docs/provider_docs.go b/pkg/config/resources/docs/provider_docs.go index a83a9e1e..1543faf6 100644 --- a/pkg/config/resources/docs/provider_docs.go +++ b/pkg/config/resources/docs/provider_docs.go @@ -94,7 +94,8 @@ func (p *DocsProvider) Create() error { return err } - return nil + // write the content + return p.Refresh() } // Destroy the documentation container @@ -114,6 +115,10 @@ func (p *DocsProvider) Destroy() error { } } + // remove the cached files + contentPath := utils.LibraryFolder("", 0775) + os.RemoveAll(contentPath) + return nil } @@ -123,14 +128,24 @@ func (p *DocsProvider) Lookup() ([]string, error) { } func (p *DocsProvider) Refresh() error { - p.log.Debug("Refresh Docs", "ref", p.config.ID) + changed, err := p.checkChanged() + if err != nil { + return fmt.Errorf("unable to check if content has changed: %s", err) + } + + // no changes return + if !changed { + return nil + } + + p.log.Info("Refresh Docs", "ref", p.config.ID) // refresh content on disk configPath := utils.LibraryFolder("config", 0775) // jumppad.config.js frontendConfigPath := filepath.Join(configPath, "jumppad.config.js") - err := p.writeConfig(frontendConfigPath) + err = p.writeConfig(frontendConfigPath) if err != nil { return err } @@ -171,9 +186,48 @@ func (p *DocsProvider) Refresh() error { } } + // store a checksum of the content + cs, err := p.generateContentChecksum() + if err != nil { + return fmt.Errorf("unable to generate checksum for content: %s", err) + } + + p.config.ContentChecksum = cs + return nil } +func (p *DocsProvider) Changed() (bool, error) { + p.log.Debug("Checking changes", "ref", p.config.ID) + + // since the content has not been processed we can not reliably determine + // if the content has changed, so we will assume it has + return true, nil +} + +// check if the content has changed +func (p *DocsProvider) checkChanged() (bool, error) { + if p.config.ContentChecksum == "" { + return true, nil + } + + cs, err := p.generateContentChecksum() + if err != nil { + return true, fmt.Errorf("unable to generate checksum for content: %s", err) + } + + return cs != p.config.ContentChecksum, nil +} + +func (p *DocsProvider) generateContentChecksum() (string, error) { + cs, err := utils.ChecksumFromInterface(p.config.Content) + if err != nil { + return "", fmt.Errorf("unable to generate checksum for content: %s", err) + } + + return cs, nil +} + func (p *DocsProvider) getDefaultPage() string { if len(p.config.Content) > 0 { book := p.config.Content[0] @@ -188,12 +242,6 @@ func (p *DocsProvider) getDefaultPage() string { return "/" } -func (p *DocsProvider) Changed() (bool, error) { - p.log.Debug("Checking changes", "ref", p.config.ID) - - return false, nil -} - func (p *DocsProvider) createDocsContainer() error { // set the FQDN fqdn := utils.FQDN(p.config.Name, p.config.Module, p.config.Type) @@ -243,25 +291,6 @@ func (p *DocsProvider) createDocsContainer() error { // ~/.jumppad/library/content contentPath := utils.LibraryFolder("content", 0775) - for _, book := range p.config.Content { - bookPath := filepath.Join(contentPath, book.Name) - - for _, chapter := range book.Chapters { - chapterPath := filepath.Join(bookPath, chapter.Name) - os.MkdirAll(chapterPath, 0755) - os.Chmod(chapterPath, 0755) - - for _, page := range chapter.Pages { - pageFile := fmt.Sprintf("%s.mdx", page.Name) - pagePath := filepath.Join(chapterPath, pageFile) - err := os.WriteFile(pagePath, []byte(page.Content), 0755) - if err != nil { - return fmt.Errorf("unable to write page %s to disk at %s", page.Name, pagePath) - } - } - } - } - // mount the content cc.Volumes = append( cc.Volumes, @@ -274,20 +303,6 @@ func (p *DocsProvider) createDocsContainer() error { // ~/.jumppad/library/config configPath := utils.LibraryFolder("config", 0775) - // write the navigation - navigationPath := filepath.Join(configPath, "navigation.jsx") - err = p.writeNavigation(navigationPath) - if err != nil { - return err - } - - // write the progress - progressPath := filepath.Join(configPath, "progress.jsx") - err = p.writeProgress(progressPath) - if err != nil { - return err - } - cc.Volumes = append( cc.Volumes, types.Volume{ diff --git a/pkg/config/resources/docs/resource_docs.go b/pkg/config/resources/docs/resource_docs.go index ff780691..383fbc0f 100644 --- a/pkg/config/resources/docs/resource_docs.go +++ b/pkg/config/resources/docs/resource_docs.go @@ -31,6 +31,10 @@ type Docs struct { // ContainerName is the fully qualified resource name for the container, this can be used // to access the container from other sources ContainerName string `hcl:"fqdn,optional" json:"fqdn,omitempty"` + + // ContentChecksum is the checksum of the content directory, this is used to determine if the + // docs need to be recreated + ContentChecksum string `hcl:"content_checksum,optional" json:"content_checksum,omitempty"` } type Logo struct { @@ -54,6 +58,7 @@ func (d *Docs) Process() error { if r != nil { kstate := r.(*Docs) d.ContainerName = kstate.ContainerName + d.ContentChecksum = kstate.ContentChecksum } } diff --git a/pkg/jumppad/engine.go b/pkg/jumppad/engine.go index 375be483..6f38c53a 100644 --- a/pkg/jumppad/engine.go +++ b/pkg/jumppad/engine.go @@ -143,8 +143,8 @@ func (e *EngineImpl) Diff(path string, variables map[string]string, variablesFil continue } - // check if the resource has changed - if cr.Metadata().Checksum != r.Metadata().Checksum { + // check if the hcl resource text has changed + if cr.Metadata().Checksum.Parsed != r.Metadata().Checksum.Parsed { // resource has changes rebuild changed = append(changed, r) continue @@ -274,9 +274,8 @@ func (e *EngineImpl) ApplyWithVariables(path string, vars map[string]string, var processErr := e.readAndProcessConfig(path, vars, variablesFile, e.createCallback) // we need to remove any resources that are in the state but not in the config - e.log.Debug("removing resources in state but not in current config") for _, r := range removed { - e.log.Debug("removing", "id", r.Metadata().ID) + e.log.Debug("removing resource in state but not current config", "id", r.Metadata().ID) p := e.providers.GetProvider(r) if p == nil { diff --git a/pkg/utils/util_test.bak b/pkg/utils/util_test.go similarity index 59% rename from pkg/utils/util_test.bak rename to pkg/utils/util_test.go index f81a2971..24ea9774 100644 --- a/pkg/utils/util_test.bak +++ b/pkg/utils/util_test.go @@ -1,6 +1,7 @@ package utils import ( + "encoding/json" "fmt" "io/ioutil" "os" @@ -8,6 +9,7 @@ import ( "testing" "github.com/gosuri/uitable/util/strutil" + "github.com/stretchr/testify/require" assert "github.com/stretchr/testify/require" ) @@ -60,14 +62,14 @@ func TestIsNotFolder(t *testing.T) { } func TestGetBlueprintFolderReturnsFolder(t *testing.T) { - dir, err := GetBlueprintFolder("github.com/org/repo//folder?ref=dfdf&foo=bah") + dir, err := BlueprintFolder("github.com/org/repo?ref=dfdf&foo=bah//folder") assert.NoError(t, err) - assert.Equal(t, "folder/ref/dfdf/foo/bah", dir) + assert.Equal(t, "folder", dir) } func TestGetBlueprintFolderReturnsError(t *testing.T) { - _, err := GetBlueprintFolder("github.com/org/repo/folder") + _, err := BlueprintFolder("github.com/org/repo/folder") assert.Error(t, err) } @@ -100,7 +102,7 @@ func TestFQDNReturnsCorrectValue(t *testing.T) { func TestFQDNReplacesInvalidChars(t *testing.T) { fq := FQDN("tes&t", "", "kubernetes_cluster") - assert.Equal(t, "tes-t.k8s-cluster.jumppad.dev", fq) + assert.Equal(t, "tes-t.kubernetes-cluster.jumppad.dev", fq) } func TestFQDNVolumeReturnsCorrectValue(t *testing.T) { @@ -115,14 +117,14 @@ func TestHomeReturnsCorrectValue(t *testing.T) { func TestStateReturnsCorrectValue(t *testing.T) { h := StateDir() - expected := filepath.Join(os.Getenv(HomeEnvName()), ".shipyard/state") + expected := filepath.Join(os.Getenv(HomeEnvName()), ".jumppad/state") assert.Equal(t, expected, h) } func TestStatePathReturnsCorrectValue(t *testing.T) { h := StatePath() - assert.Equal(t, filepath.Join(os.Getenv(HomeEnvName()), ".shipyard/state/state.json"), h) + assert.Equal(t, filepath.Join(os.Getenv(HomeEnvName()), ".jumppad/state/state.json"), h) } func TestCreateKubeConfigPathReturnsCorrectValues(t *testing.T) { @@ -133,9 +135,9 @@ func TestCreateKubeConfigPathReturnsCorrectValues(t *testing.T) { d, f, dp := CreateKubeConfigPath("testing") - assert.Equal(t, filepath.Join(tmp, ".shipyard", "config", "testing"), d) - assert.Equal(t, filepath.Join(tmp, ".shipyard", "config", "testing", "kubeconfig.yaml"), f) - assert.Equal(t, filepath.Join(tmp, ".shipyard", "config", "testing", "kubeconfig-docker.yaml"), dp) + assert.Equal(t, filepath.Join(tmp, ".jumppad", "config", "testing"), d) + assert.Equal(t, filepath.Join(tmp, ".jumppad", "config", "testing", "kubeconfig.yaml"), f) + assert.Equal(t, filepath.Join(tmp, ".jumppad", "config", "testing", "kubeconfig-docker.yaml"), dp) // check creates folder s, err := os.Stat(d) @@ -143,79 +145,6 @@ func TestCreateKubeConfigPathReturnsCorrectValues(t *testing.T) { assert.True(t, s.IsDir()) } -func setupClusterConfigTest(t *testing.T) { - home := os.Getenv(HomeEnvName()) - tmp := t.TempDir() - os.Setenv(HomeEnvName(), tmp) - - t.Cleanup(func() { - os.Setenv(HomeEnvName(), home) - }) -} -func TestGetClusterConfigReturnsExistingConfig(t *testing.T) { - setupClusterConfigTest(t) - - configDir := filepath.Join(JumppadHome(), "config", "testing") - os.MkdirAll(configDir, os.ModePerm) - - // create the temp config - cc := ClusterConfig{ - LocalAddress: "testing", - } - - err := cc.Save(filepath.Join(configDir, "config.json")) - assert.NoError(t, err) - - conf, dir := GetClusterConfig("nomad_cluster.testing") - - assert.Equal(t, cc.LocalAddress, conf.LocalAddress) - assert.Equal(t, configDir, dir) -} - -func TestGetClusterConfigReturnsEmptyWhenUnableToParseName(t *testing.T) { - setupClusterConfigTest(t) - - conf, dir := GetClusterConfig("nomad") - - assert.Equal(t, "", conf.LocalAddress) - assert.Equal(t, "", dir) -} - -func TestGetClusterConfigCreatesNewNomadConfig(t *testing.T) { - setupClusterConfigTest(t) - configDir := filepath.Join(JumppadHome(), "config", "testing") - - conf, dir := GetClusterConfig("nomad_cluster.testing") - - assert.Contains(t, GetDockerIP(), conf.LocalAddress) - assert.Equal(t, 4646, conf.RemoteAPIPort) - assert.Equal(t, "server.testing.nomad-cluster.jumppad.dev", conf.RemoteAddress) - assert.Equal(t, GetDockerIP(), conf.LocalAddress) - assert.Equal(t, configDir, dir) -} - -func TestGetClusterConfigTwiceReturnsSameConfig(t *testing.T) { - setupClusterConfigTest(t) - - conf, _ := GetClusterConfig("nomad_cluster.testing") - conf2, _ := GetClusterConfig("nomad_cluster.testing") - - assert.Equal(t, conf2.ConnectorAddress(LocalContext), conf.ConnectorAddress(LocalContext)) -} - -func TestGetClusterConfigCreatesNewKubernetesConfig(t *testing.T) { - setupClusterConfigTest(t) - configDir := filepath.Join(JumppadHome(), "config", "testing") - - conf, dir := GetClusterConfig("kubernetes_cluster.testing") - - assert.Contains(t, GetDockerIP(), conf.LocalAddress) - assert.Equal(t, conf.APIPort, conf.RemoteAPIPort) - assert.Equal(t, "server.testing.kubernetes-cluster.jumppad.dev", conf.RemoteAddress) - assert.Equal(t, GetDockerIP(), conf.LocalAddress) - assert.Equal(t, configDir, dir) -} - func TestShipyardTempReturnsPath(t *testing.T) { home := os.Getenv(HomeEnvName()) tmp, _ := ioutil.TempDir("", "") @@ -226,9 +155,9 @@ func TestShipyardTempReturnsPath(t *testing.T) { os.RemoveAll(tmp) }) - st := ShipyardTemp() + st := JumppadTemp() - assert.Equal(t, filepath.Join(tmp, ".shipyard", "/tmp"), st) + assert.Equal(t, filepath.Join(tmp, ".jumppad", "/tmp"), st) s, err := os.Stat(st) assert.NoError(t, err) @@ -245,9 +174,9 @@ func TestShipyardDataReturnsPath(t *testing.T) { os.RemoveAll(tmp) }) - d := GetDataFolder("test", 0775) + d := DataFolder("test", 0775) - assert.Equal(t, filepath.Join(tmp, ".shipyard", "/data", "/test"), d) + assert.Equal(t, filepath.Join(tmp, ".jumppad", "/data", "/test"), d) s, err := os.Stat(d) fmt.Println(d, s) @@ -258,15 +187,15 @@ func TestShipyardDataReturnsPath(t *testing.T) { func TestHelmLocalFolderReturnsPath(t *testing.T) { chart := "github.com/jetstack/cert-manager?ref=v1.2.0/deploy/charts//cert-manager" - h := GetHelmLocalFolder(chart) + h := HelmLocalFolder(chart) - assert.Equal(t, filepath.Join(os.Getenv(HomeEnvName()), ".shipyard", "/helm_charts", "github.com/jetstack/cert-manager/ref/v1.2.0/deploy/charts/cert-manager"), h) + assert.Equal(t, filepath.Join(os.Getenv(HomeEnvName()), ".jumppad", "/helm_charts", "github.com/jetstack/cert-manager/ref-v1.2.0/deploy/charts/cert-manager"), h) } func TestShipyardReleasesReturnsPath(t *testing.T) { - r := GetReleasesFolder() + r := ReleasesFolder() - assert.Equal(t, filepath.Join(os.Getenv(HomeEnvName()), ".shipyard", "/releases"), r) + assert.Equal(t, filepath.Join(os.Getenv(HomeEnvName()), ".jumppad", "/releases"), r) } func TestIsHCLFile(t *testing.T) { @@ -304,9 +233,9 @@ func TestIsHCLFile(t *testing.T) { } func TestBlueprintLocalFolder(t *testing.T) { - dst := GetBlueprintLocalFolder("github.com/shipyard-run/blueprints//vault-k8s?ref=dfdf&foo=bah") + dst := BlueprintLocalFolder("github.com/shipyard-run/blueprints?ref=dfdf&foo=bah//vault-k8s") - assert.Equal(t, filepath.Join(JumppadHome(), "/blueprints/github.com/shipyard-run/blueprints/vault-k8s/ref/dfdf/foo/bah"), dst) + assert.Equal(t, filepath.Join(JumppadHome(), "/blueprints/github.com/shipyard-run/blueprints/ref-dfdf-foo-bah/vault-k8s"), dst) } func TestDockerHostWithDefaultReturnsCorrectValue(t *testing.T) { @@ -330,13 +259,13 @@ func TestGetLocalIPAndHostnameReturnsCorrectly(t *testing.T) { func TestHTTPProxyAddressReturnsDefaultWhenEnvNotSet(t *testing.T) { proxy := HTTPProxyAddress() - assert.Equal(t, shipyardProxyAddress, proxy) + assert.Equal(t, jumppadProxyAddress, proxy) } func TestHTTPSProxyAddressReturnsDefaultWhenEnvNotSet(t *testing.T) { proxy := HTTPSProxyAddress() - assert.Equal(t, shipyardProxyAddress, proxy) + assert.Equal(t, jumppadProxyAddress, proxy) } func TestHTTPProxyAddressReturnsEnvWhenEnvSet(t *testing.T) { @@ -354,3 +283,28 @@ func TestHTTPSProxyAddressReturnsEnvWhenEnvSet(t *testing.T) { assert.Equal(t, httpsProxy, proxy) } + +var testData = ` +{ + "checks": "test", + "children": [ + { + "checks": "test" + }, + { + "checks": "test2" + } + ] +} +` + +func TestChecksumInterface(t *testing.T) { + var data map[string]interface{} + err := json.Unmarshal([]byte(testData), &data) + require.NoError(t, err) + + c, err := ChecksumFromInterface(data) + require.NoError(t, err) + + assert.Equal(t, "h1:kpp5xuYieKQMhbtP0+Y6N+dUzx9p9pGq9+WXkgbK6fs=", c) +} diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 1450e28d..0fb29678 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -4,6 +4,7 @@ import ( "bytes" "crypto/sha256" "encoding/base64" + "encoding/json" "fmt" "io" "io/ioutil" @@ -247,6 +248,9 @@ func BlueprintFolder(blueprint string) (string, error) { return "", InvalidBlueprintURIError } + // first replace any ? + parts[1] = strings.Replace(parts[1], "?", "-", -1) + return sanitize.Path(parts[1]), nil } @@ -255,6 +259,10 @@ func BlueprintFolder(blueprint string) (string, error) { func BlueprintLocalFolder(blueprint string) string { // we might have a querystring reference such has github.com/abc/cds?ref=dfdf&dfdf // replace these separators with / + + // replace any ? with / before sanitizing + blueprint = strings.Replace(blueprint, "?", "/", -1) + blueprint = sanitize.Path(blueprint) return filepath.Join(JumppadHome(), "blueprints", blueprint) @@ -263,6 +271,9 @@ func BlueprintLocalFolder(blueprint string) string { // HelmLocalFolder returns the full storage path // for the given blueprint URI func HelmLocalFolder(chart string) string { + // replace any ? with / before sanitizing + chart = strings.Replace(chart, "?", "/", -1) + chart = sanitize.Path(chart) return filepath.Join(JumppadHome(), "helm_charts", chart) @@ -543,6 +554,19 @@ func HashString(content string) (string, error) { return "h1:" + base64.StdEncoding.EncodeToString(hf.Sum(nil)), nil } +// InterfaceChecksum returns a checksum of the given interface +// Note: the checksum is positional, should an element in a map or list change +// position then a different checksum will be returned. +func ChecksumFromInterface(i interface{}) (string, error) { + // first convert the object to json + json, err := json.Marshal(i) + if err != nil { + return "", fmt.Errorf("unable to marshal interface: %w", err) + } + + return HashString(string(json)) +} + func incIP(ip net.IP) net.IP { // allocate a new IP newIp := make(net.IP, len(ip))