From 5898a3a927bee15bea1e55fd59f0167c2908aa5e Mon Sep 17 00:00:00 2001 From: Umair Idris Date: Mon, 2 Mar 2020 17:26:51 +0000 Subject: [PATCH] Add better health check (#59) * add new health check * implement new health check * simplify * minor cleanup --- cdap/health.go | 60 ++++++++++++++++++++++++++ cdap/http.go | 11 ++++- cdap/resource_application.go | 2 +- cdap/resource_gcs_artifact.go | 2 +- cdap/resource_local_artifact.go | 9 +++- cdap/resource_namespace.go | 2 +- cdap/resource_namespace_preferences.go | 2 +- cdap/resource_profile.go | 2 +- 8 files changed, 83 insertions(+), 7 deletions(-) create mode 100644 cdap/health.go diff --git a/cdap/health.go b/cdap/health.go new file mode 100644 index 0000000..3b632a8 --- /dev/null +++ b/cdap/health.go @@ -0,0 +1,60 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cdap + +import ( + "errors" + "fmt" + "log" + "time" + + "github.com/hashicorp/terraform-plugin-sdk/helper/schema" +) + +func chain(fs ...schema.CreateFunc) schema.CreateFunc { + return schema.CreateFunc(func(d *schema.ResourceData, m interface{}) error { + for _, f := range fs { + if err := f(d, m); err != nil { + return err + } + } + return nil + }) +} + +var retryErrCodes = map[int]bool{502: true, 504: true} + +// TODO(umairidris): Remove this once CDF create call returns only after all services are running. +func checkHealth(_ *schema.ResourceData, m interface{}) error { + config := m.(*Config) + for i := 0; i < 50; i++ { + log.Printf("checking system artifact attempt %d", i) + exists, err := artifactExists(config, "cdap-data-pipeline", "default") + var e *httpError + switch { + case errors.As(err, &e) && retryErrCodes[e.code]: + log.Printf("checking for system artifacts got error code %v, retrying after 10 seconds", e.code) + case err != nil: + return fmt.Errorf("failed to check for aritfact existence: %v", err) + case exists: + log.Println("system artifact exists") + return nil + default: // !exists + log.Println("system artifact not yet loaded, retrying after 10 seconds") + } + time.Sleep(10 * time.Second) + } + return errors.New("system artifact failed to come up in 50 tries") +} diff --git a/cdap/http.go b/cdap/http.go index 8b3e923..75f0f17 100644 --- a/cdap/http.go +++ b/cdap/http.go @@ -23,6 +23,15 @@ import ( "strings" ) +type httpError struct { + code int + body string +} + +func (e *httpError) Error() string { + return fmt.Sprintf("%v: %v", e.code, e.body) +} + func urlJoin(base string, paths ...string) string { p := path.Join(paths...) return fmt.Sprintf("%s/%s", strings.TrimRight(base, "/"), strings.TrimLeft(p, "/")) @@ -43,7 +52,7 @@ func httpCall(client *http.Client, req *http.Request) ([]byte, error) { } if resp.StatusCode < 200 || resp.StatusCode >= 300 { - return nil, fmt.Errorf("%v: %v", resp.Status, string(b)) + return nil, &httpError{code: resp.StatusCode, body: string(b)} } return b, nil } diff --git a/cdap/resource_application.go b/cdap/resource_application.go index 1566f9a..c56dc34 100644 --- a/cdap/resource_application.go +++ b/cdap/resource_application.go @@ -26,7 +26,7 @@ import ( // https://docs.cdap.io/cdap/current/en/reference-manual/http-restful-api/lifecycle.html. func resourceApplication() *schema.Resource { return &schema.Resource{ - Create: resourceApplicationCreate, + Create: chain(checkHealth, resourceApplicationCreate), Read: resourceApplicationRead, Delete: resourceApplicationDelete, Exists: resourceApplicationExists, diff --git a/cdap/resource_gcs_artifact.go b/cdap/resource_gcs_artifact.go index d822ca4..a02df58 100644 --- a/cdap/resource_gcs_artifact.go +++ b/cdap/resource_gcs_artifact.go @@ -32,7 +32,7 @@ var bucketPathRE = regexp.MustCompile(`^gs://(.+)/(.+)$`) // store the entire JAR's contents as a string. func resourceGCSArtifact() *schema.Resource { return &schema.Resource{ - Create: resourceGCSArtifactCreate, + Create: chain(checkHealth, resourceGCSArtifactCreate), Read: resourceLocalArtifactRead, Delete: resourceLocalArtifactDelete, Exists: resourceLocalArtifactExists, diff --git a/cdap/resource_local_artifact.go b/cdap/resource_local_artifact.go index 7e0c8b4..f17b254 100644 --- a/cdap/resource_local_artifact.go +++ b/cdap/resource_local_artifact.go @@ -19,6 +19,7 @@ import ( "encoding/json" "fmt" "io/ioutil" + "log" "net/http" "strings" @@ -30,7 +31,7 @@ import ( // store the entire JAR's contents as a string. func resourceLocalArtifact() *schema.Resource { return &schema.Resource{ - Create: resourceLocalArtifactCreate, + Create: chain(checkHealth, resourceLocalArtifactCreate), Read: resourceLocalArtifactRead, Delete: resourceLocalArtifactDelete, Exists: resourceLocalArtifactExists, @@ -200,6 +201,10 @@ func resourceLocalArtifactExists(d *schema.ResourceData, m interface{}) (bool, e return false, nil } + return artifactExists(config, name, namespace) +} + +func artifactExists(config *Config, name, namespace string) (bool, error) { addr := urlJoin(config.host, "/v3/namespaces", namespace, "/artifacts") req, err := http.NewRequest(http.MethodGet, addr, nil) @@ -221,6 +226,8 @@ func resourceLocalArtifactExists(d *schema.ResourceData, m interface{}) (bool, e return false, err } + log.Printf("got artifacts: %v", artifacts) + for _, a := range artifacts { if a.Name == name { return true, nil diff --git a/cdap/resource_namespace.go b/cdap/resource_namespace.go index eb06041..7693113 100644 --- a/cdap/resource_namespace.go +++ b/cdap/resource_namespace.go @@ -24,7 +24,7 @@ import ( // https://docs.cdap.io/cdap/current/en/reference-manual/http-restful-api/namespace.html func resourceNamespace() *schema.Resource { return &schema.Resource{ - Create: resourceNamespaceCreate, + Create: chain(checkHealth, resourceNamespaceCreate), Read: resourceNamespaceRead, Delete: resourceNamespaceDelete, Exists: resourceNamespaceExists, diff --git a/cdap/resource_namespace_preferences.go b/cdap/resource_namespace_preferences.go index 18a359d..e7e8c64 100644 --- a/cdap/resource_namespace_preferences.go +++ b/cdap/resource_namespace_preferences.go @@ -25,7 +25,7 @@ import ( // https://docs.cdap.io/cdap/current/en/reference-manual/http-restful-api/preferences.html func resourceNamespacePreferences() *schema.Resource { return &schema.Resource{ - Create: resourceNamespacePreferencesCreate, + Create: chain(checkHealth, resourceNamespacePreferencesCreate), Read: resourceNamespacePreferencesRead, Delete: resourceNamespacePreferencesDelete, Exists: resourceNamespacePreferencesExist, diff --git a/cdap/resource_profile.go b/cdap/resource_profile.go index 84f5307..4e7038b 100644 --- a/cdap/resource_profile.go +++ b/cdap/resource_profile.go @@ -26,7 +26,7 @@ import ( // https://docs.cdap.io/cdap/current/en/reference-manual/http-restful-api/profile.html func resourceProfile() *schema.Resource { return &schema.Resource{ - Create: resourceProfileCreate, + Create: chain(checkHealth, resourceProfileCreate), Read: resourceProfileRead, Delete: resourceProfileDelete, Exists: resourceProfileExists,