Add better health check (#59)

* add new health check

* implement new health check

* simplify

* minor cleanup

cdap/health.go

@@ -0,0 +1,60 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cdap
+
+import (
+	"errors"
+	"fmt"
+	"log"
+	"time"
+
+	"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
+)
+
+func chain(fs ...schema.CreateFunc) schema.CreateFunc {
+	return schema.CreateFunc(func(d *schema.ResourceData, m interface{}) error {
+		for _, f := range fs {
+			if err := f(d, m); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+}
+
+var retryErrCodes = map[int]bool{502: true, 504: true}
+
+// TODO(umairidris): Remove this once CDF create call returns only after all services are running.
+func checkHealth(_ *schema.ResourceData, m interface{}) error {
+	config := m.(*Config)
+	for i := 0; i < 50; i++ {
+		log.Printf("checking system artifact attempt %d", i)
+		exists, err := artifactExists(config, "cdap-data-pipeline", "default")
+		var e *httpError
+		switch {
+		case errors.As(err, &e) && retryErrCodes[e.code]:
+			log.Printf("checking for system artifacts got error code %v, retrying after 10 seconds", e.code)
+		case err != nil:
+			return fmt.Errorf("failed to check for aritfact existence: %v", err)
+		case exists:
+			log.Println("system artifact exists")
+			return nil
+		default: // !exists
+			log.Println("system artifact not yet loaded, retrying after 10 seconds")
+		}
+		time.Sleep(10 * time.Second)
+	}
+	return errors.New("system artifact failed to come up in 50 tries")
+}

cdap/http.go

@@ -23,6 +23,15 @@ import (
 	"strings"
 )
 
+type httpError struct {
+	code int
+	body string
+}
+
+func (e *httpError) Error() string {
+	return fmt.Sprintf("%v: %v", e.code, e.body)
+}
+
 func urlJoin(base string, paths ...string) string {
 	p := path.Join(paths...)
 	return fmt.Sprintf("%s/%s", strings.TrimRight(base, "/"), strings.TrimLeft(p, "/"))
@@ -43,7 +52,7 @@ func httpCall(client *http.Client, req *http.Request) ([]byte, error) {
 	}
 
 	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		return nil, fmt.Errorf("%v: %v", resp.Status, string(b))
+		return nil, &httpError{code: resp.StatusCode, body: string(b)}
 	}
 	return b, nil
 }

cdap/resource_application.go

@@ -26,7 +26,7 @@ import (
 // https://docs.cdap.io/cdap/current/en/reference-manual/http-restful-api/lifecycle.html.
 func resourceApplication() *schema.Resource {
 	return &schema.Resource{
-		Create: resourceApplicationCreate,
+		Create: chain(checkHealth, resourceApplicationCreate),
 		Read:   resourceApplicationRead,
 		Delete: resourceApplicationDelete,
 		Exists: resourceApplicationExists,

cdap/resource_gcs_artifact.go

@@ -32,7 +32,7 @@ var bucketPathRE = regexp.MustCompile(`^gs://(.+)/(.+)$`)
 // store the entire JAR's contents as a string.
 func resourceGCSArtifact() *schema.Resource {
 	return &schema.Resource{
-		Create: resourceGCSArtifactCreate,
+		Create: chain(checkHealth, resourceGCSArtifactCreate),
 		Read:   resourceLocalArtifactRead,
 		Delete: resourceLocalArtifactDelete,
 		Exists: resourceLocalArtifactExists,

cdap/resource_local_artifact.go

@@ -19,6 +19,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"log"
 	"net/http"
 	"strings"
 
@@ -30,7 +31,7 @@ import (
 // store the entire JAR's contents as a string.
 func resourceLocalArtifact() *schema.Resource {
 	return &schema.Resource{
-		Create: resourceLocalArtifactCreate,
+		Create: chain(checkHealth, resourceLocalArtifactCreate),
 		Read:   resourceLocalArtifactRead,
 		Delete: resourceLocalArtifactDelete,
 		Exists: resourceLocalArtifactExists,
@@ -200,6 +201,10 @@ func resourceLocalArtifactExists(d *schema.ResourceData, m interface{}) (bool, e
 		return false, nil
 	}
 
+	return artifactExists(config, name, namespace)
+}
+
+func artifactExists(config *Config, name, namespace string) (bool, error) {
 	addr := urlJoin(config.host, "/v3/namespaces", namespace, "/artifacts")
 
 	req, err := http.NewRequest(http.MethodGet, addr, nil)
@@ -221,6 +226,8 @@ func resourceLocalArtifactExists(d *schema.ResourceData, m interface{}) (bool, e
 		return false, err
 	}
 
+	log.Printf("got artifacts: %v", artifacts)
+
 	for _, a := range artifacts {
 		if a.Name == name {
 			return true, nil

cdap/resource_namespace.go

@@ -24,7 +24,7 @@ import (
 // https://docs.cdap.io/cdap/current/en/reference-manual/http-restful-api/namespace.html
 func resourceNamespace() *schema.Resource {
 	return &schema.Resource{
-		Create: resourceNamespaceCreate,
+		Create: chain(checkHealth, resourceNamespaceCreate),
 		Read:   resourceNamespaceRead,
 		Delete: resourceNamespaceDelete,
 		Exists: resourceNamespaceExists,

cdap/resource_namespace_preferences.go

@@ -25,7 +25,7 @@ import (
 // https://docs.cdap.io/cdap/current/en/reference-manual/http-restful-api/preferences.html
 func resourceNamespacePreferences() *schema.Resource {
 	return &schema.Resource{
-		Create: resourceNamespacePreferencesCreate,
+		Create: chain(checkHealth, resourceNamespacePreferencesCreate),
 		Read:   resourceNamespacePreferencesRead,
 		Delete: resourceNamespacePreferencesDelete,
 		Exists: resourceNamespacePreferencesExist,

cdap/resource_profile.go

@@ -26,7 +26,7 @@ import (
 // https://docs.cdap.io/cdap/current/en/reference-manual/http-restful-api/profile.html
 func resourceProfile() *schema.Resource {
 	return &schema.Resource{
-		Create: resourceProfileCreate,
+		Create: chain(checkHealth, resourceProfileCreate),
 		Read:   resourceProfileRead,
 		Delete: resourceProfileDelete,
 		Exists: resourceProfileExists,