Add tracing support and general improvements (#98)

This patch improves the internal configuration and service management. It adds support for distributed tracing and resolves several issues in the release pipeline and CLI.

Additionally, composable docker-compose configuration files have been added.

Several bugs have been fixed in the release management pipeline.

Signed-off-by: aeneasr <[email protected]>

.circleci/config.yml

@@ -83,6 +83,7 @@ jobs:
       - setup_remote_docker
       - run: docker login --username "$DOCKER_USERNAME" --password "$DOCKER_PASSWORD"
       - run: cp ./.releaser/LICENSE.txt ./LICENSE.txt
+      - run: go install github.com/gobuffalo/packr/packr
       - run: curl -sL https://git.io/goreleaser | bash
 
   release-docs:
@@ -95,7 +96,8 @@ jobs:
       - run: git config --global user.name "ORY Continuous Integration"
       - run: "git clone https://arekkas:[email protected]/ory/docs.git ../docs"
       - run: "cp ./docs/api.swagger.json ../docs/apis/keto.json"
-      - run: "(cd ../docs && git add -A && git commit -a -m \"Updates ORY Keto Swagger definitions\" && git push origin) || exit 0"
+      - run: ./scripts/run-configuration.sh
+      - run: "(cd ../docs && git add -A && git commit -a -m \"Updates ORY Keto Swagger and config definitions\" && git push origin) || exit 0"
 
   release-changelog:
     docker:

.gitignore

@@ -3,3 +3,4 @@ node_modules/
 coverage.txt
 vendor/
 LICENSE.txt
+*-packr.go

Dockerfile

@@ -1,31 +1,15 @@
-FROM golang:1.11-alpine
+# To compile this image manually run:
+#
+# $ packr; GO111MODULE=on GOOS=linux GOARCH=amd64 go build; docker build -t oryd/keto:latest .; rm keto; packr clean
+FROM alpine:3.9
 
-ARG git_tag
-ARG git_commit
-
-RUN apk add --no-cache git build-base
-
-WORKDIR /go/src/github.com/ory/keto
-
-RUN GO111MODULE=off go get -u github.com/gobuffalo/packr/packr
-
-ENV GO111MODULE=on
-
-ADD go.mod go.mod
-ADD go.sum go.sum
-
-RUN go mod download
-
-ADD . .
-
-RUN go mod verify
-RUN packr
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-s -X github.com/ory/keto/cmd.Version=$git_tag -X github.com/ory/keto/cmd.BuildTime=`TZ=UTC date -u '+%Y-%m-%dT%H:%M:%SZ'` -X github.com/ory/keto/cmd.GitHash=$git_commit" -a -installsuffix cgo -o keto github.com/ory/keto
+RUN apk add -U --no-cache ca-certificates
 
 FROM scratch
 
 COPY --from=0 /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
-COPY --from=0 /go/src/github.com/ory/keto/keto /usr/bin/keto
+COPY keto /usr/bin/keto
+COPY .releaser/LICENSE.txt /LICENSE.txt
 
 ENTRYPOINT ["keto"]
 

cmd/engines_acp_ory_policies_delete.go

@@ -34,7 +34,7 @@ var enginesAcpOryPoliciesDeleteCmd = &cobra.Command{
 		c := client.NewClient(cmd)
 		for _, id := range args[1:] {
 			_, err := c.Engines.DeleteOryAccessControlPolicy(engines.NewDeleteOryAccessControlPolicyParams().WithFlavor(args[0]).WithID(id))
-			cmdx.Must(err, "Unable to delete ORY Access Control Policy: %s")
+			cmdx.Must(err, "Unable to delete ORY Access Control Policy: %s", err)
 		}
 	},
 }

cmd/engines_acp_ory_policies_get.go

@@ -36,7 +36,7 @@ var enginesAcpOryPoliciesGetCmd = &cobra.Command{
 		c := client.NewClient(cmd)
 		for _, id := range args[1:] {
 			r, err := c.Engines.GetOryAccessControlPolicy(engines.NewGetOryAccessControlPolicyParams().WithFlavor(args[0]).WithID(id))
-			cmdx.Must(err, "Unable to get ORY Access Control Policy: %s")
+			cmdx.Must(err, "Unable to get ORY Access Control Policy: %s", err)
 			fmt.Println(cmdx.FormatResponse(r.Payload))
 		}
 	},

cmd/engines_acp_ory_policies_import.go

@@ -49,7 +49,7 @@ The json file(s) have to be formatted as arrays:
 				func() {
 					for _, pp := range p {
 						_, err := c.Engines.UpsertOryAccessControlPolicy(engines.NewUpsertOryAccessControlPolicyParams().WithFlavor(args[0]).WithBody(&pp))
-						cmdx.Must(err, "Unable to import ORY Access Control Policy: %s")
+						cmdx.Must(err, "Unable to import ORY Access Control Policy: %s", err)
 					}
 				},
 			)

cmd/engines_acp_ory_policies_list.go

@@ -41,8 +41,8 @@ var enginesAcpOryPoliciesListCmd = &cobra.Command{
 		r, err := c.Engines.ListOryAccessControlPolicies(
 			engines.NewListOryAccessControlPoliciesParams().WithFlavor(args[0]).WithLimit(&limit).WithOffset(&offset),
 		)
-		cmdx.Must(err, "Unable to list ORY Access Control Policies: %s")
-		fmt.Println(cmdx.FormatResponse(r))
+		cmdx.Must(err, "Unable to list ORY Access Control Policies: %s", err)
+		fmt.Println(cmdx.FormatResponse(r.Payload))
 	},
 }
 

cmd/engines_acp_ory_roles_delete.go

@@ -34,7 +34,7 @@ var deleteCmd = &cobra.Command{
 		c := client.NewClient(cmd)
 		for _, id := range args[1:] {
 			_, err := c.Engines.DeleteOryAccessControlPolicyRole(engines.NewDeleteOryAccessControlPolicyRoleParams().WithFlavor(args[0]).WithID(id))
-			cmdx.Must(err, "Unable to delete ORY Access Control Policy Role: %s")
+			cmdx.Must(err, "Unable to delete ORY Access Control Policy Role: %s", err)
 		}
 	},
 }

cmd/engines_acp_ory_roles_get.go

@@ -36,7 +36,7 @@ var getCmd = &cobra.Command{
 		c := client.NewClient(cmd)
 		for _, id := range args[1:] {
 			r, err := c.Engines.GetOryAccessControlPolicyRole(engines.NewGetOryAccessControlPolicyRoleParams().WithFlavor(args[0]).WithID(id))
-			cmdx.Must(err, "Unable to get ORY Access Control Policy Role: %s")
+			cmdx.Must(err, "Unable to get ORY Access Control Policy Role: %s", err)
 			fmt.Println(cmdx.FormatResponse(r.Payload))
 		}
 	},

cmd/engines_acp_ory_roles_import.go

@@ -49,7 +49,7 @@ The json file(s) have to be formatted as arrays:
 				func() {
 					for _, pp := range p {
 						_, err := c.Engines.UpsertOryAccessControlPolicyRole(engines.NewUpsertOryAccessControlPolicyRoleParams().WithFlavor(args[0]).WithBody(&pp))
-						cmdx.Must(err, "Unable to import ORY Access Control Policy Role: %s")
+						cmdx.Must(err, "Unable to import ORY Access Control Policy Role: %s", err)
 					}
 				},
 			)

cmd/engines_acp_ory_roles_list.go

@@ -41,8 +41,8 @@ var enginesAcpOryRolesListCmd = &cobra.Command{
 		r, err := c.Engines.ListOryAccessControlPolicyRoles(
 			engines.NewListOryAccessControlPolicyRolesParams().WithFlavor(args[0]).WithLimit(&limit).WithOffset(&offset),
 		)
-		cmdx.Must(err, "Unable to list ORY Access Control Policy Roles: %s")
-		fmt.Println(cmdx.FormatResponse(r))
+		cmdx.Must(err, "Unable to list ORY Access Control Policy Roles: %s", err)
+		fmt.Println(cmdx.FormatResponse(r.Payload))
 	},
 }
 

cmd/root.go

@@ -65,7 +65,5 @@ func init() {
 func initConfig() {
 	viper.AutomaticEnv() // read in environment variables that match
 
-	viper.SetDefault("PORT", "4466")
-
 	*logger = *logrusx.New()
 }

cmd/serve.go

@@ -15,64 +15,33 @@
 package cmd
 
 import (
-	"os"
-	"strconv"
-
 	"github.com/spf13/cobra"
 
+	"github.com/ory/x/viperx"
+
 	"github.com/ory/keto/cmd/server"
-	"github.com/ory/x/cmdx"
-	"github.com/ory/x/corsx"
 	"github.com/ory/x/logrusx"
-	"github.com/ory/x/profilex"
-	"github.com/ory/x/sqlcon"
-	"github.com/ory/x/tlsx"
 )
 
 // serveCmd represents the serve command
 var serveCmd = &cobra.Command{
 	Use:   "serve",
 	Short: "Starts the server and serves the HTTP REST API",
-	Long: cmdx.EnvVarExamplesHelpMessage("keto") + `
-
-All possible controls are listed below.
-
-CORE CONTROLS
-=============
-
-` + sqlcon.HelpMessage() + `
-
-` + logrusx.HelpMessage() + `
-
-HTTP(S) CONTROLS
-==============
-- HOST: The host to listen on. Defaults to listening on all interfaces.
-
-	Example:
-		$ export HOST=127.0.0.1
-
-- PORT: The port to listen on. Defaults to port 4466.
-
-	Example:
-		$ export PORT=4466
-
-` + tlsx.HTTPSCertificateHelpMessage() + `
-
-` + corsx.HelpMessage() + `
-
+	Long: `This command opens a network port and listens to HTTP/2 API requests.
 
-DEBUG CONTROLS
-==============
+## Configuration
 
-` + profilex.HelpMessage() + `
+ORY Keto can be configured using environment variables as well as a configuration file. For more information
+on configuration options, open the configuration documentation:
 
-`,
+>> https://github.com/ory/keto/blob/` + Version + `/docs/config.yaml <<`,
 	Run: server.RunServe(logger, Version, Commit, Date),
 }
 
 func init() {
 	RootCmd.AddCommand(serveCmd)
 
-	disableTelemetryEnv, _ := strconv.ParseBool(os.Getenv("DISABLE_TELEMETRY")) // #nosec
-	serveCmd.Flags().Bool("disable-telemetry", disableTelemetryEnv, "Disable anonymized telemetry reports - for more information please visit https://www.ory.sh/docs/ecosystem/sqa")
+	disableTelemetryEnv := viperx.GetBool(logrusx.New(), "sqa.opt_out", "DISABLE_TELEMETRY")
+	serveCmd.PersistentFlags().Bool("disable-telemetry", disableTelemetryEnv, "Disable anonymized telemetry reports - for more information please visit https://www.ory.sh/docs/ecosystem/sqa")
+	serveCmd.PersistentFlags().Bool("sqa-opt-out", disableTelemetryEnv, "Disable anonymized telemetry reports - for more information please visit https://www.ory.sh/docs/ecosystem/sqa")
 }

cmd/server/serve.go

@@ -23,11 +23,11 @@ package server
 
 import (
 	"crypto/tls"
-	"fmt"
 	"net/http"
 
-	"github.com/gobuffalo/packr"
-	"github.com/jmoiron/sqlx"
+	"github.com/ory/keto/driver"
+	"github.com/ory/x/logrusx"
+
 	"github.com/julienschmidt/httprouter"
 	negronilogrus "github.com/meatballhat/negroni-logrus"
 	"github.com/pkg/errors"
@@ -37,17 +37,13 @@ import (
 	"github.com/urfave/negroni"
 
 	"github.com/ory/graceful"
-	"github.com/ory/herodot"
-	"github.com/ory/keto/engine"
 	"github.com/ory/keto/engine/ladon"
 	"github.com/ory/x/stringslice"
 
 	// This forces go mod vendor to look for the package rego and its subpackages
 	_ "github.com/ory/keto/engine/ladon/rego"
-	"github.com/ory/keto/storage"
 	"github.com/ory/x/cmdx"
 	"github.com/ory/x/corsx"
-	"github.com/ory/x/dbal"
 	"github.com/ory/x/healthx"
 	"github.com/ory/x/metricsx"
 	"github.com/ory/x/tlsx"
@@ -56,48 +52,25 @@ import (
 // RunServe runs the Keto API HTTP server
 func RunServe(
 	logger *logrus.Logger,
-	buildVersion, buildHash string, buildTime string,
+	version, commit string, date string,
 ) func(cmd *cobra.Command, args []string) {
 	return func(cmd *cobra.Command, args []string) {
-		box := packr.NewBox("../../engine/ladon/rego")
-
-		compiler, err := engine.NewCompiler(box, logger)
-		cmdx.Must(err, "Unable to initialize compiler: %s", err)
-
-		writer := herodot.NewJSONWriter(logger)
-		//writer.ErrorEnhancer = nil
-
-		var s storage.Manager
-		checks := map[string]healthx.ReadyChecker{}
-
-		err = dbal.Connect(viper.GetString("DATABASE_URL"), logger,
-			func() error {
-				s = storage.NewMemoryManager()
-				checks["storage"] = healthx.NoopReadyChecker
-				return nil
-			},
-			func(db *sqlx.DB) error {
-				ss := storage.NewSQLManager(db)
-				checks["storage"] = db.Ping
-				s = ss
-				return nil
-			},
+		d := driver.NewDefaultDriver(
+			logrusx.New(),
+			version, commit, date,
 		)
-		if err != nil {
-			logger.WithError(err).Fatalf("DBAL was unable to connect to the database")
-			return
-		}
-
-		sh := storage.NewHandler(s, writer)
-		e := engine.NewEngine(compiler, writer)
 
 		router := httprouter.New()
-		ladon.NewEngine(s, sh, e, writer).Register(router)
-		healthx.NewHandler(writer, buildVersion, checks).SetRoutes(router, true)
+		d.Registry().LadonEngine().Register(router)
+		d.Registry().HealthHandler().SetRoutes(router, true)
 
 		n := negroni.New()
 		n.Use(negronilogrus.NewMiddlewareFromLogger(logger, "keto"))
 
+		if tracer := d.Registry().Tracer(); tracer.IsLoaded() {
+			n.Use(tracer)
+		}
+
 		metrics := metricsx.New(cmd, logger,
 			&metricsx.Options{
 				Service:       "ory-keto",
@@ -108,23 +81,27 @@ func RunServe(
 					healthx.RoutesToObserve(),
 					ladon.RoutesToObserve(),
 				),
-				BuildVersion: buildVersion,
-				BuildTime:    buildHash,
-				BuildHash:    buildTime,
+				BuildVersion: version,
+				BuildTime:    date,
+				BuildHash:    commit,
 			},
 		)
 		n.Use(metrics)
 
 		n.UseHandler(router)
-		c := corsx.Initialize(n, logger, "")
+		c := corsx.Initialize(n, logger, "serve")
 
-		addr := fmt.Sprintf("%s:%s", viper.GetString("HOST"), viper.GetString("PORT"))
 		server := graceful.WithDefaults(&http.Server{
-			Addr:    addr,
+			Addr:    d.Configuration().ListenOn(),
 			Handler: c,
 		})
 
-		cert, err := tlsx.HTTPSCertificate()
+		cert, err := tlsx.Certificate(
+			viper.GetString("serve.tls.cert.base64"),
+			viper.GetString("serve.tls.key.base64"),
+			viper.GetString("serve.tls.cert.path"),
+			viper.GetString("serve.tls.key.path"),
+		)
 		if errors.Cause(err) == tlsx.ErrNoCertificatesConfigured {
 			// do nothing
 		} else if err != nil {
@@ -133,12 +110,16 @@ func RunServe(
 			server.TLSConfig = &tls.Config{Certificates: cert}
 		}
 
+		if d.Registry().Tracer().IsLoaded() {
+			server.RegisterOnShutdown(d.Registry().Tracer().Close)
+		}
+
 		if err := graceful.Graceful(func() error {
 			if cert != nil {
-				logger.Printf("Listening on https://%s", addr)
+				logger.Printf("Listening on https://%s", d.Configuration().ListenOn())
 				return server.ListenAndServeTLS("", "")
 			}
-			logger.Printf("Listening on http://%s", addr)
+			logger.Printf("Listening on http://%s", d.Configuration().ListenOn())
 			return server.ListenAndServe()
 		}, server.Shutdown); err != nil {
 			logger.Fatalf("Unable to gracefully shutdown HTTP(s) server because %v", err)