Protos and parser for managed connectors

proto/gen/rill/runtime/v1/runtime.swagger.yaml

@@ -3953,6 +3953,10 @@ definitions:
         type: array
         items:
           type: string
+      provision:
+        type: boolean
+      provisionArgs:
+        type: object
       propertiesFromVariables:
         type: object
         additionalProperties:

proto/rill/runtime/v1/resources.proto

@@ -726,10 +726,17 @@ message CharLocation {
   uint32 line = 1;
 }
 
+message ConnectorV2 {
+  ConnectorSpec spec = 1;
+  ConnectorState state = 2;
+}
+
 message ConnectorSpec {
   string driver = 1;
   map<string, string> properties = 2;
   repeated string templated_properties = 4;
+  bool provision = 5;
+  google.protobuf.Struct provision_args = 6;
   // DEPRECATED: properties_from_variables stores properties whose value is a variable.
   // NOTE : properties_from_variables and properties both should be used to get all properties.
   map<string, string> properties_from_variables = 3;
@@ -738,8 +745,3 @@ message ConnectorSpec {
 message ConnectorState {
   string spec_hash = 1;
 }
-
-message ConnectorV2 {
-  ConnectorSpec spec = 1;
-  ConnectorState state = 2;
-}

runtime/compilers/rillv1/parse_connector.go

@@ -2,13 +2,17 @@ package rillv1
 
 import (
 	"fmt"
+
+	"google.golang.org/protobuf/types/known/structpb"
+	"gopkg.in/yaml.v3"
 )
 
 // ConnectorYAML is the raw structure of a Connector resource defined in YAML (does not include common fields)
 type ConnectorYAML struct {
 	commonYAML `yaml:",inline" mapstructure:",squash"` // Only to avoid loading common fields into Properties
 	// Driver name
 	Driver   string            `yaml:"driver"`
+	Managed  yaml.Node         `yaml:"managed"` // Boolean or map of properties
 	Defaults map[string]string `yaml:",inline" mapstructure:",remain"`
 }
 
@@ -21,6 +25,33 @@ func (p *Parser) parseConnector(node *Node) error {
 		return err
 	}
 
+	// "Managed" indicates that we should automatically provision the connector
+	var provision bool
+	var provisionArgsPB *structpb.Struct
+	if !tmp.Managed.IsZero() {
+		switch tmp.Managed.Kind {
+		case yaml.ScalarNode:
+			err := tmp.Managed.Decode(&provision)
+			if err != nil {
+				return fmt.Errorf("failed to decode 'managed': %w", err)
+			}
+		case yaml.MappingNode:
+			provision = true
+			var provisionArgs map[string]any
+			err := tmp.Managed.Decode(&provisionArgs)
+			if err != nil {
+				return fmt.Errorf("failed to decode 'managed': %w", err)
+			}
+			provisionArgsPB, err = structpb.NewStruct(provisionArgs)
+			if err != nil {
+				return fmt.Errorf("failed to convert provision args to proto: %w", err)
+			}
+		default:
+			return fmt.Errorf("invalid type for 'managed': expected boolean or map of args")
+		}
+	}
+
+	// Find out if any properties are templated
 	templatedProps, err := analyzeTemplatedProperties(tmp.Defaults)
 	if err != nil {
 		return fmt.Errorf("failed to analyze templated properties: %w", err)
@@ -36,6 +67,8 @@ func (p *Parser) parseConnector(node *Node) error {
 	r.ConnectorSpec.Driver = tmp.Driver
 	r.ConnectorSpec.Properties = tmp.Defaults
 	r.ConnectorSpec.TemplatedProperties = templatedProps
+	r.ConnectorSpec.Provision = provision
+	r.ConnectorSpec.ProvisionArgs = provisionArgsPB
 	return nil
 }
 

runtime/compilers/rillv1/parser_test.go

@@ -2003,6 +2003,79 @@ refresh:
 	requireResourcesAndErrors(t, p, []*Resource{m1, m2}, nil)
 }
 
+func TestConnector(t *testing.T) {
+	ctx := context.Background()
+	repo := makeRepo(t, map[string]string{`rill.yaml`: ``})
+
+	putRepo(t, repo, map[string]string{
+		`connectors/clickhouse.yaml`: `
+type: connector
+driver: clickhouse
+`})
+	r := &Resource{
+		Name:  ResourceName{Kind: ResourceKindConnector, Name: "clickhouse"},
+		Paths: []string{"/connectors/clickhouse.yaml"},
+		ConnectorSpec: &runtimev1.ConnectorSpec{
+			Driver: "clickhouse",
+		},
+	}
+	p, err := Parse(ctx, repo, "", "", "duckdb")
+	require.NoError(t, err)
+	requireResourcesAndErrors(t, p, []*Resource{r}, nil)
+
+	putRepo(t, repo, map[string]string{
+		`connectors/clickhouse.yaml`: `
+type: connector
+driver: clickhouse
+managed: true
+`})
+	r = &Resource{
+		Name:  ResourceName{Kind: ResourceKindConnector, Name: "clickhouse"},
+		Paths: []string{"/connectors/clickhouse.yaml"},
+		ConnectorSpec: &runtimev1.ConnectorSpec{
+			Driver:    "clickhouse",
+			Provision: true,
+		},
+	}
+	p, err = Parse(ctx, repo, "", "", "duckdb")
+	require.NoError(t, err)
+	requireResourcesAndErrors(t, p, []*Resource{r}, nil)
+
+	putRepo(t, repo, map[string]string{
+		`connectors/clickhouse.yaml`: `
+type: connector
+driver: clickhouse
+managed:
+  hello: world
+time_zone: America/Los_Angeles
+`})
+	r = &Resource{
+		Name:  ResourceName{Kind: ResourceKindConnector, Name: "clickhouse"},
+		Paths: []string{"/connectors/clickhouse.yaml"},
+		ConnectorSpec: &runtimev1.ConnectorSpec{
+			Driver:        "clickhouse",
+			Properties:    map[string]string{"time_zone": "America/Los_Angeles"},
+			Provision:     true,
+			ProvisionArgs: must(structpb.NewStruct(map[string]any{"hello": "world"})),
+		},
+	}
+	p, err = Parse(ctx, repo, "", "", "duckdb")
+	require.NoError(t, err)
+	requireResourcesAndErrors(t, p, []*Resource{r}, nil)
+
+	putRepo(t, repo, map[string]string{
+		`connectors/clickhouse.yaml`: `
+type: connector
+driver: clickhouse
+managed: 10
+`})
+	p, err = Parse(ctx, repo, "", "", "duckdb")
+	require.NoError(t, err)
+	requireResourcesAndErrors(t, p, nil, []*runtimev1.ParseError{
+		{Message: "failed to decode 'managed'", FilePath: "/connectors/clickhouse.yaml"},
+	})
+}
+
 func requireResourcesAndErrors(t testing.TB, p *Parser, wantResources []*Resource, wantErrors []*runtimev1.ParseError) {
 	// Check errors
 	// NOTE: Assumes there's at most one parse error per file path

web-common/src/proto/gen/rill/runtime/v1/resources_pb.ts

@@ -4871,6 +4871,49 @@ export class CharLocation extends Message<CharLocation> {
   }
 }
 
+/**
+ * @generated from message rill.runtime.v1.ConnectorV2
+ */
+export class ConnectorV2 extends Message<ConnectorV2> {
+  /**
+   * @generated from field: rill.runtime.v1.ConnectorSpec spec = 1;
+   */
+  spec?: ConnectorSpec;
+
+  /**
+   * @generated from field: rill.runtime.v1.ConnectorState state = 2;
+   */
+  state?: ConnectorState;
+
+  constructor(data?: PartialMessage<ConnectorV2>) {
+    super();
+    proto3.util.initPartial(data, this);
+  }
+
+  static readonly runtime: typeof proto3 = proto3;
+  static readonly typeName = "rill.runtime.v1.ConnectorV2";
+  static readonly fields: FieldList = proto3.util.newFieldList(() => [
+    { no: 1, name: "spec", kind: "message", T: ConnectorSpec },
+    { no: 2, name: "state", kind: "message", T: ConnectorState },
+  ]);
+
+  static fromBinary(bytes: Uint8Array, options?: Partial<BinaryReadOptions>): ConnectorV2 {
+    return new ConnectorV2().fromBinary(bytes, options);
+  }
+
+  static fromJson(jsonValue: JsonValue, options?: Partial<JsonReadOptions>): ConnectorV2 {
+    return new ConnectorV2().fromJson(jsonValue, options);
+  }
+
+  static fromJsonString(jsonString: string, options?: Partial<JsonReadOptions>): ConnectorV2 {
+    return new ConnectorV2().fromJsonString(jsonString, options);
+  }
+
+  static equals(a: ConnectorV2 | PlainMessage<ConnectorV2> | undefined, b: ConnectorV2 | PlainMessage<ConnectorV2> | undefined): boolean {
+    return proto3.util.equals(ConnectorV2, a, b);
+  }
+}
+
 /**
  * @generated from message rill.runtime.v1.ConnectorSpec
  */
@@ -4890,6 +4933,16 @@ export class ConnectorSpec extends Message<ConnectorSpec> {
    */
   templatedProperties: string[] = [];
 
+  /**
+   * @generated from field: bool provision = 5;
+   */
+  provision = false;
+
+  /**
+   * @generated from field: google.protobuf.Struct provision_args = 6;
+   */
+  provisionArgs?: Struct;
+
   /**
    * DEPRECATED: properties_from_variables stores properties whose value is a variable.
    * NOTE : properties_from_variables and properties both should be used to get all properties.
@@ -4909,6 +4962,8 @@ export class ConnectorSpec extends Message<ConnectorSpec> {
     { no: 1, name: "driver", kind: "scalar", T: 9 /* ScalarType.STRING */ },
     { no: 2, name: "properties", kind: "map", K: 9 /* ScalarType.STRING */, V: {kind: "scalar", T: 9 /* ScalarType.STRING */} },
     { no: 4, name: "templated_properties", kind: "scalar", T: 9 /* ScalarType.STRING */, repeated: true },
+    { no: 5, name: "provision", kind: "scalar", T: 8 /* ScalarType.BOOL */ },
+    { no: 6, name: "provision_args", kind: "message", T: Struct },
     { no: 3, name: "properties_from_variables", kind: "map", K: 9 /* ScalarType.STRING */, V: {kind: "scalar", T: 9 /* ScalarType.STRING */} },
   ]);
 
@@ -4966,46 +5021,3 @@ export class ConnectorState extends Message<ConnectorState> {
   }
 }
 
-/**
- * @generated from message rill.runtime.v1.ConnectorV2
- */
-export class ConnectorV2 extends Message<ConnectorV2> {
-  /**
-   * @generated from field: rill.runtime.v1.ConnectorSpec spec = 1;
-   */
-  spec?: ConnectorSpec;
-
-  /**
-   * @generated from field: rill.runtime.v1.ConnectorState state = 2;
-   */
-  state?: ConnectorState;
-
-  constructor(data?: PartialMessage<ConnectorV2>) {
-    super();
-    proto3.util.initPartial(data, this);
-  }
-
-  static readonly runtime: typeof proto3 = proto3;
-  static readonly typeName = "rill.runtime.v1.ConnectorV2";
-  static readonly fields: FieldList = proto3.util.newFieldList(() => [
-    { no: 1, name: "spec", kind: "message", T: ConnectorSpec },
-    { no: 2, name: "state", kind: "message", T: ConnectorState },
-  ]);
-
-  static fromBinary(bytes: Uint8Array, options?: Partial<BinaryReadOptions>): ConnectorV2 {
-    return new ConnectorV2().fromBinary(bytes, options);
-  }
-
-  static fromJson(jsonValue: JsonValue, options?: Partial<JsonReadOptions>): ConnectorV2 {
-    return new ConnectorV2().fromJson(jsonValue, options);
-  }
-
-  static fromJsonString(jsonString: string, options?: Partial<JsonReadOptions>): ConnectorV2 {
-    return new ConnectorV2().fromJsonString(jsonString, options);
-  }
-
-  static equals(a: ConnectorV2 | PlainMessage<ConnectorV2> | undefined, b: ConnectorV2 | PlainMessage<ConnectorV2> | undefined): boolean {
-    return proto3.util.equals(ConnectorV2, a, b);
-  }
-}
-

web-common/src/runtime-client/gen/index.schemas.ts

@@ -1353,10 +1353,6 @@ export interface V1MetricsViewSort {
   ascending?: boolean;
 }
 
-export interface V1MetricsViewSearchResponse {
-  results?: MetricsViewSearchResponseSearchResult[];
-}
-
 export interface V1MetricsViewSchemaResponse {
   schema?: V1StructType;
 }
@@ -1976,12 +1972,16 @@ NOTE : properties_from_variables and properties both should be used to get all p
  */
 export type V1ConnectorSpecPropertiesFromVariables = { [key: string]: string };
 
+export type V1ConnectorSpecProvisionArgs = { [key: string]: any };
+
 export type V1ConnectorSpecProperties = { [key: string]: string };
 
 export interface V1ConnectorSpec {
   driver?: string;
   properties?: V1ConnectorSpecProperties;
   templatedProperties?: string[];
+  provision?: boolean;
+  provisionArgs?: V1ConnectorSpecProvisionArgs;
   /** DEPRECATED: properties_from_variables stores properties whose value is a variable.
 NOTE : properties_from_variables and properties both should be used to get all properties. */
   propertiesFromVariables?: V1ConnectorSpecPropertiesFromVariables;
@@ -2624,6 +2624,10 @@ export interface MetricsViewSearchResponseSearchResult {
   value?: unknown;
 }
 
+export interface V1MetricsViewSearchResponse {
+  results?: MetricsViewSearchResponseSearchResult[];
+}
+
 export interface MetricsViewFilterCond {
   name?: string;
   in?: unknown[];