From 5e03dd0f2d31ff783363f90beff60a5aa9d71f11 Mon Sep 17 00:00:00 2001 From: Murad Biashimov Date: Mon, 2 Sep 2024 13:03:19 +0200 Subject: [PATCH] feat: support spec patching --- generator/main.go | 18 +-- generator/models.go | 48 ++++---- generator/patching.go | 70 +++++++++++ handler/service/service.go | 80 +++++++++++- openapi_patch.yaml | 246 +++++++++++++++++++++++++++++++++++++ 5 files changed, 424 insertions(+), 38 deletions(-) create mode 100644 generator/patching.go create mode 100644 openapi_patch.yaml diff --git a/generator/main.go b/generator/main.go index 15bbe50..785c621 100644 --- a/generator/main.go +++ b/generator/main.go @@ -30,12 +30,13 @@ const ( ) type envConfig struct { - Module string `envconfig:"MODULE" default:"github.com/aiven/go-client-codegen"` - Package string `envconfig:"PACKAGE" default:"aiven"` - HandlerDir string `envconfig:"HANDLER_DIR" default:"handler"` - ConfigFile string `envconfig:"CONFIG_FILE" default:"config.yaml"` - ClientFile string `envconfig:"CLIENT_FILE" default:"client_generated.go"` - OpenAPIFile string `envconfig:"OPENAPI_FILE" default:"openapi.json"` + Module string `envconfig:"MODULE" default:"github.com/aiven/go-client-codegen"` + Package string `envconfig:"PACKAGE" default:"aiven"` + HandlerDir string `envconfig:"HANDLER_DIR" default:"handler"` + ConfigFile string `envconfig:"CONFIG_FILE" default:"config.yaml"` + ClientFile string `envconfig:"CLIENT_FILE" default:"client_generated.go"` + OpenAPIFile string `envconfig:"OPENAPI_FILE" default:"openapi.json"` + OpenAPIPatchFile string `envconfig:"OPENAPI_PATCH_FILE" default:"openapi_patch.yaml"` } var ( @@ -81,20 +82,19 @@ func exec() error { return err } - docBytes, err := os.ReadFile(cfg.OpenAPIFile) + // Reads OpenAPI file and applies a patch + docBytes, err := readOpenAPIPatched(cfg.OpenAPIFile, cfg.OpenAPIPatchFile) if err != nil { return err } doc := new(Doc) - err = json.Unmarshal(docBytes, doc) if err != nil { return err } pkgs := make(map[string][]*Path) - for path := range doc.Paths { v := doc.Paths[path] for meth, p := range v { diff --git a/generator/models.go b/generator/models.go index 4849b10..8c94d27 100644 --- a/generator/models.go +++ b/generator/models.go @@ -134,21 +134,22 @@ const ( // Schema represents a parsed OpenAPI schema. type Schema struct { - Type SchemaType `json:"type"` - Properties map[string]*Schema `json:"properties"` - Items *Schema `json:"items"` - RequiredProps []string `json:"required"` - Enum []any `json:"enum"` - Default any `json:"default"` - MinItems int `json:"minItems"` - Ref string `json:"$ref"` - Description string `json:"description"` - CamelName string `json:"for-hash-only!"` - required bool - name string - propertyNames []string - parent *Schema - in, out bool // Request or Response DTO + Type SchemaType `json:"type"` + Properties map[string]*Schema `json:"properties"` + AdditionalProperties *Schema `json:"additionalProperties"` + Items *Schema `json:"items"` + RequiredProps []string `json:"required"` + Enum []any `json:"enum"` + Default any `json:"default"` + MinItems int `json:"minItems"` + Ref string `json:"$ref"` + Description string `json:"description"` + CamelName string `json:"for-hash-only!"` + required bool + name string + propertyNames []string + parent *Schema + in, out bool // Request or Response DTO } //nolint:funlen,gocognit,gocyclo // It is easy to maintain and read, we don't need to split it @@ -230,12 +231,6 @@ func (s *Schema) init(doc *Doc, scope map[string]*Schema, name string) { } } - // fixme: on the backend - if s.name == "topics.blacklist" && s.parent.Type != SchemaTypeArray { - s.Type = SchemaTypeArray - s.Items = &Schema{Type: SchemaTypeString} - } - if s.Type == SchemaTypeString { parts := strings.Split(s.name, "_") suffix := parts[len(parts)-1] @@ -410,7 +405,11 @@ func getType(s *Schema) *jen.Statement { a = jen.Op("*").Map(jen.String()) } - if isMapString(s) { + if s.AdditionalProperties != nil { + s.AdditionalProperties.required = true + return a.Add(getType(s.AdditionalProperties)) + } else if s.name == "tags" { + // tags are everywhere in the schema, better not to use the patch return a.String() } else { return a.Any() @@ -429,11 +428,6 @@ func mustMarshal(s any) string { return string(b) } -// isMapString for hacking schemaless maps -func isMapString(s *Schema) bool { - return s.name == "tags" -} - func lowerFirst(s string) string { return strings.ToLower(s[:1]) + s[1:] } diff --git a/generator/patching.go b/generator/patching.go new file mode 100644 index 0000000..00bede6 --- /dev/null +++ b/generator/patching.go @@ -0,0 +1,70 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + + "gopkg.in/yaml.v3" +) + +func patchDict[T map[string]any](dest, patch T) error { + for k, v := range patch { + result, ok := dest[k] + switch { + case !ok: + result = v + case reflect.TypeOf(result) != reflect.TypeOf(v): + return fmt.Errorf("type missmatch for key %s", k) + case reflect.TypeOf(result).Kind() == reflect.Map: + err := patchDict(result.(T), v.(T)) //nolint:forcetypeassert + if err != nil { + return err + } + default: + result = v + } + + dest[k] = result + } + return nil +} + +// readOpenAPIPatched Reads OpenAPI file (JSON) and applies the patch (YAML) +func readOpenAPIPatched(oaFile, patchFile string) ([]byte, error) { + dest, err := os.ReadFile(filepath.Clean(oaFile)) + if err != nil { + return nil, err + } + + patch, err := os.ReadFile(filepath.Clean(patchFile)) + if errors.Is(err, os.ErrExist) { + // No patch found, exits + return dest, nil + } + + if err != nil { + return nil, err + } + + var d, p map[string]any + err = json.Unmarshal(dest, &d) + if err != nil { + return nil, err + } + + err = yaml.Unmarshal(patch, &p) + if err != nil { + return nil, err + } + + err = patchDict(d, p) + if err != nil { + return nil, err + } + + return json.Marshal(&d) +} diff --git a/handler/service/service.go b/handler/service/service.go index 71ffa7d..fadee54 100644 --- a/handler/service/service.go +++ b/handler/service/service.go @@ -641,6 +641,66 @@ type ComponentOut struct { Ssl *bool `json:"ssl,omitempty"` // Whether the endpoint is encrypted or accepts plaintext. By default endpoints are always encrypted andthis property is only included for service components that may disable encryption. Usage UsageType `json:"usage"` // DNS usage name } + +// ConnectionInfoOut Service-specific connection information properties +type ConnectionInfoOut struct { + AggregatorHttpUri *string `json:"aggregator_http_uri,omitempty"` + Cassandra []string `json:"cassandra,omitempty"` + Clickhouse []string `json:"clickhouse,omitempty"` + Direct []string `json:"direct,omitempty"` + Elasticsearch []string `json:"elasticsearch,omitempty"` + ElasticsearchPassword *string `json:"elasticsearch_password,omitempty"` + ElasticsearchUsername *string `json:"elasticsearch_username,omitempty"` + Flink []string `json:"flink,omitempty"` + Grafana []string `json:"grafana,omitempty"` + HttpClusterUri *string `json:"http_cluster_uri,omitempty"` + HttpNodeUri *string `json:"http_node_uri,omitempty"` + Influxdb []string `json:"influxdb,omitempty"` + InfluxdbDbname *string `json:"influxdb_dbname,omitempty"` + InfluxdbPassword *string `json:"influxdb_password,omitempty"` + InfluxdbUri *string `json:"influxdb_uri,omitempty"` + InfluxdbUsername *string `json:"influxdb_username,omitempty"` + Kafka []string `json:"kafka,omitempty"` + KafkaAccessCert *string `json:"kafka_access_cert,omitempty"` + KafkaAccessKey *string `json:"kafka_access_key,omitempty"` + KafkaConnectUri *string `json:"kafka_connect_uri,omitempty"` + KafkaRestUri *string `json:"kafka_rest_uri,omitempty"` + KibanaUri *string `json:"kibana_uri,omitempty"` + Letsencrypt *bool `json:"letsencrypt,omitempty"` + M3Aggregator []string `json:"m3aggregator,omitempty"` + M3Db []string `json:"m3db,omitempty"` + Mysql []string `json:"mysql,omitempty"` + MysqlParams []MysqlParamOut `json:"mysql_params,omitempty"` + MysqlReplicaUri *string `json:"mysql_replica_uri,omitempty"` + MysqlStandby []string `json:"mysql_standby,omitempty"` + Opensearch []string `json:"opensearch,omitempty"` + OpensearchDashboardsUri *string `json:"opensearch_dashboards_uri,omitempty"` + OpensearchPassword *string `json:"opensearch_password,omitempty"` + OpensearchUsername *string `json:"opensearch_username,omitempty"` + Pg []string `json:"pg,omitempty"` + PgBouncer *string `json:"pg_bouncer,omitempty"` + PgParams []PgParamOut `json:"pg_params,omitempty"` + PgReplicaUri *string `json:"pg_replica_uri,omitempty"` + PgStandby []string `json:"pg_standby,omitempty"` + PgSyncing []string `json:"pg_syncing,omitempty"` + PrometheusRemoteReadUri *string `json:"prometheus_remote_read_uri,omitempty"` + PrometheusRemoteWriteUri *string `json:"prometheus_remote_write_uri,omitempty"` + QueryFrontendUri *string `json:"query_frontend_uri,omitempty"` + QueryUri *string `json:"query_uri,omitempty"` + ReceiverIngestingRemoteWriteUri *string `json:"receiver_ingesting_remote_write_uri,omitempty"` + ReceiverRemoteWriteUri *string `json:"receiver_remote_write_uri,omitempty"` + Redis []string `json:"redis,omitempty"` + RedisPassword *string `json:"redis_password,omitempty"` + RedisReplicaUri *string `json:"redis_replica_uri,omitempty"` + RedisSlave []string `json:"redis_slave,omitempty"` + SchemaRegistryUri *string `json:"schema_registry_uri,omitempty"` + StoreUri *string `json:"store_uri,omitempty"` + Thanos []string `json:"thanos,omitempty"` + Valkey []string `json:"valkey,omitempty"` + ValkeyPassword *string `json:"valkey_password,omitempty"` + ValkeyReplicaUri *string `json:"valkey_replica_uri,omitempty"` + ValkeySlave []string `json:"valkey_slave,omitempty"` +} type ConnectionPoolOut struct { ConnectionUri string `json:"connection_uri"` // Connection URI for the DB pool Database string `json:"database"` // Database name @@ -1057,6 +1117,14 @@ type MysqlOut struct { ServicePlans []ServicePlanOut `json:"service_plans"` // List of plans available for this type of service UserConfigSchema map[string]any `json:"user_config_schema"` // JSON-Schema for the 'user_config' properties } +type MysqlParamOut struct { + Dbname string `json:"dbname"` + Host string `json:"host"` + Password string `json:"password"` + Port string `json:"port"` + SslMode string `json:"ssl-mode"` + User string `json:"user"` +} type NodeStateOut struct { Name string `json:"name"` // Name of the service node ProgressUpdates []ProgressUpdateOut `json:"progress_updates,omitempty"` // Extra information regarding the progress for current state @@ -1133,6 +1201,14 @@ type PgOut struct { ServicePlans []ServicePlanOut `json:"service_plans"` // List of plans available for this type of service UserConfigSchema map[string]any `json:"user_config_schema"` // JSON-Schema for the 'user_config' properties } +type PgParamOut struct { + Dbname string `json:"dbname"` + Host string `json:"host"` + Password string `json:"password"` + Port string `json:"port"` + Sslmode string `json:"sslmode"` + User string `json:"user"` +} type PhaseType string const ( @@ -1376,7 +1452,7 @@ type ServiceGetOut struct { CloudDescription *string `json:"cloud_description,omitempty"` // Cloud provider and location CloudName string `json:"cloud_name"` // Target cloud Components []ComponentOut `json:"components,omitempty"` // Service component information objects - ConnectionInfo map[string]any `json:"connection_info,omitempty"` // Service-specific connection information properties + ConnectionInfo *ConnectionInfoOut `json:"connection_info,omitempty"` // Service-specific connection information properties ConnectionPools []ConnectionPoolOut `json:"connection_pools,omitempty"` // PostgreSQL PGBouncer connection pools CreateTime time.Time `json:"create_time"` // Service creation timestamp (ISO 8601) Databases []string `json:"databases,omitempty"` // List of service's user database names @@ -1398,7 +1474,7 @@ type ServiceGetOut struct { ServiceType string `json:"service_type"` // Service type code ServiceTypeDescription *string `json:"service_type_description,omitempty"` // Single line description of the service ServiceUri string `json:"service_uri"` // URI for connecting to the service (may be null) - ServiceUriParams map[string]any `json:"service_uri_params,omitempty"` // service_uri parameterized into key-value pairs + ServiceUriParams map[string]string `json:"service_uri_params,omitempty"` // service_uri parameterized into key-value pairs State ServiceStateType `json:"state"` // State of the service Tags map[string]string `json:"tags,omitempty"` // Set of resource tags TechEmails []TechEmailOut `json:"tech_emails,omitempty"` // List of service technical email addresses diff --git a/openapi_patch.yaml b/openapi_patch.yaml new file mode 100644 index 0000000..d6986c3 --- /dev/null +++ b/openapi_patch.yaml @@ -0,0 +1,246 @@ +components: + schemas: + ServiceKafkaMirrorMakerCreateReplicationFlowRequestBody: + properties: + topics.blacklist: + type: array + items: + type: string + ServiceKafkaMirrorMakerGetReplicationFlowResponse: + properties: + replication_flow: + properties: + topics.blacklist: + type: array + items: + type: string + ServiceKafkaMirrorMakerGetReplicationFlowsResponse: + properties: + replication_flows: + items: + properties: + topics.blacklist: + type: array + items: + type: string + ServiceKafkaMirrorMakerPatchReplicationFlowRequestBody: + properties: + topics.blacklist: + type: array + items: + type: string + ServiceKafkaMirrorMakerPatchReplicationFlowResponse: + properties: + replication_flow: + properties: + topics.blacklist: + type: array + items: + type: string + ServiceGetResponse: + properties: + service: + properties: + service_uri_params: + additionalProperties: + type: string + connection_info: + properties: + aggregator_http_uri: + type: string + cassandra: + items: + type: string + type: array + clickhouse: + items: + type: string + type: array + direct: + items: + type: string + type: array + elasticsearch: + items: + type: string + type: array + elasticsearch_password: + type: string + elasticsearch_username: + type: string + flink: + items: + type: string + type: array + grafana: + items: + type: string + type: array + http_cluster_uri: + type: string + http_node_uri: + type: string + influxdb: + items: + type: string + type: array + influxdb_dbname: + type: string + influxdb_password: + type: string + influxdb_uri: + type: string + influxdb_username: + type: string + kafka: + items: + type: string + type: array + kafka_access_cert: + type: string + kafka_access_key: + type: string + kafka_connect_uri: + type: string + kafka_rest_uri: + type: string + kibana_uri: + type: string + letsencrypt: + type: boolean + m3aggregator: + items: + type: string + type: array + m3db: + items: + type: string + type: array + mysql: + items: + type: string + type: array + mysql_params: + type: array + items: + properties: + dbname: + type: string + host: + type: string + password: + type: string + port: + type: string + ssl-mode: + type: string + user: + type: string + type: object + required: + - dbname + - host + - password + - port + - ssl-mode + - user + mysql_replica_uri: + type: string + mysql_standby: + items: + type: string + type: array + opensearch: + items: + type: string + type: array + opensearch_dashboards_uri: + type: string + opensearch_password: + type: string + opensearch_username: + type: string + pg: + items: + type: string + type: array + pg_bouncer: + type: string + pg_params: + type: array + items: + properties: + dbname: + type: string + host: + type: string + password: + type: string + port: + type: string + sslmode: + type: string + user: + type: string + type: object + required: + - dbname + - host + - password + - port + - sslmode + - user + pg_replica_uri: + type: string + pg_standby: + items: + type: string + type: array + pg_syncing: + items: + type: string + type: array + prometheus_remote_read_uri: + type: string + prometheus_remote_write_uri: + type: string + query_frontend_uri: + type: string + query_uri: + type: string + receiver_ingesting_remote_write_uri: + type: string + receiver_remote_write_uri: + type: string + redis: + items: + type: string + type: array + redis_password: + type: string + redis_replica_uri: + type: string + redis_slave: + items: + type: string + type: array + schema_registry_uri: + type: string + store_uri: + type: string + thanos: + items: + type: string + type: array + valkey: + items: + type: string + type: array + valkey_password: + type: string + valkey_replica_uri: + type: string + valkey_slave: + items: + type: string + type: array