From 762439f3413b860fe098596542e14c1a8432f4c4 Mon Sep 17 00:00:00 2001 From: zhouzilong <529620861@qq.com> Date: Tue, 30 Jan 2024 14:18:47 -0500 Subject: [PATCH 1/4] init api --- doc/openapi/rest-catalog-open-api.yaml | 2667 ++++++++++++++++++++++++ 1 file changed, 2667 insertions(+) create mode 100644 doc/openapi/rest-catalog-open-api.yaml diff --git a/doc/openapi/rest-catalog-open-api.yaml b/doc/openapi/rest-catalog-open-api.yaml new file mode 100644 index 0000000..b86aa8f --- /dev/null +++ b/doc/openapi/rest-catalog-open-api.yaml @@ -0,0 +1,2667 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# This is the specification for the Iceberg REST Catalog API without transaction and view. +# src: https://github.com/apache/iceberg/blob/main/open-api/rest-catalog-open-api.yaml +--- +openapi: 3.0.3 +info: + title: Apache Iceberg REST Catalog API + license: + name: Apache 2.0 + url: https://www.apache.org/licenses/LICENSE-2.0.html + version: 0.0.1 + description: + Defines the specification for the first version of the REST Catalog API. + Implementations should ideally support both Iceberg table specs v1 and v2, with priority given to v2. +servers: + - url: "{scheme}://{host}/{basePath}" + description: Server URL when the port can be inferred from the scheme + variables: + scheme: + description: The scheme of the URI, either http or https. + default: https + host: + description: The host address for the specified server + default: localhost + basePath: + description: Optional prefix to be appended to all routes + default: "" + - url: "{scheme}://{host}:{port}/{basePath}" + description: Generic base server URL, with all parts configurable + variables: + scheme: + description: The scheme of the URI, either http or https. + default: https + host: + description: The host address for the specified server + default: localhost + port: + description: The port used when addressing the host + default: "443" + basePath: + description: Optional prefix to be appended to all routes + default: "" + +paths: + /v1/config: + get: + tags: + - Configuration API + summary: List all catalog configuration settings + operationId: getConfig + parameters: + - name: warehouse + in: query + required: false + schema: + type: string + description: Warehouse location or identifier to request from the service + description: " + All REST clients should first call this route to get catalog configuration + properties from the server to configure the catalog and its HTTP client. + Configuration from the server consists of two sets of key/value pairs. + + - defaults - properties that should be used as default configuration; applied before client configuration + + - overrides - properties that should be used to override client configuration; applied after defaults and client configuration + + + Catalog configuration is constructed by setting the defaults, then client- + provided configuration, and finally overrides. The final property set is then + used to configure the catalog. + + + For example, a default configuration property might set the size of the + client pool, which can be replaced with a client-specific setting. An + override might be used to set the warehouse location, which is stored + on the server rather than in client configuration. + + + Common catalog configuration settings are documented at + https://iceberg.apache.org/configuration/#catalog-properties + " + responses: + 200: + description: Server specified configuration values. + content: + application/json: + schema: + $ref: "#/components/schemas/CatalogConfig" + example: + { + "overrides": { "warehouse": "s3://bucket/warehouse/" }, + "defaults": { "clients": "4" }, + } + 400: + $ref: "#/components/responses/BadRequestErrorResponse" + 401: + $ref: "#/components/responses/UnauthorizedResponse" + 403: + $ref: "#/components/responses/ForbiddenResponse" + 419: + $ref: "#/components/responses/AuthenticationTimeoutResponse" + 503: + $ref: "#/components/responses/ServiceUnavailableResponse" + 5XX: + $ref: "#/components/responses/ServerErrorResponse" + + /v1/{prefix}/namespaces: + parameters: + - $ref: "#/components/parameters/prefix" + + get: + tags: + - Catalog API + summary: List namespaces, optionally providing a parent namespace to list underneath + description: + List all namespaces at a certain level, optionally starting from a given parent namespace. + If table accounting.tax.paid.info exists, using 'SELECT NAMESPACE IN accounting' would + translate into `GET /namespaces?parent=accounting` and must return a namespace, ["accounting", "tax"] only. + Using 'SELECT NAMESPACE IN accounting.tax' would + translate into `GET /namespaces?parent=accounting%1Ftax` and must return a namespace, ["accounting", "tax", "paid"]. + If `parent` is not provided, all top-level namespaces should be listed. + operationId: listNamespaces + parameters: + - name: parent + in: query + description: + An optional namespace, underneath which to list namespaces. + If not provided or empty, all top-level namespaces should be listed. + If parent is a multipart namespace, the parts must be separated by the unit separator (`0x1F`) byte. + required: false + allowEmptyValue: true + schema: + type: string + example: "accounting%1Ftax" + responses: + 200: + $ref: "#/components/responses/ListNamespacesResponse" + 400: + $ref: "#/components/responses/BadRequestErrorResponse" + 401: + $ref: "#/components/responses/UnauthorizedResponse" + 403: + $ref: "#/components/responses/ForbiddenResponse" + 404: + description: Not Found - Namespace provided in the `parent` query parameter is not found. + content: + application/json: + schema: + $ref: "#/components/schemas/IcebergErrorResponse" + examples: + NoSuchNamespaceExample: + $ref: "#/components/examples/NoSuchNamespaceError" + 419: + $ref: "#/components/responses/AuthenticationTimeoutResponse" + 503: + $ref: "#/components/responses/ServiceUnavailableResponse" + 5XX: + $ref: "#/components/responses/ServerErrorResponse" + + post: + tags: + - Catalog API + summary: Create a namespace + description: Create a namespace, with an optional set of properties. + The server might also add properties, such as `last_modified_time` etc. + operationId: createNamespace + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/CreateNamespaceRequest" + responses: + 200: + $ref: "#/components/responses/CreateNamespaceResponse" + 400: + $ref: "#/components/responses/BadRequestErrorResponse" + 401: + $ref: "#/components/responses/UnauthorizedResponse" + 403: + $ref: "#/components/responses/ForbiddenResponse" + 406: + $ref: "#/components/responses/UnsupportedOperationResponse" + 409: + description: Conflict - The namespace already exists + content: + application/json: + schema: + $ref: "#/components/schemas/IcebergErrorResponse" + examples: + NamespaceAlreadyExists: + $ref: "#/components/examples/NamespaceAlreadyExistsError" + 419: + $ref: "#/components/responses/AuthenticationTimeoutResponse" + 503: + $ref: "#/components/responses/ServiceUnavailableResponse" + 5XX: + $ref: "#/components/responses/ServerErrorResponse" + + /v1/{prefix}/namespaces/{namespace}: + parameters: + - $ref: "#/components/parameters/prefix" + - $ref: "#/components/parameters/namespace" + + get: + tags: + - Catalog API + summary: Load the metadata properties for a namespace + operationId: loadNamespaceMetadata + description: Return all stored metadata properties for a given namespace + responses: + 200: + $ref: "#/components/responses/GetNamespaceResponse" + 400: + $ref: "#/components/responses/BadRequestErrorResponse" + 401: + $ref: "#/components/responses/UnauthorizedResponse" + 403: + $ref: "#/components/responses/ForbiddenResponse" + 404: + description: Not Found - Namespace not found + content: + application/json: + schema: + $ref: "#/components/schemas/IcebergErrorResponse" + examples: + NoSuchNamespaceExample: + $ref: "#/components/examples/NoSuchNamespaceError" + 419: + $ref: "#/components/responses/AuthenticationTimeoutResponse" + 503: + $ref: "#/components/responses/ServiceUnavailableResponse" + 5XX: + $ref: "#/components/responses/ServerErrorResponse" + + head: + tags: + - Catalog API + summary: Check if a namespace exists + operationId: namespaceExists + description: Check if a namespace exists. The response does not contain a body. + responses: + 204: + description: Success, no content + 400: + $ref: "#/components/responses/BadRequestErrorResponse" + 401: + $ref: "#/components/responses/UnauthorizedResponse" + 403: + $ref: "#/components/responses/ForbiddenResponse" + 404: + description: Not Found - Namespace not found + content: + application/json: + schema: + $ref: "#/components/schemas/IcebergErrorResponse" + examples: + NoSuchNamespaceExample: + $ref: "#/components/examples/NoSuchNamespaceError" + 419: + $ref: "#/components/responses/AuthenticationTimeoutResponse" + 503: + $ref: "#/components/responses/ServiceUnavailableResponse" + 5XX: + $ref: "#/components/responses/ServerErrorResponse" + + delete: + tags: + - Catalog API + summary: Drop a namespace from the catalog. Namespace must be empty. + operationId: dropNamespace + responses: + 204: + description: Success, no content + 400: + $ref: "#/components/responses/BadRequestErrorResponse" + 401: + $ref: "#/components/responses/UnauthorizedResponse" + 403: + $ref: "#/components/responses/ForbiddenResponse" + 404: + description: Not Found - Namespace to delete does not exist. + content: + application/json: + schema: + $ref: "#/components/schemas/IcebergErrorResponse" + examples: + NoSuchNamespaceExample: + $ref: "#/components/examples/NoSuchNamespaceError" + 419: + $ref: "#/components/responses/AuthenticationTimeoutResponse" + 503: + $ref: "#/components/responses/ServiceUnavailableResponse" + 5XX: + $ref: "#/components/responses/ServerErrorResponse" + + /v1/{prefix}/namespaces/{namespace}/properties: + parameters: + - $ref: "#/components/parameters/prefix" + - $ref: "#/components/parameters/namespace" + + post: + tags: + - Catalog API + summary: Set or remove properties on a namespace + operationId: updateProperties + description: Set and/or remove properties on a namespace. + The request body specifies a list of properties to remove and a map + of key value pairs to update. + + Properties that are not in the request are not modified or removed by this call. + + Server implementations are not required to support namespace properties. + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/UpdateNamespacePropertiesRequest" + examples: + UpdateAndRemoveProperties: + $ref: "#/components/examples/UpdateAndRemoveNamespacePropertiesRequest" + responses: + 200: + $ref: "#/components/responses/UpdateNamespacePropertiesResponse" + 400: + $ref: "#/components/responses/BadRequestErrorResponse" + 401: + $ref: "#/components/responses/UnauthorizedResponse" + 403: + $ref: "#/components/responses/ForbiddenResponse" + 404: + description: Not Found - Namespace not found + content: + application/json: + schema: + $ref: "#/components/schemas/IcebergErrorResponse" + examples: + NamespaceNotFound: + $ref: "#/components/examples/NoSuchNamespaceError" + 406: + $ref: "#/components/responses/UnsupportedOperationResponse" + 422: + description: Unprocessable Entity - A property key was included in both `removals` and `updates` + content: + application/json: + schema: + $ref: "#/components/schemas/IcebergErrorResponse" + examples: + UnprocessableEntityDuplicateKey: + $ref: "#/components/examples/UnprocessableEntityDuplicateKey" + 419: + $ref: "#/components/responses/AuthenticationTimeoutResponse" + 503: + $ref: "#/components/responses/ServiceUnavailableResponse" + 5XX: + $ref: "#/components/responses/ServerErrorResponse" + + /v1/{prefix}/namespaces/{namespace}/tables: + parameters: + - $ref: "#/components/parameters/prefix" + - $ref: "#/components/parameters/namespace" + + get: + tags: + - Catalog API + summary: List all table identifiers underneath a given namespace + description: Return all table identifiers under this namespace + operationId: listTables + responses: + 200: + $ref: "#/components/responses/ListTablesResponse" + 400: + $ref: "#/components/responses/BadRequestErrorResponse" + 401: + $ref: "#/components/responses/UnauthorizedResponse" + 403: + $ref: "#/components/responses/ForbiddenResponse" + 404: + description: Not Found - The namespace specified does not exist + content: + application/json: + schema: + $ref: "#/components/schemas/IcebergErrorResponse" + examples: + NamespaceNotFound: + $ref: "#/components/examples/NoSuchNamespaceError" + 419: + $ref: "#/components/responses/AuthenticationTimeoutResponse" + 503: + $ref: "#/components/responses/ServiceUnavailableResponse" + 5XX: + $ref: "#/components/responses/ServerErrorResponse" + + post: + tags: + - Catalog API + summary: Create a table in the given namespace + description: + Create a table or start a create transaction, like atomic CTAS. + + + If `stage-create` is false, the table is created immediately. + + + If `stage-create` is true, the table is not created, but table metadata is initialized and returned. + The service should prepare as needed for a commit to the table commit endpoint to complete the create + transaction. The client uses the returned metadata to begin a transaction. To commit the transaction, + the client sends all create and subsequent changes to the table commit route. Changes from the table + create operation include changes like AddSchemaUpdate and SetCurrentSchemaUpdate that set the initial + table state. + operationId: createTable + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/CreateTableRequest" + responses: + 200: + $ref: "#/components/responses/CreateTableResponse" + 400: + $ref: "#/components/responses/BadRequestErrorResponse" + 401: + $ref: "#/components/responses/UnauthorizedResponse" + 403: + $ref: "#/components/responses/ForbiddenResponse" + 404: + description: Not Found - The namespace specified does not exist + content: + application/json: + schema: + $ref: "#/components/schemas/IcebergErrorResponse" + examples: + NamespaceNotFound: + $ref: "#/components/examples/NoSuchNamespaceError" + 409: + description: Conflict - The table already exists + content: + application/json: + schema: + $ref: "#/components/schemas/IcebergErrorResponse" + examples: + NamespaceAlreadyExists: + $ref: "#/components/examples/TableAlreadyExistsError" + 419: + $ref: "#/components/responses/AuthenticationTimeoutResponse" + 503: + $ref: "#/components/responses/ServiceUnavailableResponse" + 5XX: + $ref: "#/components/responses/ServerErrorResponse" + + /v1/{prefix}/namespaces/{namespace}/register: + parameters: + - $ref: "#/components/parameters/prefix" + - $ref: "#/components/parameters/namespace" + + post: + tags: + - Catalog API + summary: Register a table in the given namespace using given metadata file location + description: Register a table using given metadata file location. + + operationId: registerTable + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/RegisterTableRequest" + responses: + 200: + $ref: "#/components/responses/LoadTableResponse" + 400: + $ref: "#/components/responses/BadRequestErrorResponse" + 401: + $ref: "#/components/responses/UnauthorizedResponse" + 403: + $ref: "#/components/responses/ForbiddenResponse" + 404: + description: Not Found - The namespace specified does not exist + content: + application/json: + schema: + $ref: "#/components/schemas/IcebergErrorResponse" + examples: + NamespaceNotFound: + $ref: "#/components/examples/NoSuchNamespaceError" + 409: + description: Conflict - The table already exists + content: + application/json: + schema: + $ref: "#/components/schemas/IcebergErrorResponse" + examples: + NamespaceAlreadyExists: + $ref: "#/components/examples/TableAlreadyExistsError" + 419: + $ref: "#/components/responses/AuthenticationTimeoutResponse" + 503: + $ref: "#/components/responses/ServiceUnavailableResponse" + 5XX: + $ref: "#/components/responses/ServerErrorResponse" + + /v1/{prefix}/namespaces/{namespace}/tables/{table}: + parameters: + - $ref: "#/components/parameters/prefix" + - $ref: "#/components/parameters/namespace" + - $ref: "#/components/parameters/table" + + get: + tags: + - Catalog API + summary: Load a table from the catalog + operationId: loadTable + description: Load a table from the catalog. + + + The response contains both configuration and table metadata. The configuration, if non-empty is used + as additional configuration for the table that overrides catalog configuration. For example, this + configuration may change the FileIO implementation to be used for the table. + + + The response also contains the table's full metadata, matching the table metadata JSON file. + + + The catalog configuration may contain credentials that should be used for subsequent requests for the + table. The configuration key "token" is used to pass an access token to be used as a bearer token + for table requests. Otherwise, a token may be passed using a RFC 8693 token type as a configuration + key. For example, "urn:ietf:params:oauth:token-type:jwt=". + parameters: + - in: query + name: snapshots + description: + The snapshots to return in the body of the metadata. Setting the value to `all` would + return the full set of snapshots currently valid for the table. Setting the value to + `refs` would load all snapshots referenced by branches or tags. + + Default if no param is provided is `all`. + required: false + schema: + type: string + enum: [all, refs] + responses: + 200: + $ref: "#/components/responses/LoadTableResponse" + 400: + $ref: "#/components/responses/BadRequestErrorResponse" + 401: + $ref: "#/components/responses/UnauthorizedResponse" + 403: + $ref: "#/components/responses/ForbiddenResponse" + 404: + description: Not Found - NoSuchTableException, table to load does not exist + content: + application/json: + schema: + $ref: "#/components/schemas/IcebergErrorResponse" + examples: + TableToLoadDoesNotExist: + $ref: "#/components/examples/NoSuchTableError" + 419: + $ref: "#/components/responses/AuthenticationTimeoutResponse" + 503: + $ref: "#/components/responses/ServiceUnavailableResponse" + 5XX: + $ref: "#/components/responses/ServerErrorResponse" + + post: + tags: + - Catalog API + summary: Commit updates to a table + operationId: updateTable + description: Commit updates to a table. + + + Commits have two parts, requirements and updates. Requirements are assertions that will be validated + before attempting to make and commit changes. For example, `assert-ref-snapshot-id` will check that a + named ref's snapshot ID has a certain value. + + + Updates are changes to make to table metadata. For example, after asserting that the current main ref + is at the expected snapshot, a commit may add a new child snapshot and set the ref to the new + snapshot id. + + + Create table transactions that are started by createTable with `stage-create` set to true are + committed using this route. Transactions should include all changes to the table, including table + initialization, like AddSchemaUpdate and SetCurrentSchemaUpdate. The `assert-create` requirement is + used to ensure that the table was not created concurrently. + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/CommitTableRequest" + responses: + 200: + $ref: "#/components/responses/CommitTableResponse" + 400: + $ref: "#/components/responses/BadRequestErrorResponse" + 401: + $ref: "#/components/responses/UnauthorizedResponse" + 403: + $ref: "#/components/responses/ForbiddenResponse" + 404: + description: Not Found - NoSuchTableException, table to load does not exist + content: + application/json: + schema: + $ref: "#/components/schemas/IcebergErrorResponse" + examples: + TableToUpdateDoesNotExist: + $ref: "#/components/examples/NoSuchTableError" + 409: + description: Conflict - CommitFailedException, one or more requirements failed. The client may retry. + content: + application/json: + schema: + $ref: "#/components/schemas/IcebergErrorResponse" + 419: + $ref: "#/components/responses/AuthenticationTimeoutResponse" + 500: + description: An unknown server-side problem occurred; the commit state is unknown. + content: + application/json: + schema: + $ref: "#/components/schemas/IcebergErrorResponse" + example: + { + "error": + { + "message": "Internal Server Error", + "type": "CommitStateUnknownException", + "code": 500, + }, + } + 503: + $ref: "#/components/responses/ServiceUnavailableResponse" + 502: + description: A gateway or proxy received an invalid response from the upstream server; the commit state is unknown. + content: + application/json: + schema: + $ref: "#/components/schemas/IcebergErrorResponse" + example: + { + "error": + { + "message": "Invalid response from the upstream server", + "type": "CommitStateUnknownException", + "code": 502, + }, + } + 504: + description: A server-side gateway timeout occurred; the commit state is unknown. + content: + application/json: + schema: + $ref: "#/components/schemas/IcebergErrorResponse" + example: + { + "error": + { + "message": "Gateway timed out during commit", + "type": "CommitStateUnknownException", + "code": 504, + }, + } + 5XX: + description: A server-side problem that might not be addressable on the client. + content: + application/json: + schema: + $ref: "#/components/schemas/IcebergErrorResponse" + example: + { + "error": + { + "message": "Bad Gateway", + "type": "InternalServerError", + "code": 502, + }, + } + + delete: + tags: + - Catalog API + summary: Drop a table from the catalog + operationId: dropTable + description: Remove a table from the catalog + parameters: + - name: purgeRequested + in: query + required: false + description: Whether the user requested to purge the underlying table's data and metadata + schema: + type: boolean + default: false + responses: + 204: + description: Success, no content + 400: + $ref: "#/components/responses/BadRequestErrorResponse" + 401: + $ref: "#/components/responses/UnauthorizedResponse" + 403: + $ref: "#/components/responses/ForbiddenResponse" + 404: + description: Not Found - NoSuchTableException, Table to drop does not exist + content: + application/json: + schema: + $ref: "#/components/schemas/IcebergErrorResponse" + examples: + TableToDeleteDoesNotExist: + $ref: "#/components/examples/NoSuchTableError" + 419: + $ref: "#/components/responses/AuthenticationTimeoutResponse" + 503: + $ref: "#/components/responses/ServiceUnavailableResponse" + 5XX: + $ref: "#/components/responses/ServerErrorResponse" + + head: + tags: + - Catalog API + summary: Check if a table exists + operationId: tableExists + description: Check if a table exists within a given namespace. The response does not contain a body. + responses: + 204: + description: Success, no content + 400: + $ref: "#/components/responses/BadRequestErrorResponse" + 401: + $ref: "#/components/responses/UnauthorizedResponse" + 403: + $ref: "#/components/responses/ForbiddenResponse" + 404: + description: Not Found - NoSuchTableException, Table not found + content: + application/json: + schema: + $ref: "#/components/schemas/IcebergErrorResponse" + examples: + TableToLoadDoesNotExist: + $ref: "#/components/examples/NoSuchTableError" + 419: + $ref: "#/components/responses/AuthenticationTimeoutResponse" + 503: + $ref: "#/components/responses/ServiceUnavailableResponse" + 5XX: + $ref: "#/components/responses/ServerErrorResponse" + + /v1/{prefix}/tables/rename: + parameters: + - $ref: "#/components/parameters/prefix" + + post: + tags: + - Catalog API + summary: Rename a table from its current name to a new name + description: + Rename a table from one identifier to another. It's valid to move a table + across namespaces, but the server implementation is not required to support it. + operationId: renameTable + requestBody: + description: Current table identifier to rename and new table identifier to rename to + content: + application/json: + schema: + $ref: "#/components/schemas/RenameTableRequest" + examples: + RenameTableSameNamespace: + $ref: "#/components/examples/RenameTableSameNamespace" + required: true + responses: + 204: + description: Success, no content + 400: + $ref: "#/components/responses/BadRequestErrorResponse" + 401: + $ref: "#/components/responses/UnauthorizedResponse" + 403: + $ref: "#/components/responses/ForbiddenResponse" + 404: + description: Not Found + - NoSuchTableException, Table to rename does not exist + - NoSuchNamespaceException, The target namespace of the new table identifier does not exist + content: + application/json: + schema: + $ref: "#/components/schemas/IcebergErrorResponse" + examples: + TableToRenameDoesNotExist: + $ref: "#/components/examples/NoSuchTableError" + NamespaceToRenameToDoesNotExist: + $ref: "#/components/examples/NoSuchNamespaceError" + 406: + $ref: "#/components/responses/UnsupportedOperationResponse" + 409: + description: Conflict - The target identifier to rename to already exists as a table or view + content: + application/json: + schema: + $ref: "#/components/schemas/IcebergErrorResponse" + example: + $ref: "#/components/examples/TableAlreadyExistsError" + 419: + $ref: "#/components/responses/AuthenticationTimeoutResponse" + 503: + $ref: "#/components/responses/ServiceUnavailableResponse" + 5XX: + $ref: "#/components/responses/ServerErrorResponse" + + /v1/{prefix}/namespaces/{namespace}/tables/{table}/metrics: + parameters: + - $ref: "#/components/parameters/prefix" + - $ref: "#/components/parameters/namespace" + - $ref: "#/components/parameters/table" + + post: + tags: + - Catalog API + summary: Send a metrics report to this endpoint to be processed by the backend + operationId: reportMetrics + requestBody: + description: The request containing the metrics report to be sent + content: + application/json: + schema: + $ref: "#/components/schemas/ReportMetricsRequest" + required: true + responses: + 204: + description: Success, no content + 400: + $ref: "#/components/responses/BadRequestErrorResponse" + 401: + $ref: "#/components/responses/UnauthorizedResponse" + 403: + $ref: "#/components/responses/ForbiddenResponse" + 404: + description: Not Found - NoSuchTableException, table to load does not exist + content: + application/json: + schema: + $ref: "#/components/schemas/IcebergErrorResponse" + examples: + TableToLoadDoesNotExist: + $ref: "#/components/examples/NoSuchTableError" + 419: + $ref: "#/components/responses/AuthenticationTimeoutResponse" + 503: + $ref: "#/components/responses/ServiceUnavailableResponse" + 5XX: + $ref: "#/components/responses/ServerErrorResponse" + +components: + ####################################################### + # Common Parameter Definitions Used In Several Routes # + ####################################################### + parameters: + namespace: + name: namespace + in: path + required: true + description: A namespace identifier as a single string. + Multipart namespace parts should be separated by the unit separator (`0x1F`) byte. + schema: + type: string + examples: + singlepart_namespace: + value: "accounting" + multipart_namespace: + value: "accounting%1Ftax" + + prefix: + name: prefix + in: path + schema: + type: string + required: true + description: An optional prefix in the path + + table: + name: table + in: path + description: A table name + required: true + schema: + type: string + example: "sales" + + ############################## + # Application Schema Objects # + ############################## + schemas: + ErrorModel: + type: object + description: JSON error payload returned in a response with further details on the error + required: + - message + - type + - code + properties: + message: + type: string + description: Human-readable error message + type: + type: string + description: Internal type definition of the error + example: NoSuchNamespaceException + code: + type: integer + minimum: 400 + maximum: 600 + description: HTTP response code + example: 404 + stack: + type: array + items: + type: string + + CatalogConfig: + type: object + description: Server-provided configuration for the catalog. + required: + - defaults + - overrides + properties: + overrides: + type: object + additionalProperties: + type: string + description: Properties that should be used to override client configuration; applied after defaults and client configuration. + defaults: + type: object + additionalProperties: + type: string + description: Properties that should be used as default configuration; applied before client configuration. + + CreateNamespaceRequest: + type: object + required: + - namespace + properties: + namespace: + $ref: "#/components/schemas/Namespace" + properties: + type: object + description: Configured string to string map of properties for the namespace + example: { "owner": "Hank Bendickson" } + default: {} + additionalProperties: + type: string + + UpdateNamespacePropertiesRequest: + type: object + properties: + removals: + type: array + uniqueItems: true + items: + type: string + example: ["department", "access_group"] + updates: + type: object + example: { "owner": "Hank Bendickson" } + additionalProperties: + type: string + + RenameTableRequest: + type: object + required: + - source + - destination + properties: + source: + $ref: "#/components/schemas/TableIdentifier" + destination: + $ref: "#/components/schemas/TableIdentifier" + + Namespace: + description: Reference to one or more levels of a namespace + type: array + items: + type: string + example: ["accounting", "tax"] + + TableIdentifier: + type: object + required: + - namespace + - name + properties: + namespace: + $ref: "#/components/schemas/Namespace" + name: + type: string + nullable: false + + PrimitiveType: + type: string + example: + - "long" + - "string" + - "fixed[16]" + - "decimal(10,2)" + + StructField: + type: object + required: + - id + - name + - type + - required + properties: + id: + type: integer + name: + type: string + type: + $ref: "#/components/schemas/Type" + required: + type: boolean + doc: + type: string + + StructType: + type: object + required: + - type + - fields + properties: + type: + type: string + enum: ["struct"] + fields: + type: array + items: + $ref: "#/components/schemas/StructField" + + ListType: + type: object + required: + - type + - element-id + - element + - element-required + properties: + type: + type: string + enum: ["list"] + element-id: + type: integer + element: + $ref: "#/components/schemas/Type" + element-required: + type: boolean + + MapType: + type: object + required: + - type + - key-id + - key + - value-id + - value + - value-required + properties: + type: + type: string + enum: ["map"] + key-id: + type: integer + key: + $ref: "#/components/schemas/Type" + value-id: + type: integer + value: + $ref: "#/components/schemas/Type" + value-required: + type: boolean + + Type: + oneOf: + - $ref: "#/components/schemas/PrimitiveType" + - $ref: "#/components/schemas/StructType" + - $ref: "#/components/schemas/ListType" + - $ref: "#/components/schemas/MapType" + + Schema: + allOf: + - $ref: "#/components/schemas/StructType" + - type: object + properties: + schema-id: + type: integer + readOnly: true + identifier-field-ids: + type: array + items: + type: integer + + Expression: + oneOf: + - $ref: "#/components/schemas/AndOrExpression" + - $ref: "#/components/schemas/NotExpression" + - $ref: "#/components/schemas/SetExpression" + - $ref: "#/components/schemas/LiteralExpression" + - $ref: "#/components/schemas/UnaryExpression" + + ExpressionType: + type: string + example: + - "eq" + - "and" + - "or" + - "not" + - "in" + - "not-in" + - "lt" + - "lt-eq" + - "gt" + - "gt-eq" + - "not-eq" + - "starts-with" + - "not-starts-with" + - "is-null" + - "not-null" + - "is-nan" + - "not-nan" + + AndOrExpression: + type: object + required: + - type + - left + - right + properties: + type: + $ref: "#/components/schemas/ExpressionType" + enum: ["and", "or"] + left: + $ref: "#/components/schemas/Expression" + right: + $ref: "#/components/schemas/Expression" + + NotExpression: + type: object + required: + - type + - child + properties: + type: + $ref: "#/components/schemas/ExpressionType" + enum: ["not"] + child: + $ref: "#/components/schemas/Expression" + + UnaryExpression: + type: object + required: + - type + - term + - value + properties: + type: + $ref: "#/components/schemas/ExpressionType" + enum: ["is-null", "not-null", "is-nan", "not-nan"] + term: + $ref: "#/components/schemas/Term" + value: + type: object + + LiteralExpression: + type: object + required: + - type + - term + - value + properties: + type: + $ref: "#/components/schemas/ExpressionType" + enum: + [ + "lt", + "lt-eq", + "gt", + "gt-eq", + "eq", + "not-eq", + "starts-with", + "not-starts-with", + ] + term: + $ref: "#/components/schemas/Term" + value: + type: object + + SetExpression: + type: object + required: + - type + - term + - values + properties: + type: + $ref: "#/components/schemas/ExpressionType" + enum: ["in", "not-in"] + term: + $ref: "#/components/schemas/Term" + values: + type: array + items: + type: object + + Term: + oneOf: + - $ref: "#/components/schemas/Reference" + - $ref: "#/components/schemas/TransformTerm" + + Reference: + type: string + example: + - "column-name" + + TransformTerm: + type: object + required: + - type + - transform + - term + properties: + type: + type: string + enum: ["transform"] + transform: + $ref: "#/components/schemas/Transform" + term: + $ref: "#/components/schemas/Reference" + + Transform: + type: string + example: + - "identity" + - "year" + - "month" + - "day" + - "hour" + - "bucket[256]" + - "truncate[16]" + + PartitionField: + type: object + required: + - source-id + - transform + - name + properties: + field-id: + type: integer + source-id: + type: integer + name: + type: string + transform: + $ref: "#/components/schemas/Transform" + + PartitionSpec: + type: object + required: + - fields + properties: + spec-id: + type: integer + readOnly: true + fields: + type: array + items: + $ref: "#/components/schemas/PartitionField" + + SortDirection: + type: string + enum: ["asc", "desc"] + + NullOrder: + type: string + enum: ["nulls-first", "nulls-last"] + + SortField: + type: object + required: + - source-id + - transform + - direction + - null-order + properties: + source-id: + type: integer + transform: + $ref: "#/components/schemas/Transform" + direction: + $ref: "#/components/schemas/SortDirection" + null-order: + $ref: "#/components/schemas/NullOrder" + + SortOrder: + type: object + required: + - order-id + - fields + properties: + order-id: + type: integer + readOnly: true + fields: + type: array + items: + $ref: "#/components/schemas/SortField" + + Snapshot: + type: object + required: + - snapshot-id + - timestamp-ms + - manifest-list + - summary + properties: + snapshot-id: + type: integer + format: int64 + parent-snapshot-id: + type: integer + format: int64 + sequence-number: + type: integer + format: int64 + timestamp-ms: + type: integer + format: int64 + manifest-list: + type: string + description: Location of the snapshot's manifest list file + summary: + type: object + required: + - operation + properties: + operation: + type: string + enum: ["append", "replace", "overwrite", "delete"] + additionalProperties: + type: string + schema-id: + type: integer + + SnapshotReference: + type: object + required: + - type + - snapshot-id + properties: + type: + type: string + enum: ["tag", "branch"] + snapshot-id: + type: integer + format: int64 + max-ref-age-ms: + type: integer + format: int64 + max-snapshot-age-ms: + type: integer + format: int64 + min-snapshots-to-keep: + type: integer + + SnapshotReferences: + type: object + additionalProperties: + $ref: "#/components/schemas/SnapshotReference" + + SnapshotLog: + type: array + items: + type: object + required: + - snapshot-id + - timestamp-ms + properties: + snapshot-id: + type: integer + format: int64 + timestamp-ms: + type: integer + format: int64 + + MetadataLog: + type: array + items: + type: object + required: + - metadata-file + - timestamp-ms + properties: + metadata-file: + type: string + timestamp-ms: + type: integer + format: int64 + + TableMetadata: + type: object + required: + - format-version + - table-uuid + properties: + format-version: + type: integer + minimum: 1 + maximum: 2 + table-uuid: + type: string + location: + type: string + last-updated-ms: + type: integer + format: int64 + properties: + type: object + additionalProperties: + type: string + # schema tracking + schemas: + type: array + items: + $ref: "#/components/schemas/Schema" + current-schema-id: + type: integer + last-column-id: + type: integer + # partition spec tracking + partition-specs: + type: array + items: + $ref: "#/components/schemas/PartitionSpec" + default-spec-id: + type: integer + last-partition-id: + type: integer + # sort order tracking + sort-orders: + type: array + items: + $ref: "#/components/schemas/SortOrder" + default-sort-order-id: + type: integer + # snapshot tracking + snapshots: + type: array + items: + $ref: "#/components/schemas/Snapshot" + refs: + $ref: "#/components/schemas/SnapshotReferences" + current-snapshot-id: + type: integer + format: int64 + last-sequence-number: + type: integer + format: int64 + # logs + snapshot-log: + $ref: "#/components/schemas/SnapshotLog" + metadata-log: + $ref: "#/components/schemas/MetadataLog" + + BaseUpdate: + discriminator: + propertyName: action + mapping: + assign-uuid: "#/components/schemas/AssignUUIDUpdate" + upgrade-format-version: "#/components/schemas/UpgradeFormatVersionUpdate" + add-schema: "#/components/schemas/AddSchemaUpdate" + set-current-schema: "#/components/schemas/SetCurrentSchemaUpdate" + add-spec: "#/components/schemas/AddPartitionSpecUpdate" + set-default-spec: "#/components/schemas/SetDefaultSpecUpdate" + add-sort-order: "#/components/schemas/AddSortOrderUpdate" + set-default-sort-order: "#/components/schemas/SetDefaultSortOrderUpdate" + add-snapshot: "#/components/schemas/AddSnapshotUpdate" + set-snapshot-ref: "#/components/schemas/SetSnapshotRefUpdate" + remove-snapshots: "#/components/schemas/RemoveSnapshotsUpdate" + remove-snapshot-ref: "#/components/schemas/RemoveSnapshotRefUpdate" + set-location: "#/components/schemas/SetLocationUpdate" + set-properties: "#/components/schemas/SetPropertiesUpdate" + remove-properties: "#/components/schemas/RemovePropertiesUpdate" + add-view-version: "#/components/schemas/AddViewVersionUpdate" + set-current-view-version: "#/components/schemas/SetCurrentViewVersionUpdate" + type: object + required: + - action + properties: + action: + type: string + + AssignUUIDUpdate: + description: Assigning a UUID to a table/view should only be done when creating the table/view. It is not safe to re-assign the UUID if a table/view already has a UUID assigned + allOf: + - $ref: "#/components/schemas/BaseUpdate" + required: + - action + - uuid + properties: + action: + type: string + enum: ["assign-uuid"] + uuid: + type: string + + UpgradeFormatVersionUpdate: + allOf: + - $ref: "#/components/schemas/BaseUpdate" + required: + - action + - format-version + properties: + action: + type: string + enum: ["upgrade-format-version"] + format-version: + type: integer + + AddSchemaUpdate: + allOf: + - $ref: "#/components/schemas/BaseUpdate" + required: + - action + - schema + properties: + action: + type: string + enum: ["add-schema"] + schema: + $ref: "#/components/schemas/Schema" + last-column-id: + type: integer + description: The highest assigned column ID for the table. This is used to ensure columns are always assigned an unused ID when evolving schemas. When omitted, it will be computed on the server side. + + SetCurrentSchemaUpdate: + allOf: + - $ref: "#/components/schemas/BaseUpdate" + required: + - action + - schema-id + properties: + action: + type: string + enum: ["set-current-schema"] + schema-id: + type: integer + description: Schema ID to set as current, or -1 to set last added schema + + AddPartitionSpecUpdate: + allOf: + - $ref: "#/components/schemas/BaseUpdate" + required: + - action + - spec + properties: + action: + type: string + enum: ["add-spec"] + spec: + $ref: "#/components/schemas/PartitionSpec" + + SetDefaultSpecUpdate: + allOf: + - $ref: "#/components/schemas/BaseUpdate" + required: + - action + - spec-id + properties: + action: + type: string + enum: ["set-default-spec"] + spec-id: + type: integer + description: Partition spec ID to set as the default, or -1 to set last added spec + + AddSortOrderUpdate: + allOf: + - $ref: "#/components/schemas/BaseUpdate" + required: + - action + - sort-order + properties: + action: + type: string + enum: ["add-sort-order"] + sort-order: + $ref: "#/components/schemas/SortOrder" + + SetDefaultSortOrderUpdate: + allOf: + - $ref: "#/components/schemas/BaseUpdate" + required: + - action + - sort-order-id + properties: + action: + type: string + enum: ["set-default-sort-order"] + sort-order-id: + type: integer + description: Sort order ID to set as the default, or -1 to set last added sort order + + AddSnapshotUpdate: + allOf: + - $ref: "#/components/schemas/BaseUpdate" + required: + - action + - snapshot + properties: + action: + type: string + enum: ["add-snapshot"] + snapshot: + $ref: "#/components/schemas/Snapshot" + + SetSnapshotRefUpdate: + allOf: + - $ref: "#/components/schemas/BaseUpdate" + - $ref: "#/components/schemas/SnapshotReference" + required: + - action + - ref-name + properties: + action: + type: string + enum: ["set-snapshot-ref"] + ref-name: + type: string + + RemoveSnapshotsUpdate: + allOf: + - $ref: "#/components/schemas/BaseUpdate" + required: + - action + - snapshot-ids + properties: + action: + type: string + enum: ["remove-snapshots"] + snapshot-ids: + type: array + items: + type: integer + format: int64 + + RemoveSnapshotRefUpdate: + allOf: + - $ref: "#/components/schemas/BaseUpdate" + required: + - action + - ref-name + properties: + action: + type: string + enum: ["remove-snapshot-ref"] + ref-name: + type: string + + SetLocationUpdate: + allOf: + - $ref: "#/components/schemas/BaseUpdate" + required: + - action + - location + properties: + action: + type: string + enum: ["set-location"] + location: + type: string + + SetPropertiesUpdate: + allOf: + - $ref: "#/components/schemas/BaseUpdate" + required: + - action + - updates + properties: + action: + type: string + enum: ["set-properties"] + updates: + type: object + additionalProperties: + type: string + + RemovePropertiesUpdate: + allOf: + - $ref: "#/components/schemas/BaseUpdate" + required: + - action + - removals + properties: + action: + type: string + enum: ["remove-properties"] + removals: + type: array + items: + type: string + + TableUpdate: + anyOf: + - $ref: "#/components/schemas/AssignUUIDUpdate" + - $ref: "#/components/schemas/UpgradeFormatVersionUpdate" + - $ref: "#/components/schemas/AddSchemaUpdate" + - $ref: "#/components/schemas/SetCurrentSchemaUpdate" + - $ref: "#/components/schemas/AddPartitionSpecUpdate" + - $ref: "#/components/schemas/SetDefaultSpecUpdate" + - $ref: "#/components/schemas/AddSortOrderUpdate" + - $ref: "#/components/schemas/SetDefaultSortOrderUpdate" + - $ref: "#/components/schemas/AddSnapshotUpdate" + - $ref: "#/components/schemas/SetSnapshotRefUpdate" + - $ref: "#/components/schemas/RemoveSnapshotsUpdate" + - $ref: "#/components/schemas/RemoveSnapshotRefUpdate" + - $ref: "#/components/schemas/SetLocationUpdate" + - $ref: "#/components/schemas/SetPropertiesUpdate" + - $ref: "#/components/schemas/RemovePropertiesUpdate" + + TableRequirement: + discriminator: + propertyName: type + mapping: + assert-create: "#/components/schemas/AssertCreate" + assert-table-uuid: "#/components/schemas/AssertTableUUID" + assert-ref-snapshot-id: "#/components/schemas/AssertRefSnapshotId" + assert-last-assigned-field-id: "#/components/schemas/AssertLastAssignedFieldId" + assert-current-schema-id: "#/components/schemas/AssertCurrentSchemaId" + assert-last-assigned-partition-id: "#/components/schemas/AssertLastAssignedPartitionId" + assert-default-spec-id: "#/components/schemas/AssertDefaultSpecId" + assert-default-sort-order-id: "#/components/schemas/AssertDefaultSortOrderId" + type: object + required: + - type + properties: + type: + type: "string" + + AssertCreate: + allOf: + - $ref: "#/components/schemas/TableRequirement" + type: object + description: The table must not already exist; used for create transactions + required: + - type + properties: + type: + type: string + enum: ["assert-create"] + + AssertTableUUID: + allOf: + - $ref: "#/components/schemas/TableRequirement" + description: The table UUID must match the requirement's `uuid` + required: + - type + - uuid + properties: + type: + type: string + enum: ["assert-table-uuid"] + uuid: + type: string + + AssertRefSnapshotId: + allOf: + - $ref: "#/components/schemas/TableRequirement" + description: + The table branch or tag identified by the requirement's `ref` must reference the requirement's `snapshot-id`; + if `snapshot-id` is `null` or missing, the ref must not already exist + required: + - type + - ref + - snapshot-id + properties: + type: + type: string + enum: ["assert-ref-snapshot-id"] + ref: + type: string + snapshot-id: + type: integer + format: int64 + + AssertLastAssignedFieldId: + allOf: + - $ref: "#/components/schemas/TableRequirement" + description: The table's last assigned column id must match the requirement's `last-assigned-field-id` + required: + - type + - last-assigned-field-id + properties: + type: + type: string + enum: ["assert-last-assigned-field-id"] + last-assigned-field-id: + type: integer + + AssertCurrentSchemaId: + allOf: + - $ref: "#/components/schemas/TableRequirement" + description: The table's current schema id must match the requirement's `current-schema-id` + required: + - type + - current-schema-id + properties: + type: + type: string + enum: ["assert-current-schema-id"] + current-schema-id: + type: integer + + AssertLastAssignedPartitionId: + allOf: + - $ref: "#/components/schemas/TableRequirement" + description: The table's last assigned partition id must match the requirement's `last-assigned-partition-id` + required: + - type + - last-assigned-partition-id + properties: + type: + type: string + enum: ["assert-last-assigned-partition-id"] + last-assigned-partition-id: + type: integer + + AssertDefaultSpecId: + allOf: + - $ref: "#/components/schemas/TableRequirement" + description: The table's default spec id must match the requirement's `default-spec-id` + required: + - type + - default-spec-id + properties: + type: + type: string + enum: ["assert-default-spec-id"] + default-spec-id: + type: integer + + AssertDefaultSortOrderId: + allOf: + - $ref: "#/components/schemas/TableRequirement" + description: The table's default sort order id must match the requirement's `default-sort-order-id` + required: + - type + - default-sort-order-id + properties: + type: + type: string + enum: ["assert-default-sort-order-id"] + default-sort-order-id: + type: integer + + LoadTableResult: + description: | + Result used when a table is successfully loaded. + + + The table metadata JSON is returned in the `metadata` field. The corresponding file location of table metadata should be returned in the `metadata-location` field, unless the metadata is not yet committed. For example, a create transaction may return metadata that is staged but not committed. + Clients can check whether metadata has changed by comparing metadata locations after the table has been created. + + + The `config` map returns table-specific configuration for the table's resources, including its HTTP client and FileIO. For example, config may contain a specific FileIO implementation class for the table depending on its underlying storage. + + + The following configurations should be respected by clients: + + ## General Configurations + + - `token`: Authorization bearer token to use for table requests if OAuth2 security is enabled + + ## AWS Configurations + + The following configurations should be respected when working with tables stored in AWS S3 + - `client.region`: region to configure client for making requests to AWS + - `s3.access-key-id`: id for for credentials that provide access to the data in S3 + - `s3.secret-access-key`: secret for credentials that provide access to data in S3 + - `s3.session-token`: if present, this value should be used for as the session token + - `s3.remote-signing-enabled`: if `true` remote signing should be performed as described in the `s3-signer-open-api.yaml` specification + type: object + required: + - metadata + properties: + metadata-location: + type: string + description: May be null if the table is staged as part of a transaction + metadata: + $ref: "#/components/schemas/TableMetadata" + config: + type: object + additionalProperties: + type: string + + CommitTableRequest: + type: object + required: + - requirements + - updates + properties: + identifier: + description: Table identifier to update; must be present for CommitTransactionRequest + $ref: "#/components/schemas/TableIdentifier" + requirements: + type: array + items: + $ref: "#/components/schemas/TableRequirement" + updates: + type: array + items: + $ref: "#/components/schemas/TableUpdate" + + CommitViewRequest: + type: object + required: + - updates + properties: + identifier: + description: View identifier to update + $ref: "#/components/schemas/TableIdentifier" + requirements: + type: array + items: + $ref: "#/components/schemas/ViewRequirement" + updates: + type: array + items: + $ref: "#/components/schemas/ViewUpdate" + + CreateTableRequest: + type: object + required: + - name + - schema + properties: + name: + type: string + location: + type: string + schema: + $ref: "#/components/schemas/Schema" + partition-spec: + $ref: "#/components/schemas/PartitionSpec" + write-order: + $ref: "#/components/schemas/SortOrder" + stage-create: + type: boolean + properties: + type: object + additionalProperties: + type: string + + RegisterTableRequest: + type: object + required: + - name + - metadata-location + properties: + name: + type: string + metadata-location: + type: string + + CreateViewRequest: + type: object + required: + - name + - schema + - view-version + - properties + properties: + name: + type: string + location: + type: string + schema: + $ref: "#/components/schemas/Schema" + view-version: + $ref: "#/components/schemas/ViewVersion" + description: The view version to create, will replace the schema-id sent within the view-version with the id assigned to the provided schema + properties: + type: object + additionalProperties: + type: string + + LoadViewResult: + description: | + Result used when a view is successfully loaded. + + + The view metadata JSON is returned in the `metadata` field. The corresponding file location of view metadata is returned in the `metadata-location` field. + Clients can check whether metadata has changed by comparing metadata locations after the view has been created. + + The `config` map returns view-specific configuration for the view's resources. + + The following configurations should be respected by clients: + + ## General Configurations + + - `token`: Authorization bearer token to use for view requests if OAuth2 security is enabled + + type: object + required: + - metadata-location + - metadata + properties: + metadata-location: + type: string + metadata: + $ref: "#/components/schemas/ViewMetadata" + config: + type: object + additionalProperties: + type: string + + TokenType: + type: string + enum: + - urn:ietf:params:oauth:token-type:access_token + - urn:ietf:params:oauth:token-type:refresh_token + - urn:ietf:params:oauth:token-type:id_token + - urn:ietf:params:oauth:token-type:saml1 + - urn:ietf:params:oauth:token-type:saml2 + - urn:ietf:params:oauth:token-type:jwt + description: Token type identifier, from RFC 8693 Section 3 + + + See https://datatracker.ietf.org/doc/html/rfc8693#section-3 + + CounterResult: + type: object + required: + - unit + - value + properties: + unit: + type: string + value: + type: integer + format: int64 + + TimerResult: + type: object + required: + - time-unit + - count + - total-duration + properties: + time-unit: + type: string + count: + type: integer + format: int64 + total-duration: + type: integer + format: int64 + + MetricResult: + anyOf: + - $ref: "#/components/schemas/CounterResult" + - $ref: "#/components/schemas/TimerResult" + + Metrics: + type: object + additionalProperties: + $ref: "#/components/schemas/MetricResult" + example: + "metrics": + { + "total-planning-duration": + { + "count": 1, + "time-unit": "nanoseconds", + "total-duration": 2644235116, + }, + "result-data-files": { "unit": "count", "value": 1 }, + "result-delete-files": { "unit": "count", "value": 0 }, + "total-data-manifests": { "unit": "count", "value": 1 }, + "total-delete-manifests": { "unit": "count", "value": 0 }, + "scanned-data-manifests": { "unit": "count", "value": 1 }, + "skipped-data-manifests": { "unit": "count", "value": 0 }, + "total-file-size-bytes": { "unit": "bytes", "value": 10 }, + "total-delete-file-size-bytes": { "unit": "bytes", "value": 0 }, + } + + ReportMetricsRequest: + anyOf: + - $ref: "#/components/schemas/ScanReport" + - $ref: "#/components/schemas/CommitReport" + required: + - report-type + properties: + report-type: + type: string + + ScanReport: + type: object + required: + - table-name + - snapshot-id + - filter + - schema-id + - projected-field-ids + - projected-field-names + - metrics + properties: + table-name: + type: string + snapshot-id: + type: integer + format: int64 + filter: + $ref: "#/components/schemas/Expression" + schema-id: + type: integer + projected-field-ids: + type: array + items: + type: integer + projected-field-names: + type: array + items: + type: string + metrics: + $ref: "#/components/schemas/Metrics" + metadata: + type: object + additionalProperties: + type: string + + CommitReport: + type: object + required: + - table-name + - snapshot-id + - sequence-number + - operation + - metrics + properties: + table-name: + type: string + snapshot-id: + type: integer + format: int64 + sequence-number: + type: integer + format: int64 + operation: + type: string + metrics: + $ref: "#/components/schemas/Metrics" + metadata: + type: object + additionalProperties: + type: string + + IcebergErrorResponse: + description: JSON wrapper for all error responses (non-2xx) + type: object + required: + - error + properties: + error: + $ref: "#/components/schemas/ErrorModel" + additionalProperties: false + example: + { + "error": + { + "message": "The server does not support this operation", + "type": "UnsupportedOperationException", + "code": 406, + }, + } + + CreateNamespaceResponse: + type: object + required: + - namespace + properties: + namespace: + $ref: "#/components/schemas/Namespace" + properties: + type: object + additionalProperties: + type: string + description: Properties stored on the namespace, if supported by the server. + example: { "owner": "Ralph", "created_at": "1452120468" } + default: {} + + GetNamespaceResponse: + type: object + required: + - namespace + properties: + namespace: + $ref: "#/components/schemas/Namespace" + properties: + type: object + description: + Properties stored on the namespace, if supported by the server. + If the server does not support namespace properties, it should return null for this field. + If namespace properties are supported, but none are set, it should return an empty object. + additionalProperties: + type: string + example: { "owner": "Ralph", "transient_lastDdlTime": "1452120468" } + default: {} + nullable: true + + ListTablesResponse: + type: object + properties: + identifiers: + type: array + uniqueItems: true + items: + $ref: "#/components/schemas/TableIdentifier" + + ListNamespacesResponse: + type: object + properties: + namespaces: + type: array + uniqueItems: true + items: + $ref: "#/components/schemas/Namespace" + + UpdateNamespacePropertiesResponse: + type: object + required: + - updated + - removed + properties: + updated: + description: List of property keys that were added or updated + type: array + uniqueItems: true + items: + type: string + removed: + description: List of properties that were removed + type: array + items: + type: string + missing: + type: array + items: + type: string + description: + List of properties requested for removal that were not found + in the namespace's properties. Represents a partial success response. + Server's do not need to implement this. + nullable: true + + CommitTableResponse: + type: object + required: + - metadata-location + - metadata + properties: + metadata-location: + type: string + metadata: + $ref: "#/components/schemas/TableMetadata" + + ############################# + # Reusable Response Objects # + ############################# + responses: + BadRequestErrorResponse: + description: + Indicates a bad request error. It could be caused by an unexpected request + body format or other forms of request validation failure, such as invalid json. + Usually serves application/json content, although in some cases simple text/plain content might + be returned by the server's middleware. + content: + application/json: + schema: + $ref: "#/components/schemas/IcebergErrorResponse" + example: + { + "error": + { + "message": "Malformed request", + "type": "BadRequestException", + "code": 400, + }, + } + + # Note that this is a representative example response for use as a shorthand in the spec. + # The fields `message` and `type` as indicated here are not presently prescriptive. + UnauthorizedResponse: + description: Unauthorized. Authentication is required and has failed or has not yet been provided. + content: + application/json: + schema: + $ref: "#/components/schemas/IcebergErrorResponse" + example: + { + "error": + { + "message": "Not authorized to make this request", + "type": "NotAuthorizedException", + "code": 401, + }, + } + + # Note that this is a representative example response for use as a shorthand in the spec. + # The fields `message` and `type` as indicated here are not presently prescriptive. + ForbiddenResponse: + description: Forbidden. Authenticated user does not have the necessary permissions. + content: + application/json: + schema: + $ref: "#/components/schemas/IcebergErrorResponse" + example: + { + "error": + { + "message": "Not authorized to make this request", + "type": "NotAuthorizedException", + "code": 403, + }, + } + + # Note that this is a representative example response for use as a shorthand in the spec. + # The fields `message` and `type` as indicated here are not presently prescriptive. + UnsupportedOperationResponse: + description: Not Acceptable / Unsupported Operation. The server does not support this operation. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorModel" + example: + { + "error": + { + "message": "The server does not support this operation", + "type": "UnsupportedOperationException", + "code": 406, + }, + } + + IcebergErrorResponse: + description: JSON wrapper for all error responses (non-2xx) + content: + application/json: + schema: + $ref: "#/components/schemas/IcebergErrorResponse" + example: + { + "error": + { + "message": "The server does not support this operation", + "type": "UnsupportedOperationException", + "code": 406, + }, + } + + CreateNamespaceResponse: + description: Represents a successful call to create a namespace. + Returns the namespace created, as well as any properties that were stored for the namespace, + including those the server might have added. Implementations are not required to support namespace + properties. + content: + application/json: + schema: + $ref: "#/components/schemas/CreateNamespaceResponse" + example: + { + "namespace": ["accounting", "tax"], + "properties": { "owner": "Ralph", "created_at": "1452120468" }, + } + + GetNamespaceResponse: + description: + Returns a namespace, as well as any properties stored on the namespace if namespace properties + are supported by the server. + content: + application/json: + schema: + $ref: "#/components/schemas/GetNamespaceResponse" + + ListTablesResponse: + description: A list of table identifiers + content: + application/json: + schema: + $ref: "#/components/schemas/ListTablesResponse" + examples: + ListTablesResponseNonEmpty: + $ref: "#/components/examples/ListTablesNonEmptyExample" + ListTablesResponseEmpty: + $ref: "#/components/examples/ListTablesEmptyExample" + + ListNamespacesResponse: + description: A list of namespaces + content: + application/json: + schema: + $ref: "#/components/schemas/ListNamespacesResponse" + examples: + NonEmptyResponse: + $ref: "#/components/examples/ListNamespacesNonEmptyExample" + EmptyResponse: + $ref: "#/components/examples/ListNamespacesEmptyExample" + + AuthenticationTimeoutResponse: + description: Credentials have timed out. If possible, the client should refresh credentials and retry. + content: + application/json: + schema: + $ref: "#/components/schemas/IcebergErrorResponse" + example: + { + "error": + { + "message": "Credentials have timed out", + "type": "AuthenticationTimeoutException", + "code": 419, + }, + } + + ServiceUnavailableResponse: + description: + The service is not ready to handle the request. The client should wait and retry. + + + The service may additionally send a Retry-After header to indicate when to retry. + content: + application/json: + schema: + $ref: "#/components/schemas/IcebergErrorResponse" + example: + { + "error": + { + "message": "Slow down", + "type": "SlowDownException", + "code": 503, + }, + } + + ServerErrorResponse: + description: + A server-side problem that might not be addressable from the client + side. Used for server 5xx errors without more specific documentation in + individual routes. + content: + application/json: + schema: + $ref: "#/components/schemas/IcebergErrorResponse" + example: + { + "error": + { + "message": "Internal Server Error", + "type": "InternalServerError", + "code": 500, + }, + } + + UpdateNamespacePropertiesResponse: + description: JSON data response for a synchronous update properties request. + content: + application/json: + schema: + $ref: "#/components/schemas/UpdateNamespacePropertiesResponse" + example: + { "updated": ["owner"], "removed": ["foo"], "missing": ["bar"] } + + CreateTableResponse: + description: Table metadata result after creating a table + content: + application/json: + schema: + $ref: "#/components/schemas/LoadTableResult" + + LoadTableResponse: + description: Table metadata result when loading a table + content: + application/json: + schema: + $ref: "#/components/schemas/LoadTableResult" + + CommitTableResponse: + description: Response used when a table is successfully updated. + + The table metadata JSON is returned in the metadata field. The corresponding file location of table metadata must be returned in the metadata-location field. Clients can check whether metadata has changed by comparing metadata locations. + content: + application/json: + schema: + $ref: "#/components/schemas/CommitTableResponse" + + ####################################### + # Common examples of different values # + ####################################### + examples: + ListTablesEmptyExample: + summary: An empty list for a namespace with no tables + value: { "identifiers": [] } + + ListNamespacesEmptyExample: + summary: An empty list of namespaces + value: { "namespaces": [] } + + ListNamespacesNonEmptyExample: + summary: A non-empty list of namespaces + value: + { "namespaces": [["accounting", "tax"], ["accounting", "credits"]] } + + ListTablesNonEmptyExample: + summary: A non-empty list of table identifiers + value: + { + "identifiers": + [ + { "namespace": ["accounting", "tax"], "name": "paid" }, + { "namespace": ["accounting", "tax"], "name": "owed" }, + ], + } + + MultipartNamespaceAsPathVariable: + summary: A multi-part namespace, as represented in a path parameter + value: "accounting%1Ftax" + + NamespaceAsPathVariable: + summary: A single part namespace, as represented in a path paremeter + value: "accounting" + + NamespaceAlreadyExistsError: + summary: The requested namespace already exists + value: + { + "error": + { + "message": "The given namespace already exists", + "type": "AlreadyExistsException", + "code": 409, + }, + } + + NoSuchTableError: + summary: The requested table does not exist + value: + { + "error": + { + "message": "The given table does not exist", + "type": "NoSuchTableException", + "code": 404, + }, + } + + NoSuchViewError: + summary: The requested view does not exist + value: + { + "error": + { + "message": "The given view does not exist", + "type": "NoSuchViewException", + "code": 404, + }, + } + + NoSuchNamespaceError: + summary: The requested namespace does not exist + value: + { + "error": + { + "message": "The given namespace does not exist", + "type": "NoSuchNamespaceException", + "code": 404, + }, + } + + RenameTableSameNamespace: + summary: Rename a table in the same namespace + value: + { + "source": { "namespace": ["accounting", "tax"], "name": "paid" }, + "destination": { "namespace": ["accounting", "tax"], "name": "owed" }, + } + + TableAlreadyExistsError: + summary: The requested table identifier already exists + value: + { + "error": + { + "message": "The given table already exists", + "type": "AlreadyExistsException", + "code": 409, + }, + } + + # This is an example response and is not meant to be prescriptive regarding the message or type. + UnprocessableEntityDuplicateKey: + summary: + The request body either has the same key multiple times in what should be a map with unique keys + or the request body has keys in two or more fields which should be disjoint sets. + value: + { + "error": + { + "message": "The request cannot be processed as there is a key present multiple times", + "type": "UnprocessableEntityException", + "code": 422, + }, + } + + UpdateAndRemoveNamespacePropertiesRequest: + summary: An update namespace properties request with both properties to remove and properties to upsert. + value: { "removals": ["foo", "bar"], "updates": { "owner": "Raoul" } } + + securitySchemes: + OAuth2: + type: oauth2 + description: This scheme is used for OAuth2 authorization. + + + For unauthorized requests, services should return an appropriate 401 or + 403 response. Implementations must not return altered success (200) + responses when a request is unauthenticated or unauthorized. + + If a separate authorization server is used, substitute the tokenUrl with + the full token path of the external authorization server, and use the + resulting token to access the resources defined in the spec. + flows: + clientCredentials: + tokenUrl: /v1/oauth/tokens + scopes: + catalog: Allows interacting with the Config and Catalog APIs + BearerAuth: + type: http + scheme: bearer From 3ea3cf667e61bd2f152fd10b9d0f2dd23c9c7bb2 Mon Sep 17 00:00:00 2001 From: zhouzilong <529620861@qq.com> Date: Tue, 30 Jan 2024 14:28:20 -0500 Subject: [PATCH 2/4] remove authentication error code --- doc/openapi/rest-catalog-open-api.yaml | 148 ------------------------- 1 file changed, 148 deletions(-) diff --git a/doc/openapi/rest-catalog-open-api.yaml b/doc/openapi/rest-catalog-open-api.yaml index b86aa8f..7e9fe4f 100644 --- a/doc/openapi/rest-catalog-open-api.yaml +++ b/doc/openapi/rest-catalog-open-api.yaml @@ -111,12 +111,6 @@ paths: } 400: $ref: "#/components/responses/BadRequestErrorResponse" - 401: - $ref: "#/components/responses/UnauthorizedResponse" - 403: - $ref: "#/components/responses/ForbiddenResponse" - 419: - $ref: "#/components/responses/AuthenticationTimeoutResponse" 503: $ref: "#/components/responses/ServiceUnavailableResponse" 5XX: @@ -155,10 +149,6 @@ paths: $ref: "#/components/responses/ListNamespacesResponse" 400: $ref: "#/components/responses/BadRequestErrorResponse" - 401: - $ref: "#/components/responses/UnauthorizedResponse" - 403: - $ref: "#/components/responses/ForbiddenResponse" 404: description: Not Found - Namespace provided in the `parent` query parameter is not found. content: @@ -168,8 +158,6 @@ paths: examples: NoSuchNamespaceExample: $ref: "#/components/examples/NoSuchNamespaceError" - 419: - $ref: "#/components/responses/AuthenticationTimeoutResponse" 503: $ref: "#/components/responses/ServiceUnavailableResponse" 5XX: @@ -192,10 +180,6 @@ paths: $ref: "#/components/responses/CreateNamespaceResponse" 400: $ref: "#/components/responses/BadRequestErrorResponse" - 401: - $ref: "#/components/responses/UnauthorizedResponse" - 403: - $ref: "#/components/responses/ForbiddenResponse" 406: $ref: "#/components/responses/UnsupportedOperationResponse" 409: @@ -207,8 +191,6 @@ paths: examples: NamespaceAlreadyExists: $ref: "#/components/examples/NamespaceAlreadyExistsError" - 419: - $ref: "#/components/responses/AuthenticationTimeoutResponse" 503: $ref: "#/components/responses/ServiceUnavailableResponse" 5XX: @@ -230,10 +212,6 @@ paths: $ref: "#/components/responses/GetNamespaceResponse" 400: $ref: "#/components/responses/BadRequestErrorResponse" - 401: - $ref: "#/components/responses/UnauthorizedResponse" - 403: - $ref: "#/components/responses/ForbiddenResponse" 404: description: Not Found - Namespace not found content: @@ -243,8 +221,6 @@ paths: examples: NoSuchNamespaceExample: $ref: "#/components/examples/NoSuchNamespaceError" - 419: - $ref: "#/components/responses/AuthenticationTimeoutResponse" 503: $ref: "#/components/responses/ServiceUnavailableResponse" 5XX: @@ -261,10 +237,6 @@ paths: description: Success, no content 400: $ref: "#/components/responses/BadRequestErrorResponse" - 401: - $ref: "#/components/responses/UnauthorizedResponse" - 403: - $ref: "#/components/responses/ForbiddenResponse" 404: description: Not Found - Namespace not found content: @@ -274,8 +246,6 @@ paths: examples: NoSuchNamespaceExample: $ref: "#/components/examples/NoSuchNamespaceError" - 419: - $ref: "#/components/responses/AuthenticationTimeoutResponse" 503: $ref: "#/components/responses/ServiceUnavailableResponse" 5XX: @@ -291,10 +261,6 @@ paths: description: Success, no content 400: $ref: "#/components/responses/BadRequestErrorResponse" - 401: - $ref: "#/components/responses/UnauthorizedResponse" - 403: - $ref: "#/components/responses/ForbiddenResponse" 404: description: Not Found - Namespace to delete does not exist. content: @@ -304,8 +270,6 @@ paths: examples: NoSuchNamespaceExample: $ref: "#/components/examples/NoSuchNamespaceError" - 419: - $ref: "#/components/responses/AuthenticationTimeoutResponse" 503: $ref: "#/components/responses/ServiceUnavailableResponse" 5XX: @@ -341,10 +305,6 @@ paths: $ref: "#/components/responses/UpdateNamespacePropertiesResponse" 400: $ref: "#/components/responses/BadRequestErrorResponse" - 401: - $ref: "#/components/responses/UnauthorizedResponse" - 403: - $ref: "#/components/responses/ForbiddenResponse" 404: description: Not Found - Namespace not found content: @@ -365,8 +325,6 @@ paths: examples: UnprocessableEntityDuplicateKey: $ref: "#/components/examples/UnprocessableEntityDuplicateKey" - 419: - $ref: "#/components/responses/AuthenticationTimeoutResponse" 503: $ref: "#/components/responses/ServiceUnavailableResponse" 5XX: @@ -388,10 +346,6 @@ paths: $ref: "#/components/responses/ListTablesResponse" 400: $ref: "#/components/responses/BadRequestErrorResponse" - 401: - $ref: "#/components/responses/UnauthorizedResponse" - 403: - $ref: "#/components/responses/ForbiddenResponse" 404: description: Not Found - The namespace specified does not exist content: @@ -401,8 +355,6 @@ paths: examples: NamespaceNotFound: $ref: "#/components/examples/NoSuchNamespaceError" - 419: - $ref: "#/components/responses/AuthenticationTimeoutResponse" 503: $ref: "#/components/responses/ServiceUnavailableResponse" 5XX: @@ -436,10 +388,6 @@ paths: $ref: "#/components/responses/CreateTableResponse" 400: $ref: "#/components/responses/BadRequestErrorResponse" - 401: - $ref: "#/components/responses/UnauthorizedResponse" - 403: - $ref: "#/components/responses/ForbiddenResponse" 404: description: Not Found - The namespace specified does not exist content: @@ -458,8 +406,6 @@ paths: examples: NamespaceAlreadyExists: $ref: "#/components/examples/TableAlreadyExistsError" - 419: - $ref: "#/components/responses/AuthenticationTimeoutResponse" 503: $ref: "#/components/responses/ServiceUnavailableResponse" 5XX: @@ -487,10 +433,6 @@ paths: $ref: "#/components/responses/LoadTableResponse" 400: $ref: "#/components/responses/BadRequestErrorResponse" - 401: - $ref: "#/components/responses/UnauthorizedResponse" - 403: - $ref: "#/components/responses/ForbiddenResponse" 404: description: Not Found - The namespace specified does not exist content: @@ -509,8 +451,6 @@ paths: examples: NamespaceAlreadyExists: $ref: "#/components/examples/TableAlreadyExistsError" - 419: - $ref: "#/components/responses/AuthenticationTimeoutResponse" 503: $ref: "#/components/responses/ServiceUnavailableResponse" 5XX: @@ -560,10 +500,6 @@ paths: $ref: "#/components/responses/LoadTableResponse" 400: $ref: "#/components/responses/BadRequestErrorResponse" - 401: - $ref: "#/components/responses/UnauthorizedResponse" - 403: - $ref: "#/components/responses/ForbiddenResponse" 404: description: Not Found - NoSuchTableException, table to load does not exist content: @@ -573,8 +509,6 @@ paths: examples: TableToLoadDoesNotExist: $ref: "#/components/examples/NoSuchTableError" - 419: - $ref: "#/components/responses/AuthenticationTimeoutResponse" 503: $ref: "#/components/responses/ServiceUnavailableResponse" 5XX: @@ -612,10 +546,6 @@ paths: $ref: "#/components/responses/CommitTableResponse" 400: $ref: "#/components/responses/BadRequestErrorResponse" - 401: - $ref: "#/components/responses/UnauthorizedResponse" - 403: - $ref: "#/components/responses/ForbiddenResponse" 404: description: Not Found - NoSuchTableException, table to load does not exist content: @@ -631,8 +561,6 @@ paths: application/json: schema: $ref: "#/components/schemas/IcebergErrorResponse" - 419: - $ref: "#/components/responses/AuthenticationTimeoutResponse" 500: description: An unknown server-side problem occurred; the commit state is unknown. content: @@ -715,10 +643,6 @@ paths: description: Success, no content 400: $ref: "#/components/responses/BadRequestErrorResponse" - 401: - $ref: "#/components/responses/UnauthorizedResponse" - 403: - $ref: "#/components/responses/ForbiddenResponse" 404: description: Not Found - NoSuchTableException, Table to drop does not exist content: @@ -728,8 +652,6 @@ paths: examples: TableToDeleteDoesNotExist: $ref: "#/components/examples/NoSuchTableError" - 419: - $ref: "#/components/responses/AuthenticationTimeoutResponse" 503: $ref: "#/components/responses/ServiceUnavailableResponse" 5XX: @@ -746,10 +668,6 @@ paths: description: Success, no content 400: $ref: "#/components/responses/BadRequestErrorResponse" - 401: - $ref: "#/components/responses/UnauthorizedResponse" - 403: - $ref: "#/components/responses/ForbiddenResponse" 404: description: Not Found - NoSuchTableException, Table not found content: @@ -759,8 +677,6 @@ paths: examples: TableToLoadDoesNotExist: $ref: "#/components/examples/NoSuchTableError" - 419: - $ref: "#/components/responses/AuthenticationTimeoutResponse" 503: $ref: "#/components/responses/ServiceUnavailableResponse" 5XX: @@ -793,10 +709,6 @@ paths: description: Success, no content 400: $ref: "#/components/responses/BadRequestErrorResponse" - 401: - $ref: "#/components/responses/UnauthorizedResponse" - 403: - $ref: "#/components/responses/ForbiddenResponse" 404: description: Not Found - NoSuchTableException, Table to rename does not exist @@ -820,8 +732,6 @@ paths: $ref: "#/components/schemas/IcebergErrorResponse" example: $ref: "#/components/examples/TableAlreadyExistsError" - 419: - $ref: "#/components/responses/AuthenticationTimeoutResponse" 503: $ref: "#/components/responses/ServiceUnavailableResponse" 5XX: @@ -850,10 +760,6 @@ paths: description: Success, no content 400: $ref: "#/components/responses/BadRequestErrorResponse" - 401: - $ref: "#/components/responses/UnauthorizedResponse" - 403: - $ref: "#/components/responses/ForbiddenResponse" 404: description: Not Found - NoSuchTableException, table to load does not exist content: @@ -863,8 +769,6 @@ paths: examples: TableToLoadDoesNotExist: $ref: "#/components/examples/NoSuchTableError" - 419: - $ref: "#/components/responses/AuthenticationTimeoutResponse" 503: $ref: "#/components/responses/ServiceUnavailableResponse" 5XX: @@ -2316,42 +2220,6 @@ components: }, } - # Note that this is a representative example response for use as a shorthand in the spec. - # The fields `message` and `type` as indicated here are not presently prescriptive. - UnauthorizedResponse: - description: Unauthorized. Authentication is required and has failed or has not yet been provided. - content: - application/json: - schema: - $ref: "#/components/schemas/IcebergErrorResponse" - example: - { - "error": - { - "message": "Not authorized to make this request", - "type": "NotAuthorizedException", - "code": 401, - }, - } - - # Note that this is a representative example response for use as a shorthand in the spec. - # The fields `message` and `type` as indicated here are not presently prescriptive. - ForbiddenResponse: - description: Forbidden. Authenticated user does not have the necessary permissions. - content: - application/json: - schema: - $ref: "#/components/schemas/IcebergErrorResponse" - example: - { - "error": - { - "message": "Not authorized to make this request", - "type": "NotAuthorizedException", - "code": 403, - }, - } - # Note that this is a representative example response for use as a shorthand in the spec. # The fields `message` and `type` as indicated here are not presently prescriptive. UnsupportedOperationResponse: @@ -2434,22 +2302,6 @@ components: EmptyResponse: $ref: "#/components/examples/ListNamespacesEmptyExample" - AuthenticationTimeoutResponse: - description: Credentials have timed out. If possible, the client should refresh credentials and retry. - content: - application/json: - schema: - $ref: "#/components/schemas/IcebergErrorResponse" - example: - { - "error": - { - "message": "Credentials have timed out", - "type": "AuthenticationTimeoutException", - "code": 419, - }, - } - ServiceUnavailableResponse: description: The service is not ready to handle the request. The client should wait and retry. From 5bc657ccaf7aab32517026061026d35b5cf43d87 Mon Sep 17 00:00:00 2001 From: Yen-Ju Wu Date: Tue, 30 Jan 2024 20:45:01 -0500 Subject: [PATCH 3/4] add design rationale and tradeoff to proposal --- doc/design_doc.md | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/doc/design_doc.md b/doc/design_doc.md index 547ab95..4fa5207 100644 --- a/doc/design_doc.md +++ b/doc/design_doc.md @@ -13,6 +13,22 @@ ## Design Rationale >Explain the goals of this design and how the design achieves these goals. Present alternatives considered and document why they are not chosen. +* Correctness: + * The catalog service ensures data consistency and correctness by adhering to the Iceberg Catalog REST interface. + * Data durability mechanisms will be implemented to prevent data loss during restarts. +* Performance: + * Optimized data retrieval and storage strategies to minimize latency in metadata access. + * Efficient indexing mechanisms to speed up metadata search operations. + * Caching strategies to reduce redundant metadata queries and enhance overall performance. +* Engineering Complexity / Maintainability: + * The microservices architecture simplifies maintenance by encapsulating each building block. + * Code modularity and clear interfaces facilitate easier updates and improvements. +* Testing: + * Comprehensive testing plans cover correctness through unit tests and performance through long-running regression tests. Unit tests focus on individual components of the catalog service, while regression tests evaluate system-wide performance and stability. +* Other Implementations: + * Considered alternative implementations, including a monolithic catalog structure and alternative REST specifications. + * Chose the Iceberg Catalog REST interface due to its industry adoption, standardization, and compatibility with various systems. + ## Testing Plan >How should the component be tested? @@ -26,10 +42,11 @@ To ensure the quality and the performance of the catalog implemented, a comprehe * Performance evaluation: We will set up a performance baseline to compare the performance of our implementation. We can measure different metrics, for example, response time, throughput, etc. * Scalability test: We will try to test our implementation under increased load and ensure the correctness and efficiency at the same time. - - ## Trade-offs and Potential Problems >Write down any conscious trade-off you made that can be problematic in the future, or any problems discovered during the design process that remain unaddressed (technical debts). +* Balancing between metadata retrieval speed and storage efficiency. +* Striking a balance between query performance and resource utilization. + ## Glossary (Optional) >If you are introducing new concepts or giving unintuitive names to components, write them down here. From 08741936a06e47cddee325eec44842754b91f28d Mon Sep 17 00:00:00 2001 From: yenjuw <107606227+yenjuw@users.noreply.github.com> Date: Wed, 31 Jan 2024 11:00:54 +0800 Subject: [PATCH 4/4] remove default descriptions --- doc/design_doc.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/doc/design_doc.md b/doc/design_doc.md index 4fa5207..cdb5871 100644 --- a/doc/design_doc.md +++ b/doc/design_doc.md @@ -11,8 +11,6 @@ >Explain the input and output of the component, describe interactions and breakdown the smaller components if any. Include diagrams if appropriate. ## Design Rationale ->Explain the goals of this design and how the design achieves these goals. Present alternatives considered and document why they are not chosen. - * Correctness: * The catalog service ensures data consistency and correctness by adhering to the Iceberg Catalog REST interface. * Data durability mechanisms will be implemented to prevent data loss during restarts. @@ -30,8 +28,6 @@ * Chose the Iceberg Catalog REST interface due to its industry adoption, standardization, and compatibility with various systems. ## Testing Plan ->How should the component be tested? - To ensure the quality and the performance of the catalog implemented, a comprehensive testing strategy is a must. Our testing strategy will include both functional and non-functional aspects of our catalog service. * Functional testing @@ -43,8 +39,6 @@ To ensure the quality and the performance of the catalog implemented, a comprehe * Scalability test: We will try to test our implementation under increased load and ensure the correctness and efficiency at the same time. ## Trade-offs and Potential Problems ->Write down any conscious trade-off you made that can be problematic in the future, or any problems discovered during the design process that remain unaddressed (technical debts). - * Balancing between metadata retrieval speed and storage efficiency. * Striking a balance between query performance and resource utilization.