From cd4031cf019aea221628b6f7ccaa5fb1ed0f4913 Mon Sep 17 00:00:00 2001 From: vikas singh Date: Tue, 6 Jun 2023 14:42:27 +0530 Subject: [PATCH 1/4] Added CosmosDiagnostics to query. (#25786) ### Packages impacted by this PR @azure/cosmos ### Issues associated with this PR ### Describe the problem that is addressed by this PR This PR adds Diagnostics to query apis. ### What are the possible designs available to address the problem? If there are more than one possible design, why was the one in this PR chosen? ### Are there test cases added in this PR? _(If not, why?)_ No, the PR is a bit large, I will raise a separate PR containing tests for diagnostics feature. ### Provide a list of related PRs _(if any)_ ### Command used to generate this PR:**_(Applicable only to SDK release request PRs)_ ### Checklists - [ ] Added impacted package name to the issue description - [ ] Does this PR needs any fixes in the SDK Generator?** _(If so, create an Issue in the [Autorest/typescript](https://github.com/Azure/autorest.typescript) repository and link it here)_ - [ ] Added a changelog (if necessary) --------- Co-authored-by: Vikas Singh --- sdk/cosmosdb/cosmos/review/cosmos.api.md | 23 ++++++---- sdk/cosmosdb/cosmos/src/ClientContext.ts | 16 +++++-- .../cosmos/src/CosmosDiagnosticsContext.ts | 19 ++++++++ sdk/cosmosdb/cosmos/src/client/ClientUtils.ts | 19 ++++++++ .../cosmos/src/client/Container/Container.ts | 8 +++- sdk/cosmosdb/cosmos/src/client/Item/Item.ts | 34 +++++++++----- sdk/cosmosdb/cosmos/src/client/Item/Items.ts | 40 ++++++++++++---- .../client/StoredProcedure/StoredProcedure.ts | 10 ++-- sdk/cosmosdb/cosmos/src/index.ts | 1 + .../GroupByEndpointComponent.ts | 19 ++++++-- .../GroupByValueEndpointComponent.ts | 25 ++++++++-- .../OffsetLimitEndpointComponent.ts | 11 +++-- .../OrderByEndpointComponent.ts | 3 +- .../OrderedDistinctEndpointComponent.ts | 6 +-- .../UnorderedDistinctEndpointComponent.ts | 6 +-- .../defaultQueryExecutionContext.ts | 46 ++++++++++++++----- .../queryExecutionContext/documentProducer.ts | 39 +++++++++++----- .../orderByQueryExecutionContext.ts | 13 +++++- .../parallelQueryExecutionContextBase.ts | 25 ++++++++-- .../pipelinedQueryExecutionContext.ts | 17 ++++--- sdk/cosmosdb/cosmos/src/queryIterator.ts | 17 +++++-- .../cosmos/src/request/FeedResponse.ts | 5 +- .../cosmos/src/request/ResourceResponse.ts | 2 +- sdk/cosmosdb/cosmos/src/request/Response.ts | 2 +- .../src/routing/partitionKeyRangeCache.ts | 20 +++++--- .../src/routing/smartRoutingMapProvider.ts | 7 ++- sdk/cosmosdb/cosmos/src/utils/batch.ts | 4 +- .../unit/defaultQueryExecutionContext.spec.ts | 16 ++++++- .../unit/smartRoutingMapProvider.spec.ts | 31 +++++++++++-- .../test/public/functional/client.spec.ts | 1 - .../functional/globalEndpointManager.spec.ts | 13 ++++-- .../public/integration/multiregion.spec.ts | 7 ++- sdk/cosmosdb/cosmos/tsconfig.strict.json | 1 + 33 files changed, 386 insertions(+), 120 deletions(-) create mode 100644 sdk/cosmosdb/cosmos/src/client/ClientUtils.ts diff --git a/sdk/cosmosdb/cosmos/review/cosmos.api.md b/sdk/cosmosdb/cosmos/review/cosmos.api.md index 06725040f63c..0a5c1fda825d 100644 --- a/sdk/cosmosdb/cosmos/review/cosmos.api.md +++ b/sdk/cosmosdb/cosmos/review/cosmos.api.md @@ -32,6 +32,11 @@ export interface Agent { // @public (undocumented) export type AggregateType = "Average" | "Count" | "Max" | "Min" | "Sum"; +// @public (undocumented) +export type BulkOperationResponse = OperationResponse[] & { + diagnostics: CosmosDiagnostics; +}; + // @public (undocumented) export const BulkOperationType: { readonly Create: "Create"; @@ -175,7 +180,7 @@ export class ClientContext { diagnosticContext?: CosmosDiagnosticContext; }): Promise>; // (undocumented) - queryPartitionKeyRanges(collectionLink: string, query?: string | SqlQuerySpec, options?: FeedOptions): QueryIterator; + queryPartitionKeyRanges(collectionLink: string, diagnosticContext: CosmosDiagnosticContext, query?: string | SqlQuerySpec, options?: FeedOptions): QueryIterator; // (undocumented) read({ path, resourceType, resourceId, options, partitionKey, diagnosticContext, }: { path: string; @@ -814,7 +819,7 @@ export interface FeedOptions extends SharedOptions { // @public (undocumented) export class FeedResponse { - constructor(resources: TResource[], headers: CosmosHeaders, hasMoreResults: boolean); + constructor(resources: TResource[], headers: CosmosHeaders, hasMoreResults: boolean, diagnostics: CosmosDiagnostics); // (undocumented) get activityId(): string; // (undocumented) @@ -822,6 +827,8 @@ export class FeedResponse { // (undocumented) get continuationToken(): string; // (undocumented) + readonly diagnostics: CosmosDiagnostics; + // (undocumented) readonly hasMoreResults: boolean; // (undocumented) get indexMetrics(): string; @@ -958,7 +965,7 @@ export class ItemResponse extends ResourceResponse>; - bulk(operations: OperationInput[], bulkOptions?: BulkOptions, options?: RequestOptions): Promise; + bulk(operations: OperationInput[], bulkOptions?: BulkOptions, options?: RequestOptions): Promise; changeFeed(partitionKey: string | number | boolean, changeFeedOptions?: ChangeFeedOptions): ChangeFeedIterator; changeFeed(changeFeedOptions?: ChangeFeedOptions): ChangeFeedIterator; changeFeed(partitionKey: string | number | boolean, changeFeedOptions?: ChangeFeedOptions): ChangeFeedIterator; @@ -1333,7 +1340,7 @@ export interface QueryInfo { // @public export class QueryIterator { // Warning: (ae-forgotten-export) The symbol "FetchFunctionCallback" needs to be exported by the entry point index.d.ts - constructor(clientContext: ClientContext, query: SqlQuerySpec | string, options: FeedOptions, fetchFunctions: FetchFunctionCallback | FetchFunctionCallback[], resourceLink?: string, resourceType?: ResourceType); + constructor(clientContext: ClientContext, query: SqlQuerySpec | string, options: FeedOptions, fetchFunctions: FetchFunctionCallback | FetchFunctionCallback[], diagnosticContext?: CosmosDiagnosticContext, resourceLink?: string, resourceType?: ResourceType); fetchAll(): Promise>; fetchNext(): Promise>; getAsyncIterator(): AsyncIterable>; @@ -1588,11 +1595,11 @@ export interface Resource { // @public (undocumented) export class ResourceResponse { - constructor(resource: TResource | undefined, headers: CosmosHeaders_2, statusCode: StatusCode, diagnostics?: CosmosDiagnostics, substatus?: SubStatusCode); + constructor(resource: TResource | undefined, headers: CosmosHeaders_2, statusCode: StatusCode, diagnostics: CosmosDiagnostics, substatus?: SubStatusCode); // (undocumented) get activityId(): string; // (undocumented) - readonly diagnostics?: CosmosDiagnostics; + readonly diagnostics: CosmosDiagnostics; // (undocumented) get etag(): string; // (undocumented) @@ -1642,7 +1649,7 @@ interface Response_2 { // (undocumented) code?: number; // (undocumented) - diagnostics?: CosmosDiagnostics; + diagnostics: CosmosDiagnostics; // (undocumented) headers: CosmosHeaders; // (undocumented) @@ -2202,7 +2209,7 @@ export class Users { // Warnings were encountered during analysis: // -// src/ClientContext.ts:90:5 - (ae-forgotten-export) The symbol "CosmosDiagnosticContext" needs to be exported by the entry point index.d.ts +// src/ClientContext.ts:91:5 - (ae-forgotten-export) The symbol "CosmosDiagnosticContext" needs to be exported by the entry point index.d.ts // (No @packageDocumentation comment for this package) diff --git a/sdk/cosmosdb/cosmos/src/ClientContext.ts b/sdk/cosmosdb/cosmos/src/ClientContext.ts index b1002c47868e..b2bd6d2c26ba 100644 --- a/sdk/cosmosdb/cosmos/src/ClientContext.ts +++ b/sdk/cosmosdb/cosmos/src/ClientContext.ts @@ -31,6 +31,7 @@ import { BulkOptions } from "./utils/batch"; import { sanitizeEndpoint } from "./utils/checkURL"; import { AzureLogger, createClientLogger } from "@azure/logger"; import { CosmosDiagnosticContext } from "./CosmosDiagnosticsContext"; +import { MetadataLookUpType } from "./CosmosDiagnostics"; const logger: AzureLogger = createClientLogger("ClientContext"); @@ -229,13 +230,17 @@ export class ClientContext { public queryPartitionKeyRanges( collectionLink: string, + diagnosticContext: CosmosDiagnosticContext, query?: string | SqlQuerySpec, options?: FeedOptions ): QueryIterator { const path = getPathFromLink(collectionLink, ResourceType.pkranges); const id = getIdFromLink(collectionLink); - const cb: FetchFunctionCallback = (innerOptions) => { - return this.queryFeed({ + const cb: FetchFunctionCallback = async ( + innerOptions, + diagnosticCtx: CosmosDiagnosticContext + ) => { + const response = await this.queryFeed({ path, resourceType: ResourceType.pkranges, resourceId: id, @@ -243,8 +248,13 @@ export class ClientContext { query, options: innerOptions, }); + diagnosticCtx.recordMetaDataLookup( + response.diagnostics, + MetadataLookUpType.PartitionKeyRangeLookUp + ); + return response; }; - return new QueryIterator(this, query, options, cb); + return new QueryIterator(this, query, options, cb, diagnosticContext); } public async delete({ diff --git a/sdk/cosmosdb/cosmos/src/CosmosDiagnosticsContext.ts b/sdk/cosmosdb/cosmos/src/CosmosDiagnosticsContext.ts index b9e6276d0071..f09feafe17af 100644 --- a/sdk/cosmosdb/cosmos/src/CosmosDiagnosticsContext.ts +++ b/sdk/cosmosdb/cosmos/src/CosmosDiagnosticsContext.ts @@ -95,6 +95,25 @@ export class CosmosDiagnosticContext { this.locationEndpointsContacted.set(location.databaseAccountEndpoint, location); } + public reset(): void { + this.requestStartTimeUTCinMs = getCurrentTimestampInMs(); + this.requestEndTimeUTCinMs = getCurrentTimestampInMs(); + this.retryStartTimeUTCinMs = 0; + this.headers = {}; + this.retryAttemptNumber = 0; + this.failedAttempts = []; + this.metadataLookups = []; + this.requestPayloadLength = 0; + this.responsePayloadLength = 0; + this.locationEndpointsContacted = new Map(); + } + + public resetAndGetDiagnostics(): CosmosDiagnostics { + const diagnostic = this.getDiagnostics(); + this.reset(); + return diagnostic; + } + public getDiagnostics(): CosmosDiagnostics { this.recordSessionEnd(); return new CosmosDiagnostics( diff --git a/sdk/cosmosdb/cosmos/src/client/ClientUtils.ts b/sdk/cosmosdb/cosmos/src/client/ClientUtils.ts new file mode 100644 index 000000000000..88f311990fe7 --- /dev/null +++ b/sdk/cosmosdb/cosmos/src/client/ClientUtils.ts @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { MetadataLookUpType } from "../CosmosDiagnostics"; +import { CosmosDiagnosticContext } from "../CosmosDiagnosticsContext"; +import { PartitionKeyDefinition } from "../documents"; +import { Container } from "./Container"; + +export async function readAndRecordPartitionKeyDefinition(container: Container): Promise<{ + diagnosticContext: CosmosDiagnosticContext; + partitionKeyDefinition: PartitionKeyDefinition; +}> { + const diagnosticContext: CosmosDiagnosticContext = new CosmosDiagnosticContext(); + + const { resource: partitionKeyDefinition, diagnostics } = + await container.readPartitionKeyDefinition(); + diagnosticContext.recordMetaDataLookup(diagnostics, MetadataLookUpType.PartitionKeyRangeLookUp); + return { diagnosticContext, partitionKeyDefinition }; +} diff --git a/sdk/cosmosdb/cosmos/src/client/Container/Container.ts b/sdk/cosmosdb/cosmos/src/client/Container/Container.ts index e27ebf3c8e11..da9dd4102aa6 100644 --- a/sdk/cosmosdb/cosmos/src/client/Container/Container.ts +++ b/sdk/cosmosdb/cosmos/src/client/Container/Container.ts @@ -25,6 +25,7 @@ import { Offer, OfferDefinition } from "../Offer"; import { OfferResponse } from "../Offer/OfferResponse"; import { Resource } from "../Resource"; import { getEmptyCosmosDiagnostics } from "../../CosmosDiagnostics"; +import { CosmosDiagnosticContext } from "../../CosmosDiagnosticsContext"; /** * Operations for reading, replacing, or deleting a specific, existing container by id. @@ -266,7 +267,12 @@ export class Container { public readPartitionKeyRanges(feedOptions?: FeedOptions): QueryIterator { feedOptions = feedOptions || {}; - return this.clientContext.queryPartitionKeyRanges(this.url, undefined, feedOptions); + return this.clientContext.queryPartitionKeyRanges( + this.url, + new CosmosDiagnosticContext(), + undefined, + feedOptions + ); } /** diff --git a/sdk/cosmosdb/cosmos/src/client/Item/Item.ts b/sdk/cosmosdb/cosmos/src/client/Item/Item.ts index b977cf016c14..aa5094628391 100644 --- a/sdk/cosmosdb/cosmos/src/client/Item/Item.ts +++ b/sdk/cosmosdb/cosmos/src/client/Item/Item.ts @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. import { ClientContext } from "../../ClientContext"; +import { CosmosDiagnosticContext } from "../../CosmosDiagnosticsContext"; import { createDocumentUri, getIdFromLink, @@ -13,6 +14,7 @@ import { PartitionKey } from "../../documents"; import { extractPartitionKey, undefinedPartitionKey } from "../../extractPartitionKey"; import { RequestOptions, Response } from "../../request"; import { PatchRequestBody } from "../../utils/patch"; +import { readAndRecordPartitionKeyDefinition } from "../ClientUtils"; import { Container } from "../Container"; import { Resource } from "../Resource"; import { ItemDefinition } from "./ItemDefinition"; @@ -74,10 +76,11 @@ export class Item { public async read( options: RequestOptions = {} ): Promise> { + let diagnosticContext: CosmosDiagnosticContext; if (this.partitionKey === undefined) { - const { resource: partitionKeyDefinition } = - await this.container.readPartitionKeyDefinition(); - this.partitionKey = undefinedPartitionKey(partitionKeyDefinition); + const partitionKeyResponse = await readAndRecordPartitionKeyDefinition(this.container); + this.partitionKey = undefinedPartitionKey(partitionKeyResponse.partitionKeyDefinition); + diagnosticContext = partitionKeyResponse.diagnosticContext; } const path = getPathFromLink(this.url); @@ -90,6 +93,7 @@ export class Item { resourceId: id, options, partitionKey: this.partitionKey, + diagnosticContext, }); } catch (error: any) { if (error.code !== StatusCodes.NotFound) { @@ -139,10 +143,11 @@ export class Item { body: T, options: RequestOptions = {} ): Promise> { + let diagnosticContext: CosmosDiagnosticContext; if (this.partitionKey === undefined) { - const { resource: partitionKeyDefinition } = - await this.container.readPartitionKeyDefinition(); - this.partitionKey = extractPartitionKey(body, partitionKeyDefinition); + const partitionKeyResponse = await readAndRecordPartitionKeyDefinition(this.container); + this.partitionKey = extractPartitionKey(body, partitionKeyResponse.partitionKeyDefinition); + diagnosticContext = partitionKeyResponse.diagnosticContext; } const err = {}; @@ -160,6 +165,7 @@ export class Item { resourceId: id, options, partitionKey: this.partitionKey, + diagnosticContext, }); return new ItemResponse( response.result, @@ -182,10 +188,11 @@ export class Item { public async delete( options: RequestOptions = {} ): Promise> { + let diagnosticContext: CosmosDiagnosticContext; if (this.partitionKey === undefined) { - const { resource: partitionKeyDefinition } = - await this.container.readPartitionKeyDefinition(); - this.partitionKey = undefinedPartitionKey(partitionKeyDefinition); + const partitionKeyResponse = await readAndRecordPartitionKeyDefinition(this.container); + this.partitionKey = undefinedPartitionKey(partitionKeyResponse.partitionKeyDefinition); + diagnosticContext = partitionKeyResponse.diagnosticContext; } const path = getPathFromLink(this.url); @@ -197,6 +204,7 @@ export class Item { resourceId: id, options, partitionKey: this.partitionKey, + diagnosticContext, }); return new ItemResponse( response.result, @@ -220,10 +228,11 @@ export class Item { body: PatchRequestBody, options: RequestOptions = {} ): Promise> { + let diagnosticContext: CosmosDiagnosticContext; if (this.partitionKey === undefined) { - const { resource: partitionKeyDefinition } = - await this.container.readPartitionKeyDefinition(); - this.partitionKey = extractPartitionKey(body, partitionKeyDefinition); + const partitionKeyResponse = await readAndRecordPartitionKeyDefinition(this.container); + this.partitionKey = extractPartitionKey(body, partitionKeyResponse.partitionKeyDefinition); + diagnosticContext = partitionKeyResponse.diagnosticContext; } const path = getPathFromLink(this.url); @@ -236,6 +245,7 @@ export class Item { resourceId: id, options, partitionKey: this.partitionKey, + diagnosticContext, }); return new ItemResponse( response.result, diff --git a/sdk/cosmosdb/cosmos/src/client/Item/Items.ts b/sdk/cosmosdb/cosmos/src/client/Item/Items.ts index 743f5c96739a..4c3ecbb02aed 100644 --- a/sdk/cosmosdb/cosmos/src/client/Item/Items.ts +++ b/sdk/cosmosdb/cosmos/src/client/Item/Items.ts @@ -25,9 +25,12 @@ import { BulkOptions, decorateBatchOperation, splitBatchBasedOnBodySize, + BulkOperationResponse, } from "../../utils/batch"; import { hashV1PartitionKey } from "../../utils/hashing/v1"; import { hashV2PartitionKey } from "../../utils/hashing/v2"; +import { CosmosDiagnosticContext } from "../../CosmosDiagnosticsContext"; +import { readAndRecordPartitionKeyDefinition } from "../ClientUtils"; /** * @hidden @@ -91,8 +94,11 @@ export class Items { const path = getPathFromLink(this.container.url, ResourceType.item); const id = getIdFromLink(this.container.url); - const fetchFunction: FetchFunctionCallback = (innerOptions: FeedOptions) => { - return this.clientContext.queryFeed({ + const fetchFunction: FetchFunctionCallback = async ( + innerOptions: FeedOptions, + diagnosticContext: CosmosDiagnosticContext + ) => { + const response = await this.clientContext.queryFeed({ path, resourceType: ResourceType.item, resourceId: id, @@ -101,6 +107,8 @@ export class Items { options: innerOptions, partitionKey: options.partitionKey, }); + diagnosticContext.mergeDiagnostics(response.diagnostics); + return response; }; return new QueryIterator( @@ -108,6 +116,7 @@ export class Items { query, options, fetchFunction, + new CosmosDiagnosticContext(), this.container.url, ResourceType.item ); @@ -266,7 +275,9 @@ export class Items { body.id = uuid(); } - const { resource: partitionKeyDefinition } = await this.container.readPartitionKeyDefinition(); + const { diagnosticContext, partitionKeyDefinition } = await readAndRecordPartitionKeyDefinition( + this.container + ); const partitionKey = extractPartitionKey(body, partitionKeyDefinition); const err = {}; @@ -284,6 +295,7 @@ export class Items { resourceId: id, options, partitionKey, + diagnosticContext, }); const ref = new Item( @@ -333,7 +345,9 @@ export class Items { body: T, options: RequestOptions = {} ): Promise> { - const { resource: partitionKeyDefinition } = await this.container.readPartitionKeyDefinition(); + const { diagnosticContext, partitionKeyDefinition } = await readAndRecordPartitionKeyDefinition( + this.container + ); const partitionKey = extractPartitionKey(body, partitionKeyDefinition); // Generate random document id if the id is missing in the payload and @@ -357,6 +371,7 @@ export class Items { resourceId: id, options, partitionKey, + diagnosticContext, }); const ref = new Item( @@ -407,11 +422,13 @@ export class Items { operations: OperationInput[], bulkOptions?: BulkOptions, options?: RequestOptions - ): Promise { + ): Promise { const { resources: partitionKeyRanges } = await this.container .readPartitionKeyRanges() .fetchAll(); - const { resource: definition } = await this.container.getPartitionKeyDefinition(); + const { diagnosticContext, partitionKeyDefinition } = await readAndRecordPartitionKeyDefinition( + this.container + ); const batches: Batch[] = partitionKeyRanges.map((keyRange: PartitionKeyRange) => { return { min: keyRange.minInclusive, @@ -422,10 +439,10 @@ export class Items { }; }); operations - .map((operation) => decorateOperation(operation, definition, options)) + .map((operation) => decorateOperation(operation, partitionKeyDefinition, options)) .forEach((operation: Operation, index: number) => { - const partitionProp = definition.paths[0].replace("/", ""); - const isV2 = definition.version && definition.version === 2; + const partitionProp = partitionKeyDefinition.paths[0].replace("/", ""); + const isV2 = partitionKeyDefinition.version && partitionKeyDefinition.version === 2; const toHashKey = getPartitionKeyToHash(operation, partitionProp); const hashed = isV2 ? hashV2PartitionKey(toHashKey) : hashV1PartitionKey(toHashKey); const batchForKey = batches.find((batch: Batch) => { @@ -455,6 +472,7 @@ export class Items { bulkOptions, options, }); + diagnosticContext.mergeDiagnostics(response.diagnostics); response.result.forEach((operationResponse: OperationResponse, index: number) => { orderedResponses[batch.indexes[index]] = operationResponse; }); @@ -471,7 +489,9 @@ export class Items { } }) ); - return orderedResponses; + const response: any = orderedResponses; + response.diagnostics = diagnosticContext.getDiagnostics(); + return response; } /** diff --git a/sdk/cosmosdb/cosmos/src/client/StoredProcedure/StoredProcedure.ts b/sdk/cosmosdb/cosmos/src/client/StoredProcedure/StoredProcedure.ts index 9e19def831e2..ad147ddefef7 100644 --- a/sdk/cosmosdb/cosmos/src/client/StoredProcedure/StoredProcedure.ts +++ b/sdk/cosmosdb/cosmos/src/client/StoredProcedure/StoredProcedure.ts @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. import { ClientContext } from "../../ClientContext"; +import { CosmosDiagnosticContext } from "../../CosmosDiagnosticsContext"; import { createStoredProcedureUri, getIdFromLink, @@ -11,6 +12,7 @@ import { import { PartitionKey } from "../../documents/PartitionKey"; import { undefinedPartitionKey } from "../../extractPartitionKey"; import { RequestOptions, ResourceResponse } from "../../request"; +import { readAndRecordPartitionKeyDefinition } from "../ClientUtils"; import { Container } from "../Container"; import { StoredProcedureDefinition } from "./StoredProcedureDefinition"; import { StoredProcedureResponse } from "./StoredProcedureResponse"; @@ -133,16 +135,18 @@ export class StoredProcedure { params?: any[], options?: RequestOptions ): Promise> { + let diagnosticContext: CosmosDiagnosticContext; if (partitionKey === undefined) { - const { resource: partitionKeyDefinition } = - await this.container.readPartitionKeyDefinition(); - partitionKey = undefinedPartitionKey(partitionKeyDefinition); + const partitionKeyResponse = await readAndRecordPartitionKeyDefinition(this.container); + diagnosticContext = partitionKeyResponse.diagnosticContext; + partitionKey = undefinedPartitionKey(partitionKeyResponse.partitionKeyDefinition); } const response = await this.clientContext.execute({ sprocLink: this.url, params, options, partitionKey, + diagnosticContext, }); return new ResourceResponse( response.result, diff --git a/sdk/cosmosdb/cosmos/src/index.ts b/sdk/cosmosdb/cosmos/src/index.ts index f668a6eb13f0..e7a9ad0089e2 100644 --- a/sdk/cosmosdb/cosmos/src/index.ts +++ b/sdk/cosmosdb/cosmos/src/index.ts @@ -17,6 +17,7 @@ export { OperationWithItem, OperationInput, BulkOperationType, + BulkOperationResponse, CreateOperationInput, UpsertOperationInput, ReplaceOperationInput, diff --git a/sdk/cosmosdb/cosmos/src/queryExecutionContext/EndpointComponent/GroupByEndpointComponent.ts b/sdk/cosmosdb/cosmos/src/queryExecutionContext/EndpointComponent/GroupByEndpointComponent.ts index 57f05acc8ba9..8c3ae2d16980 100644 --- a/sdk/cosmosdb/cosmos/src/queryExecutionContext/EndpointComponent/GroupByEndpointComponent.ts +++ b/sdk/cosmosdb/cosmos/src/queryExecutionContext/EndpointComponent/GroupByEndpointComponent.ts @@ -8,6 +8,7 @@ import { hashObject } from "../../utils/hashObject"; import { Aggregator, createAggregator } from "../Aggregators"; import { getInitialHeader, mergeHeaders } from "../headerUtils"; import { emptyGroup, extractAggregateResult } from "./emptyGroup"; +import { getEmptyCosmosDiagnostics } from "../../CosmosDiagnostics"; interface GroupByResponse { result: GroupByResult; @@ -30,11 +31,19 @@ export class GroupByEndpointComponent implements ExecutionContext { public async nextItem(): Promise> { // If we have a full result set, begin returning results if (this.aggregateResultArray.length > 0) { - return { result: this.aggregateResultArray.pop(), headers: getInitialHeader() }; + return { + result: this.aggregateResultArray.pop(), + headers: getInitialHeader(), + diagnostics: getEmptyCosmosDiagnostics(), + }; } if (this.completed) { - return { result: undefined, headers: getInitialHeader() }; + return { + result: undefined, + headers: getInitialHeader(), + diagnostics: getEmptyCosmosDiagnostics(), + }; } const aggregateHeaders = getInitialHeader(); @@ -88,7 +97,11 @@ export class GroupByEndpointComponent implements ExecutionContext { this.aggregateResultArray.push(groupResult); } this.completed = true; - return { result: this.aggregateResultArray.pop(), headers: aggregateHeaders }; + return { + result: this.aggregateResultArray.pop(), + headers: aggregateHeaders, + diagnostics: getEmptyCosmosDiagnostics(), + }; } public hasMoreResults(): boolean { diff --git a/sdk/cosmosdb/cosmos/src/queryExecutionContext/EndpointComponent/GroupByValueEndpointComponent.ts b/sdk/cosmosdb/cosmos/src/queryExecutionContext/EndpointComponent/GroupByValueEndpointComponent.ts index 0c54e304a824..e345a5a6c637 100644 --- a/sdk/cosmosdb/cosmos/src/queryExecutionContext/EndpointComponent/GroupByValueEndpointComponent.ts +++ b/sdk/cosmosdb/cosmos/src/queryExecutionContext/EndpointComponent/GroupByValueEndpointComponent.ts @@ -8,6 +8,7 @@ import { hashObject } from "../../utils/hashObject"; import { Aggregator, createAggregator } from "../Aggregators"; import { getInitialHeader, mergeHeaders } from "../headerUtils"; import { emptyGroup, extractAggregateResult } from "./emptyGroup"; +import { getEmptyCosmosDiagnostics } from "../../CosmosDiagnostics"; interface GroupByResponse { result: GroupByResult; @@ -34,11 +35,19 @@ export class GroupByValueEndpointComponent implements ExecutionContext { public async nextItem(): Promise> { // Start returning results if we have processed a full results set if (this.aggregateResultArray.length > 0) { - return { result: this.aggregateResultArray.pop(), headers: getInitialHeader() }; + return { + result: this.aggregateResultArray.pop(), + headers: getInitialHeader(), + diagnostics: getEmptyCosmosDiagnostics(), + }; } if (this.completed) { - return { result: undefined, headers: getInitialHeader() }; + return { + result: undefined, + headers: getInitialHeader(), + diagnostics: getEmptyCosmosDiagnostics(), + }; } const aggregateHeaders = getInitialHeader(); @@ -81,14 +90,22 @@ export class GroupByValueEndpointComponent implements ExecutionContext { // We bail early since we got an undefined result back `[{}]` if (this.completed) { - return { result: undefined, headers: aggregateHeaders }; + return { + result: undefined, + headers: aggregateHeaders, + diagnostics: getEmptyCosmosDiagnostics(), + }; } // If no results are left in the underlying execution context, convert our aggregate results to an array for (const aggregator of this.aggregators.values()) { this.aggregateResultArray.push(aggregator.getResult()); } this.completed = true; - return { result: this.aggregateResultArray.pop(), headers: aggregateHeaders }; + return { + result: this.aggregateResultArray.pop(), + headers: aggregateHeaders, + diagnostics: getEmptyCosmosDiagnostics(), + }; } public hasMoreResults(): boolean { diff --git a/sdk/cosmosdb/cosmos/src/queryExecutionContext/EndpointComponent/OffsetLimitEndpointComponent.ts b/sdk/cosmosdb/cosmos/src/queryExecutionContext/EndpointComponent/OffsetLimitEndpointComponent.ts index 98e215e22c20..0eba977f7640 100644 --- a/sdk/cosmosdb/cosmos/src/queryExecutionContext/EndpointComponent/OffsetLimitEndpointComponent.ts +++ b/sdk/cosmosdb/cosmos/src/queryExecutionContext/EndpointComponent/OffsetLimitEndpointComponent.ts @@ -1,5 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +import { getEmptyCosmosDiagnostics } from "../../CosmosDiagnostics"; import { Response } from "../../request"; import { ExecutionContext } from "../ExecutionContext"; import { getInitialHeader, mergeHeaders } from "../headerUtils"; @@ -21,13 +22,17 @@ export class OffsetLimitEndpointComponent implements ExecutionContext { mergeHeaders(aggregateHeaders, headers); } if (this.limit > 0) { - const { result, headers } = await this.executionContext.nextItem(); + const { result, headers, diagnostics } = await this.executionContext.nextItem(); this.limit--; mergeHeaders(aggregateHeaders, headers); - return { result, headers: aggregateHeaders }; + return { result, headers: aggregateHeaders, diagnostics }; } // If both limit and offset are 0, return nothing - return { result: undefined, headers: getInitialHeader() }; + return { + result: undefined, + headers: getInitialHeader(), + diagnostics: getEmptyCosmosDiagnostics(), + }; } public hasMoreResults(): boolean { diff --git a/sdk/cosmosdb/cosmos/src/queryExecutionContext/EndpointComponent/OrderByEndpointComponent.ts b/sdk/cosmosdb/cosmos/src/queryExecutionContext/EndpointComponent/OrderByEndpointComponent.ts index e972f654c88a..b9958830c9f0 100644 --- a/sdk/cosmosdb/cosmos/src/queryExecutionContext/EndpointComponent/OrderByEndpointComponent.ts +++ b/sdk/cosmosdb/cosmos/src/queryExecutionContext/EndpointComponent/OrderByEndpointComponent.ts @@ -17,10 +17,11 @@ export class OrderByEndpointComponent implements ExecutionContext { * Execute a provided function on the next element in the OrderByEndpointComponent. */ public async nextItem(): Promise> { - const { result: item, headers } = await this.executionContext.nextItem(); + const { result: item, headers, diagnostics } = await this.executionContext.nextItem(); return { result: item !== undefined ? item.payload : undefined, headers, + diagnostics, }; } diff --git a/sdk/cosmosdb/cosmos/src/queryExecutionContext/EndpointComponent/OrderedDistinctEndpointComponent.ts b/sdk/cosmosdb/cosmos/src/queryExecutionContext/EndpointComponent/OrderedDistinctEndpointComponent.ts index cffabb769c4d..72b8c4b9ff89 100644 --- a/sdk/cosmosdb/cosmos/src/queryExecutionContext/EndpointComponent/OrderedDistinctEndpointComponent.ts +++ b/sdk/cosmosdb/cosmos/src/queryExecutionContext/EndpointComponent/OrderedDistinctEndpointComponent.ts @@ -10,15 +10,15 @@ export class OrderedDistinctEndpointComponent implements ExecutionContext { constructor(private executionContext: ExecutionContext) {} public async nextItem(): Promise> { - const { headers, result } = await this.executionContext.nextItem(); + const { headers, result, diagnostics } = await this.executionContext.nextItem(); if (result) { const hashedResult = await hashObject(result); if (hashedResult === this.hashedLastResult) { - return { result: undefined, headers }; + return { result: undefined, headers, diagnostics }; } this.hashedLastResult = hashedResult; } - return { result, headers }; + return { result, headers, diagnostics }; } public hasMoreResults(): boolean { diff --git a/sdk/cosmosdb/cosmos/src/queryExecutionContext/EndpointComponent/UnorderedDistinctEndpointComponent.ts b/sdk/cosmosdb/cosmos/src/queryExecutionContext/EndpointComponent/UnorderedDistinctEndpointComponent.ts index 4f4d876f47a7..296abd7ee596 100644 --- a/sdk/cosmosdb/cosmos/src/queryExecutionContext/EndpointComponent/UnorderedDistinctEndpointComponent.ts +++ b/sdk/cosmosdb/cosmos/src/queryExecutionContext/EndpointComponent/UnorderedDistinctEndpointComponent.ts @@ -12,15 +12,15 @@ export class UnorderedDistinctEndpointComponent implements ExecutionContext { } public async nextItem(): Promise> { - const { headers, result } = await this.executionContext.nextItem(); + const { headers, result, diagnostics } = await this.executionContext.nextItem(); if (result) { const hashedResult = await hashObject(result); if (this.hashedResults.has(hashedResult)) { - return { result: undefined, headers }; + return { result: undefined, headers, diagnostics }; } this.hashedResults.add(hashedResult); } - return { result, headers }; + return { result, headers, diagnostics }; } public hasMoreResults(): boolean { diff --git a/sdk/cosmosdb/cosmos/src/queryExecutionContext/defaultQueryExecutionContext.ts b/sdk/cosmosdb/cosmos/src/queryExecutionContext/defaultQueryExecutionContext.ts index 55647cd31a04..f36ba4eb780e 100644 --- a/sdk/cosmosdb/cosmos/src/queryExecutionContext/defaultQueryExecutionContext.ts +++ b/sdk/cosmosdb/cosmos/src/queryExecutionContext/defaultQueryExecutionContext.ts @@ -6,10 +6,15 @@ import { ClientSideMetrics, QueryMetrics } from "../queryMetrics"; import { FeedOptions, Response } from "../request"; import { getInitialHeader } from "./headerUtils"; import { ExecutionContext } from "./index"; +import { CosmosDiagnosticContext } from "../CosmosDiagnosticsContext"; +import { getEmptyCosmosDiagnostics } from "../CosmosDiagnostics"; const logger: AzureLogger = createClientLogger("ClientContext"); /** @hidden */ -export type FetchFunctionCallback = (options: FeedOptions) => Promise>; +export type FetchFunctionCallback = ( + options: FeedOptions, + diagnosticContext: CosmosDiagnosticContext +) => Promise>; /** @hidden */ enum STATES { @@ -45,7 +50,8 @@ export class DefaultQueryExecutionContext implements ExecutionContext { */ constructor( options: FeedOptions, - fetchFunctions: FetchFunctionCallback | FetchFunctionCallback[] + fetchFunctions: FetchFunctionCallback | FetchFunctionCallback[], + private diagnosticContext: CosmosDiagnosticContext ) { this.resources = []; this.currentIndex = 0; @@ -73,24 +79,29 @@ export class DefaultQueryExecutionContext implements ExecutionContext { return { result: this.resources[this.currentIndex], headers: getInitialHeader(), + diagnostics: getEmptyCosmosDiagnostics(), }; } if (this._canFetchMore()) { - const { result: resources, headers } = await this.fetchMore(); + const { result: resources, headers, diagnostics } = await this.fetchMore(); this.resources = resources; if (this.resources.length === 0) { if (!this.continuationToken && this.currentPartitionIndex >= this.fetchFunctions.length) { this.state = DefaultQueryExecutionContext.STATES.ended; - return { result: undefined, headers }; + return { result: undefined, headers, diagnostics }; } else { return this.current(); } } - return { result: this.resources[this.currentIndex], headers }; + return { result: this.resources[this.currentIndex], headers, diagnostics }; } else { this.state = DefaultQueryExecutionContext.STATES.ended; - return { result: undefined, headers: getInitialHeader() }; + return { + result: undefined, + headers: getInitialHeader(), + diagnostics: getEmptyCosmosDiagnostics(), + }; } } @@ -114,7 +125,11 @@ export class DefaultQueryExecutionContext implements ExecutionContext { */ public async fetchMore(): Promise> { if (this.currentPartitionIndex >= this.fetchFunctions.length) { - return { headers: getInitialHeader(), result: undefined }; + return { + headers: getInitialHeader(), + result: undefined, + diagnostics: getEmptyCosmosDiagnostics(), + }; } // Keep to the original continuation and to restore the value after fetchFunction call @@ -123,11 +138,16 @@ export class DefaultQueryExecutionContext implements ExecutionContext { // Return undefined if there is no more results if (this.currentPartitionIndex >= this.fetchFunctions.length) { - return { headers: getInitialHeader(), result: undefined }; + return { + headers: getInitialHeader(), + result: undefined, + diagnostics: getEmptyCosmosDiagnostics(), + }; } let resources; let responseHeaders; + let diagnostics; try { let p: Promise>; if (this.nextFetchFunction !== undefined) { @@ -136,11 +156,12 @@ export class DefaultQueryExecutionContext implements ExecutionContext { this.nextFetchFunction = undefined; } else { logger.verbose("using fresh fetch"); - p = this.fetchFunctions[this.currentPartitionIndex](this.options); + p = this.fetchFunctions[this.currentPartitionIndex](this.options, this.diagnosticContext); } const response = await p; resources = response.result; responseHeaders = response.headers; + diagnostics = response.diagnostics; this.continuationToken = responseHeaders[Constants.HttpHeaders.Continuation]; if (!this.continuationToken) { @@ -150,7 +171,10 @@ export class DefaultQueryExecutionContext implements ExecutionContext { if (this.options && this.options.bufferItems === true) { const fetchFunction = this.fetchFunctions[this.currentPartitionIndex]; this.nextFetchFunction = fetchFunction - ? fetchFunction({ ...this.options, continuationToken: this.continuationToken }) + ? fetchFunction( + { ...this.options, continuationToken: this.continuationToken }, + this.diagnosticContext + ) : undefined; } } catch (err: any) { @@ -196,7 +220,7 @@ export class DefaultQueryExecutionContext implements ExecutionContext { responseHeaders[Constants.HttpHeaders.QueryMetrics]["0"] = queryMetrics; } - return { result: resources, headers: responseHeaders }; + return { result: resources, headers: responseHeaders, diagnostics }; } private _canFetchMore(): boolean { diff --git a/sdk/cosmosdb/cosmos/src/queryExecutionContext/documentProducer.ts b/sdk/cosmosdb/cosmos/src/queryExecutionContext/documentProducer.ts index a95e8df71827..06d9114007c9 100644 --- a/sdk/cosmosdb/cosmos/src/queryExecutionContext/documentProducer.ts +++ b/sdk/cosmosdb/cosmos/src/queryExecutionContext/documentProducer.ts @@ -10,6 +10,8 @@ import { StatusCodes, SubStatusCodes, } from "../common"; +import { getEmptyCosmosDiagnostics } from "../CosmosDiagnostics"; +import { CosmosDiagnosticContext } from "../CosmosDiagnosticsContext"; import { FeedOptions } from "../request"; import { Response } from "../request"; import { DefaultQueryExecutionContext } from "./defaultQueryExecutionContext"; @@ -44,7 +46,8 @@ export class DocumentProducer { collectionLink: string, query: SqlQuerySpec, targetPartitionKeyRange: PartitionKeyRange, - options: FeedOptions + options: FeedOptions, + diagnosticContext: CosmosDiagnosticContext ) { // TODO: any options this.collectionLink = collectionLink; @@ -59,7 +62,11 @@ export class DocumentProducer { this.continuationToken = undefined; this.respHeaders = getInitialHeader(); - this.internalExecutionContext = new DefaultQueryExecutionContext(options, this.fetchFunction); + this.internalExecutionContext = new DefaultQueryExecutionContext( + options, + this.fetchFunction, + diagnosticContext + ); } /** * Synchronously gives the contiguous buffered results (stops at the first non result) if any @@ -159,8 +166,11 @@ export class DocumentProducer { } try { - const { result: resources, headers: headerResponse } = - await this.internalExecutionContext.fetchMore(); + const { + result: resources, + headers: headerResponse, + diagnostics, + } = await this.internalExecutionContext.fetchMore(); ++this.generation; this._updateStates(undefined, resources === undefined); if (resources !== undefined) { @@ -182,7 +192,7 @@ export class DocumentProducer { queryMetrics; } - return { result: resources, headers: headerResponse }; + return { result: resources, headers: headerResponse, diagnostics }; } catch (err: any) { // TODO: any error if (DocumentProducer._needPartitionKeyRangeCacheRefresh(err)) { @@ -191,7 +201,11 @@ export class DocumentProducer { const bufferedError = new FetchResult(undefined, err); this.fetchResults.push(bufferedError); // Putting a dummy result so that the rest of code flows - return { result: [bufferedError], headers: err.headers }; + return { + result: [bufferedError], + headers: err.headers, + diagnostics: getEmptyCosmosDiagnostics(), + }; } else { this._updateStates(err, err.resources === undefined); throw err; @@ -218,7 +232,7 @@ export class DocumentProducer { } try { - const { result, headers } = await this.current(); + const { result, headers, diagnostics } = await this.current(); const fetchResult = this.fetchResults.shift(); this._updateStates(undefined, result === undefined); @@ -227,12 +241,12 @@ export class DocumentProducer { } switch (fetchResult.fetchResultType) { case FetchResultType.Done: - return { result: undefined, headers }; + return { result: undefined, headers, diagnostics }; case FetchResultType.Exception: fetchResult.error.headers = headers; throw fetchResult.error; case FetchResultType.Result: - return { result: fetchResult.feedResponse, headers }; + return { result: fetchResult.feedResponse, headers, diagnostics }; } } catch (err: any) { this._updateStates(err, err.item === undefined); @@ -253,6 +267,7 @@ export class DocumentProducer { return { result: undefined, headers: this._getAndResetActiveResponseHeaders(), + diagnostics: getEmptyCosmosDiagnostics(), }; case FetchResultType.Exception: fetchResult.error.headers = this._getAndResetActiveResponseHeaders(); @@ -261,6 +276,7 @@ export class DocumentProducer { return { result: fetchResult.feedResponse, headers: this._getAndResetActiveResponseHeaders(), + diagnostics: getEmptyCosmosDiagnostics(), }; } } @@ -270,14 +286,15 @@ export class DocumentProducer { return { result: undefined, headers: this._getAndResetActiveResponseHeaders(), + diagnostics: getEmptyCosmosDiagnostics(), }; } // If there are no more bufferd items and there are still items to be fetched then buffer more - const { result, headers } = await this.bufferMore(); + const { result, headers, diagnostics } = await this.bufferMore(); mergeHeaders(this.respHeaders, headers); if (result === undefined) { - return { result: undefined, headers: this.respHeaders }; + return { result: undefined, headers: this.respHeaders, diagnostics }; } return this.current(); } diff --git a/sdk/cosmosdb/cosmos/src/queryExecutionContext/orderByQueryExecutionContext.ts b/sdk/cosmosdb/cosmos/src/queryExecutionContext/orderByQueryExecutionContext.ts index 958e08b52a07..1836ffbd1718 100644 --- a/sdk/cosmosdb/cosmos/src/queryExecutionContext/orderByQueryExecutionContext.ts +++ b/sdk/cosmosdb/cosmos/src/queryExecutionContext/orderByQueryExecutionContext.ts @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. import { ClientContext } from "../ClientContext"; +import { CosmosDiagnosticContext } from "../CosmosDiagnosticsContext"; import { PartitionedQueryExecutionInfo } from "../request/ErrorResponse"; import { FeedOptions } from "../request/FeedOptions"; import { DocumentProducer } from "./documentProducer"; @@ -33,10 +34,18 @@ export class OrderByQueryExecutionContext collectionLink: string, query: string | SqlQuerySpec, options: FeedOptions, - partitionedQueryExecutionInfo: PartitionedQueryExecutionInfo + partitionedQueryExecutionInfo: PartitionedQueryExecutionInfo, + diagnosticContext: CosmosDiagnosticContext ) { // Calling on base class constructor - super(clientContext, collectionLink, query, options, partitionedQueryExecutionInfo); + super( + clientContext, + collectionLink, + query, + options, + partitionedQueryExecutionInfo, + diagnosticContext + ); this.orderByComparator = new OrderByDocumentProducerComparator(this.sortOrders); } // Instance members are inherited diff --git a/sdk/cosmosdb/cosmos/src/queryExecutionContext/parallelQueryExecutionContextBase.ts b/sdk/cosmosdb/cosmos/src/queryExecutionContext/parallelQueryExecutionContextBase.ts index ed113cf53a4d..8cfd4bc5688d 100644 --- a/sdk/cosmosdb/cosmos/src/queryExecutionContext/parallelQueryExecutionContextBase.ts +++ b/sdk/cosmosdb/cosmos/src/queryExecutionContext/parallelQueryExecutionContextBase.ts @@ -14,6 +14,8 @@ import { DocumentProducer } from "./documentProducer"; import { ExecutionContext } from "./ExecutionContext"; import { getInitialHeader, mergeHeaders } from "./headerUtils"; import { SqlQuerySpec } from "./SqlQuerySpec"; +import { CosmosDiagnosticContext } from "../CosmosDiagnosticsContext"; +import { getEmptyCosmosDiagnostics } from "../CosmosDiagnostics"; /** @hidden */ const logger: AzureLogger = createClientLogger("parallelQueryExecutionContextBase"); @@ -55,7 +57,8 @@ export abstract class ParallelQueryExecutionContextBase implements ExecutionCont private collectionLink: string, private query: string | SqlQuerySpec, private options: FeedOptions, - private partitionedQueryExecutionInfo: PartitionedQueryExecutionInfo + private partitionedQueryExecutionInfo: PartitionedQueryExecutionInfo, + private diagnosticContext: CosmosDiagnosticContext ) { this.clientContext = clientContext; this.collectionLink = collectionLink; @@ -187,7 +190,11 @@ export abstract class ParallelQueryExecutionContextBase implements ExecutionCont // invokes the callback when the target partition ranges are ready const parsedRanges = this.partitionedQueryExecutionInfo.queryRanges; const queryRanges = parsedRanges.map((item) => QueryRange.parseFromDict(item)); - return this.routingProvider.getOverlappingRanges(this.collectionLink, queryRanges); + return this.routingProvider.getOverlappingRanges( + this.collectionLink, + queryRanges, + this.diagnosticContext + ); } /** @@ -201,7 +208,11 @@ export abstract class ParallelQueryExecutionContextBase implements ExecutionCont this.routingProvider = new SmartRoutingMapProvider(this.clientContext); // Get the queryRange that relates to this partitionKeyRange const queryRange = QueryRange.parsePartitionKeyRange(partitionKeyRange); - return this.routingProvider.getOverlappingRanges(this.collectionLink, [queryRange]); + return this.routingProvider.getOverlappingRanges( + this.collectionLink, + [queryRange], + this.diagnosticContext + ); } // TODO: P0 Code smell - can barely tell what this is doing @@ -329,6 +340,7 @@ export abstract class ParallelQueryExecutionContextBase implements ExecutionCont return resolve({ result: undefined, headers: this._getAndResetActiveResponseHeaders(), + diagnostics: getEmptyCosmosDiagnostics(), }); } @@ -355,10 +367,12 @@ export abstract class ParallelQueryExecutionContextBase implements ExecutionCont let item: any; let headers: CosmosHeaders; + let diagnostics; try { const response = await documentProducer.nextItem(); item = response.result; headers = response.headers; + diagnostics = response.diagnostics; this._mergeWithActiveResponseHeaders(headers); if (item === undefined) { // this should never happen @@ -373,6 +387,7 @@ export abstract class ParallelQueryExecutionContextBase implements ExecutionCont return resolve({ result: undefined, headers: this._getAndResetActiveResponseHeaders(), + diagnostics, }); } } catch (err: any) { @@ -427,6 +442,7 @@ export abstract class ParallelQueryExecutionContextBase implements ExecutionCont return resolve({ result: item, headers: this._getAndResetActiveResponseHeaders(), + diagnostics, }); }; this._repairExecutionContextIfNeeded(ifCallback, elseCallback).catch(reject); @@ -479,7 +495,8 @@ export abstract class ParallelQueryExecutionContextBase implements ExecutionCont this.collectionLink, sqlQuerySpec, partitionKeyTargetRange, - options + options, + this.diagnosticContext ); } } diff --git a/sdk/cosmosdb/cosmos/src/queryExecutionContext/pipelinedQueryExecutionContext.ts b/sdk/cosmosdb/cosmos/src/queryExecutionContext/pipelinedQueryExecutionContext.ts index 40f610d44bd0..e2645f50d775 100644 --- a/sdk/cosmosdb/cosmos/src/queryExecutionContext/pipelinedQueryExecutionContext.ts +++ b/sdk/cosmosdb/cosmos/src/queryExecutionContext/pipelinedQueryExecutionContext.ts @@ -15,6 +15,7 @@ import { OrderByQueryExecutionContext } from "./orderByQueryExecutionContext"; import { ParallelQueryExecutionContext } from "./parallelQueryExecutionContext"; import { GroupByValueEndpointComponent } from "./EndpointComponent/GroupByValueEndpointComponent"; import { SqlQuerySpec } from "./SqlQuerySpec"; +import { CosmosDiagnosticContext } from "../CosmosDiagnosticsContext"; /** @hidden */ export class PipelinedQueryExecutionContext implements ExecutionContext { @@ -28,7 +29,8 @@ export class PipelinedQueryExecutionContext implements ExecutionContext { private collectionLink: string, private query: string | SqlQuerySpec, private options: FeedOptions, - private partitionedQueryExecutionInfo: PartitionedQueryExecutionInfo + private partitionedQueryExecutionInfo: PartitionedQueryExecutionInfo, + private diagnosticContext: CosmosDiagnosticContext ) { this.endpoint = null; this.pageSize = options["maxItemCount"]; @@ -47,7 +49,8 @@ export class PipelinedQueryExecutionContext implements ExecutionContext { this.collectionLink, this.query, this.options, - this.partitionedQueryExecutionInfo + this.partitionedQueryExecutionInfo, + this.diagnosticContext ) ); } else { @@ -56,7 +59,8 @@ export class PipelinedQueryExecutionContext implements ExecutionContext { this.collectionLink, this.query, this.options, - this.partitionedQueryExecutionInfo + this.partitionedQueryExecutionInfo, + this.diagnosticContext ); } if ( @@ -122,7 +126,7 @@ export class PipelinedQueryExecutionContext implements ExecutionContext { private async _fetchMoreImplementation(): Promise> { try { - const { result: item, headers } = await this.endpoint.nextItem(); + const { result: item, headers, diagnostics } = await this.endpoint.nextItem(); mergeHeaders(this.fetchMoreRespHeaders, headers); if (item === undefined) { // no more results @@ -130,12 +134,13 @@ export class PipelinedQueryExecutionContext implements ExecutionContext { return { result: undefined, headers: this.fetchMoreRespHeaders, + diagnostics, }; } else { // Just give what we have const temp = this.fetchBuffer; this.fetchBuffer = []; - return { result: temp, headers: this.fetchMoreRespHeaders }; + return { result: temp, headers: this.fetchMoreRespHeaders, diagnostics }; } } else { // append the result @@ -144,7 +149,7 @@ export class PipelinedQueryExecutionContext implements ExecutionContext { // fetched enough results const temp = this.fetchBuffer.slice(0, this.pageSize); this.fetchBuffer = this.fetchBuffer.splice(this.pageSize); - return { result: temp, headers: this.fetchMoreRespHeaders }; + return { result: temp, headers: this.fetchMoreRespHeaders, diagnostics }; } else { // recursively fetch more // TODO: is recursion a good idea? diff --git a/sdk/cosmosdb/cosmos/src/queryIterator.ts b/sdk/cosmosdb/cosmos/src/queryIterator.ts index e495335d753f..2470a53bd50f 100644 --- a/sdk/cosmosdb/cosmos/src/queryIterator.ts +++ b/sdk/cosmosdb/cosmos/src/queryIterator.ts @@ -4,6 +4,7 @@ /// import { ClientContext } from "./ClientContext"; +import { CosmosDiagnosticContext } from "./CosmosDiagnosticsContext"; import { getPathFromLink, ResourceType, StatusCodes } from "./common"; import { CosmosHeaders, @@ -39,6 +40,7 @@ export class QueryIterator { private query: SqlQuerySpec | string, private options: FeedOptions, private fetchFunctions: FetchFunctionCallback | FetchFunctionCallback[], + private diagnosticContext: CosmosDiagnosticContext = new CosmosDiagnosticContext(), private resourceLink?: string, private resourceType?: ResourceType ) { @@ -95,7 +97,8 @@ export class QueryIterator { const feedResponse = new FeedResponse( response.result, response.headers, - this.queryExecutionContext.hasMoreResults() + this.queryExecutionContext.hasMoreResults(), + this.diagnosticContext.resetAndGetDiagnostics() ); if (response.result !== undefined) { yield feedResponse; @@ -159,7 +162,8 @@ export class QueryIterator { return new FeedResponse( response.result, response.headers, - this.queryExecutionContext.hasMoreResults() + this.queryExecutionContext.hasMoreResults(), + this.diagnosticContext.resetAndGetDiagnostics() ); } @@ -170,7 +174,8 @@ export class QueryIterator { this.queryPlanPromise = undefined; this.queryExecutionContext = new DefaultQueryExecutionContext( this.options, - this.fetchFunctions + this.fetchFunctions, + this.diagnosticContext ); } @@ -202,7 +207,8 @@ export class QueryIterator { return new FeedResponse( this.fetchAllTempResources, this.fetchAllLastResHeaders, - this.queryExecutionContext.hasMoreResults() + this.queryExecutionContext.hasMoreResults(), + this.diagnosticContext.resetAndGetDiagnostics() ); } @@ -224,7 +230,8 @@ export class QueryIterator { this.resourceLink, this.query, this.options, - queryPlan + queryPlan, + this.diagnosticContext ); } diff --git a/sdk/cosmosdb/cosmos/src/request/FeedResponse.ts b/sdk/cosmosdb/cosmos/src/request/FeedResponse.ts index 70538be5f36e..6028ecef208d 100644 --- a/sdk/cosmosdb/cosmos/src/request/FeedResponse.ts +++ b/sdk/cosmosdb/cosmos/src/request/FeedResponse.ts @@ -3,13 +3,16 @@ import { Constants } from "../common"; import { CosmosHeaders } from "../queryExecutionContext"; import { IndexMetricWriter, IndexUtilizationInfo } from "../indexMetrics"; +import { CosmosDiagnostics } from "../CosmosDiagnostics"; export class FeedResponse { constructor( public readonly resources: TResource[], private readonly headers: CosmosHeaders, - public readonly hasMoreResults: boolean + public readonly hasMoreResults: boolean, + public readonly diagnostics: CosmosDiagnostics ) {} + public get continuation(): string { return this.continuationToken; } diff --git a/sdk/cosmosdb/cosmos/src/request/ResourceResponse.ts b/sdk/cosmosdb/cosmos/src/request/ResourceResponse.ts index 16d674956cbd..87e40552e680 100644 --- a/sdk/cosmosdb/cosmos/src/request/ResourceResponse.ts +++ b/sdk/cosmosdb/cosmos/src/request/ResourceResponse.ts @@ -10,7 +10,7 @@ export class ResourceResponse { public readonly resource: TResource | undefined, public readonly headers: CosmosHeaders, public readonly statusCode: StatusCode, - public readonly diagnostics?: CosmosDiagnostics, + public readonly diagnostics: CosmosDiagnostics, public readonly substatus?: SubStatusCode ) {} public get requestCharge(): number { diff --git a/sdk/cosmosdb/cosmos/src/request/Response.ts b/sdk/cosmosdb/cosmos/src/request/Response.ts index 68b9313e3594..fd7264560e36 100644 --- a/sdk/cosmosdb/cosmos/src/request/Response.ts +++ b/sdk/cosmosdb/cosmos/src/request/Response.ts @@ -10,5 +10,5 @@ export interface Response { result?: T; code?: number; substatus?: number; - diagnostics?: CosmosDiagnostics; + diagnostics: CosmosDiagnostics; } diff --git a/sdk/cosmosdb/cosmos/src/routing/partitionKeyRangeCache.ts b/sdk/cosmosdb/cosmos/src/routing/partitionKeyRangeCache.ts index 4c9c0c7118f2..00be39832df3 100644 --- a/sdk/cosmosdb/cosmos/src/routing/partitionKeyRangeCache.ts +++ b/sdk/cosmosdb/cosmos/src/routing/partitionKeyRangeCache.ts @@ -3,6 +3,7 @@ import { PartitionKeyRange } from "../client/Container/PartitionKeyRange"; import { ClientContext } from "../ClientContext"; import { getIdFromLink } from "../common/helper"; +import { CosmosDiagnosticContext } from "../CosmosDiagnosticsContext"; import { createCompleteRoutingMap } from "./CollectionRoutingMapFactory"; import { InMemoryCollectionRoutingMap } from "./inMemoryCollectionRoutingMap"; import { QueryRange } from "./QueryRange"; @@ -22,12 +23,15 @@ export class PartitionKeyRangeCache { * @hidden */ public async onCollectionRoutingMap( - collectionLink: string + collectionLink: string, + diagnosticContext: CosmosDiagnosticContext ): Promise { const collectionId = getIdFromLink(collectionLink); if (this.collectionRoutingMapByCollectionId[collectionId] === undefined) { - this.collectionRoutingMapByCollectionId[collectionId] = - this.requestCollectionRoutingMap(collectionLink); + this.collectionRoutingMapByCollectionId[collectionId] = this.requestCollectionRoutingMap( + collectionLink, + diagnosticContext + ); } return this.collectionRoutingMapByCollectionId[collectionId]; } @@ -38,17 +42,19 @@ export class PartitionKeyRangeCache { */ public async getOverlappingRanges( collectionLink: string, - queryRange: QueryRange + queryRange: QueryRange, + diagnosticContext: CosmosDiagnosticContext ): Promise { - const crm = await this.onCollectionRoutingMap(collectionLink); + const crm = await this.onCollectionRoutingMap(collectionLink, diagnosticContext); return crm.getOverlappingRanges(queryRange); } private async requestCollectionRoutingMap( - collectionLink: string + collectionLink: string, + diagnosticContext: CosmosDiagnosticContext ): Promise { const { resources } = await this.clientContext - .queryPartitionKeyRanges(collectionLink) + .queryPartitionKeyRanges(collectionLink, diagnosticContext) .fetchAll(); return createCompleteRoutingMap(resources.map((r) => [r, true])); } diff --git a/sdk/cosmosdb/cosmos/src/routing/smartRoutingMapProvider.ts b/sdk/cosmosdb/cosmos/src/routing/smartRoutingMapProvider.ts index ab992cac4a8a..3ac91a687c96 100644 --- a/sdk/cosmosdb/cosmos/src/routing/smartRoutingMapProvider.ts +++ b/sdk/cosmosdb/cosmos/src/routing/smartRoutingMapProvider.ts @@ -2,6 +2,7 @@ // Licensed under the MIT license. import { ClientContext } from "../ClientContext"; import { Constants } from "../common/constants"; +import { CosmosDiagnosticContext } from "../CosmosDiagnosticsContext"; import { PartitionKeyRangeCache } from "./partitionKeyRangeCache"; import { QueryRange } from "./QueryRange"; @@ -70,7 +71,8 @@ export class SmartRoutingMapProvider { */ public async getOverlappingRanges( collectionLink: string, - sortedRanges: QueryRange[] + sortedRanges: QueryRange[], + diagnosticContext: CosmosDiagnosticContext ): Promise { // validate if the list is non- overlapping and sorted TODO: any PartitionKeyRanges if (!SmartRoutingMapProvider._isSortedAndNonOverlapping(sortedRanges)) { @@ -84,7 +86,8 @@ export class SmartRoutingMapProvider { } const collectionRoutingMap = await this.partitionKeyRangeCache.onCollectionRoutingMap( - collectionLink + collectionLink, + diagnosticContext ); let index = 0; diff --git a/sdk/cosmosdb/cosmos/src/utils/batch.ts b/sdk/cosmosdb/cosmos/src/utils/batch.ts index 360fd702b235..041425a93c2a 100644 --- a/sdk/cosmosdb/cosmos/src/utils/batch.ts +++ b/sdk/cosmosdb/cosmos/src/utils/batch.ts @@ -4,7 +4,7 @@ import { JSONObject } from "../queryExecutionContext"; import { extractPartitionKey } from "../extractPartitionKey"; import { PartitionKeyDefinition } from "../documents"; -import { RequestOptions } from ".."; +import { CosmosDiagnostics, RequestOptions } from ".."; import { PatchRequestBody } from "./patch"; import { v4 } from "uuid"; import { bodyFromData } from "../request/request"; @@ -27,6 +27,8 @@ export interface Batch { operations: Operation[]; } +export type BulkOperationResponse = OperationResponse[] & { diagnostics: CosmosDiagnostics }; + export interface OperationResponse { statusCode: number; requestCharge: number; diff --git a/sdk/cosmosdb/cosmos/test/internal/unit/defaultQueryExecutionContext.spec.ts b/sdk/cosmosdb/cosmos/test/internal/unit/defaultQueryExecutionContext.spec.ts index 708eb1b71526..234be657ea20 100644 --- a/sdk/cosmosdb/cosmos/test/internal/unit/defaultQueryExecutionContext.spec.ts +++ b/sdk/cosmosdb/cosmos/test/internal/unit/defaultQueryExecutionContext.spec.ts @@ -7,6 +7,8 @@ import { import { FeedOptions } from "../../../src"; import assert from "assert"; import { sleep } from "../../../src/common"; +import { getEmptyCosmosDiagnostics } from "../../../src/CosmosDiagnostics"; +import { CosmosDiagnosticContext } from "../../../src/CosmosDiagnosticsContext"; describe("defaultQueryExecutionContext", function () { it("should not buffer items if bufferItems is false", async function () { @@ -24,6 +26,7 @@ describe("defaultQueryExecutionContext", function () { }, ], substatus: 0, + diagnostics: getEmptyCosmosDiagnostics(), }; }; @@ -31,7 +34,11 @@ describe("defaultQueryExecutionContext", function () { bufferItems: false, }; - const context = new DefaultQueryExecutionContext(options, fetchFunction); + const context = new DefaultQueryExecutionContext( + options, + fetchFunction, + new CosmosDiagnosticContext() + ); assert.strictEqual(calledCount, 0, "Nothing should be fetched at this point"); @@ -63,6 +70,7 @@ describe("defaultQueryExecutionContext", function () { }, ], substatus: 0, + diagnostics: getEmptyCosmosDiagnostics(), }; }; @@ -70,7 +78,11 @@ describe("defaultQueryExecutionContext", function () { bufferItems: true, }; - const context = new DefaultQueryExecutionContext(options, fetchFunction); + const context = new DefaultQueryExecutionContext( + options, + fetchFunction, + new CosmosDiagnosticContext() + ); assert.strictEqual(calledCount, 0, "Nothing should be fetched at this point"); diff --git a/sdk/cosmosdb/cosmos/test/internal/unit/smartRoutingMapProvider.spec.ts b/sdk/cosmosdb/cosmos/test/internal/unit/smartRoutingMapProvider.spec.ts index 2203641a6afe..5ac85c4115ac 100644 --- a/sdk/cosmosdb/cosmos/test/internal/unit/smartRoutingMapProvider.spec.ts +++ b/sdk/cosmosdb/cosmos/test/internal/unit/smartRoutingMapProvider.spec.ts @@ -4,6 +4,7 @@ import assert from "assert"; import { ClientContext } from "../../../src/ClientContext"; import { PartitionKeyRangeCache, QueryRange, SmartRoutingMapProvider } from "../../../src/routing"; import { MockedClientContext } from "../../public/common/MockClientContext"; +import { CosmosDiagnosticContext } from "../../../src/CosmosDiagnosticsContext"; describe("Smart Routing Map Provider OverlappingRanges", function () { const containerLink = "dbs/7JZZAA==/colls/7JZZAOS-JQA=/"; @@ -43,12 +44,20 @@ describe("Smart Routing Map Provider OverlappingRanges", function () { results1 = results2 = null; err1 = err2 = null; try { - results1 = await smartRoutingMapProvider.getOverlappingRanges(containerLink, queryRanges); + results1 = await smartRoutingMapProvider.getOverlappingRanges( + containerLink, + queryRanges, + new CosmosDiagnosticContext() + ); } catch (err: any) { err1 = err; } try { - results2 = await partitionKeyRangeCache.getOverlappingRanges(containerLink, queryRanges); + results2 = await partitionKeyRangeCache.getOverlappingRanges( + containerLink, + queryRanges, + new CosmosDiagnosticContext() + ); } catch (err: any) { err2 = err; } @@ -66,7 +75,11 @@ describe("Smart Routing Map Provider OverlappingRanges", function () { ): Promise { errorExpected = errorExpected || false; try { - const results = await provider.getOverlappingRanges(containerLink, queryRanges); + const results = await provider.getOverlappingRanges( + containerLink, + queryRanges, + new CosmosDiagnosticContext() + ); assert.deepEqual(results, expectedResults); } catch (err: any) { if (errorExpected) { @@ -137,12 +150,20 @@ describe("Smart Routing Map Provider OverlappingRanges", function () { let err1: any; let err2: any; try { - results1 = await provider.getOverlappingRanges(containerLink, queryRanges1); + results1 = await provider.getOverlappingRanges( + containerLink, + queryRanges1, + new CosmosDiagnosticContext() + ); } catch (err: any) { err1 = err; } try { - results2 = await provider.getOverlappingRanges(containerLink, queryRanges2); + results2 = await provider.getOverlappingRanges( + containerLink, + queryRanges2, + new CosmosDiagnosticContext() + ); } catch (err: any) { err2 = err; } diff --git a/sdk/cosmosdb/cosmos/test/public/functional/client.spec.ts b/sdk/cosmosdb/cosmos/test/public/functional/client.spec.ts index 18efc4ac4382..97f8362ab222 100644 --- a/sdk/cosmosdb/cosmos/test/public/functional/client.spec.ts +++ b/sdk/cosmosdb/cosmos/test/public/functional/client.spec.ts @@ -92,7 +92,6 @@ describe("Client Tests", function (this: Suite) { await client.getDatabaseAccount({ abortSignal: signal }); assert.fail("Must throw when trying to connect to database"); } catch (err: any) { - console.log(err); assert.equal(err.name, "AbortError", "client should throw exception"); } client.dispose(); diff --git a/sdk/cosmosdb/cosmos/test/public/functional/globalEndpointManager.spec.ts b/sdk/cosmosdb/cosmos/test/public/functional/globalEndpointManager.spec.ts index cfa61b410dea..d1f083b9964e 100644 --- a/sdk/cosmosdb/cosmos/test/public/functional/globalEndpointManager.spec.ts +++ b/sdk/cosmosdb/cosmos/test/public/functional/globalEndpointManager.spec.ts @@ -7,6 +7,7 @@ import { OperationType, ResourceType } from "../../../src"; import * as fakeTimers from "@sinonjs/fake-timers"; import assert from "assert"; +import { getEmptyCosmosDiagnostics } from "../../../src/CosmosDiagnostics"; const locationUnavailabilityExpiratationTime = 6 * 60 * 1000; const headers = { @@ -56,7 +57,8 @@ describe("GlobalEndpointManager", function () { const response: ResourceResponse = new ResourceResponse( new DatabaseAccount(databaseAccountBody, headers), headers, - 200 + 200, + getEmptyCosmosDiagnostics() ); return response; } @@ -89,7 +91,8 @@ describe("GlobalEndpointManager", function () { const response: ResourceResponse = new ResourceResponse( new DatabaseAccount(databaseAccountBody, headers), headers, - 200 + 200, + getEmptyCosmosDiagnostics() ); return response; } @@ -116,7 +119,8 @@ describe("GlobalEndpointManager", function () { const response: ResourceResponse = new ResourceResponse( new DatabaseAccount(databaseAccountBody, headers), headers, - 200 + 200, + getEmptyCosmosDiagnostics() ); return response; } @@ -147,7 +151,8 @@ describe("GlobalEndpointManager", function () { const response: ResourceResponse = new ResourceResponse( new DatabaseAccount(databaseAccountBody, headers), headers, - 200 + 200, + getEmptyCosmosDiagnostics() ); return response; } diff --git a/sdk/cosmosdb/cosmos/test/public/integration/multiregion.spec.ts b/sdk/cosmosdb/cosmos/test/public/integration/multiregion.spec.ts index 2e5ae4923c57..ec95d3bef632 100644 --- a/sdk/cosmosdb/cosmos/test/public/integration/multiregion.spec.ts +++ b/sdk/cosmosdb/cosmos/test/public/integration/multiregion.spec.ts @@ -6,6 +6,7 @@ import { Suite } from "mocha"; import { CosmosClient, RequestContext } from "../../../src"; import { masterKey } from "../common/_fakeTestSecrets"; import { PluginOn, PluginConfig, CosmosClientOptions } from "../../../src"; +import { getEmptyCosmosDiagnostics } from "../../../src/CosmosDiagnostics"; const endpoint = "https://failovertest.documents.azure.com/"; @@ -63,6 +64,7 @@ const databaseAccountResponse = { '{"maxSqlQueryInputLength":262144,"maxJoinsPerSqlQuery":5,"maxLogicalAndPerSqlQuery":500,"maxLogicalOrPerSqlQuery":500,"maxUdfRefPerSqlQuery":10,"maxInExpressionItemsCount":16000,"queryMaxInMemorySortDocumentCount":500,"maxQueryRequestTimeoutFraction":0.9,"sqlAllowNonFiniteNumbers":false,"sqlAllowAggregateFunctions":true,"sqlAllowSubQuery":true,"sqlAllowScalarSubQuery":true,"allowNewKeywords":true,"sqlAllowLike":false,"sqlAllowGroupByClause":true,"maxSpatialQueryCells":12,"spatialMaxGeometryPointCount":256,"sqlAllowTop":true,"enableSpatialIndexing":true}', }, code: 200, + diagnostics: getEmptyCosmosDiagnostics(), }; const collectionResponse = { @@ -106,6 +108,7 @@ const collectionResponse = { _conflicts: "conflicts/", }, code: 200, + diagnostics: getEmptyCosmosDiagnostics(), }; describe("Multi-region tests", function (this: Suite) { @@ -117,7 +120,7 @@ describe("Multi-region tests", function (this: Suite) { const responses = [ databaseAccountResponse, collectionResponse, - { code: 200, result: {}, headers: {} }, + { code: 200, result: {}, headers: {}, diagnostics: getEmptyCosmosDiagnostics() }, ]; const options: CosmosClientOptions = { endpoint, @@ -160,7 +163,7 @@ describe("Multi-region tests", function (this: Suite) { const responses = [ databaseAccountResponse, collectionResponse, - { code: 201, result: {}, headers: {} }, + { code: 201, result: {}, headers: {}, diagnostics: getEmptyCosmosDiagnostics() }, ]; const options: CosmosClientOptions = { endpoint, diff --git a/sdk/cosmosdb/cosmos/tsconfig.strict.json b/sdk/cosmosdb/cosmos/tsconfig.strict.json index e3700844b5b1..8f05b8330504 100644 --- a/sdk/cosmosdb/cosmos/tsconfig.strict.json +++ b/sdk/cosmosdb/cosmos/tsconfig.strict.json @@ -117,6 +117,7 @@ "src/client/Conflict/index.ts", "src/client/Container/index.ts", "src/client/Database/index.ts", + "src/client/ClientUtils.ts", "src/documents/index.ts", "src/request/index.ts", "src/index.ts", From 781dafc0542b639f143187ab5bc6782fb22a5fa0 Mon Sep 17 00:00:00 2001 From: Deyaaeldeen Almahallawi Date: Tue, 6 Jun 2023 17:16:42 -0700 Subject: [PATCH 2/4] [OpenAI] Update tests (#26043) To test across all authentication methods, Azure Key, openai.com Key, and AAD. --- .github/CODEOWNERS | 3 + sdk/openai/openai/.eslintrc.json | 3 +- sdk/openai/openai/CHANGELOG.md | 8 +- sdk/openai/openai/assets.json | 2 +- sdk/openai/openai/package.json | 2 +- sdk/openai/openai/test/public/openai.spec.ts | 98 ++++++++++--------- .../test/public/utils/recordedClient.ts | 1 - 7 files changed, 62 insertions(+), 55 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 74bc46c1dc1d..3e91adc9bff7 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -168,6 +168,9 @@ # PRLabel: %Cognitive - Language /sdk/cognitivelanguage/ @minhanh-phan @deyaaeldeen +# PRLabel: %OpenAI +/sdk/openai/ @deyaaeldeen @minhanh-phan + # PRLabel: %Schema Registry /sdk/schemaregistry/ @deyaaeldeen diff --git a/sdk/openai/openai/.eslintrc.json b/sdk/openai/openai/.eslintrc.json index 53002ad1ff49..f5bfc15cf5e7 100644 --- a/sdk/openai/openai/.eslintrc.json +++ b/sdk/openai/openai/.eslintrc.json @@ -2,6 +2,7 @@ "plugins": ["@azure/azure-sdk"], "extends": ["plugin:@azure/azure-sdk/azure-sdk-base"], "rules": { - "tsdoc/syntax": "warn" + "tsdoc/syntax": "warn", + "@azure/azure-sdk/ts-package-json-main-is-cjs": "warn" } } diff --git a/sdk/openai/openai/CHANGELOG.md b/sdk/openai/openai/CHANGELOG.md index f67b156d0eab..6b52e8b965b6 100644 --- a/sdk/openai/openai/CHANGELOG.md +++ b/sdk/openai/openai/CHANGELOG.md @@ -1,14 +1,10 @@ # Release History -## 1.0.0-beta.2 (Unreleased) - -### Features Added - -### Breaking Changes +## 1.0.0-beta.2 (2023-06-06) ### Bugs Fixed -### Other Changes +- Fix a bug where the customer-passed options for credentials were overwritten by the defaults values. ## 1.0.0-beta.1 (2023-05-22) diff --git a/sdk/openai/openai/assets.json b/sdk/openai/openai/assets.json index 989b6ab6840e..f5be35afdde7 100644 --- a/sdk/openai/openai/assets.json +++ b/sdk/openai/openai/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "js", "TagPrefix": "js/openai/openai", - "Tag": "js/openai/openai_2df52bb465" + "Tag": "js/openai/openai_81f8a513bb" } diff --git a/sdk/openai/openai/package.json b/sdk/openai/openai/package.json index 6261e0282498..9ff5cb945836 100644 --- a/sdk/openai/openai/package.json +++ b/sdk/openai/openai/package.json @@ -3,7 +3,7 @@ "version": "1.0.0-beta.2", "description": "An isomorphic client library for Azure OpenAI.", "sdk-type": "client", - "main": "dist/index.js", + "main": "dist/index.cjs", "module": "dist-esm/src/index.js", "browser": { "./dist-esm/src/api/getStream.js": "./dist-esm/src/api/getStream.browser.js" diff --git a/sdk/openai/openai/test/public/openai.spec.ts b/sdk/openai/openai/test/public/openai.spec.ts index 29048037269c..73246486451b 100644 --- a/sdk/openai/openai/test/public/openai.spec.ts +++ b/sdk/openai/openai/test/public/openai.spec.ts @@ -2,36 +2,43 @@ // Licensed under the MIT license. import { Recorder } from "@azure-tools/test-recorder"; -import { assert } from "@azure/test-utils"; +import { assert, isNode, matrix } from "@azure/test-utils"; import { Context } from "mocha"; import { OpenAIClient } from "../../src/OpenAIClient.js"; -import { createClient, startRecorder } from "./utils/recordedClient.js"; +import { AuthMethod, createClient, startRecorder } from "./utils/recordedClient.js"; -describe("client", () => { - let recorder: Recorder; - let client: OpenAIClient; +matrix([["AzureAPIKey", "AAD", "OpenAIKey"]] as const, async function (authMethod: AuthMethod) { + describe(`[${authMethod}] Client`, () => { + let recorder: Recorder; + let client: OpenAIClient; - beforeEach(async function (this: Context) { - recorder = new Recorder(this.currentTest); - recorder = await startRecorder(this.currentTest); - client = createClient("AzureAPIKey", { recorder }); - }); + beforeEach(async function (this: Context) { + if (!isNode && authMethod === "AAD") { + this.skip(); + } + recorder = new Recorder(this.currentTest); + recorder = await startRecorder(this.currentTest); + client = createClient(authMethod, { recorder }); + }); - afterEach(async function () { - await recorder.stop(); - }); + afterEach(async function () { + if (!isNode && authMethod === "AAD") { + this.skip(); + } + await recorder.stop(); + }); - it("completions test", async function () { - const prompt = ["This is a test"]; - const modelName = "text-davinci-003"; - const completions = await client.getCompletions(modelName, prompt); - assert.isNotNull(completions.choices); - assert.equal(completions.choices?.length, 1); - }); + it("completions test", async function () { + const prompt = ["This is a test"]; + const modelName = "text-davinci-003"; + const completions = await client.getCompletions(modelName, prompt); + assert.isNotNull(completions.choices); + assert.equal(completions.choices?.length, 1); + }); - it("stream long completions", async function () { - const prompt = [ - `##### Translate this code snippet into Python. Use Azure SDKs where possible. + it("stream long completions", async function () { + const prompt = [ + `##### Translate this code snippet into Python. Use Azure SDKs where possible. \`\`\` using System.Threading.Tasks; using Microsoft.AspNetCore.Mvc; @@ -85,29 +92,30 @@ namespace Function1 \`\`\` `, - ]; - const modelName = "text-davinci-003"; - const events = await client.listCompletions(modelName, prompt, { - maxTokens: 2048, - }); - for await (const event of events) { - if (!event?.choices) { - throw new Error("Expected choices in the response"); - } - for (const choice of event.choices) { - assert.isDefined(choice.text); + ]; + const modelName = "text-davinci-003"; + const events = await client.listCompletions(modelName, prompt, { + maxTokens: 2048, + }); + for await (const event of events) { + if (!event?.choices) { + throw new Error("Expected choices in the response"); + } + for (const choice of event.choices) { + assert.isDefined(choice.text); + } } - } - }); + }); - it("embeddings test", async function () { - const prompt = ["This is a test"]; - const modelName = "text-embedding-ada-002-2"; - const embeddings = await client.getEmbeddings(modelName, prompt); - assert.isNotNull(embeddings.data); - assert.equal(embeddings.data.length > 0, true); - assert.isNotNull(embeddings.data[0].embedding); - assert.equal(embeddings.data[0].embedding.length > 0, true); - assert.isNotNull(embeddings.usage); + it("embeddings test", async function () { + const prompt = ["This is a test"]; + const modelName = "text-embedding-ada-002"; + const embeddings = await client.getEmbeddings(modelName, prompt); + assert.isNotNull(embeddings.data); + assert.equal(embeddings.data.length > 0, true); + assert.isNotNull(embeddings.data[0].embedding); + assert.equal(embeddings.data[0].embedding.length > 0, true); + assert.isNotNull(embeddings.usage); + }); }); }); diff --git a/sdk/openai/openai/test/public/utils/recordedClient.ts b/sdk/openai/openai/test/public/utils/recordedClient.ts index 08566033f6ab..8ef5d31d99c6 100644 --- a/sdk/openai/openai/test/public/utils/recordedClient.ts +++ b/sdk/openai/openai/test/public/utils/recordedClient.ts @@ -45,7 +45,6 @@ export function createClient( } case "OpenAIKey": { return new OpenAIClient( - endpoint, new OpenAIKeyCredential(assertEnvironmentVariable("OPENAI_API_KEY")), updatedOptions ); From 9d95316322f3ea5f171ed7c15ae6fb0425ce9db5 Mon Sep 17 00:00:00 2001 From: Azure SDK Bot <53356347+azure-sdk@users.noreply.github.com> Date: Tue, 6 Jun 2023 21:22:36 -0700 Subject: [PATCH 3/4] Post release automated changes for openai releases (#26127) Post release automated changes for azure-openai --- sdk/openai/openai/CHANGELOG.md | 10 ++++++++++ sdk/openai/openai/package.json | 2 +- sdk/openai/openai/src/rest/openAIClient.ts | 2 +- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/sdk/openai/openai/CHANGELOG.md b/sdk/openai/openai/CHANGELOG.md index 6b52e8b965b6..2e15d13cb58d 100644 --- a/sdk/openai/openai/CHANGELOG.md +++ b/sdk/openai/openai/CHANGELOG.md @@ -1,5 +1,15 @@ # Release History +## 1.0.0-beta.3 (Unreleased) + +### Features Added + +### Breaking Changes + +### Bugs Fixed + +### Other Changes + ## 1.0.0-beta.2 (2023-06-06) ### Bugs Fixed diff --git a/sdk/openai/openai/package.json b/sdk/openai/openai/package.json index 9ff5cb945836..1de623cdc4cc 100644 --- a/sdk/openai/openai/package.json +++ b/sdk/openai/openai/package.json @@ -1,6 +1,6 @@ { "name": "@azure/openai", - "version": "1.0.0-beta.2", + "version": "1.0.0-beta.3", "description": "An isomorphic client library for Azure OpenAI.", "sdk-type": "client", "main": "dist/index.cjs", diff --git a/sdk/openai/openai/src/rest/openAIClient.ts b/sdk/openai/openai/src/rest/openAIClient.ts index 69363b3dc032..b4a09eae94f4 100644 --- a/sdk/openai/openai/src/rest/openAIClient.ts +++ b/sdk/openai/openai/src/rest/openAIClient.ts @@ -28,7 +28,7 @@ export default function createClient( ...options, }; - const userAgentInfo = `azsdk-js-openai-rest/1.0.0-beta.2`; + const userAgentInfo = `azsdk-js-openai-rest/1.0.0-beta.3`; const userAgentPrefix = options.userAgentOptions && options.userAgentOptions.userAgentPrefix ? `${options.userAgentOptions.userAgentPrefix} ${userAgentInfo}` From 4dae4490dfbb7e15c9966042e402b5f2d80eb255 Mon Sep 17 00:00:00 2001 From: Mary Gao Date: Wed, 7 Jun 2023 16:48:35 +0800 Subject: [PATCH 4/4] Update the emitter version to latest (#26129) Update the emitter version to latest --- eng/emitter-package.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/eng/emitter-package.json b/eng/emitter-package.json index 440be7b8a297..55a82385e880 100644 --- a/eng/emitter-package.json +++ b/eng/emitter-package.json @@ -1,7 +1,6 @@ { "main": "dist/src/index.js", "dependencies": { - "@azure-tools/typespec-ts": "0.13.0", - "@typespec/versioning": "0.44.0" + "@azure-tools/typespec-ts": "latest" } }