[MD]Allow create and distinguish index pattern with same name but fro…

…m different datasources (opensearch-project#3604) *[Multiple Datasource] Allow create and distinguish index pattern with same name but from different datasources Signed-off-by: Su <[email protected]> Signed-off-by: David Sinclair <[email protected]>
sikhote · Apr 24, 2023 · b198246 · b198246
1 parent 9ecc3dd
commit b198246
Show file tree

Hide file tree

Showing 15 changed files with 312 additions and 87 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -71,6 +71,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Use mirrors to download Node.js binaries to escape sporadic 404 errors ([#3619](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3619))
 - [Multiple DataSource] Refactor dev tool console to use opensearch-js client to send requests ([#3544](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3544))
 - [Data] Add geo shape filter field ([#3605](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3605))
+- [Multiple DataSource] Allow create and distinguish index pattern with same name but from different datasources ([#3571](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3571))
 
 ### 🐛 Bug Fixes
 

diff --git a/src/plugins/data/common/index_patterns/fields/utils.test.ts b/src/plugins/data/common/index_patterns/fields/utils.test.ts
@@ -0,0 +1,71 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Any modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { isFilterable } from '..';
+import { IFieldType } from '.';
+
+const mockField = {
+  name: 'foo',
+  scripted: false,
+  searchable: true,
+  type: 'string',
+} as IFieldType;
+
+describe('isFilterable', () => {
+  describe('types', () => {
+    it('should return true for filterable types', () => {
+      ['string', 'number', 'date', 'ip', 'boolean'].forEach((type) => {
+        expect(isFilterable({ ...mockField, type })).toBe(true);
+      });
+    });
+
+    it('should return false for filterable types if the field is not searchable', () => {
+      ['string', 'number', 'date', 'ip', 'boolean'].forEach((type) => {
+        expect(isFilterable({ ...mockField, type, searchable: false })).toBe(false);
+      });
+    });
+
+    it('should return false for un-filterable types', () => {
+      ['geo_point', 'geo_shape', 'attachment', 'murmur3', '_source', 'unknown', 'conflict'].forEach(
+        (type) => {
+          expect(isFilterable({ ...mockField, type })).toBe(false);
+        }
+      );
+    });
+  });
+
+  it('should return true for scripted fields', () => {
+    expect(isFilterable({ ...mockField, scripted: true, searchable: false })).toBe(true);
+  });
+
+  it('should return true for the _id field', () => {
+    expect(isFilterable({ ...mockField, name: '_id' })).toBe(true);
+  });
+});
diff --git a/src/plugins/data/common/index_patterns/index.ts b/src/plugins/data/common/index_patterns/index.ts
@@ -33,3 +33,4 @@ export * from './types';
 export { IndexPatternsService } from './index_patterns';
 export type { IndexPattern } from './index_patterns';
 export * from './errors';
+export { validateDataSourceReference, getIndexPatternTitle } from './utils';
diff --git a/src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts b/src/plugins/data/common/index_patterns/index_patterns/index_pattern.ts
@@ -366,7 +366,7 @@ export class IndexPattern implements IIndexPattern {
     };
   }
 
-  getSaveObjectReference() {
+  getSaveObjectReference = () => {
     return this.dataSourceRef
       ? [
           {
@@ -376,7 +376,7 @@ export class IndexPattern implements IIndexPattern {
           },
         ]
       : [];
-  }
+  };
 
   /**
    * Provide a field, get its formatter

diff --git a/src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts b/src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts
@@ -29,6 +29,7 @@
  */
 
 import { i18n } from '@osd/i18n';
+import { DataSourceAttributes } from 'src/plugins/data_source/common/data_sources';
 import { SavedObjectsClientCommon } from '../..';
 import { createIndexPatternCache } from '.';
 import { IndexPattern } from './index_pattern';
@@ -53,7 +54,7 @@ import { FieldFormatsStartCommon } from '../../field_formats';
 import { UI_SETTINGS, SavedObject } from '../../../common';
 import { SavedObjectNotFound } from '../../../../opensearch_dashboards_utils/common';
 import { IndexPatternMissingIndices } from '../lib';
-import { findByTitle } from '../utils';
+import { findByTitle, getIndexPatternTitle } from '../utils';
 import { DuplicateIndexPatternError } from '../errors';
 
 const indexPatternCache = createIndexPatternCache();
@@ -118,8 +119,28 @@ export class IndexPatternsService {
       fields: ['title'],
       perPage: 10000,
     });
+
+    this.savedObjectsCache = await Promise.all(
+      this.savedObjectsCache.map(async (obj) => {
+        if (obj.type === 'index-pattern') {
+          const result = { ...obj };
+          result.attributes.title = await getIndexPatternTitle(
+            obj.attributes.title,
+            obj.references,
+            this.getDataSource
+          );
+          return result;
+        } else {
+          return obj;
+        }
+      })
+    );
   }
 
+  getDataSource = async (id: string) => {
+    return await this.savedObjectsClient.get<DataSourceAttributes>('data-source', id);
+  };
+
   /**
    * Get list of index pattern ids
    * @param refresh Force refresh of index pattern list
@@ -557,7 +578,11 @@ export class IndexPatternsService {
    */
 
   async createSavedObject(indexPattern: IndexPattern, override = false) {
-    const dupe = await findByTitle(this.savedObjectsClient, indexPattern.title);
+    const dupe = await findByTitle(
+      this.savedObjectsClient,
+      indexPattern.title,
+      indexPattern.dataSourceRef?.id
+    );
     if (dupe) {
       if (override) {
         await this.delete(dupe.id);

diff --git a/src/plugins/data/common/index_patterns/utils.test.ts b/src/plugins/data/common/index_patterns/utils.test.ts
@@ -9,63 +9,84 @@
  * GitHub history for details.
  */
 
-/*
- * Licensed to Elasticsearch B.V. under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch B.V. licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import { isFilterable } from '.';
-import { IFieldType } from './fields';
+import { AuthType, DataSourceAttributes } from 'src/plugins/data_source/common/data_sources';
+import { IndexPatternSavedObjectAttrs } from './index_patterns';
+import { SavedObject, SavedObjectReference } from './types';
+import { getIndexPatternTitle, validateDataSourceReference } from './utils';
 
-const mockField = {
-  name: 'foo',
-  scripted: false,
-  searchable: true,
-  type: 'string',
-} as IFieldType;
+describe('test validateDataSourceReference', () => {
+  const getIndexPatternSavedObjectMock = (mockedFields: any = {}) =>
+    ({ ...mockedFields } as SavedObject<IndexPatternSavedObjectAttrs>);
+  let indexPatternSavedObjectMock;
+  const dataSourceId = 'fakeDataSourceId';
 
-describe('isFilterable', () => {
-  describe('types', () => {
-    it('should return true for filterable types', () => {
-      ['string', 'number', 'date', 'ip', 'boolean'].forEach((type) => {
-        expect(isFilterable({ ...mockField, type })).toBe(true);
-      });
+  test('ivalidateDataSourceReference should return false when datasource reference does not exist in index pattern', () => {
+    indexPatternSavedObjectMock = getIndexPatternSavedObjectMock({
+      references: [{ name: 'someReference' }],
     });
 
-    it('should return false for filterable types if the field is not searchable', () => {
-      ['string', 'number', 'date', 'ip', 'boolean'].forEach((type) => {
-        expect(isFilterable({ ...mockField, type, searchable: false })).toBe(false);
-      });
-    });
+    expect(validateDataSourceReference(indexPatternSavedObjectMock)).toBe(false);
+    expect(validateDataSourceReference(indexPatternSavedObjectMock, dataSourceId)).toBe(false);
+  });
 
-    it('should return false for un-filterable types', () => {
-      ['geo_point', 'geo_shape', 'attachment', 'murmur3', '_source', 'unknown', 'conflict'].forEach(
-        (type) => {
-          expect(isFilterable({ ...mockField, type })).toBe(false);
-        }
-      );
+  test('ivalidateDataSourceReference should return true when datasource reference exists in index pattern, and datasource id matches', () => {
+    indexPatternSavedObjectMock = getIndexPatternSavedObjectMock({
+      references: [{ type: 'data-source', id: dataSourceId }],
     });
+
+    expect(validateDataSourceReference(indexPatternSavedObjectMock)).toBe(false);
+    expect(validateDataSourceReference(indexPatternSavedObjectMock, dataSourceId)).toBe(true);
+  });
+});
+
+describe('test getIndexPatternTitle', () => {
+  const dataSourceMock: SavedObject<DataSourceAttributes> = {
+    id: 'dataSourceId',
+    type: 'data-source',
+    attributes: {
+      title: 'dataSourceMockTitle',
+      endpoint: 'https://fakeendpoint.com',
+      auth: {
+        type: AuthType.NoAuth,
+        credentials: undefined,
+      },
+    },
+    references: [],
+  };
+  const indexPatternMockTitle = 'indexPatternMockTitle';
+  const referencesMock: SavedObjectReference[] = [{ type: 'data-source', id: 'dataSourceId' }];
+
+  let getDataSourceMock: jest.Mock<any, any>;
+
+  beforeEach(() => {
+    getDataSourceMock = jest.fn().mockResolvedValue(dataSourceMock);
+  });
+
+  afterEach(() => {
+    jest.resetAllMocks();
+  });
+
+  test('getIndexPatternTitle should concat datasource title with index pattern title', async () => {
+    const res = await getIndexPatternTitle(
+      indexPatternMockTitle,
+      referencesMock,
+      getDataSourceMock
+    );
+    expect(res).toEqual('dataSourceMockTitle.indexPatternMockTitle');
   });
 
-  it('should return true for scripted fields', () => {
-    expect(isFilterable({ ...mockField, scripted: true, searchable: false })).toBe(true);
+  test('getIndexPatternTitle should return index pattern title, when index-pattern is not referenced to any datasource', async () => {
+    const res = await getIndexPatternTitle(indexPatternMockTitle, [], getDataSourceMock);
+    expect(res).toEqual('indexPatternMockTitle');
   });
 
-  it('should return true for the _id field', () => {
-    expect(isFilterable({ ...mockField, name: '_id' })).toBe(true);
+  test('getIndexPatternTitle should return index pattern title, when failing to fetch datasource info', async () => {
+    getDataSourceMock = jest.fn().mockRejectedValue(new Error('error'));
+    const res = await getIndexPatternTitle(
+      indexPatternMockTitle,
+      referencesMock,
+      getDataSourceMock
+    );
+    expect(res).toEqual('dataSourceId.indexPatternMockTitle');
   });
 });
diff --git a/src/plugins/data/common/index_patterns/utils.ts b/src/plugins/data/common/index_patterns/utils.ts
@@ -28,26 +28,81 @@
  * under the License.
  */
 
+import { DataSourceAttributes } from 'src/plugins/data_source/common/data_sources';
 import type { IndexPatternSavedObjectAttrs } from './index_patterns';
-import type { SavedObjectsClientCommon } from '../types';
+import type { SavedObject, SavedObjectReference, SavedObjectsClientCommon } from '../types';
 
 /**
  * Returns an object matching a given title
  *
  * @param client {SavedObjectsClientCommon}
  * @param title {string}
+ * @param dataSourceId {string}{optional}
  * @returns {Promise<SavedObject|undefined>}
  */
-export async function findByTitle(client: SavedObjectsClientCommon, title: string) {
+export async function findByTitle(
+  client: SavedObjectsClientCommon,
+  title: string,
+  dataSourceId?: string
+) {
   if (title) {
-    const savedObjects = await client.find<IndexPatternSavedObjectAttrs>({
-      type: 'index-pattern',
-      perPage: 10,
-      search: `"${title}"`,
-      searchFields: ['title'],
-      fields: ['title'],
+    const savedObjects = (
+      await client.find<IndexPatternSavedObjectAttrs>({
+        type: 'index-pattern',
+        perPage: 10,
+        search: `"${title}"`,
+        searchFields: ['title'],
+        fields: ['title'],
+      })
+    ).filter((obj) => {
+      return obj && obj.attributes && validateDataSourceReference(obj, dataSourceId);
     });
 
     return savedObjects.find((obj) => obj.attributes.title.toLowerCase() === title.toLowerCase());
   }
 }
+
+// This is used to validate datasource reference of index pattern
+export const validateDataSourceReference = (
+  indexPattern: SavedObject<IndexPatternSavedObjectAttrs>,
+  dataSourceId?: string
+) => {
+  const references = indexPattern.references;
+  if (dataSourceId) {
+    return references.some((ref) => ref.id === dataSourceId && ref.type === 'data-source');
+  } else {
+    // No datasource id passed as input meaning we are getting index pattern from default cluster,
+    // and it's supposed to be an empty array
+    return references.length === 0;
+  }
+};
+
+export const getIndexPatternTitle = async (
+  indexPatternTitle: string,
+  references: SavedObjectReference[],
+  getDataSource: (id: string) => Promise<SavedObject<DataSourceAttributes>>
+): Promise<string> => {
+  const DATA_SOURCE_INDEX_PATTERN_DELIMITER = '.';
+  let dataSourceTitle;
+  const dataSourceReference = references.find((ref) => ref.type === 'data-source');
+
+  // If an index-pattern references datasource, prepend data source name with index pattern name for display purpose
+  if (dataSourceReference) {
+    const dataSourceId = dataSourceReference.id;
+    try {
+      const {
+        attributes: { title },
+        error,
+      } = await getDataSource(dataSourceId);
+      dataSourceTitle = error ? dataSourceId : title;
+    } catch (e) {
+      // use datasource id as title when failing to fetch datasource
+      dataSourceTitle = dataSourceId;
+    }
+
+    return dataSourceTitle.concat(DATA_SOURCE_INDEX_PATTERN_DELIMITER).concat(indexPatternTitle);
+  } else {
+    // if index pattern doesn't reference datasource, return as it is.
+    return indexPatternTitle;
+  }
+};
diff --git a/src/plugins/data/opensearch_dashboards.json b/src/plugins/data/opensearch_dashboards.json
@@ -5,7 +5,7 @@
   "ui": true,
   "requiredPlugins": ["expressions", "uiActions"],
   "optionalPlugins": ["usageCollection", "dataSource"],
-  "extraPublicDirs": ["common", "common/utils/abort_utils"],
+  "extraPublicDirs": ["common", "common/utils/abort_utils", "common/index_patterns/utils.ts"],
   "requiredBundles": [
     "usageCollection",
     "opensearchDashboardsUtils",
-Original file line number
+Diff line change
@@ Expand Up / @@ -366,7 +366,7 @@ export class IndexPattern implements IIndexPattern { @@
         };
       }
-      getSaveObjectReference() {
+      getSaveObjectReference = () => {
         return this.dataSourceRef
           ? [
               {
@@ Expand All / @@ -376,7 +376,7 @@ export class IndexPattern implements IIndexPattern { @@
               },
             ]
           : [];
-      }
+      };
       /**
        * Provide a field, get its formatter
@@ Expand Down @@