Implement ObjectID to replace mongodb

We need to remove code related to mongodb as we are using postgrace. This is to be done in steps and first step is to implement ObjectId class and replace mongodb.ObjectId calls with that class. Signed-off-by: Ashish Pandey <[email protected]>
noobaa · Nov 27, 2024 · 01897a3 · 01897a3
1 parent e5e6a35
commit 01897a3
Show file tree

Hide file tree

Showing 26 changed files with 288 additions and 122 deletions.
diff --git a/src/agent/block_store_speed.js b/src/agent/block_store_speed.js
@@ -4,14 +4,14 @@
 // const _ = require('lodash');
 const argv = require('minimist')(process.argv);
 const cluster = require('cluster');
-const mongodb = require('mongodb');
 
 const api = require('../api');
 const config = require('../../config');
 const dotenv = require('../util/dotenv');
 const Speedometer = require('../util/speedometer');
 const { RPC_BUFFERS } = require('../rpc');
 
+const ObjectID = require('../util/objectid');
 dotenv.load();
 
 argv.email = argv.email || '[email protected]';
@@ -60,7 +60,7 @@ async function worker(client) {
 }
 
 async function write_block(client) {
-    const block_id = new mongodb.ObjectId();
+    const block_id = new ObjectID(null);
     return client.block_store.write_block({
         [RPC_BUFFERS]: { data: Buffer.allocUnsafe(argv.size) },
         block_md: {

diff --git a/src/cmd/manage_nsfs.js b/src/cmd/manage_nsfs.js
@@ -11,7 +11,7 @@ const nb_native = require('../util/nb_native');
 const { ConfigFS } = require('../sdk/config_fs');
 const cloud_utils = require('../util/cloud_utils');
 const native_fs_utils = require('../util/native_fs_utils');
-const mongo_utils = require('../util/mongo_utils');
+const ObjectID = require('../util/objectid');
 const SensitiveString = require('../util/sensitive_string');
 const { account_id_cache } = require('../sdk/accountspace_fs');
 const ManageCLIError = require('../manage_nsfs/manage_nsfs_cli_errors').ManageCLIError;
@@ -169,7 +169,7 @@ async function merge_new_and_existing_config_data(user_input_bucket_data) {
  * @returns { Promise<{ code: ManageCLIResponse.BucketCreated, detail: Object, event_arg: Object }>} 
  */
 async function add_bucket(data) {
-    data._id = mongo_utils.mongoObjectId();
+    data._id = new ObjectID(null);
     const parsed_bucket_data = await config_fs.create_bucket_config_file(data);
     await set_bucker_owner(parsed_bucket_data);
     return { code: ManageCLIResponse.BucketCreated, detail: parsed_bucket_data, event_arg: { bucket: data.name }};
@@ -413,7 +413,7 @@ async function fetch_existing_account_data(action, target, decrypt_secret_key) {
  * @returns { Promise<{ code: typeof ManageCLIResponse.AccountCreated, detail: Object, event_arg: Object  }>} 
  */
 async function add_account(data) {
-    data._id = mongo_utils.mongoObjectId();
+    data._id = new ObjectID(null);
     await config_fs.create_account_config_file(data);
     return { code: ManageCLIResponse.AccountCreated, detail: data, event_arg: { account: data.name } };
 }

diff --git a/src/nc/nc_utils.js b/src/nc/nc_utils.js
@@ -1,8 +1,7 @@
 /* Copyright (C) 2024 NooBaa */
 'use strict';
 
-const mongo_utils = require('../util/mongo_utils');
-
+const objectid = require('../util/objectid');
 /**
  * generate_id will generate an id that we use to identify entities (such as account, bucket, etc.). 
  */
@@ -11,7 +10,7 @@ const mongo_utils = require('../util/mongo_utils');
 // - this function implantation should be db_client.new_object_id(), 
 //   but to align with manage nsfs we won't change it now
 function generate_id() {
-    return mongo_utils.mongoObjectId();
+    return objectid();
 }
 
 /**

diff --git a/src/sdk/bucketspace_fs.js b/src/sdk/bucketspace_fs.js
@@ -1,6 +1,7 @@
 /* Copyright (C) 2020 NooBaa */
 'use strict';
 
+const objectid = require('../util/objectid');
 const _ = require('lodash');
 const util = require('util');
 const path = require('path');
@@ -9,7 +10,6 @@ const config = require('../../config');
 const RpcError = require('../rpc/rpc_error');
 const js_utils = require('../util/js_utils');
 const nb_native = require('../util/nb_native');
-const mongo_utils = require('../util/mongo_utils');
 const KeysSemaphore = require('../util/keys_semaphore');
 const {
     get_umasked_mode,
@@ -314,7 +314,7 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {
 
     new_bucket_defaults(account, { name, tag, lock_enabled, force_md5_etag }, create_uls, bucket_storage_path) {
         return {
-            _id: mongo_utils.mongoObjectId(),
+            _id: objectid(),
             name,
             tag: js_utils.default_value(tag, undefined),
             owner_account: account._id,

diff --git a/src/sdk/nb.d.ts b/src/sdk/nb.d.ts
@@ -5,7 +5,9 @@ import * as mongodb from 'mongodb';
 import { EventEmitter } from 'events';
 import { Readable, Writable } from 'stream';
 import { IncomingMessage, ServerResponse } from 'http';
-import { ObjectPart, Checksum} from '@aws-sdk/client-s3';
+import { ObjectPart, Checksum } from '@aws-sdk/client-s3';
+
+import ObjectID = require("../util/objectid");
 
 type Semaphore = import('../util/semaphore');
 type KeysSemaphore = import('../util/keys_semaphore');
@@ -39,14 +41,16 @@ type ReplicationLogCandidates = Record<string, { action: ReplicationLogAction, t
 
 type BucketDiffKeysDiff = { [key: string]: Array<object> };
 
+
+
 interface MapByID<T> { [id: string]: T }
 
 interface Base {
     toJSON?(): object | string;
     toString?(): string;
 }
 
-type ID = mongodb.ObjectID;
+type ID = ObjectID;
 type DBBuffer = mongodb.Binary | Buffer;
 
 interface System extends Base {
@@ -720,8 +724,8 @@ interface DBClient {
     populate(docs: object[] | object, doc_path: string, collection: DBCollection, fields: object): Promise<object[] | object>;
     resolve_object_ids_recursive(idmap: object, item: object): object;
     resolve_object_ids_paths(idmap: object, item: object, paths: string[], allow_missing: boolean): object;
-    new_object_id(): mongodb.ObjectId;
-    parse_object_id(id_str: string): mongodb.ObjectId;
+    new_object_id(): ObjectID;
+    parse_object_id(id_str: string): ObjectID;
     fix_id_type(doc: object[] | object): object[] | object;
     is_object_id(id: object[] | object): boolean;
     is_err_duplicate_key(err: object): boolean;
@@ -1150,4 +1154,4 @@ interface GetObjectAttributesParts {
     MaxParts?: number;
     IsTruncated?: boolean;
     Parts?: ObjectPart[];
-  }
+}
diff --git a/src/sdk/objectid.d.ts b/src/sdk/objectid.d.ts
@@ -0,0 +1,32 @@
+/* Copyright (C) 2016 NooBaa */
+
+/**
+ * Represents a BSON ObjectID type.
+ */
+declare interface ObjectID {
+    readonly id: string | Buffer;
+    readonly str: string;
+
+    toHexString(): string;
+    equals(other: ObjectID | string): boolean;
+    getTimestamp(): Date;
+    generate(time?: number): string;
+    toJSON(): string;
+    toString(): string;
+}
+
+/**
+ * Constructor and utility functions for the ObjectID type.
+ */
+declare interface ObjectIDConstructor {
+    new(id?: string | number | Buffer): ObjectID;
+    isValid(id: string | Buffer | ObjectID): boolean;
+    (id?: string | number | Buffer): ObjectID;
+}
+
+/**
+ * The ObjectID constructor and utilities.
+ */
+declare const ObjectID: ObjectIDConstructor;
+
+export = ObjectID;
diff --git a/src/server/analytic_services/activity_log_store.js b/src/server/analytic_services/activity_log_store.js
@@ -1,10 +1,10 @@
 /* Copyright (C) 2016 NooBaa */
 'use strict';
 
-const mongodb = require('mongodb');
 const _ = require('lodash');
 
 const db_client = require('../../util/db_client');
+const ObjectID = require('../../util/objectid');
 const P = require('../../util/promise');
 const activity_log_schema = require('./activity_log_schema');
 const activity_log_indexes = require('./activity_log_indexes');
@@ -25,7 +25,7 @@ class ActivityLogStore {
     }
 
     make_activity_log_id(id_str) {
-        return new mongodb.ObjectID(id_str);
+        return new ObjectID(id_str);
     }
 
 

diff --git a/src/server/analytic_services/history_data_store.js b/src/server/analytic_services/history_data_store.js
@@ -1,8 +1,6 @@
 /* Copyright (C) 2016 NooBaa */
 'use strict';
 
-const mongodb = require('mongodb');
-
 // const dbg = require('../../util/debug_module')(__filename);
 const config = require('../../../config.js');
 // const pkg = require('../../../package.json');
@@ -11,6 +9,7 @@ const P = require('../../util/promise');
 const db_client = require('../../util/db_client');
 const system_history_schema = require('../analytic_services/system_history_schema');
 
+const ObjectID = require('../../util/objectid');
 class HistoryDataStore {
 
     constructor() {
@@ -30,7 +29,7 @@ class HistoryDataStore {
             const time_stamp = new Date();
             const record_expiration_date = new Date(time_stamp.getTime() - config.STATISTICS_COLLECTOR_EXPIRATION);
             const record = {
-                _id: new mongodb.ObjectId(),
+                _id: new ObjectID(null),
                 time_stamp,
                 system_snapshot: item,
                 history_type: 'SYSTEM'

diff --git a/src/server/func_services/func_stats_store.js b/src/server/func_services/func_stats_store.js
@@ -2,8 +2,7 @@
 'use strict';
 
 // const _ = require('lodash');
-const mongodb = require('mongodb');
-
+const ObjectID = require('../../util/objectid');
 // const dbg = require('../../util/debug_module')(__filename);
 const db_client = require('../../util/db_client');
 
@@ -26,7 +25,7 @@ class FuncStatsStore {
     }
 
     make_func_stat_id(id_str) {
-        return new mongodb.ObjectId(id_str);
+        return new ObjectID(id_str);
     }
 
     async create_func_stat(stat) {

diff --git a/src/server/func_services/func_store.js b/src/server/func_services/func_store.js
@@ -1,13 +1,11 @@
 /* Copyright (C) 2016 NooBaa */
 'use strict';
 
-const mongodb = require('mongodb');
-
 const db_client = require('../../util/db_client');
 
 const func_schema = require('./func_schema');
 const func_indexes = require('./func_indexes');
-
+const ObjectID = require('../../util/objectid');
 class FuncStore {
 
     constructor() {
@@ -24,7 +22,7 @@ class FuncStore {
     }
 
     make_func_id(id_str) {
-        return new mongodb.ObjectId(id_str);
+        return new ObjectID(id_str);
     }
 
     async create_func(func) {

diff --git a/src/server/node_services/nodes_store.js b/src/server/node_services/nodes_store.js
@@ -2,13 +2,12 @@
 'use strict';
 
 const _ = require('lodash');
-const mongodb = require('mongodb');
 
 const dbg = require('../../util/debug_module')(__filename);
 const node_schema = require('./node_schema');
 const db_client = require('../../util/db_client');
 const P = require('../../util/promise');
-
+const ObjectID = require('../../util/objectid');
 class NodesStore {
 
     constructor(test_suffix = '') {
@@ -24,7 +23,7 @@ class NodesStore {
     }
 
     make_node_id(id_str) {
-        return new mongodb.ObjectId(id_str);
+        return new ObjectID(id_str);
     }
 
     is_connected() {

diff --git a/src/server/notifications/alerts_log_store.js b/src/server/notifications/alerts_log_store.js
@@ -1,13 +1,15 @@
 /* Copyright (C) 2016 NooBaa */
 'use strict';
 
-const mongodb = require('mongodb');
 const _ = require('lodash');
 const P = require('../../util/promise');
 const db_client = require('../../util/db_client');
 
 const alerts_log_schema = require('./alerts_log_schema');
 
+// @ts-ignore
+const ObjectID = require('../../util/objectid');
+
 class AlertsLogStore {
 
     constructor() {
@@ -22,8 +24,8 @@ class AlertsLogStore {
         return AlertsLogStore._instance;
     }
 
-    make_alert_log_id(id_str) {
-        return new mongodb.ObjectID(id_str);
+    make_alert_log_id() {
+        return new ObjectID(null);
     }
 
     create(alert_log) {
@@ -94,12 +96,12 @@ class AlertsLogStore {
 
         let _id;
         if (ids) {
-            const obj_ids = ids.map(id => new mongodb.ObjectID(id));
+            const obj_ids = ids.map(id => new ObjectID(id));
             _id = { $in: obj_ids };
         } else if (till) {
-            _id = { $lt: new mongodb.ObjectID(till) };
+            _id = { $lt: new ObjectID(till) };
         } else if (since) {
-            _id = { $gt: new mongodb.ObjectID(since) };
+            _id = { $gt: new ObjectID(since) };
         }
 
         return _.omitBy({

diff --git a/src/server/object_services/md_store.js b/src/server/object_services/md_store.js
@@ -6,9 +6,8 @@
 const _ = require('lodash');
 const assert = require('assert');
 const moment = require('moment');
-const mongodb = require('mongodb');
 const mime = require('mime');
-
+const ObjectID = require('../../util/objectid');
 const P = require('../../util/promise');
 const dbg = require('../../util/debug_module')(__filename);
 const db_client = require('../../util/db_client');
@@ -80,18 +79,18 @@ class MDStore {
         if (zero_suffix) {
             suffix = '0'.repeat(16);
         } else {
-            suffix = String(new mongodb.ObjectId()).slice(8, 24);
+            suffix = String(new ObjectID(null)).slice(8, 24);
         }
         const hex_id = padded_hex_time + suffix;
         assert(padded_hex_time.length === 8);
         assert(suffix.length === 16);
         assert(hex_id.length === 24);
         assert(parseInt(padded_hex_time, 16) === Math.floor(time / 1000));
-        return new mongodb.ObjectId(hex_id);
+        return new ObjectID(hex_id);
     }
 
     is_valid_md_id(id_str) {
-        return mongodb.ObjectId.isValid(id_str);
+        return ObjectID.isValid(id_str);
     }
 
     /////////////
@@ -1869,7 +1868,7 @@ function sort_list_uploads_with_delimiter(a, b) {
  * @returns {nb.ID}
  */
 function make_md_id(id_str) {
-    return new mongodb.ObjectId(id_str);
+    return new ObjectID(id_str);
 }
 
 

diff --git a/src/server/system_services/config_file_store.js b/src/server/system_services/config_file_store.js
@@ -4,10 +4,9 @@
 const _ = require('lodash');
 const dbg = require('../../util/debug_module')(__filename);
 const db_client = require('../../util/db_client');
-const mongodb = require('mongodb');
 const config_file_schema = require('./schemas/config_file_schema');
 const config_file_indexes = require('./schemas/config_file_indexes');
-
+const ObjectID = require('../../util/objectid');
 class ConfigFileStore {
 
     constructor() {
@@ -26,7 +25,7 @@ class ConfigFileStore {
     async insert(item) {
         dbg.log0(`insert`, item);
         _.defaults(item, {
-            _id: new mongodb.ObjectId()
+            _id: new ObjectID(null)
         });
         // There shouldn't be more than one record, this is being on the safe side
         this._config_files.validate(item);