diff --git a/bruno/Get API Root Information.bru b/bruno/Get API Root Information.bru index dc90915..e306035 100644 --- a/bruno/Get API Root Information.bru +++ b/bruno/Get API Root Information.bru @@ -29,9 +29,12 @@ tests { const data = res.getBody(); expect(data).to.have.property('title').that.is.a('string').and.is.not.empty; expect(data).to.have.property('description').that.is.a('string').and.is.not.empty; - expect(data).to.have.property('version').that.is.a('string').and.startsWith('application/taxii+json'); - expect(data).to.have.property('maxContentLength'); - const maxContentLength = data.maxContentLength; + expect(data).to.have.property('versions').that.is.an('array'); + data.versions.forEach(version => { + expect(version).startsWith('application/taxii+json'); + }); + expect(data).to.have.property('max_content_length'); + const maxContentLength = data['max_content_length']; if (typeof maxContentLength === 'string') { expect(parseInt(maxContentLength)).to.be.a('number').and.not.NaN; diff --git a/bruno/Get Object Versions.bru b/bruno/Get Object Versions.bru index 1308b60..37c6bb1 100644 --- a/bruno/Get Object Versions.bru +++ b/bruno/Get Object Versions.bru @@ -36,7 +36,9 @@ tests { test("Verify response body structure", function() { const data = res.getBody(); - expect(data).to.have.property('more').that.is.a('boolean'); + if (data.hasOwnProperty('more')) { + expect(data.more).to.be.a('boolean'); + } // The list of object versions returned by the request. if (data.hasOwnProperty('versions')) { diff --git a/src/common/middleware/content-negotiation/content-negotiation.middleware.ts b/src/common/middleware/content-negotiation/content-negotiation.middleware.ts index c18f62b..1de52c9 100644 --- a/src/common/middleware/content-negotiation/content-negotiation.middleware.ts +++ b/src/common/middleware/content-negotiation/content-negotiation.middleware.ts @@ -20,6 +20,11 @@ export class ContentNegotiationMiddleware implements NestMiddleware { // Get a hook into the request context so we can do logging const ctx: RequestContext = RequestContextModel.get(); + // Do not enforce TAXII headers on health check endpoint + if (req.path == "/health/ping") { + return next(); + } + // Extract 'Accept' header const mediaType: string = req.headers["accept"]; diff --git a/src/config/interfaces/taxii-config.service.interface.ts b/src/config/interfaces/taxii-config.service.interface.ts index ab01693..497817c 100644 --- a/src/config/interfaces/taxii-config.service.interface.ts +++ b/src/config/interfaces/taxii-config.service.interface.ts @@ -1,6 +1,6 @@ export interface TaxiiConfigServiceInterface { createAppConnectOptions(); - createCollectorConnectOptions(); + createHydrateConnectOptions(); createDatabaseConnectOptions(); createStixConnectOptions(); get APP_ADDRESS(); diff --git a/src/config/taxii-config.service.ts b/src/config/taxii-config.service.ts index 6dffea8..a94ed9a 100644 --- a/src/config/taxii-config.service.ts +++ b/src/config/taxii-config.service.ts @@ -6,8 +6,7 @@ import { StixConnectOptions } from "../stix/interfaces"; import { isDefined } from "class-validator"; import { AppConnectOptions } from "../interfaces"; import { DatabaseConnectOptions } from "../interfaces/database-connect-options.interface"; -import { CollectorConnectOptions } from "../hydrate/collector/interfaces/collector-connect.options"; - +import { HydrateConnectOptions } from "src/hydrate/interfaces/hydrate-connect.options"; /** * This provider is responsible for loading all user-definable configuration parameters (imported from * environment variables) and making them available to all necessary app services @@ -23,7 +22,7 @@ export class TaxiiConfigService implements TaxiiConfigServiceInterface { }; } - createCollectorConnectOptions(): CollectorConnectOptions { + createHydrateConnectOptions(): HydrateConnectOptions { return { hydrateOnBoot: this.HYDRATE_ON_BOOT, ...this.createAppConnectOptions(), diff --git a/src/hydrate.ts b/src/hydrate.ts index 45c4511..a240d0d 100644 --- a/src/hydrate.ts +++ b/src/hydrate.ts @@ -14,18 +14,12 @@ export async function bootstrap() { tempConfigApp.get(TaxiiConfigService); const app: NestApplication = await NestFactory.create( - HydrateModule.register(tempConfigService.createCollectorConnectOptions()) + HydrateModule.register(tempConfigService.createHydrateConnectOptions()) ); // ** Initialize the Nest application ** // await app.init(); - // Start the 'get-attack-objects' cron job to pre-populate the TAXII DB (MongoDB) with STIX - if (tempConfigService.HYDRATE_ON_BOOT) { - const provider = app.get(HydrateService); - await provider.hydrate(); - } - console.log(`Bootstrap process completed...cleaning up...`); await tempConfigApp.close(); } diff --git a/src/hydrate/collector/collector.module.ts b/src/hydrate/collector/collector.module.ts deleted file mode 100644 index 7f15d7e..0000000 --- a/src/hydrate/collector/collector.module.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { Module } from "@nestjs/common"; -import { ObjectCollectorService } from "./providers/object-collector.service"; -import { CollectionCollectorService } from "./providers/collection-collector.service"; -import { MongooseModule } from "@nestjs/mongoose"; -import { AttackObjectEntity, AttackObjectSchema } from "./schema"; -import { TaxiiCollection, TaxiiCollectionSchema } from "./schema"; - -@Module({ - imports: [ - MongooseModule.forFeature([ - { name: AttackObjectEntity.name, schema: AttackObjectSchema }, - { name: TaxiiCollection.name, schema: TaxiiCollectionSchema }, - ]), - ], - providers: [ObjectCollectorService, CollectionCollectorService], - exports: [ObjectCollectorService, CollectionCollectorService], -}) -export class CollectorModule {} diff --git a/src/hydrate/collector/constants.ts b/src/hydrate/collector/constants.ts deleted file mode 100644 index b4b674c..0000000 --- a/src/hydrate/collector/constants.ts +++ /dev/null @@ -1,2 +0,0 @@ -export const GET_ATTACK_OBJECTS_JOB_TOKEN = "GET_ATTACK_OBJECTS"; -export const GET_STIX_COLLECTIONS_JOB_TOKEN = "GET_STIX_COLLECTIONS"; diff --git a/src/hydrate/collector/interfaces/collector-connect.options.ts b/src/hydrate/collector/interfaces/collector-connect.options.ts deleted file mode 100644 index 505aae2..0000000 --- a/src/hydrate/collector/interfaces/collector-connect.options.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { StixConnectOptions } from "src/stix/interfaces"; -import { DatabaseConnectOptions } from "src/interfaces/database-connect-options.interface"; - -export interface CollectorConnectOptions { - hydrateOnBoot: boolean; - databaseConnectOptions: DatabaseConnectOptions; - stixConnectOptions: StixConnectOptions; -} diff --git a/src/hydrate/collector/providers/collection-collector.service.ts b/src/hydrate/collector/providers/collection-collector.service.ts deleted file mode 100644 index 4c0e541..0000000 --- a/src/hydrate/collector/providers/collection-collector.service.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { Inject, Injectable } from "@nestjs/common"; -import { Cron, CronExpression } from "@nestjs/schedule"; -import { InjectModel } from "@nestjs/mongoose"; -import { Logger } from "@nestjs/common"; -import { LoggerService } from "@nestjs/common/services/logger.service"; -import { FilterQuery, Model } from "mongoose"; -import { StixObjectInterface } from "src/stix/interfaces/stix-object.interface"; -import { - TaxiiCollection, - TaxiiCollectionDocument, -} from "src/hydrate/collector/schema"; -import { TaxiiCollectionDto } from "src/taxii/providers/collection/dto"; -import { STIX_REPO_TOKEN } from "src/stix/constants"; -import { StixRepositoryInterface } from "src/stix/providers/stix.repository.interface"; -import { GET_STIX_COLLECTIONS_JOB_TOKEN } from "../constants"; - -@Injectable() -export class CollectionCollectorService { - private readonly logger: LoggerService = new Logger( - CollectionCollectorService.name - ); - - constructor( - @Inject(STIX_REPO_TOKEN) private stixRepo: StixRepositoryInterface, - @InjectModel(TaxiiCollection.name) private collectionModel: Model - ) {} - - @Cron(CronExpression.EVERY_30_MINUTES, { - name: GET_STIX_COLLECTIONS_JOB_TOKEN, - }) - async findAndStoreStixCollections(): Promise { - this.logger.debug("Starting database collection hydration"); - - // If the collections are not present in the DB, then fallback to retrieving them directly - // from the STIX repository (Workbench) - const stixCollections: StixObjectInterface[] = - await this.stixRepo.getCollections(); - - this.logger.debug( - "Successfully retrieved STIX collections from the STIX repository" - ); - - // Transform STIX to TAXII - for (const stixCollection of stixCollections) { - // Transform the collection into a TAXII-compliant collection object - const taxiiCollection = new TaxiiCollectionDto(stixCollection.stix); - - // Push the TAXII collection to the database - try { - this.logger.debug( - `Pushing TAXII collection ${stixCollection.stix.id} to the database` - ); - const filter: FilterQuery = { - id: { $eq: taxiiCollection.id }, - }; - await this.collectionModel - .updateOne(filter, taxiiCollection, { - upsert: true, - strict: true, - }) - .exec(); - this.logger.debug( - `Synchronized STIX collection '${taxiiCollection.id}' to TAXII database` - ); - } catch (e) { - // Handle I/O exceptions thrown by Mongoose - this.logger.error( - `An issue occurred while writing TAXII collection ${stixCollection.stix.id} to the database` - ); - this.logger.error(e); - } - } - } -} diff --git a/src/hydrate/collector/providers/object-collector.service.ts b/src/hydrate/collector/providers/object-collector.service.ts deleted file mode 100644 index a5eeaa3..0000000 --- a/src/hydrate/collector/providers/object-collector.service.ts +++ /dev/null @@ -1,125 +0,0 @@ -import { Inject, Injectable, Logger, LoggerService } from "@nestjs/common"; -import { FilterQuery, Model } from "mongoose"; -import { InjectModel } from "@nestjs/mongoose"; -import { Cron, CronExpression } from "@nestjs/schedule"; -import { AttackObjectDto } from "src/stix/dto/attack-object.dto"; -import { AttackObjectEntity } from "../schema"; -import { STIX_REPO_TOKEN } from "../../../stix/constants"; -import { StixRepositoryInterface } from "../../../stix/providers/stix.repository.interface"; -import { GET_ATTACK_OBJECTS_JOB_TOKEN } from "../constants"; -import { StixBundleDto } from "src/stix/dto/stix-bundle.dto"; -import { WorkbenchCollectionDto } from "src/stix/dto/workbench-collection.dto"; - -@Injectable() -export class ObjectCollectorService { - private readonly logger: LoggerService = new Logger(ObjectCollectorService.name); - - constructor( - @Inject(STIX_REPO_TOKEN) private readonly workbench: StixRepositoryInterface, - @InjectModel(AttackObjectEntity.name) private stixObjectModel: Model - ) {} - - private async processStixBundles(domain: 'enterprise-attack' | 'ics-attack' | 'mobile-attack', collectionId: string) { - const objectsToWriteToDatabase = []; - - const stixBundles = [ - { version: '2.0', bundle: await this.workbench.getStixBundle(domain, '2.0') }, - { version: '2.1', bundle: await this.workbench.getStixBundle(domain, '2.1') }, - ]; - - for (const { version, bundle } of stixBundles) { - this.logger.debug(`Retrieved STIX ${version} Bundle for ATT&CK domain '${domain}' - Object Count: ${bundle.objects.length}`); - bundle.objects.forEach((stixObject) => { - objectsToWriteToDatabase.push({ - collection_id: collectionId, - stix: stixObject, - created_at: new Date().toISOString(), // Add created_at field to capture the current timestamp - }); - }); - } - - return objectsToWriteToDatabase; - } - - @Cron(CronExpression.EVERY_30_MINUTES, { name: GET_ATTACK_OBJECTS_JOB_TOKEN }) - async findAndStoreStixObjects() { - this.logger.debug("Starting database object hydration"); - - const objectsToWriteToDatabase = []; - const stixCollections = await this.workbench.getCollections(); - this.logger.debug(`Retrieved ${stixCollections.length} collections`); - - for (const collection of stixCollections) { - this.logger.debug(`Processing STIX bundle: ${collection.stix.name}`); - - let results = []; - switch (collection.stix.name) { - case "Enterprise ATT&CK": - results = await this.processStixBundles('enterprise-attack', collection.stix.id); - break; - case "ICS ATT&CK": - results = await this.processStixBundles('ics-attack', collection.stix.id); - break; - case "Mobile ATT&CK": - results = await this.processStixBundles('mobile-attack', collection.stix.id); - break; - default: - continue; - } - objectsToWriteToDatabase.push(...results); - } - - if (objectsToWriteToDatabase.length === 0) { - this.logger.debug("No objects to write to the database"); - return; - } - - this.logger.debug(`Processed ${objectsToWriteToDatabase.length} ATT&CK objects`); - // Sort the list of objects by the 'created' date in ascending order. - // This works by calculating the difference between timestamps directly: - // - If 'a' was created earlier than 'b', the result is negative (placing 'a' before 'b'). - // - If 'a' was created later than 'b', the result is positive (placing 'b' before 'a'). - // - If they were created at the same time, the result is 0 (no change in order). - objectsToWriteToDatabase.sort((a, b) => new Date(a.stix.created).valueOf() - new Date(b.stix.created).valueOf()); - - objectsToWriteToDatabase.sort((a, b) => new Date(a.stix.created).valueOf() - new Date(b.stix.created).valueOf()); - - let bulkOps = []; - for (const object of objectsToWriteToDatabase) { - const filter: FilterQuery = { - "stix.id": { $eq: object.stix.id }, - "stix.modified": { $eq: object.stix.modified }, - "stix.created": { $eq: object.stix.created }, - "stix.spec_version": { $eq: object.stix.spec_version }, - }; - - bulkOps.push({ - updateOne: { - filter, - update: { - $setOnInsert: { created_at: object.created_at }, // Only set created_at if this is a new document - $set: { - collection_id: object.collection_id, - stix: object.stix, - }, - }, - upsert: true, // If the object doesn't exist, insert a new document. If it does exist, update the existing document - strict: false, // Disabling strict mode allows us to capture properties not explicitly declared in the schema - }, - }); - - // Execute bulkWrite for every 1000 operations or at the end of the loop - if (bulkOps.length === 1000) { - await this.stixObjectModel.bulkWrite(bulkOps); - bulkOps = []; // Clear the array for the next batch of operations - } - } - - // execute the remaining operations, if any - if (bulkOps.length > 0) { - await this.stixObjectModel.bulkWrite(bulkOps); - } - - this.logger.debug("Completed database object hydration"); - } -} diff --git a/src/hydrate/collector/schema/attack-object.schema.ts b/src/hydrate/collector/schema/attack-object.schema.ts deleted file mode 100644 index a783ce2..0000000 --- a/src/hydrate/collector/schema/attack-object.schema.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { Prop, Schema, SchemaFactory } from "@nestjs/mongoose"; -import { Document } from "mongoose"; -import { StixProperties, StixPropertiesSchema } from "./stix-properties.schema"; -import * as mongoose from "mongoose"; - -@Schema({ - collection: "object-resources", - versionKey: false, - /** - * NOTE Until we integrate the attack-data-model to handle data parsing/validation, - * we need to disable strict mode to ensure that values passed to the model - * constructor that are not specified in the schema can be saved to the db. Strict - * mode is just too prone to inadvertently dropping valid STIX properties. - */ - strict: false -}) -export class AttackObjectEntity extends Document { - @Prop(mongoose.Schema.Types.String) - collection_id: string; - - @Prop({ type: StixPropertiesSchema }) - stix: StixProperties; - - @Prop(mongoose.Schema.Types.Date) - created_at: Date; -} - -export type AttackObjectDocument = AttackObjectEntity & Document; - -// Create the schema -export const AttackObjectSchema = SchemaFactory.createForClass(AttackObjectEntity); diff --git a/src/hydrate/collector/schema/taxii-collection.schema.ts b/src/hydrate/collector/schema/taxii-collection.schema.ts deleted file mode 100644 index 3d066f1..0000000 --- a/src/hydrate/collector/schema/taxii-collection.schema.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { Prop, Schema, SchemaFactory } from "@nestjs/mongoose"; -import { Document } from "mongoose"; - -export type TaxiiCollectionDocument = TaxiiCollection & Document; - -@Schema({ - collection: "collection-resources", -}) -export class TaxiiCollection { - @Prop({ - required: true, - unique: true, - message: "collection_id must be unique", - }) - id: string; - - @Prop({ required: true }) - title: string; - - @Prop({ required: false }) - description: string; - - @Prop({ required: false }) - alias: string; - - @Prop({ required: true }) - canRead: boolean; - - @Prop({ required: true }) - canWrite: boolean; - - @Prop({ type: [String], required: false }) - mediaTypes: string[]; -} - -export const TaxiiCollectionSchema = - SchemaFactory.createForClass(TaxiiCollection); diff --git a/src/hydrate/constants.ts b/src/hydrate/constants.ts new file mode 100644 index 0000000..a7d65d0 --- /dev/null +++ b/src/hydrate/constants.ts @@ -0,0 +1,2 @@ +export const GET_TAXII_RESOURCES_JOB_TOKEN = "GET_TAXII_RESOURCES"; +export const HYDRATE_OPTIONS_TOKEN = "HYDRATE_OPTIONS"; \ No newline at end of file diff --git a/src/hydrate/hydrate.module.ts b/src/hydrate/hydrate.module.ts index 4aa9816..be84254 100644 --- a/src/hydrate/hydrate.module.ts +++ b/src/hydrate/hydrate.module.ts @@ -1,27 +1,36 @@ import { DynamicModule, Global, Module } from "@nestjs/common"; import { ScheduleModule } from "@nestjs/schedule"; import { MongooseModule } from "@nestjs/mongoose"; -import { CollectorModule } from "./collector/collector.module"; -import { CollectorConnectOptions } from "./collector/interfaces/collector-connect.options"; +import { HydrateConnectOptions } from "./interfaces/hydrate-connect.options"; import { HydrateService } from "./hydrate.service"; import { StixModule } from "../stix/stix.module"; +import { AttackObjectEntity, AttackObjectSchema, TaxiiCollectionEntity, TaxiiCollectionSchema } from "./schema"; +import { HYDRATE_OPTIONS_TOKEN } from "./constants"; @Global() @Module({}) export class HydrateModule { - static register(options: CollectorConnectOptions): DynamicModule { + static register(options: HydrateConnectOptions): DynamicModule { + console.log(options); return { module: HydrateModule, imports: [ ScheduleModule.forRoot(), - MongooseModule.forRoot(options.databaseConnectOptions.mongoUri), - + MongooseModule.forFeature([ + { name: AttackObjectEntity.name, schema: AttackObjectSchema }, + { name: TaxiiCollectionEntity.name, schema: TaxiiCollectionSchema }, + ]), StixModule.register(options.stixConnectOptions), - - CollectorModule, ], - providers: [HydrateService], + providers: [ + { + provide: HYDRATE_OPTIONS_TOKEN, + useValue: options + }, + HydrateService + ], + exports: [HydrateService], }; } -} +} \ No newline at end of file diff --git a/src/hydrate/hydrate.service.ts b/src/hydrate/hydrate.service.ts index 9138ac8..a985b07 100644 --- a/src/hydrate/hydrate.service.ts +++ b/src/hydrate/hydrate.service.ts @@ -1,16 +1,317 @@ -import { Injectable } from "@nestjs/common"; -import { CollectionCollectorService } from "./collector/providers/collection-collector.service"; -import { ObjectCollectorService } from "./collector/providers/object-collector.service"; +import { Inject, Injectable, OnModuleInit } from "@nestjs/common"; +import { Cron, CronExpression } from "@nestjs/schedule"; +import { InjectModel } from "@nestjs/mongoose"; +import { Logger } from "@nestjs/common"; +import { LoggerService } from "@nestjs/common/services/logger.service"; +import { Model } from "mongoose"; +import { TaxiiCollectionEntity, TaxiiCollectionDocument, AttackObjectEntity, AttackObjectDocument } from "./schema"; +import { TaxiiCollectionDto } from "src/taxii/providers/collection/dto"; +import { WorkbenchCollectionDto } from "src/stix/dto/workbench-collection.dto"; +import { STIX_REPO_TOKEN } from "src/stix/constants"; +import { StixRepositoryInterface } from "src/stix/providers/stix.repository.interface"; +import { GET_TAXII_RESOURCES_JOB_TOKEN, HYDRATE_OPTIONS_TOKEN } from "./constants"; +import { HydrateConnectOptions } from "./interfaces/hydrate-connect.options"; +/** + * Service responsible for synchronizing TAXII collections and objects with ATT&CK Workbench. + * + * This service maintains exact synchronization with Workbench by: + * 1. Always preferring the Workbench state, regardless of version + * 2. Maintaining history through active/inactive states + * 3. Reactivating previously seen versions when possible + * 4. Creating new resources for unseen versions + */ @Injectable() -export class HydrateService { +export class HydrateService implements OnModuleInit { + private readonly logger: LoggerService = new Logger(HydrateService.name); + constructor( - private readonly collectionCollectorService: CollectionCollectorService, - private readonly objectCollectorService: ObjectCollectorService + @Inject(STIX_REPO_TOKEN) private stixRepo: StixRepositoryInterface, + @InjectModel(TaxiiCollectionEntity.name) private collectionModel: Model, + @InjectModel(AttackObjectEntity.name) private stixObjectModel: Model, + @Inject(HYDRATE_OPTIONS_TOKEN) private options: HydrateConnectOptions ) {} - async hydrate() { - await this.collectionCollectorService.findAndStoreStixCollections(); - await this.objectCollectorService.findAndStoreStixObjects(); + async onModuleInit() { + try { + await this.ensureIndex( + this.stixObjectModel.collection, + { '_meta.createdAt': 1 }, + { background: true, name: 'taxii_object_sorting' } + ); + + await this.ensureIndex( + this.stixObjectModel.collection, + { '_meta.collectionRef.id': 1, '_meta.active': 1 }, + { background: true, name: 'taxii_objects_by_collection' } + ); + + await this.ensureIndex( + this.stixObjectModel.collection, + { '_meta.collectionRef.id': 1, 'stix.id': 1, '_meta.active': 1 }, + { background: true, name: 'taxii_object_lookup' } + ); + + if (this.options.hydrateOnBoot) { + await this.hydrate(); + } + } catch (error) { + this.logger.error('Failed to ensure indexes', error); + throw error; + } + } + + private async ensureIndex( + collection: any, + indexSpec: object, + options: object = {} + ): Promise { + try { + await collection.createIndex(indexSpec, options); + } catch (error) { + if (error.code === 85 || error.code === 86) return; + throw error; + } + } + + private safeDate(date: string | Date | undefined): Date { + if (!date) return new Date(); + const parsed = new Date(date); + return isNaN(parsed.getTime()) ? new Date() : parsed; + } + + private createTaxiiCollectionEntity(workbenchCollection: WorkbenchCollectionDto): TaxiiCollectionEntity { + const taxiiDto = new TaxiiCollectionDto(workbenchCollection); + return new this.collectionModel({ + id: taxiiDto.id, + title: taxiiDto.title, + description: taxiiDto.description, + alias: taxiiDto.alias, + canRead: taxiiDto.canRead, + canWrite: taxiiDto.canWrite, + mediaTypes: taxiiDto.mediaTypes, + _meta: { + workbenchCollection: { + version: workbenchCollection.stix.x_mitre_version, + modified: this.safeDate(workbenchCollection.stix.modified) + }, + createdAt: new Date(), + active: true + } + }); + } + + private createCollectionRef(workbenchCollection: WorkbenchCollectionDto) { + return { + id: workbenchCollection.stix.id, + title: workbenchCollection.stix.name, + version: workbenchCollection.stix.x_mitre_version, + modified: this.safeDate(workbenchCollection.stix.modified) + }; + } + + private createStixObjectEntity( + stixObject: any, + collectionRef: ReturnType + ): AttackObjectEntity { + return { + stix: stixObject, + _meta: { + collectionRef, + stixSpecVersion: '2.1', + createdAt: new Date(), + active: true + } + }; + } + + /** + * Handles collection synchronization with Workbench, always preferring the Workbench state. + * + * Key behaviors: + * 1. If no active TAXII collection exists with the same title -> create new + * 2. If active TAXII collection exists: + * a. If versions match -> no action needed + * b. If versions differ: + * - Check if we've seen this version before + * - If yes -> reactivate that version + * - If no -> create new version + */ + private async handleCollectionSync( + workbenchCollection: WorkbenchCollectionDto + ): Promise<{ shouldCreateObjects: boolean }> { + const workbenchVersion = workbenchCollection.stix.x_mitre_version; + + // Find current active collection + const activeCollection = await this.collectionModel.findOne({ + title: workbenchCollection.stix.name, + '_meta.active': true + }).exec(); + + // Find any existing inactive collection matching the Workbench version + const matchingVersion = await this.collectionModel.findOne({ + title: workbenchCollection.stix.name, + '_meta.workbenchCollection.version': workbenchVersion, + '_meta.active': false + }).exec(); + + // No active collection exists - create new + if (!activeCollection) { + this.logger.debug(`Creating new TAXII collection for ${workbenchCollection.stix.name}`); + const newCollection = this.createTaxiiCollectionEntity(workbenchCollection); + await this.collectionModel.create(newCollection); + return { shouldCreateObjects: true }; + } + + // Active collection exists but versions match - no action needed + if (activeCollection._meta.workbenchCollection.version === workbenchVersion) { + this.logger.debug(`Collection ${workbenchCollection.stix.name} versions match - no action needed`); + return { shouldCreateObjects: false }; + } + + // Deactivate current collection and its objects + await this.collectionModel.findByIdAndUpdate( + activeCollection._id, + { '$set': { '_meta.active': false } } + ); + + await this.stixObjectModel.updateMany( + { + '_meta.collectionRef.id': activeCollection.id, + '_meta.active': true + }, + { '$set': { '_meta.active': false } } + ); + + // If we've seen this version before, reactivate it and its objects + if (matchingVersion) { + this.logger.debug(`Reactivating existing version ${workbenchVersion} for ${workbenchCollection.stix.name}`); + await this.collectionModel.findByIdAndUpdate( + matchingVersion._id, + { '$set': { '_meta.active': true } } + ); + + await this.stixObjectModel.updateMany( + { + '_meta.collectionRef.id': matchingVersion.id, + '_meta.collectionRef.version': workbenchVersion, + '_meta.active': false + }, + { '$set': { '_meta.active': true } } + ); + + return { shouldCreateObjects: false }; + } + + // Create new collection for unseen version + this.logger.debug(`Creating new collection for version ${workbenchVersion} of ${workbenchCollection.stix.name}`); + const newCollection = this.createTaxiiCollectionEntity(workbenchCollection); + await this.collectionModel.create(newCollection); + return { shouldCreateObjects: true }; + } + + private async handleOrphanedCollections( + workbenchCollections: WorkbenchCollectionDto[] + ): Promise { + const workbenchTitles = new Set( + workbenchCollections.map(collection => collection.stix.name) + ); + + const orphanedCollections = await this.collectionModel + .find({ + title: { $nin: Array.from(workbenchTitles) }, + '_meta.active': true + }) + .exec(); + + if (orphanedCollections.length === 0) { + this.logger.debug('No orphaned collections found'); + return; + } + + // Mark collections as inactive + await this.collectionModel.updateMany( + { _id: { $in: orphanedCollections.map(c => c._id) } }, + { '$set': { '_meta.active': false } } + ); + + // Mark their objects as inactive + await this.stixObjectModel.updateMany( + { '_meta.collectionRef.id': { $in: orphanedCollections.map(c => c.id) } }, + { '$set': { '_meta.active': false } } + ); + + this.logger.debug(`Marked ${orphanedCollections.length} orphaned collections as inactive`); + } + + private async syncCollectionObjects( + workbenchCollection: WorkbenchCollectionDto + ): Promise { + const bundle = await this.stixRepo.getCollectionBundle(workbenchCollection.stix.id); + + if (!bundle.objects || bundle.objects.length === 0) { + this.logger.debug(`No objects found in collection ${workbenchCollection.stix.id}`); + return; + } + + const collectionRef = this.createCollectionRef(workbenchCollection); + const bulkOps = bundle.objects.map(stixObject => ({ + insertOne: { + document: this.createStixObjectEntity(stixObject, collectionRef) + } + })); + + // Execute bulk operations in batches of 1000 + for (let i = 0; i < bulkOps.length; i += 1000) { + const batch = bulkOps.slice(i, i + 1000); + await this.stixObjectModel.bulkWrite(batch); + } + + this.logger.debug( + `Synchronized ${bundle.objects.length} objects for collection ${workbenchCollection.stix.id}` + ); + } + + @Cron(CronExpression.EVERY_30_MINUTES, { + name: GET_TAXII_RESOURCES_JOB_TOKEN, + }) + async findAndStoreTaxiiResources(): Promise { + this.logger.debug('Starting database collection and object hydration'); + + try { + const workbenchCollections = await this.stixRepo.getCollections(); + this.logger.debug( + `Successfully retrieved ${workbenchCollections.length} collections from Workbench` + ); + + await this.handleOrphanedCollections(workbenchCollections); + + for (const workbenchCollection of workbenchCollections) { + try { + const { shouldCreateObjects } = await this.handleCollectionSync(workbenchCollection); + + if (shouldCreateObjects) { + await this.syncCollectionObjects(workbenchCollection); + } + + this.logger.debug( + `Processed collection '${workbenchCollection.stix.id}'` + ); + } catch (e) { + this.logger.error( + `Failed to process collection ${workbenchCollection.stix.id}`, + e.stack + ); + } + } + } catch (e) { + this.logger.error("Failed to retrieve collections from Workbench", e.stack); + throw e; + } + } + + async hydrate(): Promise { + this.logger.debug('Manual hydration process triggered'); + await this.findAndStoreTaxiiResources(); } -} +} \ No newline at end of file diff --git a/src/hydrate/interfaces/hydrate-connect.options.ts b/src/hydrate/interfaces/hydrate-connect.options.ts new file mode 100644 index 0000000..2d611a2 --- /dev/null +++ b/src/hydrate/interfaces/hydrate-connect.options.ts @@ -0,0 +1,15 @@ +import { StixConnectOptions } from "src/stix/interfaces"; +import { DatabaseConnectOptions } from "src/interfaces/database-connect-options.interface"; + +/** + * Configuration options for the Hydrate module. + * + * @property hydrateOnBoot - If true, triggers hydration process when the application starts + * @property databaseConnectOptions - MongoDB connection configuration + * @property stixConnectOptions - Configuration for connecting to Workbench + */ +export interface HydrateConnectOptions { + hydrateOnBoot: boolean; + databaseConnectOptions: DatabaseConnectOptions; + stixConnectOptions: StixConnectOptions; +} \ No newline at end of file diff --git a/src/hydrate/interfaces/semver-parts.interface.ts b/src/hydrate/interfaces/semver-parts.interface.ts new file mode 100644 index 0000000..e79cc3f --- /dev/null +++ b/src/hydrate/interfaces/semver-parts.interface.ts @@ -0,0 +1,12 @@ +/** + * Interface representing the individual parts of a semantic version number. + * @interface SemverParts + * @property {number} major - The major version number + * @property {number} minor - The minor version number + * @property {number} patch - The patch version number + */ +export interface SemverParts { + major: number; + minor: number; + patch: number; +} \ No newline at end of file diff --git a/src/hydrate/schema/attack-object.schema.ts b/src/hydrate/schema/attack-object.schema.ts new file mode 100644 index 0000000..4c74c67 --- /dev/null +++ b/src/hydrate/schema/attack-object.schema.ts @@ -0,0 +1,98 @@ +import { Prop, Schema, SchemaFactory } from "@nestjs/mongoose"; +import { Document } from "mongoose"; +import * as mongoose from "mongoose"; +import { StixProperties, StixPropertiesSchema } from "./stix-properties.schema"; + +@Schema() +export class CollectionRefEntity { + @Prop({ type: mongoose.Schema.Types.String, required: true }) + id: string; + + @Prop({ type: mongoose.Schema.Types.String, required: true }) + title: string; + + @Prop({ type: mongoose.Schema.Types.String, required: true }) + version: string; + + @Prop({ type: mongoose.Schema.Types.Date, required: true }) + modified: Date; +} + +export const CollectionRefSchema = SchemaFactory.createForClass(CollectionRefEntity); + +@Schema() +export class ObjectMetaDataEntity { + @Prop({ type: CollectionRefSchema, required: true }) + collectionRef: CollectionRefEntity; + + @Prop({ type: mongoose.Schema.Types.String, required: true }) + stixSpecVersion: string; + + @Prop({ type: mongoose.Schema.Types.Date, required: true }) + createdAt: Date; + + @Prop({ type: mongoose.Schema.Types.Boolean, required: true, default: true }) + active: boolean; +} + +export const ObjectMetaDataSchema = SchemaFactory.createForClass(ObjectMetaDataEntity); + +@Schema({ + collection: "object-resources", + versionKey: false, + /** + * NOTE Until we integrate the attack-data-model to handle data parsing/validation, + * we need to disable strict mode to ensure that values passed to the model + * constructor that are not specified in the schema can be saved to the db. Strict + * mode is just too prone to inadvertently dropping valid STIX properties. + */ + strict: false +}) +export class AttackObjectEntity { + @Prop({ type: StixPropertiesSchema }) + stix: StixProperties; + + @Prop({ type: ObjectMetaDataSchema, required: true }) + _meta: ObjectMetaDataEntity; +} + +export type AttackObjectDocument = AttackObjectEntity & Document; + +export const AttackObjectSchema = SchemaFactory.createForClass(AttackObjectEntity); + +// Index for TAXII-compliant sorting +AttackObjectSchema.index({ '_meta.createdAt': 1 }, { background: true }); + +// Required for TAXII-compliant object sorting +AttackObjectSchema.index( + { '_meta.createdAt': 1 }, + { + background: true, + name: 'taxii_object_sorting' + } +); + +// Required for "Get Objects" endpoint +AttackObjectSchema.index( + { + '_meta.collectionRef.id': 1, + '_meta.active': 1 + }, + { + background: true, + name: 'taxii_objects_by_collection' + } +); + +// Required for "Get An Object" endpoint +AttackObjectSchema.index( + { + '_meta.collectionRef.id': 1, + 'stix.id': 1, + '_meta.active': 1 + }, + { + background: true, + name: 'taxii_object_lookup' + } +); \ No newline at end of file diff --git a/src/hydrate/collector/schema/index.ts b/src/hydrate/schema/index.ts similarity index 100% rename from src/hydrate/collector/schema/index.ts rename to src/hydrate/schema/index.ts diff --git a/src/hydrate/collector/schema/stix-properties.schema.ts b/src/hydrate/schema/stix-properties.schema.ts similarity index 100% rename from src/hydrate/collector/schema/stix-properties.schema.ts rename to src/hydrate/schema/stix-properties.schema.ts diff --git a/src/hydrate/schema/taxii-collection.schema.ts b/src/hydrate/schema/taxii-collection.schema.ts new file mode 100644 index 0000000..ac74391 --- /dev/null +++ b/src/hydrate/schema/taxii-collection.schema.ts @@ -0,0 +1,83 @@ +import { Prop, Schema, SchemaFactory } from "@nestjs/mongoose"; +import { Document } from "mongoose"; +import * as mongoose from "mongoose"; + +@Schema() +export class WorkbenchCollectionEntity { + @Prop({ type: mongoose.Schema.Types.String, required: true }) + version: string; + + @Prop({ type: mongoose.Schema.Types.Date, required: true }) + modified: Date; +} + +export const WorkbenchCollectionSchema = SchemaFactory.createForClass(WorkbenchCollectionEntity); + +@Schema() +export class CollectionMetaDataEntity { + @Prop({ type: WorkbenchCollectionSchema, required: true }) + workbenchCollection: WorkbenchCollectionEntity; + + @Prop({ type: mongoose.Schema.Types.Date, required: true }) + createdAt: Date; + + @Prop({ type: mongoose.Schema.Types.Boolean, required: true, default: true }) + active: boolean; +} + +export const CollectionMetaDataSchema = SchemaFactory.createForClass(CollectionMetaDataEntity); + +@Schema({ + collection: "collection-resources", +}) +export class TaxiiCollectionEntity { + @Prop({ + required: true, + unique: false, // Changed to false since same ID can exist across versions + type: mongoose.Schema.Types.String, + }) + id: string; + + @Prop({ type: mongoose.Schema.Types.String, required: true }) + title: string; + + @Prop({ type: mongoose.Schema.Types.String, required: false }) + description: string; + + @Prop({ type: mongoose.Schema.Types.String, required: false }) + alias: string; + + @Prop({ type: mongoose.Schema.Types.Boolean, required: true }) + canRead: boolean; + + @Prop({ type: mongoose.Schema.Types.Boolean, required: true }) + canWrite: boolean; + + @Prop({ type: [String], required: false }) + mediaTypes: string[]; + + @Prop({ type: CollectionMetaDataSchema, required: true }) + _meta: CollectionMetaDataEntity; +} + +export type TaxiiCollectionDocument = TaxiiCollectionEntity & Document; + +export const TaxiiCollectionSchema = SchemaFactory.createForClass(TaxiiCollectionEntity); + +// Required for "Get A Collection" endpoint +TaxiiCollectionSchema.index( + { id: 1, '_meta.active': 1 }, + { + background: true, + name: 'taxii_collection_lookup' + } +); + +// Required for collision detection in hydration +TaxiiCollectionSchema.index( + { title: 1, '_meta.active': 1 }, + { + background: true, + name: 'collection_title_lookup' + } +); \ No newline at end of file diff --git a/src/main.ts b/src/main.ts index 893955f..126c384 100644 --- a/src/main.ts +++ b/src/main.ts @@ -52,6 +52,10 @@ export async function bootstrap() { path: "/taxii2/", method: RequestMethod.GET, }, + { + path: "/health/ping", + method: RequestMethod.GET + } ], }); diff --git a/src/stix/dto/workbench-collection.dto.ts b/src/stix/dto/workbench-collection.dto.ts index 0b649e9..a63989d 100644 --- a/src/stix/dto/workbench-collection.dto.ts +++ b/src/stix/dto/workbench-collection.dto.ts @@ -1,21 +1,28 @@ import { Expose, Type } from "class-transformer"; -import { StixObjectInterface } from "src/stix/interfaces/stix-object.interface"; -export class WorkbenchCollectionDto implements StixObjectInterface { - // The commented-out properties will also be returned from Workbench (GET /api/collection) requests, but will be - // dropped because they serve no practical purpose to the TAXII server. They are dropped by virtue of the - // excludeExtraneousValues option set to true in the plainToInstance method calls within WorkbenchRepository. +export interface WorkbenchCollectionStixProperties { + modified: string; + name: string; + description?: string; + x_mitre_contents?: any[]; + x_mitre_version: string; + x_mitre_attack_spec_version: string; + type: string; + spec_version: string; + id: string; + created: string; + created_by_ref: string; + object_marking_refs?: string[]; + x_mitre_domains?: string[]; + external_references?: any[]; +} - // _id: uuid <--- WILL BE DROPPED - // workspace: { ... } <--- WILL BE DROPPED +export class WorkbenchCollectionDto { @Expose() @Type(() => Object) - stix: { [key: string]: any }; - // __t: "Collection"; <--- WILL BE DROPPED - // __v: number; <--- WILL BE DROPPED - // created_by_identity: { ... } <--- WILL BE DROPPED + readonly stix: WorkbenchCollectionStixProperties; - constructor(partial: Partial) { - Object.assign(this, partial); + constructor(stix: WorkbenchCollectionStixProperties) { + this.stix = stix; } -} +} \ No newline at end of file diff --git a/src/stix/providers/workbench/workbench.repository.ts b/src/stix/providers/workbench/workbench.repository.ts index fc6a7d0..5b7d633 100644 --- a/src/stix/providers/workbench/workbench.repository.ts +++ b/src/stix/providers/workbench/workbench.repository.ts @@ -10,8 +10,8 @@ import { TaxiiNotFoundException, TaxiiServiceUnavailableException, } from "src/common/exceptions"; -import { WorkbenchCollectionDto } from "src/stix/dto/workbench-collection.dto"; -import { plainToClass, plainToInstance } from "class-transformer"; +import { WorkbenchCollectionDto, WorkbenchCollectionStixProperties } from "src/stix/dto/workbench-collection.dto"; +import { plainToInstance } from "class-transformer"; import { WorkbenchCollectionBundleDto } from "src/stix/dto/workbench-collection-bundle.dto"; import { AttackObjectDto } from "src/stix/dto/attack-object.dto"; import { StixIdentityPrefix, WorkbenchRESTEndpoint } from "src/stix/constants"; @@ -19,6 +19,15 @@ import { WorkbenchConnectOptionsInterface } from "src/stix/interfaces/workbench- import { WORKBENCH_OPTIONS } from "src/stix/constants"; import { StixBundleDto } from "src/stix/dto/stix-bundle.dto"; +interface WorkbenchCollectionResponseDto { + _id: string; + workspace: any; + stix: WorkbenchCollectionStixProperties; + __t: string; + __v: number; + created_by_identity: any; +} + @Injectable() export class WorkbenchRepository { private readonly baseUrl: string; @@ -115,11 +124,11 @@ export class WorkbenchRepository { ***************************************/ /** - * Retrieves a STIX bundle containing all STIX objects for a specified ATT&CK domain. - * - * @param domain The ATT&CK domain to retrieve ("enterprise-attack", "mobile-attack", or "ics-attack"). - * @returns The STIX bundle for the specified domain. - */ + * Retrieves a STIX bundle containing all STIX objects for a specified ATT&CK domain. + * + * @param domain The ATT&CK domain to retrieve ("enterprise-attack", "mobile-attack", or "ics-attack"). + * @returns The STIX bundle for the specified domain. + */ async getStixBundle(domain: string, version: '2.0' | '2.1'): Promise { // Validate the domain parameter to ensure it matches one of the supported domains @@ -204,9 +213,10 @@ export class WorkbenchRepository { } // Fetch the data from Workbench - const response: AttackObjectDto[] = await this.fetchHttp(url); + const response: WorkbenchCollectionResponseDto[] = await this.fetchHttp(url); - return response.map((collection) => ({ stix: collection.stix }) as WorkbenchCollectionDto); + // Extract only the STIX data we need + return response.map(({ stix }) => new WorkbenchCollectionDto(stix)); } /** @@ -219,18 +229,17 @@ export class WorkbenchRepository { const url = `${this.baseUrl}/api/collection-bundles?collectionId=${collectionId}`; // Fetch the data from Workbench - const response: WorkbenchCollectionBundleDto = await this.fetchHttp(url); - - this.logger.debug(`Retrieved STIX data! Data will be deserialized.`); - + return await this.fetchHttp(url); + + // TODO the serialization code is not working. it strips all the STIX properties and leaves an empty object. Let's revisit this after the ADM is integrated. + // this.logger.debug(`Retrieved STIX data! Data will be deserialized.`); // Deserialize the response body - const collectionBundle: WorkbenchCollectionBundleDto = plainToInstance( - WorkbenchCollectionBundleDto, - response, - { excludeExtraneousValues: true } - ); - - return collectionBundle; + // const collectionBundle: WorkbenchCollectionBundleDto = plainToInstance( + // WorkbenchCollectionBundleDto, + // response, + // { excludeExtraneousValues: true } + // ); + // return collectionBundle; } diff --git a/src/taxii/controllers/collections/collections.controller.spec.ts b/src/taxii/controllers/collections/collections.controller.spec.ts index 25df6c9..f41e143 100644 --- a/src/taxii/controllers/collections/collections.controller.spec.ts +++ b/src/taxii/controllers/collections/collections.controller.spec.ts @@ -16,9 +16,9 @@ import { MongooseModule } from "@nestjs/mongoose"; import { AttackObjectEntity, AttackObjectSchema, - TaxiiCollection, + TaxiiCollectionEntity, TaxiiCollectionSchema, -} from "src/hydrate/collector/schema"; +} from "src/hydrate/schema"; describe("CollectionsController", () => { let controller: CollectionsController; @@ -35,7 +35,7 @@ describe("CollectionsController", () => { rootMongooseTestModule(), MongooseModule.forFeature([ { name: AttackObjectEntity.name, schema: AttackObjectSchema }, - { name: TaxiiCollection.name, schema: TaxiiCollectionSchema }, + { name: TaxiiCollectionEntity.name, schema: TaxiiCollectionSchema }, ]), ], controllers: [CollectionsController], diff --git a/src/taxii/controllers/root/root.controller.ts b/src/taxii/controllers/root/root.controller.ts index 87a146f..22bb510 100644 --- a/src/taxii/controllers/root/root.controller.ts +++ b/src/taxii/controllers/root/root.controller.ts @@ -1,4 +1,4 @@ -import { Controller, Get, Param } from "@nestjs/common"; +import { Controller, Get, HttpCode, HttpStatus, Param } from "@nestjs/common"; import { TaxiiServiceUnavailableException } from "src/common/exceptions"; import { TaxiiLoggerService as Logger } from "src/common/logger/taxii-logger.service"; import { DiscoveryService } from "src/taxii/providers"; @@ -17,6 +17,12 @@ export class RootController { logger.setContext(RootController.name); } + @Get("/health/ping") + @HttpCode(HttpStatus.NO_CONTENT) + healthPing(): void { + this.logger.debug('Health ping.'); + } + @ApiOkResponse({ description: SWAGGER.ServerDiscovery.Description, type: DiscoveryResource, diff --git a/src/taxii/providers/collection/collection.module.ts b/src/taxii/providers/collection/collection.module.ts index 61d9d93..6151398 100644 --- a/src/taxii/providers/collection/collection.module.ts +++ b/src/taxii/providers/collection/collection.module.ts @@ -2,9 +2,9 @@ import { Module } from "@nestjs/common"; import { CollectionService } from "./collection.service"; import { MongooseModule } from "@nestjs/mongoose"; import { - TaxiiCollection, + TaxiiCollectionEntity, TaxiiCollectionSchema, -} from "src/hydrate/collector/schema"; +} from "src/hydrate/schema"; import { StixModule } from "src/stix/stix.module"; import { CollectionRepository } from "./collection.repository"; @@ -12,7 +12,7 @@ import { CollectionRepository } from "./collection.repository"; imports: [ StixModule, MongooseModule.forFeature([ - { name: TaxiiCollection.name, schema: TaxiiCollectionSchema }, + { name: TaxiiCollectionEntity.name, schema: TaxiiCollectionSchema }, ]), ], providers: [CollectionService, CollectionRepository], diff --git a/src/taxii/providers/collection/collection.repository.spec.ts b/src/taxii/providers/collection/collection.repository.spec.ts index dded7d0..d6423f2 100644 --- a/src/taxii/providers/collection/collection.repository.spec.ts +++ b/src/taxii/providers/collection/collection.repository.spec.ts @@ -8,9 +8,9 @@ import { } from "src/../test/test.mongoose.module"; import { MongooseModule } from "@nestjs/mongoose"; import { - TaxiiCollection, + TaxiiCollectionEntity, TaxiiCollectionSchema, -} from "src/hydrate/collector/schema"; +} from "src/hydrate/schema"; import { CollectionService } from "./collection.service"; describe("CollectionRepository", () => { @@ -23,7 +23,7 @@ describe("CollectionRepository", () => { TaxiiConfigModule, rootMongooseTestModule(), MongooseModule.forFeature([ - { name: TaxiiCollection.name, schema: TaxiiCollectionSchema }, + { name: TaxiiCollectionEntity.name, schema: TaxiiCollectionSchema }, ]), ], providers: [CollectionService, CollectionRepository], diff --git a/src/taxii/providers/collection/collection.repository.ts b/src/taxii/providers/collection/collection.repository.ts index 490a055..99cb058 100644 --- a/src/taxii/providers/collection/collection.repository.ts +++ b/src/taxii/providers/collection/collection.repository.ts @@ -3,39 +3,40 @@ import { TaxiiLoggerService as Logger } from "src/common/logger"; import { TaxiiCollectionDto, TaxiiCollectionsDto } from "./dto"; import { InjectModel } from "@nestjs/mongoose"; import { - TaxiiCollection, + TaxiiCollectionEntity, TaxiiCollectionDocument, -} from "src/hydrate/collector/schema"; +} from "src/hydrate/schema"; import { Model } from "mongoose"; import { isNull } from "lodash"; @Injectable() export class CollectionRepository { - /** - * Instantiates an instance of the CollectionRepository service class - * @param logger - * @param collectionModel - */ constructor( private readonly logger: Logger, - @InjectModel(TaxiiCollection.name) + @InjectModel(TaxiiCollectionEntity.name) private collectionModel: Model ) { this.logger.setContext(CollectionRepository.name); } /** - * Get a summary of one STIX collection-_dto - * @param id The unique identifier of a STIX collection-_dto + * Get a specific active TAXII collection by ID. + * + * @param id TAXII/STIX ID of the collection + * @returns Promise resolving to TaxiiCollectionDto + * @throws If collection is not found */ async findOne(id: string): Promise { - const response: TaxiiCollection = await this.collectionModel - .findOne({ id: id }) + // Uses taxii_collection_lookup index + const response: TaxiiCollectionEntity = await this.collectionModel + .findOne({ + id: id, + '_meta.active': true + }) .exec(); return new Promise((resolve, reject) => { if (!isNull(response)) { - // Transform the response object to a TAXII-compliant DTO and return resolve(new TaxiiCollectionDto({ ...response["_doc"] })); } reject(`Collection ID '${id}' not available in database`); @@ -43,19 +44,19 @@ export class CollectionRepository { } /** - * Get a summary of all STIX collections + * Get all active TAXII collections. + * + * @returns Promise resolving to TaxiiCollectionsDto containing all active collections */ async findAll(): Promise { - // Initialize the parent resource - const taxiiCollectionsResource: TaxiiCollectionsDto = - new TaxiiCollectionsDto(); + const taxiiCollectionsResource = new TaxiiCollectionsDto(); - // Retrieve the list of TAXII collection resources from the database - const response: TaxiiCollectionDto[] = await this.collectionModel - .find({}) + // Only return active collections + const response: TaxiiCollectionEntity[] = await this.collectionModel + .find({ '_meta.active': true }) .exec(); - // Transform each collection resource to a TAXII-compliant DTO and push it onto the parent DTO resource + // Transform to TAXII-compliant DTOs for (const element of response) { taxiiCollectionsResource.collections.push( new TaxiiCollectionDto({ @@ -70,4 +71,4 @@ export class CollectionRepository { } return taxiiCollectionsResource; } -} +} \ No newline at end of file diff --git a/src/taxii/providers/collection/collection.service.spec.ts b/src/taxii/providers/collection/collection.service.spec.ts index 2125d63..58df506 100644 --- a/src/taxii/providers/collection/collection.service.spec.ts +++ b/src/taxii/providers/collection/collection.service.spec.ts @@ -9,9 +9,9 @@ import { } from "src/../test/test.mongoose.module"; import { MongooseModule } from "@nestjs/mongoose"; import { - TaxiiCollection, + TaxiiCollectionEntity, TaxiiCollectionSchema, -} from "src/hydrate/collector/schema"; +} from "src/hydrate/schema"; describe("CollectionService", () => { let collectionService: CollectionService; @@ -23,7 +23,7 @@ describe("CollectionService", () => { TaxiiConfigModule, rootMongooseTestModule(), MongooseModule.forFeature([ - { name: TaxiiCollection.name, schema: TaxiiCollectionSchema }, + { name: TaxiiCollectionEntity.name, schema: TaxiiCollectionSchema }, ]), ], providers: [CollectionService, CollectionRepository], diff --git a/src/taxii/providers/collection/collection.service.ts b/src/taxii/providers/collection/collection.service.ts index 7bd09b0..580ea64 100644 --- a/src/taxii/providers/collection/collection.service.ts +++ b/src/taxii/providers/collection/collection.service.ts @@ -1,5 +1,4 @@ import { Injectable } from "@nestjs/common"; -// import { CollectionWorkbenchRepository } from "./collection.workbench.repository"; import { TaxiiCollectionDto, TaxiiCollectionsDto } from "./dto"; import { TaxiiNotFoundException } from "src/common/exceptions"; import { TaxiiLoggerService as Logger } from "src/common/logger"; @@ -9,7 +8,6 @@ import { CollectionRepository } from "./collection.repository"; export class CollectionService { constructor( private readonly logger: Logger, - // private readonly stixCollectionsRepo: CollectionWorkbenchRepository, private readonly stixCollectionsRepo: CollectionRepository ) { this.logger.setContext(CollectionService.name); diff --git a/src/taxii/providers/collection/dto/taxii-collection-dto/taxii-collection.dto.ts b/src/taxii/providers/collection/dto/taxii-collection-dto/taxii-collection.dto.ts index e49c723..392ccb7 100644 --- a/src/taxii/providers/collection/dto/taxii-collection-dto/taxii-collection.dto.ts +++ b/src/taxii/providers/collection/dto/taxii-collection-dto/taxii-collection.dto.ts @@ -8,17 +8,34 @@ import { } from "class-validator"; import { Exclude, Expose, Type } from "class-transformer"; import { ALL_MEDIA_TYPES } from "src/common/middleware/content-negotiation"; +import { WorkbenchCollectionDto } from "src/stix/dto/workbench-collection.dto"; @Exclude() export class TaxiiCollectionDto { - constructor(partial: Partial) { + + constructor(partial: Partial | WorkbenchCollectionDto) { + + // If we're passed a WorkbenchCollectionDto, extract the STIX data + if (partial instanceof WorkbenchCollectionDto) { + partial = partial.stix; + } + + // Handle case where partial is the STIX object directly + if (partial && 'stix' in partial) { + partial = partial.stix; + } + Object.assign(this, partial); - if (!partial["title"]) { - this.title = this["name"] ? this["name"] : undefined; + + // Handle title/name conversion + if (!this.title) { + this.title = (partial as any)?.name; } + + // Set default values this.canRead = true; this.canWrite = false; - if (!partial["media_types"]) { + if (!this.mediaTypes) { this.mediaTypes = ALL_MEDIA_TYPES; } } diff --git a/src/taxii/providers/envelope/envelope.service.spec.ts b/src/taxii/providers/envelope/envelope.service.spec.ts index a7026e8..01dc7a2 100644 --- a/src/taxii/providers/envelope/envelope.service.spec.ts +++ b/src/taxii/providers/envelope/envelope.service.spec.ts @@ -11,7 +11,7 @@ import { rootMongooseTestModule, } from "src/../test/test.mongoose.module"; import { MongooseModule } from "@nestjs/mongoose"; -import { AttackObjectEntity, AttackObjectSchema } from "src/hydrate/collector/schema"; +import { AttackObjectEntity, AttackObjectSchema } from "src/hydrate/schema"; describe("EnvelopeService", () => { let envelopeService: EnvelopeService; diff --git a/src/taxii/providers/manifest/manifest.service.spec.ts b/src/taxii/providers/manifest/manifest.service.spec.ts index e1a0eee..2881ed3 100644 --- a/src/taxii/providers/manifest/manifest.service.spec.ts +++ b/src/taxii/providers/manifest/manifest.service.spec.ts @@ -10,7 +10,7 @@ import { rootMongooseTestModule, } from "src/../test/test.mongoose.module"; import { MongooseModule } from "@nestjs/mongoose"; -import { AttackObjectEntity, AttackObjectSchema } from "src/hydrate/collector/schema"; +import { AttackObjectEntity, AttackObjectSchema } from "src/hydrate/schema"; describe("ManifestService", () => { let manifestService: ManifestService; diff --git a/src/taxii/providers/object/object.module.ts b/src/taxii/providers/object/object.module.ts index 60ad5de..9654e83 100644 --- a/src/taxii/providers/object/object.module.ts +++ b/src/taxii/providers/object/object.module.ts @@ -6,7 +6,7 @@ import { MongooseModule } from "@nestjs/mongoose"; import { AttackObjectSchema, AttackObjectEntity, -} from "src/hydrate/collector/schema/attack-object.schema"; +} from "src/hydrate/schema/attack-object.schema"; @Module({ imports: [ diff --git a/src/taxii/providers/object/object.repository.spec.ts b/src/taxii/providers/object/object.repository.spec.ts index c4b520d..bdb10a2 100644 --- a/src/taxii/providers/object/object.repository.spec.ts +++ b/src/taxii/providers/object/object.repository.spec.ts @@ -9,7 +9,7 @@ import { rootMongooseTestModule, } from "src/../test/test.mongoose.module"; import { MongooseModule } from "@nestjs/mongoose"; -import { AttackObjectEntity, AttackObjectSchema } from "src/hydrate/collector/schema"; +import { AttackObjectEntity, AttackObjectSchema } from "src/hydrate/schema"; describe("ObjectRepository", () => { let objectRepository: ObjectRepository; diff --git a/src/taxii/providers/object/object.repository.ts b/src/taxii/providers/object/object.repository.ts index afe42c8..69900fd 100644 --- a/src/taxii/providers/object/object.repository.ts +++ b/src/taxii/providers/object/object.repository.ts @@ -1,20 +1,16 @@ +// object.repository.ts import { Injectable } from "@nestjs/common"; import { TaxiiLoggerService as Logger } from "src/common/logger"; import { InjectModel } from "@nestjs/mongoose"; import { AttackObjectEntity, AttackObjectDocument, -} from "src/hydrate/collector/schema/attack-object.schema"; +} from "src/hydrate/schema/attack-object.schema"; import { Model } from "mongoose"; import { TaxiiNotFoundException } from "src/common/exceptions"; @Injectable() export class ObjectRepository { - /** - * Instantiates an instance of the ObjectRepository service class - * @param logger - * @param attackObjectsModel - */ constructor( private readonly logger: Logger, @InjectModel(AttackObjectEntity.name) @@ -24,16 +20,21 @@ export class ObjectRepository { } /** - * Get an iterable stream of STIX objects from the database - * @param collectionId Specifies the collection of objects that should be returned + * Get an iterable stream of active STIX objects from a specific collection. + * Objects are returned in ascending order by creation date per TAXII spec. + * + * @param collectionId TAXII/STIX ID of the collection + * @returns AsyncIterableIterator of AttackObjectEntity */ async *findByCollectionId( collectionId: string ): AsyncIterableIterator { const cursor = this.attackObjectsModel .find({ - collection_id: collectionId, + '_meta.collectionRef.id': collectionId, + '_meta.active': true }) + .sort({ '_meta.createdAt': 1 }) // Uses taxii_object_sorting index .cursor(); for ( @@ -50,35 +51,34 @@ export class ObjectRepository { } /** - * Get the latest version of a STIX object - * @param collectionId Collection identifier of the requested STIX object - * @param objectId Identifier of the requested object - // * @param versions + * Get the latest version of a STIX object from a specific collection. + * + * @param collectionId TAXII/STIX ID of the collection + * @param objectId STIX ID of the requested object + * @returns Promise resolving to array of matching objects (for version history support) + * @throws TaxiiNotFoundException if no matching objects are found */ async findOne( collectionId: string, objectId: string - // versions = false ): Promise { - // Begin by retrieving all documents that match the specified object parameters - + // Uses taxii_object_lookup index const attackObjects: AttackObjectEntity[] = await this.attackObjectsModel .find({ - collection_id: collectionId, - "stix.id": { $eq: objectId }, + '_meta.collectionRef.id': collectionId, + 'stix.id': objectId, + '_meta.active': true }) + .sort({ '_meta.createdAt': 1 }) .exec(); - // Raise an exception if an empty array was received. We need at least one object to process anything. Something is - // probably broken if an empty array is getting passed around. - if (attackObjects.length === 0) { throw new TaxiiNotFoundException({ title: "No Objects Found", - description: "No objects matching the specified filters were found.", + description: `No objects found with ID '${objectId}' in collection '${collectionId}'`, }); } return attackObjects; } -} +} \ No newline at end of file diff --git a/src/taxii/providers/object/object.service.spec.ts b/src/taxii/providers/object/object.service.spec.ts index 5b59edc..372d670 100644 --- a/src/taxii/providers/object/object.service.spec.ts +++ b/src/taxii/providers/object/object.service.spec.ts @@ -9,7 +9,7 @@ import { rootMongooseTestModule, } from "src/../test/test.mongoose.module"; import { MongooseModule } from "@nestjs/mongoose"; -import { AttackObjectEntity, AttackObjectSchema } from "src/hydrate/collector/schema"; +import { AttackObjectEntity, AttackObjectSchema } from "src/hydrate/schema"; import { FilterModule } from "../filter/filter.module"; describe("ObjectService", () => { diff --git a/src/taxii/providers/object/object.service.ts b/src/taxii/providers/object/object.service.ts index b3c0970..d892645 100644 --- a/src/taxii/providers/object/object.service.ts +++ b/src/taxii/providers/object/object.service.ts @@ -6,10 +6,8 @@ import { } from "src/common/exceptions"; import { TaxiiLoggerService as Logger } from "src/common/logger"; import { FilterService } from "../filter"; -import { StixObjectPropertiesInterface } from "src/stix/interfaces/stix-object-properties.interface"; import { ObjectRepository } from "./object.repository"; -import { StixObjectDto } from "src/stix/dto/stix-object.dto"; -import { AttackObjectEntity as MongooseAttackObject } from "src/hydrate/collector/schema"; +import { AttackObjectEntity as MongooseAttackObject } from "src/hydrate/schema"; @Injectable() export class ObjectService { diff --git a/src/taxii/providers/version/version.service.spec.ts b/src/taxii/providers/version/version.service.spec.ts index e14806e..c7d88ac 100644 --- a/src/taxii/providers/version/version.service.spec.ts +++ b/src/taxii/providers/version/version.service.spec.ts @@ -9,7 +9,7 @@ import { rootMongooseTestModule, closeInMongodConnection, } from "src/../test/test.mongoose.module"; -import { AttackObjectEntity, AttackObjectSchema } from "src/hydrate/collector/schema"; +import { AttackObjectEntity, AttackObjectSchema } from "src/hydrate/schema"; describe("VersionService", () => { let versionService: VersionService;