diff --git a/src/courses/mappers/10.md b/src/courses/mappers/10.md index a49a2354c..6a75252b0 100644 --- a/src/courses/mappers/10.md +++ b/src/courses/mappers/10.md @@ -9,11 +9,31 @@ author: Charles Hu In this section, we will cover: -## Environment Set Up +- How to set up an environment for OHDF mapper development via: + - [GitHub Codespaces](#github-codespaces-environment-set-up) + - [Local installation](#local-environment-set-up) -Node.js (a runtime environment for JavaScript) and Yarn (a package manager for JavaScript/TypeScript) are external utilities which are utilized extensively within this guide. The following section details their installation process. +## Overview -Linux/Mac OS: +The rest of this course will involve a mix of hands-on guided and unguided labs to teach you the basics of the technical implementation of OHDF mappers. To facilitate this, we provide two installation guides for setting up the environment necessary for these upcoming labs: A GitHub Codespaces environment set up (recommended for beginners) and a local environment set up (recommended for experienced developers or individuals interested in manually installing to their local system). + +## GitHub Codespaces Environment Set Up + +We provide a [GitHub Codespaces environment](https://github.com/mitre/saf-training-lab-environment) that includes a simple build script that installs the necessary repositories and packages to begin OHDF mapper development. + +Follow the instructions listed in the README and you should have a verified working environment with the Heimdall and SAF CLI repositories pulled down into the `/dev_repos/` directory. Heimdall can be found under `/dev_repos/heimdall2/`, the SAF CLI can be found under `/dev_repos/saf/`, and OHDF Converters can be found under `/dev_repos/heimdall2/libs/hdf-converters/`. + +## Local Environment Set Up + +To set up the environment locally, we will need to pull down the GitHub repositories for Heimdall and the SAF CLI using Git and install their necessary dependencies using a package manager. + +You can install Git [here](https://git-scm.com/downloads). Choose your OS and follow the installation guide as necessary. + +Node.js (a runtime environment for JavaScript), NPM (a package manager for JavaScript/TypeScript), and Yarn (another package manager for JavaScript/TypeScript) are external utilities which are utilized extensively within this guide. The following section details their installation process. + +### Runtime & Package Managers + +#### Linux/Mac OS 1. Install [nvm](https://github.com/nvm-sh/nvm#install--update-script). @@ -41,22 +61,58 @@ nvm install 18 3. Install [Yarn](https://yarnpkg.com/getting-started/install) -Windows: +#### Windows 1. Install [Node.js v18 via the installer](https://nodejs.org/en/download/). 2. Install [Yarn](https://yarnpkg.com/getting-started/install): -## Repository Set Up +### Repository Set Up + +Now that we have the runtime and package managers for Javascript installed, we can pull down the necessary GitHub repositories and install their dependencies. 1. Fork/branch a development repository from the main [Heimdall2 GitHub repository](https://github.com/mitre/heimdall2). - SAF team developers have write access to the main repository and should [create a branch](https://docs.github.com/en/desktop/contributing-and-collaborating-using-github-desktop/making-changes-in-a-branch/managing-branches#creating-a-branch) on the primary development repository. Non-SAF team developers should instead [create a fork](https://docs.github.com/en/get-started/quickstart/fork-a-repo#forking-a-repository) of the main repository and create a development branch there. -2. Create a draft [pull request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request#creating-the-pull-request) for your development branch against the main repository branch. +2. Pull down your development branch from GitHub using the following command: + +```shell +git clone {URL-TO-REPO-HERE} +``` + +3. Create a draft [pull request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request#creating-the-pull-request) for your development branch against the main repository branch. + +4. Install the necessary dependencies for Heimdall2. Under the `heimdall2` directory, enter the following command in the terminal: + +```shell +yarn install --frozen-lockfile +``` + +5. Fork/branch a development repository from the main [SAF CLI GitHub repository](https://github.com/mitre/saf). + + - SAF team developers have write access to the main repository and should [create a branch](https://docs.github.com/en/desktop/contributing-and-collaborating-using-github-desktop/making-changes-in-a-branch/managing-branches#creating-a-branch) on the primary development repository. Non-SAF team developers should instead [create a fork](https://docs.github.com/en/get-started/quickstart/fork-a-repo#forking-a-repository) of the main repository and create a development branch there. + +6. Pull down your development branch from GitHub using the following command: -3. Install the necessary dependencies for Heimdall2. Under the `heimdall2` directory, enter the following command in the terminal: +```shell +git clone {URL-TO-REPO-HERE} +``` + +7. Create a draft [pull request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request#creating-the-pull-request) for your development branch against the main repository branch. + +8. Install the necessary dependencies for the SAF CLI. Under the `saf` directory, enter the following command in the terminal: ```shell -yarn install +npm install ``` + +You should now have a working environment with the Heimdall and SAF CLI repositories. Heimdall can be found under `/heimdall2/`, the SAF CLI can be found under `/saf/`, and OHDF Converters can be found under `/heimdall2/libs/hdf-converters/`. + +## A Look Back + +In this section, we covered: + +- How to set up an environment for OHDF mapper development via: + - [GitHub Codespaces](#github-codespaces-environment-set-up) + - [Local installation](#local-environment-set-up) diff --git a/src/courses/mappers/12.md b/src/courses/mappers/12.md index d8ef8cc5d..4a4a8af65 100644 --- a/src/courses/mappers/12.md +++ b/src/courses/mappers/12.md @@ -1755,812 +1755,3 @@ export class DBProtectMapper extends BaseConverter { ::: Now we have a fully implemented DbProtect-to-OHDF mapper. - -### Mapper Demo - JFrog - -This section is yet another example of implementing an OHDF mapper, namely the JFrog-Xray mapper. Again, we will assume that the [appropriate file set up for the mapper](#file-set-up) has been performed. Because we have already gone through several examples, this section will not explain too many things in detail. The goal is to simply provide more examples for reference. - -Here is our developed mapping for JFrog-Xray for reference: - -::: details JFrog-to-OHDF Mapping - -```typescript -{ - platform: { - name: 'Heimdall Tools', - release: HeimdallToolsVersion, - target_id - }, - version: HeimdallToolsVersion, - statistics: { - duration - }, - profiles: [ - { - name, - version, - sha256, - title, - maintainer, - summary, - license, - copyright, - copyright_email, - supports, - attributes, - groups, - controls: [ - { - id: 'data.id', // If ID empty, hash the summary - title: 'data.summary', - desc: 'component_versions.more_details', - descriptions, - impact: 'data.severity', - refs, - tags: { - nist: 'data.component_versions.more_details.cves[0].cwe', // Map to NIST - cci: 'data.component_versions.more_details.cves[0].cwe', // Map to CCI - cweid: 'data.component_versions.more_details.cves[0].cwe' - }, - code, - source_location, - results: [ - { - status: 'Failed', // All reported findings are failures - code_desc: ['data.source_comp_id', 'data.component_versions.vulnerable_versions', 'data.component_versions.fixed_versions', 'data.issue_type', 'data.provider'], - message, - run_time, - start_time - } - ] - }, - ], - status: 'loaded' - }, - ], - passthrough: { - auxiliary_data: [ - { - name, - data - }, - ], - raw - } -} -``` - -::: - -::: details JFrog Source Data - -```json -{ - "total_count": 30, - "data": [ - { - "id": "", - "severity": "High", - "summary": "Acorn regexp.js Regular Expression Validation UTF-16 Surrogate Handling Infinite Loop DoS", - "issue_type": "security", - "provider": "JFrog", - "component": "acorn", - "source_id": "npm://acorn", - "source_comp_id": "npm://acorn:5.7.3", - "component_versions": { - "id": "acorn", - "vulnerable_versions": [ - "5.5.0 ≤ Version < 5.7.4", - "6.0.0 ≤ Version < 6.4.1", - "7.0.0", - "7.1.0" - ], - "fixed_versions": ["5.7.4", "6.4.1", "7.1.1"], - "more_details": { - "cves": [ - { - "cvss_v2": "7.1/AV:N/AC:M/Au:N/C:N/I:N/A:C", - "cvss_v3": "7.5/CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H" - } - ], - "description": "Acorn contains an infinite loop condition in regexp.js that is triggered when handling UTF_16 surrogates while validating regular expressions. This may allow a context-dependent attacker to hang a process using the library.", - "provider": "JFrog" - } - }, - "edited": "2020-11-03T19:30:42-05:00" - } - ] -} -``` - -::: - -We will now go over the mapper code that allows us to shape this data properly. Firstly, let us handle the ID. Sometimes, JFrog source files will not have an ID field. In this case, it is appropriate to generate a hash based on the `summary` as follows: - -```typescript -function hashId(vulnerability: unknown): string { - if (_.get(vulnerability, "id") === "") { - return generateHash( - (_.get(vulnerability, "summary") as unknown as string).toString(), - "md5" - ); - } else { - return _.get(vulnerability, "id") as unknown as string; - } -} -``` - -As with most mappers, we must also define an impact mapping: - -```typescript -const IMPACT_MAPPING: Map = new Map([ - ["high", 0.7], - ["medium", 0.5], - ["low", 0.3], -]); -``` - -Recall that we wanted to format the `desc` field based on the data in the source file's `component_versions.more_details`. We can write a simple transformer function to concatenate all these keys: - -```typescript -function formatDesc(vulnerability: unknown): string { - const text = []; - if (_.has(vulnerability, "description")) { - text.push( - (_.get(vulnerability, "description") as unknown as string).toString() - ); - } - if (_.has(vulnerability, "cves")) { - const re1 = /":/gi; - const re2 = /,/gi; - text.push( - `cves: ${JSON.stringify(_.get(vulnerability, "cves")) - .replace(re1, '"=>') - .replace(re2, ", ")}` - ); - } - return text.join("
"); -} -``` - -We also must properly handle the mappings to NIST and CCI tags. Luckily, we can make use of the library's existing mappings, as well as an existing global function called `getCCIsForNISTTags()` in `global.ts`. - -```typescript -const CWE_PATH = "component_versions.more_details.cves[0].cwe"; -const CWE_NIST_MAPPING = new CweNistMapping(); - -function nistTag(identifier: Record): string[] { - const identifiers: string[] = []; - if (Array.isArray(identifier)) { - identifier.forEach((element) => { - if (element.split("CWE-")[1]) { - identifiers.push(element.split("CWE-")[1]); - } - }); - } - return CWE_NIST_MAPPING.nistFilter( - identifiers, - DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS - ); -} -``` - -Finally, we can implement the transformer function for the `code_desc`. Some of these fields are optional, so it is important to implement a base case to ensure our `code_desc` outputs are all consistent. - -```typescript -function formatCodeDesc(vulnerability: unknown): string { - const codeDescArray: string[] = []; - const re = /,/gi; - if (_.has(vulnerability, "source_comp_id")) { - codeDescArray.push( - `source_comp_id : ${_.get(vulnerability, "source_comp_id")}` - ); - } else { - codeDescArray.push("source_comp_id : "); - } - if (_.has(vulnerability, "component_versions.vulnerable_versions")) { - codeDescArray.push( - `vulnerable_versions : ${JSON.stringify( - _.get(vulnerability, "component_versions.vulnerable_versions") - )}` - ); - } else { - codeDescArray.push("vulnerable_versions : "); - } - if (_.has(vulnerability, "component_versions.fixed_versions")) { - codeDescArray.push( - `fixed_versions : ${JSON.stringify( - _.get(vulnerability, "component_versions.fixed_versions") - )}` - ); - } else { - codeDescArray.push("fixed_versions : "); - } - if (_.has(vulnerability, "issue_type")) { - codeDescArray.push(`issue_type : ${_.get(vulnerability, "issue_type")}`); - } else { - codeDescArray.push("issue_type : "); - } - if (_.has(vulnerability, "provider")) { - codeDescArray.push(`provider : ${_.get(vulnerability, "provider")}`); - } else { - codeDescArray.push("provider : "); - } - return codeDescArray.join("\n").replace(re, ", "); -} -``` - -All that's left is to incoporate all these functions into our `base-converter` structure, write some smaller transformer functions for the simpler data formatting tasks, and combine it all together! - -:::details Full Mapper Code - -```typescript -import { ExecJSON } from "inspecjs"; -import * as _ from "lodash"; -import { version as HeimdallToolsVersion } from "../package.json"; -import { - BaseConverter, - generateHash, - ILookupPath, - impactMapping, - MappedTransform, -} from "./base-converter"; -import { CweNistMapping } from "./mappings/CweNistMapping"; -import { - DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS, - getCCIsForNISTTags, -} from "./utils/global"; - -// Constants -const IMPACT_MAPPING: Map = new Map([ - ["high", 0.7], - ["medium", 0.5], - ["low", 0.3], -]); - -const CWE_PATH = "component_versions.more_details.cves[0].cwe"; - -const CWE_NIST_MAPPING = new CweNistMapping(); - -// Transformation Functions -function hashId(vulnerability: unknown): string { - if (_.get(vulnerability, "id") === "") { - return generateHash( - (_.get(vulnerability, "summary") as unknown as string).toString(), - "md5" - ); - } else { - return _.get(vulnerability, "id") as unknown as string; - } -} -function formatDesc(vulnerability: unknown): string { - const text = []; - if (_.has(vulnerability, "description")) { - text.push( - (_.get(vulnerability, "description") as unknown as string).toString() - ); - } - if (_.has(vulnerability, "cves")) { - const re1 = /":/gi; - const re2 = /,/gi; - text.push( - `cves: ${JSON.stringify(_.get(vulnerability, "cves")) - .replace(re1, '"=>') - .replace(re2, ", ")}` - ); - } - return text.join("
"); -} -function formatCodeDesc(vulnerability: unknown): string { - const codeDescArray: string[] = []; - const re = /,/gi; - if (_.has(vulnerability, "source_comp_id")) { - codeDescArray.push( - `source_comp_id : ${_.get(vulnerability, "source_comp_id")}` - ); - } else { - codeDescArray.push("source_comp_id : "); - } - if (_.has(vulnerability, "component_versions.vulnerable_versions")) { - codeDescArray.push( - `vulnerable_versions : ${JSON.stringify( - _.get(vulnerability, "component_versions.vulnerable_versions") - )}` - ); - } else { - codeDescArray.push("vulnerable_versions : "); - } - if (_.has(vulnerability, "component_versions.fixed_versions")) { - codeDescArray.push( - `fixed_versions : ${JSON.stringify( - _.get(vulnerability, "component_versions.fixed_versions") - )}` - ); - } else { - codeDescArray.push("fixed_versions : "); - } - if (_.has(vulnerability, "issue_type")) { - codeDescArray.push(`issue_type : ${_.get(vulnerability, "issue_type")}`); - } else { - codeDescArray.push("issue_type : "); - } - if (_.has(vulnerability, "provider")) { - codeDescArray.push(`provider : ${_.get(vulnerability, "provider")}`); - } else { - codeDescArray.push("provider : "); - } - return codeDescArray.join("\n").replace(re, ", "); -} -function nistTag(identifier: Record): string[] { - const identifiers: string[] = []; - if (Array.isArray(identifier)) { - identifier.forEach((element) => { - if (element.split("CWE-")[1]) { - identifiers.push(element.split("CWE-")[1]); - } - }); - } - return CWE_NIST_MAPPING.nistFilter( - identifiers, - DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS - ); -} - -// Mappings -export class JfrogXrayMapper extends BaseConverter { - withRaw: boolean; - - mappings: MappedTransform< - ExecJSON.Execution & { passthrough: unknown }, - ILookupPath - > = { - platform: { - name: "Heimdall Tools", - release: HeimdallToolsVersion, - }, - version: HeimdallToolsVersion, - statistics: {}, - profiles: [ - { - name: "JFrog Xray Scan", - title: "JFrog Xray Scan", - summary: "Continuous Security and Universal Artifact Analysis", - supports: [], - attributes: [], - groups: [], - status: "loaded", - controls: [ - { - path: "data", - key: "id", - tags: { - cci: { - path: CWE_PATH, - transformer: (identifier: Record) => - getCCIsForNISTTags(nistTag(identifier)), - }, - nist: { - path: CWE_PATH, - transformer: nistTag, - }, - cweid: { path: CWE_PATH }, - }, - refs: [], - source_location: {}, - id: { transformer: hashId }, - title: { path: "summary" }, - desc: { - path: "component_versions.more_details", - transformer: formatDesc, - }, - impact: { - path: "severity", - transformer: impactMapping(IMPACT_MAPPING), - }, - code: { - transformer: (vulnerability: Record): string => { - return JSON.stringify(vulnerability, null, 2); - }, - }, - results: [ - { - status: ExecJSON.ControlResultStatus.Failed, - code_desc: { transformer: formatCodeDesc }, - start_time: "", - }, - ], - }, - ], - sha256: "", - }, - ], - passthrough: { - transformer: (data: Record): Record => { - return { - auxiliary_data: [ - { - name: "JFrog Xray", - data: _.pick(data, ["total_count"]), - }, - ], - ...(this.withRaw && { raw: data }), - }; - }, - }, - }; - constructor(xrayJson: string, withRaw = false) { - super(JSON.parse(xrayJson), true); - this.withRaw = withRaw; - } -} -``` - -::: - -Now we have a fully implemented JFrog-to-OHDF mapper. - -### Mapper Demo - SARIF - -Here is one last example of implementing an OHDF mapper - the SARIF mapper. As in previous examples, we will assume that the [appropriate file set up for the mapper](#file-set-up) has been performed. Like with the JFrog example, we will not be going into too much detail regarding the mapping itself. The goal is to simply provide more examples for reference. - -Here is our developed mapping for SARIF for reference: - -::: details SARIF-to-OHDF Mapping - -```typescript -{ - platform: { - name: 'Heimdall Tools', - release: HeimdallToolsVersion, - target_id - }, - version: HeimdallToolsVersion, - statistics: { - duration - }, - profiles: [ - { - name, - version: 'version', - sha256, - title, - maintainer, - summary, - license, - copyright, - copyright_email, - supports, - attributes, - groups, - controls: [ - { - id: 'results.ruleId', // If ID empty, hash the summary - title: 'results.message.text', - desc: 'results.message.text', - descriptions, - impact: 'results.level', - refs, - tags: { - nist: 'results.message.text', // Map to NIST - cci: 'results.message.text', // Map to CCI - cwe: 'results.message.text' - }, - code, - source_location: ['results.locations[0].physicalLocation.artifactLocation.uri', 'results.locations[0].physicalLocation.region.startLine'], - results: [ - { - status: 'Failed', // All reported findings are failures - code_desc: 'results.locations[0].physicalLocation', - message, - run_time, - start_time - } - ] - }, - ], - status: 'loaded' - }, - ], - passthrough: { - auxiliary_data: [ - { - name, - data - }, - ], - raw - } -} -``` - -::: - -::: details SARIF Source Data - -```json -{ - "$schema": "https://schemastore.azurewebsites.net/schemas/json/sarif-2.1.0-rtm.5.json", - "version": "2.1.0", - "runs": [ - { - "results": [ - { - "ruleId": "FF1014", - "level": "error", - "message": { - "text": "buffer/gets: Does not check for buffer overflows (CWE-120, CWE-20)." - }, - "locations": [ - { - "physicalLocation": { - "artifactLocation": { - "uri": "test/test-patched.c", - "uriBaseId": "SRCROOT" - }, - "region": { - "startLine": 36, - "startColumn": 2, - "endColumn": 10, - "snippet": { - "text": " gets(f);" - } - } - } - } - ], - "fingerprints": { - "contextHash/v1": "6a5bb383fb44030b0d9428b17359e94ba3979bc1ce702be450427f85592c649a" - }, - "rank": 1 - } - ] - } - ] -} -``` - -::: - -First, let us define some common constants that might help us with our code. - -```typescript -const MESSAGE_TEXT = "message.text"; -const CWE_NIST_MAPPING = new CweNistMapping(); -``` - -We will now outline the `IMPACT_MAPPING` for the SARIF mapper: - -```typescript -const IMPACT_MAPPING: Map = new Map([ - ["error", 0.7], - ["warning", 0.5], - ["note", 0.3], -]); - -function impactMapping(severity: unknown): number { - if (typeof severity === "string" || typeof severity === "number") { - return IMPACT_MAPPING.get(severity.toString().toLowerCase()) || 0.1; - } else { - return 0.1; - } -} -``` - -Now, note that we will need to extract the CWEs from the message text, as well as map them to the proper NIST tags. For the CCIs, we can again use the global function `getCCIsForNISTTags()`. - -```typescript -function extractCwe(text: string): string[] { - let output = text.split("(").slice(-1)[0].slice(0, -2).split(", "); - if (output.length === 1) { - output = text.split("(").slice(-1)[0].slice(0, -2).split("!/"); - } - return output; -} -function nistTag(text: string): string[] { - let identifiers = extractCwe(text); - identifiers = identifiers.map((element) => element.split("-")[1]); - return CWE_NIST_MAPPING.nistFilter( - identifiers, - DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS - ); -} -``` - -Our last standalone transformer function will format our code description. This part is up to interpretation, but we decided to just concatenate them all with colon-separated key-value pairs - -```typescript -function formatCodeDesc(input: unknown): string { - const output = []; - output.push(`URL : ${_.get(input, "artifactLocation.uri")}`); - output.push(`LINE : ${_.get(input, "region.startLine")}`); - output.push(`COLUMN : ${_.get(input, "region.startColumn")}`); - return output.join(" "); -} -``` - -Now, all that's left is to combine our functions, write some mini-transformers for the small data formatting, and plug it all into the base converter! - -:::details Full Mapper Code - -```typescript -import { ExecJSON } from "inspecjs"; -import * as _ from "lodash"; -import { version as HeimdallToolsVersion } from "../package.json"; -import { BaseConverter, ILookupPath, MappedTransform } from "./base-converter"; -import { CweNistMapping } from "./mappings/CweNistMapping"; -import { - DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS, - getCCIsForNISTTags, -} from "./utils/global"; - -const IMPACT_MAPPING: Map = new Map([ - ["error", 0.7], - ["warning", 0.5], - ["note", 0.3], -]); -const MESSAGE_TEXT = "message.text"; -const CWE_NIST_MAPPING = new CweNistMapping(); - -function extractCwe(text: string): string[] { - let output = text.split("(").slice(-1)[0].slice(0, -2).split(", "); - if (output.length === 1) { - output = text.split("(").slice(-1)[0].slice(0, -2).split("!/"); - } - return output; -} -function impactMapping(severity: unknown): number { - if (typeof severity === "string" || typeof severity === "number") { - return IMPACT_MAPPING.get(severity.toString().toLowerCase()) || 0.1; - } else { - return 0.1; - } -} -function formatCodeDesc(input: unknown): string { - const output = []; - output.push(`URL : ${_.get(input, "artifactLocation.uri")}`); - output.push(`LINE : ${_.get(input, "region.startLine")}`); - output.push(`COLUMN : ${_.get(input, "region.startColumn")}`); - return output.join(" "); -} -function nistTag(text: string): string[] { - let identifiers = extractCwe(text); - identifiers = identifiers.map((element) => element.split("-")[1]); - return CWE_NIST_MAPPING.nistFilter( - identifiers, - DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS - ); -} - -export class SarifMapper extends BaseConverter { - withRaw: boolean; - - mappings: MappedTransform< - ExecJSON.Execution & { passthrough: unknown }, - ILookupPath - > = { - platform: { - name: "Heimdall Tools", - release: HeimdallToolsVersion, - target_id: "Static Analysis Results Interchange Format", - }, - version: HeimdallToolsVersion, - statistics: {}, - profiles: [ - { - path: "runs", - name: "SARIF", - version: { path: "$.version" }, - title: "Static Analysis Results Interchange Format", - supports: [], - attributes: [], - groups: [], - status: "loaded", - controls: [ - { - path: "results", - key: "id", - tags: { - cci: { - path: MESSAGE_TEXT, - transformer: (data: string) => - getCCIsForNISTTags(nistTag(data)), - }, - nist: { path: MESSAGE_TEXT, transformer: nistTag }, - cwe: { - path: MESSAGE_TEXT, - transformer: extractCwe, - }, - }, - refs: [], - source_location: { - transformer: (control: unknown) => { - return _.omitBy( - { - ref: _.get( - control, - "locations[0].physicalLocation.artifactLocation.uri" - ), - line: _.get( - control, - "locations[0].physicalLocation.region.startLine" - ), - }, - (value) => value === "" - ); - }, - }, - title: { - path: MESSAGE_TEXT, - transformer: (text: unknown): string => { - if (typeof text === "string") { - return text.split(": ")[0]; - } else { - return ""; - } - }, - }, - id: { path: "ruleId" }, - desc: { - path: MESSAGE_TEXT, - transformer: (text: unknown): string => { - if (typeof text === "string") { - return text.split(": ")[1]; - } else { - return ""; - } - }, - }, - impact: { path: "level", transformer: impactMapping }, - code: { - transformer: (vulnerability: Record): string => - JSON.stringify(vulnerability, null, 2), - }, - results: [ - { - status: ExecJSON.ControlResultStatus.Failed, - code_desc: { - path: "locations[0].physicalLocation", - transformer: formatCodeDesc, - }, - - start_time: "", - }, - ], - }, - ], - sha256: "", - }, - ], - passthrough: { - transformer: (data: Record): Record => { - let runsData = _.get(data, "runs"); - if (Array.isArray(runsData)) { - runsData = runsData.map((run: Record) => - _.omit(run, ["results"]) - ); - } - return { - auxiliary_data: [ - { - name: "SARIF", - data: { - $schema: _.get(data, "$schema"), - runs: runsData, - }, - }, - ], - ...(this.withRaw && { raw: data }), - }; - }, - }, - }; - constructor(sarifJson: string, withRaw = false) { - super(JSON.parse(sarifJson)); - this.withRaw = withRaw; - } -} -``` - -::: - -Now we have a fully implemented SARIF-to-OHDF mapper. diff --git a/src/courses/mappers/nonguided-labs.txt b/src/courses/mappers/nonguided-labs.txt new file mode 100644 index 000000000..b840ff1bb --- /dev/null +++ b/src/courses/mappers/nonguided-labs.txt @@ -0,0 +1,810 @@ +TEMP STORAGE FOR NONGUIDED LABS TO BE ADDED LATER + +### Mapper Demo - JFrog + +This section is yet another example of implementing an OHDF mapper, namely the JFrog-Xray mapper. Again, we will assume that the [appropriate file set up for the mapper](#file-set-up) has been performed. Because we have already gone through several examples, this section will not explain too many things in detail. The goal is to simply provide more examples for reference. + +Here is our developed mapping for JFrog-Xray for reference: + +::: details JFrog-to-OHDF Mapping + +```typescript +{ + platform: { + name: 'Heimdall Tools', + release: HeimdallToolsVersion, + target_id + }, + version: HeimdallToolsVersion, + statistics: { + duration + }, + profiles: [ + { + name, + version, + sha256, + title, + maintainer, + summary, + license, + copyright, + copyright_email, + supports, + attributes, + groups, + controls: [ + { + id: 'data.id', // If ID empty, hash the summary + title: 'data.summary', + desc: 'component_versions.more_details', + descriptions, + impact: 'data.severity', + refs, + tags: { + nist: 'data.component_versions.more_details.cves[0].cwe', // Map to NIST + cci: 'data.component_versions.more_details.cves[0].cwe', // Map to CCI + cweid: 'data.component_versions.more_details.cves[0].cwe' + }, + code, + source_location, + results: [ + { + status: 'Failed', // All reported findings are failures + code_desc: ['data.source_comp_id', 'data.component_versions.vulnerable_versions', 'data.component_versions.fixed_versions', 'data.issue_type', 'data.provider'], + message, + run_time, + start_time + } + ] + }, + ], + status: 'loaded' + }, + ], + passthrough: { + auxiliary_data: [ + { + name, + data + }, + ], + raw + } +} +``` + +::: + +::: details JFrog Source Data + +```json +{ + "total_count": 30, + "data": [ + { + "id": "", + "severity": "High", + "summary": "Acorn regexp.js Regular Expression Validation UTF-16 Surrogate Handling Infinite Loop DoS", + "issue_type": "security", + "provider": "JFrog", + "component": "acorn", + "source_id": "npm://acorn", + "source_comp_id": "npm://acorn:5.7.3", + "component_versions": { + "id": "acorn", + "vulnerable_versions": [ + "5.5.0 ≤ Version < 5.7.4", + "6.0.0 ≤ Version < 6.4.1", + "7.0.0", + "7.1.0" + ], + "fixed_versions": ["5.7.4", "6.4.1", "7.1.1"], + "more_details": { + "cves": [ + { + "cvss_v2": "7.1/AV:N/AC:M/Au:N/C:N/I:N/A:C", + "cvss_v3": "7.5/CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H" + } + ], + "description": "Acorn contains an infinite loop condition in regexp.js that is triggered when handling UTF_16 surrogates while validating regular expressions. This may allow a context-dependent attacker to hang a process using the library.", + "provider": "JFrog" + } + }, + "edited": "2020-11-03T19:30:42-05:00" + } + ] +} +``` + +::: + +We will now go over the mapper code that allows us to shape this data properly. Firstly, let us handle the ID. Sometimes, JFrog source files will not have an ID field. In this case, it is appropriate to generate a hash based on the `summary` as follows: + +```typescript +function hashId(vulnerability: unknown): string { + if (_.get(vulnerability, "id") === "") { + return generateHash( + (_.get(vulnerability, "summary") as unknown as string).toString(), + "md5" + ); + } else { + return _.get(vulnerability, "id") as unknown as string; + } +} +``` + +As with most mappers, we must also define an impact mapping: + +```typescript +const IMPACT_MAPPING: Map = new Map([ + ["high", 0.7], + ["medium", 0.5], + ["low", 0.3], +]); +``` + +Recall that we wanted to format the `desc` field based on the data in the source file's `component_versions.more_details`. We can write a simple transformer function to concatenate all these keys: + +```typescript +function formatDesc(vulnerability: unknown): string { + const text = []; + if (_.has(vulnerability, "description")) { + text.push( + (_.get(vulnerability, "description") as unknown as string).toString() + ); + } + if (_.has(vulnerability, "cves")) { + const re1 = /":/gi; + const re2 = /,/gi; + text.push( + `cves: ${JSON.stringify(_.get(vulnerability, "cves")) + .replace(re1, '"=>') + .replace(re2, ", ")}` + ); + } + return text.join("
"); +} +``` + +We also must properly handle the mappings to NIST and CCI tags. Luckily, we can make use of the library's existing mappings, as well as an existing global function called `getCCIsForNISTTags()` in `global.ts`. + +```typescript +const CWE_PATH = "component_versions.more_details.cves[0].cwe"; +const CWE_NIST_MAPPING = new CweNistMapping(); + +function nistTag(identifier: Record): string[] { + const identifiers: string[] = []; + if (Array.isArray(identifier)) { + identifier.forEach((element) => { + if (element.split("CWE-")[1]) { + identifiers.push(element.split("CWE-")[1]); + } + }); + } + return CWE_NIST_MAPPING.nistFilter( + identifiers, + DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS + ); +} +``` + +Finally, we can implement the transformer function for the `code_desc`. Some of these fields are optional, so it is important to implement a base case to ensure our `code_desc` outputs are all consistent. + +```typescript +function formatCodeDesc(vulnerability: unknown): string { + const codeDescArray: string[] = []; + const re = /,/gi; + if (_.has(vulnerability, "source_comp_id")) { + codeDescArray.push( + `source_comp_id : ${_.get(vulnerability, "source_comp_id")}` + ); + } else { + codeDescArray.push("source_comp_id : "); + } + if (_.has(vulnerability, "component_versions.vulnerable_versions")) { + codeDescArray.push( + `vulnerable_versions : ${JSON.stringify( + _.get(vulnerability, "component_versions.vulnerable_versions") + )}` + ); + } else { + codeDescArray.push("vulnerable_versions : "); + } + if (_.has(vulnerability, "component_versions.fixed_versions")) { + codeDescArray.push( + `fixed_versions : ${JSON.stringify( + _.get(vulnerability, "component_versions.fixed_versions") + )}` + ); + } else { + codeDescArray.push("fixed_versions : "); + } + if (_.has(vulnerability, "issue_type")) { + codeDescArray.push(`issue_type : ${_.get(vulnerability, "issue_type")}`); + } else { + codeDescArray.push("issue_type : "); + } + if (_.has(vulnerability, "provider")) { + codeDescArray.push(`provider : ${_.get(vulnerability, "provider")}`); + } else { + codeDescArray.push("provider : "); + } + return codeDescArray.join("\n").replace(re, ", "); +} +``` + +All that's left is to incoporate all these functions into our `base-converter` structure, write some smaller transformer functions for the simpler data formatting tasks, and combine it all together! + +:::details Full Mapper Code + +```typescript +import { ExecJSON } from "inspecjs"; +import * as _ from "lodash"; +import { version as HeimdallToolsVersion } from "../package.json"; +import { + BaseConverter, + generateHash, + ILookupPath, + impactMapping, + MappedTransform, +} from "./base-converter"; +import { CweNistMapping } from "./mappings/CweNistMapping"; +import { + DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS, + getCCIsForNISTTags, +} from "./utils/global"; + +// Constants +const IMPACT_MAPPING: Map = new Map([ + ["high", 0.7], + ["medium", 0.5], + ["low", 0.3], +]); + +const CWE_PATH = "component_versions.more_details.cves[0].cwe"; + +const CWE_NIST_MAPPING = new CweNistMapping(); + +// Transformation Functions +function hashId(vulnerability: unknown): string { + if (_.get(vulnerability, "id") === "") { + return generateHash( + (_.get(vulnerability, "summary") as unknown as string).toString(), + "md5" + ); + } else { + return _.get(vulnerability, "id") as unknown as string; + } +} +function formatDesc(vulnerability: unknown): string { + const text = []; + if (_.has(vulnerability, "description")) { + text.push( + (_.get(vulnerability, "description") as unknown as string).toString() + ); + } + if (_.has(vulnerability, "cves")) { + const re1 = /":/gi; + const re2 = /,/gi; + text.push( + `cves: ${JSON.stringify(_.get(vulnerability, "cves")) + .replace(re1, '"=>') + .replace(re2, ", ")}` + ); + } + return text.join("
"); +} +function formatCodeDesc(vulnerability: unknown): string { + const codeDescArray: string[] = []; + const re = /,/gi; + if (_.has(vulnerability, "source_comp_id")) { + codeDescArray.push( + `source_comp_id : ${_.get(vulnerability, "source_comp_id")}` + ); + } else { + codeDescArray.push("source_comp_id : "); + } + if (_.has(vulnerability, "component_versions.vulnerable_versions")) { + codeDescArray.push( + `vulnerable_versions : ${JSON.stringify( + _.get(vulnerability, "component_versions.vulnerable_versions") + )}` + ); + } else { + codeDescArray.push("vulnerable_versions : "); + } + if (_.has(vulnerability, "component_versions.fixed_versions")) { + codeDescArray.push( + `fixed_versions : ${JSON.stringify( + _.get(vulnerability, "component_versions.fixed_versions") + )}` + ); + } else { + codeDescArray.push("fixed_versions : "); + } + if (_.has(vulnerability, "issue_type")) { + codeDescArray.push(`issue_type : ${_.get(vulnerability, "issue_type")}`); + } else { + codeDescArray.push("issue_type : "); + } + if (_.has(vulnerability, "provider")) { + codeDescArray.push(`provider : ${_.get(vulnerability, "provider")}`); + } else { + codeDescArray.push("provider : "); + } + return codeDescArray.join("\n").replace(re, ", "); +} +function nistTag(identifier: Record): string[] { + const identifiers: string[] = []; + if (Array.isArray(identifier)) { + identifier.forEach((element) => { + if (element.split("CWE-")[1]) { + identifiers.push(element.split("CWE-")[1]); + } + }); + } + return CWE_NIST_MAPPING.nistFilter( + identifiers, + DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS + ); +} + +// Mappings +export class JfrogXrayMapper extends BaseConverter { + withRaw: boolean; + + mappings: MappedTransform< + ExecJSON.Execution & { passthrough: unknown }, + ILookupPath + > = { + platform: { + name: "Heimdall Tools", + release: HeimdallToolsVersion, + }, + version: HeimdallToolsVersion, + statistics: {}, + profiles: [ + { + name: "JFrog Xray Scan", + title: "JFrog Xray Scan", + summary: "Continuous Security and Universal Artifact Analysis", + supports: [], + attributes: [], + groups: [], + status: "loaded", + controls: [ + { + path: "data", + key: "id", + tags: { + cci: { + path: CWE_PATH, + transformer: (identifier: Record) => + getCCIsForNISTTags(nistTag(identifier)), + }, + nist: { + path: CWE_PATH, + transformer: nistTag, + }, + cweid: { path: CWE_PATH }, + }, + refs: [], + source_location: {}, + id: { transformer: hashId }, + title: { path: "summary" }, + desc: { + path: "component_versions.more_details", + transformer: formatDesc, + }, + impact: { + path: "severity", + transformer: impactMapping(IMPACT_MAPPING), + }, + code: { + transformer: (vulnerability: Record): string => { + return JSON.stringify(vulnerability, null, 2); + }, + }, + results: [ + { + status: ExecJSON.ControlResultStatus.Failed, + code_desc: { transformer: formatCodeDesc }, + start_time: "", + }, + ], + }, + ], + sha256: "", + }, + ], + passthrough: { + transformer: (data: Record): Record => { + return { + auxiliary_data: [ + { + name: "JFrog Xray", + data: _.pick(data, ["total_count"]), + }, + ], + ...(this.withRaw && { raw: data }), + }; + }, + }, + }; + constructor(xrayJson: string, withRaw = false) { + super(JSON.parse(xrayJson), true); + this.withRaw = withRaw; + } +} +``` + +::: + +Now we have a fully implemented JFrog-to-OHDF mapper. + +### Mapper Demo - SARIF + +Here is one last example of implementing an OHDF mapper - the SARIF mapper. As in previous examples, we will assume that the [appropriate file set up for the mapper](#file-set-up) has been performed. Like with the JFrog example, we will not be going into too much detail regarding the mapping itself. The goal is to simply provide more examples for reference. + +Here is our developed mapping for SARIF for reference: + +::: details SARIF-to-OHDF Mapping + +```typescript +{ + platform: { + name: 'Heimdall Tools', + release: HeimdallToolsVersion, + target_id + }, + version: HeimdallToolsVersion, + statistics: { + duration + }, + profiles: [ + { + name, + version: 'version', + sha256, + title, + maintainer, + summary, + license, + copyright, + copyright_email, + supports, + attributes, + groups, + controls: [ + { + id: 'results.ruleId', // If ID empty, hash the summary + title: 'results.message.text', + desc: 'results.message.text', + descriptions, + impact: 'results.level', + refs, + tags: { + nist: 'results.message.text', // Map to NIST + cci: 'results.message.text', // Map to CCI + cwe: 'results.message.text' + }, + code, + source_location: ['results.locations[0].physicalLocation.artifactLocation.uri', 'results.locations[0].physicalLocation.region.startLine'], + results: [ + { + status: 'Failed', // All reported findings are failures + code_desc: 'results.locations[0].physicalLocation', + message, + run_time, + start_time + } + ] + }, + ], + status: 'loaded' + }, + ], + passthrough: { + auxiliary_data: [ + { + name, + data + }, + ], + raw + } +} +``` + +::: + +::: details SARIF Source Data + +```json +{ + "$schema": "https://schemastore.azurewebsites.net/schemas/json/sarif-2.1.0-rtm.5.json", + "version": "2.1.0", + "runs": [ + { + "results": [ + { + "ruleId": "FF1014", + "level": "error", + "message": { + "text": "buffer/gets: Does not check for buffer overflows (CWE-120, CWE-20)." + }, + "locations": [ + { + "physicalLocation": { + "artifactLocation": { + "uri": "test/test-patched.c", + "uriBaseId": "SRCROOT" + }, + "region": { + "startLine": 36, + "startColumn": 2, + "endColumn": 10, + "snippet": { + "text": " gets(f);" + } + } + } + } + ], + "fingerprints": { + "contextHash/v1": "6a5bb383fb44030b0d9428b17359e94ba3979bc1ce702be450427f85592c649a" + }, + "rank": 1 + } + ] + } + ] +} +``` + +::: + +First, let us define some common constants that might help us with our code. + +```typescript +const MESSAGE_TEXT = "message.text"; +const CWE_NIST_MAPPING = new CweNistMapping(); +``` + +We will now outline the `IMPACT_MAPPING` for the SARIF mapper: + +```typescript +const IMPACT_MAPPING: Map = new Map([ + ["error", 0.7], + ["warning", 0.5], + ["note", 0.3], +]); + +function impactMapping(severity: unknown): number { + if (typeof severity === "string" || typeof severity === "number") { + return IMPACT_MAPPING.get(severity.toString().toLowerCase()) || 0.1; + } else { + return 0.1; + } +} +``` + +Now, note that we will need to extract the CWEs from the message text, as well as map them to the proper NIST tags. For the CCIs, we can again use the global function `getCCIsForNISTTags()`. + +```typescript +function extractCwe(text: string): string[] { + let output = text.split("(").slice(-1)[0].slice(0, -2).split(", "); + if (output.length === 1) { + output = text.split("(").slice(-1)[0].slice(0, -2).split("!/"); + } + return output; +} +function nistTag(text: string): string[] { + let identifiers = extractCwe(text); + identifiers = identifiers.map((element) => element.split("-")[1]); + return CWE_NIST_MAPPING.nistFilter( + identifiers, + DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS + ); +} +``` + +Our last standalone transformer function will format our code description. This part is up to interpretation, but we decided to just concatenate them all with colon-separated key-value pairs + +```typescript +function formatCodeDesc(input: unknown): string { + const output = []; + output.push(`URL : ${_.get(input, "artifactLocation.uri")}`); + output.push(`LINE : ${_.get(input, "region.startLine")}`); + output.push(`COLUMN : ${_.get(input, "region.startColumn")}`); + return output.join(" "); +} +``` + +Now, all that's left is to combine our functions, write some mini-transformers for the small data formatting, and plug it all into the base converter! + +:::details Full Mapper Code + +```typescript +import { ExecJSON } from "inspecjs"; +import * as _ from "lodash"; +import { version as HeimdallToolsVersion } from "../package.json"; +import { BaseConverter, ILookupPath, MappedTransform } from "./base-converter"; +import { CweNistMapping } from "./mappings/CweNistMapping"; +import { + DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS, + getCCIsForNISTTags, +} from "./utils/global"; + +const IMPACT_MAPPING: Map = new Map([ + ["error", 0.7], + ["warning", 0.5], + ["note", 0.3], +]); +const MESSAGE_TEXT = "message.text"; +const CWE_NIST_MAPPING = new CweNistMapping(); + +function extractCwe(text: string): string[] { + let output = text.split("(").slice(-1)[0].slice(0, -2).split(", "); + if (output.length === 1) { + output = text.split("(").slice(-1)[0].slice(0, -2).split("!/"); + } + return output; +} +function impactMapping(severity: unknown): number { + if (typeof severity === "string" || typeof severity === "number") { + return IMPACT_MAPPING.get(severity.toString().toLowerCase()) || 0.1; + } else { + return 0.1; + } +} +function formatCodeDesc(input: unknown): string { + const output = []; + output.push(`URL : ${_.get(input, "artifactLocation.uri")}`); + output.push(`LINE : ${_.get(input, "region.startLine")}`); + output.push(`COLUMN : ${_.get(input, "region.startColumn")}`); + return output.join(" "); +} +function nistTag(text: string): string[] { + let identifiers = extractCwe(text); + identifiers = identifiers.map((element) => element.split("-")[1]); + return CWE_NIST_MAPPING.nistFilter( + identifiers, + DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS + ); +} + +export class SarifMapper extends BaseConverter { + withRaw: boolean; + + mappings: MappedTransform< + ExecJSON.Execution & { passthrough: unknown }, + ILookupPath + > = { + platform: { + name: "Heimdall Tools", + release: HeimdallToolsVersion, + target_id: "Static Analysis Results Interchange Format", + }, + version: HeimdallToolsVersion, + statistics: {}, + profiles: [ + { + path: "runs", + name: "SARIF", + version: { path: "$.version" }, + title: "Static Analysis Results Interchange Format", + supports: [], + attributes: [], + groups: [], + status: "loaded", + controls: [ + { + path: "results", + key: "id", + tags: { + cci: { + path: MESSAGE_TEXT, + transformer: (data: string) => + getCCIsForNISTTags(nistTag(data)), + }, + nist: { path: MESSAGE_TEXT, transformer: nistTag }, + cwe: { + path: MESSAGE_TEXT, + transformer: extractCwe, + }, + }, + refs: [], + source_location: { + transformer: (control: unknown) => { + return _.omitBy( + { + ref: _.get( + control, + "locations[0].physicalLocation.artifactLocation.uri" + ), + line: _.get( + control, + "locations[0].physicalLocation.region.startLine" + ), + }, + (value) => value === "" + ); + }, + }, + title: { + path: MESSAGE_TEXT, + transformer: (text: unknown): string => { + if (typeof text === "string") { + return text.split(": ")[0]; + } else { + return ""; + } + }, + }, + id: { path: "ruleId" }, + desc: { + path: MESSAGE_TEXT, + transformer: (text: unknown): string => { + if (typeof text === "string") { + return text.split(": ")[1]; + } else { + return ""; + } + }, + }, + impact: { path: "level", transformer: impactMapping }, + code: { + transformer: (vulnerability: Record): string => + JSON.stringify(vulnerability, null, 2), + }, + results: [ + { + status: ExecJSON.ControlResultStatus.Failed, + code_desc: { + path: "locations[0].physicalLocation", + transformer: formatCodeDesc, + }, + + start_time: "", + }, + ], + }, + ], + sha256: "", + }, + ], + passthrough: { + transformer: (data: Record): Record => { + let runsData = _.get(data, "runs"); + if (Array.isArray(runsData)) { + runsData = runsData.map((run: Record) => + _.omit(run, ["results"]) + ); + } + return { + auxiliary_data: [ + { + name: "SARIF", + data: { + $schema: _.get(data, "$schema"), + runs: runsData, + }, + }, + ], + ...(this.withRaw && { raw: data }), + }; + }, + }, + }; + constructor(sarifJson: string, withRaw = false) { + super(JSON.parse(sarifJson)); + this.withRaw = withRaw; + } +} +``` + +::: + +Now we have a fully implemented SARIF-to-OHDF mapper.