diff --git a/src/.vuepress/navbar.ts b/src/.vuepress/navbar.ts index d0e67529f..6fbe5203e 100644 --- a/src/.vuepress/navbar.ts +++ b/src/.vuepress/navbar.ts @@ -10,7 +10,8 @@ export default navbar([ { text: "Beginner Security Automation Developer Class", link: "/courses/beginner/", icon: "creative" }, { text: "Advanced Security Automation Developer Class", link: "/courses/advanced/", icon: "creative" }, { text: "Security Guidance Developer Class", link: "/courses/guidance/", icon: "creative" }, - { text: "InSpec Profile Development & Testing", link: "/courses/profile-dev-test/", icon: "creative"} + { text: "InSpec Profile Development & Testing", link: "/courses/profile-dev-test/", icon: "creative"}, + { text: "OHDF Mapper Development Class", link: "/courses/mappers/", icon: "creative"} ]}, { text: "Resources", icon: "book", diff --git a/src/.vuepress/sidebar.ts b/src/.vuepress/sidebar.ts index 60cb04788..7f768b807 100644 --- a/src/.vuepress/sidebar.ts +++ b/src/.vuepress/sidebar.ts @@ -32,12 +32,19 @@ export default sidebar({ children: "structure", collapsible: true }, - { + { icon: "creative", text: "InSpec Profile Development & Testing", prefix: "courses/profile-dev-test/", children: "structure", collapsible: true }, + { + icon: "creative", + text: "OHDF Mapper Class", + prefix: "courses/mappers/", + children: "structure", + collapsible: true + }, ], }); diff --git a/src/README.md b/src/README.md index c304426fa..44a2c8d16 100644 --- a/src/README.md +++ b/src/README.md @@ -21,6 +21,9 @@ actions: - text: InSpec Profile Updating & Development link: /courses/profile-dev-test/ type: primary + - text: OHDF Mapper Development Class + link: /courses/mappers/ + type: primary highlights: - header: What You Will Learn diff --git a/src/assets/img/OHDF_Inputs.png b/src/assets/img/OHDF_Inputs.png new file mode 100644 index 000000000..ccc5bf452 Binary files /dev/null and b/src/assets/img/OHDF_Inputs.png differ diff --git a/src/courses/mappers/02.md b/src/courses/mappers/02.md new file mode 100644 index 000000000..718f13ea1 --- /dev/null +++ b/src/courses/mappers/02.md @@ -0,0 +1,207 @@ +--- +order: 2 +next: 03.md +title: What Is OHDF? +author: Charles Hu +--- + +## A Look Ahead + +In this section, we will cover: + +- [What OHDF is](#what-is-ohdf) +- [Why you should use OHDF](#why-ohdf) +- [How you can use OHDF in your work](#features) +- [Who benefits from OHDF and how](#benefits) + +## What is OHDF? + +The OASIS Heimdall Data Format (OHDF) is a security data format used to normalize generated data exports from various security tools into a single common data format usable by the Security Automation Framework (SAF) tool suite. The format is defined by the [OHDF schema](https://saf.mitre.org/framework/normalize/ohdf-schema) and its goal is to provide a simple and intuitive means for representing security data context, requirements, and measures. + +:::info OASIS Open +[OASIS Open](https://www.oasis-open.org/) is an international standards body that works on the development and advancement of open source technological standards. OHDF is currently in the process of becoming an OASIS Open standard. + +For more information on the OHDF charter for OASIS Open, [refer here](https://groups.oasis-open.org/communities/tc-community-home2?CommunityKey=f8888caa-8401-46f8-bf10-018dc7d3f577). +::: + +OHDF's goal is to provide: + +- Standardization: Defining data elements in a consistent and contextualized manner. +- Normalization: Processing any format's data elements into another format's data elements in a consistent and reliable manner. + +## Why OHDF? + +In a field saturated with data format standards, what sets OHDF apart from the rest? Why should you use OHDF for your project? + +In the realm of cybersecurity, many security data exports suffer from a plethora of shortcomings and oversights. This creates critical issues such as: + +- Many security tools not providing context to relevant categorization standards for comparison across security tools.​ +- Security tools typically generating data in unique formats that require multiple dashboards and utilities to process.​ +- It taking too much time to process different security data types, data in disparate locations, and inconsistent semantics of a data element between formats.​ + +What makes OHDF stand out is its goal of standardization and normalization, which sets out to address these aforementioned problems. OHDF aims to create a simple, reliable, and targetable security standard, culminating in a data format that bridges and unifies the many diverse and disparate representations of security data. + +### Features + +By using OHDF, you can leverage the plethora of built-in features that help standardize and normalize security data across your project. This includes features such as: + +#### 1. Consistent integration, aggregation, and analysis of security data from all available sources.​ + +- Enforces consistent schema fields through consciously designed data format mappings. +- Supports data format conversion from numerous established security tools such as AWS Security Hub's AWS Security Finding Format (ASFF) and Tenable Nessus' Nessus file format. +- Allows the integration of currently unsupported security tool data formats through the development of OHDF mappers for the OHDF Converters library. + +![OHDF can ingest and standardize a plethora of security data formats.](../../assets/img/OHDF_Inputs.png) + +#### 2. Preserving data integrity with original source data. + +- Uses mappings which maximize meaningful schema field conversions. +- Leverages schema fields `passthrough` and `raw` to preserve the original data in its entirety. +- Allows for bidirectional format conversions to and from OHDF. + +![A mapping between Format A and OHDF.](./img/basic_mapping.png) + +#### 3. Maximizing interoperability and data sharing.​ + +- Provides a consistent and standardized format for communication. +- Provides an easily ingestible data format and tools to improve user readability. + +#### 4. Facilitating the transformation and transport of data between security/management processes or technologies.​ + +- Provides a clear schema for technologies/processes to support. +- Includes a simple file format that technologies/processes can accept. +- Compatible with [Heimdall](./03.md#what-is-heimdall) to provide data visualization. + +![OHDF can help bridge the gap between security tool outputs and security assessors.](./img/scan_to_assessor.png) + +#### 5. Allowing for the mapping and enrichment of security data to relevant compliance standards (GDPR, NIST SP 800-53, CCIs, PCI-DSS, etc.). + +- Uses mappers which provide and append relevant categorization standards to converted security tool data. + +:::::::info What Are All These Abbreviations? +The aforementioned terms are all security related guidelines, frameworks, or implementations. The following are explanations on terms that are commonly used in this course: + +:::details NIST +The National Institute of Standards and Technology (NIST) is a U.S. agency that "promotes U.S. innovation and industrial competitiveness by advancing measurement science, standards, and technology in ways that enhance economic security and improve quality of life." + +More information can be found [here](https://www.nist.gov/). +::: + +:::details NIST RMF +The NIST Risk Management Framework (NIST RMF) provides "a process that integrates security, privacy, and cyber supply chain risk management activities into the system development life cycle." + +More information can be found [here](https://csrc.nist.gov/projects/risk-management/about-rmf). +::: + +:::::details NIST SP 800-53 Controls/Requirements +The NIST Special Publication 800-53 (NIST SP 800-53s) "provides a catalog of security and privacy controls for information systems and organizations to protect organizational operations and assets, individuals, other organizations, and the Nation from a diverse set of threats and risks." + +More information can be found [here](https://csrc.nist.gov/pubs/sp/800/53/r5/upd1/final). + +- NIST SP 800-53 control example: + +``` +AC-17 REMOTE ACCESS +Control: +a. Establish and document usage restrictions, configuration/connection requirements, and +implementation guidance for each type of remote access allowed; and +b. Authorize each type of remote access to the system prior to allowing such connections. +``` + +::::: + +:::details DISA +The Defense Information Systems Agency (DISA) is a U.S. agency conducting "DODIN (Department of Defense information networks) operations to enable lethality across all warfighting domains in defense of our nation." + +More information can be found [here](https://www.disa.mil/). +::: + +:::::details CCIs +Control Correlation Identifiers (**CCIs**) are "standard identifiers and descriptions by DISA that aim to correlate high-level policy expressions and low-level technical implementations of security requirements." They are analogous to NIST SP 800-53 controls in that they both provide security and privacy controls. CCIs specifically address a single requirement and collate corresponding NIST SP 800-53 controls. + +More information can be found [here](https://public.cyber.mil/stigs/cci/). + +- CCI example: + +``` +CCI-000002: Disseminate the organization-level; mission/business process-level; and/or system-level access control policy that addresses purpose, scope, roles, responsibilities, management commitment, coordination among organizational entities, and compliance to organization-defined personnel or roles. + +References: +- NIST: NIST SP 800-53 (v3): AC-1 a +- NIST: NIST SP 800-53 Revision 4 (v4): AC-1 a 1 +- NIST: NIST SP 800-53 Revision 5 (v5): AC-1 a 1 (a) +- NIST: NIST SP 800-53A (v1): AC-1.1 (iii) +``` + +::::: +::::::: + +### Benefits + +Depending on your role, leveraging OHDF can help streamline and improve how you interact with your project's security processes in a myriad of ways: + +- For Commercial and Vendor Cybersecurity Partners: + + - OHDF defines a standardized, interoperable target format that vendor tools can consume across their customer base consistently and that is easily managed within the product lifecycle. + +- For the Open Source Community: + + - OHDF enables easy integration with commercial solutions without the need for direct partnerships. + +- For Government Agencies: + + - OHDF can streamline business processes by having a standard, open source, machine-readable format for all security data. + +- For Academia: + + - OHDF offers a structured way to communicate and enhance research findings throughout the security community. + +- For Corporate and Federal CISOs/CIOs: + + - OHDF can increase visibility across the enterprise by taking advantage of normalized security data in a standard format that supports risk information interoperability from a broad range of inputs to support security risk decision-making. + +- For Security Engineers: + + - OHDF can reduce resource requirements for multiple security data types by standardizing formatting across disparate security tools. + +- For Risk Managers: + + - OHDF can improve decision making by using a standardized format to facilitate automation, standardize communication requirements, and inform risk-based analysis. + +- For DevSecOps/Software Engineers: + + - OHDF can streamline CI/CD processes by leveraging a standardized format to collate/aggregate normalized security data to support automated and continuous security processes. + +## What Else? + +You can read more about OHDF [here](https://saf.mitre.org/framework/normalize). + +## A Look Back + +In this section, we covered: + +- [What OHDF is](#what-is-ohdf) + + - OHDF is a security data format used to normalize generated data exports from various security tools into a standard common data format usable by the SAF tool suite. + +- [Why you should use OHDF](#why-ohdf) + + - OHDF provides a simple, reliable, and targetable security standard which bridges and unifies the many diverse and disparate representations of security data in the field of cybersecurity. + +- [How you can use OHDF in your work](#features) + + - OHDF provides a plethora of built-in features which help you achieve standardization and normalization of your project's security data. + +- [Who benefits from OHDF and how](#benefits) + + - OHDF can help you streamline and ease how you specifcally process and interact with security data in your project. + +### Knowledge Check + +:::details Where is OHDF used? +OHDF can be found in a plethora of projects but is officially used in the SAF tool suite. This includes tools such as Heimdall, a visualization tool for security data. +::: + +:::details How do security tool exports get converted into OHDF? +Security tool exports are converted into OHDF through the use of OHDF mappers, which convert the data elements of an export into the corresponding data elements in OHDF. +::: diff --git a/src/courses/mappers/03.md b/src/courses/mappers/03.md new file mode 100644 index 000000000..9f81e1a57 --- /dev/null +++ b/src/courses/mappers/03.md @@ -0,0 +1,100 @@ +--- +order: 3 +next: 04.md +title: Where Is OHDF Used? +author: Charles Hu +--- + +## A Look Ahead + +In this section, we will cover: + +- Where OHDF is used: + - [The Security Automation Framework](#what-is-the-security-automation-framework) + - [InspecJS](#what-is-inspecjs) + - [OHDF Converters](#what-is-ohdf-converters) + - [The SAF CLI](#what-is-the-saf-cli) + - [Heimdall](#what-is-heimdall) + +## Where Is OHDF Used? + +OHDF is a cornerstone of the Security Automation Framework and is officially implemented and used in a plethora of tools and libraries including InspecJS, OHDF Converters, the SAF CLI, and Heimdall. + +### What Is the Security Automation Framework? + +The Security Automation Framework (SAF) is a suite of open-source security automation tools that facilitate the development, collection, and standardization of content for use by government and industry organizations to​: + +- Accelerate ATO. +- Establish security requirements. +- Build security in. +- Assess/monitor vulnerabilities. + +SAF has five core capabilities, of which OHDF is involved in varying degrees: + +- **Plan**: OHDF is defined such that it includes all information described in the guidance, as well as relevant metadata. Thus, the information provided by the guidance is incorporated with the security tool results. + +- **Harden**: OHDF is not typically used in this capability. + +- **Validate**: InSpec's results format is a subset of the OHDF, meaning that InSpec is natively supported. Moreover, it's easy to write InSpec such that all the guidance information is passed through into both the OHDF profile and OHDF results formats. + +- **Normalize**: With the use of OHDF Converters, we are able to map many different security result formats to our standard format, OHDF. These converters can either be accessed directly or via the SAF CLI. + +- **Visualize**: OHDF is ingested by a variety of tools and transformed in many different ways. For example, a results file can be turned into a threshold file for pipeline use, turned into a POAM for ingestion by eMASS, or ingested by Heimdall directly for display purposes. + +![SAF has five core capabilities, which we have developed/identified tooling and scripts to address.](./img/saf_security_validation_lifecycle.png) + +You can read more about SAF [here](../user/03.md). + +### What Is InspecJS? + +[InspecJS](https://github.com/mitre/heimdall2/tree/master/libs/inspecjs) is a library that provides schema definitions, classes, and utilities for OHDF file handling. InspecJS plays a pivotal role in the contextualization process where it performs tasks such as converting the individual statuses for each finding into an overall status for the requirement/control. + +### What Is OHDF Converters? + +[OHDF Converters](https://github.com/mitre/heimdall2/tree/master/libs/hdf-converters) is a custom data normalization library which hosts and leverages OHDF mappers for transforming various security data formats to and from OHDF. It is currently integrated in tools such as [Heimdall](https://github.com/mitre/heimdall2) and the [SAF CLI](https://github.com/mitre/saf), which collectively are part of the [Security Automation Framework (SAF)](https://saf.mitre.org/#/), a set of tools and processes which aim to standardize and ease security testing and verification for systems such as automated build pipelines. + +### What Is the SAF CLI? + +The Security Automation Framework Command Line Interface (SAF CLI) brings together applications, techniques, libraries, and tools developed by MITRE and the security community to streamline security automation for systems and DevOps pipelines. + +One use case for the SAF CLI is automating certain security processes in a CI/CD pipeline. For instance, you can set up the SAF CLI to automatically ingest security reports generated from the build pipeline, normalize the security data into OHDF using the `saf convert` command, and then forward the newly generated OHDF files to a visualization tool such as Heimdall that allows a security assessor to review the state of the current software build. + +![SAF CLI utility overview: Attest, Convert, View, Validate, and Generate.](./img/saf_cli_features.png) + +### What Is Heimdall? + +[Heimdall](https://github.com/mitre/heimdall2) is a visualization tool that provides a GUI-based means for managing and analyzing security data. Data that is imported into Heimdall is automatically converted to OHDF through OHDF Converters, which serves as the underlying library that services data format conversion requests in Heimdall. + +![An instance of Heimdall visualizing a security result set. Results are displayed with figures, charts, and compliance level percentages to quickly convey important takeaways at a glance.](./img/heimdall_view.png) + +## A Look Back + +In this section, we covered: + +- Where OHDF is used: + + - [The Security Automation Framework](#what-is-the-security-automation-framework) + + - The Security Automation Framework is a suite of open-source security automation tools that facilitate the development, collection, and standardization of content. OHDF is an integral data format used in many of these SAF tools. + + - [InspecJS](#what-is-inspecjs) + + - InspecJS is a library that provides schema definitions, classes, and utilities for OHDF file handling. + + - [OHDF Converters](#what-is-ohdf-converters) + + - OHDF Converters is a custom data normalization library which hosts and leverages OHDF mappers for transforming various security data formats to and from OHDF. + + - [The SAF CLI](#what-is-the-saf-cli) + + - The SAF CLI provides applications, techniques, libraries, and tools for security automation in systems and DevOps pipelines. OHDF is used in SAF CLI utilities such as the `saf convert` command which allows you to convert any compatible security data format into OHDF for further use in other SAF tools. + + - [Heimdall](#what-is-heimdall) + + - Heimdall is a security data visualization tool which uses OHDF as its primary data format. + +### Knowledge Check + +:::details Where can you find OHDF mappers? +OHDF mappers can be found in the OHDF Converters library, where they are hosted. They can also be found in tools which leverage OHDF Converters such as Heimdall and the SAF CLI. +::: diff --git a/src/courses/mappers/04.md b/src/courses/mappers/04.md new file mode 100644 index 000000000..78f808633 --- /dev/null +++ b/src/courses/mappers/04.md @@ -0,0 +1,216 @@ +--- +order: 4 +next: 05.md +title: What Is an OHDF Mapper? +author: Charles Hu +--- + +## A Look Ahead + +In this section, we will cover: + +- [What a mapper is](#what-is-a-mapper) +- [What an OHDF mapper is](#what-is-an-ohdf-mapper) + +## What Is a Mapper? + +A mapper is an example of the Adapter Design Pattern which is used to correlate (or 'map') data elements in two different objects with one another. Mappers are useful in that they allow us to correlate items in objects that are nominally different but semantically the same. A result of this is that we can easily transform one object type into another through the use of mappings which define the direct relationship of data elements in each object type. An important aspect of the transformation process is that while the data elements containing the values are transformed, the values themselves typically remain the same. + +![A correlation between data elements in Format A and Format B.](./img/mapping.png) + +Mappings are the actual correlation of data elements between two different objects. They are the basis for mappers and are responsible for defining which data elements between two objects are semantically the same. Mappers are the implementation of mappings which perform the necessary processing to transform the objects according to relationships found in the mappings. In order to have a mapper that is as accurate as possible in its transformations, the mappings should ideally include correlations for all data elements in the objects. + +![A mapper converting an object from Format A to Format B using a predefined mapping.](./img/mapper_mapping.png) + +Oftentimes, however, we will encounter situations where the data elements between two objects either do not perfectly align or are not correlatable in a one-to-one relationship. In such scenarios, data elements can also be correlated to other data elements that are semantically similar enough and or correlated to multiple data elements that are appropriate in a one-to-many or many-to-one relationship. + +![An instance of a mapping between formats which do not perfectly align.](./img/mapping_bad.png) + +Here are some scenarios which demonstrate some key aspects of mappers: + +::: details Transferring Employee IDs +Say we have a business that is changing its employee identification software and needs to transfer the current credentials of its employees from Software A to Software B. The data formats the softwares use for IDs are as follows: + +```json +// Software A +{Name, DoB, Age, Title} + +// Software B +{employee, employeeBirthday, employeeAge, jobTitle} +``` + +How do we transfer John's credentials from Software A to Software B? + +```json +{ + "Name": "John Doe", + "DoB": "10-6-1992", + "Age": 32, + "Title": "Security Technician" +} +``` + +What we can do is create a mapping which correlates the items from Software A's ID scheme to Software B's: + +| Format A | Format B | +| ---------------- | -------- | +| employee | Name | +| employeeBirthday | DoB | +| employeeAge | Age | +| jobTitle | Title | + +With this, we can then develop a mapper which takes John's credentials from Software A and transforms it to Software B's format as so: + +```json +{ + "Name": "John Doe", + "DoB": "10-6-1992", + "Age": 32, + "Title": "Security Technician" +} +``` + +``` + || + \ / + \/ + + MAPPER + + || + \ / + \/ +``` + +```json +{ + "employee": "John Doe", + "employeeBirthday": "10-6-1992", + "employeeAge": 32, + "jobTitle": "Security Technician" +} +``` + +The important thing to note here is that mappers rely on underlying mappings which match semantically similar fields between two objects. These matches allow us to correctly convert each item in one object to the other. +::: + +::: details Newly Issued Licenses +The city of Gdańsk is issuing a new license for its citizens. The old and new license formats are as so: + +```json +// Old license +{name, father, mother, spouse, age, id} + +// New license +{firstName, lastName, DoB, nationalID, spouse, children} +``` + +How do we transfer Wojtek's license from the old format to the new format? + +```json +{ + "name": "Wojtek Bokiewicz", + "father": "N/A", + "mother": "Irena Bokiewicz", + "spouse": "N/A", + "age": 21, + "id": 2491 +} +``` + +Like in the previous scenario, we can create a mapping between the two formats to correlate semantically similar items. In this case, however, our formats do not perfectly align, which means that we may have multicorrelated or uncorrelated data elements. + +| Old License | New License | +| ----------- | ----------- | +| name | firstName\* | +| name | lastName\* | +| father | N/A | +| mother | N/A | +| spouse | spouse | +| age | DoB\* | +| id | nationalID | +| N/A | children | + +\*: Data element requires manipulation to apply. + +Notice that certain data elements in the new license format do not have a direct mapping back to the old license format and vice versa. This results in data loss when we convert from the old data format to the new data format where we lose possibly critical information. + +We can prevent this through data preservation, which is specifically implemented by adding new data elements to accomodate the lost ones or by adding a 'data repository' which stores all unused data elements. Data preservation in OHDF mappers will be discussed further in the next section. + +We can see this data loss when we actually apply this mapping: + +```json +{ + "name": "Wojtek Bokiewicz", + "father": "N/A", + "mother": "Irena Bokiewicz", + "spouse": "N/A", + "age": 21, + "id": 2491 +} +``` + +``` + || + \ / + \/ + + MAPPER + + || + \ / + \/ +``` + +```json +{ + "firstName": "Wojtek", + "lastName": "Bokiewicz", + "DoB": 1942, + "nationalID": 2491, + "spouse": "N/A", + "children": "" +} +``` + +Note how the information stored in the `mother` field is totally lost. + +An additional issue that may occur is empty data elements being generated by the mapper when no data is pulled over from the input data object (see `children`). There are many possible solutions for this, including pulling in the correct data from an external source, providing a default value that automatically fills out for every generated entry, or just leaving the field blank. All options are potentially valid depending on the needs of your mapper. + +Another important detail to note is how certain mappings require additional manipulations performed on the data in order to qualify the data as applicable to that data element. This can occur in both singly correlated (see `age` to `DoB`) or multicorrelated mappings (see `name` to `firstName` and `lastName`). Ideally, you should limit the amount of manipulations you perform on data unless absolutely necessary in order to preserve the integrity of your input data. If you must perform manipulations on the data, try to also preserve the original data in a backup data element or data repository. + +::: + +## What Is an OHDF Mapper? + +An OHDF mapper is a mapper specifically focused on transforming security data to and from OHDF. It consists of a mapping and a number of helper functions which facilitate the actual application of the mapping. In the case of the SAF tool suite, these mappers allow for the conversion of any given data format to OHDF (\*-to-OHDF) and vice versa (OHDF-to-\*) using helper functions and utilities provided by the existing conversion infrastructure in OHDF Converters. + +We can break down each individual OHDF mapper into three general components: + +- **Fingerprinting**: This component contains information on specific data elements unique to a given data format. It uses those 'fingerprints' to correctly identify the data format when encountered and pass it along to the actual mapper for transformation. + +![We can use the fingerprinting component to screen input files to determine if we should apply this mapper or not depending on whether the correct data elements are identified.](./img/fingerprinting.png) + +- **Mapper**: This component contains the actual mapping correlating the given data format to or from OHDF and the mapper which implements it. Mappings are built upon an understanding of the security data format (whether through formal schemas or on-hand export examples) and the OHDF schema, and the mechanics of the mapper revolve around correlating the fields in each as accurately as possible. Mappers may sometimes include preprocessing on the input file in order to ensure that the data is readable and easily usable by the mapper implementation. + +- **Testing**: This component consists of a testing suite which verifies the operability of the mapper and a variety of sample files that the testing suite ingests to perform those verifications. + +![Testing helps us verify that our generated outputs are correct and that future iterations of the mapper do not break key functionality.](./img/testing.png) + +## A Look Back + +In this section, we covered: + +- [What a mapper is](#what-is-a-mapper) + + - A mapper is a design pattern which allows us to to correlate items in objects that are nominally different but semantically the same. Mappings are the underlying correlations that define the relationship between data elements in the objects, while mappers are the actual implementation of those correlations which transforms one object into the another. + +- [What an OHDF mapper is](#what-is-an-ohdf-mapper) + + - An OHDF mapper is a mapper specifically geared towards either converting to OHDF or from OHDF and can be broken down into three general components: The fingerprinting component, the mapper component, and the testing component. + +### Knowledge Check + +:::details How does an OHDF mapper identify the correct file to normalize? +An OHDF mapper uses its fingerprinting component, which identifies data elements unique to that data format. +::: diff --git a/src/courses/mappers/05.md b/src/courses/mappers/05.md new file mode 100644 index 000000000..27e040282 --- /dev/null +++ b/src/courses/mappers/05.md @@ -0,0 +1,343 @@ +--- +order: 5 +next: 06.md +title: OHDF Schema Basics +author: Charles Hu +--- + +## A Look Ahead + +In this section, we will cover: + +- [Core structures of the OHDF schema:](#an-overview-of-the-ohdf-schema) + - Exec JSON + - Profiles + - Controls + - Results +- [A look at the overall OHDF schema hierarchy](#overall-hierarchy) + +## An Overview of the OHDF Schema + +In order to make an OHDF mapper, it is important to understand the specifics of how OHDF is actually structured through the OHDF schema. + +The OHDF schema is designed to provide a simple, structured, and hierarchal view of security tool results. OHDF actually consists of several related schema formats; however, the Exec JSON format is the most common and is typically the one being referred to when speaking of the OHDF schema. Any file or object that implements the schema can be broken down into a hierarchy of four main object structures. These structures are: + +### 1. **Exec JSON** + +This is the top level object structure that contains the security data (which is stored in the Profiles structure and related substructures), relevant metadata on the tool used to generate the OHDF file itself (stored in structures such as Platform, Version, and Statistics), and miscellaneous information that does not neatly fit in the previous structures (stored in the Passthrough structure). + +Before we discuss the Profiles structure and its related substructures, here is a brief introduction to the latter object structures contained by the Exec JSON superstructure: + +- **Platform**: This structure contains metadata involving the platform or tool that is producing the OHDF file. + +- **Version**: This structure contains metadata involving the version of the platform or tool that is producing the OHDF file. + +- **Statistics**: This structure contains metadata involving the specific statistics of the original run that produced the export. + +- **Passthrough**: The primary purpose of the Passthrough structure is to contain and transport any piece of information that cannot be properly stored in the other Exec JSON substructures. Unlike other object structures in the OHDF schema, there is no formal schema declaration for the specific fields that exist in Passthrough. This means that Passthrough is fluid and capable of accommodating user-defined object fields as needed. There are, however, commonly agreed upon fields for specific capabilities, such as data preservation, that are discussed further in the next section. + +A basic representation of this structure in JSON is as so: + +```json +{ + execjson: { + platform: {...}, + version: ..., + statistics: {...}, + profiles: [...], + passthrough: {...} + } +} +``` + +### 2. **Profiles** + +This structure contains metadata on the target system of the original security tool export and on the specific run performed by the security tool. Through this, the Profiles structure provides a high-level overview of the security tool result and the targeted system, which are formatted in a manner that is digestible and immediately accessible to the assessor. There is typically only one Profiles instance in a single Exec JSON structure (represented as a single element in an array of Profiles). Further instances of Profiles are additional overlays. + +A basic representation of this structure in JSON is as so: + +```json +{ + execjson: { + profiles: [...], + ... + } +} +``` + +### 3. **Controls** + +Controls are security requirements that are used to prevent, mitigate, and address various security risks to sensitive information, systems, and infrastructure. In the case of OHDF, the Controls structure is a collection of requirements tested for by an external security tool to ensure that the target complies with vulnerability and weakness prevention standards. Any given Profiles structure contains some number of Controls structures (i.e., an array of Controls) which were tested against the target system during the original security tool run. + +A basic representation of this structure in JSON is as so: + +```json +{ + profiles: [ + { + controls: [...], + ... + }, + ... + ], + ... +} +``` + +### 4. **Results** + +The Results structure contains information on the results of specific tests ran by the security tool on the target system against a single security requirement/control. These results will always correlate to a certain instance of a Controls structure and can report either `passed`, `failed`, `skipped`, or `error` to indicate the test status. Any given Controls structure contains some number of Results structures (i.e., an array of Results) which reflect the implemented tests to check if the target system is actually meeting the overall requirement/control. The test statuses of these results cumulatively influence the determined compliance level of the target system for some set of requirements/controls. + +A basic representation of this structure in JSON is as so: + +```json +{ + controls: [ + { + results: [...], + ... + }, + ... + ], + ... +} +``` + +## Overall Hierarchy + +These aforementioned main object structures cumulatively result in the following general structure: + +```json +{ + execjson: { + platform: {...}, + version: ..., + statistics: {...}, + profiles: [ + { + controls: [ + { + results: [...], + ... + }, + ... + ], + ... + }, + ... + ], + passthrough: {...} + } +} +``` + +## A Look Back + +In this section, we covered: + +- [Core structures of the OHDF schema:](#an-overview-of-the-ohdf-schema) + + - Exec JSON + + - This is the top level object structure that acts as a container for all of the other object structures that are specialized for specific sets of information. + + - Profiles + + - This structure is for containing high-level metadata on the security tool run. + + - Controls + + - This structure is for containing information pertaining to the set of controls/requirements that were ran against the target system in the original security tool run. + + - Results + + - This structure is for containing information pertaining to the specific test results of the application of the control/requirement set against the target system. + +- [A look at the overall OHDF schema hierarchy](#overall-hierarchy) + +### Knowledge Check + +:::details Describe the relationship between Profiles, Controls, and Results. + +The Profiles object structure contains one or more Controls structures (represented as a `controls` array within a `profiles` element). + +The Controls object structure contains one or more Results structures (represented as a `results` array within a `controls` element). +::: + +:::details Say that we map from Format A to OHDF and have data elements left over in Format A that are still uncorrelated. What do we do with those data elements? +We perform data preservation and place the uncorrelated data elements from Format A into the Passthrough structure in OHDF (the explicit field to use will be defined later). +::: + +### Knowledge Review + +Now that we're familiar with the general layout of an OHDF file, let's briefly look at an actual OHDF file generated from a Twistlock security scan export. This OHDF file in particular was created through the OHDF Converters library embedded in the Heimdall visualizer. + +Since we have not yet gone over the specific OHDF schema fields, try to focus on the actual information being stored by file instead and ask yourself: + +- What is the purpose of this specific piece of data? What is it trying to convey (file metadata, vulnerability alerts, check results, etc.)? + +- Does this piece of data fit in with the general purpose of the structure that contains it? + +- Would this piece of data fit better in another structure? + +:::note Twistlock-to-OHDF Sample +This file has been truncated for pedagogical purposes. For the full file, see [here](https://github.com/mitre/heimdall2/blob/master/libs/hdf-converters/sample_jsons/twistlock_mapper/twistlock-hdf.json) +::: + +:::details Twistlock-to-OHDF Sample + +```json +{ + "platform": { + "name": "Heimdall Tools", + "release": "2.6.29", + "target_id": "registry.io/test:1a7a431a105aa04632f5f5fbe8f753bd245add0a" + }, + "version": "2.6.29", + "statistics": {}, + "profiles": [ + { + "name": "Twistlock Scan", + "title": "Twistlock Project: All / TEST-COLLECTION", + "summary": "Package Vulnerability Summary: 97 Application Compliance Issue Total: 0", + "supports": [], + "attributes": [], + "groups": [], + "status": "loaded", + "controls": [ + { + "tags": { + "nist": ["SI-2", "RA-5"], + "cci": ["CCI-002605", "CCI-001643"], + "cveid": "CVE-2021-43529" + }, + "refs": [], + "source_location": {}, + "title": "CVE-2021-43529", + "id": "CVE-2021-43529", + "desc": "DOCUMENTATION: A remote code execution flaw was found in the way NSS verifies certificates. This flaw allows an attacker posing as an SSL/TLS server to trigger this issue in a client application compiled with NSS when it tries to initiate an SSL/TLS connection. Similarly, a server application compiled with NSS, which processes client certificates, can receive a malicious certificate via a client, triggering the flaw. The highest threat to this vulnerability is confidentiality, integrity, as well as system availability. STATEMENT: The issue is not limited to TLS. Any applications that use NSS certificate verification are vulnerable; S/MIME is impacted as well. Similarly, a server application compiled with NSS, which processes client certificates, can receive a malicious certificate via a client. Firefox is not vulnerable to this flaw as it uses the mozilla::pkix for certificate verification. Thunderbird is affected when parsing email with the S/MIME signature. Thunderbird on Red Hat Enterprise Linux 8.4 and later does not need to be updated since it uses the system NSS library, but earlier Red Hat Enterprise Linux 8 extended life streams will need to update Thunderbird as well as NSS. MITIGATION: Red Hat has investigated whether a possible mitigation exists for this issue, and has not been able to identify a practical example. Please update the affec", + "impact": 0.9, + "code": "{\n \"id\": \"CVE-2021-43529\",\n \"status\": \"affected\",\n \"cvss\": 9.8,\n \"description\": \"DOCUMENTATION: A remote code execution flaw was found in the way NSS verifies certificates. This flaw allows an attacker posing as an SSL/TLS server to trigger this issue in a client application compiled with NSS when it tries to initiate an SSL/TLS connection. Similarly, a server application compiled with NSS, which processes client certificates, can receive a malicious certificate via a client, triggering the flaw. The highest threat to this vulnerability is confidentiality, integrity, as well as system availability. STATEMENT: The issue is not limited to TLS. Any applications that use NSS certificate verification are vulnerable; S/MIME is impacted as well. Similarly, a server application compiled with NSS, which processes client certificates, can receive a malicious certificate via a client. Firefox is not vulnerable to this flaw as it uses the mozilla::pkix for certificate verification. Thunderbird is affected when parsing email with the S/MIME signature. Thunderbird on Red Hat Enterprise Linux 8.4 and later does not need to be updated since it uses the system NSS library, but earlier Red Hat Enterprise Linux 8 extended life streams will need to update Thunderbird as well as NSS. MITIGATION: Red Hat has investigated whether a possible mitigation exists for this issue, and has not been able to identify a practical example. Please update the affec\",\n \"severity\": \"critical\",\n \"packageName\": \"nss-util\",\n \"packageVersion\": \"3.67.0-7.el8_5\",\n \"link\": \"https://access.redhat.com/security/cve/CVE-2021-43529\",\n \"riskFactors\": [\n \"Remote execution\",\n \"Attack complexity: low\",\n \"Attack vector: network\",\n \"Critical severity\",\n \"Recent vulnerability\"\n ],\n \"impactedVersions\": [\n \"*\"\n ],\n \"publishedDate\": \"2021-12-01T00:00:00Z\",\n \"discoveredDate\": \"2022-05-18T12:24:22Z\",\n \"layerTime\": \"2022-05-16T23:12:25Z\"\n}", + "results": [ + { + "status": "failed", + "code_desc": "Package \"nss-util\" should be updated to latest version above impacted versions [\"*\"]", + "message": "Expected latest version of \"nss-util\"\nDetected vulnerable version \"3.67.0-7.el8_5\" of \"nss-util\"", + "start_time": "2022-05-18T12:24:22Z" + }, + { + "status": "failed", + "code_desc": "Package \"nss-sysinit\" should be updated to latest version above impacted versions [\"*\"]", + "message": "Expected latest version of \"nss-sysinit\"\nDetected vulnerable version \"3.67.0-7.el8_5\" of \"nss-sysinit\"", + "start_time": "2022-05-18T12:24:22Z" + }, + { + "status": "failed", + "code_desc": "Package \"nss\" should be updated to latest version above impacted versions [\"*\"]", + "message": "Expected latest version of \"nss\"\nDetected vulnerable version \"3.67.0-7.el8_5\" of \"nss\"", + "start_time": "2022-05-18T12:24:22Z" + }, + { + "status": "failed", + "code_desc": "Package \"nss-softokn\" should be updated to latest version above impacted versions [\"*\"]", + "message": "Expected latest version of \"nss-softokn\"\nDetected vulnerable version \"3.67.0-7.el8_5\" of \"nss-softokn\"", + "start_time": "2022-05-18T12:24:22Z" + }, + { + "status": "failed", + "code_desc": "Package \"nss-softokn-freebl\" should be updated to latest version above impacted versions [\"*\"]", + "message": "Expected latest version of \"nss-softokn-freebl\"\nDetected vulnerable version \"3.67.0-7.el8_5\" of \"nss-softokn-freebl\"", + "start_time": "2022-05-18T12:24:22Z" + } + ] + } + ], + "sha256": "807c8ef1534bb7f3e428db4a40667a6dd37d89f8a48dc6d1f6bb2426ff53f97f" + } + ], + "passthrough": { + "auxiliary_data": [ + { + "name": "Twistlock", + "data": { + "results": [ + { + "id": "sha256:b1c0237ebd29860066656372da10d8d7da33b34715986f74b3d5a7e4ba060d1b", + "distro": "Red Hat Enterprise Linux release 8.6 (Ootpa)", + "distroRelease": "RHEL8", + "digest": "sha256:c0274cb1e0c6e92d2ccc1e23bd19b7dddedbaa2da26861c225d4ad6cec5047f4", + "packages": [ + { + "type": "os", + "name": "nss-util", + "version": "3.67.0-7.el8_5", + "licenses": ["MPLv2.0"] + }, + { + "type": "os", + "name": "nss-softokn", + "version": "3.67.0-7.el8_5", + "licenses": ["MPLv2.0"] + }, + { + "type": "os", + "name": "nss", + "version": "3.67.0-7.el8_5", + "licenses": ["MPLv2.0"] + }, + { + "type": "os", + "name": "nss-softokn-freebl", + "version": "3.67.0-7.el8_5", + "licenses": ["MPLv2.0"] + }, + { + "type": "os", + "name": "nss-sysinit", + "version": "3.67.0-7.el8_5", + "licenses": ["MPLv2.0"] + } + ], + "applications": [ + { + "name": "asp.net-core", + "version": "5.0.17", + "path": "/usr/lib64/dotnet/dotnet" + }, + { + "name": ".net-core", + "version": "5.0.17", + "path": "/usr/lib64/dotnet/dotnet" + } + ], + "complianceScanPassed": true, + "vulnerabilityScanPassed": true, + "history": [ + { + "created": "2022-05-03T08:38:31Z" + }, + { + "created": "2022-05-03T08:39:27Z" + }, + { + "created": "2022-05-16T23:12:02Z", + "instruction": "ARG GOPROXY=http://nexus-repository-manager.nexus-repository-manager.svc.cluster.local:8081/repository/goproxy/ HTTP_PROXY=http://localhost:3128 http_proxy=http://localhost:3128" + }, + { + "created": "2022-05-16T23:12:02Z", + "instruction": "ARG GOPROXY=http://nexus-repository-manager.nexus-repository-manager.svc.cluster.local:8081/repository/goproxy/ GOSUMDB=sum.golang.org http://nexus-repository-manager.nexus-repository-manager.svc.cluster.local:8081/repository/gosum HTTP_PROXY=http://localhost:3128 http_proxy=http://localhost:3128" + } + ], + "scanTime": "2022-05-18T12:24:32.855444532Z", + "scanID": "6284e580d9600f8d0db159e2" + } + ], + "consoleURL": "https://twistlock.test.net/#!/monitor/vulnerabilities/images/ci?search=sha256%3Ab1c0237ebd29860066656372da10d8d7da33b34715986f74b3d5a7e4ba060d1b" + } + } + ] + } +} +``` + +::: diff --git a/src/courses/mappers/06.md b/src/courses/mappers/06.md new file mode 100644 index 000000000..089b13bca --- /dev/null +++ b/src/courses/mappers/06.md @@ -0,0 +1,742 @@ +--- +order: 6 +next: 07.md +title: Formal OHDF Schema Definition +author: Charles Hu +--- + +## A Look Ahead + +In this section, we will cover: + +- [Background context for technical aspects of the OHDF schema](#background) +- [A breakdown of the OHDF schema](#breaking-down-the-formal-ohdf-schema): + - [Exec JSON](#exec-json) + - [Platform](#platform) + - [Statistics](#statistics) + - [Passthrough](#passthrough) + - [Profiles](#profiles) + - [Controls](#controls) + - [Results](#results) + +## Background + +The OHDF schema is a JSON-based schema that uses typed values. This means that any given OHDF file must be formatted in JSON and the typing of each key-value pair in the JSON object must conform to its declared type in the OHDF schema. Nonconformance to these rules will result in an OHDF output that is incompatible with compliant OHDF-based functionality and will produce undefined and erroneous behavior. + +### Unused Fields + +:::info Required Fields +A required field in OHDF is a schema field that must always exist in the file to qualify the file as conforming to the OHDF schema. If a field is required by the OHDF schema, it does not necessarily mean that you have to provide it with meaningful data or correlate it with a data element from another source format. A file can still conform to OHDF by having a required field that has an empty value assigned to it. +::: + +Not all fields in the schema have to be filled or even used in an OHDF mapper. In practice, most implementations either provide an empty value (e.g, an empty string, an empty object, etc.) or omit an unused field entirely (i.e., by not defining the key or by passing a value of `undefined`). Whether or not you should provide an empty value or omit the key for an unused field depends on if the field is declared as required by the OHDF schema. If you have an unused field that is declared as required by the schema, you must define the key and pass it an empty value. If the unused field is not declared as required, you can safely omit the field. + +:::note Omitting Schema Fields +Just because you can omit a schema field does not mean that you should! + +Always aim to maximize the filled fields in your mapper and only skip a field as a last resort if no applicable equivalent exists in your source data. Consider required fields as priority fields that you should aim to fill out first instead of the bare minimum necessary to create a mapper. +::: + +For example, the following required key-value pairs are all considered valid: + +```json +{ + "platform": { + "name": "" + } +} +``` + +```json +{ + "profiles": [ + { + "supports": [] + } + ] +} +``` + +Here is a snippet from the previous Twistlock-to-OHDF sample file which shows some empty required fields: + +:::details Twistlock-to-OHDF Sample + +```json +{ + "statistics": {}, + "profiles": [ + { + "name": "Twistlock Scan", + "title": "Twistlock Project: All / TEST-COLLECTION", + "summary": "Package Vulnerability Summary: 97 Application Compliance Issue Total: 0", + "supports": [], + "attributes": [], + "groups": [], + "status": "loaded", + "sha256": "807c8ef1534bb7f3e428db4a40667a6dd37d89f8a48dc6d1f6bb2426ff53f97f" + } + ] +} +``` + +::: + +## Breaking Down the Formal OHDF Schema + +We can break down the formal OHDF schema in the same manner as before by observing each structure in the hierarchy from a top-down view. + +::: note Full OHDF Schema +The following section contains a breakdown of a streamlined version of the OHDF schema. + +For the full and up-to-date version, see [here](https://saf.mitre.org/framework/normalize/ohdf-schema). + +For the technical implementation of the schema, see [here](https://github.com/mitre/heimdall2/blob/master/libs/inspecjs/src/generated_parsers/v_1_0/exec-json.ts). +::: + +### **Exec JSON** + +| Field Name | Definition | Type | Required | +| ------------- | ---------------------------------------------------------------------------- | ------------------------- | -------- | +| `platform` | Platform substructure (see [**Platform**](#platform)). | `Platform` | Yes | +| `version` | Version of tool generating the file. | `string` | Yes | +| `statistics` | Statistics substructure (see [**Statistics**](#statistics)). | `Statistics` | Yes | +| `profiles` | Profiles substructure (see [**Profiles**](#profiles)). | `ExecJSONProfile[]` | Yes | +| `passthrough` | Passthrough substructure (see [**Passthrough**](#passthrough)) | `Record` | No | + +A basic representation of this structure in JSON is as so: + +```json +{ + "platform": {}, + "version": "", + "statistics": {}, + "profiles": [], + "passthrough": {} +} +``` + +### **Profiles** + +| Field Name | Definition | Type | Required | +| ----------------- | ------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------ | -------- | +| `name` | Name of profile, usually the original security tool. Must be unique. | `string` | Yes | +| `version` | Version of security tool. | `string`, `null` | No | +| `sha256` | Checksum of the profile (NOTE: AUTOMATICALLY GENERATED BY HDF CONVERTERS, DO NOT POPULATE). | `string` | Yes | +| `title` | Title of security tool; must be human readable. | `string`, `null` | No | +| `maintainer` | Maintainer(s). | `string`, `null` | No | +| `summary` | Summary of security data (e.g., the STIG header). | `string`, `null` | No | +| `license` | Copyright license. | `string`, `null` | No | +| `copyright` | Copyright holder. | `string`, `null` | No | +| `copyright_email` | Copyright holder's email. | `string`, `null` | No | +| `supports` | Supported platform targets. | [`SupportedPlatform[]`](https://github.com/mitre/heimdall2/blob/master/libs/inspecjs/src/generated_parsers/v_1_0/exec-json.ts#L396) | Yes | +| `attributes` | Inputs/attributes used in scan. | `Record[]` | Yes | +| `groups` | Set of descriptions for the control groups (e.g., control IDs). | [`ControlGroup[]`](https://github.com/mitre/heimdall2/blob/master/libs/inspecjs/src/generated_parsers/v_1_0/exec-json.ts#L377) | Yes | +| `controls` | Controls substructure (see [**Controls**](#controls)). | `ExecJSONControl[]` | Yes | +| `status` | Status of profile (typically 'loaded'). | `string`, `null` | No | +| `status_message` | Reason for the status. | `string`, `null` | No | +| `skip_message` | Reason for skipping this profile if skipped. | `string`, `null` | No | +| `depends` | Set of dependencies this profile depends on (e.g., an overlay profile is dependent on another profile). | [`Dependency[]`](https://github.com/mitre/heimdall2/blob/master/libs/inspecjs/src/generated_parsers/v_1_0/exec-json.ts#L334), `null` | No | +| `parent_profile` | Name of the parent profile if this profile is a dependency of another profile. | `string`, `null` | No | +| `description` | Description of profile; should be more detailed than summary. | `string`, `null` | No | +| `inspec_version` | The version of the Inspec tool used to generate this profile. | `string`, `null` | No | + +A basic representation of this structure in JSON is as so: + +```json +{ + "name": "", + "version": "", + "sha256": "", + "title": "", + "maintainer": "", + "summary": "", + "license": "", + "copyright": "", + "copyright_email": "", + "supports": [], + "attributes": [], + "groups": [], + "controls": [], + "status": "", + "status_message": "", + "skip_message": "", + "depends": [], + "parent_profile": "", + "description": "", + "inspec_version": "" +} +``` + +#### Knowledge Check + +Given a snippet of data, try and identify what information is applicable to the Profiles structure and which fields they would correlate with. + +- Data snippet: + +:::details GoSec Source Data + +```json +{ + "Golang errors": {}, + "Issues": [ + { + "severity": "MEDIUM", + "confidence": "HIGH", + "cwe": { + "id": "22", + "url": "https://cwe.mitre.org/data/definitions/22.html" + }, + "rule_id": "G304", + "details": "Potential file inclusion via variable", + "file": "C:\\Users\\AGILLUM\\OneDrive - The MITRE Corporation\\Documents\\Code\\grype-0.34.4\\internal\\file\\tar.go", + "code": "82: \t\tcase tar.TypeReg:\n83: \t\t\tf, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))\n84: \t\t\tif err != nil {\n", + "line": "83", + "column": "14", + "nosec": false, + "suppressions": null + } + ], + "Stats": { + "files": 199, + "lines": 12401, + "nosec": 0, + "found": 7 + }, + "GosecVersion": "dev" +} +``` + +::: + +- Converted OHDF snippet (may not perfectly align with your answers): + +:::details OHDF Converted GoSec Data + +```json +"profiles": [ + { + "name": "Gosec scanner", + "title": "gosec", + "version": "dev", + "supports": [], + "attributes": [], + "groups": [], + "status": "loaded", + "controls": [ + ... + ], + "sha256": "d0506f4b7715bf8b1cd81a8a87ab8efcce41ebeb2b5ec5fcfb23d3bdd136f48c" + } +] +``` + +::: + +### **Controls** + +| Field Name | Definition | Type | Required | +| ------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------- | -------- | +| `id` | ID of control, used for sorting. Must be unique. | `string` | Yes | +| `title` | Title of control. | `string`, `null` | No | +| `desc` | The overarching description of the control. | `string`, `null` | No | +| `descriptions` | Additional descriptions; usually 'check' and 'fix' text for control. | [`ControlDescription[]`](https://github.com/mitre/heimdall2/blob/master/libs/inspecjs/src/generated_parsers/v_1_0/exec-json.ts#L224), `null` | No | +| `impact` | Security severity of control. On a scale of 0 to 1 and typically increments by 0.1 values. This metric is aligned with the [Common Vulnerability Scoring System (CVSS)](https://www.first.org/cvss/). | `number` | Yes | +| `refs` | References to external control documentation. | [`Reference[]`](https://github.com/mitre/heimdall2/blob/master/libs/inspecjs/src/generated_parsers/v_1_0/exec-json.ts#L244) | Yes | +| `tags` | Control tags; typically correlate to existing vulnerability/weakness database (e.g., NIST, CVE, CWE). | `Record` | Yes | +| `code` | Control source code for code preservation. | `string`, `null` | No | +| `source_location` | Location of control within source code. | [`SourceLocation`](https://github.com/mitre/heimdall2/blob/master/libs/inspecjs/src/generated_parsers/v_1_0/exec-json.ts#L311) | Yes | +| `results` | Results substructure (see [**Results**](#results)). | `ControlResult[]` | Yes | +| `waiver_data` | Information on waiver if applicable. | [`WaiverData`](https://github.com/mitre/heimdall2/blob/master/libs/inspecjs/src/generated_parsers/v_1_0/exec-json.ts#L322), `null` | No | +| `attestation_data` | Information on attestation if applicable. | [`AttestationData`](https://github.com/mitre/heimdall2/blob/master/libs/inspecjs/src/generated_parsers/v_1_0/exec-json.ts#L204), `null` | No | + +A basic representation of this structure in JSON is as so: + +```json +{ + "id": "", + "title": "", + "desc": "", + "descriptions": [], + "impact": 0, + "refs": [], + "tags": {}, + "code": "", + "source_location": {}, + "results": [], + "waiver_data": {}, + "attestation_data": {} +} +``` + +#### Knowledge Check + +Given a snippet of data, try and identify what information is applicable to the Controls structure and which fields they would correlate with. + +- Data snippet: + +:::details GoSec Source Data + +```json +{ + "Golang errors": {}, + "Issues": [ + { + "severity": "MEDIUM", + "confidence": "HIGH", + "cwe": { + "id": "22", + "url": "https://cwe.mitre.org/data/definitions/22.html" + }, + "rule_id": "G304", + "details": "Potential file inclusion via variable", + "file": "C:\\Users\\AGILLUM\\OneDrive - The MITRE Corporation\\Documents\\Code\\grype-0.34.4\\internal\\file\\tar.go", + "code": "82: \t\tcase tar.TypeReg:\n83: \t\t\tf, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))\n84: \t\t\tif err != nil {\n", + "line": "83", + "column": "14", + "nosec": false, + "suppressions": null + } + ], + "Stats": { + "files": 199, + "lines": 12401, + "nosec": 0, + "found": 7 + }, + "GosecVersion": "dev" +} +``` + +::: + +- Converted OHDF snippet (may not perfectly align with your answers): + +:::details OHDF Converted GoSec Data + +```json +"controls": [ + { + "tags": { + "nist": ["SI-10"], + "cwe": { + "id": "22", + "url": "https://cwe.mitre.org/data/definitions/22.html" + }, + "nosec": "", + "suppressions": "", + "severity": "MEDIUM", + "confidence": "HIGH" + }, + "refs": [], + "source_location": {}, + "title": "Potential file inclusion via variable", + "id": "G304", + "desc": "", + "impact": 0.5, + "results": [ + ... + ] + } +], +``` + +::: + +### **Results** + +| Field Name | Definition | Type | Required | +| -------------- | --------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | -------- | +| `status` | `passed`/`failed` status of test (other status include `skipped` and `error`). | [`ControlResultStatus`](https://github.com/mitre/heimdall2/blob/master/libs/inspecjs/src/generated_parsers/v_1_0/exec-json.ts#L299), `null` | No | +| `skip_message` | Reasoning for if test has `skipped` status. | `string`, `null` | No | +| `code_desc` | Test expectations as defined by control. | `string` | Yes | +| `exception` | Type of exception if one was thrown. | `string`, `null` | No | +| `backtrace` | Backtrace of exception if one was thrown. | `string[]`, `null` | No | +| `message` | Explanation of test status; typically in the form of expected and actual results. | `string`, `null` | No | +| `resource` | Resources used in the test (e.g., Inspec). | `string`, `null` | No | +| `resource_id` | The unique ID of the used resources. | `string`, `null` | No | +| `run_time` | Overall runtime of test. | `number`, `null` | No | +| `start_time` | Starting time of test. | `string` | Yes | + +A basic representation of this structure in JSON is as so: + +```json +{ + "status": {}, + "skip_message": "", + "code_desc": "", + "exception": "", + "backtrace": [], + "message": "", + "resource": "", + "resource_id": "", + "run_time": 0, + "start_time": "" +} +``` + +#### Knowledge Review + +Given a snippet of data, try and identify what information is applicable to the Results structure and which fields they would correlate with. + +- Data snippet: + +:::details GoSec Source Data + +```json +{ + "Golang errors": {}, + "Issues": [ + { + "severity": "MEDIUM", + "confidence": "HIGH", + "cwe": { + "id": "22", + "url": "https://cwe.mitre.org/data/definitions/22.html" + }, + "rule_id": "G304", + "details": "Potential file inclusion via variable", + "file": "C:\\Users\\AGILLUM\\OneDrive - The MITRE Corporation\\Documents\\Code\\grype-0.34.4\\internal\\file\\tar.go", + "code": "82: \t\tcase tar.TypeReg:\n83: \t\t\tf, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))\n84: \t\t\tif err != nil {\n", + "line": "83", + "column": "14", + "nosec": false, + "suppressions": null + } + ], + "Stats": { + "files": 199, + "lines": 12401, + "nosec": 0, + "found": 7 + }, + "GosecVersion": "dev" +} +``` + +::: + +- Converted OHDF snippet (may not perfectly align with your answers): + +:::details OHDF Converted GoSec Data + +```json +"results": [ + { + "status": "failed", + "code_desc": "82: \t\tcase tar.TypeReg:\n83: \t\t\tf, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))\n84: \t\t\tif err != nil {\n", + "message": "C:\\Users\\AGILLUM\\OneDrive - The MITRE Corporation\\Documents\\Code\\grype-0.34.4\\internal\\file\\tar.go, line:83, column:14", + "start_time": "" + } +] +``` + +::: + +### **Platform** + +| Field Name | Definition | Type | Required | +| ----------- | ----------------------------------------------------------------------- | ---------------- | -------- | +| `name` | Name of platform export was run on. | `string` | Yes | +| `release` | Platform version. | `string` | Yes | +| `target_id` | Platform target ID (i.e., further identifying information on platform). | `string`, `null` | No | + +A basic representation of this structure in JSON is as so: + +```json +{ + "name": "", + "release": "", + "target_id": "" +} +``` + +### **Statistics** + +| Field Name | Definition | Type | Required | +| ---------- | -------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------- | -------- | +| `controls` | Break down of control statistics by result status. | [`StatisticHash`](https://github.com/mitre/heimdall2/blob/master/libs/inspecjs/src/generated_parsers/v_1_0/exec-json.ts#L442), `null` | No | +| `duration` | Duration of run (in seconds). | `number`, `null` | No | + +A basic representation of this structure in JSON is as so: + +```json +{ + "controls": {}, + "duration": 0 +} +``` + +### **Passthrough** + +There is no formal schema definition for the Passthrough structure. As such, any key-value pair can technically be used in this structure and still be compliant with the OHDF schema. There are, however, some common field naming conventions for certain types of data that are typically placed in Passthrough as defined below: + +| Field Name | Definition | Type | Required | +| ----------------------- | --------------------------------------------- | --------------------------- | ---------- | +| `auxiliary_data` | Storage for unused data from the sample file. | `Record[]` | Undeclared | +| `auxiliary_data[].name` | Name of auxiliary data source. | `string` | Undeclared | +| `auxiliary_data[].data` | Auxiliary data itself. | `Record` | Undeclared | +| `raw` | Raw data dump of input file. | `Record` | Undeclared | + +A basic representation of this structure in JSON is as so: + +```json +{ + "auxiliary_data": [ + { + "name": "", + "data": {} + } + ], + "raw": {} +} +``` + +#### Knowledge Check + +Given a snippet of data, try and identify what information is applicable to the Exec JSON, Platform, Statistics, and Passthrough structures and which fields they would correlate with. + +- Data snippet: + +:::details GoSec Source Data + +```json +{ + "Golang errors": {}, + "Issues": [ + { + "severity": "MEDIUM", + "confidence": "HIGH", + "cwe": { + "id": "22", + "url": "https://cwe.mitre.org/data/definitions/22.html" + }, + "rule_id": "G304", + "details": "Potential file inclusion via variable", + "file": "C:\\Users\\AGILLUM\\OneDrive - The MITRE Corporation\\Documents\\Code\\grype-0.34.4\\internal\\file\\tar.go", + "code": "82: \t\tcase tar.TypeReg:\n83: \t\t\tf, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))\n84: \t\t\tif err != nil {\n", + "line": "83", + "column": "14", + "nosec": false, + "suppressions": null + } + ], + "Stats": { + "files": 199, + "lines": 12401, + "nosec": 0, + "found": 7 + }, + "GosecVersion": "dev" +} +``` + +::: + +- Converted OHDF snippet (may not perfectly align with your answers): + +:::details OHDF Converted GoSec Data + +```json +{ + "platform": { "name": "Heimdall Tools", "release": "2.10.8" }, + "version": "2.10.8", + "statistics": {}, + "profiles": [], + "passthrough": { + "auxiliary_data": [{ "name": "Gosec", "data": { "Golang errors": {} } }] + } +} +``` + +::: + +## A Look Back + +In this section, we covered: + +- [Background context for technical aspects of the OHDF schema](#background) + + - Certain unused/uncorrelated fields in the final OHDF mapping can be omitted/left empty depending on whether the field is qualified as required. + +- [A breakdown of the OHDF schema](#breaking-down-the-formal-ohdf-schema): + - [Exec JSON](#exec-json) + - [Platform](#platform) + - [Statistics](#statistics) + - [Passthrough](#passthrough) + - [Profiles](#profiles) + - [Controls](#controls) + - [Results](#results) + +### Knowledge Review + +Now that we're familiar with the OHDF schema, let's revisit that Twistlock-to-OHDF sample with a specific focus on the fields that the data is assigned to. + +Here are some key questions to ask yourself while reading: + +- Does the information assigned to the schema field make sense? + +- Does the definition of the schema field match what the assigned data is trying to convey? + +- Would the data assigned to the schema field be more applicable elsewhere? + +:::note Twistlock-to-OHDF Sample +This file has been truncated for pedagogical purposes. For the full file, see [here](https://github.com/mitre/heimdall2/blob/master/libs/hdf-converters/sample_jsons/twistlock_mapper/twistlock-hdf.json) +::: + +:::details Twistlock-to-OHDF Sample + +```json +{ + "platform": { + "name": "Heimdall Tools", + "release": "2.6.29", + "target_id": "registry.io/test:1a7a431a105aa04632f5f5fbe8f753bd245add0a" + }, + "version": "2.6.29", + "statistics": {}, + "profiles": [ + { + "name": "Twistlock Scan", + "title": "Twistlock Project: All / TEST-COLLECTION", + "summary": "Package Vulnerability Summary: 97 Application Compliance Issue Total: 0", + "supports": [], + "attributes": [], + "groups": [], + "status": "loaded", + "controls": [ + { + "tags": { + "nist": ["SI-2", "RA-5"], + "cci": ["CCI-002605", "CCI-001643"], + "cveid": "CVE-2021-43529" + }, + "refs": [], + "source_location": {}, + "title": "CVE-2021-43529", + "id": "CVE-2021-43529", + "desc": "DOCUMENTATION: A remote code execution flaw was found in the way NSS verifies certificates. This flaw allows an attacker posing as an SSL/TLS server to trigger this issue in a client application compiled with NSS when it tries to initiate an SSL/TLS connection. Similarly, a server application compiled with NSS, which processes client certificates, can receive a malicious certificate via a client, triggering the flaw. The highest threat to this vulnerability is confidentiality, integrity, as well as system availability. STATEMENT: The issue is not limited to TLS. Any applications that use NSS certificate verification are vulnerable; S/MIME is impacted as well. Similarly, a server application compiled with NSS, which processes client certificates, can receive a malicious certificate via a client. Firefox is not vulnerable to this flaw as it uses the mozilla::pkix for certificate verification. Thunderbird is affected when parsing email with the S/MIME signature. Thunderbird on Red Hat Enterprise Linux 8.4 and later does not need to be updated since it uses the system NSS library, but earlier Red Hat Enterprise Linux 8 extended life streams will need to update Thunderbird as well as NSS. MITIGATION: Red Hat has investigated whether a possible mitigation exists for this issue, and has not been able to identify a practical example. Please update the affec", + "impact": 0.9, + "code": "{\n \"id\": \"CVE-2021-43529\",\n \"status\": \"affected\",\n \"cvss\": 9.8,\n \"description\": \"DOCUMENTATION: A remote code execution flaw was found in the way NSS verifies certificates. This flaw allows an attacker posing as an SSL/TLS server to trigger this issue in a client application compiled with NSS when it tries to initiate an SSL/TLS connection. Similarly, a server application compiled with NSS, which processes client certificates, can receive a malicious certificate via a client, triggering the flaw. The highest threat to this vulnerability is confidentiality, integrity, as well as system availability. STATEMENT: The issue is not limited to TLS. Any applications that use NSS certificate verification are vulnerable; S/MIME is impacted as well. Similarly, a server application compiled with NSS, which processes client certificates, can receive a malicious certificate via a client. Firefox is not vulnerable to this flaw as it uses the mozilla::pkix for certificate verification. Thunderbird is affected when parsing email with the S/MIME signature. Thunderbird on Red Hat Enterprise Linux 8.4 and later does not need to be updated since it uses the system NSS library, but earlier Red Hat Enterprise Linux 8 extended life streams will need to update Thunderbird as well as NSS. MITIGATION: Red Hat has investigated whether a possible mitigation exists for this issue, and has not been able to identify a practical example. Please update the affec\",\n \"severity\": \"critical\",\n \"packageName\": \"nss-util\",\n \"packageVersion\": \"3.67.0-7.el8_5\",\n \"link\": \"https://access.redhat.com/security/cve/CVE-2021-43529\",\n \"riskFactors\": [\n \"Remote execution\",\n \"Attack complexity: low\",\n \"Attack vector: network\",\n \"Critical severity\",\n \"Recent vulnerability\"\n ],\n \"impactedVersions\": [\n \"*\"\n ],\n \"publishedDate\": \"2021-12-01T00:00:00Z\",\n \"discoveredDate\": \"2022-05-18T12:24:22Z\",\n \"layerTime\": \"2022-05-16T23:12:25Z\"\n}", + "results": [ + { + "status": "failed", + "code_desc": "Package \"nss-util\" should be updated to latest version above impacted versions [\"*\"]", + "message": "Expected latest version of \"nss-util\"\nDetected vulnerable version \"3.67.0-7.el8_5\" of \"nss-util\"", + "start_time": "2022-05-18T12:24:22Z" + }, + { + "status": "failed", + "code_desc": "Package \"nss-sysinit\" should be updated to latest version above impacted versions [\"*\"]", + "message": "Expected latest version of \"nss-sysinit\"\nDetected vulnerable version \"3.67.0-7.el8_5\" of \"nss-sysinit\"", + "start_time": "2022-05-18T12:24:22Z" + }, + { + "status": "failed", + "code_desc": "Package \"nss\" should be updated to latest version above impacted versions [\"*\"]", + "message": "Expected latest version of \"nss\"\nDetected vulnerable version \"3.67.0-7.el8_5\" of \"nss\"", + "start_time": "2022-05-18T12:24:22Z" + }, + { + "status": "failed", + "code_desc": "Package \"nss-softokn\" should be updated to latest version above impacted versions [\"*\"]", + "message": "Expected latest version of \"nss-softokn\"\nDetected vulnerable version \"3.67.0-7.el8_5\" of \"nss-softokn\"", + "start_time": "2022-05-18T12:24:22Z" + }, + { + "status": "failed", + "code_desc": "Package \"nss-softokn-freebl\" should be updated to latest version above impacted versions [\"*\"]", + "message": "Expected latest version of \"nss-softokn-freebl\"\nDetected vulnerable version \"3.67.0-7.el8_5\" of \"nss-softokn-freebl\"", + "start_time": "2022-05-18T12:24:22Z" + } + ] + } + ], + "sha256": "807c8ef1534bb7f3e428db4a40667a6dd37d89f8a48dc6d1f6bb2426ff53f97f" + } + ], + "passthrough": { + "auxiliary_data": [ + { + "name": "Twistlock", + "data": { + "results": [ + { + "id": "sha256:b1c0237ebd29860066656372da10d8d7da33b34715986f74b3d5a7e4ba060d1b", + "distro": "Red Hat Enterprise Linux release 8.6 (Ootpa)", + "distroRelease": "RHEL8", + "digest": "sha256:c0274cb1e0c6e92d2ccc1e23bd19b7dddedbaa2da26861c225d4ad6cec5047f4", + "packages": [ + { + "type": "os", + "name": "nss-util", + "version": "3.67.0-7.el8_5", + "licenses": ["MPLv2.0"] + }, + { + "type": "os", + "name": "nss-softokn", + "version": "3.67.0-7.el8_5", + "licenses": ["MPLv2.0"] + }, + { + "type": "os", + "name": "nss", + "version": "3.67.0-7.el8_5", + "licenses": ["MPLv2.0"] + }, + { + "type": "os", + "name": "nss-softokn-freebl", + "version": "3.67.0-7.el8_5", + "licenses": ["MPLv2.0"] + }, + { + "type": "os", + "name": "nss-sysinit", + "version": "3.67.0-7.el8_5", + "licenses": ["MPLv2.0"] + } + ], + "applications": [ + { + "name": "asp.net-core", + "version": "5.0.17", + "path": "/usr/lib64/dotnet/dotnet" + }, + { + "name": ".net-core", + "version": "5.0.17", + "path": "/usr/lib64/dotnet/dotnet" + } + ], + "complianceScanPassed": true, + "vulnerabilityScanPassed": true, + "history": [ + { + "created": "2022-05-03T08:38:31Z" + }, + { + "created": "2022-05-03T08:39:27Z" + }, + { + "created": "2022-05-16T23:12:02Z", + "instruction": "ARG GOPROXY=http://nexus-repository-manager.nexus-repository-manager.svc.cluster.local:8081/repository/goproxy/ HTTP_PROXY=http://localhost:3128 http_proxy=http://localhost:3128" + }, + { + "created": "2022-05-16T23:12:02Z", + "instruction": "ARG GOPROXY=http://nexus-repository-manager.nexus-repository-manager.svc.cluster.local:8081/repository/goproxy/ GOSUMDB=sum.golang.org http://nexus-repository-manager.nexus-repository-manager.svc.cluster.local:8081/repository/gosum HTTP_PROXY=http://localhost:3128 http_proxy=http://localhost:3128" + } + ], + "scanTime": "2022-05-18T12:24:32.855444532Z", + "scanID": "6284e580d9600f8d0db159e2" + } + ], + "consoleURL": "https://twistlock.test.net/#!/monitor/vulnerabilities/images/ci?search=sha256%3Ab1c0237ebd29860066656372da10d8d7da33b34715986f74b3d5a7e4ba060d1b" + } + } + ] + } +} +``` + +::: diff --git a/src/courses/mappers/07.md b/src/courses/mappers/07.md new file mode 100644 index 000000000..ff9db3033 --- /dev/null +++ b/src/courses/mappers/07.md @@ -0,0 +1,300 @@ +--- +order: 7 +next: 08.md +title: OHDF Schema Review +author: Charles Hu +--- + +## OHDF Schema Review + +Now that you've familiarized yourself with the formal OHDF schema, let's revisit the GoSec and Twistlock examples and analyze both the original source data and how it has been converted to OHDF. + +Here are some key questions to keep note of while reading through these examples: + +- How is the source data structured/organized (e.g., by requirement, by target system, by finding)? +- Is the source data's data format perfectly aligned to OHDF? + - What should we do when multiple source data fields correspond to the same OHDF field? + - How should we handle a source data field that contains information correlating to multiple OHDF fields? + - How should we handle source data fields that don't fit into the OHDF schema? +- How is the structure of the source data interpreted to fit the OHDF hierarchy? + - What groups of fields in the source data correlate to our Profiles, Controls, and Results structures? +- Which fields in the source data are used to fill each OHDF field? Is that mapping accurate? + - Can you think of a reason for why that field is used to populate the OHDF field? + - Are there any fields in the source data that you believe are more applicable? Why? +- Which fields are unfilled/omitted? + - Can you think of a reason for why that field is unfilled/omitted? + - Are there any fields in the source data that you believe are applicable? Why? + +These questions will help you understand the conversion process for when you create your own OHDF mappers later. + +:::note Sample Files +The sample files below are truncated for pedagogical purposes. + +For the original files and more examples of actual OHDF samples used in production, see [here](https://github.com/mitre/heimdall2/tree/master/libs/hdf-converters/sample_jsons). +::: + +### GoSec + +:::details Source Data + +```json +{ + "Golang errors": {}, + "Issues": [ + { + "severity": "MEDIUM", + "confidence": "HIGH", + "cwe": { + "id": "22", + "url": "https://cwe.mitre.org/data/definitions/22.html" + }, + "rule_id": "G304", + "details": "Potential file inclusion via variable", + "file": "C:\\Users\\AGILLUM\\OneDrive - The MITRE Corporation\\Documents\\Code\\grype-0.34.4\\internal\\file\\tar.go", + "code": "82: \t\tcase tar.TypeReg:\n83: \t\t\tf, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))\n84: \t\t\tif err != nil {\n", + "line": "83", + "column": "14", + "nosec": false, + "suppressions": null + } + ], + "Stats": { + "files": 199, + "lines": 12401, + "nosec": 0, + "found": 7 + }, + "GosecVersion": "dev" +} +``` + +::: + +:::details Converted OHDF File + +```json +{ + "platform": { "name": "Heimdall Tools", "release": "2.10.8" }, + "version": "2.10.8", + "statistics": {}, + "profiles": [ + { + "name": "Gosec scanner", + "title": "gosec", + "version": "dev", + "supports": [], + "attributes": [], + "groups": [], + "status": "loaded", + "controls": [ + { + "tags": { + "nist": ["SI-10"], + "cwe": { + "id": "22", + "url": "https://cwe.mitre.org/data/definitions/22.html" + }, + "nosec": "", + "suppressions": "", + "severity": "MEDIUM", + "confidence": "HIGH" + }, + "refs": [], + "source_location": {}, + "title": "Potential file inclusion via variable", + "id": "G304", + "desc": "", + "impact": 0.5, + "results": [ + { + "status": "failed", + "code_desc": "82: \t\tcase tar.TypeReg:\n83: \t\t\tf, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))\n84: \t\t\tif err != nil {\n", + "message": "C:\\Users\\AGILLUM\\OneDrive - The MITRE Corporation\\Documents\\Code\\grype-0.34.4\\internal\\file\\tar.go, line:83, column:14", + "start_time": "" + } + ] + } + ], + "sha256": "d0506f4b7715bf8b1cd81a8a87ab8efcce41ebeb2b5ec5fcfb23d3bdd136f48c" + } + ], + "passthrough": { + "auxiliary_data": [{ "name": "Gosec", "data": { "Golang errors": {} } }] + } +} +``` + +::: + +### Twistlock + +:::details Source Data + +```json +{ + "results": [ + { + "id": "sha256:b1c0237ebd29860066656372da10d8d7da33b34715986f74b3d5a7e4ba060d1b", + "name": "registry.io/test:1a7a431a105aa04632f5f5fbe8f753bd245add0a", + "distro": "Red Hat Enterprise Linux release 8.6 (Ootpa)", + "distroRelease": "RHEL8", + "digest": "sha256:c0274cb1e0c6e92d2ccc1e23bd19b7dddedbaa2da26861c225d4ad6cec5047f4", + "collections": ["All", "TEST-COLLECTION"], + "packages": [ + { + "type": "os", + "name": "nss-util", + "version": "3.67.0-7.el8_5", + "licenses": ["MPLv2.0"] + } + ], + "applications": [ + { + "name": ".net-core", + "version": "5.0.17", + "path": "/usr/lib64/dotnet/dotnet" + } + ], + "complianceDistribution": { + "critical": 0, + "high": 0, + "medium": 0, + "low": 0, + "total": 0 + }, + "complianceScanPassed": true, + "vulnerabilities": [ + { + "id": "CVE-2021-43529", + "status": "affected", + "cvss": 9.8, + "description": "DOCUMENTATION: A remote code execution flaw was found in the way NSS verifies certificates. This flaw allows an attacker posing as an SSL/TLS server to trigger this issue in a client application compiled with NSS when it tries to initiate an SSL/TLS connection. Similarly, a server application compiled with NSS, which processes client certificates, can receive a malicious certificate via a client, triggering the flaw. The highest threat to this vulnerability is confidentiality, integrity, as well as system availability. STATEMENT: The issue is not limited to TLS. Any applications that use NSS certificate verification are vulnerable; S/MIME is impacted as well. Similarly, a server application compiled with NSS, which processes client certificates, can receive a malicious certificate via a client. Firefox is not vulnerable to this flaw as it uses the mozilla::pkix for certificate verification. Thunderbird is affected when parsing email with the S/MIME signature. Thunderbird on Red Hat Enterprise Linux 8.4 and later does not need to be updated since it uses the system NSS library, but earlier Red Hat Enterprise Linux 8 extended life streams will need to update Thunderbird as well as NSS. MITIGATION: Red Hat has investigated whether a possible mitigation exists for this issue, and has not been able to identify a practical example. Please update the affec", + "severity": "critical", + "packageName": "nss-util", + "packageVersion": "3.67.0-7.el8_5", + "link": "https://access.redhat.com/security/cve/CVE-2021-43529", + "riskFactors": [ + "Remote execution", + "Attack complexity: low", + "Attack vector: network", + "Critical severity", + "Recent vulnerability" + ], + "impactedVersions": ["*"], + "publishedDate": "2021-12-01T00:00:00Z", + "discoveredDate": "2022-05-18T12:24:22Z", + "layerTime": "2022-05-16T23:12:25Z" + } + ], + "vulnerabilityDistribution": { + "critical": 5, + "high": 1, + "medium": 86, + "low": 5, + "total": 97 + }, + "vulnerabilityScanPassed": true, + "history": [ + { + "created": "2022-05-03T08:38:31Z" + } + ], + "scanTime": "2022-05-18T12:24:32.855444532Z", + "scanID": "6284e580d9600f8d0db159e2" + } + ], + "consoleURL": "https://twistlock.test.net/#!/monitor/vulnerabilities/images/ci?search=sha256%3Ab1c0237ebd29860066656372da10d8d7da33b34715986f74b3d5a7e4ba060d1b" +} +``` + +::: + +:::details Converted OHDF File + +```json +{ + "platform": { + "name": "Heimdall Tools", + "release": "2.10.8", + "target_id": "registry.io/test:1a7a431a105aa04632f5f5fbe8f753bd245add0a" + }, + "version": "2.10.8", + "statistics": {}, + "profiles": [ + { + "name": "Twistlock Scan", + "title": "Twistlock Project: All / TEST-COLLECTION", + "summary": "Package Vulnerability Summary: 97 Application Compliance Issue Total: 0", + "supports": [], + "attributes": [], + "groups": [], + "status": "loaded", + "controls": [ + { + "tags": { + "nist": ["SI-2", "RA-5"], + "cci": ["CCI-002605", "CCI-001643"], + "cveid": "CVE-2021-43529" + }, + "refs": [], + "source_location": {}, + "title": "CVE-2021-43529", + "id": "CVE-2021-43529", + "desc": "DOCUMENTATION: A remote code execution flaw was found in the way NSS verifies certificates. This flaw allows an attacker posing as an SSL/TLS server to trigger this issue in a client application compiled with NSS when it tries to initiate an SSL/TLS connection. Similarly, a server application compiled with NSS, which processes client certificates, can receive a malicious certificate via a client, triggering the flaw. The highest threat to this vulnerability is confidentiality, integrity, as well as system availability. STATEMENT: The issue is not limited to TLS. Any applications that use NSS certificate verification are vulnerable; S/MIME is impacted as well. Similarly, a server application compiled with NSS, which processes client certificates, can receive a malicious certificate via a client. Firefox is not vulnerable to this flaw as it uses the mozilla::pkix for certificate verification. Thunderbird is affected when parsing email with the S/MIME signature. Thunderbird on Red Hat Enterprise Linux 8.4 and later does not need to be updated since it uses the system NSS library, but earlier Red Hat Enterprise Linux 8 extended life streams will need to update Thunderbird as well as NSS. MITIGATION: Red Hat has investigated whether a possible mitigation exists for this issue, and has not been able to identify a practical example. Please update the affec", + "impact": 0.9, + "code": "{\n \"id\": \"CVE-2021-43529\",\n \"status\": \"affected\",\n \"cvss\": 9.8,\n \"description\": \"DOCUMENTATION: A remote code execution flaw was found in the way NSS verifies certificates. This flaw allows an attacker posing as an SSL/TLS server to trigger this issue in a client application compiled with NSS when it tries to initiate an SSL/TLS connection. Similarly, a server application compiled with NSS, which processes client certificates, can receive a malicious certificate via a client, triggering the flaw. The highest threat to this vulnerability is confidentiality, integrity, as well as system availability. STATEMENT: The issue is not limited to TLS. Any applications that use NSS certificate verification are vulnerable; S/MIME is impacted as well. Similarly, a server application compiled with NSS, which processes client certificates, can receive a malicious certificate via a client. Firefox is not vulnerable to this flaw as it uses the mozilla::pkix for certificate verification. Thunderbird is affected when parsing email with the S/MIME signature. Thunderbird on Red Hat Enterprise Linux 8.4 and later does not need to be updated since it uses the system NSS library, but earlier Red Hat Enterprise Linux 8 extended life streams will need to update Thunderbird as well as NSS. MITIGATION: Red Hat has investigated whether a possible mitigation exists for this issue, and has not been able to identify a practical example. Please update the affec\",\n \"severity\": \"critical\",\n \"packageName\": \"nss-util\",\n \"packageVersion\": \"3.67.0-7.el8_5\",\n \"link\": \"https://access.redhat.com/security/cve/CVE-2021-43529\",\n \"riskFactors\": [\n \"Remote execution\",\n \"Attack complexity: low\",\n \"Attack vector: network\",\n \"Critical severity\",\n \"Recent vulnerability\"\n ],\n \"impactedVersions\": [\n \"*\"\n ],\n \"publishedDate\": \"2021-12-01T00:00:00Z\",\n \"discoveredDate\": \"2022-05-18T12:24:22Z\",\n \"layerTime\": \"2022-05-16T23:12:25Z\"\n}", + "results": [ + { + "status": "failed", + "code_desc": "Package \"nss-util\" should be updated to latest version above impacted versions [\"*\"]", + "message": "Expected latest version of \"nss-util\"\nDetected vulnerable version \"3.67.0-7.el8_5\" of \"nss-util\"", + "start_time": "2022-05-18T12:24:22Z" + } + ] + } + ], + "sha256": "998180601124e9d2b4f03be239a5bb68d9045ba2dc5a26fa2c8d453500288b2e" + } + ], + "passthrough": { + "auxiliary_data": [ + { + "name": "Twistlock", + "data": { + "results": [ + { + "id": "sha256:b1c0237ebd29860066656372da10d8d7da33b34715986f74b3d5a7e4ba060d1b", + "distro": "Red Hat Enterprise Linux release 8.6 (Ootpa)", + "distroRelease": "RHEL8", + "digest": "sha256:c0274cb1e0c6e92d2ccc1e23bd19b7dddedbaa2da26861c225d4ad6cec5047f4", + "packages": [ + { + "type": "os", + "name": "nss-util", + "version": "3.67.0-7.el8_5", + "licenses": ["MPLv2.0"] + } + ], + "applications": [ + { + "name": ".net-core", + "version": "5.0.17", + "path": "/usr/lib64/dotnet/dotnet" + } + ], + "complianceScanPassed": true, + "vulnerabilityScanPassed": true, + "history": [{ "created": "2022-05-03T08:38:31Z" }], + "scanTime": "2022-05-18T12:24:32.855444532Z", + "scanID": "6284e580d9600f8d0db159e2" + } + ], + "consoleURL": "https://twistlock.test.net/#!/monitor/vulnerabilities/images/ci?search=sha256%3Ab1c0237ebd29860066656372da10d8d7da33b34715986f74b3d5a7e4ba060d1b" + } + } + ] + } +} +``` + +::: diff --git a/src/courses/mappers/08.md b/src/courses/mappers/08.md new file mode 100644 index 000000000..6925732ed --- /dev/null +++ b/src/courses/mappers/08.md @@ -0,0 +1,304 @@ +--- +order: 8 +next: 09.md +title: OHDF Converters +author: Charles Hu +--- + +## A Look Ahead + +In this section, we will cover: + +- [What OHDF Converters is](#overview) +- How OHDF Converters can be used to create: + - [The fingerprinting component](#fingerprinting) + - [The mapper component](#mapper) + - [The testing component](#testing) + +## Overview + +OHDF Converters is a custom data normalization library that hosts and provides OHDF mapper normalization services to other applications in the SAF tool suite. All OHDF mappers depend on and utilize the functionality that OHDF converters provides in order to implement [the three core components of an OHDF mapper](./04.md). As such, it is important to have a general understanding of what some key OHDF Converters utilities are and how they operate in order to create the necessary components for an OHDF mapper and successfully integrate your OHDF mapper into OHDF Converters. + +### Directory Structure + +The following is a simplified depiction of the directory tree for the HDF Converters. Only noteworthy and potentially useful files and directories are included. It is not imperative to memorize the structure, but it is useful to familiarize yourself with it to better understand what exists where in the library for future reference. + +``` +hdf-converters ++-- data +| +-- converters +| | +-- csv2json.ts +| | +-- xml2json.ts ++-- sample_jsons // Sample exports for mapper testing are located here ++-- src // *-to-OHDF mappers are located here +| +-- converters-from-hdf // OHDF-to-* mappers are located here +| | +-- reverse-any-base-converter.ts +| | +-- reverse-base-converter.ts +| +-- mappings // Non-OHDF mappings are located here (e.g., CVE, CCI, NIST) +| +-- utils // Utility functions are located here +| | +-- fingerprinting.ts +| | +-- global.ts +| +-- base-converter.ts ++-- test // Mapper tests are located here +| +-- mappers +| | +-- forward // *-to-OHDF tests +| | +-- reverse // OHDF-to-* tests +| | +-- utils.ts ++-- types // Explicit data typing for known export schemas ++-- index.ts ++-- package.json +``` + +## Fingerprinting + +Fingerprinting in OHDF Converters is performed by the fingerprinting feature implemented in the `fingerprinting.ts` file under `/src/utils/`. The actual fingerprinting utility itself is provided through the following function: + +```typescript +export function fingerprint(guessOptions: { + data: string; + filename: string; +}): INPUT_TYPES; +``` + +This function takes an object which consists of the key-value pairs `data`, which contains a stringified representation of an input file of unknown data format, and `filename`, which contains the file name of the input file. Once the function receives the file to analyze, it leverages two key items: `INPUT_TYPES` and `fileTypeFingerprints`. + +`INPUT_TYPES` is an enumerated type that defines a common set of agreed upon names to identify each data format with. Members of `INPUT_TYPES` typically appear as so: + +```typescript +export enum INPUT_TYPES { + ASFF = "asff", + BURP = "burp", + CHECKLIST = "checklist", + CONVEYOR = "conveyor", + // Truncated for pedagogical purposes +} +``` + +where the enum member is the uppercase snake case form of the common name while the initialized value is the stringified lowercase form of the name. + +`fileTypeFingerprints` is a variable defining the specific set of data elements that we are to uniquely associate with a certain data format. Members of `fileTypeFingerprints` typically appear as so: + +```typescript +const fileTypeFingerprints: Record = { + [INPUT_TYPES.ASFF]: ["Findings", "AwsAccountId", "ProductArn"], + [INPUT_TYPES.CONVEYOR]: ["api_error_message", "api_response"], + [INPUT_TYPES.FORTIFY]: ["FVDL", "FVDL.EngineData.EngineVersion", "FVDL.UUID"], + // Truncated for pedagogical purposes +}; +``` + +where the key is the standard name for a data format as defined by `INPUT_TYPES`, while the value is an array of stringified field names that are cumulatively unqiue to that data format. + +Using the fields defined in `fileTypeFingerprints`, the function `fingerprint()` will search through the input file and attempt to assign it a data format according to whichever fingerprint array in `fileTypeFingerprints` has the most matches to keys found in the input file. It will then return the common name of that data format as defined by `INPUT_TYPES`. If no match is found, it will return an empty string instead (as defined by `INPUT_TYPES.NOT_FOUND`). + +Most OHDF mapper developers will not touch the `fingerprint()` function itself but will rather add the necessary members to `INPUT_TYPES` and `fileTypeFingerprints` in order to allow `fingerprint()` to correctly identify their data format. + +## Mapper + +Mappers are often self-contained in a single specialized file under `/src/`, but leverage a number of utilities spread throughout OHDF Converters to actually implement the ability to normalize input files from developer-provided mappings. + +### Mapper Base Class + +The core of OHDF Converters revolves around the [`base-converter`](https://github.com/mitre/heimdall2/blob/master/libs/hdf-converters/src/base-converter.ts) class found in the `base-converter.ts` file under `/src/`. `base-converter` is a key class that all \*-to-OHDF mappers extend. It enables the streamlined development of \*-to-OHDF mappers by abstracting the actual implementation of the underlying service that performs the data transformations (i.e., the mapper), resulting in the developer only having to write a technical implementation of their developed \*-to-OHDF mapping for the `base-converter` class to consume and use. + +Other services provided by `base-converter` include: + +#### File Format Parsing + +`base-converter` provides a number of functions which can parse a variety of file formats and convert them into a usable Javascript object equivalent. + +Currently supported file formats are as follows: + +| Format | Function | +| ------ | ------------------------------------------------------------------------------------------------------------- | +| CSV | [`parseCsv()`](https://github.com/mitre/heimdall2/blob/master/libs/hdf-converters/src/base-converter.ts#L75) | +| HTML | [`parseHtml()`](https://github.com/mitre/heimdall2/blob/master/libs/hdf-converters/src/base-converter.ts#L43) | +| XML | [`parseXml()`](https://github.com/mitre/heimdall2/blob/master/libs/hdf-converters/src/base-converter.ts#L57) | + +#### Generic Types + +`base-converter` accepts type arguments to define the typing of the data that the mapper is expected to ingest. This is defined here: + +```typescript +export class BaseConverter> { + data: D; + mappings?: MappedTransform; + collapseResults: boolean; + + constructor(data: D, collapseResults = false) { + this.data = data; + this.collapseResults = collapseResults; + } + // Truncated for pedagogical purposes +} +``` + +and can be used as so: + +```typescript +export class CycloneDXSBOMMapper extends BaseConverter +``` + +This is particularly useful for when we know the type of incoming data and want to avoid continuous manual casting of types within the mapping. + +#### Keywords + +`base-converter` provides a series of keywords that can be inserted into the technical implementation of the mapping in order to allow necessary code functions or data manipulations to occur within the mapping definition. These keywords are inherited upon extending `base-converter` and are as follows: + +- `path`: Define JSON object path to go to. Paths are found recursively (e.g., if a top-level field is set to the path `result`, then all fields below it will only see the `result` field of the source file). To escape this recursion, you can set the `$` symbol at the beginning of the path. + + - Use: + + ```typescript + path: PATH_AS_STRING; + ``` + + - Example: + + ```typescript + // Attribute `id` will be set as whatever JSON object attribute `vulnerability.id` is + id: { + path: "vulnerability.id"; + } + ``` + + - Example of recursion: + + ```typescript + // Attribute `id` will look at the path `result.id` is instead of just the path `id` + controls: [ + { + path: "result", + id: { path: "id" }, + }, + ]; + ``` + + - Example of escaping from recursion: + + ```typescript + // Attribute `id` will look at the top-level path `id`, ignoring the recursive path `result` + controls: [ + { + path: "result", + id: { path: "$id" }, + }, + ]; + ``` + +- `transformer`: Oftentimes, source data is not be formatted in a readable/desirable manner. To remedy this, we can make use of the `transformer` keyword. The `transformer` accepts a function as a parameter. This can either be an anonymous funciton or a normal, named function. This transformer acts as a callback function that will go to the specified path location and modify the data accordingly. Note that while the `transformer` keyword can accept an array as an input, it expects and will treat the input as a Javascript object. + + - Use: + + ```typescript + transformer: (PARAMETER: TYPE): OUTPUT_TYPE => { + CODE_TO_EXECUTE; + }; + ``` + + - Example: + + ```typescript + // Applying a transformer that maps NIST 800-53 tags to CCI tags + // Attribute 'cci' will be set as the returned CCI tag(s) from the ingested 'data' argument + cci: { + path: 'vulnerabilityClassifications', + transformer: (data: string) => getCCIsForNISTTags(nistTag(data)) + } + ``` + +- `arrayTransformer`: The `arrayTransformer` works similarly to the `transformer` in that it takes our in-progress mapping for the array structure, accepts a function that returns an array, and transforms the array accordingly (e.g., filtering out element items, passing in additional data, etc.). + + - Use: + + ```typescript + arrayTransformer: (PARAMETER: TYPE[]): OUTPUT_TYPE[] => { + CODE_TO_EXECUTE; + }; + ``` + + - Example: + + ```typescript + controls: [ + { + path: "...", // Some path to the array in the source file + // The function 'deduplicateId' will run against all items in the current array that the 'arrayTransformer' was called inside + arrayTransformer: deduplicateId, + }, + ]; + ``` + + In the above code block, `arrayTransformer` is used in the controls array of the mapping, which creates a control for every item in the source file's specified path location. This array, however, may contain duplicate items (with the same IDs); thus, the `arrayTransformer` is being leveraged to deduplicate items with the same ID and collapse them into one element. + +- `pathTransform`: Converts the object path structure. This preprocesses the source file before any of the other keywords are run. Only use this as a last resort if the source file needs to be drastically reworked. Note that the intermediate format generated by it is not saved to disk, so it can be difficult to work through. + + - Use: + + ```typescript + pathTransform: (PARAMETER: TYPE): OUTPUT_TYPE => { + CODE_TO_EXECUTE; + }; + ``` + + - Example: + + ```typescript + // Returns the JSON path if it is an array, otherwise returns an empty array + pathTransform: (value) => (Array.isArray(value) ? value : []), + ``` + +- `key`: Used by `base-converter` to sort an array of objects by. Currently, this is only used in the `controls` array to automatically de-duplicate elements with the same key to ensure that all controls are unique. As a result, you typically only pass control IDs to `key`. When calling the constructor for the mapper, we can set `collapseResults=true` to make use of this automatic deduplication function. + + - Use: + + ```typescript + key: KEY_AS_STRING; + ``` + + - Example: + + ```typescript + // `id` is now the key by which this array will be sorted by + key: "id"; + ``` + +### Utility Files + +Several exported utility functions and variables exist within files contained under `/src/utils/`. Some examples of commonly used ones from `global.ts` are as follows: + +- [`getCCIsForNISTTags()`](https://github.com/mitre/heimdall2/blob/master/libs/hdf-converters/src/utils/global.ts#L66): Converts a set of NIST 800-53 tags into CCI tags. + +- [`DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS`](https://github.com/mitre/heimdall2/blob/master/libs/hdf-converters/src/utils/global.ts#L12): An array of NIST 800-53s applicable to all automated configuration tests. + +- [`filterString()`](https://github.com/mitre/heimdall2/blob/master/libs/hdf-converters/src/utils/global.ts#L104): Return the input string if it is not empty, otherwise return `undefined` if the string is empty. + +You are encouraged to explore these files for further utilities and to add on more to help other developers who may require similar functionality to the one you're implementing. + +## Testing + +Testing in OHDF Converters is facilitated through the [Jest testing framework](https://jestjs.io/docs/getting-started). The tests for \*-to-OHDF mappers are contained under `/test/mappers/forward/` and depend on a set of sample files and expected OHDF output files contained under `/sample_jsons/`. + +The mapper tests operate by passing the sample files found under `/sample_jsons/` to the given mapper to create a set of observed OHDF outputs. These observed outputs will then be compared against the set of expected outputs generated from expected OHDF files also stored under `/sample_jsons/`. The tests will pass if the observed outputs match the expected outputs and will fail otherwise. The exact technical implementation of this will be discussed later on. + +## A Look Back + +In this section, we covered: + +- [What OHDF Converters is](#overview) + + - OHDF Converters is a custom data normalization library that all OHDF mappers depend upon to implement the three core components of an OHDF mapper. + +- How OHDF Converters can be used to create: + + - [The fingerprinting component](#fingerprinting) + + - OHDF Converters provides a fingerprinting service that helps identify the data format of the given input data. + + - [The mapper component](#mapper) + + - OHDF Converters provides a number of utilities provided by both the `base-converter` class and associated utility files to help streamline the OHDF mapper development experience. + + - [The testing component](#testing) + + - OHDF Converters extends the testing framework provided by Jest to ensure that the tested OHDF mappers generate the correct OHDF output. diff --git a/src/courses/mappers/09.md b/src/courses/mappers/09.md new file mode 100644 index 000000000..660ac456d --- /dev/null +++ b/src/courses/mappers/09.md @@ -0,0 +1,98 @@ +--- +order: 9 +next: 10.md +title: The Art of Mapper Making +author: Charles Hu +--- + +## A Look Ahead + +In this section, we will cover: + +- [Tips and best practices for mapper development](#overview): + - [Reference existing solutions](#reference-existing-solutions) + - [Understand your security tool and data](#understand-your-security-tool-and-data) + - [Use detailed and diverse samples](#use-detailed-and-diverse-samples) + - [Regularly check your work](#regularly-check-your-work) + - [Reach out](#reach-out) + +## Overview + +Mapper creation is a nuanced process in which every situation demands a unique and tailored solution. As a result, there is no one objective best template or process to follow as field mappings will always be designed on a case-by-case basis. + +But don't fret! Mapper making, like many forms of development, is a skill you have to cultivate and improve at. So while there is no one-size-fits-all solution, you can still train yourself to produce mappers that are the best for your needs and uses. + +To help in improving your mapper making skills, the following section details a number of tips and best practices for mapper development that you should keep in mind while creating your mapper. + +## Tips and Best Practices + +### Reference Existing Solutions + +Try to browse and reference existing solutions for other mappers in [OHDF Converters](https://github.com/mitre/heimdall2/tree/master/libs/hdf-converters) to glean some insight on how previous \*-to-OHDF mappers were implemented. By doing this, you may be able to find existing solutions to problems you may be facing or find novel ways of implementing concepts that you never thought of before. + +Some simple and generally helpful mappers to look at when starting out are the [Burp Suite mapper](https://github.com/mitre/heimdall2/blob/master/libs/hdf-converters/src/burpsuite-mapper.ts) and the [Snyk mapper](https://github.com/mitre/heimdall2/blob/master/libs/hdf-converters/src/snyk-mapper.ts). These mappers are straightforward and involve minimal data manipulation or corner cases. They represent the standard mapper experience and will likely reflect how your mapper will turn out unless your data format involves complicated or nontraditional data. + +Some more technically involved mappers are the [CycloneDX SBOM mapper](https://github.com/mitre/heimdall2/blob/master/libs/hdf-converters/src/cyclonedx-sbom-mapper.ts) and the [Nessus mapper](https://github.com/mitre/heimdall2/blob/master/libs/hdf-converters/src/nessus-mapper.ts). These mappers are much more complicated but involve handling of specific corner cases (SBOM data formatting for the CycloneDX SBOM mapper and high complexity data for the Nessus mapper). It is not advised to look at these mappers unless you are decently familiar with Typescript and software development paradigms. + +### Understand Your Security Tool and Data + +Understand how your security data is meant to be consumed in the original security tool's use case (i.e., don't just start development with the raw security data). Ask yourself the following questions: + +- How does the original security tool group security data? Is it by type of concern? Where/what component was affected? +- What associations does the tool make to common security references (e.g., NIST SP 800-53, CVE, CWE, CCE, OWASP, STIG IDs)? +- What is the purpose of the security tool (e.g., vulnerability scanning, information visualization, etc.)? +- How does the security tool organize its results (e.g., by requirement, by system, etc.)? + +These answers will help ensure that the intent of the data isn't skewed by the mapping process. It is important to try and "seek parity" with the original display style in the vendor's own tool. In other words, users must be able to recognize and navigate the security data in Heimdall just as easily as they can in the original native format. + +### Use Detailed and Diverse Samples + +:::warning Data Sanitization +Sanitize your data sample so that it is safe for public release in the OHDF Converters GitHub repository. Once sensitive data is publically released, it is difficult to effectively scrub off the Internet. You can easily prevent this from happening by thoroughly screening the data first. +::: + +Build your mapper around sample outputs that are detailed and diverse, and try to obtain the data in any machine-readable format (e.g., JSON, XML, CSV) that is natively available. Oftentimes, the schema for the data format is not publically available or is poorly documented, so it is good practice to gather a large volume of varied samples in order to accurately cover the many possible representations of the data format that may exist. + +:::note APIs +If your security tool provides an API instead of a security data export, you will need to explore the API to determine what information may need to be collected and collated for use within the mapper. Even without an explicit results file from the vendor, you should have a thorough understanding of the security data that needs to be mapped to OHDF. We recommend contacting the SAF team for further guidance in creating a specialized mapper that works with that API. +::: + +### Regularly Check Your Work + +Mapper development is an iterative process that involves many adjustments and corrections in order to achieve an \*-to-OHDF mapping that is as accurate as possible. You are encouraged to periodically check on your mapper output during development in order to check on your progress and verify that your mapper is operating as expected. + +The technical details of how to check your output will be discussed later on. + +### Reach Out + +Don't be afraid to reach out to the SAF team for assistance with mapper development if you are unable to solve an issue by yourself or are totally lost on how to continue. Our developers can help you with everything from mapping decisions to mapper integration with OHDF Converters. + +You can contact us through email (saf@groups.mitre.org) for inquiries and by creating a [GitHub issue on the Heimdall2 repository](https://github.com/mitre/heimdall2/issues/new/choose) for technical implementation issues. + +Try to be detailed and thorough in describing your issue. It is difficult to help when we have little to no context on what your problem is or how we can address it, and we certainly can't read minds to figure it out either. + +## A Look Back + +In this section, we covered: + +- [Tips and best practices for mapper development](#overview): + + - [Reference existing solutions](#reference-existing-solutions) + + - Review existing mappers for potential solutions to any development issues you may encounter. + + - [Understand your security tool and data](#understand-your-security-tool-and-data) + + - Have a thorough understanding of your data format so you can preserve the intent of your data format when mapping to OHDF. + + - [Use detailed and diverse samples](#use-detailed-and-diverse-samples) + + - Have a detailed and diverse set of samples to use for building your mapping in order to cover all possible representations of your data format. + + - [Regularly check your work](#regularly-check-your-work) + + - Check on your in-progress OHDF mapper to ensure that your current mapping is satisfactory and working as expected. + + - [Reach out](#reach-out) + + - Reach out for help if you can't work it out on your own. diff --git a/src/courses/mappers/10.md b/src/courses/mappers/10.md new file mode 100644 index 000000000..6a75252b0 --- /dev/null +++ b/src/courses/mappers/10.md @@ -0,0 +1,118 @@ +--- +order: 10 +next: 11.md +title: Environment Set Up +author: Charles Hu +--- + +## A Look Ahead + +In this section, we will cover: + +- How to set up an environment for OHDF mapper development via: + - [GitHub Codespaces](#github-codespaces-environment-set-up) + - [Local installation](#local-environment-set-up) + +## Overview + +The rest of this course will involve a mix of hands-on guided and unguided labs to teach you the basics of the technical implementation of OHDF mappers. To facilitate this, we provide two installation guides for setting up the environment necessary for these upcoming labs: A GitHub Codespaces environment set up (recommended for beginners) and a local environment set up (recommended for experienced developers or individuals interested in manually installing to their local system). + +## GitHub Codespaces Environment Set Up + +We provide a [GitHub Codespaces environment](https://github.com/mitre/saf-training-lab-environment) that includes a simple build script that installs the necessary repositories and packages to begin OHDF mapper development. + +Follow the instructions listed in the README and you should have a verified working environment with the Heimdall and SAF CLI repositories pulled down into the `/dev_repos/` directory. Heimdall can be found under `/dev_repos/heimdall2/`, the SAF CLI can be found under `/dev_repos/saf/`, and OHDF Converters can be found under `/dev_repos/heimdall2/libs/hdf-converters/`. + +## Local Environment Set Up + +To set up the environment locally, we will need to pull down the GitHub repositories for Heimdall and the SAF CLI using Git and install their necessary dependencies using a package manager. + +You can install Git [here](https://git-scm.com/downloads). Choose your OS and follow the installation guide as necessary. + +Node.js (a runtime environment for JavaScript), NPM (a package manager for JavaScript/TypeScript), and Yarn (another package manager for JavaScript/TypeScript) are external utilities which are utilized extensively within this guide. The following section details their installation process. + +### Runtime & Package Managers + +#### Linux/Mac OS + +1. Install [nvm](https://github.com/nvm-sh/nvm#install--update-script). + +- 1a. Use either of the following commands to install nvm: + + ```shell + curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash + ``` + + ```shell + wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash + ``` + +- 1b. Either restart the terminal or run the following commands to use nvm: + ```shell + export NVM_DIR="$([ -z "${XDG_CONFIG_HOME-}" ] && printf %s "${HOME}/.nvm" || printf %s "${XDG_CONFIG_HOME}/nvm")" + [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm + ``` + +2. Run the following command to install and use Node.js v18: + +```shell +nvm install 18 +``` + +3. Install [Yarn](https://yarnpkg.com/getting-started/install) + +#### Windows + +1. Install [Node.js v18 via the installer](https://nodejs.org/en/download/). + +2. Install [Yarn](https://yarnpkg.com/getting-started/install): + +### Repository Set Up + +Now that we have the runtime and package managers for Javascript installed, we can pull down the necessary GitHub repositories and install their dependencies. + +1. Fork/branch a development repository from the main [Heimdall2 GitHub repository](https://github.com/mitre/heimdall2). + + - SAF team developers have write access to the main repository and should [create a branch](https://docs.github.com/en/desktop/contributing-and-collaborating-using-github-desktop/making-changes-in-a-branch/managing-branches#creating-a-branch) on the primary development repository. Non-SAF team developers should instead [create a fork](https://docs.github.com/en/get-started/quickstart/fork-a-repo#forking-a-repository) of the main repository and create a development branch there. + +2. Pull down your development branch from GitHub using the following command: + +```shell +git clone {URL-TO-REPO-HERE} +``` + +3. Create a draft [pull request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request#creating-the-pull-request) for your development branch against the main repository branch. + +4. Install the necessary dependencies for Heimdall2. Under the `heimdall2` directory, enter the following command in the terminal: + +```shell +yarn install --frozen-lockfile +``` + +5. Fork/branch a development repository from the main [SAF CLI GitHub repository](https://github.com/mitre/saf). + + - SAF team developers have write access to the main repository and should [create a branch](https://docs.github.com/en/desktop/contributing-and-collaborating-using-github-desktop/making-changes-in-a-branch/managing-branches#creating-a-branch) on the primary development repository. Non-SAF team developers should instead [create a fork](https://docs.github.com/en/get-started/quickstart/fork-a-repo#forking-a-repository) of the main repository and create a development branch there. + +6. Pull down your development branch from GitHub using the following command: + +```shell +git clone {URL-TO-REPO-HERE} +``` + +7. Create a draft [pull request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request#creating-the-pull-request) for your development branch against the main repository branch. + +8. Install the necessary dependencies for the SAF CLI. Under the `saf` directory, enter the following command in the terminal: + +```shell +npm install +``` + +You should now have a working environment with the Heimdall and SAF CLI repositories. Heimdall can be found under `/heimdall2/`, the SAF CLI can be found under `/saf/`, and OHDF Converters can be found under `/heimdall2/libs/hdf-converters/`. + +## A Look Back + +In this section, we covered: + +- How to set up an environment for OHDF mapper development via: + - [GitHub Codespaces](#github-codespaces-environment-set-up) + - [Local installation](#local-environment-set-up) diff --git a/src/courses/mappers/11.md b/src/courses/mappers/11.md new file mode 100644 index 000000000..c990c6b32 --- /dev/null +++ b/src/courses/mappers/11.md @@ -0,0 +1,1637 @@ +--- +order: 11 +next: 12.md +title: Creating a Mapping +author: Charles Hu, Ryan Lin +--- + +## Mapping Introduction + +::: note Security Tool Type +This walkthrough is targeted at converting security scans to OHDF. If your security tool provides exports that are not security scans or are APIs, contact the SAF team for further instruction on how to develop those services into an OHDF mapper. +::: + +We begin the creation of every OHDF mapper by first planning out how we want correlate the fields in the OHDF and security tool's data format. This mapping is important for the following reasons: + +1. It gives us a good framework for guiding how we actually implement the \*-to-OHDF mapper. +2. It notifies us of potential knowledge gaps we may have of a security tool's data format. We need to close these gaps as best as we can in order to ensure that we can create field correlations that are as accurate/applicable as possible. +3. It gives us a chance to review our mapping in a format that is still human readable. +4. It helps us reduce the amount of corrections and rewrites we have to perform on the actual mapper implementation due to potential mapping errors. + +As we develop the mappings in the following examples, note that we will break down and analyze the fields according to the OHDF schema structures we previously learned about (profiles, controls, and results). This is not necessarily the only or the best way to do this and you are encouraged to develop your own methods for correlating fields that work best for you. + +::: note Use of Exported Security Tool Data +The following examples will use generated instances of exported security tool data that do not reflect the entirety of the schemas they are based on. Ideally, you will want to use the full schema when creating a mapping; however, the full schema is always not readily available. To reflect this, we will be using generated export data and will fill in missing schema data when possible. +::: + +## Mapping Demo - GoSec + +### 1. Breaking Down the Data + +Let's first start with our GoSec sample, which you should be already familiar with: + +::: details GoSec Source Data + +```json +{ + "Golang errors": {}, + "Issues": [ + { + "severity": "MEDIUM", + "confidence": "HIGH", + "cwe": { + "id": "22", + "url": "https://cwe.mitre.org/data/definitions/22.html" + }, + "rule_id": "G304", + "details": "Potential file inclusion via variable", + "file": "C:\\Users\\AGILLUM\\OneDrive - The MITRE Corporation\\Documents\\Code\\grype-0.34.4\\internal\\file\\tar.go", + "code": "82: \t\tcase tar.TypeReg:\n83: \t\t\tf, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))\n84: \t\t\tif err != nil {\n", + "line": "83", + "column": "14", + "nosec": false, + "suppressions": null + } + ], + "Stats": { + "files": 199, + "lines": 12401, + "nosec": 0, + "found": 7 + }, + "GosecVersion": "dev" +} +``` + +::: + +We can begin our mapping process by identifying the purpose of each field in the source data. This is relatively simple in this case as we have a [formal schema for GoSec](https://github.com/securego/gosec/blob/master/issue/issue.go) that defines many of its fields. While we do this, it's also important that we try to tie back each identified purpose to the three general OHDF schema structures that we learned about earlier; that is, in addition to finding the purpose of each field, we should also ask ourselves the following: + +- Is this field recording metadata? +- Is this field recording requirements? +- Is this field recording requirement testing? + +Let's apply this to our GoSec source data. Try to find the answers to these two questions for each field yourself and then refer back to our annotated solution to check your work. + +::: details GoSec Annotated Source Data + +```json +{ + // Purpose: Go compilation errors + // Recording: Metadata - Not specifically related to the requirements and will be already recorded as a security issue in 'Issues' if critical + "Golang errors": {}, + // Purpose: Container for identified security issues + // Recording: Requirements - This entity records all identified security issues in a Go source code + "Issues": [ + { + // Purpose: The severity of the identified issue + // Recording: Requirements - This is specifically related to the severity level of the requirement + "severity": "MEDIUM", + // Purpose: How sure that the identified issue if applicable to this source code + // Recording: Requirements testing - This field gives the applicability of the issue after source code testing + "confidence": "HIGH", + // Purpose: The associated CWE for the security issue + // Recording: Requirements - This gives the associated CWE for the security issue + "cwe": { + "id": "22", + "url": "https://cwe.mitre.org/data/definitions/22.html" + }, + // Purpose: The internal GoSec ID for the security issue + // Recording: Requirements - This gives an ID for the security issue + "rule_id": "G304", + // Purpose: Explanation of the security issue + // Recording: Requirements - This explains the security issue + "details": "Potential file inclusion via variable", + // Purpose: The offending file + // Recording: Requirement testing - This specifically notes which file fails the requirement after source code testing + "file": "C:\\Users\\AGILLUM\\OneDrive - The MITRE Corporation\\Documents\\Code\\grype-0.34.4\\internal\\file\\tar.go", + // Purpose: The offending code + // Recording: Requirement testing - This specifies the code that fails the requirement after source code testing + "code": "82: \t\tcase tar.TypeReg:\n83: \t\t\tf, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))\n84: \t\t\tif err != nil {\n", + // Purpose: The line number of the offending code + // Recording: Requirement testing - This field specifies the location of the failing code + "line": "83", + // Purpose: The column number of the offending code + // Recording: Requirement testing - This field specifies the location of the failing code + "column": "14", + // Purpose: Whether this security issue should be ignored + // Recording: Requirements - Specifies whether this security issue should be ignored + "nosec": false, + // Purpose: The supression level for info on the security issue + // Recording: Requirements - Specifies the info suppression level of the security issue + "suppressions": null + } + ], + // Purpose: The statistics of the GoSec scan on the source code + // Recording: Metadata - Info on the scan itself + "Stats": { + "files": 199, + "lines": 12401, + "nosec": 0, + "found": 7 + }, + // Purpose: The version of the GoSec instance currently running + // Recording: Metadata - Info on the scan itself + "GosecVersion": "dev" +} +``` + +::: + +Note that this process is subjective and your answers may vary from ours. Don't be so quick to backtrack on your own answers, as your solutions may be as equally valid as ours. There is no objective answer to this process and what's important is that you can identify and break down the source data into useful pieces that we can correlate back to the OHDF schema. + +The purpose of this annotation is to understand the intention of the security tool that produced the source data. By breaking down the source data field by field, we can gain a better understanding of how the security tool structures its export data and intends for it to be read by a user. GoSec, for example, organizes its data by security issues because GoSec is intended for code correction and hardening. This manifests in the real world through the user reading the GoSec report issue by issue and solving them as they go along. We can use this knowledge to help guide us in both correlating the schemas and reconfiguring the structure of the source data to fit OHDF. + +### 2. Correlating the Schemas + +Now that we're familiar with the specifics of the GoSec schema, we can begin to map these fields to the OHDF schema. We'll do this by taking a field in the source data and correlating it to the most applicable OHDF schema field. While performing these correlations, it is important to account for the intention and structure of the source data schema. + +This initial mapping is informal and does not need to adhere to any programming syntax since you're probably annotating the documents by hand. The mapping process entails taking the full field path from the source data and placing it as the corresponding value for the OHDF schema field. Proper implementation of these mappings will be done later in the course. An example of such mapping is as follows: + +```typescript +{ + profiles: [ + { + controls: [ + { + title: Issues.details, + }, + ], + }, + ]; +} +``` + +This shows us a mapping of the field `Issues.details` from the source data to the field `profiles.controls.title` in OHDF. + +Note that sometimes every relevant field within the OHDF schema is addressed, but the source data still contains unmapped data. In such an event, it is good practice to ensure that such data is still passed through to and recorded somewhere within the OHDF schema for data preservation. In most cases, you should place such data in the `passthrough` field. + +Let's first start with an empty OHDF schema as follows: + +::: details Empty OHDF Schema + +```typescript +{ + platform: { + name, + release, + target_id + }, + version, + statistics: { + duration + }, + profiles: [ + { + name, + version, + sha256, + title, + maintainer, + summary, + license, + copyright, + copyright_email, + supports, + attributes, + groups, + controls: [ + { + id, + title, + desc, + descriptions, + impact, + refs, + tags, + code, + source_location, + results: [ + { + status, + code_desc, + message, + run_time, + start_time + } + ] + }, + ], + status + }, + ], + passthrough: { + auxiliary_data: [ + { + name, + data + }, + ], + raw + } +} +``` + +::: + +#### Mapping Version 1 + +We can begin our schema correlation by directly porting over fields from the GoSec source data that are obvious: `GosecVersion`, `Issues.rule_id`, `Issues.details`, and `Issues.cwe`. Try to map these fields youself as you follow along and refer to our solution to check your work. + +::: details Correlated Fields +OHDF | Source Data +-|- +`profiles.version` | `GosecVersion` +`profiles.controls.id` | `Issues.rule_id` +`profiles.controls.title` | `Issues.details` +`profiles.controls.tags` | `Issues.cwe` +::: + +::: details OHDF Schema Mapping Ver. 1 + +```typescript +{ + platform: { + name, + release, + target_id + }, + version, + statistics: { + duration + }, + profiles: [ + { + name, + version: GosecVersion, // Version of GoSec instance + sha256, + title, + maintainer, + summary, + license, + copyright, + copyright_email, + supports, + attributes, + groups, + controls: [ + { + id: Issues.rule_id, // ID of the requirement + title: Issues.details, // Human readable title for the requirement + desc, + descriptions, + impact, + refs, + tags: { + Issues.cwe // Associated CWE for the requirement + }, + code, + source_location, + results: [ + { + status, + code_desc, + message, + run_time, + start_time + } + ] + }, + ], + status + }, + ], + passthrough: { + auxiliary_data: [ + { + name, + data + }, + ], + raw + } +} +``` + +::: + +::: details Remaining GoSec Source Data + +```json +{ + // Purpose: Go compilation errors + // Recording: Metadata - Not specifically related to the requirements and will be already recorded as a security issue in 'Issues' if critical + "Golang errors": {}, + // Purpose: Container for identified security issues + // Recording: Requirements - This entity records all identified security issues in a Go source code + "Issues": [ + { + // Purpose: The severity of the identified issue + // Recording: Requirements - This is specifically related to the severity level of the requirement + "severity": "MEDIUM", + // Purpose: How sure that the identified issue if applicable to this source code + // Recording: Requirements testing - This field gives the applicability of the issue after source code testing + "confidence": "HIGH", + // Purpose: The offending file + // Recording: Requirement testing - This specifically notes which file fails the requirement after source code testing + "file": "C:\\Users\\AGILLUM\\OneDrive - The MITRE Corporation\\Documents\\Code\\grype-0.34.4\\internal\\file\\tar.go", + // Purpose: The offending code + // Recording: Requirement testing - This specifies the code that fails the requirement after source code testing + "code": "82: \t\tcase tar.TypeReg:\n83: \t\t\tf, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))\n84: \t\t\tif err != nil {\n", + // Purpose: The line number of the offending code + // Recording: Requirement testing - This field specifies the location of the failing code + "line": "83", + // Purpose: The column number of the offending code + // Recording: Requirement testing - This field specifies the location of the failing code + "column": "14", + // Purpose: Whether this security issue should be ignored + // Recording: Requirements - Specifies whether this security issue should be ignored + "nosec": false, + // Purpose: The supression level for info on the security issue + // Recording: Requirements - Specifies the info suppression level of the security issue + "suppressions": null + } + ], + // Purpose: The statistics of the GoSec scan on the source code + // Recording: Metadata - Info on the scan itself + "Stats": { + "files": 199, + "lines": 12401, + "nosec": 0, + "found": 7 + } +} +``` + +::: + +::: details Reasoning + +- `version: GosecVersion`: We want to specifically place this in `version` in `profiles` because this is metadata on the actual security tool that generated these results. This is different from `version` on the top level since that refers to the platform, or the tool creating the actual OHDF file itself. + +- `id: Issues.rule_id`: `rule_id` gives us an ID used by GoSec to identify each failing requirement in the source code. + +- `title: Issues.details`: The source data doesn't provide a proper title that goes along with the failing requirement. In lieu of this, we can use `Issues.details` which does give us a human readable description of the requirement and repurpose it as a title for our control. + +- `tags: {Issues.cwe}`: `Issues.cwe` provides an association of our GoSec security issue with a known vulnerability database. This can be used in `tags` to act as metadata which can used to filter or organize controls created from these GoSec security issues. + ::: + +#### Mapping Version 2 + +Next, let's look at `Issues.code`, which can be a little bit tricky. + +::: details Correlated Fields +OHDF | Source Data +-|- +`profiles.controls.results.code_desc` | `Issues.code` +::: + +::: details OHDF Schema Mapping Ver. 2 + +```typescript +{ + platform: { + name, + release, + target_id + }, + version, + statistics: { + duration + }, + profiles: [ + { + name, + version: GosecVersion, // Version of GoSec instance + sha256, + title, + maintainer, + summary, + license, + copyright, + copyright_email, + supports, + attributes, + groups, + controls: [ + { + id: Issues.rule_id, // ID of the requirement + title: Issues.details, // Human readable title for the requirement + desc, + descriptions, + impact, + refs, + tags: { + Issues.cwe // Associated CWE for the requirement + }, + code, + source_location, + results: [ + { + status, + code_desc: Issues.code, // The code failing the requirement test + message, + run_time, + start_time + } + ] + }, + ], + status + }, + ], + passthrough: { + auxiliary_data: [ + { + name, + data + }, + ], + raw + } +} +``` + +::: + +::: details Remaining GoSec Source Data + +```json +{ + // Purpose: Go compilation errors + // Recording: Metadata - Not specifically related to the requirements and will be already recorded as a security issue in 'Issues' if critical + "Golang errors": {}, + // Purpose: Container for identified security issues + // Recording: Requirements - This entity records all identified security issues in a Go source code + "Issues": [ + { + // Purpose: The severity of the identified issue + // Recording: Requirements - This is specifically related to the severity level of the requirement + "severity": "MEDIUM", + // Purpose: How sure that the identified issue if applicable to this source code + // Recording: Requirements testing - This field gives the applicability of the issue after source code testing + "confidence": "HIGH", + // Purpose: The offending file + // Recording: Requirement testing - This specifically notes which file fails the requirement after source code testing + "file": "C:\\Users\\AGILLUM\\OneDrive - The MITRE Corporation\\Documents\\Code\\grype-0.34.4\\internal\\file\\tar.go", + // Purpose: The line number of the offending code + // Recording: Requirement testing - This field specifies the location of the failing code + "line": "83", + // Purpose: The column number of the offending code + // Recording: Requirement testing - This field specifies the location of the failing code + "column": "14", + // Purpose: Whether this security issue should be ignored + // Recording: Requirements - Specifies whether this security issue should be ignored + "nosec": false, + // Purpose: The supression level for info on the security issue + // Recording: Requirements - Specifies the info suppression level of the security issue + "suppressions": null + } + ], + // Purpose: The statistics of the GoSec scan on the source code + // Recording: Metadata - Info on the scan itself + "Stats": { + "files": 199, + "lines": 12401, + "nosec": 0, + "found": 7 + } +} +``` + +::: + +::: details Reasoning + +- `code_desc: Issues.code`: `Issues.code` refers to the source code that is explicitly failing the GoSec requirement. This does not align with our definition for `code_desc` but remember that we always aim to seek parity with the original security tool. GoSec focuses heavily on identifying a security issue and directly pinpointing where that issue occurs for immediate resolution. We want to mirror that behavior by also pinpointing where the security issue occurs at instead of providing a code description of what test went wrong. + ::: + +#### Mapping Version 3 + +Next, let's look at the fields `Issues.severity`, `Issues.confidence`, `Issues.nosec`, and `Issues.suppressions`. + +::: details Correlated Fields +OHDF | Source Data +-|- +`profiles.controls.tags` | `Issues.severity` +`profiles.controls.tags` | `Issues.confidence` +`profiles.controls.tags` | `Issues.nosec` +`profiles.controls.tags` | `Issues.suppressions` +::: + +::: details OHDF Schema Mapping Ver. 3 + +```typescript +{ + platform: { + name, + release, + target_id + }, + version, + statistics: { + duration + }, + profiles: [ + { + name, + version: GosecVersion, // Version of GoSec instance + sha256, + title, + maintainer, + summary, + license, + copyright, + copyright_email, + supports, + attributes, + groups, + controls: [ + { + id: Issues.rule_id, // ID of the requirement + title: Issues.details, // Human readable title for the requirement + desc, + descriptions, + impact, + refs, + tags: { + Issues.cwe // Associated CWE for the requirement + Issues.severity, // Severity of the requirement + Issues.confidence, // Applicability of the requirement + Issues.nosec, // Whether to ignore the requirement + Issues.suppressions // Info suppression level of the requirement + }, + code, + source_location, + results: [ + { + status, + code_desc: Issues.code, // The code for the requirement test + message, + run_time, + start_time + } + ] + }, + ], + status + }, + ], + passthrough: { + auxiliary_data: [ + { + name, + data + }, + ], + raw + } +} +``` + +::: + +::: details Remaining GoSec Source Data + +```json +{ + // Purpose: Go compilation errors + // Recording: Metadata - Not specifically related to the requirements and will be already recorded as a security issue in 'Issues' if critical + "Golang errors": {}, + // Purpose: Container for identified security issues + // Recording: Requirements - This entity records all identified security issues in a Go source code + "Issues": [ + { + // Purpose: The offending file + // Recording: Requirement testing - This specifically notes which file fails the requirement after source code testing + "file": "C:\\Users\\AGILLUM\\OneDrive - The MITRE Corporation\\Documents\\Code\\grype-0.34.4\\internal\\file\\tar.go", + // Purpose: The line number of the offending code + // Recording: Requirement testing - This field specifies the location of the failing code + "line": "83", + // Purpose: The column number of the offending code + // Recording: Requirement testing - This field specifies the location of the failing code + "column": "14" + } + ], + // Purpose: The statistics of the GoSec scan on the source code + // Recording: Metadata - Info on the scan itself + "Stats": { + "files": 199, + "lines": 12401, + "nosec": 0, + "found": 7 + } +} +``` + +::: + +::: details Reasoning +We map these fields to `tags` in the OHDF schema due to each of these fields acting as descriptive traits of the control which we can categorize and sort the controls by. + +It may be tempting to map `Issues.severity` to `impact`; however, it is important to note that `severity` is not the same as `impact`. These two fields are related but distinct metrics, and while conversion is possible, it is not performed in this case due to uncertainty on how to translate the two metrics properly. +::: + +#### Mapping Version 4 + +Next, let's look at the fields `Issues.file`, `Issues.line`, and `Issues.column`. + +::: details Correlated Fields +OHDF | Source Data +-|- +`profiles.controls.results.message` | `Issues.file` +`profiles.controls.results.message` | `Issues.line` +`profiles.controls.results.message` | `Issues.column` +::: + +::: details OHDF Schema Mapping Ver. 4 + +```typescript +{ + platform: { + name, + release, + target_id + }, + version, + statistics: { + duration + }, + profiles: [ + { + name, + version: GosecVersion, // Version of GoSec instance + sha256, + title, + maintainer, + summary, + license, + copyright, + copyright_email, + supports, + attributes, + groups, + controls: [ + { + id: Issues.rule_id, // ID of the requirement + title: Issues.details, // Human readable title for the requirement + desc, + descriptions, + impact, + refs, + tags: { + Issues.cwe // Associated CWE for the requirement + Issues.severity, // Severity of the requirement + Issues.confidence, // Applicability of the requirement + Issues.nosec, // Whether to ignore the requirement + Issues.suppressions // Info suppression level of the requirement + }, + code, + source_location, + results: [ + { + status, + code_desc: Issues.code, // The code for the requirement test + message: Issues.file + Issues.line + Issues.column, // All materials describing where the issue occurred + run_time, + start_time + } + ] + }, + ], + status + }, + ], + passthrough: { + auxiliary_data: [ + { + name, + data + }, + ], + raw + } +} +``` + +::: + +::: details Remaining GoSec Source Data + +```json +{ + // Purpose: Go compilation errors + // Recording: Metadata - Not specifically related to the requirements and will be already recorded as a security issue in 'Issues' if critical + "Golang errors": {}, + // Purpose: The statistics of the GoSec scan on the source code + // Recording: Metadata - Info on the scan itself + "Stats": { + "files": 199, + "lines": 12401, + "nosec": 0, + "found": 7 + } +} +``` + +::: + +::: details Reasoning +This is a situation similar to the one `Issues.code` is in. `Issues.file`, `Issues.line`, and `Issues.column` do not fit the definition given for `message` (i.e., they are not explanations for the test status); however, they do elaborate upon the location of the offending code snippet identified by GoSec, which we want to seek parity with. Thus, in lieu of any suitable substitutes, we instead use the locational information of the failing source code snippet as a description for the control result. +::: + +#### Mapping Version 5 + +Next, let's look at the fields `Golang errors` and `Stats`. + +::: details Correlated Fields +OHDF | Source Data +-|- +`passthrough.auxiliary_data` | `Golang errors` +`passthrough.auxiliary_data` | `Stats` +::: + +::: details OHDF Schema Mapping Ver. 5 + +```typescript +{ + platform: { + name, + release, + target_id + }, + version, + statistics: { + duration + }, + profiles: [ + { + name, + version: GosecVersion, // Version of GoSec instance + sha256, + title, + maintainer, + summary, + license, + copyright, + copyright_email, + supports, + attributes, + groups, + controls: [ + { + id: Issues.rule_id, // ID of the requirement + title: Issues.details, // Human readable title for the requirement + desc, + descriptions, + impact, + refs, + tags: { + Issues.cwe // Associated CWE for the requirement + Issues.severity, // Severity of the requirement + Issues.confidence, // Applicability of the requirement + Issues.nosec, // Whether to ignore the requirement + Issues.suppressions // Info suppression level of the requirement + }, + code, + source_location, + results: [ + { + status, + code_desc: Issues.code, // The code for the requirement test + message: Issues.file + Issues.line + Issues.column, // All materials describing where the issue occurred + run_time, + start_time + } + ] + }, + ], + status + }, + ], + passthrough: { + auxiliary_data: [ + { // Go source data compilation errors; Stats on GoSec scan + name: 'Gosec', + data: Golang errors, Stats + } + ], + raw + } +} +``` + +::: + +::: details Reasoning +For these fields, we can place them in `passthrough` as there are no reasonably applicable fields in the OHDF schema to place them in. + +`Golang errors` is not used anywhere in `controls` as it is a description of compilation errors which would've already been flagged by GoSec as security issues if they already occurred. Thus, to avoid duplication, `Golang errors` is omitted from the main OHDF structures. +::: + +#### Mapping Version 6 + +Finally, let's fill the rest of the empty fields using what available information we can use, prioritizing required fields and omitting any applicable unrequired fields. + +::: details Correlated Fields +OHDF | Source Data +-|- +`platform.name` | `'Heimdall Tools'` +`platform.release` | `HeimdallToolsVersion` +`version` | `HeimdallToolsVersion` +`profiles.name` | `'Gosec scanner'` +`profiles.title` | `'gosec'` +`profiles.controls.impact` | `0.5` +`profiles.controls.results.status` | `'Failure'` +`profiles.status` | `'loaded'` +::: + +::: details OHDF Schema Mapping Ver. 6 + +```typescript +{ + platform: { // We fill in Heimdall for the platform as it handles the generation of this OHDF file + name: 'Heimdall Tools', + release: HeimdallToolsVersion + }, + version: HeimdallToolsVersion, // See 'platform' reasoning + statistics: {}, // Not enough info to fill + profiles: [ + { + name: 'Gosec scanner', // We know that this report is generated from GoSec + version: GosecVersion, // Version of GoSec instance + sha256: '', // Leave it empty as OHDF Converters will generate one for us + title: 'gosec', // We know that this report is generated from GoSec + supports: [], // Not enough info to fill + attributes: [], // Not enough info to fill + groups: [], // Not enough info to fill + controls: [ + { + id: Issues.rule_id, // ID of the requirement + title: Issues.details, // Human readable title for the requirement + desc: '', // Not enough info to fill + impact: 0.5, // Have no solid information on impact of security issue, so we default to 0.5 + refs: [], // Not enough info to fill + tags: { + Issues.cwe // Associated CWE for the requirement + Issues.severity, // Severity of the requirement + Issues.confidence, // Applicability of the requirement + Issues.nosec, // Whether to ignore the requirement + Issues.suppressions // Info suppression level of the requirement + }, + source_location: {}, // Not enough info to fill + results: [ + { + status: 'failed', // The security scan only reports failed requirements, so all findings we receive get fail statuses + code_desc: Issues.code, // The code failing the requirement test + message: Issues.file + Issues.line + Issues.column, // All materials describing where the issue occurred + start_time // Not enough info to fill + } + ] + }, + ], + status: 'loaded' // Give loaded status to denote that profile is loaded by OHDF Converters + }, + ], + passthrough: { + auxiliary_data: [ + { // Go source data compilation errors; Stats on GoSec scan + name: 'Gosec', + data: Golang errors, Stats + } + ], + raw + } +} +``` + +::: + +::: details Reasoning +Unfilled fields are a result of a lack of sufficent information from the source data that can be used to reasonably associate with these fields. + +The reasoning for the filled fields are as follows: + +- `platform: {name: 'Heimdall Tools'}`: `platform` refers to the instance generating the OHDF file, which in this case is Heimdall2 or Heimdall Tools. + +- `platform: {release: HeimdallToolsVersion}`: Similar reasoning as above. + +- `version: HeimdallToolsVersion`: Similar reasoning as `platform` fields. + +- `name: 'Gosec scanner'`: `profiles` refers to the security tool that generated the original source data. In this case, it is the GoSec security scanning tool. + +- `title: 'gosec'`: We can give a succinct title for the profile here. + +- `impact: 0.5`: We use a default value of `0.5` since we do not have a direct mapping set up to convert the `Issues.severity` field. + +- `status: 'failed'`: GoSec only reports security vulnerabilities if it finds that the scanned source code has such security issues present. As such, every reported issue in a GoSec scan will always be a result of a failed security test. + +- `status: 'loaded'`: The status of a profile is typically `loaded` since it has been ingested and converted to OHDF. + ::: + +Now we have a finalized mapping for GoSec to OHDF. + +## Mapping Demo - DbProtect + +It is also possible to convert XML-based results formats. The OHDF library relies on `fast-xml-parser` to convert these XML files into a JSON-like format for compatibility with the base converter. Let us take a look at one such example: The DbProtect Mapper. + +### 1. Breaking Down the Data + +Let's first start with our DbProtect sample. + +::: details DbProtect Source Data + +```xml + + + + + + + + + + + + + + + + + + + + + TEST ORGANIZATION (Local DBP server) + Audit + Microsoft SQL Server + CONDS181 + 10.0.10.204, 1433, MSSQLSERVER + Heimdall Test scan report generation + DISA-STIG SQL Server 2016 V2R1-1 Audit (Built-In) + Fact + Improper Access Controls + Medium + 2986 + Schema ownership + Schema name=DatabaseMailUserRole;Database=msdb;Owner name=DatabaseMailUserRole + Feb 18 2021 15:57 + + + TEST ORGANIZATION (Local DBP server) + Audit + Microsoft SQL Server + CONDS181 + 10.0.10.204, 1433, MSSQLSERVER + Heimdall Test scan report generation + DISA-STIG SQL Server 2016 V2R1-1 Audit (Built-In) + Fact + Improper Access Controls + Medium + 2986 + Schema ownership + Schema name=db_accessadmin;Database=AppDetective;Owner name=db_accessadmin + Feb 18 2021 15:57 + + ... + + + +``` + +::: + +You may notice that the metadata field maps corresponding keys to each of the values in a row of the data field. Consider ways you might try to restructure this data, but focus mainly on the purpose of each of the values for now. We will handle data restructuring in a later portion. + +Unlike with GoSec, DbProtect does not have a published schema. Thus, we must empirically reason the mapping based on our sample data. Because we are reasoning solely off of sample data, even more so than the previous example, this process is subjective and your mapping may vary from ours. One thing we can use to inform ourselves about the purpose of each field in place of a schema, however, is the metadata fields at the top of the sample. + +#### Knowledge Check + +::: details What questions do we ask ourselves when analyzing security data fields? + +- What is the purpose of this field? + +- What is this field recording (metadata, requirements, requirement testing)? + ::: + +Let us now apply this philosophy to our DbProtect source data. Try to find the answers to these two questions for each field yourself and then refer back to our annotated solution to check your work. + +::: details DbProtect Annotated Source Data + +```xml + + + + + + + + + + + + + + + + + + + + + + + + + TEST ORGANIZATION (Local DBP server) + + + Audit + + + Microsoft SQL Server + + + CONDS181 + + + 10.0.10.204, 1433, MSSQLSERVER + + + Heimdall Test scan report generation + + + DISA-STIG SQL Server 2016 V2R1-1 Audit (Built-In) + + + Fact + + + Improper Access Controls + + + Medium + + + 2986 + + + Schema ownership + + + Schema name=DatabaseMailUserRole;Database=msdb;Owner name=DatabaseMailUserRole + + + Feb 18 2021 15:57 + + ... + + +``` + +::: + +Note that this process is subjective and your answers may vary from ours. Don't be so quick to backtrack on your own answers, as your solutions may be as equally valid as ours. There is no objective answer to this process and what's important is that you can identify and break down the source data into useful pieces that we can correlate back to the OHDF schema. + +### 2. Correlating the Schemas + +Now that we're familiar with the specifics of the DbProtect report structure, we can begin to map these fields to the OHDF schema. + +Note that since we used an export from DbProtect and are not basing our mapping off the full schema, we will inevitably miss some fields. We can rectify this by adding them into our mapping post hoc whenever needed; however, we can still capture these missed fields without manual intervention by using our `passthrough` field to capture unmapped data. + +Let's first start with an empty OHDF schema as follows: + +::: details Empty OHDF Schema + +```typescript +{ + platform: { + name, + release, + target_id + }, + version, + statistics: { + duration + }, + profiles: [ + { + name, + version, + sha256, + title, + maintainer, + summary, + license, + copyright, + copyright_email, + supports, + attributes, + groups, + controls: [ + { + id, + title, + desc, + descriptions, + impact, + refs, + tags, + code, + source_location, + results: [ + { + status, + code_desc, + message, + run_time, + start_time + } + ] + }, + ], + status + }, + ], + passthrough: { + auxiliary_data: [ + { + name, + data + }, + ], + raw + } +} +``` + +::: + +#### Mapping Version 1 + +We can begin our schema correlation by directly porting over fields from the DbProtect source data that are obvious: `Check ID`, `Result Status`, and `Date`. Try to map these fields yourself, and refer to our solution below to check your work. + +As in the previous example, note that these mapping are informal and do not need to adhere to any programming syntax. Proper implementation of these mappings will be done later in the course. + +::: details Correlated Fields +OHDF | Source Data +-|- +`profiles.controls.id` | `Check ID` +`profiles.controls.results.status` | `Result Status` +`profiles.controls.results.start_time` | `Date` +::: + +::: details OHDF Schema Mapping Ver. 1 + +```typescript +{ + platform: { + name, + release, + target_id + }, + version, + statistics: { + duration + }, + profiles: [ + { + name, + version, + sha256, + title, + maintainer, + summary, + license, + copyright, + copyright_email, + supports, + attributes, + groups, + controls: [ + { + id: 'Check ID', // ID of the requirement + title, + desc, + descriptions, + impact, + refs, + tags: { + nist, + cci + }, + code, + source_location, + results: [ + { + status: 'Result Status', // The result of the scan for that particular control + code_desc, + message, + run_time, + start_time: 'Date' // Some indication of when the scan was run + } + ] + }, + ], + status + }, + ], + passthrough: { + auxiliary_data: [ + { + name, + data + }, + ], + raw + } +} +``` + +::: + +::: details Remaining DbProtect Source Data + +```xml + + + + + + + ... + + + + + + TEST ORGANIZATION (Local DBP server) + + + Audit + + + Microsoft SQL Server + + + CONDS181 + + + 10.0.10.204, 1433, MSSQLSERVER + + + Heimdall Test scan report generation + + + DISA-STIG SQL Server 2016 V2R1-1 Audit (Built-In) + + + Improper Access Controls + + + Medium + + + Schema ownership + + + Schema name=DatabaseMailUserRole;Database=msdb;Owner name=DatabaseMailUserRole + + ... + + +``` + +::: + +::: details Reasoning + +- `id: 'Check ID'`: `Check ID` gives us an ID used by DbProtect to identify each failing requirement. + +- `status: 'Result Status'`: `Result Status` gives us information about the result of the scan, whether it passed or failed or was skipped. + +- `start_time: 'Date'`: `Date` provides information about when the scan was run. + ::: + +#### Mapping Version 2 + +Now let us handle some of the more detailed information about the specific controls, namely `Task`, `Check Category`, `Risk DV`, `Check`, and `Details`. + +::: details Correlated Fields +OHDF | Source Data +-|- +`profiles.controls.desc` | `Task`, `Check Category` +`profiles.controls.impact` | `Risk DV` +`profiles.controls.title` | `Check` +`profiles.controls.results.code_desc` | `Details` +::: + +::: details OHDF Schema Mapping Ver. 2 + +```typescript +{ + platform: { + name, + release, + target_id + }, + version, + statistics: { + duration + }, + profiles: [ + { + name, + version, + sha256, + title, + maintainer, + summary, + license, + copyright, + copyright_email, + supports, + attributes, + groups, + controls: [ + { + id: 'Check ID', // ID of the requirement + title: 'Check', + desc: ['Task', 'Check Category'], + descriptions, + impact: 'Risk DV', + refs, + tags: { + nist, + cci + }, + code, + source_location, + results: [ + { + status: 'Result Status', // The result of the scan for that particular control + code_desc: 'Details', + message, + run_time, + start_time: 'Date' // Some indication of when the scan was run + } + ] + }, + ], + status + }, + ], + passthrough: { + auxiliary_data: [ + { + name, + data + }, + ], + raw + } +} +``` + +::: + +::: details Remaining DbProtect Source Data + +```xml + + + + + + + ... + + + + + + TEST ORGANIZATION (Local DBP server) + + + Microsoft SQL Server + + + CONDS181 + + + 10.0.10.204, 1433, MSSQLSERVER + + + Heimdall Test scan report generation + + + DISA-STIG SQL Server 2016 V2R1-1 Audit (Built-In) + + ... + + +``` + +::: + +::: details Reasoning + +For the control description, we want to give information about the type of control being run. Thus, it makes sense to store the category of the check, and the particular task it was for (whether it was for an Audit, or for another reason). + +In this case, we can directly map impact to `Risk DV` because there exists an intuitive mapping: High, Medium, Low, and Informational can map to 0.7, 0.5, 0.3, and 0, respectively. + +Note that `Check` appears to also correspond to the `Check ID`, and is a brief desciption of the check being run. Thus, it would make sense to map it to the title field of a control. + +Finally, for the details, recall that the `code_desc` is looking for the test expectations as defined by the particular control. `Details` is telling us about the particular result's associated parameters. +::: + +#### Mapping Version 3 + +Finally, we are left with the more general, report-level information from our DbProtect sample file: `Organization`, `Asset Type`, `Check Asset`, `IP Address, Port, Instance`, `Job Name`, `Policy`. Notice that OHDF does not have fields that correspond to some of this additional info, so consider ways to condense this data and encode it into other fields. + +::: details Correlated Fields +OHDF | Source Data +-|- +`profiles.summary` | `Organization`, `Asset`, `Asset Type`, `IP Address, Port, Instance` +`profiles.title` | `Job Name` +`profiles.name` | `Policy` +::: + +::: details OHDF Schema Mapping Ver. 3 + +```typescript +{ + platform: { + name, + release, + target_id + }, + version, + statistics: { + duration + }, + profiles: [ + { + name: 'Policy', + version, + sha256, + title: 'Job Name', + maintainer, + summary: ['Organization', 'Asset', 'Asset Type', 'IP Address, Port, Instance'], + license, + copyright, + copyright_email, + supports, + attributes, + groups, + controls: [ + { + id: 'Check ID', // ID of the requirement + title: 'Check', + desc: ['Task', 'Check Category'], + descriptions, + impact: 'Risk DV', + refs, + tags: { + nist, + cci + }, + code, + source_location, + results: [ + { + status: 'Result Status', // The result of the scan for that particular control + code_desc: 'Details', + message, + run_time, + start_time: 'Date' // Some indication of when the scan was run + } + ] + }, + ], + status + }, + ], + passthrough: { + auxiliary_data: [ + { + name, + data + }, + ], + raw + } +} +``` + +::: + +::: details Reasoning +As mentioned previously, OHDF does not have a specific place to put information like the `Organization`, `Asset`, or `Asset Type`. However, we do have a summary field for profiles. Recall our goal of preserving as much data as possible from the source. With this in mind, it would be reasonable to map all these fields to the summary for the sake of preservation. + +The profile name and title fields are a little tricky. Note that the `Policy` field refers to the baseline guidance that the scan tested against. This aligns more with the profile name than the `Job Name`, which is just a name the customer added for the report. Thus, we map the `Policy` field to the `profiles.name` and the `Job Name` field to the `profiles.title` instead. +::: + +#### Mapping Version 4 + +Finally, let's fill the rest of the empty fields using what available information we can use, prioritizing required fields and omitting any applicable unrequired fields. + +::: details Correlated Fields +OHDF | Source Data +-|- +`platform.name` | `'Heimdall Tools'` +`platform.release` | `HeimdallToolsVersion` +`version` | `HeimdallToolsVersion` +`profiles.controls.tags.nist` | `DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS` +`profiles.controls.tags.cci` | `DEFAULT_STATIC_CODE_ANALYSIS_CCI_TAGS` +`profiles.status` | `'loaded'` +::: + +::: details OHDF Schema Mapping Ver. 4 + +```typescript +{ + platform: { + name: 'Heimdall Tools', + release: HeimdallToolsVersion, + target_id + }, + version: HeimdallToolsVersion, + statistics: { + duration + }, + profiles: [ + { + name: 'Policy', + version, + sha256, + title: 'Job Name', + maintainer, + summary: ['Organization', 'Asset', 'Asset Type', 'IP Address, Port, Instance'], + license, + copyright, + copyright_email, + supports, + attributes, + groups, + controls: [ + { + id: 'Check ID', // ID of the requirement + title: 'Check', + desc: ['Task', 'Check Category'], + descriptions, + impact: 'Risk DV', + refs, + tags: { + nist: DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS, + cci: DEFAULT_STATIC_CODE_ANALYSIS_CCI_TAGS + }, + code, + source_location, + results: [ + { + status: 'Result Status', // The result of the scan for that particular control + code_desc: 'Details', + message, + run_time, + start_time: 'Date' // Some indication of when the scan was run + } + ] + }, + ], + status: 'loaded' + }, + ], + passthrough: { + auxiliary_data: [ + { + name, + data + }, + ], + raw + } +} +``` + +::: + +::: details Reasoning +Unfilled fields are a result of a lack of sufficent information from the source data that can be used to reasonably associate with these fields. + +The reasoning for the filled fields are as follows: + +- `platform: {name: 'Heimdall Tools'}`: `platform` refers to the instance generating the OHDF file, which in this case is Heimdall2 or Heimdall Tools. + +- `platform: {release: HeimdallToolsVersion}`: Similar reasoning as above. + +- `version: HeimdallToolsVersion`: Similar reasoning as `platform` fields. + +- `profiles.controls.tags.nist: DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS`: The DbProtect sample does not have any associated NIST tags within it. Thus, we use the default tags for a static code analysis. + +- `profiles.controls.tags.cci: DEFAULT_STATIC_CODE_ANALYSIS_CCI_TAGS`: Similar reasoning as above. + +- `status: 'loaded'`: The status of a profile is typically `loaded` since it has been ingested and converted to OHDF. + ::: + +Now we have a finalized mapping for DbProtect to OHDF. diff --git a/src/courses/mappers/12.md b/src/courses/mappers/12.md new file mode 100644 index 000000000..4a4a8af65 --- /dev/null +++ b/src/courses/mappers/12.md @@ -0,0 +1,1757 @@ +--- +order: 12 +next: 13.md +title: Implementing a Mapper +author: Charles Hu +--- + +## Mapper Introduction + +::: note OHDF Converters Utilities +Special utilities from OHDF Converters are used here. [Refer here](./09.md) for a refresher on OHDF Converters. +::: + +Now that we have developed a mapping, we can implement that mapping as a mapper which applies the mapping to any compatible file for a conversion to OHDF. + +## File Set Up + +A number of crucial files that support and provide the infrastructure needed for the \*-to-OHDF mapper have to be set up first before we begin actual mapper development. + +::: note Specialized Security Tools +This guide is geared for security tools that provide scan-based export data. If your security tool provides a specialized form of export data or is an API, contact the SAF team for further guidance. +::: + +### Mapper File + +First, we need to create the file that hosts the mapper and link to it so other files in OHDF Converters can access it. + +1. Create a blank TypeScript file under the `src` directory in `hdf-converters`. It should be named: + +``` +{YOUR-EXPORT-NAME-HERE}-mapper.ts +``` + +2. Select the appropriate mapper skeleton (see below) for your export type. Place them in the file created in step 1. Replace names (`SKELETON` by default) as necessary. + +::: details JSON Mapper Skeleton + +```typescript +import { ExecJSON } from "inspecjs"; +import _ from "lodash"; +import { version as HeimdallToolsVersion } from "../package.json"; +import { + BaseConverter, + ILookupPath, + impactMapping, + MappedTransform, +} from "./base-converter"; + +export class SKELETONMapper extends BaseConverter { + withRaw: boolean; + + mappings: MappedTransform< + ExecJSON.Execution & { passthrough: unknown }, + ILookupPath + > = { + platform: { + name: "Heimdall Tools", + release: HeimdallToolsVersion, + target_id: null, //Insert data + }, + version: HeimdallToolsVersion, + statistics: { + duration: null, //Insert data + }, + profiles: [ + { + name: "", //Insert data + title: null, //Insert data + version: null, //Insert data + maintainer: null, //Insert data + summary: null, //Insert data + license: null, //Insert data + copyright: null, //Insert data + copyright_email: null, //Insert data + supports: [], //Insert data + attributes: [], //Insert data + depends: [], //Insert data + groups: [], //Insert data + status: "loaded", //Insert data + controls: [ + { + key: "id", + tags: {}, //Insert data + descriptions: [], //Insert data + refs: [], //Insert data + source_location: {}, //Insert data + title: null, //Insert data + id: "", //Insert data + desc: null, //Insert data + impact: 0, //Insert data + code: null, //Insert data + results: [ + { + status: ExecJSON.ControlResultStatus.Failed, //Insert data + code_desc: "", //Insert data + message: null, //Insert data + run_time: null, //Insert data + start_time: "", //Insert data + }, + ], + }, + ], + sha256: "", + }, + ], + passthrough: { + transformer: (data: Record): Record => { + return { + auxiliary_data: [{ name: "", data: _.omit([]) }], //Insert service name and mapped fields to be removed + ...(this.withRaw && { raw: data }), + }; + }, + }, + }; + constructor(exportJson: string, withRaw = false) { + super(JSON.parse(exportJson), true); + this.withRaw = withRaw; + } +} +``` + +::: + +3. Export your mapper class created in the previous steps by specifying its export in the index.ts file. Add the following line: + +``` +export * from './src/{YOUR-EXPORT-NAME-HERE}-mapper'; +``` + +### Sample File + +Next, we need to add a sample file for the mapper to ingest when running unit tests on it. + +1. Create a new directory named `{YOUR-EXPORT-NAME-HERE}_mapper` under the `sample_jsons` directory in `hdf-converters`. Create another directory named `sample_input_report` in the directory you just made. The directory structure should look like this: + +``` ++-- sample_jsons +| +-- {YOUR-EXPORT-NAME-HERE}-mapper +| | +-- sample_input_report +``` + +2. Place your sample export under the `sample_input_report` directory. Your sample export should be genericized to avoid any leaking of sensitive information. The directory structure should now look like this: + +``` ++-- sample_jsons +| +-- {YOUR-EXPORT-NAME-HERE}-mapper +| | +-- sample_input_report +| | | +-- {YOUR-SAMPLE-EXPORT} +``` + +### Regression Testing + +Now that we have a sample file, we can now add some regression tests which automatically test our mapper to ensure that it is producing readable and correct OHDF file outputs. + +1. Create a blank TypeScript file under the `test/mappers/forward` directory in `hdf-converters`. It should be named: + +``` +{YOUR-EXPORT-NAME-HERE}_mapper.spec.ts +``` + +2. Select the appropriate mapper test skeleton (see below) for your export type. Place it in the file created in step 1. Replace names (`SKELETON` by default) as necessary. + +::: details JSON Mapper Test Skeleton + +```typescript +import fs from "fs"; +import { SKELETONMapper } from "../../../src/SKELETON-mapper"; +import { omitVersions } from "../../utils"; + +describe("SKELETON_mapper", () => { + it("Successfully converts SKELETON targeted at a local/cloned repository data", () => { + const mapper = new SKELETONMapper( + fs.readFileSync( + "sample_jsons/SKELETON_mapper/sample_input_report/SKELETON.json", + { encoding: "utf-8" } + ) + ); + + // fs.writeFileSync( + // 'sample_jsons/SKELETON_mapper/SKELETON-hdf.json', + // JSON.stringify(mapper.toHdf(), null, 2) + // ); + + expect(omitVersions(mapper.toHdf())).toEqual( + omitVersions( + JSON.parse( + fs.readFileSync("sample_jsons/SKELETON_mapper/SKELETON-hdf.json", { + encoding: "utf-8", + }) + ) + ) + ); + }); +}); + +describe("SKELETON_mapper_withraw", () => { + it("Successfully converts withraw flagged SKELETON targeted at a local/cloned repository data", () => { + const mapper = new SKELETONMapper( + fs.readFileSync( + "sample_jsons/SKELETON_mapper/sample_input_report/SKELETON.json", + { encoding: "utf-8" } + ), + true + ); + + // fs.writeFileSync( + // 'sample_jsons/SKELETON_mapper/SKELETON-hdf-withraw.json', + // JSON.stringify(mapper.toHdf(), null, 2) + // ); + + expect(omitVersions(mapper.toHdf())).toEqual( + omitVersions( + JSON.parse( + fs.readFileSync( + "sample_jsons/SKELETON_mapper/SKELETON-hdf-withraw.json", + { + encoding: "utf-8", + } + ) + ) + ) + ); + }); +}); +``` + +::: + +### Fingerprinting + +OHDF Converters has a fingerprinting service that detects a security tool data format and automatically applies the correct mapper to convert it to OHDF. To enable this feature, we need to explicitly declare keywords unique to the security tool data format as follows: + +1. Go to the file `report_intake.ts` under the `heimdall2/apps/frontend/src/store` directory. + +2. Import your mapper file. You should be able to add the name of your mapper class to a pre-existing import statement pointing at `hdf-converters` as follows: + +```typescript +import { + ASFFResults as ASFFResultsMapper, + BurpSuiteMapper, + ... + {YOUR-MAPPER-CLASS-HERE} +} from '@mitre/hdf-converters'; +``` + +3. Instantiate your mapper class in the `convertToHdf` switch block. Add the following lines: + +```typescript +case '{YOUR-EXPORT-SERVICE-NAME-HERE}': + return new {YOUR-MAPPER-CLASS-HERE}(convertOptions.data).toHdf(); +``` + +4. Navigate to the file `fingerprinting.ts` in the `src/utils` directory in `hdf-converters`. Add keywords that are unique to your sample export to the `fileTypeFingerprints` variable. It should be formatted as follows: + +```typescript +export const fileTypeFingerprints = { + asff: ['Findings', 'AwsAccountId', 'ProductArn'], + ... + {YOUR-EXPORT-SERVICE-NAME-HERE}: [{UNIQUE KEYWORDS AS STRINGS}] +}; +``` + +## Mapper Implementation + +With the necessary files now set up, we can begin the actual creation of the OHDF mapper using the skeleton mapper base in the `{YOUR-EXPORT-NAME-HERE}-mapper.ts` file. The skeleton mapper and the `base-converter` class have been designed to provide the base functionality needed for \*-to-HDF mapper generation. For most developers, mapper creation will be limited to assigning objects from the export structure to correlating attributes in the mapper according to the mappings they developed earlier. + +::: warning File Processing +Certain security services produce exports which are not immediately usable by the skeleton mapper. In such case, pre-processing on the export and or post-processing on the generated OHDF file is necessary in order to ensure compatibility. +::: + +While developing the mapper, it's useful to occasionally check on the generated OHDF file to check that the mapper is working as intended. You can do this by starting a local instance of Heimdall with the following command: + +```shell +yarn start:dev +``` + +Note that the `yarn start:dev` command will dynamically rebuild the application upon changes to the Heimdall frontend. However, if you make any changes to OHDF-Converters, you will have to restart the command entirely. + +Import your source data and then export it as an OHDF JSON to check what your mapper is actually mapping. + +### Mapper Demo - GoSec + +This section is a demonstration on implementing an OHDF mapper for GoSec, assuming that the [appropriate file set up for the mapper](#file-set-up) has been performed. + +Here is our developed mapping for GoSec for reference: + +::: details GoSec-to-OHDF Mapping + +```typescript +{ + platform: { // We fill in Heimdall for the platform as it handles the generation of this OHDF file + name: 'Heimdall Tools', + release: HeimdallToolsVersion + }, + version: HeimdallToolsVersion, // See 'platform' reasoning + statistics: {}, // Not enough info to fill + profiles: [ + { + name: 'Gosec scanner', // We know that this report is generated from GoSec + version: GosecVersion, // Version of GoSec instance + sha256: '', // Leave it empty as OHDF Converters will generate one for us + title: 'gosec', // We know that this report is generated from GoSec + supports: [], // Not enough info to fill + attributes: [], // Not enough info to fill + groups: [], // Not enough info to fill + controls: [ + { + id: Issues.rule_id, // ID of the requirement + title: Issues.details, // Human readable title for the requirement + desc: '', // Not enough info to fill + impact: 0.5, // Have no solid information on impact of security issue, so we default to 0.5 + refs: [], // Not enough info to fill + tags: { + Issues.cwe // Associated CWE for the requirement + Issues.severity, // Severity of the requirement + Issues.confidence, // Applicability of the requirement + Issues.nosec, // Whether to ignore the requirement + Issues.suppressions // Info suppression level of the requirement + }, + source_location: {}, // Not enough info to fill + results: [ + { + status: 'failed', // The security scan only reports failed requirements, so all findings we receive get fail statuses + code_desc: Issues.code, // The code failing the requirement test + message: Issues.file + Issues.line + Issues.column, // All materials describing where the issue occurred + start_time // Not enough info to fill + } + ] + }, + ], + status: 'loaded' // Give loaded status to denote that profile is loaded by OHDF Converters + }, + ], + passthrough: { + auxiliary_data: [ + { // Go source data compilation errors; Stats on GoSec scan + name: 'Gosec', + data: Golang errors, Stats + } + ], + raw + } +} +``` + +::: + +::: details GoSec Annotated Source Data + +```json +{ + // Purpose: Go compilation errors + // Recording: Metadata - Not specifically related to the requirements and will be already recorded as a security issue in 'Issues' if critical + "Golang errors": {}, + // Purpose: Container for identified security issues + // Recording: Requirements - This entity records all identified security issues in a Go source code + "Issues": [ + { + // Purpose: The severity of the identified issue + // Recording: Requirements - This is specifically related to the severity level of the requirement + "severity": "MEDIUM", + // Purpose: How sure that the identified issue if applicable to this source code + // Recording: Requirements testing - This field gives the applicability of the issue after source code testing + "confidence": "HIGH", + // Purpose: The associated CWE for the security issue + // Recording: Requirements - This gives the associated CWE for the security issue + "cwe": { + "id": "22", + "url": "https://cwe.mitre.org/data/definitions/22.html" + }, + // Purpose: The internal GoSec ID for the security issue + // Recording: Requirements - This gives an ID for the security issue + "rule_id": "G304", + // Purpose: Explanation of the security issue + // Recording: Requirements - This explains the security issue + "details": "Potential file inclusion via variable", + // Purpose: The offending file + // Recording: Requirement testing - This specifically notes which file fails the requirement after source code testing + "file": "C:\\Users\\AGILLUM\\OneDrive - The MITRE Corporation\\Documents\\Code\\grype-0.34.4\\internal\\file\\tar.go", + // Purpose: The offending code + // Recording: Requirement testing - This specifies the code that fails the requirement after source code testing + "code": "82: \t\tcase tar.TypeReg:\n83: \t\t\tf, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))\n84: \t\t\tif err != nil {\n", + // Purpose: The line number of the offending code + // Recording: Requirement testing - This field specifies the location of the failing code + "line": "83", + // Purpose: The column number of the offending code + // Recording: Requirement testing - This field specifies the location of the failing code + "column": "14", + // Purpose: Whether this security issue should be ignored + // Recording: Requirements - Specifies whether this security issue should be ignored + "nosec": false, + // Purpose: The supression level for info on the security issue + // Recording: Requirements - Specifies the info suppression level of the security issue + "suppressions": null + } + ], + // Purpose: The statistics of the GoSec scan on the source code + // Recording: Metadata - Info on the scan itself + "Stats": { + "files": 199, + "lines": 12401, + "nosec": 0, + "found": 7 + }, + // Purpose: The version of the GoSec instance currently running + // Recording: Metadata - Info on the scan itself + "GosecVersion": "dev" +} +``` + +::: + +Mapper creation has been streamlined to be as simple as possible for a developer. Most of the work involves simple references to the object path for a field in the source data as so: + +```typescript +version: { + path: "GosecVersion"; +} +``` + +This primarily applies in cases where the field is simply carried over. Some fields from the source data need to be processed or transformed in some way, which will be elaborated upon later. + +#### Unfilled/Omitted and Hard Coded Fields + +First, let's assign mappings which are unfilled/omitted or are not dependent on the source data (i.e., hard coded data). This include fields like our mappings for `profiles.name` and `profiles.sha256` + +::: details GoSec-to-OHDF Mapper + +```typescript +import { ExecJSON } from "inspecjs"; +import _ from "lodash"; +import { version as HeimdallToolsVersion } from "../package.json"; +import { + BaseConverter, + ILookupPath, + impactMapping, + MappedTransform, +} from "./base-converter"; + +export class GoSecMapper extends BaseConverter { + withRaw: boolean; + + mappings: MappedTransform< + ExecJSON.Execution & { passthrough: unknown }, + ILookupPath + > = { + platform: { + name: "Heimdall Tools", + release: HeimdallToolsVersion, + }, + version: HeimdallToolsVersion, + statistics: {}, + profiles: [ + { + name: "Gosec scanner", + title: "gosec", + version: null, //Insert data + supports: [], + attributes: [], + groups: [], + status: "loaded", + controls: [ + { + key: "id", + tags: {}, //Insert data + refs: [], + source_location: {}, + title: null, //Insert data + id: "", //Insert data + desc: "", + impact: 0.5, + results: [ + { + status: ExecJSON.ControlResultStatus.Failed, + code_desc: "", //Insert data + message: null, //Insert data + start_time: "", + }, + ], + }, + ], + sha256: "", + }, + ], + passthrough: { + transformer: (data: Record): Record => { + return { + auxiliary_data: [{ name: "", data: _.omit([]) }], //Insert service name and mapped fields to be removed + ...(this.withRaw && { raw: data }), + }; + }, + }, + }; + constructor(exportJson: string, withRaw = false) { + super(JSON.parse(exportJson), true); + this.withRaw = withRaw; + } +} +``` + +::: + +::: details Remaining GoSec-to-OHDF Mapping + +```typescript +{ + profiles: [ + { + version: GosecVersion, // Version of GoSec instance + controls: [ + { + id: Issues.rule_id, // ID of the requirement + title: Issues.details, // Human readable title for the requirement + tags: { + Issues.cwe // Associated CWE for the requirement + Issues.severity, // Severity of the requirement + Issues.confidence, // Applicability of the requirement + Issues.nosec, // Whether to ignore the requirement + Issues.suppressions // Info suppression level of the requirement + }, + results: [ + { + code_desc: Issues.code, // The code failing the requirement test + message: Issues.file + Issues.line + Issues.column // All materials describing where the issue occurred + } + ] + }, + ] + }, + ], + passthrough: { + auxiliary_data: [ + { // Go source data compilation errors; Stats on GoSec scan + name: 'Gosec', + data: Golang errors, Stats + } + ], + raw + } +} +``` + +::: + +#### Simple Portable Fields + +Next, let's look at the fields which can be just simply be directly ported over from the source data like `GosecVersion`. To do this, we just need to invoke the `path` keyword from `base-converter` and feed the direct JSON object path as a value like so: + +```typescript +version: { + path: "GosecVersion"; +} +``` + +For nested fields (i.e., fields requiring traversal through parent fields), we need to have the mapper traverse into the level containing the fields we want to access. Think of this as a similar process as using `cd` to traverse through a directory to access a file. For example, if we can allow the access of fields using `path` in the `Issues` superfield in the source data as follows: + +```typescript +path: "Issues"; +``` + +Let's put this into practice and start implementing the mappings for simple fields that don't require transformation or processing: + +::: details GoSec-to-OHDF Mapper + +```typescript +import { ExecJSON } from "inspecjs"; +import _ from "lodash"; +import { version as HeimdallToolsVersion } from "../package.json"; +import { + BaseConverter, + ILookupPath, + impactMapping, + MappedTransform, +} from "./base-converter"; + +export class GoSecMapper extends BaseConverter { + withRaw: boolean; + + mappings: MappedTransform< + ExecJSON.Execution & { passthrough: unknown }, + ILookupPath + > = { + platform: { + name: "Heimdall Tools", + release: HeimdallToolsVersion, + }, + version: HeimdallToolsVersion, + statistics: {}, + profiles: [ + { + name: "Gosec scanner", + title: "gosec", + version: { path: "GosecVersion" }, + supports: [], + attributes: [], + groups: [], + status: "loaded", + controls: [ + { + path: "Issues", + key: "id", + tags: { + cwe: { path: "cwe" }, + nosec: { path: "nosec" }, + suppressions: { path: "suppressions" }, + severity: { path: "severity" }, + confidence: { path: "confidence" }, + }, + refs: [], + source_location: {}, + title: { path: "details" }, + id: { path: "rule_id" }, + desc: "", + impact: 0.5, + results: [ + { + status: ExecJSON.ControlResultStatus.Failed, + code_desc: { path: "code" }, + message: null, //Insert data + start_time: "", + }, + ], + }, + ], + sha256: "", + }, + ], + passthrough: { + transformer: (data: Record): Record => { + return { + auxiliary_data: [{ name: "", data: _.omit([]) }], //Insert service name and mapped fields to be removed + ...(this.withRaw && { raw: data }), + }; + }, + }, + }; + constructor(exportJson: string, withRaw = false) { + super(JSON.parse(exportJson), true); + this.withRaw = withRaw; + } +} +``` + +::: + +::: details Remaining GoSec-to-OHDF Mapping + +```typescript +{ + profiles: [ + { + controls: [ + { + results: [ + { + message: Issues.file + Issues.line + Issues.column // All materials describing where the issue occurred + } + ] + }, + ] + }, + ], + passthrough: { + auxiliary_data: [ + { // Go source data compilation errors; Stats on GoSec scan + name: 'Gosec', + data: Golang errors, Stats + } + ], + raw + } +} +``` + +::: + +#### Transformed/Processed Fields + +Finally, let's look at fields that require some level of processing before we can use them in the OHDF mapper. + +The fields like `Issues.file`, `Issues.line`, and `Issues.column` need to be combined to make a coherent locational message. We can combine them using a simple function that concatenates them together. The function to do this is provided: + +```typescript +function formatMessage(input: Record): string { + return `${_.get(input, "file")}, line:${_.get(input, "line")}, column:${_.get( + input, + "column" + )}`; +} +``` + +This function just pulls the three fields and joins them into a single string. + +To invoke this, we can use the `transformer` keyword and feed the function as a value for the transformer to invoke. This is implemented as follows: + +::: details GoSec-to-OHDF Mapper + +```typescript +import { ExecJSON } from "inspecjs"; +import _ from "lodash"; +import { version as HeimdallToolsVersion } from "../package.json"; +import { + BaseConverter, + ILookupPath, + impactMapping, + MappedTransform, +} from "./base-converter"; + +function formatMessage(input: Record): string { + return `${_.get(input, "file")}, line:${_.get(input, "line")}, column:${_.get( + input, + "column" + )}`; +} + +export class GoSecMapper extends BaseConverter { + withRaw: boolean; + + mappings: MappedTransform< + ExecJSON.Execution & { passthrough: unknown }, + ILookupPath + > = { + platform: { + name: "Heimdall Tools", + release: HeimdallToolsVersion, + }, + version: HeimdallToolsVersion, + statistics: {}, + profiles: [ + { + name: "Gosec scanner", + title: "gosec", + version: { path: "GosecVersion" }, + supports: [], + attributes: [], + groups: [], + status: "loaded", + controls: [ + { + path: "Issues", + key: "id", + tags: { + cwe: { path: "cwe" }, + nosec: { path: "nosec" }, + suppressions: { path: "suppressions" }, + severity: { path: "severity" }, + confidence: { path: "confidence" }, + }, + refs: [], + source_location: {}, + title: { path: "details" }, + id: { path: "rule_id" }, + desc: "", + impact: 0.5, + results: [ + { + status: ExecJSON.ControlResultStatus.Failed, + code_desc: { path: "code" }, + message: { transformer: formatMessage }, + start_time: "", + }, + ], + }, + ], + sha256: "", + }, + ], + passthrough: { + transformer: (data: Record): Record => { + return { + auxiliary_data: [{ name: "", data: _.omit([]) }], //Insert service name and mapped fields to be removed + ...(this.withRaw && { raw: data }), + }; + }, + }, + }; + constructor(exportJson: string, withRaw = false) { + super(JSON.parse(exportJson), true); + this.withRaw = withRaw; + } +} +``` + +::: + +::: details Remaining GoSec-to-OHDF Mapping + +```typescript +{ + passthrough: { + auxiliary_data: [ + { // Go source data compilation errors; Stats on GoSec scan + name: 'Gosec', + data: Golang errors, Stats + } + ], + raw + } +} +``` + +::: + +We can also combine keywords such as using `path` to traverse to a particular field and then apply a function using `transformer`. For example, we can target the `Issues.cwe` field specifically and apply the following function to create corresponding NIST 800-53s: + +```typescript +import { CweNistMapping } from "./mappings/CweNistMapping"; + +const CWE_NIST_MAPPING = new CweNistMapping(); +const DEFAULT_NIST_TAG = ["SI-2", "RA-5"]; + +function nistTag(input: Record): string[] { + const cwe = [`${_.get(input, "id")}`]; + return CWE_NIST_MAPPING.nistFilter(cwe, DEFAULT_NIST_TAG); +} +``` + +We can then correlate it with a `profiles.controls.tags.nist` field in the OHDF mapper as so: + +::: details GoSec-to-OHDF Mapper + +```typescript +import { ExecJSON } from "inspecjs"; +import _ from "lodash"; +import { version as HeimdallToolsVersion } from "../package.json"; +import { + BaseConverter, + ILookupPath, + impactMapping, + MappedTransform, +} from "./base-converter"; +import { CweNistMapping } from "./mappings/CweNistMapping"; + +const CWE_NIST_MAPPING = new CweNistMapping(); +const DEFAULT_NIST_TAG = ["SI-2", "RA-5"]; + +function nistTag(input: Record): string[] { + const cwe = [`${_.get(input, "id")}`]; + return CWE_NIST_MAPPING.nistFilter(cwe, DEFAULT_NIST_TAG); +} + +function formatMessage(input: Record): string { + return `${_.get(input, "file")}, line:${_.get(input, "line")}, column:${_.get( + input, + "column" + )}`; +} + +export class GoSecMapper extends BaseConverter { + withRaw: boolean; + + mappings: MappedTransform< + ExecJSON.Execution & { passthrough: unknown }, + ILookupPath + > = { + platform: { + name: "Heimdall Tools", + release: HeimdallToolsVersion, + }, + version: HeimdallToolsVersion, + statistics: {}, + profiles: [ + { + name: "Gosec scanner", + title: "gosec", + version: { path: "GosecVersion" }, + supports: [], + attributes: [], + groups: [], + status: "loaded", + controls: [ + { + path: "Issues", + key: "id", + tags: { + nist: { + path: "cwe", + transformer: nistTag, + }, + cwe: { path: "cwe" }, + nosec: { path: "nosec" }, + suppressions: { path: "suppressions" }, + severity: { path: "severity" }, + confidence: { path: "confidence" }, + }, + refs: [], + source_location: {}, + title: { path: "details" }, + id: { path: "rule_id" }, + desc: "", + impact: 0.5, + results: [ + { + status: ExecJSON.ControlResultStatus.Failed, + code_desc: { path: "code" }, + message: { transformer: formatMessage }, + start_time: "", + }, + ], + }, + ], + sha256: "", + }, + ], + passthrough: { + transformer: (data: Record): Record => { + return { + auxiliary_data: [{ name: "", data: _.omit([]) }], //Insert service name and mapped fields to be removed + ...(this.withRaw && { raw: data }), + }; + }, + }, + }; + constructor(exportJson: string, withRaw = false) { + super(JSON.parse(exportJson), true); + this.withRaw = withRaw; + } +} +``` + +::: + +::: details Remaining GoSec-to-OHDF Mapping + +```typescript +{ + passthrough: { + auxiliary_data: [ + { // Go source data compilation errors; Stats on GoSec scan + name: 'Gosec', + data: Golang errors, Stats + } + ], + raw + } +} +``` + +::: + +For the remaining fields that we want to place in `passthrough`, we need to use the [lodash library](https://lodash.com/docs/) to preserve the field in its entirety. In particular, we will use the `_.get(OBJECT, FIELD)` command to pull a field from the source data object. + +::: details GoSec-to-OHDF Mapper + +```typescript +import { ExecJSON } from "inspecjs"; +import _ from "lodash"; +import { version as HeimdallToolsVersion } from "../package.json"; +import { + BaseConverter, + ILookupPath, + impactMapping, + MappedTransform, +} from "./base-converter"; +import { CweNistMapping } from "./mappings/CweNistMapping"; + +const CWE_NIST_MAPPING = new CweNistMapping(); +const DEFAULT_NIST_TAG = ["SI-2", "RA-5"]; + +function nistTag(input: Record): string[] { + const cwe = [`${_.get(input, "id")}`]; + return CWE_NIST_MAPPING.nistFilter(cwe, DEFAULT_NIST_TAG); +} + +function formatMessage(input: Record): string { + return `${_.get(input, "file")}, line:${_.get(input, "line")}, column:${_.get( + input, + "column" + )}`; +} + +export class GoSecMapper extends BaseConverter { + withRaw: boolean; + + mappings: MappedTransform< + ExecJSON.Execution & { passthrough: unknown }, + ILookupPath + > = { + platform: { + name: "Heimdall Tools", + release: HeimdallToolsVersion, + }, + version: HeimdallToolsVersion, + statistics: {}, + profiles: [ + { + name: "Gosec scanner", + title: "gosec", + version: { path: "GosecVersion" }, + supports: [], + attributes: [], + groups: [], + status: "loaded", + controls: [ + { + path: "Issues", + key: "id", + tags: { + nist: { + path: "cwe", + transformer: nistTag, + }, + cwe: { path: "cwe" }, + nosec: { path: "nosec" }, + suppressions: { path: "suppressions" }, + severity: { path: "severity" }, + confidence: { path: "confidence" }, + }, + refs: [], + source_location: {}, + title: { path: "details" }, + id: { path: "rule_id" }, + desc: "", + impact: 0.5, + results: [ + { + status: ExecJSON.ControlResultStatus.Failed, + code_desc: { path: "code" }, + message: { transformer: formatMessage }, + start_time: "", + }, + ], + }, + ], + sha256: "", + }, + ], + passthrough: { + transformer: (data: Record): Record => { + return { + auxiliary_data: [ + { + name: "Gosec", + data: { + "Golang errors": _.get(data, "Golang errors"), + Stats: _.get(data, "Stats"), + }, + }, + ], + ...(this.withRaw && { raw: data }), + }; + }, + }, + }; + constructor(exportJson: string, withRaw = false) { + super(JSON.parse(exportJson), true); + this.withRaw = withRaw; + } +} +``` + +::: + +Now we have a fully implemented GoSec-to-OHDF mapper. + +### Mapper Demo - DbProtect + +This section is a demonstration for implementing an OHDF mapper that handles XML-based source data, namely DbProtect. As in the previous section, we assume that the [appropriate file set up for the mapper](#file-set-up) has been performed. + +Here is our developed mapping for DbProtect for reference: + +::: details DbProtect-to-OHDF Mapping + +```typescript +{ + platform: { + name: 'Heimdall Tools', + release: HeimdallToolsVersion, + target_id + }, + version: HeimdallToolsVersion, + statistics: { + duration + }, + profiles: [ + { + name: 'Policy', + version, + sha256, + title: 'Job Name', + maintainer, + summary: ['Organization', 'Asset', 'Asset Type', 'IP Address, Port, Instance'], + license, + copyright, + copyright_email, + supports, + attributes, + groups, + controls: [ + { + id: 'Check ID', // ID of the requirement + title: 'Check', + desc: ['Task', 'Check Category'], + descriptions, + impact: 'Risk DV', + refs, + tags: { + nist: DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS, + cci: DEFAULT_STATIC_CODE_ANALYSIS_CCI_TAGS + }, + code, + source_location, + results: [ + { + status: 'Result Status', // The result of the scan for that particular control + code_desc: 'Details', + message, + run_time, + start_time: 'Date' // Some indication of when the scan was run + } + ] + }, + ], + status: 'loaded' + }, + ], + passthrough: { + auxiliary_data: [ + { + name, + data + }, + ], + raw + } +} +``` + +::: + +::: details DbProtect Annotated Source Data + +```xml + + + + + + + + + + + + + + + + + + + + + + + + + TEST ORGANIZATION (Local DBP server) + + + Audit + + + Microsoft SQL Server + + + CONDS181 + + + 10.0.10.204, 1433, MSSQLSERVER + + + Heimdall Test scan report generation + + + DISA-STIG SQL Server 2016 V2R1-1 Audit (Built-In) + + + Fact + + + Improper Access Controls + + + Medium + + + 2986 + + + Schema ownership + + + Schema name=DatabaseMailUserRole;Database=msdb;Owner name=DatabaseMailUserRole + + + Feb 18 2021 15:57 + + ... + + +``` + +::: + +As in the previous GoSec example, most of the work involves simple references to the object path for a field in the source data. For example: + +```typescript +title: { + path: "Check"; +} +``` + +Again, some fields from the source data need to be processed or transformed in some way, which will be elaborated upon later. + +#### Unfilled/Omitted and Hard Coded Fields + +First, let's assign mappings which are unfilled/omitted or are not dependent on the source data (i.e., hard coded data). This include fields like our mappings for `platform.name` and `profiles.sha256`. Note that for our tags, we use the global method in the Heimdall repo, `getCCIsForNISTTags`, as well as the global constant `DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS`. We can also just directly put the source data into the passthrough. Also note the `compileFindings` function we wrote to deal with the strange structure of the XML, which has metadata correlated with each row in the data field. No need to worry about this function for now - just think of it as a way of assigning the metadata keys to each data row explicitly, so that our base converter has a way to easily access fields and know what they are. + +::: details DbProtect-to-OHDF Mapper + +```typescript +import { ExecJSON } from "inspecjs"; +import * as _ from "lodash"; +import { version as HeimdallToolsVersion } from "../package.json"; +import { + BaseConverter, + ILookupPath, + impactMapping, + MappedTransform, + parseXml, +} from "./base-converter"; + +function compileFindings( + input: Record +): Record { + const keys = _.get(input, "dataset.metadata.item"); + const findings = _.get(input, "dataset.data.row"); + + let output: unknown[] = []; + + if (Array.isArray(keys) && Array.isArray(findings)) { + const keyNames = keys.map((element: Record): string => { + return _.get(element, "name") as string; + }); + output = findings.map((element: Record) => { + return Object.fromEntries( + keyNames.map(function (name: string, i: number) { + return [name, _.get(element, `value[${i}]`)]; + }) + ); + }); + } + return Object.fromEntries([["data", output]]); +} + +export class DBProtectMapper extends BaseConverter { + withRaw: boolean; + + mappings: MappedTransform< + ExecJSON.Execution & { passthrough: unknown }, + ILookupPath + > = { + platform: { + name: "Heimdall Tools", + release: HeimdallToolsVersion, + }, + version: HeimdallToolsVersion, + statistics: {}, + profiles: [ + { + name: {}, + title: {}, + summary: {}, + supports: [], + attributes: [], + groups: [], + status: "loaded", + controls: [ + { + key: "id", + tags: { + nist: DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS, + cci: getCCIsForNISTTags(DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS), + }, + refs: [], + source_location: {}, + title: {}, + id: {}, + desc: {}, + impact: {}, + code: {}, + results: [ + { + status: {}, + code_desc: {}, + start_time: {}, + }, + ], + }, + ], + sha256: "", + }, + ], + passthrough: { + transformer: (data: Record): Record => { + return { ...(this.withRaw && { raw: data }) }; + }, + }, + }; + constructor(dbProtectXml: string, withRaw = false) { + super(compileFindings(parseXml(dbProtectXml))); + this.withRaw = withRaw; + } +} +``` + +::: + +::: details Remaining DbProtect-to-OHDF Mapping + +```typescript +{ + profiles: [ + { + name: "Policy", + title: "Job Name", + maintainer, + summary: [ + "Organization", + "Asset", + "Asset Type", + "IP Address, Port, Instance", + ], + license, + copyright, + copyright_email, + supports, + attributes, + groups, + controls: [ + { + id: "Check ID", // ID of the requirement + title: "Check", + desc: ["Task", "Check Category"], + descriptions, + impact: "Risk DV", + refs, + code, + source_location, + results: [ + { + status: "Result Status", // The result of the scan for that particular control + code_desc: "Details", + message, + run_time, + start_time: "Date", // Some indication of when the scan was run + }, + ], + }, + ], + }, + ]; +} +``` + +::: + +#### Simple Portable Fields + +Next, let's look at the fields which can be just simply be directly ported over from the source data like `Policy`. To do this, we just need to invoke the `path` keyword from `base-converter` and feed the direct JSON object path as a value like so: + +```typescript +name: { + path: "data.[0].Policy"; +} +``` + +Note that in this example, we must call the path with `data.[0]`, since even though the Policy is the same for all data rows, we still must specify an element to extract the data from. + +Let's put this into practice and start implementing the mappings for simple fields that don't require transformation or processing: + +::: details DbProtect-to-OHDF Mapper + +```typescript +import { ExecJSON } from "inspecjs"; +import * as _ from "lodash"; +import { version as HeimdallToolsVersion } from "../package.json"; +import { + BaseConverter, + ILookupPath, + impactMapping, + MappedTransform, + parseXml, +} from "./base-converter"; + +function compileFindings( + input: Record +): Record { + const keys = _.get(input, "dataset.metadata.item"); + const findings = _.get(input, "dataset.data.row"); + + let output: unknown[] = []; + + if (Array.isArray(keys) && Array.isArray(findings)) { + const keyNames = keys.map((element: Record): string => { + return _.get(element, "name") as string; + }); + output = findings.map((element: Record) => { + return Object.fromEntries( + keyNames.map(function (name: string, i: number) { + return [name, _.get(element, `value[${i}]`)]; + }) + ); + }); + } + return Object.fromEntries([["data", output]]); +} + +export class DBProtectMapper extends BaseConverter { + withRaw: boolean; + + mappings: MappedTransform< + ExecJSON.Execution & { passthrough: unknown }, + ILookupPath + > = { + platform: { + name: "Heimdall Tools", + release: HeimdallToolsVersion, + }, + version: HeimdallToolsVersion, + statistics: {}, + profiles: [ + { + name: { path: "data.[0].Policy" }, + title: { path: "data.[0].Job Name" }, + summary: {}, + supports: [], + attributes: [], + groups: [], + status: "loaded", + controls: [ + { + path: "data", + key: "id", + tags: { + nist: DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS, + cci: getCCIsForNISTTags(DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS), + }, + refs: [], + source_location: {}, + title: { path: "Check" }, + id: {}, + desc: {}, + impact: {}, + code: {}, + results: [ + { + status: {}, + code_desc: { path: "Details" }, + start_time: { path: "Date" }, + }, + ], + }, + ], + sha256: "", + }, + ], + passthrough: { + transformer: (data: Record): Record => { + return { ...(this.withRaw && { raw: data }) }; + }, + }, + }; + constructor(dbProtectXml: string, withRaw = false) { + super(compileFindings(parseXml(dbProtectXml))); + this.withRaw = withRaw; + } +} +``` + +::: + +::: details Remaining DbProtect-to-OHDF Mapping + +```typescript +{ + profiles: [ + { + maintainer, + summary: [ + "Organization", + "Asset", + "Asset Type", + "IP Address, Port, Instance", + ], + license, + copyright, + copyright_email, + supports, + attributes, + groups, + controls: [ + { + id: "Check ID", // ID of the requirement + desc: ["Task", "Check Category"], + descriptions, + impact: "Risk DV", + refs, + code, + source_location, + results: [ + { + status: "Result Status", // The result of the scan for that particular control + message, + run_time, + }, + ], + }, + ], + }, + ]; +} +``` + +::: + +#### Transformed/Processed Fields + +As in the previous example, there are several fields that need to be processed further. To do so, we can make use of the `transformer` field supported by the OHDF Converters library. Some simple transformers include casting the `id` field to a string, and mapping the `impact` fields properly (by creating a mapping and passing it into the `impactMapping` function in `base-converter`): + +```typescript +const IMPACT_MAPPING: Map = new Map([ + ["high", 0.7], + ["medium", 0.5], + ["low", 0.3], + ["informational", 0], +]); + +function idToString(id: unknown): string { + if (typeof id === "string" || typeof id === "number") { + return id.toString(); + } else { + return ""; + } +} +``` + +For the OHDF fields that concatenate information from multiple DbProtect fields, we must write corresponding transformers that format these strings: + +```typescript +function formatSummary(entry: unknown): string { + const text = []; + text.push(`Organization : ${_.get(entry, "Organization")}`); + text.push(`Asset : ${_.get(entry, "Check Asset")}`); + text.push(`Asset Type : ${_.get(entry, "Asset Type")}`); + text.push(`IP Address, Port, Instance : ${_.get(entry, "Asset Type")}`); + text.push( + `IP Address, Port, Instance : ${_.get( + entry, + "IP Address, Port, Instance" + )} ` + ); + return text.join("\n"); +} + +function formatDesc(entry: unknown): string { + const text = []; + text.push(`Task : ${_.get(entry, "Task")}`); + text.push(`Check Category : ${_.get(entry, "Check Category")}`); + return text.join("; "); +} +``` + +Finally, we write one more function to map the `Result Status` to the proper `ExecJSON` statuses: + +```typescript +function getStatus(input: unknown): ExecJSON.ControlResultStatus { + switch (input) { + case "Fact": + return ExecJSON.ControlResultStatus.Skipped; + case "Failed": + return ExecJSON.ControlResultStatus.Failed; + case "Finding": + return ExecJSON.ControlResultStatus.Failed; + case "Not A Finding": + return ExecJSON.ControlResultStatus.Passed; + } + return ExecJSON.ControlResultStatus.Skipped; +} +``` + +Writing out all these transformers and applying them to the mapping fed into `base-converter` looks something like this: + +:::details Full Mapper Code + +```typescript +import { ExecJSON } from "inspecjs"; +import * as _ from "lodash"; +import { version as HeimdallToolsVersion } from "../package.json"; +import { + BaseConverter, + ILookupPath, + impactMapping, + MappedTransform, + parseXml, +} from "./base-converter"; +import { + DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS, + getCCIsForNISTTags, +} from "./utils/global"; + +const IMPACT_MAPPING: Map = new Map([ + ["high", 0.7], + ["medium", 0.5], + ["low", 0.3], + ["informational", 0], +]); + +function compileFindings( + input: Record +): Record { + const keys = _.get(input, "dataset.metadata.item"); + const findings = _.get(input, "dataset.data.row"); + + let output: unknown[] = []; + + if (Array.isArray(keys) && Array.isArray(findings)) { + const keyNames = keys.map((element: Record): string => { + return _.get(element, "name") as string; + }); + output = findings.map((element: Record) => { + return Object.fromEntries( + keyNames.map(function (name: string, i: number) { + return [name, _.get(element, `value[${i}]`)]; + }) + ); + }); + } + return Object.fromEntries([["data", output]]); +} + +function formatSummary(entry: unknown): string { + const text = []; + text.push(`Organization : ${_.get(entry, "Organization")}`); + text.push(`Asset : ${_.get(entry, "Check Asset")}`); + text.push(`Asset Type : ${_.get(entry, "Asset Type")}`); + text.push(`IP Address, Port, Instance : ${_.get(entry, "Asset Type")}`); + text.push( + `IP Address, Port, Instance : ${_.get( + entry, + "IP Address, Port, Instance" + )} ` + ); + return text.join("\n"); +} + +function formatDesc(entry: unknown): string { + const text = []; + text.push(`Task : ${_.get(entry, "Task")}`); + text.push(`Check Category : ${_.get(entry, "Check Category")}`); + return text.join("; "); +} + +function getStatus(input: unknown): ExecJSON.ControlResultStatus { + switch (input) { + case "Fact": + return ExecJSON.ControlResultStatus.Skipped; + case "Failed": + return ExecJSON.ControlResultStatus.Failed; + case "Finding": + return ExecJSON.ControlResultStatus.Failed; + case "Not A Finding": + return ExecJSON.ControlResultStatus.Passed; + } + return ExecJSON.ControlResultStatus.Skipped; +} + +function idToString(id: unknown): string { + if (typeof id === "string" || typeof id === "number") { + return id.toString(); + } else { + return ""; + } +} + +export class DBProtectMapper extends BaseConverter { + withRaw: boolean; + + mappings: MappedTransform< + ExecJSON.Execution & { passthrough: unknown }, + ILookupPath + > = { + platform: { + name: "Heimdall Tools", + release: HeimdallToolsVersion, + }, + version: HeimdallToolsVersion, + statistics: {}, + profiles: [ + { + name: { path: "data.[0].Policy" }, + title: { path: "data.[0].Job Name" }, + summary: { path: "data.[0]", transformer: formatSummary }, + supports: [], + attributes: [], + groups: [], + status: "loaded", + controls: [ + { + path: "data", + key: "id", + tags: { + nist: DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS, + cci: getCCIsForNISTTags(DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS), + }, + refs: [], + source_location: {}, + title: { path: "Check" }, + id: { path: "Check ID", transformer: idToString }, + desc: { transformer: formatDesc }, + impact: { + path: "Risk DV", + transformer: impactMapping(IMPACT_MAPPING), + }, + code: { + transformer: (vulnerability: Record): string => + JSON.stringify(vulnerability, null, 2), + }, + results: [ + { + status: { path: "Result Status", transformer: getStatus }, + code_desc: { path: "Details" }, + start_time: { path: "Date" }, + }, + ], + }, + ], + sha256: "", + }, + ], + passthrough: { + transformer: (data: Record): Record => { + return { ...(this.withRaw && { raw: data }) }; + }, + }, + }; + constructor(dbProtectXml: string, withRaw = false) { + super(compileFindings(parseXml(dbProtectXml))); + this.withRaw = withRaw; + } +} +``` + +::: + +Now we have a fully implemented DbProtect-to-OHDF mapper. diff --git a/src/courses/mappers/13.md b/src/courses/mappers/13.md new file mode 100644 index 000000000..a5ead72d2 --- /dev/null +++ b/src/courses/mappers/13.md @@ -0,0 +1,49 @@ +--- +order: 13 +next: 14.md +title: Finalizing the Mapper +author: Charles Hu +--- + +## Mapper Finalization + +Now that we have a mapper developed that can convert files from \*-to-OHDF, we need to perform some final touches to make sure that our mapper works and can be integrated into OHDF Converters. + +### Using Regression Testing + +Regression testing is extremely useful, as we make changes and improvements to mappers over time (for example, if we have new sample files or if the schema of a source format changes). Moreover, internal library changes, like an update to `base-converter`, or major dependency updates could change the behavior of a mapper significantly. If a particular change affects a mapping, we will need to regenerate the sample file through the following process: + +First, we have to activate the regression testing suite that we created earlier to actually be able to automatically run and review your mapper. + +1. Uncomment the lines in the `{YOUR-SAMPLE-EXPORT}-hdf.json` file. This will allow the regression tests to automatically generate an OHDF output file whenever you run the tests. These files are needed for the tests to check the output of your mappers. The commented out lines should look similar to the following: + +```typescript +// fs.writeFileSync( +// 'sample_jsons/SKELETON_mapper/SKELETON-hdf.json', +// JSON.stringify(mapper.toHdf(), null, 2) +// ); +``` + +2. Using the terminal, `cd` into the `hdf-converters` directory and run the following command. This command will run your mapper against the sample export file in `sample_jsons` and tests to see if the output is generated as expected. However, this initial run will be used to create a baseline OHDF file for manual inspection to validate that we wrote the mapper properly and thereafter will be the comparison target for future runs to confirm that no regressions in the mapping occurred. + +```shell +yarn run test --verbose --silent=false ./test/mappers/forward/{YOUR-EXPORT-NAME-HERE}_mapper.spec.ts +``` + +3. After running this command, you should see two files named `SKELETON-hdf.json` and `SKELETON-hdf-withraw.json` appear in the `sample_jsons/SKELETON_mapper/` directory. Check these files to make sure that the OHDF mapper output looks correct and aligns with your mapping. `SKELETON-hdf.json` should match your mapping completely, while `SKELETON-hdf-withraw.json` should also include your original source data in the OHDF output file. + +4. Re-comment out the lines from step 1. + +5. Future uses of the command in step 2 will now leverage the baseline files you generated to perform regression testing. It will compare the contents of these generated files with temporary outputs created by the mapper instance to check for field alignment. Review the test output to ensure that the tests are all passing. + +### Documentation + +Document your new mapper in the `README.md` file for `hdf-converters` under the `Supported Formats` section. It should be formatted as follows: + +```markdown +{#}. [{YOUR-EXPORT-NAME-HERE}] - {MAPPER INPUT DESCRIPTION} +``` + +### GitHub Merging + +Commit your final changes and mark your pull request as 'ready for review'. You should request for a code review from members of the SAF team and edit your code as necessary. Once approved, your mapper will be merged into the main development branch and [scheduled for release](https://github.com/mitre/heimdall2/wiki/How-to-create-a-Heimdall2-release) as an officially supported conversion format for OHDF Converters. diff --git a/src/courses/mappers/14.md b/src/courses/mappers/14.md new file mode 100644 index 000000000..ce9ad2781 --- /dev/null +++ b/src/courses/mappers/14.md @@ -0,0 +1,219 @@ +--- +order: 14 +next: 15.md +title: More Practice +author: Charles Hu +--- + +The following are some practice source data sets to create mappers with. + +Before you begin, recall the steps we took previously to develop our GoSec and DBProtect mappers: + +1. Examine and break down the security data format. Remember to ask yourself the following questions while analyzing the fields of your security data: + +- What is the purpose of this field? + +- What is this field recording (i.e., metadata, requirements, requirement testing)? + +2. Correlate your security data to the OHDF schema. Focus on correlating fields from order of obvious to less obvious. + +::: details Empty OHDF Schema + +```typescript +{ + platform: { + name, + release, + target_id + }, + version, + statistics: { + duration + }, + profiles: [ + { + name, + version, + sha256, + title, + maintainer, + summary, + license, + copyright, + copyright_email, + supports, + attributes, + groups, + controls: [ + { + id, + title, + desc, + descriptions, + impact, + refs, + tags, + code, + source_location, + results: [ + { + status, + code_desc, + message, + run_time, + start_time + } + ] + }, + ], + status + }, + ], + passthrough: { + auxiliary_data: [ + { + name, + data + }, + ], + raw + } +} +``` + +::: + +3. Implement your mapping using the tools and resources provided in the [previous section](./14.md). + +## JFrog Xray + +[Based on this mapper](https://github.com/mitre/heimdall2/blob/master/libs/hdf-converters/src/jfrog-xray-mapper.ts). + +::: details Source Data + +```json +{ + "total_count": 1, + "data": [ + { + "id": "", + "severity": "High", + "summary": "Acorn regexp.js Regular Expression Validation UTF-16 Surrogate Handling Infinite Loop DoS", + "issue_type": "security", + "provider": "JFrog", + "component": "acorn", + "source_id": "npm://acorn", + "source_comp_id": "npm://acorn:5.7.3", + "component_versions": { + "id": "acorn", + "vulnerable_versions": [ + "5.5.0 ≤ Version < 5.7.4", + "6.0.0 ≤ Version < 6.4.1", + "7.0.0", + "7.1.0" + ], + "fixed_versions": ["5.7.4", "6.4.1", "7.1.1"], + "more_details": { + "cves": [ + { + "cvss_v2": "7.1/AV:N/AC:M/Au:N/C:N/I:N/A:C", + "cvss_v3": "7.5/CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H" + } + ], + "description": "Acorn contains an infinite loop condition in regexp.js that is triggered when handling UTF_16 surrogates while validating regular expressions. This may allow a context-dependent attacker to hang a process using the library.", + "provider": "JFrog" + } + }, + "edited": "2020-11-03T19:30:42-05:00" + } + ] +} +``` + +::: + +## Twistlock + +[Based on this mapper](https://github.com/mitre/heimdall2/blob/master/libs/hdf-converters/src/twistlock-mapper.ts). + +::: details Source Data + +```json +{ + "results": [ + { + "id": "sha256:b1c0237ebd29860066656372da10d8d7da33b34715986f74b3d5a7e4ba060d1b", + "name": "registry.io/test:1a7a431a105aa04632f5f5fbe8f753bd245add0a", + "distro": "Red Hat Enterprise Linux release 8.6 (Ootpa)", + "distroRelease": "RHEL8", + "digest": "sha256:c0274cb1e0c6e92d2ccc1e23bd19b7dddedbaa2da26861c225d4ad6cec5047f4", + "collections": ["All", "TEST-COLLECTION"], + "packages": [ + { + "type": "os", + "name": "nss-util", + "version": "3.67.0-7.el8_5", + "licenses": ["MPLv2.0"] + } + ], + "applications": [ + { + "name": ".net-core", + "version": "5.0.17", + "path": "/usr/lib64/dotnet/dotnet" + } + ], + "complianceDistribution": { + "critical": 0, + "high": 0, + "medium": 0, + "low": 0, + "total": 0 + }, + "complianceScanPassed": true, + "vulnerabilities": [ + { + "id": "CVE-2021-43529", + "status": "affected", + "cvss": 9.8, + "description": "DOCUMENTATION: A remote code execution flaw was found in the way NSS verifies certificates. This flaw allows an attacker posing as an SSL/TLS server to trigger this issue in a client application compiled with NSS when it tries to initiate an SSL/TLS connection. Similarly, a server application compiled with NSS, which processes client certificates, can receive a malicious certificate via a client, triggering the flaw. The highest threat to this vulnerability is confidentiality, integrity, as well as system availability. STATEMENT: The issue is not limited to TLS. Any applications that use NSS certificate verification are vulnerable; S/MIME is impacted as well. Similarly, a server application compiled with NSS, which processes client certificates, can receive a malicious certificate via a client. Firefox is not vulnerable to this flaw as it uses the mozilla::pkix for certificate verification. Thunderbird is affected when parsing email with the S/MIME signature. Thunderbird on Red Hat Enterprise Linux 8.4 and later does not need to be updated since it uses the system NSS library, but earlier Red Hat Enterprise Linux 8 extended life streams will need to update Thunderbird as well as NSS. MITIGATION: Red Hat has investigated whether a possible mitigation exists for this issue, and has not been able to identify a practical example. Please update the affec", + "severity": "critical", + "packageName": "nss-util", + "packageVersion": "3.67.0-7.el8_5", + "link": "https://access.redhat.com/security/cve/CVE-2021-43529", + "riskFactors": [ + "Remote execution", + "Attack complexity: low", + "Attack vector: network", + "Critical severity", + "Recent vulnerability" + ], + "impactedVersions": ["*"], + "publishedDate": "2021-12-01T00:00:00Z", + "discoveredDate": "2022-05-18T12:24:22Z", + "layerTime": "2022-05-16T23:12:25Z" + } + ], + "vulnerabilityDistribution": { + "critical": 1, + "high": 0, + "medium": 0, + "low": 0, + "total": 1 + }, + "vulnerabilityScanPassed": true, + "history": [ + { + "created": "2022-05-16T23:12:02Z", + "instruction": "ARG GOPROXY=http://nexus-repository-manager.nexus-repository-manager.svc.cluster.local:8081/repository/goproxy/ HTTP_PROXY=http://localhost:3128 http_proxy=http://localhost:3128" + } + ], + "scanTime": "2022-05-18T12:24:32.855444532Z", + "scanID": "6284e580d9600f8d0db159e2" + } + ], + "consoleURL": "https://twistlock.test.net/#!/monitor/vulnerabilities/images/ci?search=sha256%3Ab1c0237ebd29860066656372da10d8d7da33b34715986f74b3d5a7e4ba060d1b" +} +``` + +::: diff --git a/src/courses/mappers/15.md b/src/courses/mappers/15.md new file mode 100644 index 000000000..7884f90a0 --- /dev/null +++ b/src/courses/mappers/15.md @@ -0,0 +1,24 @@ +--- +order: 15 +next: 16.md +title: Next Steps +author: Charles Hu +--- + +## Next Steps + +### Take Our Survey + +Take our brief, anonymous [survey](https://forms.office.com/g/6JnMN8tYnD) to help improve the class. + +### Take Our Other Security Automation Framework Classes + +This class is part of a series of security automation content classes offered by the MITRE SAF team. If you found this content interesting and want to learn about more topics, like writing InSpec resources, we encourage you to check out the other classes provided (shown in the side bar on the left). + +### Check Out the Rest of MITRE SAF's Content + +MITRE SAF is a large collection of tools and techniques for security automation in addition to those discussed in this class. You can find utilities and libraries to support any step of the software development lifecycle by browsing our offerings at [saf.mitre.org](https://saf.mitre.org). Note that everything offered by MITRE SAF is open-source and available to use free of charge. You can also reference all of the resources listed from this class in the [Resources page](../../resources/README.md). + +### Contact Us + +The MITRE SAF team supports U.S. government sponsors in developing new tools for the Framework and implementing existing ones in DevSecOps pipelines. If you have a question about how you can use any of the content you saw in this class in your own environment, contact us at [saf@groups.mitre.org](mailto:saf@groups.mitre.org). diff --git a/src/courses/mappers/16.md b/src/courses/mappers/16.md new file mode 100644 index 000000000..5a5e2961f --- /dev/null +++ b/src/courses/mappers/16.md @@ -0,0 +1,291 @@ +--- +order: 16 +title: Appendix - SAF CLI Integration +author: Charles Hu +--- + +## SAF CLI Integration + +The following is a supplemental lesson on how to integrate your developed mapper with the SAF CLI. Integration with the SAF CLI will allow your mapper to be used in a command line environment, independent of Heimdall2 throughout. + +If you have not yet created a mapper, please follow the primary course and do so. Once completed, you will be able to continue with this lesson. + +### Set Up + +First, we need set up the necessary files to begin integrating your mapper with the SAF CLI. + +1. Create a development branch against the [SAF CLI repository](https://github.com/mitre/saf) and create a draft pull request for your new branch. + +2. In the `package.json` file, update the versions of `@mitre/hdf-converters` and `@mitre/heimdall-lite` to the [latest release of Heimdall2](https://github.com/mitre/heimdall2/releases). + +3. In the `src/commands/convert` directory, create a blank TypeScript file. It should be named: + +``` +{YOUR-EXPORT-NAME-HERE}2hdf.ts +``` + +4. In the `test/sample_data` directory, create a directory named `{YOUR-EXPORT-NAME-HERE}`. Underneath it, create a directory named `sample_input_report`. The file structure should now look like this: + +``` ++-- sample_data +| +-- {YOUR-EXPORT-NAME-HERE} +| | +-- sample_input_report +``` + +5. Place your sample export under the `sample_input_report` directory. Your sample export should be genericized to avoid any leaking of sensitive information. Under the `{YOUR-EXPORT-NAME-HERE}` directory, place your output OHDF files generated during the original testing phase of your mapper development. The file structure should now look like this: + +``` ++-- sample_data +| +-- {YOUR-EXPORT-NAME-HERE} +| | +-- sample_input_report +| | | +-- {YOUR-SAMPLE-EXPORT} +| | +-- {YOUR-EXPORT-NAME-HERE}-hdf.json +| | +-- {YOUR-EXPORT-NAME-HERE}-hdf-withraw.json +``` + +6. In the `test/commands/convert` directory, create a blank TypeScript file. It should be named: + +``` +{YOUR-EXPORT-NAME-HERE}2hdf.test.ts +``` + +### Integration + +::: note APIs +If your security tool provides an API instead of a security data export, you will need to explore the API to determine what information may need to be collected and collated for use in the `convert` command. For further guidance, refer to [previous API-based command implementations](https://github.com/mitre/saf/tree/main/src/commands/convert) or contact the SAF team for help. +::: + +With file set up out of the way, we can begin filling out the necessary files to create a `saf convert` command for our mapper. + +::: info +For further guidance on writing good usage strings for command line interfaces, refer [here](http://docopt.org/). +::: + +1. Insert the skeleton convert command file (see below) in the file `{YOUR-EXPORT-NAME-HERE}2hdf.ts` which you created earlier. Replace names (`SKELETON` by default) as necessary. + +::: details Skeleton Convert Command File + +```typescript +import {Command, Flags} from '@oclif/core' +import fs from 'fs' +import {SKELETONMapper as Mapper} from '@mitre/hdf-converters' +import {checkSuffix} from '../../utils/global' + +export default class SKELETON2HDF extends Command { + static usage = 'convert SKELETON2hdf -i -o ' + + static description = 'Translate a SKELETON output file into an HDF results set' + + static examples = ['saf convert SKELETON2hdf -i SKELETON.json -o output-hdf-name.json'] + + static flags = { + help: Flags.help({char: 'h'}), + input: Flags.string({char: 'i', required: true, description: 'Input SKELETON file'}), + output: Flags.string({char: 'o', required: true, description: 'Output HDF file'}), + 'with-raw': Flags.boolean({char: 'w', required: false}), + } + + async run() { + const {flags} = await this.parse(SKELETON2HDF) + const input = fs.readFileSync(flags.input, 'utf8') + + const converter = new Mapper(input, flags.['with-raw']) + fs.writeFileSync(checkSuffix(flags.output), JSON.stringify(converter.toHdf())) + } +} +``` + +::: + +2. Insert the appropriate skeleton convert command test file (see below) in the file `{YOUR-EXPORT-NAME-HERE}2hdf.test.ts` which you created earlier. Replace names (`SKELETON` by default) as necessary. + +::: details JSON Skeleton Convert Command Test File + +```typescript +import { expect, test } from "@oclif/test"; +import tmp from "tmp"; +import path from "path"; +import fs from "fs"; +import { omitHDFChangingFields } from "../utils"; + +describe("Test SKELETON", () => { + const tmpobj = tmp.dirSync({ unsafeCleanup: true }); + + test + .stdout() + .command([ + "convert SKELETON2hdf", + "-i", + path.resolve( + "./test/sample_data/SKELETON/sample_input_report/SKELETON_sample.json" + ), + "-o", + `${tmpobj.name}/SKELETONtest.json`, + ]) + .it("hdf-converter output test", () => { + const converted = JSON.parse( + fs.readFileSync(`${tmpobj.name}/SKELETONtest.json`, "utf8") + ); + const sample = JSON.parse( + fs.readFileSync( + path.resolve("./test/sample_data/SKELETON/SKELETON-hdf.json"), + "utf8" + ) + ); + expect(omitHDFChangingFields(converted)).to.eql( + omitHDFChangingFields(sample) + ); + }); +}); + +describe("Test SKELETON withraw flag", () => { + const tmpobj = tmp.dirSync({ unsafeCleanup: true }); + + test + .stdout() + .command([ + "convert SKELETON2hdf", + "-i", + path.resolve( + "./test/sample_data/SKELETON/sample_input_report/SKELETON_sample.json" + ), + "-o", + `${tmpobj.name}/SKELETONtest.json`, + "-w", + ]) + .it("hdf-converter withraw output test", () => { + const converted = JSON.parse( + fs.readFileSync(`${tmpobj.name}/SKELETONtest.json`, "utf8") + ); + const sample = JSON.parse( + fs.readFileSync( + path.resolve("./test/sample_data/SKELETON/SKELETON-hdf-withraw.json"), + "utf8" + ) + ); + expect(omitHDFChangingFields(converted)).to.eql( + omitHDFChangingFields(sample) + ); + }); +}); +``` + +::: + +::: details XML Skeleton Convert Command Test File + +```typescript +import { expect, test } from "@oclif/test"; +import tmp from "tmp"; +import path from "path"; +import fs from "fs"; +import { omitHDFChangingFields } from "../utils"; + +describe("Test SKELETON", () => { + const tmpobj = tmp.dirSync({ unsafeCleanup: true }); + + test + .stdout() + .command([ + "convert SKELETON2hdf", + "-i", + path.resolve( + "./test/sample_data/SKELETON/sample_input_report/SKELETON_sample.xml" + ), + "-o", + `${tmpobj.name}/SKELETONtest.json`, + ]) + .it("hdf-converter output test", () => { + const converted = JSON.parse( + fs.readFileSync(`${tmpobj.name}/SKELETONtest.json`, "utf8") + ); + const sample = JSON.parse( + fs.readFileSync( + path.resolve("./test/sample_data/SKELETON/SKELETON-hdf.json"), + "utf8" + ) + ); + expect(omitHDFChangingFields(converted)).to.eql( + omitHDFChangingFields(sample) + ); + }); +}); + +describe("Test SKELETON withraw flag", () => { + const tmpobj = tmp.dirSync({ unsafeCleanup: true }); + + test + .stdout() + .command([ + "convert SKELETON2hdf", + "-i", + path.resolve( + "./test/sample_data/SKELETON/sample_input_report/SKELETON_sample.xml" + ), + "-o", + `${tmpobj.name}/SKELETONtest.json`, + "-w", + ]) + .it("hdf-converter withraw output test", () => { + const converted = JSON.parse( + fs.readFileSync(`${tmpobj.name}/SKELETONtest.json`, "utf8") + ); + const sample = JSON.parse( + fs.readFileSync( + path.resolve("./test/sample_data/SKELETON/SKELETON-hdf-withraw.json"), + "utf8" + ) + ); + expect(omitHDFChangingFields(converted)).to.eql( + omitHDFChangingFields(sample) + ); + }); +}); +``` + +::: + +3. Navigate to the `index.ts` file under the `src/commands/convert` directory. Import your mapper using the existing import block as follows: + +```typescript +import { + ASFFResults, + ... + {YOUR-MAPPER-CLASS-HERE} +} from '@mitre/hdf-converters' +``` + +4. Under the switch block in the `getFlagsForInputFile` function, add your mapper class as it is defined for fingerprinting for the generic convert command. If the convert command for your mapper has any additional flags beyond the standard `--input` and `--output` flags, return the entire flag block in the switch case. This is demonstrated as follows: + +```typescript +switch (Convert.detectedType) { + ... + case {YOUR-EXPORT-SERVICE-NAME-HERE}: + return {YOUR-CLI-CONVERT-CLASS}.flags //Only add if special flags exist + ... + return {} +} +``` + +5. Edit the `README.md` file to reflect your newly added conversion command under the `Convert To HDF` section. It should be formatted as follows: + +````markdown +##### {YOUR-EXPORT-NAME-HERE} to HDF + +\``` +convert {YOUR-EXPORT-NAME-HERE}2hdf Translate a {YOUR-EXPORT-NAME-HERE} results {EXPORT-TYPE} into an HDF results set + +USAGE +$ saf convert {YOUR-EXPORT-NAME-HERE}hdf -i <{INPUT-NAME}> -o + +OPTIONS +-i, --input=input Input {EXPORT-TYPE} File +-o, --output=output Output HDF JSON File +-w, --with-raw Include raw input file in HDF JSON file + +EXAMPLES +$ saf convert {YOUR-EXPORT-NAME-HERE}2hdf -i {INPUT-NAME} -o output-hdf-name.json +\``` +```` + +5. Commit your changes and mark your pull request as 'ready for review'. You should request for a code review from members of the SAF team and edit your code as necessary. Once approved, merged, and [released](https://github.com/mitre/saf/wiki/Creating-a-Release-of-the-SAF-CLI), your mapper will be callable using the SAF CLI. diff --git a/src/courses/mappers/README.md b/src/courses/mappers/README.md new file mode 100644 index 000000000..89e5cd65a --- /dev/null +++ b/src/courses/mappers/README.md @@ -0,0 +1,29 @@ +--- +order: 1 +next: 02.md +title: OHDF Mapper Development Class +shortTitle: OHDF Mapper Development +author: Charles Hu +--- + +## Overview + +The purpose of this class is to provide you with the foundational knowledge needed to create data format mappers for mapping data to the [OASIS Heimdall Data Format (OHDF)](https://saf.mitre.org/framework/normalize). In the following course, you will be taught the art and science of schema translation through a combination of both guided explanations and hands-on demonstrations. + +The class is split into 2 sections. The first portion will introduce you to the OHDF schema and its specificities, while the second portion will show you how to apply the OHDF schema in mappers for conversions from any security data format to OHDF (\*-to-OHDF). + +### Objectives + +By the end of this class, you should be able to achieve the following objectives: + +- Understand the OHDF schema and its components. +- Be able to identify and break down key components of any given security tool export. +- Correlate schema fields of your security data export to schema fields in OHDF. +- Create a \*-to-OHDF mapper. +- Integrate your mapper into the [OHDF Converters](https://github.com/mitre/heimdall2/tree/master/libs/hdf-converters) library and the larger Security Automation Framework (SAF) tool suite. + +:::info Mapper Examples +This course will use examples and code derived from previously developed OHDF mappers which can be [found here](https://github.com/mitre/heimdall2/tree/master/libs/hdf-converters). + +You are encouraged to regularly refer back to this GitHub repository to check your understanding of OHDF mappers and see how mappers are actually developed and implemented in a production environment. +::: diff --git a/src/courses/mappers/img/basic_mapping.png b/src/courses/mappers/img/basic_mapping.png new file mode 100644 index 000000000..53edca089 Binary files /dev/null and b/src/courses/mappers/img/basic_mapping.png differ diff --git a/src/courses/mappers/img/fingerprinting.png b/src/courses/mappers/img/fingerprinting.png new file mode 100644 index 000000000..8a09251f7 Binary files /dev/null and b/src/courses/mappers/img/fingerprinting.png differ diff --git a/src/courses/mappers/img/heimdall_view.png b/src/courses/mappers/img/heimdall_view.png new file mode 100644 index 000000000..0ddd9b500 Binary files /dev/null and b/src/courses/mappers/img/heimdall_view.png differ diff --git a/src/courses/mappers/img/mapper_mapping.png b/src/courses/mappers/img/mapper_mapping.png new file mode 100644 index 000000000..589f6678c Binary files /dev/null and b/src/courses/mappers/img/mapper_mapping.png differ diff --git a/src/courses/mappers/img/mapping.png b/src/courses/mappers/img/mapping.png new file mode 100644 index 000000000..f8482c24c Binary files /dev/null and b/src/courses/mappers/img/mapping.png differ diff --git a/src/courses/mappers/img/mapping_bad.png b/src/courses/mappers/img/mapping_bad.png new file mode 100644 index 000000000..b4da76724 Binary files /dev/null and b/src/courses/mappers/img/mapping_bad.png differ diff --git a/src/courses/mappers/img/saf_cli_features.png b/src/courses/mappers/img/saf_cli_features.png new file mode 100644 index 000000000..91d4b9bc0 Binary files /dev/null and b/src/courses/mappers/img/saf_cli_features.png differ diff --git a/src/courses/mappers/img/saf_security_validation_lifecycle.png b/src/courses/mappers/img/saf_security_validation_lifecycle.png new file mode 100644 index 000000000..321ce20e0 Binary files /dev/null and b/src/courses/mappers/img/saf_security_validation_lifecycle.png differ diff --git a/src/courses/mappers/img/scan_to_assessor.png b/src/courses/mappers/img/scan_to_assessor.png new file mode 100644 index 000000000..277db2c78 Binary files /dev/null and b/src/courses/mappers/img/scan_to_assessor.png differ diff --git a/src/courses/mappers/img/testing.png b/src/courses/mappers/img/testing.png new file mode 100644 index 000000000..594198c7a Binary files /dev/null and b/src/courses/mappers/img/testing.png differ diff --git a/src/courses/mappers/nonguided-labs.txt b/src/courses/mappers/nonguided-labs.txt new file mode 100644 index 000000000..b840ff1bb --- /dev/null +++ b/src/courses/mappers/nonguided-labs.txt @@ -0,0 +1,810 @@ +TEMP STORAGE FOR NONGUIDED LABS TO BE ADDED LATER + +### Mapper Demo - JFrog + +This section is yet another example of implementing an OHDF mapper, namely the JFrog-Xray mapper. Again, we will assume that the [appropriate file set up for the mapper](#file-set-up) has been performed. Because we have already gone through several examples, this section will not explain too many things in detail. The goal is to simply provide more examples for reference. + +Here is our developed mapping for JFrog-Xray for reference: + +::: details JFrog-to-OHDF Mapping + +```typescript +{ + platform: { + name: 'Heimdall Tools', + release: HeimdallToolsVersion, + target_id + }, + version: HeimdallToolsVersion, + statistics: { + duration + }, + profiles: [ + { + name, + version, + sha256, + title, + maintainer, + summary, + license, + copyright, + copyright_email, + supports, + attributes, + groups, + controls: [ + { + id: 'data.id', // If ID empty, hash the summary + title: 'data.summary', + desc: 'component_versions.more_details', + descriptions, + impact: 'data.severity', + refs, + tags: { + nist: 'data.component_versions.more_details.cves[0].cwe', // Map to NIST + cci: 'data.component_versions.more_details.cves[0].cwe', // Map to CCI + cweid: 'data.component_versions.more_details.cves[0].cwe' + }, + code, + source_location, + results: [ + { + status: 'Failed', // All reported findings are failures + code_desc: ['data.source_comp_id', 'data.component_versions.vulnerable_versions', 'data.component_versions.fixed_versions', 'data.issue_type', 'data.provider'], + message, + run_time, + start_time + } + ] + }, + ], + status: 'loaded' + }, + ], + passthrough: { + auxiliary_data: [ + { + name, + data + }, + ], + raw + } +} +``` + +::: + +::: details JFrog Source Data + +```json +{ + "total_count": 30, + "data": [ + { + "id": "", + "severity": "High", + "summary": "Acorn regexp.js Regular Expression Validation UTF-16 Surrogate Handling Infinite Loop DoS", + "issue_type": "security", + "provider": "JFrog", + "component": "acorn", + "source_id": "npm://acorn", + "source_comp_id": "npm://acorn:5.7.3", + "component_versions": { + "id": "acorn", + "vulnerable_versions": [ + "5.5.0 ≤ Version < 5.7.4", + "6.0.0 ≤ Version < 6.4.1", + "7.0.0", + "7.1.0" + ], + "fixed_versions": ["5.7.4", "6.4.1", "7.1.1"], + "more_details": { + "cves": [ + { + "cvss_v2": "7.1/AV:N/AC:M/Au:N/C:N/I:N/A:C", + "cvss_v3": "7.5/CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H" + } + ], + "description": "Acorn contains an infinite loop condition in regexp.js that is triggered when handling UTF_16 surrogates while validating regular expressions. This may allow a context-dependent attacker to hang a process using the library.", + "provider": "JFrog" + } + }, + "edited": "2020-11-03T19:30:42-05:00" + } + ] +} +``` + +::: + +We will now go over the mapper code that allows us to shape this data properly. Firstly, let us handle the ID. Sometimes, JFrog source files will not have an ID field. In this case, it is appropriate to generate a hash based on the `summary` as follows: + +```typescript +function hashId(vulnerability: unknown): string { + if (_.get(vulnerability, "id") === "") { + return generateHash( + (_.get(vulnerability, "summary") as unknown as string).toString(), + "md5" + ); + } else { + return _.get(vulnerability, "id") as unknown as string; + } +} +``` + +As with most mappers, we must also define an impact mapping: + +```typescript +const IMPACT_MAPPING: Map = new Map([ + ["high", 0.7], + ["medium", 0.5], + ["low", 0.3], +]); +``` + +Recall that we wanted to format the `desc` field based on the data in the source file's `component_versions.more_details`. We can write a simple transformer function to concatenate all these keys: + +```typescript +function formatDesc(vulnerability: unknown): string { + const text = []; + if (_.has(vulnerability, "description")) { + text.push( + (_.get(vulnerability, "description") as unknown as string).toString() + ); + } + if (_.has(vulnerability, "cves")) { + const re1 = /":/gi; + const re2 = /,/gi; + text.push( + `cves: ${JSON.stringify(_.get(vulnerability, "cves")) + .replace(re1, '"=>') + .replace(re2, ", ")}` + ); + } + return text.join("
"); +} +``` + +We also must properly handle the mappings to NIST and CCI tags. Luckily, we can make use of the library's existing mappings, as well as an existing global function called `getCCIsForNISTTags()` in `global.ts`. + +```typescript +const CWE_PATH = "component_versions.more_details.cves[0].cwe"; +const CWE_NIST_MAPPING = new CweNistMapping(); + +function nistTag(identifier: Record): string[] { + const identifiers: string[] = []; + if (Array.isArray(identifier)) { + identifier.forEach((element) => { + if (element.split("CWE-")[1]) { + identifiers.push(element.split("CWE-")[1]); + } + }); + } + return CWE_NIST_MAPPING.nistFilter( + identifiers, + DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS + ); +} +``` + +Finally, we can implement the transformer function for the `code_desc`. Some of these fields are optional, so it is important to implement a base case to ensure our `code_desc` outputs are all consistent. + +```typescript +function formatCodeDesc(vulnerability: unknown): string { + const codeDescArray: string[] = []; + const re = /,/gi; + if (_.has(vulnerability, "source_comp_id")) { + codeDescArray.push( + `source_comp_id : ${_.get(vulnerability, "source_comp_id")}` + ); + } else { + codeDescArray.push("source_comp_id : "); + } + if (_.has(vulnerability, "component_versions.vulnerable_versions")) { + codeDescArray.push( + `vulnerable_versions : ${JSON.stringify( + _.get(vulnerability, "component_versions.vulnerable_versions") + )}` + ); + } else { + codeDescArray.push("vulnerable_versions : "); + } + if (_.has(vulnerability, "component_versions.fixed_versions")) { + codeDescArray.push( + `fixed_versions : ${JSON.stringify( + _.get(vulnerability, "component_versions.fixed_versions") + )}` + ); + } else { + codeDescArray.push("fixed_versions : "); + } + if (_.has(vulnerability, "issue_type")) { + codeDescArray.push(`issue_type : ${_.get(vulnerability, "issue_type")}`); + } else { + codeDescArray.push("issue_type : "); + } + if (_.has(vulnerability, "provider")) { + codeDescArray.push(`provider : ${_.get(vulnerability, "provider")}`); + } else { + codeDescArray.push("provider : "); + } + return codeDescArray.join("\n").replace(re, ", "); +} +``` + +All that's left is to incoporate all these functions into our `base-converter` structure, write some smaller transformer functions for the simpler data formatting tasks, and combine it all together! + +:::details Full Mapper Code + +```typescript +import { ExecJSON } from "inspecjs"; +import * as _ from "lodash"; +import { version as HeimdallToolsVersion } from "../package.json"; +import { + BaseConverter, + generateHash, + ILookupPath, + impactMapping, + MappedTransform, +} from "./base-converter"; +import { CweNistMapping } from "./mappings/CweNistMapping"; +import { + DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS, + getCCIsForNISTTags, +} from "./utils/global"; + +// Constants +const IMPACT_MAPPING: Map = new Map([ + ["high", 0.7], + ["medium", 0.5], + ["low", 0.3], +]); + +const CWE_PATH = "component_versions.more_details.cves[0].cwe"; + +const CWE_NIST_MAPPING = new CweNistMapping(); + +// Transformation Functions +function hashId(vulnerability: unknown): string { + if (_.get(vulnerability, "id") === "") { + return generateHash( + (_.get(vulnerability, "summary") as unknown as string).toString(), + "md5" + ); + } else { + return _.get(vulnerability, "id") as unknown as string; + } +} +function formatDesc(vulnerability: unknown): string { + const text = []; + if (_.has(vulnerability, "description")) { + text.push( + (_.get(vulnerability, "description") as unknown as string).toString() + ); + } + if (_.has(vulnerability, "cves")) { + const re1 = /":/gi; + const re2 = /,/gi; + text.push( + `cves: ${JSON.stringify(_.get(vulnerability, "cves")) + .replace(re1, '"=>') + .replace(re2, ", ")}` + ); + } + return text.join("
"); +} +function formatCodeDesc(vulnerability: unknown): string { + const codeDescArray: string[] = []; + const re = /,/gi; + if (_.has(vulnerability, "source_comp_id")) { + codeDescArray.push( + `source_comp_id : ${_.get(vulnerability, "source_comp_id")}` + ); + } else { + codeDescArray.push("source_comp_id : "); + } + if (_.has(vulnerability, "component_versions.vulnerable_versions")) { + codeDescArray.push( + `vulnerable_versions : ${JSON.stringify( + _.get(vulnerability, "component_versions.vulnerable_versions") + )}` + ); + } else { + codeDescArray.push("vulnerable_versions : "); + } + if (_.has(vulnerability, "component_versions.fixed_versions")) { + codeDescArray.push( + `fixed_versions : ${JSON.stringify( + _.get(vulnerability, "component_versions.fixed_versions") + )}` + ); + } else { + codeDescArray.push("fixed_versions : "); + } + if (_.has(vulnerability, "issue_type")) { + codeDescArray.push(`issue_type : ${_.get(vulnerability, "issue_type")}`); + } else { + codeDescArray.push("issue_type : "); + } + if (_.has(vulnerability, "provider")) { + codeDescArray.push(`provider : ${_.get(vulnerability, "provider")}`); + } else { + codeDescArray.push("provider : "); + } + return codeDescArray.join("\n").replace(re, ", "); +} +function nistTag(identifier: Record): string[] { + const identifiers: string[] = []; + if (Array.isArray(identifier)) { + identifier.forEach((element) => { + if (element.split("CWE-")[1]) { + identifiers.push(element.split("CWE-")[1]); + } + }); + } + return CWE_NIST_MAPPING.nistFilter( + identifiers, + DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS + ); +} + +// Mappings +export class JfrogXrayMapper extends BaseConverter { + withRaw: boolean; + + mappings: MappedTransform< + ExecJSON.Execution & { passthrough: unknown }, + ILookupPath + > = { + platform: { + name: "Heimdall Tools", + release: HeimdallToolsVersion, + }, + version: HeimdallToolsVersion, + statistics: {}, + profiles: [ + { + name: "JFrog Xray Scan", + title: "JFrog Xray Scan", + summary: "Continuous Security and Universal Artifact Analysis", + supports: [], + attributes: [], + groups: [], + status: "loaded", + controls: [ + { + path: "data", + key: "id", + tags: { + cci: { + path: CWE_PATH, + transformer: (identifier: Record) => + getCCIsForNISTTags(nistTag(identifier)), + }, + nist: { + path: CWE_PATH, + transformer: nistTag, + }, + cweid: { path: CWE_PATH }, + }, + refs: [], + source_location: {}, + id: { transformer: hashId }, + title: { path: "summary" }, + desc: { + path: "component_versions.more_details", + transformer: formatDesc, + }, + impact: { + path: "severity", + transformer: impactMapping(IMPACT_MAPPING), + }, + code: { + transformer: (vulnerability: Record): string => { + return JSON.stringify(vulnerability, null, 2); + }, + }, + results: [ + { + status: ExecJSON.ControlResultStatus.Failed, + code_desc: { transformer: formatCodeDesc }, + start_time: "", + }, + ], + }, + ], + sha256: "", + }, + ], + passthrough: { + transformer: (data: Record): Record => { + return { + auxiliary_data: [ + { + name: "JFrog Xray", + data: _.pick(data, ["total_count"]), + }, + ], + ...(this.withRaw && { raw: data }), + }; + }, + }, + }; + constructor(xrayJson: string, withRaw = false) { + super(JSON.parse(xrayJson), true); + this.withRaw = withRaw; + } +} +``` + +::: + +Now we have a fully implemented JFrog-to-OHDF mapper. + +### Mapper Demo - SARIF + +Here is one last example of implementing an OHDF mapper - the SARIF mapper. As in previous examples, we will assume that the [appropriate file set up for the mapper](#file-set-up) has been performed. Like with the JFrog example, we will not be going into too much detail regarding the mapping itself. The goal is to simply provide more examples for reference. + +Here is our developed mapping for SARIF for reference: + +::: details SARIF-to-OHDF Mapping + +```typescript +{ + platform: { + name: 'Heimdall Tools', + release: HeimdallToolsVersion, + target_id + }, + version: HeimdallToolsVersion, + statistics: { + duration + }, + profiles: [ + { + name, + version: 'version', + sha256, + title, + maintainer, + summary, + license, + copyright, + copyright_email, + supports, + attributes, + groups, + controls: [ + { + id: 'results.ruleId', // If ID empty, hash the summary + title: 'results.message.text', + desc: 'results.message.text', + descriptions, + impact: 'results.level', + refs, + tags: { + nist: 'results.message.text', // Map to NIST + cci: 'results.message.text', // Map to CCI + cwe: 'results.message.text' + }, + code, + source_location: ['results.locations[0].physicalLocation.artifactLocation.uri', 'results.locations[0].physicalLocation.region.startLine'], + results: [ + { + status: 'Failed', // All reported findings are failures + code_desc: 'results.locations[0].physicalLocation', + message, + run_time, + start_time + } + ] + }, + ], + status: 'loaded' + }, + ], + passthrough: { + auxiliary_data: [ + { + name, + data + }, + ], + raw + } +} +``` + +::: + +::: details SARIF Source Data + +```json +{ + "$schema": "https://schemastore.azurewebsites.net/schemas/json/sarif-2.1.0-rtm.5.json", + "version": "2.1.0", + "runs": [ + { + "results": [ + { + "ruleId": "FF1014", + "level": "error", + "message": { + "text": "buffer/gets: Does not check for buffer overflows (CWE-120, CWE-20)." + }, + "locations": [ + { + "physicalLocation": { + "artifactLocation": { + "uri": "test/test-patched.c", + "uriBaseId": "SRCROOT" + }, + "region": { + "startLine": 36, + "startColumn": 2, + "endColumn": 10, + "snippet": { + "text": " gets(f);" + } + } + } + } + ], + "fingerprints": { + "contextHash/v1": "6a5bb383fb44030b0d9428b17359e94ba3979bc1ce702be450427f85592c649a" + }, + "rank": 1 + } + ] + } + ] +} +``` + +::: + +First, let us define some common constants that might help us with our code. + +```typescript +const MESSAGE_TEXT = "message.text"; +const CWE_NIST_MAPPING = new CweNistMapping(); +``` + +We will now outline the `IMPACT_MAPPING` for the SARIF mapper: + +```typescript +const IMPACT_MAPPING: Map = new Map([ + ["error", 0.7], + ["warning", 0.5], + ["note", 0.3], +]); + +function impactMapping(severity: unknown): number { + if (typeof severity === "string" || typeof severity === "number") { + return IMPACT_MAPPING.get(severity.toString().toLowerCase()) || 0.1; + } else { + return 0.1; + } +} +``` + +Now, note that we will need to extract the CWEs from the message text, as well as map them to the proper NIST tags. For the CCIs, we can again use the global function `getCCIsForNISTTags()`. + +```typescript +function extractCwe(text: string): string[] { + let output = text.split("(").slice(-1)[0].slice(0, -2).split(", "); + if (output.length === 1) { + output = text.split("(").slice(-1)[0].slice(0, -2).split("!/"); + } + return output; +} +function nistTag(text: string): string[] { + let identifiers = extractCwe(text); + identifiers = identifiers.map((element) => element.split("-")[1]); + return CWE_NIST_MAPPING.nistFilter( + identifiers, + DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS + ); +} +``` + +Our last standalone transformer function will format our code description. This part is up to interpretation, but we decided to just concatenate them all with colon-separated key-value pairs + +```typescript +function formatCodeDesc(input: unknown): string { + const output = []; + output.push(`URL : ${_.get(input, "artifactLocation.uri")}`); + output.push(`LINE : ${_.get(input, "region.startLine")}`); + output.push(`COLUMN : ${_.get(input, "region.startColumn")}`); + return output.join(" "); +} +``` + +Now, all that's left is to combine our functions, write some mini-transformers for the small data formatting, and plug it all into the base converter! + +:::details Full Mapper Code + +```typescript +import { ExecJSON } from "inspecjs"; +import * as _ from "lodash"; +import { version as HeimdallToolsVersion } from "../package.json"; +import { BaseConverter, ILookupPath, MappedTransform } from "./base-converter"; +import { CweNistMapping } from "./mappings/CweNistMapping"; +import { + DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS, + getCCIsForNISTTags, +} from "./utils/global"; + +const IMPACT_MAPPING: Map = new Map([ + ["error", 0.7], + ["warning", 0.5], + ["note", 0.3], +]); +const MESSAGE_TEXT = "message.text"; +const CWE_NIST_MAPPING = new CweNistMapping(); + +function extractCwe(text: string): string[] { + let output = text.split("(").slice(-1)[0].slice(0, -2).split(", "); + if (output.length === 1) { + output = text.split("(").slice(-1)[0].slice(0, -2).split("!/"); + } + return output; +} +function impactMapping(severity: unknown): number { + if (typeof severity === "string" || typeof severity === "number") { + return IMPACT_MAPPING.get(severity.toString().toLowerCase()) || 0.1; + } else { + return 0.1; + } +} +function formatCodeDesc(input: unknown): string { + const output = []; + output.push(`URL : ${_.get(input, "artifactLocation.uri")}`); + output.push(`LINE : ${_.get(input, "region.startLine")}`); + output.push(`COLUMN : ${_.get(input, "region.startColumn")}`); + return output.join(" "); +} +function nistTag(text: string): string[] { + let identifiers = extractCwe(text); + identifiers = identifiers.map((element) => element.split("-")[1]); + return CWE_NIST_MAPPING.nistFilter( + identifiers, + DEFAULT_STATIC_CODE_ANALYSIS_NIST_TAGS + ); +} + +export class SarifMapper extends BaseConverter { + withRaw: boolean; + + mappings: MappedTransform< + ExecJSON.Execution & { passthrough: unknown }, + ILookupPath + > = { + platform: { + name: "Heimdall Tools", + release: HeimdallToolsVersion, + target_id: "Static Analysis Results Interchange Format", + }, + version: HeimdallToolsVersion, + statistics: {}, + profiles: [ + { + path: "runs", + name: "SARIF", + version: { path: "$.version" }, + title: "Static Analysis Results Interchange Format", + supports: [], + attributes: [], + groups: [], + status: "loaded", + controls: [ + { + path: "results", + key: "id", + tags: { + cci: { + path: MESSAGE_TEXT, + transformer: (data: string) => + getCCIsForNISTTags(nistTag(data)), + }, + nist: { path: MESSAGE_TEXT, transformer: nistTag }, + cwe: { + path: MESSAGE_TEXT, + transformer: extractCwe, + }, + }, + refs: [], + source_location: { + transformer: (control: unknown) => { + return _.omitBy( + { + ref: _.get( + control, + "locations[0].physicalLocation.artifactLocation.uri" + ), + line: _.get( + control, + "locations[0].physicalLocation.region.startLine" + ), + }, + (value) => value === "" + ); + }, + }, + title: { + path: MESSAGE_TEXT, + transformer: (text: unknown): string => { + if (typeof text === "string") { + return text.split(": ")[0]; + } else { + return ""; + } + }, + }, + id: { path: "ruleId" }, + desc: { + path: MESSAGE_TEXT, + transformer: (text: unknown): string => { + if (typeof text === "string") { + return text.split(": ")[1]; + } else { + return ""; + } + }, + }, + impact: { path: "level", transformer: impactMapping }, + code: { + transformer: (vulnerability: Record): string => + JSON.stringify(vulnerability, null, 2), + }, + results: [ + { + status: ExecJSON.ControlResultStatus.Failed, + code_desc: { + path: "locations[0].physicalLocation", + transformer: formatCodeDesc, + }, + + start_time: "", + }, + ], + }, + ], + sha256: "", + }, + ], + passthrough: { + transformer: (data: Record): Record => { + let runsData = _.get(data, "runs"); + if (Array.isArray(runsData)) { + runsData = runsData.map((run: Record) => + _.omit(run, ["results"]) + ); + } + return { + auxiliary_data: [ + { + name: "SARIF", + data: { + $schema: _.get(data, "$schema"), + runs: runsData, + }, + }, + ], + ...(this.withRaw && { raw: data }), + }; + }, + }, + }; + constructor(sarifJson: string, withRaw = false) { + super(JSON.parse(sarifJson)); + this.withRaw = withRaw; + } +} +``` + +::: + +Now we have a fully implemented SARIF-to-OHDF mapper.