Merge branch 'develop' into 'main'

ATLAS Data 4.1.0 - case study refreshes and new technique

.gitlab-ci.yml

@@ -39,12 +39,28 @@ lint yaml:
         - "*.yaml"
         - "*.yml"
 
+check spelling and syntax:
+  stage: test
+  script:
+    - pip install -r tests/requirements.txt
+    # Run tests with minimal console output, produce report, and error on warnings
+    - pytest tests/test_syntax.py --tb=line --junitxml=report.xml -W error::UserWarning
+  allow_failure:
+    exit_codes:
+      - 1   # Tests were collected and run but some tests failed https://docs.pytest.org/en/latest/reference/exit-codes.html
+  rules:
+    - changes:
+        - data/*.yaml   # Source data was updated
+        - tests/*.py    # Any tests changed
+        - tests/custom_words.txt  # Exclusion words updated
+        - conftest.py   # Any test fixtures changed
+
 validate data:
   stage: test
   script:
     - pip install -r tests/requirements.txt
-    # Run tests with minimal console output, produce report, and fail on warnings
-    - pytest --tb=line --junitxml=report.xml -W error::UserWarning
+    # Run tests with minimal console output, produce report, and output warnings
+    - pytest --tb=line --junitxml=report.xml -W default::UserWarning
     - yamllint -c tests/.yamllint .
   artifacts:
     when: always

CHANGELOG.md

@@ -1,5 +1,21 @@
 # ATLAS Data Changelog
 
+## [4.1.0]() (2022-10-27)
+
+Refreshed existing case studies
+
+#### Tactics and techniques
+- Added a ATLAS technique
+    + System Misuse for External Effect
+- Updated descriptions
+
+#### Case studies
+- Updated existing case study content
+- New case study fields: case study type (exercise or incident), actor, target, and reporter
+
+#### Tests
+- Added test for mismatched tactics and techniques in case study procedure steps
+
 ## [4.0.1]() (2022-07-12)
 
 #### Tools

conftest.py

@@ -71,8 +71,31 @@ def all_data_objects(request):
     """Represents IDs in data objects, such as tactics, techniques, and case studies. """
     return request.param
 
+@pytest.fixture(scope='session')
+def procedure_steps(request):
+    """Represents each procedure step."""
+    return request.param
+
+@pytest.fixture(scope='session')
+def technique_id_to_tactic_ids(request):
+    """Represents a dictionary of technique ID to a list of tactic IDs."""
+    return request.param
+
 #endregion
 
+def add_label_entries(collection, obj, keys):
+    """
+    Adds a tuple of (label, value) to the specified list,
+    which identifies key-values of the object.
+    """
+    for key in keys:
+        if key in obj:
+            # Ex. "AML.CS0000 Name"
+            label = f"{obj['id']} {key.capitalize()}"
+            value = obj[key]
+            entry = (label, value)
+            collection.append(entry)
+
 def pytest_generate_tests(metafunc):
     """Enables test functions that use the above fixtures to operate on a
     single dictionary, where each test function is automatically run once
@@ -121,14 +144,18 @@ def pytest_generate_tests(metafunc):
     text_with_possible_markdown_syntax = []
     text_to_be_spellchecked = []
     all_values = []
+    procedure_steps = []
 
     for fixture_name in fixture_names:
         # Handle the key 'case_studies' really being 'case-studies' in the input
         key = fixture_name.replace('_','-')
         # List of tuples that hold the ID and the corresponding object
+        # For tactics and techniques
         values = [(obj['id'], obj) for matrix in matrices if key in matrix for obj in matrix[key]]
+
         # Creates a list of tuples across all fixture names
         all_values.extend(values)
+        # For case studies
         if key in data:
             id_to_obj = [(obj['id'], obj) for obj in data[key]]
             all_values.extend(id_to_obj)
@@ -137,7 +164,6 @@ def pytest_generate_tests(metafunc):
     if 'all_data_objects' in metafunc.fixturenames:
         metafunc.parametrize('all_data_objects', [all_values], indirect=True, scope='session')
 
-
     # Parameterize based on data objects
     for fixture_name in fixture_names:
 
@@ -146,23 +172,47 @@ def pytest_generate_tests(metafunc):
 
         # Construct a list of objects across all matrices under the specified key
         values = [obj for matrix in matrices if key in matrix for obj in matrix[key]]
+
         # Add top-level objects, if exists, ex. case-studies appended to an empty list from above
         if key in data:
             values.extend(data[key])
 
+        # Keys expected to be text strings in case study objects
+        # Used for spellcheck purposes
+        text_cs_keys = [
+            'name',
+            'summary',
+            'reporter',
+            'actor',
+            'target'
+        ]
+        # Collect technique objects
+        if 'technique_id_to_tactic_ids' in metafunc.fixturenames and key == 'techniques':
+            technique_id_to_tactic_ids = {obj['id']: obj['tactics'] for obj in values if 'subtechnique-of' not in obj}
+            metafunc.parametrize('technique_id_to_tactic_ids', [technique_id_to_tactic_ids], ids=[''],indirect=True, scope='session')
+
         # Build up text parameters
         # Parameter format is (test_identifier, text)
         if key == 'case-studies':
+
             for cs in values:
-                cs_id = cs['id']
+                # Add each of the specified keys defined above to spellcheck list
+                add_label_entries(text_to_be_spellchecked, cs, text_cs_keys)
+
+                # Process each procedure step
+                for i, step in enumerate(cs['procedure']):
+
+                    # Example tuple is of the form (AML.CS0000 Procedure #3, <procedure step description>)
+                    step_id = f"{cs['id']} Procedure #{i+1}"
 
-                text_to_be_spellchecked.append((f"{cs_id} Name", cs['name']))
-                text_to_be_spellchecked.append((f"{cs_id} Summary", cs['summary']))
+                    # Track the step itself
+                    procedure_steps.append((step_id, step))
+
+                    # And the description for text syntax
+                    step_description = (step_id, step['description'])
+                    text_to_be_spellchecked.append(step_description)
+                    text_with_possible_markdown_syntax.append(step_description)
 
-                # AML.CS0000 Procedure #3, <procedure step description>
-                procedure_step_texts = [(f"{cs_id} Procedure #{i+1}", p['description']) for i, p in enumerate(cs['procedure'])]
-                text_to_be_spellchecked.extend(procedure_step_texts)
-                text_with_possible_markdown_syntax.extend(procedure_step_texts)
         else:
             # This based off of a default ATLAS data object
             for t in values:
@@ -190,6 +240,12 @@ def pytest_generate_tests(metafunc):
     if 'text_to_be_spellchecked' in metafunc.fixturenames:
         metafunc.parametrize('text_to_be_spellchecked', text_to_be_spellchecked, ids=lambda x: x[0], indirect=True, scope='session')
 
+    ## Create parameterized fixtures for each procedure step
+
+    # Parametrize when called for via test signature
+    if 'procedure_steps' in metafunc.fixturenames:
+        metafunc.parametrize('procedure_steps', procedure_steps, ids=lambda x: x[0], indirect=True, scope='session')
+
 #region Schemas
 @pytest.fixture(scope='session')
 def output_schema():

data/README.md

@@ -13,7 +13,7 @@ For example, the ATLAS `data.yaml` is as follows:
 
 id: ATLAS
 name: Adversarial Threat Landscape for AI Systems
-version: 4.0.0
+version: 4.1.0
 
 matrices:
   - !include .

data/case-studies/AML.CS0000.yaml

@@ -1,74 +1,57 @@
 ---
 id: AML.CS0000
-name: Evasion of Deep Learning Detector for Malware C&C Traffic
 object-type: case-study
-summary: 'Palo Alto Networks Security AI research team tested a deep learning model
+name: Evasion of Deep Learning Detector for Malware C&C Traffic
+summary: 'The Palo Alto Networks Security AI research team tested a deep learning model
   for malware command and control (C&C) traffic detection in HTTP traffic.
 
-  Based on the publicly available paper by Le et al.  [1], we built a model that was
-  trained on a similar dataset as our production model and had performance similar
-  to it.
-
-  Then we crafted adversarial samples and queried the model and adjusted the adversarial
-  sample accordingly till the model was evaded.
+  Based on the publicly available [paper by Le et al.](https://arxiv.org/abs/1802.03162), we built a model that was
+  trained on a similar dataset as our production model and had similar performance.
 
-  '
+  Then we crafted adversarial samples, queried the model, and adjusted the adversarial
+  sample accordingly until the model was evaded.'
 incident-date: 2020-01-01
 incident-date-granularity: YEAR
 procedure:
 - tactic: '{{reconnaissance.id}}'
   technique: '{{victim_research_preprint.id}}'
   description: 'We identified a machine learning based approach to malicious URL detection
-    as a representative approach and potential target from the paper "URLNet: Learning
-    a URL representation with deep learning for malicious URL detection" [1], which
-    was found on arXiv (a pre-print repository).
-
-    '
+    as a representative approach and potential target from the paper [URLNet: Learning
+    a URL representation with deep learning for malicious URL detection](https://arxiv.org/abs/1802.03162),
+    which was found on arXiv (a pre-print repository).'
 - tactic: '{{resource_development.id}}'
   technique: '{{acquire_ml_artifacts_data.id}}'
-  description: 'We acquired a similar dataset to the target production model.
-
-    '
+  description: 'We acquired a command and control HTTP traffic  dataset consisting of approximately 33 million benign and 27 million malicious HTTP packet headers.'
 - tactic: '{{ml_attack_staging.id}}'
   technique: '{{train_proxy_model.id}}'
-  description: 'We built a model that was trained on a similar dataset as the production
-    model.
+  description: 'We trained a model on the HTTP traffic dataset to use as a proxy for the target model.
 
-    We trained the model on ~ 33 million benign and ~ 27 million malicious HTTP packet
-    headers.
-
-    Evaluation showed a true positive rate of ~ 99% and false positive rate of ~0.01%,
+    Evaluation showed a true positive rate of ~ 99% and false positive rate of ~ 0.01%,
     on average.
 
     Testing the model with a HTTP packet header from known malware command and control
-    traffic samples was detected as malicious with high confidence (> 99%).
-
-    '
+    traffic samples was detected as malicious with high confidence (> 99%).'
 - tactic: '{{ml_attack_staging.id}}'
   technique: '{{craft_adv_manual.id}}'
-  description: 'We crafted evasion samples by removing fields from packet header which
+  description: We crafted evasion samples by removing fields from packet header which
     are typically not used for C&C communication (e.g. cache-control, connection,
-    etc.)
-
-    '
+    etc.).
 - tactic: '{{ml_attack_staging.id}}'
   technique: '{{verify_attack.id}}'
-  description: 'We queried the model with our adversarial examples and adjusted them
+  description: We queried the model with our adversarial examples and adjusted them
     until the model was evaded.
-
-    '
 - tactic: '{{defense_evasion.id}}'
   technique: '{{evade_model.id}}'
-  description: 'With the crafted samples we performed online evasion of the ML-based
+  description: 'With the crafted samples, we performed online evasion of the ML-based
     spyware detection model.
 
-    The crafted packets were identified as benign with >80% confidence.
+    The crafted packets were identified as benign with > 80% confidence.
 
     This evaluation demonstrates that adversaries are able to bypass advanced ML detection
-    techniques, by crafting samples that are misclassified by an ML model.
-
-    '
-reported-by: Palo Alto Networks (Network Security AI Research Team)
+    techniques, by crafting samples that are misclassified by an ML model.'
+actor: Palo Alto Networks AI Research Team
+target: Palo Alto Networks malware detection system
+case-study-type: exercise
 references:
 - title: 'Le, Hung, et al. "URLNet: Learning a URL representation with deep learning
     for malicious URL detection." arXiv preprint arXiv:1802.03162 (2018).'