MAPEX-190 Optimize large files

If a mappings table has over 500 rows, we now truncate to 500 rows, show
a message, and link to the full size file. This improves initial page
load, expecially for NIST page which is the first on featured on the
home page and is our most popular project.

src/mappings_explorer/site_builder.py

@@ -12,7 +12,13 @@
 )
 
 from .attack_query import create_attack_jsons, get_attack_data, load_tactic_structure
-from .template import DATA_DIR, PUBLIC_DIR, ROOT_DIR, TEMPLATE_DIR, load_template
+from .template import (
+    DATA_DIR,
+    PUBLIC_DIR,
+    ROOT_DIR,
+    TEMPLATE_DIR,
+    load_template,
+)
 
 
 class Capability:
@@ -778,6 +784,7 @@ def build_external_landing(
     project_id = project.id
     if project_id == "nist":
         project_id = "nist_800_53"
+    table_max_count = 500
     stream = template.stream(
         title=project.label + " Landing",
         url_prefix=url_prefix,
@@ -800,8 +807,47 @@ def build_external_landing(
         non_mappables=project.non_mappables,
         project=project,
         group_artifact=group_artifact,
+        table_max_count=999_999,
+        full_link="",
+        full_size=0,
     )
     stream.dump(str(output_path))
+
+    if len(mappings) > table_max_count:
+        full_size = output_path.stat().st_size
+        full_path = output_path.parent / "all-data.html"
+        output_path.rename(full_path)
+
+        stream = template.stream(
+            title=project.label + " Landing",
+            url_prefix=url_prefix,
+            control=project.label,
+            description=project.description,
+            project_version=project_version.replace("/", "."),
+            project_id=project_id,
+            versions=project.versions,
+            attack_version=attack_version,
+            attackVersions=project.attackVersions,
+            attack_domain=attack_domain,
+            domains=project.attackDomains,
+            mappings=mappings,
+            headers=headers,
+            group_headers=capability_group_headers,
+            capability_groups=[
+                g for g in project.capability_groups if g.num_mappings > 0
+            ],
+            valid_versions=project.validVersions,
+            breadcrumbs=breadcrumbs,
+            non_mappable_headers=non_mappable_headers,
+            non_mappables=project.non_mappables,
+            project=project,
+            group_artifact=group_artifact,
+            table_max_count=table_max_count,
+            full_link="all-data.html",
+            full_size=full_size,
+        )
+        stream.dump(str(output_path))
+
     logger.trace(
         "Created {project_id} landing: ATT&CK version {attack_version}, ",
         "control version {project_version}, ATT&CK domain {attack_domain}",
@@ -985,6 +1031,9 @@ def build_capability_group(
         breadcrumbs=breadcrumbs,
         capability_group_headers=capability_group_headers,
         previous_link=previous_link,
+        table_max_count=999_999,
+        full_link="",
+        full_size=0,
     )
     stream.dump(str(output_path))
     logger.trace(
@@ -1042,6 +1091,9 @@ def build_external_capability(
         capability=capability,
         breadcrumbs=breadcrumbs,
         previous_link=previous_link,
+        table_max_count=999_999,
+        full_link="",
+        full_size=0,
     )
     stream.dump(str(output_path))
     logger.trace("          Created capability page {id}", id=capability.id)
@@ -1344,6 +1396,9 @@ def build_technique_page(
         subtechniques=technique.subtechniques,
         breadcrumbs=nav,
         previous_link=attack_prefix,
+        table_max_count=999_999,
+        full_link="",
+        full_size=0,
     )
     stream.dump(str(output_path))
     logger.trace("          Created technique page {id}", id=technique.id)
@@ -1401,6 +1456,9 @@ def build_tactic_page(
         prev_page=prev_page,
         breadcrumbs=nav,
         previous_link=previous_link,
+        table_max_count=999_999,
+        full_link="",
+        full_size=0,
     )
     stream.dump(str(output_path))
     logger.trace("          Created tactic page {id}", id=tactic.id)
@@ -1474,6 +1532,9 @@ def build_technique_landing_page(
         breadcrumbs=technique_nav,
         non_mappable_headers=non_mappable_headers,
         non_mappables=non_mappables,
+        table_max_count=999_999,
+        full_link="",
+        full_size=0,
     )
     stream.dump(str(output_path))
     description = """Tactics represent the "why" of a MITRE ATT&CK® technique or
@@ -1512,6 +1573,9 @@ def build_technique_landing_page(
         domains=attack_domains,
         valid_versions=valid_versions,
         breadcrumbs=tactic_nav,
+        table_max_count=999_999,
+        full_link="",
+        full_size=0,
     )
     stream.dump(str(output_path))
     logger.trace("Built techniques and tactics landing pages ")

src/mappings_explorer/template.py

@@ -40,3 +40,40 @@ def build_capability_url(mapping: dict, url_prefix: str, id: str):
 
 
 _environment.filters["build_capability_url"] = build_capability_url
+
+
+def format_int(value: int) -> str:
+    """
+    Format an integer value.
+
+    Args:
+        value: an integer to format
+
+    Returns:
+        formatted value
+    """
+    return "{:,d}".format(value)
+
+
+_environment.filters["format_int"] = format_int
+
+
+def data_size(value):
+    """
+    Format a number of bytes as a more readable unit.
+
+    Args:
+        value; a byte count
+
+    Returns:
+        A human-friendly string like "4.3MB"
+    """
+    units = ["bytes", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"]
+    unit_idx = 0
+    while value > 1000:
+        value /= 1000
+        unit_idx += 1
+    return f"{value:0.1f} {units[unit_idx]}"
+
+
+_environment.filters["data_size"] = data_size

src/mappings_explorer/templates/_navigation.html.j2

@@ -59,7 +59,6 @@
 </div>
 <script>
   function navSearchWebsite(url) {
-    console.log("searching website");
     window.location.href = `${url}?search=${document.getElementById("nav-search-input").value}`;
   }
   function navHandleKeypress(event, url) {

src/mappings_explorer/templates/_set_versions.html.j2

@@ -83,8 +83,6 @@ for (i = 0; i < l; i++) {
     c = document.createElement("DIV");
     c.innerHTML = selElmnt.options[j].innerHTML;
     c.addEventListener("click", function(e) {
-      console.log("click on  ", e.srcElement)
-
       // add our own handling of option states to the boilerplate click handling
       let control_version = document.getElementById("control_version_select").value;
       let attack_version = document.getElementById("attack_version_select").value;

src/mappings_explorer/templates/_set_versions_attack.html.j2

@@ -70,8 +70,6 @@ for (i = 0; i < l; i++) {
     c = document.createElement("DIV");
     c.innerHTML = selElmnt.options[j].innerHTML;
     c.addEventListener("click", function(e) {
-      console.log("click on  ", e.srcElement)
-
       // add our own handling of option states to the boilerplate click handling
       let attack_version = document.getElementById("attack_version_select").value;
       let attack_domain = document.getElementById("attack_domain_select").value;

src/mappings_explorer/templates/about.html.j2

@@ -2,7 +2,7 @@
 
 {% block content %}
 <section id="about-header" class="header-container">
-  <div>
+  <div class="container aos-init aos-animate">
     {% include "_breadcrumbs.html.j2" %}
     <div class="row justify-content-center">
       <div class="col-lg-8">

src/mappings_explorer/templates/attack_landing.html.j2

@@ -35,7 +35,7 @@
           </div>
         </div>
         <div class="col-12">
-            {{ macros.table(headers, mappings) }}
+            {{ macros.table(headers, mappings, url_prefix, table_max_count, full_link, full_size) }}
         </div>
       </div>
     </div>
@@ -54,7 +54,7 @@
               <span class="tooltip-text">Non-mappable techniques cannot be easily mitigated with preventive controls since they are based on behaviors performed outside of the scope of enterprise defenses and controls</span>
             </div>
           </div>
-          {{ macros.table(non_mappable_headers, non_mappables, url_prefix) }}
+          {{ macros.table(non_mappable_headers, non_mappables, url_prefix, table_max_count, full_link, full_size) }}
       </div>
     </div>
   </section>

src/mappings_explorer/templates/capability.html.j2

@@ -30,7 +30,7 @@
             </div>
         </div>
         <div class="col-12">
-            {{ macros.table(headers, mappings, url_prefix) }}
+            {{ macros.table(headers, mappings, url_prefix, table_max_count, full_link, full_size) }}
         </div>
       </div>
     </div>

src/mappings_explorer/templates/capability_group.html.j2

@@ -32,7 +32,7 @@
           </div>
         </div>
         <div class="col-12">
-            {{ macros.table(headers, mappings, url_prefix) }}
+            {{ macros.table(headers, mappings, url_prefix, table_max_count, full_link, full_size) }}
         </div>
       </div>
     </div>
@@ -49,7 +49,7 @@
             </div>
         </div>
         <div class="col-12">
-            {{ macros.table(capability_group_headers, capability_group.capabilities, url_prefix) }}
+            {{ macros.table(capability_group_headers, capability_group.capabilities, url_prefix, table_max_count, full_link, full_size) }}
       </div>
     </div>
   </section>

src/mappings_explorer/templates/framework_landing.html.j2

@@ -110,7 +110,7 @@
     <div class="row justify-content-left">
       <div class="col-12">
         <h2>Capability Groups</h2>
-        {{ macros.table(group_headers, capability_groups, url_prefix) }}
+        {{ macros.table(group_headers, capability_groups, url_prefix, table_max_count, full_link, full_size) }}
       </div>
     </div>
 </section>
@@ -120,7 +120,7 @@
     <div class="row justify-content-left">
       <div class="col-12">
         <h2>All Mappings</h2>
-        {{ macros.table(headers, mappings, url_prefix) }}
+        {{ macros.table(headers, mappings, url_prefix, table_max_count, full_link, full_size) }}
       </div>
     </div>
 </section>
@@ -137,7 +137,7 @@
               ATT&CK objects</span>
           </div>
         </div>
-        {{ macros.table(non_mappable_headers, non_mappables, url_prefix) }}
+        {{ macros.table(non_mappable_headers, non_mappables, url_prefix, table_max_count, full_link, full_size) }}
       </div>
     </div>
 </section>

src/mappings_explorer/templates/macros.html.j2

@@ -5,8 +5,15 @@
     </a>
 {% endmacro %}
 
-{% macro table(headers, mappings, url_prefix) %}
+{% macro table(headers, mappings, url_prefix, table_max_count, full_link, full_size) %}
 <div class="table-responsive table-outer">
+{% if mappings | length > table_max_count %}
+<p class="notice">
+  <strong>This is a very large mapping.</strong> To reduce the size, we have only
+  downloaded the first {{ table_max_count }} of {{ mappings | length | format_int }} mappings.
+  <a class="load-all" href="./{{ full_link }}">Load all data ({{ full_size | data_size }})</a>
+</p>
+{% endif %}
 <div class="form-outline mb-4">
   <table class="table table-borderless table-hover table-responsive text-truncate"
     data-toggle="table"
@@ -16,27 +23,27 @@
     data-search-align="left">
   <thead>
     <tr>
-        {%- for header in headers -%}
+        {%- for header in headers %}
             <th id={{id}} data-field={{header[0]}} data-sortable="true">{{header[1]}}</th>
         {% endfor %}
       </tr>
   </thead>
   <tbody>
-    {% for mapping in mappings %}
+    {%- for mapping in mappings[:table_max_count] %}
       <tr>
-        {%- for header in headers -%}
-            {% if header[2] and header[3] %}
+        {% for header in headers -%}
+            {%- if header[2] and header[3] -%}
                 <td><a href="{{header[3]}}{{mapping[header[2]] | replace(' ', '_') }}/">{{mapping[header[0]]}}</a></td>
-            {% endif %}
-            {% if header[2] and not header[3] %}
+            {% endif -%}
+            {%- if header[2] and not header[3] -%}
                 <td><a href="{{mapping | build_capability_url(url_prefix, mapping[header[2]])}}">{{mapping[header[0]]}}</a></td>
-            {% endif %}
-            {% if not header[2] %}
+            {% endif -%}
+            {%- if not header[2] -%}
                 <td>{{mapping[header[0]]}}</td>
-            {% endif %}
-        {% endfor %}
+            {% endif -%}
+        {% endfor -%}
       </tr>
-    {% endfor %}
+    {% endfor -%}
   </tbody>
 </table>
 <script>

src/mappings_explorer/templates/matrix.html.j2

@@ -390,7 +390,6 @@
       },
       setMatrixColumnAmount(){
         let matrixWidth = document.getElementById("matrix-table-scrollable").clientWidth;
-        console.log(matrixWidth)
         // based off of bootsrap breakpoint sizes
         // matrix column size is set to col-lg-1 col-md-2 col-sm-4
         this.amountMatrixColsShown = matrixWidth <= 576 ? 3 : matrixWidth <= 768 ? 5 : 12;

src/mappings_explorer/templates/static/main.css

@@ -303,10 +303,6 @@ div.banner {
   }
 
   div.banner {
-    top: 58px;
-  }
-
-  .banner.sticked {
     top: 54px;
   }
 }
@@ -1020,6 +1016,12 @@ section#user-stories button.accordion-button div {
 /**
   # Mapping Table Section
   **/
+p.notice {
+  padding: 1rem;
+  border: 1px solid var(--bs-border-color);
+  background-color: var(--bs-gray-200);
+}
+
 .mapping-table h2,
 .references h2 {
   text-transform: uppercase;

src/mappings_explorer/templates/tactic.html.j2

@@ -33,7 +33,7 @@
             </div>
         </div>
         <div class="col-12">
-            {{ macros.table(headers, mappings, url_prefix) }}
+            {{ macros.table(headers, mappings, url_prefix, table_max_count, full_link, full_size) }}
         </div>
       </div>
     </div>

src/mappings_explorer/templates/technique.html.j2

@@ -33,7 +33,7 @@
           </div>
         </div>
         <div class="col-12">
-            {{ macros.table(headers, mappings, url_prefix) }}
+            {{ macros.table(headers, mappings, url_prefix, table_max_count, full_link, full_size) }}
         </div>
       </div>
     </div>
@@ -51,7 +51,7 @@
           </div>
         </div>
         <div class="col-12">
-            {{ macros.table(technique_headers, subtechniques, url_prefix) }}
+            {{ macros.table(technique_headers, subtechniques, url_prefix, table_max_count, full_link, full_size) }}
         </div>
       </div>
     </div>