merge master

.gitignore

@@ -121,3 +121,7 @@ ENV/
 
 # PyCharm metadata
 .idea/
+
+# Vi
+*.swp
+*.swo

CHANGELOG.rst

@@ -6,14 +6,68 @@ dcicutils
 Change Log
 ----------
 
-7.11.0
+7.14.0
 ======
 
 * New module ``transfer_utils``:
 
   * Creates new utilities for downloading files and patching a location back to the portal
 
 
+7.13.0
+======
+
+* In ``license_utils``:
+
+  * Add an ``RLanguageFramework``.
+
+  * Add various additional checker classes, and a registry to catalog them. Refactor so that pre-existing
+    classes better share information in an inherited way.
+
+    +------------------------------------------+--------------------------------+----------------+
+    |                 Class                    |          Checker Name          |    Status      |
+    +==========================================+================================+================+
+    | ``ParkLabCommonLicenseChecker``          | ``park-lab-common``            | New            |
+    +------------------------------------------+--------------------------------+----------------+
+    | ``ParkLabGplPipelineLicenseChecker``     | ``park-lab-gpl-pipeline``      | New            |
+    +------------------------------------------+--------------------------------+----------------+
+    | ``ParkLabCommonServerLicenseChecker``    | ``park-lab-common-server``     | New            |
+    +------------------------------------------+--------------------------------+----------------+
+    | ``C4InfrastructureLicenseChecker``       | ``c4-infastructure``           | Refactored     |
+    +------------------------------------------+--------------------------------+----------------+
+    | ``C4PythonInfrastructureLicenseChecker`` | ``c4-python-infrastructure``   | Refactored     |
+    +------------------------------------------+--------------------------------+----------------+
+    | ``Scan2PipelineLicenseChecker``          | ``scan2-pipeline``             | New            |
+    +------------------------------------------+--------------------------------+----------------+
+
+* In ``misc_utils``:
+
+  * New function ``json_file_contents``
+
+* In ``scripts``:
+
+  * Add a ``run-license-checker`` script, implemented by ``run_license_checker.py``,
+    that runs the license checker whose "checker name" is given as an argument.
+
+
+7.12.0
+======
+
+* In ``glacier_utils``:
+
+  * Add functionality for KMS key encrypted accounts
+
+
+7.11.0
+======
+
+* In ``ff_utils``:
+
+  * Fix in ``get_schema`` and ``get_schemas`` for the ``portal_vapp`` case needing a leading slash on the URL.
+  * Fix in ``get_schema`` and ``get_schemas`` for the ``portal_vapp`` returning webtest.response.TestResponse
+    which has a ``json`` object property rather than a function.
+
+
 7.10.0
 ======
 

dcicutils/common.py

@@ -1,4 +1,5 @@
 import os
+import re
 
 from typing import Dict, Union, Tuple, List, Any
 from typing_extensions import Literal
@@ -36,6 +37,8 @@
 
 LIBRARY_DIR = os.path.dirname(__file__)
 
+Regexp = type(re.compile("sample"))
+
 # ===== Auth Data =====
 
 AuthStr = str

dcicutils/ff_utils.py

@@ -17,7 +17,7 @@
     # S3BucketName, S3KeyName,
 )
 from .lang_utils import disjoined_list
-from .misc_utils import PRINT, to_camel_case, remove_suffix, VirtualApp
+from .misc_utils import PRINT, to_camel_case, remove_suffix, VirtualApp, VirtualAppResponse
 
 
 # TODO (C4-92, C4-102): Probably to centralize this information in env_utils. Also figure out relation to CGAP.
@@ -1006,7 +1006,7 @@ def get_schema(name, key=None, ff_env: Optional[str] = None, portal_env: Optiona
     base_url = f"profiles/{to_camel_case(name)}.json"
     add_on = 'frame=raw'
     if portal_vapp:
-        full_url = f"{base_url}?{add_on}"
+        full_url = f"/{base_url}?{add_on}"
         res = portal_vapp.get(full_url)
         return get_response_json(res)
     else:
@@ -1038,7 +1038,7 @@ def get_schemas(key=None, ff_env: Optional[str] = None, *, allow_abstract: bool
     base_url = 'profiles/'
     add_on = 'frame=raw'
     if portal_vapp:
-        full_url = f"{base_url}?{add_on}"
+        full_url = f"/{base_url}?{add_on}"
         schemas: Dict[str, Dict] = portal_vapp.get(full_url)
     else:
         schemas: Dict[str, Dict] = get_metadata(obj_id=base_url, key=key, ff_env=portal_env, add_on=add_on)
@@ -1504,7 +1504,10 @@ def get_response_json(res):
     it is not present. Used with the metadata functions.
     """
     try:
-        res_json = res.json()
+        if isinstance(res, VirtualAppResponse):
+            res_json = res.json
+        else:
+            res_json = res.json()
     except Exception:
         raise Exception('Cannot get json for request to %s. Status'
                         ' code: %s. Response text: %s' %

dcicutils/glacier_utils.py

@@ -58,6 +58,10 @@ def __init__(self, env_name: str):
         self.env_key = self.key_manager.get_keydict_for_env(env_name)
         self.health_page = get_health_page(key=self.env_key, ff_env=env_name)
 
+    @property
+    def kms_key_id(self) -> str:
+        return self.health_page.get("s3_encrypt_key_id", "")
+
     @classmethod
     def is_glacier_storage_class(cls, storage_class: S3StorageClass):
         return storage_class in S3_GLACIER_CLASSES
@@ -295,6 +299,9 @@ def _do_multipart_upload(self, bucket: str, key: str, total_size: int, part_size
             }
             if tags:
                 cmu['Tagging'] = tags
+            if self.kms_key_id:
+                cmu['ServerSideEncryption'] = 'aws:kms'
+                cmu['SSEKMSKeyId'] = self.kms_key_id
             mpu = self.s3.create_multipart_upload(**cmu)
             mpu_upload_id = mpu['UploadId']
         except Exception as e:
@@ -381,16 +388,21 @@ def copy_object_back_to_original_location(self, bucket: str, key: str, storage_c
             else:
                 # Force copy the object into standard in a single operation
                 copy_source = {'Bucket': bucket, 'Key': key}
-                copy_target = {
+                copy_args = {
                     'Bucket': bucket, 'Key': key,
                     'StorageClass': storage_class,
                 }
                 if version_id:
                     copy_source['VersionId'] = version_id
-                    copy_target['CopySourceVersionId'] = version_id
+                    copy_args['CopySourceVersionId'] = version_id
                 if tags:
-                    copy_target['Tagging'] = tags
-                response = self.s3.copy_object(CopySource=copy_source, **copy_target)
+                    copy_args['Tagging'] = tags
+                if self.kms_key_id:
+                    copy_args['ServerSideEncryption'] = 'aws:kms'
+                    copy_args['SSEKMSKeyId'] = self.kms_key_id
+                response = self.s3.copy_object(
+                    **copy_args, CopySource=copy_source
+                )
                 PRINT(f'Response from boto3 copy:\n{response}')
                 PRINT(f'Object {bucket}/{key} copied back to its original location in S3')
                 return response

dcicutils/license_policies/c4-infrastructure.jsonc

@@ -0,0 +1,8 @@
+{
+    "class_key": "c4-infrastructure",
+    "class_name": "C4InfrastructureLicenseChecker",
+    "inherits_from": ["park-lab-common-server"],
+    "description": "Checker for C4 infrastructure (Fourfront, CGAP, SMaHT) from Park Lab.",
+
+    "LICENSE_TITLE": "(The )?MIT License"
+}

dcicutils/license_policies/c4-python-infrastructure.jsonc

@@ -0,0 +1,8 @@
+{
+    "class_key": "c4-python-infrastructure",
+    "class_name": "C4PythonInfrastructureLicenseChecker",
+    "inherits_from": ["c4-infrastructure"],
+    "description": "Checker for C4 python library infrastructure (Fourfront, CGAP, SMaHT) from Park Lab.",
+
+    "LICENSE_FRAMEWORKS": ["python"]
+}

dcicutils/license_policies/park-lab-common-server.jsonc

@@ -0,0 +1,104 @@
+{
+    "class_key": "park-lab-common-server",
+    "inherits_from": ["park-lab-common"],
+    "description": "Minimal/generic checker for servers from Park Lab.",
+
+    "LICENSE_FRAMEWORKS": ["python", "javascript"],
+
+    "EXCEPTIONS": {
+        "BSD*": [
+            // Although modified to insert the author name into the license text itself,
+            // the license for these libraries are essentially BSD-3-Clause.
+            "formatio",
+            "samsam",
+
+            // There are some slightly different versions of what appear to be BSD licenses here,
+            // but clearly the license is permissive.
+            // Ref: https://www.npmjs.com/package/mutation-observer?activeTab=readme
+            "mutation-observer"
+        ],
+        "Custom: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global": [
+            // The use of this URL appears to be a syntax error in the definition of entries-ponyfill
+            // In fact this seems to be covered by a CC0-1.0 license.
+            // Ref: https://unpkg.com/browse/[email protected]/LICENSE
+            "object.entries-ponyfill"
+        ],
+        "Custom: https://github.com/saikocat/colorbrewer.": [
+            // The use of this URL appears to be a syntax error in the definition of cartocolor
+            // In fact, this seems to be covered by a CC-BY-3.0 license.
+            // Ref: https://www.npmjs.com/package/cartocolor?activeTab=readme
+            "cartocolor"
+        ],
+        "Custom: https://travis-ci.org/component/emitter.png": [
+            // The use of this png appears to be a syntax error in the definition of emitter-component.
+            // In fact, emitter-component uses an MIT License
+            // Ref: https://www.npmjs.com/package/emitter-component
+            // Ref: https://github.com/component/emitter/blob/master/LICENSE
+            "emitter-component"
+        ],
+        "Custom: https://travis-ci.org/DenisCarriere/turf-jsts.svg": [        
+            // The 'turfs-jsts' repository (https://github.com/DenisCarriere/turf-jsts/blob/master/README.md)
+            // seems to lack a license, but appears to be forked from the jsts library that uses
+            // the Eclipse Public License 1.0 and Eclipse Distribution License 1.0, so probably a permissive
+            // license is intended.
+            "turf-jsts"
+        ],
+        "GNU General Public License (GPL)": [
+            "docutils"  // Used only privately as a separate documentation-generation task for ReadTheDocs
+        ],
+        "GNU Library or Lesser General Public License (LGPL)": [
+
+            // Linking = With Restrictions, Private Use = Yes
+            // Ref: https://en.wikipedia.org/wiki/Comparison_of_free_and_open-source_software_licenses
+            // "GNU Lesser General Public License v3 or later (LGPLv3+)",
+            // Linking = With Restrictions, Private Use = Yes
+            // Ref: https://en.wikipedia.org/wiki/Comparison_of_free_and_open-source_software_licenses
+            "psycopg2",         // Used at runtime during server operation, but not modified or distributed
+            "psycopg2-binary",  // Used at runtime during server operation, but not modified or distributed
+            "chardet",          // Potentially used downstream in loadxl to detect charset for text files
+            "pyzmq"             // Used in post-deploy-perf-tests, not distributed, and not modified or distributed
+        ],
+        "GPL-2.0": [
+            // The license file for the node-forge javascript library says:
+            //
+            //   "You may use the Forge project under the terms of either the BSD License or the
+            //   GNU General Public License (GPL) Version 2."
+            //
+            // (We choose to use it under the BSD license.)
+            // Ref: https://www.npmjs.com/package/node-forge?activeTab=code
+            "node-forge"
+        ],
+        "MIT*": [
+            // This library uses a mix of licenses, but they (MIT, CC0) generally seem permissive.
+            // (It also mentions that some tools for building/testing use other libraries.)
+            // Ref: https://github.com/requirejs/domReady/blob/master/LICENSE
+            "domready",
+
+            // This library is under "COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.1"
+            // Ref: https://github.com/javaee/jsonp/blob/master/LICENSE.txt
+            // About CDDL ...
+            // Linking = Permissive, Private Use = ?
+            // Ref: https://en.wikipedia.org/wiki/Comparison_of_free_and_open-source_software_licenses
+            "jsonp",
+
+            // This library says pretty clearly it intends MIT license.
+            // Ref: https://www.npmjs.com/package/component-indexof
+            // Linking = Permissive, Private Use = Yes
+            // Ref: https://en.wikipedia.org/wiki/Comparison_of_free_and_open-source_software_licenses
+            "component-indexof",
+
+            // These look like a pretty straight MIT license.
+            // Linking = Permissive, Private Use = Yes
+            // Ref: https://en.wikipedia.org/wiki/Comparison_of_free_and_open-source_software_licenses
+            "mixin",           // LICENSE file at https://www.npmjs.com/package/mixin?activeTab=code
+            "stack-trace",     // https://github.com/stacktracejs/stacktrace.js/blob/master/LICENSE
+            "typed-function"   // LICENSE at https://www.npmjs.com/package/typed-function?activeTab=code
+        ],
+        "UNLICENSED": [
+            // The udn-browser library is our own and has been observed to sometimes show up in some contexts
+            // as UNLICENSED, when really it is MIT.
+            // Ref: https://github.com/dbmi-bgm/udn-browser/blob/main/LICENSE
+            "udn-browser"
+        ]
+    }
+}