Merge branch 'mda-main' into share-inttest-3

.checkov.baseline

@@ -192,6 +192,13 @@
                         "CKV_AWS_115"
                     ]
                 },
+                {
+                    "resource": "AWS::Lambda::Function.CustomAuthorizerFunctiondevB38B5CCB",
+                    "check_ids": [
+                        "CKV_AWS_115",
+                        "CKV_AWS_116"
+                    ]
+                },
                 {
                     "resource": "AWS::Lambda::Function.ElasticSearchProxyHandlerDBDE7574",
                     "check_ids": [
@@ -210,6 +217,12 @@
                         "CKV_AWS_158"
                     ]
                 },
+                {
+                    "resource": "AWS::Logs::LogGroup.customauthorizerloggroup8F3B5B9D",
+                    "check_ids": [
+                        "CKV_AWS_158"
+                    ]
+                },
                 {
                     "resource": "AWS::Logs::LogGroup.dataalldevapigateway2625FE76",
                     "check_ids": [
@@ -404,7 +417,7 @@
             ]
         },
         {
-            "file": "/cdk.out/asset.3045cb6b4340be1e173df6dcf6248d565aa849ceda3e2cf2c2f221ccee4bc1d6/pivotRole.yaml",
+            "file": "/cdk.out/asset.05d71d8b69cd4483d3c9db9120b556b718c72f349debbb79d461c74c4964b350/pivotRole.yaml",
             "findings": [
                 {
                     "resource": "AWS::IAM::ManagedPolicy.PivotRolePolicy0",
@@ -477,12 +490,6 @@
         {
             "file": "/checkov_environment_synth.json",
             "findings": [
-                {
-                    "resource": "AWS::IAM::ManagedPolicy.dataallanothergroup111111servicespolicy19AC37181",
-                    "check_ids": [
-                        "CKV_AWS_111"
-                    ]
-                },
                 {
                     "resource": "AWS::IAM::ManagedPolicy.dataallanothergroup111111servicespolicy2E85AF510",
                     "check_ids": [
@@ -495,24 +502,6 @@
                         "CKV_AWS_111"
                     ]
                 },
-                {
-                    "resource": "AWS::IAM::ManagedPolicy.dataallanothergroup111111servicespolicy5A19E75CA",
-                    "check_ids": [
-                        "CKV_AWS_109"
-                    ]
-                },
-                {
-                    "resource": "AWS::IAM::ManagedPolicy.dataallanothergroup111111servicespolicyCC720210",
-                    "check_ids": [
-                        "CKV_AWS_109"
-                    ]
-                },
-                {
-                    "resource": "AWS::IAM::ManagedPolicy.dataalltestadmins111111servicespolicy1A0C96958",
-                    "check_ids": [
-                        "CKV_AWS_111"
-                    ]
-                },
                 {
                     "resource": "AWS::IAM::ManagedPolicy.dataalltestadmins111111servicespolicy2B12D381A",
                     "check_ids": [
@@ -525,18 +514,6 @@
                         "CKV_AWS_111"
                     ]
                 },
-                {
-                    "resource": "AWS::IAM::ManagedPolicy.dataalltestadmins111111servicespolicy3E3CBA9E",
-                    "check_ids": [
-                        "CKV_AWS_109"
-                    ]
-                },
-                {
-                    "resource": "AWS::IAM::ManagedPolicy.dataalltestadmins111111servicespolicy56D7DC525",
-                    "check_ids": [
-                        "CKV_AWS_109"
-                    ]
-                },
                 {
                     "resource": "AWS::Lambda::Function.CustomCDKBucketDeployment8693BB64968944B69AAFB0CC9EB8756C81C01536",
                     "check_ids": [
@@ -550,38 +527,34 @@
                     "resource": "AWS::Lambda::Function.GlueDatabaseLFCustomResourceHandler7FAF0F82",
                     "check_ids": [
                         "CKV_AWS_115",
-                        "CKV_AWS_117",
-                        "CKV_AWS_173"
+                        "CKV_AWS_117"
                     ]
                 },
                 {
                     "resource": "AWS::Lambda::Function.LakeformationDefaultSettingsHandler2CBEDB06",
                     "check_ids": [
                         "CKV_AWS_115",
-                        "CKV_AWS_117",
-                        "CKV_AWS_173"
+                        "CKV_AWS_117"
                     ]
                 },
                 {
                     "resource": "AWS::Lambda::Function.dataallGlueDbCustomResourceProviderframeworkonEventF8347BA7",
                     "check_ids": [
                         "CKV_AWS_115",
                         "CKV_AWS_116",
-                        "CKV_AWS_117",
-                        "CKV_AWS_173"
+                        "CKV_AWS_117"
                     ]
                 },
                 {
                     "resource": "AWS::Lambda::Function.dataallLakeformationDefaultSettingsProviderframeworkonEventBB660E32",
                     "check_ids": [
                         "CKV_AWS_115",
                         "CKV_AWS_116",
-                        "CKV_AWS_117",
-                        "CKV_AWS_173"
+                        "CKV_AWS_117"
                     ]
                 },
                 {
-                    "resource": "AWS::S3::Bucket.EnvironmentDefaultBucket78C3A8B0",
+                    "resource": "AWS::S3::Bucket.EnvironmentDefaultLogBucket7F0EFAB3",
                     "check_ids": [
                         "CKV_AWS_18"
                     ]
@@ -640,6 +613,25 @@
                 }
             ]
         },
+        {
+            "file": "/checkov_pipeline_synth.json",
+            "findings": [
+                {
+                    "resource": "AWS::IAM::Role.PipelineRoleDCFDBB91",
+                    "check_ids": [
+                        "CKV_AWS_107",
+                        "CKV_AWS_108",
+                        "CKV_AWS_111"
+                    ]
+                },
+                {
+                    "resource": "AWS::S3::Bucket.thistableartifactsbucketDB1C8C64",
+                    "check_ids": [
+                        "CKV_AWS_18"
+                    ]
+                }
+            ]
+        },
         {
             "file": "/frontend/docker/prod/Dockerfile",
             "findings": [

.github/workflows/snyk.yaml

@@ -0,0 +1,32 @@
+name: Snyk
+
+on:
+  workflow_dispatch:
+
+  schedule:
+    - cron: "0 9 * * 1"  # runs each Monday at 9:00 UTC
+
+permissions:
+  contents: read
+  security-events: write
+
+jobs:
+  security:
+    strategy:
+      matrix:
+        python-version: [3.9]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install All Requirements
+        run: make install
+      - name: Run Snyk to check for vulnerabilities
+        run: snyk test --all-projects --detection-depth=5 --severity-threshold=high
+        env:
+          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+        with:
+          args: --all-projects --detection-depth=5 --severity-threshold=high

Makefile

@@ -16,7 +16,7 @@ venv:
 	@python3 -m venv "venv"
 	@/bin/bash -c "source venv/bin/activate"
 
-install: upgrade-pip install-deploy install-backend install-cdkproxy install-tests
+install: upgrade-pip install-deploy install-backend install-cdkproxy install-tests install-integration-tests install-custom-auth install-userguide
 
 upgrade-pip:
 	pip install --upgrade pip setuptools
@@ -36,6 +36,12 @@ install-tests:
 install-integration-tests:
 	pip install -r tests_new/integration_tests/requirements.txt
 
+install-custom-auth:
+	pip install -r deploy/custom_resources/custom_authorizer/requirements.txt
+
+install-userguide:
+	pip install -r documentation/userguide/requirements.txt
+
 lint:
 	pip install ruff
 	ruff check --fix

backend/api_handler.py

@@ -15,13 +15,15 @@
     attach_tenant_policy_for_groups,
     check_reauth,
     validate_and_block_if_maintenance_window,
+    redact_creds,
 )
 from dataall.core.tasks.service_handlers import Worker
 from dataall.base.aws.sqs import SqsQueue
 from dataall.base.context import set_context, dispose_context, RequestContext
 from dataall.base.db import get_engine
 from dataall.base.loader import load_modules, ImportMode
 
+from graphql.pyutils import did_you_mean
 
 logger = logging.getLogger()
 logger.setLevel(os.environ.get('LOG_LEVEL', 'INFO'))
@@ -31,6 +33,11 @@
 for name in ['boto3', 's3transfer', 'botocore', 'boto']:
     logging.getLogger(name).setLevel(logging.ERROR)
 
+ALLOW_INTROSPECTION = True if os.getenv('ALLOW_INTROSPECTION') == 'True' else False
+
+if not ALLOW_INTROSPECTION:
+    did_you_mean.__globals__['MAX_LENGTH'] = 0
+
 load_modules(modes={ImportMode.API})
 SCHEMA = bootstrap_schema()
 TYPE_DEFS = gql(SCHEMA.gql(with_directives=False))
@@ -84,6 +91,7 @@ def handler(event, context):
         Return doc: https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html
     """
 
+    event = redact_creds(event)
     log.info('Lambda Event %s', event)
     log.debug('Env name %s', ENVNAME)
     log.debug('Engine %s', ENGINE.engine.url)
@@ -136,7 +144,9 @@ def handler(event, context):
     else:
         raise Exception(f'Could not initialize user context from event {event}')
 
-    success, response = graphql_sync(schema=executable_schema, data=query, context_value=app_context)
+    success, response = graphql_sync(
+        schema=executable_schema, data=query, context_value=app_context, introspection=ALLOW_INTROSPECTION
+    )
 
     dispose_context()
     response = json.dumps(response)

backend/dataall/__init__.py

@@ -1,2 +1,13 @@
 from . import core, version
 from .base import utils, db, api
+import logging
+import os
+import sys
+
+logging.basicConfig(
+    level=os.environ.get('LOG_LEVEL', 'INFO'),
+    handlers=[logging.StreamHandler(sys.stdout)],
+    format='[%(levelname)s] %(message)s',
+)
+for name in ['boto3', 's3transfer', 'botocore', 'boto', 'urllib3']:
+    logging.getLogger(name).setLevel(logging.ERROR)

backend/dataall/base/cdkproxy/requirements.txt

@@ -2,7 +2,7 @@ aws-cdk-lib==2.160.0
 boto3==1.35.26
 boto3-stubs==1.35.26
 cdk-nag==2.7.2
-fastapi == 0.115.0
+fastapi == 0.115.5
 PyYAML==6.0
 requests==2.32.2
 tabulate==0.8.9

backend/dataall/base/feature_toggle_checker.py

@@ -2,6 +2,7 @@
 Contains decorators that check if a feature has been enabled or not
 """
 
+import functools
 from typing import List, Any, Optional, Callable
 
 from dataall.base.config import config
@@ -12,6 +13,7 @@ def is_feature_enabled(config_property: str):
     def decorator(f):
         fn, fn_decorator = process_func(f)
 
+        @functools.wraps(fn)
         def decorated(*args, **kwargs):
             value = config.get_property(config_property)
             if not value:
@@ -33,6 +35,7 @@ def is_feature_enabled_for_allowed_values(
     def decorator(f):
         fn, fn_decorator = process_func(f)
 
+        @functools.wraps(fn)
         def decorated(*args, **kwargs):
             config_property_value = None
             if config_property is None and resolve_property is None:

backend/dataall/base/utils/api_handler_utils.py

@@ -24,6 +24,16 @@
 ]
 ENGINE = get_engine(envname=ENVNAME)
 ALLOWED_ORIGINS = os.getenv('ALLOWED_ORIGINS', '*')
+AWS_REGION = os.getenv('AWS_REGION')
+
+
+def redact_creds(event):
+    if event.get('headers', {}).get('Authorization'):
+        event['headers']['Authorization'] = 'XXXXXXXXXXXX'
+
+    if event.get('multiValueHeaders', {}).get('Authorization'):
+        event['multiValueHeaders']['Authorization'] = 'XXXXXXXXXXXX'
+    return event
 
 
 def get_cognito_groups(claims):
@@ -107,7 +117,7 @@ def check_reauth(query, auth_time, username):
     # Determine if there are any Operations that Require ReAuth From SSM Parameter
     try:
         reauth_apis = ParameterStoreManager.get_parameter_value(
-            region=os.getenv('AWS_REGION', 'eu-west-1'), parameter_path=f'/dataall/{ENVNAME}/reauth/apis'
+            region=AWS_REGION, parameter_path=f'/dataall/{ENVNAME}/reauth/apis'
         ).split(',')
     except Exception:
         log.info('No ReAuth APIs Found in SSM')

backend/dataall/base/utils/naming_convention.py

@@ -23,9 +23,10 @@ class NamingConventionPattern(Enum):
     NOTEBOOK = {'regex': '[^a-zA-Z0-9-]', 'separator': '-', 'max_length': 63}
     MLSTUDIO_DOMAIN = {'regex': '[^a-zA-Z0-9-]', 'separator': '-', 'max_length': 63}
     DEFAULT = {'regex': '[^a-zA-Z0-9-_]', 'separator': '-', 'max_length': 63}
+    DEFAULT_SEARCH = {'regex': '[^a-zA-Z0-9-_:. ]'}
     OPENSEARCH = {'regex': '[^a-z0-9-]', 'separator': '-', 'max_length': 27}
     OPENSEARCH_SERVERLESS = {'regex': '[^a-z0-9-]', 'separator': '-', 'max_length': 31}
-    DATA_FILTERS = {'regex': '^[a-z0-9_]*$', 'separator': '_', 'max_length': 31}
+    DATA_FILTERS = {'regex': '[^a-z0-9_]', 'separator': '_', 'max_length': 31}
     REDSHIFT_DATASHARE = {
         'regex': '[^a-zA-Z0-9_]',
         'separator': '_',
@@ -59,9 +60,7 @@ def build_compliant_name(self) -> str:
     def validate_name(self):
         regex = NamingConventionPattern[self.service].value['regex']
         max_length = NamingConventionPattern[self.service].value['max_length']
-        if 'arn:aws:' in self.target_label:
-            raise Exception(f'An error occurred (InvalidInput): name expected, arn-like string received: {regex}')
-        if not re.search(regex, self.target_label):
+        if re.search(regex, self.target_label):
             raise Exception(
                 f'An error occurred (InvalidInput): label value {self.target_label} must match the pattern {regex}'
             )
@@ -70,6 +69,10 @@ def validate_name(self):
                 f'An error occurred (InvalidInput): label value {self.target_label} must be less than {max_length} characters'
             )
 
+    def sanitize(self):
+        regex = NamingConventionPattern[self.service].value['regex']
+        return re.sub(regex, '', self.target_label)
+
     def validate_imported_name(self):
         max_length = NamingConventionPattern[self.service].value['max_length']
         valid_external_regex = NamingConventionPattern[self.service].value.get('valid_external_regex', '.*')

backend/dataall/core/environment/api/queries.py

@@ -32,6 +32,7 @@
 
 getTrustAccount = gql.QueryField(
     name='getTrustAccount',
+    args=[gql.Argument(name='organizationUri', type=gql.NonNullableType(gql.String))],
     type=gql.String,
     resolver=get_trust_account,
     test_scope='Environment',