WIP investigating problem and possible solutions

superset-frontend/src/features/databases/DatabaseModal/index.tsx

@@ -592,6 +592,9 @@ const DatabaseModal: FunctionComponent<DatabaseModalProps> = ({
   const [importingModal, setImportingModal] = useState<boolean>(false);
   const [importingErrorMessage, setImportingErrorMessage] = useState<string>();
   const [passwordFields, setPasswordFields] = useState<string[]>([]);
+  const [encryptedExtraFields, setEncryptedExtraFields] = useState<string[]>(
+    [],
+  );
   const [sshTunnelPasswordFields, setSSHTunnelPasswordFields] = useState<
     string[]
   >([]);
@@ -1342,6 +1345,7 @@ const DatabaseModal: FunctionComponent<DatabaseModalProps> = ({
   const onDbImport = async (info: UploadChangeParam) => {
     setImportingErrorMessage('');
     setPasswordFields([]);
+    setEncryptedExtraFields([]);
     setSSHTunnelPasswordFields([]);
     setSSHTunnelPrivateKeyFields([]);
     setSSHTunnelPrivateKeyPasswordFields([]);

superset-frontend/src/views/CRUD/hooks.ts

@@ -30,6 +30,7 @@ import {
   createErrorHandler,
   getAlreadyExists,
   getPasswordsNeeded,
+  getEncryptedExtraNeeded,
   hasTerminalValidation,
   getSSHPasswordsNeeded,
   getSSHPrivateKeysNeeded,
@@ -400,6 +401,7 @@ export function useSingleViewResource<D extends object = any>(
 interface ImportResourceState {
   loading: boolean;
   passwordsNeeded: string[];
+  encryptedExtraNeeded: string[];
   alreadyExists: string[];
   sshPasswordNeeded: string[];
   sshPrivateKeyNeeded: string[];
@@ -415,6 +417,7 @@ export function useImportResource(
   const [state, setState] = useState<ImportResourceState>({
     loading: false,
     passwordsNeeded: [],
+    encryptedExtraNeeded: [],
     alreadyExists: [],
     sshPasswordNeeded: [],
     sshPrivateKeyNeeded: [],
@@ -533,6 +536,7 @@ export function useImportResource(
             } else {
               updateState({
                 passwordsNeeded: getPasswordsNeeded(error.errors),
+                encryptedExtraNeeded: getEncryptedExtraNeeded(error.errors),
                 sshPasswordNeeded: getSSHPasswordsNeeded(error.errors),
                 sshPrivateKeyNeeded: getSSHPrivateKeysNeeded(error.errors),
                 sshPrivateKeyPasswordNeeded: getSSHPrivateKeyPasswordsNeeded(

superset-frontend/src/views/CRUD/utils.tsx

@@ -395,6 +395,14 @@ const isNeedsPassword = (payload: any) =>
     (e: string) => e === 'Must provide a password for the database',
   );
 
+export /* eslint-disable no-underscore-dangle */
+const isNeedsEncryptedExtra = (payload: any) =>
+  typeof payload === 'object' &&
+  Array.isArray(payload._schema) &&
+  !!payload._schema?.find(
+    (e: string) => e === 'Must provide encrypted credentials for the database',
+  );
+
 export /* eslint-disable no-underscore-dangle */
 const isNeedsSSHPassword = (payload: any) =>
   typeof payload === 'object' &&
@@ -433,6 +441,15 @@ export const getPasswordsNeeded = (errors: Record<string, any>[]) =>
     )
     .flat();
 
+export const getEncryptedExtraNeeded = (errors: Record<string, any>[]) =>
+  errors
+    .map(error =>
+      Object.entries(error.extra)
+        .filter(([, payload]) => isNeedsEncryptedExtra(payload))
+        .map(([fileName]) => fileName),
+    )
+    .flat();
+
 export const getSSHPasswordsNeeded = (errors: Record<string, any>[]) =>
   errors
     .map(error =>
@@ -482,6 +499,7 @@ export const hasTerminalValidation = (errors: Record<string, any>[]) =>
     return !noIssuesCodes.every(
       ([, payload]) =>
         isNeedsPassword(payload) ||
+        isNeedsEncryptedExtra(payload) ||
         isAlreadyExists(payload) ||
         isNeedsSSHPassword(payload) ||
         isNeedsSSHPrivateKey(payload) ||

superset/commands/chart/importers/v1/__init__.py

@@ -15,6 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import logging
 from typing import Any
 
 from marshmallow import Schema
@@ -31,6 +32,8 @@
 from superset.databases.schemas import ImportV1DatabaseSchema
 from superset.datasets.schemas import ImportV1DatasetSchema
 
+logger = logging.getLogger(__name__)
+
 
 class ImportChartsCommand(ImportModelsCommand):
     """Import charts"""
@@ -49,22 +52,28 @@ class ImportChartsCommand(ImportModelsCommand):
     def _import(configs: dict[str, Any], overwrite: bool = False) -> None:
         # discover datasets associated with charts
         dataset_uuids: set[str] = set()
+        logger.info("parsing datasets")
         for file_name, config in configs.items():
             if file_name.startswith("charts/"):
                 dataset_uuids.add(config["dataset_uuid"])
+        logger.info("datasets parsed: %s", dataset_uuids)
 
         # discover databases associated with datasets
+        logger.info("discovering databases associated with datasets")
+
         database_uuids: set[str] = set()
         for file_name, config in configs.items():
             if file_name.startswith("datasets/") and config["uuid"] in dataset_uuids:
                 database_uuids.add(config["database_uuid"])
 
         # import related databases
+        logger.info("importing related dbs")
         database_ids: dict[str, int] = {}
         for file_name, config in configs.items():
             if file_name.startswith("databases/") and config["uuid"] in database_uuids:
                 database = import_database(config, overwrite=False)
                 database_ids[str(database.uuid)] = database.id
+        logger.info("related dbs: %s", database_ids)
 
         # import datasets with the correct parent ref
         datasets: dict[str, SqlaTable] = {}

superset/commands/importers/v1/utils.py

@@ -184,6 +184,10 @@ def load_configs(
                         db_ssh_tunnel_priv_key_passws[config["uuid"]]
                     )
 
+                logger.info(
+                    "attempting to load content into schema %s %s", schema, content
+                )
+
                 schema.load(config)
                 configs[file_name] = config
             except ValidationError as exc:

superset/databases/schemas.py

@@ -20,6 +20,7 @@
 from __future__ import annotations
 
 import inspect
+import logging
 import os
 from pathlib import Path
 from typing import Any, TypedDict
@@ -55,6 +56,8 @@
 from superset.utils import json
 from superset.utils.core import markdown, parse_ssl_cert
 
+logger = logging.getLogger(__name__)
+
 database_schemas_query_schema = {
     "type": "object",
     "properties": {
@@ -859,6 +862,7 @@ def fix_allow_csv_upload(
     allow_csv_upload = fields.Boolean()
     impersonate_user = fields.Boolean()
     extra = fields.Nested(ImportV1DatabaseExtraSchema)
+    encrypted_extra = fields.String(allow_none=True)
     uuid = fields.UUID(required=True)
     version = fields.String(required=True)
     is_managed_externally = fields.Boolean(allow_none=True, dump_default=False)
@@ -878,6 +882,18 @@ def validate_password(self, data: dict[str, Any], **kwargs: Any) -> None:
         if password == PASSWORD_MASK and data.get("password") is None:
             raise ValidationError("Must provide a password for the database")
 
+    @validates_schema
+    def validate_encrypted_extra(self, data: dict[str, Any], **kwargs: Any) -> None:
+        """If encrypted extra contains masked data, encrypted extra is required"""
+        uuid = data["uuid"]
+        existing = db.session.query(Database).filter_by(uuid=uuid).first()
+        if existing:
+            return
+        encrypted_extra = data.get("encrypted_extra", None)
+        logger.info("encrypted_extra loaded as %s", encrypted_extra)
+        if not encrypted_extra:
+            raise ValidationError("Must provide encrypted credentials for the database")
+
     @validates_schema
     def validate_ssh_tunnel_credentials(
         self, data: dict[str, Any], **kwargs: Any

superset/models/core.py

@@ -172,6 +172,7 @@ class Database(Model, AuditMixinNullable, ImportExportMixin):  # pylint: disable
         "allow_file_upload",
         "extra",
         "impersonate_user",
+        "encrypted_extra",
     ]
     extra_import_fields = [
         "password",