diff --git a/.startup/env_var_defaults.json b/.startup/env_var_defaults.json index b732268..2ad1769 100644 --- a/.startup/env_var_defaults.json +++ b/.startup/env_var_defaults.json @@ -328,5 +328,264 @@ "": ".env.superset::POSTGRES_PASSWORD", "": ".env.superset::POSTGRES_DB" } + }, + ".env.om_db::POSTGRES_USER": { + "file": ".env.om_db", + "name": "POSTGRES_USER", + "group": "Open Metadata DB", + "default_value": "POSTGRES_USER", + "user_input": true, + "valid_pattern": null, + "invalid_substrings": "[\\s]", + "dependant_on_other_env_vars": false + }, + ".env.om_db::DB_USER": { + "file": ".env.om_db", + "name": "DB_USER", + "group": "Open Metadata DB", + "default_value": "DB_USER", + "user_input": false, + "dependant_on_other_env_vars": true, + "is_uri": false, + "env_var_mappings": { + "DWH_POSTGRES_USER": ".env.om_db::POSTGRES_USER" + } + }, + ".env.om_db::POSTGRES_PASSWORD": { + "file": ".env.om_db", + "name": "POSTGRES_PASSWORD", + "group": "Open Metadata DB", + "default_value": "POSTGRES_PASSWORD", + "user_input": true, + "valid_pattern": null, + "invalid_substrings": "[\\s]", + "dependant_on_other_env_vars": false + }, + ".env.om_db::DB_USER_PASSWORD": { + "file": ".env.om_db", + "name": "DB_USER_PASSWORD", + "group": "Open Metadata DB", + "default_value": "DB_USER_PASSWORD", + "user_input": false, + "dependant_on_other_env_vars": true, + "is_uri": false, + "env_var_mappings": { + "DWH_POSTGRES_USER": ".env.om_db::POSTGRES_PASSWORD" + } + }, + ".env.om_db::POSTGRES_DB": { + "file": ".env.om_db", + "name": "POSTGRES_DB", + "group": "Open Metadata DB", + "default_value": "POSTGRES_DB", + "user_input": true, + "valid_pattern": null, + "invalid_substrings": "[\\s]", + "dependant_on_other_env_vars": false + }, + ".env.om_db::OM_DATABASE": { + "file": ".env.om_db", + "name": "OM_DATABASE", + "group": "Open Metadata DB", + "default_value": "openmetadata_db", + "user_input": false, + "dependant_on_other_env_vars": true, + "is_uri": false, + "env_var_mappings": { + "DWH_POSTGRES_USER": ".env.om_db::POSTGRES_DB" + } + }, + + ".env.om_server::SERVER_PORT": { + "file": ".env.om_server", + "name": "SERVER_PORT", + "group": "Open Metadata Server", + "default_value": "8585", + "user_input": true, + "valid_pattern": "^[\\d]{4,5}$", + "invalid_substrings": "[\\s]", + "dependant_on_other_env_vars": false + }, + ".env.om_server::SERVER_ADMIN_PORT": { + "file": ".env.om_server", + "name": "SERVER_ADMIN_PORT", + "group": "Open Metadata Server", + "default_value": "8586", + "user_input": true, + "valid_pattern": "^[\\d]{4,5}$", + "invalid_substrings": "[\\s]", + "dependant_on_other_env_vars": false + }, + ".env.om_server::OPENMETADATA_CLUSTER_NAME": { + "file": ".env.om_server", + "name": "OPENMETADATA_CLUSTER_NAME", + "group": "Open Metadata Server", + "default_value": "openmetadata", + "user_input": true, + "valid_pattern": null, + "invalid_substrings": "[\\s]", + "dependant_on_other_env_vars": false + }, + ".env.om_server::PIPELINE_SERVICE_CLIENT_ENDPOINT": { + "file": ".env.om_server", + "name": "PIPELINE_SERVICE_CLIENT_ENDPOINT", + "group": "Open Metadata Server", + "default_value": "http://airflow-webserver:8080", + "user_input": false, + "dependant_on_other_env_vars": false + }, + ".env.om_server::SERVER_HOST_API_URL": { + "file": ".env.om_server", + "name": "SERVER_HOST_API_URL", + "group": "Open Metadata Server", + "default_value": "http://om_server:8585/api", + "user_input": false, + "dependant_on_other_env_vars": false + }, + ".env.om_server::DB_DRIVER_CLASS": { + "file": ".env.om_server", + "name": "DB_DRIVER_CLASS", + "group": "Open Metadata Server", + "default_value": "org.postgresql.Driver", + "user_input": false, + "dependant_on_other_env_vars": false + }, + ".env.om_server::DB_SCHEME": { + "file": ".env.om_server", + "name": "DB_SCHEME", + "group": "Open Metadata Server", + "default_value": "postgresql", + "user_input": false, + "dependant_on_other_env_vars": false + }, + ".env.om_server::DB_HOST": { + "file": ".env.om_server", + "name": "DB_HOST", + "group": "Open Metadata Server", + "default_value": "om_db", + "user_input": false, + "dependant_on_other_env_vars": false + }, + ".env.om_server::DB_PORT": { + "file": ".env.om_server", + "name": "DB_PORT", + "group": "Open Metadata Server", + "default_value": "5432", + "user_input": false, + "dependant_on_other_env_vars": false + }, + ".env.om_server::AUTHENTICATION_PROVIDER": { + "file": ".env.om_server", + "name": "AUTHENTICATION_PROVIDER", + "group": "Open Metadata Server", + "default_value": "basic", + "user_input": false, + "dependant_on_other_env_vars": false + }, + ".env.om_server::AUTHORIZER_CLASS_NAME": { + "file": ".env.om_server", + "name": "AUTHORIZER_CLASS_NAME", + "group": "Open Metadata Server", + "default_value": "org.openmetadata.service.security.DefaultAuthorizer", + "user_input": false, + "dependant_on_other_env_vars": false + }, + ".env.om_server::AUTHORIZER_REQUEST_FILTER": { + "file": ".env.om_server", + "name": "AUTHORIZER_REQUEST_FILTER", + "group": "Open Metadata Server", + "default_value": "org.openmetadata.service.security.JwtFilter", + "user_input": false, + "dependant_on_other_env_vars": false + }, + ".env.om_server::RSA_PUBLIC_KEY_FILE_PATH": { + "file": ".env.om_server", + "name": "RSA_PUBLIC_KEY_FILE_PATH", + "group": "Open Metadata Server", + "default_value": "/opt/openmetadata/conf/public_key.der", + "user_input": false, + "dependant_on_other_env_vars": false + }, + ".env.om_server::RSA_PRIVATE_KEY_FILE_PATH": { + "file": ".env.om_server", + "name": "RSA_PRIVATE_KEY_FILE_PATH", + "group": "Open Metadata Server", + "default_value": "/opt/openmetadata/conf/private_key.der", + "user_input": false, + "dependant_on_other_env_vars": false + }, + ".env.om_server::JWT_ISSUER": { + "file": ".env.om_server", + "name": "JWT_ISSUER", + "group": "Open Metadata Server", + "default_value": "local-openmetadata", + "user_input": false, + "dependant_on_other_env_vars": false + }, + ".env.om_server::AIRFLOW_USERNAME": { + "file": ".env.om_server", + "name": "AIRFLOW_USERNAME", + "group": "Open Metadata Server", + "default_value": "AIRFLOW_USERNAME", + "user_input": false, + "dependant_on_other_env_vars": true, + "is_uri": false, + "env_var_mappings": { + "DWH_POSTGRES_USER": ".env::_AIRFLOW_WWW_USER_USERNAME" + } + }, + ".env.om_server::AIRFLOW_PASSWORD": { + "file": ".env.om_server", + "name": "AIRFLOW_PASSWORD", + "group": "Open Metadata Server", + "default_value": "AIRFLOW_PASSWORD", + "user_input": false, + "dependant_on_other_env_vars": true, + "is_uri": false, + "env_var_mappings": { + "DWH_POSTGRES_USER": ".env::_AIRFLOW_WWW_USER_PASSWORD" + } + }, + ".env.om_server::WEB_CONF_URI_PATH": { + "file": ".env.om_server", + "name": "WEB_CONF_URI_PATH", + "group": "Open Metadata Server", + "default_value": "/api", + "user_input": false, + "dependant_on_other_env_vars": false + }, + ".env.elastic::ELASTICSEARCH_HOST": { + "file": ".env.elastic", + "name": "ELASTICSEARCH_HOST", + "group": "Open Metadata Elasticsearch", + "default_value": "om_elastic", + "user_input": false, + "dependant_on_other_env_vars": false + }, + ".env.elastic::ELASTICSEARCH_PORT": { + "file": ".env.elastic", + "name": "ELASTICSEARCH_PORT", + "group": "Open Metadata Elasticsearch", + "default_value": "9200", + "user_input": false, + "dependant_on_other_env_vars": false + }, + ".env.elastic::ELASTICSEARCH_SCHEME": { + "file": ".env.elastic", + "name": "ELASTICSEARCH_SCHEME", + "group": "Open Metadata Elasticsearch", + "default_value": "http", + "user_input": false, + "dependant_on_other_env_vars": false + }, + ".env.elastic::ELASTICSEARCH_USER": { + "file": ".env.elastic", + "name": "ELASTICSEARCH_USER", + "group": "Open Metadata Elasticsearch", + "default_value": "elastic", + "user_input": true, + "valid_pattern": null, + "invalid_substrings": "[\\s]", + "dependant_on_other_env_vars": false } } \ No newline at end of file diff --git a/.startup/make_env_docker.py b/.startup/make_env_docker.py index 6504f30..87a176a 100644 --- a/.startup/make_env_docker.py +++ b/.startup/make_env_docker.py @@ -4,6 +4,7 @@ import re import subprocess from typing import Dict, List, Optional +import uuid import urllib from make_fernet_key import generate_fernet_key_value @@ -117,8 +118,16 @@ def get_distinct_dot_env_file_names(env_var_payloads: List) -> List: def create_dot_env_files(output_dir: Path, env_var_dict: Dict) -> None: + all_lines_all_files = prepare_dot_env_file_lines(output_dir, env_var_dict) + for file_name, lines in all_lines_all_files.items(): + with open(file_name, "x") as f: + f.write(lines) + + +def prepare_dot_env_file_lines(output_dir: Path, env_var_dict: Dict) -> None: env_var_payloads = get_env_var_payloads(env_var_dict=env_var_dict) dot_env_file_names = get_distinct_dot_env_file_names(env_var_payloads=env_var_payloads) + all_lines_all_files = {} for file_name in dot_env_file_names: file_payloads = [p for p in env_var_payloads if p["file"] == file_name] distinct_groups_in_file = list(set([p["group"] for p in file_payloads])) @@ -131,18 +140,20 @@ def create_dot_env_files(output_dir: Path, env_var_dict: Dict) -> None: file_lines.append(f"{file_group_payload['name']}={file_group_payload['set_value']}") file_lines.append("") file_out_path = output_dir.joinpath(file_name) - with open(file_out_path, "x") as f: - all_file_lines = "".join([f"{line}\n" for line in file_lines]) - all_file_lines = all_file_lines.replace("\n\n\n", "\n\n") - all_file_lines = re.sub(r"(\n\n)$", "\n", all_file_lines) - f.write(all_file_lines) + all_file_lines = "".join([f"{line}\n" for line in file_lines]) + all_file_lines = all_file_lines.replace("\n\n\n", "\n\n") + all_file_lines = re.sub(r"(\n\n)$", "\n", all_file_lines) + all_lines_all_files[str(file_out_path)] = all_file_lines + return all_lines_all_files if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--startup_dir", default=".", help="The project's top-level directory") parser.add_argument( - "--mode", default="interactive", help="Credential-defining process: options: ['interactive', 'dev']" + "--mode", + default="interactive", + help="Credential-defining process: options: ['interactive', 'dev']", ) args = parser.parse_args() @@ -202,4 +213,18 @@ def create_dot_env_files(output_dir: Path, env_var_dict: Dict) -> None: "group": "Superset", "set_value": "", } + env_var_dict[".env.om_server::FERNET_KEY"] = { + "file": ".env.om_server", + "name": "FERNET_KEY", + "group": "Open Metadata Server", + "set_value": secret_key, + } + env_var_dict[".env.om_server::JWT_KEY_ID"] = { + "file": ".env.om_server", + "name": "JWT_KEY_ID", + "group": "Open Metadata Server", + "set_value": str(uuid.uuid4()), + } + # file_lines = prepare_dot_env_file_lines(output_dir=output_dir, env_var_dict=env_var_dict) + # print(file_lines) create_dot_env_files(output_dir=output_dir, env_var_dict=env_var_dict) diff --git a/makefile b/makefile index 4ece85a..fe22f7e 100644 --- a/makefile +++ b/makefile @@ -3,7 +3,7 @@ SHELL := /bin/bash build_images init_airflow initialize_system create_warehouse_infra update_dbt_packages \ dbt_generate_docs get_py_utils_shell make_fernet_key run_tests \ build_images_no_cache - + .DEFAULT_GOAL: startup MAKEFILE_FILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) @@ -16,7 +16,7 @@ run_time := "$(shell date '+%Y_%m_%d__%H_%M_%S')" PROJECT_NAME := $(shell basename $(MAKEFILE_DIR_PATH) | tr '[:upper:]' '[:lower:]') ADWH_SERVICES = $(shell docker compose config --services) -make_credentials: +make_open_metadata_pks: @if [ -f "${MAKEFILE_DIR_PATH}/config/private_key.pem" ]; then \ echo "private_keys for openmetadata auth already exist, doing nothing"; \ else \ @@ -24,7 +24,9 @@ make_credentials: openssl rsa -in "${MAKEFILE_DIR_PATH}/config/private_key.pem" -outform DER -pubout -out "${MAKEFILE_DIR_PATH}/config/public_key.der"; \ openssl pkcs8 -topk8 -inform PEM -outform DER -in "${MAKEFILE_DIR_PATH}/config/private_key.pem" -out "${MAKEFILE_DIR_PATH}/config/private_key.der" -nocrypt; \ fi - @if [ -f .env ] || [ -f .env.dwh ] || [ -f .env.superset ]; then \ + +make_credentials: make_open_metadata_pks + @if ls .env* >/dev/null 2>&1; then \ echo "Some .env files already exist. Remove or rename them to rerun startup process."; \ else \ echo "Running startup scripts to create .env files with ADWH credentials."; \ @@ -33,6 +35,8 @@ make_credentials: mv "${STARTUP_DIR}/.env" "${MAKEFILE_DIR_PATH}/.env"; \ mv "${STARTUP_DIR}/.env.dwh" "${MAKEFILE_DIR_PATH}/.env.dwh"; \ mv "${STARTUP_DIR}/.env.superset" "${MAKEFILE_DIR_PATH}/.env.superset"; \ + mv "${STARTUP_DIR}/.env.om_db" "${MAKEFILE_DIR_PATH}/.env.om_db"; \ + mv "${STARTUP_DIR}/.env.om_server" "${MAKEFILE_DIR_PATH}/.env.om_server"; \ fi