From ef914a0e3f53d7f1a480684680fd7fb806c006b7 Mon Sep 17 00:00:00 2001 From: Ian Knox Date: Fri, 15 Jul 2022 13:47:09 -0500 Subject: [PATCH 1/8] CLI modeling RAT, commands and global params --- core/dbt/cli/README.md | 1 + core/dbt/cli/__init__.py | 0 core/dbt/cli/global_params.py | 114 ++++++++++++++++++++ core/dbt/cli/main.py | 191 ++++++++++++++++++++++++++++++++++ core/setup.py | 4 +- 5 files changed, 307 insertions(+), 3 deletions(-) create mode 100644 core/dbt/cli/README.md create mode 100644 core/dbt/cli/__init__.py create mode 100644 core/dbt/cli/global_params.py create mode 100644 core/dbt/cli/main.py diff --git a/core/dbt/cli/README.md b/core/dbt/cli/README.md new file mode 100644 index 00000000000..1333ed77b7e --- /dev/null +++ b/core/dbt/cli/README.md @@ -0,0 +1 @@ +TODO diff --git a/core/dbt/cli/__init__.py b/core/dbt/cli/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/core/dbt/cli/global_params.py b/core/dbt/cli/global_params.py new file mode 100644 index 00000000000..753834496d1 --- /dev/null +++ b/core/dbt/cli/global_params.py @@ -0,0 +1,114 @@ +import click +from os import path + +# Commonly used global options (sort alphabeticaly) +profiles_dir = click.option( + "--profiles-dir", + help="Which directory to look in for the profiles.yml file. Default = " + + path.join(path.expanduser("~"), ".dbt"), + default=path.join(path.expanduser("~"), ".dbt"), +) + +version = click.option("--version", help="Show version information", is_flag=True, default=False) + +# Global options that override config (sort alphabeticaly) +cache_selected_only = click.option( + "--cache-selected-only/--no-cache-selected-only", + help="Pre cache database objects relevant to selected resource only.", + default=False, +) + +debug = click.option( + "--debug/--no-debug", + "-d/ ", + help="Display debug logging during dbt execution. Useful for debugging and making bug reports.", + default=False, +) + +fail_fast = click.option( + "--fail-fast/--no-fail-fast", "-x/ ", help="Stop execution on first failure.", default=False +) + +log_format = click.option( + "--log-format", + help="Specify the log format, overriding the command's default.", + type=click.Choice(["text", "json", "default"], case_sensitive=False), + default="default", +) + +partial_parse = click.option( + "--partial-parse/--no-partial-parse", + help="Allow for partial parsing by looking for and writing to a pickle file in the target directory. This overrides the user configuration file.", + default=True, +) + +print = click.option( + "--print/--no-print", help="Output all {{ print() }} macro calls.", default=True +) + +printer_width = click.option( + "--printer_width", help="Sets the width of terminal output", type=click.INT, default=80 +) + +quiet = click.option( + "--quiet/--no-quiet", + help="Suppress all non-error logging to stdout. Does not affect {{ print() }} macro calls.", + default=False, +) + +send_anonymous_usage_stats = click.option( + "--anonymous-usage-stats/--no-anonymous-usage-stats", + help="Send anonymous usage stats to dbt Labs.", + default=True, +) + +static_parser = click.option( + "--static-parser/--no-static-parser", help="Use the static parser.", default=True +) + +use_colors = click.option( + "--use-colors/--no-use-colors", + help="Output is colorized by default and may also be set in a profile or at the command line.", + default=True, +) + +use_experimental_parser = click.option( + "--use-experimental-parser/--no-use-experimental-parser", + help="Enable experimental parsing features.", + default=False, +) + +version_check = click.option( + "--version-check/--no-version-check", + help="Ensure dbt's version matches the one specified in the dbt_project.yml file ('require-dbt-version')", + default=True, +) + +warn_error = click.option( + "--warn-error/--no-warn-error", + help="If dbt would normally warn, instead raise an exception. Examples include --models that selects nothing, deprecations, configurations with no associated models, invalid test configurations, and missing sources/refs in tests.", + default=False, +) + +write_json = click.option( + "--write-json/--no-write-json", + help="Writing the manifest and run_results.json files to disk", + default=True, +) + + +# Rarely used global options +event_buffer_size = click.option( + "--event-buffer-size", + help="Sets the max number of events to buffer in EVENT_HISTORY.", + default=100000, + type=click.INT, +) + +record_timing = click.option( + "-r", + "--record-timing-info", + help="When this option is passed, dbt will output low-level timing stats to the specified file. Example: `--record-timing-info output.profile`", + is_flag=True, + default=False, +) diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py new file mode 100644 index 00000000000..1fcbdab2c53 --- /dev/null +++ b/core/dbt/cli/main.py @@ -0,0 +1,191 @@ +import click +import dbt.cli.global_params as global_params +import inspect # This is temporary for RAT-ing +import sys + + +# dbt +@click.group( + invoke_without_command=True, + no_args_is_help=True, + epilog="Specify one of these sub-commands and you can find more help from there.", +) +@click.pass_context +@global_params.profiles_dir +@global_params.version +@global_params.cache_selected_only +@global_params.debug +@global_params.fail_fast +@global_params.log_format +@global_params.partial_parse +@global_params.print +@global_params.printer_width +@global_params.quiet +@global_params.send_anonymous_usage_stats +@global_params.static_parser +@global_params.use_colors +@global_params.use_experimental_parser +@global_params.version_check +@global_params.warn_error +@global_params.write_json +@global_params.event_buffer_size +@global_params.record_timing +def cli(ctx, **kwargs): + """An ELT tool for managing your SQL transformations and data models. + For more documentation on these commands, visit: docs.getdbt.com + """ + + if kwargs.get("version", False): + click.echo(f"`version` called\n ctx.params: {ctx.params}") + sys.exit() + else: + del ctx.params["version"] + + +# dbt build +@cli.command("build") +@click.pass_context +def build(ctx, **kwargs): + """Run all Seeds, Models, Snapshots, and tests in DAG order""" + click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + + +# dbt clean +@cli.command("clean") +@click.pass_context +def clean(ctx, **kwargs): + """Delete all folders in the clean-targets list (usually the dbt_packages and target directories.)""" + click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + + +# dbt docs +@cli.group() +@click.pass_context +def docs(ctx, **kwargs): + """Generate or serve the documentation website for your project""" + + +# dbt docs generate +@docs.command("generate") +@click.pass_context +def docs_generate(ctx, **kwargs): + """Generate the documentation website for your project""" + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.parent.params}" + ) + + +# dbt docs serve +@docs.command("serve") +@click.pass_context +def docs_serve(ctx, **kwargs): + """Serve the documentation website for your project""" + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.parent.params}" + ) + + +# dbt compile +@cli.command("compile") +@click.pass_context +def compile(ctx, **kwargs): + """Generates executable SQL from source, model, test, and analysis files. Compiled SQL files are written to the target/ directory.""" + click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + + +# dbt debug +@cli.command("debug") +@click.pass_context +def debug(ctx, **kwargs): + """Show some helpful information about dbt for debugging. Not to be confused with the --debug option which increases verbosity.""" + click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + + +# dbt deps +@cli.command("deps") +@click.pass_context +def deps(ctx, **kwargs): + """Pull the most recent version of the dependencies listed in packages.yml""" + click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + + +# dbt init +@cli.command("init") +@click.pass_context +def init(ctx, **kwargs): + """Initialize a new DBT project.""" + click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + + +# dbt list +# dbt TODO: Figure out aliasing for ls (or just c/p?) +@cli.command("list") +@click.pass_context +def list(ctx, **kwargs): + """List the resources in your project""" + click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + + +# dbt parse +@cli.command("parse") +@click.pass_context +def parse(ctx, **kwargs): + """Parses the project and provides information on performance""" + click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + + +# dbt run +@cli.command("run") +@click.pass_context +def run(ctx, **kwargs): + """Compile SQL and execute against the current target database.""" + click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + + +# dbt run operation +@cli.command("run-operation") +@click.pass_context +def run_operation(ctx, **kwargs): + """Run the named macro with any supplied arguments.""" + click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + + +# dbt seed +@cli.command("seed") +@click.pass_context +def seed(ctx, **kwargs): + """Load data from csv files into your data warehouse.""" + click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + + +# dbt snapshot +@cli.command("snapshot") +@click.pass_context +def snapshot(ctx, **kwargs): + """Execute snapshots defined in your project""" + click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + + +# dbt source +@cli.group() +@click.pass_context +def source(ctx, **kwargs): + """Manage your project's sources""" + + +# dbt source freshness +@source.command("freshness") +@click.pass_context +def freshness(ctx, **kwargs): + """Snapshots the current freshness of the project's sources""" + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.parent.params}" + ) + + +# dbt test +@cli.command("test") +@click.pass_context +def test(ctx, **kwargs): + """Runs tests on data in deployed models. Run this after `dbt run`""" + click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") diff --git a/core/setup.py b/core/setup.py index 776a71897a7..5e0a5bde2e4 100644 --- a/core/setup.py +++ b/core/setup.py @@ -43,9 +43,7 @@ include_package_data=True, test_suite="test", entry_points={ - "console_scripts": [ - "dbt = dbt.main:main", - ], + "console_scripts": ["dbt = dbt.main:main", "dbt-cli = dbt.cli.main:cli"], }, install_requires=[ "Jinja2==2.11.3", From 4476158eb6c16d0cbd0c43c161075719103ff514 Mon Sep 17 00:00:00 2001 From: Ian Knox Date: Fri, 15 Jul 2022 14:02:41 -0500 Subject: [PATCH 2/8] pretty output --- core/dbt/cli/main.py | 61 ++++++++++++++++++++++++++++++++------------ 1 file changed, 44 insertions(+), 17 deletions(-) diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index 1fcbdab2c53..08a5c28502c 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -1,6 +1,7 @@ import click import dbt.cli.global_params as global_params import inspect # This is temporary for RAT-ing +from pprint import pformat as pf import sys @@ -36,7 +37,7 @@ def cli(ctx, **kwargs): """ if kwargs.get("version", False): - click.echo(f"`version` called\n ctx.params: {ctx.params}") + click.echo(f"`version` called\n ctx.params: {pf(ctx.params)}") sys.exit() else: del ctx.params["version"] @@ -47,7 +48,9 @@ def cli(ctx, **kwargs): @click.pass_context def build(ctx, **kwargs): """Run all Seeds, Models, Snapshots, and tests in DAG order""" - click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" + ) # dbt clean @@ -55,7 +58,9 @@ def build(ctx, **kwargs): @click.pass_context def clean(ctx, **kwargs): """Delete all folders in the clean-targets list (usually the dbt_packages and target directories.)""" - click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" + ) # dbt docs @@ -71,7 +76,7 @@ def docs(ctx, **kwargs): def docs_generate(ctx, **kwargs): """Generate the documentation website for your project""" click.echo( - f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.parent.params}" + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.parent.params)}" ) @@ -81,7 +86,7 @@ def docs_generate(ctx, **kwargs): def docs_serve(ctx, **kwargs): """Serve the documentation website for your project""" click.echo( - f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.parent.params}" + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.parent.params)}" ) @@ -90,7 +95,9 @@ def docs_serve(ctx, **kwargs): @click.pass_context def compile(ctx, **kwargs): """Generates executable SQL from source, model, test, and analysis files. Compiled SQL files are written to the target/ directory.""" - click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" + ) # dbt debug @@ -98,7 +105,9 @@ def compile(ctx, **kwargs): @click.pass_context def debug(ctx, **kwargs): """Show some helpful information about dbt for debugging. Not to be confused with the --debug option which increases verbosity.""" - click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" + ) # dbt deps @@ -106,7 +115,9 @@ def debug(ctx, **kwargs): @click.pass_context def deps(ctx, **kwargs): """Pull the most recent version of the dependencies listed in packages.yml""" - click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" + ) # dbt init @@ -114,7 +125,9 @@ def deps(ctx, **kwargs): @click.pass_context def init(ctx, **kwargs): """Initialize a new DBT project.""" - click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" + ) # dbt list @@ -123,7 +136,9 @@ def init(ctx, **kwargs): @click.pass_context def list(ctx, **kwargs): """List the resources in your project""" - click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" + ) # dbt parse @@ -131,7 +146,9 @@ def list(ctx, **kwargs): @click.pass_context def parse(ctx, **kwargs): """Parses the project and provides information on performance""" - click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" + ) # dbt run @@ -139,7 +156,9 @@ def parse(ctx, **kwargs): @click.pass_context def run(ctx, **kwargs): """Compile SQL and execute against the current target database.""" - click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" + ) # dbt run operation @@ -147,7 +166,9 @@ def run(ctx, **kwargs): @click.pass_context def run_operation(ctx, **kwargs): """Run the named macro with any supplied arguments.""" - click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" + ) # dbt seed @@ -155,7 +176,9 @@ def run_operation(ctx, **kwargs): @click.pass_context def seed(ctx, **kwargs): """Load data from csv files into your data warehouse.""" - click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" + ) # dbt snapshot @@ -163,7 +186,9 @@ def seed(ctx, **kwargs): @click.pass_context def snapshot(ctx, **kwargs): """Execute snapshots defined in your project""" - click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" + ) # dbt source @@ -179,7 +204,7 @@ def source(ctx, **kwargs): def freshness(ctx, **kwargs): """Snapshots the current freshness of the project's sources""" click.echo( - f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.parent.params}" + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.parent.params)}" ) @@ -188,4 +213,6 @@ def freshness(ctx, **kwargs): @click.pass_context def test(ctx, **kwargs): """Runs tests on data in deployed models. Run this after `dbt run`""" - click.echo(f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {ctx.parent.params}") + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" + ) From 3d36e8a6923f25011c1ff6e7f63e0aa4864a40aa Mon Sep 17 00:00:00 2001 From: Ian Knox Date: Tue, 19 Jul 2022 14:18:12 -0500 Subject: [PATCH 3/8] updates --- core/dbt/cli/main.py | 10 ++++-- core/dbt/cli/params/__init__.py | 0 .../{global_params.py => params/_global.py} | 10 +----- core/dbt/cli/params/project.py | 31 +++++++++++++++++++ 4 files changed, 39 insertions(+), 12 deletions(-) create mode 100644 core/dbt/cli/params/__init__.py rename core/dbt/cli/{global_params.py => params/_global.py} (91%) create mode 100644 core/dbt/cli/params/project.py diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index 08a5c28502c..fef5e09c10e 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -1,5 +1,6 @@ import click -import dbt.cli.global_params as global_params +from dbt.cli.params import _global as global_params +from dbt.cli.params import project as project_params import inspect # This is temporary for RAT-ing from pprint import pformat as pf import sys @@ -12,7 +13,6 @@ epilog="Specify one of these sub-commands and you can find more help from there.", ) @click.pass_context -@global_params.profiles_dir @global_params.version @global_params.cache_selected_only @global_params.debug @@ -35,7 +35,6 @@ def cli(ctx, **kwargs): """An ELT tool for managing your SQL transformations and data models. For more documentation on these commands, visit: docs.getdbt.com """ - if kwargs.get("version", False): click.echo(f"`version` called\n ctx.params: {pf(ctx.params)}") sys.exit() @@ -56,6 +55,11 @@ def build(ctx, **kwargs): # dbt clean @cli.command("clean") @click.pass_context +@project_params.profile +@project_params.profiles_dir +@project_params.project_dir +@project_params.target +@project_params.vars def clean(ctx, **kwargs): """Delete all folders in the clean-targets list (usually the dbt_packages and target directories.)""" click.echo( diff --git a/core/dbt/cli/params/__init__.py b/core/dbt/cli/params/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/core/dbt/cli/global_params.py b/core/dbt/cli/params/_global.py similarity index 91% rename from core/dbt/cli/global_params.py rename to core/dbt/cli/params/_global.py index 753834496d1..7ae3d07c501 100644 --- a/core/dbt/cli/global_params.py +++ b/core/dbt/cli/params/_global.py @@ -1,14 +1,6 @@ import click -from os import path - -# Commonly used global options (sort alphabeticaly) -profiles_dir = click.option( - "--profiles-dir", - help="Which directory to look in for the profiles.yml file. Default = " - + path.join(path.expanduser("~"), ".dbt"), - default=path.join(path.expanduser("~"), ".dbt"), -) +# Version is a special snowflake, list it first version = click.option("--version", help="Show version information", is_flag=True, default=False) # Global options that override config (sort alphabeticaly) diff --git a/core/dbt/cli/params/project.py b/core/dbt/cli/params/project.py new file mode 100644 index 00000000000..1be04952068 --- /dev/null +++ b/core/dbt/cli/params/project.py @@ -0,0 +1,31 @@ +import click +from pathlib import Path, PurePath + +project_dir = click.option( + "--project-dir", + help="Which directory to look in for the dbt_project.yml file. Default is the current working directory and its parents.", + default=Path.cwd(), + type=click.Path(exists=True), +) + +profiles_dir = click.option( + "--profiles-dir", + help=f"Which directory to look in for the profiles.yml file. Default = {PurePath.joinpath(Path.home(), '.dbt')}", + default=PurePath.joinpath(Path.home(), ".dbt"), + type=click.Path( + exists=True, + ), +) + +profile = click.option( + "--profile", + help="Which profile to load. Overrides setting in dbt_project.yml.", +) + +target = click.option("-t", "--target", help="Which target to load for the given profile") + +# TODO validate the yaml (validation callback + custom type) +vars = click.option( + "--vars", + help="Supply variables to the project. This argument overrides variables defined in your dbt_project.yml file. This argument should be a YAML string, eg. '{my_variable: my_value}'", +) From 03af44502a9b264d2c91d293bb767ad3ecf5ccb5 Mon Sep 17 00:00:00 2001 From: Ian Knox Date: Tue, 19 Jul 2022 15:10:28 -0500 Subject: [PATCH 4/8] added multi-flag example and a custom type --- core/dbt/cli/main.py | 17 ++++++++++++++ core/dbt/cli/params/project.py | 2 ++ core/dbt/cli/params/run.py | 43 ++++++++++++++++++++++++++++++++++ core/dbt/cli/params/types.py | 17 ++++++++++++++ 4 files changed, 79 insertions(+) create mode 100644 core/dbt/cli/params/run.py create mode 100644 core/dbt/cli/params/types.py diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index fef5e09c10e..276590dc8f7 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -1,6 +1,7 @@ import click from dbt.cli.params import _global as global_params from dbt.cli.params import project as project_params +from dbt.cli.params import run as run_params import inspect # This is temporary for RAT-ing from pprint import pformat as pf import sys @@ -158,6 +159,22 @@ def parse(ctx, **kwargs): # dbt run @cli.command("run") @click.pass_context +@global_params.fail_fast +@global_params.version_check +@project_params.profile +@project_params.profiles_dir +@project_params.project_dir +@project_params.target +@project_params.vars +@run_params.log_path +@run_params.target_path +@run_params.threads +@run_params.models +@run_params.exclude +@run_params.selector +@run_params.state +@run_params.defer +@run_params.full_refresh def run(ctx, **kwargs): """Compile SQL and execute against the current target database.""" click.echo( diff --git a/core/dbt/cli/params/project.py b/core/dbt/cli/params/project.py index 1be04952068..8e10fc8eea3 100644 --- a/core/dbt/cli/params/project.py +++ b/core/dbt/cli/params/project.py @@ -1,5 +1,6 @@ import click from pathlib import Path, PurePath +from dbt.cli.params.types import YAML project_dir = click.option( "--project-dir", @@ -28,4 +29,5 @@ vars = click.option( "--vars", help="Supply variables to the project. This argument overrides variables defined in your dbt_project.yml file. This argument should be a YAML string, eg. '{my_variable: my_value}'", + type=YAML(), ) diff --git a/core/dbt/cli/params/run.py b/core/dbt/cli/params/run.py new file mode 100644 index 00000000000..ccaae943637 --- /dev/null +++ b/core/dbt/cli/params/run.py @@ -0,0 +1,43 @@ +import click + +log_path = click.option( + "--log-path", + help="Configure the 'log-path'. Only applies this setting for the current run. Overrides the 'DBT_LOG_PATH' if it is set.", + type=click.Path(), +) + +target_path = click.option( + "--target-path", + help="Configure the 'target-path'. Only applies this setting for the current run. Overrides the 'DBT_TARGET_PATH' if it is set.", + type=click.Path(), +) + +threads = click.option( + "--threads", + help="Specify number of threads to use while executing models. Overrides settings in profiles.yml.", + default=1, + type=click.INT, +) + +models = click.option("-m", "-s", help="Specify the nodes to include.", multiple=True) + +exclude = click.option("--exclude", help="Specify the nodes to exclude.") + +selector = click.option("--selector", help="The selector name to use, as defined in selectors.yml") + +state = click.option( + "--state", + help="If set, use the given directory as the source for json files to compare with this project.", +) + +defer = click.option( + "--defer/--no-defer", + help="If set, defer to the state variable for resolving unselected nodes.", + default=True, +) + +full_refresh = click.option( + "--full_refresh", + help="If specified, dbt will drop incremental models and fully-recalculate the incremental table from the model definition.", + is_flag=True, +) diff --git a/core/dbt/cli/params/types.py b/core/dbt/cli/params/types.py new file mode 100644 index 00000000000..6c024b4966b --- /dev/null +++ b/core/dbt/cli/params/types.py @@ -0,0 +1,17 @@ +from click import ParamType +import yaml + + +class YAML(ParamType): + """The YAML type converts YAML strings into objects.""" + + name = "YAML" + + def convert(self, value, param, ctx): + # assume non-string values are a problem + if not isinstance(value, str): + self.fail(f"Cannot load YAML from type {type(value)}", param, ctx) + try: + return yaml.load(value, Loader=yaml.Loader) + except yaml.parser.ParserError: + self.fail(f"String '{value}' is not valid YAML", param, ctx) From 83f84ef206c3690b88096f0a7fcba8fa41796732 Mon Sep 17 00:00:00 2001 From: Ian Knox Date: Thu, 11 Aug 2022 14:23:14 -0500 Subject: [PATCH 5/8] feature complete --- core/dbt/cli/main.py | 147 ++++++++++++++++++++++++++++++++- core/dbt/cli/params/docs.py | 17 ++++ core/dbt/cli/params/misc.py | 46 +++++++++++ core/dbt/cli/params/parse.py | 13 +++ core/dbt/cli/params/project.py | 1 - core/dbt/cli/params/run.py | 43 +++++++++- 6 files changed, 263 insertions(+), 4 deletions(-) create mode 100644 core/dbt/cli/params/docs.py create mode 100644 core/dbt/cli/params/misc.py create mode 100644 core/dbt/cli/params/parse.py diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index 276590dc8f7..b7de2eaaccd 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -1,5 +1,8 @@ import click from dbt.cli.params import _global as global_params +from dbt.cli.params import docs as docs_params +from dbt.cli.params import misc as misc_params +from dbt.cli.params import parse as parse_params from dbt.cli.params import project as project_params from dbt.cli.params import run as run_params import inspect # This is temporary for RAT-ing @@ -56,9 +59,9 @@ def build(ctx, **kwargs): # dbt clean @cli.command("clean") @click.pass_context -@project_params.profile -@project_params.profiles_dir @project_params.project_dir +@project_params.profiles_dir +@project_params.profile @project_params.target @project_params.vars def clean(ctx, **kwargs): @@ -78,6 +81,21 @@ def docs(ctx, **kwargs): # dbt docs generate @docs.command("generate") @click.pass_context +@global_params.version_check +@project_params.project_dir +@project_params.profiles_dir +@project_params.profile +@project_params.target +@project_params.vars +@docs_params.compile +@run_params.defer +@run_params.threads +@run_params.target_path +@run_params.log_path +@run_params.models +@run_params.exclude +@run_params.selector +@run_params.state def docs_generate(ctx, **kwargs): """Generate the documentation website for your project""" click.echo( @@ -88,6 +106,13 @@ def docs_generate(ctx, **kwargs): # dbt docs serve @docs.command("serve") @click.pass_context +@project_params.project_dir +@project_params.profiles_dir +@project_params.profile +@project_params.target +@project_params.vars +@docs_params.port +@docs_params.browser def docs_serve(ctx, **kwargs): """Serve the documentation website for your project""" click.echo( @@ -98,6 +123,22 @@ def docs_serve(ctx, **kwargs): # dbt compile @cli.command("compile") @click.pass_context +@global_params.version_check +@project_params.project_dir +@project_params.profiles_dir +@project_params.profile +@project_params.target +@project_params.vars +@run_params.parse_only +@run_params.threads +@run_params.target_path +@run_params.log_path +@run_params.models +@run_params.exclude +@run_params.selector +@run_params.state +@run_params.defer +@run_params.full_refresh def compile(ctx, **kwargs): """Generates executable SQL from source, model, test, and analysis files. Compiled SQL files are written to the target/ directory.""" click.echo( @@ -108,6 +149,13 @@ def compile(ctx, **kwargs): # dbt debug @cli.command("debug") @click.pass_context +@global_params.version_check +@project_params.project_dir +@project_params.profiles_dir +@project_params.profile +@project_params.target +@project_params.vars +@misc_params.config_dir def debug(ctx, **kwargs): """Show some helpful information about dbt for debugging. Not to be confused with the --debug option which increases verbosity.""" click.echo( @@ -118,6 +166,11 @@ def debug(ctx, **kwargs): # dbt deps @cli.command("deps") @click.pass_context +@project_params.profile +@project_params.profiles_dir +@project_params.project_dir +@project_params.target +@project_params.vars def deps(ctx, **kwargs): """Pull the most recent version of the dependencies listed in packages.yml""" click.echo( @@ -128,6 +181,12 @@ def deps(ctx, **kwargs): # dbt init @cli.command("init") @click.pass_context +@project_params.profile +@project_params.profiles_dir +@project_params.project_dir +@project_params.target +@project_params.vars +@misc_params.skip_profile_setup def init(ctx, **kwargs): """Initialize a new DBT project.""" click.echo( @@ -139,6 +198,19 @@ def init(ctx, **kwargs): # dbt TODO: Figure out aliasing for ls (or just c/p?) @cli.command("list") @click.pass_context +@project_params.profile +@project_params.profiles_dir +@project_params.project_dir +@project_params.target +@project_params.vars +@misc_params.output +@misc_params.ouptut_keys +@run_params.resource_type +@run_params.models +@run_params.indirect_selection +@run_params.exclude +@run_params.selector +@run_params.state def list(ctx, **kwargs): """List the resources in your project""" click.echo( @@ -149,6 +221,17 @@ def list(ctx, **kwargs): # dbt parse @cli.command("parse") @click.pass_context +@project_params.profile +@project_params.profiles_dir +@project_params.project_dir +@project_params.target +@project_params.vars +@parse_params.write_manifest +@parse_params.compile +@run_params.threads +@run_params.target_path +@run_params.log_path +@global_params.version_check def parse(ctx, **kwargs): """Parses the project and provides information on performance""" click.echo( @@ -185,6 +268,12 @@ def run(ctx, **kwargs): # dbt run operation @cli.command("run-operation") @click.pass_context +@project_params.profile +@project_params.profiles_dir +@project_params.project_dir +@project_params.target +@project_params.vars +@run_params.args def run_operation(ctx, **kwargs): """Run the named macro with any supplied arguments.""" click.echo( @@ -195,6 +284,21 @@ def run_operation(ctx, **kwargs): # dbt seed @cli.command("seed") @click.pass_context +@global_params.version_check +@project_params.profile +@project_params.profiles_dir +@project_params.project_dir +@project_params.target +@project_params.vars +@run_params.full_refresh +@run_params.log_path +@run_params.target_path +@run_params.threads +@run_params.models +@run_params.exclude +@run_params.selector +@run_params.state +@misc_params.show def seed(ctx, **kwargs): """Load data from csv files into your data warehouse.""" click.echo( @@ -205,6 +309,17 @@ def seed(ctx, **kwargs): # dbt snapshot @cli.command("snapshot") @click.pass_context +@project_params.profile +@project_params.profiles_dir +@project_params.project_dir +@project_params.target +@project_params.vars +@run_params.threads +@run_params.models +@run_params.exclude +@run_params.selector +@run_params.state +@run_params.defer def snapshot(ctx, **kwargs): """Execute snapshots defined in your project""" click.echo( @@ -222,6 +337,17 @@ def source(ctx, **kwargs): # dbt source freshness @source.command("freshness") @click.pass_context +@project_params.profile +@project_params.profiles_dir +@project_params.project_dir +@project_params.target +@project_params.vars +@run_params.threads +@run_params.models +@run_params.exclude +@run_params.selector +@run_params.state +@misc_params.output_path # TODO: Is this ok to re-use? We have three different output params, how much can we consolidate? def freshness(ctx, **kwargs): """Snapshots the current freshness of the project's sources""" click.echo( @@ -232,6 +358,23 @@ def freshness(ctx, **kwargs): # dbt test @cli.command("test") @click.pass_context +@global_params.fail_fast +@global_params.version_check +@misc_params.store_failures +@project_params.profile +@project_params.profiles_dir +@project_params.project_dir +@project_params.target +@project_params.vars +@run_params.indirect_selection +@run_params.log_path +@run_params.target_path +@run_params.threads +@run_params.models +@run_params.exclude +@run_params.selector +@run_params.state +@run_params.defer def test(ctx, **kwargs): """Runs tests on data in deployed models. Run this after `dbt run`""" click.echo( diff --git a/core/dbt/cli/params/docs.py b/core/dbt/cli/params/docs.py new file mode 100644 index 00000000000..05425824dc3 --- /dev/null +++ b/core/dbt/cli/params/docs.py @@ -0,0 +1,17 @@ +import click + +compile = click.option( + "--compile/--no-compile", + help="Wether or not to run 'dbt compile' as part of docs generation", + default=True, +) + +port = click.option( + "--port", help="Specify the port number for the docs server", default=8080, type=click.INT +) + +browser = click.option( + "--browser/--no-browser", + help="Wether or not to open a local web browser after starting the server", + default=True, +) diff --git a/core/dbt/cli/params/misc.py b/core/dbt/cli/params/misc.py new file mode 100644 index 00000000000..6e85d3d2760 --- /dev/null +++ b/core/dbt/cli/params/misc.py @@ -0,0 +1,46 @@ +import click +from pathlib import Path, PurePath + +config_dir = click.option( + "--config-dir", + help="If specified, DBT will show path information for this project", + type=click.STRING, +) + +skip_profile_setup = click.option( + "--skip-profile-setup", + "-s", + help="Skip interative profile setup.", + default=False, +) + +output = click.option( + "--output", + help="TODO: No current help text", + type=click.Choice(["json", "name", "path", "selector"], case_sensitive=False), + default="name", +) + +ouptut_keys = click.option( + "--output-keys", + help="TODO: No current help text", + default=False, +) + +show = click.option( + "--show", + help="Show a sample of the loaded data in the terminal", + default=False, +) + +output_path = click.option( + "--output", + "-o", + help="Specify the output path for the json report. By default, outputs to 'target/sources.json'", + type=click.Path(file_okay=True, dir_okay=False, writable=True), + default=PurePath.joinpath(Path.cwd(), "target/sources.json"), +) + +store_failures = click.option( + "--store-failures", help="Store test results (failing rows) in the database", default=False +) diff --git a/core/dbt/cli/params/parse.py b/core/dbt/cli/params/parse.py new file mode 100644 index 00000000000..0f376d91e31 --- /dev/null +++ b/core/dbt/cli/params/parse.py @@ -0,0 +1,13 @@ +import click + +compile = click.option( + "--compile/--no-compile", + help="TODO: No help text currently available", + default=True, +) + +write_manifest = click.option( + "--write-manifest/--no-write-manifest", + help="TODO: No help text currently available", + default=True, +) diff --git a/core/dbt/cli/params/project.py b/core/dbt/cli/params/project.py index 8e10fc8eea3..00a3494732f 100644 --- a/core/dbt/cli/params/project.py +++ b/core/dbt/cli/params/project.py @@ -25,7 +25,6 @@ target = click.option("-t", "--target", help="Which target to load for the given profile") -# TODO validate the yaml (validation callback + custom type) vars = click.option( "--vars", help="Supply variables to the project. This argument overrides variables defined in your dbt_project.yml file. This argument should be a YAML string, eg. '{my_variable: my_value}'", diff --git a/core/dbt/cli/params/run.py b/core/dbt/cli/params/run.py index ccaae943637..035bfe06f75 100644 --- a/core/dbt/cli/params/run.py +++ b/core/dbt/cli/params/run.py @@ -1,4 +1,5 @@ import click +from dbt.cli.params.types import YAML log_path = click.option( "--log-path", @@ -37,7 +38,47 @@ ) full_refresh = click.option( - "--full_refresh", + "--full-refresh", help="If specified, dbt will drop incremental models and fully-recalculate the incremental table from the model definition.", is_flag=True, ) + +parse_only = click.option( + "--parse-only", + help="TODO: No help text currently available", + is_flag=True, +) + +resource_type = click.option( + "--resource-type", + help="TODO: No current help text", + type=click.Choice( + [ + "metric", + "source", + "analysis", + "model", + "test", + "exposure", + "snapshot", + "seed", + "default", + "all", + ], + case_sensitive=False, + ), + default="default", +) + +indirect_selection = click.option( + "--indirect_selection", + help="Select all tests that are adjacent to selected resources, even if they those resources have been explicitly selected.", + type=click.Choice(["eager", "cautious"], case_sensitive=False), + default="eager", +) + +args = click.option( + "--args", + help="Supply arguments to the macro. This dictionary will be mapped to the keyword arguments defined in the selected macro. This argument should be a YAML string, eg. '{my_variable: my_value}'", + type=YAML(), +) From 61650cafd91c11556351fb125dbfbef3d7db48b4 Mon Sep 17 00:00:00 2001 From: Ian Knox Date: Thu, 11 Aug 2022 16:04:25 -0500 Subject: [PATCH 6/8] consolidate params --- core/dbt/cli/main.py | 371 ++++++++++++++++---------------- core/dbt/cli/params.py | 304 ++++++++++++++++++++++++++ core/dbt/cli/params/__init__.py | 0 core/dbt/cli/params/_global.py | 106 --------- core/dbt/cli/params/docs.py | 17 -- core/dbt/cli/params/misc.py | 46 ---- core/dbt/cli/params/parse.py | 13 -- core/dbt/cli/params/project.py | 32 --- core/dbt/cli/params/run.py | 84 -------- core/dbt/cli/params/types.py | 17 -- 10 files changed, 488 insertions(+), 502 deletions(-) create mode 100644 core/dbt/cli/params.py delete mode 100644 core/dbt/cli/params/__init__.py delete mode 100644 core/dbt/cli/params/_global.py delete mode 100644 core/dbt/cli/params/docs.py delete mode 100644 core/dbt/cli/params/misc.py delete mode 100644 core/dbt/cli/params/parse.py delete mode 100644 core/dbt/cli/params/project.py delete mode 100644 core/dbt/cli/params/run.py delete mode 100644 core/dbt/cli/params/types.py diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index b7de2eaaccd..3f11c001827 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -1,14 +1,11 @@ import click -from dbt.cli.params import _global as global_params -from dbt.cli.params import docs as docs_params -from dbt.cli.params import misc as misc_params -from dbt.cli.params import parse as parse_params -from dbt.cli.params import project as project_params -from dbt.cli.params import run as run_params -import inspect # This is temporary for RAT-ing -from pprint import pformat as pf +from dbt.cli import params as p import sys +# This is temporary for RAT-ing +import inspect +from pprint import pformat as pf + # dbt @click.group( @@ -17,24 +14,24 @@ epilog="Specify one of these sub-commands and you can find more help from there.", ) @click.pass_context -@global_params.version -@global_params.cache_selected_only -@global_params.debug -@global_params.fail_fast -@global_params.log_format -@global_params.partial_parse -@global_params.print -@global_params.printer_width -@global_params.quiet -@global_params.send_anonymous_usage_stats -@global_params.static_parser -@global_params.use_colors -@global_params.use_experimental_parser -@global_params.version_check -@global_params.warn_error -@global_params.write_json -@global_params.event_buffer_size -@global_params.record_timing +@p.version +@p.cache_selected_only +@p.debug +@p.fail_fast +@p.log_format +@p.partial_parse +@p.print +@p.printer_width +@p.quiet +@p.send_anonymous_usage_stats +@p.static_parser +@p.use_colors +@p.use_experimental_parser +@p.version_check +@p.warn_error +@p.write_json +@p.event_buffer_size +@p.record_timing def cli(ctx, **kwargs): """An ELT tool for managing your SQL transformations and data models. For more documentation on these commands, visit: docs.getdbt.com @@ -59,11 +56,11 @@ def build(ctx, **kwargs): # dbt clean @cli.command("clean") @click.pass_context -@project_params.project_dir -@project_params.profiles_dir -@project_params.profile -@project_params.target -@project_params.vars +@p.project_dir +@p.profiles_dir +@p.profile +@p.target +@p.vars def clean(ctx, **kwargs): """Delete all folders in the clean-targets list (usually the dbt_packages and target directories.)""" click.echo( @@ -81,21 +78,21 @@ def docs(ctx, **kwargs): # dbt docs generate @docs.command("generate") @click.pass_context -@global_params.version_check -@project_params.project_dir -@project_params.profiles_dir -@project_params.profile -@project_params.target -@project_params.vars -@docs_params.compile -@run_params.defer -@run_params.threads -@run_params.target_path -@run_params.log_path -@run_params.models -@run_params.exclude -@run_params.selector -@run_params.state +@p.version_check +@p.project_dir +@p.profiles_dir +@p.profile +@p.target +@p.vars +@p.compile_docs +@p.defer +@p.threads +@p.target_path +@p.log_path +@p.models +@p.exclude +@p.selector +@p.state def docs_generate(ctx, **kwargs): """Generate the documentation website for your project""" click.echo( @@ -106,13 +103,13 @@ def docs_generate(ctx, **kwargs): # dbt docs serve @docs.command("serve") @click.pass_context -@project_params.project_dir -@project_params.profiles_dir -@project_params.profile -@project_params.target -@project_params.vars -@docs_params.port -@docs_params.browser +@p.project_dir +@p.profiles_dir +@p.profile +@p.target +@p.vars +@p.port +@p.browser def docs_serve(ctx, **kwargs): """Serve the documentation website for your project""" click.echo( @@ -123,22 +120,22 @@ def docs_serve(ctx, **kwargs): # dbt compile @cli.command("compile") @click.pass_context -@global_params.version_check -@project_params.project_dir -@project_params.profiles_dir -@project_params.profile -@project_params.target -@project_params.vars -@run_params.parse_only -@run_params.threads -@run_params.target_path -@run_params.log_path -@run_params.models -@run_params.exclude -@run_params.selector -@run_params.state -@run_params.defer -@run_params.full_refresh +@p.version_check +@p.project_dir +@p.profiles_dir +@p.profile +@p.target +@p.vars +@p.parse_only +@p.threads +@p.target_path +@p.log_path +@p.models +@p.exclude +@p.selector +@p.state +@p.defer +@p.full_refresh def compile(ctx, **kwargs): """Generates executable SQL from source, model, test, and analysis files. Compiled SQL files are written to the target/ directory.""" click.echo( @@ -149,13 +146,13 @@ def compile(ctx, **kwargs): # dbt debug @cli.command("debug") @click.pass_context -@global_params.version_check -@project_params.project_dir -@project_params.profiles_dir -@project_params.profile -@project_params.target -@project_params.vars -@misc_params.config_dir +@p.version_check +@p.project_dir +@p.profiles_dir +@p.profile +@p.target +@p.vars +@p.config_dir def debug(ctx, **kwargs): """Show some helpful information about dbt for debugging. Not to be confused with the --debug option which increases verbosity.""" click.echo( @@ -166,11 +163,11 @@ def debug(ctx, **kwargs): # dbt deps @cli.command("deps") @click.pass_context -@project_params.profile -@project_params.profiles_dir -@project_params.project_dir -@project_params.target -@project_params.vars +@p.profile +@p.profiles_dir +@p.project_dir +@p.target +@p.vars def deps(ctx, **kwargs): """Pull the most recent version of the dependencies listed in packages.yml""" click.echo( @@ -181,12 +178,12 @@ def deps(ctx, **kwargs): # dbt init @cli.command("init") @click.pass_context -@project_params.profile -@project_params.profiles_dir -@project_params.project_dir -@project_params.target -@project_params.vars -@misc_params.skip_profile_setup +@p.profile +@p.profiles_dir +@p.project_dir +@p.target +@p.vars +@p.skip_profile_setup def init(ctx, **kwargs): """Initialize a new DBT project.""" click.echo( @@ -198,19 +195,19 @@ def init(ctx, **kwargs): # dbt TODO: Figure out aliasing for ls (or just c/p?) @cli.command("list") @click.pass_context -@project_params.profile -@project_params.profiles_dir -@project_params.project_dir -@project_params.target -@project_params.vars -@misc_params.output -@misc_params.ouptut_keys -@run_params.resource_type -@run_params.models -@run_params.indirect_selection -@run_params.exclude -@run_params.selector -@run_params.state +@p.profile +@p.profiles_dir +@p.project_dir +@p.target +@p.vars +@p.output +@p.ouptut_keys +@p.resource_type +@p.models +@p.indirect_selection +@p.exclude +@p.selector +@p.state def list(ctx, **kwargs): """List the resources in your project""" click.echo( @@ -221,17 +218,17 @@ def list(ctx, **kwargs): # dbt parse @cli.command("parse") @click.pass_context -@project_params.profile -@project_params.profiles_dir -@project_params.project_dir -@project_params.target -@project_params.vars -@parse_params.write_manifest -@parse_params.compile -@run_params.threads -@run_params.target_path -@run_params.log_path -@global_params.version_check +@p.profile +@p.profiles_dir +@p.project_dir +@p.target +@p.vars +@p.write_manifest +@p.compile_parse +@p.threads +@p.target_path +@p.log_path +@p.version_check def parse(ctx, **kwargs): """Parses the project and provides information on performance""" click.echo( @@ -242,22 +239,22 @@ def parse(ctx, **kwargs): # dbt run @cli.command("run") @click.pass_context -@global_params.fail_fast -@global_params.version_check -@project_params.profile -@project_params.profiles_dir -@project_params.project_dir -@project_params.target -@project_params.vars -@run_params.log_path -@run_params.target_path -@run_params.threads -@run_params.models -@run_params.exclude -@run_params.selector -@run_params.state -@run_params.defer -@run_params.full_refresh +@p.fail_fast +@p.version_check +@p.profile +@p.profiles_dir +@p.project_dir +@p.target +@p.vars +@p.log_path +@p.target_path +@p.threads +@p.models +@p.exclude +@p.selector +@p.state +@p.defer +@p.full_refresh def run(ctx, **kwargs): """Compile SQL and execute against the current target database.""" click.echo( @@ -268,12 +265,12 @@ def run(ctx, **kwargs): # dbt run operation @cli.command("run-operation") @click.pass_context -@project_params.profile -@project_params.profiles_dir -@project_params.project_dir -@project_params.target -@project_params.vars -@run_params.args +@p.profile +@p.profiles_dir +@p.project_dir +@p.target +@p.vars +@p.args def run_operation(ctx, **kwargs): """Run the named macro with any supplied arguments.""" click.echo( @@ -284,21 +281,21 @@ def run_operation(ctx, **kwargs): # dbt seed @cli.command("seed") @click.pass_context -@global_params.version_check -@project_params.profile -@project_params.profiles_dir -@project_params.project_dir -@project_params.target -@project_params.vars -@run_params.full_refresh -@run_params.log_path -@run_params.target_path -@run_params.threads -@run_params.models -@run_params.exclude -@run_params.selector -@run_params.state -@misc_params.show +@p.version_check +@p.profile +@p.profiles_dir +@p.project_dir +@p.target +@p.vars +@p.full_refresh +@p.log_path +@p.target_path +@p.threads +@p.models +@p.exclude +@p.selector +@p.state +@p.show def seed(ctx, **kwargs): """Load data from csv files into your data warehouse.""" click.echo( @@ -309,17 +306,17 @@ def seed(ctx, **kwargs): # dbt snapshot @cli.command("snapshot") @click.pass_context -@project_params.profile -@project_params.profiles_dir -@project_params.project_dir -@project_params.target -@project_params.vars -@run_params.threads -@run_params.models -@run_params.exclude -@run_params.selector -@run_params.state -@run_params.defer +@p.profile +@p.profiles_dir +@p.project_dir +@p.target +@p.vars +@p.threads +@p.models +@p.exclude +@p.selector +@p.state +@p.defer def snapshot(ctx, **kwargs): """Execute snapshots defined in your project""" click.echo( @@ -337,17 +334,17 @@ def source(ctx, **kwargs): # dbt source freshness @source.command("freshness") @click.pass_context -@project_params.profile -@project_params.profiles_dir -@project_params.project_dir -@project_params.target -@project_params.vars -@run_params.threads -@run_params.models -@run_params.exclude -@run_params.selector -@run_params.state -@misc_params.output_path # TODO: Is this ok to re-use? We have three different output params, how much can we consolidate? +@p.profile +@p.profiles_dir +@p.project_dir +@p.target +@p.vars +@p.threads +@p.models +@p.exclude +@p.selector +@p.state +@p.output_path # TODO: Is this ok to re-use? We have three different output params, how much can we consolidate? def freshness(ctx, **kwargs): """Snapshots the current freshness of the project's sources""" click.echo( @@ -358,23 +355,23 @@ def freshness(ctx, **kwargs): # dbt test @cli.command("test") @click.pass_context -@global_params.fail_fast -@global_params.version_check -@misc_params.store_failures -@project_params.profile -@project_params.profiles_dir -@project_params.project_dir -@project_params.target -@project_params.vars -@run_params.indirect_selection -@run_params.log_path -@run_params.target_path -@run_params.threads -@run_params.models -@run_params.exclude -@run_params.selector -@run_params.state -@run_params.defer +@p.fail_fast +@p.version_check +@p.store_failures +@p.profile +@p.profiles_dir +@p.project_dir +@p.target +@p.vars +@p.indirect_selection +@p.log_path +@p.target_path +@p.threads +@p.models +@p.exclude +@p.selector +@p.state +@p.defer def test(ctx, **kwargs): """Runs tests on data in deployed models. Run this after `dbt run`""" click.echo( diff --git a/core/dbt/cli/params.py b/core/dbt/cli/params.py new file mode 100644 index 00000000000..f991359ecf7 --- /dev/null +++ b/core/dbt/cli/params.py @@ -0,0 +1,304 @@ +import click +import yaml +from pathlib import Path, PurePath +from click import ParamType + + +class YAML(ParamType): + """The Click YAML type. Converts YAML strings into objects.""" + + name = "YAML" + + def convert(self, value, param, ctx): + # assume non-string values are a problem + if not isinstance(value, str): + self.fail(f"Cannot load YAML from type {type(value)}", param, ctx) + try: + return yaml.load(value, Loader=yaml.Loader) + except yaml.parser.ParserError: + self.fail(f"String '{value}' is not valid YAML", param, ctx) + + +args = click.option( + "--args", + help="Supply arguments to the macro. This dictionary will be mapped to the keyword arguments defined in the selected macro. This argument should be a YAML string, eg. '{my_variable: my_value}'", + type=YAML(), +) + +browser = click.option( + "--browser/--no-browser", + help="Wether or not to open a local web browser after starting the server", + default=True, +) + +cache_selected_only = click.option( + "--cache-selected-only/--no-cache-selected-only", + help="Pre cache database objects relevant to selected resource only.", + default=False, +) + +compile_docs = click.option( + "--compile/--no-compile", + help="Wether or not to run 'dbt compile' as part of docs generation", + default=True, +) + +compile_parse = click.option( + "--compile/--no-compile", + help="TODO: No help text currently available", + default=True, +) + +config_dir = click.option( + "--config-dir", + help="If specified, DBT will show path information for this project", + type=click.STRING, +) + +debug = click.option( + "--debug/--no-debug", + "-d/ ", + help="Display debug logging during dbt execution. Useful for debugging and making bug reports.", + default=False, +) + +defer = click.option( + "--defer/--no-defer", + help="If set, defer to the state variable for resolving unselected nodes.", + default=True, +) + +event_buffer_size = click.option( + "--event-buffer-size", + help="Sets the max number of events to buffer in EVENT_HISTORY.", + default=100000, + type=click.INT, +) + +exclude = click.option("--exclude", help="Specify the nodes to exclude.") + +fail_fast = click.option( + "--fail-fast/--no-fail-fast", "-x/ ", help="Stop execution on first failure.", default=False +) + +full_refresh = click.option( + "--full-refresh", + help="If specified, dbt will drop incremental models and fully-recalculate the incremental table from the model definition.", + is_flag=True, +) + +indirect_selection = click.option( + "--indirect_selection", + help="Select all tests that are adjacent to selected resources, even if they those resources have been explicitly selected.", + type=click.Choice(["eager", "cautious"], case_sensitive=False), + default="eager", +) + +log_format = click.option( + "--log-format", + help="Specify the log format, overriding the command's default.", + type=click.Choice(["text", "json", "default"], case_sensitive=False), + default="default", +) + +log_path = click.option( + "--log-path", + help="Configure the 'log-path'. Only applies this setting for the current run. Overrides the 'DBT_LOG_PATH' if it is set.", + type=click.Path(), +) + +models = click.option("-m", "-s", help="Specify the nodes to include.", multiple=True) + +output = click.option( + "--output", + help="TODO: No current help text", + type=click.Choice(["json", "name", "path", "selector"], case_sensitive=False), + default="name", +) + +ouptut_keys = click.option( + "--output-keys", + help="TODO: No current help text", + default=False, +) + +output_path = click.option( + "--output", + "-o", + help="Specify the output path for the json report. By default, outputs to 'target/sources.json'", + type=click.Path(file_okay=True, dir_okay=False, writable=True), + default=PurePath.joinpath(Path.cwd(), "target/sources.json"), +) + +parse_only = click.option( + "--parse-only", + help="TODO: No help text currently available", + is_flag=True, +) + +partial_parse = click.option( + "--partial-parse/--no-partial-parse", + help="Allow for partial parsing by looking for and writing to a pickle file in the target directory. This overrides the user configuration file.", + default=True, +) + +port = click.option( + "--port", help="Specify the port number for the docs server", default=8080, type=click.INT +) + +print = click.option( + "--print/--no-print", help="Output all {{ print() }} macro calls.", default=True +) + +printer_width = click.option( + "--printer_width", help="Sets the width of terminal output", type=click.INT, default=80 +) + +profile = click.option( + "--profile", + help="Which profile to load. Overrides setting in dbt_project.yml.", +) + +profiles_dir = click.option( + "--profiles-dir", + help=f"Which directory to look in for the profiles.yml file. Default = {PurePath.joinpath(Path.home(), '.dbt')}", + default=PurePath.joinpath(Path.home(), ".dbt"), + type=click.Path( + exists=True, + ), +) + +project_dir = click.option( + "--project-dir", + help="Which directory to look in for the dbt_project.yml file. Default is the current working directory and its parents.", + default=Path.cwd(), + type=click.Path(exists=True), +) + +quiet = click.option( + "--quiet/--no-quiet", + help="Suppress all non-error logging to stdout. Does not affect {{ print() }} macro calls.", + default=False, +) + +record_timing = click.option( + "-r", + "--record-timing-info", + help="When this option is passed, dbt will output low-level timing stats to the specified file. Example: `--record-timing-info output.profile`", + is_flag=True, + default=False, +) + +resource_type = click.option( + "--resource-type", + help="TODO: No current help text", + type=click.Choice( + [ + "metric", + "source", + "analysis", + "model", + "test", + "exposure", + "snapshot", + "seed", + "default", + "all", + ], + case_sensitive=False, + ), + default="default", +) + +selector = click.option("--selector", help="The selector name to use, as defined in selectors.yml") + +send_anonymous_usage_stats = click.option( + "--anonymous-usage-stats/--no-anonymous-usage-stats", + help="Send anonymous usage stats to dbt Labs.", + default=True, +) + +show = click.option( + "--show", + help="Show a sample of the loaded data in the terminal", + default=False, +) + +skip_profile_setup = click.option( + "--skip-profile-setup", + "-s", + help="Skip interative profile setup.", + default=False, +) + +state = click.option( + "--state", + help="If set, use the given directory as the source for json files to compare with this project.", +) + +static_parser = click.option( + "--static-parser/--no-static-parser", help="Use the static parser.", default=True +) + +store_failures = click.option( + "--store-failures", help="Store test results (failing rows) in the database", default=False +) + +target = click.option("-t", "--target", help="Which target to load for the given profile") + +target_path = click.option( + "--target-path", + help="Configure the 'target-path'. Only applies this setting for the current run. Overrides the 'DBT_TARGET_PATH' if it is set.", + type=click.Path(), +) + +threads = click.option( + "--threads", + help="Specify number of threads to use while executing models. Overrides settings in profiles.yml.", + default=1, + type=click.INT, +) + +use_colors = click.option( + "--use-colors/--no-use-colors", + help="Output is colorized by default and may also be set in a profile or at the command line.", + default=True, +) + +use_experimental_parser = click.option( + "--use-experimental-parser/--no-use-experimental-parser", + help="Enable experimental parsing features.", + default=False, +) + +vars = click.option( + "--vars", + help="Supply variables to the project. This argument overrides variables defined in your dbt_project.yml file. This argument should be a YAML string, eg. '{my_variable: my_value}'", + type=YAML(), +) + +version = click.option("--version", help="Show version information", is_flag=True, default=False) + +version_check = click.option( + "--version-check/--no-version-check", + help="Ensure dbt's version matches the one specified in the dbt_project.yml file ('require-dbt-version')", + default=True, +) + +warn_error = click.option( + "--warn-error/--no-warn-error", + help="If dbt would normally warn, instead raise an exception. Examples include --models that selects nothing, deprecations, configurations with no associated models, invalid test configurations, and missing sources/refs in tests.", + default=False, +) + +write_json = click.option( + "--write-json/--no-write-json", + help="Writing the manifest and run_results.json files to disk", + default=True, +) + +write_manifest = click.option( + "--write-manifest/--no-write-manifest", + help="TODO: No help text currently available", + default=True, +) diff --git a/core/dbt/cli/params/__init__.py b/core/dbt/cli/params/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/core/dbt/cli/params/_global.py b/core/dbt/cli/params/_global.py deleted file mode 100644 index 7ae3d07c501..00000000000 --- a/core/dbt/cli/params/_global.py +++ /dev/null @@ -1,106 +0,0 @@ -import click - -# Version is a special snowflake, list it first -version = click.option("--version", help="Show version information", is_flag=True, default=False) - -# Global options that override config (sort alphabeticaly) -cache_selected_only = click.option( - "--cache-selected-only/--no-cache-selected-only", - help="Pre cache database objects relevant to selected resource only.", - default=False, -) - -debug = click.option( - "--debug/--no-debug", - "-d/ ", - help="Display debug logging during dbt execution. Useful for debugging and making bug reports.", - default=False, -) - -fail_fast = click.option( - "--fail-fast/--no-fail-fast", "-x/ ", help="Stop execution on first failure.", default=False -) - -log_format = click.option( - "--log-format", - help="Specify the log format, overriding the command's default.", - type=click.Choice(["text", "json", "default"], case_sensitive=False), - default="default", -) - -partial_parse = click.option( - "--partial-parse/--no-partial-parse", - help="Allow for partial parsing by looking for and writing to a pickle file in the target directory. This overrides the user configuration file.", - default=True, -) - -print = click.option( - "--print/--no-print", help="Output all {{ print() }} macro calls.", default=True -) - -printer_width = click.option( - "--printer_width", help="Sets the width of terminal output", type=click.INT, default=80 -) - -quiet = click.option( - "--quiet/--no-quiet", - help="Suppress all non-error logging to stdout. Does not affect {{ print() }} macro calls.", - default=False, -) - -send_anonymous_usage_stats = click.option( - "--anonymous-usage-stats/--no-anonymous-usage-stats", - help="Send anonymous usage stats to dbt Labs.", - default=True, -) - -static_parser = click.option( - "--static-parser/--no-static-parser", help="Use the static parser.", default=True -) - -use_colors = click.option( - "--use-colors/--no-use-colors", - help="Output is colorized by default and may also be set in a profile or at the command line.", - default=True, -) - -use_experimental_parser = click.option( - "--use-experimental-parser/--no-use-experimental-parser", - help="Enable experimental parsing features.", - default=False, -) - -version_check = click.option( - "--version-check/--no-version-check", - help="Ensure dbt's version matches the one specified in the dbt_project.yml file ('require-dbt-version')", - default=True, -) - -warn_error = click.option( - "--warn-error/--no-warn-error", - help="If dbt would normally warn, instead raise an exception. Examples include --models that selects nothing, deprecations, configurations with no associated models, invalid test configurations, and missing sources/refs in tests.", - default=False, -) - -write_json = click.option( - "--write-json/--no-write-json", - help="Writing the manifest and run_results.json files to disk", - default=True, -) - - -# Rarely used global options -event_buffer_size = click.option( - "--event-buffer-size", - help="Sets the max number of events to buffer in EVENT_HISTORY.", - default=100000, - type=click.INT, -) - -record_timing = click.option( - "-r", - "--record-timing-info", - help="When this option is passed, dbt will output low-level timing stats to the specified file. Example: `--record-timing-info output.profile`", - is_flag=True, - default=False, -) diff --git a/core/dbt/cli/params/docs.py b/core/dbt/cli/params/docs.py deleted file mode 100644 index 05425824dc3..00000000000 --- a/core/dbt/cli/params/docs.py +++ /dev/null @@ -1,17 +0,0 @@ -import click - -compile = click.option( - "--compile/--no-compile", - help="Wether or not to run 'dbt compile' as part of docs generation", - default=True, -) - -port = click.option( - "--port", help="Specify the port number for the docs server", default=8080, type=click.INT -) - -browser = click.option( - "--browser/--no-browser", - help="Wether or not to open a local web browser after starting the server", - default=True, -) diff --git a/core/dbt/cli/params/misc.py b/core/dbt/cli/params/misc.py deleted file mode 100644 index 6e85d3d2760..00000000000 --- a/core/dbt/cli/params/misc.py +++ /dev/null @@ -1,46 +0,0 @@ -import click -from pathlib import Path, PurePath - -config_dir = click.option( - "--config-dir", - help="If specified, DBT will show path information for this project", - type=click.STRING, -) - -skip_profile_setup = click.option( - "--skip-profile-setup", - "-s", - help="Skip interative profile setup.", - default=False, -) - -output = click.option( - "--output", - help="TODO: No current help text", - type=click.Choice(["json", "name", "path", "selector"], case_sensitive=False), - default="name", -) - -ouptut_keys = click.option( - "--output-keys", - help="TODO: No current help text", - default=False, -) - -show = click.option( - "--show", - help="Show a sample of the loaded data in the terminal", - default=False, -) - -output_path = click.option( - "--output", - "-o", - help="Specify the output path for the json report. By default, outputs to 'target/sources.json'", - type=click.Path(file_okay=True, dir_okay=False, writable=True), - default=PurePath.joinpath(Path.cwd(), "target/sources.json"), -) - -store_failures = click.option( - "--store-failures", help="Store test results (failing rows) in the database", default=False -) diff --git a/core/dbt/cli/params/parse.py b/core/dbt/cli/params/parse.py deleted file mode 100644 index 0f376d91e31..00000000000 --- a/core/dbt/cli/params/parse.py +++ /dev/null @@ -1,13 +0,0 @@ -import click - -compile = click.option( - "--compile/--no-compile", - help="TODO: No help text currently available", - default=True, -) - -write_manifest = click.option( - "--write-manifest/--no-write-manifest", - help="TODO: No help text currently available", - default=True, -) diff --git a/core/dbt/cli/params/project.py b/core/dbt/cli/params/project.py deleted file mode 100644 index 00a3494732f..00000000000 --- a/core/dbt/cli/params/project.py +++ /dev/null @@ -1,32 +0,0 @@ -import click -from pathlib import Path, PurePath -from dbt.cli.params.types import YAML - -project_dir = click.option( - "--project-dir", - help="Which directory to look in for the dbt_project.yml file. Default is the current working directory and its parents.", - default=Path.cwd(), - type=click.Path(exists=True), -) - -profiles_dir = click.option( - "--profiles-dir", - help=f"Which directory to look in for the profiles.yml file. Default = {PurePath.joinpath(Path.home(), '.dbt')}", - default=PurePath.joinpath(Path.home(), ".dbt"), - type=click.Path( - exists=True, - ), -) - -profile = click.option( - "--profile", - help="Which profile to load. Overrides setting in dbt_project.yml.", -) - -target = click.option("-t", "--target", help="Which target to load for the given profile") - -vars = click.option( - "--vars", - help="Supply variables to the project. This argument overrides variables defined in your dbt_project.yml file. This argument should be a YAML string, eg. '{my_variable: my_value}'", - type=YAML(), -) diff --git a/core/dbt/cli/params/run.py b/core/dbt/cli/params/run.py deleted file mode 100644 index 035bfe06f75..00000000000 --- a/core/dbt/cli/params/run.py +++ /dev/null @@ -1,84 +0,0 @@ -import click -from dbt.cli.params.types import YAML - -log_path = click.option( - "--log-path", - help="Configure the 'log-path'. Only applies this setting for the current run. Overrides the 'DBT_LOG_PATH' if it is set.", - type=click.Path(), -) - -target_path = click.option( - "--target-path", - help="Configure the 'target-path'. Only applies this setting for the current run. Overrides the 'DBT_TARGET_PATH' if it is set.", - type=click.Path(), -) - -threads = click.option( - "--threads", - help="Specify number of threads to use while executing models. Overrides settings in profiles.yml.", - default=1, - type=click.INT, -) - -models = click.option("-m", "-s", help="Specify the nodes to include.", multiple=True) - -exclude = click.option("--exclude", help="Specify the nodes to exclude.") - -selector = click.option("--selector", help="The selector name to use, as defined in selectors.yml") - -state = click.option( - "--state", - help="If set, use the given directory as the source for json files to compare with this project.", -) - -defer = click.option( - "--defer/--no-defer", - help="If set, defer to the state variable for resolving unselected nodes.", - default=True, -) - -full_refresh = click.option( - "--full-refresh", - help="If specified, dbt will drop incremental models and fully-recalculate the incremental table from the model definition.", - is_flag=True, -) - -parse_only = click.option( - "--parse-only", - help="TODO: No help text currently available", - is_flag=True, -) - -resource_type = click.option( - "--resource-type", - help="TODO: No current help text", - type=click.Choice( - [ - "metric", - "source", - "analysis", - "model", - "test", - "exposure", - "snapshot", - "seed", - "default", - "all", - ], - case_sensitive=False, - ), - default="default", -) - -indirect_selection = click.option( - "--indirect_selection", - help="Select all tests that are adjacent to selected resources, even if they those resources have been explicitly selected.", - type=click.Choice(["eager", "cautious"], case_sensitive=False), - default="eager", -) - -args = click.option( - "--args", - help="Supply arguments to the macro. This dictionary will be mapped to the keyword arguments defined in the selected macro. This argument should be a YAML string, eg. '{my_variable: my_value}'", - type=YAML(), -) diff --git a/core/dbt/cli/params/types.py b/core/dbt/cli/params/types.py deleted file mode 100644 index 6c024b4966b..00000000000 --- a/core/dbt/cli/params/types.py +++ /dev/null @@ -1,17 +0,0 @@ -from click import ParamType -import yaml - - -class YAML(ParamType): - """The YAML type converts YAML strings into objects.""" - - name = "YAML" - - def convert(self, value, param, ctx): - # assume non-string values are a problem - if not isinstance(value, str): - self.fail(f"Cannot load YAML from type {type(value)}", param, ctx) - try: - return yaml.load(value, Loader=yaml.Loader) - except yaml.parser.ParserError: - self.fail(f"String '{value}' is not valid YAML", param, ctx) From 421472656563efd3fb740b22a1e18d17f98701c1 Mon Sep 17 00:00:00 2001 From: Ian Knox Date: Thu, 11 Aug 2022 18:10:32 -0500 Subject: [PATCH 7/8] tests --- tests/unit/test_cli.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 tests/unit/test_cli.py diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py new file mode 100644 index 00000000000..294961c98e7 --- /dev/null +++ b/tests/unit/test_cli.py @@ -0,0 +1,25 @@ +from dbt.cli.main import cli +import click + + +class TestCLI: + def test_commands_have_docstrings(self): + def run_test(commands): + for _, command in commands.items(): + if type(command) is click.core.Command: + assert command.__doc__ is not None + if type(command) is click.core.Group: + run_test(command.commands) + + run_test(cli.commands) + + def test_params_have_help_texts(self): + def run_test(commands): + for _, command in commands.items(): + if type(command) is click.core.Command: + for param in command.params: + assert param.help is not None + if type(command) is click.core.Group: + run_test(command.commands) + + run_test(cli.commands) From 54f198c8aea2ed35291933d6a26d7638e1b15bda Mon Sep 17 00:00:00 2001 From: Ian Knox Date: Thu, 8 Sep 2022 10:42:14 -0500 Subject: [PATCH 8/8] changelog --- .changes/unreleased/Features-20220908-104149.yaml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changes/unreleased/Features-20220908-104149.yaml diff --git a/.changes/unreleased/Features-20220908-104149.yaml b/.changes/unreleased/Features-20220908-104149.yaml new file mode 100644 index 00000000000..c975dfb1446 --- /dev/null +++ b/.changes/unreleased/Features-20220908-104149.yaml @@ -0,0 +1,7 @@ +kind: Features +body: Adds new cli framework +time: 2022-09-08T10:41:49.375734-05:00 +custom: + Author: iknox-fa + Issue: "5526" + PR: "5647"