diff --git a/.changes/unreleased/Features-20220908-104149.yaml b/.changes/unreleased/Features-20220908-104149.yaml new file mode 100644 index 00000000000..c975dfb1446 --- /dev/null +++ b/.changes/unreleased/Features-20220908-104149.yaml @@ -0,0 +1,7 @@ +kind: Features +body: Adds new cli framework +time: 2022-09-08T10:41:49.375734-05:00 +custom: + Author: iknox-fa + Issue: "5526" + PR: "5647" diff --git a/core/dbt/cli/README.md b/core/dbt/cli/README.md new file mode 100644 index 00000000000..1333ed77b7e --- /dev/null +++ b/core/dbt/cli/README.md @@ -0,0 +1 @@ +TODO diff --git a/core/dbt/cli/__init__.py b/core/dbt/cli/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py new file mode 100644 index 00000000000..3f11c001827 --- /dev/null +++ b/core/dbt/cli/main.py @@ -0,0 +1,379 @@ +import click +from dbt.cli import params as p +import sys + +# This is temporary for RAT-ing +import inspect +from pprint import pformat as pf + + +# dbt +@click.group( + invoke_without_command=True, + no_args_is_help=True, + epilog="Specify one of these sub-commands and you can find more help from there.", +) +@click.pass_context +@p.version +@p.cache_selected_only +@p.debug +@p.fail_fast +@p.log_format +@p.partial_parse +@p.print +@p.printer_width +@p.quiet +@p.send_anonymous_usage_stats +@p.static_parser +@p.use_colors +@p.use_experimental_parser +@p.version_check +@p.warn_error +@p.write_json +@p.event_buffer_size +@p.record_timing +def cli(ctx, **kwargs): + """An ELT tool for managing your SQL transformations and data models. + For more documentation on these commands, visit: docs.getdbt.com + """ + if kwargs.get("version", False): + click.echo(f"`version` called\n ctx.params: {pf(ctx.params)}") + sys.exit() + else: + del ctx.params["version"] + + +# dbt build +@cli.command("build") +@click.pass_context +def build(ctx, **kwargs): + """Run all Seeds, Models, Snapshots, and tests in DAG order""" + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" + ) + + +# dbt clean +@cli.command("clean") +@click.pass_context +@p.project_dir +@p.profiles_dir +@p.profile +@p.target +@p.vars +def clean(ctx, **kwargs): + """Delete all folders in the clean-targets list (usually the dbt_packages and target directories.)""" + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" + ) + + +# dbt docs +@cli.group() +@click.pass_context +def docs(ctx, **kwargs): + """Generate or serve the documentation website for your project""" + + +# dbt docs generate +@docs.command("generate") +@click.pass_context +@p.version_check +@p.project_dir +@p.profiles_dir +@p.profile +@p.target +@p.vars +@p.compile_docs +@p.defer +@p.threads +@p.target_path +@p.log_path +@p.models +@p.exclude +@p.selector +@p.state +def docs_generate(ctx, **kwargs): + """Generate the documentation website for your project""" + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.parent.params)}" + ) + + +# dbt docs serve +@docs.command("serve") +@click.pass_context +@p.project_dir +@p.profiles_dir +@p.profile +@p.target +@p.vars +@p.port +@p.browser +def docs_serve(ctx, **kwargs): + """Serve the documentation website for your project""" + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.parent.params)}" + ) + + +# dbt compile +@cli.command("compile") +@click.pass_context +@p.version_check +@p.project_dir +@p.profiles_dir +@p.profile +@p.target +@p.vars +@p.parse_only +@p.threads +@p.target_path +@p.log_path +@p.models +@p.exclude +@p.selector +@p.state +@p.defer +@p.full_refresh +def compile(ctx, **kwargs): + """Generates executable SQL from source, model, test, and analysis files. Compiled SQL files are written to the target/ directory.""" + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" + ) + + +# dbt debug +@cli.command("debug") +@click.pass_context +@p.version_check +@p.project_dir +@p.profiles_dir +@p.profile +@p.target +@p.vars +@p.config_dir +def debug(ctx, **kwargs): + """Show some helpful information about dbt for debugging. Not to be confused with the --debug option which increases verbosity.""" + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" + ) + + +# dbt deps +@cli.command("deps") +@click.pass_context +@p.profile +@p.profiles_dir +@p.project_dir +@p.target +@p.vars +def deps(ctx, **kwargs): + """Pull the most recent version of the dependencies listed in packages.yml""" + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" + ) + + +# dbt init +@cli.command("init") +@click.pass_context +@p.profile +@p.profiles_dir +@p.project_dir +@p.target +@p.vars +@p.skip_profile_setup +def init(ctx, **kwargs): + """Initialize a new DBT project.""" + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" + ) + + +# dbt list +# dbt TODO: Figure out aliasing for ls (or just c/p?) +@cli.command("list") +@click.pass_context +@p.profile +@p.profiles_dir +@p.project_dir +@p.target +@p.vars +@p.output +@p.ouptut_keys +@p.resource_type +@p.models +@p.indirect_selection +@p.exclude +@p.selector +@p.state +def list(ctx, **kwargs): + """List the resources in your project""" + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" + ) + + +# dbt parse +@cli.command("parse") +@click.pass_context +@p.profile +@p.profiles_dir +@p.project_dir +@p.target +@p.vars +@p.write_manifest +@p.compile_parse +@p.threads +@p.target_path +@p.log_path +@p.version_check +def parse(ctx, **kwargs): + """Parses the project and provides information on performance""" + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" + ) + + +# dbt run +@cli.command("run") +@click.pass_context +@p.fail_fast +@p.version_check +@p.profile +@p.profiles_dir +@p.project_dir +@p.target +@p.vars +@p.log_path +@p.target_path +@p.threads +@p.models +@p.exclude +@p.selector +@p.state +@p.defer +@p.full_refresh +def run(ctx, **kwargs): + """Compile SQL and execute against the current target database.""" + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" + ) + + +# dbt run operation +@cli.command("run-operation") +@click.pass_context +@p.profile +@p.profiles_dir +@p.project_dir +@p.target +@p.vars +@p.args +def run_operation(ctx, **kwargs): + """Run the named macro with any supplied arguments.""" + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" + ) + + +# dbt seed +@cli.command("seed") +@click.pass_context +@p.version_check +@p.profile +@p.profiles_dir +@p.project_dir +@p.target +@p.vars +@p.full_refresh +@p.log_path +@p.target_path +@p.threads +@p.models +@p.exclude +@p.selector +@p.state +@p.show +def seed(ctx, **kwargs): + """Load data from csv files into your data warehouse.""" + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" + ) + + +# dbt snapshot +@cli.command("snapshot") +@click.pass_context +@p.profile +@p.profiles_dir +@p.project_dir +@p.target +@p.vars +@p.threads +@p.models +@p.exclude +@p.selector +@p.state +@p.defer +def snapshot(ctx, **kwargs): + """Execute snapshots defined in your project""" + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" + ) + + +# dbt source +@cli.group() +@click.pass_context +def source(ctx, **kwargs): + """Manage your project's sources""" + + +# dbt source freshness +@source.command("freshness") +@click.pass_context +@p.profile +@p.profiles_dir +@p.project_dir +@p.target +@p.vars +@p.threads +@p.models +@p.exclude +@p.selector +@p.state +@p.output_path # TODO: Is this ok to re-use? We have three different output params, how much can we consolidate? +def freshness(ctx, **kwargs): + """Snapshots the current freshness of the project's sources""" + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.parent.params)}" + ) + + +# dbt test +@cli.command("test") +@click.pass_context +@p.fail_fast +@p.version_check +@p.store_failures +@p.profile +@p.profiles_dir +@p.project_dir +@p.target +@p.vars +@p.indirect_selection +@p.log_path +@p.target_path +@p.threads +@p.models +@p.exclude +@p.selector +@p.state +@p.defer +def test(ctx, **kwargs): + """Runs tests on data in deployed models. Run this after `dbt run`""" + click.echo( + f"`{inspect.stack()[0][3]}` called\n kwargs: {kwargs}\n ctx: {pf(ctx.parent.params)}" + ) diff --git a/core/dbt/cli/params.py b/core/dbt/cli/params.py new file mode 100644 index 00000000000..f991359ecf7 --- /dev/null +++ b/core/dbt/cli/params.py @@ -0,0 +1,304 @@ +import click +import yaml +from pathlib import Path, PurePath +from click import ParamType + + +class YAML(ParamType): + """The Click YAML type. Converts YAML strings into objects.""" + + name = "YAML" + + def convert(self, value, param, ctx): + # assume non-string values are a problem + if not isinstance(value, str): + self.fail(f"Cannot load YAML from type {type(value)}", param, ctx) + try: + return yaml.load(value, Loader=yaml.Loader) + except yaml.parser.ParserError: + self.fail(f"String '{value}' is not valid YAML", param, ctx) + + +args = click.option( + "--args", + help="Supply arguments to the macro. This dictionary will be mapped to the keyword arguments defined in the selected macro. This argument should be a YAML string, eg. '{my_variable: my_value}'", + type=YAML(), +) + +browser = click.option( + "--browser/--no-browser", + help="Wether or not to open a local web browser after starting the server", + default=True, +) + +cache_selected_only = click.option( + "--cache-selected-only/--no-cache-selected-only", + help="Pre cache database objects relevant to selected resource only.", + default=False, +) + +compile_docs = click.option( + "--compile/--no-compile", + help="Wether or not to run 'dbt compile' as part of docs generation", + default=True, +) + +compile_parse = click.option( + "--compile/--no-compile", + help="TODO: No help text currently available", + default=True, +) + +config_dir = click.option( + "--config-dir", + help="If specified, DBT will show path information for this project", + type=click.STRING, +) + +debug = click.option( + "--debug/--no-debug", + "-d/ ", + help="Display debug logging during dbt execution. Useful for debugging and making bug reports.", + default=False, +) + +defer = click.option( + "--defer/--no-defer", + help="If set, defer to the state variable for resolving unselected nodes.", + default=True, +) + +event_buffer_size = click.option( + "--event-buffer-size", + help="Sets the max number of events to buffer in EVENT_HISTORY.", + default=100000, + type=click.INT, +) + +exclude = click.option("--exclude", help="Specify the nodes to exclude.") + +fail_fast = click.option( + "--fail-fast/--no-fail-fast", "-x/ ", help="Stop execution on first failure.", default=False +) + +full_refresh = click.option( + "--full-refresh", + help="If specified, dbt will drop incremental models and fully-recalculate the incremental table from the model definition.", + is_flag=True, +) + +indirect_selection = click.option( + "--indirect_selection", + help="Select all tests that are adjacent to selected resources, even if they those resources have been explicitly selected.", + type=click.Choice(["eager", "cautious"], case_sensitive=False), + default="eager", +) + +log_format = click.option( + "--log-format", + help="Specify the log format, overriding the command's default.", + type=click.Choice(["text", "json", "default"], case_sensitive=False), + default="default", +) + +log_path = click.option( + "--log-path", + help="Configure the 'log-path'. Only applies this setting for the current run. Overrides the 'DBT_LOG_PATH' if it is set.", + type=click.Path(), +) + +models = click.option("-m", "-s", help="Specify the nodes to include.", multiple=True) + +output = click.option( + "--output", + help="TODO: No current help text", + type=click.Choice(["json", "name", "path", "selector"], case_sensitive=False), + default="name", +) + +ouptut_keys = click.option( + "--output-keys", + help="TODO: No current help text", + default=False, +) + +output_path = click.option( + "--output", + "-o", + help="Specify the output path for the json report. By default, outputs to 'target/sources.json'", + type=click.Path(file_okay=True, dir_okay=False, writable=True), + default=PurePath.joinpath(Path.cwd(), "target/sources.json"), +) + +parse_only = click.option( + "--parse-only", + help="TODO: No help text currently available", + is_flag=True, +) + +partial_parse = click.option( + "--partial-parse/--no-partial-parse", + help="Allow for partial parsing by looking for and writing to a pickle file in the target directory. This overrides the user configuration file.", + default=True, +) + +port = click.option( + "--port", help="Specify the port number for the docs server", default=8080, type=click.INT +) + +print = click.option( + "--print/--no-print", help="Output all {{ print() }} macro calls.", default=True +) + +printer_width = click.option( + "--printer_width", help="Sets the width of terminal output", type=click.INT, default=80 +) + +profile = click.option( + "--profile", + help="Which profile to load. Overrides setting in dbt_project.yml.", +) + +profiles_dir = click.option( + "--profiles-dir", + help=f"Which directory to look in for the profiles.yml file. Default = {PurePath.joinpath(Path.home(), '.dbt')}", + default=PurePath.joinpath(Path.home(), ".dbt"), + type=click.Path( + exists=True, + ), +) + +project_dir = click.option( + "--project-dir", + help="Which directory to look in for the dbt_project.yml file. Default is the current working directory and its parents.", + default=Path.cwd(), + type=click.Path(exists=True), +) + +quiet = click.option( + "--quiet/--no-quiet", + help="Suppress all non-error logging to stdout. Does not affect {{ print() }} macro calls.", + default=False, +) + +record_timing = click.option( + "-r", + "--record-timing-info", + help="When this option is passed, dbt will output low-level timing stats to the specified file. Example: `--record-timing-info output.profile`", + is_flag=True, + default=False, +) + +resource_type = click.option( + "--resource-type", + help="TODO: No current help text", + type=click.Choice( + [ + "metric", + "source", + "analysis", + "model", + "test", + "exposure", + "snapshot", + "seed", + "default", + "all", + ], + case_sensitive=False, + ), + default="default", +) + +selector = click.option("--selector", help="The selector name to use, as defined in selectors.yml") + +send_anonymous_usage_stats = click.option( + "--anonymous-usage-stats/--no-anonymous-usage-stats", + help="Send anonymous usage stats to dbt Labs.", + default=True, +) + +show = click.option( + "--show", + help="Show a sample of the loaded data in the terminal", + default=False, +) + +skip_profile_setup = click.option( + "--skip-profile-setup", + "-s", + help="Skip interative profile setup.", + default=False, +) + +state = click.option( + "--state", + help="If set, use the given directory as the source for json files to compare with this project.", +) + +static_parser = click.option( + "--static-parser/--no-static-parser", help="Use the static parser.", default=True +) + +store_failures = click.option( + "--store-failures", help="Store test results (failing rows) in the database", default=False +) + +target = click.option("-t", "--target", help="Which target to load for the given profile") + +target_path = click.option( + "--target-path", + help="Configure the 'target-path'. Only applies this setting for the current run. Overrides the 'DBT_TARGET_PATH' if it is set.", + type=click.Path(), +) + +threads = click.option( + "--threads", + help="Specify number of threads to use while executing models. Overrides settings in profiles.yml.", + default=1, + type=click.INT, +) + +use_colors = click.option( + "--use-colors/--no-use-colors", + help="Output is colorized by default and may also be set in a profile or at the command line.", + default=True, +) + +use_experimental_parser = click.option( + "--use-experimental-parser/--no-use-experimental-parser", + help="Enable experimental parsing features.", + default=False, +) + +vars = click.option( + "--vars", + help="Supply variables to the project. This argument overrides variables defined in your dbt_project.yml file. This argument should be a YAML string, eg. '{my_variable: my_value}'", + type=YAML(), +) + +version = click.option("--version", help="Show version information", is_flag=True, default=False) + +version_check = click.option( + "--version-check/--no-version-check", + help="Ensure dbt's version matches the one specified in the dbt_project.yml file ('require-dbt-version')", + default=True, +) + +warn_error = click.option( + "--warn-error/--no-warn-error", + help="If dbt would normally warn, instead raise an exception. Examples include --models that selects nothing, deprecations, configurations with no associated models, invalid test configurations, and missing sources/refs in tests.", + default=False, +) + +write_json = click.option( + "--write-json/--no-write-json", + help="Writing the manifest and run_results.json files to disk", + default=True, +) + +write_manifest = click.option( + "--write-manifest/--no-write-manifest", + help="TODO: No help text currently available", + default=True, +) diff --git a/core/setup.py b/core/setup.py index f893b7d5e43..2c254dcf6b0 100644 --- a/core/setup.py +++ b/core/setup.py @@ -43,9 +43,7 @@ include_package_data=True, test_suite="test", entry_points={ - "console_scripts": [ - "dbt = dbt.main:main", - ], + "console_scripts": ["dbt = dbt.main:main", "dbt-cli = dbt.cli.main:cli"], }, install_requires=[ "Jinja2==3.1.2", diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py new file mode 100644 index 00000000000..294961c98e7 --- /dev/null +++ b/tests/unit/test_cli.py @@ -0,0 +1,25 @@ +from dbt.cli.main import cli +import click + + +class TestCLI: + def test_commands_have_docstrings(self): + def run_test(commands): + for _, command in commands.items(): + if type(command) is click.core.Command: + assert command.__doc__ is not None + if type(command) is click.core.Group: + run_test(command.commands) + + run_test(cli.commands) + + def test_params_have_help_texts(self): + def run_test(commands): + for _, command in commands.items(): + if type(command) is click.core.Command: + for param in command.params: + assert param.help is not None + if type(command) is click.core.Group: + run_test(command.commands) + + run_test(cli.commands)