diff --git a/README.md b/README.md index f108915..cb535d3 100644 --- a/README.md +++ b/README.md @@ -242,12 +242,6 @@ This command prints a dbt Cloud run status JSON response. For more information o ```bash >> dbt-cloud run get --run-id 36053848 -Job 43167 run 34929305: QUEUED ... -Job 43167 run 34929305: QUEUED ... -Job 43167 run 34929305: QUEUED ... -Job 43167 run 34929305: STARTING ... -Job 43167 run 34929305: RUNNING ... -Job 43167 run 34929305: SUCCESS ... { "status": { "code": 200, @@ -302,8 +296,8 @@ Job 43167 run 34929305: SUCCESS ... "duration_humanized": "1 minute, 50 seconds", "queued_duration_humanized": "1 minute, 37 seconds", "run_duration_humanized": "12 seconds", - "created_at_humanized": "34 minutes, 20 seconds ago", - "finished_at_humanized": "32 minutes, 29 seconds ago", + "created_at_humanized": "2 weeks, 1 day ago", + "finished_at_humanized": "2 weeks, 1 day ago", "job_id": 43167 } } diff --git a/dbt_cloud/__init__.py b/dbt_cloud/__init__.py index e69de29..04336b7 100644 --- a/dbt_cloud/__init__.py +++ b/dbt_cloud/__init__.py @@ -0,0 +1 @@ +from .job import DbtCloudJob, DbtCloudRun diff --git a/dbt_cloud/cli.py b/dbt_cloud/cli.py index 04d075d..029db15 100644 --- a/dbt_cloud/cli.py +++ b/dbt_cloud/cli.py @@ -50,6 +50,7 @@ def run(wait, **kwargs): ) time.sleep(5) click.echo(json.dumps(response.json(), indent=2)) + response.raise_for_status() @job.command() @@ -59,6 +60,7 @@ def get(**kwargs): job = DbtCloudJob(**args.dict()) response = job.get(order_by=args.order_by) click.echo(json.dumps(response.json(), indent=2)) + response.raise_for_status() @job.command() @@ -68,7 +70,7 @@ def create(**kwargs): job = DbtCloudJob(job_id=None, **args.dict()) response = job.create(args) click.echo(json.dumps(response.json(), indent=2)) - response.raise_for_status() # TODO: Align all commands with this control flow + response.raise_for_status() @job_run.command() @@ -78,3 +80,4 @@ def get(**kwargs): run = args.get_run() response, _ = run.get_status() click.echo(json.dumps(response.json(), indent=2)) + response.raise_for_status() diff --git a/dbt_cloud/job.py b/dbt_cloud/job.py index 13fc4bf..0907b1e 100644 --- a/dbt_cloud/job.py +++ b/dbt_cloud/job.py @@ -134,13 +134,12 @@ def get_api_url(self) -> str: return f"{super().get_api_url()}/jobs/{self.job_id}" return f"{super().get_api_url()}/jobs" - def get(self, order_by: str) -> requests.Response: + def get(self, order_by: str = None) -> requests.Response: response = requests.get( url=f"{self.get_api_url()}/", headers={"Authorization": f"Token {self.api_token}"}, params={"order_by": order_by}, ) - response.raise_for_status() return response def create(self, args: DbtCloudJobCreateArgs) -> requests.Response: @@ -152,16 +151,15 @@ def create(self, args: DbtCloudJobCreateArgs) -> requests.Response: return response def run(self, args: DbtCloudJobRunArgs) -> Tuple[requests.Response, DbtCloudRun]: + assert str(args.job_id) == str(self.job_id), f"{args.job_id} != {self.job_id}" response = requests.post( url=f"{self.get_api_url()}/run/", headers={"Authorization": f"Token {self.api_token}"}, json=args.get_payload(), ) - response.raise_for_status() run_id = response.json()["data"]["id"] return response, DbtCloudRun( run_id=run_id, - args=args, account_id=self.account_id, api_token=self.api_token, ) diff --git a/dbt_cloud/run.py b/dbt_cloud/run.py index 498c1d4..1a2f93a 100644 --- a/dbt_cloud/run.py +++ b/dbt_cloud/run.py @@ -41,5 +41,4 @@ def get_status(self) -> Tuple[requests.Response, DbtCloudRunStatus]: url=f"{self.get_api_url()}/", headers={"Authorization": f"Token {self.api_token}"}, ) - response.raise_for_status() return response, DbtCloudRunStatus(response.json()["data"]["status"]) diff --git a/setup.py b/setup.py index 7b5eac0..6e1a730 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ packages=["dbt_cloud"], install_requires=["requests", "click", "pydantic"], extras_require={ - "test": ["pytest", "pytest-cov", "pytest-datadir"], + "test": ["pytest", "pytest-cov", "pytest-datadir", "requests-mock"], "lint": ["black"], }, scripts=[], diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..5b266e9 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,49 @@ +import json +import pytest +from dbt_cloud import DbtCloudJob, DbtCloudRun + + +def load_response(shared_datadir, response_name): + response_file = shared_datadir / f"{response_name}.json" + response_json = response_file.read_text() + return json.loads(response_json) + + +@pytest.fixture +def job_get_response(shared_datadir): + return load_response(shared_datadir, "job_get_response") + + +@pytest.fixture +def job_create_response(shared_datadir): + return load_response(shared_datadir, "job_create_response") + + +@pytest.fixture +def job_run_response(shared_datadir): + return load_response(shared_datadir, "job_run_response") + + +@pytest.fixture +def run_get_response(shared_datadir): + return load_response(shared_datadir, "run_get_response") + + +@pytest.fixture +def project_id(): + return 123457 + + +@pytest.fixture +def environment_id(): + return 49819 + + +@pytest.fixture +def job(): + return DbtCloudJob(api_token="foo", account_id=123456, job_id=43167) + + +@pytest.fixture +def run(job): + return DbtCloudRun(run_id=36053848, **job.dict()) diff --git a/tests/data/job_create_response.json b/tests/data/job_create_response.json new file mode 100644 index 0000000..48fb50f --- /dev/null +++ b/tests/data/job_create_response.json @@ -0,0 +1,56 @@ +{ + "status": { + "code": 201, + "is_success": true, + "user_message": "Success!", + "developer_message": "" + }, + "data": { + "execution": { + "timeout_seconds": 0 + }, + "generate_docs": false, + "run_generate_sources": false, + "id": 48180, + "account_id": 123456, + "project_id": 123457, + "environment_id": 49819, + "name": "Dummy job", + "dbt_version": null, + "created_at": "2021-12-22T11:23:26.968076+00:00", + "updated_at": "2021-12-22T11:23:26.968094+00:00", + "execute_steps": [ + "dbt seed", + "dbt run" + ], + "state": 1, + "deferring_job_definition_id": null, + "lifecycle_webhooks": false, + "lifecycle_webhooks_url": null, + "triggers": { + "github_webhook": false, + "git_provider_webhook": null, + "custom_branch_only": false, + "schedule": false + }, + "settings": { + "threads": 1, + "target_name": "default" + }, + "schedule": { + "cron": "0 * * * *", + "date": { + "type": "every_day" + }, + "time": { + "type": "every_hour", + "interval": 1 + } + }, + "is_deferrable": false, + "generate_sources": false, + "cron_humanized": "Every hour", + "next_run": null, + "next_run_humanized": null + } + } \ No newline at end of file diff --git a/tests/data/job_run_response.json b/tests/data/job_run_response.json new file mode 100644 index 0000000..bac99dd --- /dev/null +++ b/tests/data/job_run_response.json @@ -0,0 +1,59 @@ +{ + "status": { + "code": 200, + "is_success": true, + "user_message": "Success!", + "developer_message": "" + }, + "data": { + "id": 34929305, + "trigger_id": 35644346, + "account_id": 123456, + "environment_id": 49819, + "project_id": 123457, + "job_definition_id": 43167, + "status": 10, + "dbt_version": "0.21.0", + "git_branch": "main", + "git_sha": "981c5cf1ba299e942c6c277c38c8dec9b0738dd0", + "status_message": null, + "owner_thread_id": null, + "executed_by_thread_id": "dbt-run-34929305-dcmbq", + "deferring_run_id": null, + "artifacts_saved": true, + "artifact_s3_path": "prod/runs/34929305/artifacts/target", + "has_docs_generated": false, + "has_sources_generated": false, + "notifications_sent": true, + "blocked_by": [], + "scribe_enabled": true, + "created_at": "2021-11-26 16:48:41.431645+00:00", + "updated_at": "2021-11-26 16:49:33.078918+00:00", + "dequeued_at": "2021-11-26 16:49:15.670558+00:00", + "started_at": "2021-11-26 16:49:20.535987+00:00", + "finished_at": "2021-11-26 16:49:32.996703+00:00", + "last_checked_at": null, + "last_heartbeat_at": null, + "should_start_at": null, + "trigger": null, + "job": null, + "environment": null, + "run_steps": [], + "status_humanized": "Success", + "in_progress": false, + "is_complete": true, + "is_success": true, + "is_error": false, + "is_cancelled": false, + "href": "REDACTED", + "duration": "00:00:51", + "queued_duration": "00:00:39", + "run_duration": "00:00:12", + "duration_humanized": "51 seconds", + "queued_duration_humanized": "39 seconds", + "run_duration_humanized": "12 seconds", + "created_at_humanized": "1 minute ago", + "finished_at_humanized": "9 seconds ago", + "job_id": 43167 + } + } \ No newline at end of file diff --git a/tests/data/run_get_response.json b/tests/data/run_get_response.json new file mode 100644 index 0000000..7336282 --- /dev/null +++ b/tests/data/run_get_response.json @@ -0,0 +1,59 @@ +{ + "status": { + "code": 200, + "is_success": true, + "user_message": "Success!", + "developer_message": "" + }, + "data": { + "id": 36053848, + "trigger_id": 36768889, + "account_id": 123456, + "environment_id": 49819, + "project_id": 123457, + "job_definition_id": 43167, + "status": 10, + "dbt_version": "0.21.0", + "git_branch": "main", + "git_sha": "981c5cf1ba299e942c6c277c38c8dec9b0738dd0", + "status_message": null, + "owner_thread_id": null, + "executed_by_thread_id": "dbt-run-36053848-84vsp", + "deferring_run_id": null, + "artifacts_saved": true, + "artifact_s3_path": "prod/runs/36053848/artifacts/target", + "has_docs_generated": false, + "has_sources_generated": false, + "notifications_sent": true, + "blocked_by": [], + "scribe_enabled": true, + "created_at": "2021-12-07 10:32:24.326116+00:00", + "updated_at": "2021-12-07 10:34:14.507280+00:00", + "dequeued_at": "2021-12-07 10:33:54.599925+00:00", + "started_at": "2021-12-07 10:34:01.982824+00:00", + "finished_at": "2021-12-07 10:34:14.435474+00:00", + "last_checked_at": null, + "last_heartbeat_at": null, + "should_start_at": null, + "trigger": null, + "job": null, + "environment": null, + "run_steps": [], + "status_humanized": "Success", + "in_progress": false, + "is_complete": true, + "is_success": true, + "is_error": false, + "is_cancelled": false, + "href": "REDACTED", + "duration": "00:01:50", + "queued_duration": "00:01:37", + "run_duration": "00:00:12", + "duration_humanized": "1 minute, 50 seconds", + "queued_duration_humanized": "1 minute, 37 seconds", + "run_duration_humanized": "12 seconds", + "created_at_humanized": "2 weeks, 1 day ago", + "finished_at_humanized": "2 weeks, 1 day ago", + "job_id": 43167 + } + } \ No newline at end of file diff --git a/tests/test_job.py b/tests/test_job.py new file mode 100644 index 0000000..91a8317 --- /dev/null +++ b/tests/test_job.py @@ -0,0 +1,32 @@ +from dbt_cloud.job import DbtCloudJob, DbtCloudJobCreateArgs, DbtCloudJobRunArgs + + +def test_mock_job_get(requests_mock, job, job_get_response): + url = job.get_api_url() + "/" + requests_mock.get(url, json=job_get_response, status_code=200) + response = job.get() + assert response.json() == job_get_response + + +def test_mock_job_create( + requests_mock, job, job_create_response, project_id, environment_id +): + url = job.get_api_url() + "/" + requests_mock.post(url, json=job_create_response, status_code=201) + args = DbtCloudJobCreateArgs( + project_id=project_id, + environment_id=environment_id, + name="Dummy job", + execute_steps=["dbt seed", "dbt run"], + ) + response = job.create(args) + assert response.json() == job_create_response + + +def test_mock_job_run(requests_mock, job, job_run_response): + url = job.get_api_url() + "/run/" + requests_mock.post(url, json=job_run_response, status_code=200) + args = DbtCloudJobRunArgs() + response, job_run = job.run(args) + assert response.json() == job_run_response + assert job_run.run_id == job_run_response["data"]["id"] diff --git a/tests/test_job_import.py b/tests/test_job_import.py index 872500a..2ed4ed8 100644 --- a/tests/test_job_import.py +++ b/tests/test_job_import.py @@ -2,11 +2,8 @@ from dbt_cloud.job import DbtCloudJobCreateArgs -def test_job_args_import_from_json(shared_datadir): - response_file = shared_datadir / "job_get_response.json" - response_json = response_file.read_text() - response_dict = json.loads(response_json) - job_dict = response_dict["data"] +def test_job_args_import_from_json(job_get_response): + job_dict = job_get_response["data"] args = DbtCloudJobCreateArgs(**job_dict) assert args.environment_id == 49819 assert args.account_id == 123456 diff --git a/tests/test_run.py b/tests/test_run.py new file mode 100644 index 0000000..af96a1e --- /dev/null +++ b/tests/test_run.py @@ -0,0 +1,9 @@ +from dbt_cloud.run import DbtCloudRun, DbtCloudRunStatus + + +def test_mock_run_get_status(requests_mock, run, run_get_response): + url = run.get_api_url() + "/" + requests_mock.get(url, json=run_get_response, status_code=200) + response, run_status = run.get_status() + assert response.json() == run_get_response + assert run_status == DbtCloudRunStatus.SUCCESS