Skip to content

Commit

Permalink
[Backport 1.0.latest] Partial parsing bug with empty schema file - en… (
Browse files Browse the repository at this point in the history
#6517)

* [Backport 1.0.latest] Partial parsing bug with empty schema file - ensure None is not passe… (#6514)

Co-authored-by: Gerda Shank <[email protected]>
(cherry picked from commit 6e9ff28)

Co-authored-by: leahwicz <[email protected]>

* Cleanup flags.py

* Change ubuntu-latest to ubuntu-20.04

* Change more ubuntu-latest to ubuntu-20.04

* Change ubuntu-latest in integration-test-matrix.js to ubuntu-20.04

Co-authored-by: leahwicz <[email protected]>
  • Loading branch information
gshank and leahwicz authored Jan 4, 2023
1 parent 3fbc5a7 commit b8323fc
Show file tree
Hide file tree
Showing 13 changed files with 59 additions and 36 deletions.
6 changes: 6 additions & 0 deletions .changes/unreleased/Fixes-20230101-223405.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Fixes
body: Bug when partial parsing with an empty schema file
time: 2023-01-01T22:34:05.97322-05:00
custom:
Author: gshank
Issue: "4850"
4 changes: 2 additions & 2 deletions .github/scripts/integration-test-matrix.js
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ module.exports = ({ context }) => {
) {
// always run tests on ubuntu by default
include.push({
os: "ubuntu-latest",
os: "ubuntu-20.04",
adapter,
"python-version": pythonVersion,
});
Expand Down Expand Up @@ -68,7 +68,7 @@ module.exports = ({ context }) => {
for (const adapter of supportedAdapters) {
for (const pythonVersion of supportedPythonVersions) {
include.push({
os: 'ubuntu-latest',
os: 'ubuntu-20.04',
adapter: adapter,
"python-version": pythonVersion,
});
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/changelog-check.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ jobs:
changelog:
name: changelog

runs-on: ubuntu-latest
runs-on: ubuntu-20.04

steps:
- name: Check if changelog file was added
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/integration.yml
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ jobs:
github.event.pull_request.head.repo.full_name == github.repository ||
contains(github.event.pull_request.labels.*.name, 'ok to test')
runs-on: ubuntu-latest
runs-on: ubuntu-20.04

outputs:
matrix: ${{ steps.generate-matrix.outputs.result }}
Expand Down Expand Up @@ -199,7 +199,7 @@ jobs:
path: integration_results.csv

require-label-comment:
runs-on: ubuntu-latest
runs-on: ubuntu-20.04

needs: test

Expand Down
8 changes: 4 additions & 4 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ jobs:
code-quality:
name: ${{ matrix.toxenv }}

runs-on: ubuntu-latest
runs-on: ubuntu-20.04

strategy:
fail-fast: false
Expand Down Expand Up @@ -74,7 +74,7 @@ jobs:
unit:
name: unit test / python ${{ matrix.python-version }}

runs-on: ubuntu-latest
runs-on: ubuntu-20.04

strategy:
fail-fast: false
Expand Down Expand Up @@ -120,7 +120,7 @@ jobs:
build:
name: build packages

runs-on: ubuntu-latest
runs-on: ubuntu-20.04

steps:
- name: Check out the repository
Expand Down Expand Up @@ -168,7 +168,7 @@ jobs:
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
os: [ubuntu-20.04, macos-latest, windows-latest]
python-version: [3.7, 3.8, 3.9]

steps:
Expand Down
12 changes: 6 additions & 6 deletions .github/workflows/performance.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ jobs:
# will block merging, but not prevent developing
fmt:
name: Cargo fmt
runs-on: ubuntu-latest
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
- uses: actions-rs/toolchain@v1
Expand All @@ -31,7 +31,7 @@ jobs:
# these tests make sure the runner logic is correct
test-runner:
name: Test Runner
runs-on: ubuntu-latest
runs-on: ubuntu-20.04
env:
# turns errors into warnings
RUSTFLAGS: "-D warnings"
Expand All @@ -51,7 +51,7 @@ jobs:
build-runner:
needs: [test-runner]
name: Build Runner
runs-on: ubuntu-latest
runs-on: ubuntu-20.04
env:
RUSTFLAGS: "-D warnings"
steps:
Expand All @@ -74,7 +74,7 @@ jobs:
measure-dev:
needs: [build-runner]
name: Measure Dev Branch
runs-on: ubuntu-latest
runs-on: ubuntu-20.04
steps:
- name: checkout dev
uses: actions/checkout@v2
Expand Down Expand Up @@ -109,7 +109,7 @@ jobs:
measure-baseline:
needs: [build-runner]
name: Measure Baseline Branch
runs-on: ubuntu-latest
runs-on: ubuntu-20.04
steps:
- name: checkout latest
uses: actions/checkout@v2
Expand Down Expand Up @@ -149,7 +149,7 @@ jobs:
calculate-regressions:
needs: [measure-dev, measure-baseline]
name: Compare Results
runs-on: ubuntu-latest
runs-on: ubuntu-20.04
steps:
- uses: actions/download-artifact@v2
with:
Expand Down
10 changes: 5 additions & 5 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ jobs:
unit:
name: Unit test

runs-on: ubuntu-latest
runs-on: ubuntu-20.04

env:
TOXENV: "unit"
Expand Down Expand Up @@ -58,7 +58,7 @@ jobs:
build:
name: build packages

runs-on: ubuntu-latest
runs-on: ubuntu-20.04

steps:
- name: Check out the repository
Expand Down Expand Up @@ -104,7 +104,7 @@ jobs:

needs: [build, unit]

runs-on: ubuntu-latest
runs-on: ubuntu-20.04

steps:
- name: Set up Python
Expand Down Expand Up @@ -148,7 +148,7 @@ jobs:

needs: test-build

runs-on: ubuntu-latest
runs-on: ubuntu-20.04

steps:
- uses: actions/download-artifact@v2
Expand Down Expand Up @@ -183,7 +183,7 @@ jobs:
pypi-release:
name: Pypi release

runs-on: ubuntu-latest
runs-on: ubuntu-20.04

needs: github-release

Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/schema-check.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ env:
jobs:
checking-schemas:
name: "Checking schemas"
runs-on: ubuntu-latest
runs-on: ubuntu-20.04

steps:
- name: Set up Python
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/version-bump.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ on:

jobs:
bump:
runs-on: ubuntu-latest
runs-on: ubuntu-20.04
steps:
- name: Check out the repository
uses: actions/checkout@v2
Expand Down
3 changes: 1 addition & 2 deletions core/dbt/context/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -536,9 +536,8 @@ def flags(self) -> Any:
{% endif %}
This supports all flags defined in flags submodule (core/dbt/flags.py)
TODO: Replace with object that provides read-only access to flag values
"""
return flags
return flags.get_flag_obj()


def generate_base_context(cli_vars: Dict[str, Any]) -> Dict[str, Any]:
Expand Down
37 changes: 26 additions & 11 deletions core/dbt/flags.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
import os
# Do not import the os package because we expose this package in jinja
from os import name as os_name, path as os_path, getenv as os_getenv
import multiprocessing
if os.name != 'nt':
from argparse import Namespace

if os_name != "nt":
# https://bugs.python.org/issue41567
import multiprocessing.popen_spawn_posix # type: ignore
from pathlib import Path
Expand All @@ -9,10 +12,8 @@
# PROFILES_DIR must be set before the other flags
# It also gets set in main.py and in set_from_args because the rpc server
# doesn't go through exactly the same main arg processing.
DEFAULT_PROFILES_DIR = os.path.join(os.path.expanduser('~'), '.dbt')
PROFILES_DIR = os.path.expanduser(
os.getenv('DBT_PROFILES_DIR', DEFAULT_PROFILES_DIR)
)
DEFAULT_PROFILES_DIR = os_path.join(os_path.expanduser("~"), ".dbt")
PROFILES_DIR = os_path.expanduser(os_getenv("DBT_PROFILES_DIR", DEFAULT_PROFILES_DIR))

STRICT_MODE = False # Only here for backwards compatibility
FULL_REFRESH = False # subcommand
Expand Down Expand Up @@ -63,7 +64,7 @@ def env_set_truthy(key: str) -> Optional[str]:
"""Return the value if it was set to a "truthy" string value, or None
otherwise.
"""
value = os.getenv(key)
value = os_getenv(key)
if not value or value.lower() in ('0', 'false', 'f'):
return None
return value
Expand All @@ -76,7 +77,7 @@ def env_set_bool(env_value):


def env_set_path(key: str) -> Optional[Path]:
value = os.getenv(key)
value = os_getenv(key)
if value is None:
return value
else:
Expand Down Expand Up @@ -136,7 +137,7 @@ def get_flag_value(flag, args, user_config):
if flag_value is None:
# Environment variables use pattern 'DBT_{flag name}'
env_flag = f"DBT_{flag}"
env_value = os.getenv(env_flag)
env_value = os_getenv(env_flag)
if env_value is not None and env_value != '':
env_value = env_value.lower()
# non Boolean values
Expand All @@ -157,7 +158,7 @@ def get_flag_value(flag, args, user_config):
if flag in ['PRINTER_WIDTH', 'EVENT_BUFFER_SIZE']: # must be ints
flag_value = int(flag_value)
if flag == 'PROFILES_DIR':
flag_value = os.path.abspath(flag_value)
flag_value = os_path.abspath(flag_value)

return flag_value

Expand All @@ -179,5 +180,19 @@ def get_flag_dict():
"printer_width": PRINTER_WIDTH,
"indirect_selection": INDIRECT_SELECTION,
"log_cache_events": LOG_CACHE_EVENTS,
"event_buffer_size": EVENT_BUFFER_SIZE
"event_buffer_size": EVENT_BUFFER_SIZE,
}


# This is used by core/dbt/context/base.py to return a flag object
# in Jinja.
def get_flag_obj():
new_flags = Namespace()
for k, v in get_flag_dict().items():
setattr(new_flags, k.upper(), v)
# The following 3 are CLI arguments only so they're not full-fledged flags,
# but we put in flags for users.
setattr(new_flags, "FULL_REFRESH", FULL_REFRESH)
setattr(new_flags, "STORE_FAILURES", STORE_FAILURES)
setattr(new_flags, "WHICH", WHICH)
return new_flags
3 changes: 2 additions & 1 deletion core/dbt/parser/schemas.py
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,8 @@ def yaml_from_file(
"""
path = source_file.path.relative_path
try:
return load_yaml_text(source_file.contents)
# source_file.contents can sometimes be None
return load_yaml_text(source_file.contents or "")
except ValidationException as e:
reason = validator_error_message(e)
raise ParsingException(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -457,6 +457,8 @@ def test_postgres_skip_macros(self):
# initial run so we have a msgpack file
self.setup_directories()
self.copy_file('test-files/model_one.sql', 'models/model_one.sql')
# use empty_schema file for bug #4850
self.copy_file('test-files/empty_schema.yml', 'models/eschema.yml')
results = self.run_dbt()

# add a new ref override macro
Expand Down

0 comments on commit b8323fc

Please sign in to comment.