From c8c70e5a211bbd30f1610c82bd297656c90077b6 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Mon, 15 Apr 2024 12:27:15 -0500 Subject: [PATCH 1/6] be less explicit (#9936) (#9940) --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b850078b0a6..efd7a20ec14 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -225,7 +225,7 @@ jobs: - name: Install source distributions # ignore dbt-1.0.0, which intentionally raises an error when installed from source run: | - find ./dist/dbt-[a-z]*.gz -maxdepth 1 -type f | xargs pip install --force-reinstall --find-links=dist/ + find ./dist/*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ - name: Check source distributions run: | From ba59fff449b856b05637dfd325fb67672dde23b4 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Thu, 25 Apr 2024 10:13:48 -0500 Subject: [PATCH 2/6] pin macos test runners to macos-12 (#10031) (#10042) # Conflicts: # .github/workflows/main.yml # .github/workflows/test-repeater.yml Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index efd7a20ec14..97433f886bd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -123,7 +123,7 @@ jobs: - python-version: 3.8 os: windows-latest - python-version: 3.8 - os: macos-latest + os: macos-12 env: TOXENV: integration From a5c57cbae3cf0be4465ca08c6b560dbe398e30ff Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Tue, 21 May 2024 21:02:04 +0100 Subject: [PATCH 3/6] [Backport 1.1.latest] Fix #9907: Add retry to tox to reduce flaky tests due to network failures (#10179) * [Backport 1.1.latest] Fix #9907: Add retry to tox to reduce flaky tests due to network failures * Update main.yml --- .github/workflows/main.yml | 16 ++++++++++++---- .../structured-logging-schema-check.yml | 6 +++++- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 97433f886bd..a15dbbe1e3d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -93,8 +93,12 @@ jobs: pip install tox tox --version - - name: Run tox - run: tox + - name: Run unit tests + uses: nick-fields/retry@v3 + with: + timeout_minutes: 10 + max_attempts: 3 + command: tox -e unit - name: Get current date if: always() @@ -158,8 +162,12 @@ jobs: pip install tox tox --version - - name: Run tests - run: tox + - name: Run integration tests + uses: nick-fields/retry@v3 + with: + timeout_minutes: 30 + max_attempts: 3 + command: tox - name: Get current date if: always() diff --git a/.github/workflows/structured-logging-schema-check.yml b/.github/workflows/structured-logging-schema-check.yml index 340e5831f88..ff981226e05 100644 --- a/.github/workflows/structured-logging-schema-check.yml +++ b/.github/workflows/structured-logging-schema-check.yml @@ -63,7 +63,11 @@ jobs: # integration tests generate a ton of logs in different files. the next step will find them all. # we actually care if these pass, because the normal test run doesn't usually include many json log outputs - name: Run integration tests - run: tox -e integration -- -nauto + uses: nick-fields/retry@v3 + with: + timeout_minutes: 30 + max_attempts: 3 + command: tox -e integration -- -nauto # apply our schema tests to every log event from the previous step # skips any output that isn't valid json From 3782dcfd5e835f42f8f3fc0c71534d201e059a8c Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Tue, 21 May 2024 23:38:42 +0100 Subject: [PATCH 4/6] [Backport 1.1.latest] [Tidy First] Remove wheel and twine from dev-requirements.txt (#10200) --- dev-requirements.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index d519365aeeb..ce1d29fdaea 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -15,9 +15,7 @@ pytest-logbook pytest-mock pytest-xdist tox>=3.13 -twine types-python-dateutil types-pytz types-PyYAML types-requests -wheel From cc336cba2c4f176b2d2e680dfc28be16bf861472 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Thu, 18 Jul 2024 15:14:58 -0400 Subject: [PATCH 5/6] install & check source distributions before wheels (#10467) --- .github/workflows/main.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a15dbbe1e3d..e9eafd3d705 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -218,6 +218,15 @@ jobs: run: | twine check dist/* + - name: Install source distributions + # ignore dbt-1.0.0, which intentionally raises an error when installed from source + run: | + find ./dist/*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ + + - name: Check source distributions + run: | + dbt --version + - name: Check wheel contents run: | check-wheel-contents dist/*.whl --ignore W007,W008 @@ -229,12 +238,3 @@ jobs: - name: Check wheel distributions run: | dbt --version - - - name: Install source distributions - # ignore dbt-1.0.0, which intentionally raises an error when installed from source - run: | - find ./dist/*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ - - - name: Check source distributions - run: | - dbt --version From dc750b327beda17182bc0a59c9ec2d03ee08c944 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 10 Sep 2024 13:34:42 -0400 Subject: [PATCH 6/6] [1.3.latest] Avoid packaging for semver (#10679) (#10684) * CT 2000 fix semver prerelease comparisons (#6838) (#6958) * Modify semver.py to not use packaging.version.parse * Changie (cherry picked from commit d9424cc7100cb3fcf92155c7d4fb4f7e030f19fe) * Fix regression in semver comparison logic (#7040) (#7041) (cherry picked from commit 915585c36e5798a8d81dbb74afde5a818b1c7803) * Fix changed name of exception --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Jeremy Cohen --- .../unreleased/Fixes-20230201-154418.yaml | 6 +++ .../unreleased/Fixes-20230224-001338.yaml | 6 +++ core/dbt/semver.py | 47 +++++++++++++++---- core/setup.py | 2 +- test/unit/test_semver.py | 20 ++++++-- 5 files changed, 68 insertions(+), 13 deletions(-) create mode 100644 .changes/unreleased/Fixes-20230201-154418.yaml create mode 100644 .changes/unreleased/Fixes-20230224-001338.yaml diff --git a/.changes/unreleased/Fixes-20230201-154418.yaml b/.changes/unreleased/Fixes-20230201-154418.yaml new file mode 100644 index 00000000000..dc2099f94b1 --- /dev/null +++ b/.changes/unreleased/Fixes-20230201-154418.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Remove pin on packaging and stop using it for prerelease comparisons +time: 2023-02-01T15:44:18.279158-05:00 +custom: + Author: gshank + Issue: "6834" diff --git a/.changes/unreleased/Fixes-20230224-001338.yaml b/.changes/unreleased/Fixes-20230224-001338.yaml new file mode 100644 index 00000000000..28f33bef515 --- /dev/null +++ b/.changes/unreleased/Fixes-20230224-001338.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Fix semver comparison logic by ensuring numeric values +time: 2023-02-24T00:13:38.23242+01:00 +custom: + Author: jtcohen6 + Issue: "7039" diff --git a/core/dbt/semver.py b/core/dbt/semver.py index a98e18c3afe..63b474c5396 100644 --- a/core/dbt/semver.py +++ b/core/dbt/semver.py @@ -1,10 +1,7 @@ from dataclasses import dataclass import re -import warnings from typing import List -from packaging import version as packaging_version - from dbt.exceptions import VersionsNotCompatibleException import dbt.utils @@ -70,6 +67,11 @@ class VersionSpecification(dbtClassMixin): _VERSION_REGEX = re.compile(_VERSION_REGEX_PAT_STR, re.VERBOSE) +def _cmp(a, b): + """Return negative if ab.""" + return (a > b) - (a < b) + + @dataclass class VersionSpecifier(VersionSpecification): def to_version_string(self, skip_matcher=False): @@ -142,13 +144,19 @@ def compare(self, other): return 1 if b is None: return -1 - # This suppresses the LegacyVersion deprecation warning - with warnings.catch_warnings(): - warnings.simplefilter("ignore", category=DeprecationWarning) - if packaging_version.parse(a) > packaging_version.parse(b): + + # Check the prerelease component only + prcmp = self._nat_cmp(a, b) + if prcmp != 0: # either -1 or 1 + return prcmp + # else is equal and will fall through + + else: # major/minor/patch, should all be numbers + if int(a) > int(b): return 1 - elif packaging_version.parse(a) < packaging_version.parse(b): + elif int(a) < int(b): return -1 + # else is equal and will fall through equal = ( self.matcher == Matchers.GREATER_THAN_OR_EQUAL @@ -212,6 +220,29 @@ def is_upper_bound(self): def is_exact(self): return self.matcher == Matchers.EXACT + @classmethod + def _nat_cmp(cls, a, b): + def cmp_prerelease_tag(a, b): + if isinstance(a, int) and isinstance(b, int): + return _cmp(a, b) + elif isinstance(a, int): + return -1 + elif isinstance(b, int): + return 1 + else: + return _cmp(a, b) + + a, b = a or "", b or "" + a_parts, b_parts = a.split("."), b.split(".") + a_parts = [int(x) if re.match(r"^\d+$", x) else x for x in a_parts] + b_parts = [int(x) if re.match(r"^\d+$", x) else x for x in b_parts] + for sub_a, sub_b in zip(a_parts, b_parts): + cmp_result = cmp_prerelease_tag(sub_a, sub_b) + if cmp_result != 0: + return cmp_result + else: + return _cmp(len(a), len(b)) + @dataclass class VersionRange: diff --git a/core/setup.py b/core/setup.py index 728f4eb2e7c..2efc6b1ade5 100644 --- a/core/setup.py +++ b/core/setup.py @@ -62,7 +62,7 @@ "mashumaro==2.9", "minimal-snowplow-tracker==0.0.2", "networkx>=2.3,<2.8.4", - "packaging>=20.9,<22.0", + "packaging>20.9", "sqlparse>=0.2.3,<0.4.4", "dbt-extractor~=0.4.1", "typing-extensions>=3.7.4", diff --git a/test/unit/test_semver.py b/test/unit/test_semver.py index cb19e6fe1ff..601f285ca3a 100644 --- a/test/unit/test_semver.py +++ b/test/unit/test_semver.py @@ -173,13 +173,25 @@ def test__resolve_to_specific_version(self): ['1.0.0', '1.1.0a1', '1.1.0', '1.2.0a1', '1.2.0']), '1.1.0') + self.assertEqual( + resolve_to_specific_version( + # https://github.com/dbt-labs/dbt-core/issues/7039 + # 10 is greater than 9 + create_range('>0.9.0', '<0.10.0'), + ['0.9.0', '0.9.1', '0.10.0']), + '0.9.1') + def test__filter_installable(self): - assert filter_installable( + installable = filter_installable( ['1.1.0', '1.2.0a1', '1.0.0','2.1.0-alpha','2.2.0asdf','2.1.0','2.2.0','2.2.0-fishtown-beta','2.2.0-2'], install_prerelease=True - ) == ['1.0.0', '1.1.0', '1.2.0a1','2.1.0-alpha','2.1.0','2.2.0asdf','2.2.0-fishtown-beta','2.2.0-2','2.2.0'] + ) + expected = ['1.0.0', '1.1.0', '1.2.0a1','2.1.0-alpha','2.1.0','2.2.0-2','2.2.0asdf','2.2.0-fishtown-beta','2.2.0'] + assert installable == expected - assert filter_installable( + installable = filter_installable( ['1.1.0', '1.2.0a1', '1.0.0','2.1.0-alpha','2.2.0asdf','2.1.0','2.2.0','2.2.0-fishtown-beta'], install_prerelease=False - ) == ['1.0.0', '1.1.0','2.1.0','2.2.0'] + ) + expected = ['1.0.0', '1.1.0','2.1.0','2.2.0'] + assert installable == expected