From 1e6feb1ab4e76a1c61e701e28b094cfaa711519f Mon Sep 17 00:00:00 2001 From: Tatiana Al-Chueyr Date: Mon, 4 Dec 2023 15:35:03 +0000 Subject: [PATCH 1/2] Update doc with conflicts between Airflow and dbt doc (#731) Update the docs describing conflicts between Airflow and dbt until Airflow 2.7 and dbt 1.7. --- .pre-commit-config.yaml | 1 + .../execution-modes-local-conflicts.rst | 52 +++++++++++++------ 2 files changed, 37 insertions(+), 16 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 890afb0b7..0ea06541b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,6 +34,7 @@ repos: args: - --exclude-file=tests/sample/manifest_model_version.json - --skip=**/manifest.json + - -L connexion - repo: https://github.com/pre-commit/pygrep-hooks rev: v1.10.0 hooks: diff --git a/docs/getting_started/execution-modes-local-conflicts.rst b/docs/getting_started/execution-modes-local-conflicts.rst index 0f23ef824..3e201bef8 100644 --- a/docs/getting_started/execution-modes-local-conflicts.rst +++ b/docs/getting_started/execution-modes-local-conflicts.rst @@ -4,25 +4,27 @@ Airflow and DBT dependencies conflicts ====================================== When using the `Local Execution Mode `__, users may face dependency conflicts between -Apache Airflow and DBT. The amount of conflicts may increase depending on the Airflow providers and DBT plugins being used. +Apache Airflow and DBT. The conflicts may increase depending on the Airflow providers and DBT plugins being used. If you find errors, we recommend users look into using `alternative execution modes `__. In the following table, ``x`` represents combinations that lead to conflicts (vanilla ``apache-airflow`` and ``dbt-core`` packages): -+---------------+-----+-----+-----+-----+-----+-----+---------+ -| Airflow \ DBT | 1.0 | 1.1 | 1.2 | 1.3 | 1.4 | 1.5 | 1.6.0b6 | -+===============+=====+=====+=====+=====+=====+=====+=========+ -| 2.2 | | | | x | x | x | x | -+---------------+-----+-----+-----+-----+-----+-----+---------+ -| 2.3 | x | x | | x | x | x | x | -+---------------+-----+-----+-----+-----+-----+-----+---------+ -| 2.4 | x | x | x | | | | | -+---------------+-----+-----+-----+-----+-----+-----+---------+ -| 2.5 | x | x | x | | | | | -+---------------+-----+-----+-----+-----+-----+-----+---------+ -| 2.6 | x | x | x | x | x | | x | -+---------------+-----+-----+-----+-----+-----+-----+---------+ ++---------------+-----+-----+-----+-----+-----+-----+-----+-----+ +| Airflow / DBT | 1.0 | 1.1 | 1.2 | 1.3 | 1.4 | 1.5 | 1.6 | 1.7 | ++===============+=====+=====+=====+=====+=====+=====+=====+=====+ +| 2.2 | | | | x | x | x | x | x | ++---------------+-----+-----+-----+-----+-----+-----+-----+-----+ +| 2.3 | x | x | | x | x | x | x | x | ++---------------+-----+-----+-----+-----+-----+-----+-----+-----+ +| 2.4 | x | x | x | | | | | | ++---------------+-----+-----+-----+-----+-----+-----+-----+-----+ +| 2.5 | x | x | x | | | | | | ++---------------+-----+-----+-----+-----+-----+-----+-----+-----+ +| 2.6 | x | x | x | x | x | | | | ++---------------+-----+-----+-----+-----+-----+-----+-----+-----+ +| 2.7 | x | x | x | x | x | | | | ++---------------+-----+-----+-----+-----+-----+-----+-----+-----+ Examples of errors ----------------------------------- @@ -41,6 +43,22 @@ Examples of errors apache-airflow 2.6.0 depends on importlib-metadata<5.0.0 and >=1.7; python_version < "3.9" dbt-semantic-interfaces 0.1.0.dev7 depends on importlib-metadata==6.6.0 +.. code-block:: bash + + ERROR: Cannot install apache-airflow, apache-airflow==2.7.0 and dbt-core==1.4.0 because these package versions have conflicting dependencies. + + The conflict is caused by: + dbt-core 1.4.0 depends on pyyaml>=6.0 + connexion 2.12.0 depends on PyYAML<6 and >=5.1 + dbt-core 1.4.0 depends on pyyaml>=6.0 + connexion 2.11.2 depends on PyYAML<6 and >=5.1 + dbt-core 1.4.0 depends on pyyaml>=6.0 + connexion 2.11.1 depends on PyYAML<6 and >=5.1 + dbt-core 1.4.0 depends on pyyaml>=6.0 + connexion 2.11.0 depends on PyYAML<6 and >=5.1 + apache-airflow 2.7.0 depends on jsonschema>=4.18.0 + flask-appbuilder 4.3.3 depends on jsonschema<5 and >=3 + connexion 2.10.0 depends on jsonschema<4 and >=2.5.1 How to reproduce ---------------- @@ -57,8 +75,10 @@ The table was created by running `nox `__ wi @nox.session(python=["3.10"]) - @nox.parametrize("dbt_version", ["1.0", "1.1", "1.2", "1.3", "1.4", "1.5", "1.6.0b6"]) - @nox.parametrize("airflow_version", ["2.2.4", "2.3", "2.4", "2.5", "2.6"]) + @nox.parametrize( + "dbt_version", ["1.0", "1.1", "1.2", "1.3", "1.4", "1.5", "1.6", "1.7"] + ) + @nox.parametrize("airflow_version", ["2.2.4", "2.3", "2.4", "2.5", "2.6", "2.7"]) def compatibility(session: nox.Session, airflow_version, dbt_version) -> None: """Run both unit and integration tests.""" session.run( From e1f34ea424b8940f1c06660554298008876ea11b Mon Sep 17 00:00:00 2001 From: Tatiana Al-Chueyr Date: Mon, 4 Dec 2023 15:43:12 +0000 Subject: [PATCH 2/2] Document contributor roles and add maintainers list (#734) Add documentation about Astronomer Cosmos contributor roles and maintainers until now. --------- Co-authored-by: Harel Shein Co-authored-by: Julian LaNeve --- docs/contributing.rst | 4 +++ docs/contributors-roles.rst | 56 +++++++++++++++++++++++++++++++++++++ docs/contributors.rst | 26 +++++++++++++++++ docs/index.rst | 3 +- 4 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 docs/contributors-roles.rst create mode 100644 docs/contributors.rst diff --git a/docs/contributing.rst b/docs/contributing.rst index f87553883..3e2ab6fe3 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -1,3 +1,5 @@ +.. _contributing: + Cosmos Contributing Guide ========================= @@ -6,6 +8,8 @@ All contributions, bug reports, bug fixes, documentation improvements, enhanceme As contributors and maintainers to this project, you are expected to abide by the `Contributor Code of Conduct `_. +Learn more about the contributors roles in :ref:`contributors-roles`. + Overview ________ diff --git a/docs/contributors-roles.rst b/docs/contributors-roles.rst new file mode 100644 index 000000000..7cddfb95f --- /dev/null +++ b/docs/contributors-roles.rst @@ -0,0 +1,56 @@ +.. _contributors-roles: + +Contributor roles +================== + +Contributors are welcome and are greatly appreciated! Every little bit helps, and we give credit to them. + +This document aims to explain the current roles in the Astronomer Cosmos project. +For more information, check :ref:`contributing` and :ref:`contributors`. + + +Contributors +------------ + +A contributor is anyone who wants to contribute code, documentation, tests, ideas, or anything to the Astronomer Cosmos project. + +Cosmos contributors can be found in the Astronomer Cosmos Github `insights page `_ and in the `#airflow-dbt `_ Slack channel. + +Contributors are responsible for: + +* Fixing bugs +* Refactoring code +* Improving processes and tooling +* Adding features +* Improving the documentation +* Making/answering questions in the #airflow-dbt Slack channel + + +Committers +---------------------- + +Committers are community members with write access to the `Astronomer Cosmos Github repository `_. +They can modify the code and the documentation and accept others' contributions to the repo. + +Check :ref:`contributors` for the official list of Astronomer Cosmos committers. + +Committers have the same responsibilities as standard contributors and also perform the following actions: + +* Reviewing & merging pull-requests +* Scanning and responding to GitHub issues, helping triaging them + +If you know you are not going to be able to contribute for a long time (for instance, due to a change of job or circumstances), you should inform other maintainers, and we will mark you as "emeritus". +Emeritus committers will no longer have write access to the repo. +As merit earned never expires, once an emeritus committer becomes active again, they can simply email another maintainer from Astronomer and ask to be reinstated. + +Pre-requisites to becoming a committer +....................................... + +General prerequisites that we look for in all candidates: + +1. Consistent contribution over last few months +2. Visibility on discussions on the Slack channel or GitHub issues/discussions +3. Contributions to community health and project's sustainability for the long-term +4. Understands the project's contributors guidelines :ref:`contributing`. +Astronomer is responsible and accountable for releasing new versions of Cosmos in PyPI , following the milestones . +Astronomer has the right to grant and revoke write access permissions to the project's official repository for any reason it sees fit. diff --git a/docs/contributors.rst b/docs/contributors.rst new file mode 100644 index 000000000..273358d3c --- /dev/null +++ b/docs/contributors.rst @@ -0,0 +1,26 @@ +.. _contributors: +Contributors +============ + +There are different ways people can contribute to Astronomer Cosmos. +Learn more about the project contributors roles in :ref:`contributors-roles`. + +Committers +---------------------- + +* Chris Hronek (`@chrishronek `_) +* Harel Shein (`@harels `_) +* Julian LaNeve (`@jlaneve `_) +* Tatiana Al-Chueyr (`@tatiana `_) + + +Emeritus Committers +------------------------------- + +(none at the moment) + +Contributors +------------ + +Many people are improving Astronomer Cosmos each day. +Find more contributors `in our Github page `_ and in the `#airflow-dbt `_ Slack channel. diff --git a/docs/index.rst b/docs/index.rst index f5cd3673d..0c7ab506c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -73,7 +73,8 @@ __________________ All contributions, bug reports, bug fixes, documentation improvements, enhancements are welcome. -A detailed overview an how to contribute can be found in the `Contributing Guide `_. +A detailed overview on how to contribute can be found in the `Contributing Guide `_. +Find out more about `our contributors `_. As contributors and maintainers to this project, you are expected to abide by the `Contributor Code of Conduct `_.