Package-Requirements passes along public URL constraints in packaged_requirements.txt

README.md

@@ -139,6 +139,8 @@ For example usage see [Installing Python dependencies using PyPi.org Requirement
 - There is a directory at the root of this repository called plugins. 
 - In this directory, create a file for your new custom plugin.
 - Add any Python dependencies to `requirements/requirements.txt`.
+- Adds a local `constraints.txt` file to the `plugins/` directory, this is zipped together into the `plugins.zip` artefact.
+- Creates a new `packaged_requirements.txt` file with the correct configuration for `--find-links` and `--constraint`. This file should be the one you rename to `requirements.txt` and upload to S3 to be used by MWAA.
 
 **Note**: this step assumes you have a DAG that corresponds to the custom plugin. For example usage [MWAA Code Examples](https://docs.aws.amazon.com/mwaa/latest/userguide/sample-code.html).
 

docker/Dockerfile

@@ -21,6 +21,7 @@ ARG INDEX_URL=""
 ENV AIRFLOW_HOME=${AIRFLOW_USER_HOME}
 ENV PATH="$PATH:/usr/local/airflow/.local/bin:/root/.local/bin:/usr/local/airflow/.local/lib/python3.10/site-packages"
 ENV PYTHON_VERSION=3.11.6
+ENV AIRFLOW_VERSION=${AIRFLOW_VERSION}
 
 COPY script/bootstrap.sh /bootstrap.sh
 COPY script/systemlibs.sh /systemlibs.sh

docker/script/entrypoint.sh

@@ -41,10 +41,11 @@ package_requirements() {
     if [[ -e "$AIRFLOW_HOME/$REQUIREMENTS_FILE" ]]; then
         echo "Packaging requirements.txt into plugins"
         pip3 download -r "$AIRFLOW_HOME/$REQUIREMENTS_FILE" -d "$AIRFLOW_HOME/plugins"
+        wget "https://raw.githubusercontent.com/aws/aws-mwaa-local-runner/v${AIRFLOW_VERSION}/docker/config/constraints.txt" -O $AIRFLOW_HOME/plugins/constraints.txt
         cd "$AIRFLOW_HOME/plugins"
         zip "$AIRFLOW_HOME/requirements/plugins.zip" *
         printf '%s\n%s\n' "--no-index" "$(cat $AIRFLOW_HOME/$REQUIREMENTS_FILE)" > "$AIRFLOW_HOME/requirements/packaged_requirements.txt"
-        printf '%s\n%s\n' "--find-links /usr/local/airflow/plugins" "$(cat $AIRFLOW_HOME/requirements/packaged_requirements.txt)" > "$AIRFLOW_HOME/requirements/packaged_requirements.txt"
+        printf '%s\n%s\n%s\n%s\n' "--find-links /usr/local/airflow/plugins" "--constraint /usr/local/airflow/plugins/constraints.txt" "$(cat $AIRFLOW_HOME/requirements/packaged_requirements.txt | grep -v '^--constraint')" > "$AIRFLOW_HOME/requirements/packaged_requirements.txt" 
     fi
 }
 

requirements/requirements.txt

@@ -1,4 +1,4 @@
---constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.7.2/constraints-3.11.txt"
+--constraint "https://raw.githubusercontent.com/aws/aws-mwaa-local-runner/v2.7.2/docker/config/constraints.txt"
 
 apache-airflow-providers-snowflake==5.0.1
 apache-airflow-providers-mysql==5.3.1