Install pip requirements with --require-hashes and add requirements t…

…xt for all platforms
google · Jun 27, 2024 · 46c6892 · 46c6892
1 parent d34e87b
commit 46c6892
Show file tree

Hide file tree

Showing 38 changed files with 4,148 additions and 409 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -28,5 +28,4 @@ grr/var
 grr_server*.tar.gz
 LICENSE
 README.md
-travis
 vagrant
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -87,14 +87,23 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: '3.9'
+      - name: Verify requirements
+        run: |
+          ./travis/compile_requirements.sh ubuntu
+      - name: Upload requirements to GitHub artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: ubuntu-requirements
+          path: requirements/
+          retention-days: 3
       - name: Set up
         run: |
           sudo apt install fakeroot debhelper libffi-dev libssl-dev
           pip install virtualenv
           virtualenv "${HOME}/INSTALL"
       - name: Build
         run: |
-          travis/install_client_builder.sh
+          travis/install_client_builder.sh ubuntu
           travis/build_templates.sh
           ls -la gcs_upload_dir
       - name: Upload installers to GitHub artifacts
@@ -115,9 +124,18 @@ jobs:
         run: |
           pip install --upgrade setuptools virtualenv
           virtualenv "${HOME}/INSTALL"
+      - name: Verify requirements
+        run: |
+          ./travis/compile_requirements.sh osx
+      - name: Upload requirements to GitHub artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: osx-requirements
+          path: requirements/
+          retention-days: 3
       - name: Build installers
         run: |
-          travis/install_client_builder.sh
+          travis/install_client_builder.sh osx
           travis/build_templates.sh
           ls -la gcs_upload_dir
       - name: Upload installers to GitHub artifacts
@@ -136,6 +154,15 @@ jobs:
       DOCKER_USER: grrbot
     steps:
       - uses: actions/checkout@v4
+      - name: Verify requirements
+        run: |
+          ./travis/compile_requirements.sh centos
+      - name: Upload requirements to GitHub artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: centos-requirements
+          path: requirements/
+          retention-days: 3
       - name: Build installers
         run: |
           docker run -dit \
@@ -150,7 +177,7 @@ jobs:
           # registered in the environment variables.
           docker exec "${DOCKER_CONTAINER}" bash -l travis/set_up_test_user.sh
           docker exec --user "${DOCKER_USER}" "${DOCKER_CONTAINER}" bash -l -c '/usr/local/bin/python3.9 -m venv "/home/${DOCKER_USER}/INSTALL"'
-          docker exec --user "${DOCKER_USER}" "${DOCKER_CONTAINER}" bash -l travis/install_client_builder.sh
+          docker exec --user "${DOCKER_USER}" "${DOCKER_CONTAINER}" bash -l travis/install_client_builder.sh centos
           docker exec --user "${DOCKER_USER}" "${DOCKER_CONTAINER}" bash -l travis/build_templates.sh
           docker exec "${DOCKER_CONTAINER}" rpm -vih gcs_upload_dir/*.rpm
           ls -la gcs_upload_dir
@@ -168,6 +195,15 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: '3.9'
+      - name: Verify requirements
+        run: |
+          .\travis\compile_requirements.bat
+      - name: Upload requirements to GitHub artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: windows-requirements
+          path: requirements/
+          retention-days: 3
       - name: Build installers
         shell: bash
         run: |

diff --git a/Dockerfile b/Dockerfile
@@ -69,14 +69,17 @@ ENV GRR_SOURCE /usr/src/grr
 RUN python -m venv --system-site-packages $VIRTUAL_ENV
 ENV PATH=${VIRTUAL_ENV}/bin:${PATH}
 
+RUN ${VIRTUAL_ENV}/bin/python -m pip install wheel nodeenv grpcio-tools==1.60
+
+RUN ${VIRTUAL_ENV}/bin/nodeenv -p --prebuilt --node=16.13.0
+
 RUN mkdir ${GRR_SOURCE}
 ADD . ${GRR_SOURCE}
 
 WORKDIR ${GRR_SOURCE}
 
-RUN ${VIRTUAL_ENV}/bin/python -m pip install --require-hashes -r requirements.txt
-
-RUN ${VIRTUAL_ENV}/bin/nodeenv -p --prebuilt --node=16.13.0
+RUN ${VIRTUAL_ENV}/bin/python -m pip install --require-hashes \
+  -r travis/requirements/ubuntu.txt
 
 RUN ${VIRTUAL_ENV}/bin/python -m pip install --no-deps --no-index \
   -e grr/proto \

diff --git a/api_client/python/requirements.in b/api_client/python/requirements.in
@@ -0,0 +1,5 @@
+cryptography>=3.3.2
+requests>=2.25.1,<3
+Werkzeug>=2.1.2,<3
+# only required for shell
+ipython==7.15.0
diff --git a/api_client/python/setup.py b/api_client/python/setup.py
@@ -1,6 +1,8 @@
 #!/usr/bin/env python
 """setup.py file for a GRR API client library."""
 
+from typing import List
+
 import configparser
 import os
 import shutil
@@ -41,43 +43,47 @@ def make_release_tree(self, base_dir, files):
     if os.path.exists(sdist_version_ini):
       os.unlink(sdist_version_ini)
     shutil.copy(
-        os.path.join(THIS_DIRECTORY, "../../version.ini"), sdist_version_ini)
+      os.path.join(THIS_DIRECTORY, "../../version.ini"), sdist_version_ini
+    )
 
 
 VERSION = get_config()
 
+
+def parse_requirements(filename: str) -> List[str]:
+  requirements = []
+  with open(filename) as file:
+    for line in file:
+      requirement = line.strip()
+      if (comment := requirement.find("#")) >= 0:
+        requirement = requirement[:comment].strip()
+      requirements.append(requirement)
+
+  return requirements
+
+
 setup_args = dict(
-    name="grr-api-client",
-    version=VERSION.get("Version", "packageversion"),
-    description="GRR API client library",
-    license="Apache License, Version 2.0",
-    url="https://github.com/google/grr/tree/master/api_client/python",
-    maintainer="GRR Development Team",
-    maintainer_email="[email protected]",
-    cmdclass={
-        "sdist": Sdist,
-    },
-    packages=find_packages(),
-    entry_points={
-        "console_scripts": [
-            "grr_api_shell = grr_api_client.api_shell:main",
-        ]
-    },
-    install_requires=[
-        "grr_response_proto==%s" % VERSION.get("Version", "packagedepends"),
-        # Note: grr-api-client might very much be used as a library and
-        # therefore shouldn't pin dependencies that might be shared with other
-        # pip packages.
-        "cryptography>=3.3.2",
-        "requests>=2.25.1,<3",
-        "Werkzeug>=2.1.2,<3",
-    ],
-    extra_requires={
-        "shell": [
-            "ipython==7.15.0",
-        ],
-    },
-    data=["version.ini"],
+  name="grr-api-client",
+  version=VERSION.get("Version", "packageversion"),
+  description="GRR API client library",
+  license="Apache License, Version 2.0",
+  url="https://github.com/google/grr/tree/master/api_client/python",
+  maintainer="GRR Development Team",
+  maintainer_email="[email protected]",
+  cmdclass={
+    "sdist": Sdist,
+  },
+  packages=find_packages(),
+  entry_points={
+    "console_scripts": [
+      "grr_api_shell = grr_api_client.api_shell:main",
+    ]
+  },
+  install_requires=[
+    "grr_response_proto==%s" % VERSION.get("Version", "packagedepends"),
+  ]
+  + parse_requirements("requirements.in"),
+  data=["version.ini", "requirements.in"],
 )
 
 setup(**setup_args)