Move pip compile to Dockerfile.

If any dependencies had variants based on the platform then the wrong requirements would be rendered out. For example, if I used Chassis on a Windows machine and one of the dependencies had a variant that used the `win32` APIs, then the generated `requirements.txt` would include those packages which would not work when they were installed during the Docker build.

Now the dependencies are generated in the same environment that they will be installed in and the `pip compile` and dependency normalization happens in a new pre-stage of the Docker build process.

packages/chassisml/src/chassis/builder/buildable.py

@@ -47,6 +47,8 @@ def _copy_libraries(context: BuildContext, server: str, ignore_patterns: List[st
              os.path.join(context.chassis_dir, "ftypes"), ignore=ignore)
     copytree(os.path.join(root, "chassis", "server", server),
              os.path.join(context.chassis_dir, "server", server), ignore=ignore)
+    copytree(os.path.join(root, "chassis", "scripts"),
+             os.path.join(context.chassis_dir, "scripts"), ignore=ignore)
 
 
 class Buildable(metaclass=abc.ABCMeta):
@@ -294,23 +296,23 @@ def _write_requirements(self, context: BuildContext, options: BuildOptions):
             additional_requirements="\n".join(additional_requirements)
         )
         requirements_in = os.path.join(context.base_dir, "requirements.in")
-        requirements_txt = os.path.join(context.base_dir, "requirements.txt")
+        # requirements_txt = os.path.join(context.base_dir, "requirements.txt")
         with open(requirements_in, "wb") as f:
             f.write(rendered_template.encode("utf-8"))
-        # Use pip-tools to expand the list out to a frozen and pinned list.
-        subprocess.run([
-            sys.executable, "-m", "piptools", "compile", "-q",
-            "-o", requirements_txt, requirements_in,
-        ])
-        # Post-process the full requirements.txt with automatic replacements.
-        with open(requirements_txt, "rb") as f:
-            reqs = f.read().decode()
-        for old, new in REQUIREMENTS_SUBSTITUTIONS.items():
-            reqs = reqs.replace(old, new)
-        if "torch" in reqs and options.cuda_version is None:
-            reqs = "--extra-index-url https://download.pytorch.org/whl/cpu\n\n" + reqs
-        with open(requirements_txt, "wb") as f:
-            f.write(reqs.encode())
+        # # Use pip-tools to expand the list out to a frozen and pinned list.
+        # subprocess.run([
+        #     sys.executable, "-m", "piptools", "compile", "-q",
+        #     "-o", requirements_txt, requirements_in,
+        # ])
+        # # Post-process the full requirements.txt with automatic replacements.
+        # with open(requirements_txt, "rb") as f:
+        #     reqs = f.read().decode()
+        # for old, new in REQUIREMENTS_SUBSTITUTIONS.items():
+        #     reqs = reqs.replace(old, new)
+        # if "torch" in reqs and options.cuda_version is None:
+        #     reqs = "--extra-index-url https://download.pytorch.org/whl/cpu\n\n" + reqs
+        # with open(requirements_txt, "wb") as f:
+        #     f.write(reqs.encode())
 
     def _write_python_modules(self, context: BuildContext):
         for key, m in self.python_modules.items():

packages/chassisml/src/chassis/builder/templates/Dockerfile

@@ -1,4 +1,23 @@
-{%- if cuda_version -%}
+FROM python:{{ python_version }}-slim-bullseye AS pip-compile
+
+RUN pip install --no-cache-dir pip-tools
+
+{% if cuda_version -%}
+ENV GPU=true
+{%- else -%}
+ENV GPU=false
+{%- endif %}
+
+COPY chassis/scripts scripts
+
+COPY requirements.in .
+RUN python -m piptools compile -q -o requirements.txt requirements.in
+
+RUN python scripts/normalize_requirements.py
+
+# ------------------------------------------------------------------------------
+
+{% if cuda_version -%}
 FROM nvidia/cuda:{{ cuda_version }}-runtime-ubuntu20.04
 
 # Install Python
@@ -19,7 +38,7 @@ LABEL {{key}}="{{value}}"
 
 
 # Copy requirements file and pip install.
-COPY requirements.txt .
+COPY --from=pip-compile requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
 
 # Copy the entrypoint file and gRPC server implementation.

packages/chassisml/src/chassis/scripts/normalize_requirements.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+
+import os
+
+"""
+A list of pip requirements that are modified when writing out the container's
+`requirements.txt`.
+
+Items in this list are primarily to ensure that large packages that have a
+headless variant use the headless variant since the container doesn't use a
+display.
+"""
+REQUIREMENTS_SUBSTITUTIONS = {
+    "opencv-python=": "opencv-python-headless="
+}
+
+requirements_txt = "requirements.txt"
+
+# Post-process the full requirements.txt with automatic replacements.
+with open(requirements_txt, "rb") as f:
+    reqs = f.read().decode()
+for old, new in REQUIREMENTS_SUBSTITUTIONS.items():
+    reqs = reqs.replace(old, new)
+if "torch" in reqs and not os.getenv("GPU").strip() == "true":
+    reqs = "--extra-index-url https://download.pytorch.org/whl/cpu\n\n" + reqs
+with open(requirements_txt, "wb") as f:
+    f.write(reqs.encode())