Revert "Auto-generate GRPC client bindings as part of pip install (#4041

)"

This reverts commit bafbbda.

.github/workflows/python-client-release-to-pypi.yml

@@ -19,8 +19,8 @@ jobs:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
       - uses: ./.github/workflows/python-tests
         with:
-          python-version: '3.9'
-          tox-env: 'py39'
+          python-version: '3.8'
+          tox-env: 'py38'
           path: 'client/python'
           github-token: ${{secrets.GITHUB_TOKEN}}
       - name: Publish package to PyPI

.github/workflows/python-client.yml

@@ -34,16 +34,13 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        python: [ '3.9', '3.10', '3.11', '3.12' ]
+        python: [ '3.8', '3.9', '3.10' ]
         include:
+          - tox-env: 'py38'
           - tox-env: 'py39'
             python: '3.9'
           - tox-env: 'py310'
             python: '3.10'
-          - tox-env: 'py311'
-            python: '3.11'
-          - tox-env: 'py312'
-            python: '3.12'
     steps:
       - uses: actions/checkout@v4
       - name: Setup Go

.github/workflows/python-tests/action.yml

@@ -21,7 +21,7 @@ runs:
       with:
         python-version: ${{ inputs.python-version }}
     # Tox to run tests; build to build the wheel after tests pass
-    - run: pip install tox==4.17.0 build twine setuptools
+    - run: pip install tox==3.27.1 build twine
       shell: bash
     - name: Install Protoc
       uses: arduino/setup-protoc@v3
@@ -45,7 +45,7 @@ runs:
       working-directory: ${{ inputs.path }}
     - name: Build and verify wheel
       run: |
-        python -m build --sdist
+        python -m build --wheel
         twine check dist/*
       shell: bash
       working-directory: ${{ inputs.path }}

.gitignore

@@ -77,7 +77,6 @@ client/python/dist
 *_pb2.py
 *_pb2.pyi
 *_pb2_grpc.py
-client/python/armada_client/proto/
 client/python/armada_client/armada/
 .tox
 proto-airflow

build/python-client/Dockerfile

@@ -1,13 +1,13 @@
 ARG PLATFORM=x86_64
-ARG BASE_IMAGE=python:3.9.20-bookworm
+ARG BASE_IMAGE=python:3.8.18-bookworm
 
 FROM --platform=$PLATFORM ${BASE_IMAGE}
 
 RUN mkdir /proto
 
 COPY client/python/pyproject.toml /code/pyproject.toml
 
-RUN pip install setuptools "/code[test]"
+RUN pip install "/code[test]"
 
 # Creating folders, and files for a project:
 COPY client/python /code

client/python/armada_client/gen/event_typings.py

@@ -1,5 +1,3 @@
-import argparse
-from pathlib import Path
 import sys
 
 from armada_client.armada import event_pb2, submit_pb2
@@ -65,7 +63,15 @@ def gen_file(states, classes, jobstates):
     return import_text, states_text, union_text, jobstates_text
 
 
-def main(typings_file: Path):
+def write_file(import_text, states_text, union_text, jobstates_text, file):
+    with open(f"{file}", "w", encoding="utf-8") as f:
+        f.write(import_text)
+        f.write(states_text)
+        f.write(jobstates_text)
+        f.write(union_text)
+
+
+def main():
     states = get_event_states()
     print("Done creating EventStates")
 
@@ -78,16 +84,13 @@ def main(typings_file: Path):
     import_text, states_text, union_text, jobstates_text = gen_file(
         states, classes, jobstates
     )
-    typings_file.write_text(import_text + states_text + jobstates_text + union_text)
+    write_file(import_text, states_text, union_text, jobstates_text, typings_file)
 
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("typings_file", type=Path, help="Path to typings file")
-
-    args = parser.parse_args()
-    print(f"{args}")
-    typings_file = args.typings_file or Path("armada_client") / "typings.py"
-    print(f"{typings_file}")
-    main(typings_file)
+    # get path to this files location
+    root = f"{sys.path[0]}/../../"
+    typings_file = f"{root}/armada_client/typings.py"
+
+    main()
     sys.exit(0)

client/python/docs/source/conf.py

@@ -13,6 +13,8 @@
 import os
 import sys
 
+sys.path.insert(0, os.path.abspath("../.."))
+
 
 # -- Project information -----------------------------------------------------
 

client/python/pyproject.toml

@@ -1,21 +1,21 @@
 [project]
 name = "armada_client"
-version = "0.4.5"
+version = "0.3.5"
 description = "Armada gRPC API python client"
 readme = "README.md"
-requires-python = ">=3.9"
-dependencies = ["grpcio-tools", "protobuf>3.20,<5.0"]
+requires-python = ">=3.7"
+dependencies = ["grpcio==1.66.1", "grpcio-tools==1.66.1", "mypy-protobuf>=3.2.0", "protobuf>=5.26.1,<6.0dev" ]
 license = { text = "Apache Software License" }
 authors = [{ name = "G-Research Open Source Software", email = "[email protected]" }]
 
 [project.optional-dependencies]
 format = ["black==23.7.0", "flake8==7.0.0", "pylint==2.17.5"]
 # note(JayF): sphinx-jekyll-builder was broken by sphinx-markdown-builder 0.6 -- so pin to 0.5.5
 docs = ["sphinx==7.1.2", "sphinx-jekyll-builder==0.3.0", "sphinx-toolbox==3.2.0b1", "sphinx-markdown-builder==0.5.5"]
-test = ["pytest==7.3.1", "pytest-cov", "pytest-asyncio==0.21.1"]
+test = ["pytest==7.3.1", "coverage>=6.5.0", "pytest-asyncio==0.21.1"]
 
 [build-system]
-requires = ["setuptools", "wheel", "grpcio-tools", "mypy-protobuf", "protobuf>3.20,<5.0"]
+requires = ["setuptools"]
 build-backend = "setuptools.build_meta"
 
 [tool.mypy]
@@ -39,4 +39,4 @@ omit = [
 # py.typed is required for mypy to find type hints in the package
 # from: https://mypy.readthedocs.io/en/stable/installed_packages.html#making-pep-561-compatible-packages
 [tool.setuptools.package-data]
-"*" = ["*.pyi", "py.typed", "proto/**/*.proto"]
+"*" = ["*.pyi", "py.typed"]

client/python/tox.ini

@@ -1,15 +1,16 @@
 [tox]
+isolated_build = true
 envlist =
     format
+    py38
     py39
     py310
-    py311
-    py312
 
 [testenv]
 extras = test
 commands =
-    pytest --cov={envsitepackagesdir}/armada_client --cov-report=xml --cov-report=term tests/unit/
+    coverage run -m pytest tests/unit/
+    coverage xml
 
 [testenv:docs]
 extras = docs

scripts/build-python-client.sh

@@ -5,4 +5,37 @@
 mkdir -p proto/armada
 cp pkg/api/event.proto pkg/api/submit.proto pkg/api/health.proto pkg/api/job.proto pkg/api/binoculars/binoculars.proto proto/armada
 sed -i 's/\([^\/]\)pkg\/api/\1armada/g' proto/armada/*.proto
-cp -rf proto/* client/python/armada_client/proto/
+
+# generate python stubs
+cd proto
+python3 -m grpc_tools.protoc -I. --plugin=protoc-gen-mypy=$(which protoc-gen-mypy) --python_out=../client/python/armada_client --grpc_python_out=../client/python/armada_client --mypy_out=../client/python/armada_client \
+    google/api/annotations.proto \
+    google/api/http.proto \
+    armada/event.proto armada/submit.proto armada/health.proto armada/job.proto armada/binoculars.proto \
+    github.com/gogo/protobuf/gogoproto/gogo.proto \
+    k8s.io/api/core/v1/generated.proto \
+    k8s.io/apimachinery/pkg/api/resource/generated.proto \
+    k8s.io/apimachinery/pkg/apis/meta/v1/generated.proto \
+    k8s.io/apimachinery/pkg/runtime/generated.proto \
+    k8s.io/apimachinery/pkg/runtime/schema/generated.proto \
+    k8s.io/apimachinery/pkg/util/intstr/generated.proto \
+    k8s.io/api/networking/v1/generated.proto
+
+cd ..
+# This hideous code is because we can't use python package option in grpc.
+# See https://github.com/protocolbuffers/protobuf/issues/7061 for an explanation.
+# We need to import these packages as a module.
+sed -i 's/from armada/from armada_client.armada/g' client/python/armada_client/armada/*.py
+sed -i 's/from github.com/from armada_client.github.com/g' client/python/armada_client/armada/*.py
+sed -i 's/from google.api/from armada_client.google.api/g' client/python/armada_client/armada/*.py
+sed -i 's/from google.api/from armada_client.google.api/g' client/python/armada_client/google/api/*.py
+
+find client/python/armada_client/ -name '*.py' | xargs sed -i 's/from k8s.io/from armada_client.k8s.io/g'
+
+# Generate better docs for the client
+export PYTHONPATH=${PWD}/client/python
+python3 ${PWD}/client/python/armada_client/gen/event_typings.py
+
+find client/python/armada_client/k8s -name '*.pyi' | xargs sed -i 's/ k8s/ armada_client.k8s/g'
+find client/python/armada_client/k8s -name '*.pyi' | xargs sed -i 's/\[k8s/\[armada_client.k8s/g'
+find client/python/armada_client/k8s/io -name '*.pyi' | xargs sed -i 's/import k8s.io/import armada_client.k8s.io/g'