Skip to content

Commit

Permalink
requirements
Browse files Browse the repository at this point in the history
  • Loading branch information
andrewtruong committed Aug 25, 2024
1 parent 626f359 commit 331b1fd
Show file tree
Hide file tree
Showing 18 changed files with 46 additions and 44 deletions.
38 changes: 19 additions & 19 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -18,31 +18,31 @@ ENV CONDA_VERSION=4.9.2 \
# - Remove all conda managed *.pyc files
# - Cleanup conda files
# - Uninstall miniconda install dependencies
RUN mkdir /weave \
&& apk add --no-cache wget bzip2 \
&& wget --quiet https://repo.continuum.io/miniconda/Miniconda3-py${PYTHON_VERSION}_$CONDA_VERSION-Linux-x86_64.sh \
&& echo "${CONDA_MD5} Miniconda3-py${PYTHON_VERSION}_$CONDA_VERSION-Linux-x86_64.sh" > miniconda.md5 \
&& if [ $(md5sum -c miniconda.md5 | awk '{print $2}') != "OK" ] ; then exit 1; fi \
&& mv Miniconda3-py${PYTHON_VERSION}_$CONDA_VERSION-Linux-x86_64.sh miniconda.sh \
&& sh ./miniconda.sh -b -p /opt/conda \
&& rm miniconda.sh miniconda.md5 \
&& ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh \
&& echo ". /opt/conda/etc/profile.d/conda.sh" >> /weave/.profile \
&& echo "conda activate base" >> /weave/.profile \
&& /opt/conda/bin/conda install conda==$CONDA_VERSION \
&& echo "conda == $CONDA_VERSION" >> /opt/conda/conda-meta/pinned \
&& /opt/conda/bin/conda install --freeze-installed tini pip gunicorn python=3.9.7 -y \
&& find /opt/conda/ -follow -type f -name '*.a' -delete \
&& find /opt/conda/ -follow -type f -name '*.pyc' -delete \
&& /opt/conda/bin/conda clean -afy \
&& apk del wget bzip2
RUN mkdir /weave &&
apk add --no-cache wget bzip2 &&
wget --quiet https://repo.continuum.io/miniconda/Miniconda3-py${PYTHON_VERSION}_$CONDA_VERSION-Linux-x86_64.sh &&
echo "${CONDA_MD5} Miniconda3-py${PYTHON_VERSION}_$CONDA_VERSION-Linux-x86_64.sh" >miniconda.md5 &&
if [ $(md5sum -c miniconda.md5 | awk '{print $2}') != "OK" ]; then exit 1; fi &&
mv Miniconda3-py${PYTHON_VERSION}_$CONDA_VERSION-Linux-x86_64.sh miniconda.sh &&
sh ./miniconda.sh -b -p /opt/conda &&
rm miniconda.sh miniconda.md5 &&
ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh &&
echo ". /opt/conda/etc/profile.d/conda.sh" >>/weave/.profile &&
echo "conda activate base" >>/weave/.profile &&
/opt/conda/bin/conda install conda==$CONDA_VERSION &&
echo "conda == $CONDA_VERSION" >>/opt/conda/conda-meta/pinned &&
/opt/conda/bin/conda install --freeze-installed tini pip gunicorn python=3.9.7 -y &&
find /opt/conda/ -follow -type f -name '*.a' -delete &&
find /opt/conda/ -follow -type f -name '*.pyc' -delete &&
/opt/conda/bin/conda clean -afy &&
apk del wget bzip2

ENV PORT 9239

WORKDIR /weave
ADD . .

RUN pip install -r requirements.engine.txt
RUN pip install -r weave/legacy/requirements.engine.txt
RUN mkdir /local-artifacts

EXPOSE 9239
Expand Down
16 changes: 8 additions & 8 deletions Dockerfile.test
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,12 @@ FROM cypress/included:10.11.0 as builder
RUN apt update
RUN apt -y install python3-venv python3-dev gcc g++ xz-utils
RUN rm /bin/sh && ln -s /bin/bash /bin/sh
COPY requirements.* /root/
COPY weave/legacy/requirements.* /root/
WORKDIR /root
RUN python3 -m venv venv
RUN --mount=type=cache,target=/root/.cache /bin/bash -c "source venv/bin/activate && \
pip install --upgrade pip && \
pip install -r requirements.test.txt -r requirements.dev.txt && \
pip install -r weave/legacy/requirements.test.txt -r weave/legacy/requirements.dev.txt && \
pip install \"spacy>=3.0.0,<4.0.0\" && python3 -m spacy download en_core_web_sm"

ENTRYPOINT "/bin/bash"
Expand All @@ -22,19 +22,19 @@ COPY weave-js /root/weave-js
COPY wb_schema.gql /root/wb_schema.gql
WORKDIR /root/weave-js
RUN --mount=type=cache,target=/usr/local/share/.cache \
SHA1=$(find $JS_DIR -not -path "*/.vite-cache/*" -not -path "*/node_modules/*" -not -path "*/build/*" -type f -print0 | sort -z | xargs -0 sha1sum | sha1sum | cut -d " " -f1) && \
yarn install --frozen-lockfile && \
yarn build && \
echo $SHA1 > ./build/sha1.txt && \
SHA1=$(find $JS_DIR -not -path "*/.vite-cache/*" -not -path "*/node_modules/*" -not -path "*/build/*" -type f -print0 | sort -z | xargs -0 sha1sum | sha1sum | cut -d " " -f1) &&
yarn install --frozen-lockfile &&
yarn build &&
echo $SHA1 >./build/sha1.txt &&
rm -rf node_modules

# final stage
FROM builder
WORKDIR /root

COPY requirements.ecosystem.txt /root
COPY weave/legacy/requirements.ecosystem.txt /root
RUN --mount=type=cache,target=/root/.cache /bin/bash -c "source venv/bin/activate && \
pip install -r requirements.ecosystem.txt"
pip install -r weave/legacy/requirements.ecosystem.txt"

ENV RECORD_REPLAY_DIRECTORY=/root/.replay/
RUN mkdir -p $RECORD_REPLAY_DIRECTORY
Expand Down
2 changes: 1 addition & 1 deletion MANIFEST.in
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
include requirements.*
include weave/legacy/requirements.*
include wb_schema.gql
graft weave/client_context
graft weave/deploy
Expand Down
6 changes: 3 additions & 3 deletions dev_docs/DEVELOPMENT.md
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,9 @@ Install primary package deps

```
pip install -e .
pip install -r requirements.dev.txt
pip install -r requirements.test.txt
pip install -r requirements.ecosystem.txt
pip install -r weave/legacy/requirements.dev.txt
pip install -r weave/legacy/requirements.test.txt
pip install -r weave/legacy/requirements.ecosystem.txt
```

If you have an issue that looks like you need a rust compiler:
Expand Down
10 changes: 6 additions & 4 deletions docs/CONTRIBUTING_DOCS.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ Satisfy the following dependencies to create, build, and locally serve Weave Doc
npm install --global yarn
```
- Make sure your python environment is setup by running the following from the repro root:
- `pip install -r requirements.dev.txt`
- `pip install -e .`
- `pip install -r weave/legacy/requirements.dev.txt`
- `pip install -e .`
- Install an IDE (e.g. VS Code) or Text Editor (e.g. Sublime)

&nbsp;
Expand Down Expand Up @@ -82,7 +82,9 @@ git push origin <your-feature-branch>
8. Open a pull request from the new branch to the original repo.

## DocGen

Currently, we have 3 forms of doc generation:

1. Python Doc Gen
2. Service Doc Gen
3. Notebook Doc Gen
Expand All @@ -108,7 +110,7 @@ Python doc gen uses `lazydocs` as the core library for building markdown docs fr

### Service Doc Gen

See `docs/scripts/generate_service_api_spec.py` and `./docs/reference/service-api`
See `docs/scripts/generate_service_api_spec.py` and `./docs/reference/service-api`

Service doc generation loads the `openapi.json` file describing the server, processes it, then uses the `docusaurus-plugin-openapi-docs` plugin to generate markdown files from that specification.

Expand All @@ -119,11 +121,11 @@ To improve docs, basically follow FastAPI's instructions to create good Swagger
See `docs/scripts/generate_notebooks.py`, `./docs/notebooks`, and `./docs/reference/gen_notebooks`.

This script will load all notebooks in `./docs/notebooks`, transforming them into viable markdown docs in `./docs/reference/gen_notebooks` which can be referenced by docusaurus just like any other markdown file. If you need header metadata, you can add a markdown block at the top of your notebook with:

```
<!-- docusaurus_head_meta::start
---
title: Head Metadata
---
docusaurus_head_meta::end -->
```

4 changes: 2 additions & 2 deletions weave/deploy/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ FROM $BASE_IMAGE
ENV PYTHONUNBUFFERED 1
WORKDIR /app

COPY requirements/* .
RUN pip install --no-cache-dir -r requirements.txt
COPY weave/legacy/requirements/* .
RUN pip install --no-cache-dir -r weave/legacy/requirements.txt
ENV PROJECT_NAME $PROJECT_NAME

EXPOSE 8080
Expand Down
4 changes: 2 additions & 2 deletions weave/deploy/gcp/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -107,9 +107,9 @@ def compile(
) -> str:
"""Compile the weave application."""
dir = tempfile.mkdtemp()
reqs = os.path.join(dir, "requirements")
reqs = os.path.join(dir, "weave/legacy/requirements")
os.mkdir(reqs)
with open(os.path.join(reqs, "requirements.txt"), "w") as f:
with open(os.path.join(reqs, "weave/legacy/requirements.txt"), "w") as f:
f.write(generate_requirements_txt(model_ref, reqs, dev))
with open(os.path.join(dir, "Dockerfile"), "w") as f:
f.write(
Expand Down
4 changes: 2 additions & 2 deletions weave/legacy/.devcontainer/postCreateCommand.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,12 @@ sudo apt update
sudo apt install -y libsndfile1

# Install python packages
pip install -r requirements.dev.txt -r requirements.test.txt
pip install -r weave/legacy/requirements.dev.txt -r weave/legacy/requirements.test.txt
pip install -e .[ecosystem]
pre-commit install

# pre-commit hooks cause permission weirdness
git config --global --add safe.directory /workspaces/weave

# Move extension installed black to black-latest
mv /usr/local/py-utils/bin/black /usr/local/py-utils/bin/black-latest
mv /usr/local/py-utils/bin/black /usr/local/py-utils/bin/black-latest
2 changes: 1 addition & 1 deletion weave/legacy/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ ENV PORT 9239
WORKDIR /weave
ADD . .

RUN pip install -r requirements.engine.txt
RUN pip install -r weave/legacy/requirements.engine.txt
RUN mkdir /local-artifacts

EXPOSE 9239
Expand Down
4 changes: 2 additions & 2 deletions weave/legacy/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@ generate_panel_instructions:
jupyter nbconvert --to markdown examples/template_instructions/*.ipynb --output-dir weave/panels_py/instructions/


.integration-deps: requirements.test.txt requirements.txt
pip install -r requirements.test.txt
.integration-deps: weave/legacy/requirements.test.txt weave/legacy/requirements.txt
pip install -r weave/legacy/requirements.test.txt
touch .integration-deps


Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.

0 comments on commit 331b1fd

Please sign in to comment.