From 7aaa2386b14bd107be66b7b806e6acee5486254a Mon Sep 17 00:00:00 2001 From: luksi1 Date: Wed, 15 Mar 2023 17:52:35 +0100 Subject: [PATCH 1/2] AI-844: Adds researcher user and renv.lock to R --- tests/test_pytorch_env.py | 2 +- tests/test_r_env.py | 8 +- tests/test_tensorflow_env.py | 2 +- {{ cookiecutter.repo_name }}/Dockerfile | 29 +++-- {{ cookiecutter.repo_name }}/README.md | 3 + {{ cookiecutter.repo_name }}/r-start.sh | 6 ++ {{ cookiecutter.repo_name }}/renv.lock | 135 +++++++++++++++++++++++- 7 files changed, 172 insertions(+), 13 deletions(-) create mode 100755 {{ cookiecutter.repo_name }}/r-start.sh diff --git a/tests/test_pytorch_env.py b/tests/test_pytorch_env.py index 2e730cd6a..5371d526c 100644 --- a/tests/test_pytorch_env.py +++ b/tests/test_pytorch_env.py @@ -88,7 +88,7 @@ def test_dockerfile(self): dockerfile_text = Path(dockerfile_path).read_text() assert lines[1] == "FROM registry.git.vgregion.se/aiplattform/images/pytorch:0.3.2" - assert lines[-1] == 'WORKDIR /workspace' + assert lines[-2] == 'WORKDIR /workspace' assert "ADD http://aiav2.vgregion.se/VGC%20Root%20CA%20v2.crt /tmp/vgc_root.der" in lines assert "ADD http://aiav2.vgregion.se/VGC%20Issuing%201%20CA%20v2.crt /tmp/vgc_issuing1.der" in lines assert "ADD http://aiav2.vgregion.se/VGC%20Issuing%202%20CA%20v2.crt /tmp/vgc_issuing2.der" in lines diff --git a/tests/test_r_env.py b/tests/test_r_env.py index 1012c29f8..1f3bfb49a 100644 --- a/tests/test_r_env.py +++ b/tests/test_r_env.py @@ -51,7 +51,11 @@ def test_dockerfile(self): lines = list(map(lambda x: x.strip(), fin.readlines())) dockerfile_text = Path(dockerfile_path).read_text() - assert lines[2] == "FROM registry.git.vgregion.se/aiplattform/images/r:0.2.0" + assert lines[2] == "FROM registry.git.vgregion.se/aiplattform/images/r:0.3.0" + assert lines[-5] == 'RUN R -e \'renv::activate()\' && R -e \'renv::restore()\'' + assert lines[-3] == 'USER root' + assert lines[-2] == 'EXPOSE 8787' + assert "ADD http://aiav2.vgregion.se/VGC%20Root%20CA%20v2.crt /tmp/vgc_root.der" in lines assert "ADD http://aiav2.vgregion.se/VGC%20Issuing%201%20CA%20v2.crt /tmp/vgc_issuing1.der" in lines assert "ADD http://aiav2.vgregion.se/VGC%20Issuing%202%20CA%20v2.crt /tmp/vgc_issuing2.der" in lines @@ -61,6 +65,8 @@ def test_dockerfile(self): assert "openssl x509 -inform der -in /tmp/vgc_issuing1_2.der -out /usr/local/share/ca-certificates/vgc_issuing1_2.crt" assert "openssl x509 -inform der -in /tmp/vgc_issuing2.der -out /usr/local/share/ca-certificates/vgc_issuing2.crt" assert "update-ca-certificates" + assert 'echo "setwd(\"/workspace/\")" > /home/rstudio/.Rprofile' + assert 'COPY renv.lock /workspace/renv.lock' def test_folders(self): module_name = pytest.param.get("project_name") diff --git a/tests/test_tensorflow_env.py b/tests/test_tensorflow_env.py index 22df18204..c812cd44b 100644 --- a/tests/test_tensorflow_env.py +++ b/tests/test_tensorflow_env.py @@ -88,7 +88,7 @@ def test_dockerfile(self): dockerfile_text = Path(dockerfile_path).read_text() assert lines[0] == "FROM registry.git.vgregion.se/aiplattform/images/tensorflow:0.2.1" - assert lines[-1] == 'WORKDIR /workspace' + assert lines[-2] == 'WORKDIR /workspace' assert "ADD http://aiav2.vgregion.se/VGC%20Root%20CA%20v2.crt /tmp/vgc_root.der" in lines assert "ADD http://aiav2.vgregion.se/VGC%20Issuing%201%20CA%20v2.crt /tmp/vgc_issuing1.der" in lines assert "ADD http://aiav2.vgregion.se/VGC%20Issuing%202%20CA%20v2.crt /tmp/vgc_issuing2.der" in lines diff --git a/{{ cookiecutter.repo_name }}/Dockerfile b/{{ cookiecutter.repo_name }}/Dockerfile index 9c724af03..dd7c77fed 100644 --- a/{{ cookiecutter.repo_name }}/Dockerfile +++ b/{{ cookiecutter.repo_name }}/Dockerfile @@ -1,29 +1,42 @@ {% if cookiecutter.image == 'Tensorflow' %}FROM registry.git.vgregion.se/aiplattform/images/tensorflow:0.2.1{% endif %} {% if cookiecutter.image == 'PyTorch' %}FROM registry.git.vgregion.se/aiplattform/images/pytorch:0.3.2{% endif %} -{% if cookiecutter.image == 'R' %}FROM registry.git.vgregion.se/aiplattform/images/r:0.2.0{% endif %} +{% if cookiecutter.image == 'R' %}FROM registry.git.vgregion.se/aiplattform/images/r:0.3.0{% endif %} ENV GID=1000 ENV UID=1000 +{% if cookiecutter.image == 'R' %}ENV EDITOR_FOCUS_DIR="/workspace"{% endif %} RUN apt -y update \ - && apt install -y --no-install-recommends python3-pip git openssl {% if cookiecutter.image != 'R' %}\ - && addgroup --gid $GID researcher \ - && adduser --home /workspace --shell /bin/bash --disabled-password --gecos '' --uid $UID --gid $GID researcher \ + && apt install -y --no-install-recommends python3-pip git openssl \{% if cookiecutter.image != 'R' %} + && addgroup --gid $GID researcher \{% endif %} + && useradd --home-dir /workspace --create-home --shell /bin/bash --non-unique --comment '' --uid $UID --gid $GID researcher \ && chown -R 1000:1000 /workspace - {% endif %} ADD http://aiav2.vgregion.se/VGC%20Root%20CA%20v2.crt /tmp/vgc_root.der ADD http://aiav2.vgregion.se/VGC%20Issuing%201%20CA%20v2.crt /tmp/vgc_issuing1.der ADD http://aiav2.vgregion.se/VGC%20Issuing%202%20CA%20v2.crt /tmp/vgc_issuing2.der ADD http://aiav2.vgregion.se/VGC%20Issuing%201%20CA%20v2(1).crt /tmp/vgc_issuing1_2.der -COPY requirements.txt {% if cookiecutter.image != 'R' %} setup.py {% endif %} ./ +COPY requirements.txt {% if cookiecutter.image != 'R' %} setup.py {% endif %} ./{% if cookiecutter.image == 'R' %} +COPY renv.lock /workspace/renv.lock +COPY r-start.sh /usr/local/bin/r-start{% endif %} + RUN pip install -r requirements.txt --no-cache-dir \ && openssl x509 -inform der -in /tmp/vgc_root.der -out /usr/local/share/ca-certificates/vgc_root.crt \ && openssl x509 -inform der -in /tmp/vgc_issuing1.der -out /usr/local/share/ca-certificates/vgc_issuing1.crt \ && openssl x509 -inform der -in /tmp/vgc_issuing1_2.der -out /usr/local/share/ca-certificates/vgc_issuing1_2.crt \ && openssl x509 -inform der -in /tmp/vgc_issuing2.der -out /usr/local/share/ca-certificates/vgc_issuing2.crt \ && update-ca-certificates \ - && rm requirements.txt setup.py + {% if cookiecutter.image == 'R' %}&& chown -R 1000:1000 /workspace/ \ + && echo "setwd(\"/workspace/\")" > /home/rstudio/.Rprofile \ + && chmod +x /usr/local/bin/r-start \{% endif %} + && rm requirements.txt {% if cookiecutter.image != 'R' %} setup.py {% endif %} USER researcher -WORKDIR /workspace \ No newline at end of file +WORKDIR /workspace + +{% if cookiecutter.image == 'R' %}RUN R -e 'renv::activate()' && R -e 'renv::restore()' + +USER root +EXPOSE 8787 + +{% endif %} \ No newline at end of file diff --git a/{{ cookiecutter.repo_name }}/README.md b/{{ cookiecutter.repo_name }}/README.md index 101d8c8a6..fb6f81d8e 100644 --- a/{{ cookiecutter.repo_name }}/README.md +++ b/{{ cookiecutter.repo_name }}/README.md @@ -83,6 +83,9 @@ Project Organization ├── renv.lock <- R packages for reproducing the analysis environment, e.g. | generated with `renv::restore()` | + ├── r-start.sh <- An R start script that will be available in your Docker image as + | /usr/local/bin/r-start + | ├── requirements.txt <- Python requirements file for reproducing the analysis environment. | Used in **addition** to R for things like documentation and DVC. │ diff --git a/{{ cookiecutter.repo_name }}/r-start.sh b/{{ cookiecutter.repo_name }}/r-start.sh new file mode 100755 index 000000000..d4d939743 --- /dev/null +++ b/{{ cookiecutter.repo_name }}/r-start.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +su - researcher +R -e 'renv::activate()' + +# Insert additional code here diff --git a/{{ cookiecutter.repo_name }}/renv.lock b/{{ cookiecutter.repo_name }}/renv.lock index 20ea6ba96..79dcd5180 100644 --- a/{{ cookiecutter.repo_name }}/renv.lock +++ b/{{ cookiecutter.repo_name }}/renv.lock @@ -1,6 +1,6 @@ { "R": { - "Version": "4.1.2", + "Version": "4.2.1", "Repositories": [ { "Name": "CRAN", @@ -9,6 +9,117 @@ ] }, "Packages": { + "DBI": { + "Package": "DBI", + "Version": "1.1.2", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "dcd1743af4336156873e3ce3c950b8b9", + "Requirements": [] + }, + "Rcpp": { + "Package": "Rcpp", + "Version": "1.0.8.3", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "32e79b908fda56ee57fe518a8d37b864", + "Requirements": [] + }, + "bit": { + "Package": "bit", + "Version": "4.0.4", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "f36715f14d94678eea9933af927bc15d", + "Requirements": [] + }, + "bit64": { + "Package": "bit64", + "Version": "4.0.5", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "9fe98599ca456d6552421db0d6772d8f", + "Requirements": [ + "bit" + ] + }, + "blob": { + "Package": "blob", + "Version": "1.2.2", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "dc5f7a6598bb025d20d66bb758f12879", + "Requirements": [ + "rlang", + "vctrs" + ] + }, + "ellipsis": { + "Package": "ellipsis", + "Version": "0.3.2", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "bb0eec2fe32e88d9e2836c2f73ea2077", + "Requirements": [ + "rlang" + ] + }, + "glue": { + "Package": "glue", + "Version": "1.6.2", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "4f2596dfb05dac67b9dc558e5c6fba2e", + "Requirements": [] + }, + "hms": { + "Package": "hms", + "Version": "1.1.1", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "5b8a2dd0fdbe2ab4f6081e6c7be6dfca", + "Requirements": [ + "ellipsis", + "lifecycle", + "pkgconfig", + "rlang", + "vctrs" + ] + }, + "lifecycle": { + "Package": "lifecycle", + "Version": "1.0.1", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "a6b6d352e3ed897373ab19d8395c98d0", + "Requirements": [ + "glue", + "rlang" + ] + }, + "odbc": { + "Package": "odbc", + "Version": "1.3.3", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "3f9efc29c48dca3aebf00b61a472d543", + "Requirements": [ + "DBI", + "Rcpp", + "bit64", + "blob", + "hms", + "rlang" + ] + }, + "pkgconfig": { + "Package": "pkgconfig", + "Version": "2.0.3", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "01f28d4278f15c76cddbea05899c5d6f", + "Requirements": [] + }, "renv": { "Package": "renv", "Version": "0.15.2-2", @@ -21,6 +132,26 @@ "RemoteSha": "4a12af7cd16e5d4120d2515a2318bd5315dca8b4", "Hash": "93afeb9fb3a032439684997363b7423a", "Requirements": [] + }, + "rlang": { + "Package": "rlang", + "Version": "1.0.2", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "04884d9a75d778aca22c7154b8333ec9", + "Requirements": [] + }, + "vctrs": { + "Package": "vctrs", + "Version": "0.3.8", + "Source": "Repository", + "Repository": "CRAN", + "Hash": "ecf749a1b39ea72bd9b51b76292261f1", + "Requirements": [ + "ellipsis", + "glue", + "rlang" + ] } } -} +} \ No newline at end of file From f6f8716f4b1a805930d7c8a590fe6026ba6859c3 Mon Sep 17 00:00:00 2001 From: luksi1 Date: Fri, 17 Mar 2023 17:04:22 +0100 Subject: [PATCH 2/2] AI-849: Fixes FROM formatting --- tests/test_pytorch_env.py | 2 +- tests/test_r_env.py | 2 +- {{ cookiecutter.repo_name }}/Dockerfile | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_pytorch_env.py b/tests/test_pytorch_env.py index 5371d526c..23f0aea0f 100644 --- a/tests/test_pytorch_env.py +++ b/tests/test_pytorch_env.py @@ -87,7 +87,7 @@ def test_dockerfile(self): lines = list(map(lambda x: x.strip(), fin.readlines())) dockerfile_text = Path(dockerfile_path).read_text() - assert lines[1] == "FROM registry.git.vgregion.se/aiplattform/images/pytorch:0.3.2" + assert lines[0] == "FROM registry.git.vgregion.se/aiplattform/images/pytorch:0.3.2" assert lines[-2] == 'WORKDIR /workspace' assert "ADD http://aiav2.vgregion.se/VGC%20Root%20CA%20v2.crt /tmp/vgc_root.der" in lines assert "ADD http://aiav2.vgregion.se/VGC%20Issuing%201%20CA%20v2.crt /tmp/vgc_issuing1.der" in lines diff --git a/tests/test_r_env.py b/tests/test_r_env.py index 1f3bfb49a..20a4b2b37 100644 --- a/tests/test_r_env.py +++ b/tests/test_r_env.py @@ -51,7 +51,7 @@ def test_dockerfile(self): lines = list(map(lambda x: x.strip(), fin.readlines())) dockerfile_text = Path(dockerfile_path).read_text() - assert lines[2] == "FROM registry.git.vgregion.se/aiplattform/images/r:0.3.0" + assert lines[0] == "FROM registry.git.vgregion.se/aiplattform/images/r:0.3.0" assert lines[-5] == 'RUN R -e \'renv::activate()\' && R -e \'renv::restore()\'' assert lines[-3] == 'USER root' assert lines[-2] == 'EXPOSE 8787' diff --git a/{{ cookiecutter.repo_name }}/Dockerfile b/{{ cookiecutter.repo_name }}/Dockerfile index dd7c77fed..d49d9ff3b 100644 --- a/{{ cookiecutter.repo_name }}/Dockerfile +++ b/{{ cookiecutter.repo_name }}/Dockerfile @@ -1,6 +1,6 @@ -{% if cookiecutter.image == 'Tensorflow' %}FROM registry.git.vgregion.se/aiplattform/images/tensorflow:0.2.1{% endif %} -{% if cookiecutter.image == 'PyTorch' %}FROM registry.git.vgregion.se/aiplattform/images/pytorch:0.3.2{% endif %} -{% if cookiecutter.image == 'R' %}FROM registry.git.vgregion.se/aiplattform/images/r:0.3.0{% endif %} +{% if cookiecutter.image == 'Tensorflow' -%}FROM registry.git.vgregion.se/aiplattform/images/tensorflow:0.2.1 +{% elif cookiecutter.image == 'PyTorch' -%}FROM registry.git.vgregion.se/aiplattform/images/pytorch:0.3.2 +{% elif cookiecutter.image == 'R' -%}FROM registry.git.vgregion.se/aiplattform/images/r:0.3.0{%- endif %} ENV GID=1000 ENV UID=1000 {% if cookiecutter.image == 'R' %}ENV EDITOR_FOCUS_DIR="/workspace"{% endif %}