Big update to v2.0 (#39)

.devcontainer/devcontainer.json

@@ -1,50 +1,83 @@
 // For format details, see https://aka.ms/devcontainer.json. For config options, see the
 // README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-dockerfile
 {
-	"name": "Existing Dockerfile",
-	"build": {
-		// Sets the run context to one level up instead of the .devcontainer folder.
-		"context": "..",
-		// Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename.
-		"dockerfile": "../Dockerfile",
-		"args": {
-			"OS_SELECTOR": "ubuntu" // force ubuntu base OS for devcontainer
-		}
-	},
-	// Specify which devices you want to use (e.g "device=0")
+	"name": "nlp-research-template:latest",
+	// Put your own built docker image here
+	"image": "konstantinjdobler/nlp-research-template:latest",
+	//-------------------------------------------------
+	// ---- Alternative: let VS Code build the image---
+	// ------------------------------------------------
+	// "build": {
+	// 	// Sets the run context to one level up instead of the .devcontainer folder.
+	// 	"context": "..",
+	// 	// Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename.
+	// 	"dockerfile": "../Dockerfile",
+	//	// "args": {
+	//	// 	"OS_SELECTOR": "ubuntu" // force ubuntu base OS for devcontainer, necessary  if you want to use "features"
+	//	// }
+	// },
+	////////////////////////////////////////////////////////////
+	// -----------------------------------
+	// CUSTOMIZE SOME THINGS BELOW HERE !!!!!
+	// -----------------------------------
+	////////////////////////////////////////////////////////////
+	// Specify which GPU device you want to use (e.g "device=0")
 	"runArgs": [
 		"--ipc=host",
-	//	"--gpus",
-	//	"device=CHANGE_ME"
+		//	"--gpus",
+		//	"device=CHANGE_ME"
 	],
-	// Optionally mount cache files like HuggingFace datasets cache. The user directory inside the container is /home/mamba/
+	// ------------------------------------------------------------
+	// Setup your mounts and env vars like for a docker run command
+	// ------------------------------------------------------------
+	// Mount cache files like HuggingFace datasets cache. The user directory inside the container is /home/mamba/
+	// Paths need to be absolute, so we use the localEnv variable to get the local user home directory
+	// Check HF_DATASETS_CACHE, WANDB_DATA_DIR, XDG_CACHE_HOME etc. env vars to see what you need to mount
 	// "mounts": [
-	// 	"source=/MY_HOME_DIR/.cache,target=/home/mamba/.cache,type=bind"
+	// "source=/scratch/YOUR_USERNAME/data,target=/scratch/YOUR_USERNAME/data,type=bind",
+	// "source=/scratch/YOUR_USERNAME/cache,target=/scratch/YOUR_USERNAME/cache,type=bind"
 	// ],
+	"mounts": [
+		"source=${localEnv:HOME}/.netrc,target=/home/mamba/.netrc,type=bind", // this is to get WANDB_API_KEY from .netrc
+		"source=${localEnv:HOME}/.gitconfig,target=/home/mamba/.gitconfig,type=bind", // this is to get git credentials
+		"source=${localEnv:HOME}/.cache,target=/home/mamba/.cache,type=bind", // general cache directory
+	],
 	"remoteEnv": {
-		"WANDB_API_KEY": "${localEnv:WANDB_API_KEY}"
+		"WANDB_API_KEY": "${localEnv:WANDB_API_KEY}",
+		// "WANDB_DATA_DIR": "${localEnv:WANDB_DATA_DIR}",
+		// "WANDB_DIR": "${localEnv:WANDB_DIR}",
+		// "HF_DATASETS_CACHE": "${localEnv:HF_DATASETS_CACHE}",
+		// "XDG_CACHE_HOME": "${localEnv:XDG_CACHE_HOME}", // if you use a custom cache dir
 	},
+	////////////////////////////////////////////////////////////////////
+	// -----------------------------------------------------
+	// END CUSTOMIZATION OF ENV VARS, MOUNTS, RUN ARGS etc.
+	// -----------------------------------------------------
+	////////////////////////////////////////////////////////////////////
 	// Features to add to the dev container. More info: https://containers.dev/features.
-	"features": {
-		"ghcr.io/devcontainers/features/git:1.1.5": {},
-		"ghcr.io/devcontainers-contrib/features/flake8:2": {},
-		"ghcr.io/devcontainers-contrib/features/black:2": {},
-		"ghcr.io/devcontainers-contrib/features/isort:2": {}
-	},
+	// Doesn't work with centOS / RHEL base images, which we use
+	// "features": {
+	// "ghcr.io/devcontainers/features/git:1.1.5": {},
+	// },
 	"customizations": {
 		// Configure properties specific to VS Code.
 		"vscode": {
 			// Add the IDs of extensions you want installed when the container is created.
 			// These depend on your preferences, we just want to give our recommendations here.
 			"extensions": [
 				"ms-python.python",
+				"VisualStudioExptTeam.vscodeintellicode",
+				"ms-python.vscode-pylance",
 				"github.copilot",
-				"percy.vscode-pydata-viewer",
 				"kevinrose.vsc-python-indent",
+				"ms-python.black-formatter",
+				"charliermarsh.ruff",
+				"ms-azuretools.vscode-docker",
+				// nice to have below
+				"jgclark.vscode-todo-highlight",
+				"percy.vscode-pydata-viewer",
 				"njqdev.vscode-python-typehint",
-				"ms-python.flake8",
-				"foxundermoon.shell-format",
-				"ms-azuretools.vscode-docker"
+				"foxundermoon.shell-format"
 			]
 		}
 	},

.gitignore

@@ -1,5 +1,7 @@
-data/**/*
-
+data/*
+!data/mock/
+data/mock/tokenized/*
+logs/
 # experiment output logs
 wandb/
 artifacts/
@@ -136,8 +138,5 @@ dmypy.json
 # Pyre type checker
 .pyre/
 
-# data for testing
-/data
-
-# everything that has to do with run-results 
-/template
+# everything that has to do with run-results
+/template

Dockerfile

@@ -2,8 +2,8 @@
 
 # This Dockerfile produces a container with all dependencies installed into an environment called "research"
 # Additionally, the container has a full-fledged micromamba installation, which is a faster drop-in replacement for conda
-# When inside the container, you can install additional dependencies with `conda install <package>`, e.g. `conda install scipy`
-# The actual installation is done by micromamba, we have simply provided an alias to link the conda command to micromamba
+# When inside the container, you can install additional dependencies with `mamba install <package>`, e.g. `mamba install scipy`
+# The actual installation is done by micromamba, we have simply provided an alias to link the mamba command to micromamba
 
 # The syntax line above is crucial to enable variable expansion for type=cache=mount commands
 
@@ -23,7 +23,7 @@ FROM --platform=$TARGETPLATFORM mambaorg/micromamba:1.4.2 as micromamba
 # -----------------
 FROM --platform=linux/amd64 nvidia/cuda:11.8.0-cudnn8-runtime-ubi8 as amd64ubi8
 # Install compiler for .compile() with PyTorch 2.0 and nano for devcontainers
-RUN yum install -y gcc gcc-c++ nano && yum clean all
+RUN yum install -y git gcc gcc-c++ nano && yum clean all
 # Copy lockfile to container
 COPY conda-lock.yml /locks/conda-lock.yml
 
@@ -35,7 +35,7 @@ COPY conda-lock.yml /locks/conda-lock.yml
 # -----------------
 FROM --platform=linux/amd64 nvidia/cuda:11.8.0-cudnn8-runtime-ubuntu22.04 as amd64ubuntu
 # Install compiler for .compile() with PyTorch 2.0 and nano for devcontainers
-RUN apt-get update && apt-get install -y gcc g++ nano openssh-client && apt-get clean
+RUN apt-get update && apt-get install -y git gcc g++ nano openssh-client && apt-get clean
 # Copy lockfile to container
 COPY conda-lock.yml /locks/conda-lock.yml
 
@@ -111,8 +111,8 @@ RUN chown $MAMBA_USER:$MAMBA_USER /usr/bin/gcc
 # Necessary to prevent permission error when micromamba tries to install pip dependencies from lockfile
 RUN chown $MAMBA_USER:$MAMBA_USER /locks/
 RUN chown $MAMBA_USER:$MAMBA_USER /locks/conda-lock.yml
-# Provide conda alias for micromamba
-RUN echo "alias conda=micromamba" >> /usr/local/bin/_activate_current_env.sh
+# Provide mamba alias for micromamba
+RUN echo "alias mamba=micromamba" >> /usr/local/bin/_activate_current_env.sh
 
 # Give permission to everyone for e.g. caching
 RUN mkdir /home/mamba/.cache && chmod -R 777 /home/mamba/.cache/