Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Upgraded self supervised pipeline to fairseq version 0.12.2 #3

Open
wants to merge 13 commits into
base: master
Choose a base branch
from
Open

Upgraded self supervised pipeline to fairseq version 0.12.2 #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
fc9ce84
working pipeline
rrichajalota Mar 13, 2023
b0a6173
added conda env file
rrichajalota Mar 13, 2023
a933f73
fixed error at the end of last epoch
rrichajalota Mar 14, 2023
2ae389b
slurm scripts for bart pretraining, fix fine-tuning errors
rrichajalota Mar 15, 2023
6e89986
added evaluation, get_bt data scripts
rrichajalota Apr 24, 2023
3b5db4b
unsupervised tuning, faiss support for 5M index
rrichajalota May 31, 2023
5292936
fixed nll_loss_sum in reduce_metrics()
rrichajalota May 31, 2023
bcfb2ae
added modified train file for unsupervised tuning, scripts for prepar…
rrichajalota May 31, 2023
9190a9b
emnlp_submission
rrichajalota Jun 23, 2023
99516ee
Update README.md
rrichajalota Jun 24, 2023
7d7e028
Update README.md
rrichajalota Jun 24, 2023
0bb5b98
Update README.md
rrichajalota Jun 24, 2023
2f2705a
thesis code
rrichajalota Aug 2, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
334 changes: 128 additions & 206 deletions README.md
View file
Open in desktop

Large diffs are not rendered by default.

251 changes: 251 additions & 0 deletions environment.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,251 @@
name: fsq012
channels:
- pytorch
- nvidia
- conda-forge
- defaults
dependencies:
- _libgcc_mutex=0.1=main
- _openmp_mutex=5.1=1_gnu
- binutils_impl_linux-64=2.33.1=he6710b0_7
- binutils_linux-64=2.33.1=h9595d00_15
- blas=1.0=mkl
- brotlipy=0.7.0=py38h27cfd23_1003
- bzip2=1.0.8=h7b6447c_0
- ca-certificates=2023.5.7=hbcca054_0
- certifi=2023.5.7=pyhd8ed1ab_0
- cffi=1.15.1=py38h5eee18b_3
- charset-normalizer=2.0.4=pyhd3eb1b0_0
- cryptography=39.0.1=py38h9ce1e76_0
- cuda=11.7.1=0
- cuda-cccl=11.7.91=0
- cuda-command-line-tools=11.7.1=0
- cuda-compiler=11.7.1=0
- cuda-cudart=11.7.99=0
- cuda-cudart-dev=11.7.99=0
- cuda-cuobjdump=11.7.91=0
- cuda-cupti=11.7.101=0
- cuda-cuxxfilt=11.7.91=0
- cuda-demo-suite=12.1.55=0
- cuda-documentation=12.1.55=0
- cuda-driver-dev=11.7.99=0
- cuda-gdb=12.1.55=0
- cuda-libraries=11.7.1=0
- cuda-libraries-dev=11.7.1=0
- cuda-memcheck=11.8.86=0
- cuda-nsight=12.1.55=0
- cuda-nsight-compute=12.1.0=0
- cuda-nvcc=11.7.99=0
- cuda-nvdisasm=12.1.55=0
- cuda-nvml-dev=11.7.91=0
- cuda-nvprof=12.1.55=0
- cuda-nvprune=11.7.91=0
- cuda-nvrtc=11.7.99=0
- cuda-nvrtc-dev=11.7.99=0
- cuda-nvtx=11.7.91=0
- cuda-nvvp=12.1.55=0
- cuda-runtime=11.7.1=0
- cuda-sanitizer-api=12.1.55=0
- cuda-toolkit=11.7.1=0
- cuda-tools=11.7.1=0
- cuda-visual-tools=11.7.1=0
- cudatoolkit=11.7.0=hd8887f6_10
- faiss=1.7.2=py38cuda112h5d0fea0_0_cuda
- faiss-gpu=1.7.2=h788eb59_3
- ffmpeg=4.3=hf484d3e_0
- flit-core=3.6.0=pyhd3eb1b0_0
- freetype=2.12.1=h4a9f257_0
- gcc_impl_linux-64=7.3.0=habb00fd_1
- gcc_linux-64=7.3.0=h553295d_15
- gds-tools=1.6.0.25=0
- giflib=5.2.1=h5eee18b_3
- gmp=6.2.1=h295c915_3
- gnutls=3.6.15=he1e5248_0
- gxx_impl_linux-64=7.3.0=hdf63c60_1
- gxx_linux-64=7.3.0=h553295d_15
- idna=3.4=py38h06a4308_0
- intel-openmp=2021.4.0=h06a4308_3561
- jpeg=9e=h5eee18b_1
- lame=3.100=h7b6447c_0
- lcms2=2.12=h3be6417_0
- ld_impl_linux-64=2.33.1=h53a641e_7
- lerc=3.0=h295c915_0
- libblas=3.9.0=12_linux64_mkl
- libcublas=11.10.3.66=0
- libcublas-dev=11.10.3.66=0
- libcufft=10.7.2.124=h4fbf590_0
- libcufft-dev=10.7.2.124=h98a8f43_0
- libcufile=1.6.0.25=0
- libcufile-dev=1.6.0.25=0
- libcurand=10.3.2.56=0
- libcurand-dev=10.3.2.56=0
- libcusolver=11.4.0.1=0
- libcusolver-dev=11.4.0.1=0
- libcusparse=11.7.4.91=0
- libcusparse-dev=11.7.4.91=0
- libdeflate=1.17=h5eee18b_0
- libfaiss=1.7.2=cuda112hc9ed507_0_cuda
- libfaiss-avx2=1.7.2=cuda112h1234567_0_cuda
- libffi=3.4.2=h6a678d5_6
- libgcc=7.2.0=h69d50b8_2
- libgcc-ng=11.2.0=h1234567_1
- libgomp=11.2.0=h1234567_1
- libiconv=1.16=h7f8727e_2
- libidn2=2.3.2=h7f8727e_0
- liblapack=3.9.0=12_linux64_mkl
- libnpp=11.7.4.75=0
- libnpp-dev=11.7.4.75=0
- libnvjpeg=11.8.0.2=0
- libnvjpeg-dev=11.8.0.2=0
- libpng=1.6.39=h5eee18b_0
- libstdcxx-ng=11.2.0=h1234567_1
- libtasn1=4.16.0=h27cfd23_0
- libtiff=4.5.0=h6a678d5_2
- libunistring=0.9.10=h27cfd23_0
- libwebp=1.2.4=h11a3e52_1
- libwebp-base=1.2.4=h5eee18b_1
- lz4-c=1.9.4=h6a678d5_0
- mkl=2021.4.0=h06a4308_640
- mkl-service=2.4.0=py38h7f8727e_0
- mkl_fft=1.3.1=py38hd3c417c_0
- mkl_random=1.2.2=py38h51133e4_0
- ncurses=6.4=h6a678d5_0
- nettle=3.7.3=hbbd107a_1
- nsight-compute=2023.1.0.15=0
- numpy-base=1.23.5=py38h31eccc5_0
- openh264=2.1.1=h4ff587b_0
- openssl=1.1.1t=h7f8727e_0
- pillow=9.4.0=py38h6a678d5_0
- pycparser=2.21=pyhd3eb1b0_0
- pyopenssl=23.0.0=py38h06a4308_0
- pysocks=1.7.1=py38h06a4308_0
- python=3.8.16=h7a1cb2a_2
- python_abi=3.8=2_cp38
- pytorch=1.13.1=py3.8_cuda11.7_cudnn8.5.0_0
- pytorch-cuda=11.7=h67b0de4_1
- pytorch-mutex=1.0=cuda
- readline=8.2=h5eee18b_0
- requests=2.28.1=py38h06a4308_0
- six=1.16.0=pyhd3eb1b0_1
- sqlite=3.40.1=h5082296_0
- tk=8.6.12=h1ccaba5_0
- torchvision=0.14.1=py38_cu117
- typing_extensions=4.4.0=py38h06a4308_0
- urllib3=1.26.14=py38h06a4308_0
- xz=5.2.10=h5eee18b_1
- zlib=1.2.13=h5eee18b_0
- zstd=1.5.2=ha4553b6_0
- pip:
- accelerate==0.19.0
- aiohttp==3.8.4
- aiosignal==1.3.1
- antlr4-python3-runtime==4.8
- apex==0.1
- appdirs==1.4.4
- async-timeout==4.0.2
- attrs==22.2.0
- bert-score==0.3.13
- bitarray==2.7.3
- blis==0.7.9
- catalogue==2.0.8
- click==8.1.3
- colorama==0.4.6
- confection==0.0.4
- contourpy==1.0.7
- cycler==0.11.0
- cymem==2.0.7
- cython==0.29.33
- datasets==2.10.1
- de-dep-news-trf==3.5.0
- deepspeed==0.8.2
- dill==0.3.6
- docker-pycreds==0.4.0
- easynmt==2.0.2
- en-core-web-trf==3.5.0
- evaluate==0.4.0
- fairscale==0.4.13
- fairseq==0.12.2
- fastbpe==0.1.0
- fasttext==0.9.2
- filelock==3.9.1
- fonttools==4.39.4
- frozenlist==1.3.3
- fsspec==2023.3.0
- gensim==4.3.1
- gitdb==4.0.10
- gitpython==3.1.31
- hjson==3.1.0
- huggingface-hub==0.13.2
- hydra-core==1.0.7
- importlib-resources==5.12.0
- jinja2==3.1.2
- joblib==1.2.0
- kiwisolver==1.4.4
- langcodes==3.3.0
- langdetect==1.0.9
- lxml==4.9.2
- markupsafe==2.1.3
- matplotlib==3.7.1
- multidict==6.0.4
- multiprocess==0.70.14
- murmurhash==1.0.9
- ninja==1.11.1
- nltk==3.8.1
- numpy==1.24.2
- nvidia-cublas-cu11==11.10.3.66
- nvidia-cuda-nvrtc-cu11==11.7.99
- nvidia-cuda-runtime-cu11==11.7.99
- nvidia-cudnn-cu11==8.5.0.96
- omegaconf==2.0.6
- packaging==23.0
- pandas==1.5.3
- pathtools==0.1.2
- pathy==0.10.1
- pip==23.1.2
- portalocker==2.7.0
- preshed==3.0.8
- protobuf==4.22.1
- psutil==5.9.4
- py-cpuinfo==9.0.0
- pyarrow==11.0.0
- pybind11==2.10.4
- pydantic==1.10.6
- pyparsing==3.0.9
- python-dateutil==2.8.2
- pytz==2022.7.1
- pyyaml==6.0
- regex==2022.10.31
- responses==0.18.0
- sacrebleu==2.3.1
- sacremoses==0.0.53
- scikit-learn==1.2.1
- scipy==1.10.1
- sentencepiece==0.1.97
- sentry-sdk==1.25.0
- setproctitle==1.3.2
- setuptools==67.8.0
- smart-open==6.3.0
- smmap==5.0.0
- spacy==3.5.3
- spacy-alignments==0.9.0
- spacy-legacy==3.0.12
- spacy-loggers==1.0.4
- spacy-transformers==1.2.4
- srsly==2.4.6
- tabulate==0.9.0
- thinc==8.1.10
- threadpoolctl==3.1.0
- tokenizers==0.13.2
- torch==1.13.1
- torchaudio==0.13.1
- tqdm==4.64.1
- transformers==4.27.0
- typer==0.7.0
- typing-extensions==4.5.0
- wandb==0.15.3
- wasabi==1.1.2
- wheel==0.40.0
- xxhash==3.2.0
- yarl==1.8.2
- zipp==3.15.0
prefix: /netscratch/jalota/miniconda3/envs/fsq012
Loading