Skip to content

Commit

Permalink
Update fork-maintenance.yml
Browse files Browse the repository at this point in the history
  • Loading branch information
@Cemberk
Cemberk authored Nov 12, 2024
1 parent 8d36a70 commit 673bb7e
Showing 1 changed file with 15 additions and 318 deletions.
333 changes: 15 additions & 318 deletions .github/workflows/fork-maintenance.yml
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: Run Scheduled Events
name: Run Scheduled Events Docker

permissions:
actions: write
Expand All @@ -9,323 +9,20 @@ permissions:
on:
workflow_dispatch:
schedule:
- cron: '0 0 * * *'
- cron: '0 0 10 * *'

jobs:
run-scheduled-events:
runs-on: ubuntu-latest
steps:
- name: Fork Maintenance System
uses: Cemberk/Fork-Maintenance-System@conda
with:
github_token: ${{ secrets.CRED_TOKEN }}
upstream_repo: "https://github.com/huggingface/transformers"
schedule_json: |
{
"upstream_main_branch": "main",
"upstream_release_branch": "v4.43-release",
"downstream_main_branch": "upstream_sync",
"downstream_testing_branch": "rocm6.3_testing_rel4.43_testing",
"downstream_develop_branch": "develop",
"commits": [
"731f0308bb0a2796e28b68664f4ed050eab6dbfd",
"0c1b311bff616a4faf214461584a758fbab14a6f",
"5b2d4fec5abb8ffe755fe66ce0856fc066c561a6"
]
}
pr_branch_prefix: "scheduled-merge"
conda_environment: |
name: my_new_env
channels:
- defaults
dependencies:
- _libgcc_mutex=0.1=main
- _openmp_mutex=5.1=1_gnu
- ca-certificates=2024.7.2=h06a4308_0
- ld_impl_linux-64=2.38=h1181459_1
- libffi=3.4.4=h6a678d5_1
- libgcc-ng=11.2.0=h1234567_1
- libgomp=11.2.0=h1234567_1
- libstdcxx-ng=11.2.0=h1234567_1
- ncurses=6.4=h6a678d5_0
- openssl=3.0.14=h5eee18b_0
- pip=24.2=py39h06a4308_0
- python=3.9.19=h955ad1f_1
- readline=8.2=h5eee18b_0
- setuptools=72.1.0=py39h06a4308_0
- sqlite=3.45.3=h5eee18b_0
- tk=8.6.14=h39e8969_0
- wheel=0.43.0=py39h06a4308_0
- xz=5.4.6=h5eee18b_1
- zlib=1.2.13=h5eee18b_1
- pip:
- absl-py==2.1.0
- accelerate==0.33.0
- adal==1.2.7
- aiohappyeyeballs==2.3.4
- aiohttp==3.10.0
- aiosignal==1.3.1
- albucore==0.0.14
- albumentations==1.4.14
- altair==5.4.1
- annotated-types==0.7.0
- antlr4-python3-runtime==4.9.3
- argcomplete==3.5.0
- array-record==0.5.1
- arrow==1.3.0
- asgiref==3.8.1
- astunparse==1.6.3
- async-timeout==4.0.3
- attrs==23.2.0
- audioread==3.0.1
- av==9.2.0
- azure-common==1.1.28
- azure-core==1.30.2
- azure-graphrbac==0.61.1
- azure-mgmt-authorization==4.0.0
- azure-mgmt-containerregistry==10.3.0
- azure-mgmt-core==1.4.0
- azure-mgmt-keyvault==10.3.1
- azure-mgmt-network==26.0.0
- azure-mgmt-resource==23.1.1
- azure-mgmt-storage==21.2.1
- azureml==0.2.7
- azureml-core
- backports-tempfile==1.0
- backports-weakref==1.0.post1
- bcrypt==4.2.0
- beautifulsoup4==4.12.3
- binaryornot==0.4.4
- black==24.8.0
- blinker==1.8.2
- boto3==1.26.0
- botocore==1.29.8
- cachetools==5.4.0
- cerberus==1.3.5
- certifi==2024.7.4
- cffi==1.16.0
- chardet==5.2.0
- charset-normalizer==3.3.2
- clang==17.0.6
- click==8.1.7
- cloudpickle==3.0.0
- colorama==0.4.6
- conllu==5.0.1
- contextlib2==21.6.0
- contourpy==1.3.0
- cookiecutter==1.7.3
- coremltools==5.0b5
- cryptography==43.0.0
- cycler==0.12.1
- cython==3.0.10
- datasets==2.21.0
- decorator==5.1.1
- decord==0.6.0
- dill==0.3.4
- django==4.2.14
- dm-tree==0.1.8
- docker==7.1.0
- elastic-transport
- elasticsearch==8.15.0
- etils==1.5.2
- eval-type-backport==0.2.0
- evaluate==0.4.2
- exceptiongroup==1.2.2
- execnet==2.1.1
- expecttest==0.1.6
- faiss-cpu==1.8.0.post1
- filelock==3.13.1
- fire==0.6.0
- flatbuffers==2.0
- fonttools==4.53.1
- frozenlist==1.4.1
- fsspec==2024.6.1
- future==1.0.0
- fvcore==0.1.5.post20221221
- geojson==2.5.0
- ghstack==0.8.0
- git-python==1.0.3
- gitdb==4.0.11
- gitpython==3.1.18
- google-auth==2.32.0
- google-auth-oauthlib==1.0.0
- gputil==1.4.0
- grpcio==1.65.4
- huggingface-hub==0.24.6
- humanfriendly==10.0
- hydra-core==1.3.2
- hypothesis==5.35.1
- idna==3.7
- image==1.5.33
- imageio==2.34.2
- importlib-metadata==8.2.0
- importlib-resources==6.4.4
- iniconfig==2.0.0
- iopath==0.1.9
- isodate==0.6.1
- itsdangerous==2.0.1
- jeepney==0.8.0
- jinja2==3.1.3
- jinja2-time==0.2.0
- jiwer==3.0.4
- jmespath==0.10.0
- joblib==1.4.2
- jsonpickle==3.2.2
- jsonschema==4.23.0
- jsonschema-specifications==2023.12.1
- junitparser==2.1.1
- kiwisolver==1.4.5
- knack==0.11.0
- lark==0.12.0
- lazy-loader==0.4
- librosa==0.10.2.post1
- lightning-utilities==0.11.6
- lintrunner==0.10.7
- llvmlite==0.43.0
- lxml==5.0.0
- markdown==3.6
- markdown-it-py==3.0.0
- markupsafe==2.1.5
- matplotlib==3.9.2
- mdurl==0.1.2
- mpmath==1.3.0
- msal==1.30.0
- msal-extensions==1.2.0
- msgpack==1.0.8
- msrest==0.7.1
- msrestazure==0.6.4.post1
- multidict==6.0.5
- multiprocess==0.70.12.2
- mypy==1.9.0
- mypy-extensions==1.0.0
- narwhals==1.5.5
- ndg-httpsclient==0.5.1
- networkx==2.8.8
- ninja==1.11.1.1
- nltk==3.9.1
- numba==0.60.0
- numpy==1.26.4
- oauthlib==3.2.2
- omegaconf==2.3.0
- opencv-python-headless==4.10.0.84
- opt-einsum==3.3.0
- optionloop==1.0.7
- optree==0.11.0
- packaging==24.1
- pandas==2.2.2
- parameterized==0.9.0
- paramiko==3.4.1
- pathspec==0.12.1
- pillow==10.2.0
- pkginfo==1.11.1
- platformdirs==4.2.2
- pluggy==1.5.0
- pooch==1.8.2
- portalocker==2.0.0
- poyo==0.5.0
- promise==2.3
- protobuf==3.20.3
- psutil==6.0.0
- pyarrow==17.0.0
- pyasn1==0.6.0
- pyasn1-modules==0.4.0
- pycocotools==2.0.8
- pycparser==2.22
- pydantic==2.8.2
- pydantic-core==2.20.1
- pydeck==0.9.1
- pygments==2.15.0
- pyjwt==2.9.0
- pynacl==1.5.0
- pyopenssl==24.2.1
- pyparsing==3.1.4
- pysocks==1.7.1
- pytesseract==0.3.13
- pytest==7.3.2
- pytest-cpp==2.3.0
- pytest-flakefinder==1.1.0
- pytest-rerunfailures==14.0
- pytest-rich==0.1.1
- pytest-timeout==2.3.1
- pytest-xdist==3.3.1
- python-dateutil==2.9.0.post0
- python-slugify==8.0.4
- pytz==2024.1
- pywavelets==1.4.1
- pyyaml==6.0.2
- rapidfuzz==3.9.6
- referencing==0.35.1
- regex==2024.7.24
- requests==2.32.2
- requests-oauthlib==2.0.0
- rich==13.8.0
- rjieba==0.1.11
- rockset==1.0.3
- rouge-score==0.1.2
- rpds-py==0.20.0
- rsa==4.9
- ruff==0.4.4
- s3transfer==0.5.2
- sacrebleu==1.5.1
- sacremoses==0.1.1
- safetensors==0.4.4
- scikit-image==0.24.0
- scikit-learn==1.5.1
- scipy==1.13.1
- secretstorage==3.3.3
- sentencepiece==0.2.0
- seqeval==1.2.2
- six==1.16.0
- smmap==5.0.1
- sortedcontainers==2.4.0
- soundfile==0.12.1
- soupsieve==2.6
- soxr==0.4.0
- sqlparse==0.5.1
- streamlit==1.38.0
- sympy==1.12.1
- tabulate==0.9.0
- tb-nightly==2.13.0a20230426
- tenacity==8.5.0
- tensorboard==2.13.0
- tensorboard-data-server==0.7.2
- tensorflow-datasets==4.9.3
- tensorflow-metadata==1.15.0
- termcolor==2.4.0
- text-unidecode==1.3
- threadpoolctl==3.5.0
- tifffile==2024.7.24
- timeout-decorator==0.5.0
- timm==1.0.9
- tokenizers
- toml==0.10.2
- tomli==2.0.1
- torch==2.4.0 --extra-index-url https://download.pytorch.org/whl/rocm6.1
- torchaudio==2.4.0 --extra-index-url https://download.pytorch.org/whl/rocm6.1
- torchvision==0.19.0 --extra-index-url https://download.pytorch.org/whl/rocm6.1
- tornado==6.4.1
- tqdm==4.66.4
- transformers==4.43.0
- triton==3.0.0
- types-python-dateutil==2.9.0.20240821
- typing-extensions==4.12.2
- tzdata==2024.1
- unittest-xml-reporting==3.2.0
- urllib3==1.26
- watchdog==4.0.2
- werkzeug==3.0.3
- wrapt==1.16.0
- xdoctest==1.1.0
- xxhash==3.5.0
- yacs==0.1.8
- yarl==1.9.4
- z3-solver==4.12.2.0
- zipp==3.19.2
prefix: /opt/conda/envs/my_new_env
unit_test_command: >
cd tests; folders=($(python3 -c 'import os; tests = os.getcwd(); model_tests = os.listdir(os.path.join(tests, "models")); d1 = sorted(list(filter(os.path.isdir, os.listdir(tests)))); d2 = sorted(list(filter(os.path.isdir, [f"models/{x}" for x in model_tests]))); d1.remove("models"); d = d2 + d1; print("\n".join(d))')); cd ..; for folder in "${folders[@]}"; do pytest tests/${folder} -v --make-reports="huggingface_unit_tests_run_models_gpu_${folder}" -rfEsx --continue-on-collection-errors -m "not not_device_test" -p no:cacheprovider; done; allstats=($(find reports -name stats.txt)); for stat in "${allstats[@]}"; do echo "$stat"; cat $stat; done
performance_test_command: >
python examples/pytorch/language-modeling/run_mlm.py --model_name_or_path bert-base-uncased --dataset_name wikitext --dataset_config_name wikitext-2-raw-v1 --do_train --do_eval --output_dir /tmp/test-mlm --per_device_train_batch_size 8 --per_device_eval_batch_size 8 --max_steps 500
extra_install_command: >
pip install tokenizers==0.19
uses: Cemberk/fork-maintenance-system/.github/workflows/fork-maintenance-action.yml@artifacts
with:
platform: 'gfx90a'
upstream_repo: 'https://github.com/huggingface/transformers'
pr_branch_prefix: 'scheduled-merge'
requirements_command: 'sudo sed -i \"s/torchaudio//g\" examples/pytorch/_tests_requirements.txt && pip install -r examples/pytorch/_tests_requirements.txt && git restore examples/pytorch/_tests_requirements.txt && pip install --no-cache-dir GPUtil azureml azureml-core tokenizers ninja cerberus sympy sacremoses sacrebleu==1.5.1 sentencepiece scipy scikit-learn urllib3 && pip install huggingface_hub datasets && pip install parameterized && pip install -e .'
unit_test_command: cd tests; folders=\$(python3 -c \import os; tests = os.getcwd(); models = \"models\"; model_tests = os.listdir(os.path.join(tests, models)); d1 = sorted(list(filter(os.path.isdir, os.listdir(tests)))); d2 = sorted(list(filter(os.path.isdir, [os.path.join(models, x) for x in model_tests]))); d1.remove(models); d = d2 + d1; print(\" \".join(d[:5]))' ); cd ..; for folder in \${folders[@]}; do pytest tests/\${folder} -v --make-reports=huggingface_unit_tests_\${machine_type}_run_models_gpu_\${folder} -rfEs --continue-on-collection-errors -m \"not not_device_test\" -p no:cacheprovider; done; allstats=\$(find reports -name stats.txt); for stat in \${allstats[@]}; do echo \$stat; cat \$stat; done
performance_test_command: 'echo \"python examples/pytorch/language-modeling/run_mlm.py --model_name_or_path bert-base-uncased --dataset_name wikitext --dataset_config_name wikitext-2-raw-v1 --do_train --do_eval --output_dir /tmp/test-mlm --per_device_train_batch_size 8 --per_device_eval_batch_size 8 --max_steps 500\"'
docker_image: 'rocm/pytorch:latest'
docker_options: '--device=/dev/kfd --device=/dev/dri --group-add video --shm-size 16G --network=host'
secrets:
GIT_TOKEN: ${{ secrets.CRED_TOKEN }}
schedule_json: ${{ secrets.SCHEDULE_CONFIG }}

0 comments on commit 673bb7e

Please sign in to comment.