run with py

huggingface · Sep 5, 2024 · d5cfaa5 · d5cfaa5
1 parent bbe1c29
commit d5cfaa5
Show file tree

Hide file tree

Showing 2 changed files with 47 additions and 21 deletions.
diff --git a/.github/workflows/model_jobs.yml b/.github/workflows/model_jobs.yml
@@ -114,29 +114,33 @@ jobs:
         working-directory: /transformers
         run: rm -rf /mnt/cache/datasets/downloads/0f3877f8b8ff8f882b5032f93535a4d21bd7475946970fbf306f79c2769f3d0d.incomplete
 
-      - name: Create dir
+      - name: Run python script
         working-directory: /transformers
-        run: |
-          python3 -c 'import os; lpath = "/mnt/cache/datasets/downloads/0f3877f8b8ff8f882b5032f93535a4d21bd7475946970fbf306f79c2769f3d0d.incomplete"; os.makedirs(os.path.dirname(lpath), exist_ok=True)'
-
-      - name: Show dir
-        working-directory: /transformers
-        run: |
-          ls -l /mnt/cache/datasets/downloads/
+        run: python3 my_script.py
 
-      - name: Remove cache 1
-        working-directory: /transformers
-        run: |
-          python3 -c 'lpath = "/mnt/cache/datasets/downloads/0f3877f8b8ff8f882b5032f93535a4d21bd7475946970fbf306f79c2769f3d0d.incomplete"; outfile = open(lpath, "wb"); print(outfile)'
-
-      - name: Show file
-        working-directory: /transformers
-        run: |
-          ls -l /mnt/cache/datasets/downloads/0f3877f8b8ff8f882b5032f93535a4d21bd7475946970fbf306f79c2769f3d0d.incomplete
-
-      - name: Run all tests on GPU
-        working-directory: /transformers
-        run: python3 -m pytest tests/${{ matrix.folders }} -k "ASTFeatureExtractionTest and test_integration"
+#      - name: Create dir
+#        working-directory: /transformers
+#        run: |
+#          python3 -c 'import os; lpath = "/mnt/cache/datasets/downloads/0f3877f8b8ff8f882b5032f93535a4d21bd7475946970fbf306f79c2769f3d0d.incomplete"; os.makedirs(os.path.dirname(lpath), exist_ok=True)'
+#
+#      - name: Show dir
+#        working-directory: /transformers
+#        run: |
+#          ls -l /mnt/cache/datasets/downloads/
+#
+#      - name: Remove cache 1
+#        working-directory: /transformers
+#        run: |
+#          python3 -c 'lpath = "/mnt/cache/datasets/downloads/0f3877f8b8ff8f882b5032f93535a4d21bd7475946970fbf306f79c2769f3d0d.incomplete"; outfile = open(lpath, "wb"); print(outfile)'
+#
+#      - name: Show file
+#        working-directory: /transformers
+#        run: |
+#          ls -l /mnt/cache/datasets/downloads/0f3877f8b8ff8f882b5032f93535a4d21bd7475946970fbf306f79c2769f3d0d.incomplete
+#
+#      - name: Run all tests on GPU
+#        working-directory: /transformers
+#        run: python3 -m pytest tests/${{ matrix.folders }} -k "ASTFeatureExtractionTest and test_integration"
 
 #        run: python3 -m pytest -rsfE -v --make-reports=${{ inputs.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports tests/${{ matrix.folders }}
 

diff --git a/my_script.py b/my_script.py
@@ -0,0 +1,22 @@
+import os
+from pathlib import Path
+
+
+def get_file(self, rpath, lpath, callback=_DEFAULT_CALLBACK, outfile=None, **kwargs) -> None:
+    # # Taken from https://github.com/fsspec/filesystem_spec/blob/47b445ae4c284a82dd15e0287b1ffc410e8fc470/fsspec/spec.py#L883
+    # if isfilelike(lpath):
+    #     outfile = lpath
+    # elif self.isdir(rpath):
+    #     os.makedirs(lpath, exist_ok=True)
+    #     return None
+
+    if isinstance(lpath, (str, Path)):  # otherwise, let's assume it's a file-like object
+        os.makedirs(os.path.dirname(lpath), exist_ok=True)
+        # import time
+        # time.sleep(10)
+
+    # Open file if not already open
+    close_file = False
+    if outfile is None:
+        outfile = open(lpath, "wb")
+        print(outfile)