Fix dataset caching in CI (#5)

* Create DATA_PATH variable and adjust CI * Install package before getting files * Use the same key for all caches * Run ubuntu first * Forgot fi * Forgot another fi, rename jobs * Rename depends * Cache after running tests * Use restore cache * Remove second cache * Undo DATA_PATH and update checkout in CI * Use other check for OS * Set env differently per dataset * Use apostrophe
ScaDS · Mar 27, 2023 · 6706d3e · 6706d3e
1 parent f91fb3b
commit 6706d3e
Show file tree

Hide file tree

Showing 2 changed files with 44 additions and 13 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -14,17 +14,43 @@ on:
 
 
 jobs:
-  build:
+  ubuntu:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install nox
+    - name: Set pystow dir
+      run: |
+            echo "PYSTOW_HOME=" >> $GITHUB_ENV
+            echo "PYSTOW_NAME=data" >> $GITHUB_ENV
+    - uses: actions/cache@v3
+      id: cache
+      with:
+        path: data
+        key: alwaysthesame
+        enableCrossOsArchive: True
+    - name: Test
+      run:
+        nox
 
+  other:
+    needs: ubuntu
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.7, '3.11']
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: [3.7]
+        os: [macos-latest, windows-latest]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v2
       with:
@@ -33,16 +59,22 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         python -m pip install nox
-    - name: Set PYTHONPATH
+    - name: Set pystow dir windows
+      if: runner.os == 'Windows'
+      run: |
+            echo "PYSTOW_HOME=" >> $env:GITHUB_ENV
+            echo "PYSTOW_NAME=data" >> $env:GITHUB_ENV
+    - name: Set pystow dir mac
+      if: runner.os == 'macOS'
       run: |
-        echo "PYTHONPATH=src" >> $GITHUB_ENV
-        echo "${{ env.PYTHONPATH }}"
-    - name: Cache dataset downloads
-      uses: actions/cache@v3
+            echo "PYSTOW_HOME=" >> $GITHUB_ENV
+            echo "PYSTOW_NAME=data" >> $GITHUB_ENV
+    - uses: actions/cache/restore@v3
+      id: cache
       with:
-        # datasets are stored in `~/.data/moviegraphbenchmark/data`
-        path: ~/.data/moviegraphbenchmark/data
-        key: ${{ runner.os }}-build-${{ matrix.python-version }}
+        path: data
+        key: alwaysthesame
+        enableCrossOsArchive: True
     - name: Test
       run:
         nox
diff --git a/src/moviegraphbenchmark/create_graph.py b/src/moviegraphbenchmark/create_graph.py
@@ -1,7 +1,6 @@
 import ast
 import logging
 import os
-import sys
 import zipfile
 from typing import List, Set, Tuple