From a71aaa8b0b62bd7fe8831533be65ff80441f6cf9 Mon Sep 17 00:00:00 2001
From: Samuel Boehm <samuel-boehm@web.de>
Date: Wed, 11 Dec 2024 19:31:58 +0100
Subject: [PATCH] handle pagination - needed for Stieger2021 ds

---
 moabb/datasets/download.py | 33 +++++++++++++++++++++------------
 1 file changed, 21 insertions(+), 12 deletions(-)

diff --git a/moabb/datasets/download.py b/moabb/datasets/download.py
index 4684a89fe..48507203c 100644
--- a/moabb/datasets/download.py
+++ b/moabb/datasets/download.py
@@ -206,30 +206,39 @@ def fs_issue_request(method, url, headers, data=None, binary=False):
 
 def fs_get_file_list(article_id, version=None):
     """List all the files associated with a given article.
-
     Parameters
     ----------
     article_id : str or int
         Figshare article ID
     version : str or id, default is None
         Figshare article version. If None, selects the most recent version.
-
     Returns
     -------
     response : dict
         HTTP request response as a python dict
     """
     fsurl = "https://api.figshare.com/v2"
-    if version is None:
-        url = fsurl + "/articles/{}/files".format(article_id)
-        headers = {"Content-Type": "application/json"}
-        response = fs_issue_request("GET", url, headers=headers)
-        return response
-    else:
-        url = fsurl + "/articles/{}/versions/{}".format(article_id, version)
-        headers = {"Content-Type": "application/json"}
-        request = fs_issue_request("GET", url, headers=headers)
-        return request["files"]
+    all_files = []
+    page = 1
+    
+    while True:
+        if version is None:
+            url = f"{fsurl}/articles/{article_id}/files?page={page}&page_size=100"
+            headers = {"Content-Type": "application/json"}
+            response = fs_issue_request("GET", url, headers=headers)
+            
+            if not response:  # If response is empty, we've got all files
+                break
+                
+            all_files.extend(response)
+            page += 1
+        else:
+            url = f"{fsurl}/articles/{article_id}/versions/{version}"
+            headers = {"Content-Type": "application/json"}
+            request = fs_issue_request("GET", url, headers=headers)
+            return request["files"]
+            
+    return all_files
 
 
 def fs_get_file_hash(filelist):