Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix pylint errors #4

Merged
merged 5 commits into from
Jul 27, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/doit.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ on:
- cron: "33 18 * * *"

env:
PYTHON_VERSION: "3.10"
PYTHON_VERSION: "graalpy-24"

permissions:
contents: read
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/pull.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: do it
name: pull request lint and test

on:
pull_request:
Expand Down Expand Up @@ -47,6 +47,6 @@ jobs:
flake8 . --count --max-complexity=10 --max-line-length=127 --statistics

- name: Run tests
if: true
if: false # TODO no unittests for now
run: |
pytest
50 changes: 17 additions & 33 deletions main.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,12 @@
import argparse
import logging
import pathlib
import shutil
from pathlib import Path

from pathvalidate import FileNameValidator
from sortedcontainers import SortedSet
from tempfile import TemporaryDirectory
from kaggle.api.kaggle_api_extended import KaggleApi

import pathvalidate
from kaggle.api.kaggle_api_extended import KaggleApi
from sortedcontainers import SortedSet

MAX_PAGE_SIZE = 100

Expand Down Expand Up @@ -46,13 +44,9 @@ def validate_filename(value):


def get_kernels(api, user, page=1, include_private=True, page_size=MAX_PAGE_SIZE):
kernels = api.kernels_list(
page=page,
user=user or api.get_config_value(api.CONFIG_NAME_USER) or api.api_client.configuration.username,
sort_by="dateRun",
page_size=page_size,
mine=True
)
kernels = api.kernels_list(page=page,
user=user or api.get_config_value(api.CONFIG_NAME_USER),
sort_by="dateRun", page_size=page_size, mine=True)
if not include_private:
return [k for k in kernels if not getattr(k, 'isPrivate', getattr(k, 'isPrivateNullable'))]
else:
Expand All @@ -70,26 +64,16 @@ def kernel_to_path(kernel):
def main(include_private=True, max_page_size=MAX_PAGE_SIZE, user=None, output_name="kernels.zip",
tmp_dir_prefix="kaggle_", tmp_dir=None):
parser = argparse.ArgumentParser(description="Download All Kaggle Kernels")
parser.add_argument(
"-o", "--output", type=validate_filename, default=output_name,
help="Name of the output zip file (default: kernels.zip)"
)
parser.add_argument(
"-p", "--include-private", action="store_true", default=include_private,
help="Include private kernels in the download (default: True)"
)
parser.add_argument(
"-u", "--user", type=str, default=user,
help="Username of the Kaggle user to search kernels for (default: current user)"
)
parser.add_argument(
"-s", "--max-page-size", type=validate_positive_int, default=max_page_size,
help="Maximum number of kernels to download per page (default: 100)"
)
parser.add_argument(
"-t", "--tmp-dir", type=str, default=tmp_dir,
help="Path to the temporary directory (default: None)"
)
parser.add_argument("-o", "--output", type=validate_filename, default=output_name,
help="Name of the output zip file (default: kernels.zip)")
parser.add_argument("-p", "--include-private", action="store_true", default=include_private,
help="Include private kernels in the download (default: True)")
parser.add_argument("-u", "--user", type=str, default=user,
help="Username of the Kaggle user to search kernels for (default: current user)")
parser.add_argument("-s", "--max-page-size", type=validate_positive_int, default=max_page_size,
help="Maximum number of kernels to download per page (default: 100)")
parser.add_argument("-t", "--tmp-dir", type=str, default=tmp_dir,
help="Path to the temporary directory (default: None)")

args = parser.parse_args()

Expand Down Expand Up @@ -128,7 +112,7 @@ def main(include_private=True, max_page_size=MAX_PAGE_SIZE, user=None, output_na
path = kernel_to_path(kernel)
path.mkdir(parents=True, exist_ok=True)
api.kernels_pull(kernel.ref, path=path, metadata=True)
except Exception as e:
except Exception:
logging.warning("Failed to download %r", getattr(kernel, 'ref', getattr(kernel, 'title')),
exc_info=True)

Expand Down
Loading