From 94688b89c296f2d19f74e008a0f952027322d2b4 Mon Sep 17 00:00:00 2001 From: maxisoft Date: Sat, 27 Jul 2024 19:47:58 +0200 Subject: [PATCH 1/6] Removed setup 7z actions --- .github/workflows/doit.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/doit.yml b/.github/workflows/doit.yml index cbff29d..d29d8da 100644 --- a/.github/workflows/doit.yml +++ b/.github/workflows/doit.yml @@ -73,9 +73,6 @@ jobs: run: | python main.py - - - uses: milliewalky/setup-7-zip@v1 - - name: ReZip kernels if: ${{ github.actor == github.repository_owner }} run: | From 719bec5028edf7a44fe82b3f024192a9721e8864 Mon Sep 17 00:00:00 2001 From: maxisoft Date: Sat, 27 Jul 2024 20:40:59 +0200 Subject: [PATCH 2/6] docs: update Readme to include ARCHIVE_PASSWORD secret - Added instructions to set the ARCHIVE_PASSWORD secret for artifact encryption --- Readme.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Readme.md b/Readme.md index 834c6f5..c1dccac 100644 --- a/Readme.md +++ b/Readme.md @@ -44,9 +44,10 @@ To enable this functionality, you'll need to configure secrets within your repos 4. **Set Up Secrets (Required):** - Navigate to your repository's settings and then to "Secrets." - - Create two secrets: + - Create secrets: - `KAGGLE_USERNAME`: Your Kaggle username. - `KAGGLE_KEY`: Your Kaggle API key (obtainable from your profile settings). + - `ARCHIVE_PASSWORD`: The artifact archive password to use 5. **Manual Trigger:** You can manually trigger the workflow by navigating to the “Actions” tab in your repository, selecting the `doit.yml` workflow, and clicking the “Run workflow” button. 6. **Output as GitHub Artifact:** From 122b06400e212e6a6f6ef3f15557bedb3ecb7c53 Mon Sep 17 00:00:00 2001 From: maxisoft Date: Sat, 27 Jul 2024 20:41:40 +0200 Subject: [PATCH 3/6] feat: enhance workflow with conditional archive creation and deployment - Added conditional check for ARCHIVE_PASSWORD secret - Implemented high compression and password protection for kernels archive - Updated artifact upload to use the new kernels.7z file - Added conditional steps for deploying to another repository if SSH_DEPLOY_KEY and DESTINATION_REPOSITORY_NAME are set - Improved cleanup steps to remove temporary files and directories --- .github/workflows/doit.yml | 65 +++++++++++++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 8 deletions(-) diff --git a/.github/workflows/doit.yml b/.github/workflows/doit.yml index d29d8da..e62417c 100644 --- a/.github/workflows/doit.yml +++ b/.github/workflows/doit.yml @@ -78,24 +78,73 @@ jobs: run: | if [ -z "${{ secrets.ARCHIVE_PASSWORD }}" ]; then echo "ARCHIVE_PASSWORD secret is not set" - exit 1 + echo "ARCHIVE_PASSWORD secret is not set" >> $GITHUB_STEP_SUMMARY + exit 0 fi mkdir -p tmp 7z x kernels.zip -otmp 7z a -t7z -mx=9 -p"${{ secrets.ARCHIVE_PASSWORD }}" kernels.7z tmp/* - - - name: Cleanup - if: ${{ always() }} - run: | + echo "archive_created=true" >> $GITHUB_ENV + echo "kernels.7z has been created" >> $GITHUB_STEP_SUMMARY rm -rf tmp || : - rm -rf kernels.zip || : - exit 0 - - name: Upload Artifact + - name: Upload Archive Artifact uses: actions/upload-artifact@v4 + if: ${{ env.archive_created == 'true' }} with: name: kernels path: kernels.7z if-no-files-found: error retention-days: 90 + - if: ${{ github.actor == github.repository_owner }} + run: | + if [ -z "${{ secrets.SSH_DEPLOY_KEY }}" ]; then + echo "SSH_DEPLOY_KEY secret is not set" >> $GITHUB_STEP_SUMMARY + exit 0 + fi + + if [ -z "${{ env.DESTINATION_REPOSITORY_NAME }}" ]; then + if [ -z "${{ secrets.DESTINATION_REPOSITORY_NAME }}" ]; then + echo "DESTINATION_REPOSITORY_NAME secret is not set" + else + export DESTINATION_REPOSITORY_NAME="${{ secrets.DESTINATION_REPOSITORY_NAME }}" + fi + fi + + if [ -z "$DESTINATION_REPOSITORY_NAME" ]; then + echo "DESTINATION_REPOSITORY_NAME is not set" + echo "DESTINATION_REPOSITORY_NAME is not set" >> $GITHUB_STEP_SUMMARY + exit 0 + fi + + echo "DESTINATION_REPOSITORY_NAME=$DESTINATION_REPOSITORY_NAME" >> $GITHUB_ENV + + rm -rf tmp || : + mkdir -p tmp + echo "Extract archive" >> $GITHUB_STEP_SUMMARY + + 7z x kernels.zip -otmp + + echo "DEPLOY_TO_REPOSITORY=true" >> $GITHUB_ENV + + - name: Push kernelsdirectory to another repository + uses: cpina/github-action-push-to-another-repository@v1 + if: ${{ env.DEPLOY_TO_REPOSITORY == 'true' && github.actor == github.repository_owner }} + with: + source-directory: tmp + destination-github-username: ${{ env.DESTINATION_REPOSITORY_USERNAME || github.actor }} + user-name: ${{ env.DESTINATION_REPOSITORY_USERNAME || github.actor }} + user-email: ${{ env.DESTINATION_REPOSITORY_USERNAME || github.actor }}@users.noreply.github.com + destination-repository-name: ${{ env.DESTINATION_REPOSITORY_NAME }} + env: + SSH_DEPLOY_KEY: ${{ secrets.SSH_DEPLOY_KEY }} + + - name: Cleanup + if: ${{ always() }} + run: | + rm -rf kernels.zip || : + rm -rf kernels.7z || : + rm -rf tmp || : + exit 0 + From 0c0037811f5cec6b2fc6f1e20ee66bd9453b5acd Mon Sep 17 00:00:00 2001 From: maxisoft Date: Sat, 27 Jul 2024 21:00:26 +0200 Subject: [PATCH 4/6] fix: update main.py to use environment variable for include_private - Added os import to main.py - Changed default value of include_private to False - Updated main function to use environment variable KAGGLE_KERNELS_PRIVATE for include_private parameter --- main.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index 78731ef..bd8375c 100644 --- a/main.py +++ b/main.py @@ -1,5 +1,6 @@ import argparse import logging +import os import shutil from pathlib import Path from tempfile import TemporaryDirectory @@ -61,13 +62,13 @@ def kernel_to_path(kernel): return Path(pathvalidate.sanitize_filename(f"{kernel.ref}#{kernel.id}", replacement_text="_")) -def main(include_private=True, max_page_size=MAX_PAGE_SIZE, user=None, output_name="kernels.zip", +def main(include_private=False, max_page_size=MAX_PAGE_SIZE, user=None, output_name="kernels.zip", tmp_dir_prefix="kaggle_", tmp_dir=None): parser = argparse.ArgumentParser(description="Download All Kaggle Kernels") parser.add_argument("-o", "--output", type=validate_filename, default=output_name, help="Name of the output zip file (default: kernels.zip)") parser.add_argument("-p", "--include-private", action="store_true", default=include_private, - help="Include private kernels in the download (default: True)") + help="Include private kernels in the download (default: False)") parser.add_argument("-u", "--user", type=str, default=user, help="Username of the Kaggle user to search kernels for (default: current user)") parser.add_argument("-s", "--max-page-size", type=validate_positive_int, default=max_page_size, @@ -120,4 +121,4 @@ def main(include_private=True, max_page_size=MAX_PAGE_SIZE, user=None, output_na if __name__ == '__main__': - main() + main(include_private=os.getenv('KAGGLE_KERNELS_PRIVATE', '').lower() in ('true', '1', 'y', 'yes', 'ok')) From 49351693a1c8e42f2a1660f7a7a4163b6eb4800a Mon Sep 17 00:00:00 2001 From: maxisoft Date: Sat, 27 Jul 2024 21:01:08 +0200 Subject: [PATCH 5/6] feat: enhance workflow with detailed step summaries and improved deployment - Added step summaries to provide detailed information in the workflow summary - Updated conditional checks and environment variable handling for better clarity - Improved cleanup steps to ensure temporary files and directories are removed - Corrected the name of the step for pushing the kernels directory to another repository --- .github/workflows/doit.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/doit.yml b/.github/workflows/doit.yml index e62417c..dbc5e8f 100644 --- a/.github/workflows/doit.yml +++ b/.github/workflows/doit.yml @@ -71,7 +71,9 @@ jobs: - name: Run app if: ${{ github.actor == github.repository_owner }} run: | + echo "kaggle kernels download process" > $GITHUB_STEP_SUMMARY python main.py + echo "kaggle kernels downloaded" > $GITHUB_STEP_SUMMARY - name: ReZip kernels if: ${{ github.actor == github.repository_owner }} @@ -83,9 +85,10 @@ jobs: fi mkdir -p tmp 7z x kernels.zip -otmp + echo "kernels.7z creation" > $GITHUB_STEP_SUMMARY 7z a -t7z -mx=9 -p"${{ secrets.ARCHIVE_PASSWORD }}" kernels.7z tmp/* echo "archive_created=true" >> $GITHUB_ENV - echo "kernels.7z has been created" >> $GITHUB_STEP_SUMMARY + echo "kernels.7z has been created" > $GITHUB_STEP_SUMMARY rm -rf tmp || : - name: Upload Archive Artifact @@ -109,6 +112,7 @@ jobs: echo "DESTINATION_REPOSITORY_NAME secret is not set" else export DESTINATION_REPOSITORY_NAME="${{ secrets.DESTINATION_REPOSITORY_NAME }}" + echo "DESTINATION_REPOSITORY_NAME set from secrets" fi fi @@ -122,20 +126,21 @@ jobs: rm -rf tmp || : mkdir -p tmp - echo "Extract archive" >> $GITHUB_STEP_SUMMARY + echo "Extract archive" > $GITHUB_STEP_SUMMARY 7z x kernels.zip -otmp echo "DEPLOY_TO_REPOSITORY=true" >> $GITHUB_ENV + echo "DEPLOY_TO_REPOSITORY prepared" >> $GITHUB_STEP_SUMMARY - - name: Push kernelsdirectory to another repository + - name: Push kernels directory to another repository uses: cpina/github-action-push-to-another-repository@v1 if: ${{ env.DEPLOY_TO_REPOSITORY == 'true' && github.actor == github.repository_owner }} with: source-directory: tmp destination-github-username: ${{ env.DESTINATION_REPOSITORY_USERNAME || github.actor }} - user-name: ${{ env.DESTINATION_REPOSITORY_USERNAME || github.actor }} - user-email: ${{ env.DESTINATION_REPOSITORY_USERNAME || github.actor }}@users.noreply.github.com + user-name: ${{ github.actor }} + user-email: ${{ github.actor }}@users.noreply.github.com destination-repository-name: ${{ env.DESTINATION_REPOSITORY_NAME }} env: SSH_DEPLOY_KEY: ${{ secrets.SSH_DEPLOY_KEY }} From a67fa8d7a16a9024e66f9a5f358002bf01127a95 Mon Sep 17 00:00:00 2001 From: maxisoft Date: Sat, 27 Jul 2024 21:10:56 +0200 Subject: [PATCH 6/6] feat: add KAGGLE_KERNELS_PRIVATE environment variable and update main.py - Added KAGGLE_KERNELS_PRIVATE environment variable to GitHub Actions workflow - Updated argparse help messages in main.py to use formatted default values --- .github/workflows/doit.yml | 1 + main.py | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/doit.yml b/.github/workflows/doit.yml index dbc5e8f..2a6aa5c 100644 --- a/.github/workflows/doit.yml +++ b/.github/workflows/doit.yml @@ -8,6 +8,7 @@ on: workflow_dispatch: env: + KAGGLE_KERNELS_PRIVATE: "no" PYTHON_VERSION: "3.11" permissions: diff --git a/main.py b/main.py index bd8375c..13e0561 100644 --- a/main.py +++ b/main.py @@ -66,15 +66,15 @@ def main(include_private=False, max_page_size=MAX_PAGE_SIZE, user=None, output_n tmp_dir_prefix="kaggle_", tmp_dir=None): parser = argparse.ArgumentParser(description="Download All Kaggle Kernels") parser.add_argument("-o", "--output", type=validate_filename, default=output_name, - help="Name of the output zip file (default: kernels.zip)") + help=f"Name of the output zip file (default: {output_name})") parser.add_argument("-p", "--include-private", action="store_true", default=include_private, - help="Include private kernels in the download (default: False)") + help=f"Include private kernels in the download (default: {include_private})") parser.add_argument("-u", "--user", type=str, default=user, help="Username of the Kaggle user to search kernels for (default: current user)") parser.add_argument("-s", "--max-page-size", type=validate_positive_int, default=max_page_size, - help="Maximum number of kernels to download per page (default: 100)") + help=f"Maximum number of kernels to download per page (default: {max_page_size})") parser.add_argument("-t", "--tmp-dir", type=str, default=tmp_dir, - help="Path to the temporary directory (default: None)") + help=f"Path to the temporary directory (default: {tmp_dir})") args = parser.parse_args()