From ac3f334f00710c82c10a7432c2e9ce3310b6bee9 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Fri, 4 Aug 2023 15:55:57 -0500 Subject: [PATCH 1/6] Update GitHub Actions --- .github/workflows/release.yaml | 18 ++++++++++ .github/workflows/test.yaml | 33 +++++++++++++++++++ .../workflows/u24_element_before_release.yaml | 17 ---------- .../workflows/u24_element_release_call.yaml | 28 ---------------- .../workflows/u24_element_tag_to_release.yaml | 14 -------- CHANGELOG.md | 5 +++ element_deeplabcut/version.py | 2 +- 7 files changed, 57 insertions(+), 60 deletions(-) create mode 100644 .github/workflows/release.yaml create mode 100644 .github/workflows/test.yaml delete mode 100644 .github/workflows/u24_element_before_release.yaml delete mode 100644 .github/workflows/u24_element_release_call.yaml delete mode 100644 .github/workflows/u24_element_tag_to_release.yaml diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..4fb78cb --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,18 @@ +name: Release +on: + workflow_dispatch: +jobs: + make_github_release: + uses: datajoint/.github/.github/workflows/make_github_release.yaml@main + pypi_release: + needs: make_github_release + uses: datajoint/.github/.github/workflows/pypi_release.yaml@main + secrets: + TWINE_USERNAME: ${{secrets.TWINE_USERNAME}} + TWINE_PASSWORD: ${{secrets.TWINE_PASSWORD}} + with: + UPLOAD_URL: ${{needs.make_github_release.outputs.release_upload_url}} + mkdocs_release: + uses: datajoint/.github/.github/workflows/mkdocs_release.yaml@main + permissions: + contents: write diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000..ac7af16 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,33 @@ +name: Test +on: + push: + pull_request: + workflow_dispatch: + schedule: + - cron: "0 8 * * 1" +jobs: + tests: + runs-on: ubuntu-latest + strategy: + matrix: + py_ver: ["3.9", "3.10"] + mysql_ver: ["8.0", "5.7"] + include: + - py_ver: "3.8" + mysql_ver: "5.7" + - py_ver: "3.7" + mysql_ver: "5.7" + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{matrix.py_ver}} + uses: actions/setup-python@v4 + with: + python-version: ${{matrix.py_ver}} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 "black[jupyter]" + - name: Run style tests + run: | + python_version=${{matrix.py_ver}} + black element_deeplabcut --check --verbose --target-version py${python_version//.} diff --git a/.github/workflows/u24_element_before_release.yaml b/.github/workflows/u24_element_before_release.yaml deleted file mode 100644 index 692cf82..0000000 --- a/.github/workflows/u24_element_before_release.yaml +++ /dev/null @@ -1,17 +0,0 @@ -name: u24_element_before_release -on: - pull_request: - push: - branches: - - '**' - tags-ignore: - - '**' - workflow_dispatch: -jobs: - call_context_check: - uses: dj-sciops/djsciops-cicd/.github/workflows/context_check.yaml@main - call_u24_elements_build_alpine: - uses: dj-sciops/djsciops-cicd/.github/workflows/u24_element_build.yaml@main - with: - py_ver: 3.9 - image: djbase diff --git a/.github/workflows/u24_element_release_call.yaml b/.github/workflows/u24_element_release_call.yaml deleted file mode 100644 index 4324cca..0000000 --- a/.github/workflows/u24_element_release_call.yaml +++ /dev/null @@ -1,28 +0,0 @@ -name: u24_element_release_call -on: - workflow_run: - workflows: ["u24_element_tag_to_release"] - types: - - completed -jobs: - call_context_check: - uses: dj-sciops/djsciops-cicd/.github/workflows/context_check.yaml@main - test_call_u24_elements_release_alpine: - if: >- - github.event.workflow_run.conclusion == 'success' && ( contains(github.event.workflow_run.head_branch, 'test') || (github.event.workflow_run.event == 'pull_request')) - uses: dj-sciops/djsciops-cicd/.github/workflows/u24_element_release.yaml@main - with: - py_ver: 3.9 - twine_repo: testpypi - secrets: - TWINE_USERNAME: ${{secrets.TWINE_TEST_USERNAME}} - TWINE_PASSWORD: ${{secrets.TWINE_TEST_PASSWORD}} - call_u24_elements_release_alpine: - if: >- - github.event.workflow_run.conclusion == 'success' && github.repository_owner == 'datajoint' && !contains(github.event.workflow_run.head_branch, 'test') - uses: dj-sciops/djsciops-cicd/.github/workflows/u24_element_release.yaml@main - with: - py_ver: 3.9 - secrets: - TWINE_USERNAME: ${{secrets.TWINE_USERNAME}} - TWINE_PASSWORD: ${{secrets.TWINE_PASSWORD}} diff --git a/.github/workflows/u24_element_tag_to_release.yaml b/.github/workflows/u24_element_tag_to_release.yaml deleted file mode 100644 index 57334e9..0000000 --- a/.github/workflows/u24_element_tag_to_release.yaml +++ /dev/null @@ -1,14 +0,0 @@ -name: u24_element_tag_to_release -on: - push: - tags: - - '*.*.*' - - 'test*.*.*' -jobs: - call_context_check: - uses: dj-sciops/djsciops-cicd/.github/workflows/context_check.yaml@main - call_u24_elements_build_alpine: - uses: dj-sciops/djsciops-cicd/.github/workflows/u24_element_build.yaml@main - with: - py_ver: 3.9 - image: djbase diff --git a/CHANGELOG.md b/CHANGELOG.md index b264da9..d8d961a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention. +## [0.2.7] - 2023-08-04 + ++ Update - GitHub Actions with new reusable workflows + ## [0.2.6] - 2023-05-22 + Add - DeepLabCut, NWB, and DANDI citations @@ -68,6 +72,7 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and graciously provided by the Mathis Lab. + Add - Support for 2d single-animal models +[0.2.7]: https://github.com/datajoint/element-deeplabcut/releases/tag/0.2.7 [0.2.6]: https://github.com/datajoint/element-deeplabcut/releases/tag/0.2.6 [0.2.5]: https://github.com/datajoint/element-deeplabcut/releases/tag/0.2.5 [0.2.4]: https://github.com/datajoint/element-deeplabcut/releases/tag/0.2.4 diff --git a/element_deeplabcut/version.py b/element_deeplabcut/version.py index 21ce3f8..617ac09 100644 --- a/element_deeplabcut/version.py +++ b/element_deeplabcut/version.py @@ -1,4 +1,4 @@ """ Package metadata """ -__version__ = "0.2.6" +__version__ = "0.2.7" From cc2cbfcbc79fc38da6fe2eadcad189c9732190d4 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Fri, 4 Aug 2023 16:07:43 -0500 Subject: [PATCH 2/6] Rename images --- images/{diagram_flowchart.drawio => flowchart.drawio} | 0 images/{diagram_flowchart.svg => flowchart.svg} | 0 images/{diagram_dlc.svg => pipeline.svg} | 0 images/{diagram_dlc_model.svg => pipeline_model.svg} | 0 images/{diagram_dlc_train.svg => pipeline_train.svg} | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename images/{diagram_flowchart.drawio => flowchart.drawio} (100%) rename images/{diagram_flowchart.svg => flowchart.svg} (100%) rename images/{diagram_dlc.svg => pipeline.svg} (100%) rename images/{diagram_dlc_model.svg => pipeline_model.svg} (100%) rename images/{diagram_dlc_train.svg => pipeline_train.svg} (100%) diff --git a/images/diagram_flowchart.drawio b/images/flowchart.drawio similarity index 100% rename from images/diagram_flowchart.drawio rename to images/flowchart.drawio diff --git a/images/diagram_flowchart.svg b/images/flowchart.svg similarity index 100% rename from images/diagram_flowchart.svg rename to images/flowchart.svg diff --git a/images/diagram_dlc.svg b/images/pipeline.svg similarity index 100% rename from images/diagram_dlc.svg rename to images/pipeline.svg diff --git a/images/diagram_dlc_model.svg b/images/pipeline_model.svg similarity index 100% rename from images/diagram_dlc_model.svg rename to images/pipeline_model.svg diff --git a/images/diagram_dlc_train.svg b/images/pipeline_train.svg similarity index 100% rename from images/diagram_dlc_train.svg rename to images/pipeline_train.svg From fcd026a86be2d22d74f70a5ffdd6a82de1a77626 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Fri, 4 Aug 2023 16:20:11 -0500 Subject: [PATCH 3/6] Update readme --- README.md | 30 ++++++++++++++++++++++++++---- docs/src/concepts.md | 2 +- docs/src/index.md | 2 +- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 3392f20..e5bd971 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,6 @@ -# DataJoint Element - DeepLabCut +[![PyPI version](https://badge.fury.io/py/element-deeplabcut.svg)](http://badge.fury.io/py/element-deeplabcut) + +# DataJoint Element for Pose Estimation with DeepLabCut DataJoint Element for markerless pose estimation with [DeepLabCut](https://www.deeplabcut.org/). DataJoint Elements collectively standardize @@ -6,7 +8,27 @@ and automate data collection and analysis for neuroscience experiments. Each El a modular pipeline for data storage and processing with corresponding database tables that can be combined with other Elements to assemble a fully functional pipeline. -![diagram](https://raw.githubusercontent.com/datajoint/element-deeplabcut/main/images/diagram_flowchart.svg) +## Experiment Flowchart + +![flowchart](https://raw.githubusercontent.com/datajoint/element-deeplabcut/main/images/flowchart.svg) + +## Data Pipeline Diagram + +![pipeline](https://raw.githubusercontent.com/datajoint/element-deeplabcut/main/images/pipeline.svg) + +## Getting Started + ++ Install from PyPI + + ```bash + pip install element-deeplabcut + ``` + ++ [Interactive tutorial](https://github.com/datajoint/workflow-deeplabcut) + ++ [Documentation](https://datajoint.com/docs/elements/element-deeplabcut) + +## Support -Installation and usage instructions can be found at the -[Element documentation](https://datajoint.com/docs/elements/element-deeplabcut). ++ If you need help getting started or run into any errors, please open a GitHub Issue +or contact our team by email at support@datajoint.com. diff --git a/docs/src/concepts.md b/docs/src/concepts.md index 15d055f..02000bd 100644 --- a/docs/src/concepts.md +++ b/docs/src/concepts.md @@ -88,7 +88,7 @@ corresponding tables in the database. Within the workflow, Element DeepLabCut c to upstream Elements including Lab, Animal, and Session. For more detailed documentation on each table, see the API docs for the respective schemas. -![element-deeplabcut diagram](https://raw.githubusercontent.com/datajoint/element-deeplabcut/main/images/diagram_dlc.svg) +![pipeline](https://raw.githubusercontent.com/datajoint/element-deeplabcut/main/images/pipeline.svg) ### `lab` schema ([API docs](../api/workflow_deeplabcut/pipeline/#workflow_deeplabcut.pipeline.Device)) diff --git a/docs/src/index.md b/docs/src/index.md index 8e493e6..5732762 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -6,7 +6,7 @@ and automate data collection and analysis for neuroscience experiments. Each El a modular pipeline for data storage and processing with corresponding database tables that can be combined with other Elements to assemble a fully functional pipeline. -![diagram](https://raw.githubusercontent.com/datajoint/element-deeplabcut/main/images/diagram_flowchart.svg) +![flowchart](https://raw.githubusercontent.com/datajoint/element-deeplabcut/main/images/flowchart.svg) Element DeepLabCut runs DeepLabCut which uses image recognition machine learning models to generate animal position estimates from consumer grade video equipment. The Element From dff5b8776d12dc3c9332f60514fa926f7e3b12ae Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Fri, 4 Aug 2023 16:21:57 -0500 Subject: [PATCH 4/6] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8d961a..e3ac610 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and ## [0.2.7] - 2023-08-04 + Update - GitHub Actions with new reusable workflows ++ Update - Readme instructions ## [0.2.6] - 2023-05-22 From f37a17f2f59bc974b82b6d4c0dffbde72be34ec1 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Fri, 4 Aug 2023 16:26:21 -0500 Subject: [PATCH 5/6] Format with black --- element_deeplabcut/model.py | 4 +++- element_deeplabcut/train.py | 16 ++++++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/element_deeplabcut/model.py b/element_deeplabcut/model.py index ef49333..097a5c8 100644 --- a/element_deeplabcut/model.py +++ b/element_deeplabcut/model.py @@ -630,7 +630,9 @@ def generate( """ processed_dir = get_dlc_processed_data_dir() output_dir = cls.infer_output_dir( - {**video_recording_key, "model_name": model_name}, relative=False, mkdir=True + {**video_recording_key, "model_name": model_name}, + relative=False, + mkdir=True, ) if task_mode is None: diff --git a/element_deeplabcut/train.py b/element_deeplabcut/train.py index ff85146..f63c756 100644 --- a/element_deeplabcut/train.py +++ b/element_deeplabcut/train.py @@ -241,13 +241,16 @@ class ModelTraining(dj.Computed): # https://github.com/DeepLabCut/DeepLabCut/issues/70 def make(self, key): - from deeplabcut import train_network # isort:skip + from deeplabcut import train_network # isort:skip + try: - from deeplabcut.utils.auxiliaryfunctions import get_model_folder # isort:skip + from deeplabcut.utils.auxiliaryfunctions import ( + get_model_folder, + ) # isort:skip except ImportError: from deeplabcut.utils.auxiliaryfunctions import ( - GetModelFolder as get_model_folder - ) # isort:skip + GetModelFolder as get_model_folder, + ) # isort:skip """Launch training for each train.TrainingTask training_id via `.populate()`.""" project_path, model_prefix = (TrainingTask & key).fetch1( @@ -278,8 +281,9 @@ def make(self, key): # ---- Trigger DLC model training job ---- train_network_input_args = list(inspect.signature(train_network).parameters) train_network_kwargs = { - k: int(v) if k in ("shuffle", "trainingsetindex", "maxiters") else v - for k, v in dlc_config.items() if k in train_network_input_args + k: int(v) if k in ("shuffle", "trainingsetindex", "maxiters") else v + for k, v in dlc_config.items() + if k in train_network_input_args } for k in ["shuffle", "trainingsetindex", "maxiters"]: train_network_kwargs[k] = int(train_network_kwargs[k]) From b16c3861c5a4f51aaabb6300999b0fc4a0b92641 Mon Sep 17 00:00:00 2001 From: Kabilar Gunalan Date: Fri, 4 Aug 2023 16:28:19 -0500 Subject: [PATCH 6/6] Format with black --- element_deeplabcut/train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/element_deeplabcut/train.py b/element_deeplabcut/train.py index f63c756..89d1047 100644 --- a/element_deeplabcut/train.py +++ b/element_deeplabcut/train.py @@ -21,7 +21,7 @@ def activate( *, create_schema: bool = True, create_tables: bool = True, - linking_module: str = None + linking_module: str = None, ): """Activate this schema.