From ceda919739da0ecc8738bef4efd60a5998b06551 Mon Sep 17 00:00:00 2001 From: InnopolisU Date: Tue, 27 Aug 2024 20:51:21 +0300 Subject: [PATCH 1/3] Fixed errors and improved Dockerfiles --- DockerfileMED | 25 +++++++---------- DockerfileNM | 25 +++++++---------- DockerfilePROM | 23 +++++++--------- .../example_image_segmentation_unet.yaml | 4 +-- .../example_image_detection_yolo.yaml | 9 ++++--- .../detection/yolov5fruits_pretrain.yaml | 4 +++ dui/assets/dui.js | 27 ++++++++++++++----- dui/pages/config_edition.py | 2 ++ dui/pages/config_saving.py | 4 ++- 9 files changed, 65 insertions(+), 58 deletions(-) create mode 100644 config/models/detection/yolov5fruits_pretrain.yaml diff --git a/DockerfileMED b/DockerfileMED index 4755a376..82263bb8 100755 --- a/DockerfileMED +++ b/DockerfileMED @@ -1,28 +1,23 @@ FROM pytorch/pytorch:1.11.0-cuda11.3-cudnn8-runtime -ENV UI_TITLE="Фреймворк для Медицины" -ENV WANDB_DISABLED="true" - -RUN apt-get update && apt-get upgrade -y -RUN apt-get install ffmpeg libsm6 libxext6 -y -RUN apt-get install curl -y -RUN pip3 install --upgrade pip -# works with 1.2.2 +RUN export PYTHONPATH=. && export PATH=$PATH +RUN apt-get update \ + && apt-get upgrade -y \ + && apt-get install ffmpeg libsm6 libxext6 curl iotop -y \ + && rm -rf /var/lib/{apt,dpkg,cache,log}/ +RUN pip3 install --upgrade pip pyopenssl cryptography RUN /bin/bash -c 'curl -sSL https://install.python-poetry.org | python - --version 1.3.0' COPY . /code RUN rm -r /code/config/experiments/* -COPY ./config/experiments/demo_prom /code/config/experiments/ +COPY ./config/experiments/demo_med /code/config/experiments/ WORKDIR /code RUN /bin/bash -c '/root/.local/bin/poetry config virtualenvs.create false \ - && /root/.local/bin/poetry lock --no-update && /root/.local/bin/poetry install --no-interaction --no-ansi' + && /root/.local/bin/poetry lock --no-update && /root/.local/bin/poetry install --no-interaction --no-ansi --no-cache' WORKDIR /code/dui -RUN python3 -m pip install -U pyopenssl cryptography -RUN apt-get -y install iotop -RUN export PYTHONPATH=. -RUN export PATH=$PATH - +ENV UI_TITLE="Фреймворк для Медицины" +ENV WANDB_DISABLED="true" ENTRYPOINT PYTHONUNBUFFERED=1 python3 app.py \ No newline at end of file diff --git a/DockerfileNM b/DockerfileNM index 33cb400c..4b0549a4 100755 --- a/DockerfileNM +++ b/DockerfileNM @@ -1,28 +1,23 @@ FROM pytorch/pytorch:1.11.0-cuda11.3-cudnn8-runtime -ENV UI_TITLE="Фреймворк для Новых Материалов" -ENV WANDB_DISABLED="true" - -RUN apt-get update && apt-get upgrade -y -RUN apt-get install ffmpeg libsm6 libxext6 -y -RUN apt-get install curl -y -RUN pip3 install --upgrade pip -# works with 1.2.2 +RUN export PYTHONPATH=. && export PATH=$PATH +RUN apt-get update \ + && apt-get upgrade -y \ + && apt-get install ffmpeg libsm6 libxext6 curl iotop -y \ + && rm -rf /var/lib/{apt,dpkg,cache,log}/ +RUN pip3 install --upgrade pip pyopenssl cryptography RUN /bin/bash -c 'curl -sSL https://install.python-poetry.org | python - --version 1.3.0' COPY . /code RUN rm -r /code/config/experiments/* -COPY ./config/experiments/demo_prom /code/config/experiments/ +COPY ./config/experiments/demo_nm /code/config/experiments/ WORKDIR /code RUN /bin/bash -c '/root/.local/bin/poetry config virtualenvs.create false \ - && /root/.local/bin/poetry lock --no-update && /root/.local/bin/poetry install --no-interaction --no-ansi' + && /root/.local/bin/poetry lock --no-update && /root/.local/bin/poetry install --no-interaction --no-ansi --no-cache' WORKDIR /code/dui -RUN python3 -m pip install -U pyopenssl cryptography -RUN apt-get -y install iotop -RUN export PYTHONPATH=. -RUN export PATH=$PATH - +ENV UI_TITLE="Фреймворк для Новых Материалов" +ENV WANDB_DISABLED="true" ENTRYPOINT PYTHONUNBUFFERED=1 python3 app.py \ No newline at end of file diff --git a/DockerfilePROM b/DockerfilePROM index 9e8f8898..f3154c33 100755 --- a/DockerfilePROM +++ b/DockerfilePROM @@ -1,13 +1,11 @@ FROM pytorch/pytorch:1.11.0-cuda11.3-cudnn8-runtime -ENV UI_TITLE="Фреймворк для Промышленности" -ENV WANDB_DISABLED="true" - -RUN apt-get update && apt-get upgrade -y -RUN apt-get install ffmpeg libsm6 libxext6 -y -RUN apt-get install curl -y -RUN pip3 install --upgrade pip -# works with 1.2.2 +RUN export PYTHONPATH=. && export PATH=$PATH +RUN apt-get update \ + && apt-get upgrade -y \ + && apt-get install ffmpeg libsm6 libxext6 curl iotop -y \ + && rm -rf /var/lib/{apt,dpkg,cache,log}/ +RUN pip3 install --upgrade pip pyopenssl cryptography RUN /bin/bash -c 'curl -sSL https://install.python-poetry.org | python - --version 1.3.0' COPY . /code @@ -16,13 +14,10 @@ COPY ./config/experiments/demo_prom /code/config/experiments/ WORKDIR /code RUN /bin/bash -c '/root/.local/bin/poetry config virtualenvs.create false \ - && /root/.local/bin/poetry lock --no-update && /root/.local/bin/poetry install --no-interaction --no-ansi' + && /root/.local/bin/poetry lock --no-update && /root/.local/bin/poetry install --no-interaction --no-ansi --no-cache' WORKDIR /code/dui -RUN python3 -m pip install -U pyopenssl cryptography -RUN apt-get -y install iotop -RUN export PYTHONPATH=. -RUN export PATH=$PATH - +ENV UI_TITLE="Фреймворк для Промышленности" +ENV WANDB_DISABLED="true" ENTRYPOINT PYTHONUNBUFFERED=1 python3 app.py \ No newline at end of file diff --git a/config/experiments/demo_med/example_image_segmentation_unet.yaml b/config/experiments/demo_med/example_image_segmentation_unet.yaml index 54dc16dc..41c45138 100644 --- a/config/experiments/demo_med/example_image_segmentation_unet.yaml +++ b/config/experiments/demo_med/example_image_segmentation_unet.yaml @@ -2,7 +2,7 @@ defaults: - override /models: semantic-segmentation/unet - override /datasets: semantic-segmentation/segmentation_brain.yaml - - override /augmentations_train: none #segmentation_stroke.yaml + - override /augmentations_train: none - override /augmentations_val: none - override /augmentations_test: none - override /losses: segmentation_losses.yaml @@ -12,7 +12,7 @@ project: "segmentation" task: "image-segmentation" random_seed: 42 stop_param: 10 -epochs: 300 +epochs: 35 weights_freq: 1 batch_size: 10 ckpt_path: https://api.blackhole.ai.innopolis.university/pretrained/segmentation_unet_brain.pt \ No newline at end of file diff --git a/config/experiments/demo_prom/example_image_detection_yolo.yaml b/config/experiments/demo_prom/example_image_detection_yolo.yaml index 20eaeaca..e440f6af 100644 --- a/config/experiments/demo_prom/example_image_detection_yolo.yaml +++ b/config/experiments/demo_prom/example_image_detection_yolo.yaml @@ -1,12 +1,13 @@ # @package _global_ accelerator: gpu batch_size: 16 -ckpt_path: https://api.blackhole.ai.innopolis.university/pretrained/prom/fruits/new_baseline.pt +ckpt_path: /home/qazybek/restart/innofw/dui/logs/train/rotten_fruits/example_image_detection_yolo.yaml/20240826-203849/checkpoints/best.pt defaults: -- override /models: detection/yolov5 +- override /models: detection/yolov5fruits_pretrain - override /datasets: detection/detection_rotten_fruits -epochs: 100 -gpus: 3 +device: 0 +epochs: 10 +gpus: 1 project: rotten_fruits random_seed: 43 task: image-detection diff --git a/config/models/detection/yolov5fruits_pretrain.yaml b/config/models/detection/yolov5fruits_pretrain.yaml new file mode 100644 index 00000000..bec4077b --- /dev/null +++ b/config/models/detection/yolov5fruits_pretrain.yaml @@ -0,0 +1,4 @@ +_target_: ultralytics.YOLO +description: yolov5 by ultralytics +model: /home/qazybek/restart/innofw/dui/logs/train/rotten_fruits/example_image_detection_yolo.yaml/20240826-203849/checkpoints/best.pt +name: yolov5 \ No newline at end of file diff --git a/dui/assets/dui.js b/dui/assets/dui.js index 81de4fa8..02614852 100644 --- a/dui/assets/dui.js +++ b/dui/assets/dui.js @@ -11,6 +11,7 @@ let tooltipDict = { "gpus": "Количество видеокарт, которые можно использовать", "in_channels": "Количество входных каналов в модель. Например для RGB изображения - 3", "devices": "Номера видеокарт, которые можно использовать", + "device": "Номера видеокарт, которые можно использовать(YOLO)", "learning_rate": "Скорость обучения модели", "models": "Расширение или изменение параметров модели", @@ -422,10 +423,17 @@ function onKeyFieldChange(callback) { }(keyFields[i]); } } - +function processKeyFieldChange(keyfield){ + changeSpan(keyfield); + addEditionButtonOnOverridePresence(keyfield); +} function changeSpan(keyfield){ - let tooltiptext = keyfield.parentNode.getElementsByClassName("tooltiptext")[0]; - tooltiptext.textContent = tooltipDict[keyfield.value]; + try{ + let tooltiptext = keyfield.parentNode.getElementsByClassName("tooltiptext")[0]; + tooltiptext.textContent = tooltipDict[keyfield.value]; + }catch (e){ + console.log(e) + } } function addEditionButtonOnOverridePresence(keyfield){ @@ -655,8 +663,8 @@ function set_editionpg_callbacks(){ onDeleteParameterRowClick(deleteParameterRow); onAddParameterRowClick(addParameterRow); onSaveConfigButtonClick(saveConfig); - onKeyFieldChange(addEditionButtonOnOverridePresence); - onKeyFieldChange(changeSpan); + + onKeyFieldChange(processKeyFieldChange); onEditButtonClick(openModalWindowOnEditButtonClick); onInputFieldClick(); @@ -677,8 +685,13 @@ function set_editionpg_callbacks(){ let keyfields = document.getElementsByClassName("keyfield"); for(let k of keyfields){ - let tooltiptext = k.parentNode.getElementsByClassName("tooltiptext")[0]; - tooltiptext.textContent = tooltipDict[k.value]; + try { + let tooltiptext = k.parentNode.getElementsByClassName("tooltiptext")[0]; + tooltiptext.textContent = tooltipDict[k.value]; + } + catch (e){ + console.log(e) + } }; diff --git a/dui/pages/config_edition.py b/dui/pages/config_edition.py index 181896e0..aab1da18 100644 --- a/dui/pages/config_edition.py +++ b/dui/pages/config_edition.py @@ -144,6 +144,8 @@ def layout(config_name=None): dash.html.Option("accelerator"), dash.html.Option("gpus"), dash.html.Option("devices"), + dash.html.Option("device"), + dash.html.Option("ckpt_path"), ], id="parameters")])) diff --git a/dui/pages/config_saving.py b/dui/pages/config_saving.py index 3b2f5c90..9796ff19 100644 --- a/dui/pages/config_saving.py +++ b/dui/pages/config_saving.py @@ -16,7 +16,9 @@ @flaskapp.route("/save_config", methods = ['GET', 'POST']) def layout(): request_body = request.json - with open(configs_path / request_body["config_name"], "w+") as stream: + config_name = str(request_body["config_name"]) + config_name += ".yaml" if not config_name.endswith(".yaml") else "" + with open(configs_path / config_name, "w+") as stream: try: if "experiments/" in str(request_body["config_name"]): s = "# @package _global_\n" + yaml.dump(request_body["html"], allow_unicode=True, From c74c8df3efd70fbeb95031d2aebc306937e72221 Mon Sep 17 00:00:00 2001 From: InnopolisU Date: Tue, 27 Aug 2024 21:26:20 +0300 Subject: [PATCH 2/3] Corrected ckpt paths --- config/experiments/demo_prom/example_image_detection_yolo.yaml | 2 +- config/models/detection/yolov5fruits_pretrain.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/experiments/demo_prom/example_image_detection_yolo.yaml b/config/experiments/demo_prom/example_image_detection_yolo.yaml index e440f6af..b6ae9e21 100644 --- a/config/experiments/demo_prom/example_image_detection_yolo.yaml +++ b/config/experiments/demo_prom/example_image_detection_yolo.yaml @@ -1,7 +1,7 @@ # @package _global_ accelerator: gpu batch_size: 16 -ckpt_path: /home/qazybek/restart/innofw/dui/logs/train/rotten_fruits/example_image_detection_yolo.yaml/20240826-203849/checkpoints/best.pt +ckpt_path: https://api.blackhole.ai.innopolis.university/pretrained/prom/fruits/new_baseline.pt defaults: - override /models: detection/yolov5fruits_pretrain - override /datasets: detection/detection_rotten_fruits diff --git a/config/models/detection/yolov5fruits_pretrain.yaml b/config/models/detection/yolov5fruits_pretrain.yaml index bec4077b..87692d05 100644 --- a/config/models/detection/yolov5fruits_pretrain.yaml +++ b/config/models/detection/yolov5fruits_pretrain.yaml @@ -1,4 +1,4 @@ _target_: ultralytics.YOLO description: yolov5 by ultralytics -model: /home/qazybek/restart/innofw/dui/logs/train/rotten_fruits/example_image_detection_yolo.yaml/20240826-203849/checkpoints/best.pt +model: yolov5xu name: yolov5 \ No newline at end of file From e3221eddec2c02ab13b7b4e93c821893c11fd9c6 Mon Sep 17 00:00:00 2001 From: InnopolisU Date: Tue, 27 Aug 2024 22:53:45 +0300 Subject: [PATCH 3/3] Removed gpus from dockerfile --- config/experiments/demo_prom/example_image_detection_yolo.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/config/experiments/demo_prom/example_image_detection_yolo.yaml b/config/experiments/demo_prom/example_image_detection_yolo.yaml index b6ae9e21..d309f351 100644 --- a/config/experiments/demo_prom/example_image_detection_yolo.yaml +++ b/config/experiments/demo_prom/example_image_detection_yolo.yaml @@ -7,7 +7,6 @@ defaults: - override /datasets: detection/detection_rotten_fruits device: 0 epochs: 10 -gpus: 1 project: rotten_fruits random_seed: 43 task: image-detection