diff --git a/Chicken_Disease_Classification/Dockerfile b/Chicken_Disease_Classification/Dockerfile new file mode 100644 index 0000000000..e46419162b --- /dev/null +++ b/Chicken_Disease_Classification/Dockerfile @@ -0,0 +1,9 @@ +FROM python:3.8-slim-buster + +RUN apt update -y && apt install awscli -y +WORKDIR /app + +COPY . /app +RUN pip install -r requirements.txt + +CMD ["python3", "app.py"] \ No newline at end of file diff --git a/Chicken_Disease_Classification/LICENSE b/Chicken_Disease_Classification/LICENSE new file mode 100644 index 0000000000..93bc26f8cc --- /dev/null +++ b/Chicken_Disease_Classification/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 BAPPY AHMED + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Chicken_Disease_Classification/README.md b/Chicken_Disease_Classification/README.md new file mode 100644 index 0000000000..c4eaa013f2 --- /dev/null +++ b/Chicken_Disease_Classification/README.md @@ -0,0 +1,203 @@ +# Chicken-Disease-Classification-Project + + +## Workflows + +1. Update config.yaml +2. Update secrets.yaml [Optional] +3. Update params.yaml +4. Update the entity +5. Update the configuration manager in src config +6. Update the components +7. Update the pipeline +8. Update the main.py +9. Update the dvc.yaml +While Building the project, we have to follow these workflows, make sure we are updating everthing at every step. + +# How to run? +### STEPS: + +Clone the repository + +```bash +https://github.com/entbappy/Chicken-Disease-Classification--Project +``` +### STEP 01- Create a conda environment after opening the repository + +```bash +conda create -n cnncls python=3.8 -y +``` + +```bash +conda activate cnncls +``` + + +### STEP 02- install the requirements +```bash +pip install -r requirements.txt +``` + + +```bash +# Finally run the following command +python app.py +``` + +Now, +```bash +open up you local host and port +``` + + +### DVC cmd + +1. dvc init +2. dvc repro +3. dvc dag + + +## MLflow + +[Documentation](https://mlflow.org/docs/latest/index.html) + + +##### cmd +- mlflow ui + +### dagshub +[dagshub](https://dagshub.com/) + +MLFLOW_TRACKING_URI=https://dagshub.com/entbappy/MLflow-DVC-Chicken-Disease-Classification.mlflow +MLFLOW_TRACKING_USERNAME=entbappy +MLFLOW_TRACKING_PASSWORD=6824692c47a369aa6f9eac5b10041d5c8edbcef0 +python script.py + +Run this to export as env variables: + +```bash + +export MLFLOW_TRACKING_URI=https://dagshub.com/entbappy/MLflow-DVC-Chicken-Disease-Classification.mlflow + +export MLFLOW_TRACKING_USERNAME=entbappy + +export MLFLOW_TRACKING_PASSWORD=6824692c47a369aa6f9eac5b10041d5c8edbcef0 + +``` + + +# AWS-CICD-Deployment-with-Github-Actions + +## 1. Login to AWS console. + +## 2. Create IAM user for deployment + + #with specific access + + 1. EC2 access : It is virtual machine + + 2. ECR: Elastic Container registry to save your docker image in aws + + + #Description: About the deployment + + 1. Build docker image of the source code + + 2. Push your docker image to ECR + + 3. Launch Your EC2 + + 4. Pull Your image from ECR in EC2 + + 5. Lauch your docker image in EC2 + + #Policy: + + 1. AmazonEC2ContainerRegistryFullAccess + + 2. AmazonEC2FullAccess + + +## 3. Create ECR repo to store/save docker image + - Save the URI: 566373416292.dkr.ecr.us-east-1.amazonaws.com/chicken + + +## 4. Create EC2 machine (Ubuntu) + +## 5. Open EC2 and Install docker in EC2 Machine: + + + #optinal + + sudo apt-get update -y + + sudo apt-get upgrade + + #required + + curl -fsSL https://get.docker.com -o get-docker.sh + + sudo sh get-docker.sh + + sudo usermod -aG docker ubuntu + + newgrp docker + +# 6. Configure EC2 as self-hosted runner: + setting>actions>runner>new self hosted runner> choose os> then run command one by one + + +# 7. Setup github secrets: + + AWS_ACCESS_KEY_ID= + + AWS_SECRET_ACCESS_KEY= + + AWS_REGION = us-east-1 + + AWS_ECR_LOGIN_URI = demo>> 566373416292.dkr.ecr.ap-south-1.amazonaws.com + + ECR_REPOSITORY_NAME = simple-app + + + + +# AZURE-CICD-Deployment-with-Github-Actions + +## Save pass: + +s3cEZKH5yytiVnJ3h+eI3qhhzf9q1vNwEi6+q+WGdd+ACRCZ7JD6 + + +## Run from terminal: + +docker build -t chickenapp.azurecr.io/chicken:latest . + +docker login chickenapp.azurecr.io + +docker push chickenapp.azurecr.io/chicken:latest + + +## Deployment Steps: + +1. Build the Docker image of the Source Code +2. Push the Docker image to Container Registry +3. Launch the Web App Server in Azure +4. Pull the Docker image from the container registry to Web App server and run + + + +## About MLflow & DVC + +MLflow + + - Its Production Grade + - Trace all of your expriements + - Logging & taging your model + + +DVC + + - Its very lite weight for POC only + - lite weight expriements tracker + - It can perform Orchestration (Creating Pipelines) diff --git a/Chicken_Disease_Classification/app.py b/Chicken_Disease_Classification/app.py new file mode 100644 index 0000000000..695c23b65d --- /dev/null +++ b/Chicken_Disease_Classification/app.py @@ -0,0 +1,49 @@ +from flask import Flask, request, jsonify, render_template +import os +from flask_cors import CORS, cross_origin +from cnnClassifier.utils.common import decodeImage +from cnnClassifier.pipeline.predict import PredictionPipeline + + +os.putenv('LANG', 'en_US.UTF-8') +os.putenv('LC_ALL', 'en_US.UTF-8') + +app = Flask(__name__) +CORS(app) + + +class ClientApp: + def __init__(self): + self.filename = "inputImage.jpg" + self.classifier = PredictionPipeline(self.filename) + + +@app.route("/", methods=['GET']) +@cross_origin() +def home(): + return render_template('index.html') + + +@app.route("/train", methods=['GET','POST']) +@cross_origin() +def trainRoute(): + os.system("python main.py") + return "Training done successfully!" + + + +@app.route("/predict", methods=['POST']) +@cross_origin() +def predictRoute(): + image = request.json['image'] + decodeImage(image, clApp.filename) + result = clApp.classifier.predict() + return jsonify(result) + + +if __name__ == "__main__": + clApp = ClientApp() + # app.run(host='0.0.0.0', port=8080) #local host + # app.run(host='0.0.0.0', port=8080) #for AWS + app.run(host='0.0.0.0', port=80) #for AZURE + diff --git a/Chicken_Disease_Classification/config/config.yaml b/Chicken_Disease_Classification/config/config.yaml new file mode 100644 index 0000000000..0171fed733 --- /dev/null +++ b/Chicken_Disease_Classification/config/config.yaml @@ -0,0 +1,30 @@ +artifacts_root: artifacts + + +data_ingestion: + root_dir: artifacts/data_ingestion + source_URL: https://github.com/entbappy/Branching-tutorial/raw/master/Chicken-fecal-images.zip + local_data_file: artifacts/data_ingestion/data.zip + unzip_dir: artifacts/data_ingestion + + + +prepare_base_model: + root_dir: artifacts/prepare_base_model + base_model_path: artifacts/prepare_base_model/base_model.h5 + updated_base_model_path: artifacts/prepare_base_model/base_model_updated.h5 + + + + +prepare_callbacks: + root_dir: artifacts/prepare_callbacks + tensorboard_root_log_dir: artifacts/prepare_callbacks/tensorboard_log_dir + checkpoint_model_filepath: artifacts/prepare_callbacks/checkpoint_dir/model.h5 + + + + +training: + root_dir: artifacts/training + trained_model_path: artifacts/training/model.h5 diff --git a/Chicken_Disease_Classification/dvc.lock b/Chicken_Disease_Classification/dvc.lock new file mode 100644 index 0000000000..9b9d183155 --- /dev/null +++ b/Chicken_Disease_Classification/dvc.lock @@ -0,0 +1,100 @@ +schema: '2.0' +stages: + data_ingestion: + cmd: python src/cnnClassifier/pipeline/stage_01_data_ingestion.py + deps: + - path: config/config.yaml + md5: 3f6dc733e0735665409e293214355568 + size: 818 + - path: src/cnnClassifier/pipeline/stage_01_data_ingestion.py + md5: b0ff51aae578695575ba98f83c03d86e + size: 914 + outs: + - path: artifacts/data_ingestion/Chicken-fecal-images + md5: 9e1f8dd2eae3c29e9d635df89d438ae4.dir + size: 12207807 + nfiles: 390 + prepare_base_model: + cmd: python src/cnnClassifier/pipeline/stage_02_prepare_base_model.py + deps: + - path: config/config.yaml + md5: 3f6dc733e0735665409e293214355568 + size: 818 + - path: src/cnnClassifier/pipeline/stage_02_prepare_base_model.py + md5: 5bbb3b3d7f36b28620ae359473f09153 + size: 1003 + params: + params.yaml: + CLASSES: 2 + IMAGE_SIZE: + - 224 + - 224 + - 3 + INCLUDE_TOP: false + LEARNING_RATE: 0.01 + WEIGHTS: imagenet + outs: + - path: artifacts/prepare_base_model + md5: 0774dbc0f4a82fe0f16c2b4f8b2ad70c.dir + size: 118054560 + nfiles: 2 + training: + cmd: python src/cnnClassifier/pipeline/stage_03_training.py + deps: + - path: artifacts/data_ingestion/Chicken-fecal-images + md5: 9e1f8dd2eae3c29e9d635df89d438ae4.dir + size: 12207807 + nfiles: 390 + - path: artifacts/prepare_base_model + md5: 0774dbc0f4a82fe0f16c2b4f8b2ad70c.dir + size: 118054560 + nfiles: 2 + - path: config/config.yaml + md5: 3f6dc733e0735665409e293214355568 + size: 818 + - path: src/cnnClassifier/components/prepare_callbacks.py + md5: dfc1909f7e54d14bb10ff488888367f4 + size: 1008 + - path: src/cnnClassifier/pipeline/stage_03_training.py + md5: 040b50b39a7a78cd8f367c558ebf6dd1 + size: 1268 + params: + params.yaml: + AUGMENTATION: true + BATCH_SIZE: 16 + EPOCHS: 1 + IMAGE_SIZE: + - 224 + - 224 + - 3 + outs: + - path: artifacts/training/model.h5 + md5: 36c31aaf79f22816bd5e8e5df9d87d03 + size: 59337520 + evaluation: + cmd: python src/cnnClassifier/pipeline/stage_04_evaluation.py + deps: + - path: artifacts/data_ingestion/Chicken-fecal-images + md5: 9e1f8dd2eae3c29e9d635df89d438ae4.dir + size: 12207807 + nfiles: 390 + - path: artifacts/training/model.h5 + md5: 36c31aaf79f22816bd5e8e5df9d87d03 + size: 59337520 + - path: config/config.yaml + md5: 3f6dc733e0735665409e293214355568 + size: 818 + - path: src/cnnClassifier/pipeline/stage_04_evaluation.py + md5: f5c5da091d33ae0dc6b6c753ff6a6f82 + size: 923 + params: + params.yaml: + BATCH_SIZE: 16 + IMAGE_SIZE: + - 224 + - 224 + - 3 + outs: + - path: scores.json + md5: 85d9b2cdb3d44e4a7fbe93252e3f060a + size: 73 diff --git a/Chicken_Disease_Classification/dvc.yaml b/Chicken_Disease_Classification/dvc.yaml new file mode 100644 index 0000000000..0a7bc686f4 --- /dev/null +++ b/Chicken_Disease_Classification/dvc.yaml @@ -0,0 +1,58 @@ +stages: + data_ingestion: + cmd: python src/cnnClassifier/pipeline/stage_01_data_ingestion.py + deps: + - src/cnnClassifier/pipeline/stage_01_data_ingestion.py + - config/config.yaml + outs: + - artifacts/data_ingestion/Chicken-fecal-images + + + prepare_base_model: + cmd: python src/cnnClassifier/pipeline/stage_02_prepare_base_model.py + deps: + - src/cnnClassifier/pipeline/stage_02_prepare_base_model.py + - config/config.yaml + params: + - IMAGE_SIZE + - INCLUDE_TOP + - CLASSES + - WEIGHTS + - LEARNING_RATE + outs: + - artifacts/prepare_base_model + + + + training: + cmd: python src/cnnClassifier/pipeline/stage_03_training.py + deps: + - src/cnnClassifier/pipeline/stage_03_training.py + - src/cnnClassifier/components/prepare_callbacks.py + - config/config.yaml + - artifacts/data_ingestion/Chicken-fecal-images + - artifacts/prepare_base_model + params: + - IMAGE_SIZE + - EPOCHS + - BATCH_SIZE + - AUGMENTATION + outs: + - artifacts/training/model.h5 + + + + + evaluation: + cmd: python src/cnnClassifier/pipeline/stage_04_evaluation.py + deps: + - src/cnnClassifier/pipeline/stage_04_evaluation.py + - config/config.yaml + - artifacts/data_ingestion/Chicken-fecal-images + - artifacts/training/model.h5 + params: + - IMAGE_SIZE + - BATCH_SIZE + metrics: + - scores.json: + cache: false diff --git a/Chicken_Disease_Classification/inputImage.jpg b/Chicken_Disease_Classification/inputImage.jpg new file mode 100644 index 0000000000..d60489d8a1 Binary files /dev/null and b/Chicken_Disease_Classification/inputImage.jpg differ diff --git a/Chicken_Disease_Classification/main.py b/Chicken_Disease_Classification/main.py new file mode 100644 index 0000000000..f460fb691d --- /dev/null +++ b/Chicken_Disease_Classification/main.py @@ -0,0 +1,65 @@ +from cnnClassifier import logger +from cnnClassifier.pipeline.stage_01_data_ingestion import DataIngestionTrainingPipeline +from cnnClassifier.pipeline.stage_02_prepare_base_model import PrepareBaseModelTrainingPipeline +from cnnClassifier.pipeline.stage_03_training import ModelTrainingPipeline +from cnnClassifier.pipeline.stage_04_evaluation import EvaluationPipeline + + +STAGE_NAME = "Data Ingestion stage" +try: + logger.info(f">>>>>> stage {STAGE_NAME} started <<<<<<") + data_ingestion = DataIngestionTrainingPipeline() + data_ingestion.main() + logger.info(f">>>>>> stage {STAGE_NAME} completed <<<<<<\n\nx==========x") +except Exception as e: + logger.exception(e) + raise e + + + + +STAGE_NAME = "Prepare base model" +try: + logger.info(f"*******************") + logger.info(f">>>>>> stage {STAGE_NAME} started <<<<<<") + prepare_base_model = PrepareBaseModelTrainingPipeline() + prepare_base_model.main() + logger.info(f">>>>>> stage {STAGE_NAME} completed <<<<<<\n\nx==========x") +except Exception as e: + logger.exception(e) + raise e + + + + +STAGE_NAME = "Training" +try: + logger.info(f"*******************") + logger.info(f">>>>>> stage {STAGE_NAME} started <<<<<<") + model_trainer = ModelTrainingPipeline() + model_trainer.main() + logger.info(f">>>>>> stage {STAGE_NAME} completed <<<<<<\n\nx==========x") +except Exception as e: + logger.exception(e) + raise e + + + + + + +STAGE_NAME = "Evaluation stage" +try: + logger.info(f"*******************") + logger.info(f">>>>>> stage {STAGE_NAME} started <<<<<<") + model_evalution = EvaluationPipeline() + model_evalution.main() + logger.info(f">>>>>> stage {STAGE_NAME} completed <<<<<<\n\nx==========x") + +except Exception as e: + logger.exception(e) + raise e + + + + diff --git a/Chicken_Disease_Classification/params.yaml b/Chicken_Disease_Classification/params.yaml new file mode 100644 index 0000000000..c69ece0aa6 --- /dev/null +++ b/Chicken_Disease_Classification/params.yaml @@ -0,0 +1,8 @@ +AUGMENTATION: True +IMAGE_SIZE: [224, 224, 3] # as per VGG 16 model +BATCH_SIZE: 16 +INCLUDE_TOP: False +EPOCHS: 1 +CLASSES: 2 +WEIGHTS: imagenet +LEARNING_RATE: 0.01 diff --git a/Chicken_Disease_Classification/requirements.txt b/Chicken_Disease_Classification/requirements.txt new file mode 100644 index 0000000000..c3897101ec --- /dev/null +++ b/Chicken_Disease_Classification/requirements.txt @@ -0,0 +1,18 @@ +tensorflow +pandas +dvc +mlflow==2.2.2 +notebook +numpy +matplotlib +seaborn +python-box==6.0.2 +pyYAML +tqdm +ensure==1.0.2 +joblib +types-PyYAML +scipy +Flask +Flask-Cors +-e . \ No newline at end of file diff --git a/Chicken_Disease_Classification/research/01_data_ingestion.ipynb b/Chicken_Disease_Classification/research/01_data_ingestion.ipynb new file mode 100644 index 0000000000..c1d01080b9 --- /dev/null +++ b/Chicken_Disease_Classification/research/01_data_ingestion.ipynb @@ -0,0 +1,257 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import os" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'d:\\\\Bappy\\\\YouTube\\\\Chicken-Disease-Classification--Project\\\\research'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%pwd" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "os.chdir(\"../\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'d:\\\\Bappy\\\\YouTube\\\\Chicken-Disease-Classification--Project'" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%pwd" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "from dataclasses import dataclass\n", + "from pathlib import Path\n", + "\n", + "\n", + "@dataclass(frozen=True)\n", + "class DataIngestionConfig:\n", + " root_dir: Path\n", + " source_URL: str\n", + " local_data_file: Path\n", + " unzip_dir: Path" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "from cnnClassifier.constants import *\n", + "from cnnClassifier.utils.common import read_yaml, create_directories" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "class ConfigurationManager:\n", + " def __init__(\n", + " self,\n", + " config_filepath = CONFIG_FILE_PATH,\n", + " params_filepath = PARAMS_FILE_PATH):\n", + "\n", + " self.config = read_yaml(config_filepath)\n", + " self.params = read_yaml(params_filepath)\n", + "\n", + " create_directories([self.config.artifacts_root])\n", + "\n", + "\n", + " \n", + " def get_data_ingestion_config(self) -> DataIngestionConfig:\n", + " config = self.config.data_ingestion\n", + "\n", + " create_directories([config.root_dir])\n", + "\n", + " data_ingestion_config = DataIngestionConfig(\n", + " root_dir=config.root_dir,\n", + " source_URL=config.source_URL,\n", + " local_data_file=config.local_data_file,\n", + " unzip_dir=config.unzip_dir \n", + " )\n", + "\n", + " return data_ingestion_config\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import urllib.request as request\n", + "import zipfile\n", + "from cnnClassifier import logger\n", + "from cnnClassifier.utils.common import get_size" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "class DataIngestion:\n", + " def __init__(self, config: DataIngestionConfig):\n", + " self.config = config\n", + "\n", + "\n", + " \n", + " def download_file(self):\n", + " if not os.path.exists(self.config.local_data_file):\n", + " filename, headers = request.urlretrieve(\n", + " url = self.config.source_URL,\n", + " filename = self.config.local_data_file\n", + " )\n", + " logger.info(f\"{filename} download! with following info: \\n{headers}\")\n", + " else:\n", + " logger.info(f\"File already exists of size: {get_size(Path(self.config.local_data_file))}\") \n", + "\n", + "\n", + " \n", + " def extract_zip_file(self):\n", + " \"\"\"\n", + " zip_file_path: str\n", + " Extracts the zip file into the data directory\n", + " Function returns None\n", + " \"\"\"\n", + " unzip_path = self.config.unzip_dir\n", + " os.makedirs(unzip_path, exist_ok=True)\n", + " with zipfile.ZipFile(self.config.local_data_file, 'r') as zip_ref:\n", + " zip_ref.extractall(unzip_path)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[2023-05-31 10:49:27,686: INFO: common: yaml file: config\\config.yaml loaded successfully]\n", + "[2023-05-31 10:49:27,688: INFO: common: yaml file: params.yaml loaded successfully]\n", + "[2023-05-31 10:49:27,689: INFO: common: created directory at: artifacts]\n", + "[2023-05-31 10:49:27,691: INFO: common: created directory at: artifacts/data_ingestion]\n", + "[2023-05-31 10:49:48,873: INFO: 1170291011: artifacts/data_ingestion/data.zip download! with following info: \n", + "Connection: close\n", + "Content-Length: 11616915\n", + "Cache-Control: max-age=300\n", + "Content-Security-Policy: default-src 'none'; style-src 'unsafe-inline'; sandbox\n", + "Content-Type: application/zip\n", + "ETag: \"adf745abc03891fe493c3be264ec012691fe3fa21d861f35a27edbe6d86a76b1\"\n", + "Strict-Transport-Security: max-age=31536000\n", + "X-Content-Type-Options: nosniff\n", + "X-Frame-Options: deny\n", + "X-XSS-Protection: 1; mode=block\n", + "X-GitHub-Request-Id: DD26:594D:46A0F2:51208A:6476D1D8\n", + "Accept-Ranges: bytes\n", + "Date: Wed, 31 May 2023 04:49:29 GMT\n", + "Via: 1.1 varnish\n", + "X-Served-By: cache-hkg17929-HKG\n", + "X-Cache: MISS\n", + "X-Cache-Hits: 0\n", + "X-Timer: S1685508569.655993,VS0,VE951\n", + "Vary: Authorization,Accept-Encoding,Origin\n", + "Access-Control-Allow-Origin: *\n", + "X-Fastly-Request-ID: ef2c1333b2cc4998b0020b1c7edf46d7a603aa90\n", + "Expires: Wed, 31 May 2023 04:54:29 GMT\n", + "Source-Age: 1\n", + "\n", + "]\n" + ] + } + ], + "source": [ + "try:\n", + " config = ConfigurationManager()\n", + " data_ingestion_config = config.get_data_ingestion_config()\n", + " data_ingestion = DataIngestion(config=data_ingestion_config)\n", + " data_ingestion.download_file()\n", + " data_ingestion.extract_zip_file()\n", + "except Exception as e:\n", + " raise e" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "chicken", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.16" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/Chicken_Disease_Classification/research/02_prepare_base_model.ipynb b/Chicken_Disease_Classification/research/02_prepare_base_model.ipynb new file mode 100644 index 0000000000..8d71752aa2 --- /dev/null +++ b/Chicken_Disease_Classification/research/02_prepare_base_model.ipynb @@ -0,0 +1,321 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import os" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'d:\\\\Bappy\\\\YouTube\\\\Chicken-Disease-Classification--Project\\\\research'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%pwd" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "os.chdir(\"../\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'d:\\\\Bappy\\\\YouTube\\\\Chicken-Disease-Classification--Project'" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%pwd" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "from dataclasses import dataclass\n", + "from pathlib import Path\n", + "\n", + "\n", + "@dataclass(frozen=True)\n", + "class PrepareBaseModelConfig:\n", + " root_dir: Path\n", + " base_model_path: Path\n", + " updated_base_model_path: Path\n", + " params_image_size: list\n", + " params_learning_rate: float\n", + " params_include_top: bool\n", + " params_weights: str\n", + " params_classes: int" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "from cnnClassifier.constants import *\n", + "from cnnClassifier.utils.common import read_yaml, create_directories" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "class ConfigurationManager:\n", + " def __init__(\n", + " self, \n", + " config_filepath = CONFIG_FILE_PATH,\n", + " params_filepath = PARAMS_FILE_PATH):\n", + " self.config = read_yaml(config_filepath)\n", + " self.params = read_yaml(params_filepath)\n", + " create_directories([self.config.artifacts_root])\n", + "\n", + "\n", + " def get_prepare_base_model_config(self) -> PrepareBaseModelConfig:\n", + " config = self.config.prepare_base_model\n", + " \n", + " create_directories([config.root_dir])\n", + "\n", + " prepare_base_model_config = PrepareBaseModelConfig(\n", + " root_dir=Path(config.root_dir),\n", + " base_model_path=Path(config.base_model_path),\n", + " updated_base_model_path=Path(config.updated_base_model_path),\n", + " params_image_size=self.params.IMAGE_SIZE,\n", + " params_learning_rate=self.params.LEARNING_RATE,\n", + " params_include_top=self.params.INCLUDE_TOP,\n", + " params_weights=self.params.WEIGHTS,\n", + " params_classes=self.params.CLASSES\n", + " )\n", + "\n", + " return prepare_base_model_config\n" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import urllib.request as request\n", + "from zipfile import ZipFile\n", + "import tensorflow as tf" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "class PrepareBaseModel:\n", + " def __init__(self, config: PrepareBaseModelConfig):\n", + " self.config = config\n", + "\n", + "\n", + " \n", + " def get_base_model(self):\n", + " self.model = tf.keras.applications.vgg16.VGG16(\n", + " input_shape=self.config.params_image_size,\n", + " weights=self.config.params_weights,\n", + " include_top=self.config.params_include_top\n", + " )\n", + "\n", + " self.save_model(path=self.config.base_model_path, model=self.model)\n", + "\n", + "\n", + " \n", + " @staticmethod\n", + " def _prepare_full_model(model, classes, freeze_all, freeze_till, learning_rate):\n", + " if freeze_all:\n", + " for layer in model.layers:\n", + " model.trainable = False\n", + " elif (freeze_till is not None) and (freeze_till > 0):\n", + " for layer in model.layers[:-freeze_till]:\n", + " model.trainable = False\n", + "\n", + " flatten_in = tf.keras.layers.Flatten()(model.output)\n", + " prediction = tf.keras.layers.Dense(\n", + " units=classes,\n", + " activation=\"softmax\"\n", + " )(flatten_in)\n", + "\n", + " full_model = tf.keras.models.Model(\n", + " inputs=model.input,\n", + " outputs=prediction\n", + " )\n", + "\n", + " full_model.compile(\n", + " optimizer=tf.keras.optimizers.SGD(learning_rate=learning_rate),\n", + " loss=tf.keras.losses.CategoricalCrossentropy(),\n", + " metrics=[\"accuracy\"]\n", + " )\n", + "\n", + " full_model.summary()\n", + " return full_model\n", + " \n", + "\n", + " def update_base_model(self):\n", + " self.full_model = self._prepare_full_model(\n", + " model=self.model,\n", + " classes=self.config.params_classes,\n", + " freeze_all=True,\n", + " freeze_till=None,\n", + " learning_rate=self.config.params_learning_rate\n", + " )\n", + "\n", + " self.save_model(path=self.config.updated_base_model_path, model=self.full_model)\n", + "\n", + " \n", + " @staticmethod\n", + " def save_model(path: Path, model: tf.keras.Model):\n", + " model.save(path)\n", + "\n", + " \n" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[2023-05-31 13:21:13,143: INFO: common: yaml file: config\\config.yaml loaded successfully]\n", + "[2023-05-31 13:21:13,145: INFO: common: yaml file: params.yaml loaded successfully]\n", + "[2023-05-31 13:21:13,147: INFO: common: created directory at: artifacts]\n", + "[2023-05-31 13:21:13,148: INFO: common: created directory at: artifacts/prepare_base_model]\n", + "[2023-05-31 13:21:13,571: WARNING: saving_utils: Compiled the loaded model, but the compiled metrics have yet to be built. `model.compile_metrics` will be empty until you train or evaluate the model.]\n", + "Model: \"model\"\n", + "_________________________________________________________________\n", + " Layer (type) Output Shape Param # \n", + "=================================================================\n", + " input_1 (InputLayer) [(None, 224, 224, 3)] 0 \n", + " \n", + " block1_conv1 (Conv2D) (None, 224, 224, 64) 1792 \n", + " \n", + " block1_conv2 (Conv2D) (None, 224, 224, 64) 36928 \n", + " \n", + " block1_pool (MaxPooling2D) (None, 112, 112, 64) 0 \n", + " \n", + " block2_conv1 (Conv2D) (None, 112, 112, 128) 73856 \n", + " \n", + " block2_conv2 (Conv2D) (None, 112, 112, 128) 147584 \n", + " \n", + " block2_pool (MaxPooling2D) (None, 56, 56, 128) 0 \n", + " \n", + " block3_conv1 (Conv2D) (None, 56, 56, 256) 295168 \n", + " \n", + " block3_conv2 (Conv2D) (None, 56, 56, 256) 590080 \n", + " \n", + " block3_conv3 (Conv2D) (None, 56, 56, 256) 590080 \n", + " \n", + " block3_pool (MaxPooling2D) (None, 28, 28, 256) 0 \n", + " \n", + " block4_conv1 (Conv2D) (None, 28, 28, 512) 1180160 \n", + " \n", + " block4_conv2 (Conv2D) (None, 28, 28, 512) 2359808 \n", + " \n", + " block4_conv3 (Conv2D) (None, 28, 28, 512) 2359808 \n", + " \n", + " block4_pool (MaxPooling2D) (None, 14, 14, 512) 0 \n", + " \n", + " block5_conv1 (Conv2D) (None, 14, 14, 512) 2359808 \n", + " \n", + " block5_conv2 (Conv2D) (None, 14, 14, 512) 2359808 \n", + " \n", + " block5_conv3 (Conv2D) (None, 14, 14, 512) 2359808 \n", + " \n", + " block5_pool (MaxPooling2D) (None, 7, 7, 512) 0 \n", + " \n", + " flatten (Flatten) (None, 25088) 0 \n", + " \n", + " dense (Dense) (None, 2) 50178 \n", + " \n", + "=================================================================\n", + "Total params: 14,764,866\n", + "Trainable params: 50,178\n", + "Non-trainable params: 14,714,688\n", + "_________________________________________________________________\n" + ] + } + ], + "source": [ + "try:\n", + " config = ConfigurationManager()\n", + " prepare_base_model_config = config.get_prepare_base_model_config()\n", + " prepare_base_model = PrepareBaseModel(config=prepare_base_model_config)\n", + " prepare_base_model.get_base_model()\n", + " prepare_base_model.update_base_model()\n", + "except Exception as e:\n", + " raise e" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "chicken", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.16" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/Chicken_Disease_Classification/research/03_prepare_callbacks.ipynb b/Chicken_Disease_Classification/research/03_prepare_callbacks.ipynb new file mode 100644 index 0000000000..b1a0deed6c --- /dev/null +++ b/Chicken_Disease_Classification/research/03_prepare_callbacks.ipynb @@ -0,0 +1,254 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import os" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'d:\\\\Bappy\\\\YouTube\\\\Chicken-Disease-Classification--Project\\\\research'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%pwd" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "os.chdir(\"../\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'d:\\\\Bappy\\\\YouTube\\\\Chicken-Disease-Classification--Project'" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%pwd" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "from dataclasses import dataclass\n", + "from pathlib import Path\n", + "\n", + "\n", + "@dataclass(frozen=True)\n", + "class PrepareCallbacksConfig:\n", + " root_dir: Path\n", + " tensorboard_root_log_dir: Path\n", + " checkpoint_model_filepath: Path" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "from cnnClassifier.constants import *\n", + "from cnnClassifier.utils.common import read_yaml, create_directories" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "class ConfigurationManager:\n", + " def __init__(\n", + " self, \n", + " config_filepath = CONFIG_FILE_PATH,\n", + " params_filepath = PARAMS_FILE_PATH):\n", + " self.config = read_yaml(config_filepath)\n", + " self.params = read_yaml(params_filepath)\n", + " create_directories([self.config.artifacts_root])\n", + "\n", + "\n", + " \n", + " def get_prepare_callback_config(self) -> PrepareCallbacksConfig:\n", + " config = self.config.prepare_callbacks\n", + " model_ckpt_dir = os.path.dirname(config.checkpoint_model_filepath)\n", + " create_directories([\n", + " Path(model_ckpt_dir),\n", + " Path(config.tensorboard_root_log_dir)\n", + " ])\n", + "\n", + " prepare_callback_config = PrepareCallbacksConfig(\n", + " root_dir=Path(config.root_dir),\n", + " tensorboard_root_log_dir=Path(config.tensorboard_root_log_dir),\n", + " checkpoint_model_filepath=Path(config.checkpoint_model_filepath)\n", + " )\n", + "\n", + " return prepare_callback_config\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import urllib.request as request\n", + "from zipfile import ZipFile\n", + "import tensorflow as tf\n", + "import time" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "class PrepareCallback:\n", + " def __init__(self, config: PrepareCallbacksConfig):\n", + " self.config = config\n", + "\n", + "\n", + " \n", + " @property\n", + " def _create_tb_callbacks(self):\n", + " timestamp = time.strftime(\"%Y-%m-%d-%H-%M-%S\")\n", + " tb_running_log_dir = os.path.join(\n", + " self.config.tensorboard_root_log_dir,\n", + " f\"tb_logs_at_{timestamp}\",\n", + " )\n", + " return tf.keras.callbacks.TensorBoard(log_dir=tb_running_log_dir)\n", + " \n", + "\n", + " @property\n", + " def _create_ckpt_callbacks(self):\n", + " return tf.keras.callbacks.ModelCheckpoint(\n", + " filepath=self.config.checkpoint_model_filepath,\n", + " save_best_only=True\n", + " )\n", + "\n", + "\n", + " def get_tb_ckpt_callbacks(self):\n", + " return [\n", + " self._create_tb_callbacks,\n", + " self._create_ckpt_callbacks\n", + " ]\n" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[2023-05-31 14:37:12,294: INFO: common: yaml file: config\\config.yaml loaded successfully]\n", + "[2023-05-31 14:37:12,303: INFO: common: yaml file: params.yaml loaded successfully]\n", + "[2023-05-31 14:37:12,305: INFO: common: created directory at: artifacts]\n", + "[2023-05-31 14:37:12,307: INFO: common: created directory at: artifacts\\prepare_callbacks\\checkpoint_dir]\n", + "[2023-05-31 14:37:12,310: INFO: common: created directory at: artifacts\\prepare_callbacks\\tensorboard_log_dir]\n" + ] + } + ], + "source": [ + "try:\n", + " config = ConfigurationManager()\n", + " prepare_callbacks_config = config.get_prepare_callback_config()\n", + " prepare_callbacks = PrepareCallback(config=prepare_callbacks_config)\n", + " callback_list = prepare_callbacks.get_tb_ckpt_callbacks()\n", + " \n", + "except Exception as e:\n", + " raise e" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'tb_logs_at_2023-05-31-14-38-20'" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import time\n", + "timestamp = time.strftime(\"%Y-%m-%d-%H-%M-%S\")\n", + "f\"tb_logs_at_{timestamp}\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "chicken", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.16" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/Chicken_Disease_Classification/research/04_training.ipynb b/Chicken_Disease_Classification/research/04_training.ipynb new file mode 100644 index 0000000000..5135b5efaf --- /dev/null +++ b/Chicken_Disease_Classification/research/04_training.ipynb @@ -0,0 +1,375 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import os" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'d:\\\\Bappy\\\\YouTube\\\\Chicken-Disease-Classification--Project\\\\research'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%pwd" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "os.chdir(\"../\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'d:\\\\Bappy\\\\YouTube\\\\Chicken-Disease-Classification--Project'" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%pwd" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "from dataclasses import dataclass\n", + "from pathlib import Path\n", + "\n", + "\n", + "@dataclass(frozen=True)\n", + "class TrainingConfig:\n", + " root_dir: Path\n", + " trained_model_path: Path\n", + " updated_base_model_path: Path\n", + " training_data: Path\n", + " params_epochs: int\n", + " params_batch_size: int\n", + " params_is_augmentation: bool\n", + " params_image_size: list\n", + "\n", + "\n", + "\n", + "@dataclass(frozen=True)\n", + "class PrepareCallbacksConfig:\n", + " root_dir: Path\n", + " tensorboard_root_log_dir: Path\n", + " checkpoint_model_filepath: Path" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "from cnnClassifier.constants import *\n", + "from cnnClassifier.utils.common import read_yaml, create_directories\n", + "import tensorflow as tf" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "class ConfigurationManager:\n", + " def __init__(\n", + " self, \n", + " config_filepath = CONFIG_FILE_PATH,\n", + " params_filepath = PARAMS_FILE_PATH):\n", + " self.config = read_yaml(config_filepath)\n", + " self.params = read_yaml(params_filepath)\n", + " create_directories([self.config.artifacts_root])\n", + "\n", + "\n", + " \n", + " def get_prepare_callback_config(self) -> PrepareCallbacksConfig:\n", + " config = self.config.prepare_callbacks\n", + " model_ckpt_dir = os.path.dirname(config.checkpoint_model_filepath)\n", + " create_directories([\n", + " Path(model_ckpt_dir),\n", + " Path(config.tensorboard_root_log_dir)\n", + " ])\n", + "\n", + " prepare_callback_config = PrepareCallbacksConfig(\n", + " root_dir=Path(config.root_dir),\n", + " tensorboard_root_log_dir=Path(config.tensorboard_root_log_dir),\n", + " checkpoint_model_filepath=Path(config.checkpoint_model_filepath)\n", + " )\n", + "\n", + " return prepare_callback_config\n", + " \n", + "\n", + "\n", + "\n", + "\n", + " def get_training_config(self) -> TrainingConfig:\n", + " training = self.config.training\n", + " prepare_base_model = self.config.prepare_base_model\n", + " params = self.params\n", + " training_data = os.path.join(self.config.data_ingestion.unzip_dir, \"Chicken-fecal-images\")\n", + " create_directories([\n", + " Path(training.root_dir)\n", + " ])\n", + "\n", + " training_config = TrainingConfig(\n", + " root_dir=Path(training.root_dir),\n", + " trained_model_path=Path(training.trained_model_path),\n", + " updated_base_model_path=Path(prepare_base_model.updated_base_model_path),\n", + " training_data=Path(training_data),\n", + " params_epochs=params.EPOCHS,\n", + " params_batch_size=params.BATCH_SIZE,\n", + " params_is_augmentation=params.AUGMENTATION,\n", + " params_image_size=params.IMAGE_SIZE\n", + " )\n", + "\n", + " return training_config" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "import time" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "class PrepareCallback:\n", + " def __init__(self, config: PrepareCallbacksConfig):\n", + " self.config = config\n", + "\n", + "\n", + " \n", + " @property\n", + " def _create_tb_callbacks(self):\n", + " timestamp = time.strftime(\"%Y-%m-%d-%H-%M-%S\")\n", + " tb_running_log_dir = os.path.join(\n", + " self.config.tensorboard_root_log_dir,\n", + " f\"tb_logs_at_{timestamp}\",\n", + " )\n", + " return tf.keras.callbacks.TensorBoard(log_dir=tb_running_log_dir)\n", + " \n", + "\n", + " @property\n", + " def _create_ckpt_callbacks(self):\n", + " return tf.keras.callbacks.ModelCheckpoint(\n", + " filepath=self.config.checkpoint_model_filepath,\n", + " save_best_only=True\n", + " )\n", + "\n", + "\n", + " def get_tb_ckpt_callbacks(self):\n", + " return [\n", + " self._create_tb_callbacks,\n", + " self._create_ckpt_callbacks\n", + " ]\n" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import urllib.request as request\n", + "from zipfile import ZipFile\n", + "import tensorflow as tf\n", + "import time" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "class Training:\n", + " def __init__(self, config: TrainingConfig):\n", + " self.config = config\n", + " \n", + " def get_base_model(self):\n", + " self.model = tf.keras.models.load_model(\n", + " self.config.updated_base_model_path\n", + " )\n", + " \n", + " def train_valid_generator(self):\n", + "\n", + " datagenerator_kwargs = dict(\n", + " rescale = 1./255,\n", + " validation_split=0.20\n", + " )\n", + "\n", + " dataflow_kwargs = dict(\n", + " target_size=self.config.params_image_size[:-1],\n", + " batch_size=self.config.params_batch_size,\n", + " interpolation=\"bilinear\"\n", + " )\n", + "\n", + " valid_datagenerator = tf.keras.preprocessing.image.ImageDataGenerator(\n", + " **datagenerator_kwargs\n", + " )\n", + "\n", + " self.valid_generator = valid_datagenerator.flow_from_directory(\n", + " directory=self.config.training_data,\n", + " subset=\"validation\",\n", + " shuffle=False,\n", + " **dataflow_kwargs\n", + " )\n", + "\n", + " if self.config.params_is_augmentation:\n", + " train_datagenerator = tf.keras.preprocessing.image.ImageDataGenerator(\n", + " rotation_range=40,\n", + " horizontal_flip=True,\n", + " width_shift_range=0.2,\n", + " height_shift_range=0.2,\n", + " shear_range=0.2,\n", + " zoom_range=0.2,\n", + " **datagenerator_kwargs\n", + " )\n", + " else:\n", + " train_datagenerator = valid_datagenerator\n", + "\n", + " self.train_generator = train_datagenerator.flow_from_directory(\n", + " directory=self.config.training_data,\n", + " subset=\"training\",\n", + " shuffle=True,\n", + " **dataflow_kwargs\n", + " )\n", + "\n", + " @staticmethod\n", + " def save_model(path: Path, model: tf.keras.Model):\n", + " model.save(path)\n", + "\n", + "\n", + " def train(self, callback_list: list):\n", + " self.steps_per_epoch = self.train_generator.samples // self.train_generator.batch_size\n", + " self.validation_steps = self.valid_generator.samples // self.valid_generator.batch_size\n", + "\n", + " self.model.fit(\n", + " self.train_generator,\n", + " epochs=self.config.params_epochs,\n", + " steps_per_epoch=self.steps_per_epoch,\n", + " validation_steps=self.validation_steps,\n", + " validation_data=self.valid_generator,\n", + " callbacks=callback_list\n", + " )\n", + "\n", + " self.save_model(\n", + " path=self.config.trained_model_path,\n", + " model=self.model\n", + " )\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[2023-06-01 10:18:59,787: INFO: common: yaml file: config\\config.yaml loaded successfully]\n", + "[2023-06-01 10:18:59,792: INFO: common: yaml file: params.yaml loaded successfully]\n", + "[2023-06-01 10:18:59,794: INFO: common: created directory at: artifacts]\n", + "[2023-06-01 10:18:59,795: INFO: common: created directory at: artifacts\\prepare_callbacks\\checkpoint_dir]\n", + "[2023-06-01 10:18:59,796: INFO: common: created directory at: artifacts\\prepare_callbacks\\tensorboard_log_dir]\n", + "[2023-06-01 10:18:59,797: INFO: common: created directory at: artifacts\\training]\n", + "Found 78 images belonging to 2 classes.\n", + "Found 312 images belonging to 2 classes.\n", + "19/19 [==============================] - 28s 1s/step - loss: 13.6681 - accuracy: 0.5270 - val_loss: 19.1089 - val_accuracy: 0.3906\n" + ] + } + ], + "source": [ + "try:\n", + " config = ConfigurationManager()\n", + " prepare_callbacks_config = config.get_prepare_callback_config()\n", + " prepare_callbacks = PrepareCallback(config=prepare_callbacks_config)\n", + " callback_list = prepare_callbacks.get_tb_ckpt_callbacks()\n", + "\n", + " training_config = config.get_training_config()\n", + " training = Training(config=training_config)\n", + " training.get_base_model()\n", + " training.train_valid_generator()\n", + " training.train(\n", + " callback_list=callback_list\n", + " )\n", + " \n", + "except Exception as e:\n", + " raise e" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "chicken", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.16" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/Chicken_Disease_Classification/research/05_model_evaluation_with_mlflow.ipynb b/Chicken_Disease_Classification/research/05_model_evaluation_with_mlflow.ipynb new file mode 100644 index 0000000000..5dc98d47bf --- /dev/null +++ b/Chicken_Disease_Classification/research/05_model_evaluation_with_mlflow.ipynb @@ -0,0 +1,324 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import os" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'d:\\\\Bappy\\\\YouTube\\\\Chicken-mlflow-dvc\\\\research'" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%pwd" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "os.chdir(\"../\")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'d:\\\\Bappy\\\\YouTube\\\\Chicken-mlflow-dvc'" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%pwd" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "os.environ[\"MLFLOW_TRACKING_URI\"]=\"https://dagshub.com/entbappy/MLflow-DVC-Chicken-Disease-Classification.mlflow\"\n", + "os.environ[\"MLFLOW_TRACKING_USERNAME\"]=\"entbappy\"\n", + "os.environ[\"MLFLOW_TRACKING_PASSWORD\"]=\"6824692c47a369aa6f9eac5b10041d5c8edbcef0\"" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "import tensorflow as tf" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "model = tf.keras.models.load_model(\"artifacts/training/model.h5\")" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "from dataclasses import dataclass\n", + "from pathlib import Path\n", + "\n", + "@dataclass(frozen=True)\n", + "class EvaluationConfig:\n", + " path_of_model: Path\n", + " training_data: Path\n", + " all_params: dict\n", + " mlflow_uri: str\n", + " params_image_size: list\n", + " params_batch_size: int\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "from cnnClassifier.constants import *\n", + "from cnnClassifier.utils.common import read_yaml, create_directories, save_json" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "class ConfigurationManager:\n", + " def __init__(\n", + " self, \n", + " config_filepath = CONFIG_FILE_PATH,\n", + " params_filepath = PARAMS_FILE_PATH):\n", + " self.config = read_yaml(config_filepath)\n", + " self.params = read_yaml(params_filepath)\n", + " create_directories([self.config.artifacts_root])\n", + "\n", + " \n", + " def get_evaluation_config(self) -> EvaluationConfig:\n", + " eval_config = EvaluationConfig(\n", + " path_of_model=\"artifacts/training/model.h5\",\n", + " training_data=\"artifacts/data_ingestion/Chicken-fecal-images\",\n", + " mlflow_uri=\"https://dagshub.com/entbappy/MLflow-DVC-Chicken-Disease-Classification.mlflow\",\n", + " all_params=self.params,\n", + " params_image_size=self.params.IMAGE_SIZE,\n", + " params_batch_size=self.params.BATCH_SIZE\n", + " )\n", + " return eval_config\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "import tensorflow as tf\n", + "from pathlib import Path\n", + "import mlflow\n", + "import mlflow.keras\n", + "from urllib.parse import urlparse" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "class Evaluation:\n", + " def __init__(self, config: EvaluationConfig):\n", + " self.config = config\n", + "\n", + " \n", + " def _valid_generator(self):\n", + "\n", + " datagenerator_kwargs = dict(\n", + " rescale = 1./255,\n", + " validation_split=0.30\n", + " )\n", + "\n", + " dataflow_kwargs = dict(\n", + " target_size=self.config.params_image_size[:-1],\n", + " batch_size=self.config.params_batch_size,\n", + " interpolation=\"bilinear\"\n", + " )\n", + "\n", + " valid_datagenerator = tf.keras.preprocessing.image.ImageDataGenerator(\n", + " **datagenerator_kwargs\n", + " )\n", + "\n", + " self.valid_generator = valid_datagenerator.flow_from_directory(\n", + " directory=self.config.training_data,\n", + " subset=\"validation\",\n", + " shuffle=False,\n", + " **dataflow_kwargs\n", + " )\n", + "\n", + " \n", + " @staticmethod\n", + " def load_model(path: Path) -> tf.keras.Model:\n", + " return tf.keras.models.load_model(path)\n", + " \n", + "\n", + " def evaluation(self):\n", + " self.model = self.load_model(self.config.path_of_model)\n", + " self._valid_generator()\n", + " self.score = model.evaluate(self.valid_generator)\n", + "\n", + " \n", + " def save_score(self):\n", + " scores = {\"loss\": self.score[0], \"accuracy\": self.score[1]}\n", + " save_json(path=Path(\"scores.json\"), data=scores)\n", + "\n", + " \n", + "\n", + " def log_into_mlflow(self):\n", + " mlflow.set_registry_uri(self.config.mlflow_uri)\n", + " tracking_url_type_store = urlparse(mlflow.get_tracking_uri()).scheme\n", + " \n", + " with mlflow.start_run():\n", + " mlflow.log_params(self.config.all_params)\n", + " mlflow.log_metrics(\n", + " {\"loss\": self.score[0], \"accuracy\": self.score[1]}\n", + " )\n", + " # Model registry does not work with file store\n", + " if tracking_url_type_store != \"file\":\n", + "\n", + " # Register the model\n", + " # There are other ways to use the Model Registry, which depends on the use case,\n", + " # please refer to the doc for more information:\n", + " # https://mlflow.org/docs/latest/model-registry.html#api-workflow\n", + " mlflow.keras.log_model(self.model, \"model\", registered_model_name=\"VGG16Model\")\n", + " else:\n", + " mlflow.keras.log_model(self.model, \"model\")\n", + "\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[2023-06-17 12:10:47,659: INFO: common: yaml file: config\\config.yaml loaded successfully]\n", + "[2023-06-17 12:10:47,661: INFO: common: yaml file: params.yaml loaded successfully]\n", + "[2023-06-17 12:10:47,662: INFO: common: created directory at: artifacts]\n", + "Found 116 images belonging to 2 classes.\n", + "8/8 [==============================] - 7s 833ms/step - loss: 1.8001 - accuracy: 0.7328\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023/06/17 12:10:56 WARNING mlflow.tensorflow: You are saving a TensorFlow Core model or Keras model without a signature. Inference with mlflow.pyfunc.spark_udf() will not work unless the model's pyfunc representation accepts pandas DataFrames as inference inputs.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[2023-06-17 12:10:57,841: WARNING: save: Found untraced functions such as _jit_compiled_convolution_op, _jit_compiled_convolution_op, _jit_compiled_convolution_op, _jit_compiled_convolution_op, _jit_compiled_convolution_op while saving (showing 5 of 14). These functions will not be directly callable after loading.]\n", + "INFO:tensorflow:Assets written to: C:\\Users\\bokti\\AppData\\Local\\Temp\\tmp6vzrgib0\\model\\data\\model\\assets\n", + "[2023-06-17 12:10:58,318: INFO: builder_impl: Assets written to: C:\\Users\\bokti\\AppData\\Local\\Temp\\tmp6vzrgib0\\model\\data\\model\\assets]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "d:\\Softwares\\anaconda3\\envs\\chicken\\lib\\site-packages\\_distutils_hack\\__init__.py:33: UserWarning: Setuptools is replacing distutils.\n", + " warnings.warn(\"Setuptools is replacing distutils.\")\n", + "Registered model 'VGG16Model' already exists. Creating a new version of this model...\n", + "2023/06/17 12:12:04 INFO mlflow.tracking._model_registry.client: Waiting up to 300 seconds for model version to finish creation. Model name: VGG16Model, version 2\n", + "Created version '2' of model 'VGG16Model'.\n" + ] + } + ], + "source": [ + "try:\n", + " config = ConfigurationManager()\n", + " eval_config = config.get_evaluation_config()\n", + " evaluation = Evaluation(eval_config)\n", + " evaluation.evaluation()\n", + " evaluation.log_into_mlflow()\n", + "\n", + "except Exception as e:\n", + " raise e" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "chicken", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.16" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/Chicken_Disease_Classification/research/mlflow_exp/argv_demo.py b/Chicken_Disease_Classification/research/mlflow_exp/argv_demo.py new file mode 100644 index 0000000000..0831243680 --- /dev/null +++ b/Chicken_Disease_Classification/research/mlflow_exp/argv_demo.py @@ -0,0 +1,11 @@ +import sys + +default = 0.3 + +alpha = float(sys.argv[1]) if len(sys.argv) > 1 else default +l1_ratio = float(sys.argv[2]) if len(sys.argv) > 2 else default + +args = sys.argv +print(args) + +print(alpha, l1_ratio) \ No newline at end of file diff --git a/Chicken_Disease_Classification/research/mlflow_exp/example.py b/Chicken_Disease_Classification/research/mlflow_exp/example.py new file mode 100644 index 0000000000..292e2f247f --- /dev/null +++ b/Chicken_Disease_Classification/research/mlflow_exp/example.py @@ -0,0 +1,96 @@ +# The data set used in this example is from http://archive.ics.uci.edu/ml/datasets/Wine+Quality +# P. Cortez, A. Cerdeira, F. Almeida, T. Matos and J. Reis. +# Modeling wine preferences by data mining from physicochemical properties. In Decision Support Systems, Elsevier, 47(4):547-553, 2009. + +import os +import warnings +import sys + +import pandas as pd +import numpy as np +from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score +from sklearn.model_selection import train_test_split +from sklearn.linear_model import ElasticNet +from urllib.parse import urlparse +import mlflow +import mlflow.sklearn + +import logging + +logging.basicConfig(level=logging.WARN) +logger = logging.getLogger(__name__) + + +def eval_metrics(actual, pred): + rmse = np.sqrt(mean_squared_error(actual, pred)) + mae = mean_absolute_error(actual, pred) + r2 = r2_score(actual, pred) + return rmse, mae, r2 + + +if __name__ == "__main__": + warnings.filterwarnings("ignore") + np.random.seed(40) + + # Read the wine-quality csv file from the URL + csv_url = ( + "http://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-red.csv" + ) + try: + data = pd.read_csv(csv_url, sep=";") + except Exception as e: + logger.exception( + "Unable to download training & test CSV, check your internet connection. Error: %s", e + ) + + # Split the data into training and test sets. (0.75, 0.25) split. + train, test = train_test_split(data) + + # The predicted column is "quality" which is a scalar from [3, 9] + train_x = train.drop(["quality"], axis=1) + test_x = test.drop(["quality"], axis=1) + train_y = train[["quality"]] + test_y = test[["quality"]] + + alpha = float(sys.argv[1]) if len(sys.argv) > 1 else 0.5 + l1_ratio = float(sys.argv[2]) if len(sys.argv) > 2 else 0.5 + + with mlflow.start_run(): + lr = ElasticNet(alpha=alpha, l1_ratio=l1_ratio, random_state=42) + lr.fit(train_x, train_y) + + predicted_qualities = lr.predict(test_x) + + (rmse, mae, r2) = eval_metrics(test_y, predicted_qualities) + + print("Elasticnet model (alpha=%f, l1_ratio=%f):" % (alpha, l1_ratio)) + print(" RMSE: %s" % rmse) + print(" MAE: %s" % mae) + print(" R2: %s" % r2) + + mlflow.log_param("alpha", alpha) + mlflow.log_param("l1_ratio", l1_ratio) + mlflow.log_metric("rmse", rmse) + mlflow.log_metric("r2", r2) + mlflow.log_metric("mae", mae) + + + + # For remote server only + remote_server_uri = "https://dagshub.com/entbappy/MLflow-DVC-Chicken-Disease-Classification.mlflow" + mlflow.set_tracking_uri(remote_server_uri) + + + + tracking_url_type_store = urlparse(mlflow.get_tracking_uri()).scheme + + # Model registry does not work with file store + if tracking_url_type_store != "file": + + # Register the model + # There are other ways to use the Model Registry, which depends on the use case, + # please refer to the doc for more information: + # https://mlflow.org/docs/latest/model-registry.html#api-workflow + mlflow.sklearn.log_model(lr, "model", registered_model_name="ElasticnetWineModel") + else: + mlflow.sklearn.log_model(lr, "model") \ No newline at end of file diff --git a/Chicken_Disease_Classification/research/trials.ipynb b/Chicken_Disease_Classification/research/trials.ipynb new file mode 100644 index 0000000000..e5e6c9fee3 --- /dev/null +++ b/Chicken_Disease_Classification/research/trials.ipynb @@ -0,0 +1,253 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "d = {\"key\":\"value\", \"key1\":\"value1\"}" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'value'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "d['key']" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "ename": "AttributeError", + "evalue": "'dict' object has no attribute 'key'", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mAttributeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[1;32mIn[3], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m d\u001b[39m.\u001b[39;49mkey\n", + "\u001b[1;31mAttributeError\u001b[0m: 'dict' object has no attribute 'key'" + ] + } + ], + "source": [ + "d.key" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "from box import ConfigBox" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "d2 = ConfigBox({\"key\":\"value\", \"key1\":\"value1\"})" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "ConfigBox({'key': 'value', 'key1': 'value1'})" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "d2" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'value'" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "d2.key" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "def get_prodict(x:int, y:int) -> int:\n", + " return x*y" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "8" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "get_prodict(x=2, y=4)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'44'" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "get_prodict(x=2, y=\"4\")" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "from ensure import ensure_annotations" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "@ensure_annotations\n", + "def get_prodict(x:int, y:int) -> int:\n", + " return x*y" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "8" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "get_prodict(x=2, y=4)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "ename": "EnsureError", + "evalue": "Argument y of type to does not match annotation type ", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mEnsureError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[1;32mIn[14], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m get_prodict(x\u001b[39m=\u001b[39;49m\u001b[39m2\u001b[39;49m, y\u001b[39m=\u001b[39;49m\u001b[39m\"\u001b[39;49m\u001b[39m4\u001b[39;49m\u001b[39m\"\u001b[39;49m)\n", + "File \u001b[1;32md:\\Softwares\\anaconda3\\envs\\chicken\\lib\\site-packages\\ensure\\main.py:845\u001b[0m, in \u001b[0;36mWrappedFunctionReturn.__call__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m 840\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39misinstance\u001b[39m(value, templ):\n\u001b[0;32m 841\u001b[0m msg \u001b[39m=\u001b[39m (\n\u001b[0;32m 842\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mArgument \u001b[39m\u001b[39m{arg}\u001b[39;00m\u001b[39m of type \u001b[39m\u001b[39m{valt}\u001b[39;00m\u001b[39m to \u001b[39m\u001b[39m{f}\u001b[39;00m\u001b[39m \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[0;32m 843\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mdoes not match annotation type \u001b[39m\u001b[39m{t}\u001b[39;00m\u001b[39m\"\u001b[39m\n\u001b[0;32m 844\u001b[0m )\n\u001b[1;32m--> 845\u001b[0m \u001b[39mraise\u001b[39;00m EnsureError(msg\u001b[39m.\u001b[39mformat(\n\u001b[0;32m 846\u001b[0m arg\u001b[39m=\u001b[39marg, f\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mf, t\u001b[39m=\u001b[39mtempl, valt\u001b[39m=\u001b[39m\u001b[39mtype\u001b[39m(value)\n\u001b[0;32m 847\u001b[0m ))\n\u001b[0;32m 849\u001b[0m return_val \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mf(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[0;32m 850\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39misinstance\u001b[39m(return_val, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mreturn_templ):\n", + "\u001b[1;31mEnsureError\u001b[0m: Argument y of type to does not match annotation type " + ] + } + ], + "source": [ + "get_prodict(x=2, y=\"4\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "chicken", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.16" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/Chicken_Disease_Classification/scores.json b/Chicken_Disease_Classification/scores.json new file mode 100644 index 0000000000..f3e758cef5 --- /dev/null +++ b/Chicken_Disease_Classification/scores.json @@ -0,0 +1,4 @@ +{ + "loss": 0.4874950647354126, + "accuracy": 0.9137930870056152 +} \ No newline at end of file diff --git a/Chicken_Disease_Classification/setup.py b/Chicken_Disease_Classification/setup.py new file mode 100644 index 0000000000..3a348dca02 --- /dev/null +++ b/Chicken_Disease_Classification/setup.py @@ -0,0 +1,29 @@ +import setuptools + +with open("README.md", "r", encoding="utf-8") as f: + long_description = f.read() + + +__version__ = "0.0.0" + +REPO_NAME = "Chicken-Disease-Classification--Project" +AUTHOR_USER_NAME = "nvrcharan" +SRC_REPO = "cnnClassifier" +AUTHOR_EMAIL = "venkataraghucharan1@gmail.com" + + +setuptools.setup( + name=SRC_REPO, + version=__version__, + author=AUTHOR_USER_NAME, + author_email=AUTHOR_EMAIL, + description="A small python package for CNN app", + long_description=long_description, + long_description_content="text/markdown", + url=f"https://github.com/{AUTHOR_USER_NAME}/{REPO_NAME}", + project_urls={ + "Bug Tracker": f"https://github.com/{AUTHOR_USER_NAME}/{REPO_NAME}/issues", + }, + package_dir={"": "src"}, + packages=setuptools.find_packages(where="src") +) diff --git a/Chicken_Disease_Classification/src/cnnClassifier/__init__.py b/Chicken_Disease_Classification/src/cnnClassifier/__init__.py new file mode 100644 index 0000000000..258f281c36 --- /dev/null +++ b/Chicken_Disease_Classification/src/cnnClassifier/__init__.py @@ -0,0 +1,23 @@ +import os +import sys +import logging + +logging_str = "[%(asctime)s: %(levelname)s: %(module)s: %(message)s]" + +log_dir = "logs" +log_filepath = os.path.join(log_dir,"running_logs.log") +os.makedirs(log_dir, exist_ok=True) + + +logging.basicConfig( + level= logging.INFO, + format= logging_str, + + handlers=[ + logging.FileHandler(log_filepath), + logging.StreamHandler(sys.stdout) + ] +) + +logger = logging.getLogger("cnnClassifierLogger") + diff --git a/Chicken_Disease_Classification/src/cnnClassifier/components/__init__.py b/Chicken_Disease_Classification/src/cnnClassifier/components/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Chicken_Disease_Classification/src/cnnClassifier/components/data_ingestion.py b/Chicken_Disease_Classification/src/cnnClassifier/components/data_ingestion.py new file mode 100644 index 0000000000..638559d3d7 --- /dev/null +++ b/Chicken_Disease_Classification/src/cnnClassifier/components/data_ingestion.py @@ -0,0 +1,38 @@ +import os +import urllib.request as request +import zipfile +from cnnClassifier import logger +from cnnClassifier.utils.common import get_size +from cnnClassifier.entity.config_entity import DataIngestionConfig +from pathlib import Path + + +class DataIngestion: + def __init__(self, config: DataIngestionConfig): + self.config = config + + + + def download_file(self): + if not os.path.exists(self.config.local_data_file): + filename, headers = request.urlretrieve( + url = self.config.source_URL, + filename = self.config.local_data_file + ) + logger.info(f"{filename} download! with following info: \n{headers}") + else: + logger.info(f"File already exists of size: {get_size(Path(self.config.local_data_file))}") + + + + def extract_zip_file(self): + """ + zip_file_path: str + Extracts the zip file into the data directory + Function returns None + """ + unzip_path = self.config.unzip_dir + os.makedirs(unzip_path, exist_ok=True) + with zipfile.ZipFile(self.config.local_data_file, 'r') as zip_ref: + zip_ref.extractall(unzip_path) + diff --git a/Chicken_Disease_Classification/src/cnnClassifier/components/evaluation.py b/Chicken_Disease_Classification/src/cnnClassifier/components/evaluation.py new file mode 100644 index 0000000000..07730717e1 --- /dev/null +++ b/Chicken_Disease_Classification/src/cnnClassifier/components/evaluation.py @@ -0,0 +1,78 @@ +import tensorflow as tf +import mlflow +import mlflow.keras +from urllib.parse import urlparse +from pathlib import Path +from cnnClassifier.entity.config_entity import EvaluationConfig +from cnnClassifier.utils.common import save_json + + +class Evaluation: + def __init__(self, config: EvaluationConfig): + self.config = config + + + def _valid_generator(self): + + datagenerator_kwargs = dict( + rescale = 1./255, + validation_split=0.30 + ) + + dataflow_kwargs = dict( + target_size=self.config.params_image_size[:-1], + batch_size=self.config.params_batch_size, + interpolation="bilinear" + ) + + valid_datagenerator = tf.keras.preprocessing.image.ImageDataGenerator( + **datagenerator_kwargs + ) + + self.valid_generator = valid_datagenerator.flow_from_directory( + directory=self.config.training_data, + subset="validation", + shuffle=False, + **dataflow_kwargs + ) + + + @staticmethod + def load_model(path: Path) -> tf.keras.Model: + return tf.keras.models.load_model(path) + + + def evaluation(self): + self.model = self.load_model(self.config.path_of_model) + self._valid_generator() + self.score = self.model.evaluate(self.valid_generator) + + + def save_score(self): + scores = {"loss": self.score[0], "accuracy": self.score[1]} + save_json(path=Path("scores.json"), data=scores) + + + + def log_into_mlflow(self): + mlflow.set_registry_uri(self.config.mlflow_uri) + tracking_url_type_store = urlparse(mlflow.get_tracking_uri()).scheme + + with mlflow.start_run(): + mlflow.log_params(self.config.all_params) + mlflow.log_metrics( + {"loss": self.score[0], "accuracy": self.score[1]} + ) + # Model registry does not work with file store + if tracking_url_type_store != "file": + + # Register the model + # There are other ways to use the Model Registry, which depends on the use case, + # please refer to the doc for more information: + # https://mlflow.org/docs/latest/model-registry.html#api-workflow + mlflow.keras.log_model(self.model, "model", registered_model_name="VGG16Model") + else: + mlflow.keras.log_model(self.model, "model") + + + diff --git a/Chicken_Disease_Classification/src/cnnClassifier/components/prepare_base_model.py b/Chicken_Disease_Classification/src/cnnClassifier/components/prepare_base_model.py new file mode 100644 index 0000000000..f03c7871c0 --- /dev/null +++ b/Chicken_Disease_Classification/src/cnnClassifier/components/prepare_base_model.py @@ -0,0 +1,73 @@ +import os +import urllib.request as request +from zipfile import ZipFile +import tensorflow as tf +from pathlib import Path +from cnnClassifier.entity.config_entity import PrepareBaseModelConfig + +class PrepareBaseModel: + def __init__(self, config: PrepareBaseModelConfig): + self.config = config + + + + def get_base_model(self): + self.model = tf.keras.applications.vgg16.VGG16( + input_shape=self.config.params_image_size, + weights=self.config.params_weights, + include_top=self.config.params_include_top + ) + + self.save_model(path=self.config.base_model_path, model=self.model) + + + + @staticmethod + def _prepare_full_model(model, classes, freeze_all, freeze_till, learning_rate): + if freeze_all: + for layer in model.layers: + model.trainable = False + elif (freeze_till is not None) and (freeze_till > 0): + for layer in model.layers[:-freeze_till]: + model.trainable = False + + flatten_in = tf.keras.layers.Flatten()(model.output) + prediction = tf.keras.layers.Dense( + units=classes, + activation="softmax" + )(flatten_in) + + full_model = tf.keras.models.Model( + inputs=model.input, + outputs=prediction + ) + + full_model.compile( + optimizer=tf.keras.optimizers.SGD(learning_rate=learning_rate), + loss=tf.keras.losses.CategoricalCrossentropy(), + metrics=["accuracy"] + ) + + full_model.summary() + return full_model + + + def update_base_model(self): + self.full_model = self._prepare_full_model( + model=self.model, + classes=self.config.params_classes, + freeze_all=True, + freeze_till=None, + learning_rate=self.config.params_learning_rate + ) + + self.save_model(path=self.config.updated_base_model_path, model=self.full_model) + + + @staticmethod + def save_model(path: Path, model: tf.keras.Model): + model.save(path) + + + + diff --git a/Chicken_Disease_Classification/src/cnnClassifier/components/prepare_callbacks.py b/Chicken_Disease_Classification/src/cnnClassifier/components/prepare_callbacks.py new file mode 100644 index 0000000000..032f432cdb --- /dev/null +++ b/Chicken_Disease_Classification/src/cnnClassifier/components/prepare_callbacks.py @@ -0,0 +1,37 @@ +import os +import urllib.request as request +from zipfile import ZipFile +import tensorflow as tf +import time +from cnnClassifier.entity.config_entity import PrepareCallbacksConfig + + +class PrepareCallback: + def __init__(self, config: PrepareCallbacksConfig): + self.config = config + + + + @property + def _create_tb_callbacks(self): + timestamp = time.strftime("%Y-%m-%d-%H-%M-%S") + tb_running_log_dir = os.path.join( + self.config.tensorboard_root_log_dir, + f"tb_logs_at_{timestamp}", + ) + return tf.keras.callbacks.TensorBoard(log_dir=tb_running_log_dir) + + + @property + def _create_ckpt_callbacks(self): + return tf.keras.callbacks.ModelCheckpoint( + filepath=self.config.checkpoint_model_filepath, + save_best_only=True + ) + + + def get_tb_ckpt_callbacks(self): + return [ + self._create_tb_callbacks, + self._create_ckpt_callbacks + ] diff --git a/Chicken_Disease_Classification/src/cnnClassifier/components/training.py b/Chicken_Disease_Classification/src/cnnClassifier/components/training.py new file mode 100644 index 0000000000..a753500638 --- /dev/null +++ b/Chicken_Disease_Classification/src/cnnClassifier/components/training.py @@ -0,0 +1,81 @@ +from cnnClassifier.entity.config_entity import TrainingConfig +import tensorflow as tf +from pathlib import Path + + +class Training: + def __init__(self, config: TrainingConfig): + self.config = config + + def get_base_model(self): + self.model = tf.keras.models.load_model( + self.config.updated_base_model_path + ) + + def train_valid_generator(self): + + datagenerator_kwargs = dict( + rescale = 1./255, + validation_split=0.20 + ) + + dataflow_kwargs = dict( + target_size=self.config.params_image_size[:-1], + batch_size=self.config.params_batch_size, + interpolation="bilinear" + ) + + valid_datagenerator = tf.keras.preprocessing.image.ImageDataGenerator( + **datagenerator_kwargs + ) + + self.valid_generator = valid_datagenerator.flow_from_directory( + directory=self.config.training_data, + subset="validation", + shuffle=False, + **dataflow_kwargs + ) + + if self.config.params_is_augmentation: + train_datagenerator = tf.keras.preprocessing.image.ImageDataGenerator( + rotation_range=40, + horizontal_flip=True, + width_shift_range=0.2, + height_shift_range=0.2, + shear_range=0.2, + zoom_range=0.2, + **datagenerator_kwargs + ) + else: + train_datagenerator = valid_datagenerator + + self.train_generator = train_datagenerator.flow_from_directory( + directory=self.config.training_data, + subset="training", + shuffle=True, + **dataflow_kwargs + ) + + @staticmethod + def save_model(path: Path, model: tf.keras.Model): + model.save(path) + + + def train(self, callback_list: list): + self.steps_per_epoch = self.train_generator.samples // self.train_generator.batch_size + self.validation_steps = self.valid_generator.samples // self.valid_generator.batch_size + + self.model.fit( + self.train_generator, + epochs=self.config.params_epochs, + steps_per_epoch=self.steps_per_epoch, + validation_steps=self.validation_steps, + validation_data=self.valid_generator, + callbacks=callback_list + ) + + self.save_model( + path=self.config.trained_model_path, + model=self.model + ) + diff --git a/Chicken_Disease_Classification/src/cnnClassifier/config/__init__.py b/Chicken_Disease_Classification/src/cnnClassifier/config/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Chicken_Disease_Classification/src/cnnClassifier/config/configuration.py b/Chicken_Disease_Classification/src/cnnClassifier/config/configuration.py new file mode 100644 index 0000000000..a191029062 --- /dev/null +++ b/Chicken_Disease_Classification/src/cnnClassifier/config/configuration.py @@ -0,0 +1,117 @@ +from cnnClassifier.constants import * +import os +from pathlib import Path +from cnnClassifier.utils.common import read_yaml, create_directories +from cnnClassifier.entity.config_entity import (DataIngestionConfig, + PrepareBaseModelConfig, + PrepareCallbacksConfig, + TrainingConfig, + EvaluationConfig) + + + +class ConfigurationManager: + def __init__( + self, + config_filepath = CONFIG_FILE_PATH, + params_filepath = PARAMS_FILE_PATH): + + self.config = read_yaml(config_filepath) + self.params = read_yaml(params_filepath) + + create_directories([self.config.artifacts_root]) + + + + def get_data_ingestion_config(self) -> DataIngestionConfig: + config = self.config.data_ingestion + + create_directories([config.root_dir]) + + data_ingestion_config = DataIngestionConfig( + root_dir=config.root_dir, + source_URL=config.source_URL, + local_data_file=config.local_data_file, + unzip_dir=config.unzip_dir + ) + + return data_ingestion_config + + + + + def get_prepare_base_model_config(self) -> PrepareBaseModelConfig: + config = self.config.prepare_base_model + + create_directories([config.root_dir]) + + prepare_base_model_config = PrepareBaseModelConfig( + root_dir=Path(config.root_dir), + base_model_path=Path(config.base_model_path), + updated_base_model_path=Path(config.updated_base_model_path), + params_image_size=self.params.IMAGE_SIZE, + params_learning_rate=self.params.LEARNING_RATE, + params_include_top=self.params.INCLUDE_TOP, + params_weights=self.params.WEIGHTS, + params_classes=self.params.CLASSES + ) + + return prepare_base_model_config + + + + def get_prepare_callback_config(self) -> PrepareCallbacksConfig: + config = self.config.prepare_callbacks + model_ckpt_dir = os.path.dirname(config.checkpoint_model_filepath) + create_directories([ + Path(model_ckpt_dir), + Path(config.tensorboard_root_log_dir) + ]) + + prepare_callback_config = PrepareCallbacksConfig( + root_dir=Path(config.root_dir), + tensorboard_root_log_dir=Path(config.tensorboard_root_log_dir), + checkpoint_model_filepath=Path(config.checkpoint_model_filepath) + ) + + return prepare_callback_config + + + + def get_training_config(self) -> TrainingConfig: + training = self.config.training + prepare_base_model = self.config.prepare_base_model + params = self.params + training_data = os.path.join(self.config.data_ingestion.unzip_dir, "Chicken-fecal-images") + create_directories([ + Path(training.root_dir) + ]) + + training_config = TrainingConfig( + root_dir=Path(training.root_dir), + trained_model_path=Path(training.trained_model_path), + updated_base_model_path=Path(prepare_base_model.updated_base_model_path), + training_data=Path(training_data), + params_epochs=params.EPOCHS, + params_batch_size=params.BATCH_SIZE, + params_is_augmentation=params.AUGMENTATION, + params_image_size=params.IMAGE_SIZE + ) + + return training_config + + + + + def get_evaluation_config(self) -> EvaluationConfig: + eval_config = EvaluationConfig( + path_of_model="artifacts/training/model.h5", + training_data="artifacts/data_ingestion/Chicken-fecal-images", + mlflow_uri="https://dagshub.com/entbappy/MLflow-DVC-Chicken-Disease-Classification.mlflow", + all_params=self.params, + params_image_size=self.params.IMAGE_SIZE, + params_batch_size=self.params.BATCH_SIZE + ) + return eval_config + + \ No newline at end of file diff --git a/Chicken_Disease_Classification/src/cnnClassifier/constants/__init__.py b/Chicken_Disease_Classification/src/cnnClassifier/constants/__init__.py new file mode 100644 index 0000000000..de2d074b35 --- /dev/null +++ b/Chicken_Disease_Classification/src/cnnClassifier/constants/__init__.py @@ -0,0 +1,4 @@ +from pathlib import Path + +CONFIG_FILE_PATH = Path("config/config.yaml") +PARAMS_FILE_PATH = Path("params.yaml") \ No newline at end of file diff --git a/Chicken_Disease_Classification/src/cnnClassifier/entity/__init__.py b/Chicken_Disease_Classification/src/cnnClassifier/entity/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Chicken_Disease_Classification/src/cnnClassifier/entity/config_entity.py b/Chicken_Disease_Classification/src/cnnClassifier/entity/config_entity.py new file mode 100644 index 0000000000..6c3a78c129 --- /dev/null +++ b/Chicken_Disease_Classification/src/cnnClassifier/entity/config_entity.py @@ -0,0 +1,54 @@ +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class DataIngestionConfig: + root_dir: Path + source_URL: str + local_data_file: Path + unzip_dir: Path + + +@dataclass(frozen=True) +class PrepareBaseModelConfig: + root_dir: Path + base_model_path: Path + updated_base_model_path: Path + params_image_size: list + params_learning_rate: float + params_include_top: bool + params_weights: str + params_classes: int + + + +@dataclass(frozen=True) +class PrepareCallbacksConfig: + root_dir: Path + tensorboard_root_log_dir: Path + checkpoint_model_filepath: Path + + + +@dataclass(frozen=True) +class TrainingConfig: + root_dir: Path + trained_model_path: Path + updated_base_model_path: Path + training_data: Path + params_epochs: int + params_batch_size: int + params_is_augmentation: bool + params_image_size: list + + + +@dataclass(frozen=True) +class EvaluationConfig: + path_of_model: Path + training_data: Path + all_params: dict + mlflow_uri: str + params_image_size: list + params_batch_size: int diff --git a/Chicken_Disease_Classification/src/cnnClassifier/pipeline/__init__.py b/Chicken_Disease_Classification/src/cnnClassifier/pipeline/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Chicken_Disease_Classification/src/cnnClassifier/pipeline/predict.py b/Chicken_Disease_Classification/src/cnnClassifier/pipeline/predict.py new file mode 100644 index 0000000000..8386276883 --- /dev/null +++ b/Chicken_Disease_Classification/src/cnnClassifier/pipeline/predict.py @@ -0,0 +1,30 @@ +import numpy as np +from tensorflow.keras.models import load_model +from tensorflow.keras.preprocessing import image +import os + + + +class PredictionPipeline: + def __init__(self,filename): + self.filename =filename + + + + def predict(self): + # load model + model = load_model(os.path.join("artifacts","training", "model.h5")) + + imagename = self.filename + test_image = image.load_img(imagename, target_size = (224,224)) + test_image = image.img_to_array(test_image) + test_image = np.expand_dims(test_image, axis = 0) + result = np.argmax(model.predict(test_image), axis=1) + print(result) + + if result[0] == 1: + prediction = 'Healthy' + return [{ "image" : prediction}] + else: + prediction = 'Coccidiosis' + return [{ "image" : prediction}] diff --git a/Chicken_Disease_Classification/src/cnnClassifier/pipeline/stage_01_data_ingestion.py b/Chicken_Disease_Classification/src/cnnClassifier/pipeline/stage_01_data_ingestion.py new file mode 100644 index 0000000000..41e986dd35 --- /dev/null +++ b/Chicken_Disease_Classification/src/cnnClassifier/pipeline/stage_01_data_ingestion.py @@ -0,0 +1,31 @@ +from cnnClassifier.config.configuration import ConfigurationManager +from cnnClassifier.components.data_ingestion import DataIngestion +from cnnClassifier import logger + + +STAGE_NAME = "Data Ingestion stage" + +class DataIngestionTrainingPipeline: + def __init__(self): + pass + + def main(self): + config = ConfigurationManager() + data_ingestion_config = config.get_data_ingestion_config() + data_ingestion = DataIngestion(config=data_ingestion_config) + data_ingestion.download_file() + data_ingestion.extract_zip_file() + + + + +if __name__ == '__main__': + try: + logger.info(f">>>>>> stage {STAGE_NAME} started <<<<<<") + obj = DataIngestionTrainingPipeline() + obj.main() + logger.info(f">>>>>> stage {STAGE_NAME} completed <<<<<<\n\nx==========x") + except Exception as e: + logger.exception(e) + raise e + diff --git a/Chicken_Disease_Classification/src/cnnClassifier/pipeline/stage_02_prepare_base_model.py b/Chicken_Disease_Classification/src/cnnClassifier/pipeline/stage_02_prepare_base_model.py new file mode 100644 index 0000000000..c6916a895a --- /dev/null +++ b/Chicken_Disease_Classification/src/cnnClassifier/pipeline/stage_02_prepare_base_model.py @@ -0,0 +1,34 @@ +from cnnClassifier.config.configuration import ConfigurationManager +from cnnClassifier.components.prepare_base_model import PrepareBaseModel +from cnnClassifier import logger + + +STAGE_NAME = "Prepare base model" + +class PrepareBaseModelTrainingPipeline: + def __init__(self): + pass + + def main(self): + config = ConfigurationManager() + prepare_base_model_config = config.get_prepare_base_model_config() + prepare_base_model = PrepareBaseModel(config=prepare_base_model_config) + prepare_base_model.get_base_model() + prepare_base_model.update_base_model() + + + + + +if __name__ == '__main__': + try: + logger.info(f"*******************") + logger.info(f">>>>>> stage {STAGE_NAME} started <<<<<<") + obj = PrepareBaseModelTrainingPipeline() + obj.main() + logger.info(f">>>>>> stage {STAGE_NAME} completed <<<<<<\n\nx==========x") + except Exception as e: + logger.exception(e) + raise e + + diff --git a/Chicken_Disease_Classification/src/cnnClassifier/pipeline/stage_03_training.py b/Chicken_Disease_Classification/src/cnnClassifier/pipeline/stage_03_training.py new file mode 100644 index 0000000000..0aae4e65f0 --- /dev/null +++ b/Chicken_Disease_Classification/src/cnnClassifier/pipeline/stage_03_training.py @@ -0,0 +1,43 @@ +from cnnClassifier.config.configuration import ConfigurationManager +from cnnClassifier.components.prepare_callbacks import PrepareCallback +from cnnClassifier.components.training import Training +from cnnClassifier import logger + + + +STAGE_NAME = "Training" + + +class ModelTrainingPipeline: + def __init__(self): + pass + + def main(self): + config = ConfigurationManager() + prepare_callbacks_config = config.get_prepare_callback_config() + prepare_callbacks = PrepareCallback(config=prepare_callbacks_config) + callback_list = prepare_callbacks.get_tb_ckpt_callbacks() + + + training_config = config.get_training_config() + training = Training(config=training_config) + training.get_base_model() + training.train_valid_generator() + training.train( + callback_list=callback_list + ) + + + + +if __name__ == '__main__': + try: + logger.info(f"*******************") + logger.info(f">>>>>> stage {STAGE_NAME} started <<<<<<") + obj = ModelTrainingPipeline() + obj.main() + logger.info(f">>>>>> stage {STAGE_NAME} completed <<<<<<\n\nx==========x") + except Exception as e: + logger.exception(e) + raise e + diff --git a/Chicken_Disease_Classification/src/cnnClassifier/pipeline/stage_04_evaluation.py b/Chicken_Disease_Classification/src/cnnClassifier/pipeline/stage_04_evaluation.py new file mode 100644 index 0000000000..1348dae306 --- /dev/null +++ b/Chicken_Disease_Classification/src/cnnClassifier/pipeline/stage_04_evaluation.py @@ -0,0 +1,35 @@ +from cnnClassifier.config.configuration import ConfigurationManager +from cnnClassifier.components.evaluation import Evaluation +from cnnClassifier import logger + + + + +STAGE_NAME = "Evaluation stage" + + +class EvaluationPipeline: + def __init__(self): + pass + + def main(self): + config = ConfigurationManager() + eval_config = config.get_evaluation_config() + evaluation = Evaluation(eval_config) + evaluation.evaluation() + evaluation.save_score() + evaluation.log_into_mlflow() + + + +if __name__ == '__main__': + try: + logger.info(f"*******************") + logger.info(f">>>>>> stage {STAGE_NAME} started <<<<<<") + obj = EvaluationPipeline() + obj.main() + logger.info(f">>>>>> stage {STAGE_NAME} completed <<<<<<\n\nx==========x") + except Exception as e: + logger.exception(e) + raise e + \ No newline at end of file diff --git a/Chicken_Disease_Classification/src/cnnClassifier/utils/__init__.py b/Chicken_Disease_Classification/src/cnnClassifier/utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Chicken_Disease_Classification/src/cnnClassifier/utils/common.py b/Chicken_Disease_Classification/src/cnnClassifier/utils/common.py new file mode 100644 index 0000000000..7b3a795cb4 --- /dev/null +++ b/Chicken_Disease_Classification/src/cnnClassifier/utils/common.py @@ -0,0 +1,138 @@ +import os +from box.exceptions import BoxValueError +import yaml +from cnnClassifier import logger +import json +import joblib +from ensure import ensure_annotations +from box import ConfigBox +from pathlib import Path +from typing import Any +import base64 + + + +@ensure_annotations +def read_yaml(path_to_yaml: Path) -> ConfigBox: + """reads yaml file and returns + + Args: + path_to_yaml (str): path like input + + Raises: + ValueError: if yaml file is empty + e: empty file + + Returns: + ConfigBox: ConfigBox type + """ + try: + with open(path_to_yaml) as yaml_file: + content = yaml.safe_load(yaml_file) + logger.info(f"yaml file: {path_to_yaml} loaded successfully") + return ConfigBox(content) + except BoxValueError: + raise ValueError("yaml file is empty") + except Exception as e: + raise e + + + +@ensure_annotations +def create_directories(path_to_directories: list, verbose=True): + """create list of directories + + Args: + path_to_directories (list): list of path of directories + ignore_log (bool, optional): ignore if multiple dirs is to be created. Defaults to False. + """ + for path in path_to_directories: + os.makedirs(path, exist_ok=True) + if verbose: + logger.info(f"created directory at: {path}") + + +@ensure_annotations +def save_json(path: Path, data: dict): + """save json data + + Args: + path (Path): path to json file + data (dict): data to be saved in json file + """ + with open(path, "w") as f: + json.dump(data, f, indent=4) + + logger.info(f"json file saved at: {path}") + + + + +@ensure_annotations +def load_json(path: Path) -> ConfigBox: + """load json files data + + Args: + path (Path): path to json file + + Returns: + ConfigBox: data as class attributes instead of dict + """ + with open(path) as f: + content = json.load(f) + + logger.info(f"json file loaded succesfully from: {path}") + return ConfigBox(content) + + +@ensure_annotations +def save_bin(data: Any, path: Path): + """save binary file + + Args: + data (Any): data to be saved as binary + path (Path): path to binary file + """ + joblib.dump(value=data, filename=path) + logger.info(f"binary file saved at: {path}") + + +@ensure_annotations +def load_bin(path: Path) -> Any: + """load binary data + + Args: + path (Path): path to binary file + + Returns: + Any: object stored in the file + """ + data = joblib.load(path) + logger.info(f"binary file loaded from: {path}") + return data + +@ensure_annotations +def get_size(path: Path) -> str: + """get size in KB + + Args: + path (Path): path of the file + + Returns: + str: size in KB + """ + size_in_kb = round(os.path.getsize(path)/1024) + return f"~ {size_in_kb} KB" + + +def decodeImage(imgstring, fileName): + imgdata = base64.b64decode(imgstring) + with open(fileName, 'wb') as f: + f.write(imgdata) + f.close() + + +def encodeImageIntoBase64(croppedImagePath): + with open(croppedImagePath, "rb") as f: + return base64.b64encode(f.read()) + diff --git a/Chicken_Disease_Classification/template.py b/Chicken_Disease_Classification/template.py new file mode 100644 index 0000000000..e09892ddaa --- /dev/null +++ b/Chicken_Disease_Classification/template.py @@ -0,0 +1,48 @@ +import os +from pathlib import Path +import logging + +logging.basicConfig(level=logging.INFO, format='[%(asctime)s]: %(message)s:') + + +project_name = "cnnClassifier" + +list_of_files = [ + ".github/workflows/.gitkeep", + f"src/{project_name}/__init__.py", + f"src/{project_name}/components/__init__.py", + f"src/{project_name}/utils/__init__.py", + f"src/{project_name}/config/__init__.py", + f"src/{project_name}/config/configuration.py", + f"src/{project_name}/pipeline/__init__.py", + f"src/{project_name}/entity/__init__.py", + f"src/{project_name}/constants/__init__.py", + "config/config.yaml", + "dvc.yaml", + "params.yaml", + "requirements.txt", + "setup.py", + "research/trials.ipynb", + "templates/index.html" + + +] + + +for filepath in list_of_files: + filepath = Path(filepath) + filedir, filename = os.path.split(filepath) + + + if filedir !="": + os.makedirs(filedir, exist_ok=True) + logging.info(f"Creating directory; {filedir} for the file: {filename}") + + if (not os.path.exists(filepath)) or (os.path.getsize(filepath) == 0): + with open(filepath, "w") as f: + pass + logging.info(f"Creating empty file: {filepath}") + + + else: + logging.info(f"{filename} is already exists") \ No newline at end of file diff --git a/Chicken_Disease_Classification/templates/index.html b/Chicken_Disease_Classification/templates/index.html new file mode 100644 index 0000000000..437f62be6e --- /dev/null +++ b/Chicken_Disease_Classification/templates/index.html @@ -0,0 +1,234 @@ + + + + + + + cnncls + + + + + +
+
+

Object Classification

+
+
+
+ + + +
+
+
+
+ + +
+ + + + + + + + + + + + +
+
+
+
+
+
Prediction Results
+
+
+
+
+
+
+
+
+ + + + +
+ + + + + + + \ No newline at end of file