From f90fbf152b58bc0e61d3161b4814c2da00cda9eb Mon Sep 17 00:00:00 2001 From: Sebastian Echeverria Date: Fri, 4 Oct 2024 10:26:34 -0400 Subject: [PATCH] Demo: formatting, cleanup and consistency with model size --- demo/scenarios/1_requirements.ipynb | 34 +-- .../2h_evidence_monitorability.ipynb | 4 +- demo/scenarios/garden.py | 212 +++++++++--------- demo/scenarios/model_predict.py | 29 ++- demo/scenarios/properties/__init__.py | 0 demo/scenarios/values/__init__.py | 0 6 files changed, 145 insertions(+), 134 deletions(-) create mode 100644 demo/scenarios/properties/__init__.py create mode 100644 demo/scenarios/values/__init__.py diff --git a/demo/scenarios/1_requirements.ipynb b/demo/scenarios/1_requirements.ipynb index 248a0d88d..5fced8d98 100644 --- a/demo/scenarios/1_requirements.ipynb +++ b/demo/scenarios/1_requirements.ipynb @@ -32,7 +32,7 @@ "* **Robustness - Model Robust to Noise (Channel Loss)**\n", " * The model receives a picture taken at a garden using a loaned device. These devices are known to sometimes lose a channel (i.e., RGB channel). The model should still be able to successfully identify the flower at the same rate as full images. Test data needs to include images with a missing channel. Test images will be generated by removing the R, G and B channels in the original test data using ImageMagic, therefore producing three data sets. Images with a missing channel are successfully identified at rates equal to that of original images. This will be measured using the Wilcoxon Rank-Sum test, with significance at p-value <=0.05.\n", "* **Performance on Operational Platform**\n", - " * The model will need to run on the devices loaned out by the garden centers to visitors. These are small, inexpensive devices with limited CPU power, as well as limited memory and disk space (512 MB and 3 KB, respectively). The original test dataset can be used. 1- Executing the model on the loaned platform will not exceed maximum CPU usage of 30% to ensure reasonable response time. CPU usage will be measure using ps. 2- Memory usage at inference time will not exceed available memory of 512 MB. This will be measured using pmap. 3 - Disk usage will not exceed available disk space of 3 KB. This will be measured using by adding the size of each file in the path for the model code.\n", + " * The model will need to run on the devices loaned out by the garden centers to visitors. These are small, inexpensive devices with limited CPU power, as well as limited memory and disk space (512 MB and 150 MB, respectively). The original test dataset can be used. 1- Executing the model on the loaned platform will not exceed maximum CPU usage of 30% to ensure reasonable response time. CPU usage will be measure using ps. 2- Memory usage at inference time will not exceed available memory of 512 MB. This will be measured using pmap. 3 - Disk usage will not exceed available disk space of 150 MB. This will be measured using by adding the size of each file in the path for the model code.\n", "* **Interpretability - Understanding Model Results**\n", " * The application that runs on the loaned device should indicate the main features that were used to recognize the flower, as part of the educational experience. The app will display the image highlighting the most informative features in flower identification, in addition to the flower name. The original test data set can be used. The model needs to return evidence, in this case a heat map implementing the Integrated Gradients algorithm, showing the pixels that were most informative in the classification decision. This evidence should be returned with each inference. \n", "\n", @@ -69,7 +69,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -96,7 +96,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -208,7 +208,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -425,11 +425,11 @@ " stimulus=\"The model receives a picture taken at the garden using a loaned device. The model will need to run on this device.\",\n", " source=\"By a member of the general public.\",\n", " environment=\"The original test dataset can be used.\",\n", - " response=\"The model will not exceed the limited CPU power, as well as limited memory and disk space (512 MB and 128 GB, respectively) available.\",\n", + " response=\"The model will not exceed the limited CPU power, as well as limited memory and disk space (512 MB and 150 MB, respectively) available.\",\n", " measure=(\n", " \"1- Executing the model on the loaned platform will not exceed maximum CPU usage of 30% to ensure reasonable response time. CPU usage will be measured using ps.\"\n", " + \"2- Memory usage at inference time will not exceed available memory of 512 MB. This will be measured using pmap.\"\n", - " + \"3 - Disk usage will not exceed available disk space of 128 GB. This will be measured using by adding the size of each file in the path for the model code.\"\n", + " + \"3 - Disk usage will not exceed available disk space of 150 MB. This will be measured using by adding the size of each file in the path for the model code.\"\n", " ),\n", " ),\n", " QASDescriptor(\n", @@ -496,7 +496,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -532,7 +532,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -552,14 +552,14 @@ "from mlte.measurement.memory import LocalProcessMemoryConsumption\n", "from mlte.value.types.image import Image\n", "from mlte.value.types.real import Real\n", - "from values.multiple_accuracy import MultipleAccuracy\n", - "from values.ranksums import RankSums\n", - "from values.multiple_ranksums import MultipleRanksums\n", - "from properties.monitorability import Monitorability\n", - "from properties.interoperability import Interoperability\n", - "from properties.resilience import Resilience\n", - "from properties.accuracy import Accuracy\n", - "from values.string import String\n", + "from demo.scenarios.values.multiple_accuracy import MultipleAccuracy\n", + "from demo.scenarios.values.ranksums import RankSums\n", + "from demo.scenarios.values.multiple_ranksums import MultipleRanksums\n", + "from demo.scenarios.properties.monitorability import Monitorability\n", + "from demo.scenarios.properties.interoperability import Interoperability\n", + "from demo.scenarios.properties.resilience import Resilience\n", + "from demo.scenarios.properties.accuracy import Accuracy\n", + "from demo.scenarios.values.string import String\n", "\n", "\n", "# The full spec. Note that the Robustness Property contains conditions for both Robustness scenarios.\n", @@ -584,7 +584,7 @@ " ),\n", " },\n", " StorageCost(\"Critical since model will be in an embedded device\"): {\n", - " \"model size\": LocalObjectSize.value().less_than(3000)\n", + " \"model size\": LocalObjectSize.value().less_than(150000000)\n", " },\n", " PredictingMemoryCost(\n", " \"Useful to evaluate resources needed when predicting\"\n", diff --git a/demo/scenarios/2h_evidence_monitorability.ipynb b/demo/scenarios/2h_evidence_monitorability.ipynb index 67b57d167..7db45493b 100644 --- a/demo/scenarios/2h_evidence_monitorability.ipynb +++ b/demo/scenarios/2h_evidence_monitorability.ipynb @@ -22,7 +22,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -40,7 +40,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ diff --git a/demo/scenarios/garden.py b/demo/scenarios/garden.py index 6880a6a50..6d32ea9be 100644 --- a/demo/scenarios/garden.py +++ b/demo/scenarios/garden.py @@ -2,111 +2,113 @@ import pandas as pd - # set up a data label dictionary to switch between data label number and english name +# set up a data label dictionary to switch between data label number and english name label_dict = { - 0: "_mexican_aster", - 1: "alpine_sea_holly", - 2: "anthurium", - 3: "artichoke", - 4: "arum_lily", - 5: "azalea", - 6: "ball_moss", - 7: "ballon_flower", - 8: "barberton_daisy", - 9: "bearded_iris", - 10: "bee_balm", - 11: "bird_of_paradise", - 12: "bishop_of_llandaf_dahlia", - 13: "black-eyed_susan", - 14: "blackberry_lily", - 15: "blanket_flower", - 16: "bolero_deep_blue", - 17: "bougainvillea", - 18: "bromelia", - 19: "buttercup", - 20: "california_poppy", - 21: "camellia", - 22: "canna_lily", - 23: "canterbury_bells", - 24: "cape_flower", - 25: "carnation", - 26: "cattleya", - 27: "cautleya_spicata", - 28: "clematis", - 29: "coltsfoot", - 30: "columbine", - 31: "common_dandelion", - 32: "corn_poppy", - 33: "cyclamen", - 34: "daffodil", - 35: "dahlia", - 36: "desert_rose", - 37: "english_marigold", - 38: "fire_lily", - 39: "foxglove", - 40: "frangipani", - 41: "fritillary", - 42: "garden_phlox", - 43: "gaura", - 44: "gazania", - 45: "geranium", - 46: "globe_flower", - 47: "globe_thistle", - 48: "grape_hyacinth", - 49: "great_masterwort", - 50: "hard-leaved_pocket_orchid", - 51: "hibiscus", - 52: "hippesatrum", - 53: "japanese_anemone", - 54: "king_protea", - 55: "lenten_rose", - 56: "lotus", - 57: "love_in_the_mist", - 58: "magnolia", - 59: "mallow", - 60: "marigold", - 61: "mexican_petunia", - 62: "monkshood", - 63: "moon_orchild", - 64: "morning_glory", - 65: "osteospermum", - 66: "oxeye_daisy", - 67: "passion_flower", - 68: "pelargonium", - 69: "peruvian_lily", - 70: "petunia", - 71: "pincushion_flower", - 72: "poinsettia", - 73: "primrose", - 74: "primula", - 75: "prince_of_whales_feather", - 76: "purple_coneflower", - 77: "red_ginger", - 78: "rose", - 79: "siam_tulip", - 80: "silverbush", - 81: "snapdragon", - 82: "spear_thistle", - 83: "spring_crocus", - 84: "stemless_gentain", - 85: "sunflower", - 86: "swear_pea", - 87: "sweet_william", - 88: "sword_lily", - 89: "thorn_apple", - 90: "tiger_lily", - 91: "orange_dahlia", - 92: "toad_lily", - 93: "tree_mallow", - 94: "tree_poppy", - 95: "trumpet_creeper", - 96: "wallflower", - 97: "water_lily", - 98: "watercress", - 99: "wild_pansy", - 100: "windflower", - 101: "yellow_iris", - } + 0: "_mexican_aster", + 1: "alpine_sea_holly", + 2: "anthurium", + 3: "artichoke", + 4: "arum_lily", + 5: "azalea", + 6: "ball_moss", + 7: "ballon_flower", + 8: "barberton_daisy", + 9: "bearded_iris", + 10: "bee_balm", + 11: "bird_of_paradise", + 12: "bishop_of_llandaf_dahlia", + 13: "black-eyed_susan", + 14: "blackberry_lily", + 15: "blanket_flower", + 16: "bolero_deep_blue", + 17: "bougainvillea", + 18: "bromelia", + 19: "buttercup", + 20: "california_poppy", + 21: "camellia", + 22: "canna_lily", + 23: "canterbury_bells", + 24: "cape_flower", + 25: "carnation", + 26: "cattleya", + 27: "cautleya_spicata", + 28: "clematis", + 29: "coltsfoot", + 30: "columbine", + 31: "common_dandelion", + 32: "corn_poppy", + 33: "cyclamen", + 34: "daffodil", + 35: "dahlia", + 36: "desert_rose", + 37: "english_marigold", + 38: "fire_lily", + 39: "foxglove", + 40: "frangipani", + 41: "fritillary", + 42: "garden_phlox", + 43: "gaura", + 44: "gazania", + 45: "geranium", + 46: "globe_flower", + 47: "globe_thistle", + 48: "grape_hyacinth", + 49: "great_masterwort", + 50: "hard-leaved_pocket_orchid", + 51: "hibiscus", + 52: "hippesatrum", + 53: "japanese_anemone", + 54: "king_protea", + 55: "lenten_rose", + 56: "lotus", + 57: "love_in_the_mist", + 58: "magnolia", + 59: "mallow", + 60: "marigold", + 61: "mexican_petunia", + 62: "monkshood", + 63: "moon_orchild", + 64: "morning_glory", + 65: "osteospermum", + 66: "oxeye_daisy", + 67: "passion_flower", + 68: "pelargonium", + 69: "peruvian_lily", + 70: "petunia", + 71: "pincushion_flower", + 72: "poinsettia", + 73: "primrose", + 74: "primula", + 75: "prince_of_whales_feather", + 76: "purple_coneflower", + 77: "red_ginger", + 78: "rose", + 79: "siam_tulip", + 80: "silverbush", + 81: "snapdragon", + 82: "spear_thistle", + 83: "spring_crocus", + 84: "stemless_gentain", + 85: "sunflower", + 86: "swear_pea", + 87: "sweet_william", + 88: "sword_lily", + 89: "thorn_apple", + 90: "tiger_lily", + 91: "orange_dahlia", + 92: "toad_lily", + 93: "tree_mallow", + 94: "tree_poppy", + 95: "trumpet_creeper", + 96: "wallflower", + 97: "water_lily", + 98: "watercress", + 99: "wild_pansy", + 100: "windflower", + 101: "yellow_iris", +} + + def load_base_results(data_folder: str) -> pd.DataFrame: df_results = pd.read_csv( path.join(data_folder, "FlowerModelv1_TestSetResults.csv") @@ -129,8 +131,6 @@ def load_taxonomy(data_folder: str) -> pd.DataFrame: ) df_labels.head() - - df_dict = pd.DataFrame.from_dict( {"Label": list(label_dict), "Label Name": label_dict.values()} ) diff --git a/demo/scenarios/model_predict.py b/demo/scenarios/model_predict.py index 2606deed0..af43adbfd 100644 --- a/demo/scenarios/model_predict.py +++ b/demo/scenarios/model_predict.py @@ -5,8 +5,10 @@ import time from resource import * from typing import Optional -import tensorflow as tf + import garden +import tensorflow as tf + def setup_log(): logging.basicConfig( @@ -115,8 +117,10 @@ def run_model(image_folder_path, model_file, weights_file): image_np = image.numpy() if len(image_np.shape) == 3: print_and_log("Model - Input Validation Okay - RGB image loaded") - else: - print_and_log(f"Model - Input Validation Error - RGB image expected but {image} has wrong number of channels") + else: + print_and_log( + f"Model - Input Validation Error - RGB image expected but {image} has wrong number of channels" + ) # Not sure if this is the best way to deal with the spec: "input specification it will generate the output "N/A" break # OOD @@ -156,13 +160,16 @@ def run_model(image_folder_path, model_file, weights_file): total_elapsed_time += elapsed_time total_inference_memory += inference_memory - # Output validation if pred.shape[1] == 102: - print_and_log(f"Model - Output Validation Pass - {pred.shape} - validated") + print_and_log( + f"Model - Output Validation Pass - {pred.shape} - validated" + ) else: - print_and_log(f"Model - Output Validation Error - Output shape: {pred.shape}") - + print_and_log( + f"Model - Output Validation Error - Output shape: {pred.shape}" + ) + # Get prediction probs = tf.nn.softmax(pred) predicted_class = tf.argmax(probs, axis=-1) @@ -170,9 +177,13 @@ def run_model(image_folder_path, model_file, weights_file): if confidence[0] > 0.8: label = garden.label_dict[int(predicted_class[0])] - print_and_log(f"Model - file: {image_folder_path} predicted to be {label} (class {predicted_class}) with confidence {confidence}") + print_and_log( + f"Model - file: {image_folder_path} predicted to be {label} (class {predicted_class}) with confidence {confidence}" + ) else: - print_and_log(f"Model - Output Confidence Error - max class confidence of {confidence} too low") + print_and_log( + f"Model - Output Confidence Error - max class confidence of {confidence} too low" + ) avg_elapsed_time = total_elapsed_time / num_samples avg_inference_memory = total_inference_memory / num_samples diff --git a/demo/scenarios/properties/__init__.py b/demo/scenarios/properties/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/demo/scenarios/values/__init__.py b/demo/scenarios/values/__init__.py new file mode 100644 index 000000000..e69de29bb