From 41e1d65bc5e37f5975fc6d7b827df03a75412d5d Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Wed, 18 Sep 2024 00:18:16 +0000 Subject: [PATCH 01/13] testing adding startup --- .github/workflows/kserve_cpu_tests.yml | 6 ++++-- .../java/org/pytorch/serve/snapshot/SnapshotManager.java | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/kserve_cpu_tests.yml b/.github/workflows/kserve_cpu_tests.yml index 0f56b392a6..6b7b5f4c4c 100644 --- a/.github/workflows/kserve_cpu_tests.yml +++ b/.github/workflows/kserve_cpu_tests.yml @@ -3,8 +3,10 @@ name: KServe CPU Nightly Tests on: workflow_dispatch: # runs everyday at 5:15am - schedule: - - cron: '15 5 * * *' + # schedule: + # - cron: '15 5 * * *' + push: + -branches: "snap_fix" jobs: kserve-cpu-tests: diff --git a/frontend/server/src/main/java/org/pytorch/serve/snapshot/SnapshotManager.java b/frontend/server/src/main/java/org/pytorch/serve/snapshot/SnapshotManager.java index 6d0aca353f..b9b2c8e07b 100644 --- a/frontend/server/src/main/java/org/pytorch/serve/snapshot/SnapshotManager.java +++ b/frontend/server/src/main/java/org/pytorch/serve/snapshot/SnapshotManager.java @@ -112,8 +112,9 @@ public Snapshot getSnapshot(String snapshotName) throws SnapshotReadException { } public void restore(String modelSnapshot) throws InvalidSnapshotException, IOException { - logger.info("Started restoring models from snapshot {}", modelSnapshot); - Snapshot snapshot = snapshotSerializer.getSnapshot(modelSnapshot); + String result = modelSnapshot.substring(0, modelSnapshot.length() - 1) + ",\n \"startupTimeout\": 120 }"; + logger.info("Started restoring models from snapshot {}", result); + Snapshot snapshot = snapshotSerializer.getSnapshot(result); // Validate snapshot validate(snapshot); // Init. models From c943c5e9ecebd65bced80275985864cf3a26c1e1 Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Wed, 18 Sep 2024 00:47:51 +0000 Subject: [PATCH 02/13] testing adding default --- .../java/org/pytorch/serve/archive/model/ModelConfig.java | 5 ++++- .../java/org/pytorch/serve/snapshot/SnapshotManager.java | 5 ++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/frontend/archive/src/main/java/org/pytorch/serve/archive/model/ModelConfig.java b/frontend/archive/src/main/java/org/pytorch/serve/archive/model/ModelConfig.java index c83da0523c..dcfa786e9d 100644 --- a/frontend/archive/src/main/java/org/pytorch/serve/archive/model/ModelConfig.java +++ b/frontend/archive/src/main/java/org/pytorch/serve/archive/model/ModelConfig.java @@ -127,9 +127,12 @@ public static ModelConfig build(Map yamlMap) { case "startupTimeout": if (v instanceof Integer) { modelConfig.setStartupTimeout((int) v); - } else { + } else if { logger.warn("Invalid startupTimeout: {}, should be integer", v); } + else { + modelConfig.setStartupTimeout(startupTimeout); + } break; case "deviceType": if (v instanceof String) { diff --git a/frontend/server/src/main/java/org/pytorch/serve/snapshot/SnapshotManager.java b/frontend/server/src/main/java/org/pytorch/serve/snapshot/SnapshotManager.java index b9b2c8e07b..6d0aca353f 100644 --- a/frontend/server/src/main/java/org/pytorch/serve/snapshot/SnapshotManager.java +++ b/frontend/server/src/main/java/org/pytorch/serve/snapshot/SnapshotManager.java @@ -112,9 +112,8 @@ public Snapshot getSnapshot(String snapshotName) throws SnapshotReadException { } public void restore(String modelSnapshot) throws InvalidSnapshotException, IOException { - String result = modelSnapshot.substring(0, modelSnapshot.length() - 1) + ",\n \"startupTimeout\": 120 }"; - logger.info("Started restoring models from snapshot {}", result); - Snapshot snapshot = snapshotSerializer.getSnapshot(result); + logger.info("Started restoring models from snapshot {}", modelSnapshot); + Snapshot snapshot = snapshotSerializer.getSnapshot(modelSnapshot); // Validate snapshot validate(snapshot); // Init. models From e2bec9a7c9a7d4d78713642be4fbab92359ca562 Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Wed, 18 Sep 2024 18:30:29 +0000 Subject: [PATCH 03/13] testing newer docker --- kubernetes/kserve/tests/configs/mnist_v1_cpu.yaml | 2 +- kubernetes/kserve/tests/configs/mnist_v1_gpu.yaml | 2 +- kubernetes/kserve/tests/configs/mnist_v2_cpu.yaml | 2 +- kubernetes/kserve/tests/configs/mnist_v2_gpu.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/kubernetes/kserve/tests/configs/mnist_v1_cpu.yaml b/kubernetes/kserve/tests/configs/mnist_v1_cpu.yaml index 559a27b96a..6ff08ea3e4 100644 --- a/kubernetes/kserve/tests/configs/mnist_v1_cpu.yaml +++ b/kubernetes/kserve/tests/configs/mnist_v1_cpu.yaml @@ -6,7 +6,7 @@ spec: predictor: pytorch: storageUri: gs://kfserving-examples/models/torchserve/image_classifier/v1 - image: pytorch/torchserve-kfs-nightly:latest-cpu + image: udaij12/torchserve-kfs-nightly:latest-cpu resources: limits: cpu: "1" diff --git a/kubernetes/kserve/tests/configs/mnist_v1_gpu.yaml b/kubernetes/kserve/tests/configs/mnist_v1_gpu.yaml index 364ccc2716..5f8540eeb4 100644 --- a/kubernetes/kserve/tests/configs/mnist_v1_gpu.yaml +++ b/kubernetes/kserve/tests/configs/mnist_v1_gpu.yaml @@ -6,7 +6,7 @@ spec: predictor: pytorch: storageUri: gs://kfserving-examples/models/torchserve/image_classifier/v1 - image: pytorch/torchserve-kfs-nightly:latest-gpu + image: udaij12/torchserve-kfs-nightly:latest-gpu resources: limits: cpu: "1" diff --git a/kubernetes/kserve/tests/configs/mnist_v2_cpu.yaml b/kubernetes/kserve/tests/configs/mnist_v2_cpu.yaml index 851b69a1e5..6a35a6489c 100644 --- a/kubernetes/kserve/tests/configs/mnist_v2_cpu.yaml +++ b/kubernetes/kserve/tests/configs/mnist_v2_cpu.yaml @@ -7,7 +7,7 @@ spec: pytorch: protocolVersion: v2 storageUri: gs://kfserving-examples/models/torchserve/image_classifier/v2 - image: pytorch/torchserve-kfs-nightly:latest-cpu + image: udaij12/torchserve-kfs-nightly:latest-cpu resources: limits: cpu: "1" diff --git a/kubernetes/kserve/tests/configs/mnist_v2_gpu.yaml b/kubernetes/kserve/tests/configs/mnist_v2_gpu.yaml index 00703ef833..4d5294d350 100644 --- a/kubernetes/kserve/tests/configs/mnist_v2_gpu.yaml +++ b/kubernetes/kserve/tests/configs/mnist_v2_gpu.yaml @@ -7,7 +7,7 @@ spec: pytorch: protocolVersion: v2 storageUri: gs://kfserving-examples/models/torchserve/image_classifier/v2 - image: pytorch/torchserve-kfs-nightly:latest-gpu + image: udaij12/torchserve-kfs-nightly:latest-gpu resources: limits: cpu: "1" From 9c826adc71b7d04fe03b2fb4ca98b15af40984c7 Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Fri, 20 Sep 2024 20:42:34 +0000 Subject: [PATCH 04/13] adding default values for model --- .../serve/archive/model/ModelConfig.java | 5 +--- .../java/org/pytorch/serve/wlm/Model.java | 29 +++++++++++++++---- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/frontend/archive/src/main/java/org/pytorch/serve/archive/model/ModelConfig.java b/frontend/archive/src/main/java/org/pytorch/serve/archive/model/ModelConfig.java index dcfa786e9d..c83da0523c 100644 --- a/frontend/archive/src/main/java/org/pytorch/serve/archive/model/ModelConfig.java +++ b/frontend/archive/src/main/java/org/pytorch/serve/archive/model/ModelConfig.java @@ -127,12 +127,9 @@ public static ModelConfig build(Map yamlMap) { case "startupTimeout": if (v instanceof Integer) { modelConfig.setStartupTimeout((int) v); - } else if { + } else { logger.warn("Invalid startupTimeout: {}, should be integer", v); } - else { - modelConfig.setStartupTimeout(startupTimeout); - } break; case "deviceType": if (v instanceof String) { diff --git a/frontend/server/src/main/java/org/pytorch/serve/wlm/Model.java b/frontend/server/src/main/java/org/pytorch/serve/wlm/Model.java index c41a495260..8bd9d92641 100644 --- a/frontend/server/src/main/java/org/pytorch/serve/wlm/Model.java +++ b/frontend/server/src/main/java/org/pytorch/serve/wlm/Model.java @@ -190,12 +190,29 @@ public JsonObject getModelState(boolean isDefaultVersion) { } public void setModelState(JsonObject modelInfo) { - minWorkers = modelInfo.get(MIN_WORKERS).getAsInt(); - maxWorkers = modelInfo.get(MAX_WORKERS).getAsInt(); - maxBatchDelay = modelInfo.get(MAX_BATCH_DELAY).getAsInt(); - responseTimeout = modelInfo.get(RESPONSE_TIMEOUT).getAsInt(); - startupTimeout = modelInfo.get(STARTUP_TIMEOUT).getAsInt(); - batchSize = modelInfo.get(BATCH_SIZE).getAsInt(); + minWorkers = modelInfo.has(MIN_WORKERS) && !modelInfo.get(MIN_WORKERS).isJsonNull() + ? modelInfo.get(MIN_WORKERS).getAsInt() + : 1; // default value for minWorkers + + maxWorkers = modelInfo.has(MAX_WORKERS) && !modelInfo.get(MAX_WORKERS).isJsonNull() + ? modelInfo.get(MAX_WORKERS).getAsInt() + : 5; // default value for maxWorkers + + maxBatchDelay = modelInfo.has(MAX_BATCH_DELAY) && !modelInfo.get(MAX_BATCH_DELAY).isJsonNull() + ? modelInfo.get(MAX_BATCH_DELAY).getAsInt() + : 100; // default value for maxBatchDelay + + responseTimeout = modelInfo.has(RESPONSE_TIMEOUT) && !modelInfo.get(RESPONSE_TIMEOUT).isJsonNull() + ? modelInfo.get(RESPONSE_TIMEOUT).getAsInt() + : 120; // default value for responseTimeout + + startupTimeout = modelInfo.has(STARTUP_TIMEOUT) && !modelInfo.get(STARTUP_TIMEOUT).isJsonNull() + ? modelInfo.get(STARTUP_TIMEOUT).getAsInt() + : 120; // default value for startupTimeout + + batchSize = modelInfo.has(BATCH_SIZE) && !modelInfo.get(BATCH_SIZE).isJsonNull() + ? modelInfo.get(BATCH_SIZE).getAsInt() + : 1; // default value for batchSize JsonElement runtime = modelInfo.get(RUNTIME_TYPE); String runtime_str = Manifest.RuntimeType.PYTHON.getValue(); From dd210479071b7c1ee7df66b0fad77154f0d6c557 Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Fri, 20 Sep 2024 21:30:59 +0000 Subject: [PATCH 05/13] testing specific docker --- kubernetes/kserve/tests/configs/mnist_v1_cpu.yaml | 2 +- kubernetes/kserve/tests/configs/mnist_v2_cpu.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/kserve/tests/configs/mnist_v1_cpu.yaml b/kubernetes/kserve/tests/configs/mnist_v1_cpu.yaml index 6ff08ea3e4..c6d975f1ed 100644 --- a/kubernetes/kserve/tests/configs/mnist_v1_cpu.yaml +++ b/kubernetes/kserve/tests/configs/mnist_v1_cpu.yaml @@ -6,7 +6,7 @@ spec: predictor: pytorch: storageUri: gs://kfserving-examples/models/torchserve/image_classifier/v1 - image: udaij12/torchserve-kfs-nightly:latest-cpu + image: udaij12/torchserve-kfs-nightly@sha256:d82f45d338f09ad0376703ae00637c8cdb9647783001600e6b3436284205a4df resources: limits: cpu: "1" diff --git a/kubernetes/kserve/tests/configs/mnist_v2_cpu.yaml b/kubernetes/kserve/tests/configs/mnist_v2_cpu.yaml index 6a35a6489c..41193c7f95 100644 --- a/kubernetes/kserve/tests/configs/mnist_v2_cpu.yaml +++ b/kubernetes/kserve/tests/configs/mnist_v2_cpu.yaml @@ -7,7 +7,7 @@ spec: pytorch: protocolVersion: v2 storageUri: gs://kfserving-examples/models/torchserve/image_classifier/v2 - image: udaij12/torchserve-kfs-nightly:latest-cpu + image: udaij12/torchserve-kfs-nightly@sha256:d82f45d338f09ad0376703ae00637c8cdb9647783001600e6b3436284205a4df resources: limits: cpu: "1" From a32eda0621a79f87bc8880dc69bab5d4417d625c Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Fri, 20 Sep 2024 21:52:38 +0000 Subject: [PATCH 06/13] testing specific docker image --- kubernetes/kserve/tests/configs/mnist_v1_cpu.yaml | 2 +- kubernetes/kserve/tests/configs/mnist_v2_cpu.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/kserve/tests/configs/mnist_v1_cpu.yaml b/kubernetes/kserve/tests/configs/mnist_v1_cpu.yaml index c6d975f1ed..6ff08ea3e4 100644 --- a/kubernetes/kserve/tests/configs/mnist_v1_cpu.yaml +++ b/kubernetes/kserve/tests/configs/mnist_v1_cpu.yaml @@ -6,7 +6,7 @@ spec: predictor: pytorch: storageUri: gs://kfserving-examples/models/torchserve/image_classifier/v1 - image: udaij12/torchserve-kfs-nightly@sha256:d82f45d338f09ad0376703ae00637c8cdb9647783001600e6b3436284205a4df + image: udaij12/torchserve-kfs-nightly:latest-cpu resources: limits: cpu: "1" diff --git a/kubernetes/kserve/tests/configs/mnist_v2_cpu.yaml b/kubernetes/kserve/tests/configs/mnist_v2_cpu.yaml index 41193c7f95..6a35a6489c 100644 --- a/kubernetes/kserve/tests/configs/mnist_v2_cpu.yaml +++ b/kubernetes/kserve/tests/configs/mnist_v2_cpu.yaml @@ -7,7 +7,7 @@ spec: pytorch: protocolVersion: v2 storageUri: gs://kfserving-examples/models/torchserve/image_classifier/v2 - image: udaij12/torchserve-kfs-nightly@sha256:d82f45d338f09ad0376703ae00637c8cdb9647783001600e6b3436284205a4df + image: udaij12/torchserve-kfs-nightly:latest-cpu resources: limits: cpu: "1" From 2d711481c967a87eb8285f61f0e75bbc26b07abe Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Mon, 23 Sep 2024 17:47:29 +0000 Subject: [PATCH 07/13] fixed format --- .../java/org/pytorch/serve/wlm/Model.java | 52 +++++++++++-------- 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/frontend/server/src/main/java/org/pytorch/serve/wlm/Model.java b/frontend/server/src/main/java/org/pytorch/serve/wlm/Model.java index 8bd9d92641..b2019cf765 100644 --- a/frontend/server/src/main/java/org/pytorch/serve/wlm/Model.java +++ b/frontend/server/src/main/java/org/pytorch/serve/wlm/Model.java @@ -190,29 +190,35 @@ public JsonObject getModelState(boolean isDefaultVersion) { } public void setModelState(JsonObject modelInfo) { - minWorkers = modelInfo.has(MIN_WORKERS) && !modelInfo.get(MIN_WORKERS).isJsonNull() - ? modelInfo.get(MIN_WORKERS).getAsInt() - : 1; // default value for minWorkers - - maxWorkers = modelInfo.has(MAX_WORKERS) && !modelInfo.get(MAX_WORKERS).isJsonNull() - ? modelInfo.get(MAX_WORKERS).getAsInt() - : 5; // default value for maxWorkers - - maxBatchDelay = modelInfo.has(MAX_BATCH_DELAY) && !modelInfo.get(MAX_BATCH_DELAY).isJsonNull() - ? modelInfo.get(MAX_BATCH_DELAY).getAsInt() - : 100; // default value for maxBatchDelay - - responseTimeout = modelInfo.has(RESPONSE_TIMEOUT) && !modelInfo.get(RESPONSE_TIMEOUT).isJsonNull() - ? modelInfo.get(RESPONSE_TIMEOUT).getAsInt() - : 120; // default value for responseTimeout - - startupTimeout = modelInfo.has(STARTUP_TIMEOUT) && !modelInfo.get(STARTUP_TIMEOUT).isJsonNull() - ? modelInfo.get(STARTUP_TIMEOUT).getAsInt() - : 120; // default value for startupTimeout - - batchSize = modelInfo.has(BATCH_SIZE) && !modelInfo.get(BATCH_SIZE).isJsonNull() - ? modelInfo.get(BATCH_SIZE).getAsInt() - : 1; // default value for batchSize + minWorkers = + modelInfo.has(MIN_WORKERS) && !modelInfo.get(MIN_WORKERS).isJsonNull() + ? modelInfo.get(MIN_WORKERS).getAsInt() + : 1; // default value for minWorkers + + maxWorkers = + modelInfo.has(MAX_WORKERS) && !modelInfo.get(MAX_WORKERS).isJsonNull() + ? modelInfo.get(MAX_WORKERS).getAsInt() + : 5; // default value for maxWorkers + + maxBatchDelay = + modelInfo.has(MAX_BATCH_DELAY) && !modelInfo.get(MAX_BATCH_DELAY).isJsonNull() + ? modelInfo.get(MAX_BATCH_DELAY).getAsInt() + : 100; // default value for maxBatchDelay + + responseTimeout = + modelInfo.has(RESPONSE_TIMEOUT) && !modelInfo.get(RESPONSE_TIMEOUT).isJsonNull() + ? modelInfo.get(RESPONSE_TIMEOUT).getAsInt() + : 120; // default value for responseTimeout + + startupTimeout = + modelInfo.has(STARTUP_TIMEOUT) && !modelInfo.get(STARTUP_TIMEOUT).isJsonNull() + ? modelInfo.get(STARTUP_TIMEOUT).getAsInt() + : 120; // default value for startupTimeout + + batchSize = + modelInfo.has(BATCH_SIZE) && !modelInfo.get(BATCH_SIZE).isJsonNull() + ? modelInfo.get(BATCH_SIZE).getAsInt() + : 1; // default value for batchSize JsonElement runtime = modelInfo.get(RUNTIME_TYPE); String runtime_str = Manifest.RuntimeType.PYTHON.getValue(); From 9a002959c278fe807bc1750c30ad563d90535e37 Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Mon, 23 Sep 2024 18:18:21 +0000 Subject: [PATCH 08/13] merge ready --- .github/workflows/kserve_cpu_tests.yml | 6 ++---- kubernetes/kserve/tests/configs/mnist_v1_cpu.yaml | 2 +- kubernetes/kserve/tests/configs/mnist_v1_gpu.yaml | 2 +- kubernetes/kserve/tests/configs/mnist_v2_cpu.yaml | 2 +- kubernetes/kserve/tests/configs/mnist_v2_gpu.yaml | 2 +- 5 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/kserve_cpu_tests.yml b/.github/workflows/kserve_cpu_tests.yml index 6b7b5f4c4c..0f56b392a6 100644 --- a/.github/workflows/kserve_cpu_tests.yml +++ b/.github/workflows/kserve_cpu_tests.yml @@ -3,10 +3,8 @@ name: KServe CPU Nightly Tests on: workflow_dispatch: # runs everyday at 5:15am - # schedule: - # - cron: '15 5 * * *' - push: - -branches: "snap_fix" + schedule: + - cron: '15 5 * * *' jobs: kserve-cpu-tests: diff --git a/kubernetes/kserve/tests/configs/mnist_v1_cpu.yaml b/kubernetes/kserve/tests/configs/mnist_v1_cpu.yaml index 6ff08ea3e4..559a27b96a 100644 --- a/kubernetes/kserve/tests/configs/mnist_v1_cpu.yaml +++ b/kubernetes/kserve/tests/configs/mnist_v1_cpu.yaml @@ -6,7 +6,7 @@ spec: predictor: pytorch: storageUri: gs://kfserving-examples/models/torchserve/image_classifier/v1 - image: udaij12/torchserve-kfs-nightly:latest-cpu + image: pytorch/torchserve-kfs-nightly:latest-cpu resources: limits: cpu: "1" diff --git a/kubernetes/kserve/tests/configs/mnist_v1_gpu.yaml b/kubernetes/kserve/tests/configs/mnist_v1_gpu.yaml index 5f8540eeb4..364ccc2716 100644 --- a/kubernetes/kserve/tests/configs/mnist_v1_gpu.yaml +++ b/kubernetes/kserve/tests/configs/mnist_v1_gpu.yaml @@ -6,7 +6,7 @@ spec: predictor: pytorch: storageUri: gs://kfserving-examples/models/torchserve/image_classifier/v1 - image: udaij12/torchserve-kfs-nightly:latest-gpu + image: pytorch/torchserve-kfs-nightly:latest-gpu resources: limits: cpu: "1" diff --git a/kubernetes/kserve/tests/configs/mnist_v2_cpu.yaml b/kubernetes/kserve/tests/configs/mnist_v2_cpu.yaml index 6a35a6489c..851b69a1e5 100644 --- a/kubernetes/kserve/tests/configs/mnist_v2_cpu.yaml +++ b/kubernetes/kserve/tests/configs/mnist_v2_cpu.yaml @@ -7,7 +7,7 @@ spec: pytorch: protocolVersion: v2 storageUri: gs://kfserving-examples/models/torchserve/image_classifier/v2 - image: udaij12/torchserve-kfs-nightly:latest-cpu + image: pytorch/torchserve-kfs-nightly:latest-cpu resources: limits: cpu: "1" diff --git a/kubernetes/kserve/tests/configs/mnist_v2_gpu.yaml b/kubernetes/kserve/tests/configs/mnist_v2_gpu.yaml index 4d5294d350..00703ef833 100644 --- a/kubernetes/kserve/tests/configs/mnist_v2_gpu.yaml +++ b/kubernetes/kserve/tests/configs/mnist_v2_gpu.yaml @@ -7,7 +7,7 @@ spec: pytorch: protocolVersion: v2 storageUri: gs://kfserving-examples/models/torchserve/image_classifier/v2 - image: udaij12/torchserve-kfs-nightly:latest-gpu + image: pytorch/torchserve-kfs-nightly:latest-gpu resources: limits: cpu: "1" From 970db0e894d0c8d2e48d1c884fd120befc67f093 Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Mon, 23 Sep 2024 20:16:45 +0000 Subject: [PATCH 09/13] adding source of truth --- .../serve/archive/model/ModelConfig.java | 13 +++++++---- .../java/org/pytorch/serve/wlm/Model.java | 23 +++++-------------- 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/frontend/archive/src/main/java/org/pytorch/serve/archive/model/ModelConfig.java b/frontend/archive/src/main/java/org/pytorch/serve/archive/model/ModelConfig.java index c83da0523c..2eaf52e27d 100644 --- a/frontend/archive/src/main/java/org/pytorch/serve/archive/model/ModelConfig.java +++ b/frontend/archive/src/main/java/org/pytorch/serve/archive/model/ModelConfig.java @@ -11,18 +11,23 @@ public class ModelConfig { private static final Logger logger = LoggerFactory.getLogger(ModelConfig.class); + public static final int defaultMinWorkers = 1; + public static final int defaultBatchSize = 1; + public static final int defaultStartupTimeout = 120; // unit: sec + public static final int defaultResponseTimeout = 120; // unit: sec + /** the minimum number of workers of a model */ - private int minWorkers; + private int minWorkers = defaultMinWorkers; /** the maximum number of workers of a model */ private int maxWorkers; /** the batch size of a model */ - private int batchSize; + private int batchSize = defaultBatchSize; /** the maximum delay in msec of a batch of a model */ private int maxBatchDelay; /** the timeout in sec of a specific model's response. */ - private int responseTimeout = 120; // unit: sec + private int responseTimeout = defaultResponseTimeout; /** the timeout in sec of a specific model's startup. */ - private int startupTimeout = 120; // unit: sec + private int startupTimeout = defaultStartupTimeout; /** * the device type where the model is loaded. It can be gpu, cpu. The model is loaded on CPU if * deviceType: "cpu" is set on a GPU host. diff --git a/frontend/server/src/main/java/org/pytorch/serve/wlm/Model.java b/frontend/server/src/main/java/org/pytorch/serve/wlm/Model.java index b2019cf765..da8f5bdc26 100644 --- a/frontend/server/src/main/java/org/pytorch/serve/wlm/Model.java +++ b/frontend/server/src/main/java/org/pytorch/serve/wlm/Model.java @@ -193,32 +193,21 @@ public void setModelState(JsonObject modelInfo) { minWorkers = modelInfo.has(MIN_WORKERS) && !modelInfo.get(MIN_WORKERS).isJsonNull() ? modelInfo.get(MIN_WORKERS).getAsInt() - : 1; // default value for minWorkers - - maxWorkers = - modelInfo.has(MAX_WORKERS) && !modelInfo.get(MAX_WORKERS).isJsonNull() - ? modelInfo.get(MAX_WORKERS).getAsInt() - : 5; // default value for maxWorkers - - maxBatchDelay = - modelInfo.has(MAX_BATCH_DELAY) && !modelInfo.get(MAX_BATCH_DELAY).isJsonNull() - ? modelInfo.get(MAX_BATCH_DELAY).getAsInt() - : 100; // default value for maxBatchDelay - + : modelArchive.getModelConfig().defaultMinWorkers; // default value for minWorkers + maxWorkers = modelInfo.get(MAX_WORKERS).getAsInt(); + maxBatchDelay = modelInfo.get(MAX_BATCH_DELAY).getAsInt(); responseTimeout = modelInfo.has(RESPONSE_TIMEOUT) && !modelInfo.get(RESPONSE_TIMEOUT).isJsonNull() ? modelInfo.get(RESPONSE_TIMEOUT).getAsInt() - : 120; // default value for responseTimeout - + : modelArchive.getModelConfig().defaultResponseTimeout; // default value for responseTimeout startupTimeout = modelInfo.has(STARTUP_TIMEOUT) && !modelInfo.get(STARTUP_TIMEOUT).isJsonNull() ? modelInfo.get(STARTUP_TIMEOUT).getAsInt() - : 120; // default value for startupTimeout - + : modelArchive.getModelConfig().defaultStartupTimeout; // default value for startupTimeout batchSize = modelInfo.has(BATCH_SIZE) && !modelInfo.get(BATCH_SIZE).isJsonNull() ? modelInfo.get(BATCH_SIZE).getAsInt() - : 1; // default value for batchSize + : modelArchive.getModelConfig().defaultBatchSize; // default value for batchSize JsonElement runtime = modelInfo.get(RUNTIME_TYPE); String runtime_str = Manifest.RuntimeType.PYTHON.getValue(); From 198fcf20fcdcbf03ba12d0d754e0ffec070517d3 Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Mon, 23 Sep 2024 20:22:37 +0000 Subject: [PATCH 10/13] adding source of truth --- .../main/java/org/pytorch/serve/archive/model/ModelConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/archive/src/main/java/org/pytorch/serve/archive/model/ModelConfig.java b/frontend/archive/src/main/java/org/pytorch/serve/archive/model/ModelConfig.java index 2eaf52e27d..6186f40fcd 100644 --- a/frontend/archive/src/main/java/org/pytorch/serve/archive/model/ModelConfig.java +++ b/frontend/archive/src/main/java/org/pytorch/serve/archive/model/ModelConfig.java @@ -11,7 +11,7 @@ public class ModelConfig { private static final Logger logger = LoggerFactory.getLogger(ModelConfig.class); - public static final int defaultMinWorkers = 1; + public static final int defaultMinWorkers; public static final int defaultBatchSize = 1; public static final int defaultStartupTimeout = 120; // unit: sec public static final int defaultResponseTimeout = 120; // unit: sec From ed9aefc798f9e56b4c7f26fc4d70ca853b666b77 Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Mon, 23 Sep 2024 20:28:24 +0000 Subject: [PATCH 11/13] fixing tests --- .../java/org/pytorch/serve/archive/model/ModelConfig.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/archive/src/main/java/org/pytorch/serve/archive/model/ModelConfig.java b/frontend/archive/src/main/java/org/pytorch/serve/archive/model/ModelConfig.java index 6186f40fcd..dd0a93b21a 100644 --- a/frontend/archive/src/main/java/org/pytorch/serve/archive/model/ModelConfig.java +++ b/frontend/archive/src/main/java/org/pytorch/serve/archive/model/ModelConfig.java @@ -11,13 +11,13 @@ public class ModelConfig { private static final Logger logger = LoggerFactory.getLogger(ModelConfig.class); - public static final int defaultMinWorkers; + public static final int defaultMinWorkers = 1; public static final int defaultBatchSize = 1; public static final int defaultStartupTimeout = 120; // unit: sec public static final int defaultResponseTimeout = 120; // unit: sec /** the minimum number of workers of a model */ - private int minWorkers = defaultMinWorkers; + private int minWorkers; /** the maximum number of workers of a model */ private int maxWorkers; /** the batch size of a model */ From 83ebb39fe572b62bd3d211edcacb15271357a6f9 Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Mon, 23 Sep 2024 20:54:21 +0000 Subject: [PATCH 12/13] format --- .../src/main/java/org/pytorch/serve/wlm/Model.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/frontend/server/src/main/java/org/pytorch/serve/wlm/Model.java b/frontend/server/src/main/java/org/pytorch/serve/wlm/Model.java index da8f5bdc26..692912df5a 100644 --- a/frontend/server/src/main/java/org/pytorch/serve/wlm/Model.java +++ b/frontend/server/src/main/java/org/pytorch/serve/wlm/Model.java @@ -193,21 +193,25 @@ public void setModelState(JsonObject modelInfo) { minWorkers = modelInfo.has(MIN_WORKERS) && !modelInfo.get(MIN_WORKERS).isJsonNull() ? modelInfo.get(MIN_WORKERS).getAsInt() - : modelArchive.getModelConfig().defaultMinWorkers; // default value for minWorkers + : modelArchive.getModelConfig() + .defaultMinWorkers; // default value for minWorkers maxWorkers = modelInfo.get(MAX_WORKERS).getAsInt(); maxBatchDelay = modelInfo.get(MAX_BATCH_DELAY).getAsInt(); responseTimeout = modelInfo.has(RESPONSE_TIMEOUT) && !modelInfo.get(RESPONSE_TIMEOUT).isJsonNull() ? modelInfo.get(RESPONSE_TIMEOUT).getAsInt() - : modelArchive.getModelConfig().defaultResponseTimeout; // default value for responseTimeout + : modelArchive.getModelConfig() + .defaultResponseTimeout; // default value for responseTimeout startupTimeout = modelInfo.has(STARTUP_TIMEOUT) && !modelInfo.get(STARTUP_TIMEOUT).isJsonNull() ? modelInfo.get(STARTUP_TIMEOUT).getAsInt() - : modelArchive.getModelConfig().defaultStartupTimeout; // default value for startupTimeout + : modelArchive.getModelConfig() + .defaultStartupTimeout; // default value for startupTimeout batchSize = modelInfo.has(BATCH_SIZE) && !modelInfo.get(BATCH_SIZE).isJsonNull() ? modelInfo.get(BATCH_SIZE).getAsInt() - : modelArchive.getModelConfig().defaultBatchSize; // default value for batchSize + : modelArchive.getModelConfig() + .defaultBatchSize; // default value for batchSize JsonElement runtime = modelInfo.get(RUNTIME_TYPE); String runtime_str = Manifest.RuntimeType.PYTHON.getValue(); From 5095ba804d44301f82d9cfefe00c2d4fbb96fde4 Mon Sep 17 00:00:00 2001 From: Udai Jain Date: Mon, 23 Sep 2024 23:10:30 +0000 Subject: [PATCH 13/13] removing defaults --- .../org/pytorch/serve/archive/model/ModelConfig.java | 4 +--- .../src/main/java/org/pytorch/serve/wlm/Model.java | 12 ++---------- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/frontend/archive/src/main/java/org/pytorch/serve/archive/model/ModelConfig.java b/frontend/archive/src/main/java/org/pytorch/serve/archive/model/ModelConfig.java index dd0a93b21a..1693bc8424 100644 --- a/frontend/archive/src/main/java/org/pytorch/serve/archive/model/ModelConfig.java +++ b/frontend/archive/src/main/java/org/pytorch/serve/archive/model/ModelConfig.java @@ -11,8 +11,6 @@ public class ModelConfig { private static final Logger logger = LoggerFactory.getLogger(ModelConfig.class); - public static final int defaultMinWorkers = 1; - public static final int defaultBatchSize = 1; public static final int defaultStartupTimeout = 120; // unit: sec public static final int defaultResponseTimeout = 120; // unit: sec @@ -21,7 +19,7 @@ public class ModelConfig { /** the maximum number of workers of a model */ private int maxWorkers; /** the batch size of a model */ - private int batchSize = defaultBatchSize; + private int batchSize; /** the maximum delay in msec of a batch of a model */ private int maxBatchDelay; /** the timeout in sec of a specific model's response. */ diff --git a/frontend/server/src/main/java/org/pytorch/serve/wlm/Model.java b/frontend/server/src/main/java/org/pytorch/serve/wlm/Model.java index 692912df5a..32ae5bcc06 100644 --- a/frontend/server/src/main/java/org/pytorch/serve/wlm/Model.java +++ b/frontend/server/src/main/java/org/pytorch/serve/wlm/Model.java @@ -190,13 +190,10 @@ public JsonObject getModelState(boolean isDefaultVersion) { } public void setModelState(JsonObject modelInfo) { - minWorkers = - modelInfo.has(MIN_WORKERS) && !modelInfo.get(MIN_WORKERS).isJsonNull() - ? modelInfo.get(MIN_WORKERS).getAsInt() - : modelArchive.getModelConfig() - .defaultMinWorkers; // default value for minWorkers + minWorkers = modelInfo.get(MIN_WORKERS).getAsInt(); maxWorkers = modelInfo.get(MAX_WORKERS).getAsInt(); maxBatchDelay = modelInfo.get(MAX_BATCH_DELAY).getAsInt(); + batchSize = modelInfo.get(BATCH_SIZE).getAsInt(); responseTimeout = modelInfo.has(RESPONSE_TIMEOUT) && !modelInfo.get(RESPONSE_TIMEOUT).isJsonNull() ? modelInfo.get(RESPONSE_TIMEOUT).getAsInt() @@ -207,11 +204,6 @@ public void setModelState(JsonObject modelInfo) { ? modelInfo.get(STARTUP_TIMEOUT).getAsInt() : modelArchive.getModelConfig() .defaultStartupTimeout; // default value for startupTimeout - batchSize = - modelInfo.has(BATCH_SIZE) && !modelInfo.get(BATCH_SIZE).isJsonNull() - ? modelInfo.get(BATCH_SIZE).getAsInt() - : modelArchive.getModelConfig() - .defaultBatchSize; // default value for batchSize JsonElement runtime = modelInfo.get(RUNTIME_TYPE); String runtime_str = Manifest.RuntimeType.PYTHON.getValue();