[serving][python] Backport for pull #2173 to v0.28, support for non-2… (

#2466)

Co-authored-by: Siddharth Venkatesan <[email protected]>

engines/python/src/main/java/ai/djl/python/engine/RollingBatch.java

@@ -26,6 +26,7 @@
 import ai.djl.util.JsonUtils;
 import ai.djl.util.PairList;
 import ai.djl.util.RandomUtils;
+import ai.djl.util.Utils;
 
 import com.google.gson.JsonObject;
 
@@ -63,6 +64,9 @@ class RollingBatch implements Runnable {
                         return t;
                     });
 
+    private static boolean isBackportForNonStreamingHttpErrorCodes =
+            getBackportForNonStreamingHttpErrorCodesOption();
+
     private PyProcess process;
     private int maxRollingBatchSize;
     private int timeout;
@@ -249,6 +253,21 @@ public void shutdown() {
         currentThread.interrupt();
     }
 
+    private static boolean getBackportForNonStreamingHttpErrorCodesOption() {
+
+        // Option the allows non-streaming requests to return non-200 error code on error
+        boolean result =
+                Boolean.parseBoolean(
+                        Utils.getEnvOrSystemProperty(
+                                "SERVING_BACKPORT_FOR_NON_STREAMING_HTTP_ERROR_CODES"));
+        if (result) {
+            logger.info(
+                    "SERVING_BACKPORT_FOR_NON_STREAMING_HTTP_ERROR_CODES is enabled."
+                            + " See https://github.com/deepjavalibrary/djl-serving/pull/2173");
+        }
+        return result;
+    }
+
     private static final class Request {
 
         Input input;
@@ -348,6 +367,11 @@ void addResponse(byte[] json, Map<String, String> properties) {
                         break;
                 }
             }
+            if (isBackportForNonStreamingHttpErrorCodes && !last) {
+                // in non-streaming cases, do not return content until generation is finished
+                return;
+            }
+
             if (code != null) {
                 Map<String, Object> map = new ConcurrentHashMap<>(2);
                 map.put("code", Integer.parseInt(code));