update New daily release

docker-dev-canary/main.py

@@ -9,7 +9,6 @@
 from RouterTable import RouterTable
 from eval_model import eval_model
 
-global model_status
 model_status = "No_experiment"
 
 app = Flask(__name__)
@@ -34,6 +33,8 @@ def index():
 @app.route('/recommend/<user_id>')
 def recommend(user_id):
     global CONFIG_EXPERIMENTS, EXPERIMENT_LOG
+    global model_status
+
     try:
         user_id = int(user_id)
     except:
@@ -85,6 +86,7 @@ def recommend(user_id):
 @app.route('/test', methods=['GET', 'POST'])
 def test():
     global CONFIG_EXPERIMENTS
+    global model_status
     test_conf = request.json
     isSuccessful, config = parse_test_config(test_conf)
     print(config)
@@ -94,16 +96,23 @@ def test():
         ROUTER.set_new_treatment(BASE_PORT + 2, config['ModelInfo']['ExpPercentage'] if 'ExpPercentage' in config['ModelInfo'] else 0)
         # ROUTER.set_new_treatment(BASE_PORT + 2, config['ModelInfo']['ExpPercentage'])
         global model_status
-        model_staus = "Pending"
+        model_status = "Pending"
+        print(model_status)
         start_test(config["ModelInfo"]["ModelContainerName"], 8082, BASE_PORT + 2)
         print(config["ModelInfo"]["ModelContainerName"])
         return 'Test started successfully'
     else:
         return 'Test was not started, no changes have been applied'
 
-@app.route('/model_status', methods=['GET', 'POST'])
+@app.route('/model_status', methods=['GET'])
 def jenkins_query():
     return model_status
 
+@app.route('/reset_model_status', methods=['GET', 'POST'])
+def reset_status():
+    global model_status
+    model_status = "No_experiment"
+    return ""
+
 if __name__ == '__main__':
-    app.run(host='0.0.0.0', port=8092, debug=True)
+    app.run(host='0.0.0.0', port=8082, debug=True)

workflow/CanaryConfig.json

@@ -2,8 +2,8 @@
     "DeploymentId": "2",
     "DeploymentType": "CanaryTest",
     "DeploymentParam": {
-      "Step": 0.1,
-      "interval": 10000
+      "Step": 0.2,
+      "interval": 5
     },
     "ModelInfo":
     {

workflow/check_model_status.py

@@ -0,0 +1,51 @@
+import sys
+import requests
+import time
+hostname = "http://128.2.204.234"
+port = "8082"
+
+def reset_status():
+    url = hostname + ":" + port + "/reset_model_status"
+    requests.post(url)
+
+if __name__=='__main__':
+    url = hostname + ":" + port + "/model_status"
+    response = requests.get(url)
+    status = str(response.content.decode("utf-8"))
+    print(status)
+    report = ""
+    if status == "No_experiment":
+        report = "Supervisor Error. No experiment detected!"
+        print(report)
+        sys.exit(1)
+    counter = 0
+    while status == "Pending":
+        if counter <= 100:
+            status = str(requests.get(url).content.decode("utf-8"))
+            print('Pending')
+            if status == "Success":
+                report = "Model evaluation success! Model is ready to release!"
+                print(report)
+                reset_status()
+                sys.exit(0)
+            if status == "Failed":
+                report = "Model evaluation failed! Model is going to rollback!"
+                print(report)
+                reset_status()
+                sys.exit(1)
+            counter += 1
+            time.sleep(10)
+        else:
+            break
+    if status == "Pending" or status == "No_experiment" or status == "Failed":
+        report = "Runtime Error! Model Evaluation failed! Model is going to rollback!"
+        print(report)
+        reset_status()
+        sys.exit(1)
+    else:
+        report = "Model evaluation success! Model is ready to release!"
+        print(report)
+        reset_status()
+        sys.exit(0)
+
+

workflow/daily_relase_workflow.sh

@@ -32,11 +32,20 @@ mv workflow/run_server.sh web_server/run_server.sh
 
 current=`date "+%Y-%m-%d %H:%M:%S"`
 timeStamp=`date -d "$current" +%s`
-image_name='Webservice:'${DailyActiveModelSetting}'-'${timeStamp}
+image_name=${DailyActiveModelSetting}'-'${timeStamp}
 
 echo 'Building container:'$image_name
 sudo docker build -t web-service:$image_name .
 imageId=`sudo docker images -q web-service:${image_name}`
 echo 'Build docker image successfully, imageId is: '$imageId
 
-python3 workflow/send_new_release_to_supervisor.py 'Canary' $imageId
+python3 workflow/send_new_release_to_supervisor.py 'Canary' 'web-service:'${image_name}
+
+python3 workflow/check_model_status.py
+if [ $? -eq 0 ]
+then
+   echo 'Canary Test Succeeded! Model is Ready to Release!'
+else
+   echo 'Canary Test Failed! Model is going to rollback!'
+   exit 1 
+fi

workflow/send_new_release_to_supervisor.py

@@ -6,12 +6,12 @@
     exp_type, image_name = sys.argv[1], sys.argv[2]
     json_str = None
     if exp_type == 'Canary':
-        with open("CanaryConfig.json",'r') as f:
+        with open("workflow/CanaryConfig.json",'r') as f:
             d = json.load(f)
             d["ModelInfo"]["ModelContainerName"] = image_name
             json_str = json.dumps(d) 
     elif exp_type == 'ABTest':
-        with open("CanaryConfig.json",'r') as f:
+        with open("workflow/ABTestConfig.json",'r') as f:
             d = json.load(f)
             d["ModelInfo"]["ModelContainerName"] = image_name
             json_str = json.dumps(d)