TEST tmp modif

[phymbert]

No description provided.

[github-actions]

📈 llama.cpp server for bench-server-baseline on Standard_NC4as_T4_v3 for phi-2-q4_0: 950 iterations 🚀

Expand details for performance related PR only

• Concurrent users: 8, duration: 10m
• HTTP request : avg=4781.41ms p(90)=13466.66ms fails=0, finish reason: stop=950 truncated=0
• Prompt processing (pp): avg=236.13tk/s p(90)=692.7tk/s total=367.77tk/s
• Token generation (tg): avg=175.11tk/s p(90)=492.15tk/s total=239.68tk/s
• ggml-org/models/phi-2/ggml-model-q4_0.gguf parallel=8 ctx-size=16384 ngl=33 batch-size=2048 ubatch-size=256 pp=1024 pp+tg=2048 branch=hp/server/bench/test-new-job commit=84bff92dac69dbb8be2b06e747305f8ff31331dc

More

---
config:
    xyChart:
        titleFontSize: 12
        width: 900
        height: 600
    themeVariables:
        xyChart:
            titleColor: "#000000"
---
xychart-beta
    title "llama.cpp bench-server-baseline on Standard_NC4as_T4_v3
 duration=10m 950 iterations"
    y-axis "llamacpp:prompt_tokens_seconds"
    x-axis "llamacpp:prompt_tokens_seconds" 1712175263 --> 1712175893
    line [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 453.84, 453.84, 453.84, 453.84, 453.84, 541.56, 541.56, 541.56, 541.56, 541.56, 630.02, 630.02, 630.02, 630.02, 630.02, 670.08, 670.08, 670.08, 670.08, 670.08, 711.4, 711.4, 711.4, 711.4, 711.4, 774.34, 774.34, 774.34, 774.34, 774.34, 832.6, 832.6, 832.6, 832.6, 832.6, 901.46, 901.46, 901.46, 901.46, 901.46, 920.69, 920.69, 920.69, 920.69, 920.69, 953.42, 953.42, 953.42, 953.42, 953.42, 960.72, 960.72, 960.72, 960.72, 960.72, 966.11, 966.11, 966.11, 966.11, 966.11, 998.54, 998.54, 998.54, 998.54, 998.54, 1018.41, 1018.41, 1018.41, 1018.41, 1018.41, 1039.77, 1039.77, 1039.77, 1039.77, 1039.77, 1029.2, 1029.2, 1029.2, 1029.2, 1029.2, 1043.27, 1043.27, 1043.27, 1043.27, 1043.27, 1043.94, 1043.94, 1043.94, 1043.94, 1043.94, 1051.65, 1051.65, 1051.65, 1051.65, 1051.65, 1066.32, 1066.32, 1066.32, 1066.32, 1066.32, 1074.84, 1074.84, 1074.84, 1074.84, 1074.84, 1075.12, 1075.12, 1075.12, 1075.12, 1075.12, 1081.9, 1081.9, 1081.9, 1081.9, 1081.9, 1090.51, 1090.51, 1090.51, 1090.51, 1090.51, 1034.37, 1034.37, 1034.37, 1034.37, 1034.37, 1030.08, 1030.08, 1030.08, 1030.08, 1030.08, 1030.01, 1030.01, 1030.01, 1030.01, 1030.01, 1037.4, 1037.4, 1037.4, 1037.4, 1037.4, 1033.65, 1033.65, 1033.65, 1033.65, 1033.65, 1036.63, 1036.63, 1036.63, 1036.63, 1036.63, 1048.54, 1048.54, 1048.54, 1048.54, 1048.54, 1056.87, 1056.87, 1056.87, 1056.87, 1056.87, 1061.17, 1061.17, 1061.17, 1061.17, 1061.17, 1068.19, 1068.19, 1068.19, 1068.19, 1068.19, 1068.57, 1068.57, 1068.57, 1068.57, 1068.57, 1072.96, 1072.96, 1072.96, 1072.96, 1072.96, 1082.05, 1082.05, 1082.05, 1082.05, 1082.05, 1091.99, 1091.99, 1091.99, 1091.99, 1091.99, 1087.83, 1087.83, 1087.83, 1087.83, 1087.83, 1090.18, 1090.18, 1090.18, 1090.18, 1090.18, 1086.63, 1086.63, 1086.63, 1086.63, 1086.63, 1089.36, 1089.36, 1089.36, 1089.36, 1089.36, 1086.65, 1086.65, 1086.65, 1086.65, 1086.65, 1080.45, 1080.45, 1080.45, 1080.45, 1080.45, 1081.76, 1081.76, 1081.76, 1081.76, 1081.76, 1084.23, 1084.23, 1084.23, 1084.23, 1084.23, 1090.85, 1090.85, 1090.85, 1090.85, 1090.85, 1093.94, 1093.94, 1093.94, 1093.94, 1093.94, 1101.98, 1101.98, 1101.98, 1101.98, 1101.98, 1106.9, 1106.9, 1106.9, 1106.9, 1106.9, 1111.38, 1111.38, 1111.38, 1111.38, 1111.38, 1113.91, 1113.91, 1113.91, 1113.91, 1113.91, 1114.24, 1114.24, 1114.24, 1114.24, 1114.24, 1117.25, 1117.25, 1117.25, 1117.25, 1117.25, 1118.45, 1118.45, 1118.45, 1118.45, 1118.45, 1118.07, 1118.07, 1118.07, 1118.07, 1118.07, 1121.34, 1121.34, 1121.34, 1121.34, 1121.34, 1122.47, 1122.47, 1122.47, 1122.47, 1122.47, 1125.1, 1125.1, 1125.1, 1125.1, 1125.1, 1128.6, 1128.6, 1128.6, 1128.6, 1128.6, 1129.58]
                    

Loading

More

---
config:
    xyChart:
        titleFontSize: 12
        width: 900
        height: 600
    themeVariables:
        xyChart:
            titleColor: "#000000"
---
xychart-beta
    title "llama.cpp bench-server-baseline on Standard_NC4as_T4_v3
 duration=10m 950 iterations"
    y-axis "llamacpp:predicted_tokens_seconds"
    x-axis "llamacpp:predicted_tokens_seconds" 1712175263 --> 1712175893
    line [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 46.54, 46.54, 46.54, 46.54, 46.54, 31.0, 31.0, 31.0, 31.0, 31.0, 31.52, 31.52, 31.52, 31.52, 31.52, 34.48, 34.48, 34.48, 34.48, 34.48, 35.59, 35.59, 35.59, 35.59, 35.59, 36.15, 36.15, 36.15, 36.15, 36.15, 35.42, 35.42, 35.42, 35.42, 35.42, 33.97, 33.97, 33.97, 33.97, 33.97, 32.78, 32.78, 32.78, 32.78, 32.78, 32.97, 32.97, 32.97, 32.97, 32.97, 32.77, 32.77, 32.77, 32.77, 32.77, 32.64, 32.64, 32.64, 32.64, 32.64, 32.62, 32.62, 32.62, 32.62, 32.62, 32.72, 32.72, 32.72, 32.72, 32.72, 32.81, 32.81, 32.81, 32.81, 32.81, 32.89, 32.89, 32.89, 32.89, 32.89, 33.27, 33.27, 33.27, 33.27, 33.27, 33.68, 33.68, 33.68, 33.68, 33.68, 34.06, 34.06, 34.06, 34.06, 34.06, 34.02, 34.02, 34.02, 34.02, 34.02, 33.78, 33.78, 33.78, 33.78, 33.78, 33.99, 33.99, 33.99, 33.99, 33.99, 34.21, 34.21, 34.21, 34.21, 34.21, 34.11, 34.11, 34.11, 34.11, 34.11, 33.67, 33.67, 33.67, 33.67, 33.67, 33.34, 33.34, 33.34, 33.34, 33.34, 32.23, 32.23, 32.23, 32.23, 32.23, 32.16, 32.16, 32.16, 32.16, 32.16, 32.36, 32.36, 32.36, 32.36, 32.36, 32.47, 32.47, 32.47, 32.47, 32.47, 32.55, 32.55, 32.55, 32.55, 32.55, 32.72, 32.72, 32.72, 32.72, 32.72, 32.98, 32.98, 32.98, 32.98, 32.98, 33.18, 33.18, 33.18, 33.18, 33.18, 33.47, 33.47, 33.47, 33.47, 33.47, 33.76, 33.76, 33.76, 33.76, 33.76, 33.87, 33.87, 33.87, 33.87, 33.87, 33.99, 33.99, 33.99, 33.99, 33.99, 33.84, 33.84, 33.84, 33.84, 33.84, 33.82, 33.82, 33.82, 33.82, 33.82, 33.83, 33.83, 33.83, 33.83, 33.83, 33.58, 33.58, 33.58, 33.58, 33.58, 33.22, 33.22, 33.22, 33.22, 33.22, 32.83, 32.83, 32.83, 32.83, 32.83, 32.84, 32.84, 32.84, 32.84, 32.84, 32.99, 32.99, 32.99, 32.99, 32.99, 33.14, 33.14, 33.14, 33.14, 33.14, 33.15, 33.15, 33.15, 33.15, 33.15, 33.22, 33.22, 33.22, 33.22, 33.22, 33.22, 33.22, 33.22, 33.22, 33.22, 33.34, 33.34, 33.34, 33.34, 33.34, 33.37, 33.37, 33.37, 33.37, 33.37, 33.4, 33.4, 33.4, 33.4, 33.4, 33.56, 33.56, 33.56, 33.56, 33.56, 33.63, 33.63, 33.63, 33.63, 33.63, 33.75, 33.75, 33.75, 33.75, 33.75, 33.88, 33.88, 33.88, 33.88, 33.88, 34.03, 34.03, 34.03, 34.03, 34.03, 34.16, 34.16, 34.16, 34.16, 34.16, 34.2, 34.2, 34.2, 34.2, 34.2, 34.1]
                    

Loading

Details

More

---
config:
    xyChart:
        titleFontSize: 12
        width: 900
        height: 600
    themeVariables:
        xyChart:
            titleColor: "#000000"
---
xychart-beta
    title "llama.cpp bench-server-baseline on Standard_NC4as_T4_v3
 duration=10m 950 iterations"
    y-axis "llamacpp:kv_cache_usage_ratio"
    x-axis "llamacpp:kv_cache_usage_ratio" 1712175263 --> 1712175893
    line [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.3, 0.3, 0.3, 0.3, 0.3, 0.18, 0.18, 0.18, 0.18, 0.18, 0.15, 0.15, 0.15, 0.15, 0.15, 0.16, 0.16, 0.16, 0.16, 0.16, 0.11, 0.11, 0.11, 0.11, 0.11, 0.27, 0.27, 0.27, 0.27, 0.27, 0.25, 0.25, 0.25, 0.25, 0.25, 0.25, 0.25, 0.25, 0.25, 0.25, 0.17, 0.17, 0.17, 0.17, 0.17, 0.25, 0.25, 0.25, 0.25, 0.25, 0.25, 0.25, 0.25, 0.25, 0.25, 0.19, 0.19, 0.19, 0.19, 0.19, 0.24, 0.24, 0.24, 0.24, 0.24, 0.1, 0.1, 0.1, 0.1, 0.1, 0.26, 0.26, 0.26, 0.26, 0.26, 0.26, 0.26, 0.26, 0.26, 0.26, 0.12, 0.12, 0.12, 0.12, 0.12, 0.16, 0.16, 0.16, 0.16, 0.16, 0.24, 0.24, 0.24, 0.24, 0.24, 0.21, 0.21, 0.21, 0.21, 0.21, 0.16, 0.16, 0.16, 0.16, 0.16, 0.15, 0.15, 0.15, 0.15, 0.15, 0.13, 0.13, 0.13, 0.13, 0.13, 0.41, 0.41, 0.41, 0.41, 0.41, 0.45, 0.45, 0.45, 0.45, 0.45, 0.5, 0.5, 0.5, 0.5, 0.5, 0.18, 0.18, 0.18, 0.18, 0.18, 0.13, 0.13, 0.13, 0.13, 0.13, 0.14, 0.14, 0.14, 0.14, 0.14, 0.17, 0.17, 0.17, 0.17, 0.17, 0.11, 0.11, 0.11, 0.11, 0.11, 0.14, 0.14, 0.14, 0.14, 0.14, 0.12, 0.12, 0.12, 0.12, 0.12, 0.17, 0.17, 0.17, 0.17, 0.17, 0.07, 0.07, 0.07, 0.07, 0.07, 0.13, 0.13, 0.13, 0.13, 0.13, 0.17, 0.17, 0.17, 0.17, 0.17, 0.17, 0.17, 0.17, 0.17, 0.17, 0.25, 0.25, 0.25, 0.25, 0.25, 0.23, 0.23, 0.23, 0.23, 0.23, 0.33, 0.33, 0.33, 0.33, 0.33, 0.25, 0.25, 0.25, 0.25, 0.25, 0.3, 0.3, 0.3, 0.3, 0.3, 0.17, 0.17, 0.17, 0.17, 0.17, 0.15, 0.15, 0.15, 0.15, 0.15, 0.14, 0.14, 0.14, 0.14, 0.14, 0.25, 0.25, 0.25, 0.25, 0.25, 0.18, 0.18, 0.18, 0.18, 0.18, 0.22, 0.22, 0.22, 0.22, 0.22, 0.17, 0.17, 0.17, 0.17, 0.17, 0.16, 0.16, 0.16, 0.16, 0.16, 0.14, 0.14, 0.14, 0.14, 0.14, 0.16, 0.16, 0.16, 0.16, 0.16, 0.23, 0.23, 0.23, 0.23, 0.23, 0.16, 0.16, 0.16, 0.16, 0.16, 0.16, 0.16, 0.16, 0.16, 0.16, 0.19, 0.19, 0.19, 0.19, 0.19, 0.15, 0.15, 0.15, 0.15, 0.15, 0.2, 0.2, 0.2, 0.2, 0.2, 0.24, 0.24, 0.24, 0.24, 0.24, 0.36]
                    

Loading

More

---
config:
    xyChart:
        titleFontSize: 12
        width: 900
        height: 600
    themeVariables:
        xyChart:
            titleColor: "#000000"
---
xychart-beta
    title "llama.cpp bench-server-baseline on Standard_NC4as_T4_v3
 duration=10m 950 iterations"
    y-axis "llamacpp:requests_processing"
    x-axis "llamacpp:requests_processing" 1712175263 --> 1712175893
    line [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 7.0, 7.0, 7.0, 7.0, 7.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 7.0, 7.0, 7.0, 7.0, 7.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 7.0, 7.0, 7.0, 7.0, 7.0, 8.0, 8.0, 8.0, 8.0, 8.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 7.0, 7.0, 7.0, 7.0, 7.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 7.0, 7.0, 7.0, 7.0, 7.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 8.0, 8.0, 8.0, 8.0, 8.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 6.0, 6.0, 6.0, 6.0, 6.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 7.0, 7.0, 7.0, 7.0, 7.0, 1.0]
                    

Loading