tests : minor bash stuff

[ggerganov]

#6688 cont

[github-actions]

📈 llama.cpp server for bench-server-baseline on Standard_NC4as_T4_v3 for phi-2-q4_0: 198 iterations 🚀

Expand details for performance related PR only

• Concurrent users: 8, duration: 10m
• HTTP request : avg=24356.93ms p(95)=41716.68ms fails=, finish reason: stop=68 truncated=130
• Prompt processing (pp): avg=274.94tk/s p(95)=776.96tk/s
• Token generation (tg): avg=23.84tk/s p(95)=24.92tk/s
• ggml-org/models/phi-2/ggml-model-q4_0.gguf parallel=8 ctx-size=16384 ngl=33 batch-size=2048 ubatch-size=256 pp=1024 pp+tg=2048 branch=gg/tests-minor commit=3e4a3e4fa8394b1b91950a534d3b8e23bbe6c922

More

---
config:
    xyChart:
        titleFontSize: 12
        width: 900
        height: 600
    themeVariables:
        xyChart:
            titleColor: "#000000"
---
xychart-beta
    title "llama.cpp bench-server-baseline on Standard_NC4as_T4_v3
 duration=10m 198 iterations"
    y-axis "llamacpp:prompt_tokens_seconds"
    x-axis "llamacpp:prompt_tokens_seconds" 1714043552 --> 1714044188
    line [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 234.92, 234.92, 234.92, 234.92, 234.92, 265.52, 265.52, 265.52, 265.52, 265.52, 277.64, 277.64, 277.64, 277.64, 277.64, 311.15, 311.15, 311.15, 311.15, 311.15, 402.16, 402.16, 402.16, 402.16, 402.16, 406.37, 406.37, 406.37, 406.37, 406.37, 406.37, 406.37, 406.37, 406.37, 406.37, 405.85, 405.85, 405.85, 405.85, 405.85, 394.67, 394.67, 394.67, 394.67, 394.67, 406.8, 406.8, 406.8, 406.8, 406.8, 410.39, 410.39, 410.39, 410.39, 410.39, 422.1, 422.1, 422.1, 422.1, 422.1, 425.74, 425.74, 425.74, 425.74, 425.74, 455.1, 455.1, 455.1, 455.1, 455.1, 468.0, 468.0, 468.0, 468.0, 468.0, 469.23, 469.23, 469.23, 469.23, 469.23, 470.56, 470.56, 470.56, 470.56, 470.56, 471.44, 471.44, 471.44, 471.44, 471.44, 475.49, 475.49, 475.49, 475.49, 475.49, 482.26, 482.26, 482.26, 482.26, 482.26, 491.3, 491.3, 491.3, 491.3, 491.3, 498.62, 498.62, 498.62, 498.62, 498.62, 504.71, 504.71, 504.71, 504.71, 504.71, 505.44, 505.44, 505.44, 505.44, 505.44, 508.04, 508.04, 508.04, 508.04, 508.04, 519.45, 519.45, 519.45, 519.45, 519.45, 524.05, 524.05, 524.05, 524.05, 524.05, 524.0, 524.0, 524.0, 524.0, 524.0, 526.6, 526.6, 526.6, 526.6, 526.6, 541.62, 541.62, 541.62, 541.62, 541.62, 541.32, 541.32, 541.32, 541.32, 541.32, 540.77, 540.77, 540.77, 540.77, 540.77, 554.47, 554.47, 554.47, 554.47, 554.47, 568.71, 568.71, 568.71, 568.71, 568.71, 569.05, 569.05, 569.05, 569.05, 569.05, 569.05, 569.05, 569.05, 569.05, 569.05, 568.12, 568.12, 568.12, 568.12, 568.12, 575.0, 575.0, 575.0, 575.0, 575.0, 584.96, 584.96, 584.96, 584.96, 584.96, 587.75, 587.75, 587.75, 587.75, 587.75, 588.25, 588.25, 588.25, 588.25, 588.25, 586.54, 586.54, 586.54, 586.54, 586.54, 585.71, 585.71, 585.71, 585.71, 585.71, 586.53, 586.53, 586.53, 586.53, 586.53, 589.21, 589.21, 589.21, 589.21, 589.21, 588.94, 588.94, 588.94, 588.94, 588.94, 589.95, 589.95, 589.95, 589.95, 589.95, 591.15, 591.15, 591.15, 591.15, 591.15, 590.89, 590.89, 590.89, 590.89, 590.89, 591.09, 591.09, 591.09, 591.09, 591.09, 591.02, 591.02, 591.02, 591.02, 591.02, 602.64, 602.64, 602.64, 602.64, 602.64, 604.68, 604.68, 604.68, 604.68, 604.68, 604.0, 604.0, 604.0, 604.0, 604.0, 602.32, 602.32, 602.32, 602.32, 602.32, 605.55, 605.55, 605.55, 605.55, 605.55, 605.1, 605.1, 605.1, 605.1, 605.1, 606.57, 606.57, 606.57, 606.57, 606.57, 607.03, 607.03, 607.03, 607.03, 607.03, 611.15, 611.15, 611.15, 611.15, 611.15, 610.97, 610.97, 610.97, 610.97, 610.97, 610.97, 610.97, 610.97, 610.97]
                    

Loading

More

---
config:
    xyChart:
        titleFontSize: 12
        width: 900
        height: 600
    themeVariables:
        xyChart:
            titleColor: "#000000"
---
xychart-beta
    title "llama.cpp bench-server-baseline on Standard_NC4as_T4_v3
 duration=10m 198 iterations"
    y-axis "llamacpp:predicted_tokens_seconds"
    x-axis "llamacpp:predicted_tokens_seconds" 1714043552 --> 1714044188
    line [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 30.99, 30.99, 30.99, 30.99, 30.99, 25.36, 25.36, 25.36, 25.36, 25.36, 25.25, 25.25, 25.25, 25.25, 25.25, 21.65, 21.65, 21.65, 21.65, 21.65, 21.65, 21.65, 21.65, 21.65, 21.65, 21.65, 21.65, 21.65, 21.65, 21.65, 20.91, 20.91, 20.91, 20.91, 20.91, 16.68, 16.68, 16.68, 16.68, 16.68, 16.79, 16.79, 16.79, 16.79, 16.79, 18.13, 18.13, 18.13, 18.13, 18.13, 18.17, 18.17, 18.17, 18.17, 18.17, 18.49, 18.49, 18.49, 18.49, 18.49, 18.92, 18.92, 18.92, 18.92, 18.92, 18.88, 18.88, 18.88, 18.88, 18.88, 18.8, 18.8, 18.8, 18.8, 18.8, 18.75, 18.75, 18.75, 18.75, 18.75, 18.66, 18.66, 18.66, 18.66, 18.66, 18.81, 18.81, 18.81, 18.81, 18.81, 19.11, 19.11, 19.11, 19.11, 19.11, 19.19, 19.19, 19.19, 19.19, 19.19, 19.33, 19.33, 19.33, 19.33, 19.33, 19.35, 19.35, 19.35, 19.35, 19.35, 19.36, 19.36, 19.36, 19.36, 19.36, 19.43, 19.43, 19.43, 19.43, 19.43, 19.48, 19.48, 19.48, 19.48, 19.48, 19.54, 19.54, 19.54, 19.54, 19.54, 19.57, 19.57, 19.57, 19.57, 19.57, 19.7, 19.7, 19.7, 19.7, 19.7, 19.7, 19.7, 19.7, 19.7, 19.7, 19.68, 19.68, 19.68, 19.68, 19.68, 19.59, 19.59, 19.59, 19.59, 19.59, 19.46, 19.46, 19.46, 19.46, 19.46, 19.29, 19.29, 19.29, 19.29, 19.29, 19.19, 19.19, 19.19, 19.19, 19.19, 19.19, 19.19, 19.19, 19.19, 19.19, 19.04, 19.04, 19.04, 19.04, 19.04, 18.89, 18.89, 18.89, 18.89, 18.89, 18.75, 18.75, 18.75, 18.75, 18.75, 18.7, 18.7, 18.7, 18.7, 18.7, 18.64, 18.64, 18.64, 18.64, 18.64, 18.37, 18.37, 18.37, 18.37, 18.37, 18.23, 18.23, 18.23, 18.23, 18.23, 18.01, 18.01, 18.01, 18.01, 18.01, 17.78, 17.78, 17.78, 17.78, 17.78, 17.75, 17.75, 17.75, 17.75, 17.75, 17.7, 17.7, 17.7, 17.7, 17.7, 17.69, 17.69, 17.69, 17.69, 17.69, 17.7, 17.7, 17.7, 17.7, 17.7, 17.77, 17.77, 17.77, 17.77, 17.77, 17.83, 17.83, 17.83, 17.83, 17.83, 17.86, 17.86, 17.86, 17.86, 17.86, 17.86, 17.86, 17.86, 17.86, 17.86, 17.8, 17.8, 17.8, 17.8, 17.8, 17.72, 17.72, 17.72, 17.72, 17.72, 17.67, 17.67, 17.67, 17.67, 17.67, 17.64, 17.64, 17.64, 17.64, 17.64, 17.68, 17.68, 17.68, 17.68, 17.68, 17.73, 17.73, 17.73, 17.73, 17.73, 17.75, 17.75, 17.75, 17.75, 17.75, 17.78, 17.78, 17.78, 17.78, 17.78, 17.81, 17.81, 17.81, 17.81]
                    

Loading

Details

More

---
config:
    xyChart:
        titleFontSize: 12
        width: 900
        height: 600
    themeVariables:
        xyChart:
            titleColor: "#000000"
---
xychart-beta
    title "llama.cpp bench-server-baseline on Standard_NC4as_T4_v3
 duration=10m 198 iterations"
    y-axis "llamacpp:kv_cache_usage_ratio"
    x-axis "llamacpp:kv_cache_usage_ratio" 1714043552 --> 1714044188
    line [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.07, 0.07, 0.07, 0.07, 0.07, 0.18, 0.18, 0.18, 0.18, 0.18, 0.27, 0.27, 0.27, 0.27, 0.27, 0.31, 0.31, 0.31, 0.31, 0.31, 0.39, 0.39, 0.39, 0.39, 0.39, 0.44, 0.44, 0.44, 0.44, 0.44, 0.51, 0.51, 0.51, 0.51, 0.51, 0.53, 0.53, 0.53, 0.53, 0.53, 0.1, 0.1, 0.1, 0.1, 0.1, 0.23, 0.23, 0.23, 0.23, 0.23, 0.1, 0.1, 0.1, 0.1, 0.1, 0.17, 0.17, 0.17, 0.17, 0.17, 0.2, 0.2, 0.2, 0.2, 0.2, 0.27, 0.27, 0.27, 0.27, 0.27, 0.24, 0.24, 0.24, 0.24, 0.24, 0.23, 0.23, 0.23, 0.23, 0.23, 0.3, 0.3, 0.3, 0.3, 0.3, 0.14, 0.14, 0.14, 0.14, 0.14, 0.12, 0.12, 0.12, 0.12, 0.12, 0.18, 0.18, 0.18, 0.18, 0.18, 0.2, 0.2, 0.2, 0.2, 0.2, 0.22, 0.22, 0.22, 0.22, 0.22, 0.18, 0.18, 0.18, 0.18, 0.18, 0.19, 0.19, 0.19, 0.19, 0.19, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.14, 0.14, 0.14, 0.14, 0.14, 0.19, 0.19, 0.19, 0.19, 0.19, 0.19, 0.19, 0.19, 0.19, 0.19, 0.23, 0.23, 0.23, 0.23, 0.23, 0.29, 0.29, 0.29, 0.29, 0.29, 0.21, 0.21, 0.21, 0.21, 0.21, 0.22, 0.22, 0.22, 0.22, 0.22, 0.37, 0.37, 0.37, 0.37, 0.37, 0.3, 0.3, 0.3, 0.3, 0.3, 0.38, 0.38, 0.38, 0.38, 0.38, 0.29, 0.29, 0.29, 0.29, 0.29, 0.25, 0.25, 0.25, 0.25, 0.25, 0.31, 0.31, 0.31, 0.31, 0.31, 0.42, 0.42, 0.42, 0.42, 0.42, 0.48, 0.48, 0.48, 0.48, 0.48, 0.45, 0.45, 0.45, 0.45, 0.45, 0.43, 0.43, 0.43, 0.43, 0.43, 0.28, 0.28, 0.28, 0.28, 0.28, 0.21, 0.21, 0.21, 0.21, 0.21, 0.25, 0.25, 0.25, 0.25, 0.25, 0.21, 0.21, 0.21, 0.21, 0.21, 0.16, 0.16, 0.16, 0.16, 0.16, 0.23, 0.23, 0.23, 0.23, 0.23, 0.16, 0.16, 0.16, 0.16, 0.16, 0.14, 0.14, 0.14, 0.14, 0.14, 0.31, 0.31, 0.31, 0.31, 0.31, 0.37, 0.37, 0.37, 0.37, 0.37, 0.39, 0.39, 0.39, 0.39, 0.39, 0.27, 0.27, 0.27, 0.27, 0.27, 0.2, 0.2, 0.2, 0.2, 0.2, 0.16, 0.16, 0.16, 0.16, 0.16, 0.15, 0.15, 0.15, 0.15, 0.15, 0.19, 0.19, 0.19, 0.19, 0.19, 0.26, 0.26, 0.26, 0.26, 0.26, 0.23, 0.23, 0.23, 0.23, 0.23, 0.3, 0.3, 0.3, 0.3]
                    

Loading

More

---
config:
    xyChart:
        titleFontSize: 12
        width: 900
        height: 600
    themeVariables:
        xyChart:
            titleColor: "#000000"
---
xychart-beta
    title "llama.cpp bench-server-baseline on Standard_NC4as_T4_v3
 duration=10m 198 iterations"
    y-axis "llamacpp:requests_processing"
    x-axis "llamacpp:requests_processing" 1714043552 --> 1714044188
    line [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 6.0, 6.0, 6.0, 6.0, 6.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 7.0, 7.0, 7.0, 7.0, 7.0, 8.0, 8.0, 8.0, 8.0, 8.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 8.0, 8.0, 8.0, 8.0, 8.0, 7.0, 7.0, 7.0, 7.0, 7.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 6.0, 6.0, 6.0, 6.0, 6.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 6.0, 6.0, 6.0, 6.0, 6.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 7.0, 7.0, 7.0, 7.0, 7.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 8.0, 8.0, 8.0, 8.0, 8.0, 7.0, 7.0, 7.0, 7.0, 7.0, 3.0, 3.0, 3.0, 3.0]
                    

Loading