docs/release/results/euphrates_fraser_comparison.rst

   1 .. This work is licensed under a Creative Commons Attribution 4.0 International
   2 .. License.
   3 .. http://creativecommons.org/licenses/by/4.0
   4
   5 =======================================================
   6 Test results analysis for Euphrates and Fraser releases
   7 =======================================================
   8
   9 TC002
  10 -----
  11
  12 The round-trip-time (RTT) between 2 VMs on different blades is measured using
  13 ping.
  14
  15 Most test run measurements result on average between 0.39 and 4.00 ms.
  16 Compared with Euphrates release, the average RTT result of the same pod experiences
  17 a slight decline in Fraser release. For example, the average RTT of arm-pod5 is
  18 1.518 in Ehphrates and 1.714 in Fraser. The average RTT of intel-pod18 is 1.6575
  19 ms in Ehphrates and 1.856 ms in Fraser.
  20
  21 {
  22
  23     "huawei-pod2:stable/euphrates": [0.3925],
  24
  25     "lf-pod2:stable/euphrates": [0.5315],
  26
  27     "lf-pod1:stable/euphrates": [0.62],
  28
  29     "huawei-pod2:stable/fraser": [0.677],
  30
  31     "lf-pod1:stable/fraser": [0.725],
  32
  33     "flex-pod2:stable/euphrates": [0.795],
  34
  35     "huawei-pod12:stable/euphrates": [0.87],
  36
  37     "ericsson-pod1:stable/fraser": [0.9165],
  38
  39     "huawei-pod12:stable/fraser": [1.0465],
  40
  41     "lf-pod2:stable/fraser": [1.2325],
  42
  43     "intel-pod5:stable/euphrates": [1.25],
  44
  45     "ericsson-virtual3:stable/euphrates": [1.2655],
  46
  47     "ericsson-pod1:stable/euphrates": [1.372],
  48
  49     "zte-pod2:stable/fraser": [1.395],
  50
  51     "arm-pod5:stable/euphrates": [1.518],
  52
  53     "huawei-virtual4:stable/euphrates": [1.5355],
  54
  55     "ericsson-virtual4:stable/fraser": [1.582],
  56
  57     "huawei-virtual3:stable/euphrates": [1.606],
  58
  59     "intel-pod18:stable/euphrates": [1.6575],
  60
  61     "huawei-virtual4:stable/fraser": [1.697],
  62
  63     "huawei-virtual8:stable/euphrates": [1.709],
  64
  65     "arm-pod5:stable/fraser": [1.714],
  66
  67     "huawei-virtual3:stable/fraser": [1.716],
  68
  69     "intel-pod18:stable/fraser": [1.856],
  70
  71     "huawei-virtual2:stable/euphrates": [1.872],
  72
  73     "arm-pod6:stable/euphrates": [1.895],
  74
  75     "huawei-virtual2:stable/fraser": [1.964],
  76
  77     "huawei-virtual1:stable/fraser": [1.9765],
  78
  79     "huawei-virtual9:stable/euphrates": [2.0745],
  80
  81     "arm-pod6:stable/fraser": [2.209],
  82
  83     "huawei-virtual1:stable/euphrates": [2.495],
  84
  85     "ericsson-virtual2:stable/euphrates": [2.7895],
  86
  87     "ericsson-virtual4:stable/euphrates": [3.768],
  88
  89     "ericsson-virtual1:stable/euphrates": [3.8035],
  90
  91     "ericsson-virtual3:stable/fraser": [3.9175],
  92
  93     "ericsson-virtual2:stable/fraser": [4.004]
  94
  95 }
  96
  97 TC010
  98 -----
  99
 100 The tool we use to measure memory read latency is lmbench, which is a series of
 101 micro benchmarks intended to measure basic operating system and hardware system
 102 metrics. Compared with Euphrates release, the memory read latency of the same pod
 103 also experience a slight decline. Virtual pods seem to have a higher memory read
 104 latency than physical pods. Compared with X86 pods, the memory read latency of
 105 arm pods is significant higher.
 106
 107 {
 108
 109     "ericsson-pod1:stable/euphrates": [5.7785],
 110
 111     "flex-pod2:stable/euphrates": [5.908],
 112
 113     "ericsson-virtual1:stable/euphrates": [6.412],
 114
 115     "intel-pod18:stable/euphrates": [6.5905],
 116
 117     "intel-pod5:stable/euphrates": [6.6975],
 118
 119     "ericsson-pod1:stable/fraser": [7.0645],
 120
 121     "ericsson-virtual4:stable/euphrates": [7.183],
 122
 123     "intel-pod18:stable/fraser": [7.4465],
 124
 125     "zte-pod2:stable/fraser": [8.1865],
 126
 127     "ericsson-virtual2:stable/euphrates": [8.4985],
 128
 129     "huawei-pod2:stable/euphrates": [8.877],
 130
 131     "huawei-pod12:stable/euphrates": [9.091],
 132
 133     "huawei-pod2:stable/fraser": [9.236],
 134
 135     "huawei-pod12:stable/fraser": [9.615],
 136
 137     "ericsson-virtual3:stable/euphrates": [9.719],
 138
 139     "ericsson-virtual2:stable/fraser": [9.8925],
 140
 141     "huawei-virtual4:stable/euphrates": [10.1195],
 142
 143     "huawei-virtual3:stable/euphrates": [10.19],
 144
 145     "huawei-virtual2:stable/fraser": [10.22],
 146
 147     "huawei-virtual1:stable/euphrates": [10.3045],
 148
 149     "huawei-virtual9:stable/euphrates": [10.318],
 150
 151     "ericsson-virtual4:stable/fraser": [10.5465],
 152
 153     "ericsson-virtual3:stable/fraser": [10.9355],
 154
 155     "huawei-virtual3:stable/fraser": [10.95],
 156
 157     "huawei-virtual2:stable/euphrates": [11.274],
 158
 159     "huawei-virtual4:stable/fraser": [11.557],
 160
 161     "lf-pod1:stable/euphrates": [15.7025],
 162
 163     "lf-pod2:stable/euphrates": [15.8495],
 164
 165     "lf-pod2:stable/fraser": [16.5595],
 166
 167     "lf-pod1:stable/fraser": [16.8395],
 168
 169     "arm-pod5:stable/euphrates": [18.092],
 170
 171     "arm-pod5:stable/fraser": [18.744],
 172
 173     "huawei-virtual1:stable/fraser": [19.8235],
 174
 175     "huawei-virtual8:stable/euphrates": [33.999],
 176
 177     "arm-pod6:stable/euphrates": [41.5605],
 178
 179     "arm-pod6:stable/fraser": [55.804]
 180
 181 }
 182
 183 TC011
 184 -----
 185
 186 Iperf3 is a tool for evaluating the packet delay variation between 2 VMs on
 187 different blades. In general, the packet delay variation of the two releases
 188 look similar.
 189
 190 {
 191
 192     "arm-pod6:stable/fraser": [1],
 193
 194     "ericsson-pod1:stable/fraser": [1],
 195
 196     "ericsson-virtual2:stable/fraser": [1],
 197
 198     "ericsson-virtual3:stable/fraser": [1],
 199
 200     "lf-pod2:stable/fraser": [1],
 201
 202     "huawei-virtual1:stable/fraser": [2997],
 203
 204     "huawei-virtual2:stable/euphrates": [2997],
 205
 206     "flex-pod2:stable/euphrates": [2997.5],
 207
 208     "huawei-virtual3:stable/euphrates": [2998],
 209
 210     "huawei-virtual3:stable/fraser": [2999],
 211
 212     "huawei-virtual9:stable/euphrates": [3000],
 213
 214     "huawei-virtual8:stable/euphrates": [3001],
 215
 216     "huawei-virtual4:stable/euphrates": [3002],
 217
 218     "huawei-virtual4:stable/fraser": [3002],
 219
 220     "ericsson-virtual3:stable/euphrates": [3006],
 221
 222     "huawei-virtual1:stable/euphrates": [3007],
 223
 224     "ericsson-virtual2:stable/euphrates": [3009],
 225
 226     "intel-pod18:stable/euphrates": [3010],
 227
 228     "ericsson-virtual4:stable/euphrates": [3017],
 229
 230     "lf-pod2:stable/euphrates": [3021],
 231
 232     "arm-pod5:stable/euphrates": [3022],
 233
 234     "arm-pod6:stable/euphrates": [3022],
 235
 236     "ericsson-pod1:stable/euphrates": [3022],
 237
 238     "huawei-pod12:stable/euphrates": [3022],
 239
 240     "huawei-pod12:stable/fraser": [3022],
 241
 242     "huawei-pod2:stable/euphrates": [3022],
 243
 244     "huawei-pod2:stable/fraser": [3022],
 245
 246     "intel-pod18:stable/fraser": [3022],
 247
 248     "intel-pod5:stable/euphrates": [3022],
 249
 250     "lf-pod1:stable/euphrates": [3022],
 251
 252     "lf-pod1:stable/fraser": [3022],
 253
 254     "zte-pod2:stable/fraser": [3022],
 255
 256     "huawei-virtual2:stable/fraser": [3025]
 257
 258 }
 259
 260 TC012
 261 -----
 262
 263 Lmbench is also used to measure the memory read and write bandwidth.
 264 Like TC010, compared with Euphrates release, the memory read and write bandwidth
 265 of the same pod also experience a slight decline. And compared with X86 pods, the memory
 266 read and write bandwidth of arm pods is significant lower.
 267
 268 {
 269
 270     "lf-pod1:stable/euphrates": [22912.39],
 271
 272     "lf-pod2:stable/euphrates": [22637.67],
 273
 274     "lf-pod1:stable/fraser": [20552.9],
 275
 276     "flex-pod2:stable/euphrates": [20229.99],
 277
 278     "lf-pod2:stable/fraser": [20058.925],
 279
 280     "ericsson-pod1:stable/fraser": [18930.78],
 281
 282     "intel-pod18:stable/fraser": [18757.545],
 283
 284     "ericsson-virtual1:stable/euphrates": [17474.965],
 285
 286     "ericsson-pod1:stable/euphrates": [17127.38],
 287
 288     "ericsson-virtual4:stable/euphrates": [16219.97],
 289
 290     "ericsson-virtual2:stable/euphrates": [15652.28],
 291
 292     "ericsson-virtual3:stable/euphrates": [15551.26],
 293
 294     "ericsson-virtual4:stable/fraser": [15389.465],
 295
 296     "ericsson-virtual2:stable/fraser": [15343.79],
 297
 298     "huawei-pod2:stable/euphrates": [15017.2],
 299
 300     "huawei-pod2:stable/fraser": [14870.78],
 301
 302     "huawei-virtual4:stable/euphrates": [14266.34],
 303
 304     "huawei-virtual1:stable/euphrates": [14233.035],
 305
 306     "huawei-virtual3:stable/euphrates": [14227.63],
 307
 308     "zte-pod2:stable/fraser": [14157.99],
 309
 310     "huawei-pod12:stable/euphrates": [14147.245],
 311
 312     "huawei-pod12:stable/fraser": [14126.99],
 313
 314     "intel-pod18:stable/euphrates": [14058.33],
 315
 316     "huawei-virtual3:stable/fraser": [13929.67],
 317
 318     "huawei-virtual2:stable/euphrates": [13862.85],
 319
 320     "huawei-virtual4:stable/fraser": [13847.155],
 321
 322     "huawei-virtual2:stable/fraser": [13702.92],
 323
 324     "huawei-virtual1:stable/fraser": [13496.45],
 325
 326     "intel-pod5:stable/euphrates": [13280.32],
 327
 328     "ericsson-virtual3:stable/fraser": [12733.19],
 329
 330     "huawei-virtual9:stable/euphrates": [12559.445],
 331
 332     "huawei-virtual8:stable/euphrates": [8998.02],
 333
 334     "arm-pod5:stable/euphrates": [4388.875],
 335
 336     "arm-pod5:stable/fraser": [4326.11],
 337
 338     "arm-pod6:stable/euphrates": [4260.2],
 339
 340     "arm-pod6:stable/fraser": [3809.885]
 341
 342 }
 343
 344 TC014
 345 -----
 346
 347 The Unixbench is used to evaluate the IaaS processing speed with regards to
 348 score of single CPU running and parallel running. Below are the single CPU running
 349 scores. It can be seen that the processing test results vary from scores 715 to 3737.
 350 In general, the single CPU score of the two releases look similar.
 351
 352 {
 353
 354     "lf-pod2:stable/fraser": [3737.6],
 355
 356     "lf-pod2:stable/euphrates": [3723.95],
 357
 358     "lf-pod1:stable/fraser": [3702.7],
 359
 360     "lf-pod1:stable/euphrates": [3669],
 361
 362     "intel-pod5:stable/euphrates": [3388.6],
 363
 364     "intel-pod18:stable/euphrates": [3298.4],
 365
 366     "flex-pod2:stable/euphrates": [3208.6],
 367
 368     "ericsson-pod1:stable/fraser": [3131.6],
 369
 370     "intel-pod18:stable/fraser": [3098.1],
 371
 372     "ericsson-virtual1:stable/euphrates": [2988.9],
 373
 374     "zte-pod2:stable/fraser": [2831.4],
 375
 376     "ericsson-pod1:stable/euphrates": [2669.1],
 377
 378     "ericsson-virtual4:stable/euphrates": [2598.5],
 379
 380     "ericsson-virtual2:stable/fraser": [2559.7],
 381
 382     "ericsson-virtual3:stable/euphrates": [2553.15],
 383
 384     "huawei-pod2:stable/euphrates": [2531.2],
 385
 386     "huawei-pod2:stable/fraser": [2528.9],
 387
 388     "ericsson-virtual4:stable/fraser": [2527.8],
 389
 390     "ericsson-virtual2:stable/euphrates": [2526.9],
 391
 392     "huawei-virtual4:stable/euphrates": [2407.4],
 393
 394     "huawei-virtual3:stable/fraser": [2379.1],
 395
 396     "huawei-virtual3:stable/euphrates": [2374.6],
 397
 398     "huawei-virtual4:stable/fraser": [2362.1],
 399
 400     "huawei-virtual2:stable/euphrates": [2326.4],
 401
 402     "huawei-virtual9:stable/euphrates": [2324.95],
 403
 404     "huawei-virtual1:stable/euphrates": [2302.6],
 405
 406     "huawei-virtual2:stable/fraser": [2299.3],
 407
 408     "huawei-pod12:stable/euphrates": [2232.2],
 409
 410     "huawei-pod12:stable/fraser": [2229],
 411
 412     "huawei-virtual1:stable/fraser": [2171.3],
 413
 414     "ericsson-virtual3:stable/fraser": [2104.8],
 415
 416     "huawei-virtual8:stable/euphrates": [2085.3],
 417
 418     "arm-pod5:stable/fraser": [1764.2],
 419
 420     "arm-pod5:stable/euphrates": [1754.4],
 421
 422     "arm-pod6:stable/euphrates": [716.15],
 423
 424     "arm-pod6:stable/fraser": [715.4]
 425
 426 }
 427
 428 TC069
 429 -----
 430
 431 With the block size changing from 1 kb to 512 kb, the memory write bandwidth
 432 tends to become larger first and then smaller within every run test. Below are
 433 the scores for 32mb block array.
 434
 435 {
 436
 437     "intel-pod18:stable/euphrates": [18871.79],
 438
 439     "intel-pod18:stable/fraser": [16939.24],
 440
 441     "intel-pod5:stable/euphrates": [16055.79],
 442
 443     "arm-pod6:stable/euphrates": [13327.02],
 444
 445     "arm-pod6:stable/fraser": [11895.71],
 446
 447     "flex-pod2:stable/euphrates": [9384.585],
 448
 449     "zte-pod2:stable/fraser": [9375.33],
 450
 451     "ericsson-pod1:stable/euphrates": [9331.535],
 452
 453     "huawei-pod12:stable/euphrates": [9164.88],
 454
 455     "ericsson-pod1:stable/fraser": [9140.42],
 456
 457     "huawei-pod2:stable/euphrates": [9026.52],
 458
 459     "huawei-pod12:stable/fraser": [8993.37],
 460
 461     "huawei-virtual9:stable/euphrates": [8825.805],
 462
 463     "huawei-pod2:stable/fraser": [8794.01],
 464
 465     "huawei-virtual2:stable/fraser": [7670.21],
 466
 467     "ericsson-virtual1:stable/euphrates": [7615.97],
 468
 469     "ericsson-virtual4:stable/euphrates": [7539.23],
 470
 471     "arm-pod5:stable/fraser": [7479.32],
 472
 473     "arm-pod5:stable/euphrates": [7403.38],
 474
 475     "huawei-virtual3:stable/euphrates": [7247.89],
 476
 477     "ericsson-virtual2:stable/fraser": [7219.21],
 478
 479     "huawei-virtual2:stable/euphrates": [7205.35],
 480
 481     "huawei-virtual1:stable/euphrates": [7196.405],
 482
 483     "ericsson-virtual3:stable/euphrates": [7173.72],
 484
 485     "huawei-virtual4:stable/euphrates": [7131.47],
 486
 487     "ericsson-virtual2:stable/euphrates": [7129.08],
 488
 489     "huawei-virtual4:stable/fraser": [7059.045],
 490
 491     "huawei-virtual3:stable/fraser": [7023.57],
 492
 493     "lf-pod1:stable/euphrates": [6928.18],
 494
 495     "lf-pod2:stable/euphrates": [6875.88],
 496
 497     "lf-pod2:stable/fraser": [6834.7],
 498
 499     "lf-pod1:stable/fraser": [6775.27],
 500
 501     "ericsson-virtual4:stable/fraser": [6522.86],
 502
 503     "ericsson-virtual3:stable/fraser": [5835.59],
 504
 505     "huawei-virtual8:stable/euphrates": [5729.705],
 506
 507     "huawei-virtual1:stable/fraser": [5617.12]
 508
 509 }
 510
 511 TC082
 512 -----
 513
 514 For this test case, we use perf to measure context-switches under load.
 515 High context switch rates are not themselves an issue, but they may point the
 516 way to a more significant problem.
 517
 518 {
 519
 520     "zte-pod2:stable/fraser": [306.5],
 521
 522     "huawei-pod12:stable/euphrates": [316],
 523
 524     "lf-pod2:stable/fraser": [337.5],
 525
 526     "intel-pod18:stable/euphrates": [340],
 527
 528     "intel-pod18:stable/fraser": [343.5],
 529
 530     "intel-pod5:stable/euphrates": [357.5],
 531
 532     "ericsson-pod1:stable/euphrates": [384],
 533
 534     "lf-pod2:stable/euphrates": [394.5],
 535
 536     "huawei-pod12:stable/fraser": [399],
 537
 538     "lf-pod1:stable/euphrates": [435],
 539
 540     "lf-pod1:stable/fraser": [454],
 541
 542     "flex-pod2:stable/euphrates": [476],
 543
 544     "huawei-pod2:stable/euphrates": [518],
 545
 546     "huawei-pod2:stable/fraser": [544.5],
 547
 548     "arm-pod5:stable/euphrates": [869.5],
 549
 550     "huawei-virtual9:stable/euphrates": [1002],
 551
 552     "huawei-virtual4:stable/fraser": [1138],
 553
 554     "huawei-virtual4:stable/euphrates": [1174],
 555
 556     "huawei-virtual3:stable/euphrates": [1239],
 557
 558     "ericsson-pod1:stable/fraser": [1305],
 559
 560     "huawei-virtual2:stable/euphrates": [1430],
 561
 562     "huawei-virtual3:stable/fraser": [1433],
 563
 564     "huawei-virtual1:stable/fraser": [1470],
 565
 566     "huawei-virtual1:stable/euphrates": [1489],
 567
 568     "arm-pod6:stable/fraser": [1738.5],
 569
 570     "arm-pod6:stable/euphrates": [1883.5]
 571
 572 }
 573
 574 TC083
 575 -----
 576
 577 TC083 measures network latency and throughput between VMs using netperf.
 578 The test results shown below are for UDP throughout.
 579
 580 {
 581
 582     "lf-pod1:stable/euphrates": [2204.42],
 583
 584     "lf-pod2:stable/fraser": [1893.39],
 585
 586     "intel-pod18:stable/euphrates": [1835.55],
 587
 588     "lf-pod2:stable/euphrates": [1676.705],
 589
 590     "intel-pod5:stable/euphrates": [1612.555],
 591
 592     "zte-pod2:stable/fraser": [1543.995],
 593
 594     "lf-pod1:stable/fraser": [1480.86],
 595
 596     "intel-pod18:stable/fraser": [1417.015],
 597
 598     "flex-pod2:stable/euphrates": [1370.23],
 599
 600     "huawei-pod12:stable/euphrates": [1300.12]
 601
 602 }