docs/release/results/euphrates_fraser_comparison.rst

   1 .. This work is licensed under a Creative Commons Attribution 4.0 International
   2 .. License.
   3 .. http://creativecommons.org/licenses/by/4.0
   4
   5 ..
   6       Convention for heading levels in Yardstick:
   7       =======  Heading 0 (reserved for the title in a document)
   8       -------  Heading 1
   9       ^^^^^^^  Heading 2
  10       +++++++  Heading 3
  11       '''''''  Heading 4
  12       Avoid deeper levels because they do not render well.
  13
  14 Test results analysis for Euphrates and Fraser releases
  15 =======================================================
  16
  17 TC002
  18 -----
  19
  20 The round-trip-time (RTT) between 2 VMs on different blades is measured using
  21 ping.
  22
  23 Most test run measurements result on average between 0.39 and 4.00 ms.
  24 Compared with Euphrates release, the average RTT result of the same pod experiences
  25 a slight decline in Fraser release. For example, the average RTT of arm-pod5 is
  26 1.518 in Ehphrates and 1.714 in Fraser. The average RTT of intel-pod18 is 1.6575
  27 ms in Ehphrates and 1.856 ms in Fraser.
  28
  29 {
  30
  31     "huawei-pod2:stable/euphrates": [0.3925],
  32
  33     "lf-pod2:stable/euphrates": [0.5315],
  34
  35     "lf-pod1:stable/euphrates": [0.62],
  36
  37     "huawei-pod2:stable/fraser": [0.677],
  38
  39     "lf-pod1:stable/fraser": [0.725],
  40
  41     "flex-pod2:stable/euphrates": [0.795],
  42
  43     "huawei-pod12:stable/euphrates": [0.87],
  44
  45     "ericsson-pod1:stable/fraser": [0.9165],
  46
  47     "huawei-pod12:stable/fraser": [1.0465],
  48
  49     "lf-pod2:stable/fraser": [1.2325],
  50
  51     "intel-pod5:stable/euphrates": [1.25],
  52
  53     "ericsson-virtual3:stable/euphrates": [1.2655],
  54
  55     "ericsson-pod1:stable/euphrates": [1.372],
  56
  57     "zte-pod2:stable/fraser": [1.395],
  58
  59     "arm-pod5:stable/euphrates": [1.518],
  60
  61     "huawei-virtual4:stable/euphrates": [1.5355],
  62
  63     "ericsson-virtual4:stable/fraser": [1.582],
  64
  65     "huawei-virtual3:stable/euphrates": [1.606],
  66
  67     "intel-pod18:stable/euphrates": [1.6575],
  68
  69     "huawei-virtual4:stable/fraser": [1.697],
  70
  71     "huawei-virtual8:stable/euphrates": [1.709],
  72
  73     "arm-pod5:stable/fraser": [1.714],
  74
  75     "huawei-virtual3:stable/fraser": [1.716],
  76
  77     "intel-pod18:stable/fraser": [1.856],
  78
  79     "huawei-virtual2:stable/euphrates": [1.872],
  80
  81     "arm-pod6:stable/euphrates": [1.895],
  82
  83     "huawei-virtual2:stable/fraser": [1.964],
  84
  85     "huawei-virtual1:stable/fraser": [1.9765],
  86
  87     "huawei-virtual9:stable/euphrates": [2.0745],
  88
  89     "arm-pod6:stable/fraser": [2.209],
  90
  91     "huawei-virtual1:stable/euphrates": [2.495],
  92
  93     "ericsson-virtual2:stable/euphrates": [2.7895],
  94
  95     "ericsson-virtual4:stable/euphrates": [3.768],
  96
  97     "ericsson-virtual1:stable/euphrates": [3.8035],
  98
  99     "ericsson-virtual3:stable/fraser": [3.9175],
 100
 101     "ericsson-virtual2:stable/fraser": [4.004]
 102
 103 }
 104
 105 TC010
 106 -----
 107
 108 The tool we use to measure memory read latency is lmbench, which is a series of
 109 micro benchmarks intended to measure basic operating system and hardware system
 110 metrics. Compared with Euphrates release, the memory read latency of the same pod
 111 also experience a slight decline. Virtual pods seem to have a higher memory read
 112 latency than physical pods. Compared with X86 pods, the memory read latency of
 113 arm pods is significant higher.
 114
 115 {
 116
 117     "ericsson-pod1:stable/euphrates": [5.7785],
 118
 119     "flex-pod2:stable/euphrates": [5.908],
 120
 121     "ericsson-virtual1:stable/euphrates": [6.412],
 122
 123     "intel-pod18:stable/euphrates": [6.5905],
 124
 125     "intel-pod5:stable/euphrates": [6.6975],
 126
 127     "ericsson-pod1:stable/fraser": [7.0645],
 128
 129     "ericsson-virtual4:stable/euphrates": [7.183],
 130
 131     "intel-pod18:stable/fraser": [7.4465],
 132
 133     "zte-pod2:stable/fraser": [8.1865],
 134
 135     "ericsson-virtual2:stable/euphrates": [8.4985],
 136
 137     "huawei-pod2:stable/euphrates": [8.877],
 138
 139     "huawei-pod12:stable/euphrates": [9.091],
 140
 141     "huawei-pod2:stable/fraser": [9.236],
 142
 143     "huawei-pod12:stable/fraser": [9.615],
 144
 145     "ericsson-virtual3:stable/euphrates": [9.719],
 146
 147     "ericsson-virtual2:stable/fraser": [9.8925],
 148
 149     "huawei-virtual4:stable/euphrates": [10.1195],
 150
 151     "huawei-virtual3:stable/euphrates": [10.19],
 152
 153     "huawei-virtual2:stable/fraser": [10.22],
 154
 155     "huawei-virtual1:stable/euphrates": [10.3045],
 156
 157     "huawei-virtual9:stable/euphrates": [10.318],
 158
 159     "ericsson-virtual4:stable/fraser": [10.5465],
 160
 161     "ericsson-virtual3:stable/fraser": [10.9355],
 162
 163     "huawei-virtual3:stable/fraser": [10.95],
 164
 165     "huawei-virtual2:stable/euphrates": [11.274],
 166
 167     "huawei-virtual4:stable/fraser": [11.557],
 168
 169     "lf-pod1:stable/euphrates": [15.7025],
 170
 171     "lf-pod2:stable/euphrates": [15.8495],
 172
 173     "lf-pod2:stable/fraser": [16.5595],
 174
 175     "lf-pod1:stable/fraser": [16.8395],
 176
 177     "arm-pod5:stable/euphrates": [18.092],
 178
 179     "arm-pod5:stable/fraser": [18.744],
 180
 181     "huawei-virtual1:stable/fraser": [19.8235],
 182
 183     "huawei-virtual8:stable/euphrates": [33.999],
 184
 185     "arm-pod6:stable/euphrates": [41.5605],
 186
 187     "arm-pod6:stable/fraser": [55.804]
 188
 189 }
 190
 191 TC011
 192 -----
 193
 194 Iperf3 is a tool for evaluating the packet delay variation between 2 VMs on
 195 different blades. In general, the packet delay variation of the two releases
 196 look similar.
 197
 198 {
 199
 200     "arm-pod6:stable/fraser": [1],
 201
 202     "ericsson-pod1:stable/fraser": [1],
 203
 204     "ericsson-virtual2:stable/fraser": [1],
 205
 206     "ericsson-virtual3:stable/fraser": [1],
 207
 208     "lf-pod2:stable/fraser": [1],
 209
 210     "huawei-virtual1:stable/fraser": [2997],
 211
 212     "huawei-virtual2:stable/euphrates": [2997],
 213
 214     "flex-pod2:stable/euphrates": [2997.5],
 215
 216     "huawei-virtual3:stable/euphrates": [2998],
 217
 218     "huawei-virtual3:stable/fraser": [2999],
 219
 220     "huawei-virtual9:stable/euphrates": [3000],
 221
 222     "huawei-virtual8:stable/euphrates": [3001],
 223
 224     "huawei-virtual4:stable/euphrates": [3002],
 225
 226     "huawei-virtual4:stable/fraser": [3002],
 227
 228     "ericsson-virtual3:stable/euphrates": [3006],
 229
 230     "huawei-virtual1:stable/euphrates": [3007],
 231
 232     "ericsson-virtual2:stable/euphrates": [3009],
 233
 234     "intel-pod18:stable/euphrates": [3010],
 235
 236     "ericsson-virtual4:stable/euphrates": [3017],
 237
 238     "lf-pod2:stable/euphrates": [3021],
 239
 240     "arm-pod5:stable/euphrates": [3022],
 241
 242     "arm-pod6:stable/euphrates": [3022],
 243
 244     "ericsson-pod1:stable/euphrates": [3022],
 245
 246     "huawei-pod12:stable/euphrates": [3022],
 247
 248     "huawei-pod12:stable/fraser": [3022],
 249
 250     "huawei-pod2:stable/euphrates": [3022],
 251
 252     "huawei-pod2:stable/fraser": [3022],
 253
 254     "intel-pod18:stable/fraser": [3022],
 255
 256     "intel-pod5:stable/euphrates": [3022],
 257
 258     "lf-pod1:stable/euphrates": [3022],
 259
 260     "lf-pod1:stable/fraser": [3022],
 261
 262     "zte-pod2:stable/fraser": [3022],
 263
 264     "huawei-virtual2:stable/fraser": [3025]
 265
 266 }
 267
 268 TC012
 269 -----
 270
 271 Lmbench is also used to measure the memory read and write bandwidth.
 272 Like TC010, compared with Euphrates release, the memory read and write bandwidth
 273 of the same pod also experience a slight decline. And compared with X86 pods, the memory
 274 read and write bandwidth of arm pods is significant lower.
 275
 276 {
 277
 278     "lf-pod1:stable/euphrates": [22912.39],
 279
 280     "lf-pod2:stable/euphrates": [22637.67],
 281
 282     "lf-pod1:stable/fraser": [20552.9],
 283
 284     "flex-pod2:stable/euphrates": [20229.99],
 285
 286     "lf-pod2:stable/fraser": [20058.925],
 287
 288     "ericsson-pod1:stable/fraser": [18930.78],
 289
 290     "intel-pod18:stable/fraser": [18757.545],
 291
 292     "ericsson-virtual1:stable/euphrates": [17474.965],
 293
 294     "ericsson-pod1:stable/euphrates": [17127.38],
 295
 296     "ericsson-virtual4:stable/euphrates": [16219.97],
 297
 298     "ericsson-virtual2:stable/euphrates": [15652.28],
 299
 300     "ericsson-virtual3:stable/euphrates": [15551.26],
 301
 302     "ericsson-virtual4:stable/fraser": [15389.465],
 303
 304     "ericsson-virtual2:stable/fraser": [15343.79],
 305
 306     "huawei-pod2:stable/euphrates": [15017.2],
 307
 308     "huawei-pod2:stable/fraser": [14870.78],
 309
 310     "huawei-virtual4:stable/euphrates": [14266.34],
 311
 312     "huawei-virtual1:stable/euphrates": [14233.035],
 313
 314     "huawei-virtual3:stable/euphrates": [14227.63],
 315
 316     "zte-pod2:stable/fraser": [14157.99],
 317
 318     "huawei-pod12:stable/euphrates": [14147.245],
 319
 320     "huawei-pod12:stable/fraser": [14126.99],
 321
 322     "intel-pod18:stable/euphrates": [14058.33],
 323
 324     "huawei-virtual3:stable/fraser": [13929.67],
 325
 326     "huawei-virtual2:stable/euphrates": [13862.85],
 327
 328     "huawei-virtual4:stable/fraser": [13847.155],
 329
 330     "huawei-virtual2:stable/fraser": [13702.92],
 331
 332     "huawei-virtual1:stable/fraser": [13496.45],
 333
 334     "intel-pod5:stable/euphrates": [13280.32],
 335
 336     "ericsson-virtual3:stable/fraser": [12733.19],
 337
 338     "huawei-virtual9:stable/euphrates": [12559.445],
 339
 340     "huawei-virtual8:stable/euphrates": [8998.02],
 341
 342     "arm-pod5:stable/euphrates": [4388.875],
 343
 344     "arm-pod5:stable/fraser": [4326.11],
 345
 346     "arm-pod6:stable/euphrates": [4260.2],
 347
 348     "arm-pod6:stable/fraser": [3809.885]
 349
 350 }
 351
 352 TC014
 353 -----
 354
 355 The Unixbench is used to evaluate the IaaS processing speed with regards to
 356 score of single CPU running and parallel running. Below are the single CPU running
 357 scores. It can be seen that the processing test results vary from scores 715 to 3737.
 358 In general, the single CPU score of the two releases look similar.
 359
 360 {
 361
 362     "lf-pod2:stable/fraser": [3737.6],
 363
 364     "lf-pod2:stable/euphrates": [3723.95],
 365
 366     "lf-pod1:stable/fraser": [3702.7],
 367
 368     "lf-pod1:stable/euphrates": [3669],
 369
 370     "intel-pod5:stable/euphrates": [3388.6],
 371
 372     "intel-pod18:stable/euphrates": [3298.4],
 373
 374     "flex-pod2:stable/euphrates": [3208.6],
 375
 376     "ericsson-pod1:stable/fraser": [3131.6],
 377
 378     "intel-pod18:stable/fraser": [3098.1],
 379
 380     "ericsson-virtual1:stable/euphrates": [2988.9],
 381
 382     "zte-pod2:stable/fraser": [2831.4],
 383
 384     "ericsson-pod1:stable/euphrates": [2669.1],
 385
 386     "ericsson-virtual4:stable/euphrates": [2598.5],
 387
 388     "ericsson-virtual2:stable/fraser": [2559.7],
 389
 390     "ericsson-virtual3:stable/euphrates": [2553.15],
 391
 392     "huawei-pod2:stable/euphrates": [2531.2],
 393
 394     "huawei-pod2:stable/fraser": [2528.9],
 395
 396     "ericsson-virtual4:stable/fraser": [2527.8],
 397
 398     "ericsson-virtual2:stable/euphrates": [2526.9],
 399
 400     "huawei-virtual4:stable/euphrates": [2407.4],
 401
 402     "huawei-virtual3:stable/fraser": [2379.1],
 403
 404     "huawei-virtual3:stable/euphrates": [2374.6],
 405
 406     "huawei-virtual4:stable/fraser": [2362.1],
 407
 408     "huawei-virtual2:stable/euphrates": [2326.4],
 409
 410     "huawei-virtual9:stable/euphrates": [2324.95],
 411
 412     "huawei-virtual1:stable/euphrates": [2302.6],
 413
 414     "huawei-virtual2:stable/fraser": [2299.3],
 415
 416     "huawei-pod12:stable/euphrates": [2232.2],
 417
 418     "huawei-pod12:stable/fraser": [2229],
 419
 420     "huawei-virtual1:stable/fraser": [2171.3],
 421
 422     "ericsson-virtual3:stable/fraser": [2104.8],
 423
 424     "huawei-virtual8:stable/euphrates": [2085.3],
 425
 426     "arm-pod5:stable/fraser": [1764.2],
 427
 428     "arm-pod5:stable/euphrates": [1754.4],
 429
 430     "arm-pod6:stable/euphrates": [716.15],
 431
 432     "arm-pod6:stable/fraser": [715.4]
 433
 434 }
 435
 436 TC069
 437 -----
 438
 439 With the block size changing from 1 kb to 512 kb, the memory write bandwidth
 440 tends to become larger first and then smaller within every run test. Below are
 441 the scores for 32mb block array.
 442
 443 {
 444
 445     "intel-pod18:stable/euphrates": [18871.79],
 446
 447     "intel-pod18:stable/fraser": [16939.24],
 448
 449     "intel-pod5:stable/euphrates": [16055.79],
 450
 451     "arm-pod6:stable/euphrates": [13327.02],
 452
 453     "arm-pod6:stable/fraser": [11895.71],
 454
 455     "flex-pod2:stable/euphrates": [9384.585],
 456
 457     "zte-pod2:stable/fraser": [9375.33],
 458
 459     "ericsson-pod1:stable/euphrates": [9331.535],
 460
 461     "huawei-pod12:stable/euphrates": [9164.88],
 462
 463     "ericsson-pod1:stable/fraser": [9140.42],
 464
 465     "huawei-pod2:stable/euphrates": [9026.52],
 466
 467     "huawei-pod12:stable/fraser": [8993.37],
 468
 469     "huawei-virtual9:stable/euphrates": [8825.805],
 470
 471     "huawei-pod2:stable/fraser": [8794.01],
 472
 473     "huawei-virtual2:stable/fraser": [7670.21],
 474
 475     "ericsson-virtual1:stable/euphrates": [7615.97],
 476
 477     "ericsson-virtual4:stable/euphrates": [7539.23],
 478
 479     "arm-pod5:stable/fraser": [7479.32],
 480
 481     "arm-pod5:stable/euphrates": [7403.38],
 482
 483     "huawei-virtual3:stable/euphrates": [7247.89],
 484
 485     "ericsson-virtual2:stable/fraser": [7219.21],
 486
 487     "huawei-virtual2:stable/euphrates": [7205.35],
 488
 489     "huawei-virtual1:stable/euphrates": [7196.405],
 490
 491     "ericsson-virtual3:stable/euphrates": [7173.72],
 492
 493     "huawei-virtual4:stable/euphrates": [7131.47],
 494
 495     "ericsson-virtual2:stable/euphrates": [7129.08],
 496
 497     "huawei-virtual4:stable/fraser": [7059.045],
 498
 499     "huawei-virtual3:stable/fraser": [7023.57],
 500
 501     "lf-pod1:stable/euphrates": [6928.18],
 502
 503     "lf-pod2:stable/euphrates": [6875.88],
 504
 505     "lf-pod2:stable/fraser": [6834.7],
 506
 507     "lf-pod1:stable/fraser": [6775.27],
 508
 509     "ericsson-virtual4:stable/fraser": [6522.86],
 510
 511     "ericsson-virtual3:stable/fraser": [5835.59],
 512
 513     "huawei-virtual8:stable/euphrates": [5729.705],
 514
 515     "huawei-virtual1:stable/fraser": [5617.12]
 516
 517 }
 518
 519 TC082
 520 -----
 521
 522 For this test case, we use perf to measure context-switches under load.
 523 High context switch rates are not themselves an issue, but they may point the
 524 way to a more significant problem.
 525
 526 {
 527
 528     "zte-pod2:stable/fraser": [306.5],
 529
 530     "huawei-pod12:stable/euphrates": [316],
 531
 532     "lf-pod2:stable/fraser": [337.5],
 533
 534     "intel-pod18:stable/euphrates": [340],
 535
 536     "intel-pod18:stable/fraser": [343.5],
 537
 538     "intel-pod5:stable/euphrates": [357.5],
 539
 540     "ericsson-pod1:stable/euphrates": [384],
 541
 542     "lf-pod2:stable/euphrates": [394.5],
 543
 544     "huawei-pod12:stable/fraser": [399],
 545
 546     "lf-pod1:stable/euphrates": [435],
 547
 548     "lf-pod1:stable/fraser": [454],
 549
 550     "flex-pod2:stable/euphrates": [476],
 551
 552     "huawei-pod2:stable/euphrates": [518],
 553
 554     "huawei-pod2:stable/fraser": [544.5],
 555
 556     "arm-pod5:stable/euphrates": [869.5],
 557
 558     "huawei-virtual9:stable/euphrates": [1002],
 559
 560     "huawei-virtual4:stable/fraser": [1138],
 561
 562     "huawei-virtual4:stable/euphrates": [1174],
 563
 564     "huawei-virtual3:stable/euphrates": [1239],
 565
 566     "ericsson-pod1:stable/fraser": [1305],
 567
 568     "huawei-virtual2:stable/euphrates": [1430],
 569
 570     "huawei-virtual3:stable/fraser": [1433],
 571
 572     "huawei-virtual1:stable/fraser": [1470],
 573
 574     "huawei-virtual1:stable/euphrates": [1489],
 575
 576     "arm-pod6:stable/fraser": [1738.5],
 577
 578     "arm-pod6:stable/euphrates": [1883.5]
 579
 580 }
 581
 582 TC083
 583 -----
 584
 585 TC083 measures network latency and throughput between VMs using netperf.
 586 The test results shown below are for UDP throughout.
 587
 588 {
 589
 590     "lf-pod1:stable/euphrates": [2204.42],
 591
 592     "lf-pod2:stable/fraser": [1893.39],
 593
 594     "intel-pod18:stable/euphrates": [1835.55],
 595
 596     "lf-pod2:stable/euphrates": [1676.705],
 597
 598     "intel-pod5:stable/euphrates": [1612.555],
 599
 600     "zte-pod2:stable/fraser": [1543.995],
 601
 602     "lf-pod1:stable/fraser": [1480.86],
 603
 604     "intel-pod18:stable/fraser": [1417.015],
 605
 606     "flex-pod2:stable/euphrates": [1370.23],
 607
 608     "huawei-pod12:stable/euphrates": [1300.12]
 609
 610 }