functest/ci/testcases.yaml

   1 ---
   2 tiers:
   3     -
   4         name: healthcheck
   5         order: 0
   6         description: >-
   7             First tier to be executed to verify the basic
   8             operations in the VIM.
   9         testcases:
  10             -
  11                 case_name: connection_check
  12                 project_name: functest
  13                 criteria: 100
  14                 blocking: true
  15                 description: >-
  16                     This test case verifies the retrieval of OpenStack clients:
  17                     Keystone, Glance, Neutron and Nova and may perform some
  18                     simple queries.
  19                 run:
  20                     name: connection_check
  21
  22             -
  23                 case_name: tenantnetwork1
  24                 project_name: functest
  25                 criteria: 100
  26                 blocking: true
  27                 description: >-
  28                     It creates and configures all tenant network ressources
  29                     required by advanced testcases (subnet, network and
  30                     router).
  31                 run:
  32                     name: tenantnetwork1
  33
  34             -
  35                 case_name: tenantnetwork2
  36                 project_name: functest
  37                 criteria: 100
  38                 blocking: true
  39                 description: >-
  40                     It creates new user/project before creating and configuring
  41                     all tenant network ressources required by a testcase
  42                     (subnet, network and router).
  43                 run:
  44                     name: tenantnetwork2
  45
  46             -
  47                 case_name: vmready1
  48                 project_name: functest
  49                 criteria: 100
  50                 blocking: true
  51                 description: >-
  52                     It inherits from TenantNetwork1 which creates all network
  53                     resources and prepares a future VM attached to that
  54                     network.
  55                 run:
  56                     name: vmready1
  57
  58             -
  59                 case_name: vmready2
  60                 project_name: functest
  61                 criteria: 100
  62                 blocking: true
  63                 description: >-
  64                     It creates new user/project before creating and configuring
  65                     all tenant network ressources, flavors, images, etc.
  66                     required by advanced testcases.
  67                 run:
  68                     name: vmready2
  69
  70             -
  71                 case_name: singlevm1
  72                 project_name: functest
  73                 criteria: 100
  74                 blocking: true
  75                 description: >-
  76                     It inherits from TenantNetwork1 which creates all network
  77                     resources and completes it by booting a VM attached to that
  78                     network.
  79                 run:
  80                     name: singlevm1
  81
  82             -
  83                 case_name: singlevm2
  84                 project_name: functest
  85                 criteria: 100
  86                 blocking: true
  87                 description: >-
  88                     It creates new user/project before creating and configuring
  89                     all tenant network ressources and vms required by advanced
  90                     testcases.
  91                 run:
  92                     name: singlevm2
  93
  94             -
  95                 case_name: vping_ssh
  96                 project_name: functest
  97                 criteria: 100
  98                 blocking: true
  99                 description: >-
 100                     This test case verifies: 1) SSH to an instance using
 101                     floating IPs over the public network. 2) Connectivity
 102                     between 2 instances over a private network.
 103                 run:
 104                     name: vping_ssh
 105
 106             -
 107                 case_name: vping_userdata
 108                 project_name: functest
 109                 criteria: 100
 110                 blocking: true
 111                 description: >-
 112                     This test case verifies:  1) Boot a VM with given userdata.
 113                     2) Connectivity between 2 instances over a private network.
 114                 run:
 115                     name: vping_userdata
 116
 117             -
 118                 case_name: cinder_test
 119                 project_name: functest
 120                 criteria: 100
 121                 blocking: true
 122                 description: >-
 123                     This test case verifies: 1) Attach volume and to 1
 124                     instance; 2) Write data on volume 3) Detach volume
 125                     from instance 1, attach it on instance 2 3) Read volume
 126                     data
 127                 run:
 128                     name: cinder_test
 129
 130             -
 131                 case_name: odl
 132                 project_name: functest
 133                 criteria: 100
 134                 blocking: true
 135                 description: >-
 136                     Test Suite for the OpenDaylight SDN Controller. It
 137                     integrates some test suites from upstream using
 138                     Robot as the test framework.
 139                 dependencies:
 140                     - DEPLOY_SCENARIO: 'odl'
 141                 run:
 142                     name: odl
 143                     args:
 144                         suites:
 145                             - /src/odl_test/csit/suites/integration/basic
 146                             - /src/odl_test/csit/suites/openstack/neutron
 147
 148             -
 149                 case_name: tempest_smoke
 150                 project_name: functest
 151                 criteria: 100
 152                 blocking: false
 153                 description: >-
 154                     This test case runs the smoke subset of the OpenStack
 155                     Tempest suite. The list of test cases is generated by
 156                     Tempest automatically and depends on the parameters of
 157                     the OpenStack deplopyment.
 158                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L114
 159                 run:
 160                     name: tempest_common
 161                     args:
 162                         mode: '(?=.*\[.*\bsmoke\b.*\])(^tempest\.api)'
 163                         option:
 164                             - '--concurrency=3'
 165
 166             -
 167                 case_name: tempest_horizon
 168                 project_name: functest
 169                 criteria: 100
 170                 blocking: false
 171                 description: >-
 172                     This test case runs the Tempest suite proposed by the
 173                     Horizon project.
 174                 dependencies:
 175                     - DASHBOARD_URL: '^(?!\s*$).+'
 176                 run:
 177                     name: tempest_horizon
 178                     args:
 179                         mode: '^tempest_horizon.'
 180
 181     -
 182         name: smoke
 183         order: 1
 184         description: >-
 185             Set of basic Functional tests to validate the OPNFV scenarios.
 186         testcases:
 187             -
 188                 case_name: tempest_neutron
 189                 project_name: functest
 190                 criteria: 100
 191                 blocking: false
 192                 description: >-
 193                     This test case runs the Tempest suite proposed by the
 194                     Neutron project. The list of test cases is generated by
 195                     Tempest automatically and depends on the parameters of
 196                     the OpenStack deployment.
 197                 run:
 198                     name: tempest_common
 199                     args:
 200                         mode: '^neutron_tempest_plugin\.api'
 201                         option:
 202                             - '--concurrency=3'
 203
 204             -
 205                 case_name: tempest_cinder
 206                 project_name: functest
 207                 criteria: 100
 208                 blocking: false
 209                 description: >-
 210                     This test case runs the Tempest suite proposed by the
 211                     Cinder project.
 212                 run:
 213                     name: tempest_common
 214                     args:
 215                         mode:
 216                             '(?!.*test_incremental_backup)(^cinder_tempest_plugin.)'
 217                         option:
 218                             - '--concurrency=3'
 219
 220             -
 221                 case_name: tempest_keystone
 222                 project_name: functest
 223                 criteria: 100
 224                 blocking: false
 225                 description: >-
 226                     This test case runs the Tempest suite proposed by the
 227                     Keystone project.
 228                 run:
 229                     name: tempest_common
 230                     args:
 231                         mode: 'keystone_tempest_plugin.'
 232                         option:
 233                             - '--concurrency=3'
 234
 235             -
 236                 case_name: tempest_heat
 237                 project_name: functest
 238                 criteria: 100
 239                 blocking: false
 240                 description: >-
 241                     This test case runs the Tempest suite proposed by the
 242                     Heat project.
 243                 run:
 244                     name: tempest_heat
 245                     args:
 246                         mode: '^heat_tempest_plugin.tests'
 247                         option:
 248                             - '--concurrency=1'
 249
 250             -
 251                 case_name: rally_sanity
 252                 project_name: functest
 253                 criteria: 100
 254                 blocking: false
 255                 description: >-
 256                     This test case runs a sub group of tests of the OpenStack
 257                     Rally suite in smoke mode.
 258                 run:
 259                     name: rally_sanity
 260                     args:
 261                         optional:
 262                             - 'gnocchi'
 263                             - 'barbican'
 264
 265             -
 266                 case_name: refstack_compute
 267                 project_name: functest
 268                 criteria: 100
 269                 blocking: false
 270                 deny_skipping: true
 271                 tests_count: 222
 272                 description: >-
 273                     This test case runs a sub group of tests of the OpenStack
 274                     Compute testcases.
 275                 run:
 276                     name: refstack
 277                     args:
 278                         target: compute
 279                         option:
 280                             - '--concurrency=3'
 281
 282             -
 283                 case_name: refstack_object
 284                 project_name: functest
 285                 criteria: 100
 286                 blocking: false
 287                 deny_skipping: true
 288                 tests_count: 58
 289                 description: >-
 290                     This test case runs a sub group of tests of the OpenStack
 291                     Object testcases.
 292                 run:
 293                     name: refstack
 294                     args:
 295                         target: object
 296                         option:
 297                             - '--concurrency=3'
 298
 299             -
 300                 case_name: refstack_platform
 301                 project_name: functest
 302                 criteria: 100
 303                 blocking: false
 304                 deny_skipping: true
 305                 tests_count: 278
 306                 description: >-
 307                     This test case runs a sub group of tests of the OpenStack
 308                     Platform testcases.
 309                 run:
 310                     name: refstack
 311                     args:
 312                         target: platform
 313                         option:
 314                             - '--concurrency=3'
 315
 316             -
 317                 case_name: tempest_full
 318                 project_name: functest
 319                 criteria: 100
 320                 blocking: false
 321                 description: >-
 322                     The list of test cases is generated by
 323                     Tempest automatically and depends on the parameters of
 324                     the OpenStack deployment.
 325                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83
 326                 run:
 327                     name: tempest_common
 328                     args:
 329                         mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
 330                         option:
 331                             - '--concurrency=3'
 332
 333             -
 334                 case_name: tempest_scenario
 335                 project_name: functest
 336                 criteria: 100
 337                 blocking: false
 338                 description: >-
 339                     The list of test cases is generated by
 340                     Tempest automatically and depends on the parameters of
 341                     the OpenStack deployment.
 342                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
 343                 run:
 344                     name: tempest_scenario
 345                     args:
 346                         mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)'
 347                         option:
 348                             - '--concurrency=1'
 349
 350             -
 351                 case_name: tempest_slow
 352                 project_name: functest
 353                 criteria: 100
 354                 blocking: false
 355                 description: >-
 356                     The list of test cases is generated by
 357                     Tempest automatically and depends on the parameters of
 358                     the OpenStack deployment.
 359                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
 360                 run:
 361                     name: tempest_common
 362                     args:
 363                         mode: '(?=.*\[.*\bslow\b.*\])(^tempest\.)'
 364                         option:
 365                             - '--concurrency=1'
 366
 367             -
 368                 case_name: patrole
 369                 project_name: functest
 370                 criteria: 100
 371                 blocking: false
 372                 description: >-
 373                     Test suite from Patrole project.
 374                 run:
 375                     name: patrole
 376                     args:
 377                         role: 'admin'
 378                         includes:
 379                             - 'image'
 380                             - 'network'
 381                         excludes:
 382                             - 'network.test_flavors_rbac'
 383                             - 'network.test_network_ip_availability_rbac'
 384                             - 'test_networks_multiprovider_rbac'
 385                             - 'test_auto_allocated_topology_rbac'
 386                             - 'test_create_network_provider_segmentation_id'
 387                             - 'test_update_network_provider_physical_network'
 388                             - 'test_update_network_provider_segmentation_id'
 389                             - 'network.test_agents_rbac'
 390                             - 'test_list_metadef_namespaces'
 391
 392             -
 393                 case_name: neutron_trunk
 394                 project_name: functest
 395                 criteria: 100
 396                 blocking: false
 397                 description: >-
 398                     This test case runs the neutron trunk subtest of the
 399                     OpenStack Tempest suite. The list of test cases is
 400                     generated by Tempest having as input the relevant
 401                     testcase list file.
 402                 run:
 403                     name: tempest_common
 404                     args:
 405                         mode: 'neutron_tempest_plugin.(api|scenario).test_trunk'
 406                         neutron_extensions:
 407                             - trunk
 408                             - trunk-details
 409
 410             -
 411                 case_name: networking-bgpvpn
 412                 project_name: functest
 413                 criteria: 100
 414                 blocking: false
 415                 description: >-
 416                     This test case runs OpenStack networking-bgpvpn Tempest
 417                     suite. The list of test cases is generated by Tempest
 418                     having as input the relevant regex.
 419                 run:
 420                     name: tempest_common
 421                     args:
 422                         mode: '^neutron_tempest_plugin.bgpvpn.api\.'
 423                         neutron_extensions:
 424                             - bgpvpn
 425                         option:
 426                             - '--concurrency=3'
 427
 428             -
 429                 case_name: networking-sfc
 430                 project_name: functest
 431                 criteria: 100
 432                 blocking: false
 433                 description: >-
 434                     This test case runs OpenStack networking-sfc Tempest
 435                     suite. The list of test cases is generated by Tempest
 436                     having as input the relevant regex.
 437                 run:
 438                     name: tempest_common
 439                     args:
 440                         mode:
 441                             '^(?:neutron_tempest_plugin\.sfc\.tests\.api).*$'
 442                         neutron_extensions:
 443                             - sfc
 444                         option:
 445                             - '--concurrency=0'
 446
 447             -
 448                 case_name: barbican
 449                 project_name: functest
 450                 criteria: 100
 451                 blocking: false
 452                 description: >-
 453                     It leverages on the tempest plugin containing tests used to
 454                     verify the functionality of a barbican installation.
 455                 run:
 456                     name: barbican
 457                     args:
 458                         mode:
 459                             '^barbican_tempest_plugin.((?!test_signed_image_upload_boot_failure).)*$'
 460                         services:
 461                             - barbican
 462
 463             -
 464                 case_name: octavia
 465                 project_name: functest
 466                 criteria: 100
 467                 blocking: false
 468                 description: >-
 469                     It leverages on the tempest plugin containing tests used to
 470                     verify the functionality of an octavia installation.
 471                 run:
 472                     name: tempest_common
 473                     args:
 474                         mode:
 475                             '^octavia_tempest_plugin.tests.api.v2'
 476                         services:
 477                             - octavia
 478                         option:
 479                             - '--concurrency=2'
 480
 481     -
 482         name: benchmarking
 483         order: 2
 484         description: >-
 485             Run several OpenStack performance tools
 486             https://docs.openstack.org/performance-docs/latest/methodologies/tools.html
 487         testcases:
 488             -
 489                 case_name: rally_full
 490                 project_name: functest
 491                 criteria: 100
 492                 blocking: false
 493                 description: >-
 494                     This test case runs the full suite of scenarios of the
 495                     OpenStack Rally suite using several threads and iterations.
 496                 run:
 497                     name: rally_full
 498                     args:
 499                         optional:
 500                             - 'gnocchi'
 501                             - 'barbican'
 502
 503             -
 504                 case_name: rally_jobs
 505                 project_name: functest
 506                 criteria: 100
 507                 blocking: false
 508                 description: >-
 509                     This test case runs a group of Rally jobs used in
 510                     OpenStack gating
 511                 run:
 512                     name: rally_jobs
 513                     args:
 514                         optional:
 515                             - 'gnocchi'
 516
 517             -
 518                 case_name: vmtp
 519                 project_name: functest
 520                 criteria: 100
 521                 blocking: false
 522                 description: >-
 523                     VMTP is a small python application that will automatically
 524                     perform ping connectivity, round trip time measurement
 525                     (latency) and TCP/UDP throughput
 526                 dependencies:
 527                     - POD_ARCH: '^(?!aarch64$)'
 528                 run:
 529                     name: vmtp
 530
 531             -
 532                 case_name: shaker
 533                 project_name: functest
 534                 criteria: 100
 535                 blocking: false
 536                 description: >-
 537                     Shaker wraps around popular system network testing tools
 538                     like iperf, iperf3 and netperf (with help of flent). Shaker
 539                     is able to deploy OpenStack instances and networks in
 540                     different topologies.
 541                 dependencies:
 542                     - POD_ARCH: '^(?!aarch64$)'
 543                 run:
 544                     name: shaker
 545
 546     -
 547         name: vnf
 548         order: 4
 549         description: >-
 550             Collection of VNF test cases.
 551         testcases:
 552             -
 553                 case_name: cloudify
 554                 project_name: functest
 555                 criteria: 100
 556                 blocking: false
 557                 description: >-
 558                     This test case deploys the Cloudify orchestrator.
 559                 run:
 560                     name: cloudify
 561             -
 562                 case_name: cloudify_ims
 563                 project_name: functest
 564                 criteria: 100
 565                 blocking: false
 566                 description: >-
 567                     This test case deploys an OpenSource vIMS solution from
 568                     Clearwater using the Cloudify orchestrator. It also runs
 569                     some signaling traffic.
 570                 dependencies:
 571                     - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha'
 572                 run:
 573                     name: cloudify_ims
 574
 575             -
 576                 case_name: heat_ims
 577                 project_name: functest
 578                 criteria: 100
 579                 blocking: false
 580                 description: >-
 581                     This test case deploys an OpenSource vIMS solution from
 582                     Clearwater using the OpenStack Heat orchestrator.
 583                     It also runs some signaling traffic.
 584                 run:
 585                     name: heat_ims
 586
 587             -
 588                 case_name: vyos_vrouter
 589                 project_name: functest
 590                 criteria: 100
 591                 blocking: false
 592                 description: >-
 593                     This test case is vRouter testing.
 594                 run:
 595                     name: vyos_vrouter
 596
 597             -
 598                 case_name: juju_epc
 599                 project_name: functest
 600                 criteria: 100
 601                 blocking: false
 602                 description: >-
 603                     vEPC validation with Juju as VNF manager and ABoT as test
 604                     executor.
 605                 run:
 606                     name: juju_epc