functest/ci/testcases.yaml

   1 ---
   2 tiers:
   3     -
   4         name: healthcheck
   5         order: 0
   6         description: >-
   7             First tier to be executed to verify the basic
   8             operations in the VIM.
   9         testcases:
  10             -
  11                 case_name: connection_check
  12                 project_name: functest
  13                 criteria: 100
  14                 blocking: true
  15                 description: >-
  16                     This test case verifies the retrieval of OpenStack clients:
  17                     Keystone, Glance, Neutron and Nova and may perform some
  18                     simple queries.
  19                 run:
  20                     name: connection_check
  21
  22             -
  23                 case_name: tenantnetwork1
  24                 project_name: functest
  25                 criteria: 100
  26                 blocking: true
  27                 description: >-
  28                     It creates and configures all tenant network ressources
  29                     required by advanced testcases (subnet, network and
  30                     router).
  31                 run:
  32                     name: tenantnetwork1
  33
  34             -
  35                 case_name: tenantnetwork2
  36                 project_name: functest
  37                 criteria: 100
  38                 blocking: true
  39                 description: >-
  40                     It creates new user/project before creating and configuring
  41                     all tenant network ressources required by a testcase
  42                     (subnet, network and router).
  43                 run:
  44                     name: tenantnetwork2
  45
  46             -
  47                 case_name: vmready1
  48                 project_name: functest
  49                 criteria: 100
  50                 blocking: true
  51                 description: >-
  52                     It inherits from TenantNetwork1 which creates all network
  53                     resources and prepares a future VM attached to that
  54                     network.
  55                 run:
  56                     name: vmready1
  57
  58             -
  59                 case_name: vmready2
  60                 project_name: functest
  61                 criteria: 100
  62                 blocking: true
  63                 description: >-
  64                     It creates new user/project before creating and configuring
  65                     all tenant network ressources, flavors, images, etc.
  66                     required by advanced testcases.
  67                 run:
  68                     name: vmready2
  69
  70             -
  71                 case_name: singlevm1
  72                 project_name: functest
  73                 criteria: 100
  74                 blocking: true
  75                 description: >-
  76                     It inherits from TenantNetwork1 which creates all network
  77                     resources and completes it by booting a VM attached to that
  78                     network.
  79                 run:
  80                     name: singlevm1
  81
  82             -
  83                 case_name: singlevm2
  84                 project_name: functest
  85                 criteria: 100
  86                 blocking: true
  87                 description: >-
  88                     It creates new user/project before creating and configuring
  89                     all tenant network ressources and vms required by advanced
  90                     testcases.
  91                 run:
  92                     name: singlevm2
  93
  94             -
  95                 case_name: vping_ssh
  96                 project_name: functest
  97                 criteria: 100
  98                 blocking: true
  99                 description: >-
 100                     This test case verifies: 1) SSH to an instance using
 101                     floating IPs over the public network. 2) Connectivity
 102                     between 2 instances over a private network.
 103                 run:
 104                     name: vping_ssh
 105
 106             -
 107                 case_name: vping_userdata
 108                 project_name: functest
 109                 criteria: 100
 110                 blocking: true
 111                 description: >-
 112                     This test case verifies:  1) Boot a VM with given userdata.
 113                     2) Connectivity between 2 instances over a private network.
 114                 run:
 115                     name: vping_userdata
 116
 117             -
 118                 case_name: cinder_test
 119                 project_name: functest
 120                 criteria: 100
 121                 blocking: true
 122                 description: >-
 123                     This test case verifies: 1) Attach volume and to 1
 124                     instance; 2) Write data on volume 3) Detach volume
 125                     from instance 1, attach it on instance 2 3) Read volume
 126                     data
 127                 run:
 128                     name: cinder_test
 129
 130             -
 131                 case_name: odl
 132                 project_name: functest
 133                 criteria: 100
 134                 blocking: true
 135                 description: >-
 136                     Test Suite for the OpenDaylight SDN Controller. It
 137                     integrates some test suites from upstream using
 138                     Robot as the test framework.
 139                 dependencies:
 140                     - DEPLOY_SCENARIO: 'odl'
 141                 run:
 142                     name: odl
 143                     args:
 144                         suites:
 145                             - /src/odl_test/csit/suites/integration/basic
 146                             - /src/odl_test/csit/suites/openstack/neutron
 147
 148             -
 149                 case_name: tempest_smoke
 150                 project_name: functest
 151                 criteria: 100
 152                 blocking: false
 153                 description: >-
 154                     This test case runs the smoke subset of the OpenStack
 155                     Tempest suite. The list of test cases is generated by
 156                     Tempest automatically and depends on the parameters of
 157                     the OpenStack deplopyment.
 158                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L114
 159                 run:
 160                     name: tempest_common
 161                     args:
 162                         mode: '(?=.*\[.*\bsmoke\b.*\])(^tempest\.api)'
 163                         option:
 164                             - '--concurrency=4'
 165
 166             -
 167                 case_name: tempest_horizon
 168                 project_name: functest
 169                 criteria: 100
 170                 blocking: false
 171                 description: >-
 172                     This test case runs the Tempest suite proposed by the
 173                     Horizon project.
 174                 dependencies:
 175                     - DASHBOARD_URL: '^(?!\s*$).+'
 176                 run:
 177                     name: tempest_horizon
 178                     args:
 179                         mode: '^tempest_horizon.'
 180
 181     -
 182         name: smoke
 183         order: 1
 184         description: >-
 185             Set of basic Functional tests to validate the OPNFV scenarios.
 186         testcases:
 187             -
 188                 case_name: tempest_neutron
 189                 project_name: functest
 190                 criteria: 100
 191                 blocking: false
 192                 description: >-
 193                     This test case runs the Tempest suite proposed by the
 194                     Neutron project. The list of test cases is generated by
 195                     Tempest automatically and depends on the parameters of
 196                     the OpenStack deployment.
 197                 run:
 198                     name: tempest_common
 199                     args:
 200                         mode: '^neutron_tempest_plugin\.api'
 201                         option:
 202                             - '--concurrency=4'
 203
 204             -
 205                 case_name: tempest_cinder
 206                 project_name: functest
 207                 criteria: 100
 208                 blocking: false
 209                 description: >-
 210                     This test case runs the Tempest suite proposed by the
 211                     Cinder project.
 212                 run:
 213                     name: tempest_common
 214                     args:
 215                         mode:
 216                             '(?!.*test_incremental_backup)(^cinder_tempest_plugin.)'
 217                         option:
 218                             - '--concurrency=4'
 219
 220             -
 221                 case_name: tempest_keystone
 222                 project_name: functest
 223                 criteria: 100
 224                 blocking: false
 225                 description: >-
 226                     This test case runs the Tempest suite proposed by the
 227                     Keystone project.
 228                 run:
 229                     name: tempest_common
 230                     args:
 231                         mode: 'keystone_tempest_plugin.'
 232                         option:
 233                             - '--concurrency=4'
 234
 235             -
 236                 case_name: tempest_heat
 237                 project_name: functest
 238                 criteria: 100
 239                 blocking: false
 240                 description: >-
 241                     This test case runs the Tempest suite proposed by the
 242                     Heat project.
 243                 run:
 244                     name: tempest_heat
 245                     args:
 246                         mode: '^heat_tempest_plugin.tests'
 247                         option:
 248                             - '--concurrency=1'
 249
 250             -
 251                 case_name: tempest_telemetry
 252                 project_name: functest
 253                 criteria: 100
 254                 blocking: false
 255                 description: >-
 256                     This test case runs the Tempest suite proposed by the
 257                     Telemetry project.
 258                 run:
 259                     name: tempest_common
 260                     args:
 261                         mode: "(?!.*test_autoscaling)(?!.*test_live)\
 262                             (^telemetry_tempest_plugin)"
 263                         services:
 264                             - aodh
 265                             - gnocchi
 266                             - panko
 267                         option:
 268                             - '--concurrency=4'
 269
 270             -
 271                 case_name: rally_sanity
 272                 project_name: functest
 273                 criteria: 100
 274                 blocking: false
 275                 description: >-
 276                     This test case runs a sub group of tests of the OpenStack
 277                     Rally suite in smoke mode.
 278                 run:
 279                     name: rally_sanity
 280                     args:
 281                         optional:
 282                             - 'gnocchi'
 283                             - 'barbican'
 284
 285             -
 286                 case_name: refstack_compute
 287                 project_name: functest
 288                 criteria: 100
 289                 blocking: false
 290                 deny_skipping: true
 291                 tests_count: 222
 292                 description: >-
 293                     This test case runs a sub group of tests of the OpenStack
 294                     Compute testcases.
 295                 run:
 296                     name: refstack
 297                     args:
 298                         target: compute
 299                         option:
 300                             - '--concurrency=4'
 301
 302             -
 303                 case_name: refstack_object
 304                 project_name: functest
 305                 criteria: 100
 306                 blocking: false
 307                 deny_skipping: true
 308                 tests_count: 58
 309                 description: >-
 310                     This test case runs a sub group of tests of the OpenStack
 311                     Object testcases.
 312                 run:
 313                     name: refstack
 314                     args:
 315                         target: object
 316                         option:
 317                             - '--concurrency=4'
 318
 319             -
 320                 case_name: refstack_platform
 321                 project_name: functest
 322                 criteria: 100
 323                 blocking: false
 324                 deny_skipping: true
 325                 tests_count: 278
 326                 description: >-
 327                     This test case runs a sub group of tests of the OpenStack
 328                     Platform testcases.
 329                 run:
 330                     name: refstack
 331                     args:
 332                         target: platform
 333                         option:
 334                             - '--concurrency=4'
 335
 336             -
 337                 case_name: tempest_full
 338                 project_name: functest
 339                 criteria: 100
 340                 blocking: false
 341                 description: >-
 342                     The list of test cases is generated by
 343                     Tempest automatically and depends on the parameters of
 344                     the OpenStack deployment.
 345                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83
 346                 run:
 347                     name: tempest_common
 348                     args:
 349                         mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
 350                         option:
 351                             - '--concurrency=4'
 352
 353             -
 354                 case_name: tempest_scenario
 355                 project_name: functest
 356                 criteria: 100
 357                 blocking: false
 358                 description: >-
 359                     The list of test cases is generated by
 360                     Tempest automatically and depends on the parameters of
 361                     the OpenStack deployment.
 362                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
 363                 run:
 364                     name: tempest_common
 365                     args:
 366                         mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)'
 367                         option:
 368                             - '--concurrency=1'
 369
 370             -
 371                 case_name: tempest_slow
 372                 project_name: functest
 373                 criteria: 100
 374                 blocking: false
 375                 description: >-
 376                     The list of test cases is generated by
 377                     Tempest automatically and depends on the parameters of
 378                     the OpenStack deployment.
 379                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
 380                 run:
 381                     name: tempest_common
 382                     args:
 383                         mode: '(?=.*\[.*\bslow\b.*\])(^tempest\.)'
 384                         option:
 385                             - '--concurrency=1'
 386
 387             -
 388                 case_name: patrole_admin
 389                 project_name: functest
 390                 criteria: 100
 391                 blocking: false
 392                 description: >-
 393                     Test suite from Patrole project.
 394                 run:
 395                     name: patrole
 396                     args:
 397                         roles: 'admin'
 398                         mode: "(?!.*test_unmanage_snapshot_rbac)\
 399                           (?!.*test_show_auto_allocated_topology)\
 400                           (?!.*test_delete_auto_allocated_topology)\
 401                           (?!.*compute.test_floating_ips_rbac)\
 402                           (?=.*[.*\bslow\b.*])\
 403                           (^patrole_tempest_plugin.tests.api.(compute|image|network|volume))"
 404                         option:
 405                             - '--concurrency=4'
 406
 407             -
 408                 case_name: patrole_member
 409                 project_name: functest
 410                 criteria: 100
 411                 blocking: false
 412                 description: >-
 413                     Test suite from Patrole project.
 414                 run:
 415                     name: patrole
 416                     args:
 417                         roles: 'member'
 418                         mode: "(?!.*test_unmanage_snapshot_rbac)\
 419                           (?!.*test_show_auto_allocated_topology)\
 420                           (?!.*test_delete_auto_allocated_topology)\
 421                           (?!.*compute.test_floating_ips_rbac)\
 422                           (?=.*[.*\bslow\b.*])\
 423                           (^patrole_tempest_plugin.tests.api.(compute|image|network|volume))"
 424                         option:
 425                             - '--concurrency=4'
 426
 427             -
 428                 case_name: patrole_reader
 429                 project_name: functest
 430                 criteria: 100
 431                 blocking: false
 432                 description: >-
 433                     Test suite from Patrole project.
 434                 run:
 435                     name: patrole
 436                     args:
 437                         roles: 'reader'
 438                         mode: "(?!.*test_unmanage_snapshot_rbac)\
 439                           (?!.*test_show_auto_allocated_topology)\
 440                           (?!.*test_delete_auto_allocated_topology)\
 441                           (?!.*compute.test_floating_ips_rbac)\
 442                           (?=.*[.*\bslow\b.*])\
 443                           (^patrole_tempest_plugin.tests.api.(compute|image|network|volume))"
 444                         option:
 445                             - '--concurrency=4'
 446
 447             -
 448                 case_name: tempest_barbican
 449                 project_name: functest
 450                 criteria: 100
 451                 blocking: false
 452                 description: >-
 453                     It leverages on the tempest plugin containing tests used to
 454                     verify the functionality of a barbican installation.
 455                 run:
 456                     name: barbican
 457                     args:
 458                         mode:
 459                             '^barbican_tempest_plugin.((?!test_signed_image_upload_boot_failure).)*$'
 460                         services:
 461                             - barbican
 462                         option:
 463                             - '--concurrency=4'
 464
 465             -
 466                 case_name: tempest_octavia
 467                 project_name: functest
 468                 criteria: 100
 469                 blocking: false
 470                 description: >-
 471                     It leverages on the tempest plugin containing tests used to
 472                     verify the functionality of an octavia installation.
 473                 run:
 474                     name: tempest_common
 475                     args:
 476                         mode: "(?!.*api.v2.test_availability_zone)\
 477                             (?!.*api.v2.test_availability_zone_profile)\
 478                             (?!.*api.v2.test_member.MemberAPITest.test_member_ipv4_create)\
 479                             (?!.*api.v2.test_member.MemberAPITest.test_member_ipv6_create)\
 480                             (^octavia_tempest_plugin.tests.(api|scenario))"
 481                         services:
 482                             - octavia
 483                         option:
 484                             - '--concurrency=2'
 485
 486             -
 487                 case_name: tempest_cyborg
 488                 project_name: functest
 489                 criteria: 100
 490                 blocking: false
 491                 description: >-
 492                     It leverages on the tempest plugin containing tests used to
 493                     verify the functionality of a cyborg installation.
 494                 run:
 495                     name: tempest_common
 496                     args:
 497                         mode: '^cyborg_tempest_plugin'
 498                         services:
 499                             - cyborg
 500                         option:
 501                             - '--concurrency=4'
 502
 503     -
 504         name: smoke_cntt
 505         order: 2
 506         description: >-
 507             Set of basic Functional tests to validate the OPNFV scenarios.
 508         testcases:
 509             -
 510                 case_name: tempest_neutron_cntt
 511                 project_name: functest
 512                 criteria: 100
 513                 blocking: false
 514                 deny_skipping: true
 515                 tests_count: 517
 516                 description: >-
 517                     This test case runs the Tempest suite proposed by the
 518                     Neutron project. The list of test cases is generated by
 519                     Tempest automatically and depends on the parameters of
 520                     the OpenStack deployment.
 521                 run:
 522                     name: tempest_common
 523                     args:
 524                         mode: "(?!.*admin.test_agent_availability_zone)\
 525                             (?!.*admin.test_dhcp_agent_scheduler)\
 526                             (?!.*admin.test_l3_agent_scheduler)\
 527                             (?!.*admin.test_logging)\
 528                             (?!.*admin.test_logging_negative)\
 529                             (?!.*admin.test_network_segment_range)\
 530                             (?!.*admin.test_ports.PortTestCasesAdmin.test_regenerate_mac_address)\
 531                             (?!.*admin.test_ports.PortTestCasesResourceRequest)\
 532                             (?!.*admin.test_routers_dvr)\
 533                             (?!.*admin.test_routers_flavors)\
 534                             (?!.*admin.test_routers_ha)\
 535                             (?!.*test_conntrack_helper)\
 536                             (?!.*test_floating_ips.FloatingIPPoolTestJSON)\
 537                             (?!.*test_floating_ips.FloatingIPTestJSON.test_create_update_floatingip_port_details)\
 538                             (?!.*test_metering_extensions)\
 539                             (?!.*test_metering_negative)\
 540                             (?!.*test_networks.NetworksSearchCriteriaTest.test_list_validation_filters)\
 541                             (?!.*test_networks.NetworksTestAdmin.test_create_tenant_network_vxlan)\
 542                             (?!.*test_networks.NetworksTestJSON.test_create_update_network_dns_domain)\
 543                             (?!.*test_port_forwardings)\
 544                             (?!.*test_ports.PortsTaggingOnCreation)\
 545                             (?!.*test_ports.PortsTestJSON.test_create_port_with_propagate_uplink_status)\
 546                             (?!.*test_ports.PortsTestJSON.test_create_port_without_propagate_uplink_status)\
 547                             (?!.*test_ports.PortsTestJSON.test_create_update_port_with_dns_domain)\
 548                             (?!.*test_ports.PortsTestJSON.test_create_update_port_with_dns_name)\
 549                             (?!.*test_ports.PortsTestJSON.test_create_update_port_with_no_dns_name)\
 550                             (?!.*test_qos.QosMinimumBandwidthRuleTestJSON)\
 551                             (?!.*test_revisions.TestRevisions.test_update_dns_domain_bumps_revision)\
 552                             (?!.*test_revisions.TestRevisions.test_update_router_extra_attributes_bumps_revision)\
 553                             (?!.*test_router_interface_fip)\
 554                             (?!.*test_routers.DvrRoutersTest)\
 555                             (?!.*test_routers.HaRoutersTest)\
 556                             (?!.*test_routers.RoutersIpV6Test.test_extra_routes_atomic)\
 557                             (?!.*test_routers.RoutersTest.test_extra_routes_atomic)\
 558                             (?!.*test_routers_negative.DvrRoutersNegativeTest)\
 559                             (?!.*test_routers_negative.DvrRoutersNegativeTestExtended)\
 560                             (?!.*test_routers_negative.HaRoutersNegativeTest)\
 561                             (?!.*test_security_groups.RbacSharedSecurityGroupTest)\
 562                             (?!.*test_subnetpool_prefix_ops)\
 563                             (?!.*test_subnetpools.RbacSubnetPoolTest)\
 564                             (?!.*test_subnetpools.SubnetPoolsSearchCriteriaTest.test_list_validation_filters)\
 565                             (?!.*test_subnetpools_negative.SubnetPoolsNegativeTestJSON.test_tenant_create_subnetpool_associate_shared_address_scope)\
 566                             (?!.*test_subnets.SubnetsSearchCriteriaTest.test_list_validation_filters)\
 567                             (?!.*test_timestamp.TestTimeStamp.test_segment_with_timestamp)\
 568                             (?!.*test_trunk.TrunkTestInheritJSONBase.test_add_subport)\
 569                             (?!.*test_trunk.TrunkTestMtusJSON)\
 570                             (?!.*test_trunk_negative.TrunkTestJSON.test_create_subport_invalid_inherit_network_segmentation_type)\
 571                             (?!.*test_trunk_negative.TrunkTestMtusJSON)\
 572                             (^neutron_tempest_plugin.api)"
 573                         option:
 574                             - '--concurrency=4'
 575
 576             -
 577                 case_name: tempest_cinder_cntt
 578                 project_name: functest
 579                 criteria: 100
 580                 blocking: false
 581                 deny_skipping: true
 582                 tests_count: 8
 583                 description: >-
 584                     This test case runs the Tempest suite proposed by the
 585                     Cinder project.
 586                 run:
 587                     name: tempest_common
 588                     args:
 589                         mode: "(?!.*test_incremental_backup)\
 590                             (?!.*test_consistencygroups)\
 591                             (^cinder_tempest_plugin.)"
 592                         option:
 593                             - '--concurrency=4'
 594
 595             -
 596                 case_name: tempest_keystone_cntt
 597                 project_name: functest
 598                 criteria: 100
 599                 blocking: false
 600                 deny_skipping: true
 601                 tests_count: 27
 602                 description: >-
 603                     This test case runs the Tempest suite proposed by the
 604                     Keystone project.
 605                 run:
 606                     name: tempest_common
 607                     args:
 608                         mode: "(?!.*api.identity.v3.test_oauth1_tokens)\
 609                             (?!.*scenario.test_federated_authentication)\
 610                             keystone_tempest_plugin."
 611                         option:
 612                             - '--concurrency=4'
 613
 614             -
 615                 case_name: tempest_heat_cntt
 616                 project_name: functest
 617                 criteria: 100
 618                 blocking: false
 619                 deny_skipping: true
 620                 tests_count: 119
 621                 description: >-
 622                     This test case runs the Tempest suite proposed by the
 623                     Heat project.
 624                 run:
 625                     name: tempest_heat
 626                     args:
 627                         mode: "(?!.*functional.test_lbaasv2)\
 628                           (?!.*functional.test_encryption_vol_type)\
 629                           (?!.*RemoteStackTest.test_stack_create_with_cloud_credential)\
 630                           (?!.*scenario.test_aodh_alarm)\
 631                           (?!.*tests.scenario.test_autoscaling_lb)\
 632                           (?!.*scenario.test_autoscaling_lbv2)\
 633                           (?!.*scenario.test_server_software_config)\
 634                           (?!.*test_volumes.VolumeBackupRestoreIntegrationTest)\
 635                           (?!.*scenario.test_octavia_lbaas)\
 636                           (?!.*scenario.test_server_cfn_init)\
 637                           ^heat_tempest_plugin.tests"
 638                         option:
 639                             - '--concurrency=1'
 640
 641             -
 642                 case_name: rally_sanity_cntt
 643                 project_name: functest
 644                 criteria: 100
 645                 blocking: false
 646                 description: >-
 647                     This test case runs a sub group of tests of the OpenStack
 648                     Rally suite in smoke mode.
 649                 run:
 650                     name: rally_sanity
 651                     args:
 652                         tests:
 653                             - 'authenticate'
 654                             - 'glance'
 655                             - 'cinder'
 656                             - 'heat'
 657                             - 'keystone'
 658                             - 'neutron'
 659                             - 'nova'
 660                             - 'quotas'
 661                             - 'swift'
 662
 663             -
 664                 case_name: tempest_full_cntt
 665                 project_name: functest
 666                 criteria: 100
 667                 blocking: false
 668                 deny_skipping: true
 669                 tests_count: 1280
 670                 description: >-
 671                     The list of test cases is generated by
 672                     Tempest automatically and depends on the parameters of
 673                     the OpenStack deployment.
 674                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83
 675                 run:
 676                     name: tempest_common
 677                     args:
 678                         mode: "(?!.*test_fixed_ips)\
 679                           (?!.*test_fixed_ips_negative)\
 680                           (?!.*test_auto_allocate_network)(?!.*test_floating_ips_bulk)\
 681                           (?!.*test_flavors_microversions.FlavorsV255TestJSON)\
 682                           (?!.*test_flavors_microversions.FlavorsV261TestJSON)\
 683                           (?!.*test_live_migration.LiveAutoBlockMigrationV225Test.test_iscsi_volume)\
 684                           (?!.*test_live_migration.LiveAutoBlockMigrationV225Test.test_live_block_migration)\
 685                           (?!.*test_live_migration.LiveAutoBlockMigrationV225Test.test_live_block_migration_paused)\
 686                           (?!.*test_live_migration.LiveAutoBlockMigrationV225Test.test_volume_backed_live_migration)\
 687                           (?!.*test_live_migration.LiveMigrationTest.test_iscsi_volume)\
 688                           (?!.*test_live_migration.LiveMigrationTest.test_live_block_migration)\
 689                           (?!.*test_live_migration.LiveMigrationTest.test_live_block_migration_paused)\
 690                           (?!.*test_live_migration.LiveMigrationTest.test_volume_backed_live_migration)\
 691                           (?!.*test_live_migration.LiveMigrationRemoteConsolesV26Test)\
 692                           (?!.*test_quotas.QuotasAdminTestV257)\
 693                           (?!.*certificates.test_certificates)\
 694                           (?!.*test_quotas_negative.QuotasSecurityGroupAdminNegativeTest)\
 695                           (?!.*test_novnc)(?!.*test_server_personality)\
 696                           (?!.*test_servers.ServerShowV263Test.test_show_update_rebuild_list_server)\
 697                           (?!.*test_servers_microversions.ServerShowV254Test)\
 698                           (?!.*test_servers_microversions.ServerShowV257Test)\
 699                           (?!.*test_servers_negative.ServersNegativeTestJSON.test_personality_file_contents_not_encoded)\
 700                           (?!.*servers.test_virtual_interfaces)\
 701                           (?!.*test_server_actions.ServerActionsTestJSON.test_change_server_password)\
 702                           (?!.*test_server_actions.ServerActionsTestJSON.test_get_vnc_console)\
 703                           (?!.*test_server_actions.ServerActionsTestJSON.test_reboot_server_soft)\
 704                           (?!.*test_server_rescue.ServerBootFromVolumeStableRescueTest)\
 705                           (?!.*test_server_rescue.ServerStableDeviceRescueTest)\
 706                           (?!.*test_security_group_default_rules)\
 707                           (?!.*test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_security_group_create_with_duplicate_name)\
 708                           (?!.*test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_security_group_create_with_invalid_group_description)\
 709                           (?!.*test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_security_group_create_with_invalid_group_name)\
 710                           (?!.*test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_update_security_group_with_invalid_sg_des)\
 711                           (?!.*test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_update_security_group_with_invalid_sg_id)\
 712                           (?!.*test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_update_security_group_with_invalid_sg_name)\
 713                           (?!.*test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filtered_by_ip_regex)\
 714                           (?!.*compute.test_virtual_interfaces)(?!.*compute.test_virtual_interfaces_negative)\
 715                           (?!.*compute.test_networks)\
 716                           (?!.*test_attach_volume.AttachVolumeMultiAttach)\
 717                           (?!.*identity.admin.v2)(?!.*identity.v2)\
 718                           (?!.*image.v1)\
 719                           (?!.*image.v2.test_images.ImportImagesTest)\
 720                           (?!.*admin.test_dhcp_agent_scheduler)\
 721                           (?!.*admin.test_routers_dvr)\
 722                           (?!.*test_metering_extensions)(?!.*network.test_tags)\
 723                           (?!.*test_routers_negative.DvrRoutersNegativeTest)\
 724                           (?!.*test_routers.RoutersIpV6Test.test_create_router_set_gateway_with_fixed_ip)\
 725                           (?!.*test_routers.RoutersTest.test_create_router_set_gateway_with_fixed_ip)\
 726                           (?!.*test_group_snapshots.GroupSnapshotsV319Test.test_reset_group_snapshot_status)\
 727                           (?!.*test_multi_backend)\
 728                           (?!.*test_volume_retype.VolumeRetypeWithMigrationTest)\
 729                           (?!.*test_volume_delete_cascade.VolumesDeleteCascade.test_volume_from_snapshot_cascade_delete)\
 730                           (?!.*test_volumes_backup.VolumesBackupsTest.test_volume_backup_create_get_detailed_list_restore_delete)\
 731                           (?!.*test_volumes_negative.UpdateMultiattachVolumeNegativeTest.test_multiattach_rw_volume_update_failure)\
 732                           (?!.*test_volumes_extend.VolumesExtendAttachedTest.test_extend_attached_volume)\
 733                           (?!.*\\[.*\\bslow\\b.*\\])(^tempest.api)"
 734                         option:
 735                             - '--concurrency=4'
 736
 737             -
 738                 case_name: tempest_scenario_cntt
 739                 project_name: functest
 740                 criteria: 100
 741                 blocking: false
 742                 deny_skipping: true
 743                 tests_count: 9
 744                 description: >-
 745                     The list of test cases is generated by
 746                     Tempest automatically and depends on the parameters of
 747                     the OpenStack deployment.
 748                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
 749                 run:
 750                     name: tempest_scenario
 751                     args:
 752                         mode: "\
 753                           (?!.*test_volume_boot_pattern.TestVolumeBootPattern.test_boot_server_from_encrypted_volume_luks)\
 754                           (?!.*\\[.*\\bslow\\b.*\\])(^tempest.scenario)"
 755                         option:
 756                             - '--concurrency=1'
 757
 758             -
 759                 case_name: tempest_slow_cntt
 760                 project_name: functest
 761                 criteria: 100
 762                 blocking: false
 763                 deny_skipping: true
 764                 tests_count: 43
 765                 description: >-
 766                     The list of test cases is generated by
 767                     Tempest automatically and depends on the parameters of
 768                     the OpenStack deployment.
 769                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
 770                 run:
 771                     name: tempest_common
 772                     args:
 773                         mode: "(?!.*test_volume_swap)\
 774                           (?!.*test_server_personality)\
 775                           (?!.*test_server_rescue.ServerBootFromVolumeStableRescueTest)\
 776                           (?!.*test_container_sync.ContainerSyncTest.test_container_synchronization)\
 777                           (?!.*test_container_sync_middleware.ContainerSyncMiddlewareTest.test_container_synchronization)\
 778                           (?!.*test_encrypted_cinder_volumes)\
 779                           (?!.*test_minbw_allocation_placement)\
 780                           (?!.*test_network_basic_ops.TestNetworkBasicOps.test_router_rescheduling)\
 781                           (?!.*test_volume_migrate_attached)\
 782                           (?!.*test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_cold_migration_revert)\
 783                           (?=.*\\[.*\\bslow\\b.*\\])(^tempest.)"
 784                         option:
 785                             - '--concurrency=1'
 786
 787     -
 788         name: benchmarking
 789         order: 3
 790         description: >-
 791             Run several OpenStack performance tools
 792             https://docs.openstack.org/performance-docs/latest/methodologies/tools.html
 793         testcases:
 794             -
 795                 case_name: rally_full
 796                 project_name: functest
 797                 criteria: 100
 798                 blocking: false
 799                 description: >-
 800                     This test case runs the full suite of scenarios of the
 801                     OpenStack Rally suite using several threads and iterations.
 802                 run:
 803                     name: rally_full
 804                     args:
 805                         optional:
 806                             - 'gnocchi'
 807                             - 'barbican'
 808
 809             -
 810                 case_name: rally_jobs
 811                 project_name: functest
 812                 criteria: 100
 813                 blocking: false
 814                 description: >-
 815                     This test case runs a group of Rally jobs used in
 816                     OpenStack gating
 817                 run:
 818                     name: rally_jobs
 819                     args:
 820                         optional:
 821                             - 'gnocchi'
 822
 823             -
 824                 case_name: vmtp
 825                 project_name: functest
 826                 criteria: 100
 827                 blocking: false
 828                 description: >-
 829                     VMTP is a small python application that will automatically
 830                     perform ping connectivity, round trip time measurement
 831                     (latency) and TCP/UDP throughput
 832                 dependencies:
 833                     - POD_ARCH: '^(?!aarch64$)'
 834                 run:
 835                     name: vmtp
 836
 837             -
 838                 case_name: shaker
 839                 project_name: functest
 840                 criteria: 100
 841                 blocking: false
 842                 description: >-
 843                     Shaker wraps around popular system network testing tools
 844                     like iperf, iperf3 and netperf (with help of flent). Shaker
 845                     is able to deploy OpenStack instances and networks in
 846                     different topologies.
 847                 dependencies:
 848                     - POD_ARCH: '^(?!aarch64$)'
 849                 run:
 850                     name: shaker
 851
 852     -
 853         name: benchmarking_cntt
 854         order: 4
 855         description: >-
 856             Run several OpenStack performance tools
 857             https://docs.openstack.org/performance-docs/latest/methodologies/tools.html
 858         testcases:
 859             -
 860                 case_name: rally_full_cntt
 861                 project_name: functest
 862                 criteria: 100
 863                 blocking: false
 864                 description: >-
 865                     This test case runs the full suite of scenarios of the
 866                     OpenStack Rally suite using several threads and iterations.
 867                 run:
 868                     name: rally_full
 869                     args:
 870                         tests:
 871                             - 'authenticate'
 872                             - 'glance'
 873                             - 'cinder'
 874                             - 'heat'
 875                             - 'keystone'
 876                             - 'neutron'
 877                             - 'nova'
 878                             - 'quotas'
 879                             - 'swift'
 880
 881             -
 882                 case_name: rally_jobs_cntt
 883                 project_name: functest
 884                 criteria: 100
 885                 blocking: false
 886                 description: >-
 887                     This test case runs a group of Rally jobs used in
 888                     OpenStack gating
 889                 run:
 890                     name: rally_jobs
 891                     args:
 892                         tests:
 893                             - 'neutron'
 894
 895     -
 896         name: vnf
 897         order: 5
 898         description: >-
 899             Collection of VNF test cases.
 900         testcases:
 901             -
 902                 case_name: cloudify
 903                 project_name: functest
 904                 criteria: 100
 905                 blocking: false
 906                 description: >-
 907                     This test case deploys the Cloudify orchestrator.
 908                 run:
 909                     name: cloudify
 910             -
 911                 case_name: cloudify_ims
 912                 project_name: functest
 913                 criteria: 100
 914                 blocking: false
 915                 description: >-
 916                     This test case deploys an OpenSource vIMS solution from
 917                     Clearwater using the Cloudify orchestrator. It also runs
 918                     some signaling traffic.
 919                 dependencies:
 920                     - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha'
 921                 run:
 922                     name: cloudify_ims
 923
 924             -
 925                 case_name: heat_ims
 926                 project_name: functest
 927                 criteria: 100
 928                 blocking: false
 929                 description: >-
 930                     This test case deploys an OpenSource vIMS solution from
 931                     Clearwater using the OpenStack Heat orchestrator.
 932                     It also runs some signaling traffic.
 933                 run:
 934                     name: heat_ims
 935
 936             -
 937                 case_name: vyos_vrouter
 938                 project_name: functest
 939                 criteria: 100
 940                 blocking: false
 941                 description: >-
 942                     This test case is vRouter testing.
 943                 run:
 944                     name: vyos_vrouter
 945
 946             -
 947                 case_name: juju_epc
 948                 project_name: functest
 949                 criteria: 100
 950                 blocking: false
 951                 description: >-
 952                     vEPC validation with Juju as VNF manager and ABoT as test
 953                     executor.
 954                 run:
 955                     name: juju_epc