functest/ci/testcases.yaml

   1 ---
   2 tiers:
   3     -
   4         name: healthcheck
   5         description: >-
   6             First tier to be executed to verify the basic
   7             operations in the VIM.
   8         testcases:
   9             -
  10                 case_name: connection_check
  11                 project_name: functest
  12                 criteria: 100
  13                 blocking: true
  14                 description: >-
  15                     This test case verifies the retrieval of OpenStack clients:
  16                     Keystone, Glance, Neutron and Nova and may perform some
  17                     simple queries.
  18                 run:
  19                     name: connection_check
  20
  21             -
  22                 case_name: tenantnetwork1
  23                 project_name: functest
  24                 criteria: 100
  25                 blocking: true
  26                 description: >-
  27                     It creates and configures all tenant network ressources
  28                     required by advanced testcases (subnet, network and
  29                     router).
  30                 dependencies:
  31                     - NO_TENANT_NETWORK: '^(?![tT]rue$)'
  32                 run:
  33                     name: tenantnetwork1
  34
  35             -
  36                 case_name: tenantnetwork2
  37                 project_name: functest
  38                 criteria: 100
  39                 blocking: true
  40                 description: >-
  41                     It creates new user/project before creating and configuring
  42                     all tenant network ressources required by a testcase
  43                     (subnet, network and router).
  44                 dependencies:
  45                     - NO_TENANT_NETWORK: '^(?![tT]rue$)'
  46                 run:
  47                     name: tenantnetwork2
  48
  49             -
  50                 case_name: vmready1
  51                 project_name: functest
  52                 criteria: 100
  53                 blocking: true
  54                 description: >-
  55                     It inherits from TenantNetwork1 which creates all network
  56                     resources and prepares a future VM attached to that
  57                     network.
  58                 run:
  59                     name: vmready1
  60
  61             -
  62                 case_name: vmready2
  63                 project_name: functest
  64                 criteria: 100
  65                 blocking: true
  66                 description: >-
  67                     It creates new user/project before creating and configuring
  68                     all tenant network ressources, flavors, images, etc.
  69                     required by advanced testcases.
  70                 run:
  71                     name: vmready2
  72
  73             -
  74                 case_name: singlevm1
  75                 project_name: functest
  76                 criteria: 100
  77                 blocking: true
  78                 description: >-
  79                     It inherits from TenantNetwork1 which creates all network
  80                     resources and completes it by booting a VM attached to that
  81                     network.
  82                 run:
  83                     name: singlevm1
  84
  85             -
  86                 case_name: singlevm2
  87                 project_name: functest
  88                 criteria: 100
  89                 blocking: true
  90                 description: >-
  91                     It creates new user/project before creating and configuring
  92                     all tenant network ressources and vms required by advanced
  93                     testcases.
  94                 run:
  95                     name: singlevm2
  96
  97             -
  98                 case_name: vping_ssh
  99                 project_name: functest
 100                 criteria: 100
 101                 blocking: true
 102                 description: >-
 103                     This test case verifies: 1) SSH to an instance using
 104                     floating IPs over the public network. 2) Connectivity
 105                     between 2 instances over a private network.
 106                 run:
 107                     name: vping_ssh
 108
 109             -
 110                 case_name: vping_userdata
 111                 project_name: functest
 112                 criteria: 100
 113                 blocking: true
 114                 description: >-
 115                     This test case verifies:  1) Boot a VM with given userdata.
 116                     2) Connectivity between 2 instances over a private network.
 117                 run:
 118                     name: vping_userdata
 119
 120             -
 121                 case_name: cinder_test
 122                 project_name: functest
 123                 criteria: 100
 124                 blocking: true
 125                 description: >-
 126                     This test case verifies: 1) Attach volume and to 1
 127                     instance; 2) Write data on volume 3) Detach volume
 128                     from instance 1, attach it on instance 2 3) Read volume
 129                     data
 130                 run:
 131                     name: cinder_test
 132
 133             -
 134                 case_name: odl
 135                 project_name: functest
 136                 criteria: 100
 137                 blocking: true
 138                 description: >-
 139                     Test Suite for the OpenDaylight SDN Controller. It
 140                     integrates some test suites from upstream using
 141                     Robot as the test framework.
 142                 dependencies:
 143                     - DEPLOY_SCENARIO: 'odl'
 144                 run:
 145                     name: odl
 146                     args:
 147                         suites:
 148                             - /src/odl_test/csit/suites/integration/basic
 149                             - /src/odl_test/csit/suites/openstack/neutron
 150
 151             -
 152                 case_name: tempest_smoke
 153                 project_name: functest
 154                 criteria: 100
 155                 blocking: false
 156                 description: >-
 157                     This test case runs the smoke subset of the OpenStack
 158                     Tempest suite. The list of test cases is generated by
 159                     Tempest automatically and depends on the parameters of
 160                     the OpenStack deplopyment.
 161                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L114
 162                 run:
 163                     name: tempest_common
 164                     args:
 165                         mode: '(?=.*\[.*\bsmoke\b.*\])(^tempest\.api)'
 166                         option:
 167                             - '--concurrency=4'
 168
 169             -
 170                 case_name: tempest_horizon
 171                 project_name: functest
 172                 criteria: 100
 173                 blocking: false
 174                 description: >-
 175                     This test case runs the Tempest suite proposed by the
 176                     Horizon project.
 177                 dependencies:
 178                     - DASHBOARD_URL: '^(?!\s*$).+'
 179                 run:
 180                     name: tempest_horizon
 181                     args:
 182                         mode: '^tempest.scenario.test_dashboard_basic_ops.'
 183
 184     -
 185         name: smoke
 186         description: >-
 187             Set of basic Functional tests to validate the OPNFV scenarios.
 188         testcases:
 189             -
 190                 case_name: tempest_neutron
 191                 project_name: functest
 192                 criteria: 100
 193                 blocking: false
 194                 description: >-
 195                     This test case runs the Tempest suite proposed by the
 196                     Neutron project. The list of test cases is generated by
 197                     Tempest automatically and depends on the parameters of
 198                     the OpenStack deployment.
 199                 run:
 200                     name: tempest_common
 201                     args:
 202                         mode: '^neutron_tempest_plugin\.api'
 203                         option:
 204                             - '--concurrency=4'
 205
 206             -
 207                 case_name: tempest_cinder
 208                 project_name: functest
 209                 criteria: 100
 210                 blocking: false
 211                 description: >-
 212                     This test case runs the Tempest suite proposed by the
 213                     Cinder project.
 214                 run:
 215                     name: tempest_common
 216                     args:
 217                         mode: "(?!.*test_incremental_backup)\
 218                             (?!.*test_backup_crossproject_admin_negative)\
 219                             (?!.*test_backup_crossproject_user_negative)\
 220                             (^cinder_tempest_plugin.)"
 221                         option:
 222                             - '--concurrency=4'
 223
 224             -
 225                 case_name: tempest_keystone
 226                 project_name: functest
 227                 criteria: 100
 228                 blocking: false
 229                 description: >-
 230                     This test case runs the Tempest suite proposed by the
 231                     Keystone project.
 232                 run:
 233                     name: tempest_common
 234                     args:
 235                         mode: 'keystone_tempest_plugin.'
 236                         option:
 237                             - '--concurrency=4'
 238
 239             -
 240                 case_name: tempest_heat
 241                 project_name: functest
 242                 criteria: 100
 243                 blocking: false
 244                 description: >-
 245                     This test case runs the Tempest suite proposed by the
 246                     Heat project.
 247                 run:
 248                     name: tempest_heat
 249                     args:
 250                         mode: '^heat_tempest_plugin.tests'
 251                         option:
 252                             - '--concurrency=1'
 253
 254             -
 255                 case_name: tempest_telemetry
 256                 project_name: functest
 257                 criteria: 100
 258                 blocking: false
 259                 description: >-
 260                     This test case runs the Tempest suite proposed by the
 261                     Telemetry project.
 262                 run:
 263                     name: tempest_common
 264                     args:
 265                         mode: "(?!.*test_autoscaling)(?!.*test_live)\
 266                             (^telemetry_tempest_plugin)"
 267                         services:
 268                             - aodh
 269                             - gnocchi
 270                             - panko
 271                         option:
 272                             - '--concurrency=4'
 273
 274             -
 275                 case_name: rally_sanity
 276                 project_name: functest
 277                 criteria: 100
 278                 blocking: false
 279                 description: >-
 280                     This test case runs a sub group of tests of the OpenStack
 281                     Rally suite in smoke mode.
 282                 run:
 283                     name: rally_sanity
 284                     args:
 285                         optional:
 286                             - 'gnocchi'
 287                             - 'barbican'
 288
 289             -
 290                 case_name: refstack_compute
 291                 project_name: functest
 292                 enabled: false
 293                 criteria: 100
 294                 blocking: false
 295                 deny_skipping: true
 296                 tests_count: 222
 297                 description: >-
 298                     This test case runs a sub group of tests of the OpenStack
 299                     Compute testcases.
 300                 dependencies:
 301                     - NO_TENANT_NETWORK: '^(?![tT]rue$)'
 302                 run:
 303                     name: refstack
 304                     args:
 305                         target: compute
 306                         option:
 307                             - '--concurrency=4'
 308
 309             -
 310                 case_name: refstack_object
 311                 project_name: functest
 312                 criteria: 100
 313                 blocking: false
 314                 deny_skipping: true
 315                 tests_count: 58
 316                 description: >-
 317                     This test case runs a sub group of tests of the OpenStack
 318                     Object testcases.
 319                 run:
 320                     name: refstack
 321                     args:
 322                         target: object
 323                         option:
 324                             - '--concurrency=4'
 325
 326             -
 327                 case_name: refstack_platform
 328                 project_name: functest
 329                 enabled: false
 330                 criteria: 100
 331                 blocking: false
 332                 deny_skipping: true
 333                 tests_count: 278
 334                 description: >-
 335                     This test case runs a sub group of tests of the OpenStack
 336                     Platform testcases.
 337                 dependencies:
 338                     - NO_TENANT_NETWORK: '^(?![tT]rue$)'
 339                 run:
 340                     name: refstack
 341                     args:
 342                         target: platform
 343                         option:
 344                             - '--concurrency=4'
 345
 346             -
 347                 case_name: tempest_full
 348                 project_name: functest
 349                 criteria: 100
 350                 blocking: false
 351                 description: >-
 352                     The list of test cases is generated by
 353                     Tempest automatically and depends on the parameters of
 354                     the OpenStack deployment.
 355                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83
 356                 run:
 357                     name: tempest_common
 358                     args:
 359                         mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
 360                         option:
 361                             - '--concurrency=4'
 362
 363             -
 364                 case_name: tempest_scenario
 365                 project_name: functest
 366                 criteria: 100
 367                 blocking: false
 368                 description: >-
 369                     The list of test cases is generated by
 370                     Tempest automatically and depends on the parameters of
 371                     the OpenStack deployment.
 372                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
 373                 run:
 374                     name: tempest_common
 375                     args:
 376                         mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)'
 377                         option:
 378                             - '--concurrency=1'
 379
 380             -
 381                 case_name: tempest_slow
 382                 project_name: functest
 383                 criteria: 100
 384                 blocking: false
 385                 description: >-
 386                     The list of test cases is generated by
 387                     Tempest automatically and depends on the parameters of
 388                     the OpenStack deployment.
 389                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
 390                 run:
 391                     name: tempest_common
 392                     args:
 393                         mode: '(?=.*\[.*\bslow\b.*\])(^tempest\.)'
 394                         option:
 395                             - '--concurrency=1'
 396
 397             -
 398                 case_name: patrole_admin
 399                 project_name: functest
 400                 criteria: 100
 401                 blocking: false
 402                 description: >-
 403                     Test suite from Patrole project.
 404                 run:
 405                     name: patrole
 406                     args:
 407                         roles: 'admin'
 408                         mode: "(?!.*test_unmanage_snapshot_rbac)\
 409                           (?!.*test_show_auto_allocated_topology)\
 410                           (?!.*test_delete_auto_allocated_topology)\
 411                           (?!.*test_create_network_provider_segmentation_id)\
 412                           (?!.*compute.test_floating_ips_rbac)\
 413                           (?!.*test_reset_network)\
 414                           (?!.*test_create_image_from_volume_backed_server)\
 415                           (?=.*[.*\bslow\b.*])\
 416                           (^patrole_tempest_plugin.tests.api.(compute|image|network|volume))"
 417                         option:
 418                             - '--concurrency=4'
 419
 420             -
 421                 case_name: patrole_member
 422                 project_name: functest
 423                 criteria: 100
 424                 blocking: false
 425                 description: >-
 426                     Test suite from Patrole project.
 427                 run:
 428                     name: patrole
 429                     args:
 430                         roles: 'member'
 431                         mode: "(?!.*test_unmanage_snapshot_rbac)\
 432                           (?!.*test_show_auto_allocated_topology)\
 433                           (?!.*test_delete_auto_allocated_topology)\
 434                           (?!.*test_create_network_provider_segmentation_id)\
 435                           (?!.*compute.test_floating_ips_rbac)\
 436                           (?!.*test_reset_network)\
 437                           (?!.*test_create_image_from_volume_backed_server)\
 438                           (?=.*[.*\bslow\b.*])\
 439                           (^patrole_tempest_plugin.tests.api.(compute|image|network|volume))"
 440                         option:
 441                             - '--concurrency=4'
 442
 443             -
 444                 case_name: patrole_reader
 445                 project_name: functest
 446                 criteria: 100
 447                 blocking: false
 448                 description: >-
 449                     Test suite from Patrole project.
 450                 run:
 451                     name: patrole
 452                     args:
 453                         roles: 'reader'
 454                         mode: "(?!.*test_unmanage_snapshot_rbac)\
 455                           (?!.*test_show_auto_allocated_topology)\
 456                           (?!.*test_delete_auto_allocated_topology)\
 457                           (?!.*test_create_network_provider_segmentation_id)\
 458                           (?!.*compute.test_floating_ips_rbac)\
 459                           (?!.*test_reset_network)\
 460                           (?!.*test_create_image_from_volume_backed_server)\
 461                           (?=.*[.*\bslow\b.*])\
 462                           (^patrole_tempest_plugin.tests.api.(compute|image|network|volume))"
 463                         option:
 464                             - '--concurrency=4'
 465
 466             -
 467                 case_name: tempest_barbican
 468                 project_name: functest
 469                 criteria: 100
 470                 blocking: false
 471                 description: >-
 472                     It leverages on the tempest plugin containing tests used to
 473                     verify the functionality of a barbican installation.
 474                 run:
 475                     name: barbican
 476                     args:
 477                         mode:
 478                             '^barbican_tempest_plugin.((?!test_signed_image_upload_boot_failure).)*$'
 479                         services:
 480                             - barbican
 481                         option:
 482                             - '--concurrency=4'
 483
 484             -
 485                 case_name: tempest_octavia
 486                 project_name: functest
 487                 criteria: 100
 488                 blocking: false
 489                 description: >-
 490                     It leverages on the tempest plugin containing tests used to
 491                     verify the functionality of an octavia installation.
 492                 run:
 493                     name: tempest_common
 494                     args:
 495                         mode: "(?!.*api.v2.test_availability_zone)\
 496                             (?!.*api.v2.test_availability_zone_profile)\
 497                             (?!.*api.v2.test_member.MemberAPITest.test_member_ipv4_create)\
 498                             (?!.*api.v2.test_member.MemberAPITest.test_member_ipv6_create)\
 499                             (^octavia_tempest_plugin.tests.(api|scenario))"
 500                         services:
 501                             - octavia
 502                         option:
 503                             - '--concurrency=2'
 504
 505             -
 506                 case_name: tempest_cyborg
 507                 project_name: functest
 508                 criteria: 100
 509                 blocking: false
 510                 description: >-
 511                     It leverages on the tempest plugin containing tests used to
 512                     verify the functionality of a cyborg installation.
 513                 run:
 514                     name: tempest_common
 515                     args:
 516                         mode: '^cyborg_tempest_plugin'
 517                         services:
 518                             - cyborg
 519                         option:
 520                             - '--concurrency=4'
 521
 522     -
 523         name: smoke_cntt
 524         description: >-
 525             Set of basic Functional tests to validate the OPNFV scenarios.
 526         testcases:
 527             -
 528                 case_name: tempest_neutron_cntt
 529                 project_name: functest
 530                 criteria: 100
 531                 blocking: false
 532                 deny_skipping: true
 533                 tests_count: 545
 534                 description: >-
 535                     This test case runs the Tempest suite proposed by the
 536                     Neutron project. The list of test cases is generated by
 537                     Tempest automatically and depends on the parameters of
 538                     the OpenStack deployment.
 539                 run:
 540                     name: tempest_common
 541                     args:
 542                         mode: "(?!.*admin.test_agent_availability_zone)\
 543                             (?!.*admin.test_dhcp_agent_scheduler)\
 544                             (?!.*admin.test_l3_agent_scheduler)\
 545                             (?!.*admin.test_logging)\
 546                             (?!.*admin.test_logging_negative)\
 547                             (?!.*admin.test_network_segment_range)\
 548                             (?!.*admin.test_ports.PortTestCasesAdmin.test_regenerate_mac_address)\
 549                             (?!.*admin.test_ports.PortTestCasesResourceRequest)\
 550                             (?!.*admin.test_routers_dvr)\
 551                             (?!.*admin.test_routers_flavors)\
 552                             (?!.*admin.test_routers_ha)\
 553                             (?!.*test_conntrack_helper)\
 554                             (?!.*test_floating_ips.FloatingIPPoolTestJSON)\
 555                             (?!.*test_floating_ips.FloatingIPTestJSON.test_create_update_floatingip_port_details)\
 556                             (?!.*test_metering_extensions)\
 557                             (?!.*test_metering_negative)\
 558                             (?!.*test_networks.NetworksSearchCriteriaTest.test_list_validation_filters)\
 559                             (?!.*test_networks.NetworksTestAdmin.test_create_tenant_network_vxlan)\
 560                             (?!.*test_networks.NetworksTestJSON.test_create_update_network_dns_domain)\
 561                             (?!.*test_port_forwardings)\
 562                             (?!.*test_ports.PortsTaggingOnCreation)\
 563                             (?!.*test_ports.PortsTestJSON.test_create_port_with_propagate_uplink_status)\
 564                             (?!.*test_ports.PortsTestJSON.test_create_port_without_propagate_uplink_status)\
 565                             (?!.*test_ports.PortsTestJSON.test_create_update_port_with_dns_domain)\
 566                             (?!.*test_ports.PortsTestJSON.test_create_update_port_with_dns_name)\
 567                             (?!.*test_ports.PortsTestJSON.test_create_update_port_with_no_dns_name)\
 568                             (?!.*test_qos.QosMinimumBandwidthRuleTestJSON)\
 569                             (?!.*test_revisions.TestRevisions.test_update_dns_domain_bumps_revision)\
 570                             (?!.*test_revisions.TestRevisions.test_update_router_extra_attributes_bumps_revision)\
 571                             (?!.*test_router_interface_fip)\
 572                             (?!.*test_routers.DvrRoutersTest)\
 573                             (?!.*test_routers.HaRoutersTest)\
 574                             (?!.*test_routers.RoutersIpV6Test.test_extra_routes_atomic)\
 575                             (?!.*test_routers.RoutersTest.test_extra_routes_atomic)\
 576                             (?!.*test_routers_negative.DvrRoutersNegativeTest)\
 577                             (?!.*test_routers_negative.DvrRoutersNegativeTestExtended)\
 578                             (?!.*test_routers_negative.HaRoutersNegativeTest)\
 579                             (?!.*test_security_groups.RbacSharedSecurityGroupTest)\
 580                             (?!.*test_subnetpool_prefix_ops)\
 581                             (?!.*test_subnetpools.RbacSubnetPoolTest)\
 582                             (?!.*test_subnetpools.SubnetPoolsSearchCriteriaTest.test_list_validation_filters)\
 583                             (?!.*test_subnetpools_negative.SubnetPoolsNegativeTestJSON.test_tenant_create_subnetpool_associate_shared_address_scope)\
 584                             (?!.*test_subnets.SubnetsSearchCriteriaTest.test_list_validation_filters)\
 585                             (?!.*test_timestamp.TestTimeStamp.test_segment_with_timestamp)\
 586                             (?!.*test_trunk.TrunkTestInheritJSONBase.test_add_subport)\
 587                             (?!.*test_trunk.TrunkTestMtusJSON)\
 588                             (?!.*test_trunk_negative.TrunkTestJSON.test_create_subport_invalid_inherit_network_segmentation_type)\
 589                             (?!.*test_trunk_negative.TrunkTestMtusJSON)\
 590                             (^neutron_tempest_plugin.api)"
 591                         option:
 592                             - '--concurrency=4'
 593
 594             -
 595                 case_name: tempest_cinder_cntt
 596                 project_name: functest
 597                 criteria: 100
 598                 blocking: false
 599                 deny_skipping: true
 600                 tests_count: 10
 601                 description: >-
 602                     This test case runs the Tempest suite proposed by the
 603                     Cinder project.
 604                 run:
 605                     name: tempest_common
 606                     args:
 607                         mode: "(?!.*test_incremental_backup)\
 608                             (?!.*test_consistencygroups)\
 609                             (?!.*test_backup_crossproject_admin_negative)\
 610                             (?!.*test_backup_crossproject_user_negative)\
 611                             (?!.*test_volume_encrypted.TestEncryptedCinderVolumes)\
 612                             (^cinder_tempest_plugin.)"
 613                         option:
 614                             - '--concurrency=4'
 615
 616             -
 617                 case_name: tempest_keystone_cntt
 618                 project_name: functest
 619                 criteria: 100
 620                 blocking: false
 621                 deny_skipping: true
 622                 tests_count: 27
 623                 description: >-
 624                     This test case runs the Tempest suite proposed by the
 625                     Keystone project.
 626                 run:
 627                     name: tempest_common
 628                     args:
 629                         mode: "(?!.*api.identity.v3.test_oauth1_tokens)\
 630                             (?!.*scenario.test_federated_authentication)\
 631                             keystone_tempest_plugin."
 632                         option:
 633                             - '--concurrency=4'
 634
 635             -
 636                 case_name: tempest_heat_cntt
 637                 project_name: functest
 638                 criteria: 100
 639                 blocking: false
 640                 deny_skipping: true
 641                 tests_count: 125
 642                 description: >-
 643                     This test case runs the Tempest suite proposed by the
 644                     Heat project.
 645                 run:
 646                     name: tempest_heat
 647                     args:
 648                         mode: "(?!.*functional.test_lbaasv2)\
 649                           (?!.*functional.test_encryption_vol_type)\
 650                           (?!.*RemoteStackTest.test_stack_create_with_cloud_credential)\
 651                           (?!.*scenario.test_aodh_alarm)\
 652                           (?!.*tests.scenario.test_autoscaling_lb)\
 653                           (?!.*scenario.test_autoscaling_lbv2)\
 654                           (?!.*scenario.test_server_software_config)\
 655                           (?!.*test_volumes.VolumeBackupRestoreIntegrationTest)\
 656                           (?!.*scenario.test_octavia_lbaas)\
 657                           (?!.*scenario.test_server_cfn_init)\
 658                           ^heat_tempest_plugin.tests"
 659                         option:
 660                             - '--concurrency=1'
 661
 662             -
 663                 case_name: rally_sanity_cntt
 664                 project_name: functest
 665                 criteria: 100
 666                 blocking: false
 667                 description: >-
 668                     This test case runs a sub group of tests of the OpenStack
 669                     Rally suite in smoke mode.
 670                 run:
 671                     name: rally_sanity
 672                     args:
 673                         tests:
 674                             - 'authenticate'
 675                             - 'glance'
 676                             - 'cinder'
 677                             - 'heat'
 678                             - 'keystone'
 679                             - 'neutron'
 680                             - 'nova'
 681                             - 'quotas'
 682                             - 'swift'
 683
 684             -
 685                 case_name: tempest_full_cntt
 686                 project_name: functest
 687                 criteria: 100
 688                 blocking: false
 689                 deny_skipping: true
 690                 tests_count: 1260
 691                 description: >-
 692                     The list of test cases is generated by
 693                     Tempest automatically and depends on the parameters of
 694                     the OpenStack deployment.
 695                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83
 696                 run:
 697                     name: tempest_common
 698                     args:
 699                         mode: "(?!.*admin.test_agents)(?!.*test_fixed_ips)\
 700                           (?!.*test_fixed_ips_negative)\
 701                           (?!.*test_auto_allocate_network)(?!.*test_floating_ips_bulk)\
 702                           (?!.*test_flavors_microversions.FlavorsV255TestJSON)\
 703                           (?!.*test_flavors_microversions.FlavorsV261TestJSON)\
 704                           (?!.*test_live_migration.LiveAutoBlockMigrationV225Test.test_iscsi_volume)\
 705                           (?!.*test_live_migration.LiveAutoBlockMigrationV225Test.test_live_block_migration)\
 706                           (?!.*test_live_migration.LiveAutoBlockMigrationV225Test.test_live_block_migration_paused)\
 707                           (?!.*test_live_migration.LiveAutoBlockMigrationV225Test.test_volume_backed_live_migration)\
 708                           (?!.*test_live_migration.LiveMigrationTest.test_iscsi_volume)\
 709                           (?!.*test_live_migration.LiveMigrationTest.test_live_block_migration)\
 710                           (?!.*test_live_migration.LiveMigrationTest.test_live_block_migration_paused)\
 711                           (?!.*test_live_migration.LiveMigrationTest.test_volume_backed_live_migration)\
 712                           (?!.*test_live_migration.LiveMigrationRemoteConsolesV26Test)\
 713                           (?!.*test_quotas.QuotasAdminTestV257)\
 714                           (?!.*test_servers.ServersAdminTestJSON.test_reset_network_inject_network_info)\
 715                           (?!.*certificates.test_certificates)\
 716                           (?!.*test_quotas_negative.QuotasSecurityGroupAdminNegativeTest)\
 717                           (?!.*test_novnc)(?!.*test_server_personality)\
 718                           (?!.*test_servers.ServerShowV263Test.test_show_update_rebuild_list_server)\
 719                           (?!.*test_servers_microversions.ServerShowV254Test)\
 720                           (?!.*test_servers_microversions.ServerShowV257Test)\
 721                           (?!.*test_servers_negative.ServersNegativeTestJSON.test_personality_file_contents_not_encoded)\
 722                           (?!.*servers.test_virtual_interfaces)\
 723                           (?!.*test_server_actions.ServerActionsTestJSON.test_change_server_password)\
 724                           (?!.*test_server_actions.ServerActionsTestJSON.test_get_vnc_console)\
 725                           (?!.*test_server_actions.ServerActionsTestJSON.test_reboot_server_soft)\
 726                           (?!.*test_server_rescue.ServerBootFromVolumeStableRescueTest)\
 727                           (?!.*test_server_rescue.ServerStableDeviceRescueTest)\
 728                           (?!.*test_security_group_default_rules)\
 729                           (?!.*test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_security_group_create_with_duplicate_name)\
 730                           (?!.*test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_security_group_create_with_invalid_group_description)\
 731                           (?!.*test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_security_group_create_with_invalid_group_name)\
 732                           (?!.*test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_update_security_group_with_invalid_sg_des)\
 733                           (?!.*test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_update_security_group_with_invalid_sg_id)\
 734                           (?!.*test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_update_security_group_with_invalid_sg_name)\
 735                           (?!.*test_server_metadata.ServerMetadataTestJSON)\
 736                           (?!.*test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_delete_metadata_non_existent_server)\
 737                           (?!.*test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_metadata_items_limit)\
 738                           (?!.*test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_set_metadata_invalid_key)\
 739                           (?!.*test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_set_metadata_non_existent_server)\
 740                           (?!.*test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_set_server_metadata_blank_key)\
 741                           (?!.*test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_set_server_metadata_missing_metadata)\
 742                           (?!.*test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_update_metadata_non_existent_server)\
 743                           (?!.*test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_update_metadata_with_blank_key)\
 744                           (?!.*test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filtered_by_ip_regex)\
 745                           (?!.*compute.test_virtual_interfaces)(?!.*compute.test_virtual_interfaces_negative)\
 746                           (?!.*compute.test_networks)\
 747                           (?!.*test_attach_volume.AttachVolumeMultiAttach)\
 748                           (?!.*identity.admin.v2)(?!.*identity.v2)\
 749                           (?!.*identity.v3.test_access_rules)\
 750                           (?!.*identity.v3.test_application_credentials.ApplicationCredentialsV3Test.test_create_application_credential_access_rules)\
 751                           (?!.*image.v1)\
 752                           (?!.*image.v2.admin.test_images.ImportCopyImagesTest)\
 753                           (?!.*image.v2.test_images.ImportImagesTest)\
 754                           (?!.*image.v2.test_images.MultiStoresImportImages)\
 755                           (?!.*admin.test_dhcp_agent_scheduler)\
 756                           (?!.*admin.test_routers_dvr)\
 757                           (?!.*test_metering_extensions)(?!.*network.test_tags)\
 758                           (?!.*test_routers_negative.DvrRoutersNegativeTest)\
 759                           (?!.*test_routers.RoutersIpV6Test.test_create_router_set_gateway_with_fixed_ip)\
 760                           (?!.*test_routers.RoutersTest.test_create_router_set_gateway_with_fixed_ip)\
 761                           (?!.*test_encrypted_volumes_extend)\
 762                           (?!.*test_group_snapshots.GroupSnapshotsV319Test.test_reset_group_snapshot_status)\
 763                           (?!.*test_multi_backend)\
 764                           (?!.*test_volume_retype.VolumeRetypeWithMigrationTest)\
 765                           (?!.*test_volume_delete_cascade.VolumesDeleteCascade.test_volume_from_snapshot_cascade_delete)\
 766                           (?!.*test_volumes_backup.VolumesBackupsTest.test_volume_backup_create_get_detailed_list_restore_delete)\
 767                           (?!.*test_volumes_negative.UpdateMultiattachVolumeNegativeTest.test_multiattach_rw_volume_update_failure)\
 768                           (?!.*test_volumes_extend.VolumesExtendAttachedTest.test_extend_attached_volume)\
 769                           (?!.*\\[.*\\bslow\\b.*\\])(^tempest.api)"
 770                         option:
 771                             - '--concurrency=4'
 772
 773             -
 774                 case_name: tempest_scenario_cntt
 775                 project_name: functest
 776                 criteria: 100
 777                 blocking: false
 778                 deny_skipping: true
 779                 tests_count: 9
 780                 description: >-
 781                     The list of test cases is generated by
 782                     Tempest automatically and depends on the parameters of
 783                     the OpenStack deployment.
 784                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
 785                 run:
 786                     name: tempest_scenario
 787                     args:
 788                         mode: "\
 789                           (?!.*test_minbw_allocation_placement)\
 790                           (?!.*test_volume_boot_pattern.TestVolumeBootPattern.test_boot_server_from_encrypted_volume_luks)\
 791                           (?!.*\\[.*\\bslow\\b.*\\])(^tempest.scenario)"
 792                         option:
 793                             - '--concurrency=1'
 794
 795             -
 796                 case_name: tempest_slow_cntt
 797                 project_name: functest
 798                 criteria: 100
 799                 blocking: false
 800                 deny_skipping: true
 801                 tests_count: 43
 802                 description: >-
 803                     The list of test cases is generated by
 804                     Tempest automatically and depends on the parameters of
 805                     the OpenStack deployment.
 806                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
 807                 run:
 808                     name: tempest_common
 809                     args:
 810                         mode: "(?!.*test_volume_swap)\
 811                           (?!.*test_server_personality)\
 812                           (?!.*test_server_rescue.ServerBootFromVolumeStableRescueTest)\
 813                           (?!.*test_container_sync.ContainerSyncTest.test_container_synchronization)\
 814                           (?!.*test_container_sync_middleware.ContainerSyncMiddlewareTest.test_container_synchronization)\
 815                           (?!.*test_encrypted_cinder_volumes)\
 816                           (?!.*test_minbw_allocation_placement)\
 817                           (?!.*test_network_basic_ops.TestNetworkBasicOps.test_router_rescheduling)\
 818                           (?!.*test_shelve_instance.TestShelveInstance.test_cold_migrate_unshelved_instance)\
 819                           (?!.*test_volume_migrate_attached)\
 820                           (?!.*test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_cold_migration_revert)\
 821                           (?=.*\\[.*\\bslow\\b.*\\])(^tempest.)"
 822                         option:
 823                             - '--concurrency=1'
 824
 825     -
 826         name: benchmarking
 827         description: >-
 828             Run several OpenStack performance tools
 829             https://docs.openstack.org/performance-docs/latest/methodologies/tools.html
 830         testcases:
 831             -
 832                 case_name: rally_full
 833                 project_name: functest
 834                 criteria: 100
 835                 blocking: false
 836                 description: >-
 837                     This test case runs the full suite of scenarios of the
 838                     OpenStack Rally suite using several threads and iterations.
 839                 run:
 840                     name: rally_full
 841                     args:
 842                         optional:
 843                             - 'gnocchi'
 844                             - 'barbican'
 845
 846             -
 847                 case_name: rally_jobs
 848                 project_name: functest
 849                 criteria: 100
 850                 blocking: false
 851                 description: >-
 852                     This test case runs a group of Rally jobs used in
 853                     OpenStack gating
 854                 dependencies:
 855                     - NO_TENANT_NETWORK: '^(?![tT]rue$)'
 856                 run:
 857                     name: rally_jobs
 858                     args:
 859                         optional:
 860                             - 'gnocchi'
 861
 862             -
 863                 case_name: vmtp
 864                 project_name: functest
 865                 criteria: 100
 866                 blocking: false
 867                 description: >-
 868                     VMTP is a small python application that will automatically
 869                     perform ping connectivity, round trip time measurement
 870                     (latency) and TCP/UDP throughput
 871                 dependencies:
 872                     - NO_TENANT_NETWORK: '^(?![tT]rue$)'
 873                 run:
 874                     name: vmtp
 875
 876             -
 877                 case_name: shaker
 878                 project_name: functest
 879                 criteria: 100
 880                 blocking: false
 881                 description: >-
 882                     Shaker wraps around popular system network testing tools
 883                     like iperf, iperf3 and netperf (with help of flent). Shaker
 884                     is able to deploy OpenStack instances and networks in
 885                     different topologies.
 886                 dependencies:
 887                     - NO_TENANT_NETWORK: '^(?![tT]rue$)'
 888                 run:
 889                     name: shaker
 890
 891     -
 892         name: benchmarking_cntt
 893         description: >-
 894             Run several OpenStack performance tools
 895             https://docs.openstack.org/performance-docs/latest/methodologies/tools.html
 896         testcases:
 897             -
 898                 case_name: rally_full_cntt
 899                 project_name: functest
 900                 criteria: 100
 901                 blocking: false
 902                 description: >-
 903                     This test case runs the full suite of scenarios of the
 904                     OpenStack Rally suite using several threads and iterations.
 905                 run:
 906                     name: rally_full
 907                     args:
 908                         tests:
 909                             - 'authenticate'
 910                             - 'glance'
 911                             - 'cinder'
 912                             - 'heat'
 913                             - 'keystone'
 914                             - 'neutron'
 915                             - 'nova'
 916                             - 'quotas'
 917                             - 'swift'
 918
 919             -
 920                 case_name: rally_jobs_cntt
 921                 project_name: functest
 922                 criteria: 100
 923                 blocking: false
 924                 description: >-
 925                     This test case runs a group of Rally jobs used in
 926                     OpenStack gating
 927                 run:
 928                     name: rally_jobs
 929                     args:
 930                         tests:
 931                             - 'neutron'
 932
 933     -
 934         name: vnf
 935         description: >-
 936             Collection of VNF test cases.
 937         testcases:
 938             -
 939                 case_name: cloudify
 940                 project_name: functest
 941                 criteria: 100
 942                 blocking: false
 943                 description: >-
 944                     This test case deploys the Cloudify orchestrator.
 945                 run:
 946                     name: cloudify
 947             -
 948                 case_name: cloudify_ims
 949                 project_name: functest
 950                 criteria: 100
 951                 blocking: false
 952                 description: >-
 953                     This test case deploys an OpenSource vIMS solution from
 954                     Clearwater using the Cloudify orchestrator. It also runs
 955                     some signaling traffic.
 956                 dependencies:
 957                     - NO_TENANT_NETWORK: '^(?![tT]rue$)'
 958                 run:
 959                     name: cloudify_ims
 960
 961             -
 962                 case_name: heat_ims
 963                 project_name: functest
 964                 criteria: 100
 965                 blocking: false
 966                 description: >-
 967                     This test case deploys an OpenSource vIMS solution from
 968                     Clearwater using the OpenStack Heat orchestrator.
 969                     It also runs some signaling traffic.
 970                 dependencies:
 971                     - NO_TENANT_NETWORK: '^(?![tT]rue$)'
 972                 run:
 973                     name: heat_ims
 974
 975             -
 976                 case_name: vyos_vrouter
 977                 project_name: functest
 978                 criteria: 100
 979                 blocking: false
 980                 description: >-
 981                     This test case is vRouter testing.
 982                 dependencies:
 983                     - NO_TENANT_NETWORK: '^(?![tT]rue$)'
 984                 run:
 985                     name: vyos_vrouter
 986
 987             -
 988                 case_name: juju_epc
 989                 project_name: functest
 990                 criteria: 100
 991                 blocking: false
 992                 description: >-
 993                     vEPC validation with Juju as VNF manager and ABoT as test
 994                     executor.
 995                 dependencies:
 996                     - NO_TENANT_NETWORK: '^(?![tT]rue$)'
 997                 run:
 998                     name: juju_epc