functest/ci/testcases.yaml

   1 ---
   2 tiers:
   3     -
   4         name: healthcheck
   5         description: >-
   6             First tier to be executed to verify the basic
   7             operations in the VIM.
   8         testcases:
   9             -
  10                 case_name: connection_check
  11                 project_name: functest
  12                 criteria: 100
  13                 blocking: true
  14                 description: >-
  15                     This test case verifies the retrieval of OpenStack clients:
  16                     Keystone, Glance, Neutron and Nova and may perform some
  17                     simple queries.
  18                 run:
  19                     name: connection_check
  20
  21             -
  22                 case_name: tenantnetwork1
  23                 project_name: functest
  24                 criteria: 100
  25                 blocking: true
  26                 description: >-
  27                     It creates and configures all tenant network ressources
  28                     required by advanced testcases (subnet, network and
  29                     router).
  30                 dependencies:
  31                     - NO_TENANT_NETWORK: '^(?![tT]rue$)'
  32                 run:
  33                     name: tenantnetwork1
  34
  35             -
  36                 case_name: tenantnetwork2
  37                 project_name: functest
  38                 criteria: 100
  39                 blocking: true
  40                 description: >-
  41                     It creates new user/project before creating and configuring
  42                     all tenant network ressources required by a testcase
  43                     (subnet, network and router).
  44                 dependencies:
  45                     - NO_TENANT_NETWORK: '^(?![tT]rue$)'
  46                 run:
  47                     name: tenantnetwork2
  48
  49             -
  50                 case_name: vmready1
  51                 project_name: functest
  52                 criteria: 100
  53                 blocking: true
  54                 description: >-
  55                     It inherits from TenantNetwork1 which creates all network
  56                     resources and prepares a future VM attached to that
  57                     network.
  58                 run:
  59                     name: vmready1
  60
  61             -
  62                 case_name: vmready2
  63                 project_name: functest
  64                 criteria: 100
  65                 blocking: true
  66                 description: >-
  67                     It creates new user/project before creating and configuring
  68                     all tenant network ressources, flavors, images, etc.
  69                     required by advanced testcases.
  70                 run:
  71                     name: vmready2
  72
  73             -
  74                 case_name: singlevm1
  75                 project_name: functest
  76                 criteria: 100
  77                 blocking: true
  78                 description: >-
  79                     It inherits from TenantNetwork1 which creates all network
  80                     resources and completes it by booting a VM attached to that
  81                     network.
  82                 run:
  83                     name: singlevm1
  84
  85             -
  86                 case_name: singlevm2
  87                 project_name: functest
  88                 criteria: 100
  89                 blocking: true
  90                 description: >-
  91                     It creates new user/project before creating and configuring
  92                     all tenant network ressources and vms required by advanced
  93                     testcases.
  94                 run:
  95                     name: singlevm2
  96
  97             -
  98                 case_name: vping_ssh
  99                 project_name: functest
 100                 criteria: 100
 101                 blocking: true
 102                 description: >-
 103                     This test case verifies: 1) SSH to an instance using
 104                     floating IPs over the public network. 2) Connectivity
 105                     between 2 instances over a private network.
 106                 run:
 107                     name: vping_ssh
 108
 109             -
 110                 case_name: vping_userdata
 111                 project_name: functest
 112                 criteria: 100
 113                 blocking: true
 114                 description: >-
 115                     This test case verifies:  1) Boot a VM with given userdata.
 116                     2) Connectivity between 2 instances over a private network.
 117                 run:
 118                     name: vping_userdata
 119
 120             -
 121                 case_name: cinder_test
 122                 project_name: functest
 123                 criteria: 100
 124                 blocking: true
 125                 description: >-
 126                     This test case verifies: 1) Attach volume and to 1
 127                     instance; 2) Write data on volume 3) Detach volume
 128                     from instance 1, attach it on instance 2 3) Read volume
 129                     data
 130                 run:
 131                     name: cinder_test
 132
 133             -
 134                 case_name: odl
 135                 project_name: functest
 136                 criteria: 100
 137                 blocking: true
 138                 description: >-
 139                     Test Suite for the OpenDaylight SDN Controller. It
 140                     integrates some test suites from upstream using
 141                     Robot as the test framework.
 142                 dependencies:
 143                     - DEPLOY_SCENARIO: 'odl'
 144                 run:
 145                     name: odl
 146                     args:
 147                         suites:
 148                             - /src/odl_test/csit/suites/integration/basic
 149                             - /src/odl_test/csit/suites/openstack/neutron
 150
 151             -
 152                 case_name: tempest_smoke
 153                 project_name: functest
 154                 criteria: 100
 155                 blocking: false
 156                 description: >-
 157                     This test case runs the smoke subset of the OpenStack
 158                     Tempest suite. The list of test cases is generated by
 159                     Tempest automatically and depends on the parameters of
 160                     the OpenStack deplopyment.
 161                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L114
 162                 run:
 163                     name: tempest_common
 164                     args:
 165                         mode: '(?=.*\[.*\bsmoke\b.*\])(^tempest\.api)'
 166                         option:
 167                             - '--concurrency=4'
 168
 169             -
 170                 case_name: tempest_horizon
 171                 project_name: functest
 172                 criteria: 100
 173                 blocking: false
 174                 description: >-
 175                     This test case runs the Tempest suite proposed by the
 176                     Horizon project.
 177                 dependencies:
 178                     - DASHBOARD_URL: '^(?!\s*$).+'
 179                 run:
 180                     name: tempest_common
 181                     args:
 182                         mode: '^tempest.scenario.test_dashboard_basic_ops.'
 183
 184     -
 185         name: smoke
 186         description: >-
 187             Set of basic Functional tests to validate the OPNFV scenarios.
 188         testcases:
 189             -
 190                 case_name: tempest_neutron
 191                 project_name: functest
 192                 criteria: 100
 193                 blocking: false
 194                 description: >-
 195                     This test case runs the Tempest suite proposed by the
 196                     Neutron project. The list of test cases is generated by
 197                     Tempest automatically and depends on the parameters of
 198                     the OpenStack deployment.
 199                 run:
 200                     name: tempest_common
 201                     args:
 202                         mode: '^neutron_tempest_plugin\.api'
 203                         option:
 204                             - '--concurrency=4'
 205
 206             -
 207                 case_name: tempest_cinder
 208                 project_name: functest
 209                 criteria: 100
 210                 blocking: false
 211                 description: >-
 212                     This test case runs the Tempest suite proposed by the
 213                     Cinder project.
 214                 run:
 215                     name: tempest_common
 216                     args:
 217                         mode: "(?!.*test_incremental_backup)\
 218                             (?!.*test_backup_crossproject_admin_negative)\
 219                             (?!.*test_backup_crossproject_user_negative)\
 220                             (^cinder_tempest_plugin.)"
 221                         option:
 222                             - '--concurrency=4'
 223
 224             -
 225                 case_name: tempest_keystone
 226                 project_name: functest
 227                 criteria: 100
 228                 blocking: false
 229                 description: >-
 230                     This test case runs the Tempest suite proposed by the
 231                     Keystone project.
 232                 run:
 233                     name: tempest_common
 234                     args:
 235                         mode: 'keystone_tempest_plugin.'
 236                         option:
 237                             - '--concurrency=4'
 238
 239             -
 240                 case_name: tempest_heat
 241                 project_name: functest
 242                 criteria: 100
 243                 blocking: false
 244                 description: >-
 245                     This test case runs the Tempest suite proposed by the
 246                     Heat project.
 247                 run:
 248                     name: tempest_heat
 249                     args:
 250                         mode: '^heat_tempest_plugin.tests'
 251                         option:
 252                             - '--concurrency=1'
 253
 254             -
 255                 case_name: tempest_telemetry
 256                 project_name: functest
 257                 criteria: 100
 258                 blocking: false
 259                 description: >-
 260                     This test case runs the Tempest suite proposed by the
 261                     Telemetry project.
 262                 run:
 263                     name: tempest_common
 264                     args:
 265                         mode: "(?!.*test_autoscaling)(?!.*test_live)\
 266                             (^telemetry_tempest_plugin)"
 267                         services:
 268                             - aodh
 269                             - gnocchi
 270                             - panko
 271                         option:
 272                             - '--concurrency=4'
 273
 274             -
 275                 case_name: rally_sanity
 276                 project_name: functest
 277                 criteria: 100
 278                 blocking: false
 279                 description: >-
 280                     This test case runs a sub group of tests of the OpenStack
 281                     Rally suite in smoke mode.
 282                 run:
 283                     name: rally_sanity
 284                     args:
 285                         optional:
 286                             - 'gnocchi'
 287                             - 'barbican'
 288
 289             -
 290                 case_name: refstack_compute
 291                 project_name: functest
 292                 criteria: 100
 293                 blocking: false
 294                 description: >-
 295                     This test case runs a sub group of tests of the OpenStack
 296                     Compute testcases.
 297                 dependencies:
 298                     - NO_TENANT_NETWORK: '^(?![tT]rue$)'
 299                 run:
 300                     name: refstack
 301                     args:
 302                         target: compute
 303                         option:
 304                             - '--concurrency=4'
 305
 306             -
 307                 case_name: refstack_object
 308                 project_name: functest
 309                 criteria: 100
 310                 blocking: false
 311                 deny_skipping: true
 312                 tests_count: 58
 313                 description: >-
 314                     This test case runs a sub group of tests of the OpenStack
 315                     Object testcases.
 316                 run:
 317                     name: refstack
 318                     args:
 319                         target: object
 320                         option:
 321                             - '--concurrency=4'
 322
 323             -
 324                 case_name: refstack_platform
 325                 project_name: functest
 326                 criteria: 100
 327                 blocking: false
 328                 description: >-
 329                     This test case runs a sub group of tests of the OpenStack
 330                     Platform testcases.
 331                 dependencies:
 332                     - NO_TENANT_NETWORK: '^(?![tT]rue$)'
 333                 run:
 334                     name: refstack
 335                     args:
 336                         target: platform
 337                         option:
 338                             - '--concurrency=4'
 339
 340             -
 341                 case_name: tempest_full
 342                 project_name: functest
 343                 criteria: 100
 344                 blocking: false
 345                 description: >-
 346                     The list of test cases is generated by
 347                     Tempest automatically and depends on the parameters of
 348                     the OpenStack deployment.
 349                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83
 350                 run:
 351                     name: tempest_common
 352                     args:
 353                         mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
 354                         option:
 355                             - '--concurrency=4'
 356
 357             -
 358                 case_name: tempest_scenario
 359                 project_name: functest
 360                 criteria: 100
 361                 blocking: false
 362                 description: >-
 363                     The list of test cases is generated by
 364                     Tempest automatically and depends on the parameters of
 365                     the OpenStack deployment.
 366                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
 367                 run:
 368                     name: tempest_common
 369                     args:
 370                         mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)'
 371                         option:
 372                             - '--concurrency=1'
 373
 374             -
 375                 case_name: tempest_slow
 376                 project_name: functest
 377                 criteria: 100
 378                 blocking: false
 379                 description: >-
 380                     The list of test cases is generated by
 381                     Tempest automatically and depends on the parameters of
 382                     the OpenStack deployment.
 383                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
 384                 run:
 385                     name: tempest_common
 386                     args:
 387                         mode: '(?=.*\[.*\bslow\b.*\])(^tempest\.)'
 388                         option:
 389                             - '--concurrency=1'
 390
 391             -
 392                 case_name: patrole_admin
 393                 project_name: functest
 394                 criteria: 100
 395                 blocking: false
 396                 description: >-
 397                     Test suite from Patrole project.
 398                 run:
 399                     name: patrole
 400                     args:
 401                         roles: 'admin'
 402                         mode: "(?!.*test_unmanage_snapshot_rbac)\
 403                           (?!.*test_show_auto_allocated_topology)\
 404                           (?!.*test_delete_auto_allocated_topology)\
 405                           (?!.*test_create_network_provider_segmentation_id)\
 406                           (?!.*compute.test_floating_ips_rbac)\
 407                           (?!.*test_reset_network)\
 408                           (?!.*test_create_image_from_volume_backed_server)\
 409                           (?!.*test_network_ip_availability_rbac.NetworkIpAvailabilityExtRbacTest.test_get_network_ip_availabilities)\
 410                           (?!.*test_policy_bandwidth_limit_rule_rbac)\
 411                           (?!.*test_policy_minimum_bandwidth_rule_rbac)\
 412                           (?!.*test_group_type_specs)\
 413                           (?!.*test_groups_rbac.GroupTypesV3RbacTest.test_create_group_type)\
 414                           (?!.*test_groups_rbac.GroupTypesV3RbacTest.test_delete_group_type)\
 415                           (?!.*test_groups_rbac.GroupTypesV3RbacTest.test_update_group_type)\
 416                           (?!.*test_group_snapshots_rbac)\
 417                           (?!.*test_groups_rbac)\
 418                           (?!.*test_quota_classes_rbac)\
 419                           (?!.*test_server_misc_policy_actions_rbac.MiscPolicyActionsNetworkRbacTest.test_create_interface)\
 420                           (?!.*test_server_misc_policy_actions_rbac.MiscPolicyActionsNetworkRbacTest.test_delete_interface)\
 421                           (?!.*test_server_misc_policy_actions_rbac.MiscPolicyActionsNetworkRbacTest.test_show_interface)\
 422                           (?!.*test_user_messages_rbac)\
 423                           (?!.*test_volume_actions_rbac.VolumesActionsV310RbacTest)\
 424                           (?!.*test_volume_actions_rbac.VolumesActionsV312RbacTest)\
 425                           (?!.*test_volume_metadata_rbac.VolumeMetadataV3RbacTest.test_delete_volume_image_metadata)\
 426                           (?!.*test_volume_metadata_rbac.VolumeMetadataV3RbacTest.test_list_volumes_details_image_metadata)\
 427                           (?!.*test_volume_metadata_rbac.VolumeMetadataV3RbacTest.test_show_volume_details_image_metadata)\
 428                           (?!.*test_volume_metadata_rbac.VolumeMetadataV3RbacTest.test_update_volume_image_metadata)\
 429                           (?!.*test_volumes_backup_rbac.VolumesBackupsV318RbacTest)\
 430                           (?!.*test_volumes_backup_rbac.VolumesBackupsV39RbacTest)\
 431                           (?!.*test_volume_types_rbac)\
 432                           (?=.*[.*\bslow\b.*])\
 433                           (^patrole_tempest_plugin.tests.api.(compute|image|network|volume))"
 434                         option:
 435                             - '--concurrency=4'
 436
 437             -
 438                 case_name: patrole_member
 439                 project_name: functest
 440                 criteria: 100
 441                 blocking: false
 442                 description: >-
 443                     Test suite from Patrole project.
 444                 run:
 445                     name: patrole
 446                     args:
 447                         roles: 'member'
 448                         mode: "(?!.*test_unmanage_snapshot_rbac)\
 449                           (?!.*test_show_auto_allocated_topology)\
 450                           (?!.*test_delete_auto_allocated_topology)\
 451                           (?!.*test_create_network_provider_segmentation_id)\
 452                           (?!.*compute.test_floating_ips_rbac)\
 453                           (?!.*test_reset_network)\
 454                           (?!.*test_create_image_from_volume_backed_server)\
 455                           (?!.*test_network_ip_availability_rbac.NetworkIpAvailabilityExtRbacTest.test_get_network_ip_availabilities)\
 456                           (?!.*test_policy_bandwidth_limit_rule_rbac)\
 457                           (?!.*test_policy_minimum_bandwidth_rule_rbac)\
 458                           (?!.*test_group_type_specs)\
 459                           (?!.*test_groups_rbac.GroupTypesV3RbacTest.test_create_group_type)\
 460                           (?!.*test_groups_rbac.GroupTypesV3RbacTest.test_delete_group_type)\
 461                           (?!.*test_groups_rbac.GroupTypesV3RbacTest.test_update_group_type)\
 462                           (?!.*test_group_snapshots_rbac)\
 463                           (?!.*test_groups_rbac)\
 464                           (?!.*test_quota_classes_rbac)\
 465                           (?!.*test_server_misc_policy_actions_rbac.MiscPolicyActionsNetworkRbacTest.test_create_interface)\
 466                           (?!.*test_server_misc_policy_actions_rbac.MiscPolicyActionsNetworkRbacTest.test_delete_interface)\
 467                           (?!.*test_server_misc_policy_actions_rbac.MiscPolicyActionsNetworkRbacTest.test_show_interface)\
 468                           (?!.*test_user_messages_rbac)\
 469                           (?!.*test_volume_actions_rbac.VolumesActionsV310RbacTest)\
 470                           (?!.*test_volume_actions_rbac.VolumesActionsV312RbacTest)\
 471                           (?!.*test_volume_metadata_rbac.VolumeMetadataV3RbacTest.test_delete_volume_image_metadata)\
 472                           (?!.*test_volume_metadata_rbac.VolumeMetadataV3RbacTest.test_list_volumes_details_image_metadata)\
 473                           (?!.*test_volume_metadata_rbac.VolumeMetadataV3RbacTest.test_show_volume_details_image_metadata)\
 474                           (?!.*test_volume_metadata_rbac.VolumeMetadataV3RbacTest.test_update_volume_image_metadata)\
 475                           (?!.*test_volumes_backup_rbac.VolumesBackupsV318RbacTest)\
 476                           (?!.*test_volumes_backup_rbac.VolumesBackupsV39RbacTest)\
 477                           (?!.*test_volume_types_rbac)\
 478                           (?=.*[.*\bslow\b.*])\
 479                           (^patrole_tempest_plugin.tests.api.(compute|image|network|volume))"
 480                         option:
 481                             - '--concurrency=4'
 482
 483             -
 484                 case_name: patrole_reader
 485                 project_name: functest
 486                 criteria: 100
 487                 blocking: false
 488                 description: >-
 489                     Test suite from Patrole project.
 490                 run:
 491                     name: patrole
 492                     args:
 493                         roles: 'reader'
 494                         mode: "(?!.*test_unmanage_snapshot_rbac)\
 495                           (?!.*test_show_auto_allocated_topology)\
 496                           (?!.*test_delete_auto_allocated_topology)\
 497                           (?!.*test_create_network_provider_segmentation_id)\
 498                           (?!.*compute.test_floating_ips_rbac)\
 499                           (?!.*test_reset_network)\
 500                           (?!.*test_create_image_from_volume_backed_server)\
 501                           (?!.*test_network_ip_availability_rbac.NetworkIpAvailabilityExtRbacTest.test_get_network_ip_availabilities)\
 502                           (?!.*test_policy_bandwidth_limit_rule_rbac)\
 503                           (?!.*test_policy_minimum_bandwidth_rule_rbac)\
 504                           (?!.*test_group_type_specs)\
 505                           (?!.*test_groups_rbac.GroupTypesV3RbacTest.test_create_group_type)\
 506                           (?!.*test_groups_rbac.GroupTypesV3RbacTest.test_delete_group_type)\
 507                           (?!.*test_groups_rbac.GroupTypesV3RbacTest.test_update_group_type)\
 508                           (?!.*test_group_snapshots_rbac)\
 509                           (?!.*test_groups_rbac)\
 510                           (?!.*test_quota_classes_rbac)\
 511                           (?!.*test_server_misc_policy_actions_rbac.MiscPolicyActionsNetworkRbacTest.test_create_interface)\
 512                           (?!.*test_server_misc_policy_actions_rbac.MiscPolicyActionsNetworkRbacTest.test_delete_interface)\
 513                           (?!.*test_server_misc_policy_actions_rbac.MiscPolicyActionsNetworkRbacTest.test_show_interface)\
 514                           (?!.*test_user_messages_rbac)\
 515                           (?!.*test_volume_actions_rbac.VolumesActionsV310RbacTest)\
 516                           (?!.*test_volume_actions_rbac.VolumesActionsV312RbacTest)\
 517                           (?!.*test_volume_metadata_rbac.VolumeMetadataV3RbacTest.test_delete_volume_image_metadata)\
 518                           (?!.*test_volume_metadata_rbac.VolumeMetadataV3RbacTest.test_list_volumes_details_image_metadata)\
 519                           (?!.*test_volume_metadata_rbac.VolumeMetadataV3RbacTest.test_show_volume_details_image_metadata)\
 520                           (?!.*test_volume_metadata_rbac.VolumeMetadataV3RbacTest.test_update_volume_image_metadata)\
 521                           (?!.*test_volumes_backup_rbac.VolumesBackupsV318RbacTest)\
 522                           (?!.*test_volumes_backup_rbac.VolumesBackupsV39RbacTest)\
 523                           (?!.*test_volume_types_rbac)\
 524                           (?=.*[.*\bslow\b.*])\
 525                           (^patrole_tempest_plugin.tests.api.(compute|image|network|volume))"
 526                         option:
 527                             - '--concurrency=4'
 528
 529             -
 530                 case_name: tempest_barbican
 531                 project_name: functest
 532                 criteria: 100
 533                 blocking: false
 534                 description: >-
 535                     It leverages on the tempest plugin containing tests used to
 536                     verify the functionality of a barbican installation.
 537                 run:
 538                     name: barbican
 539                     args:
 540                         mode:
 541                             '^barbican_tempest_plugin.((?!test_signed_image_upload_boot_failure).)*$'
 542                         services:
 543                             - barbican
 544                         option:
 545                             - '--concurrency=4'
 546
 547             -
 548                 case_name: tempest_octavia
 549                 project_name: functest
 550                 criteria: 100
 551                 blocking: false
 552                 description: >-
 553                     It leverages on the tempest plugin containing tests used to
 554                     verify the functionality of an octavia installation.
 555                 run:
 556                     name: tempest_common
 557                     args:
 558                         mode: "(?!.*api.v2.test_availability_zone)\
 559                             (?!.*api.v2.test_availability_zone_profile)\
 560                             (?!.*api.v2.test_member.MemberAPITest.test_member_ipv4_create)\
 561                             (?!.*api.v2.test_member.MemberAPITest.test_member_ipv6_create)\
 562                             (^octavia_tempest_plugin.tests.(api|scenario))"
 563                         services:
 564                             - octavia
 565                         option:
 566                             - '--concurrency=2'
 567
 568             -
 569                 case_name: tempest_cyborg
 570                 project_name: functest
 571                 criteria: 100
 572                 blocking: false
 573                 description: >-
 574                     It leverages on the tempest plugin containing tests used to
 575                     verify the functionality of a cyborg installation.
 576                 run:
 577                     name: tempest_common
 578                     args:
 579                         mode: '^cyborg_tempest_plugin'
 580                         services:
 581                             - cyborg
 582                         option:
 583                             - '--concurrency=4'
 584
 585     -
 586         name: smoke_cntt
 587         description: >-
 588             Set of basic Functional tests to validate the OPNFV scenarios.
 589         testcases:
 590             -
 591                 case_name: tempest_neutron_cntt
 592                 project_name: functest
 593                 criteria: 100
 594                 blocking: false
 595                 deny_skipping: true
 596                 tests_count: 564
 597                 description: >-
 598                     This test case runs the Tempest suite proposed by the
 599                     Neutron project. The list of test cases is generated by
 600                     Tempest automatically and depends on the parameters of
 601                     the OpenStack deployment.
 602                 run:
 603                     name: tempest_common
 604                     args:
 605                         mode: "(?!.*admin.test_agent_availability_zone)\
 606                             (?!.*admin.test_dhcp_agent_scheduler)\
 607                             (?!.*admin.test_l3_agent_scheduler)\
 608                             (?!.*admin.test_logging)\
 609                             (?!.*admin.test_logging_negative)\
 610                             (?!.*admin.test_network_segment_range)\
 611                             (?!.*admin.test_ports.PortTestCasesAdmin.test_regenerate_mac_address)\
 612                             (?!.*admin.test_ports.PortTestCasesResourceRequest)\
 613                             (?!.*admin.test_routers_dvr)\
 614                             (?!.*admin.test_routers_flavors)\
 615                             (?!.*admin.test_routers_ha)\
 616                             (?!.*test_conntrack_helper)\
 617                             (?!.*test_floating_ips.FloatingIPPoolTestJSON)\
 618                             (?!.*test_floating_ips.FloatingIPTestJSON.test_create_update_floatingip_port_details)\
 619                             (?!.*test_local_ip)\
 620                             (?!.*test_metering_extensions)\
 621                             (?!.*test_metering_negative)\
 622                             (?!.*test_networks.NetworksSearchCriteriaTest.test_list_validation_filters)\
 623                             (?!.*test_networks.NetworksTestAdmin.test_create_tenant_network_vxlan)\
 624                             (?!.*test_networks.NetworksTestJSON.test_create_update_network_dns_domain)\
 625                             (?!.*test_port_forwardings)\
 626                             (?!.*test_ports.PortsTaggingOnCreation)\
 627                             (?!.*test_ports.PortsTestJSON.test_create_port_with_propagate_uplink_status)\
 628                             (?!.*test_ports.PortsTestJSON.test_create_port_without_propagate_uplink_status)\
 629                             (?!.*test_ports.PortsTestJSON.test_create_update_port_with_dns_domain)\
 630                             (?!.*test_ports.PortsTestJSON.test_create_update_port_with_dns_name)\
 631                             (?!.*test_ports.PortsTestJSON.test_create_update_port_with_no_dns_name)\
 632                             (?!.*test_qos.QosMinimumBandwidthRuleTestJSON)\
 633                             (?!.*test_revisions.TestRevisions.test_update_dns_domain_bumps_revision)\
 634                             (?!.*test_revisions.TestRevisions.test_update_router_extra_attributes_bumps_revision)\
 635                             (?!.*test_router_interface_fip)\
 636                             (?!.*test_routers.DvrRoutersTest)\
 637                             (?!.*test_routers.HaRoutersTest)\
 638                             (?!.*test_routers.RoutersIpV6Test.test_extra_routes_atomic)\
 639                             (?!.*test_routers.RoutersTest.test_extra_routes_atomic)\
 640                             (?!.*test_routers_negative.DvrRoutersNegativeTest)\
 641                             (?!.*test_routers_negative.DvrRoutersNegativeTestExtended)\
 642                             (?!.*test_routers_negative.HaRoutersNegativeTest)\
 643                             (?!.*test_security_groups.RbacSharedSecurityGroupTest)\
 644                             (?!.*test_subnetpool_prefix_ops)\
 645                             (?!.*test_subnetpools.RbacSubnetPoolTest)\
 646                             (?!.*test_subnetpools.SubnetPoolsSearchCriteriaTest.test_list_validation_filters)\
 647                             (?!.*test_subnetpools_negative.SubnetPoolsNegativeTestJSON.test_tenant_create_subnetpool_associate_shared_address_scope)\
 648                             (?!.*test_subnets.SubnetsSearchCriteriaTest.test_list_validation_filters)\
 649                             (?!.*test_timestamp.TestTimeStamp.test_segment_with_timestamp)\
 650                             (?!.*test_trunk.TrunkTestInheritJSONBase.test_add_subport)\
 651                             (?!.*test_trunk.TrunkTestMtusJSON)\
 652                             (?!.*test_trunk_negative.TrunkTestJSON.test_create_subport_invalid_inherit_network_segmentation_type)\
 653                             (?!.*test_trunk_negative.TrunkTestMtusJSON)\
 654                             (^neutron_tempest_plugin.api)"
 655                         option:
 656                             - '--concurrency=4'
 657
 658             -
 659                 case_name: tempest_cinder_cntt
 660                 project_name: functest
 661                 criteria: 100
 662                 blocking: false
 663                 deny_skipping: true
 664                 tests_count: 10
 665                 description: >-
 666                     This test case runs the Tempest suite proposed by the
 667                     Cinder project.
 668                 run:
 669                     name: tempest_common
 670                     args:
 671                         mode: "(?!.*test_incremental_backup)\
 672                             (?!.*test_consistencygroups)\
 673                             (?!.*test_backup_crossproject_admin_negative)\
 674                             (?!.*test_backup_crossproject_user_negative)\
 675                             (?!.*test_volume_encrypted.TestEncryptedCinderVolumes)\
 676                             (?!.*rbac)\
 677                             (^cinder_tempest_plugin.)"
 678                         option:
 679                             - '--concurrency=4'
 680
 681             -
 682                 case_name: tempest_keystone_cntt
 683                 project_name: functest
 684                 criteria: 100
 685                 blocking: false
 686                 deny_skipping: true
 687                 tests_count: 27
 688                 description: >-
 689                     This test case runs the Tempest suite proposed by the
 690                     Keystone project.
 691                 run:
 692                     name: tempest_common
 693                     args:
 694                         mode: "(?!.*api.identity.v3.test_oauth1_tokens)\
 695                             (?!.*rbac)\
 696                             (?!.*scenario.test_federated_authentication)\
 697                             keystone_tempest_plugin."
 698                         option:
 699                             - '--concurrency=4'
 700
 701             -
 702                 case_name: tempest_heat_cntt
 703                 project_name: functest
 704                 criteria: 100
 705                 blocking: false
 706                 deny_skipping: true
 707                 tests_count: 124
 708                 description: >-
 709                     This test case runs the Tempest suite proposed by the
 710                     Heat project.
 711                 run:
 712                     name: tempest_heat
 713                     args:
 714                         mode: "(?!.*functional.test_lbaasv2)\
 715                           (?!.*functional.test_encryption_vol_type)\
 716                           (?!.*functional.test_event_sinks)\
 717                           (?!.*functional.test_software_config.ZaqarSignalTransportTest)\
 718                           (?!.*functional.test_stack_events)\
 719                           (?!.*functional.test_waitcondition)\
 720                           (?!.*RemoteStackTest.test_stack_create_with_cloud_credential)\
 721                           (?!.*scenario.test_aodh_alarm)\
 722                           (?!.*tests.scenario.test_autoscaling_lb)\
 723                           (?!.*scenario.test_autoscaling_lbv2)\
 724                           (?!.*scenario.test_server_software_config)\
 725                           (?!.*test_volumes.VolumeBackupRestoreIntegrationTest)\
 726                           (?!.*scenario.test_octavia_lbaas)\
 727                           (?!.*scenario.test_server_cfn_init)\
 728                           ^heat_tempest_plugin.tests"
 729                         option:
 730                             - '--concurrency=1'
 731
 732             -
 733                 case_name: rally_sanity_cntt
 734                 project_name: functest
 735                 criteria: 100
 736                 blocking: false
 737                 description: >-
 738                     This test case runs a sub group of tests of the OpenStack
 739                     Rally suite in smoke mode.
 740                 run:
 741                     name: rally_sanity
 742                     args:
 743                         tests:
 744                             - 'authenticate'
 745                             - 'glance'
 746                             - 'cinder'
 747                             - 'heat'
 748                             - 'keystone'
 749                             - 'neutron'
 750                             - 'nova'
 751                             - 'quotas'
 752                             - 'swift'
 753
 754             -
 755                 case_name: tempest_full_cntt
 756                 project_name: functest
 757                 criteria: 100
 758                 blocking: false
 759                 deny_skipping: true
 760                 tests_count: 1271
 761                 description: >-
 762                     The list of test cases is generated by
 763                     Tempest automatically and depends on the parameters of
 764                     the OpenStack deployment.
 765                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83
 766                 run:
 767                     name: tempest_common
 768                     args:
 769                         mode: "(?!.*admin.test_agents)(?!.*test_fixed_ips)\
 770                           (?!.*test_fixed_ips_negative)\
 771                           (?!.*test_auto_allocate_network)(?!.*test_floating_ips_bulk)\
 772                           (?!.*test_flavors_microversions.FlavorsV255TestJSON)\
 773                           (?!.*test_flavors_microversions.FlavorsV261TestJSON)\
 774                           (?!.*test_live_migration.LiveAutoBlockMigrationV225Test.test_iscsi_volume)\
 775                           (?!.*test_live_migration.LiveAutoBlockMigrationV225Test.test_live_block_migration)\
 776                           (?!.*test_live_migration.LiveAutoBlockMigrationV225Test.test_live_block_migration_paused)\
 777                           (?!.*test_live_migration.LiveAutoBlockMigrationV225Test.test_volume_backed_live_migration)\
 778                           (?!.*test_live_migration.LiveMigrationTest.test_iscsi_volume)\
 779                           (?!.*test_live_migration.LiveMigrationTest.test_live_block_migration)\
 780                           (?!.*test_live_migration.LiveMigrationTest.test_live_block_migration_paused)\
 781                           (?!.*test_live_migration.LiveMigrationTest.test_volume_backed_live_migration)\
 782                           (?!.*test_live_migration.LiveMigrationRemoteConsolesV26Test)\
 783                           (?!.*test_quotas.QuotasAdminTestV257)\
 784                           (?!.*test_servers.ServersAdminTestJSON.test_reset_network_inject_network_info)\
 785                           (?!.*certificates.test_certificates)\
 786                           (?!.*test_quotas_negative.QuotasSecurityGroupAdminNegativeTest)\
 787                           (?!.*test_novnc)(?!.*test_server_personality)\
 788                           (?!.*test_servers.ServerShowV263Test.test_show_update_rebuild_list_server)\
 789                           (?!.*test_servers_microversions.ServerShowV254Test)\
 790                           (?!.*test_servers_microversions.ServerShowV257Test)\
 791                           (?!.*test_servers_negative.ServersNegativeTestJSON.test_personality_file_contents_not_encoded)\
 792                           (?!.*servers.test_virtual_interfaces)\
 793                           (?!.*test_server_actions.ServerActionsTestJSON.test_change_server_password)\
 794                           (?!.*test_server_actions.ServerActionsTestJSON.test_get_vnc_console)\
 795                           (?!.*test_server_actions.ServerActionsTestJSON.test_reboot_server_soft)\
 796                           (?!.*test_server_rescue.ServerBootFromVolumeStableRescueTest)\
 797                           (?!.*test_server_rescue.ServerStableDeviceRescueTest)\
 798                           (?!.*test_security_group_default_rules)\
 799                           (?!.*test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_security_group_create_with_duplicate_name)\
 800                           (?!.*test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_security_group_create_with_invalid_group_description)\
 801                           (?!.*test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_security_group_create_with_invalid_group_name)\
 802                           (?!.*test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_update_security_group_with_invalid_sg_des)\
 803                           (?!.*test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_update_security_group_with_invalid_sg_id)\
 804                           (?!.*test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_update_security_group_with_invalid_sg_name)\
 805                           (?!.*test_create_server.ServersTestFqdnHostnames.test_create_server_with_fqdn_name)\
 806                           (?!.*test_server_metadata.ServerMetadataTestJSON)\
 807                           (?!.*test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_delete_metadata_non_existent_server)\
 808                           (?!.*test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_metadata_items_limit)\
 809                           (?!.*test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_set_metadata_invalid_key)\
 810                           (?!.*test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_set_metadata_non_existent_server)\
 811                           (?!.*test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_set_server_metadata_blank_key)\
 812                           (?!.*test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_set_server_metadata_missing_metadata)\
 813                           (?!.*test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_update_metadata_non_existent_server)\
 814                           (?!.*test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_update_metadata_with_blank_key)\
 815                           (?!.*test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filtered_by_ip_regex)\
 816                           (?!.*compute.test_virtual_interfaces)(?!.*compute.test_virtual_interfaces_negative)\
 817                           (?!.*compute.test_networks)\
 818                           (?!.*test_attach_volume.AttachVolumeMultiAttach)\
 819                           (?!.*identity.admin.v2)(?!.*identity.v2)\
 820                           (?!.*identity.v3.test_access_rules)\
 821                           (?!.*identity.v3.test_application_credentials.ApplicationCredentialsV3Test.test_create_application_credential_access_rules)\
 822                           (?!.*image.v1)\
 823                           (?!.*image.v2.admin.test_images.ImportCopyImagesTest)\
 824                           (?!.*image.v2.test_images_negative.ImagesNegativeTest.test_create_image_reserved_property)\
 825                           (?!.*image.v2.test_images_negative.ImagesNegativeTest.test_update_image_reserved_property)\
 826                           (?!.*image.v2.test_images_negative.ImportImagesNegativeTest.test_image_web_download_import_with_bad_url)\
 827                           (?!.*image.v2.test_images.ImportImagesTest)\
 828                           (?!.*image.v2.test_images.MultiStoresImportImages)\
 829                           (?!.*admin.test_dhcp_agent_scheduler)\
 830                           (?!.*admin.test_routers_dvr)\
 831                           (?!.*test_metering_extensions)(?!.*network.test_tags)\
 832                           (?!.*test_routers_negative.DvrRoutersNegativeTest)\
 833                           (?!.*test_routers.RoutersIpV6Test.test_create_router_set_gateway_with_fixed_ip)\
 834                           (?!.*test_routers.RoutersTest.test_create_router_set_gateway_with_fixed_ip)\
 835                           (?!.*test_object_services.ObjectTest.test_create_object_with_transfer_encoding)\
 836                           (?!.*test_encrypted_volumes_extend)\
 837                           (?!.*test_group_snapshots.GroupSnapshotsV319Test.test_reset_group_snapshot_status)\
 838                           (?!.*test_multi_backend)\
 839                           (?!.*test_volume_retype.VolumeRetypeWithMigrationTest)\
 840                           (?!.*test_volume_delete_cascade.VolumesDeleteCascade.test_volume_from_snapshot_cascade_delete)\
 841                           (?!.*test_volumes_backup.VolumesBackupsTest.test_volume_backup_create_get_detailed_list_restore_delete)\
 842                           (?!.*test_volumes_negative.UpdateMultiattachVolumeNegativeTest.test_multiattach_rw_volume_update_failure)\
 843                           (?!.*test_volumes_extend.VolumesExtendAttachedTest.test_extend_attached_volume)\
 844                           (?!.*\\[.*\\bslow\\b.*\\])(^tempest.api)"
 845                         option:
 846                             - '--concurrency=4'
 847
 848             -
 849                 case_name: tempest_scenario_cntt
 850                 project_name: functest
 851                 criteria: 100
 852                 blocking: false
 853                 deny_skipping: true
 854                 tests_count: 13
 855                 description: >-
 856                     The list of test cases is generated by
 857                     Tempest automatically and depends on the parameters of
 858                     the OpenStack deployment.
 859                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
 860                 run:
 861                     name: tempest_scenario
 862                     args:
 863                         mode: "\
 864                           (?!.*test_compute_unified_limits)\
 865                           (?!.*test_minbw_allocation_placement)\
 866                           (?!.*test_network_qos_placement)\
 867                           (?!.*test_unified_limits.ImageQuotaTest.test_image_count_uploading_quota)\
 868                           (?!.*test_unified_limits.ImageQuotaTest.test_image_stage_quota)\
 869                           (?!.*test_volume_boot_pattern.TestVolumeBootPattern.test_boot_server_from_encrypted_volume_luks)\
 870                           (?!.*\\[.*\\bslow\\b.*\\])(^tempest.scenario)"
 871                         option:
 872                             - '--concurrency=1'
 873
 874             -
 875                 case_name: tempest_slow_cntt
 876                 project_name: functest
 877                 criteria: 100
 878                 blocking: false
 879                 deny_skipping: true
 880                 tests_count: 43
 881                 description: >-
 882                     The list of test cases is generated by
 883                     Tempest automatically and depends on the parameters of
 884                     the OpenStack deployment.
 885                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
 886                 run:
 887                     name: tempest_common
 888                     args:
 889                         mode: "(?!.*test_volume_swap)\
 890                           (?!.*test_server_personality)\
 891                           (?!.*test_server_rescue.ServerBootFromVolumeStableRescueTest)\
 892                           (?!.*test_container_sync.ContainerSyncTest.test_container_synchronization)\
 893                           (?!.*test_container_sync_middleware.ContainerSyncMiddlewareTest.test_container_synchronization)\
 894                           (?!.*test_encrypted_cinder_volumes)\
 895                           (?!.*test_minbw_allocation_placement)\
 896                           (?!.*test_network_basic_ops.TestNetworkBasicOps.test_router_rescheduling)\
 897                           (?!.*test_shelve_instance.TestShelveInstance.test_cold_migrate_unshelved_instance)\
 898                           (?!.*test_volume_migrate_attached)\
 899                           (?!.*test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_cold_migration_revert)\
 900                           (?=.*\\[.*\\bslow\\b.*\\])(^tempest.)"
 901                         option:
 902                             - '--concurrency=1'
 903
 904     -
 905         name: benchmarking
 906         description: >-
 907             Run several OpenStack performance tools
 908             https://docs.openstack.org/performance-docs/latest/methodologies/tools.html
 909         testcases:
 910             -
 911                 case_name: rally_full
 912                 project_name: functest
 913                 criteria: 100
 914                 blocking: false
 915                 description: >-
 916                     This test case runs the full suite of scenarios of the
 917                     OpenStack Rally suite using several threads and iterations.
 918                 run:
 919                     name: rally_full
 920                     args:
 921                         optional:
 922                             - 'gnocchi'
 923                             - 'barbican'
 924
 925             -
 926                 case_name: rally_jobs
 927                 project_name: functest
 928                 criteria: 100
 929                 blocking: false
 930                 description: >-
 931                     This test case runs a group of Rally jobs used in
 932                     OpenStack gating
 933                 dependencies:
 934                     - NO_TENANT_NETWORK: '^(?![tT]rue$)'
 935                 run:
 936                     name: rally_jobs
 937                     args:
 938                         optional:
 939                             - 'gnocchi'
 940
 941             -
 942                 case_name: vmtp
 943                 project_name: functest
 944                 criteria: 100
 945                 blocking: false
 946                 description: >-
 947                     VMTP is a small python application that will automatically
 948                     perform ping connectivity, round trip time measurement
 949                     (latency) and TCP/UDP throughput
 950                 dependencies:
 951                     - NO_TENANT_NETWORK: '^(?![tT]rue$)'
 952                 run:
 953                     name: vmtp
 954
 955             -
 956                 case_name: shaker
 957                 project_name: functest
 958                 criteria: 100
 959                 blocking: false
 960                 description: >-
 961                     Shaker wraps around popular system network testing tools
 962                     like iperf, iperf3 and netperf (with help of flent). Shaker
 963                     is able to deploy OpenStack instances and networks in
 964                     different topologies.
 965                 dependencies:
 966                     - NO_TENANT_NETWORK: '^(?![tT]rue$)'
 967                 run:
 968                     name: shaker
 969
 970     -
 971         name: benchmarking_cntt
 972         description: >-
 973             Run several OpenStack performance tools
 974             https://docs.openstack.org/performance-docs/latest/methodologies/tools.html
 975         testcases:
 976             -
 977                 case_name: rally_full_cntt
 978                 project_name: functest
 979                 criteria: 100
 980                 blocking: false
 981                 description: >-
 982                     This test case runs the full suite of scenarios of the
 983                     OpenStack Rally suite using several threads and iterations.
 984                 run:
 985                     name: rally_full
 986                     args:
 987                         tests:
 988                             - 'authenticate'
 989                             - 'glance'
 990                             - 'cinder'
 991                             - 'heat'
 992                             - 'keystone'
 993                             - 'neutron'
 994                             - 'nova'
 995                             - 'quotas'
 996                             - 'swift'
 997
 998             -
 999                 case_name: rally_jobs_cntt
1000                 project_name: functest
1001                 criteria: 100
1002                 blocking: false
1003                 description: >-
1004                     This test case runs a group of Rally jobs used in
1005                     OpenStack gating
1006                 run:
1007                     name: rally_jobs
1008                     args:
1009                         tests:
1010                             - 'neutron'
1011
1012     -
1013         name: vnf
1014         description: >-
1015             Collection of VNF test cases.
1016         testcases:
1017             -
1018                 case_name: cloudify
1019                 project_name: functest
1020                 criteria: 100
1021                 blocking: false
1022                 description: >-
1023                     This test case deploys the Cloudify orchestrator.
1024                 run:
1025                     name: cloudify
1026             -
1027                 case_name: cloudify_ims
1028                 project_name: functest
1029                 criteria: 100
1030                 blocking: false
1031                 description: >-
1032                     This test case deploys an OpenSource vIMS solution from
1033                     Clearwater using the Cloudify orchestrator. It also runs
1034                     some signaling traffic.
1035                 dependencies:
1036                     - NO_TENANT_NETWORK: '^(?![tT]rue$)'
1037                 run:
1038                     name: cloudify_ims
1039
1040             -
1041                 case_name: heat_ims
1042                 project_name: functest
1043                 criteria: 100
1044                 blocking: false
1045                 description: >-
1046                     This test case deploys an OpenSource vIMS solution from
1047                     Clearwater using the OpenStack Heat orchestrator.
1048                     It also runs some signaling traffic.
1049                 dependencies:
1050                     - NO_TENANT_NETWORK: '^(?![tT]rue$)'
1051                 run:
1052                     name: heat_ims
1053
1054             -
1055                 case_name: vyos_vrouter
1056                 project_name: functest
1057                 criteria: 100
1058                 blocking: false
1059                 description: >-
1060                     This test case is vRouter testing.
1061                 dependencies:
1062                     - NO_TENANT_NETWORK: '^(?![tT]rue$)'
1063                 run:
1064                     name: vyos_vrouter
1065
1066             -
1067                 case_name: juju_epc
1068                 project_name: functest
1069                 criteria: 100
1070                 blocking: false
1071                 description: >-
1072                     vEPC validation with Juju as VNF manager and ABoT as test
1073                     executor.
1074                 dependencies:
1075                     - NO_TENANT_NETWORK: '^(?![tT]rue$)'
1076                 run:
1077                     name: juju_epc