functest/ci/testcases.yaml

   1 ---
   2 tiers:
   3     -
   4         name: healthcheck
   5         description: >-
   6             First tier to be executed to verify the basic
   7             operations in the VIM.
   8         testcases:
   9             -
  10                 case_name: connection_check
  11                 project_name: functest
  12                 criteria: 100
  13                 blocking: true
  14                 description: >-
  15                     This test case verifies the retrieval of OpenStack clients:
  16                     Keystone, Glance, Neutron and Nova and may perform some
  17                     simple queries.
  18                 run:
  19                     name: connection_check
  20
  21             -
  22                 case_name: tenantnetwork1
  23                 project_name: functest
  24                 criteria: 100
  25                 blocking: true
  26                 description: >-
  27                     It creates and configures all tenant network ressources
  28                     required by advanced testcases (subnet, network and
  29                     router).
  30                 dependencies:
  31                     - NO_TENANT_NETWORK: '^(?![tT]rue$)'
  32                 run:
  33                     name: tenantnetwork1
  34
  35             -
  36                 case_name: tenantnetwork2
  37                 project_name: functest
  38                 criteria: 100
  39                 blocking: true
  40                 description: >-
  41                     It creates new user/project before creating and configuring
  42                     all tenant network ressources required by a testcase
  43                     (subnet, network and router).
  44                 dependencies:
  45                     - NO_TENANT_NETWORK: '^(?![tT]rue$)'
  46                 run:
  47                     name: tenantnetwork2
  48
  49             -
  50                 case_name: vmready1
  51                 project_name: functest
  52                 criteria: 100
  53                 blocking: true
  54                 description: >-
  55                     It inherits from TenantNetwork1 which creates all network
  56                     resources and prepares a future VM attached to that
  57                     network.
  58                 run:
  59                     name: vmready1
  60
  61             -
  62                 case_name: vmready2
  63                 project_name: functest
  64                 criteria: 100
  65                 blocking: true
  66                 description: >-
  67                     It creates new user/project before creating and configuring
  68                     all tenant network ressources, flavors, images, etc.
  69                     required by advanced testcases.
  70                 run:
  71                     name: vmready2
  72
  73             -
  74                 case_name: singlevm1
  75                 project_name: functest
  76                 criteria: 100
  77                 blocking: true
  78                 description: >-
  79                     It inherits from TenantNetwork1 which creates all network
  80                     resources and completes it by booting a VM attached to that
  81                     network.
  82                 run:
  83                     name: singlevm1
  84
  85             -
  86                 case_name: singlevm2
  87                 project_name: functest
  88                 criteria: 100
  89                 blocking: true
  90                 description: >-
  91                     It creates new user/project before creating and configuring
  92                     all tenant network ressources and vms required by advanced
  93                     testcases.
  94                 run:
  95                     name: singlevm2
  96
  97             -
  98                 case_name: vping_ssh
  99                 project_name: functest
 100                 criteria: 100
 101                 blocking: true
 102                 description: >-
 103                     This test case verifies: 1) SSH to an instance using
 104                     floating IPs over the public network. 2) Connectivity
 105                     between 2 instances over a private network.
 106                 run:
 107                     name: vping_ssh
 108
 109             -
 110                 case_name: vping_userdata
 111                 project_name: functest
 112                 criteria: 100
 113                 blocking: true
 114                 description: >-
 115                     This test case verifies:  1) Boot a VM with given userdata.
 116                     2) Connectivity between 2 instances over a private network.
 117                 run:
 118                     name: vping_userdata
 119
 120             -
 121                 case_name: cinder_test
 122                 project_name: functest
 123                 criteria: 100
 124                 blocking: true
 125                 description: >-
 126                     This test case verifies: 1) Attach volume and to 1
 127                     instance; 2) Write data on volume 3) Detach volume
 128                     from instance 1, attach it on instance 2 3) Read volume
 129                     data
 130                 run:
 131                     name: cinder_test
 132
 133             -
 134                 case_name: odl
 135                 project_name: functest
 136                 criteria: 100
 137                 blocking: true
 138                 description: >-
 139                     Test Suite for the OpenDaylight SDN Controller. It
 140                     integrates some test suites from upstream using
 141                     Robot as the test framework.
 142                 dependencies:
 143                     - DEPLOY_SCENARIO: 'odl'
 144                 run:
 145                     name: odl
 146                     args:
 147                         suites:
 148                             - /src/odl_test/csit/suites/integration/basic
 149                             - /src/odl_test/csit/suites/openstack/neutron
 150
 151             -
 152                 case_name: tempest_smoke
 153                 project_name: functest
 154                 criteria: 100
 155                 blocking: false
 156                 description: >-
 157                     This test case runs the smoke subset of the OpenStack
 158                     Tempest suite. The list of test cases is generated by
 159                     Tempest automatically and depends on the parameters of
 160                     the OpenStack deplopyment.
 161                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L114
 162                 run:
 163                     name: tempest_common
 164                     args:
 165                         mode: '(?=.*\[.*\bsmoke\b.*\])(^tempest\.api)'
 166                         option:
 167                             - '--concurrency=4'
 168
 169             -
 170                 case_name: tempest_horizon
 171                 project_name: functest
 172                 criteria: 100
 173                 blocking: false
 174                 description: >-
 175                     This test case runs the Tempest suite proposed by the
 176                     Horizon project.
 177                 dependencies:
 178                     - DASHBOARD_URL: '^(?!\s*$).+'
 179                 run:
 180                     name: tempest_common
 181                     args:
 182                         mode: '^tempest.scenario.test_dashboard_basic_ops.'
 183
 184     -
 185         name: smoke
 186         description: >-
 187             Set of basic Functional tests to validate the OPNFV scenarios.
 188         testcases:
 189             -
 190                 case_name: tempest_neutron
 191                 project_name: functest
 192                 criteria: 100
 193                 blocking: false
 194                 description: >-
 195                     This test case runs the Tempest suite proposed by the
 196                     Neutron project. The list of test cases is generated by
 197                     Tempest automatically and depends on the parameters of
 198                     the OpenStack deployment.
 199                 run:
 200                     name: tempest_common
 201                     args:
 202                         mode: '^neutron_tempest_plugin\.api'
 203                         option:
 204                             - '--concurrency=4'
 205
 206             -
 207                 case_name: tempest_cinder
 208                 project_name: functest
 209                 criteria: 100
 210                 blocking: false
 211                 description: >-
 212                     This test case runs the Tempest suite proposed by the
 213                     Cinder project.
 214                 run:
 215                     name: tempest_common
 216                     args:
 217                         mode: "(?!.*test_incremental_backup)\
 218                             (?!.*test_backup_crossproject_admin_negative)\
 219                             (?!.*test_backup_crossproject_user_negative)\
 220                             (^cinder_tempest_plugin.)"
 221                         option:
 222                             - '--concurrency=4'
 223
 224             -
 225                 case_name: tempest_keystone
 226                 project_name: functest
 227                 criteria: 100
 228                 blocking: false
 229                 description: >-
 230                     This test case runs the Tempest suite proposed by the
 231                     Keystone project.
 232                 run:
 233                     name: tempest_common
 234                     args:
 235                         mode: 'keystone_tempest_plugin.'
 236                         option:
 237                             - '--concurrency=4'
 238
 239             -
 240                 case_name: tempest_heat
 241                 project_name: functest
 242                 criteria: 100
 243                 blocking: false
 244                 description: >-
 245                     This test case runs the Tempest suite proposed by the
 246                     Heat project.
 247                 run:
 248                     name: tempest_heat
 249                     args:
 250                         mode: '^heat_tempest_plugin.tests'
 251                         option:
 252                             - '--concurrency=1'
 253
 254             -
 255                 case_name: tempest_telemetry
 256                 project_name: functest
 257                 criteria: 100
 258                 blocking: false
 259                 description: >-
 260                     This test case runs the Tempest suite proposed by the
 261                     Telemetry project.
 262                 run:
 263                     name: tempest_common
 264                     args:
 265                         mode: "(?!.*test_autoscaling)(?!.*test_live)\
 266                             (^telemetry_tempest_plugin)"
 267                         services:
 268                             - aodh
 269                             - gnocchi
 270                             - panko
 271                         option:
 272                             - '--concurrency=4'
 273
 274             -
 275                 case_name: rally_sanity
 276                 project_name: functest
 277                 criteria: 100
 278                 blocking: false
 279                 description: >-
 280                     This test case runs a sub group of tests of the OpenStack
 281                     Rally suite in smoke mode.
 282                 run:
 283                     name: rally_sanity
 284                     args:
 285                         optional:
 286                             - 'gnocchi'
 287                             - 'barbican'
 288
 289             -
 290                 case_name: refstack_compute
 291                 project_name: functest
 292                 criteria: 100
 293                 blocking: false
 294                 description: >-
 295                     This test case runs a sub group of tests of the OpenStack
 296                     Compute testcases.
 297                 dependencies:
 298                     - NO_TENANT_NETWORK: '^(?![tT]rue$)'
 299                 run:
 300                     name: refstack
 301                     args:
 302                         target: compute
 303                         option:
 304                             - '--concurrency=4'
 305
 306             -
 307                 case_name: refstack_object
 308                 project_name: functest
 309                 criteria: 100
 310                 blocking: false
 311                 description: >-
 312                     This test case runs a sub group of tests of the OpenStack
 313                     Object testcases.
 314                 run:
 315                     name: refstack
 316                     args:
 317                         target: object
 318                         option:
 319                             - '--concurrency=4'
 320
 321             -
 322                 case_name: refstack_platform
 323                 project_name: functest
 324                 criteria: 100
 325                 blocking: false
 326                 description: >-
 327                     This test case runs a sub group of tests of the OpenStack
 328                     Platform testcases.
 329                 dependencies:
 330                     - NO_TENANT_NETWORK: '^(?![tT]rue$)'
 331                 run:
 332                     name: refstack
 333                     args:
 334                         target: platform
 335                         option:
 336                             - '--concurrency=4'
 337
 338             -
 339                 case_name: tempest_full
 340                 project_name: functest
 341                 criteria: 100
 342                 blocking: false
 343                 description: >-
 344                     The list of test cases is generated by
 345                     Tempest automatically and depends on the parameters of
 346                     the OpenStack deployment.
 347                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83
 348                 run:
 349                     name: tempest_common
 350                     args:
 351                         mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
 352                         option:
 353                             - '--concurrency=4'
 354
 355             -
 356                 case_name: tempest_scenario
 357                 project_name: functest
 358                 criteria: 100
 359                 blocking: false
 360                 description: >-
 361                     The list of test cases is generated by
 362                     Tempest automatically and depends on the parameters of
 363                     the OpenStack deployment.
 364                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
 365                 run:
 366                     name: tempest_common
 367                     args:
 368                         mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)'
 369                         option:
 370                             - '--concurrency=1'
 371
 372             -
 373                 case_name: tempest_slow
 374                 project_name: functest
 375                 criteria: 100
 376                 blocking: false
 377                 description: >-
 378                     The list of test cases is generated by
 379                     Tempest automatically and depends on the parameters of
 380                     the OpenStack deployment.
 381                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
 382                 run:
 383                     name: tempest_common
 384                     args:
 385                         mode: '(?=.*\[.*\bslow\b.*\])(^tempest\.)'
 386                         option:
 387                             - '--concurrency=1'
 388
 389             -
 390                 case_name: patrole_admin
 391                 project_name: functest
 392                 criteria: 100
 393                 blocking: false
 394                 description: >-
 395                     Test suite from Patrole project.
 396                 run:
 397                     name: patrole
 398                     args:
 399                         roles: 'admin'
 400                         mode: "(?!.*test_unmanage_snapshot_rbac)\
 401                           (?!.*test_show_auto_allocated_topology)\
 402                           (?!.*test_delete_auto_allocated_topology)\
 403                           (?!.*test_create_network_provider_segmentation_id)\
 404                           (?!.*compute.test_floating_ips_rbac)\
 405                           (?!.*test_reset_network)\
 406                           (?!.*test_create_image_from_volume_backed_server)\
 407                           (?!.*test_network_ip_availability_rbac.NetworkIpAvailabilityExtRbacTest.test_get_network_ip_availabilities)\
 408                           (?!.*test_policy_bandwidth_limit_rule_rbac)\
 409                           (?!.*test_policy_minimum_bandwidth_rule_rbac)\
 410                           (?!.*test_group_type_specs)\
 411                           (?!.*test_groups_rbac.GroupTypesV3RbacTest.test_create_group_type)\
 412                           (?!.*test_groups_rbac.GroupTypesV3RbacTest.test_delete_group_type)\
 413                           (?!.*test_groups_rbac.GroupTypesV3RbacTest.test_update_group_type)\
 414                           (?!.*test_group_snapshots_rbac)\
 415                           (?!.*test_groups_rbac)\
 416                           (?!.*test_quota_classes_rbac)\
 417                           (?!.*test_server_misc_policy_actions_rbac.MiscPolicyActionsNetworkRbacTest.test_create_interface)\
 418                           (?!.*test_server_misc_policy_actions_rbac.MiscPolicyActionsNetworkRbacTest.test_delete_interface)\
 419                           (?!.*test_server_misc_policy_actions_rbac.MiscPolicyActionsNetworkRbacTest.test_show_interface)\
 420                           (?!.*test_user_messages_rbac)\
 421                           (?!.*test_volume_actions_rbac.VolumesActionsV310RbacTest)\
 422                           (?!.*test_volume_actions_rbac.VolumesActionsV312RbacTest)\
 423                           (?!.*test_volume_metadata_rbac.VolumeMetadataV3RbacTest.test_delete_volume_image_metadata)\
 424                           (?!.*test_volume_metadata_rbac.VolumeMetadataV3RbacTest.test_list_volumes_details_image_metadata)\
 425                           (?!.*test_volume_metadata_rbac.VolumeMetadataV3RbacTest.test_show_volume_details_image_metadata)\
 426                           (?!.*test_volume_metadata_rbac.VolumeMetadataV3RbacTest.test_update_volume_image_metadata)\
 427                           (?!.*test_volumes_backup_rbac.VolumesBackupsV318RbacTest)\
 428                           (?!.*test_volumes_backup_rbac.VolumesBackupsV39RbacTest)\
 429                           (?!.*test_volume_types_rbac)\
 430                           (?=.*[.*\bslow\b.*])\
 431                           (^patrole_tempest_plugin.tests.api.(compute|image|network|volume))"
 432                         option:
 433                             - '--concurrency=4'
 434
 435             -
 436                 case_name: patrole_member
 437                 project_name: functest
 438                 criteria: 100
 439                 blocking: false
 440                 description: >-
 441                     Test suite from Patrole project.
 442                 run:
 443                     name: patrole
 444                     args:
 445                         roles: 'member'
 446                         mode: "(?!.*test_unmanage_snapshot_rbac)\
 447                           (?!.*test_show_auto_allocated_topology)\
 448                           (?!.*test_delete_auto_allocated_topology)\
 449                           (?!.*test_create_network_provider_segmentation_id)\
 450                           (?!.*compute.test_floating_ips_rbac)\
 451                           (?!.*test_reset_network)\
 452                           (?!.*test_create_image_from_volume_backed_server)\
 453                           (?!.*test_network_ip_availability_rbac.NetworkIpAvailabilityExtRbacTest.test_get_network_ip_availabilities)\
 454                           (?!.*test_policy_bandwidth_limit_rule_rbac)\
 455                           (?!.*test_policy_minimum_bandwidth_rule_rbac)\
 456                           (?!.*test_group_type_specs)\
 457                           (?!.*test_groups_rbac.GroupTypesV3RbacTest.test_create_group_type)\
 458                           (?!.*test_groups_rbac.GroupTypesV3RbacTest.test_delete_group_type)\
 459                           (?!.*test_groups_rbac.GroupTypesV3RbacTest.test_update_group_type)\
 460                           (?!.*test_group_snapshots_rbac)\
 461                           (?!.*test_groups_rbac)\
 462                           (?!.*test_quota_classes_rbac)\
 463                           (?!.*test_server_misc_policy_actions_rbac.MiscPolicyActionsNetworkRbacTest.test_create_interface)\
 464                           (?!.*test_server_misc_policy_actions_rbac.MiscPolicyActionsNetworkRbacTest.test_delete_interface)\
 465                           (?!.*test_server_misc_policy_actions_rbac.MiscPolicyActionsNetworkRbacTest.test_show_interface)\
 466                           (?!.*test_user_messages_rbac)\
 467                           (?!.*test_volume_actions_rbac.VolumesActionsV310RbacTest)\
 468                           (?!.*test_volume_actions_rbac.VolumesActionsV312RbacTest)\
 469                           (?!.*test_volume_metadata_rbac.VolumeMetadataV3RbacTest.test_delete_volume_image_metadata)\
 470                           (?!.*test_volume_metadata_rbac.VolumeMetadataV3RbacTest.test_list_volumes_details_image_metadata)\
 471                           (?!.*test_volume_metadata_rbac.VolumeMetadataV3RbacTest.test_show_volume_details_image_metadata)\
 472                           (?!.*test_volume_metadata_rbac.VolumeMetadataV3RbacTest.test_update_volume_image_metadata)\
 473                           (?!.*test_volumes_backup_rbac.VolumesBackupsV318RbacTest)\
 474                           (?!.*test_volumes_backup_rbac.VolumesBackupsV39RbacTest)\
 475                           (?!.*test_volume_types_rbac)\
 476                           (?=.*[.*\bslow\b.*])\
 477                           (^patrole_tempest_plugin.tests.api.(compute|image|network|volume))"
 478                         option:
 479                             - '--concurrency=4'
 480
 481             -
 482                 case_name: patrole_reader
 483                 project_name: functest
 484                 criteria: 100
 485                 blocking: false
 486                 description: >-
 487                     Test suite from Patrole project.
 488                 run:
 489                     name: patrole
 490                     args:
 491                         roles: 'reader'
 492                         mode: "(?!.*test_unmanage_snapshot_rbac)\
 493                           (?!.*test_show_auto_allocated_topology)\
 494                           (?!.*test_delete_auto_allocated_topology)\
 495                           (?!.*test_create_network_provider_segmentation_id)\
 496                           (?!.*compute.test_floating_ips_rbac)\
 497                           (?!.*test_reset_network)\
 498                           (?!.*test_create_image_from_volume_backed_server)\
 499                           (?!.*test_network_ip_availability_rbac.NetworkIpAvailabilityExtRbacTest.test_get_network_ip_availabilities)\
 500                           (?!.*test_policy_bandwidth_limit_rule_rbac)\
 501                           (?!.*test_policy_minimum_bandwidth_rule_rbac)\
 502                           (?!.*test_group_type_specs)\
 503                           (?!.*test_groups_rbac.GroupTypesV3RbacTest.test_create_group_type)\
 504                           (?!.*test_groups_rbac.GroupTypesV3RbacTest.test_delete_group_type)\
 505                           (?!.*test_groups_rbac.GroupTypesV3RbacTest.test_update_group_type)\
 506                           (?!.*test_group_snapshots_rbac)\
 507                           (?!.*test_groups_rbac)\
 508                           (?!.*test_quota_classes_rbac)\
 509                           (?!.*test_server_misc_policy_actions_rbac.MiscPolicyActionsNetworkRbacTest.test_create_interface)\
 510                           (?!.*test_server_misc_policy_actions_rbac.MiscPolicyActionsNetworkRbacTest.test_delete_interface)\
 511                           (?!.*test_server_misc_policy_actions_rbac.MiscPolicyActionsNetworkRbacTest.test_show_interface)\
 512                           (?!.*test_user_messages_rbac)\
 513                           (?!.*test_volume_actions_rbac.VolumesActionsV310RbacTest)\
 514                           (?!.*test_volume_actions_rbac.VolumesActionsV312RbacTest)\
 515                           (?!.*test_volume_metadata_rbac.VolumeMetadataV3RbacTest.test_delete_volume_image_metadata)\
 516                           (?!.*test_volume_metadata_rbac.VolumeMetadataV3RbacTest.test_list_volumes_details_image_metadata)\
 517                           (?!.*test_volume_metadata_rbac.VolumeMetadataV3RbacTest.test_show_volume_details_image_metadata)\
 518                           (?!.*test_volume_metadata_rbac.VolumeMetadataV3RbacTest.test_update_volume_image_metadata)\
 519                           (?!.*test_volumes_backup_rbac.VolumesBackupsV318RbacTest)\
 520                           (?!.*test_volumes_backup_rbac.VolumesBackupsV39RbacTest)\
 521                           (?!.*test_volume_types_rbac)\
 522                           (?=.*[.*\bslow\b.*])\
 523                           (^patrole_tempest_plugin.tests.api.(compute|image|network|volume))"
 524                         option:
 525                             - '--concurrency=4'
 526
 527             -
 528                 case_name: tempest_barbican
 529                 project_name: functest
 530                 criteria: 100
 531                 blocking: false
 532                 description: >-
 533                     It leverages on the tempest plugin containing tests used to
 534                     verify the functionality of a barbican installation.
 535                 run:
 536                     name: barbican
 537                     args:
 538                         mode:
 539                             '^barbican_tempest_plugin.((?!test_signed_image_upload_boot_failure).)*$'
 540                         services:
 541                             - barbican
 542                         option:
 543                             - '--concurrency=4'
 544
 545             -
 546                 case_name: tempest_octavia
 547                 project_name: functest
 548                 criteria: 100
 549                 blocking: false
 550                 description: >-
 551                     It leverages on the tempest plugin containing tests used to
 552                     verify the functionality of an octavia installation.
 553                 run:
 554                     name: tempest_common
 555                     args:
 556                         mode: "(?!.*api.v2.test_availability_zone)\
 557                             (?!.*api.v2.test_availability_zone_profile)\
 558                             (?!.*api.v2.test_member.MemberAPITest.test_member_ipv4_create)\
 559                             (?!.*api.v2.test_member.MemberAPITest.test_member_ipv6_create)\
 560                             (^octavia_tempest_plugin.tests.(api|scenario))"
 561                         services:
 562                             - octavia
 563                         option:
 564                             - '--concurrency=2'
 565
 566             -
 567                 case_name: tempest_cyborg
 568                 project_name: functest
 569                 criteria: 100
 570                 blocking: false
 571                 description: >-
 572                     It leverages on the tempest plugin containing tests used to
 573                     verify the functionality of a cyborg installation.
 574                 run:
 575                     name: tempest_common
 576                     args:
 577                         mode: '^cyborg_tempest_plugin'
 578                         services:
 579                             - cyborg
 580                         option:
 581                             - '--concurrency=4'
 582
 583     -
 584         name: smoke_cntt
 585         description: >-
 586             Set of basic Functional tests to validate the OPNFV scenarios.
 587         testcases:
 588             -
 589                 case_name: tempest_neutron_cntt
 590                 project_name: functest
 591                 criteria: 100
 592                 blocking: false
 593                 deny_skipping: true
 594                 tests_count: 564
 595                 description: >-
 596                     This test case runs the Tempest suite proposed by the
 597                     Neutron project. The list of test cases is generated by
 598                     Tempest automatically and depends on the parameters of
 599                     the OpenStack deployment.
 600                 run:
 601                     name: tempest_common
 602                     args:
 603                         mode: "(?!.*admin.test_agent_availability_zone)\
 604                             (?!.*admin.test_dhcp_agent_scheduler)\
 605                             (?!.*admin.test_l3_agent_scheduler)\
 606                             (?!.*admin.test_logging)\
 607                             (?!.*admin.test_logging_negative)\
 608                             (?!.*admin.test_network_segment_range)\
 609                             (?!.*admin.test_ports.PortTestCasesAdmin.test_regenerate_mac_address)\
 610                             (?!.*admin.test_ports.PortTestCasesResourceRequest)\
 611                             (?!.*admin.test_routers_dvr)\
 612                             (?!.*admin.test_routers_flavors)\
 613                             (?!.*admin.test_routers_ha)\
 614                             (?!.*test_conntrack_helper)\
 615                             (?!.*test_floating_ips.FloatingIPPoolTestJSON)\
 616                             (?!.*test_floating_ips.FloatingIPTestJSON.test_create_update_floatingip_port_details)\
 617                             (?!.*test_local_ip)\
 618                             (?!.*test_metering_extensions)\
 619                             (?!.*test_metering_negative)\
 620                             (?!.*test_networks.NetworksSearchCriteriaTest.test_list_validation_filters)\
 621                             (?!.*test_networks.NetworksTestAdmin.test_create_tenant_network_vxlan)\
 622                             (?!.*test_networks.NetworksTestJSON.test_create_update_network_dns_domain)\
 623                             (?!.*test_port_forwardings)\
 624                             (?!.*test_ports.PortsTaggingOnCreation)\
 625                             (?!.*test_ports.PortsTestJSON.test_create_port_with_propagate_uplink_status)\
 626                             (?!.*test_ports.PortsTestJSON.test_create_port_without_propagate_uplink_status)\
 627                             (?!.*test_ports.PortsTestJSON.test_create_update_port_with_dns_domain)\
 628                             (?!.*test_ports.PortsTestJSON.test_create_update_port_with_dns_name)\
 629                             (?!.*test_ports.PortsTestJSON.test_create_update_port_with_no_dns_name)\
 630                             (?!.*test_qos.QosMinimumBandwidthRuleTestJSON)\
 631                             (?!.*test_revisions.TestRevisions.test_update_dns_domain_bumps_revision)\
 632                             (?!.*test_revisions.TestRevisions.test_update_router_extra_attributes_bumps_revision)\
 633                             (?!.*test_router_interface_fip)\
 634                             (?!.*test_routers.DvrRoutersTest)\
 635                             (?!.*test_routers.HaRoutersTest)\
 636                             (?!.*test_routers.RoutersIpV6Test.test_extra_routes_atomic)\
 637                             (?!.*test_routers.RoutersTest.test_extra_routes_atomic)\
 638                             (?!.*test_routers_negative.DvrRoutersNegativeTest)\
 639                             (?!.*test_routers_negative.DvrRoutersNegativeTestExtended)\
 640                             (?!.*test_routers_negative.HaRoutersNegativeTest)\
 641                             (?!.*test_security_groups.RbacSharedSecurityGroupTest)\
 642                             (?!.*test_subnetpool_prefix_ops)\
 643                             (?!.*test_subnetpools.RbacSubnetPoolTest)\
 644                             (?!.*test_subnetpools.SubnetPoolsSearchCriteriaTest.test_list_validation_filters)\
 645                             (?!.*test_subnetpools_negative.SubnetPoolsNegativeTestJSON.test_tenant_create_subnetpool_associate_shared_address_scope)\
 646                             (?!.*test_subnets.SubnetsSearchCriteriaTest.test_list_validation_filters)\
 647                             (?!.*test_timestamp.TestTimeStamp.test_segment_with_timestamp)\
 648                             (?!.*test_trunk.TrunkTestInheritJSONBase.test_add_subport)\
 649                             (?!.*test_trunk.TrunkTestMtusJSON)\
 650                             (?!.*test_trunk_negative.TrunkTestJSON.test_create_subport_invalid_inherit_network_segmentation_type)\
 651                             (?!.*test_trunk_negative.TrunkTestMtusJSON)\
 652                             (^neutron_tempest_plugin.api)"
 653                         option:
 654                             - '--concurrency=4'
 655
 656             -
 657                 case_name: tempest_cinder_cntt
 658                 project_name: functest
 659                 criteria: 100
 660                 blocking: false
 661                 deny_skipping: true
 662                 tests_count: 10
 663                 description: >-
 664                     This test case runs the Tempest suite proposed by the
 665                     Cinder project.
 666                 run:
 667                     name: tempest_common
 668                     args:
 669                         mode: "(?!.*test_incremental_backup)\
 670                             (?!.*test_consistencygroups)\
 671                             (?!.*test_backup_crossproject_admin_negative)\
 672                             (?!.*test_backup_crossproject_user_negative)\
 673                             (?!.*test_volume_encrypted.TestEncryptedCinderVolumes)\
 674                             (?!.*rbac)\
 675                             (^cinder_tempest_plugin.)"
 676                         option:
 677                             - '--concurrency=4'
 678
 679             -
 680                 case_name: tempest_keystone_cntt
 681                 project_name: functest
 682                 criteria: 100
 683                 blocking: false
 684                 deny_skipping: true
 685                 tests_count: 27
 686                 description: >-
 687                     This test case runs the Tempest suite proposed by the
 688                     Keystone project.
 689                 run:
 690                     name: tempest_common
 691                     args:
 692                         mode: "(?!.*api.identity.v3.test_oauth1_tokens)\
 693                             (?!.*rbac)\
 694                             (?!.*scenario.test_federated_authentication)\
 695                             keystone_tempest_plugin."
 696                         option:
 697                             - '--concurrency=4'
 698
 699             -
 700                 case_name: tempest_heat_cntt
 701                 project_name: functest
 702                 criteria: 100
 703                 blocking: false
 704                 deny_skipping: true
 705                 tests_count: 124
 706                 description: >-
 707                     This test case runs the Tempest suite proposed by the
 708                     Heat project.
 709                 run:
 710                     name: tempest_heat
 711                     args:
 712                         mode: "(?!.*functional.test_lbaasv2)\
 713                           (?!.*functional.test_encryption_vol_type)\
 714                           (?!.*functional.test_event_sinks)\
 715                           (?!.*functional.test_software_config.ZaqarSignalTransportTest)\
 716                           (?!.*functional.test_stack_events)\
 717                           (?!.*functional.test_waitcondition)\
 718                           (?!.*RemoteStackTest.test_stack_create_with_cloud_credential)\
 719                           (?!.*scenario.test_aodh_alarm)\
 720                           (?!.*tests.scenario.test_autoscaling_lb)\
 721                           (?!.*scenario.test_autoscaling_lbv2)\
 722                           (?!.*scenario.test_server_software_config)\
 723                           (?!.*test_volumes.VolumeBackupRestoreIntegrationTest)\
 724                           (?!.*scenario.test_octavia_lbaas)\
 725                           (?!.*scenario.test_server_cfn_init)\
 726                           ^heat_tempest_plugin.tests"
 727                         option:
 728                             - '--concurrency=1'
 729
 730             -
 731                 case_name: rally_sanity_cntt
 732                 project_name: functest
 733                 criteria: 100
 734                 blocking: false
 735                 description: >-
 736                     This test case runs a sub group of tests of the OpenStack
 737                     Rally suite in smoke mode.
 738                 run:
 739                     name: rally_sanity
 740                     args:
 741                         tests:
 742                             - 'authenticate'
 743                             - 'glance'
 744                             - 'cinder'
 745                             - 'heat'
 746                             - 'keystone'
 747                             - 'neutron'
 748                             - 'nova'
 749                             - 'quotas'
 750                             - 'swift'
 751
 752             -
 753                 case_name: tempest_full_cntt
 754                 project_name: functest
 755                 criteria: 100
 756                 blocking: false
 757                 deny_skipping: true
 758                 tests_count: 1271
 759                 description: >-
 760                     The list of test cases is generated by
 761                     Tempest automatically and depends on the parameters of
 762                     the OpenStack deployment.
 763                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83
 764                 run:
 765                     name: tempest_common
 766                     args:
 767                         mode: "(?!.*admin.test_agents)(?!.*test_fixed_ips)\
 768                           (?!.*test_fixed_ips_negative)\
 769                           (?!.*test_auto_allocate_network)(?!.*test_floating_ips_bulk)\
 770                           (?!.*test_flavors_microversions.FlavorsV255TestJSON)\
 771                           (?!.*test_flavors_microversions.FlavorsV261TestJSON)\
 772                           (?!.*test_live_migration.LiveAutoBlockMigrationV225Test.test_iscsi_volume)\
 773                           (?!.*test_live_migration.LiveAutoBlockMigrationV225Test.test_live_block_migration)\
 774                           (?!.*test_live_migration.LiveAutoBlockMigrationV225Test.test_live_block_migration_paused)\
 775                           (?!.*test_live_migration.LiveAutoBlockMigrationV225Test.test_volume_backed_live_migration)\
 776                           (?!.*test_live_migration.LiveMigrationTest.test_iscsi_volume)\
 777                           (?!.*test_live_migration.LiveMigrationTest.test_live_block_migration)\
 778                           (?!.*test_live_migration.LiveMigrationTest.test_live_block_migration_paused)\
 779                           (?!.*test_live_migration.LiveMigrationTest.test_volume_backed_live_migration)\
 780                           (?!.*test_live_migration.LiveMigrationRemoteConsolesV26Test)\
 781                           (?!.*test_quotas.QuotasAdminTestV257)\
 782                           (?!.*test_servers.ServersAdminTestJSON.test_reset_network_inject_network_info)\
 783                           (?!.*certificates.test_certificates)\
 784                           (?!.*test_quotas_negative.QuotasSecurityGroupAdminNegativeTest)\
 785                           (?!.*test_novnc)(?!.*test_server_personality)\
 786                           (?!.*test_servers.ServerShowV263Test.test_show_update_rebuild_list_server)\
 787                           (?!.*test_servers_microversions.ServerShowV254Test)\
 788                           (?!.*test_servers_microversions.ServerShowV257Test)\
 789                           (?!.*test_servers_negative.ServersNegativeTestJSON.test_personality_file_contents_not_encoded)\
 790                           (?!.*servers.test_virtual_interfaces)\
 791                           (?!.*test_server_actions.ServerActionsTestJSON.test_change_server_password)\
 792                           (?!.*test_server_actions.ServerActionsTestJSON.test_get_vnc_console)\
 793                           (?!.*test_server_actions.ServerActionsTestJSON.test_reboot_server_soft)\
 794                           (?!.*test_server_rescue.ServerBootFromVolumeStableRescueTest)\
 795                           (?!.*test_server_rescue.ServerStableDeviceRescueTest)\
 796                           (?!.*test_security_group_default_rules)\
 797                           (?!.*test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_security_group_create_with_duplicate_name)\
 798                           (?!.*test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_security_group_create_with_invalid_group_description)\
 799                           (?!.*test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_security_group_create_with_invalid_group_name)\
 800                           (?!.*test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_update_security_group_with_invalid_sg_des)\
 801                           (?!.*test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_update_security_group_with_invalid_sg_id)\
 802                           (?!.*test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_update_security_group_with_invalid_sg_name)\
 803                           (?!.*test_create_server.ServersTestFqdnHostnames.test_create_server_with_fqdn_name)\
 804                           (?!.*test_server_metadata.ServerMetadataTestJSON)\
 805                           (?!.*test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_delete_metadata_non_existent_server)\
 806                           (?!.*test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_metadata_items_limit)\
 807                           (?!.*test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_set_metadata_invalid_key)\
 808                           (?!.*test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_set_metadata_non_existent_server)\
 809                           (?!.*test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_set_server_metadata_blank_key)\
 810                           (?!.*test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_set_server_metadata_missing_metadata)\
 811                           (?!.*test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_update_metadata_non_existent_server)\
 812                           (?!.*test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_update_metadata_with_blank_key)\
 813                           (?!.*test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filtered_by_ip_regex)\
 814                           (?!.*compute.test_virtual_interfaces)(?!.*compute.test_virtual_interfaces_negative)\
 815                           (?!.*compute.test_networks)\
 816                           (?!.*test_attach_volume.AttachVolumeMultiAttach)\
 817                           (?!.*identity.admin.v2)(?!.*identity.v2)\
 818                           (?!.*identity.v3.test_access_rules)\
 819                           (?!.*identity.v3.test_application_credentials.ApplicationCredentialsV3Test.test_create_application_credential_access_rules)\
 820                           (?!.*image.v1)\
 821                           (?!.*image.v2.admin.test_images.ImportCopyImagesTest)\
 822                           (?!.*image.v2.test_images_negative.ImagesNegativeTest.test_create_image_reserved_property)\
 823                           (?!.*image.v2.test_images_negative.ImagesNegativeTest.test_update_image_reserved_property)\
 824                           (?!.*image.v2.test_images_negative.ImportImagesNegativeTest.test_image_web_download_import_with_bad_url)\
 825                           (?!.*image.v2.test_images.ImportImagesTest)\
 826                           (?!.*image.v2.test_images.MultiStoresImportImages)\
 827                           (?!.*admin.test_dhcp_agent_scheduler)\
 828                           (?!.*admin.test_routers_dvr)\
 829                           (?!.*test_metering_extensions)(?!.*network.test_tags)\
 830                           (?!.*test_routers_negative.DvrRoutersNegativeTest)\
 831                           (?!.*test_routers.RoutersIpV6Test.test_create_router_set_gateway_with_fixed_ip)\
 832                           (?!.*test_routers.RoutersTest.test_create_router_set_gateway_with_fixed_ip)\
 833                           (?!.*test_object_services.ObjectTest.test_create_object_with_transfer_encoding)\
 834                           (?!.*test_encrypted_volumes_extend)\
 835                           (?!.*test_group_snapshots.GroupSnapshotsV319Test.test_reset_group_snapshot_status)\
 836                           (?!.*test_multi_backend)\
 837                           (?!.*test_volume_retype.VolumeRetypeWithMigrationTest)\
 838                           (?!.*test_volume_delete_cascade.VolumesDeleteCascade.test_volume_from_snapshot_cascade_delete)\
 839                           (?!.*test_volumes_backup.VolumesBackupsTest.test_volume_backup_create_get_detailed_list_restore_delete)\
 840                           (?!.*test_volumes_negative.UpdateMultiattachVolumeNegativeTest.test_multiattach_rw_volume_update_failure)\
 841                           (?!.*test_volumes_extend.VolumesExtendAttachedTest.test_extend_attached_volume)\
 842                           (?!.*\\[.*\\bslow\\b.*\\])(^tempest.api)"
 843                         option:
 844                             - '--concurrency=4'
 845
 846             -
 847                 case_name: tempest_scenario_cntt
 848                 project_name: functest
 849                 criteria: 100
 850                 blocking: false
 851                 deny_skipping: true
 852                 tests_count: 13
 853                 description: >-
 854                     The list of test cases is generated by
 855                     Tempest automatically and depends on the parameters of
 856                     the OpenStack deployment.
 857                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
 858                 run:
 859                     name: tempest_scenario
 860                     args:
 861                         mode: "\
 862                           (?!.*test_compute_unified_limits)\
 863                           (?!.*test_minbw_allocation_placement)\
 864                           (?!.*test_network_qos_placement)\
 865                           (?!.*test_unified_limits.ImageQuotaTest.test_image_count_uploading_quota)\
 866                           (?!.*test_unified_limits.ImageQuotaTest.test_image_stage_quota)\
 867                           (?!.*test_volume_boot_pattern.TestVolumeBootPattern.test_boot_server_from_encrypted_volume_luks)\
 868                           (?!.*\\[.*\\bslow\\b.*\\])(^tempest.scenario)"
 869                         option:
 870                             - '--concurrency=1'
 871
 872             -
 873                 case_name: tempest_slow_cntt
 874                 project_name: functest
 875                 criteria: 100
 876                 blocking: false
 877                 deny_skipping: true
 878                 tests_count: 43
 879                 description: >-
 880                     The list of test cases is generated by
 881                     Tempest automatically and depends on the parameters of
 882                     the OpenStack deployment.
 883                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
 884                 run:
 885                     name: tempest_common
 886                     args:
 887                         mode: "(?!.*test_volume_swap)\
 888                           (?!.*test_server_personality)\
 889                           (?!.*test_server_rescue.ServerBootFromVolumeStableRescueTest)\
 890                           (?!.*test_container_sync.ContainerSyncTest.test_container_synchronization)\
 891                           (?!.*test_container_sync_middleware.ContainerSyncMiddlewareTest.test_container_synchronization)\
 892                           (?!.*test_encrypted_cinder_volumes)\
 893                           (?!.*test_minbw_allocation_placement)\
 894                           (?!.*test_network_basic_ops.TestNetworkBasicOps.test_router_rescheduling)\
 895                           (?!.*test_shelve_instance.TestShelveInstance.test_cold_migrate_unshelved_instance)\
 896                           (?!.*test_volume_migrate_attached)\
 897                           (?!.*test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_cold_migration_revert)\
 898                           (?=.*\\[.*\\bslow\\b.*\\])(^tempest.)"
 899                         option:
 900                             - '--concurrency=1'
 901
 902     -
 903         name: benchmarking
 904         description: >-
 905             Run several OpenStack performance tools
 906             https://docs.openstack.org/performance-docs/latest/methodologies/tools.html
 907         testcases:
 908             -
 909                 case_name: rally_full
 910                 project_name: functest
 911                 criteria: 100
 912                 blocking: false
 913                 description: >-
 914                     This test case runs the full suite of scenarios of the
 915                     OpenStack Rally suite using several threads and iterations.
 916                 run:
 917                     name: rally_full
 918                     args:
 919                         optional:
 920                             - 'gnocchi'
 921                             - 'barbican'
 922
 923             -
 924                 case_name: rally_jobs
 925                 project_name: functest
 926                 criteria: 100
 927                 blocking: false
 928                 description: >-
 929                     This test case runs a group of Rally jobs used in
 930                     OpenStack gating
 931                 dependencies:
 932                     - NO_TENANT_NETWORK: '^(?![tT]rue$)'
 933                 run:
 934                     name: rally_jobs
 935                     args:
 936                         optional:
 937                             - 'gnocchi'
 938
 939             -
 940                 case_name: vmtp
 941                 project_name: functest
 942                 criteria: 100
 943                 blocking: false
 944                 description: >-
 945                     VMTP is a small python application that will automatically
 946                     perform ping connectivity, round trip time measurement
 947                     (latency) and TCP/UDP throughput
 948                 dependencies:
 949                     - NO_TENANT_NETWORK: '^(?![tT]rue$)'
 950                 run:
 951                     name: vmtp
 952
 953             -
 954                 case_name: shaker
 955                 project_name: functest
 956                 criteria: 100
 957                 blocking: false
 958                 description: >-
 959                     Shaker wraps around popular system network testing tools
 960                     like iperf, iperf3 and netperf (with help of flent). Shaker
 961                     is able to deploy OpenStack instances and networks in
 962                     different topologies.
 963                 dependencies:
 964                     - NO_TENANT_NETWORK: '^(?![tT]rue$)'
 965                 run:
 966                     name: shaker
 967
 968     -
 969         name: benchmarking_cntt
 970         description: >-
 971             Run several OpenStack performance tools
 972             https://docs.openstack.org/performance-docs/latest/methodologies/tools.html
 973         testcases:
 974             -
 975                 case_name: rally_full_cntt
 976                 project_name: functest
 977                 criteria: 100
 978                 blocking: false
 979                 description: >-
 980                     This test case runs the full suite of scenarios of the
 981                     OpenStack Rally suite using several threads and iterations.
 982                 run:
 983                     name: rally_full
 984                     args:
 985                         tests:
 986                             - 'authenticate'
 987                             - 'glance'
 988                             - 'cinder'
 989                             - 'heat'
 990                             - 'keystone'
 991                             - 'neutron'
 992                             - 'nova'
 993                             - 'quotas'
 994                             - 'swift'
 995
 996             -
 997                 case_name: rally_jobs_cntt
 998                 project_name: functest
 999                 criteria: 100
1000                 blocking: false
1001                 description: >-
1002                     This test case runs a group of Rally jobs used in
1003                     OpenStack gating
1004                 run:
1005                     name: rally_jobs
1006                     args:
1007                         tests:
1008                             - 'neutron'
1009
1010     -
1011         name: vnf
1012         description: >-
1013             Collection of VNF test cases.
1014         testcases:
1015             -
1016                 case_name: cloudify
1017                 project_name: functest
1018                 criteria: 100
1019                 blocking: false
1020                 description: >-
1021                     This test case deploys the Cloudify orchestrator.
1022                 run:
1023                     name: cloudify
1024             -
1025                 case_name: cloudify_ims
1026                 project_name: functest
1027                 criteria: 100
1028                 blocking: false
1029                 description: >-
1030                     This test case deploys an OpenSource vIMS solution from
1031                     Clearwater using the Cloudify orchestrator. It also runs
1032                     some signaling traffic.
1033                 dependencies:
1034                     - NO_TENANT_NETWORK: '^(?![tT]rue$)'
1035                 run:
1036                     name: cloudify_ims
1037
1038             -
1039                 case_name: heat_ims
1040                 project_name: functest
1041                 criteria: 100
1042                 blocking: false
1043                 description: >-
1044                     This test case deploys an OpenSource vIMS solution from
1045                     Clearwater using the OpenStack Heat orchestrator.
1046                     It also runs some signaling traffic.
1047                 dependencies:
1048                     - NO_TENANT_NETWORK: '^(?![tT]rue$)'
1049                 run:
1050                     name: heat_ims
1051
1052             -
1053                 case_name: vyos_vrouter
1054                 project_name: functest
1055                 criteria: 100
1056                 blocking: false
1057                 description: >-
1058                     This test case is vRouter testing.
1059                 dependencies:
1060                     - NO_TENANT_NETWORK: '^(?![tT]rue$)'
1061                 run:
1062                     name: vyos_vrouter
1063
1064             -
1065                 case_name: juju_epc
1066                 project_name: functest
1067                 criteria: 100
1068                 blocking: false
1069                 description: >-
1070                     vEPC validation with Juju as VNF manager and ABoT as test
1071                     executor.
1072                 dependencies:
1073                     - NO_TENANT_NETWORK: '^(?![tT]rue$)'
1074                 run:
1075                     name: juju_epc