functest/ci/testcases.yaml

   1 ---
   2 tiers:
   3     -
   4         name: healthcheck
   5         order: 0
   6         description: >-
   7             First tier to be executed to verify the basic
   8             operations in the VIM.
   9         testcases:
  10             -
  11                 case_name: connection_check
  12                 project_name: functest
  13                 criteria: 100
  14                 blocking: true
  15                 description: >-
  16                     This test case verifies the retrieval of OpenStack clients:
  17                     Keystone, Glance, Neutron and Nova and may perform some
  18                     simple queries.
  19                 run:
  20                     name: connection_check
  21
  22             -
  23                 case_name: tenantnetwork1
  24                 project_name: functest
  25                 criteria: 100
  26                 blocking: true
  27                 description: >-
  28                     It creates and configures all tenant network ressources
  29                     required by advanced testcases (subnet, network and
  30                     router).
  31                 run:
  32                     name: tenantnetwork1
  33
  34             -
  35                 case_name: tenantnetwork2
  36                 project_name: functest
  37                 criteria: 100
  38                 blocking: true
  39                 description: >-
  40                     It creates new user/project before creating and configuring
  41                     all tenant network ressources required by a testcase
  42                     (subnet, network and router).
  43                 run:
  44                     name: tenantnetwork2
  45
  46             -
  47                 case_name: vmready1
  48                 project_name: functest
  49                 criteria: 100
  50                 blocking: true
  51                 description: >-
  52                     It inherits from TenantNetwork1 which creates all network
  53                     resources and prepares a future VM attached to that
  54                     network.
  55                 run:
  56                     name: vmready1
  57
  58             -
  59                 case_name: vmready2
  60                 project_name: functest
  61                 criteria: 100
  62                 blocking: true
  63                 description: >-
  64                     It creates new user/project before creating and configuring
  65                     all tenant network ressources, flavors, images, etc.
  66                     required by advanced testcases.
  67                 run:
  68                     name: vmready2
  69
  70             -
  71                 case_name: singlevm1
  72                 project_name: functest
  73                 criteria: 100
  74                 blocking: true
  75                 description: >-
  76                     It inherits from TenantNetwork1 which creates all network
  77                     resources and completes it by booting a VM attached to that
  78                     network.
  79                 run:
  80                     name: singlevm1
  81
  82             -
  83                 case_name: singlevm2
  84                 project_name: functest
  85                 criteria: 100
  86                 blocking: true
  87                 description: >-
  88                     It creates new user/project before creating and configuring
  89                     all tenant network ressources and vms required by advanced
  90                     testcases.
  91                 run:
  92                     name: singlevm2
  93
  94             -
  95                 case_name: vping_ssh
  96                 project_name: functest
  97                 criteria: 100
  98                 blocking: true
  99                 description: >-
 100                     This test case verifies: 1) SSH to an instance using
 101                     floating IPs over the public network. 2) Connectivity
 102                     between 2 instances over a private network.
 103                 run:
 104                     name: vping_ssh
 105
 106             -
 107                 case_name: vping_userdata
 108                 project_name: functest
 109                 criteria: 100
 110                 blocking: true
 111                 description: >-
 112                     This test case verifies:  1) Boot a VM with given userdata.
 113                     2) Connectivity between 2 instances over a private network.
 114                 run:
 115                     name: vping_userdata
 116
 117             -
 118                 case_name: cinder_test
 119                 project_name: functest
 120                 criteria: 100
 121                 blocking: true
 122                 description: >-
 123                     This test case verifies: 1) Attach volume and to 1
 124                     instance; 2) Write data on volume 3) Detach volume
 125                     from instance 1, attach it on instance 2 3) Read volume
 126                     data
 127                 run:
 128                     name: cinder_test
 129
 130             -
 131                 case_name: odl
 132                 project_name: functest
 133                 criteria: 100
 134                 blocking: true
 135                 description: >-
 136                     Test Suite for the OpenDaylight SDN Controller. It
 137                     integrates some test suites from upstream using
 138                     Robot as the test framework.
 139                 dependencies:
 140                     - DEPLOY_SCENARIO: 'odl'
 141                 run:
 142                     name: odl
 143                     args:
 144                         suites:
 145                             - /src/odl_test/csit/suites/integration/basic
 146                             - /src/odl_test/csit/suites/openstack/neutron
 147
 148             -
 149                 case_name: tempest_smoke
 150                 project_name: functest
 151                 criteria: 100
 152                 blocking: false
 153                 description: >-
 154                     This test case runs the smoke subset of the OpenStack
 155                     Tempest suite. The list of test cases is generated by
 156                     Tempest automatically and depends on the parameters of
 157                     the OpenStack deplopyment.
 158                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L114
 159                 run:
 160                     name: tempest_common
 161                     args:
 162                         mode: '(?=.*\[.*\bsmoke\b.*\])(^tempest\.api)'
 163                         option:
 164                             - '--concurrency=3'
 165
 166             -
 167                 case_name: tempest_horizon
 168                 project_name: functest
 169                 criteria: 100
 170                 blocking: false
 171                 description: >-
 172                     This test case runs the Tempest suite proposed by the
 173                     Horizon project.
 174                 dependencies:
 175                     - DASHBOARD_URL: '^(?!\s*$).+'
 176                 run:
 177                     name: tempest_horizon
 178                     args:
 179                         mode: '^tempest_horizon.'
 180
 181     -
 182         name: smoke
 183         order: 1
 184         description: >-
 185             Set of basic Functional tests to validate the OPNFV scenarios.
 186         testcases:
 187             -
 188                 case_name: neutron-tempest-plugin-api
 189                 project_name: functest
 190                 criteria: 100
 191                 blocking: false
 192                 description: >-
 193                     This test case runs the Tempest suite proposed by the
 194                     Neutron project. The list of test cases is generated by
 195                     Tempest automatically and depends on the parameters of
 196                     the OpenStack deployment.
 197                 run:
 198                     name: tempest_common
 199                     args:
 200                         mode: '^neutron_tempest_plugin\.api'
 201                         option:
 202                             - '--concurrency=3'
 203
 204             -
 205                 case_name: tempest_cinder
 206                 project_name: functest
 207                 criteria: 100
 208                 blocking: false
 209                 description: >-
 210                     This test case runs the Tempest suite proposed by the
 211                     Cinder project.
 212                 run:
 213                     name: tempest_common
 214                     args:
 215                         mode:
 216                             '(?!.*test_incremental_backup)(^cinder_tempest_plugin.)'
 217                         option:
 218                             - '--concurrency=3'
 219
 220             -
 221                 case_name: tempest_keystone
 222                 project_name: functest
 223                 criteria: 100
 224                 blocking: false
 225                 description: >-
 226                     This test case runs the Tempest suite proposed by the
 227                     Keystone project.
 228                 run:
 229                     name: tempest_common
 230                     args:
 231                         mode: 'keystone_tempest_plugin.'
 232                         option:
 233                             - '--concurrency=3'
 234
 235             -
 236                 case_name: rally_sanity
 237                 project_name: functest
 238                 criteria: 100
 239                 blocking: false
 240                 description: >-
 241                     This test case runs a sub group of tests of the OpenStack
 242                     Rally suite in smoke mode.
 243                 run:
 244                     name: rally_sanity
 245                     args:
 246                         optional:
 247                             - 'gnocchi'
 248                             - 'barbican'
 249
 250             -
 251                 case_name: refstack_compute
 252                 project_name: functest
 253                 criteria: 100
 254                 blocking: false
 255                 deny_skipping: true
 256                 tests_count: 222
 257                 description: >-
 258                     This test case runs a sub group of tests of the OpenStack
 259                     Compute testcases.
 260                 run:
 261                     name: refstack
 262                     args:
 263                         target: compute
 264                         option:
 265                             - '--concurrency=3'
 266
 267             -
 268                 case_name: refstack_object
 269                 project_name: functest
 270                 criteria: 100
 271                 blocking: false
 272                 deny_skipping: true
 273                 tests_count: 58
 274                 description: >-
 275                     This test case runs a sub group of tests of the OpenStack
 276                     Object testcases.
 277                 run:
 278                     name: refstack
 279                     args:
 280                         target: object
 281                         option:
 282                             - '--concurrency=3'
 283
 284             -
 285                 case_name: refstack_platform
 286                 project_name: functest
 287                 criteria: 100
 288                 blocking: false
 289                 deny_skipping: true
 290                 tests_count: 278
 291                 description: >-
 292                     This test case runs a sub group of tests of the OpenStack
 293                     Platform testcases.
 294                 run:
 295                     name: refstack
 296                     args:
 297                         target: platform
 298                         option:
 299                             - '--concurrency=3'
 300
 301             -
 302                 case_name: tempest_full
 303                 project_name: functest
 304                 criteria: 100
 305                 blocking: false
 306                 description: >-
 307                     The list of test cases is generated by
 308                     Tempest automatically and depends on the parameters of
 309                     the OpenStack deployment.
 310                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83
 311                 run:
 312                     name: tempest_common
 313                     args:
 314                         mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
 315                         option:
 316                             - '--concurrency=3'
 317
 318             -
 319                 case_name: tempest_scenario
 320                 project_name: functest
 321                 criteria: 100
 322                 blocking: false
 323                 description: >-
 324                     The list of test cases is generated by
 325                     Tempest automatically and depends on the parameters of
 326                     the OpenStack deployment.
 327                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
 328                 run:
 329                     name: tempest_scenario
 330                     args:
 331                         mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)'
 332                         option:
 333                             - '--concurrency=1'
 334
 335             -
 336                 case_name: tempest_slow
 337                 project_name: functest
 338                 criteria: 100
 339                 blocking: false
 340                 description: >-
 341                     The list of test cases is generated by
 342                     Tempest automatically and depends on the parameters of
 343                     the OpenStack deployment.
 344                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
 345                 run:
 346                     name: tempest_common
 347                     args:
 348                         mode: '(?=.*\[.*\bslow\b.*\])(^tempest\.)'
 349                         option:
 350                             - '--concurrency=1'
 351
 352             -
 353                 case_name: patrole
 354                 project_name: functest
 355                 criteria: 100
 356                 blocking: false
 357                 description: >-
 358                     Test suite from Patrole project.
 359                 run:
 360                     name: patrole
 361                     args:
 362                         role: 'admin'
 363                         includes:
 364                             - 'image'
 365                             - 'network'
 366                         excludes:
 367                             - 'network.test_flavors_rbac'
 368                             - 'network.test_network_ip_availability_rbac'
 369                             - 'test_networks_multiprovider_rbac'
 370                             - 'test_auto_allocated_topology_rbac'
 371                             - 'test_create_network_provider_segmentation_id'
 372                             - 'test_update_network_provider_physical_network'
 373                             - 'test_update_network_provider_segmentation_id'
 374                             - 'network.test_agents_rbac'
 375                             - 'test_list_metadef_namespaces'
 376
 377             -
 378                 case_name: neutron_trunk
 379                 project_name: functest
 380                 criteria: 100
 381                 blocking: false
 382                 description: >-
 383                     This test case runs the neutron trunk subtest of the
 384                     OpenStack Tempest suite. The list of test cases is
 385                     generated by Tempest having as input the relevant
 386                     testcase list file.
 387                 run:
 388                     name: tempest_common
 389                     args:
 390                         mode: 'neutron_tempest_plugin.(api|scenario).test_trunk'
 391                         neutron_extensions:
 392                             - trunk
 393                             - trunk-details
 394
 395             -
 396                 case_name: networking-bgpvpn
 397                 project_name: functest
 398                 criteria: 100
 399                 blocking: false
 400                 description: >-
 401                     This test case runs OpenStack networking-bgpvpn Tempest
 402                     suite. The list of test cases is generated by Tempest
 403                     having as input the relevant regex.
 404                 run:
 405                     name: tempest_common
 406                     args:
 407                         mode: '^neutron_tempest_plugin.bgpvpn.api\.'
 408                         neutron_extensions:
 409                             - bgpvpn
 410                         option:
 411                             - '--concurrency=3'
 412
 413             -
 414                 case_name: networking-sfc
 415                 project_name: functest
 416                 criteria: 100
 417                 blocking: false
 418                 description: >-
 419                     This test case runs OpenStack networking-sfc Tempest
 420                     suite. The list of test cases is generated by Tempest
 421                     having as input the relevant regex.
 422                 run:
 423                     name: tempest_common
 424                     args:
 425                         mode:
 426                             '^(?:neutron_tempest_plugin\.sfc\.tests\.api).*$'
 427                         neutron_extensions:
 428                             - sfc
 429                         option:
 430                             - '--concurrency=0'
 431
 432             -
 433                 case_name: barbican
 434                 project_name: functest
 435                 criteria: 100
 436                 blocking: false
 437                 description: >-
 438                     It leverages on the tempest plugin containing tests used to
 439                     verify the functionality of a barbican installation.
 440                 run:
 441                     name: barbican
 442                     args:
 443                         mode:
 444                             '^barbican_tempest_plugin.((?!test_signed_image_upload_boot_failure).)*$'
 445                         services:
 446                             - barbican
 447
 448             -
 449                 case_name: octavia
 450                 project_name: functest
 451                 criteria: 100
 452                 blocking: false
 453                 description: >-
 454                     It leverages on the tempest plugin containing tests used to
 455                     verify the functionality of an octavia installation.
 456                 run:
 457                     name: tempest_common
 458                     args:
 459                         mode:
 460                             '^octavia_tempest_plugin.tests.api.v2'
 461                         services:
 462                             - octavia
 463                         option:
 464                             - '--concurrency=2'
 465
 466     -
 467         name: benchmarking
 468         order: 2
 469         description: >-
 470             Run several OpenStack performance tools
 471             https://docs.openstack.org/performance-docs/latest/methodologies/tools.html
 472         testcases:
 473             -
 474                 case_name: rally_full
 475                 project_name: functest
 476                 criteria: 100
 477                 blocking: false
 478                 description: >-
 479                     This test case runs the full suite of scenarios of the
 480                     OpenStack Rally suite using several threads and iterations.
 481                 run:
 482                     name: rally_full
 483                     args:
 484                         optional:
 485                             - 'gnocchi'
 486                             - 'barbican'
 487
 488             -
 489                 case_name: rally_jobs
 490                 project_name: functest
 491                 criteria: 100
 492                 blocking: false
 493                 description: >-
 494                     This test case runs a group of Rally jobs used in
 495                     OpenStack gating
 496                 run:
 497                     name: rally_jobs
 498                     args:
 499                         optional:
 500                             - 'gnocchi'
 501
 502             -
 503                 case_name: vmtp
 504                 project_name: functest
 505                 criteria: 100
 506                 blocking: false
 507                 description: >-
 508                     VMTP is a small python application that will automatically
 509                     perform ping connectivity, round trip time measurement
 510                     (latency) and TCP/UDP throughput
 511                 dependencies:
 512                     - POD_ARCH: '^(?!aarch64$)'
 513                 run:
 514                     name: vmtp
 515
 516             -
 517                 case_name: shaker
 518                 project_name: functest
 519                 criteria: 100
 520                 blocking: false
 521                 description: >-
 522                     Shaker wraps around popular system network testing tools
 523                     like iperf, iperf3 and netperf (with help of flent). Shaker
 524                     is able to deploy OpenStack instances and networks in
 525                     different topologies.
 526                 dependencies:
 527                     - POD_ARCH: '^(?!aarch64$)'
 528                 run:
 529                     name: shaker
 530
 531     -
 532         name: vnf
 533         order: 4
 534         description: >-
 535             Collection of VNF test cases.
 536         testcases:
 537             -
 538                 case_name: cloudify
 539                 project_name: functest
 540                 criteria: 100
 541                 blocking: false
 542                 description: >-
 543                     This test case deploys the Cloudify orchestrator.
 544                 run:
 545                     name: cloudify
 546             -
 547                 case_name: cloudify_ims
 548                 project_name: functest
 549                 criteria: 100
 550                 blocking: false
 551                 description: >-
 552                     This test case deploys an OpenSource vIMS solution from
 553                     Clearwater using the Cloudify orchestrator. It also runs
 554                     some signaling traffic.
 555                 dependencies:
 556                     - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha'
 557                 run:
 558                     name: cloudify_ims
 559
 560             -
 561                 case_name: heat_ims
 562                 project_name: functest
 563                 criteria: 100
 564                 blocking: false
 565                 description: >-
 566                     This test case deploys an OpenSource vIMS solution from
 567                     Clearwater using the OpenStack Heat orchestrator.
 568                     It also runs some signaling traffic.
 569                 run:
 570                     name: heat_ims
 571
 572             -
 573                 case_name: vyos_vrouter
 574                 project_name: functest
 575                 criteria: 100
 576                 blocking: false
 577                 description: >-
 578                     This test case is vRouter testing.
 579                 run:
 580                     name: vyos_vrouter
 581
 582             -
 583                 case_name: juju_epc
 584                 project_name: functest
 585                 criteria: 100
 586                 blocking: false
 587                 description: >-
 588                     vEPC validation with Juju as VNF manager and ABoT as test
 589                     executor.
 590                 run:
 591                     name: juju_epc