functest/ci/testcases.yaml

   1 ---
   2 tiers:
   3     -
   4         name: healthcheck
   5         order: 0
   6         description: >-
   7             First tier to be executed to verify the basic
   8             operations in the VIM.
   9         testcases:
  10             -
  11                 case_name: connection_check
  12                 project_name: functest
  13                 criteria: 100
  14                 blocking: true
  15                 description: >-
  16                     This test case verifies the retrieval of OpenStack clients:
  17                     Keystone, Glance, Neutron and Nova and may perform some
  18                     simple queries.
  19                 run:
  20                     name: connection_check
  21
  22             -
  23                 case_name: tenantnetwork1
  24                 project_name: functest
  25                 criteria: 100
  26                 blocking: true
  27                 description: >-
  28                     It creates and configures all tenant network ressources
  29                     required by advanced testcases (subnet, network and
  30                     router).
  31                 run:
  32                     name: tenantnetwork1
  33
  34             -
  35                 case_name: tenantnetwork2
  36                 project_name: functest
  37                 criteria: 100
  38                 blocking: true
  39                 description: >-
  40                     It creates new user/project before creating and configuring
  41                     all tenant network ressources required by a testcase
  42                     (subnet, network and router).
  43                 run:
  44                     name: tenantnetwork2
  45
  46             -
  47                 case_name: vmready1
  48                 project_name: functest
  49                 criteria: 100
  50                 blocking: true
  51                 description: >-
  52                     It inherits from TenantNetwork1 which creates all network
  53                     resources and prepares a future VM attached to that
  54                     network.
  55                 run:
  56                     name: vmready1
  57
  58             -
  59                 case_name: vmready2
  60                 project_name: functest
  61                 criteria: 100
  62                 blocking: true
  63                 description: >-
  64                     It creates new user/project before creating and configuring
  65                     all tenant network ressources, flavors, images, etc.
  66                     required by advanced testcases.
  67                 run:
  68                     name: vmready2
  69
  70             -
  71                 case_name: singlevm1
  72                 project_name: functest
  73                 criteria: 100
  74                 blocking: true
  75                 description: >-
  76                     It inherits from TenantNetwork1 which creates all network
  77                     resources and completes it by booting a VM attached to that
  78                     network.
  79                 run:
  80                     name: singlevm1
  81
  82             -
  83                 case_name: singlevm2
  84                 project_name: functest
  85                 criteria: 100
  86                 blocking: true
  87                 description: >-
  88                     It creates new user/project before creating and configuring
  89                     all tenant network ressources and vms required by advanced
  90                     testcases.
  91                 run:
  92                     name: singlevm2
  93
  94             -
  95                 case_name: vping_ssh
  96                 project_name: functest
  97                 criteria: 100
  98                 blocking: true
  99                 description: >-
 100                     This test case verifies: 1) SSH to an instance using
 101                     floating IPs over the public network. 2) Connectivity
 102                     between 2 instances over a private network.
 103                 run:
 104                     name: vping_ssh
 105
 106             -
 107                 case_name: vping_userdata
 108                 project_name: functest
 109                 criteria: 100
 110                 blocking: true
 111                 description: >-
 112                     This test case verifies:  1) Boot a VM with given userdata.
 113                     2) Connectivity between 2 instances over a private network.
 114                 run:
 115                     name: vping_userdata
 116
 117             -
 118                 case_name: cinder_test
 119                 project_name: functest
 120                 criteria: 100
 121                 blocking: true
 122                 description: >-
 123                     This test case verifies: 1) Attach volume and to 1
 124                     instance; 2) Write data on volume 3) Detach volume
 125                     from instance 1, attach it on instance 2 3) Read volume
 126                     data
 127                 run:
 128                     name: cinder_test
 129
 130             -
 131                 case_name: odl
 132                 project_name: functest
 133                 criteria: 100
 134                 blocking: true
 135                 description: >-
 136                     Test Suite for the OpenDaylight SDN Controller. It
 137                     integrates some test suites from upstream using
 138                     Robot as the test framework.
 139                 dependencies:
 140                     - DEPLOY_SCENARIO: 'odl'
 141                 run:
 142                     name: odl
 143                     args:
 144                         suites:
 145                             - /src/odl_test/csit/suites/integration/basic
 146                             - /src/odl_test/csit/suites/openstack/neutron
 147
 148             -
 149                 case_name: tempest_smoke
 150                 project_name: functest
 151                 criteria: 100
 152                 blocking: false
 153                 description: >-
 154                     This test case runs the smoke subset of the OpenStack
 155                     Tempest suite. The list of test cases is generated by
 156                     Tempest automatically and depends on the parameters of
 157                     the OpenStack deplopyment.
 158                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L114
 159                 run:
 160                     name: tempest_common
 161                     args:
 162                         mode: '(?=.*\[.*\bsmoke\b.*\])(^tempest\.api)'
 163                         option:
 164                             - '--concurrency=4'
 165
 166     -
 167         name: smoke
 168         order: 1
 169         description: >-
 170             Set of basic Functional tests to validate the OPNFV scenarios.
 171         testcases:
 172             -
 173                 case_name: neutron-tempest-plugin-api
 174                 project_name: functest
 175                 criteria: 100
 176                 blocking: false
 177                 description: >-
 178                     This test case runs the Tempest suite proposed by the
 179                     Neutron project. The list of test cases is generated by
 180                     Tempest automatically and depends on the parameters of
 181                     the OpenStack deployment.
 182                 run:
 183                     name: tempest_common
 184                     args:
 185                         mode: '^neutron_tempest_plugin\.api'
 186                         option:
 187                             - '--concurrency=4'
 188
 189             -
 190                 case_name: rally_sanity
 191                 project_name: functest
 192                 criteria: 100
 193                 blocking: false
 194                 description: >-
 195                     This test case runs a sub group of tests of the OpenStack
 196                     Rally suite in smoke mode.
 197                 run:
 198                     name: rally_sanity
 199                     args:
 200                         optional:
 201                             - 'gnocchi'
 202                             - 'barbican'
 203
 204             -
 205                 case_name: refstack_compute
 206                 project_name: functest
 207                 criteria: 100
 208                 blocking: false
 209                 deny_skipping: true
 210                 tests_count: 219
 211                 description: >-
 212                     This test case runs a sub group of tests of the OpenStack
 213                     Compute testcases.
 214                 run:
 215                     name: refstack
 216                     args:
 217                         target: compute
 218                         option:
 219                             - '--concurrency=4'
 220
 221             -
 222                 case_name: refstack_object
 223                 project_name: functest
 224                 criteria: 100
 225                 blocking: false
 226                 deny_skipping: true
 227                 tests_count: 57
 228                 description: >-
 229                     This test case runs a sub group of tests of the OpenStack
 230                     Object testcases.
 231                 run:
 232                     name: refstack
 233                     args:
 234                         target: object
 235                         option:
 236                             - '--concurrency=4'
 237
 238             -
 239                 case_name: refstack_platform
 240                 project_name: functest
 241                 criteria: 100
 242                 blocking: false
 243                 deny_skipping: true
 244                 tests_count: 275
 245                 description: >-
 246                     This test case runs a sub group of tests of the OpenStack
 247                     Platform testcases.
 248                 run:
 249                     name: refstack
 250                     args:
 251                         target: platform
 252                         option:
 253                             - '--concurrency=4'
 254
 255             -
 256                 case_name: tempest_full
 257                 project_name: functest
 258                 criteria: 100
 259                 blocking: false
 260                 description: >-
 261                     The list of test cases is generated by
 262                     Tempest automatically and depends on the parameters of
 263                     the OpenStack deployment.
 264                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83
 265                 run:
 266                     name: tempest_common
 267                     args:
 268                         mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
 269                         option:
 270                             - '--concurrency=4'
 271
 272             -
 273                 case_name: tempest_scenario
 274                 project_name: functest
 275                 criteria: 100
 276                 blocking: false
 277                 description: >-
 278                     The list of test cases is generated by
 279                     Tempest automatically and depends on the parameters of
 280                     the OpenStack deployment.
 281                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
 282                 run:
 283                     name: tempest_scenario
 284                     args:
 285                         mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)'
 286                         option:
 287                             - '--concurrency=1'
 288
 289             -
 290                 case_name: tempest_slow
 291                 project_name: functest
 292                 criteria: 100
 293                 blocking: false
 294                 description: >-
 295                     The list of test cases is generated by
 296                     Tempest automatically and depends on the parameters of
 297                     the OpenStack deployment.
 298                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
 299                 run:
 300                     name: tempest_common
 301                     args:
 302                         mode: '(?=.*\[.*\bslow\b.*\])(^tempest\.)'
 303                         option:
 304                             - '--concurrency=2'
 305
 306             -
 307                 case_name: patrole
 308                 project_name: functest
 309                 criteria: 100
 310                 blocking: false
 311                 description: >-
 312                     Test suite from Patrole project.
 313                 run:
 314                     name: patrole
 315                     args:
 316                         role: 'admin'
 317                         includes:
 318                             - 'image'
 319                             - 'network'
 320                         excludes:
 321                             - 'network.test_flavors_rbac'
 322                             - 'network.test_network_ip_availability_rbac'
 323                             - 'test_networks_multiprovider_rbac'
 324                             - 'test_auto_allocated_topology_rbac'
 325                             - 'test_create_network_provider_segmentation_id'
 326                             - 'test_update_network_provider_physical_network'
 327                             - 'test_update_network_provider_segmentation_id'
 328                             - 'network.test_agents_rbac'
 329                             - 'test_list_metadef_namespaces'
 330
 331             -
 332                 case_name: neutron_trunk
 333                 project_name: functest
 334                 criteria: 100
 335                 blocking: false
 336                 description: >-
 337                     This test case runs the neutron trunk subtest of the
 338                     OpenStack Tempest suite. The list of test cases is
 339                     generated by Tempest having as input the relevant
 340                     testcase list file.
 341                 run:
 342                     name: tempest_common
 343                     args:
 344                         mode: 'neutron_tempest_plugin.(api|scenario).test_trunk'
 345                         neutron_extensions:
 346                             - trunk
 347                             - trunk-details
 348
 349             -
 350                 case_name: networking-bgpvpn
 351                 project_name: functest
 352                 criteria: 100
 353                 blocking: false
 354                 description: >-
 355                     This test case runs OpenStack networking-bgpvpn Tempest
 356                     suite. The list of test cases is generated by Tempest
 357                     having as input the relevant regex.
 358                 run:
 359                     name: tempest_common
 360                     args:
 361                         mode: '^neutron_tempest_plugin.bgpvpn.api\.'
 362                         neutron_extensions:
 363                             - bgpvpn
 364                         option:
 365                             - '--concurrency=4'
 366
 367             -
 368                 case_name: networking-sfc
 369                 project_name: functest
 370                 criteria: 100
 371                 blocking: false
 372                 description: >-
 373                     This test case runs OpenStack networking-sfc Tempest
 374                     suite. The list of test cases is generated by Tempest
 375                     having as input the relevant regex.
 376                 run:
 377                     name: tempest_common
 378                     args:
 379                         mode:
 380                             '^(?:neutron_tempest_plugin\.sfc\.tests\.api).*$'
 381                         neutron_extensions:
 382                             - sfc
 383                         option:
 384                             - '--concurrency=0'
 385
 386             -
 387                 case_name: barbican
 388                 project_name: functest
 389                 criteria: 100
 390                 blocking: false
 391                 description: >-
 392                     It leverages on the tempest plugin containing tests used to
 393                     verify the functionality of a barbican installation.
 394                 run:
 395                     name: barbican
 396                     args:
 397                         mode:
 398                             '^barbican_tempest_plugin.((?!test_signed_image_upload_boot_failure).)*$'
 399                         services:
 400                             - barbican
 401
 402             -
 403                 case_name: octavia
 404                 project_name: functest
 405                 criteria: 100
 406                 blocking: false
 407                 description: >-
 408                     It leverages on the tempest plugin containing tests used to
 409                     verify the functionality of an octavia installation.
 410                 run:
 411                     name: tempest_common
 412                     args:
 413                         mode:
 414                             '^octavia_tempest_plugin.tests.api.v2'
 415                         services:
 416                             - octavia
 417                         option:
 418                             - '--concurrency=2'
 419
 420     -
 421         name: benchmarking
 422         order: 2
 423         description: >-
 424             Run several OpenStack performance tools
 425             https://docs.openstack.org/performance-docs/latest/methodologies/tools.html
 426         testcases:
 427             -
 428                 case_name: rally_full
 429                 project_name: functest
 430                 criteria: 100
 431                 blocking: false
 432                 description: >-
 433                     This test case runs the full suite of scenarios of the
 434                     OpenStack Rally suite using several threads and iterations.
 435                 run:
 436                     name: rally_full
 437                     args:
 438                         optional:
 439                             - 'gnocchi'
 440                             - 'barbican'
 441
 442             -
 443                 case_name: rally_jobs
 444                 project_name: functest
 445                 criteria: 100
 446                 blocking: false
 447                 description: >-
 448                     This test case runs a group of Rally jobs used in
 449                     OpenStack gating
 450                 run:
 451                     name: rally_jobs
 452                     args:
 453                         optional:
 454                             - 'gnocchi'
 455
 456             -
 457                 case_name: vmtp
 458                 project_name: functest
 459                 criteria: 100
 460                 blocking: false
 461                 description: >-
 462                     VMTP is a small python application that will automatically
 463                     perform ping connectivity, round trip time measurement
 464                     (latency) and TCP/UDP throughput
 465                 dependencies:
 466                     - POD_ARCH: '^(?!aarch64$)'
 467                 run:
 468                     name: vmtp
 469
 470             -
 471                 case_name: shaker
 472                 project_name: functest
 473                 criteria: 100
 474                 blocking: false
 475                 description: >-
 476                     Shaker wraps around popular system network testing tools
 477                     like iperf, iperf3 and netperf (with help of flent). Shaker
 478                     is able to deploy OpenStack instances and networks in
 479                     different topologies.
 480                 dependencies:
 481                     - POD_ARCH: '^(?!aarch64$)'
 482                 run:
 483                     name: shaker
 484
 485     -
 486         name: vnf
 487         order: 4
 488         description: >-
 489             Collection of VNF test cases.
 490         testcases:
 491             -
 492                 case_name: cloudify
 493                 project_name: functest
 494                 criteria: 100
 495                 blocking: false
 496                 description: >-
 497                     This test case deploys the Cloudify orchestrator.
 498                 run:
 499                     name: cloudify
 500             -
 501                 case_name: cloudify_ims
 502                 project_name: functest
 503                 criteria: 100
 504                 blocking: false
 505                 description: >-
 506                     This test case deploys an OpenSource vIMS solution from
 507                     Clearwater using the Cloudify orchestrator. It also runs
 508                     some signaling traffic.
 509                 dependencies:
 510                     - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha'
 511                 run:
 512                     name: cloudify_ims
 513
 514             -
 515                 case_name: heat_ims
 516                 project_name: functest
 517                 criteria: 100
 518                 blocking: false
 519                 description: >-
 520                     This test case deploys an OpenSource vIMS solution from
 521                     Clearwater using the OpenStack Heat orchestrator.
 522                     It also runs some signaling traffic.
 523                 run:
 524                     name: heat_ims
 525
 526             -
 527                 case_name: vyos_vrouter
 528                 project_name: functest
 529                 criteria: 100
 530                 blocking: false
 531                 description: >-
 532                     This test case is vRouter testing.
 533                 run:
 534                     name: vyos_vrouter
 535
 536             -
 537                 case_name: juju_epc
 538                 project_name: functest
 539                 criteria: 100
 540                 blocking: false
 541                 description: >-
 542                     vEPC validation with Juju as VNF manager and ABoT as test
 543                     executor.
 544                 run:
 545                     name: juju_epc