functest/ci/testcases.yaml

   1 ---
   2 tiers:
   3     -
   4         name: healthcheck
   5         order: 0
   6         description: >-
   7             First tier to be executed to verify the basic
   8             operations in the VIM.
   9         testcases:
  10             -
  11                 case_name: connection_check
  12                 project_name: functest
  13                 criteria: 100
  14                 blocking: true
  15                 description: >-
  16                     This test case verifies the retrieval of OpenStack clients:
  17                     Keystone, Glance, Neutron and Nova and may perform some
  18                     simple queries.
  19                 run:
  20                     module:
  21                         'functest.opnfv_tests.openstack.api.connection_check'
  22                     class: 'ConnectionCheck'
  23
  24             -
  25                 case_name: tenantnetwork1
  26                 project_name: functest
  27                 criteria: 100
  28                 blocking: true
  29                 description: >-
  30                     It creates and configures all tenant network ressources
  31                     required by advanced testcases (subnet, network and
  32                     router).
  33                 run:
  34                     module: 'functest.core.tenantnetwork'
  35                     class: 'TenantNetwork1'
  36
  37             -
  38                 case_name: tenantnetwork2
  39                 project_name: functest
  40                 criteria: 100
  41                 blocking: true
  42                 description: >-
  43                     It creates new user/project before creating and configuring
  44                     all tenant network ressources required by a testcase
  45                     (subnet, network and router).
  46                 run:
  47                     module: 'functest.core.tenantnetwork'
  48                     class: 'TenantNetwork2'
  49
  50             -
  51                 case_name: vmready1
  52                 project_name: functest
  53                 criteria: 100
  54                 blocking: true
  55                 description: >-
  56                     It inherits from TenantNetwork1 which creates all network
  57                     resources and prepares a future VM attached to that
  58                     network.
  59                 run:
  60                     module: 'functest.core.singlevm'
  61                     class: 'VmReady1'
  62
  63             -
  64                 case_name: vmready2
  65                 project_name: functest
  66                 criteria: 100
  67                 blocking: true
  68                 description: >-
  69                     It creates new user/project before creating and configuring
  70                     all tenant network ressources, flavors, images, etc.
  71                     required by advanced testcases.
  72                 run:
  73                     module: 'functest.core.singlevm'
  74                     class: 'VmReady2'
  75
  76             -
  77                 case_name: singlevm1
  78                 project_name: functest
  79                 criteria: 100
  80                 blocking: true
  81                 description: >-
  82                     It inherits from TenantNetwork1 which creates all network
  83                     resources and completes it by booting a VM attached to that
  84                     network.
  85                 run:
  86                     module: 'functest.core.singlevm'
  87                     class: 'SingleVm1'
  88
  89             -
  90                 case_name: singlevm2
  91                 project_name: functest
  92                 criteria: 100
  93                 blocking: true
  94                 description: >-
  95                     It creates new user/project before creating and configuring
  96                     all tenant network ressources and vms required by advanced
  97                     testcases.
  98                 run:
  99                     module: 'functest.core.singlevm'
 100                     class: 'SingleVm2'
 101
 102             -
 103                 case_name: vping_ssh
 104                 project_name: functest
 105                 criteria: 100
 106                 blocking: true
 107                 description: >-
 108                     This test case verifies: 1) SSH to an instance using
 109                     floating IPs over the public network. 2) Connectivity
 110                     between 2 instances over a private network.
 111                 run:
 112                     module: 'functest.opnfv_tests.openstack.vping.vping_ssh'
 113                     class: 'VPingSSH'
 114
 115             -
 116                 case_name: vping_userdata
 117                 project_name: functest
 118                 criteria: 100
 119                 blocking: true
 120                 description: >-
 121                     This test case verifies:  1) Boot a VM with given userdata.
 122                     2) Connectivity between 2 instances over a private network.
 123                 run:
 124                     module:
 125                         'functest.opnfv_tests.openstack.vping.vping_userdata'
 126                     class: 'VPingUserdata'
 127
 128             -
 129                 case_name: cinder_test
 130                 project_name: functest
 131                 criteria: 100
 132                 blocking: true
 133                 description: >-
 134                     This test case verifies: 1) Attach volume and to 1
 135                     instance; 2) Write data on volume 3) Detach volume
 136                     from instance 1, attach it on instance 2 3) Read volume
 137                     data
 138                 run:
 139                     module: 'functest.opnfv_tests.openstack.cinder.cinder_test'
 140                     class: 'CinderCheck'
 141
 142             -
 143                 case_name: odl
 144                 project_name: functest
 145                 criteria: 100
 146                 blocking: true
 147                 description: >-
 148                     Test Suite for the OpenDaylight SDN Controller. It
 149                     integrates some test suites from upstream using
 150                     Robot as the test framework.
 151                 dependencies:
 152                     - DEPLOY_SCENARIO: 'odl'
 153                 run:
 154                     module: 'functest.opnfv_tests.sdn.odl.odl'
 155                     class: 'ODLTests'
 156                     args:
 157                         suites:
 158                             - /src/odl_test/csit/suites/integration/basic
 159                             - /src/odl_test/csit/suites/openstack/neutron
 160
 161             -
 162                 case_name: api_check
 163                 project_name: functest
 164                 criteria: 100
 165                 blocking: true
 166                 description: >-
 167                     This test case verifies the retrieval of OpenStack clients:
 168                     Keystone, Glance, Neutron and Nova and may perform some
 169                     simple queries. When the config value of
 170                     snaps.use_keystone is True, functest must have access to
 171                     the cloud's private network.
 172                 run:
 173                     module: 'functest.opnfv_tests.openstack.snaps.api_check'
 174                     class: 'ApiCheck'
 175
 176             -
 177                 case_name: snaps_health_check
 178                 project_name: functest
 179                 criteria: 100
 180                 blocking: true
 181                 description: >-
 182                     This test case creates executes the SimpleHealthCheck
 183                     Python test class which creates an, image, flavor, network,
 184                     and Cirros VM instance and observes the console output to
 185                     validate the single port obtains the correct IP address.
 186                 run:
 187                     module: 'functest.opnfv_tests.openstack.snaps.health_check'
 188                     class: 'HealthCheck'
 189
 190     -
 191         name: smoke
 192         order: 1
 193         description: >-
 194             Set of basic Functional tests to validate the OPNFV scenarios.
 195         testcases:
 196             -
 197                 case_name: tempest_smoke
 198                 project_name: functest
 199                 criteria: 100
 200                 blocking: false
 201                 description: >-
 202                     This test case runs the smoke subset of the OpenStack
 203                     Tempest suite. The list of test cases is generated by
 204                     Tempest automatically and depends on the parameters of
 205                     the OpenStack deplopyment.
 206                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L114
 207                 run:
 208                     module: 'functest.opnfv_tests.openstack.tempest.tempest'
 209                     class: 'TempestCommon'
 210                     args:
 211                         mode: '(?=.*\[.*\bsmoke\b.*\])(^tempest\.api)'
 212                         option:
 213                             - '--concurrency=4'
 214
 215             -
 216                 case_name: neutron-tempest-plugin-api
 217                 project_name: functest
 218                 criteria: 100
 219                 blocking: false
 220                 description: >-
 221                     This test case runs the Tempest suite proposed by the
 222                     Neutron project. The list of test cases is generated by
 223                     Tempest automatically and depends on the parameters of
 224                     the OpenStack deployment.
 225                 run:
 226                     module: 'functest.opnfv_tests.openstack.tempest.tempest'
 227                     class: 'TempestCommon'
 228                     args:
 229                         mode: '^neutron_tempest_plugin\.api'
 230                         option:
 231                             - '--concurrency=4'
 232
 233             -
 234                 case_name: rally_sanity
 235                 project_name: functest
 236                 criteria: 100
 237                 blocking: false
 238                 description: >-
 239                     This test case runs a sub group of tests of the OpenStack
 240                     Rally suite in smoke mode.
 241                 run:
 242                     module: 'functest.opnfv_tests.openstack.rally.rally'
 243                     class: 'RallySanity'
 244
 245             -
 246                 case_name: refstack_defcore
 247                 project_name: functest
 248                 criteria: 100
 249                 blocking: false
 250                 description: >-
 251                     This test case runs a sub group of tests of the OpenStack
 252                     Defcore testcases.
 253                 run:
 254                     module:
 255                         'functest.opnfv_tests.openstack.refstack.refstack'
 256                     class: 'Refstack'
 257                     args:
 258                         option:
 259                             - '--concurrency=4'
 260
 261             -
 262                 case_name: patrole
 263                 project_name: functest
 264                 criteria: 100
 265                 blocking: false
 266                 description: >-
 267                     Test suite from Patrole project.
 268                 run:
 269                     module: 'functest.opnfv_tests.openstack.patrole.patrole'
 270                     class: 'Patrole'
 271                     args:
 272                         role: 'admin'
 273                         includes:
 274                             - 'image'
 275                             - 'network'
 276                         excludes:
 277                             - 'test_networks_multiprovider_rbac'
 278
 279             -
 280                 case_name: vmtp
 281                 project_name: functest
 282                 criteria: 100
 283                 blocking: false
 284                 description: >-
 285                     VMTP is a small python application that will automatically
 286                     perform ping connectivity, round trip time measurement
 287                     (latency) and TCP/UDP throughput
 288                 dependencies:
 289                     - POD_ARCH: '^(?!aarch64$)'
 290                 run:
 291                     module:
 292                         'functest.opnfv_tests.openstack.vmtp.vmtp'
 293                     class: 'Vmtp'
 294
 295             -
 296                 case_name: shaker
 297                 project_name: functest
 298                 criteria: 100
 299                 blocking: false
 300                 description: >-
 301                     Shaker wraps around popular system network testing tools
 302                     like iperf, iperf3 and netperf (with help of flent). Shaker
 303                     is able to deploy OpenStack instances and networks in
 304                     different topologies.
 305                 run:
 306                     module:
 307                         'functest.opnfv_tests.openstack.shaker.shaker'
 308                     class: 'Shaker'
 309
 310             -
 311                 case_name: snaps_smoke
 312                 project_name: functest
 313                 criteria: 100
 314                 blocking: false
 315                 description: >-
 316                     This test case contains tests that setup and destroy
 317                     environments with VMs with and without Floating IPs
 318                     with a newly created user and project. Set the config
 319                     value snaps.use_floating_ips (True|False) to toggle
 320                     this functionality. When the config value of
 321                     snaps.use_keystone is True, functest must have access to
 322                     the cloud's private network.
 323                 run:
 324                     module: 'functest.opnfv_tests.openstack.snaps.smoke'
 325                     class: 'SnapsSmoke'
 326
 327             -
 328                 case_name: neutron_trunk
 329                 project_name: functest
 330                 criteria: 100
 331                 blocking: false
 332                 description: >-
 333                     This test case runs the neutron trunk subtest of the
 334                     OpenStack Tempest suite. The list of test cases is
 335                     generated by Tempest having as input the relevant
 336                     testcase list file.
 337                 run:
 338                     module: 'functest.opnfv_tests.openstack.tempest.tempest'
 339                     class: 'TempestCommon'
 340                     args:
 341                         mode: 'neutron_tempest_plugin.(api|scenario).test_trunk'
 342                         neutron_extensions:
 343                             - trunk
 344                             - trunk-details
 345
 346             -
 347                 case_name: networking-bgpvpn
 348                 project_name: functest
 349                 criteria: 100
 350                 blocking: false
 351                 description: >-
 352                     This test case runs OpenStack networking-bgpvpn Tempest
 353                     suite. The list of test cases is generated by Tempest
 354                     having as input the relevant regex.
 355                 run:
 356                     module: 'functest.opnfv_tests.openstack.tempest.tempest'
 357                     class: 'TempestCommon'
 358                     args:
 359                         mode: '^networking_bgpvpn_tempest\.'
 360                         neutron_extensions:
 361                             - bgpvpn
 362                         option:
 363                             - '--concurrency=4'
 364
 365             -
 366                 case_name: networking-sfc
 367                 project_name: functest
 368                 criteria: 100
 369                 blocking: false
 370                 description: >-
 371                     This test case runs OpenStack networking-sfc Tempest
 372                     suite. The list of test cases is generated by Tempest
 373                     having as input the relevant regex.
 374                 run:
 375                     module: 'functest.opnfv_tests.openstack.tempest.tempest'
 376                     class: 'TempestCommon'
 377                     args:
 378                         mode:
 379                             '^(?:networking_sfc\.tests\.tempest_plugin.tests.api).*$'
 380                         neutron_extensions:
 381                             - sfc
 382                         option:
 383                             - '--concurrency=0'
 384
 385             -
 386                 case_name: barbican
 387                 project_name: functest
 388                 criteria: 100
 389                 blocking: false
 390                 description: >-
 391                     It leverages on the tempest plugin containing tests used to
 392                     verify the functionality of a barbican installation.
 393                 run:
 394                     module: 'functest.opnfv_tests.openstack.tempest.tempest'
 395                     class: 'TempestCommon'
 396                     args:
 397                         mode:
 398                             'barbican_tempest_plugin.tests.(api|scenario).(?!test_image_signing)'
 399                         services:
 400                             - barbican
 401
 402     -
 403         name: features
 404         order: 2
 405         description: >-
 406             Test suites from feature projects
 407             integrated in functest
 408         testcases:
 409             -
 410                 case_name: doctor-notification
 411                 project_name: doctor
 412                 criteria: 100
 413                 blocking: false
 414                 description: >-
 415                     Test suite from Doctor project.
 416                 dependencies:
 417                     - INSTALLER_TYPE: '(apex)|(fuel)|(daisy)'
 418                     - DEPLOY_SCENARIO: '^((?!fdio|nofeature).)*$'
 419                 run:
 420                     module: 'xtesting.core.feature'
 421                     class: 'BashFeature'
 422                     args:
 423                         cmd: 'doctor-test'
 424
 425             -
 426                 case_name: bgpvpn
 427                 project_name: sdnvpn
 428                 criteria: 100
 429                 blocking: false
 430                 description: >-
 431                     Test suite from SDNVPN project.
 432                 dependencies:
 433                     - DEPLOY_SCENARIO: 'bgpvpn'
 434                 run:
 435                     module: 'sdnvpn.test.functest.run_sdnvpn_tests'
 436                     class: 'SdnvpnFunctest'
 437
 438             -
 439                 case_name: functest-odl-sfc
 440                 project_name: sfc
 441                 criteria: 100
 442                 blocking: false
 443                 description: >-
 444                     Test suite for odl-sfc to test two chains with one SF and
 445                     one chain with two SFs
 446                 dependencies:
 447                     - DEPLOY_SCENARIO: 'odl.*sfc'
 448                 run:
 449                     module: 'sfc.tests.functest.run_sfc_tests'
 450                     class: 'SfcFunctest'
 451
 452             -
 453                 case_name: barometercollectd
 454                 project_name: barometer
 455                 criteria: 100
 456                 blocking: false
 457                 description: >-
 458                     Test suite for the Barometer project. Separate tests verify
 459                     the proper configuration and basic functionality of all the
 460                     collectd plugins as described in the Project Release Plan
 461                 dependencies:
 462                     - DEPLOY_SCENARIO: 'bar'
 463                 run:
 464                     module: 'baro_tests.barometer'
 465                     class: 'BarometerCollectd'
 466
 467             -
 468                 case_name: fds
 469                 project_name: fastdatastacks
 470                 criteria: 100
 471                 blocking: false
 472                 description: >-
 473                     Test Suite for the OpenDaylight SDN Controller when GBP
 474                     features are installed. It integrates some test suites from
 475                     upstream using Robot as the test framework.
 476                 dependencies:
 477                     - DEPLOY_SCENARIO: 'odl.*-fdio'
 478                 run:
 479                     module: 'functest.opnfv_tests.sdn.odl.odl'
 480                     class: 'ODLTests'
 481                     args:
 482                         suites:
 483                             - /src/fds/testing/robot
 484             -
 485                 case_name: vgpu
 486                 project_name: functest
 487                 criteria: 100
 488                 blocking: false
 489                 description: >-
 490                     Test suite for the OpenStack vGPU feature
 491                 dependencies:
 492                     - DEPLOY_SCENARIO: 'vgpu'
 493                 run:
 494                     module: 'functest.opnfv_tests.openstack.vgpu.vgpu'
 495                     class: 'vGPU'
 496
 497             -
 498                 case_name: stor4nfv_os
 499                 project_name: stor4nfv
 500                 criteria: 100
 501                 blocking: false
 502                 description: >-
 503                     This tests are necessary demonstrate conformance
 504                     of the OpenStack+Stor4NFV deployment.
 505                 dependencies:
 506                     - DEPLOY_SCENARIO: 'stor4nfv'
 507                 run:
 508                     module: 'stor4nfv_tests.stor4nfv_os'
 509                     class: 'OsStor4nfvTest'
 510
 511     -
 512         name: components
 513         order: 3
 514         description: >-
 515             Extensive testing of OpenStack API.
 516         testcases:
 517             -
 518                 case_name: tempest_full
 519                 project_name: functest
 520                 criteria: 100
 521                 blocking: false
 522                 description: >-
 523                     The list of test cases is generated by
 524                     Tempest automatically and depends on the parameters of
 525                     the OpenStack deployment.
 526                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83
 527                 run:
 528                     module: 'functest.opnfv_tests.openstack.tempest.tempest'
 529                     class: 'TempestCommon'
 530                     args:
 531                         mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
 532                         option:
 533                             - '--concurrency=4'
 534
 535             -
 536                 case_name: tempest_scenario
 537                 project_name: functest
 538                 criteria: 100
 539                 blocking: false
 540                 description: >-
 541                     The list of test cases is generated by
 542                     Tempest automatically and depends on the parameters of
 543                     the OpenStack deployment.
 544                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
 545                 run:
 546                     module: 'functest.opnfv_tests.openstack.tempest.tempest'
 547                     class: 'TempestCommon'
 548                     args:
 549                         mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)'
 550                         option:
 551                             - '--concurrency=1'
 552
 553             -
 554                 case_name: rally_full
 555                 project_name: functest
 556                 criteria: 100
 557                 blocking: false
 558                 description: >-
 559                     This test case runs the full suite of scenarios of the
 560                     OpenStack Rally suite using several threads and iterations.
 561                 run:
 562                     module: 'functest.opnfv_tests.openstack.rally.rally'
 563                     class: 'RallyFull'
 564
 565     -
 566         name: vnf
 567         order: 4
 568         description: >-
 569             Collection of VNF test cases.
 570         testcases:
 571             -
 572                 case_name: cloudify
 573                 project_name: functest
 574                 criteria: 100
 575                 blocking: false
 576                 description: >-
 577                     This test case deploys the Cloudify orchestrator.
 578                 dependencies:
 579                     - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha'
 580                 run:
 581                     module: 'functest.core.cloudify'
 582                     class: 'Cloudify'
 583             -
 584                 case_name: cloudify_ims
 585                 project_name: functest
 586                 criteria: 100
 587                 blocking: false
 588                 description: >-
 589                     This test case deploys an OpenSource vIMS solution from
 590                     Clearwater using the Cloudify orchestrator. It also runs
 591                     some signaling traffic.
 592                 dependencies:
 593                     - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha'
 594                 run:
 595                     module: 'functest.opnfv_tests.vnf.ims.cloudify_ims'
 596                     class: 'CloudifyIms'
 597
 598             -
 599                 case_name: heat_ims
 600                 project_name: functest
 601                 criteria: 100
 602                 blocking: false
 603                 description: >-
 604                     This test case deploys an OpenSource vIMS solution from
 605                     Clearwater using the OpenStack Heat orchestrator.
 606                     It also runs some signaling traffic.
 607                 dependencies:
 608                     - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha'
 609                 run:
 610                     module: 'functest.opnfv_tests.vnf.ims.heat_ims'
 611                     class: 'HeatIms'
 612
 613             -
 614                 case_name: vyos_vrouter
 615                 project_name: functest
 616                 criteria: 100
 617                 blocking: false
 618                 description: >-
 619                     This test case is vRouter testing.
 620                 dependencies:
 621                     - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha'
 622                 run:
 623                     module: 'functest.opnfv_tests.vnf.router.cloudify_vrouter'
 624                     class: 'CloudifyVrouter'
 625
 626             -
 627                 case_name: juju_epc
 628                 project_name: functest
 629                 criteria: 100
 630                 blocking: false
 631                 description: >-
 632                     vEPC validation with Juju as VNF manager and ABoT as test
 633                     executor.
 634                 dependencies:
 635                     - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha'
 636                 run:
 637                     module: 'functest.opnfv_tests.vnf.epc.juju_epc'
 638                     class: 'JujuEpc'