functest/ci/testcases.yaml

   1 ---
   2 tiers:
   3     -
   4         name: healthcheck
   5         order: 0
   6         description: >-
   7             First tier to be executed to verify the basic
   8             operations in the VIM.
   9         testcases:
  10             -
  11                 case_name: connection_check
  12                 project_name: functest
  13                 criteria: 100
  14                 blocking: true
  15                 description: >-
  16                     This test case verifies the retrieval of OpenStack clients:
  17                     Keystone, Glance, Neutron and Nova and may perform some
  18                     simple queries.
  19                 run:
  20                     name: connection_check
  21
  22             -
  23                 case_name: tenantnetwork1
  24                 project_name: functest
  25                 criteria: 100
  26                 blocking: true
  27                 description: >-
  28                     It creates and configures all tenant network ressources
  29                     required by advanced testcases (subnet, network and
  30                     router).
  31                 run:
  32                     name: tenantnetwork1
  33
  34             -
  35                 case_name: tenantnetwork2
  36                 project_name: functest
  37                 criteria: 100
  38                 blocking: true
  39                 description: >-
  40                     It creates new user/project before creating and configuring
  41                     all tenant network ressources required by a testcase
  42                     (subnet, network and router).
  43                 run:
  44                     name: tenantnetwork2
  45
  46             -
  47                 case_name: vmready1
  48                 project_name: functest
  49                 criteria: 100
  50                 blocking: true
  51                 description: >-
  52                     It inherits from TenantNetwork1 which creates all network
  53                     resources and prepares a future VM attached to that
  54                     network.
  55                 run:
  56                     name: vmready1
  57
  58             -
  59                 case_name: vmready2
  60                 project_name: functest
  61                 criteria: 100
  62                 blocking: true
  63                 description: >-
  64                     It creates new user/project before creating and configuring
  65                     all tenant network ressources, flavors, images, etc.
  66                     required by advanced testcases.
  67                 run:
  68                     name: vmready2
  69
  70             -
  71                 case_name: singlevm1
  72                 project_name: functest
  73                 criteria: 100
  74                 blocking: true
  75                 description: >-
  76                     It inherits from TenantNetwork1 which creates all network
  77                     resources and completes it by booting a VM attached to that
  78                     network.
  79                 run:
  80                     name: singlevm1
  81
  82             -
  83                 case_name: singlevm2
  84                 project_name: functest
  85                 criteria: 100
  86                 blocking: true
  87                 description: >-
  88                     It creates new user/project before creating and configuring
  89                     all tenant network ressources and vms required by advanced
  90                     testcases.
  91                 run:
  92                     name: singlevm2
  93
  94             -
  95                 case_name: vping_ssh
  96                 project_name: functest
  97                 criteria: 100
  98                 blocking: true
  99                 description: >-
 100                     This test case verifies: 1) SSH to an instance using
 101                     floating IPs over the public network. 2) Connectivity
 102                     between 2 instances over a private network.
 103                 run:
 104                     name: vping_ssh
 105
 106             -
 107                 case_name: vping_userdata
 108                 project_name: functest
 109                 criteria: 100
 110                 blocking: true
 111                 description: >-
 112                     This test case verifies:  1) Boot a VM with given userdata.
 113                     2) Connectivity between 2 instances over a private network.
 114                 run:
 115                     name: vping_userdata
 116
 117             -
 118                 case_name: cinder_test
 119                 project_name: functest
 120                 criteria: 100
 121                 blocking: true
 122                 description: >-
 123                     This test case verifies: 1) Attach volume and to 1
 124                     instance; 2) Write data on volume 3) Detach volume
 125                     from instance 1, attach it on instance 2 3) Read volume
 126                     data
 127                 run:
 128                     name: cinder_test
 129
 130             -
 131                 case_name: odl
 132                 project_name: functest
 133                 criteria: 100
 134                 blocking: true
 135                 description: >-
 136                     Test Suite for the OpenDaylight SDN Controller. It
 137                     integrates some test suites from upstream using
 138                     Robot as the test framework.
 139                 dependencies:
 140                     - DEPLOY_SCENARIO: 'odl'
 141                 run:
 142                     name: odl
 143                     args:
 144                         suites:
 145                             - /src/odl_test/csit/suites/integration/basic
 146                             - /src/odl_test/csit/suites/openstack/neutron
 147
 148             -
 149                 case_name: api_check
 150                 project_name: functest
 151                 criteria: 100
 152                 blocking: true
 153                 description: >-
 154                     This test case verifies the retrieval of OpenStack clients:
 155                     Keystone, Glance, Neutron and Nova and may perform some
 156                     simple queries. When the config value of
 157                     snaps.use_keystone is True, functest must have access to
 158                     the cloud's private network.
 159                 run:
 160                     name: api_check
 161
 162             -
 163                 case_name: snaps_health_check
 164                 project_name: functest
 165                 criteria: 100
 166                 blocking: true
 167                 description: >-
 168                     This test case creates executes the SimpleHealthCheck
 169                     Python test class which creates an, image, flavor, network,
 170                     and Cirros VM instance and observes the console output to
 171                     validate the single port obtains the correct IP address.
 172                 run:
 173                     name: snaps_health_check
 174
 175     -
 176         name: smoke
 177         order: 1
 178         description: >-
 179             Set of basic Functional tests to validate the OPNFV scenarios.
 180         testcases:
 181             -
 182                 case_name: tempest_smoke
 183                 project_name: functest
 184                 criteria: 100
 185                 blocking: false
 186                 description: >-
 187                     This test case runs the smoke subset of the OpenStack
 188                     Tempest suite. The list of test cases is generated by
 189                     Tempest automatically and depends on the parameters of
 190                     the OpenStack deplopyment.
 191                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L114
 192                 run:
 193                     name: tempest_common
 194                     args:
 195                         mode: '(?=.*\[.*\bsmoke\b.*\])(^tempest\.api)'
 196                         option:
 197                             - '--concurrency=4'
 198
 199             -
 200                 case_name: neutron-tempest-plugin-api
 201                 project_name: functest
 202                 criteria: 100
 203                 blocking: false
 204                 description: >-
 205                     This test case runs the Tempest suite proposed by the
 206                     Neutron project. The list of test cases is generated by
 207                     Tempest automatically and depends on the parameters of
 208                     the OpenStack deployment.
 209                 run:
 210                     name: tempest_common
 211                     args:
 212                         mode: '^neutron_tempest_plugin\.api'
 213                         option:
 214                             - '--concurrency=4'
 215
 216             -
 217                 case_name: rally_sanity
 218                 project_name: functest
 219                 criteria: 100
 220                 blocking: false
 221                 description: >-
 222                     This test case runs a sub group of tests of the OpenStack
 223                     Rally suite in smoke mode.
 224                 run:
 225                     name: rally_sanity
 226
 227             -
 228                 case_name: refstack_defcore
 229                 project_name: functest
 230                 criteria: 100
 231                 blocking: false
 232                 description: >-
 233                     This test case runs a sub group of tests of the OpenStack
 234                     Defcore testcases.
 235                 run:
 236                     name: refstack_defcore
 237                     args:
 238                         option:
 239                             - '--concurrency=4'
 240
 241             -
 242                 case_name: patrole
 243                 project_name: functest
 244                 criteria: 100
 245                 blocking: false
 246                 description: >-
 247                     Test suite from Patrole project.
 248                 run:
 249                     name: patrole
 250                     args:
 251                         role: 'admin'
 252                         includes:
 253                             - 'image'
 254                             - 'network'
 255                         excludes:
 256                             - 'test_networks_multiprovider_rbac'
 257
 258             -
 259                 case_name: snaps_smoke
 260                 project_name: functest
 261                 criteria: 100
 262                 blocking: false
 263                 description: >-
 264                     This test case contains tests that setup and destroy
 265                     environments with VMs with and without Floating IPs
 266                     with a newly created user and project. Set the config
 267                     value snaps.use_floating_ips (True|False) to toggle
 268                     this functionality. When the config value of
 269                     snaps.use_keystone is True, functest must have access to
 270                     the cloud's private network.
 271                 run:
 272                     name: snaps_smoke
 273
 274             -
 275                 case_name: neutron_trunk
 276                 project_name: functest
 277                 criteria: 100
 278                 blocking: false
 279                 description: >-
 280                     This test case runs the neutron trunk subtest of the
 281                     OpenStack Tempest suite. The list of test cases is
 282                     generated by Tempest having as input the relevant
 283                     testcase list file.
 284                 run:
 285                     name: tempest_common
 286                     args:
 287                         mode: 'neutron_tempest_plugin.(api|scenario).test_trunk'
 288                         neutron_extensions:
 289                             - trunk
 290                             - trunk-details
 291
 292             -
 293                 case_name: networking-bgpvpn
 294                 project_name: functest
 295                 criteria: 100
 296                 blocking: false
 297                 description: >-
 298                     This test case runs OpenStack networking-bgpvpn Tempest
 299                     suite. The list of test cases is generated by Tempest
 300                     having as input the relevant regex.
 301                 run:
 302                     name: tempest_common
 303                     args:
 304                         mode: '^networking_bgpvpn_tempest\.'
 305                         neutron_extensions:
 306                             - bgpvpn
 307                         option:
 308                             - '--concurrency=4'
 309
 310             -
 311                 case_name: networking-sfc
 312                 project_name: functest
 313                 criteria: 100
 314                 blocking: false
 315                 description: >-
 316                     This test case runs OpenStack networking-sfc Tempest
 317                     suite. The list of test cases is generated by Tempest
 318                     having as input the relevant regex.
 319                 run:
 320                     name: tempest_common
 321                     args:
 322                         mode:
 323                             '^(?:networking_sfc\.tests\.tempest_plugin.tests.api).*$'
 324                         neutron_extensions:
 325                             - sfc
 326                         option:
 327                             - '--concurrency=0'
 328
 329             -
 330                 case_name: barbican
 331                 project_name: functest
 332                 criteria: 100
 333                 blocking: false
 334                 description: >-
 335                     It leverages on the tempest plugin containing tests used to
 336                     verify the functionality of a barbican installation.
 337                 run:
 338                     name: tempest_common
 339                     args:
 340                         mode:
 341                             'barbican_tempest_plugin.tests.(api|scenario).(?!test_image_signing)'
 342                         services:
 343                             - barbican
 344
 345     -
 346         name: benchmarking
 347         order: 2
 348         description: >-
 349             Run several OpenStack performance tools
 350             https://docs.openstack.org/performance-docs/latest/methodologies/tools.html
 351         testcases:
 352             -
 353                 case_name: vmtp
 354                 project_name: functest
 355                 criteria: 100
 356                 blocking: false
 357                 description: >-
 358                     VMTP is a small python application that will automatically
 359                     perform ping connectivity, round trip time measurement
 360                     (latency) and TCP/UDP throughput
 361                 dependencies:
 362                     - POD_ARCH: '^(?!aarch64$)'
 363                 run:
 364                     name: vmtp
 365
 366             -
 367                 case_name: shaker
 368                 project_name: functest
 369                 criteria: 100
 370                 blocking: false
 371                 description: >-
 372                     Shaker wraps around popular system network testing tools
 373                     like iperf, iperf3 and netperf (with help of flent). Shaker
 374                     is able to deploy OpenStack instances and networks in
 375                     different topologies.
 376                 run:
 377                     name: shaker
 378
 379     -
 380         name: features
 381         order: 3
 382         description: >-
 383             Test suites from feature projects
 384             integrated in functest
 385         testcases:
 386             -
 387                 case_name: doctor-notification
 388                 project_name: doctor
 389                 criteria: 100
 390                 blocking: false
 391                 description: >-
 392                     Test suite from Doctor project.
 393                 dependencies:
 394                     - INSTALLER_TYPE: '(apex)|(fuel)|(daisy)'
 395                     - DEPLOY_SCENARIO: '^((?!fdio|nofeature).)*$'
 396                 run:
 397                     name: bashfeature
 398                     args:
 399                         cmd: 'doctor-test'
 400
 401             -
 402                 case_name: bgpvpn
 403                 project_name: sdnvpn
 404                 criteria: 100
 405                 blocking: false
 406                 description: >-
 407                     Test suite from SDNVPN project.
 408                 dependencies:
 409                     - DEPLOY_SCENARIO: 'bgpvpn'
 410                 run:
 411                     name: bgpvpn
 412
 413             -
 414                 case_name: functest-odl-sfc
 415                 project_name: sfc
 416                 criteria: 100
 417                 blocking: false
 418                 description: >-
 419                     Test suite for odl-sfc to test two chains with one SF and
 420                     one chain with two SFs
 421                 dependencies:
 422                     - DEPLOY_SCENARIO: 'odl.*sfc'
 423                 run:
 424                     name: functest-odl-sfc
 425
 426             -
 427                 case_name: barometercollectd
 428                 project_name: barometer
 429                 criteria: 100
 430                 blocking: false
 431                 description: >-
 432                     Test suite for the Barometer project. Separate tests verify
 433                     the proper configuration and basic functionality of all the
 434                     collectd plugins as described in the Project Release Plan
 435                 dependencies:
 436                     - DEPLOY_SCENARIO: 'bar'
 437                 run:
 438                     name: barometercollectd
 439
 440             -
 441                 case_name: fds
 442                 project_name: fastdatastacks
 443                 criteria: 100
 444                 blocking: false
 445                 description: >-
 446                     Test Suite for the OpenDaylight SDN Controller when GBP
 447                     features are installed. It integrates some test suites from
 448                     upstream using Robot as the test framework.
 449                 dependencies:
 450                     - DEPLOY_SCENARIO: 'odl.*-fdio'
 451                 run:
 452                     name: odl
 453                     args:
 454                         suites:
 455                             - /src/fds/testing/robot
 456             -
 457                 case_name: vgpu
 458                 project_name: functest
 459                 criteria: 100
 460                 blocking: false
 461                 description: >-
 462                     Test suite for the OpenStack vGPU feature
 463                 dependencies:
 464                     - DEPLOY_SCENARIO: 'vgpu'
 465                 run:
 466                     name: vgpu
 467
 468             -
 469                 case_name: stor4nfv_os
 470                 project_name: stor4nfv
 471                 criteria: 100
 472                 blocking: false
 473                 description: >-
 474                     This tests are necessary demonstrate conformance
 475                     of the OpenStack+Stor4NFV deployment.
 476                 dependencies:
 477                     - DEPLOY_SCENARIO: 'stor4nfv'
 478                 run:
 479                     name: stor4nfv_os
 480
 481     -
 482         name: components
 483         order: 4
 484         description: >-
 485             Extensive testing of OpenStack API.
 486         testcases:
 487             -
 488                 case_name: tempest_full
 489                 project_name: functest
 490                 criteria: 100
 491                 blocking: false
 492                 description: >-
 493                     The list of test cases is generated by
 494                     Tempest automatically and depends on the parameters of
 495                     the OpenStack deployment.
 496                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83
 497                 run:
 498                     name: tempest_common
 499                     args:
 500                         mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
 501                         option:
 502                             - '--concurrency=4'
 503
 504             -
 505                 case_name: tempest_scenario
 506                 project_name: functest
 507                 criteria: 100
 508                 blocking: false
 509                 description: >-
 510                     The list of test cases is generated by
 511                     Tempest automatically and depends on the parameters of
 512                     the OpenStack deployment.
 513                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
 514                 run:
 515                     name: tempest_common
 516                     args:
 517                         mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)'
 518                         option:
 519                             - '--concurrency=1'
 520
 521             -
 522                 case_name: rally_full
 523                 project_name: functest
 524                 criteria: 100
 525                 blocking: false
 526                 description: >-
 527                     This test case runs the full suite of scenarios of the
 528                     OpenStack Rally suite using several threads and iterations.
 529                 run:
 530                     name: rally_full
 531
 532     -
 533         name: vnf
 534         order: 5
 535         description: >-
 536             Collection of VNF test cases.
 537         testcases:
 538             -
 539                 case_name: cloudify
 540                 project_name: functest
 541                 criteria: 100
 542                 blocking: false
 543                 description: >-
 544                     This test case deploys the Cloudify orchestrator.
 545                 dependencies:
 546                     - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha'
 547                 run:
 548                     name: cloudify
 549             -
 550                 case_name: cloudify_ims
 551                 project_name: functest
 552                 criteria: 100
 553                 blocking: false
 554                 description: >-
 555                     This test case deploys an OpenSource vIMS solution from
 556                     Clearwater using the Cloudify orchestrator. It also runs
 557                     some signaling traffic.
 558                 dependencies:
 559                     - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha'
 560                 run:
 561                     name: cloudify_ims
 562
 563             -
 564                 case_name: heat_ims
 565                 project_name: functest
 566                 criteria: 100
 567                 blocking: false
 568                 description: >-
 569                     This test case deploys an OpenSource vIMS solution from
 570                     Clearwater using the OpenStack Heat orchestrator.
 571                     It also runs some signaling traffic.
 572                 dependencies:
 573                     - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha'
 574                 run:
 575                     name: heat_ims
 576
 577             -
 578                 case_name: vyos_vrouter
 579                 project_name: functest
 580                 criteria: 100
 581                 blocking: false
 582                 description: >-
 583                     This test case is vRouter testing.
 584                 dependencies:
 585                     - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha'
 586                 run:
 587                     name: vyos_vrouter
 588
 589             -
 590                 case_name: juju_epc
 591                 project_name: functest
 592                 criteria: 100
 593                 blocking: false
 594                 description: >-
 595                     vEPC validation with Juju as VNF manager and ABoT as test
 596                     executor.
 597                 dependencies:
 598                     - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha'
 599                 run:
 600                     name: juju_epc