functest/ci/testcases.yaml

   1 ---
   2 tiers:
   3     -
   4         name: healthcheck
   5         order: 0
   6         description: >-
   7             First tier to be executed to verify the basic
   8             operations in the VIM.
   9         testcases:
  10             -
  11                 case_name: connection_check
  12                 project_name: functest
  13                 criteria: 100
  14                 blocking: true
  15                 description: >-
  16                     This test case verifies the retrieval of OpenStack clients:
  17                     Keystone, Glance, Neutron and Nova and may perform some
  18                     simple queries.
  19                 run:
  20                     name: connection_check
  21
  22             -
  23                 case_name: tenantnetwork1
  24                 project_name: functest
  25                 criteria: 100
  26                 blocking: true
  27                 description: >-
  28                     It creates and configures all tenant network ressources
  29                     required by advanced testcases (subnet, network and
  30                     router).
  31                 run:
  32                     name: tenantnetwork1
  33
  34             -
  35                 case_name: tenantnetwork2
  36                 project_name: functest
  37                 criteria: 100
  38                 blocking: true
  39                 description: >-
  40                     It creates new user/project before creating and configuring
  41                     all tenant network ressources required by a testcase
  42                     (subnet, network and router).
  43                 run:
  44                     name: tenantnetwork2
  45
  46             -
  47                 case_name: vmready1
  48                 project_name: functest
  49                 criteria: 100
  50                 blocking: true
  51                 description: >-
  52                     It inherits from TenantNetwork1 which creates all network
  53                     resources and prepares a future VM attached to that
  54                     network.
  55                 run:
  56                     name: vmready1
  57
  58             -
  59                 case_name: vmready2
  60                 project_name: functest
  61                 criteria: 100
  62                 blocking: true
  63                 description: >-
  64                     It creates new user/project before creating and configuring
  65                     all tenant network ressources, flavors, images, etc.
  66                     required by advanced testcases.
  67                 run:
  68                     name: vmready2
  69
  70             -
  71                 case_name: singlevm1
  72                 project_name: functest
  73                 criteria: 100
  74                 blocking: true
  75                 description: >-
  76                     It inherits from TenantNetwork1 which creates all network
  77                     resources and completes it by booting a VM attached to that
  78                     network.
  79                 run:
  80                     name: singlevm1
  81
  82             -
  83                 case_name: singlevm2
  84                 project_name: functest
  85                 criteria: 100
  86                 blocking: true
  87                 description: >-
  88                     It creates new user/project before creating and configuring
  89                     all tenant network ressources and vms required by advanced
  90                     testcases.
  91                 run:
  92                     name: singlevm2
  93
  94             -
  95                 case_name: vping_ssh
  96                 project_name: functest
  97                 criteria: 100
  98                 blocking: true
  99                 description: >-
 100                     This test case verifies: 1) SSH to an instance using
 101                     floating IPs over the public network. 2) Connectivity
 102                     between 2 instances over a private network.
 103                 run:
 104                     name: vping_ssh
 105
 106             -
 107                 case_name: vping_userdata
 108                 project_name: functest
 109                 criteria: 100
 110                 blocking: true
 111                 description: >-
 112                     This test case verifies:  1) Boot a VM with given userdata.
 113                     2) Connectivity between 2 instances over a private network.
 114                 run:
 115                     name: vping_userdata
 116
 117             -
 118                 case_name: cinder_test
 119                 project_name: functest
 120                 criteria: 100
 121                 blocking: true
 122                 description: >-
 123                     This test case verifies: 1) Attach volume and to 1
 124                     instance; 2) Write data on volume 3) Detach volume
 125                     from instance 1, attach it on instance 2 3) Read volume
 126                     data
 127                 run:
 128                     name: cinder_test
 129
 130             -
 131                 case_name: odl
 132                 project_name: functest
 133                 criteria: 100
 134                 blocking: true
 135                 description: >-
 136                     Test Suite for the OpenDaylight SDN Controller. It
 137                     integrates some test suites from upstream using
 138                     Robot as the test framework.
 139                 dependencies:
 140                     - DEPLOY_SCENARIO: 'odl'
 141                 run:
 142                     name: odl
 143                     args:
 144                         suites:
 145                             - /src/odl_test/csit/suites/integration/basic
 146                             - /src/odl_test/csit/suites/openstack/neutron
 147
 148             -
 149                 case_name: api_check
 150                 project_name: functest
 151                 criteria: 100
 152                 blocking: true
 153                 description: >-
 154                     This test case verifies the retrieval of OpenStack clients:
 155                     Keystone, Glance, Neutron and Nova and may perform some
 156                     simple queries. When the config value of
 157                     snaps.use_keystone is True, functest must have access to
 158                     the cloud's private network.
 159                 run:
 160                     name: api_check
 161
 162             -
 163                 case_name: snaps_health_check
 164                 project_name: functest
 165                 criteria: 100
 166                 blocking: true
 167                 description: >-
 168                     This test case creates executes the SimpleHealthCheck
 169                     Python test class which creates an, image, flavor, network,
 170                     and Cirros VM instance and observes the console output to
 171                     validate the single port obtains the correct IP address.
 172                 run:
 173                     name: snaps_health_check
 174
 175             -
 176                 case_name: tempest_smoke
 177                 project_name: functest
 178                 criteria: 100
 179                 blocking: false
 180                 description: >-
 181                     This test case runs the smoke subset of the OpenStack
 182                     Tempest suite. The list of test cases is generated by
 183                     Tempest automatically and depends on the parameters of
 184                     the OpenStack deplopyment.
 185                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L114
 186                 run:
 187                     name: tempest_common
 188                     args:
 189                         mode: '(?=.*\[.*\bsmoke\b.*\])(^tempest\.api)'
 190                         option:
 191                             - '--concurrency=4'
 192
 193     -
 194         name: smoke
 195         order: 1
 196         description: >-
 197             Set of basic Functional tests to validate the OPNFV scenarios.
 198         testcases:
 199             -
 200                 case_name: neutron-tempest-plugin-api
 201                 project_name: functest
 202                 criteria: 100
 203                 blocking: false
 204                 description: >-
 205                     This test case runs the Tempest suite proposed by the
 206                     Neutron project. The list of test cases is generated by
 207                     Tempest automatically and depends on the parameters of
 208                     the OpenStack deployment.
 209                 run:
 210                     name: tempest_common
 211                     args:
 212                         mode: '^neutron_tempest_plugin\.api'
 213                         option:
 214                             - '--concurrency=4'
 215
 216             -
 217                 case_name: rally_sanity
 218                 project_name: functest
 219                 criteria: 100
 220                 blocking: false
 221                 description: >-
 222                     This test case runs a sub group of tests of the OpenStack
 223                     Rally suite in smoke mode.
 224                 run:
 225                     name: rally_sanity
 226
 227             -
 228                 case_name: refstack_defcore
 229                 project_name: functest
 230                 criteria: 100
 231                 blocking: false
 232                 description: >-
 233                     This test case runs a sub group of tests of the OpenStack
 234                     Defcore testcases.
 235                 run:
 236                     name: refstack_defcore
 237                     args:
 238                         option:
 239                             - '--concurrency=4'
 240
 241             -
 242                 case_name: tempest_full
 243                 project_name: functest
 244                 criteria: 100
 245                 blocking: false
 246                 description: >-
 247                     The list of test cases is generated by
 248                     Tempest automatically and depends on the parameters of
 249                     the OpenStack deployment.
 250                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83
 251                 run:
 252                     name: tempest_common
 253                     args:
 254                         mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
 255                         option:
 256                             - '--concurrency=4'
 257
 258             -
 259                 case_name: tempest_scenario
 260                 project_name: functest
 261                 criteria: 100
 262                 blocking: false
 263                 description: >-
 264                     The list of test cases is generated by
 265                     Tempest automatically and depends on the parameters of
 266                     the OpenStack deployment.
 267                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
 268                 run:
 269                     name: tempest_common
 270                     args:
 271                         mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)'
 272                         option:
 273                             - '--concurrency=1'
 274
 275             -
 276                 case_name: patrole
 277                 project_name: functest
 278                 criteria: 100
 279                 blocking: false
 280                 description: >-
 281                     Test suite from Patrole project.
 282                 run:
 283                     name: patrole
 284                     args:
 285                         role: 'admin'
 286                         includes:
 287                             - 'image'
 288                             - 'network'
 289                         excludes:
 290                             - 'network.test_flavors_rbac'
 291                             - 'network.test_network_ip_availability_rbac'
 292                             - 'test_networks_multiprovider_rbac'
 293                             - 'test_auto_allocated_topology_rbac'
 294
 295             -
 296                 case_name: snaps_smoke
 297                 project_name: functest
 298                 criteria: 100
 299                 blocking: false
 300                 description: >-
 301                     This test case contains tests that setup and destroy
 302                     environments with VMs with and without Floating IPs
 303                     with a newly created user and project. Set the config
 304                     value snaps.use_floating_ips (True|False) to toggle
 305                     this functionality. When the config value of
 306                     snaps.use_keystone is True, functest must have access to
 307                     the cloud's private network.
 308                 run:
 309                     name: snaps_smoke
 310
 311             -
 312                 case_name: neutron_trunk
 313                 project_name: functest
 314                 criteria: 100
 315                 blocking: false
 316                 description: >-
 317                     This test case runs the neutron trunk subtest of the
 318                     OpenStack Tempest suite. The list of test cases is
 319                     generated by Tempest having as input the relevant
 320                     testcase list file.
 321                 run:
 322                     name: tempest_common
 323                     args:
 324                         mode: 'neutron_tempest_plugin.(api|scenario).test_trunk'
 325                         neutron_extensions:
 326                             - trunk
 327                             - trunk-details
 328
 329             -
 330                 case_name: networking-bgpvpn
 331                 project_name: functest
 332                 criteria: 100
 333                 blocking: false
 334                 description: >-
 335                     This test case runs OpenStack networking-bgpvpn Tempest
 336                     suite. The list of test cases is generated by Tempest
 337                     having as input the relevant regex.
 338                 run:
 339                     name: tempest_common
 340                     args:
 341                         mode: '^networking_bgpvpn_tempest\.'
 342                         neutron_extensions:
 343                             - bgpvpn
 344                         option:
 345                             - '--concurrency=4'
 346
 347             -
 348                 case_name: networking-sfc
 349                 project_name: functest
 350                 criteria: 100
 351                 blocking: false
 352                 description: >-
 353                     This test case runs OpenStack networking-sfc Tempest
 354                     suite. The list of test cases is generated by Tempest
 355                     having as input the relevant regex.
 356                 run:
 357                     name: tempest_common
 358                     args:
 359                         mode:
 360                             '^(?:networking_sfc\.tests\.tempest_plugin.tests.api).*$'
 361                         neutron_extensions:
 362                             - sfc
 363                         option:
 364                             - '--concurrency=0'
 365
 366             -
 367                 case_name: barbican
 368                 project_name: functest
 369                 criteria: 100
 370                 blocking: false
 371                 description: >-
 372                     It leverages on the tempest plugin containing tests used to
 373                     verify the functionality of a barbican installation.
 374                 run:
 375                     name: barbican
 376                     args:
 377                         mode:
 378                             '^barbican_tempest_plugin.((?!test_signed_image_upload_boot_failure).)*$'
 379                         services:
 380                             - barbican
 381
 382     -
 383         name: benchmarking
 384         order: 2
 385         description: >-
 386             Run several OpenStack performance tools
 387             https://docs.openstack.org/performance-docs/latest/methodologies/tools.html
 388         testcases:
 389             -
 390                 case_name: rally_jobs
 391                 project_name: functest
 392                 criteria: 100
 393                 blocking: false
 394                 description: >-
 395                     This test case runs a group of Rally jobs used in
 396                     OpenStack gating
 397                 run:
 398                     name: rally_jobs
 399
 400             -
 401                 case_name: vmtp
 402                 project_name: functest
 403                 criteria: 100
 404                 blocking: false
 405                 description: >-
 406                     VMTP is a small python application that will automatically
 407                     perform ping connectivity, round trip time measurement
 408                     (latency) and TCP/UDP throughput
 409                 dependencies:
 410                     - POD_ARCH: '^(?!aarch64$)'
 411                 run:
 412                     name: vmtp
 413
 414             -
 415                 case_name: shaker
 416                 project_name: functest
 417                 criteria: 100
 418                 blocking: false
 419                 description: >-
 420                     Shaker wraps around popular system network testing tools
 421                     like iperf, iperf3 and netperf (with help of flent). Shaker
 422                     is able to deploy OpenStack instances and networks in
 423                     different topologies.
 424                 run:
 425                     name: shaker
 426
 427     -
 428         name: features
 429         order: 3
 430         description: >-
 431             Test suites from feature projects
 432             integrated in functest
 433         testcases:
 434             -
 435                 case_name: doctor-notification
 436                 project_name: doctor
 437                 enabled: false
 438                 criteria: 100
 439                 blocking: false
 440                 description: >-
 441                     Test suite from Doctor project.
 442                 dependencies:
 443                     - INSTALLER_TYPE: '(apex)|(fuel)|(daisy)'
 444                     - DEPLOY_SCENARIO: '^((?!fdio|nofeature).)*$'
 445                 run:
 446                     name: bashfeature
 447                     args:
 448                         cmd: 'doctor-test'
 449
 450             -
 451                 case_name: bgpvpn
 452                 project_name: sdnvpn
 453                 enabled: false
 454                 criteria: 100
 455                 blocking: false
 456                 description: >-
 457                     Test suite from SDNVPN project.
 458                 dependencies:
 459                     - DEPLOY_SCENARIO: 'bgpvpn'
 460                 run:
 461                     name: bgpvpn
 462
 463             -
 464                 case_name: functest-odl-sfc
 465                 project_name: sfc
 466                 enabled: false
 467                 criteria: 100
 468                 blocking: false
 469                 description: >-
 470                     Test suite for odl-sfc to test two chains with one SF and
 471                     one chain with two SFs
 472                 dependencies:
 473                     - DEPLOY_SCENARIO: 'odl.*sfc'
 474                 run:
 475                     name: functest-odl-sfc
 476
 477             -
 478                 case_name: barometercollectd
 479                 project_name: barometer
 480                 enabled: false
 481                 criteria: 100
 482                 blocking: false
 483                 description: >-
 484                     Test suite for the Barometer project. Separate tests verify
 485                     the proper configuration and basic functionality of all the
 486                     collectd plugins as described in the Project Release Plan
 487                 dependencies:
 488                     - DEPLOY_SCENARIO: 'bar'
 489                 run:
 490                     name: barometercollectd
 491
 492             -
 493                 case_name: vgpu
 494                 project_name: functest
 495                 criteria: 100
 496                 blocking: false
 497                 description: >-
 498                     Test suite for the OpenStack vGPU feature
 499                 dependencies:
 500                     - DEPLOY_SCENARIO: 'vgpu'
 501                 run:
 502                     name: vgpu
 503
 504             -
 505                 case_name: stor4nfv_os
 506                 project_name: stor4nfv
 507                 enabled: false
 508                 criteria: 100
 509                 blocking: false
 510                 description: >-
 511                     This tests are necessary demonstrate conformance
 512                     of the OpenStack+Stor4NFV deployment.
 513                 dependencies:
 514                     - DEPLOY_SCENARIO: 'stor4nfv'
 515                 run:
 516                     name: stor4nfv_os
 517
 518     -
 519         name: components
 520         order: 4
 521         description: >-
 522             Extensive testing of OpenStack API.
 523         testcases:
 524             -
 525                 case_name: rally_full
 526                 project_name: functest
 527                 criteria: 100
 528                 blocking: false
 529                 description: >-
 530                     This test case runs the full suite of scenarios of the
 531                     OpenStack Rally suite using several threads and iterations.
 532                 run:
 533                     name: rally_full
 534
 535     -
 536         name: vnf
 537         order: 5
 538         description: >-
 539             Collection of VNF test cases.
 540         testcases:
 541             -
 542                 case_name: cloudify
 543                 project_name: functest
 544                 criteria: 100
 545                 blocking: false
 546                 description: >-
 547                     This test case deploys the Cloudify orchestrator.
 548                 run:
 549                     name: cloudify
 550             -
 551                 case_name: cloudify_ims
 552                 project_name: functest
 553                 criteria: 100
 554                 blocking: false
 555                 enabled: false
 556                 description: >-
 557                     This test case deploys an OpenSource vIMS solution from
 558                     Clearwater using the Cloudify orchestrator. It also runs
 559                     some signaling traffic.
 560                 dependencies:
 561                     - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha'
 562                 run:
 563                     name: cloudify_ims
 564
 565             -
 566                 case_name: heat_ims
 567                 project_name: functest
 568                 criteria: 100
 569                 blocking: false
 570                 description: >-
 571                     This test case deploys an OpenSource vIMS solution from
 572                     Clearwater using the OpenStack Heat orchestrator.
 573                     It also runs some signaling traffic.
 574                 run:
 575                     name: heat_ims
 576
 577             -
 578                 case_name: vyos_vrouter
 579                 project_name: functest
 580                 criteria: 100
 581                 blocking: false
 582                 enabled: false
 583                 description: >-
 584                     This test case is vRouter testing.
 585                 run:
 586                     name: vyos_vrouter
 587
 588             -
 589                 case_name: juju_epc
 590                 project_name: functest
 591                 criteria: 100
 592                 blocking: false
 593                 description: >-
 594                     vEPC validation with Juju as VNF manager and ABoT as test
 595                     executor.
 596                 run:
 597                     name: juju_epc