functest/ci/testcases.yaml

   1 ---
   2 tiers:
   3     -
   4         name: healthcheck
   5         order: 0
   6         description: >-
   7             First tier to be executed to verify the basic
   8             operations in the VIM.
   9         testcases:
  10             -
  11                 case_name: connection_check
  12                 project_name: functest
  13                 criteria: 100
  14                 blocking: true
  15                 description: >-
  16                     This test case verifies the retrieval of OpenStack clients:
  17                     Keystone, Glance, Neutron and Nova and may perform some
  18                     simple queries.
  19                 run:
  20                     name: connection_check
  21
  22             -
  23                 case_name: tenantnetwork1
  24                 project_name: functest
  25                 criteria: 100
  26                 blocking: true
  27                 description: >-
  28                     It creates and configures all tenant network ressources
  29                     required by advanced testcases (subnet, network and
  30                     router).
  31                 run:
  32                     name: tenantnetwork1
  33
  34             -
  35                 case_name: tenantnetwork2
  36                 project_name: functest
  37                 criteria: 100
  38                 blocking: true
  39                 description: >-
  40                     It creates new user/project before creating and configuring
  41                     all tenant network ressources required by a testcase
  42                     (subnet, network and router).
  43                 run:
  44                     name: tenantnetwork2
  45
  46             -
  47                 case_name: vmready1
  48                 project_name: functest
  49                 criteria: 100
  50                 blocking: true
  51                 description: >-
  52                     It inherits from TenantNetwork1 which creates all network
  53                     resources and prepares a future VM attached to that
  54                     network.
  55                 run:
  56                     name: vmready1
  57
  58             -
  59                 case_name: vmready2
  60                 project_name: functest
  61                 criteria: 100
  62                 blocking: true
  63                 description: >-
  64                     It creates new user/project before creating and configuring
  65                     all tenant network ressources, flavors, images, etc.
  66                     required by advanced testcases.
  67                 run:
  68                     name: vmready2
  69
  70             -
  71                 case_name: singlevm1
  72                 project_name: functest
  73                 criteria: 100
  74                 blocking: true
  75                 description: >-
  76                     It inherits from TenantNetwork1 which creates all network
  77                     resources and completes it by booting a VM attached to that
  78                     network.
  79                 run:
  80                     name: singlevm1
  81
  82             -
  83                 case_name: singlevm2
  84                 project_name: functest
  85                 criteria: 100
  86                 blocking: true
  87                 description: >-
  88                     It creates new user/project before creating and configuring
  89                     all tenant network ressources and vms required by advanced
  90                     testcases.
  91                 run:
  92                     name: singlevm2
  93
  94             -
  95                 case_name: vping_ssh
  96                 project_name: functest
  97                 criteria: 100
  98                 blocking: true
  99                 description: >-
 100                     This test case verifies: 1) SSH to an instance using
 101                     floating IPs over the public network. 2) Connectivity
 102                     between 2 instances over a private network.
 103                 run:
 104                     name: vping_ssh
 105
 106             -
 107                 case_name: vping_userdata
 108                 project_name: functest
 109                 criteria: 100
 110                 blocking: true
 111                 description: >-
 112                     This test case verifies:  1) Boot a VM with given userdata.
 113                     2) Connectivity between 2 instances over a private network.
 114                 run:
 115                     name: vping_userdata
 116
 117             -
 118                 case_name: cinder_test
 119                 project_name: functest
 120                 criteria: 100
 121                 blocking: true
 122                 description: >-
 123                     This test case verifies: 1) Attach volume and to 1
 124                     instance; 2) Write data on volume 3) Detach volume
 125                     from instance 1, attach it on instance 2 3) Read volume
 126                     data
 127                 run:
 128                     name: cinder_test
 129
 130             -
 131                 case_name: odl
 132                 project_name: functest
 133                 criteria: 100
 134                 blocking: true
 135                 description: >-
 136                     Test Suite for the OpenDaylight SDN Controller. It
 137                     integrates some test suites from upstream using
 138                     Robot as the test framework.
 139                 dependencies:
 140                     - DEPLOY_SCENARIO: 'odl'
 141                 run:
 142                     name: odl
 143                     args:
 144                         suites:
 145                             - /src/odl_test/csit/suites/integration/basic
 146                             - /src/odl_test/csit/suites/openstack/neutron
 147
 148             -
 149                 case_name: api_check
 150                 project_name: functest
 151                 criteria: 100
 152                 blocking: true
 153                 description: >-
 154                     This test case verifies the retrieval of OpenStack clients:
 155                     Keystone, Glance, Neutron and Nova and may perform some
 156                     simple queries. When the config value of
 157                     snaps.use_keystone is True, functest must have access to
 158                     the cloud's private network.
 159                 run:
 160                     name: api_check
 161
 162             -
 163                 case_name: snaps_health_check
 164                 project_name: functest
 165                 criteria: 100
 166                 blocking: true
 167                 description: >-
 168                     This test case creates executes the SimpleHealthCheck
 169                     Python test class which creates an, image, flavor, network,
 170                     and Cirros VM instance and observes the console output to
 171                     validate the single port obtains the correct IP address.
 172                 run:
 173                     name: snaps_health_check
 174
 175             -
 176                 case_name: tempest_smoke
 177                 project_name: functest
 178                 criteria: 100
 179                 blocking: false
 180                 description: >-
 181                     This test case runs the smoke subset of the OpenStack
 182                     Tempest suite. The list of test cases is generated by
 183                     Tempest automatically and depends on the parameters of
 184                     the OpenStack deplopyment.
 185                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L114
 186                 run:
 187                     name: tempest_common
 188                     args:
 189                         mode: '(?=.*\[.*\bsmoke\b.*\])(^tempest\.api)'
 190                         option:
 191                             - '--concurrency=4'
 192
 193     -
 194         name: smoke
 195         order: 1
 196         description: >-
 197             Set of basic Functional tests to validate the OPNFV scenarios.
 198         testcases:
 199             -
 200                 case_name: neutron-tempest-plugin-api
 201                 project_name: functest
 202                 criteria: 100
 203                 blocking: false
 204                 description: >-
 205                     This test case runs the Tempest suite proposed by the
 206                     Neutron project. The list of test cases is generated by
 207                     Tempest automatically and depends on the parameters of
 208                     the OpenStack deployment.
 209                 run:
 210                     name: tempest_common
 211                     args:
 212                         mode: '^neutron_tempest_plugin\.api'
 213                         option:
 214                             - '--concurrency=4'
 215
 216             -
 217                 case_name: rally_sanity
 218                 project_name: functest
 219                 criteria: 100
 220                 blocking: false
 221                 description: >-
 222                     This test case runs a sub group of tests of the OpenStack
 223                     Rally suite in smoke mode.
 224                 run:
 225                     name: rally_sanity
 226                     args:
 227                         optional:
 228                             - 'gnocchi'
 229
 230             -
 231                 case_name: refstack_defcore
 232                 project_name: functest
 233                 criteria: 100
 234                 blocking: false
 235                 description: >-
 236                     This test case runs a sub group of tests of the OpenStack
 237                     Defcore testcases.
 238                 run:
 239                     name: refstack_defcore
 240                     args:
 241                         option:
 242                             - '--concurrency=4'
 243
 244             -
 245                 case_name: tempest_full
 246                 project_name: functest
 247                 criteria: 100
 248                 blocking: false
 249                 description: >-
 250                     The list of test cases is generated by
 251                     Tempest automatically and depends on the parameters of
 252                     the OpenStack deployment.
 253                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83
 254                 run:
 255                     name: tempest_common
 256                     args:
 257                         mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
 258                         option:
 259                             - '--concurrency=4'
 260
 261             -
 262                 case_name: tempest_scenario
 263                 project_name: functest
 264                 criteria: 100
 265                 blocking: false
 266                 description: >-
 267                     The list of test cases is generated by
 268                     Tempest automatically and depends on the parameters of
 269                     the OpenStack deployment.
 270                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
 271                 run:
 272                     name: tempest_common
 273                     args:
 274                         mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)'
 275                         option:
 276                             - '--concurrency=1'
 277
 278             -
 279                 case_name: patrole
 280                 project_name: functest
 281                 criteria: 100
 282                 blocking: false
 283                 description: >-
 284                     Test suite from Patrole project.
 285                 run:
 286                     name: patrole
 287                     args:
 288                         role: 'admin'
 289                         includes:
 290                             - 'image'
 291                             - 'network'
 292                         excludes:
 293                             - 'network.test_flavors_rbac'
 294                             - 'network.test_network_ip_availability_rbac'
 295                             - 'test_networks_multiprovider_rbac'
 296                             - 'test_auto_allocated_topology_rbac'
 297
 298             -
 299                 case_name: snaps_smoke
 300                 project_name: functest
 301                 criteria: 100
 302                 blocking: false
 303                 description: >-
 304                     This test case contains tests that setup and destroy
 305                     environments with VMs with and without Floating IPs
 306                     with a newly created user and project. Set the config
 307                     value snaps.use_floating_ips (True|False) to toggle
 308                     this functionality. When the config value of
 309                     snaps.use_keystone is True, functest must have access to
 310                     the cloud's private network.
 311                 run:
 312                     name: snaps_smoke
 313
 314             -
 315                 case_name: neutron_trunk
 316                 project_name: functest
 317                 criteria: 100
 318                 blocking: false
 319                 description: >-
 320                     This test case runs the neutron trunk subtest of the
 321                     OpenStack Tempest suite. The list of test cases is
 322                     generated by Tempest having as input the relevant
 323                     testcase list file.
 324                 run:
 325                     name: tempest_common
 326                     args:
 327                         mode: 'neutron_tempest_plugin.(api|scenario).test_trunk'
 328                         neutron_extensions:
 329                             - trunk
 330                             - trunk-details
 331
 332             -
 333                 case_name: networking-bgpvpn
 334                 project_name: functest
 335                 criteria: 100
 336                 blocking: false
 337                 description: >-
 338                     This test case runs OpenStack networking-bgpvpn Tempest
 339                     suite. The list of test cases is generated by Tempest
 340                     having as input the relevant regex.
 341                 run:
 342                     name: tempest_common
 343                     args:
 344                         mode: '^networking_bgpvpn_tempest.tests.api\.'
 345                         neutron_extensions:
 346                             - bgpvpn
 347                         option:
 348                             - '--concurrency=4'
 349
 350             -
 351                 case_name: networking-sfc
 352                 project_name: functest
 353                 criteria: 100
 354                 blocking: false
 355                 description: >-
 356                     This test case runs OpenStack networking-sfc Tempest
 357                     suite. The list of test cases is generated by Tempest
 358                     having as input the relevant regex.
 359                 run:
 360                     name: tempest_common
 361                     args:
 362                         mode:
 363                             '^(?:networking_sfc\.tests\.tempest_plugin.tests.api).*$'
 364                         neutron_extensions:
 365                             - sfc
 366                         option:
 367                             - '--concurrency=0'
 368
 369             -
 370                 case_name: barbican
 371                 project_name: functest
 372                 criteria: 100
 373                 blocking: false
 374                 description: >-
 375                     It leverages on the tempest plugin containing tests used to
 376                     verify the functionality of a barbican installation.
 377                 run:
 378                     name: barbican
 379                     args:
 380                         mode:
 381                             '^barbican_tempest_plugin.((?!test_signed_image_upload_boot_failure).)*$'
 382                         services:
 383                             - barbican
 384
 385     -
 386         name: benchmarking
 387         order: 2
 388         description: >-
 389             Run several OpenStack performance tools
 390             https://docs.openstack.org/performance-docs/latest/methodologies/tools.html
 391         testcases:
 392             -
 393                 case_name: rally_full
 394                 project_name: functest
 395                 criteria: 100
 396                 blocking: false
 397                 description: >-
 398                     This test case runs the full suite of scenarios of the
 399                     OpenStack Rally suite using several threads and iterations.
 400                 run:
 401                     name: rally_full
 402                     args:
 403                         optional:
 404                             - 'gnocchi'
 405
 406             -
 407                 case_name: rally_jobs
 408                 project_name: functest
 409                 criteria: 100
 410                 blocking: false
 411                 description: >-
 412                     This test case runs a group of Rally jobs used in
 413                     OpenStack gating
 414                 run:
 415                     name: rally_jobs
 416                     args:
 417                         optional:
 418                             - 'gnocchi'
 419
 420             -
 421                 case_name: vmtp
 422                 project_name: functest
 423                 criteria: 100
 424                 blocking: false
 425                 description: >-
 426                     VMTP is a small python application that will automatically
 427                     perform ping connectivity, round trip time measurement
 428                     (latency) and TCP/UDP throughput
 429                 dependencies:
 430                     - POD_ARCH: '^(?!aarch64$)'
 431                 run:
 432                     name: vmtp
 433
 434             -
 435                 case_name: shaker
 436                 project_name: functest
 437                 criteria: 100
 438                 blocking: false
 439                 description: >-
 440                     Shaker wraps around popular system network testing tools
 441                     like iperf, iperf3 and netperf (with help of flent). Shaker
 442                     is able to deploy OpenStack instances and networks in
 443                     different topologies.
 444                 run:
 445                     name: shaker
 446
 447     -
 448         name: features
 449         order: 3
 450         description: >-
 451             Test suites from feature projects
 452             integrated in functest
 453         testcases:
 454             -
 455                 case_name: doctor-notification
 456                 project_name: doctor
 457                 enabled: false
 458                 criteria: 100
 459                 blocking: false
 460                 description: >-
 461                     Test suite from Doctor project.
 462                 dependencies:
 463                     - INSTALLER_TYPE: '(apex)|(fuel)|(daisy)'
 464                     - DEPLOY_SCENARIO: '^((?!fdio|nofeature).)*$'
 465                 run:
 466                     name: bashfeature
 467                     args:
 468                         cmd: 'doctor-test'
 469
 470             -
 471                 case_name: bgpvpn
 472                 project_name: sdnvpn
 473                 enabled: false
 474                 criteria: 100
 475                 blocking: false
 476                 description: >-
 477                     Test suite from SDNVPN project.
 478                 dependencies:
 479                     - DEPLOY_SCENARIO: 'bgpvpn'
 480                 run:
 481                     name: bgpvpn
 482
 483             -
 484                 case_name: functest-odl-sfc
 485                 project_name: sfc
 486                 criteria: 100
 487                 blocking: false
 488                 description: >-
 489                     Test suite for odl-sfc to test two chains with one SF and
 490                     one chain with two SFs
 491                 dependencies:
 492                     - DEPLOY_SCENARIO: 'odl.*sfc'
 493                 run:
 494                     name: functest-odl-sfc
 495
 496             -
 497                 case_name: barometercollectd
 498                 project_name: barometer
 499                 enabled: false
 500                 criteria: 100
 501                 blocking: false
 502                 description: >-
 503                     Test suite for the Barometer project. Separate tests verify
 504                     the proper configuration and basic functionality of all the
 505                     collectd plugins as described in the Project Release Plan
 506                 dependencies:
 507                     - DEPLOY_SCENARIO: 'bar'
 508                 run:
 509                     name: barometercollectd
 510
 511             -
 512                 case_name: vgpu
 513                 project_name: functest
 514                 criteria: 100
 515                 blocking: false
 516                 description: >-
 517                     Test suite for the OpenStack vGPU feature
 518                 dependencies:
 519                     - DEPLOY_SCENARIO: 'vgpu'
 520                 run:
 521                     name: vgpu
 522
 523             -
 524                 case_name: stor4nfv_os
 525                 project_name: stor4nfv
 526                 enabled: false
 527                 criteria: 100
 528                 blocking: false
 529                 description: >-
 530                     This tests are necessary demonstrate conformance
 531                     of the OpenStack+Stor4NFV deployment.
 532                 dependencies:
 533                     - DEPLOY_SCENARIO: 'stor4nfv'
 534                 run:
 535                     name: stor4nfv_os
 536
 537     -
 538         name: vnf
 539         order: 4
 540         description: >-
 541             Collection of VNF test cases.
 542         testcases:
 543             -
 544                 case_name: cloudify
 545                 project_name: functest
 546                 criteria: 100
 547                 blocking: false
 548                 description: >-
 549                     This test case deploys the Cloudify orchestrator.
 550                 run:
 551                     name: cloudify
 552             -
 553                 case_name: cloudify_ims
 554                 project_name: functest
 555                 criteria: 100
 556                 blocking: false
 557                 description: >-
 558                     This test case deploys an OpenSource vIMS solution from
 559                     Clearwater using the Cloudify orchestrator. It also runs
 560                     some signaling traffic.
 561                 dependencies:
 562                     - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha'
 563                 run:
 564                     name: cloudify_ims
 565
 566             -
 567                 case_name: heat_ims
 568                 project_name: functest
 569                 criteria: 100
 570                 blocking: false
 571                 description: >-
 572                     This test case deploys an OpenSource vIMS solution from
 573                     Clearwater using the OpenStack Heat orchestrator.
 574                     It also runs some signaling traffic.
 575                 run:
 576                     name: heat_ims
 577
 578             -
 579                 case_name: vyos_vrouter
 580                 project_name: functest
 581                 criteria: 100
 582                 blocking: false
 583                 description: >-
 584                     This test case is vRouter testing.
 585                 run:
 586                     name: vyos_vrouter
 587
 588             -
 589                 case_name: juju_epc
 590                 project_name: functest
 591                 criteria: 100
 592                 blocking: false
 593                 description: >-
 594                     vEPC validation with Juju as VNF manager and ABoT as test
 595                     executor.
 596                 run:
 597                     name: juju_epc