functest/ci/testcases.yaml

   1 ---
   2 tiers:
   3     -
   4         name: healthcheck
   5         order: 0
   6         description: >-
   7             First tier to be executed to verify the basic
   8             operations in the VIM.
   9         testcases:
  10             -
  11                 case_name: connection_check
  12                 project_name: functest
  13                 criteria: 100
  14                 blocking: true
  15                 description: >-
  16                     This test case verifies the retrieval of OpenStack clients:
  17                     Keystone, Glance, Neutron and Nova and may perform some
  18                     simple queries.
  19                 run:
  20                     name: connection_check
  21
  22             -
  23                 case_name: tenantnetwork1
  24                 project_name: functest
  25                 criteria: 100
  26                 blocking: true
  27                 description: >-
  28                     It creates and configures all tenant network ressources
  29                     required by advanced testcases (subnet, network and
  30                     router).
  31                 run:
  32                     name: tenantnetwork1
  33
  34             -
  35                 case_name: tenantnetwork2
  36                 project_name: functest
  37                 criteria: 100
  38                 blocking: true
  39                 description: >-
  40                     It creates new user/project before creating and configuring
  41                     all tenant network ressources required by a testcase
  42                     (subnet, network and router).
  43                 run:
  44                     name: tenantnetwork2
  45
  46             -
  47                 case_name: vmready1
  48                 project_name: functest
  49                 criteria: 100
  50                 blocking: true
  51                 description: >-
  52                     It inherits from TenantNetwork1 which creates all network
  53                     resources and prepares a future VM attached to that
  54                     network.
  55                 run:
  56                     name: vmready1
  57
  58             -
  59                 case_name: vmready2
  60                 project_name: functest
  61                 criteria: 100
  62                 blocking: true
  63                 description: >-
  64                     It creates new user/project before creating and configuring
  65                     all tenant network ressources, flavors, images, etc.
  66                     required by advanced testcases.
  67                 run:
  68                     name: vmready2
  69
  70             -
  71                 case_name: singlevm1
  72                 project_name: functest
  73                 criteria: 100
  74                 blocking: true
  75                 description: >-
  76                     It inherits from TenantNetwork1 which creates all network
  77                     resources and completes it by booting a VM attached to that
  78                     network.
  79                 run:
  80                     name: singlevm1
  81
  82             -
  83                 case_name: singlevm2
  84                 project_name: functest
  85                 criteria: 100
  86                 blocking: true
  87                 description: >-
  88                     It creates new user/project before creating and configuring
  89                     all tenant network ressources and vms required by advanced
  90                     testcases.
  91                 run:
  92                     name: singlevm2
  93
  94             -
  95                 case_name: vping_ssh
  96                 project_name: functest
  97                 criteria: 100
  98                 blocking: true
  99                 description: >-
 100                     This test case verifies: 1) SSH to an instance using
 101                     floating IPs over the public network. 2) Connectivity
 102                     between 2 instances over a private network.
 103                 run:
 104                     name: vping_ssh
 105
 106             -
 107                 case_name: vping_userdata
 108                 project_name: functest
 109                 criteria: 100
 110                 blocking: true
 111                 description: >-
 112                     This test case verifies:  1) Boot a VM with given userdata.
 113                     2) Connectivity between 2 instances over a private network.
 114                 run:
 115                     name: vping_userdata
 116
 117             -
 118                 case_name: cinder_test
 119                 project_name: functest
 120                 criteria: 100
 121                 blocking: true
 122                 description: >-
 123                     This test case verifies: 1) Attach volume and to 1
 124                     instance; 2) Write data on volume 3) Detach volume
 125                     from instance 1, attach it on instance 2 3) Read volume
 126                     data
 127                 run:
 128                     name: cinder_test
 129
 130             -
 131                 case_name: odl
 132                 project_name: functest
 133                 criteria: 100
 134                 blocking: true
 135                 description: >-
 136                     Test Suite for the OpenDaylight SDN Controller. It
 137                     integrates some test suites from upstream using
 138                     Robot as the test framework.
 139                 dependencies:
 140                     - DEPLOY_SCENARIO: 'odl'
 141                 run:
 142                     name: odl
 143                     args:
 144                         suites:
 145                             - /src/odl_test/csit/suites/integration/basic
 146                             - /src/odl_test/csit/suites/openstack/neutron
 147
 148             -
 149                 case_name: api_check
 150                 project_name: functest
 151                 criteria: 100
 152                 blocking: true
 153                 description: >-
 154                     This test case verifies the retrieval of OpenStack clients:
 155                     Keystone, Glance, Neutron and Nova and may perform some
 156                     simple queries. When the config value of
 157                     snaps.use_keystone is True, functest must have access to
 158                     the cloud's private network.
 159                 run:
 160                     name: api_check
 161
 162             -
 163                 case_name: snaps_health_check
 164                 project_name: functest
 165                 criteria: 100
 166                 blocking: true
 167                 description: >-
 168                     This test case creates executes the SimpleHealthCheck
 169                     Python test class which creates an, image, flavor, network,
 170                     and Cirros VM instance and observes the console output to
 171                     validate the single port obtains the correct IP address.
 172                 run:
 173                     name: snaps_health_check
 174
 175     -
 176         name: smoke
 177         order: 1
 178         description: >-
 179             Set of basic Functional tests to validate the OPNFV scenarios.
 180         testcases:
 181             -
 182                 case_name: tempest_smoke
 183                 project_name: functest
 184                 criteria: 100
 185                 blocking: false
 186                 description: >-
 187                     This test case runs the smoke subset of the OpenStack
 188                     Tempest suite. The list of test cases is generated by
 189                     Tempest automatically and depends on the parameters of
 190                     the OpenStack deplopyment.
 191                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L114
 192                 run:
 193                     name: tempest_common
 194                     args:
 195                         mode: '(?=.*\[.*\bsmoke\b.*\])(^tempest\.api)'
 196                         option:
 197                             - '--concurrency=4'
 198
 199             -
 200                 case_name: neutron-tempest-plugin-api
 201                 project_name: functest
 202                 criteria: 100
 203                 blocking: false
 204                 description: >-
 205                     This test case runs the Tempest suite proposed by the
 206                     Neutron project. The list of test cases is generated by
 207                     Tempest automatically and depends on the parameters of
 208                     the OpenStack deployment.
 209                 run:
 210                     name: tempest_common
 211                     args:
 212                         mode: '^neutron_tempest_plugin\.api'
 213                         option:
 214                             - '--concurrency=4'
 215
 216             -
 217                 case_name: rally_sanity
 218                 project_name: functest
 219                 criteria: 100
 220                 blocking: false
 221                 description: >-
 222                     This test case runs a sub group of tests of the OpenStack
 223                     Rally suite in smoke mode.
 224                 run:
 225                     name: rally_sanity
 226
 227             -
 228                 case_name: rally_jobs
 229                 project_name: functest
 230                 criteria: 100
 231                 blocking: false
 232                 description: >-
 233                     This test case runs a group of Rally jobs used in
 234                     OpenStack gating
 235                 run:
 236                     name: rally_jobs
 237
 238             -
 239                 case_name: refstack_defcore
 240                 project_name: functest
 241                 criteria: 100
 242                 blocking: false
 243                 description: >-
 244                     This test case runs a sub group of tests of the OpenStack
 245                     Defcore testcases.
 246                 run:
 247                     name: refstack_defcore
 248                     args:
 249                         option:
 250                             - '--concurrency=4'
 251
 252             -
 253                 case_name: patrole
 254                 project_name: functest
 255                 criteria: 100
 256                 blocking: false
 257                 description: >-
 258                     Test suite from Patrole project.
 259                 run:
 260                     name: patrole
 261                     args:
 262                         role: 'admin'
 263                         includes:
 264                             - 'image'
 265                             - 'network'
 266                         excludes:
 267                             - 'test_networks_multiprovider_rbac'
 268                             - 'test_auto_allocated_topology_rbac'
 269                             - 'test_address_scope_rbac'
 270
 271             -
 272                 case_name: snaps_smoke
 273                 project_name: functest
 274                 criteria: 100
 275                 blocking: false
 276                 description: >-
 277                     This test case contains tests that setup and destroy
 278                     environments with VMs with and without Floating IPs
 279                     with a newly created user and project. Set the config
 280                     value snaps.use_floating_ips (True|False) to toggle
 281                     this functionality. When the config value of
 282                     snaps.use_keystone is True, functest must have access to
 283                     the cloud's private network.
 284                 run:
 285                     name: snaps_smoke
 286
 287             -
 288                 case_name: neutron_trunk
 289                 project_name: functest
 290                 criteria: 100
 291                 blocking: false
 292                 description: >-
 293                     This test case runs the neutron trunk subtest of the
 294                     OpenStack Tempest suite. The list of test cases is
 295                     generated by Tempest having as input the relevant
 296                     testcase list file.
 297                 run:
 298                     name: tempest_common
 299                     args:
 300                         mode: 'neutron_tempest_plugin.(api|scenario).test_trunk'
 301                         neutron_extensions:
 302                             - trunk
 303                             - trunk-details
 304
 305             -
 306                 case_name: networking-bgpvpn
 307                 project_name: functest
 308                 criteria: 100
 309                 blocking: false
 310                 description: >-
 311                     This test case runs OpenStack networking-bgpvpn Tempest
 312                     suite. The list of test cases is generated by Tempest
 313                     having as input the relevant regex.
 314                 run:
 315                     name: tempest_common
 316                     args:
 317                         mode: '^networking_bgpvpn_tempest\.'
 318                         neutron_extensions:
 319                             - bgpvpn
 320                         option:
 321                             - '--concurrency=4'
 322
 323             -
 324                 case_name: networking-sfc
 325                 project_name: functest
 326                 criteria: 100
 327                 blocking: false
 328                 description: >-
 329                     This test case runs OpenStack networking-sfc Tempest
 330                     suite. The list of test cases is generated by Tempest
 331                     having as input the relevant regex.
 332                 run:
 333                     name: tempest_common
 334                     args:
 335                         mode:
 336                             '^(?:networking_sfc\.tests\.tempest_plugin.tests.api).*$'
 337                         neutron_extensions:
 338                             - sfc
 339                         option:
 340                             - '--concurrency=0'
 341
 342             -
 343                 case_name: barbican
 344                 project_name: functest
 345                 criteria: 100
 346                 blocking: false
 347                 description: >-
 348                     It leverages on the tempest plugin containing tests used to
 349                     verify the functionality of a barbican installation.
 350                 run:
 351                     name: barbican
 352                     args:
 353                         mode:
 354                             '^barbican_tempest_plugin.((?!test_signed_image_upload_boot_failure).)*$'
 355                         services:
 356                             - barbican
 357
 358     -
 359         name: benchmarking
 360         order: 2
 361         description: >-
 362             Run several OpenStack performance tools
 363             https://docs.openstack.org/performance-docs/latest/methodologies/tools.html
 364         testcases:
 365             -
 366                 case_name: vmtp
 367                 project_name: functest
 368                 criteria: 100
 369                 blocking: false
 370                 description: >-
 371                     VMTP is a small python application that will automatically
 372                     perform ping connectivity, round trip time measurement
 373                     (latency) and TCP/UDP throughput
 374                 dependencies:
 375                     - POD_ARCH: '^(?!aarch64$)'
 376                 run:
 377                     name: vmtp
 378
 379             -
 380                 case_name: shaker
 381                 project_name: functest
 382                 criteria: 100
 383                 blocking: false
 384                 description: >-
 385                     Shaker wraps around popular system network testing tools
 386                     like iperf, iperf3 and netperf (with help of flent). Shaker
 387                     is able to deploy OpenStack instances and networks in
 388                     different topologies.
 389                 run:
 390                     name: shaker
 391
 392     -
 393         name: features
 394         order: 3
 395         description: >-
 396             Test suites from feature projects
 397             integrated in functest
 398         testcases:
 399             -
 400                 case_name: doctor-notification
 401                 project_name: doctor
 402                 enabled: false
 403                 criteria: 100
 404                 blocking: false
 405                 description: >-
 406                     Test suite from Doctor project.
 407                 dependencies:
 408                     - INSTALLER_TYPE: '(apex)|(fuel)|(daisy)'
 409                     - DEPLOY_SCENARIO: '^((?!fdio|nofeature).)*$'
 410                 run:
 411                     name: bashfeature
 412                     args:
 413                         cmd: 'doctor-test'
 414
 415             -
 416                 case_name: bgpvpn
 417                 project_name: sdnvpn
 418                 enabled: false
 419                 criteria: 100
 420                 blocking: false
 421                 description: >-
 422                     Test suite from SDNVPN project.
 423                 dependencies:
 424                     - DEPLOY_SCENARIO: 'bgpvpn'
 425                 run:
 426                     name: bgpvpn
 427
 428             -
 429                 case_name: functest-odl-sfc
 430                 project_name: sfc
 431                 enabled: false
 432                 criteria: 100
 433                 blocking: false
 434                 description: >-
 435                     Test suite for odl-sfc to test two chains with one SF and
 436                     one chain with two SFs
 437                 dependencies:
 438                     - DEPLOY_SCENARIO: 'odl.*sfc'
 439                 run:
 440                     name: functest-odl-sfc
 441
 442             -
 443                 case_name: barometercollectd
 444                 project_name: barometer
 445                 enabled: false
 446                 criteria: 100
 447                 blocking: false
 448                 description: >-
 449                     Test suite for the Barometer project. Separate tests verify
 450                     the proper configuration and basic functionality of all the
 451                     collectd plugins as described in the Project Release Plan
 452                 dependencies:
 453                     - DEPLOY_SCENARIO: 'bar'
 454                 run:
 455                     name: barometercollectd
 456
 457             -
 458                 case_name: fds
 459                 project_name: fastdatastacks
 460                 enabled: false
 461                 criteria: 100
 462                 blocking: false
 463                 description: >-
 464                     Test Suite for the OpenDaylight SDN Controller when GBP
 465                     features are installed. It integrates some test suites from
 466                     upstream using Robot as the test framework.
 467                 dependencies:
 468                     - DEPLOY_SCENARIO: 'odl.*-fdio'
 469                 run:
 470                     name: odl
 471                     args:
 472                         suites:
 473                             - /src/fds/testing/robot
 474             -
 475                 case_name: vgpu
 476                 project_name: functest
 477                 criteria: 100
 478                 blocking: false
 479                 description: >-
 480                     Test suite for the OpenStack vGPU feature
 481                 dependencies:
 482                     - DEPLOY_SCENARIO: 'vgpu'
 483                 run:
 484                     name: vgpu
 485
 486             -
 487                 case_name: stor4nfv_os
 488                 project_name: stor4nfv
 489                 enabled: false
 490                 criteria: 100
 491                 blocking: false
 492                 description: >-
 493                     This tests are necessary demonstrate conformance
 494                     of the OpenStack+Stor4NFV deployment.
 495                 dependencies:
 496                     - DEPLOY_SCENARIO: 'stor4nfv'
 497                 run:
 498                     name: stor4nfv_os
 499
 500     -
 501         name: components
 502         order: 4
 503         description: >-
 504             Extensive testing of OpenStack API.
 505         testcases:
 506             -
 507                 case_name: tempest_full
 508                 project_name: functest
 509                 criteria: 100
 510                 blocking: false
 511                 description: >-
 512                     The list of test cases is generated by
 513                     Tempest automatically and depends on the parameters of
 514                     the OpenStack deployment.
 515                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83
 516                 run:
 517                     name: tempest_common
 518                     args:
 519                         mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
 520                         option:
 521                             - '--concurrency=4'
 522
 523             -
 524                 case_name: tempest_scenario
 525                 project_name: functest
 526                 criteria: 100
 527                 blocking: false
 528                 description: >-
 529                     The list of test cases is generated by
 530                     Tempest automatically and depends on the parameters of
 531                     the OpenStack deployment.
 532                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
 533                 run:
 534                     name: tempest_common
 535                     args:
 536                         mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)'
 537                         option:
 538                             - '--concurrency=1'
 539
 540             -
 541                 case_name: rally_full
 542                 project_name: functest
 543                 criteria: 100
 544                 blocking: false
 545                 description: >-
 546                     This test case runs the full suite of scenarios of the
 547                     OpenStack Rally suite using several threads and iterations.
 548                 run:
 549                     name: rally_full
 550
 551     -
 552         name: vnf
 553         order: 5
 554         description: >-
 555             Collection of VNF test cases.
 556         testcases:
 557             -
 558                 case_name: cloudify
 559                 project_name: functest
 560                 criteria: 100
 561                 blocking: false
 562                 description: >-
 563                     This test case deploys the Cloudify orchestrator.
 564                 dependencies:
 565                     - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha'
 566                 run:
 567                     name: cloudify
 568             -
 569                 case_name: cloudify_ims
 570                 project_name: functest
 571                 criteria: 100
 572                 blocking: false
 573                 description: >-
 574                     This test case deploys an OpenSource vIMS solution from
 575                     Clearwater using the Cloudify orchestrator. It also runs
 576                     some signaling traffic.
 577                 dependencies:
 578                     - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha'
 579                 run:
 580                     name: cloudify_ims
 581
 582             -
 583                 case_name: heat_ims
 584                 project_name: functest
 585                 criteria: 100
 586                 blocking: false
 587                 description: >-
 588                     This test case deploys an OpenSource vIMS solution from
 589                     Clearwater using the OpenStack Heat orchestrator.
 590                     It also runs some signaling traffic.
 591                 dependencies:
 592                     - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha'
 593                 run:
 594                     name: heat_ims
 595
 596             -
 597                 case_name: vyos_vrouter
 598                 project_name: functest
 599                 criteria: 100
 600                 blocking: false
 601                 description: >-
 602                     This test case is vRouter testing.
 603                 dependencies:
 604                     - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha'
 605                 run:
 606                     name: vyos_vrouter
 607
 608             -
 609                 case_name: juju_epc
 610                 project_name: functest
 611                 criteria: 100
 612                 blocking: false
 613                 description: >-
 614                     vEPC validation with Juju as VNF manager and ABoT as test
 615                     executor.
 616                 dependencies:
 617                     - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha'
 618                 run:
 619                     name: juju_epc