functest/ci/testcases.yaml

   1 ---
   2 tiers:
   3     -
   4         name: healthcheck
   5         order: 0
   6         description: >-
   7             First tier to be executed to verify the basic
   8             operations in the VIM.
   9         testcases:
  10             -
  11                 case_name: connection_check
  12                 project_name: functest
  13                 criteria: 100
  14                 blocking: true
  15                 description: >-
  16                     This test case verifies the retrieval of OpenStack clients:
  17                     Keystone, Glance, Neutron and Nova and may perform some
  18                     simple queries.
  19                 run:
  20                     name: connection_check
  21
  22             -
  23                 case_name: tenantnetwork1
  24                 project_name: functest
  25                 criteria: 100
  26                 blocking: true
  27                 description: >-
  28                     It creates and configures all tenant network ressources
  29                     required by advanced testcases (subnet, network and
  30                     router).
  31                 run:
  32                     name: tenantnetwork1
  33
  34             -
  35                 case_name: tenantnetwork2
  36                 project_name: functest
  37                 criteria: 100
  38                 blocking: true
  39                 description: >-
  40                     It creates new user/project before creating and configuring
  41                     all tenant network ressources required by a testcase
  42                     (subnet, network and router).
  43                 run:
  44                     name: tenantnetwork2
  45
  46             -
  47                 case_name: vmready1
  48                 project_name: functest
  49                 criteria: 100
  50                 blocking: true
  51                 description: >-
  52                     It inherits from TenantNetwork1 which creates all network
  53                     resources and prepares a future VM attached to that
  54                     network.
  55                 run:
  56                     name: vmready1
  57
  58             -
  59                 case_name: vmready2
  60                 project_name: functest
  61                 criteria: 100
  62                 blocking: true
  63                 description: >-
  64                     It creates new user/project before creating and configuring
  65                     all tenant network ressources, flavors, images, etc.
  66                     required by advanced testcases.
  67                 run:
  68                     name: vmready2
  69
  70             -
  71                 case_name: singlevm1
  72                 project_name: functest
  73                 criteria: 100
  74                 blocking: true
  75                 description: >-
  76                     It inherits from TenantNetwork1 which creates all network
  77                     resources and completes it by booting a VM attached to that
  78                     network.
  79                 run:
  80                     name: singlevm1
  81
  82             -
  83                 case_name: singlevm2
  84                 project_name: functest
  85                 criteria: 100
  86                 blocking: true
  87                 description: >-
  88                     It creates new user/project before creating and configuring
  89                     all tenant network ressources and vms required by advanced
  90                     testcases.
  91                 run:
  92                     name: singlevm2
  93
  94             -
  95                 case_name: vping_ssh
  96                 project_name: functest
  97                 criteria: 100
  98                 blocking: true
  99                 description: >-
 100                     This test case verifies: 1) SSH to an instance using
 101                     floating IPs over the public network. 2) Connectivity
 102                     between 2 instances over a private network.
 103                 run:
 104                     name: vping_ssh
 105
 106             -
 107                 case_name: vping_userdata
 108                 project_name: functest
 109                 criteria: 100
 110                 blocking: true
 111                 description: >-
 112                     This test case verifies:  1) Boot a VM with given userdata.
 113                     2) Connectivity between 2 instances over a private network.
 114                 run:
 115                     name: vping_userdata
 116
 117             -
 118                 case_name: cinder_test
 119                 project_name: functest
 120                 criteria: 100
 121                 blocking: true
 122                 description: >-
 123                     This test case verifies: 1) Attach volume and to 1
 124                     instance; 2) Write data on volume 3) Detach volume
 125                     from instance 1, attach it on instance 2 3) Read volume
 126                     data
 127                 run:
 128                     name: cinder_test
 129
 130             -
 131                 case_name: odl
 132                 project_name: functest
 133                 criteria: 100
 134                 blocking: true
 135                 description: >-
 136                     Test Suite for the OpenDaylight SDN Controller. It
 137                     integrates some test suites from upstream using
 138                     Robot as the test framework.
 139                 dependencies:
 140                     - DEPLOY_SCENARIO: 'odl'
 141                 run:
 142                     name: odl
 143                     args:
 144                         suites:
 145                             - /src/odl_test/csit/suites/integration/basic
 146                             - /src/odl_test/csit/suites/openstack/neutron
 147
 148             -
 149                 case_name: tempest_smoke
 150                 project_name: functest
 151                 criteria: 100
 152                 blocking: false
 153                 description: >-
 154                     This test case runs the smoke subset of the OpenStack
 155                     Tempest suite. The list of test cases is generated by
 156                     Tempest automatically and depends on the parameters of
 157                     the OpenStack deplopyment.
 158                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L114
 159                 run:
 160                     name: tempest_common
 161                     args:
 162                         mode: '(?=.*\[.*\bsmoke\b.*\])(^tempest\.api)'
 163                         option:
 164                             - '--concurrency=4'
 165
 166     -
 167         name: smoke
 168         order: 1
 169         description: >-
 170             Set of basic Functional tests to validate the OPNFV scenarios.
 171         testcases:
 172             -
 173                 case_name: neutron-tempest-plugin-api
 174                 project_name: functest
 175                 criteria: 100
 176                 blocking: false
 177                 description: >-
 178                     This test case runs the Tempest suite proposed by the
 179                     Neutron project. The list of test cases is generated by
 180                     Tempest automatically and depends on the parameters of
 181                     the OpenStack deployment.
 182                 run:
 183                     name: tempest_common
 184                     args:
 185                         mode: '^neutron_tempest_plugin\.api'
 186                         option:
 187                             - '--concurrency=4'
 188
 189             -
 190                 case_name: rally_sanity
 191                 project_name: functest
 192                 criteria: 100
 193                 blocking: false
 194                 description: >-
 195                     This test case runs a sub group of tests of the OpenStack
 196                     Rally suite in smoke mode.
 197                 run:
 198                     name: rally_sanity
 199                     args:
 200                         optional:
 201                             - 'gnocchi'
 202
 203             -
 204                 case_name: refstack_defcore
 205                 project_name: functest
 206                 criteria: 100
 207                 blocking: false
 208                 deny_skipping: true
 209                 description: >-
 210                     This test case runs a sub group of tests of the OpenStack
 211                     Defcore testcases.
 212                 run:
 213                     name: refstack_defcore
 214                     args:
 215                         option:
 216                             - '--concurrency=4'
 217
 218             -
 219                 case_name: tempest_full
 220                 project_name: functest
 221                 criteria: 100
 222                 blocking: false
 223                 description: >-
 224                     The list of test cases is generated by
 225                     Tempest automatically and depends on the parameters of
 226                     the OpenStack deployment.
 227                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83
 228                 run:
 229                     name: tempest_common
 230                     args:
 231                         mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
 232                         option:
 233                             - '--concurrency=4'
 234
 235             -
 236                 case_name: tempest_scenario
 237                 project_name: functest
 238                 criteria: 100
 239                 blocking: false
 240                 description: >-
 241                     The list of test cases is generated by
 242                     Tempest automatically and depends on the parameters of
 243                     the OpenStack deployment.
 244                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
 245                 run:
 246                     name: tempest_common
 247                     args:
 248                         mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)'
 249                         option:
 250                             - '--concurrency=1'
 251
 252             -
 253                 case_name: patrole
 254                 project_name: functest
 255                 criteria: 100
 256                 blocking: false
 257                 description: >-
 258                     Test suite from Patrole project.
 259                 run:
 260                     name: patrole
 261                     args:
 262                         role: 'admin'
 263                         includes:
 264                             - 'image'
 265                             - 'network'
 266                         excludes:
 267                             - 'network.test_flavors_rbac'
 268                             - 'network.test_network_ip_availability_rbac'
 269                             - 'test_networks_multiprovider_rbac'
 270                             - 'test_auto_allocated_topology_rbac'
 271                             - 'test_create_network_provider_segmentation_id'
 272                             - 'test_update_network_provider_physical_network'
 273                             - 'test_update_network_provider_segmentation_id'
 274                             - 'network.test_agents_rbac'
 275
 276             -
 277                 case_name: neutron_trunk
 278                 project_name: functest
 279                 criteria: 100
 280                 blocking: false
 281                 description: >-
 282                     This test case runs the neutron trunk subtest of the
 283                     OpenStack Tempest suite. The list of test cases is
 284                     generated by Tempest having as input the relevant
 285                     testcase list file.
 286                 run:
 287                     name: tempest_common
 288                     args:
 289                         mode: 'neutron_tempest_plugin.(api|scenario).test_trunk'
 290                         neutron_extensions:
 291                             - trunk
 292                             - trunk-details
 293
 294             -
 295                 case_name: networking-bgpvpn
 296                 project_name: functest
 297                 criteria: 100
 298                 blocking: false
 299                 description: >-
 300                     This test case runs OpenStack networking-bgpvpn Tempest
 301                     suite. The list of test cases is generated by Tempest
 302                     having as input the relevant regex.
 303                 run:
 304                     name: tempest_common
 305                     args:
 306                         mode: '^networking_bgpvpn_tempest.tests.api\.'
 307                         neutron_extensions:
 308                             - bgpvpn
 309                         option:
 310                             - '--concurrency=4'
 311
 312             -
 313                 case_name: networking-sfc
 314                 project_name: functest
 315                 criteria: 100
 316                 blocking: false
 317                 description: >-
 318                     This test case runs OpenStack networking-sfc Tempest
 319                     suite. The list of test cases is generated by Tempest
 320                     having as input the relevant regex.
 321                 run:
 322                     name: tempest_common
 323                     args:
 324                         mode:
 325                             '^(?:networking_sfc\.tests\.tempest_plugin.tests.api).*$'
 326                         neutron_extensions:
 327                             - sfc
 328                         option:
 329                             - '--concurrency=0'
 330
 331             -
 332                 case_name: barbican
 333                 project_name: functest
 334                 criteria: 100
 335                 blocking: false
 336                 description: >-
 337                     It leverages on the tempest plugin containing tests used to
 338                     verify the functionality of a barbican installation.
 339                 run:
 340                     name: barbican
 341                     args:
 342                         mode:
 343                             '^barbican_tempest_plugin.((?!test_signed_image_upload_boot_failure).)*$'
 344                         services:
 345                             - barbican
 346
 347     -
 348         name: benchmarking
 349         order: 2
 350         description: >-
 351             Run several OpenStack performance tools
 352             https://docs.openstack.org/performance-docs/latest/methodologies/tools.html
 353         testcases:
 354             -
 355                 case_name: rally_full
 356                 project_name: functest
 357                 criteria: 100
 358                 blocking: false
 359                 description: >-
 360                     This test case runs the full suite of scenarios of the
 361                     OpenStack Rally suite using several threads and iterations.
 362                 run:
 363                     name: rally_full
 364                     args:
 365                         optional:
 366                             - 'gnocchi'
 367
 368             -
 369                 case_name: rally_jobs
 370                 project_name: functest
 371                 criteria: 100
 372                 blocking: false
 373                 description: >-
 374                     This test case runs a group of Rally jobs used in
 375                     OpenStack gating
 376                 run:
 377                     name: rally_jobs
 378                     args:
 379                         optional:
 380                             - 'gnocchi'
 381
 382             -
 383                 case_name: vmtp
 384                 project_name: functest
 385                 enabled: false
 386                 criteria: 100
 387                 blocking: false
 388                 description: >-
 389                     VMTP is a small python application that will automatically
 390                     perform ping connectivity, round trip time measurement
 391                     (latency) and TCP/UDP throughput
 392                 dependencies:
 393                     - POD_ARCH: '^(?!aarch64$)'
 394                 run:
 395                     name: vmtp
 396
 397             -
 398                 case_name: shaker
 399                 project_name: functest
 400                 criteria: 100
 401                 blocking: false
 402                 description: >-
 403                     Shaker wraps around popular system network testing tools
 404                     like iperf, iperf3 and netperf (with help of flent). Shaker
 405                     is able to deploy OpenStack instances and networks in
 406                     different topologies.
 407                 run:
 408                     name: shaker
 409
 410     -
 411         name: features
 412         order: 3
 413         description: >-
 414             Test suites from feature projects
 415             integrated in functest
 416         testcases:
 417             -
 418                 case_name: doctor-notification
 419                 project_name: doctor
 420                 enabled: false
 421                 criteria: 100
 422                 blocking: false
 423                 description: >-
 424                     Test suite from Doctor project.
 425                 dependencies:
 426                     - INSTALLER_TYPE: '(apex)|(fuel)|(daisy)'
 427                     - DEPLOY_SCENARIO: '^((?!fdio|nofeature).)*$'
 428                 run:
 429                     name: bashfeature
 430                     args:
 431                         cmd: 'doctor-test'
 432
 433             -
 434                 case_name: functest-odl-sfc
 435                 project_name: sfc
 436                 criteria: 100
 437                 blocking: false
 438                 description: >-
 439                     Test suite for odl-sfc to test two chains with one SF and
 440                     one chain with two SFs
 441                 dependencies:
 442                     - DEPLOY_SCENARIO: 'odl.*sfc'
 443                 run:
 444                     name: functest-odl-sfc
 445
 446             -
 447                 case_name: barometercollectd
 448                 project_name: barometer
 449                 enabled: false
 450                 criteria: 100
 451                 blocking: false
 452                 description: >-
 453                     Test suite for the Barometer project. Separate tests verify
 454                     the proper configuration and basic functionality of all the
 455                     collectd plugins as described in the Project Release Plan
 456                 dependencies:
 457                     - DEPLOY_SCENARIO: 'bar'
 458                 run:
 459                     name: barometercollectd
 460
 461             -
 462                 case_name: vgpu
 463                 project_name: functest
 464                 criteria: 100
 465                 blocking: false
 466                 description: >-
 467                     Test suite for the OpenStack vGPU feature
 468                 dependencies:
 469                     - DEPLOY_SCENARIO: 'vgpu'
 470                 run:
 471                     name: vgpu
 472
 473             -
 474                 case_name: stor4nfv_os
 475                 project_name: stor4nfv
 476                 enabled: false
 477                 criteria: 100
 478                 blocking: false
 479                 description: >-
 480                     This tests are necessary demonstrate conformance
 481                     of the OpenStack+Stor4NFV deployment.
 482                 dependencies:
 483                     - DEPLOY_SCENARIO: 'stor4nfv'
 484                 run:
 485                     name: stor4nfv_os
 486
 487     -
 488         name: vnf
 489         order: 4
 490         description: >-
 491             Collection of VNF test cases.
 492         testcases:
 493             -
 494                 case_name: cloudify
 495                 project_name: functest
 496                 criteria: 100
 497                 blocking: false
 498                 description: >-
 499                     This test case deploys the Cloudify orchestrator.
 500                 run:
 501                     name: cloudify
 502             -
 503                 case_name: cloudify_ims
 504                 project_name: functest
 505                 criteria: 100
 506                 blocking: false
 507                 description: >-
 508                     This test case deploys an OpenSource vIMS solution from
 509                     Clearwater using the Cloudify orchestrator. It also runs
 510                     some signaling traffic.
 511                 dependencies:
 512                     - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha'
 513                 run:
 514                     name: cloudify_ims
 515
 516             -
 517                 case_name: heat_ims
 518                 project_name: functest
 519                 criteria: 100
 520                 blocking: false
 521                 description: >-
 522                     This test case deploys an OpenSource vIMS solution from
 523                     Clearwater using the OpenStack Heat orchestrator.
 524                     It also runs some signaling traffic.
 525                 run:
 526                     name: heat_ims
 527
 528             -
 529                 case_name: vyos_vrouter
 530                 project_name: functest
 531                 criteria: 100
 532                 blocking: false
 533                 description: >-
 534                     This test case is vRouter testing.
 535                 run:
 536                     name: vyos_vrouter
 537
 538             -
 539                 case_name: juju_epc
 540                 project_name: functest
 541                 criteria: 100
 542                 blocking: false
 543                 description: >-
 544                     vEPC validation with Juju as VNF manager and ABoT as test
 545                     executor.
 546                 run:
 547                     name: juju_epc