functest/ci/testcases.yaml

   1 ---
   2 tiers:
   3     -
   4         name: healthcheck
   5         order: 0
   6         description: >-
   7             First tier to be executed to verify the basic
   8             operations in the VIM.
   9         testcases:
  10             -
  11                 case_name: connection_check
  12                 project_name: functest
  13                 criteria: 100
  14                 blocking: true
  15                 description: >-
  16                     This test case verifies the retrieval of OpenStack clients:
  17                     Keystone, Glance, Neutron and Nova and may perform some
  18                     simple queries.
  19                 run:
  20                     name: connection_check
  21
  22             -
  23                 case_name: tenantnetwork1
  24                 project_name: functest
  25                 criteria: 100
  26                 blocking: true
  27                 description: >-
  28                     It creates and configures all tenant network ressources
  29                     required by advanced testcases (subnet, network and
  30                     router).
  31                 run:
  32                     name: tenantnetwork1
  33
  34             -
  35                 case_name: tenantnetwork2
  36                 project_name: functest
  37                 criteria: 100
  38                 blocking: true
  39                 description: >-
  40                     It creates new user/project before creating and configuring
  41                     all tenant network ressources required by a testcase
  42                     (subnet, network and router).
  43                 run:
  44                     name: tenantnetwork2
  45
  46             -
  47                 case_name: vmready1
  48                 project_name: functest
  49                 criteria: 100
  50                 blocking: true
  51                 description: >-
  52                     It inherits from TenantNetwork1 which creates all network
  53                     resources and prepares a future VM attached to that
  54                     network.
  55                 run:
  56                     name: vmready1
  57
  58             -
  59                 case_name: vmready2
  60                 project_name: functest
  61                 criteria: 100
  62                 blocking: true
  63                 description: >-
  64                     It creates new user/project before creating and configuring
  65                     all tenant network ressources, flavors, images, etc.
  66                     required by advanced testcases.
  67                 run:
  68                     name: vmready2
  69
  70             -
  71                 case_name: singlevm1
  72                 project_name: functest
  73                 criteria: 100
  74                 blocking: true
  75                 description: >-
  76                     It inherits from TenantNetwork1 which creates all network
  77                     resources and completes it by booting a VM attached to that
  78                     network.
  79                 run:
  80                     name: singlevm1
  81
  82             -
  83                 case_name: singlevm2
  84                 project_name: functest
  85                 criteria: 100
  86                 blocking: true
  87                 description: >-
  88                     It creates new user/project before creating and configuring
  89                     all tenant network ressources and vms required by advanced
  90                     testcases.
  91                 run:
  92                     name: singlevm2
  93
  94             -
  95                 case_name: vping_ssh
  96                 project_name: functest
  97                 criteria: 100
  98                 blocking: true
  99                 description: >-
 100                     This test case verifies: 1) SSH to an instance using
 101                     floating IPs over the public network. 2) Connectivity
 102                     between 2 instances over a private network.
 103                 run:
 104                     name: vping_ssh
 105
 106             -
 107                 case_name: vping_userdata
 108                 project_name: functest
 109                 criteria: 100
 110                 blocking: true
 111                 description: >-
 112                     This test case verifies:  1) Boot a VM with given userdata.
 113                     2) Connectivity between 2 instances over a private network.
 114                 run:
 115                     name: vping_userdata
 116
 117             -
 118                 case_name: cinder_test
 119                 project_name: functest
 120                 criteria: 100
 121                 blocking: true
 122                 description: >-
 123                     This test case verifies: 1) Attach volume and to 1
 124                     instance; 2) Write data on volume 3) Detach volume
 125                     from instance 1, attach it on instance 2 3) Read volume
 126                     data
 127                 run:
 128                     name: cinder_test
 129
 130             -
 131                 case_name: odl
 132                 project_name: functest
 133                 criteria: 100
 134                 blocking: true
 135                 description: >-
 136                     Test Suite for the OpenDaylight SDN Controller. It
 137                     integrates some test suites from upstream using
 138                     Robot as the test framework.
 139                 dependencies:
 140                     - DEPLOY_SCENARIO: 'odl'
 141                 run:
 142                     name: odl
 143                     args:
 144                         suites:
 145                             - /src/odl_test/csit/suites/integration/basic
 146                             - /src/odl_test/csit/suites/openstack/neutron
 147
 148             -
 149                 case_name: tempest_smoke
 150                 project_name: functest
 151                 criteria: 100
 152                 blocking: false
 153                 description: >-
 154                     This test case runs the smoke subset of the OpenStack
 155                     Tempest suite. The list of test cases is generated by
 156                     Tempest automatically and depends on the parameters of
 157                     the OpenStack deplopyment.
 158                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L114
 159                 run:
 160                     name: tempest_common
 161                     args:
 162                         mode: '(?=.*\[.*\bsmoke\b.*\])(^tempest\.api)'
 163                         option:
 164                             - '--concurrency=4'
 165
 166     -
 167         name: smoke
 168         order: 1
 169         description: >-
 170             Set of basic Functional tests to validate the OPNFV scenarios.
 171         testcases:
 172             -
 173                 case_name: neutron-tempest-plugin-api
 174                 project_name: functest
 175                 criteria: 100
 176                 blocking: false
 177                 description: >-
 178                     This test case runs the Tempest suite proposed by the
 179                     Neutron project. The list of test cases is generated by
 180                     Tempest automatically and depends on the parameters of
 181                     the OpenStack deployment.
 182                 run:
 183                     name: tempest_common
 184                     args:
 185                         mode: '^neutron_tempest_plugin\.api'
 186                         option:
 187                             - '--concurrency=4'
 188
 189             -
 190                 case_name: rally_sanity
 191                 project_name: functest
 192                 criteria: 100
 193                 blocking: false
 194                 description: >-
 195                     This test case runs a sub group of tests of the OpenStack
 196                     Rally suite in smoke mode.
 197                 run:
 198                     name: rally_sanity
 199                     args:
 200                         optional:
 201                             - 'gnocchi'
 202
 203             -
 204                 case_name: refstack_compute
 205                 project_name: functest
 206                 criteria: 100
 207                 blocking: false
 208                 deny_skipping: true
 209                 description: >-
 210                     This test case runs a sub group of tests of the OpenStack
 211                     Compute testcases.
 212                 run:
 213                     name: refstack
 214                     args:
 215                         target: compute
 216                         option:
 217                             - '--concurrency=4'
 218
 219             -
 220                 case_name: refstack_object
 221                 project_name: functest
 222                 criteria: 100
 223                 blocking: false
 224                 deny_skipping: true
 225                 description: >-
 226                     This test case runs a sub group of tests of the OpenStack
 227                     Object testcases.
 228                 run:
 229                     name: refstack
 230                     args:
 231                         target: object
 232                         option:
 233                             - '--concurrency=4'
 234
 235             -
 236                 case_name: refstack_platform
 237                 project_name: functest
 238                 criteria: 100
 239                 blocking: false
 240                 deny_skipping: true
 241                 description: >-
 242                     This test case runs a sub group of tests of the OpenStack
 243                     Platform testcases.
 244                 run:
 245                     name: refstack
 246                     args:
 247                         target: platform
 248                         option:
 249                             - '--concurrency=4'
 250
 251             -
 252                 case_name: tempest_full
 253                 project_name: functest
 254                 criteria: 100
 255                 blocking: false
 256                 description: >-
 257                     The list of test cases is generated by
 258                     Tempest automatically and depends on the parameters of
 259                     the OpenStack deployment.
 260                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83
 261                 run:
 262                     name: tempest_common
 263                     args:
 264                         mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
 265                         option:
 266                             - '--concurrency=4'
 267
 268             -
 269                 case_name: tempest_scenario
 270                 project_name: functest
 271                 criteria: 100
 272                 blocking: false
 273                 description: >-
 274                     The list of test cases is generated by
 275                     Tempest automatically and depends on the parameters of
 276                     the OpenStack deployment.
 277                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
 278                 run:
 279                     name: tempest_common
 280                     args:
 281                         mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)'
 282                         option:
 283                             - '--concurrency=1'
 284
 285             -
 286                 case_name: patrole
 287                 project_name: functest
 288                 criteria: 100
 289                 blocking: false
 290                 description: >-
 291                     Test suite from Patrole project.
 292                 run:
 293                     name: patrole
 294                     args:
 295                         role: 'admin'
 296                         includes:
 297                             - 'image'
 298                             - 'network'
 299                         excludes:
 300                             - 'network.test_flavors_rbac'
 301                             - 'network.test_network_ip_availability_rbac'
 302                             - 'test_networks_multiprovider_rbac'
 303                             - 'test_auto_allocated_topology_rbac'
 304                             - 'test_create_network_provider_segmentation_id'
 305                             - 'test_update_network_provider_physical_network'
 306                             - 'test_update_network_provider_segmentation_id'
 307                             - 'network.test_agents_rbac'
 308                             - 'test_list_metadef_namespaces'
 309
 310             -
 311                 case_name: neutron_trunk
 312                 project_name: functest
 313                 criteria: 100
 314                 blocking: false
 315                 description: >-
 316                     This test case runs the neutron trunk subtest of the
 317                     OpenStack Tempest suite. The list of test cases is
 318                     generated by Tempest having as input the relevant
 319                     testcase list file.
 320                 run:
 321                     name: tempest_common
 322                     args:
 323                         mode: 'neutron_tempest_plugin.(api|scenario).test_trunk'
 324                         neutron_extensions:
 325                             - trunk
 326                             - trunk-details
 327
 328             -
 329                 case_name: networking-bgpvpn
 330                 project_name: functest
 331                 criteria: 100
 332                 blocking: false
 333                 description: >-
 334                     This test case runs OpenStack networking-bgpvpn Tempest
 335                     suite. The list of test cases is generated by Tempest
 336                     having as input the relevant regex.
 337                 run:
 338                     name: tempest_common
 339                     args:
 340                         mode: '^neutron_tempest_plugin.bgpvpn.api\.'
 341                         neutron_extensions:
 342                             - bgpvpn
 343                         option:
 344                             - '--concurrency=4'
 345
 346             -
 347                 case_name: networking-sfc
 348                 project_name: functest
 349                 criteria: 100
 350                 blocking: false
 351                 description: >-
 352                     This test case runs OpenStack networking-sfc Tempest
 353                     suite. The list of test cases is generated by Tempest
 354                     having as input the relevant regex.
 355                 run:
 356                     name: tempest_common
 357                     args:
 358                         mode:
 359                             '^(?:neutron_tempest_plugin\.sfc\.tests\.api).*$'
 360                         neutron_extensions:
 361                             - sfc
 362                         option:
 363                             - '--concurrency=0'
 364
 365             -
 366                 case_name: barbican
 367                 project_name: functest
 368                 criteria: 100
 369                 blocking: false
 370                 description: >-
 371                     It leverages on the tempest plugin containing tests used to
 372                     verify the functionality of a barbican installation.
 373                 run:
 374                     name: barbican
 375                     args:
 376                         mode:
 377                             '^barbican_tempest_plugin.((?!test_signed_image_upload_boot_failure).)*$'
 378                         services:
 379                             - barbican
 380
 381     -
 382         name: benchmarking
 383         order: 2
 384         description: >-
 385             Run several OpenStack performance tools
 386             https://docs.openstack.org/performance-docs/latest/methodologies/tools.html
 387         testcases:
 388             -
 389                 case_name: rally_full
 390                 project_name: functest
 391                 criteria: 100
 392                 blocking: false
 393                 description: >-
 394                     This test case runs the full suite of scenarios of the
 395                     OpenStack Rally suite using several threads and iterations.
 396                 run:
 397                     name: rally_full
 398                     args:
 399                         optional:
 400                             - 'gnocchi'
 401
 402             -
 403                 case_name: rally_jobs
 404                 project_name: functest
 405                 criteria: 100
 406                 blocking: false
 407                 description: >-
 408                     This test case runs a group of Rally jobs used in
 409                     OpenStack gating
 410                 run:
 411                     name: rally_jobs
 412                     args:
 413                         optional:
 414                             - 'gnocchi'
 415
 416             -
 417                 case_name: vmtp
 418                 project_name: functest
 419                 criteria: 100
 420                 blocking: false
 421                 description: >-
 422                     VMTP is a small python application that will automatically
 423                     perform ping connectivity, round trip time measurement
 424                     (latency) and TCP/UDP throughput
 425                 dependencies:
 426                     - POD_ARCH: '^(?!aarch64$)'
 427                 run:
 428                     name: vmtp
 429
 430             -
 431                 case_name: shaker
 432                 project_name: functest
 433                 criteria: 100
 434                 blocking: false
 435                 description: >-
 436                     Shaker wraps around popular system network testing tools
 437                     like iperf, iperf3 and netperf (with help of flent). Shaker
 438                     is able to deploy OpenStack instances and networks in
 439                     different topologies.
 440                 run:
 441                     name: shaker
 442
 443     -
 444         name: features
 445         order: 3
 446         description: >-
 447             Test suites from feature projects
 448             integrated in functest
 449         testcases:
 450             -
 451                 case_name: doctor-notification
 452                 project_name: doctor
 453                 enabled: false
 454                 criteria: 100
 455                 blocking: false
 456                 description: >-
 457                     Test suite from Doctor project.
 458                 dependencies:
 459                     - INSTALLER_TYPE: '(apex)|(fuel)|(daisy)'
 460                     - DEPLOY_SCENARIO: '^((?!fdio|nofeature).)*$'
 461                 run:
 462                     name: bashfeature
 463                     args:
 464                         cmd: 'doctor-test'
 465
 466             -
 467                 case_name: functest-odl-sfc
 468                 project_name: sfc
 469                 criteria: 100
 470                 blocking: false
 471                 description: >-
 472                     Test suite for odl-sfc to test two chains with one SF and
 473                     one chain with two SFs
 474                 dependencies:
 475                     - DEPLOY_SCENARIO: 'odl.*sfc'
 476                 run:
 477                     name: functest-odl-sfc
 478
 479             -
 480                 case_name: barometercollectd
 481                 project_name: barometer
 482                 enabled: false
 483                 criteria: 100
 484                 blocking: false
 485                 description: >-
 486                     Test suite for the Barometer project. Separate tests verify
 487                     the proper configuration and basic functionality of all the
 488                     collectd plugins as described in the Project Release Plan
 489                 dependencies:
 490                     - DEPLOY_SCENARIO: 'bar'
 491                 run:
 492                     name: barometercollectd
 493
 494             -
 495                 case_name: vgpu
 496                 project_name: functest
 497                 criteria: 100
 498                 blocking: false
 499                 description: >-
 500                     Test suite for the OpenStack vGPU feature
 501                 dependencies:
 502                     - DEPLOY_SCENARIO: 'vgpu'
 503                 run:
 504                     name: vgpu
 505
 506             -
 507                 case_name: stor4nfv_os
 508                 project_name: stor4nfv
 509                 enabled: false
 510                 criteria: 100
 511                 blocking: false
 512                 description: >-
 513                     This tests are necessary demonstrate conformance
 514                     of the OpenStack+Stor4NFV deployment.
 515                 dependencies:
 516                     - DEPLOY_SCENARIO: 'stor4nfv'
 517                 run:
 518                     name: stor4nfv_os
 519
 520     -
 521         name: vnf
 522         order: 4
 523         description: >-
 524             Collection of VNF test cases.
 525         testcases:
 526             -
 527                 case_name: cloudify
 528                 project_name: functest
 529                 criteria: 100
 530                 blocking: false
 531                 description: >-
 532                     This test case deploys the Cloudify orchestrator.
 533                 run:
 534                     name: cloudify
 535             -
 536                 case_name: cloudify_ims
 537                 project_name: functest
 538                 criteria: 100
 539                 blocking: false
 540                 description: >-
 541                     This test case deploys an OpenSource vIMS solution from
 542                     Clearwater using the Cloudify orchestrator. It also runs
 543                     some signaling traffic.
 544                 dependencies:
 545                     - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha'
 546                 run:
 547                     name: cloudify_ims
 548
 549             -
 550                 case_name: heat_ims
 551                 project_name: functest
 552                 criteria: 100
 553                 blocking: false
 554                 description: >-
 555                     This test case deploys an OpenSource vIMS solution from
 556                     Clearwater using the OpenStack Heat orchestrator.
 557                     It also runs some signaling traffic.
 558                 run:
 559                     name: heat_ims
 560
 561             -
 562                 case_name: vyos_vrouter
 563                 project_name: functest
 564                 criteria: 100
 565                 blocking: false
 566                 description: >-
 567                     This test case is vRouter testing.
 568                 run:
 569                     name: vyos_vrouter
 570
 571             -
 572                 case_name: juju_epc
 573                 project_name: functest
 574                 criteria: 100
 575                 blocking: false
 576                 description: >-
 577                     vEPC validation with Juju as VNF manager and ABoT as test
 578                     executor.
 579                 run:
 580                     name: juju_epc