functest/ci/testcases.yaml

   1 ---
   2 tiers:
   3     -
   4         name: healthcheck
   5         order: 0
   6         description: >-
   7             First tier to be executed to verify the basic
   8             operations in the VIM.
   9         testcases:
  10             -
  11                 case_name: connection_check
  12                 project_name: functest
  13                 criteria: 100
  14                 blocking: true
  15                 description: >-
  16                     This test case verifies the retrieval of OpenStack clients:
  17                     Keystone, Glance, Neutron and Nova and may perform some
  18                     simple queries.
  19                 run:
  20                     name: connection_check
  21
  22             -
  23                 case_name: tenantnetwork1
  24                 project_name: functest
  25                 criteria: 100
  26                 blocking: true
  27                 description: >-
  28                     It creates and configures all tenant network ressources
  29                     required by advanced testcases (subnet, network and
  30                     router).
  31                 run:
  32                     name: tenantnetwork1
  33
  34             -
  35                 case_name: tenantnetwork2
  36                 project_name: functest
  37                 criteria: 100
  38                 blocking: true
  39                 description: >-
  40                     It creates new user/project before creating and configuring
  41                     all tenant network ressources required by a testcase
  42                     (subnet, network and router).
  43                 run:
  44                     name: tenantnetwork2
  45
  46             -
  47                 case_name: vmready1
  48                 project_name: functest
  49                 criteria: 100
  50                 blocking: true
  51                 description: >-
  52                     It inherits from TenantNetwork1 which creates all network
  53                     resources and prepares a future VM attached to that
  54                     network.
  55                 run:
  56                     name: vmready1
  57
  58             -
  59                 case_name: vmready2
  60                 project_name: functest
  61                 criteria: 100
  62                 blocking: true
  63                 description: >-
  64                     It creates new user/project before creating and configuring
  65                     all tenant network ressources, flavors, images, etc.
  66                     required by advanced testcases.
  67                 run:
  68                     name: vmready2
  69
  70             -
  71                 case_name: singlevm1
  72                 project_name: functest
  73                 criteria: 100
  74                 blocking: true
  75                 description: >-
  76                     It inherits from TenantNetwork1 which creates all network
  77                     resources and completes it by booting a VM attached to that
  78                     network.
  79                 run:
  80                     name: singlevm1
  81
  82             -
  83                 case_name: singlevm2
  84                 project_name: functest
  85                 criteria: 100
  86                 blocking: true
  87                 description: >-
  88                     It creates new user/project before creating and configuring
  89                     all tenant network ressources and vms required by advanced
  90                     testcases.
  91                 run:
  92                     name: singlevm2
  93
  94             -
  95                 case_name: vping_ssh
  96                 project_name: functest
  97                 criteria: 100
  98                 blocking: true
  99                 description: >-
 100                     This test case verifies: 1) SSH to an instance using
 101                     floating IPs over the public network. 2) Connectivity
 102                     between 2 instances over a private network.
 103                 run:
 104                     name: vping_ssh
 105
 106             -
 107                 case_name: vping_userdata
 108                 project_name: functest
 109                 criteria: 100
 110                 blocking: true
 111                 description: >-
 112                     This test case verifies:  1) Boot a VM with given userdata.
 113                     2) Connectivity between 2 instances over a private network.
 114                 run:
 115                     name: vping_userdata
 116
 117             -
 118                 case_name: cinder_test
 119                 project_name: functest
 120                 criteria: 100
 121                 blocking: true
 122                 description: >-
 123                     This test case verifies: 1) Attach volume and to 1
 124                     instance; 2) Write data on volume 3) Detach volume
 125                     from instance 1, attach it on instance 2 3) Read volume
 126                     data
 127                 run:
 128                     name: cinder_test
 129
 130             -
 131                 case_name: odl
 132                 project_name: functest
 133                 criteria: 100
 134                 blocking: true
 135                 description: >-
 136                     Test Suite for the OpenDaylight SDN Controller. It
 137                     integrates some test suites from upstream using
 138                     Robot as the test framework.
 139                 dependencies:
 140                     - DEPLOY_SCENARIO: 'odl'
 141                 run:
 142                     name: odl
 143                     args:
 144                         suites:
 145                             - /src/odl_test/csit/suites/integration/basic
 146                             - /src/odl_test/csit/suites/openstack/neutron
 147
 148             -
 149                 case_name: tempest_smoke
 150                 project_name: functest
 151                 criteria: 100
 152                 blocking: false
 153                 description: >-
 154                     This test case runs the smoke subset of the OpenStack
 155                     Tempest suite. The list of test cases is generated by
 156                     Tempest automatically and depends on the parameters of
 157                     the OpenStack deplopyment.
 158                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L114
 159                 run:
 160                     name: tempest_common
 161                     args:
 162                         mode: '(?=.*\[.*\bsmoke\b.*\])(^tempest\.api)'
 163                         option:
 164                             - '--concurrency=4'
 165
 166     -
 167         name: smoke
 168         order: 1
 169         description: >-
 170             Set of basic Functional tests to validate the OPNFV scenarios.
 171         testcases:
 172             -
 173                 case_name: neutron-tempest-plugin-api
 174                 project_name: functest
 175                 criteria: 100
 176                 blocking: false
 177                 description: >-
 178                     This test case runs the Tempest suite proposed by the
 179                     Neutron project. The list of test cases is generated by
 180                     Tempest automatically and depends on the parameters of
 181                     the OpenStack deployment.
 182                 run:
 183                     name: tempest_common
 184                     args:
 185                         mode: '^neutron_tempest_plugin\.api'
 186                         option:
 187                             - '--concurrency=4'
 188
 189             -
 190                 case_name: rally_sanity
 191                 project_name: functest
 192                 criteria: 100
 193                 blocking: false
 194                 description: >-
 195                     This test case runs a sub group of tests of the OpenStack
 196                     Rally suite in smoke mode.
 197                 run:
 198                     name: rally_sanity
 199                     args:
 200                         optional:
 201                             - 'gnocchi'
 202                             - 'barbican'
 203
 204             -
 205                 case_name: refstack_compute
 206                 project_name: functest
 207                 criteria: 100
 208                 blocking: false
 209                 deny_skipping: true
 210                 description: >-
 211                     This test case runs a sub group of tests of the OpenStack
 212                     Compute testcases.
 213                 run:
 214                     name: refstack
 215                     args:
 216                         target: compute
 217                         option:
 218                             - '--concurrency=4'
 219
 220             -
 221                 case_name: refstack_object
 222                 project_name: functest
 223                 criteria: 100
 224                 blocking: false
 225                 deny_skipping: true
 226                 description: >-
 227                     This test case runs a sub group of tests of the OpenStack
 228                     Object testcases.
 229                 run:
 230                     name: refstack
 231                     args:
 232                         target: object
 233                         option:
 234                             - '--concurrency=4'
 235
 236             -
 237                 case_name: refstack_platform
 238                 project_name: functest
 239                 criteria: 100
 240                 blocking: false
 241                 deny_skipping: true
 242                 description: >-
 243                     This test case runs a sub group of tests of the OpenStack
 244                     Platform testcases.
 245                 run:
 246                     name: refstack
 247                     args:
 248                         target: platform
 249                         option:
 250                             - '--concurrency=4'
 251
 252             -
 253                 case_name: tempest_full
 254                 project_name: functest
 255                 criteria: 100
 256                 blocking: false
 257                 description: >-
 258                     The list of test cases is generated by
 259                     Tempest automatically and depends on the parameters of
 260                     the OpenStack deployment.
 261                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83
 262                 run:
 263                     name: tempest_common
 264                     args:
 265                         mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
 266                         option:
 267                             - '--concurrency=4'
 268
 269             -
 270                 case_name: tempest_scenario
 271                 project_name: functest
 272                 criteria: 100
 273                 blocking: false
 274                 description: >-
 275                     The list of test cases is generated by
 276                     Tempest automatically and depends on the parameters of
 277                     the OpenStack deployment.
 278                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
 279                 run:
 280                     name: tempest_common
 281                     args:
 282                         mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)'
 283                         option:
 284                             - '--concurrency=1'
 285
 286             -
 287                 case_name: tempest_slow
 288                 project_name: functest
 289                 criteria: 100
 290                 blocking: false
 291                 description: >-
 292                     The list of test cases is generated by
 293                     Tempest automatically and depends on the parameters of
 294                     the OpenStack deployment.
 295                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
 296                 run:
 297                     name: tempest_common
 298                     args:
 299                         mode: '(?=.*\[.*\bslow\b.*\])(^tempest\.)'
 300                         option:
 301                             - '--concurrency=2'
 302
 303             -
 304                 case_name: patrole
 305                 project_name: functest
 306                 criteria: 100
 307                 blocking: false
 308                 description: >-
 309                     Test suite from Patrole project.
 310                 run:
 311                     name: patrole
 312                     args:
 313                         role: 'admin'
 314                         includes:
 315                             - 'image'
 316                             - 'network'
 317                         excludes:
 318                             - 'network.test_flavors_rbac'
 319                             - 'network.test_network_ip_availability_rbac'
 320                             - 'test_networks_multiprovider_rbac'
 321                             - 'test_auto_allocated_topology_rbac'
 322                             - 'test_create_network_provider_segmentation_id'
 323                             - 'test_update_network_provider_physical_network'
 324                             - 'test_update_network_provider_segmentation_id'
 325                             - 'network.test_agents_rbac'
 326                             - 'test_list_metadef_namespaces'
 327
 328             -
 329                 case_name: neutron_trunk
 330                 project_name: functest
 331                 criteria: 100
 332                 blocking: false
 333                 description: >-
 334                     This test case runs the neutron trunk subtest of the
 335                     OpenStack Tempest suite. The list of test cases is
 336                     generated by Tempest having as input the relevant
 337                     testcase list file.
 338                 run:
 339                     name: tempest_common
 340                     args:
 341                         mode: 'neutron_tempest_plugin.(api|scenario).test_trunk'
 342                         neutron_extensions:
 343                             - trunk
 344                             - trunk-details
 345
 346             -
 347                 case_name: networking-bgpvpn
 348                 project_name: functest
 349                 criteria: 100
 350                 blocking: false
 351                 description: >-
 352                     This test case runs OpenStack networking-bgpvpn Tempest
 353                     suite. The list of test cases is generated by Tempest
 354                     having as input the relevant regex.
 355                 run:
 356                     name: tempest_common
 357                     args:
 358                         mode: '^neutron_tempest_plugin.bgpvpn.api\.'
 359                         neutron_extensions:
 360                             - bgpvpn
 361                         option:
 362                             - '--concurrency=4'
 363
 364             -
 365                 case_name: networking-sfc
 366                 project_name: functest
 367                 criteria: 100
 368                 blocking: false
 369                 description: >-
 370                     This test case runs OpenStack networking-sfc Tempest
 371                     suite. The list of test cases is generated by Tempest
 372                     having as input the relevant regex.
 373                 run:
 374                     name: tempest_common
 375                     args:
 376                         mode:
 377                             '^(?:neutron_tempest_plugin\.sfc\.tests\.api).*$'
 378                         neutron_extensions:
 379                             - sfc
 380                         option:
 381                             - '--concurrency=0'
 382
 383             -
 384                 case_name: barbican
 385                 project_name: functest
 386                 criteria: 100
 387                 blocking: false
 388                 description: >-
 389                     It leverages on the tempest plugin containing tests used to
 390                     verify the functionality of a barbican installation.
 391                 run:
 392                     name: barbican
 393                     args:
 394                         mode:
 395                             '^barbican_tempest_plugin.((?!test_signed_image_upload_boot_failure).)*$'
 396                         services:
 397                             - barbican
 398
 399     -
 400         name: benchmarking
 401         order: 2
 402         description: >-
 403             Run several OpenStack performance tools
 404             https://docs.openstack.org/performance-docs/latest/methodologies/tools.html
 405         testcases:
 406             -
 407                 case_name: rally_full
 408                 project_name: functest
 409                 criteria: 100
 410                 blocking: false
 411                 description: >-
 412                     This test case runs the full suite of scenarios of the
 413                     OpenStack Rally suite using several threads and iterations.
 414                 run:
 415                     name: rally_full
 416                     args:
 417                         optional:
 418                             - 'gnocchi'
 419                             - 'barbican'
 420
 421             -
 422                 case_name: rally_jobs
 423                 project_name: functest
 424                 criteria: 100
 425                 blocking: false
 426                 description: >-
 427                     This test case runs a group of Rally jobs used in
 428                     OpenStack gating
 429                 run:
 430                     name: rally_jobs
 431                     args:
 432                         optional:
 433                             - 'gnocchi'
 434
 435             -
 436                 case_name: vmtp
 437                 project_name: functest
 438                 criteria: 100
 439                 blocking: false
 440                 description: >-
 441                     VMTP is a small python application that will automatically
 442                     perform ping connectivity, round trip time measurement
 443                     (latency) and TCP/UDP throughput
 444                 dependencies:
 445                     - POD_ARCH: '^(?!aarch64$)'
 446                 run:
 447                     name: vmtp
 448
 449             -
 450                 case_name: shaker
 451                 project_name: functest
 452                 criteria: 100
 453                 blocking: false
 454                 description: >-
 455                     Shaker wraps around popular system network testing tools
 456                     like iperf, iperf3 and netperf (with help of flent). Shaker
 457                     is able to deploy OpenStack instances and networks in
 458                     different topologies.
 459                 run:
 460                     name: shaker
 461
 462     -
 463         name: features
 464         order: 3
 465         description: >-
 466             Test suites from feature projects
 467             integrated in functest
 468         testcases:
 469             -
 470                 case_name: doctor-notification
 471                 project_name: doctor
 472                 enabled: false
 473                 criteria: 100
 474                 blocking: false
 475                 description: >-
 476                     Test suite from Doctor project.
 477                 dependencies:
 478                     - INSTALLER_TYPE: '(apex)|(fuel)|(daisy)'
 479                     - DEPLOY_SCENARIO: '^((?!fdio|nofeature).)*$'
 480                 run:
 481                     name: bashfeature
 482                     args:
 483                         cmd: 'doctor-test'
 484
 485             -
 486                 case_name: functest-odl-sfc
 487                 project_name: sfc
 488                 criteria: 100
 489                 blocking: false
 490                 description: >-
 491                     Test suite for odl-sfc to test two chains with one SF and
 492                     one chain with two SFs
 493                 dependencies:
 494                     - DEPLOY_SCENARIO: 'odl.*sfc'
 495                 run:
 496                     name: functest-odl-sfc
 497
 498             -
 499                 case_name: barometercollectd
 500                 project_name: barometer
 501                 enabled: false
 502                 criteria: 100
 503                 blocking: false
 504                 description: >-
 505                     Test suite for the Barometer project. Separate tests verify
 506                     the proper configuration and basic functionality of all the
 507                     collectd plugins as described in the Project Release Plan
 508                 dependencies:
 509                     - DEPLOY_SCENARIO: 'bar'
 510                 run:
 511                     name: barometercollectd
 512
 513             -
 514                 case_name: vgpu
 515                 project_name: functest
 516                 criteria: 100
 517                 blocking: false
 518                 description: >-
 519                     Test suite for the OpenStack vGPU feature
 520                 dependencies:
 521                     - DEPLOY_SCENARIO: 'vgpu'
 522                 run:
 523                     name: vgpu
 524
 525             -
 526                 case_name: stor4nfv_os
 527                 project_name: stor4nfv
 528                 enabled: false
 529                 criteria: 100
 530                 blocking: false
 531                 description: >-
 532                     This tests are necessary demonstrate conformance
 533                     of the OpenStack+Stor4NFV deployment.
 534                 dependencies:
 535                     - DEPLOY_SCENARIO: 'stor4nfv'
 536                 run:
 537                     name: stor4nfv_os
 538
 539     -
 540         name: vnf
 541         order: 4
 542         description: >-
 543             Collection of VNF test cases.
 544         testcases:
 545             -
 546                 case_name: cloudify
 547                 project_name: functest
 548                 criteria: 100
 549                 blocking: false
 550                 description: >-
 551                     This test case deploys the Cloudify orchestrator.
 552                 run:
 553                     name: cloudify
 554             -
 555                 case_name: cloudify_ims
 556                 project_name: functest
 557                 criteria: 100
 558                 blocking: false
 559                 description: >-
 560                     This test case deploys an OpenSource vIMS solution from
 561                     Clearwater using the Cloudify orchestrator. It also runs
 562                     some signaling traffic.
 563                 dependencies:
 564                     - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha'
 565                 run:
 566                     name: cloudify_ims
 567
 568             -
 569                 case_name: heat_ims
 570                 project_name: functest
 571                 criteria: 100
 572                 blocking: false
 573                 description: >-
 574                     This test case deploys an OpenSource vIMS solution from
 575                     Clearwater using the OpenStack Heat orchestrator.
 576                     It also runs some signaling traffic.
 577                 run:
 578                     name: heat_ims
 579
 580             -
 581                 case_name: vyos_vrouter
 582                 project_name: functest
 583                 criteria: 100
 584                 blocking: false
 585                 description: >-
 586                     This test case is vRouter testing.
 587                 run:
 588                     name: vyos_vrouter
 589
 590             -
 591                 case_name: juju_epc
 592                 project_name: functest
 593                 criteria: 100
 594                 blocking: false
 595                 description: >-
 596                     vEPC validation with Juju as VNF manager and ABoT as test
 597                     executor.
 598                 run:
 599                     name: juju_epc