functest/ci/testcases.yaml

   1 ---
   2 tiers:
   3     -
   4         name: healthcheck
   5         order: 0
   6         description: >-
   7             First tier to be executed to verify the basic
   8             operations in the VIM.
   9         testcases:
  10             -
  11                 case_name: connection_check
  12                 project_name: functest
  13                 criteria: 100
  14                 blocking: true
  15                 description: >-
  16                     This test case verifies the retrieval of OpenStack clients:
  17                     Keystone, Glance, Neutron and Nova and may perform some
  18                     simple queries.
  19                 run:
  20                     name: connection_check
  21
  22             -
  23                 case_name: tenantnetwork1
  24                 project_name: functest
  25                 criteria: 100
  26                 blocking: true
  27                 description: >-
  28                     It creates and configures all tenant network ressources
  29                     required by advanced testcases (subnet, network and
  30                     router).
  31                 run:
  32                     name: tenantnetwork1
  33
  34             -
  35                 case_name: tenantnetwork2
  36                 project_name: functest
  37                 criteria: 100
  38                 blocking: true
  39                 description: >-
  40                     It creates new user/project before creating and configuring
  41                     all tenant network ressources required by a testcase
  42                     (subnet, network and router).
  43                 run:
  44                     name: tenantnetwork2
  45
  46             -
  47                 case_name: vmready1
  48                 project_name: functest
  49                 criteria: 100
  50                 blocking: true
  51                 description: >-
  52                     It inherits from TenantNetwork1 which creates all network
  53                     resources and prepares a future VM attached to that
  54                     network.
  55                 run:
  56                     name: vmready1
  57
  58             -
  59                 case_name: vmready2
  60                 project_name: functest
  61                 criteria: 100
  62                 blocking: true
  63                 description: >-
  64                     It creates new user/project before creating and configuring
  65                     all tenant network ressources, flavors, images, etc.
  66                     required by advanced testcases.
  67                 run:
  68                     name: vmready2
  69
  70             -
  71                 case_name: singlevm1
  72                 project_name: functest
  73                 criteria: 100
  74                 blocking: true
  75                 description: >-
  76                     It inherits from TenantNetwork1 which creates all network
  77                     resources and completes it by booting a VM attached to that
  78                     network.
  79                 run:
  80                     name: singlevm1
  81
  82             -
  83                 case_name: singlevm2
  84                 project_name: functest
  85                 criteria: 100
  86                 blocking: true
  87                 description: >-
  88                     It creates new user/project before creating and configuring
  89                     all tenant network ressources and vms required by advanced
  90                     testcases.
  91                 run:
  92                     name: singlevm2
  93
  94             -
  95                 case_name: vping_ssh
  96                 project_name: functest
  97                 criteria: 100
  98                 blocking: true
  99                 description: >-
 100                     This test case verifies: 1) SSH to an instance using
 101                     floating IPs over the public network. 2) Connectivity
 102                     between 2 instances over a private network.
 103                 run:
 104                     name: vping_ssh
 105
 106             -
 107                 case_name: vping_userdata
 108                 project_name: functest
 109                 criteria: 100
 110                 blocking: true
 111                 description: >-
 112                     This test case verifies:  1) Boot a VM with given userdata.
 113                     2) Connectivity between 2 instances over a private network.
 114                 run:
 115                     name: vping_userdata
 116
 117             -
 118                 case_name: cinder_test
 119                 project_name: functest
 120                 criteria: 100
 121                 blocking: true
 122                 description: >-
 123                     This test case verifies: 1) Attach volume and to 1
 124                     instance; 2) Write data on volume 3) Detach volume
 125                     from instance 1, attach it on instance 2 3) Read volume
 126                     data
 127                 run:
 128                     name: cinder_test
 129
 130             -
 131                 case_name: odl
 132                 project_name: functest
 133                 criteria: 100
 134                 blocking: true
 135                 description: >-
 136                     Test Suite for the OpenDaylight SDN Controller. It
 137                     integrates some test suites from upstream using
 138                     Robot as the test framework.
 139                 dependencies:
 140                     - DEPLOY_SCENARIO: 'odl'
 141                 run:
 142                     name: odl
 143                     args:
 144                         suites:
 145                             - /src/odl_test/csit/suites/integration/basic
 146                             - /src/odl_test/csit/suites/openstack/neutron
 147
 148             -
 149                 case_name: tempest_smoke
 150                 project_name: functest
 151                 criteria: 100
 152                 blocking: false
 153                 description: >-
 154                     This test case runs the smoke subset of the OpenStack
 155                     Tempest suite. The list of test cases is generated by
 156                     Tempest automatically and depends on the parameters of
 157                     the OpenStack deplopyment.
 158                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L114
 159                 run:
 160                     name: tempest_common
 161                     args:
 162                         mode: '(?=.*\[.*\bsmoke\b.*\])(^tempest\.api)'
 163                         option:
 164                             - '--concurrency=4'
 165
 166             -
 167                 case_name: tempest_horizon
 168                 project_name: functest
 169                 criteria: 100
 170                 blocking: false
 171                 description: >-
 172                     This test case runs the Tempest suite proposed by the
 173                     Horizon project.
 174                 dependencies:
 175                     - DASHBOARD_URL: '^(?!\s*$).+'
 176                 run:
 177                     name: tempest_horizon
 178                     args:
 179                         mode: '^tempest_horizon.'
 180
 181     -
 182         name: smoke
 183         order: 1
 184         description: >-
 185             Set of basic Functional tests to validate the OPNFV scenarios.
 186         testcases:
 187             -
 188                 case_name: neutron-tempest-plugin-api
 189                 project_name: functest
 190                 criteria: 100
 191                 blocking: false
 192                 description: >-
 193                     This test case runs the Tempest suite proposed by the
 194                     Neutron project. The list of test cases is generated by
 195                     Tempest automatically and depends on the parameters of
 196                     the OpenStack deployment.
 197                 run:
 198                     name: tempest_common
 199                     args:
 200                         mode: '^neutron_tempest_plugin\.api'
 201                         option:
 202                             - '--concurrency=4'
 203
 204             -
 205                 case_name: rally_sanity
 206                 project_name: functest
 207                 criteria: 100
 208                 blocking: false
 209                 description: >-
 210                     This test case runs a sub group of tests of the OpenStack
 211                     Rally suite in smoke mode.
 212                 run:
 213                     name: rally_sanity
 214                     args:
 215                         optional:
 216                             - 'gnocchi'
 217                             - 'barbican'
 218
 219             -
 220                 case_name: refstack_compute
 221                 project_name: functest
 222                 criteria: 100
 223                 blocking: false
 224                 deny_skipping: true
 225                 tests_count: 222
 226                 description: >-
 227                     This test case runs a sub group of tests of the OpenStack
 228                     Compute testcases.
 229                 run:
 230                     name: refstack
 231                     args:
 232                         target: compute
 233                         option:
 234                             - '--concurrency=4'
 235
 236             -
 237                 case_name: refstack_object
 238                 project_name: functest
 239                 criteria: 100
 240                 blocking: false
 241                 deny_skipping: true
 242                 tests_count: 58
 243                 description: >-
 244                     This test case runs a sub group of tests of the OpenStack
 245                     Object testcases.
 246                 run:
 247                     name: refstack
 248                     args:
 249                         target: object
 250                         option:
 251                             - '--concurrency=4'
 252
 253             -
 254                 case_name: refstack_platform
 255                 project_name: functest
 256                 criteria: 100
 257                 blocking: false
 258                 deny_skipping: true
 259                 tests_count: 278
 260                 description: >-
 261                     This test case runs a sub group of tests of the OpenStack
 262                     Platform testcases.
 263                 run:
 264                     name: refstack
 265                     args:
 266                         target: platform
 267                         option:
 268                             - '--concurrency=4'
 269
 270             -
 271                 case_name: tempest_full
 272                 project_name: functest
 273                 criteria: 100
 274                 blocking: false
 275                 description: >-
 276                     The list of test cases is generated by
 277                     Tempest automatically and depends on the parameters of
 278                     the OpenStack deployment.
 279                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83
 280                 run:
 281                     name: tempest_common
 282                     args:
 283                         mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
 284                         option:
 285                             - '--concurrency=4'
 286
 287             -
 288                 case_name: tempest_scenario
 289                 project_name: functest
 290                 criteria: 100
 291                 blocking: false
 292                 description: >-
 293                     The list of test cases is generated by
 294                     Tempest automatically and depends on the parameters of
 295                     the OpenStack deployment.
 296                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
 297                 run:
 298                     name: tempest_scenario
 299                     args:
 300                         mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)'
 301                         option:
 302                             - '--concurrency=1'
 303
 304             -
 305                 case_name: tempest_slow
 306                 project_name: functest
 307                 criteria: 100
 308                 blocking: false
 309                 description: >-
 310                     The list of test cases is generated by
 311                     Tempest automatically and depends on the parameters of
 312                     the OpenStack deployment.
 313                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
 314                 run:
 315                     name: tempest_common
 316                     args:
 317                         mode: '(?=.*\[.*\bslow\b.*\])(^tempest\.)'
 318                         option:
 319                             - '--concurrency=2'
 320
 321             -
 322                 case_name: patrole
 323                 project_name: functest
 324                 criteria: 100
 325                 blocking: false
 326                 description: >-
 327                     Test suite from Patrole project.
 328                 run:
 329                     name: patrole
 330                     args:
 331                         role: 'admin'
 332                         includes:
 333                             - 'image'
 334                             - 'network'
 335                         excludes:
 336                             - 'network.test_flavors_rbac'
 337                             - 'network.test_network_ip_availability_rbac'
 338                             - 'test_networks_multiprovider_rbac'
 339                             - 'test_auto_allocated_topology_rbac'
 340                             - 'test_create_network_provider_segmentation_id'
 341                             - 'test_update_network_provider_physical_network'
 342                             - 'test_update_network_provider_segmentation_id'
 343                             - 'network.test_agents_rbac'
 344                             - 'test_list_metadef_namespaces'
 345
 346             -
 347                 case_name: neutron_trunk
 348                 project_name: functest
 349                 criteria: 100
 350                 blocking: false
 351                 description: >-
 352                     This test case runs the neutron trunk subtest of the
 353                     OpenStack Tempest suite. The list of test cases is
 354                     generated by Tempest having as input the relevant
 355                     testcase list file.
 356                 run:
 357                     name: tempest_common
 358                     args:
 359                         mode: 'neutron_tempest_plugin.(api|scenario).test_trunk'
 360                         neutron_extensions:
 361                             - trunk
 362                             - trunk-details
 363
 364             -
 365                 case_name: networking-bgpvpn
 366                 project_name: functest
 367                 criteria: 100
 368                 blocking: false
 369                 description: >-
 370                     This test case runs OpenStack networking-bgpvpn Tempest
 371                     suite. The list of test cases is generated by Tempest
 372                     having as input the relevant regex.
 373                 run:
 374                     name: tempest_common
 375                     args:
 376                         mode: '^neutron_tempest_plugin.bgpvpn.api\.'
 377                         neutron_extensions:
 378                             - bgpvpn
 379                         option:
 380                             - '--concurrency=4'
 381
 382             -
 383                 case_name: networking-sfc
 384                 project_name: functest
 385                 criteria: 100
 386                 blocking: false
 387                 description: >-
 388                     This test case runs OpenStack networking-sfc Tempest
 389                     suite. The list of test cases is generated by Tempest
 390                     having as input the relevant regex.
 391                 run:
 392                     name: tempest_common
 393                     args:
 394                         mode:
 395                             '^(?:neutron_tempest_plugin\.sfc\.tests\.api).*$'
 396                         neutron_extensions:
 397                             - sfc
 398                         option:
 399                             - '--concurrency=0'
 400
 401             -
 402                 case_name: barbican
 403                 project_name: functest
 404                 criteria: 100
 405                 blocking: false
 406                 description: >-
 407                     It leverages on the tempest plugin containing tests used to
 408                     verify the functionality of a barbican installation.
 409                 run:
 410                     name: barbican
 411                     args:
 412                         mode:
 413                             '^barbican_tempest_plugin.((?!test_signed_image_upload_boot_failure).)*$'
 414                         services:
 415                             - barbican
 416
 417             -
 418                 case_name: octavia
 419                 project_name: functest
 420                 criteria: 100
 421                 blocking: false
 422                 description: >-
 423                     It leverages on the tempest plugin containing tests used to
 424                     verify the functionality of an octavia installation.
 425                 run:
 426                     name: tempest_common
 427                     args:
 428                         mode:
 429                             '^octavia_tempest_plugin.tests.api.v2'
 430                         services:
 431                             - octavia
 432                         option:
 433                             - '--concurrency=2'
 434
 435     -
 436         name: benchmarking
 437         order: 2
 438         description: >-
 439             Run several OpenStack performance tools
 440             https://docs.openstack.org/performance-docs/latest/methodologies/tools.html
 441         testcases:
 442             -
 443                 case_name: rally_full
 444                 project_name: functest
 445                 criteria: 100
 446                 blocking: false
 447                 description: >-
 448                     This test case runs the full suite of scenarios of the
 449                     OpenStack Rally suite using several threads and iterations.
 450                 run:
 451                     name: rally_full
 452                     args:
 453                         optional:
 454                             - 'gnocchi'
 455                             - 'barbican'
 456
 457             -
 458                 case_name: rally_jobs
 459                 project_name: functest
 460                 criteria: 100
 461                 blocking: false
 462                 description: >-
 463                     This test case runs a group of Rally jobs used in
 464                     OpenStack gating
 465                 run:
 466                     name: rally_jobs
 467                     args:
 468                         optional:
 469                             - 'gnocchi'
 470
 471             -
 472                 case_name: vmtp
 473                 project_name: functest
 474                 criteria: 100
 475                 blocking: false
 476                 description: >-
 477                     VMTP is a small python application that will automatically
 478                     perform ping connectivity, round trip time measurement
 479                     (latency) and TCP/UDP throughput
 480                 dependencies:
 481                     - POD_ARCH: '^(?!aarch64$)'
 482                 run:
 483                     name: vmtp
 484
 485             -
 486                 case_name: shaker
 487                 project_name: functest
 488                 criteria: 100
 489                 blocking: false
 490                 description: >-
 491                     Shaker wraps around popular system network testing tools
 492                     like iperf, iperf3 and netperf (with help of flent). Shaker
 493                     is able to deploy OpenStack instances and networks in
 494                     different topologies.
 495                 dependencies:
 496                     - POD_ARCH: '^(?!aarch64$)'
 497                 run:
 498                     name: shaker
 499
 500     -
 501         name: vnf
 502         order: 4
 503         description: >-
 504             Collection of VNF test cases.
 505         testcases:
 506             -
 507                 case_name: cloudify
 508                 project_name: functest
 509                 criteria: 100
 510                 blocking: false
 511                 description: >-
 512                     This test case deploys the Cloudify orchestrator.
 513                 run:
 514                     name: cloudify
 515             -
 516                 case_name: cloudify_ims
 517                 project_name: functest
 518                 criteria: 100
 519                 blocking: false
 520                 description: >-
 521                     This test case deploys an OpenSource vIMS solution from
 522                     Clearwater using the Cloudify orchestrator. It also runs
 523                     some signaling traffic.
 524                 dependencies:
 525                     - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha'
 526                 run:
 527                     name: cloudify_ims
 528
 529             -
 530                 case_name: heat_ims
 531                 project_name: functest
 532                 criteria: 100
 533                 blocking: false
 534                 description: >-
 535                     This test case deploys an OpenSource vIMS solution from
 536                     Clearwater using the OpenStack Heat orchestrator.
 537                     It also runs some signaling traffic.
 538                 run:
 539                     name: heat_ims
 540
 541             -
 542                 case_name: vyos_vrouter
 543                 project_name: functest
 544                 criteria: 100
 545                 blocking: false
 546                 description: >-
 547                     This test case is vRouter testing.
 548                 run:
 549                     name: vyos_vrouter
 550
 551             -
 552                 case_name: juju_epc
 553                 project_name: functest
 554                 criteria: 100
 555                 blocking: false
 556                 description: >-
 557                     vEPC validation with Juju as VNF manager and ABoT as test
 558                     executor.
 559                 run:
 560                     name: juju_epc