functest/ci/testcases.yaml

   1 ---
   2 tiers:
   3     -
   4         name: healthcheck
   5         order: 0
   6         description: >-
   7             First tier to be executed to verify the basic
   8             operations in the VIM.
   9         testcases:
  10             -
  11                 case_name: connection_check
  12                 project_name: functest
  13                 criteria: 100
  14                 blocking: true
  15                 description: >-
  16                     This test case verifies the retrieval of OpenStack clients:
  17                     Keystone, Glance, Neutron and Nova and may perform some
  18                     simple queries.
  19                 run:
  20                     name: connection_check
  21
  22             -
  23                 case_name: tenantnetwork1
  24                 project_name: functest
  25                 criteria: 100
  26                 blocking: true
  27                 description: >-
  28                     It creates and configures all tenant network ressources
  29                     required by advanced testcases (subnet, network and
  30                     router).
  31                 run:
  32                     name: tenantnetwork1
  33
  34             -
  35                 case_name: tenantnetwork2
  36                 project_name: functest
  37                 criteria: 100
  38                 blocking: true
  39                 description: >-
  40                     It creates new user/project before creating and configuring
  41                     all tenant network ressources required by a testcase
  42                     (subnet, network and router).
  43                 run:
  44                     name: tenantnetwork2
  45
  46             -
  47                 case_name: vmready1
  48                 project_name: functest
  49                 criteria: 100
  50                 blocking: true
  51                 description: >-
  52                     It inherits from TenantNetwork1 which creates all network
  53                     resources and prepares a future VM attached to that
  54                     network.
  55                 run:
  56                     name: vmready1
  57
  58             -
  59                 case_name: vmready2
  60                 project_name: functest
  61                 criteria: 100
  62                 blocking: true
  63                 description: >-
  64                     It creates new user/project before creating and configuring
  65                     all tenant network ressources, flavors, images, etc.
  66                     required by advanced testcases.
  67                 run:
  68                     name: vmready2
  69
  70             -
  71                 case_name: singlevm1
  72                 project_name: functest
  73                 criteria: 100
  74                 blocking: true
  75                 description: >-
  76                     It inherits from TenantNetwork1 which creates all network
  77                     resources and completes it by booting a VM attached to that
  78                     network.
  79                 run:
  80                     name: singlevm1
  81
  82             -
  83                 case_name: singlevm2
  84                 project_name: functest
  85                 criteria: 100
  86                 blocking: true
  87                 description: >-
  88                     It creates new user/project before creating and configuring
  89                     all tenant network ressources and vms required by advanced
  90                     testcases.
  91                 run:
  92                     name: singlevm2
  93
  94             -
  95                 case_name: vping_ssh
  96                 project_name: functest
  97                 criteria: 100
  98                 blocking: true
  99                 description: >-
 100                     This test case verifies: 1) SSH to an instance using
 101                     floating IPs over the public network. 2) Connectivity
 102                     between 2 instances over a private network.
 103                 run:
 104                     name: vping_ssh
 105
 106             -
 107                 case_name: vping_userdata
 108                 project_name: functest
 109                 criteria: 100
 110                 blocking: true
 111                 description: >-
 112                     This test case verifies:  1) Boot a VM with given userdata.
 113                     2) Connectivity between 2 instances over a private network.
 114                 run:
 115                     name: vping_userdata
 116
 117             -
 118                 case_name: cinder_test
 119                 project_name: functest
 120                 criteria: 100
 121                 blocking: true
 122                 description: >-
 123                     This test case verifies: 1) Attach volume and to 1
 124                     instance; 2) Write data on volume 3) Detach volume
 125                     from instance 1, attach it on instance 2 3) Read volume
 126                     data
 127                 run:
 128                     name: cinder_test
 129
 130             -
 131                 case_name: odl
 132                 project_name: functest
 133                 criteria: 100
 134                 blocking: true
 135                 description: >-
 136                     Test Suite for the OpenDaylight SDN Controller. It
 137                     integrates some test suites from upstream using
 138                     Robot as the test framework.
 139                 dependencies:
 140                     - DEPLOY_SCENARIO: 'odl'
 141                 run:
 142                     name: odl
 143                     args:
 144                         suites:
 145                             - /src/odl_test/csit/suites/integration/basic
 146                             - /src/odl_test/csit/suites/openstack/neutron
 147
 148             -
 149                 case_name: tempest_smoke
 150                 project_name: functest
 151                 criteria: 100
 152                 blocking: false
 153                 description: >-
 154                     This test case runs the smoke subset of the OpenStack
 155                     Tempest suite. The list of test cases is generated by
 156                     Tempest automatically and depends on the parameters of
 157                     the OpenStack deplopyment.
 158                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L114
 159                 run:
 160                     name: tempest_common
 161                     args:
 162                         mode: '(?=.*\[.*\bsmoke\b.*\])(^tempest\.api)'
 163                         option:
 164                             - '--concurrency=4'
 165
 166             -
 167                 case_name: tempest_horizon
 168                 project_name: functest
 169                 criteria: 100
 170                 blocking: false
 171                 description: >-
 172                     This test case runs the Tempest suite proposed by the
 173                     Horizon project.
 174                 dependencies:
 175                     - DASHBOARD_URL: '^(?!\s*$).+'
 176                 run:
 177                     name: tempest_horizon
 178                     args:
 179                         mode: '^tempest_horizon.'
 180
 181     -
 182         name: smoke
 183         order: 1
 184         description: >-
 185             Set of basic Functional tests to validate the OPNFV scenarios.
 186         testcases:
 187             -
 188                 case_name: neutron-tempest-plugin-api
 189                 project_name: functest
 190                 criteria: 100
 191                 blocking: false
 192                 description: >-
 193                     This test case runs the Tempest suite proposed by the
 194                     Neutron project. The list of test cases is generated by
 195                     Tempest automatically and depends on the parameters of
 196                     the OpenStack deployment.
 197                 run:
 198                     name: tempest_common
 199                     args:
 200                         mode: '^neutron_tempest_plugin\.api'
 201                         option:
 202                             - '--concurrency=4'
 203
 204             -
 205                 case_name: tempest_cinder
 206                 project_name: functest
 207                 criteria: 100
 208                 blocking: false
 209                 description: >-
 210                     This test case runs the Tempest suite proposed by the
 211                     Cinder project.
 212                 run:
 213                     name: tempest_common
 214                     args:
 215                         mode:
 216                             '(?!.*test_incremental_backup)(^cinder_tempest_plugin.)'
 217                         option:
 218                             - '--concurrency=4'
 219
 220             -
 221                 case_name: rally_sanity
 222                 project_name: functest
 223                 criteria: 100
 224                 blocking: false
 225                 description: >-
 226                     This test case runs a sub group of tests of the OpenStack
 227                     Rally suite in smoke mode.
 228                 run:
 229                     name: rally_sanity
 230                     args:
 231                         optional:
 232                             - 'gnocchi'
 233                             - 'barbican'
 234
 235             -
 236                 case_name: refstack_compute
 237                 project_name: functest
 238                 criteria: 100
 239                 blocking: false
 240                 deny_skipping: true
 241                 tests_count: 222
 242                 description: >-
 243                     This test case runs a sub group of tests of the OpenStack
 244                     Compute testcases.
 245                 run:
 246                     name: refstack
 247                     args:
 248                         target: compute
 249                         option:
 250                             - '--concurrency=4'
 251
 252             -
 253                 case_name: refstack_object
 254                 project_name: functest
 255                 criteria: 100
 256                 blocking: false
 257                 deny_skipping: true
 258                 tests_count: 58
 259                 description: >-
 260                     This test case runs a sub group of tests of the OpenStack
 261                     Object testcases.
 262                 run:
 263                     name: refstack
 264                     args:
 265                         target: object
 266                         option:
 267                             - '--concurrency=4'
 268
 269             -
 270                 case_name: refstack_platform
 271                 project_name: functest
 272                 criteria: 100
 273                 blocking: false
 274                 deny_skipping: true
 275                 tests_count: 278
 276                 description: >-
 277                     This test case runs a sub group of tests of the OpenStack
 278                     Platform testcases.
 279                 run:
 280                     name: refstack
 281                     args:
 282                         target: platform
 283                         option:
 284                             - '--concurrency=4'
 285
 286             -
 287                 case_name: tempest_full
 288                 project_name: functest
 289                 criteria: 100
 290                 blocking: false
 291                 description: >-
 292                     The list of test cases is generated by
 293                     Tempest automatically and depends on the parameters of
 294                     the OpenStack deployment.
 295                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83
 296                 run:
 297                     name: tempest_common
 298                     args:
 299                         mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
 300                         option:
 301                             - '--concurrency=4'
 302
 303             -
 304                 case_name: tempest_scenario
 305                 project_name: functest
 306                 criteria: 100
 307                 blocking: false
 308                 description: >-
 309                     The list of test cases is generated by
 310                     Tempest automatically and depends on the parameters of
 311                     the OpenStack deployment.
 312                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
 313                 run:
 314                     name: tempest_scenario
 315                     args:
 316                         mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)'
 317                         option:
 318                             - '--concurrency=1'
 319
 320             -
 321                 case_name: tempest_slow
 322                 project_name: functest
 323                 criteria: 100
 324                 blocking: false
 325                 description: >-
 326                     The list of test cases is generated by
 327                     Tempest automatically and depends on the parameters of
 328                     the OpenStack deployment.
 329                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
 330                 run:
 331                     name: tempest_common
 332                     args:
 333                         mode: '(?=.*\[.*\bslow\b.*\])(^tempest\.)'
 334                         option:
 335                             - '--concurrency=2'
 336
 337             -
 338                 case_name: patrole
 339                 project_name: functest
 340                 criteria: 100
 341                 blocking: false
 342                 description: >-
 343                     Test suite from Patrole project.
 344                 run:
 345                     name: patrole
 346                     args:
 347                         role: 'admin'
 348                         includes:
 349                             - 'image'
 350                             - 'network'
 351                         excludes:
 352                             - 'network.test_flavors_rbac'
 353                             - 'network.test_network_ip_availability_rbac'
 354                             - 'test_networks_multiprovider_rbac'
 355                             - 'test_auto_allocated_topology_rbac'
 356                             - 'test_create_network_provider_segmentation_id'
 357                             - 'test_update_network_provider_physical_network'
 358                             - 'test_update_network_provider_segmentation_id'
 359                             - 'network.test_agents_rbac'
 360                             - 'test_list_metadef_namespaces'
 361
 362             -
 363                 case_name: neutron_trunk
 364                 project_name: functest
 365                 criteria: 100
 366                 blocking: false
 367                 description: >-
 368                     This test case runs the neutron trunk subtest of the
 369                     OpenStack Tempest suite. The list of test cases is
 370                     generated by Tempest having as input the relevant
 371                     testcase list file.
 372                 run:
 373                     name: tempest_common
 374                     args:
 375                         mode: 'neutron_tempest_plugin.(api|scenario).test_trunk'
 376                         neutron_extensions:
 377                             - trunk
 378                             - trunk-details
 379
 380             -
 381                 case_name: networking-bgpvpn
 382                 project_name: functest
 383                 criteria: 100
 384                 blocking: false
 385                 description: >-
 386                     This test case runs OpenStack networking-bgpvpn Tempest
 387                     suite. The list of test cases is generated by Tempest
 388                     having as input the relevant regex.
 389                 run:
 390                     name: tempest_common
 391                     args:
 392                         mode: '^neutron_tempest_plugin.bgpvpn.api\.'
 393                         neutron_extensions:
 394                             - bgpvpn
 395                         option:
 396                             - '--concurrency=4'
 397
 398             -
 399                 case_name: networking-sfc
 400                 project_name: functest
 401                 criteria: 100
 402                 blocking: false
 403                 description: >-
 404                     This test case runs OpenStack networking-sfc Tempest
 405                     suite. The list of test cases is generated by Tempest
 406                     having as input the relevant regex.
 407                 run:
 408                     name: tempest_common
 409                     args:
 410                         mode:
 411                             '^(?:neutron_tempest_plugin\.sfc\.tests\.api).*$'
 412                         neutron_extensions:
 413                             - sfc
 414                         option:
 415                             - '--concurrency=0'
 416
 417             -
 418                 case_name: barbican
 419                 project_name: functest
 420                 criteria: 100
 421                 blocking: false
 422                 description: >-
 423                     It leverages on the tempest plugin containing tests used to
 424                     verify the functionality of a barbican installation.
 425                 run:
 426                     name: barbican
 427                     args:
 428                         mode:
 429                             '^barbican_tempest_plugin.((?!test_signed_image_upload_boot_failure).)*$'
 430                         services:
 431                             - barbican
 432
 433             -
 434                 case_name: octavia
 435                 project_name: functest
 436                 criteria: 100
 437                 blocking: false
 438                 description: >-
 439                     It leverages on the tempest plugin containing tests used to
 440                     verify the functionality of an octavia installation.
 441                 run:
 442                     name: tempest_common
 443                     args:
 444                         mode:
 445                             '^octavia_tempest_plugin.tests.api.v2'
 446                         services:
 447                             - octavia
 448                         option:
 449                             - '--concurrency=2'
 450
 451     -
 452         name: benchmarking
 453         order: 2
 454         description: >-
 455             Run several OpenStack performance tools
 456             https://docs.openstack.org/performance-docs/latest/methodologies/tools.html
 457         testcases:
 458             -
 459                 case_name: rally_full
 460                 project_name: functest
 461                 criteria: 100
 462                 blocking: false
 463                 description: >-
 464                     This test case runs the full suite of scenarios of the
 465                     OpenStack Rally suite using several threads and iterations.
 466                 run:
 467                     name: rally_full
 468                     args:
 469                         optional:
 470                             - 'gnocchi'
 471                             - 'barbican'
 472
 473             -
 474                 case_name: rally_jobs
 475                 project_name: functest
 476                 criteria: 100
 477                 blocking: false
 478                 description: >-
 479                     This test case runs a group of Rally jobs used in
 480                     OpenStack gating
 481                 run:
 482                     name: rally_jobs
 483                     args:
 484                         optional:
 485                             - 'gnocchi'
 486
 487             -
 488                 case_name: vmtp
 489                 project_name: functest
 490                 criteria: 100
 491                 blocking: false
 492                 description: >-
 493                     VMTP is a small python application that will automatically
 494                     perform ping connectivity, round trip time measurement
 495                     (latency) and TCP/UDP throughput
 496                 dependencies:
 497                     - POD_ARCH: '^(?!aarch64$)'
 498                 run:
 499                     name: vmtp
 500
 501             -
 502                 case_name: shaker
 503                 project_name: functest
 504                 criteria: 100
 505                 blocking: false
 506                 description: >-
 507                     Shaker wraps around popular system network testing tools
 508                     like iperf, iperf3 and netperf (with help of flent). Shaker
 509                     is able to deploy OpenStack instances and networks in
 510                     different topologies.
 511                 dependencies:
 512                     - POD_ARCH: '^(?!aarch64$)'
 513                 run:
 514                     name: shaker
 515
 516     -
 517         name: vnf
 518         order: 4
 519         description: >-
 520             Collection of VNF test cases.
 521         testcases:
 522             -
 523                 case_name: cloudify
 524                 project_name: functest
 525                 criteria: 100
 526                 blocking: false
 527                 description: >-
 528                     This test case deploys the Cloudify orchestrator.
 529                 run:
 530                     name: cloudify
 531             -
 532                 case_name: cloudify_ims
 533                 project_name: functest
 534                 criteria: 100
 535                 blocking: false
 536                 description: >-
 537                     This test case deploys an OpenSource vIMS solution from
 538                     Clearwater using the Cloudify orchestrator. It also runs
 539                     some signaling traffic.
 540                 dependencies:
 541                     - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha'
 542                 run:
 543                     name: cloudify_ims
 544
 545             -
 546                 case_name: heat_ims
 547                 project_name: functest
 548                 criteria: 100
 549                 blocking: false
 550                 description: >-
 551                     This test case deploys an OpenSource vIMS solution from
 552                     Clearwater using the OpenStack Heat orchestrator.
 553                     It also runs some signaling traffic.
 554                 run:
 555                     name: heat_ims
 556
 557             -
 558                 case_name: vyos_vrouter
 559                 project_name: functest
 560                 criteria: 100
 561                 blocking: false
 562                 description: >-
 563                     This test case is vRouter testing.
 564                 run:
 565                     name: vyos_vrouter
 566
 567             -
 568                 case_name: juju_epc
 569                 project_name: functest
 570                 criteria: 100
 571                 blocking: false
 572                 description: >-
 573                     vEPC validation with Juju as VNF manager and ABoT as test
 574                     executor.
 575                 run:
 576                     name: juju_epc