c15bde7982fdf4f0db55cf79d8410f4edee965ff
[functest.git] / functest / ci / testcases.yaml
1 ---
2 tiers:
3     -
4         name: healthcheck
5         order: 0
6         description: >-
7             First tier to be executed to verify the basic
8             operations in the VIM.
9         testcases:
10             -
11                 case_name: connection_check
12                 project_name: functest
13                 criteria: 100
14                 blocking: true
15                 description: >-
16                     This test case verifies the retrieval of OpenStack clients:
17                     Keystone, Glance, Neutron and Nova and may perform some
18                     simple queries.
19                 run:
20                     name: connection_check
21
22             -
23                 case_name: tenantnetwork1
24                 project_name: functest
25                 criteria: 100
26                 blocking: true
27                 description: >-
28                     It creates and configures all tenant network ressources
29                     required by advanced testcases (subnet, network and
30                     router).
31                 run:
32                     name: tenantnetwork1
33
34             -
35                 case_name: tenantnetwork2
36                 project_name: functest
37                 criteria: 100
38                 blocking: true
39                 description: >-
40                     It creates new user/project before creating and configuring
41                     all tenant network ressources required by a testcase
42                     (subnet, network and router).
43                 run:
44                     name: tenantnetwork2
45
46             -
47                 case_name: vmready1
48                 project_name: functest
49                 criteria: 100
50                 blocking: true
51                 description: >-
52                     It inherits from TenantNetwork1 which creates all network
53                     resources and prepares a future VM attached to that
54                     network.
55                 run:
56                     name: vmready1
57
58             -
59                 case_name: vmready2
60                 project_name: functest
61                 criteria: 100
62                 blocking: true
63                 description: >-
64                     It creates new user/project before creating and configuring
65                     all tenant network ressources, flavors, images, etc.
66                     required by advanced testcases.
67                 run:
68                     name: vmready2
69
70             -
71                 case_name: singlevm1
72                 project_name: functest
73                 criteria: 100
74                 blocking: true
75                 description: >-
76                     It inherits from TenantNetwork1 which creates all network
77                     resources and completes it by booting a VM attached to that
78                     network.
79                 run:
80                     name: singlevm1
81
82             -
83                 case_name: singlevm2
84                 project_name: functest
85                 criteria: 100
86                 blocking: true
87                 description: >-
88                     It creates new user/project before creating and configuring
89                     all tenant network ressources and vms required by advanced
90                     testcases.
91                 run:
92                     name: singlevm2
93
94             -
95                 case_name: vping_ssh
96                 project_name: functest
97                 criteria: 100
98                 blocking: true
99                 description: >-
100                     This test case verifies: 1) SSH to an instance using
101                     floating IPs over the public network. 2) Connectivity
102                     between 2 instances over a private network.
103                 run:
104                     name: vping_ssh
105
106             -
107                 case_name: vping_userdata
108                 project_name: functest
109                 criteria: 100
110                 blocking: true
111                 description: >-
112                     This test case verifies:  1) Boot a VM with given userdata.
113                     2) Connectivity between 2 instances over a private network.
114                 run:
115                     name: vping_userdata
116
117             -
118                 case_name: cinder_test
119                 project_name: functest
120                 criteria: 100
121                 blocking: true
122                 description: >-
123                     This test case verifies: 1) Attach volume and to 1
124                     instance; 2) Write data on volume 3) Detach volume
125                     from instance 1, attach it on instance 2 3) Read volume
126                     data
127                 run:
128                     name: cinder_test
129
130             -
131                 case_name: odl
132                 project_name: functest
133                 criteria: 100
134                 blocking: true
135                 description: >-
136                     Test Suite for the OpenDaylight SDN Controller. It
137                     integrates some test suites from upstream using
138                     Robot as the test framework.
139                 dependencies:
140                     - DEPLOY_SCENARIO: 'odl'
141                 run:
142                     name: odl
143                     args:
144                         suites:
145                             - /src/odl_test/csit/suites/integration/basic
146                             - /src/odl_test/csit/suites/openstack/neutron
147
148             -
149                 case_name: api_check
150                 project_name: functest
151                 criteria: 100
152                 blocking: true
153                 description: >-
154                     This test case verifies the retrieval of OpenStack clients:
155                     Keystone, Glance, Neutron and Nova and may perform some
156                     simple queries. When the config value of
157                     snaps.use_keystone is True, functest must have access to
158                     the cloud's private network.
159                 run:
160                     name: api_check
161
162             -
163                 case_name: snaps_health_check
164                 project_name: functest
165                 criteria: 100
166                 blocking: true
167                 description: >-
168                     This test case creates executes the SimpleHealthCheck
169                     Python test class which creates an, image, flavor, network,
170                     and Cirros VM instance and observes the console output to
171                     validate the single port obtains the correct IP address.
172                 run:
173                     name: snaps_health_check
174
175     -
176         name: smoke
177         order: 1
178         description: >-
179             Set of basic Functional tests to validate the OPNFV scenarios.
180         testcases:
181             -
182                 case_name: tempest_smoke
183                 project_name: functest
184                 criteria: 100
185                 blocking: false
186                 description: >-
187                     This test case runs the smoke subset of the OpenStack
188                     Tempest suite. The list of test cases is generated by
189                     Tempest automatically and depends on the parameters of
190                     the OpenStack deplopyment.
191                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L114
192                 run:
193                     name: tempest_common
194                     args:
195                         mode: '(?=.*\[.*\bsmoke\b.*\])(^tempest\.api)'
196                         option:
197                             - '--concurrency=4'
198
199             -
200                 case_name: neutron-tempest-plugin-api
201                 project_name: functest
202                 criteria: 100
203                 blocking: false
204                 description: >-
205                     This test case runs the Tempest suite proposed by the
206                     Neutron project. The list of test cases is generated by
207                     Tempest automatically and depends on the parameters of
208                     the OpenStack deployment.
209                 run:
210                     name: tempest_common
211                     args:
212                         mode: '^neutron_tempest_plugin\.api'
213                         option:
214                             - '--concurrency=4'
215
216             -
217                 case_name: rally_sanity
218                 project_name: functest
219                 criteria: 100
220                 blocking: false
221                 description: >-
222                     This test case runs a sub group of tests of the OpenStack
223                     Rally suite in smoke mode.
224                 run:
225                     name: rally_sanity
226
227             -
228                 case_name: refstack_defcore
229                 project_name: functest
230                 criteria: 100
231                 blocking: false
232                 description: >-
233                     This test case runs a sub group of tests of the OpenStack
234                     Defcore testcases.
235                 run:
236                     name: refstack_defcore
237                     args:
238                         option:
239                             - '--concurrency=4'
240
241             -
242                 case_name: patrole
243                 project_name: functest
244                 criteria: 100
245                 blocking: false
246                 description: >-
247                     Test suite from Patrole project.
248                 run:
249                     name: patrole
250                     args:
251                         role: 'admin'
252                         includes:
253                             - 'image'
254                             - 'network'
255                         excludes:
256                             - 'test_networks_multiprovider_rbac'
257
258             -
259                 case_name: snaps_smoke
260                 project_name: functest
261                 criteria: 100
262                 blocking: false
263                 description: >-
264                     This test case contains tests that setup and destroy
265                     environments with VMs with and without Floating IPs
266                     with a newly created user and project. Set the config
267                     value snaps.use_floating_ips (True|False) to toggle
268                     this functionality. When the config value of
269                     snaps.use_keystone is True, functest must have access to
270                     the cloud's private network.
271                 run:
272                     name: snaps_smoke
273
274             -
275                 case_name: neutron_trunk
276                 project_name: functest
277                 criteria: 100
278                 blocking: false
279                 description: >-
280                     This test case runs the neutron trunk subtest of the
281                     OpenStack Tempest suite. The list of test cases is
282                     generated by Tempest having as input the relevant
283                     testcase list file.
284                 run:
285                     name: tempest_common
286                     args:
287                         mode: 'neutron_tempest_plugin.(api|scenario).test_trunk'
288                         neutron_extensions:
289                             - trunk
290                             - trunk-details
291
292             -
293                 case_name: networking-bgpvpn
294                 project_name: functest
295                 criteria: 100
296                 blocking: false
297                 description: >-
298                     This test case runs OpenStack networking-bgpvpn Tempest
299                     suite. The list of test cases is generated by Tempest
300                     having as input the relevant regex.
301                 run:
302                     name: tempest_common
303                     args:
304                         mode: '^networking_bgpvpn_tempest\.'
305                         neutron_extensions:
306                             - bgpvpn
307                         option:
308                             - '--concurrency=4'
309
310             -
311                 case_name: networking-sfc
312                 project_name: functest
313                 criteria: 100
314                 blocking: false
315                 description: >-
316                     This test case runs OpenStack networking-sfc Tempest
317                     suite. The list of test cases is generated by Tempest
318                     having as input the relevant regex.
319                 run:
320                     name: tempest_common
321                     args:
322                         mode:
323                             '^(?:networking_sfc\.tests\.tempest_plugin.tests.api).*$'
324                         neutron_extensions:
325                             - sfc
326                         option:
327                             - '--concurrency=0'
328
329             -
330                 case_name: barbican
331                 project_name: functest
332                 criteria: 100
333                 blocking: false
334                 description: >-
335                     It leverages on the tempest plugin containing tests used to
336                     verify the functionality of a barbican installation.
337                 run:
338                     name: barbican
339                     args:
340                         mode:
341                             '^barbican_tempest_plugin.((?!test_signed_image_upload_boot_failure).)*$'
342                         services:
343                             - barbican
344
345     -
346         name: benchmarking
347         order: 2
348         description: >-
349             Run several OpenStack performance tools
350             https://docs.openstack.org/performance-docs/latest/methodologies/tools.html
351         testcases:
352             -
353                 case_name: vmtp
354                 project_name: functest
355                 criteria: 100
356                 blocking: false
357                 description: >-
358                     VMTP is a small python application that will automatically
359                     perform ping connectivity, round trip time measurement
360                     (latency) and TCP/UDP throughput
361                 dependencies:
362                     - POD_ARCH: '^(?!aarch64$)'
363                 run:
364                     name: vmtp
365
366             -
367                 case_name: shaker
368                 project_name: functest
369                 criteria: 100
370                 blocking: false
371                 description: >-
372                     Shaker wraps around popular system network testing tools
373                     like iperf, iperf3 and netperf (with help of flent). Shaker
374                     is able to deploy OpenStack instances and networks in
375                     different topologies.
376                 run:
377                     name: shaker
378
379     -
380         name: features
381         order: 3
382         description: >-
383             Test suites from feature projects
384             integrated in functest
385         testcases:
386             -
387                 case_name: doctor-notification
388                 project_name: doctor
389                 enabled: false
390                 criteria: 100
391                 blocking: false
392                 description: >-
393                     Test suite from Doctor project.
394                 dependencies:
395                     - INSTALLER_TYPE: '(apex)|(fuel)|(daisy)'
396                     - DEPLOY_SCENARIO: '^((?!fdio|nofeature).)*$'
397                 run:
398                     name: bashfeature
399                     args:
400                         cmd: 'doctor-test'
401
402             -
403                 case_name: bgpvpn
404                 project_name: sdnvpn
405                 enabled: false
406                 criteria: 100
407                 blocking: false
408                 description: >-
409                     Test suite from SDNVPN project.
410                 dependencies:
411                     - DEPLOY_SCENARIO: 'bgpvpn'
412                 run:
413                     name: bgpvpn
414
415             -
416                 case_name: functest-odl-sfc
417                 project_name: sfc
418                 enabled: false
419                 criteria: 100
420                 blocking: false
421                 description: >-
422                     Test suite for odl-sfc to test two chains with one SF and
423                     one chain with two SFs
424                 dependencies:
425                     - DEPLOY_SCENARIO: 'odl.*sfc'
426                 run:
427                     name: functest-odl-sfc
428
429             -
430                 case_name: barometercollectd
431                 project_name: barometer
432                 enabled: false
433                 criteria: 100
434                 blocking: false
435                 description: >-
436                     Test suite for the Barometer project. Separate tests verify
437                     the proper configuration and basic functionality of all the
438                     collectd plugins as described in the Project Release Plan
439                 dependencies:
440                     - DEPLOY_SCENARIO: 'bar'
441                 run:
442                     name: barometercollectd
443
444             -
445                 case_name: fds
446                 project_name: fastdatastacks
447                 enabled: false
448                 criteria: 100
449                 blocking: false
450                 description: >-
451                     Test Suite for the OpenDaylight SDN Controller when GBP
452                     features are installed. It integrates some test suites from
453                     upstream using Robot as the test framework.
454                 dependencies:
455                     - DEPLOY_SCENARIO: 'odl.*-fdio'
456                 run:
457                     name: odl
458                     args:
459                         suites:
460                             - /src/fds/testing/robot
461             -
462                 case_name: vgpu
463                 project_name: functest
464                 criteria: 100
465                 blocking: false
466                 description: >-
467                     Test suite for the OpenStack vGPU feature
468                 dependencies:
469                     - DEPLOY_SCENARIO: 'vgpu'
470                 run:
471                     name: vgpu
472
473             -
474                 case_name: stor4nfv_os
475                 project_name: stor4nfv
476                 enabled: false
477                 criteria: 100
478                 blocking: false
479                 description: >-
480                     This tests are necessary demonstrate conformance
481                     of the OpenStack+Stor4NFV deployment.
482                 dependencies:
483                     - DEPLOY_SCENARIO: 'stor4nfv'
484                 run:
485                     name: stor4nfv_os
486
487     -
488         name: components
489         order: 4
490         description: >-
491             Extensive testing of OpenStack API.
492         testcases:
493             -
494                 case_name: tempest_full
495                 project_name: functest
496                 criteria: 100
497                 blocking: false
498                 description: >-
499                     The list of test cases is generated by
500                     Tempest automatically and depends on the parameters of
501                     the OpenStack deployment.
502                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83
503                 run:
504                     name: tempest_common
505                     args:
506                         mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
507                         option:
508                             - '--concurrency=4'
509
510             -
511                 case_name: tempest_scenario
512                 project_name: functest
513                 criteria: 100
514                 blocking: false
515                 description: >-
516                     The list of test cases is generated by
517                     Tempest automatically and depends on the parameters of
518                     the OpenStack deployment.
519                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
520                 run:
521                     name: tempest_common
522                     args:
523                         mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)'
524                         option:
525                             - '--concurrency=1'
526
527             -
528                 case_name: rally_full
529                 project_name: functest
530                 criteria: 100
531                 blocking: false
532                 description: >-
533                     This test case runs the full suite of scenarios of the
534                     OpenStack Rally suite using several threads and iterations.
535                 run:
536                     name: rally_full
537
538     -
539         name: vnf
540         order: 5
541         description: >-
542             Collection of VNF test cases.
543         testcases:
544             -
545                 case_name: cloudify
546                 project_name: functest
547                 criteria: 100
548                 blocking: false
549                 description: >-
550                     This test case deploys the Cloudify orchestrator.
551                 dependencies:
552                     - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha'
553                 run:
554                     name: cloudify
555             -
556                 case_name: cloudify_ims
557                 project_name: functest
558                 criteria: 100
559                 blocking: false
560                 description: >-
561                     This test case deploys an OpenSource vIMS solution from
562                     Clearwater using the Cloudify orchestrator. It also runs
563                     some signaling traffic.
564                 dependencies:
565                     - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha'
566                 run:
567                     name: cloudify_ims
568
569             -
570                 case_name: heat_ims
571                 project_name: functest
572                 criteria: 100
573                 blocking: false
574                 description: >-
575                     This test case deploys an OpenSource vIMS solution from
576                     Clearwater using the OpenStack Heat orchestrator.
577                     It also runs some signaling traffic.
578                 dependencies:
579                     - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha'
580                 run:
581                     name: heat_ims
582
583             -
584                 case_name: vyos_vrouter
585                 project_name: functest
586                 criteria: 100
587                 blocking: false
588                 description: >-
589                     This test case is vRouter testing.
590                 dependencies:
591                     - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha'
592                 run:
593                     name: vyos_vrouter
594
595             -
596                 case_name: juju_epc
597                 project_name: functest
598                 criteria: 100
599                 blocking: false
600                 description: >-
601                     vEPC validation with Juju as VNF manager and ABoT as test
602                     executor.
603                 dependencies:
604                     - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha'
605                 run:
606                     name: juju_epc