ede9350eaee3fcc3c26963f1d208979c9ce3f3f7
[functest.git] / functest / ci / testcases.yaml
1 ---
2 tiers:
3     -
4         name: healthcheck
5         order: 0
6         description: >-
7             First tier to be executed to verify the basic
8             operations in the VIM.
9         testcases:
10             -
11                 case_name: connection_check
12                 project_name: functest
13                 criteria: 100
14                 blocking: true
15                 description: >-
16                     This test case verifies the retrieval of OpenStack clients:
17                     Keystone, Glance, Neutron and Nova and may perform some
18                     simple queries.
19                 run:
20                     name: connection_check
21
22             -
23                 case_name: tenantnetwork1
24                 project_name: functest
25                 criteria: 100
26                 blocking: true
27                 description: >-
28                     It creates and configures all tenant network ressources
29                     required by advanced testcases (subnet, network and
30                     router).
31                 run:
32                     name: tenantnetwork1
33
34             -
35                 case_name: tenantnetwork2
36                 project_name: functest
37                 criteria: 100
38                 blocking: true
39                 description: >-
40                     It creates new user/project before creating and configuring
41                     all tenant network ressources required by a testcase
42                     (subnet, network and router).
43                 run:
44                     name: tenantnetwork2
45
46             -
47                 case_name: vmready1
48                 project_name: functest
49                 criteria: 100
50                 blocking: true
51                 description: >-
52                     It inherits from TenantNetwork1 which creates all network
53                     resources and prepares a future VM attached to that
54                     network.
55                 run:
56                     name: vmready1
57
58             -
59                 case_name: vmready2
60                 project_name: functest
61                 criteria: 100
62                 blocking: true
63                 description: >-
64                     It creates new user/project before creating and configuring
65                     all tenant network ressources, flavors, images, etc.
66                     required by advanced testcases.
67                 run:
68                     name: vmready2
69
70             -
71                 case_name: singlevm1
72                 project_name: functest
73                 criteria: 100
74                 blocking: true
75                 description: >-
76                     It inherits from TenantNetwork1 which creates all network
77                     resources and completes it by booting a VM attached to that
78                     network.
79                 run:
80                     name: singlevm1
81
82             -
83                 case_name: singlevm2
84                 project_name: functest
85                 criteria: 100
86                 blocking: true
87                 description: >-
88                     It creates new user/project before creating and configuring
89                     all tenant network ressources and vms required by advanced
90                     testcases.
91                 run:
92                     name: singlevm2
93
94             -
95                 case_name: vping_ssh
96                 project_name: functest
97                 criteria: 100
98                 blocking: true
99                 description: >-
100                     This test case verifies: 1) SSH to an instance using
101                     floating IPs over the public network. 2) Connectivity
102                     between 2 instances over a private network.
103                 run:
104                     name: vping_ssh
105
106             -
107                 case_name: vping_userdata
108                 project_name: functest
109                 criteria: 100
110                 blocking: true
111                 description: >-
112                     This test case verifies:  1) Boot a VM with given userdata.
113                     2) Connectivity between 2 instances over a private network.
114                 run:
115                     name: vping_userdata
116
117             -
118                 case_name: cinder_test
119                 project_name: functest
120                 criteria: 100
121                 blocking: true
122                 description: >-
123                     This test case verifies: 1) Attach volume and to 1
124                     instance; 2) Write data on volume 3) Detach volume
125                     from instance 1, attach it on instance 2 3) Read volume
126                     data
127                 run:
128                     name: cinder_test
129
130             -
131                 case_name: odl
132                 project_name: functest
133                 criteria: 100
134                 blocking: true
135                 description: >-
136                     Test Suite for the OpenDaylight SDN Controller. It
137                     integrates some test suites from upstream using
138                     Robot as the test framework.
139                 dependencies:
140                     - DEPLOY_SCENARIO: 'odl'
141                 run:
142                     name: odl
143                     args:
144                         suites:
145                             - /src/odl_test/csit/suites/integration/basic
146                             - /src/odl_test/csit/suites/openstack/neutron
147
148             -
149                 case_name: tempest_smoke
150                 project_name: functest
151                 criteria: 100
152                 blocking: false
153                 description: >-
154                     This test case runs the smoke subset of the OpenStack
155                     Tempest suite. The list of test cases is generated by
156                     Tempest automatically and depends on the parameters of
157                     the OpenStack deplopyment.
158                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L114
159                 run:
160                     name: tempest_common
161                     args:
162                         mode: '(?=.*\[.*\bsmoke\b.*\])(^tempest\.api)'
163                         option:
164                             - '--concurrency=4'
165
166     -
167         name: smoke
168         order: 1
169         description: >-
170             Set of basic Functional tests to validate the OPNFV scenarios.
171         testcases:
172             -
173                 case_name: neutron-tempest-plugin-api
174                 project_name: functest
175                 criteria: 100
176                 blocking: false
177                 description: >-
178                     This test case runs the Tempest suite proposed by the
179                     Neutron project. The list of test cases is generated by
180                     Tempest automatically and depends on the parameters of
181                     the OpenStack deployment.
182                 run:
183                     name: tempest_common
184                     args:
185                         mode: '^neutron_tempest_plugin\.api'
186                         option:
187                             - '--concurrency=4'
188
189             -
190                 case_name: rally_sanity
191                 project_name: functest
192                 criteria: 100
193                 blocking: false
194                 description: >-
195                     This test case runs a sub group of tests of the OpenStack
196                     Rally suite in smoke mode.
197                 run:
198                     name: rally_sanity
199                     args:
200                         optional:
201                             - 'gnocchi'
202
203             -
204                 case_name: refstack_defcore
205                 project_name: functest
206                 criteria: 100
207                 blocking: false
208                 deny_skipping: true
209                 description: >-
210                     This test case runs a sub group of tests of the OpenStack
211                     Defcore testcases.
212                 run:
213                     name: refstack_defcore
214                     args:
215                         option:
216                             - '--concurrency=4'
217
218             -
219                 case_name: tempest_full
220                 project_name: functest
221                 criteria: 100
222                 blocking: false
223                 description: >-
224                     The list of test cases is generated by
225                     Tempest automatically and depends on the parameters of
226                     the OpenStack deployment.
227                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83
228                 run:
229                     name: tempest_common
230                     args:
231                         mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
232                         option:
233                             - '--concurrency=4'
234
235             -
236                 case_name: tempest_scenario
237                 project_name: functest
238                 criteria: 100
239                 blocking: false
240                 description: >-
241                     The list of test cases is generated by
242                     Tempest automatically and depends on the parameters of
243                     the OpenStack deployment.
244                     https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
245                 run:
246                     name: tempest_common
247                     args:
248                         mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)'
249                         option:
250                             - '--concurrency=1'
251
252             -
253                 case_name: patrole
254                 project_name: functest
255                 criteria: 100
256                 blocking: false
257                 description: >-
258                     Test suite from Patrole project.
259                 run:
260                     name: patrole
261                     args:
262                         role: 'admin'
263                         includes:
264                             - 'image'
265                             - 'network'
266                         excludes:
267                             - 'network.test_flavors_rbac'
268                             - 'network.test_network_ip_availability_rbac'
269                             - 'test_networks_multiprovider_rbac'
270                             - 'test_auto_allocated_topology_rbac'
271
272             -
273                 case_name: neutron_trunk
274                 project_name: functest
275                 criteria: 100
276                 blocking: false
277                 description: >-
278                     This test case runs the neutron trunk subtest of the
279                     OpenStack Tempest suite. The list of test cases is
280                     generated by Tempest having as input the relevant
281                     testcase list file.
282                 run:
283                     name: tempest_common
284                     args:
285                         mode: 'neutron_tempest_plugin.(api|scenario).test_trunk'
286                         neutron_extensions:
287                             - trunk
288                             - trunk-details
289
290             -
291                 case_name: networking-bgpvpn
292                 project_name: functest
293                 criteria: 100
294                 blocking: false
295                 description: >-
296                     This test case runs OpenStack networking-bgpvpn Tempest
297                     suite. The list of test cases is generated by Tempest
298                     having as input the relevant regex.
299                 run:
300                     name: tempest_common
301                     args:
302                         mode: '^networking_bgpvpn_tempest.tests.api\.'
303                         neutron_extensions:
304                             - bgpvpn
305                         option:
306                             - '--concurrency=4'
307
308             -
309                 case_name: networking-sfc
310                 project_name: functest
311                 criteria: 100
312                 blocking: false
313                 description: >-
314                     This test case runs OpenStack networking-sfc Tempest
315                     suite. The list of test cases is generated by Tempest
316                     having as input the relevant regex.
317                 run:
318                     name: tempest_common
319                     args:
320                         mode:
321                             '^(?:networking_sfc\.tests\.tempest_plugin.tests.api).*$'
322                         neutron_extensions:
323                             - sfc
324                         option:
325                             - '--concurrency=0'
326
327             -
328                 case_name: barbican
329                 project_name: functest
330                 criteria: 100
331                 blocking: false
332                 description: >-
333                     It leverages on the tempest plugin containing tests used to
334                     verify the functionality of a barbican installation.
335                 run:
336                     name: barbican
337                     args:
338                         mode:
339                             '^barbican_tempest_plugin.((?!test_signed_image_upload_boot_failure).)*$'
340                         services:
341                             - barbican
342
343     -
344         name: benchmarking
345         order: 2
346         description: >-
347             Run several OpenStack performance tools
348             https://docs.openstack.org/performance-docs/latest/methodologies/tools.html
349         testcases:
350             -
351                 case_name: rally_full
352                 project_name: functest
353                 criteria: 100
354                 blocking: false
355                 description: >-
356                     This test case runs the full suite of scenarios of the
357                     OpenStack Rally suite using several threads and iterations.
358                 run:
359                     name: rally_full
360                     args:
361                         optional:
362                             - 'gnocchi'
363
364             -
365                 case_name: rally_jobs
366                 project_name: functest
367                 criteria: 100
368                 blocking: false
369                 description: >-
370                     This test case runs a group of Rally jobs used in
371                     OpenStack gating
372                 run:
373                     name: rally_jobs
374                     args:
375                         optional:
376                             - 'gnocchi'
377
378             -
379                 case_name: vmtp
380                 project_name: functest
381                 criteria: 100
382                 blocking: false
383                 description: >-
384                     VMTP is a small python application that will automatically
385                     perform ping connectivity, round trip time measurement
386                     (latency) and TCP/UDP throughput
387                 dependencies:
388                     - POD_ARCH: '^(?!aarch64$)'
389                 run:
390                     name: vmtp
391
392             -
393                 case_name: shaker
394                 project_name: functest
395                 criteria: 100
396                 blocking: false
397                 description: >-
398                     Shaker wraps around popular system network testing tools
399                     like iperf, iperf3 and netperf (with help of flent). Shaker
400                     is able to deploy OpenStack instances and networks in
401                     different topologies.
402                 run:
403                     name: shaker
404
405     -
406         name: features
407         order: 3
408         description: >-
409             Test suites from feature projects
410             integrated in functest
411         testcases:
412             -
413                 case_name: doctor-notification
414                 project_name: doctor
415                 enabled: false
416                 criteria: 100
417                 blocking: false
418                 description: >-
419                     Test suite from Doctor project.
420                 dependencies:
421                     - INSTALLER_TYPE: '(apex)|(fuel)|(daisy)'
422                     - DEPLOY_SCENARIO: '^((?!fdio|nofeature).)*$'
423                 run:
424                     name: bashfeature
425                     args:
426                         cmd: 'doctor-test'
427
428             -
429                 case_name: functest-odl-sfc
430                 project_name: sfc
431                 criteria: 100
432                 blocking: false
433                 description: >-
434                     Test suite for odl-sfc to test two chains with one SF and
435                     one chain with two SFs
436                 dependencies:
437                     - DEPLOY_SCENARIO: 'odl.*sfc'
438                 run:
439                     name: functest-odl-sfc
440
441             -
442                 case_name: barometercollectd
443                 project_name: barometer
444                 enabled: false
445                 criteria: 100
446                 blocking: false
447                 description: >-
448                     Test suite for the Barometer project. Separate tests verify
449                     the proper configuration and basic functionality of all the
450                     collectd plugins as described in the Project Release Plan
451                 dependencies:
452                     - DEPLOY_SCENARIO: 'bar'
453                 run:
454                     name: barometercollectd
455
456             -
457                 case_name: vgpu
458                 project_name: functest
459                 criteria: 100
460                 blocking: false
461                 description: >-
462                     Test suite for the OpenStack vGPU feature
463                 dependencies:
464                     - DEPLOY_SCENARIO: 'vgpu'
465                 run:
466                     name: vgpu
467
468             -
469                 case_name: stor4nfv_os
470                 project_name: stor4nfv
471                 enabled: false
472                 criteria: 100
473                 blocking: false
474                 description: >-
475                     This tests are necessary demonstrate conformance
476                     of the OpenStack+Stor4NFV deployment.
477                 dependencies:
478                     - DEPLOY_SCENARIO: 'stor4nfv'
479                 run:
480                     name: stor4nfv_os
481
482     -
483         name: vnf
484         order: 4
485         description: >-
486             Collection of VNF test cases.
487         testcases:
488             -
489                 case_name: cloudify
490                 project_name: functest
491                 criteria: 100
492                 blocking: false
493                 description: >-
494                     This test case deploys the Cloudify orchestrator.
495                 run:
496                     name: cloudify
497             -
498                 case_name: cloudify_ims
499                 project_name: functest
500                 criteria: 100
501                 blocking: false
502                 description: >-
503                     This test case deploys an OpenSource vIMS solution from
504                     Clearwater using the Cloudify orchestrator. It also runs
505                     some signaling traffic.
506                 dependencies:
507                     - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha'
508                 run:
509                     name: cloudify_ims
510
511             -
512                 case_name: heat_ims
513                 project_name: functest
514                 criteria: 100
515                 blocking: false
516                 description: >-
517                     This test case deploys an OpenSource vIMS solution from
518                     Clearwater using the OpenStack Heat orchestrator.
519                     It also runs some signaling traffic.
520                 run:
521                     name: heat_ims
522
523             -
524                 case_name: vyos_vrouter
525                 project_name: functest
526                 criteria: 100
527                 blocking: false
528                 description: >-
529                     This test case is vRouter testing.
530                 run:
531                     name: vyos_vrouter
532
533             -
534                 case_name: juju_epc
535                 project_name: functest
536                 criteria: 100
537                 blocking: false
538                 description: >-
539                     vEPC validation with Juju as VNF manager and ABoT as test
540                     executor.
541                 run:
542                     name: juju_epc