JIRA: BOTTLENECKS-29
[bottlenecks.git] / vstf / vstf / controller / database / dbinterface.py
1 ##############################################################################
2 # Copyright (c) 2015 Huawei Technologies Co.,Ltd and others.
3 #
4 # All rights reserved. This program and the accompanying materials
5 # are made available under the terms of the Apache License, Version 2.0
6 # which accompanies this distribution, and is available at
7 # http://www.apache.org/licenses/LICENSE-2.0
8 ##############################################################################
9
10 import os
11 import logging
12
13 from sqlalchemy import create_engine
14 from sqlalchemy.orm import sessionmaker
15 from sqlalchemy import and_
16 import vstf.controller.database.tables as table
17 from vstf.controller.database import constants as const
18
19 LOG = logging.getLogger(__name__)
20
21 """
22 @event.listens_for(Engine, "before_cursor_execute")
23 def before_cursor_execute(conn, cursor, statement,
24     parameters, context, executemany):
25     conn.info.setdefault('query_start_time', []).append(time.time())
26     logging.debug("Start Query: %s", statement)
27 @event.listens_for(Engine, "after_cursor_execute")
28 def after_cursor_execute(conn, cursor, statement,
29     parameters, context, executemany):
30     total = time.time() - conn.info['query_start_time'].pop(-1)
31     logging.debug("Query Complete!")
32     logging.debug("Total Time: %f", total)"""
33
34
35 class DbManage(object):
36     def __init__(self, db_name=const.DBPATH):
37         db_exists = os.path.exists(db_name)
38         try:
39             self._engine = create_engine('sqlite:///%s' % db_name, echo=False)
40             db_session = sessionmaker(bind=self._engine)
41             self._session = db_session()
42         except Exception as e:
43             raise e
44
45         # if the db is new , create all tables and init static tables
46         if not db_exists:
47             self.create_tables()
48             self.init_tables()
49
50     def __delete__(self):
51         self._engine.close_all()
52
53     def create_tables(self):
54         table.Base.metadata.create_all(self._engine)
55         self._session.commit()
56
57     def drop_tables(self):
58         table.Base.metadata.drop_all(self._engine)
59         self._session.commit()
60
61     def init_tables(self):
62         self.init_casetable()
63         self.init_scenario_table()
64         self._session.commit()
65
66     def init_scenario_table(self):
67         items = []
68         for values in const.SCENARIO_INFO_LIST:
69             item = table.TblScenarioInfo(ScenarioName=values[0],
70                                          FigurePath=values[1],
71                                          Description=values[2])
72             items.append(item)
73         self._session.add_all(items)
74
75     # Single TblCaseInfo API
76     def init_casetable(self):
77         items = []
78         for values in const.CASE_INFO_LIST:
79             item = table.TblCaseInfo(CaseTag=values[0],
80                                      ScenarioName=values[1],
81                                      CaseName=values[2],
82                                      FigurePath=values[3],
83                                      Description=values[4],
84                                      Direction=values[5],
85                                      Directiontag=values[6],
86                                      Configure=values[7])
87             items.append(item)
88         self._session.add_all(items)
89
90     def query_caseinfo(self):
91         query = self._session.query(table.TblCaseInfo.ScenarioName,
92                                     table.TblCaseInfo.CaseTag,
93                                     table.TblCaseInfo.CaseName,
94                                     table.TblCaseInfo.Direction,
95                                     table.TblCaseInfo.Configure)
96         return query.all()
97
98     def query_case(self, casetag):
99         query = self._session.query(table.TblCaseInfo.ScenarioName,
100                                     table.TblCaseInfo.Directiontag)
101         return query.first()
102
103     # Single TblTaskList API
104     def get_last_taskid(self):
105         query = self._session.query(table.TblTaskList.TaskID)
106         if query:
107             return query.all()[-1][0]
108         else:
109             return 0
110
111     def query_tasklist(self):
112         query = self._session.query(table.TblTaskList)
113         return query.all()
114
115     def query_taskdate(self, taskid):
116         query = self._session.query(table.TblTaskList.Date).filter(and_(
117             table.TblTaskList.TaskID == taskid))
118         result = ""
119         if query:
120             result += query.first()[0]
121         return result
122
123     def query_taskname(self, taskid):
124         query = self._session.query(table.TblTaskList.TaskName).filter(and_(
125             table.TblTaskList.TaskID == taskid))
126         result = ""
127         if query:
128             result += query.first()[0]
129         return result
130
131     def create_task(self, name, date, desc):
132         try:
133             item = table.TblTaskList(name, date, desc)
134             self._session.add(item)
135             self._session.commit()
136         except Exception:
137             return -1
138
139         return self.get_last_taskid()
140
141     # Single TblHostInfo API
142     def add_host_2task(self, taskid, name, machine, cpu, men, nic, os):
143         """All var except task must be string"""
144         item = table.TblHostInfo(taskid, name, machine, cpu, men, nic, os)
145
146         self._session.add(item)
147         self._session.commit()
148
149     def query_task_host_list(self, taskid):
150         query = self._session.query(table.TblHostInfo.HostName,
151                                     table.TblHostInfo.Server,
152                                     table.TblHostInfo.CPU,
153                                     table.TblHostInfo.MEM,
154                                     table.TblHostInfo.NIC,
155                                     table.TblHostInfo.OS).filter(
156             table.TblHostInfo.TaskID == taskid)
157         return query.all()
158
159     # Single TblTestList API
160     def get_last_testid(self):
161         query = self._session.query(table.TblTestList.TestID)
162         print query.all()
163         if query:
164             return query.all()[-1][0]
165         else:
166             return 0
167
168     def add_test_2task(self, task, case, protocol, typ, switch, provider, tool):
169         try:
170             item = table.TblTestList(task, case, protocol, typ, switch, provider, tool)
171             self._session.add(item)
172             self._session.commit()
173         except Exception:
174             return -1
175
176         return self.get_last_testid()
177
178     def get_test_type(self, testid):
179         query = self._session.query(table.TblTestList.Type).filter(
180             table.TblTestList.TestID == testid)
181         return query.first()
182
183     def add_extent_2task(self, task, name, content, description):
184         item = table.TblEXTInfo(task, name, content, description)
185         self._session.add(item)
186         self._session.commit()
187
188     def add_data_2test(self, testid, data):
189         """
190         :data example {'64':{
191                             'AvgFrameSize':0
192                             'OfferedLoad':0
193                             'PercentLoss':0
194                             'Bandwidth':0
195                             'MinimumLatency':0
196                             'MaximumLatency':0
197                             'AverageLatency':0
198                             'TxFrameCount':0
199                             'RxFrameCount':0
200                             'Duration':0
201                             'CPU':0
202                             'MppspGhz':0
203                             }}
204         """
205         ptype = self.get_test_type(testid)
206         instance_map = {
207             'throughput': table.TblThroughput,
208             'frameloss': table.TblFrameloss,
209             'latency': table.TblLatency
210         }
211
212         if ptype and ptype[0] not in instance_map:
213             print "cant find this test(id=%d)" % (testid)
214             return False
215
216         test_table_instance = instance_map[ptype[0]]
217         for pktlen in data.iterkeys():
218             args = data.get(pktlen)
219             query = self._session.query(test_table_instance).filter(and_(
220                 test_table_instance.TestID == testid,
221                 test_table_instance.AvgFrameSize == pktlen))
222             if query.all():
223                 data_dict = {}
224                 for key, value in data.items():
225                     if key in test_table_instance.__dict__:
226                         data_dict[test_table_instance.__dict__[key]] = value
227                 query.update(data_dict)
228             else:
229                 print args
230                 tester = test_table_instance(testid, pktlen, **args)
231                 self._session.add(tester)
232         self._session.commit()
233
234     def query_tasks(self):
235         result = []
236         ret = self._session.query(table.TblTaskList)
237         if ret:
238             for tmp in ret.all():
239                 result.append([tmp.TaskID, tmp.TaskName, tmp.Date, tmp.EXTInfo])
240         return result
241
242     def query_all_task_id(self):
243         query = self._session.query(table.TblTaskList.TaskID)
244         if query:
245             return query.all()
246         else:
247             return []
248
249     def get_caseinfo(self):
250         query = self._session.query(table.TblCaseInfo.ScenarioName,
251                                     table.TblCaseInfo.CaseTag,
252                                     table.TblCaseInfo.CaseName,
253                                     table.TblCaseInfo.Direction,
254                                     table.TblCaseInfo.Configure)
255         return query.all()
256
257     def query_scenario(self, casetag):
258         query = self._session.query(table.TblCaseInfo.ScenarioName).filter(table.TblCaseInfo.CaseTag == casetag)
259         ret = ""
260         if query and query.first():
261             ret = query.first()[0]
262         return ret
263
264     def query_casefigure(self, casetag, tools):
265         query = self._session.query(table.TblCaseInfo.FigurePath).filter(and_(
266             table.TblCaseInfo.CaseTag == casetag))
267         result = ""
268         if query:
269             result += query.first()[0]
270         print tools, casetag
271         result += tools + '/' + casetag + '.jpg'
272         return result
273
274     def query_casename(self, casetag):
275         query = self._session.query(table.TblCaseInfo.CaseName).filter(and_(
276             table.TblCaseInfo.CaseTag == casetag))
277         result = ""
278         if query:
279             result += query.first()[0]
280         return result
281
282     # Single TblTestList API
283
284     def query_caselist(self, taskid, scenario):
285         query = self._session.query(table.TblTestList.CaseTag).filter(and_(
286             table.TblTestList.CaseTag == table.TblCaseInfo.CaseTag,
287             table.TblCaseInfo.ScenarioName == scenario,
288             table.TblTestList.TaskID == taskid)).group_by(table.TblCaseInfo.CaseTag)
289         return query.all()
290
291     def query_testlist(self, taskid, scenario):
292         """
293
294         :rtype : object
295         """
296         query = self._session.query(table.TblTestList).filter(and_(
297             table.TblTestList.CaseTag == table.TblCaseInfo.CaseTag,
298             table.TblCaseInfo.ScenarioName == scenario,
299             table.TblTestList.TaskID == taskid))
300         return query.all()
301
302     def query_casetool(self, taskid, casetag, provider, ptype):
303         query = self._session.query(table.TblTestList.Tools).filter(and_(
304             table.TblTestList.TaskID == taskid,
305             table.TblTestList.CaseTag == casetag,
306             table.TblTestList.Provider == provider,
307             table.TblTestList.Type == ptype))
308         return query.all()
309
310     def query_casetools(self, taskid, casetag):
311         query = self._session.query(table.TblTestList.Tools).filter(and_(
312             table.TblTestList.CaseTag == casetag,
313             table.TblTestList.TaskID == taskid)).group_by(table.TblTestList.Tools)
314         return query.all()
315
316     def query_scenariolist(self, taskid):
317         query = self._session.query(table.TblCaseInfo.ScenarioName).filter(and_(
318             table.TblTestList.CaseTag == table.TblCaseInfo.CaseTag,
319             table.TblTestList.TaskID == taskid)).group_by(table.TblCaseInfo.ScenarioName)
320         return query.all()
321
322     def query_throughput_load(self, taskid, casetag, provider):
323         ptype = 'throughput'
324         query = self._session.query(table.TblThroughput.AvgFrameSize, table.TblThroughput.OfferedLoad).filter(and_(
325             table.TblTestList.TaskID == taskid,
326             table.TblTestList.CaseTag == casetag,
327             table.TblTestList.Provider == provider, table.TblTestList.Type == ptype,
328             table.TblTestList.TestID == table.TblThroughput.TestID))
329         return query.all()
330
331     def query_throughput_bandwidth(self, taskid, casetag, provider):
332         ptype = 'throughput'
333         query = self._session.query(table.TblThroughput.AvgFrameSize, table.TblThroughput.Bandwidth).filter(and_(
334             table.TblTestList.TaskID == taskid,
335             table.TblTestList.CaseTag == casetag,
336             table.TblTestList.Provider == provider, table.TblTestList.Type == ptype,
337             table.TblTestList.TestID == table.TblThroughput.TestID))
338         return query.all()
339
340     def query_throughput_table(self, taskid, casetag, provider):
341         ptype = 'throughput'
342         query = self._session.query(table.TblThroughput.AvgFrameSize,
343                                     table.TblThroughput.Bandwidth,
344                                     table.TblThroughput.OfferedLoad,
345                                     table.TblThroughput.CPU,
346                                     table.TblThroughput.MppspGhz,
347                                     table.TblThroughput.MinimumLatency,
348                                     table.TblThroughput.MaximumLatency,
349                                     table.TblThroughput.AverageLatency,
350                                     ).filter(and_(
351             table.TblTestList.TaskID == taskid,
352             table.TblTestList.CaseTag == casetag,
353             table.TblTestList.Provider == provider, table.TblTestList.Type == ptype,
354             table.TblTestList.TestID == table.TblThroughput.TestID))
355         return query.all()
356
357     def query_throughput_simpletable(self, taskid, casetag, provider):
358         ptype = 'throughput'
359         query = self._session.query(table.TblThroughput.AvgFrameSize,
360                                     table.TblThroughput.Bandwidth,
361                                     table.TblThroughput.OfferedLoad,
362                                     table.TblThroughput.CPU,
363                                     table.TblThroughput.MppspGhz,
364                                     table.TblThroughput.AverageLatency,
365                                     ).filter(and_(
366             table.TblTestList.TaskID == taskid,
367             table.TblTestList.CaseTag == casetag,
368             table.TblTestList.Provider == provider, table.TblTestList.Type == ptype,
369             table.TblTestList.TestID == table.TblThroughput.TestID))
370         return query.all()
371
372     def query_testdata(self, testid, ptype):
373         obj = getattr(table, "Tbl%s" % (ptype.title()))
374         query = self._session.query(obj).filter(obj.TestID == testid)
375         return query.all()
376
377     def query_throughput_avg(self, taskid, casetag, provider):
378         ptype = 'throughput'
379         query = self._session.query(table.TblThroughput.AvgFrameSize, table.TblThroughput.AverageLatency).filter(and_(
380             table.TblTestList.TaskID == taskid,
381             table.TblTestList.CaseTag == casetag,
382             table.TblTestList.Provider == provider, table.TblTestList.Type == ptype,
383             table.TblTestList.TestID == table.TblThroughput.TestID))
384         return query.all()
385
386     def query_frameloss_bandwidth(self, taskid, casetag, provider):
387         ptype = 'frameloss'
388         query = self._session.query(table.TblFrameloss.AvgFrameSize, table.TblFrameloss.Bandwidth).filter(and_(
389             table.TblTestList.TaskID == taskid,
390             table.TblTestList.CaseTag == casetag,
391             table.TblTestList.Provider == provider, table.TblTestList.Type == ptype,
392             table.TblTestList.TestID == table.TblFrameloss.TestID))
393         return query.all()
394
395     def query_frameloss_load(self, taskid, casetag, provider):
396         ptype = 'frameloss'
397         query = self._session.query(table.TblFrameloss.AvgFrameSize, table.TblFrameloss.OfferedLoad).filter(and_(
398             table.TblTestList.TaskID == taskid,
399             table.TblTestList.CaseTag == casetag,
400             table.TblTestList.Provider == provider, table.TblTestList.Type == ptype,
401             table.TblTestList.TestID == table.TblFrameloss.TestID))
402         return query.all()
403
404     def query_frameloss_table(self, taskid, casetag, provider):
405         ptype = 'frameloss'
406         query = self._session.query(table.TblFrameloss.AvgFrameSize,
407                                     table.TblFrameloss.Bandwidth,
408                                     table.TblFrameloss.OfferedLoad,
409                                     table.TblFrameloss.CPU,
410                                     table.TblFrameloss.MppspGhz,
411                                     table.TblFrameloss.MinimumLatency,
412                                     table.TblFrameloss.MaximumLatency,
413                                     table.TblFrameloss.AverageLatency
414                                     ).filter(and_(
415             table.TblTestList.TaskID == taskid,
416             table.TblTestList.CaseTag == casetag,
417             table.TblTestList.Provider == provider, table.TblTestList.Type == ptype,
418             table.TblTestList.TestID == table.TblFrameloss.TestID))
419         return query.all()
420
421     def query_frameloss_simpletable(self, taskid, casetag, provider):
422         ptype = 'frameloss'
423         query = self._session.query(table.TblFrameloss.AvgFrameSize,
424                                     table.TblFrameloss.Bandwidth,
425                                     table.TblFrameloss.OfferedLoad,
426                                     table.TblFrameloss.CPU,
427                                     table.TblFrameloss.MppspGhz,
428                                     table.TblFrameloss.AverageLatency
429                                     ).filter(and_(
430             table.TblTestList.TaskID == taskid,
431             table.TblTestList.CaseTag == casetag,
432             table.TblTestList.Provider == provider, table.TblTestList.Type == ptype,
433             table.TblTestList.TestID == table.TblFrameloss.TestID))
434         return query.all()
435
436     def query_frameloss_avg(self, taskid, casetag, provider):
437         ptype = 'frameloss'
438         query = self._session.query(table.TblFrameloss.AvgFrameSize, table.TblFrameloss.AverageLatency).filter(and_(
439             table.TblTestList.TaskID == taskid,
440             table.TblTestList.CaseTag == casetag,
441             table.TblTestList.Provider == provider, table.TblTestList.Type == ptype,
442             table.TblTestList.TestID == table.TblFrameloss.TestID))
443         return query.all()
444
445     def query_latency_avg(self, taskid, casetag, provider):
446         ptype = 'latency'
447         query = self._session.query(table.TblLatency.AvgFrameSize, table.TblLatency.AverageLatency).filter(and_(
448             table.TblTestList.TaskID == taskid,
449             table.TblTestList.CaseTag == casetag,
450             table.TblTestList.Provider == provider, table.TblTestList.Type == ptype,
451             table.TblTestList.TestID == table.TblLatency.TestID))
452         return query.all()
453
454     def query_summary_table(self, taskid, casetag, provider, ptype):
455         if ptype in ['throughput', 'frameloss']:
456             qfunc = getattr(self, "query_%s_table" % (ptype))
457             return qfunc(taskid, casetag, provider)
458         return []
459
460     def query_summary_simpletable(self, taskid, casetag, provider, ptype):
461         if ptype in ['throughput', 'frameloss']:
462             qfunc = getattr(self, "query_%s_simpletable" % (ptype))
463             return qfunc(taskid, casetag, provider)
464         return []
465
466     def query_bandwidth(self, taskid, casetag, provider, ptype):
467         if ptype in ['throughput', 'frameloss']:
468             qfunc = getattr(self, "query_%s_bandwidth" % (ptype))
469             return qfunc(taskid, casetag, provider)
470         return []
471
472     def query_load(self, taskid, casetag, provider, ptype):
473         if ptype in ['throughput', 'frameloss']:
474             qfunc = getattr(self, "query_%s_load" % (ptype))
475             return qfunc(taskid, casetag, provider)
476         return []
477
478     def query_avglatency(self, taskid, casetag, provider, ptype):
479         if ptype in ['throughput', 'frameloss', 'latency']:
480             qfunc = getattr(self, "query_%s_avg" % (ptype))
481             return qfunc(taskid, casetag, provider)
482         return []
483
484     def query_throughput_provider(self, taskid, casetag, provider):
485         query = self._session.query(table.TblThroughput).filter(and_(table.TblTestList.CaseTag == casetag,
486                                                                      table.TblTestList.Provider == provider,
487                                                                      table.TblTestList.TaskID == taskid,
488                                                                      table.TblTestList.TestID == table.TblThroughput.TestID))
489         return query.all()
490
491     def query_frameloss_provider(self, taskid, casetag, provider):
492         query = self._session.query(table.TblFrameloss).filter(and_(table.TblTestList.CaseTag == casetag,
493                                                                     table.TblTestList.Provider == provider,
494                                                                     table.TblTestList.TaskID == taskid,
495                                                                     table.TblTestList.TestID == table.TblFrameloss.TestID))
496         return query.all()
497
498     def query_latency_provider(self, taskid, casetag, provider):
499         query = self._session.query(table.TblLatency).filter(and_(table.TblTestList.CaseTag == casetag,
500                                                                   table.TblTestList.Provider == provider,
501                                                                   table.TblTestList.TaskID == taskid,
502                                                                   table.TblTestList.TestID == table.TblLatency.TestID))
503         return query.all()
504
505     def query_case_type_count(self, taskid, casetag, ptype):
506         query = self._session.query(table.TblTestList).filter(and_(table.TblTestList.CaseTag == casetag,
507                                                                    table.TblTestList.Type == ptype,
508                                                                    table.TblTestList.TaskID == taskid))
509
510         return query.count()
511
512     def query_case_provider_count(self, taskid, casetag, provider):
513         query = self._session.query(table.TblTestList).filter(and_(table.TblTestList.CaseTag == casetag,
514                                                                    table.TblTestList.Provider == provider,
515                                                                    table.TblTestList.TaskID == taskid))
516         return query.count()
517
518     def query_case_type_provider_count(self, taskid, casetag, provider, ptype):
519         query = self._session.query(table.TblTestList).filter(and_(table.TblTestList.CaseTag == casetag,
520                                                                    table.TblTestList.Type == ptype,
521                                                                    table.TblTestList.Provider == provider,
522                                                                    table.TblTestList.TaskID == taskid))
523
524         return query.count()
525
526     def query_exten_info(self, taskid):
527         query = self._session.query(table.TblEXTInfo.EXTName,
528                                     table.TblEXTInfo.EXTContent,
529                                     table.TblEXTInfo.Description).filter(table.TblEXTInfo.TaskID == taskid)
530         return query.all()
531
532
533 def unit_test():
534     import time
535     dbase = DbManage()
536
537     taskid = dbase.create_task("test", str(time.ctime()), "this is a unit test")
538     dbase.add_host_2task(taskid, "hosta", "hw82576", "xxx", "x", "82599", "ubuntu")
539     dbase.add_extent_2task(taskid, "CETH", "driver", "version 2.0")
540     dbase.add_extent_2task(taskid, "EVS", "switch", "version 3.0")
541
542     testid = dbase.add_test_2task(taskid, "Tn-1", 'udp', "throughput", "ovs", None, "netperf")
543     data = {
544         '64': {
545             'OfferedLoad': 2,
546             'PercentLoss': 3,
547             'Bandwidth': 4,
548             'MinimumLatency': 5,
549             'MaximumLatency': 6,
550             'AverageLatency': 7,
551             'TxFrameCount': 8,
552             'RxFrameCount': 9,
553             'Duration': 10,
554             'CPU': 11,
555             'MppspGhz': 12,
556         }
557     }
558     dbase.add_data_2test(testid, data)
559
560     testid = dbase.add_test_2task(taskid, "Tn-1", 'udp', "frameloss", "ovs", None, "netperf")
561     data = {
562         '64': {
563             'OfferedLoad': 2,
564             'PercentLoss': 3,
565             'Bandwidth': 4,
566             'MinimumLatency': 5,
567             'MaximumLatency': 6,
568             'AverageLatency': 7,
569             'TxFrameCount': 8,
570             'RxFrameCount': 9,
571             'Duration': 10,
572             'CPU': 11,
573             'MppspGhz': 12,
574         }
575     }
576     dbase.add_data_2test(testid, data)
577
578     testid = dbase.add_test_2task(taskid, "Tn-1", 'udp', "latency", "ovs", None, "netperf")
579     data = {
580         64: {'MaximumLatency': 0.0, 'AverageLatency': 0.0, 'MinimumLatency': 0.0, 'OfferedLoad': 0.0},
581         128: {'MaximumLatency': 0.0, 'AverageLatency': 0.0, 'MinimumLatency': 0.0, 'OfferedLoad': 0.0},
582         512: {'MaximumLatency': 0.0, 'AverageLatency': 0.0, 'MinimumLatency': 0.0, 'OfferedLoad': 0.0},
583         1024: {'MaximumLatency': 0.0, 'AverageLatency': 0.0, 'MinimumLatency': 0.0, 'OfferedLoad': 0.0}
584     }
585     dbase.add_data_2test(testid, data)
586     query = dbase.query_testlist(1, "Tn")
587     for item in query:
588         print item.Protocol
589
590     query = dbase.query_testdata(1, "throughput")
591     print query
592
593 if __name__ == '__main__':
594     unit_test()