12 from dashboard.common import elastic_access
13 from dashboard.common import logger_utils
14 from dashboard.conf import testcases
15 from dashboard.conf.config import APIConfig
16 from dashboard.mongo2elastic import format
18 logger = logger_utils.DashboardLogger('mongo2elastic').get
20 parser = argparse.ArgumentParser()
21 parser.add_argument("-c", "--config-file",
23 help="Config file location")
24 parser.add_argument('-ld', '--latest-days',
28 help='get entries old at most N days from mongodb and'
29 ' parse those that are not already in elasticsearch.'
30 ' If not present, will get everything from mongodb, which is the default')
32 args = parser.parse_args()
33 CONF = APIConfig().parse(args.config_file)
36 tmp_docs_file = './mongo-{}.json'.format(uuid.uuid4())
39 class DocumentVerification(object):
40 def __init__(self, doc):
41 super(DocumentVerification, self).__init__()
43 self.doc_id = doc['_id'] if '_id' in doc else None
46 def mandatory_fields_exist(self):
47 mandatory_fields = ['installer',
55 for key, value in self.doc.items():
56 if key in mandatory_fields:
58 logger.info("Skip testcase '%s' because field '%s' missing" %
62 mandatory_fields.remove(key)
66 if len(mandatory_fields) > 0:
67 logger.info("Skip testcase '%s' because field(s) '%s' missing" %
68 (self.doc_id, mandatory_fields))
73 def modify_start_date(self):
76 self.doc[field] = self._fix_date(self.doc[field])
80 def modify_scenario(self):
84 if (scenario not in self.doc) or \
85 (scenario in self.doc and self.doc[scenario] is None):
86 self.doc[scenario] = self.doc[version]
93 def _fix_date(self, date_string):
94 if date_string == 'None':
96 if isinstance(date_string, dict):
97 return date_string['$date']
98 if 'T' not in date_string:
99 date_string = date_string[:-3].replace(' ', 'T')
100 if not date_string.endswith('Z'):
106 class DocumentPublisher(object):
108 def __init__(self, doc, fmt, exist_docs, creds, elastic_url):
112 self.exist_docs = exist_docs
113 self.elastic_url = elastic_url
114 self.is_formatted = True
118 if self._verify_document() and self.fmt:
119 self.is_formatted = vars(format)[self.fmt](self.doc)
121 self.is_formatted = False
123 logger.error("Fail in format testcase[%s]\nerror message: %s" %
124 (self.doc, traceback.format_exc()))
125 self.is_formatted = False
130 if self.is_formatted and self.doc not in self.exist_docs:
134 status, data = elastic_access.publish_docs(self.elastic_url, self.creds, self.doc)
136 logger.error('Publish record[{}] failed, due to [{}]'
137 .format(self.doc, json.loads(data)['error']['reason']))
139 def _fix_date(self, date_string):
140 if isinstance(date_string, dict):
141 return date_string['$date']
143 return date_string[:-3].replace(' ', 'T') + 'Z'
145 def _verify_document(self):
146 return not (DocumentVerification(self.doc)
149 .mandatory_fields_exist()
153 class DocumentsPublisher(object):
155 def __init__(self, project, case, fmt, days, elastic_url, creds):
156 self.project = project
160 self.elastic_url = elastic_url
162 self.existed_docs = []
166 past_time = datetime.datetime.today() - datetime.timedelta(days=self.days)
168 "project_name": "{}",
170 "start_date": {{"$gt" : "{}"}}
171 }}'''.format(self.project, self.case, past_time)
174 "project_name": "{}",
176 }}'''.format(self.project, self.case)
177 cmd = ['mongoexport',
178 '--db', 'test_results_collection',
179 '--collection', 'results',
180 '--query', '{}'.format(query),
181 '--out', '{}'.format(tmp_docs_file)]
183 subprocess.check_call(cmd)
185 except Exception, err:
186 logger.error("export mongodb failed: %s" % err)
190 def get_exists(self):
196 {{ "match": {{ "project_name": "{}" }} }},
197 {{ "match": {{ "case_name": "{}" }} }}
201 }}'''.format(self.project, self.case)
207 {{ "match": {{ "project_name": "{}" }} }},
208 {{ "match": {{ "case_name": "{}" }} }}
212 "start_date": {{ "gte": "now-{}d" }}
217 }}'''.format(self.project, self.case, self.days)
219 raise Exception('Update days must be non-negative')
220 self.existed_docs = elastic_access.get_docs(self.elastic_url, self.creds, body)
226 with open(tmp_docs_file) as fdocs:
227 for doc_line in fdocs:
228 DocumentPublisher(json.loads(doc_line),
232 self.elastic_url).format().publish()
239 if os.path.exists(tmp_docs_file):
240 os.remove(tmp_docs_file)
244 for project, case_dicts in testcases.testcases_yaml.items():
245 for case_dict in case_dicts:
246 case = case_dict.get('name')
247 fmt = testcases.compose_format(case_dict.get('format'))
248 DocumentsPublisher(project,
253 CONF.es_creds).export().get_exists().publish()