12 from dashboard.common import elastic_access
13 from dashboard.common import logger_utils
14 from dashboard.conf import testcases
15 from dashboard.conf.config import APIConfig
16 from dashboard.mongo2elastic import format
18 logger = logger_utils.DashboardLogger('mongo2elastic').get
20 parser = argparse.ArgumentParser()
21 parser.add_argument("-c", "--config-file",
23 help="Config file location")
24 parser.add_argument('-ld', '--latest-days',
28 help='get entries old at most N days from mongodb and'
29 ' parse those that are not already in elasticsearch.'
30 ' If not present, will get everything from mongodb,'
31 ' which is the default')
33 args = parser.parse_args()
34 CONF = APIConfig().parse(args.config_file)
37 tmp_docs_file = './mongo-{}.json'.format(uuid.uuid4())
40 class DocumentVerification(object):
42 def __init__(self, doc):
43 super(DocumentVerification, self).__init__()
45 self.doc_id = doc['_id'] if '_id' in doc else None
48 def mandatory_fields_exist(self):
49 mandatory_fields = ['installer',
57 for key, value in self.doc.items():
58 if key in mandatory_fields:
60 logger.info("Skip testcase '%s' because field "
61 "'%s' missing" % (self.doc_id, key))
64 mandatory_fields.remove(key)
68 if len(mandatory_fields) > 0:
69 logger.info("Skip testcase '%s' because field(s) '%s' missing" %
70 (self.doc_id, mandatory_fields))
75 def modify_start_date(self):
78 self.doc[field] = self._fix_date(self.doc[field])
82 def modify_scenario(self):
86 if (scenario not in self.doc) or \
87 (scenario in self.doc and self.doc[scenario] is None):
88 self.doc[scenario] = self.doc[version]
95 def _fix_date(self, date_string):
96 if date_string == 'None':
98 if isinstance(date_string, dict):
99 return date_string['$date']
100 if 'T' not in date_string:
101 date_string = date_string[:-3].replace(' ', 'T')
102 if not date_string.endswith('Z'):
108 class DocumentPublisher(object):
110 def __init__(self, doc, fmt, exist_docs, creds, elastic_url):
114 self.exist_docs = exist_docs
115 self.elastic_url = elastic_url
116 self.is_formatted = True
120 if self._verify_document() and self.fmt:
121 self.is_formatted = vars(format)[self.fmt](self.doc)
123 self.is_formatted = False
125 logger.error("Fail in format testcase[%s]\nerror message: %s" %
126 (self.doc, traceback.format_exc()))
127 self.is_formatted = False
132 if self.is_formatted and self.doc not in self.exist_docs:
136 status, data = elastic_access.publish_docs(
137 self.elastic_url, self.creds, self.doc)
139 logger.error('Publish record[{}] failed, due to [{}]'
141 json.loads(data)['error']['reason']))
143 def _fix_date(self, date_string):
144 if isinstance(date_string, dict):
145 return date_string['$date']
147 return date_string[:-3].replace(' ', 'T') + 'Z'
149 def _verify_document(self):
150 return not (DocumentVerification(self.doc)
153 .mandatory_fields_exist()
157 class DocumentsPublisher(object):
159 def __init__(self, project, case, fmt, days, elastic_url, creds):
160 self.project = project
164 self.elastic_url = elastic_url
166 self.existed_docs = []
170 past_time = datetime.datetime.today(
171 ) - datetime.timedelta(days=self.days)
173 "project_name": "{}",
175 "start_date": {{"$gt" : "{}"}}
176 }}'''.format(self.project, self.case, past_time)
179 "project_name": "{}",
181 }}'''.format(self.project, self.case)
182 cmd = ['mongoexport',
183 '--db', 'test_results_collection',
184 '--collection', 'results',
185 '--query', '{}'.format(query),
186 '--out', '{}'.format(tmp_docs_file)]
188 subprocess.check_call(cmd)
190 except Exception as err:
191 logger.error("export mongodb failed: %s" % err)
195 def get_exists(self):
201 {{ "match": {{ "project_name": "{}" }} }},
202 {{ "match": {{ "case_name": "{}" }} }}
206 }}'''.format(self.project, self.case)
212 {{ "match": {{ "project_name": "{}" }} }},
213 {{ "match": {{ "case_name": "{}" }} }}
217 "start_date": {{ "gte": "now-{}d" }}
222 }}'''.format(self.project, self.case, self.days)
224 raise Exception('Update days must be non-negative')
225 self.existed_docs = elastic_access.get_docs(
226 self.elastic_url, self.creds, body)
232 with open(tmp_docs_file) as fdocs:
233 for doc_line in fdocs:
234 DocumentPublisher(json.loads(doc_line),
238 self.elastic_url).format().publish()
245 if os.path.exists(tmp_docs_file):
246 os.remove(tmp_docs_file)
250 for project, case_dicts in testcases.testcases_yaml.items():
251 for case_dict in case_dicts:
252 case = case_dict.get('name')
253 fmt = testcases.compose_format(case_dict.get('format'))
254 DocumentsPublisher(project,
259 CONF.es_creds).export().get_exists().publish()