13 from common import logger_utils, elastic_access
14 from conf import testcases
15 from conf.config import APIConfig
16 from mongo2elastic import format
18 logger = logger_utils.DashboardLogger('mongo2elastic').get
20 parser = argparse.ArgumentParser()
21 parser.add_argument("-c", "--config-file",
23 help="Config file location")
24 parser.add_argument('-ld', '--latest-days',
28 help='get entries old at most N days from mongodb and'
29 ' parse those that are not already in elasticsearch.'
30 ' If not present, will get everything from mongodb, which is the default')
32 args = parser.parse_args()
33 CONF = APIConfig().parse(args.config_file)
36 tmp_docs_file = './mongo-{}.json'.format(uuid.uuid4())
39 class DocumentVerification(object):
40 def __init__(self, doc):
41 super(DocumentVerification, self).__init__()
43 self.doc_id = doc['_id'] if '_id' in doc else None
46 def mandatory_fields_exist(self):
47 mandatory_fields = ['installer',
55 for key, value in self.doc.items():
56 if key in mandatory_fields:
58 logger.info("Skip testcase '%s' because field '%s' missing" %
62 mandatory_fields.remove(key)
66 if len(mandatory_fields) > 0:
67 logger.info("Skip testcase '%s' because field(s) '%s' missing" %
68 (self.doc_id, mandatory_fields))
73 def modify_start_date(self):
76 self.doc[field] = self._fix_date(self.doc[field])
80 def modify_scenario(self):
84 if (scenario not in self.doc) or \
85 (scenario in self.doc and self.doc[scenario] is None):
86 self.doc[scenario] = self.doc[version]
93 def _fix_date(self, date_string):
94 if isinstance(date_string, dict):
95 return date_string['$date']
97 return date_string[:-3].replace(' ', 'T') + 'Z'
100 class DocumentPublisher(object):
102 def __init__(self, doc, fmt, exist_docs, creds, elastic_url):
106 self.exist_docs = exist_docs
107 self.elastic_url = elastic_url
108 self.is_formatted = True
112 if self._verify_document() and self.fmt:
113 self.is_formatted = vars(format)[self.fmt](self.doc)
115 self.is_formatted = False
117 logger.error("Fail in format testcase[%s]\nerror message: %s" %
118 (self.doc, traceback.format_exc()))
119 self.is_formatted = False
124 if self.is_formatted and self.doc not in self.exist_docs:
128 status, data = elastic_access.publish_docs(self.elastic_url, self.creds, self.doc)
130 logger.error('Publish record[{}] failed, due to [{}]'
131 .format(self.doc, json.loads(data)['error']['reason']))
133 def _fix_date(self, date_string):
134 if isinstance(date_string, dict):
135 return date_string['$date']
137 return date_string[:-3].replace(' ', 'T') + 'Z'
139 def _verify_document(self):
140 return not (DocumentVerification(self.doc)
143 .mandatory_fields_exist()
147 class DocumentsPublisher(object):
149 def __init__(self, project, case, fmt, days, elastic_url, creds):
150 self.project = project
154 self.elastic_url = elastic_url
156 self.existed_docs = []
160 past_time = datetime.datetime.today() - datetime.timedelta(days=self.days)
162 "project_name": "{}",
164 "start_date": {{"$gt" : "{}"}}
165 }}'''.format(self.project, self.case, past_time)
168 "project_name": "{}",
170 }}'''.format(self.project, self.case)
171 cmd = ['mongoexport',
172 '--db', 'test_results_collection',
173 '--collection', 'results',
174 '--query', '{}'.format(query),
175 '--out', '{}'.format(tmp_docs_file)]
177 subprocess.check_call(cmd)
179 except Exception, err:
180 logger.error("export mongodb failed: %s" % err)
184 def get_existed_docs(self):
190 {{ "match": {{ "project_name": "{}" }} }},
191 {{ "match": {{ "case_name": "{}" }} }}
195 }}'''.format(self.project, self.case)
201 {{ "match": {{ "project_name": "{}" }} }},
202 {{ "match": {{ "case_name": "{}" }} }}
206 "start_date": {{ "gte": "now-{}d" }}
211 }}'''.format(self.project, self.case, self.days)
213 raise Exception('Update days must be non-negative')
214 self.existed_docs = elastic_access.get_docs(self.elastic_url, self.creds, body)
220 with open(tmp_docs_file) as fdocs:
221 for doc_line in fdocs:
222 DocumentPublisher(json.loads(doc_line),
226 self.elastic_url).format().publish()
233 if os.path.exists(tmp_docs_file):
234 os.remove(tmp_docs_file)
238 base_elastic_url = urlparse.urljoin(CONF.es_url, '/test_results/mongo2elastic')
239 days = args.latest_days
240 es_creds = CONF.es_creds
242 for project, case_dicts in testcases.testcases_yaml.items():
243 for case_dict in case_dicts:
244 case = case_dict.get('name')
245 fmt = testcases.compose_format(case_dict.get('format'))
246 DocumentsPublisher(project,
251 es_creds).export().get_existed_docs().publish()