import os
import subprocess
import traceback
-import urlparse
import uuid
import argparse
metavar='N',
help='get entries old at most N days from mongodb and'
' parse those that are not already in elasticsearch.'
- ' If not present, will get everything from mongodb, which is the default')
+ ' If not present, will get everything from mongodb,'
+ ' which is the default')
args = parser.parse_args()
CONF = APIConfig().parse(args.config_file)
class DocumentVerification(object):
+
def __init__(self, doc):
super(DocumentVerification, self).__init__()
self.doc = doc
for key, value in self.doc.items():
if key in mandatory_fields:
if value is None:
- logger.info("Skip testcase '%s' because field '%s' missing" %
- (self.doc_id, key))
+ logger.info("Skip testcase '%s' because field "
+ "'%s' missing" % (self.doc_id, key))
self.skip = True
else:
mandatory_fields.remove(key)
self._publish()
def _publish(self):
- status, data = elastic_access.publish_docs(self.elastic_url, self.creds, self.doc)
+ status, data = elastic_access.publish_docs(
+ self.elastic_url, self.creds, self.doc)
if status > 300:
logger.error('Publish record[{}] failed, due to [{}]'
- .format(self.doc, json.loads(data)['error']['reason']))
+ .format(self.doc,
+ json.loads(data)['error']['reason']))
def _fix_date(self, date_string):
if isinstance(date_string, dict):
def export(self):
if self.days > 0:
- past_time = datetime.datetime.today() - datetime.timedelta(days=self.days)
+ past_time = datetime.datetime.today(
+ ) - datetime.timedelta(days=self.days)
query = '''{{
"project_name": "{}",
"case_name": "{}",
try:
subprocess.check_call(cmd)
return self
- except Exception, err:
+ except Exception as err:
logger.error("export mongodb failed: %s" % err)
self._remove()
exit(-1)
- def get_existed_docs(self):
+ def get_exists(self):
if self.days == 0:
body = '''{{
"query": {{
}}'''.format(self.project, self.case, self.days)
else:
raise Exception('Update days must be non-negative')
- self.existed_docs = elastic_access.get_docs(self.elastic_url, self.creds, body)
+ self.existed_docs = elastic_access.get_docs(
+ self.elastic_url, self.creds, body)
return self
def publish(self):
def main():
- base_elastic_url = urlparse.urljoin(CONF.es_url, '/test_results/mongo2elastic')
- days = args.latest_days
- es_creds = CONF.es_creds
-
for project, case_dicts in testcases.testcases_yaml.items():
for case_dict in case_dicts:
case = case_dict.get('name')
DocumentsPublisher(project,
case,
fmt,
- days,
- base_elastic_url,
- es_creds).export().get_existed_docs().publish()
+ args.latest_days,
+ CONF.index_url,
+ CONF.es_creds).export().get_exists().publish()