Merge "Add qtip job to pod zte-virtual6"

[releng.git] / utils / test / dashboard / dashboard / mongo2elastic / main.py

diff --git a/utils/test/dashboard/dashboard/mongo2elastic/main.py b/utils/test/dashboard/dashboard/mongo2elastic/main.py
index 8c7300b..e33252d 100644 (file)
--- a/utils/test/dashboard/dashboard/mongo2elastic/main.py
+++ b/utils/test/dashboard/dashboard/mongo2elastic/main.py
@@ -5,15 +5,15 @@ import json
 import os
 import subprocess
 import traceback
-import urlparse
 import uuid
 
 import argparse
 
-from common import logger_utils, elastic_access
-from conf import testcases
-from conf.config import APIConfig
-from mongo2elastic import format
+from dashboard.common import elastic_access
+from dashboard.common import logger_utils
+from dashboard.conf import testcases
+from dashboard.conf.config import APIConfig
+from dashboard.mongo2elastic import format
 
 logger = logger_utils.DashboardLogger('mongo2elastic').get
 
@@ -27,7 +27,8 @@ parser.add_argument('-ld', '--latest-days',
                     metavar='N',
                     help='get entries old at most N days from mongodb and'
                          ' parse those that are not already in elasticsearch.'
-                         ' If not present, will get everything from mongodb, which is the default')
+                         ' If not present, will get everything from mongodb,'
+                         ' which is the default')
 
 args = parser.parse_args()
 CONF = APIConfig().parse(args.config_file)
@@ -37,6 +38,7 @@ tmp_docs_file = './mongo-{}.json'.format(uuid.uuid4())
 
 
 class DocumentVerification(object):
+
     def __init__(self, doc):
         super(DocumentVerification, self).__init__()
         self.doc = doc
@@ -55,8 +57,8 @@ class DocumentVerification(object):
         for key, value in self.doc.items():
             if key in mandatory_fields:
                 if value is None:
-                    logger.info("Skip testcase '%s' because field '%s' missing" %
-                                (self.doc_id, key))
+                    logger.info("Skip testcase '%s' because field "
+                                "'%s' missing" % (self.doc_id, key))
                     self.skip = True
                 else:
                     mandatory_fields.remove(key)
@@ -131,10 +133,12 @@ class DocumentPublisher(object):
             self._publish()
 
     def _publish(self):
-        status, data = elastic_access.publish_docs(self.elastic_url, self.creds, self.doc)
+        status, data = elastic_access.publish_docs(
+            self.elastic_url, self.creds, self.doc)
         if status > 300:
             logger.error('Publish record[{}] failed, due to [{}]'
-                         .format(self.doc, json.loads(data)['error']['reason']))
+                         .format(self.doc,
+                                 json.loads(data)['error']['reason']))
 
     def _fix_date(self, date_string):
         if isinstance(date_string, dict):
@@ -163,7 +167,8 @@ class DocumentsPublisher(object):
 
     def export(self):
         if self.days > 0:
-            past_time = datetime.datetime.today() - datetime.timedelta(days=self.days)
+            past_time = datetime.datetime.today(
+            ) - datetime.timedelta(days=self.days)
             query = '''{{
                           "project_name": "{}",
                           "case_name": "{}",
@@ -182,12 +187,12 @@ class DocumentsPublisher(object):
         try:
             subprocess.check_call(cmd)
             return self
-        except Exception, err:
+        except Exception as err:
             logger.error("export mongodb failed: %s" % err)
             self._remove()
             exit(-1)
 
-    def get_existed_docs(self):
+    def get_exists(self):
         if self.days == 0:
             body = '''{{
                         "query": {{
@@ -217,7 +222,8 @@ class DocumentsPublisher(object):
                    }}'''.format(self.project, self.case, self.days)
         else:
             raise Exception('Update days must be non-negative')
-        self.existed_docs = elastic_access.get_docs(self.elastic_url, self.creds, body)
+        self.existed_docs = elastic_access.get_docs(
+            self.elastic_url, self.creds, body)
         return self
 
     def publish(self):
@@ -241,10 +247,6 @@ class DocumentsPublisher(object):
 
 
 def main():
-    base_elastic_url = urlparse.urljoin(CONF.es_url, '/testapi/results')
-    days = args.latest_days
-    es_creds = CONF.es_creds
-
     for project, case_dicts in testcases.testcases_yaml.items():
         for case_dict in case_dicts:
             case = case_dict.get('name')
@@ -252,6 +254,6 @@ def main():
             DocumentsPublisher(project,
                                case,
                                fmt,
-                               days,
-                               base_elastic_url,
-                               es_creds).export().get_existed_docs().publish()
+                               args.latest_days,
+                               CONF.index_url,
+                               CONF.es_creds).export().get_exists().publish()