seperate exception rules for releng project

add optional project exception configuration file.

for one project:
configuration in project exception file and in gatechecks.yaml will be
merged together for one project

JIRA:RELENG-234

Change-Id: I7fff3082e837a478f155769fadd75533583543cd
Signed-off-by: Julien <zhang.jun3g@zte.com.cn>

diff --git a/anteater/src/get_lists.py b/anteater/src/get_lists.py
index 8941510..7d6d3f1 100644 (file)
--- a/anteater/src/get_lists.py
+++ b/anteater/src/get_lists.py
@@ -16,6 +16,8 @@
 
 import anteater.utils.anteater_logger as antlog
 import ConfigParser
+import copy
+import os
 import yaml
 import re
 
@@ -27,14 +29,45 @@ gate_checks = config.get('config', 'gate_checks')
 with open(gate_checks, 'r') as f:
     yl = yaml.safe_load(f)
 
+def _remove_nullvalue(contents):
+    if contents and len(contents) > 2 and 'nullvalue' in contents:
+        contents.remove('nullvalue')
+
+def _merge(org, ded):
+    ret = copy.deepcopy(org)
+    for key in list(set([k for k in org] + [k for k in ded])):
+        if key in org and key in ded:
+            ret[key] = list(set(ret[key] + ded[key]))
+            _remove_nullvalue(ret[key])
+        elif key in ded:
+            ret[key] = ded[key]
+    return ret
 
 class GetLists(object):
     def __init__(self, *args):
         # Placeholder for future args if more filters are needed
         self.args = args
+        self.loaded = False
+
+    def load_project_exception_file(self, project_config, project):
+        if self.loaded:
+            return
+        exception_file = None
+        for item in project_config:
+            if project in item:
+                exception_file = item.get(project)
+        if exception_file is not None:
+            with open(exception_file, 'r') as f:
+                ex = yaml.safe_load(f)
+            for key in ex:
+                if key in yl:
+                    yl[key][project] = _merge(yl[key][project], ex.get(key, None)) \
+                            if project in yl[key] else ex.get(key, None)
+            self.loaded = True
 
     def binary_list(self, project):
         project_list = False
+        self.load_project_exception_file(yl.get('project_config'), project)
         try:
             default_list = (yl['binaries']['binary_ignore'])
         except KeyError:
@@ -58,6 +91,7 @@ class GetLists(object):
 
     def file_audit_list(self, project):
         project_list = False
+        self.load_project_exception_file(yl.get('project_config'), project)
         try:
             default_list = set((yl['file_audits']['file_names']))
         except KeyError:
@@ -83,6 +117,7 @@ class GetLists(object):
 
     def file_content_list(self,  project):
         project_list = False
+        self.load_project_exception_file(yl.get('project_config'), project)
         try:
             default_list = set((yl['file_audits']['file_contents']))
         except KeyError:

diff --git a/gatechecks.yaml b/gatechecks.yaml
index 23b5bd5..243d5aa 100644 (file)
--- a/gatechecks.yaml
+++ b/gatechecks.yaml
@@ -7,41 +7,18 @@
 binaries:
   binary_ignore: [\.pdf,\.png,\.jpeg,\.jpg,\.gif,\.idx,\.pack,\.woff,\.ttf,\.eot,\.DS_Store,\.git/objects,\.git/index,
                   \.otf,\.ico]
-  releng:
-    binary_ignore: [nullvalue]
-  sandbox:
-    binary_ignore: [nullvalue]
-  testproj:
-    binary_ignore: [nullvalue]
 
 file_audits:
   file_names: [\.gpg$,\.key$,\.asc$,\.sig$,\.md5,aws_access_key_id,aws_secret_access_key,id_rsa]
   file_contents: [-----BEGIN\sRSA\sPRIVATE\sKEY----,secret,ssh_key,private_key,md5,wget,"curl(.*?)bash",eval,
                  "sh(.*?)curl","git(.*?)clone",gost,md2,md4,md5,rc4,sha0,streebog,dual_ec_drbg,
                  snefru,panama,ripemd,sslv1,sslv2,tlsv1]
-  releng:
-    file_names: [nullvalue]
-    file_contents: [slave_secret,self\.local_ssh_key,self\.proxy_ssh_key,jh_ssh_key='/root/\.ssh/id_rsa',fa-user-secret,
-                     -s  set secret key,paramiko\.RSAKey\.from_private_key_file\(pkey_file\),git clone the Openstack-Ansible,
-                     secret not defined,user_secrets\.yml,wget -O /tmp/get-pip\.py,"PKG_MAP\\[wget\\]",^wget \\,
-                     "git clone(.*)gerrit\\.opnfv\\.org","git clone(.*)\\.openstack\\.org",wget(.*)build.opnfv.org,
-                     wget --version,"\\[wget\\]=wget",git repositories of XCI,"packages = \\['parted', 'puppet', 'wget'",
-                     bifrost_ssh_private_key,github\.com/maestrodev/puppet-wget,"wget http://repo1\\.maven\\.org",
-                     elif type wget >/dev/null,wget \$get_pip_url,"wget(.*)puppetlabs\\.com",git_clone(.*)MOD(.*)MODULE_PATH(.*)MODULE_NAME,
-                     git clone \$MOD \$DEST,function git_clone,"name: SSH_KEY","name: GIT_CLONE_BASE",
-                     wget -q -O \$ODL_ZIP \$NETVIRT_ARTIFACT_URL,git clone(.*)\$GIT_BASE(.*)\$WORKSPACE,
-                     git clone --quiet --branch \$BRANCH \$LAB_CONFIG_URL lab-config,git clone(.*)\$OPENSTACK_OSA_GIT_URL,
-                     wget --directory-prefix=(.*)SNAP_CACHE(.*)snap_url(.*),sudo yum -y install wget,cacheValues,isInfiniteValue,
-                     wget(.*)WORKSPACE/opnfv\.properties(.*)GS_URL(.*)properties,wget > /dev/null,fileValidationError,eval_collection,
-                     eval_db,if \(name === 'evals'\),evaluate,value,ast\.literal_eva,_eval_pods_db,new = eval\(obj\)]
-  sandbox:
-    file_names: [nullvalue]
-    file_contents: [nullvalue]
-  testproj:
-    file_names: [nullvalue]
-    file_contents: [nullvalue]
 
 licence:
   licence_ext: ['.rst','.md','.py','.sh','.java','.rb']
   licence_ignore: ['__init__.py']
 
+project_config:
+  - releng: releng_exception.yaml
+  - sandbox: sandbox_exception.yaml
+  - testproj: sandbox_exception.yaml

diff --git a/releng_exception.yaml b/releng_exception.yaml
new file mode 100644 (file)
index 0000000..d860ce7
--- /dev/null
+++ b/releng_exception.yaml
@@ -0,0 +1,61 @@
+# When adding projects all `arrays: []` sections must have
+# a value, Use 'nullvalue' if no waivers are available.
+#
+# This file uses standard regular expression syntax,  however be mindful
+# of escaping YAML delimiters too (such as `:`) using double quotes "".
+
+binaries:
+  binary_ignore: [nullvalue]
+file_audits:
+  file_names: [nullvalue]
+  file_contents:
+    - -s  set secret key
+    - "PKG_MAP\\[wget\\]"
+    - "\\[wget\\]=wget"
+    - "git clone(.*)\\.openstack\\.org"
+    - "git clone(.*)gerrit\\.opnfv\\.org"
+    - "name: GIT_CLONE_BASE"
+    - "name: SSH_KEY"
+    - "packages = \\['parted', 'puppet', 'wget'"
+    - "wget http://repo1\\.maven\\.org"
+    - "wget(.*)puppetlabs\\.com"
+    - ^wget \\
+    - _eval_pods_db
+    - ast\.literal_eva
+    - bifrost_ssh_private_key
+    - cacheValues
+    - elif type wget >/dev/null
+    - eval_collection
+    - eval_db
+    - evaluate
+    - fa-user-secret
+    - fileValidationError
+    - function git_clone
+    - git clone --quiet --branch \$BRANCH \$LAB_CONFIG_URL lab-config
+    - git clone \$MOD \$DEST
+    - git clone the Openstack-Ansible
+    - git clone(.*)\$GIT_BASE(.*)\$WORKSPACE
+    - git clone(.*)\$OPENSTACK_OSA_GIT_URL
+    - git repositories of XCI
+    - git_clone(.*)MOD(.*)MODULE_PATH(.*)MODULE_NAME
+    - github\.com/maestrodev/puppet-wget
+    - if \(name === 'evals'\)
+    - isInfiniteValue
+    - jh_ssh_key='/root/\.ssh/id_rsa'
+    - new = eval\(obj\)
+    - paramiko\.RSAKey\.from_private_key_file\(pkey_file\)
+    - secret not defined
+    - self\.local_ssh_key
+    - self\.proxy_ssh_key
+    - slave_secret
+    - sudo yum -y install wget
+    - user_secrets\.yml
+    - value
+    - wget --directory-prefix=(.*)SNAP_CACHE(.*)snap_url(.*)
+    - wget --version
+    - wget -O /tmp/get-pip\.py
+    - wget -q -O \$ODL_ZIP \$NETVIRT_ARTIFACT_URL
+    - wget > /dev/null
+    - wget \$get_pip_url
+    - wget(.*)WORKSPACE/opnfv\.properties(.*)GS_URL(.*)properties
+    - wget(.*)build.opnfv.org

diff --git a/sandbox_exception.yaml b/sandbox_exception.yaml
new file mode 100644 (file)
index 0000000..14f0e81
--- /dev/null
+++ b/sandbox_exception.yaml
@@ -0,0 +1,11 @@
+# When adding projects all `arrays: []` sections must have
+# a value, Use 'nullvalue' if no waivers are available.
+#
+# This file uses standard regular expression syntax,  however be mindful
+# of escaping YAML delimiters too (such as `:`) using double quotes "".
+
+binaries:
+  binary_ignore: [nullvalue]
+file_audits:
+  file_names: [nullvalue]
+  file_contents: [nullvalue]