Merge "Add wget from artifacts.opnfv.org to exceptions"

diff --git a/anteater/src/get_lists.py b/anteater/src/get_lists.py
index 713321f..b7b9aea 100644 (file)
--- a/anteater/src/get_lists.py
+++ b/anteater/src/get_lists.py
@@ -69,28 +69,27 @@ class GetLists(object):
             self.loaded = True
 
     def binary_list(self, project):
-        project_list = False
-        self.load_project_exception_file(yl.get('project_exceptions'), project)
         try:
             default_list = (yl['binaries']['binary_ignore'])
         except KeyError:
             logger.error('Key Error processing binary list values')
-        try:
-            project_list = (yl['binaries'][project]['binary_ignore'])
-        except KeyError:
-            logger.info('No binary waivers found for {0}'.
-                        format(project))
 
         binary_re = re.compile("|".join(default_list),
-                flags=re.IGNORECASE)
+                               flags=re.IGNORECASE)
+        return binary_re
+
+    def binary_hash(self, project, patch_file):
+        self.load_project_exception_file(yl.get('project_exceptions'), project)
+        file_name = os.path.basename(patch_file)
+        try:
+            binary_hash = (yl['binaries'][project][file_name])
+            return binary_hash
+        except KeyError:
+            logger.info('No checksum entries found for {0}'.
+                        format(file_name))
+            binary_hash = 'null'
+            return binary_hash
 
-        if project_list:
-            binary_project_re = re.compile("|".join(project_list),
-                                           flags=re.IGNORECASE)
-            return binary_re, binary_project_re
-        else:
-            binary_project_re = re.compile("")
-            return binary_re, binary_project_re
 
     def file_audit_list(self, project):
         project_list = False

diff --git a/anteater/src/patch_scan.py b/anteater/src/patch_scan.py
index 48c78fd..f8ef225 100644 (file)
--- a/anteater/src/patch_scan.py
+++ b/anteater/src/patch_scan.py
@@ -21,6 +21,7 @@ from binaryornot.check import is_binary
 import anteater.utils.anteater_logger as antlog
 import anteater.src.get_lists as get_lists
 import ConfigParser
+import hashlib
 import sys
 import re
 
@@ -30,6 +31,7 @@ config = ConfigParser.RawConfigParser()
 config.read('anteater.conf')
 reports_dir = config.get('config', 'reports_dir')
 failure = False
+hasher = hashlib.sha256()
 
 
 def prepare_patchset(project, patchset):
@@ -39,7 +41,7 @@ def prepare_patchset(project, patchset):
     # Get Various Lists / Project Waivers
     lists = get_lists.GetLists()
     # Get binary white list
-    binary_list, binary_project_list = lists.binary_list(project)
+    binary_list = lists.binary_list(project)
 
     # Get file name black list and project waivers
     file_audit_list, file_audit_project_list = lists.file_audit_list(project)
@@ -59,7 +61,7 @@ def prepare_patchset(project, patchset):
     for line in lines:
         patch_file = line.strip('\n')
         # Perform binary and file / content checks
-        scan_patch(project, patch_file, binary_list, binary_project_list,
+        scan_patch(project, patch_file, binary_list,
                    file_audit_list, file_audit_project_list,
                    file_content_list, file_content_project_list, licence_ext,
                    licence_ignore)
@@ -69,16 +71,26 @@ def prepare_patchset(project, patchset):
     process_failure()
 
 
-def scan_patch(project, patch_file, binary_list, binary_project_list,
-               file_audit_list, file_audit_project_list, file_content_list,
+def scan_patch(project, patch_file, binary_list, file_audit_list,
+               file_audit_project_list, file_content_list,
                file_content_project_list, licence_ext, licence_ignore):
     """ Scan actions for each commited file in patch set """
     global failure
     if is_binary(patch_file):
-        if not binary_list.search(patch_file) and not binary_project_list\
-                .search(patch_file):
-            logger.error('Non Whitelisted Binary file: {0}'.
-                         format(patch_file))
+        hashlist = get_lists.GetLists()
+        binary_hash = hashlist.binary_hash(project, patch_file)
+        if not binary_list.search(patch_file):
+            with open(patch_file, 'rb') as afile:
+                buf = afile.read()
+                hasher.update(buf)
+            if hasher.hexdigest() in binary_hash:
+                logger.info('Found matching file hash for file: {0}'.
+                            format(patch_file))
+            else:
+                logger.error('Non Whitelisted Binary file: {0}'.
+                             format(patch_file))
+                logger.error('Submit patch with the following hash: {0}'.
+                             format(hasher.hexdigest()))
             failure = True
             with open(reports_dir + "binaries-" + project + ".log", "a") \
                     as gate_report:

diff --git a/anteater/src/project_scan.py b/anteater/src/project_scan.py
index c7c6f28..15498f1 100644 (file)
--- a/anteater/src/project_scan.py
+++ b/anteater/src/project_scan.py
@@ -17,6 +17,7 @@
 
 from __future__ import division, print_function, absolute_import
 import ConfigParser
+import hashlib
 import os
 import re
 import anteater.utils.anteater_logger as antlog
@@ -29,6 +30,7 @@ config.read('anteater.conf')
 reports_dir = config.get('config', 'reports_dir')
 master_list = config.get('config', 'master_list')
 ignore_dirs = ['.git']
+hasher = hashlib.sha256()
 
 
 def prepare_project(project, project_dir):
@@ -38,7 +40,7 @@ def prepare_project(project, project_dir):
     lists = get_lists.GetLists()
 
     # Get binary white list
-    binary_list, binary_project_list = lists.binary_list(project)
+    binary_list = lists.binary_list(project)
 
     # Get file name black list and project waivers
     file_audit_list, file_audit_project_list = lists.file_audit_list(project)
@@ -51,8 +53,8 @@ def prepare_project(project, project_dir):
     licence_ignore = lists.licence_ignore()
 
     # Perform rudimentary scans
-    scan_file(project_dir, project, binary_list, binary_project_list,
-              file_audit_list, file_audit_project_list, file_content_list,
+    scan_file(project_dir, project, binary_list,file_audit_list,
+              file_audit_project_list, file_content_list,
               project_content_list)
 
     # Perform licence header checks
@@ -60,8 +62,8 @@ def prepare_project(project, project_dir):
     licence_root_check(project_dir, project)
 
 
-def scan_file(project_dir, project, binary_list, binary_project_list,
-              file_audit_list, file_audit_project_list, file_content_list,
+def scan_file(project_dir, project, binary_list, file_audit_list,
+              file_audit_project_list, file_content_list,
               project_content_list):
     """Searches for banned strings and files that are listed """
     for root, dirs, files in os.walk(project_dir):
@@ -114,15 +116,25 @@ def scan_file(project_dir, project, binary_list, binary_project_list,
                                               format(match.group()))
             else:
                 # Check if Binary is whitelisted
-                if not binary_list.search(full_path) \
-                        and not binary_project_list.search(full_path):
-                    logger.error('Non Whitelisted Binary: {0}'.
-                                 format(full_path))
-                    with open(reports_dir + "binaries-" + project + ".log",
-                              "a") \
-                            as gate_report:
-                        gate_report.write('Non Whitelisted Binary: {0}\n'.
-                                          format(full_path))
+                hashlist = get_lists.GetLists()
+                binary_hash = hashlist.binary_hash(project, full_path)
+                if not binary_list.search(full_path):
+                    with open(full_path, 'rb') as afile:
+                        buf = afile.read()
+                        hasher.update(buf)
+                    if hasher.hexdigest() in binary_hash:
+                        logger.info('Found matching file hash for file: {0}'.
+                                    format(full_path))
+                    else:
+                        logger.error('Non Whitelisted Binary file: {0}'.
+                                     format(full_path))
+                        logger.error('Please submit patch with this hash: {0}'.
+                                     format(hasher.hexdigest()))
+                        with open(reports_dir + "binaries-" + project + ".log",
+                                  "a") \
+                                as gate_report:
+                            gate_report.write('Non Whitelisted Binary: {0}\n'.
+                                              format(full_path))
 
 
 def licence_root_check(project_dir, project):

diff --git a/docker/Dockerfile b/docker/Dockerfile
index 7a82583..424d926 100644 (file)
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -26,7 +26,7 @@ ENV ANTEATER_HOME ${HOME}/anteater
 # Packaged dependencies
 RUN yum -y install epel-release
 RUN yum -y update
-RUN yum -y install git python-devel python-pip
+RUN yum -y install git python-devel python-pip python-virtualenv
 RUN yum clean all
 
 # Run all following commands and container as non-root user
@@ -36,5 +36,7 @@ USER ${ANTEATER_USER}
 RUN mkdir -p ${ANTEATER_HOME}
 RUN git clone https://gerrit.opnfv.org/gerrit/releng-anteater ${ANTEATER_HOME}
 WORKDIR ${ANTEATER_HOME}
-RUN /usr/bin/pip install -r ${ANTEATER_HOME}/requirements.txt
-RUN python ${ANTEATER_HOME}/setup.py install
+RUN virtualenv ~/venv
+RUN . ~/venv/bin/activate
+RUN ~/venv/bin/pip install -r ${ANTEATER_HOME}/requirements.txt
+RUN ~/venv/bin/python ${ANTEATER_HOME}/setup.py install

diff --git a/exceptions/sandbox.yaml b/exceptions/sandbox.yaml
index 97d41ab..3152816 100644 (file)
--- a/exceptions/sandbox.yaml
+++ b/exceptions/sandbox.yaml
@@ -6,7 +6,9 @@
 # of escaping YAML delimiters too (such as `:`) using double quotes "".
 
 binaries:
-  binary_ignore: [nullvalue]
+  ping:
+    - d0d7dfc73e0fac09d920ebbdf8cd4e0ef623f15d6246ff20d7a6d12c9a48bf41
+
 file_audits:
   file_names: [nullvalue]
   file_contents:

diff --git a/master_list.yaml b/master_list.yaml
index 5b34af9..6306011 100644 (file)
--- a/master_list.yaml
+++ b/master_list.yaml
@@ -7,19 +7,7 @@
 
 binaries:
   binary_ignore:
-    - \.DS_Store
-    - \.eot
-    - \.gif
     - \.git/(index|objects)
-    - \.ico
-    - \.idx
-    - \.jp(e?)g
-    - \.otf
-    - \.pack
-    - \.pdf
-    - \.png
-    - \.ttf
-    - \.woff
 
 file_audits:
   file_names: