Merge "move logging to runtime init"
[releng-anteater.git] / anteater / src / patch_scan.py
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3 ##############################################################################
4 # Copyright (c) 2017 Luke Hinds <lhinds@redhat.com>, Red Hat
5 #
6 # All rights reserved. This program and the accompanying materials
7 # are made available under the terms of the Apache License, Version 2.0
8 # which accompanies this distribution, and is available at
9 # http://www.apache.org/licenses/LICENSE-2.0
10 ##############################################################################
11
12 """
13     Accepts the --patchset argument and iterates through each line of the
14     patchset file to perform various checks such as if the file is a binary, or
15     contains a blacklisted string. If any violations are found, the script
16     exits with code 1 and logs the violation(s) found.
17 """
18
19 from __future__ import division, print_function, absolute_import
20 from binaryornot.check import is_binary
21 import logging
22 import hashlib
23 import six.moves.configparser
24 import sys
25 import re
26
27 from . import get_lists
28
29 logger = logging.getLogger(__name__)
30 config = six.moves.configparser.RawConfigParser()
31 config.read('anteater.conf')
32 reports_dir = config.get('config', 'reports_dir')
33 failure = False
34 hasher = hashlib.sha256()
35
36
37 def prepare_patchset(project, patchset):
38     """ Create black/white lists and default / project waivers
39         and iterates over patchset file """
40
41     # Get Various Lists / Project Waivers
42     lists = get_lists.GetLists()
43     # Get binary white list
44     binary_list = lists.binary_list(project)
45
46     # Get file name black list and project waivers
47     file_audit_list, file_audit_project_list = lists.file_audit_list(project)
48
49     # Get file content black list and project waivers
50     file_content_list, \
51         file_content_project_list = lists.file_content_list(project)
52
53     # Get Licence Lists
54     licence_ext = lists.licence_extensions()
55     licence_ignore = lists.licence_ignore()
56
57     # Open patch set to get file list
58     fo = open(patchset, 'r')
59     lines = fo.readlines()
60
61     for line in lines:
62         patch_file = line.strip('\n')
63         # Perform binary and file / content checks
64         scan_patch(project, patch_file, binary_list,
65                    file_audit_list, file_audit_project_list,
66                    file_content_list, file_content_project_list, licence_ext,
67                    licence_ignore)
68
69     # Process each file in patch set using waivers generated above
70     # Process final result
71     process_failure()
72
73
74 def scan_patch(project, patch_file, binary_list, file_audit_list,
75                file_audit_project_list, file_content_list,
76                file_content_project_list, licence_ext, licence_ignore):
77     """ Scan actions for each commited file in patch set """
78     global failure
79     if is_binary(patch_file):
80         hashlist = get_lists.GetLists()
81         binary_hash = hashlist.binary_hash(project, patch_file)
82         if not binary_list.search(patch_file):
83             with open(patch_file, 'rb') as afile:
84                 buf = afile.read()
85                 hasher.update(buf)
86             if hasher.hexdigest() in binary_hash:
87                 logger.info('Found matching file hash for file: %s',
88                             patch_file)
89             else:
90                 logger.error('Non Whitelisted Binary file: %s',
91                              patch_file)
92                 logger.error('Submit patch with the following hash: %s',
93                              hasher.hexdigest())
94             failure = True
95             with open(reports_dir + "binaries-" + project + ".log", "a") \
96                     as gate_report:
97                 gate_report.write('Non Whitelisted Binary file: {0}\n'.
98                                   format(patch_file))
99     else:
100         # Check file names / extensions
101         if file_audit_list.search(patch_file) and not \
102                     file_audit_project_list.search(patch_file):
103             match = file_audit_list.search(patch_file)
104             logger.error('Blacklisted file: %s', patch_file)
105             logger.error('Matched String: %s', match.group())
106             failure = True
107             with open(reports_dir + "file-names_" + project + ".log", "a") \
108                     as gate_report:
109                 gate_report.write('Blacklisted file: {0}\n'.
110                                   format(patch_file))
111                 gate_report.write('Matched String: {0}'.
112                                   format(match.group()))
113
114         # Open file to check for blacklisted content
115         fo = open(patch_file, 'r')
116         lines = fo.readlines()
117
118         for line in lines:
119             if file_content_list.search(line) and not \
120                     file_content_project_list.search(line):
121                 match = file_content_list.search(line)
122                 logger.error('File contains violation: %s', patch_file)
123                 logger.error('Flagged Content: %s', line.rstrip())
124                 logger.error('Matched String: %s', match.group())
125                 failure = True
126                 with open(reports_dir + "contents_" + project + ".log",
127                           "a") as gate_report:
128                     gate_report.write('File contains violation: {0}\n'.
129                                       format(patch_file))
130                     gate_report.write('Flagged Content: {0}'.
131                                       format(line))
132                     gate_report.write('Matched String: {0}\n'.
133                                       format(match.group()))
134
135         # Run license check
136         licence_check(project, licence_ext, licence_ignore, patch_file)
137
138
139 def licence_check(project, licence_ext,
140                   licence_ignore, patch_file):
141     """ Performs licence checks """
142     global failure
143     if patch_file.endswith(tuple(licence_ext)) \
144             and patch_file not in licence_ignore:
145         fo = open(patch_file, 'r')
146         content = fo.read()
147         # Note: Hardcoded use of 'copyright' & 'spdx' is the result
148         # of a decision made at 2017 plugfest to limit searches to
149         # just these two strings.
150         if re.search("copyright", content, re.IGNORECASE):
151             logger.info('Contains needed Licence string: %s', patch_file)
152         elif re.search("spdx", content, re.IGNORECASE):
153             logger.info('Contains needed Licence string: %s', patch_file)
154         else:
155             logger.error('Licence header missing in file: %s', patch_file)
156             failure = True
157             with open(reports_dir + "licence-" + project + ".log", "a") \
158                     as gate_report:
159                 gate_report.write('Licence header missing in file: {0}\n'.
160                                   format(patch_file))
161
162
163 def process_failure():
164     """ If any scan operations register a failure, sys.exit(1) is called
165         to allow jjb to register a failure"""
166     if failure:
167         logger.error('Please visit: https://wiki.opnfv.org/x/5oey')
168         sys.exit(1)