reports_dir = .reports/
anteater_log = .reports/anteater.log
master_list = ./master_list.yaml
+ignore_list = ./ignore_list.yaml
config.read('anteater.conf')
logger = logging.getLogger(__name__)
master_list = config.get('config', 'master_list')
+ignore_list = config.get('config', 'ignore_list')
with open(master_list, 'r') as f:
- yl = yaml.safe_load(f)
+ ml = yaml.safe_load(f)
+
+with open(ignore_list, 'r') as f:
+ il = yaml.safe_load(f)
def _remove_nullvalue(contents):
with open(exception_file, 'r') as f:
ex = yaml.safe_load(f)
for key in ex:
- if key in yl:
- yl[key][project] = _merge(yl[key][project], ex.get(key, None)) \
- if project in yl[key] else ex.get(key, None)
+ if key in ml:
+ ml[key][project] = _merge(ml[key][project], ex.get(key, None)) \
+ if project in ml[key] else ex.get(key, None)
self.loaded = True
def binary_list(self, project):
try:
- default_list = (yl['binaries']['binary_ignore'])
+ default_list = (ml['binaries']['binary_ignore'])
except KeyError:
logger.error('Key Error processing binary list values')
return binary_re
def binary_hash(self, project, patch_file):
- self.load_project_exception_file(yl.get('project_exceptions'), project)
+ self.load_project_exception_file(ml.get('project_exceptions'), project)
file_name = os.path.basename(patch_file)
try:
- binary_hash = (yl['binaries'][project][file_name])
+ binary_hash = (ml['binaries'][project][file_name])
return binary_hash
except KeyError:
binary_hash = 'null'
def file_audit_list(self, project):
project_list = False
- self.load_project_exception_file(yl.get('project_exceptions'), project)
+ self.load_project_exception_file(ml.get('project_exceptions'), project)
try:
- default_list = set((yl['file_audits']['file_names']))
+ default_list = set((ml['file_audits']['file_names']))
except KeyError:
logger.error('Key Error processing file_names list values')
try:
- project_list = set((yl['file_audits'][project]['file_names']))
+ project_list = set((ml['file_audits'][project]['file_names']))
logger.info('file_names waivers found for %s', project)
except KeyError:
logger.info('No file_names waivers found for %s', project)
def file_content_list(self, project):
project_list = False
- self.load_project_exception_file(yl.get('project_exceptions'), project)
+ self.load_project_exception_file(ml.get('project_exceptions'), project)
try:
- master_list = (yl['file_audits']['file_contents'])
+ master_list = (ml['file_audits']['file_contents'])
except KeyError:
logger.error('Key Error processing file_contents list values')
try:
- project_list = set((yl['file_audits'][project]['file_contents']))
- project_list_re = re.compile("|".join(project_list),
- flags=re.IGNORECASE)
+ ignore_list = il['file_audits']['file_contents']
+
+ except KeyError:
+ logger.error('Key Error processing file_contents list values')
+
+ try:
+ project_list = ml['file_audits'][project]['file_contents']
+
except KeyError:
logger.info('No file_contents waivers found for %s', project)
- return master_list, project_list_re
+ ignore_list_merge = project_list + ignore_list
+
+ ignore_list_re = re.compile("|".join(ignore_list_merge), flags=re.IGNORECASE)
+
+ return master_list, ignore_list_re
def file_ignore(self):
try:
- file_ignore = (yl['file_ignore'])
+ file_ignore = (ml['file_ignore'])
except KeyError:
logger.error('Key Error processing file_ignore list values')
return file_ignore
def licence_extensions(self):
try:
- licence_extensions = (yl['licence']['licence_ext'])
+ licence_extensions = (ml['licence']['licence_ext'])
except KeyError:
logger.error('Key Error processing licence_extensions list values')
return licence_extensions
def licence_ignore(self):
try:
- licence_ignore = (yl['licence']['licence_ignore'])
+ licence_ignore = (ml['licence']['licence_ignore'])
except KeyError:
logger.error('Key Error processing licence_ignore list values')
return licence_ignore
file_audit_list, file_audit_project_list = lists.file_audit_list(project)
# Get file content black list and project waivers
- master_list, project_list_re = lists.file_content_list(project)
+ master_list, ignore_list = lists.file_content_list(project)
# Get File Ignore Lists
file_ignore = lists.file_ignore()
# Perform binary and file / content checks
scan_patch(project, patch_file, binary_list,
file_audit_list, file_audit_project_list,
- master_list, project_list_re, licence_ext,
+ master_list, ignore_list, licence_ext,
file_ignore, licence_ignore)
# Process each file in patch set using waivers generated above
def scan_patch(project, patch_file, binary_list, file_audit_list,
file_audit_project_list, master_list,
- project_list_re, licence_ext, file_ignore, licence_ignore):
+ ignore_list, licence_ext, file_ignore, licence_ignore):
""" Scan actions for each commited file in patch set """
global failure
if is_binary(patch_file):
for key, value in master_list.iteritems():
regex = value['regex']
desc = value['desc']
- if re.search(regex, line) and not re.search(project_list_re, line):
+ if re.search(regex, line) and not re.search(
+ ignore_list, line):
logger.error('File contains violation: %s', patch_file)
logger.error('Flagged Content: %s', line.rstrip())
logger.error('Matched Regular Exp: %s', regex)
config.read('anteater.conf')
reports_dir = config.get('config', 'reports_dir')
master_list = config.get('config', 'master_list')
+ignore_list = config.get('config', 'master_list')
ignore_dirs = ['.git']
hasher = hashlib.sha256()
file_audit_list, file_audit_project_list = lists.file_audit_list(project)
# Get file content black list and project waivers
- master_list, project_list = lists.file_content_list(project)
+ master_list, ignore_list = lists.file_content_list(project)
# Get File Ignore Lists
file_ignore = lists.file_ignore()
# Perform rudimentary scans
scan_file(project_dir, project, binary_list,file_audit_list,
- file_audit_project_list, master_list, file_ignore,
- project_list)
+ file_audit_project_list, master_list, ignore_list,
+ file_ignore)
# Perform licence header checks
licence_check(licence_ext, licence_ignore, project, project_dir)
def scan_file(project_dir, project, binary_list, file_audit_list,
- file_audit_project_list, master_list, file_ignore,
- project_list):
+ file_audit_project_list, master_list, ignore_list,
+ file_ignore):
"""Searches for banned strings and files that are listed """
for root, dirs, files in os.walk(project_dir):
# Filter out ignored directories from list.
write('Matched String: {0}'.
format(match.group()))
- # Check if Binary is whitelisted
+ # Check if Binary is whitelisted
hashlist = get_lists.GetLists()
binary_hash = hashlist.binary_hash(project, full_path)
+
if is_binary(full_path) and not binary_list.search(full_path):
with open(full_path, 'rb') as afile:
buf = afile.read()
regex = value['regex']
desc = value['desc']
if re.search(regex, line) and not re.search(
- project_list, line):
+ ignore_list, line):
logger.error('File contains violation: %s',
full_path)
logger.error('Flagged Content: %s',
- network_settings.py
- deploy_settings.py
file_contents:
- - ^#
- - git clone.+\.openstack\.org
- - git clone.+gerrit\.opnfv\.org
- - wget.+build\.opnfv\.org
- - wget.+artifacts\.opnfv\.org
- - wget.+git\.opnfv.org
- - wget.+git\.openstack.org
- - git clone.+https:\/\/git.opendaylight\.org\/gerrit\/
- rpm-build wget libvirt
- wget git gcc
- def clean_ssh_keys\(key_file=\'\/root\/\.ssh\/authorized\_keys
file_audits:
file_names: [nullvalue]
file_contents:
- - ^.+#
- - git clone.+\.openstack\.org
- - git clone.+gerrit\.opnfv\.org
- - wget.+build\.opnfv\.org
- - wget.+artifacts\.opnfv\.org
- - wget.+git\.opnfv.org
- - wget.+git\.openstack.org
- - git clone.+https:\/\/git.opendaylight\.org\/gerrit\/
- curl.+\$local_env
- password.+salt.+opnfv_user_password
file_audits:
file_names: [nullvalue]
- file_contents:
- - ^#
- - git clone.+\.openstack\.org
- - git clone.+gerrit\.opnfv\.org
- - wget.+build\.opnfv\.org
- - wget.+artifacts\.opnfv\.org
- - wget.+git\.opnfv.org
- - wget.+git\.openstack.org
- - git clone.+https:\/\/git.opendaylight\.org\/gerrit\/
+ file_contents: [nullvalue]
file_audits:
file_names: [nullvalue]
- file_contents:
- - ^#
- - git clone.+\.openstack\.org
- - git clone.+gerrit\.opnfv\.org
- - wget.+build\.opnfv\.org
- - wget.+artifacts\.opnfv\.org
- - wget.+git\.opnfv.org
- - wget.+git\.openstack.org
- - git clone.+https:\/\/git.opendaylight\.org\/gerrit\/
+ file_contents: [nullvalue]
file_audits:
file_names: [nullvalue]
- file_contents:
- - ^#
- - git clone.+\.openstack\.org
- - git clone.+gerrit\.opnfv\.org
- - wget.+build\.opnfv\.org
- - wget.+artifacts\.opnfv\.org
- - wget.+git\.opnfv.org
- - wget.+git\.openstack.org
- - git clone.+https:\/\/git.opendaylight\.org\/gerrit\/
+ file_contents: [nullvalue]
file_audits:
file_names: [nullvalue]
- file_contents:
- - ^#
- - git clone.+\.openstack\.org
- - git clone.+gerrit\.opnfv\.org
- - wget.+build\.opnfv\.org
- - wget.+artifacts\.opnfv\.org
- - wget.+git\.opnfv.org
- - wget.+git\.openstack.org
- - git clone.+https:\/\/git.opendaylight\.org\/gerrit\/
+ file_contents: [nullvalue]
file_audits:
file_names: [nullvalue]
file_contents:
- - ^#
- - git clone.+\.openstack\.org
- - git clone.+gerrit\.opnfv\.org
- - wget.+build\.opnfv\.org
- - wget.+artifacts\.opnfv\.org
- - wget.+git\.opnfv.org
- - wget.+git\.openstack.org
- - git clone.+https:\/\/git.opendaylight\.org\/gerrit\/
- \.login-button#login-buttons-(.*)
- <input class="mdl-textfield__input" type="password" id="apiPassword">
- "password: { type: String }"
file_audits:
file_names: [nullvalue]
file_contents:
- - ^#
- - git clone.+\.openstack\.org
- - git clone.+gerrit\.opnfv\.org
- - wget.+build\.opnfv\.org
- - wget.+artifacts\.opnfv\.org
- - wget.+git\.opnfv.org
- - wget.+git\.openstack.org
- - git clone.+https:\/\/git.opendaylight\.org\/gerrit\/
- publicURL
- server_password
- username\,.password
file_audits:
file_names: [nullvalue]
- file_contents:
- - ^#
- - git clone.+\.openstack\.org
- - git clone.+gerrit\.opnfv\.org
- - wget.+build\.opnfv\.org
- - wget.+artifacts\.opnfv\.org
- - wget.+git\.opnfv.org
- - wget.+git\.openstack.org
- - git clone.+https:\/\/git.opendaylight\.org\/gerrit\/
+ file_contents: [nullvalue]
file_audits:
file_names: [nullvalue]
- file_contents:
- - ^#
- - git clone.+\.openstack\.org
- - git clone.+gerrit\.opnfv\.org
- - wget.+build\.opnfv\.org
- - wget.+artifacts\.opnfv\.org
- - wget.+git\.opnfv.org
- - wget.+git\.openstack.org
- - git clone.+https:\/\/git.opendaylight\.org\/gerrit\/
+ file_contents: [nullvalue]
file_audits:
file_names: [nullvalue]
- file_contents:
- - ^#
- - git clone.+\.openstack\.org
- - git clone.+gerrit\.opnfv\.org
- - wget.+build\.opnfv\.org
- - wget.+artifacts\.opnfv\.org
- - wget.+git\.opnfv.org
- - wget.+git\.openstack.org
- - git clone.+https:\/\/git.opendaylight\.org\/gerrit\/
+ file_contents: [nullvalue]
file_audits:
file_names: [nullvalue]
- file_contents:
- - ^#
- - git clone.+\.openstack\.org
- - git clone.+gerrit\.opnfv\.org
- - wget.+build\.opnfv\.org
- - wget.+artifacts\.opnfv\.org
- - wget.+git\.opnfv.org
- - wget.+git\.openstack.org
- - git clone.+https:\/\/git.opendaylight\.org\/gerrit\/
+ file_contents: [nullvalue]
file_audits:
file_names: [nullvalue]
- file_contents:
- - ^#
- - git clone.+\.openstack\.org
- - git clone.+gerrit\.opnfv\.org
- - wget.+build\.opnfv\.org
- - wget.+artifacts\.opnfv\.org
- - wget.+git\.opnfv.org
- - wget.+git\.openstack.org
- - git clone.+https:\/\/git.opendaylight\.org\/gerrit\/
+ file_contents: [nullvalue]
file_audits:
file_names: [nullvalue]
- file_contents:
- - ^#
- - git clone.+\.openstack\.org
- - git clone.+gerrit\.opnfv\.org
- - wget.+build\.opnfv\.org
- - wget.+artifacts\.opnfv\.org
- - wget.+git\.opnfv.org
- - wget.+git\.openstack.org
- - git clone.+https:\/\/git.opendaylight\.org\/gerrit\/
+ file_contents: [nullvalue]
file_audits:
file_names: [nullvalue]
- file_contents:
- - ^#
- - git clone.+\.openstack\.org
- - git clone.+gerrit\.opnfv\.org
- - wget.+build\.opnfv\.org
- - wget.+artifacts\.opnfv\.org
- - wget.+git\.opnfv.org
- - wget.+git\.openstack.org
- - git clone.+https:\/\/git.opendaylight\.org\/gerrit\/
+ file_contents: [nullvalue]
file_audits:
file_names: [nullvalue]
- file_contents:
- - ^#
- - git clone.+\.openstack\.org
- - git clone.+gerrit\.opnfv\.org
- - wget.+build\.opnfv\.org
- - wget.+artifacts\.opnfv\.org
- - wget.+git\.opnfv.org
- - wget.+git\.openstack.org
- - git clone.+https:\/\/git.opendaylight\.org\/gerrit\/
+ file_contents: [nullvalue]
file_audits:
file_names: [nullvalue]
- file_contents:
- - ^#
- - git clone.+\.openstack\.org
- - git clone.+gerrit\.opnfv\.org
- - wget.+build\.opnfv\.org
- - wget.+artifacts\.opnfv\.org
- - wget.+git\.opnfv.org
- - wget.+git\.openstack.org
- - git clone.+https:\/\/git.opendaylight\.org\/gerrit\/
+ file_contents: [nullvalue]
file_audits:
file_names: [nullvalue]
- file_contents:
- - ^#
- - git clone.+\.openstack\.org
- - git clone.+gerrit\.opnfv\.org
- - wget.+build\.opnfv\.org
- - wget.+artifacts\.opnfv\.org
- - wget.+git\.opnfv.org
- - wget.+git\.openstack.org
- - git clone.+https:\/\/git.opendaylight\.org\/gerrit\/
+ file_contents: [nullvalue]
file_audits:
file_names: [nullvalue]
- file_contents:
- - ^#
- - git clone.+\.openstack\.org
- - git clone.+gerrit\.opnfv\.org
- - wget.+build\.opnfv\.org
- - wget.+artifacts\.opnfv\.org
- - wget.+git\.opnfv.org
- - wget.+git\.openstack.org
- - git clone.+https:\/\/git.opendaylight\.org\/gerrit\/
+ file_contents: [nullvalue]
file_audits:
file_names: [nullvalue]
file_contents:
- - ^#
- - git clone.+\.openstack\.org
- - git clone.+gerrit\.opnfv\.org
- - wget.+build\.opnfv\.org
- - wget.+artifacts\.opnfv\.org
- - wget.+git\.opnfv.org
- - wget.+git\.openstack.org
- - git clone.+https:\/\/git.opendaylight\.org\/gerrit\/
- openssl-dev.libjpeg-turbo-dev.git.wget.&
- RUN.+curl.*https\:\/\/get\.rvm\.io.*bash.*stable
- grep.sed.wget.ca-certificates.git.\&&
- dca00ca0c823938e3fca1889ae366e86e6ce2279e4fc689b437d43978cfbe1c9
file_audits:
file_names: [nullvalue]
- file_contents:
- - ^#
- - git clone.+\.openstack\.org
- - git clone.+gerrit\.opnfv\.org
- - wget.+build\.opnfv\.org
- - wget.+artifacts\.opnfv\.org
- - wget.+git\.opnfv.org
- - wget.+git\.openstack.org
- - git clone.+https:\/\/git.opendaylight\.org\/gerrit\/
+ file_contents: [nullvalue]
binary_ignore: [nullvalue]
file_audits:
file_names: [nullvalue]
- file_contents:
- - ^#
- - git clone.+\.openstack\.org
- - git clone.+gerrit\.opnfv\.org
- - wget.+build\.opnfv\.org
- - wget.+artifacts\.opnfv\.org
- - wget.+git\.opnfv.org
- - wget.+git\.openstack.org
- - git clone.+https:\/\/git.opendaylight\.org\/gerrit\/
+ file_contents: [nullvalue]
file_audits:
file_names: [nullvalue]
file_contents:
- - ^#
- -s set secret key
- "PKG_MAP\\[wget\\]"
- "\\[wget\\]=wget"
- - "git clone(.*)\\.openstack\\.org"
- - "git clone(.*)gerrit\\.opnfv\\.org"
- "name: GIT_CLONE_BASE"
- "name: SSH_KEY"
- "packages = \\['parted', 'puppet', 'wget'"
- wget > /dev/null
- wget \$get_pip_url
- wget(.*)WORKSPACE/opnfv\.properties(.*)GS_URL(.*)properties
- - wget(.*)build\.opnfv\.org
- - wget.+git\.opnfv.org
- - wget(.*)artifacts\.opnfv\.org
file_audits:
file_names: [nullvalue]
- file_contents:
- - ^#
- - git clone.+\.openstack\.org
- - git clone.+gerrit\.opnfv\.org
- - wget.+build\.opnfv\.org
- - wget.+artifacts\.opnfv\.org
- - wget.+git\.opnfv.org
- - wget.+git\.openstack.org
- - git clone.+https:\/\/git.opendaylight\.org\/gerrit\/
+ file_contents: [nullvalue]
file_audits:
file_names: [nullvalue]
- file_contents:
- - ^#
- - git clone.+\.openstack\.org
- - git clone.+gerrit\.opnfv\.org
- - wget.+build\.opnfv\.org
- - wget.+artifacts\.opnfv\.org
- - wget.+git\.opnfv.org
- - wget.+git\.openstack.org
- - git clone.+https:\/\/git.opendaylight\.org\/gerrit\/
+ file_contents: [nullvalue]
file_audits:
file_names: [nullvalue]
- file_contents:
- - ^#
- - git clone.+\.openstack\.org
- - git clone.+gerrit\.opnfv\.org
- - wget.+build\.opnfv\.org
- - wget.+artifacts\.opnfv\.org
- - wget.+git\.opnfv.org
- - wget.+git\.openstack.org
- - git clone.+https:\/\/git.opendaylight\.org\/gerrit\/
+ file_contents: [nullvalue]
--- /dev/null
+---
+binaries:
+ binary_ignore: [nullvalue]
+
+file_audits:
+ file_names: [nullvalue]
+ file_contents:
+ - ^#
+ - git clone.+\.openstack\.org
+ - git clone.+gerrit\.opnfv\.org
+ - wget.+build\.opnfv\.org
+ - wget.+artifacts\.opnfv\.org
+ - wget.+git\.opnfv.org
+ - wget.+git\.openstack.org
+ - git clone.+https:\/\/git.opendaylight\.org\/gerrit\/