Adding packet forward test cases involving multiple VMs
[kvmfornfv.git] / kernel / scripts / checkkconfigsymbols.py
1 #!/usr/bin/env python2
2
3 """Find Kconfig symbols that are referenced but not defined."""
4
5 # (c) 2014-2015 Valentin Rothberg <valentinrothberg@gmail.com>
6 # (c) 2014 Stefan Hengelein <stefan.hengelein@fau.de>
7 #
8 # Licensed under the terms of the GNU GPL License version 2
9
10
11 import os
12 import re
13 import sys
14 from subprocess import Popen, PIPE, STDOUT
15 from optparse import OptionParser
16
17
18 # regex expressions
19 OPERATORS = r"&|\(|\)|\||\!"
20 FEATURE = r"(?:\w*[A-Z0-9]\w*){2,}"
21 DEF = r"^\s*(?:menu){,1}config\s+(" + FEATURE + r")\s*"
22 EXPR = r"(?:" + OPERATORS + r"|\s|" + FEATURE + r")+"
23 DEFAULT = r"default\s+.*?(?:if\s.+){,1}"
24 STMT = r"^\s*(?:if|select|depends\s+on|(?:" + DEFAULT + r"))\s+" + EXPR
25 SOURCE_FEATURE = r"(?:\W|\b)+[D]{,1}CONFIG_(" + FEATURE + r")"
26
27 # regex objects
28 REGEX_FILE_KCONFIG = re.compile(r".*Kconfig[\.\w+\-]*$")
29 REGEX_FEATURE = re.compile(r'(?!\B"[^"]*)' + FEATURE + r'(?![^"]*"\B)')
30 REGEX_SOURCE_FEATURE = re.compile(SOURCE_FEATURE)
31 REGEX_KCONFIG_DEF = re.compile(DEF)
32 REGEX_KCONFIG_EXPR = re.compile(EXPR)
33 REGEX_KCONFIG_STMT = re.compile(STMT)
34 REGEX_KCONFIG_HELP = re.compile(r"^\s+(help|---help---)\s*$")
35 REGEX_FILTER_FEATURES = re.compile(r"[A-Za-z0-9]$")
36 REGEX_NUMERIC = re.compile(r"0[xX][0-9a-fA-F]+|[0-9]+")
37
38
39 def parse_options():
40     """The user interface of this module."""
41     usage = "%prog [options]\n\n"                                              \
42             "Run this tool to detect Kconfig symbols that are referenced but " \
43             "not defined in\nKconfig.  The output of this tool has the "       \
44             "format \'Undefined symbol\\tFile list\'\n\n"                      \
45             "If no option is specified, %prog will default to check your\n"    \
46             "current tree.  Please note that specifying commits will "         \
47             "\'git reset --hard\'\nyour current tree!  You may save "          \
48             "uncommitted changes to avoid losing data."
49
50     parser = OptionParser(usage=usage)
51
52     parser.add_option('-c', '--commit', dest='commit', action='store',
53                       default="",
54                       help="Check if the specified commit (hash) introduces "
55                            "undefined Kconfig symbols.")
56
57     parser.add_option('-d', '--diff', dest='diff', action='store',
58                       default="",
59                       help="Diff undefined symbols between two commits.  The "
60                            "input format bases on Git log's "
61                            "\'commmit1..commit2\'.")
62
63     parser.add_option('-f', '--find', dest='find', action='store_true',
64                       default=False,
65                       help="Find and show commits that may cause symbols to be "
66                            "missing.  Required to run with --diff.")
67
68     parser.add_option('-i', '--ignore', dest='ignore', action='store',
69                       default="",
70                       help="Ignore files matching this pattern.  Note that "
71                            "the pattern needs to be a Python regex.  To "
72                            "ignore defconfigs, specify -i '.*defconfig'.")
73
74     parser.add_option('', '--force', dest='force', action='store_true',
75                       default=False,
76                       help="Reset current Git tree even when it's dirty.")
77
78     (opts, _) = parser.parse_args()
79
80     if opts.commit and opts.diff:
81         sys.exit("Please specify only one option at once.")
82
83     if opts.diff and not re.match(r"^[\w\-\.]+\.\.[\w\-\.]+$", opts.diff):
84         sys.exit("Please specify valid input in the following format: "
85                  "\'commmit1..commit2\'")
86
87     if opts.commit or opts.diff:
88         if not opts.force and tree_is_dirty():
89             sys.exit("The current Git tree is dirty (see 'git status').  "
90                      "Running this script may\ndelete important data since it "
91                      "calls 'git reset --hard' for some performance\nreasons. "
92                      " Please run this script in a clean Git tree or pass "
93                      "'--force' if you\nwant to ignore this warning and "
94                      "continue.")
95
96     if opts.commit:
97         opts.find = False
98
99     if opts.ignore:
100         try:
101             re.match(opts.ignore, "this/is/just/a/test.c")
102         except:
103             sys.exit("Please specify a valid Python regex.")
104
105     return opts
106
107
108 def main():
109     """Main function of this module."""
110     opts = parse_options()
111
112     if opts.commit or opts.diff:
113         head = get_head()
114
115         # get commit range
116         commit_a = None
117         commit_b = None
118         if opts.commit:
119             commit_a = opts.commit + "~"
120             commit_b = opts.commit
121         elif opts.diff:
122             split = opts.diff.split("..")
123             commit_a = split[0]
124             commit_b = split[1]
125             undefined_a = {}
126             undefined_b = {}
127
128         # get undefined items before the commit
129         execute("git reset --hard %s" % commit_a)
130         undefined_a = check_symbols(opts.ignore)
131
132         # get undefined items for the commit
133         execute("git reset --hard %s" % commit_b)
134         undefined_b = check_symbols(opts.ignore)
135
136         # report cases that are present for the commit but not before
137         for feature in sorted(undefined_b):
138             # feature has not been undefined before
139             if not feature in undefined_a:
140                 files = sorted(undefined_b.get(feature))
141                 print "%s\t%s" % (yel(feature), ", ".join(files))
142                 if opts.find:
143                     commits = find_commits(feature, opts.diff)
144                     print red(commits)
145             # check if there are new files that reference the undefined feature
146             else:
147                 files = sorted(undefined_b.get(feature) -
148                                undefined_a.get(feature))
149                 if files:
150                     print "%s\t%s" % (yel(feature), ", ".join(files))
151                     if opts.find:
152                         commits = find_commits(feature, opts.diff)
153                         print red(commits)
154
155         # reset to head
156         execute("git reset --hard %s" % head)
157
158     # default to check the entire tree
159     else:
160         undefined = check_symbols(opts.ignore)
161         for feature in sorted(undefined):
162             files = sorted(undefined.get(feature))
163             print "%s\t%s" % (yel(feature), ", ".join(files))
164
165
166 def yel(string):
167     """
168     Color %string yellow.
169     """
170     return "\033[33m%s\033[0m" % string
171
172
173 def red(string):
174     """
175     Color %string red.
176     """
177     return "\033[31m%s\033[0m" % string
178
179
180 def execute(cmd):
181     """Execute %cmd and return stdout.  Exit in case of error."""
182     pop = Popen(cmd, stdout=PIPE, stderr=STDOUT, shell=True)
183     (stdout, _) = pop.communicate()  # wait until finished
184     if pop.returncode != 0:
185         sys.exit(stdout)
186     return stdout
187
188
189 def find_commits(symbol, diff):
190     """Find commits changing %symbol in the given range of %diff."""
191     commits = execute("git log --pretty=oneline --abbrev-commit -G %s %s"
192                       % (symbol, diff))
193     return commits
194
195
196 def tree_is_dirty():
197     """Return true if the current working tree is dirty (i.e., if any file has
198     been added, deleted, modified, renamed or copied but not committed)."""
199     stdout = execute("git status --porcelain")
200     for line in stdout:
201         if re.findall(r"[URMADC]{1}", line[:2]):
202             return True
203     return False
204
205
206 def get_head():
207     """Return commit hash of current HEAD."""
208     stdout = execute("git rev-parse HEAD")
209     return stdout.strip('\n')
210
211
212 def check_symbols(ignore):
213     """Find undefined Kconfig symbols and return a dict with the symbol as key
214     and a list of referencing files as value.  Files matching %ignore are not
215     checked for undefined symbols."""
216     source_files = []
217     kconfig_files = []
218     defined_features = set()
219     referenced_features = dict()  # {feature: [files]}
220
221     # use 'git ls-files' to get the worklist
222     stdout = execute("git ls-files")
223     if len(stdout) > 0 and stdout[-1] == "\n":
224         stdout = stdout[:-1]
225
226     for gitfile in stdout.rsplit("\n"):
227         if ".git" in gitfile or "ChangeLog" in gitfile or      \
228                 ".log" in gitfile or os.path.isdir(gitfile) or \
229                 gitfile.startswith("tools/"):
230             continue
231         if REGEX_FILE_KCONFIG.match(gitfile):
232             kconfig_files.append(gitfile)
233         else:
234             # all non-Kconfig files are checked for consistency
235             source_files.append(gitfile)
236
237     for sfile in source_files:
238         if ignore and re.match(ignore, sfile):
239             # do not check files matching %ignore
240             continue
241         parse_source_file(sfile, referenced_features)
242
243     for kfile in kconfig_files:
244         if ignore and re.match(ignore, kfile):
245             # do not collect references for files matching %ignore
246             parse_kconfig_file(kfile, defined_features, dict())
247         else:
248             parse_kconfig_file(kfile, defined_features, referenced_features)
249
250     undefined = {}  # {feature: [files]}
251     for feature in sorted(referenced_features):
252         # filter some false positives
253         if feature == "FOO" or feature == "BAR" or \
254                 feature == "FOO_BAR" or feature == "XXX":
255             continue
256         if feature not in defined_features:
257             if feature.endswith("_MODULE"):
258                 # avoid false positives for kernel modules
259                 if feature[:-len("_MODULE")] in defined_features:
260                     continue
261             undefined[feature] = referenced_features.get(feature)
262     return undefined
263
264
265 def parse_source_file(sfile, referenced_features):
266     """Parse @sfile for referenced Kconfig features."""
267     lines = []
268     with open(sfile, "r") as stream:
269         lines = stream.readlines()
270
271     for line in lines:
272         if not "CONFIG_" in line:
273             continue
274         features = REGEX_SOURCE_FEATURE.findall(line)
275         for feature in features:
276             if not REGEX_FILTER_FEATURES.search(feature):
277                 continue
278             sfiles = referenced_features.get(feature, set())
279             sfiles.add(sfile)
280             referenced_features[feature] = sfiles
281
282
283 def get_features_in_line(line):
284     """Return mentioned Kconfig features in @line."""
285     return REGEX_FEATURE.findall(line)
286
287
288 def parse_kconfig_file(kfile, defined_features, referenced_features):
289     """Parse @kfile and update feature definitions and references."""
290     lines = []
291     skip = False
292
293     with open(kfile, "r") as stream:
294         lines = stream.readlines()
295
296     for i in range(len(lines)):
297         line = lines[i]
298         line = line.strip('\n')
299         line = line.split("#")[0]  # ignore comments
300
301         if REGEX_KCONFIG_DEF.match(line):
302             feature_def = REGEX_KCONFIG_DEF.findall(line)
303             defined_features.add(feature_def[0])
304             skip = False
305         elif REGEX_KCONFIG_HELP.match(line):
306             skip = True
307         elif skip:
308             # ignore content of help messages
309             pass
310         elif REGEX_KCONFIG_STMT.match(line):
311             features = get_features_in_line(line)
312             # multi-line statements
313             while line.endswith("\\"):
314                 i += 1
315                 line = lines[i]
316                 line = line.strip('\n')
317                 features.extend(get_features_in_line(line))
318             for feature in set(features):
319                 if REGEX_NUMERIC.match(feature):
320                     # ignore numeric values
321                     continue
322                 paths = referenced_features.get(feature, set())
323                 paths.add(kfile)
324                 referenced_features[feature] = paths
325
326
327 if __name__ == "__main__":
328     main()