firmware_TouchMTB: print metrics with missing values
[chromiumos/third_party/autotest.git] / client / site_tests / platform_ToolchainOptions / platform_ToolchainOptions.py
1 # Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file.
4
5 import os
6
7 import logging
8 from autotest_lib.client.bin import test, utils
9 from autotest_lib.client.common_lib import error
10 from optparse import OptionParser
11
12 FILE_CMD="file -m /usr/local/share/misc/magic.mgc"
13
14 class ToolchainOptionSet:
15     def __init__(self, description, bad_files, whitelist_file):
16         self.description = description
17         self.bad_set = set(bad_files.splitlines())
18         self.whitelist_set = set([])
19         self.process_whitelist_with_private(whitelist_file)
20
21
22     def process_whitelist_with_private(self, whitelist_file):
23         whitelist_files = [whitelist_file]
24         private_file = os.path.join(os.path.dirname(whitelist_file),
25                                     "private_" +
26                                     os.path.basename(whitelist_file))
27         whitelist_files.append(private_file)
28         self.process_whitelists(whitelist_files)
29
30
31     def process_whitelist(self, whitelist_file):
32         if not os.path.isfile(whitelist_file):
33             self.whitelist_set = self.whitelist_set.union(set([]))
34         else:
35             f = open(whitelist_file)
36             whitelist = [x for x in f.read().splitlines()
37                                     if not x.startswith('#')]
38             f.close()
39             self.whitelist_set = self.whitelist_set.union(set(whitelist))
40         self.filtered_set = self.bad_set.difference(self.whitelist_set)
41         self.new_passes = self.whitelist_set.difference(self.bad_set)
42
43
44     def process_whitelists(self, whitelist_files):
45         for whitelist_file in whitelist_files:
46             self.process_whitelist(whitelist_file)
47
48
49     def get_fail_summary_message(self):
50         m = "Test %s " % self.description
51         m += "%d failures\n" % len(self.filtered_set)
52         return m
53
54
55     def get_fail_message(self):
56         m = self.get_fail_summary_message()
57         sorted_list = list(self.filtered_set)
58         sorted_list.sort()
59         m += "FAILED:\n%s\n\n" % "\n".join(sorted_list)
60         return m
61
62
63     def __str__(self):
64         m = "Test %s " % self.description
65         m += ("%d failures, %d in whitelist, %d in filtered, %d new passes " %
66               (len(self.bad_set),
67                len(self.whitelist_set),
68                len(self.filtered_set),
69                len(self.new_passes)))
70
71         if len(self.filtered_set):
72             sorted_list = list(self.filtered_set)
73             sorted_list.sort()
74             m += "FAILED:\n%s" % "\n".join(sorted_list)
75         else:
76             m += "PASSED!"
77
78         if len(self.new_passes):
79             sorted_list = list(self.new_passes)
80             sorted_list.sort()
81             m += ("\nNew passes (remove these from the whitelist):\n%s" %
82                   "\n".join(sorted_list))
83         logging.debug(m)
84         return m
85
86
87 class platform_ToolchainOptions(test.test):
88     version = 2
89
90     def get_cmd(self, test_cmd, find_options=""):
91         base_cmd = ("find '%s' -wholename %s -prune -o "
92                     " -wholename /proc -prune -o "
93                     " -wholename /dev -prune -o "
94                     " -wholename /sys -prune -o "
95                     " -wholename /mnt/stateful_partition -prune -o "
96                     " -wholename /usr/local -prune -o "
97                     # There are files in /home/chronos that cause false
98                     # positives, and since that's noexec anyways, it should
99                     # be skipped.
100                     " -wholename '/home/chronos' -prune -o "
101                     " %s "
102                     " -not -name 'libstdc++.so.*' "
103                     " -not -name 'libgcc_s.so.*' "
104                     " -type f -executable -exec "
105                     "sh -c '%s "
106                     "{} | grep -q ELF && "
107                     "(%s || echo {})' ';'")
108         rootdir = "/"
109         cmd = base_cmd % (rootdir, self.autodir, find_options, FILE_CMD,
110                           test_cmd)
111         return cmd
112
113
114     # http://build.chromium.org/mirror/chromiumos/mirror/distfiles/
115     # binutils-2.19.1.tar.bz2
116     def setup(self, tarball="binutils-2.19.1.tar.bz2"):
117         # clean
118         if os.path.exists(self.srcdir):
119             utils.system("rm -rf %s" % self.srcdir)
120
121         tarball = utils.unmap_url(self.bindir, tarball, self.tmpdir)
122         utils.extract_tarball_to_dir(tarball, self.srcdir)
123
124         os.chdir(self.srcdir)
125         utils.system("patch -p1 < ../binutils-2.19-arm.patch");
126         utils.configure()
127         utils.make(extra="CFLAGS+=\"-w\"")
128
129
130     def create_and_filter(self, description, cmd, whitelist_file,
131                           find_options=""):
132         full_cmd = self.get_cmd(cmd, find_options)
133         bad_files = utils.system_output(full_cmd)
134         cso = ToolchainOptionSet(description, bad_files, whitelist_file)
135         cso.process_whitelist_with_private(whitelist_file)
136         return cso
137
138
139     def run_once(self, rootdir="/", args=[]):
140         """
141         Do a find for all the ELF files on the system.
142         For each one, test for compiler options that should have been used
143         when compiling the file.
144
145         For missing compiler options, print the files.
146         """
147
148         parser = OptionParser()
149         parser.add_option('--hardfp',
150                           dest='enable_hardfp',
151                           default=False,
152                           action='store_true',
153                           help='Whether to check for hardfp binaries.')
154         (options, args) = parser.parse_args(args)
155
156         option_sets = []
157
158         libc_glob = "/lib/libc-[0-9]*"
159         os.chdir(self.srcdir)
160
161         # We do not test binaries if they are built with Address Sanitizer
162         # because it is a separate testing tool.
163         no_asan_used = utils.system_output("binutils/readelf -s "
164                                            "/opt/google/chrome/chrome | "
165                                            "egrep -q \"__asan_init\" || "
166                                            "echo no ASAN")
167         if not no_asan_used:
168           logging.debug("ASAN detected on /opt/google/chrome/chrome. "
169                         "Will skip all checks.")
170           return
171
172         # Check that gold was used to build binaries.
173         gold_cmd = ("binutils/readelf -S {} 2>&1 | "
174                     "egrep -q \".note.gnu.gold-ve\"")
175         gold_find_options = ""
176         if utils.get_cpu_arch() == "arm":
177           # gold is only enabled for Chrome on arm.
178           gold_find_options = "-path \"/opt/google/chrome/chrome\""
179         gold_whitelist = os.path.join(self.bindir, "gold_whitelist")
180         option_sets.append(self.create_and_filter("gold",
181                                                   gold_cmd,
182                                                   gold_whitelist,
183                                                   gold_find_options))
184
185         # ARM arch doesn't have hardened.
186         if utils.get_cpu_arch() != "arm":
187             # Verify non-static binaries have BIND_NOW in dynamic section.
188             now_cmd = ("(%s {} | grep -q statically) ||"
189                        "binutils/readelf -d {} 2>&1 | "
190                        "egrep -q \"BIND_NOW\"" % FILE_CMD)
191             now_whitelist = os.path.join(self.bindir, "now_whitelist")
192             option_sets.append(self.create_and_filter("-Wl,-z,now",
193                                                       now_cmd,
194                                                       now_whitelist))
195
196             # Verify non-static binaries have RELRO program header.
197             relro_cmd = ("(%s {} | grep -q statically) ||"
198                          "binutils/readelf -l {} 2>&1 | "
199                          "egrep -q \"GNU_RELRO\"" % FILE_CMD)
200             relro_whitelist = os.path.join(self.bindir, "relro_whitelist")
201             option_sets.append(self.create_and_filter("-Wl,-z,relro",
202                                                       relro_cmd,
203                                                       relro_whitelist))
204
205             # Verify non-static binaries are dynamic (built PIE).
206             pie_cmd = ("(%s {} | grep -q statically) ||"
207                        "binutils/readelf -l {} 2>&1 | "
208                        "egrep -q \"Elf file type is DYN\"" % FILE_CMD)
209             pie_whitelist = os.path.join(self.bindir, "pie_whitelist")
210             option_sets.append(self.create_and_filter("-fPIE",
211                                                       pie_cmd,
212                                                       pie_whitelist))
213
214             # Verify all binaries have non-exec STACK program header.
215             stack_cmd = ("binutils/readelf -lW {} 2>&1 | "
216                          "egrep -q \"GNU_STACK.*RW \"")
217             stack_whitelist = os.path.join(self.bindir, "stack_whitelist")
218             option_sets.append(self.create_and_filter("Executable Stack",
219                                                       stack_cmd,
220                                                       stack_whitelist))
221
222         if (options.enable_hardfp and utils.get_cpu_arch() == 'arm'):
223             hardfp_cmd = ("binutils/readelf -A {} 2>&1 | "
224                           "egrep -q \"Tag_ABI_VFP_args: VFP registers\"")
225             hardfp_whitelist = os.path.join(self.bindir, "hardfp_whitelist")
226             option_sets.append(self.create_and_filter("hardfp", hardfp_cmd,
227                                                       hardfp_whitelist))
228
229         fail_msg = ""
230
231         # There is currently no way to clear binary prebuilts for all devs.
232         # Thus, when a new check is added to this test, the test might fail
233         # for users who have old prebuilts which have not been compiled
234         # in the correct manner. Warn the user that if a test fails,
235         # they might have to clear their prebuilts to make it pass.
236         fail_summary_msg = "The following tests failed. If you expected " \
237                            "the test to pass you may have stale binary " \
238                            "prebuilts which are causing the failure. Try " \
239                            "clearing binary prebuilts and rebuilding by " \
240                            " running: ./setup_board --board=... --force\n\n"
241         full_msg = "Test results:"
242         num_fails = 0
243         for cos in option_sets:
244             if len(cos.filtered_set):
245                 num_fails += 1
246                 fail_msg += cos.get_fail_message() + "\n"
247                 fail_summary_msg += cos.get_fail_summary_message() + "\n"
248             full_msg += str(cos) + "\n\n"
249
250         logging.error(fail_msg)
251         logging.debug(full_msg)
252         if num_fails:
253             raise error.TestFail(fail_summary_msg)