mirror of
https://sourceware.org/git/binutils-gdb.git
synced 2024-12-27 04:52:05 +08:00
178 lines
6.5 KiB
Python
Executable File
178 lines
6.5 KiB
Python
Executable File
#!/usr/bin/env python
|
|
|
|
# Copyright (C) 2016-2018 Free Software Foundation, Inc.
|
|
#
|
|
# This file is part of GDB.
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation; either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
|
|
# This program is used to analyze the test results (i.e., *.sum files)
|
|
# generated by GDB's testsuite, and print the testcases that are found
|
|
# to be racy.
|
|
#
|
|
# Racy testcases are considered as being testcases which can
|
|
# intermittently FAIL (or PASS) when run two or more times
|
|
# consecutively, i.e., tests whose results are not deterministic.
|
|
#
|
|
# This program is invoked when the user runs "make check" and
|
|
# specifies the RACY_ITER environment variable.
|
|
|
|
import sys
|
|
import os
|
|
import re
|
|
|
|
# The (global) dictionary that stores the associations between a *.sum
|
|
# file and its results. The data inside it will be stored as:
|
|
#
|
|
# files_and_tests = { 'file1.sum' : { 'PASS' : { 'test1', 'test2' ... },
|
|
# 'FAIL' : { 'test5', 'test6' ... },
|
|
# ...
|
|
# },
|
|
# { 'file2.sum' : { 'PASS' : { 'test1', 'test3' ... },
|
|
# ...
|
|
# }
|
|
# }
|
|
|
|
files_and_tests = dict ()
|
|
|
|
# The relatioships between various states of the same tests that
|
|
# should be ignored. For example, if the same test PASSes on a
|
|
# testcase run but KFAILs on another, this test should be considered
|
|
# racy because a known-failure is... known.
|
|
|
|
ignore_relations = { 'PASS' : 'KFAIL' }
|
|
|
|
# We are interested in lines that start with '.?(PASS|FAIL)'. In
|
|
# other words, we don't process errors (maybe we should).
|
|
|
|
sum_matcher = re.compile('^(.?(PASS|FAIL)): (.*)$')
|
|
|
|
def parse_sum_line (line, dic):
|
|
"""Parse a single LINE from a sumfile, and store the results in the
|
|
dictionary referenced by DIC."""
|
|
global sum_matcher
|
|
|
|
line = line.rstrip ()
|
|
m = re.match (sum_matcher, line)
|
|
if m:
|
|
result = m.group (1)
|
|
test_name = m.group (3)
|
|
# Remove tail parentheses. These are likely to be '(timeout)'
|
|
# and other extra information that will only confuse us.
|
|
test_name = re.sub ('(\s+)?\(.*$', '', test_name)
|
|
if result not in dic.keys ():
|
|
dic[result] = set ()
|
|
if test_name in dic[result]:
|
|
# If the line is already present in the dictionary, then
|
|
# we include a unique identifier in the end of it, in the
|
|
# form or '<<N>>' (where N is a number >= 2). This is
|
|
# useful because the GDB testsuite is full of non-unique
|
|
# test messages; however, if you process the racy summary
|
|
# file you will also need to perform this same operation
|
|
# in order to identify the racy test.
|
|
i = 2
|
|
while True:
|
|
nname = test_name + ' <<' + str (i) + '>>'
|
|
if nname not in dic[result]:
|
|
break
|
|
i += 1
|
|
test_name = nname
|
|
dic[result].add (test_name)
|
|
|
|
def read_sum_files (files):
|
|
"""Read the sumfiles (passed as a list in the FILES variable), and
|
|
process each one, filling the FILES_AND_TESTS global dictionary with
|
|
information about them. """
|
|
global files_and_tests
|
|
|
|
for x in files:
|
|
with open (x, 'r') as f:
|
|
files_and_tests[x] = dict ()
|
|
for line in f.readlines ():
|
|
parse_sum_line (line, files_and_tests[x])
|
|
|
|
def identify_racy_tests ():
|
|
"""Identify and print the racy tests. This function basically works
|
|
on sets, and the idea behind it is simple. It takes all the sets that
|
|
refer to the same result (for example, all the sets that contain PASS
|
|
tests), and compare them. If a test is present in all PASS sets, then
|
|
it is not racy. Otherwise, it is.
|
|
|
|
This function does that for all sets (PASS, FAIL, KPASS, KFAIL, etc.),
|
|
and then print a sorted list (without duplicates) of all the tests
|
|
that were found to be racy."""
|
|
global files_and_tests
|
|
|
|
# First, construct two dictionaries that will hold one set of
|
|
# testcases for each state (PASS, FAIL, etc.).
|
|
#
|
|
# Each set in NONRACY_TESTS will contain only the non-racy
|
|
# testcases for that state. A non-racy testcase is a testcase
|
|
# that has the same state in all test runs.
|
|
#
|
|
# Each set in ALL_TESTS will contain all tests, racy or not, for
|
|
# that state.
|
|
nonracy_tests = dict ()
|
|
all_tests = dict ()
|
|
for f in files_and_tests:
|
|
for state in files_and_tests[f]:
|
|
try:
|
|
nonracy_tests[state] &= files_and_tests[f][state].copy ()
|
|
except KeyError:
|
|
nonracy_tests[state] = files_and_tests[f][state].copy ()
|
|
|
|
try:
|
|
all_tests[state] |= files_and_tests[f][state].copy ()
|
|
except KeyError:
|
|
all_tests[state] = files_and_tests[f][state].copy ()
|
|
|
|
# Now, we eliminate the tests that are present in states that need
|
|
# to be ignored. For example, tests both in the PASS and KFAIL
|
|
# states should not be considered racy.
|
|
ignored_tests = set ()
|
|
for s1, s2 in ignore_relations.iteritems ():
|
|
try:
|
|
ignored_tests |= (all_tests[s1] & all_tests[s2])
|
|
except:
|
|
continue
|
|
|
|
racy_tests = set ()
|
|
for f in files_and_tests:
|
|
for state in files_and_tests[f]:
|
|
racy_tests |= files_and_tests[f][state] - nonracy_tests[state]
|
|
|
|
racy_tests = racy_tests - ignored_tests
|
|
|
|
# Print the header.
|
|
print "\t\t=== gdb racy tests ===\n"
|
|
|
|
# Print each test.
|
|
for line in sorted (racy_tests):
|
|
print line
|
|
|
|
# Print the summary.
|
|
print "\n"
|
|
print "\t\t=== gdb Summary ===\n"
|
|
print "# of racy tests:\t\t%d" % len (racy_tests)
|
|
|
|
if __name__ == '__main__':
|
|
if len (sys.argv) < 3:
|
|
# It only makes sense to invoke this program if you pass two
|
|
# or more files to be analyzed.
|
|
sys.exit ("Usage: %s [FILE] [FILE] ..." % sys.argv[0])
|
|
read_sum_files (sys.argv[1:])
|
|
identify_racy_tests ()
|
|
exit (0)
|