]>
gcc.gnu.org Git - gcc.git/blob - contrib/dg-extract-results.py
3 # Copyright (C) 2014 Free Software Foundation, Inc.
5 # This script is free software; you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License as published by
7 # the Free Software Foundation; either version 3, or (at your option)
13 from datetime
import datetime
15 # True if unrecognised lines should cause a fatal error. Might want to turn
16 # this on by default later.
19 # True if the order of .log segments should match the .sum file, false if
20 # they should keep the original order.
24 def __init__ (self
, name
):
27 def __cmp__ (self
, other
):
28 return cmp (self
.name
, other
.name
)
30 class ToolRun (Named
):
31 def __init__ (self
, name
):
32 Named
.__init
__ (self
, name
)
33 # The variations run for this tool, mapped by --target_board name.
34 self
.variations
= dict()
36 # Return the VariationRun for variation NAME.
37 def get_variation (self
, name
):
38 if name
not in self
.variations
:
39 self
.variations
[name
] = VariationRun (name
)
40 return self
.variations
[name
]
42 class VariationRun (Named
):
43 def __init__ (self
, name
):
44 Named
.__init
__ (self
, name
)
45 # A segment of text before the harness runs start, describing which
46 # baseboard files were loaded for the target.
48 # The harnesses run for this variation, mapped by filename.
49 self
.harnesses
= dict()
50 # A list giving the number of times each type of result has
54 # Return the HarnessRun for harness NAME.
55 def get_harness (self
, name
):
56 if name
not in self
.harnesses
:
57 self
.harnesses
[name
] = HarnessRun (name
)
58 return self
.harnesses
[name
]
60 class HarnessRun (Named
):
61 def __init__ (self
, name
):
62 Named
.__init
__ (self
, name
)
63 # Segments of text that make up the harness run, mapped by a test-based
64 # key that can be used to order them.
65 self
.segments
= dict()
66 # Segments of text that make up the harness run but which have
67 # no recognized test results. These are typically harnesses that
68 # are completely skipped for the target.
70 # A list of results. Each entry is a pair in which the first element
71 # is a unique sorting key and in which the second is the full
75 # Add a segment of text to the harness run. If the segment includes
76 # test results, KEY is an example of one of them, and can be used to
77 # combine the individual segments in order. If the segment has no
78 # test results (e.g. because the harness doesn't do anything for the
79 # current configuration) then KEY is None instead. In that case
80 # just collect the segments in the order that we see them.
81 def add_segment (self
, key
, segment
):
83 assert key
not in self
.segments
84 self
.segments
[key
] = segment
86 self
.empty
.append (segment
)
89 def __init__ (self
, filename
, start
):
90 self
.filename
= filename
96 # The variations specified on the command line.
98 # The variations seen in the input files.
99 self
.known_variations
= set()
100 # The tools specified on the command line.
102 # Whether to create .sum rather than .log output.
104 # Regexps used while parsing.
105 self
.test_run_re
= re
.compile (r
'^Test Run By (\S+) on (.*)$')
106 self
.tool_re
= re
.compile (r
'^\t\t=== (.*) tests ===$')
107 self
.result_re
= re
.compile (r
'^(PASS|XPASS|FAIL|XFAIL|UNRESOLVED'
108 r
'|WARNING|ERROR|UNSUPPORTED|UNTESTED'
110 self
.completed_re
= re
.compile (r
'.* completed at (.*)')
111 # Pieces of text to write at the head of the output.
112 # start_line is a pair in which the first element is a datetime
113 # and in which the second is the associated 'Test Run By' line.
114 self
.start_line
= None
115 self
.native_line
= ''
116 self
.target_line
= ''
118 self
.acats_premable
= ''
119 # Pieces of text to write at the end of the output.
120 # end_line is like start_line but for the 'runtest completed' line.
121 self
.acats_failures
= []
122 self
.version_output
= ''
124 # Known summary types.
126 '# of expected passes\t\t',
127 '# of unexpected failures\t',
128 '# of unexpected successes\t',
129 '# of expected failures\t\t',
130 '# of unknown successes\t\t',
131 '# of known failures\t\t',
132 '# of untested testcases\t\t',
133 '# of unresolved testcases\t',
134 '# of unsupported tests\t\t'
140 sys
.stderr
.write ('Usage: ' + name
141 + ''' [-t tool] [-l variant-list] [-L] log-or-sum-file ...
143 tool The tool (e.g. g++, libffi) for which to create a
144 new test summary file. If not specified then output
145 is created for all tools.
146 variant-list One or more test variant names. If the list is
147 not specified then one is constructed from all
148 variants in the files for <tool>.
149 sum-file A test summary file with the format of those
150 created by runtest from DejaGnu.
151 If -L is used, merge *.log files instead of *.sum. In this
152 mode the exact order of lines may not be preserved, just different
153 Running *.exp chunks should be in correct order.
157 def fatal (self
, what
, string
):
160 sys
.stderr
.write (what
+ ': ' + string
+ '\n')
163 # Parse the command-line arguments.
164 def parse_cmdline (self
):
166 (options
, self
.files
) = getopt
.getopt (sys
.argv
[1:], 'l:t:L')
167 if len (self
.files
) == 0:
169 for (option
, value
) in options
:
171 self
.variations
.append (value
)
173 self
.tools
.append (value
)
176 except getopt
.GetoptError
as e
:
177 self
.fatal (None, e
.msg
)
179 # Try to parse time string TIME, returning an arbitrary time on failure.
180 # Getting this right is just a nice-to-have so failures should be silent.
181 def parse_time (self
, time
):
183 return datetime
.strptime (time
, '%c')
185 return datetime
.now()
187 # Parse an integer and abort on failure.
188 def parse_int (self
, filename
, value
):
192 self
.fatal (filename
, 'expected an integer, got: ' + value
)
194 # Return a list that represents no test results.
195 def zero_counts (self
):
196 return [0 for x
in self
.count_names
]
198 # Return the ToolRun for tool NAME.
199 def get_tool (self
, name
):
200 if name
not in self
.runs
:
201 self
.runs
[name
] = ToolRun (name
)
202 return self
.runs
[name
]
204 # Add the result counts in list FROMC to TOC.
205 def accumulate_counts (self
, toc
, fromc
):
206 for i
in range (len (self
.count_names
)):
209 # Parse the list of variations after 'Schedule of variations:'.
210 # Return the number seen.
211 def parse_variations (self
, filename
, file):
214 line
= file.readline()
216 self
.fatal (filename
, 'could not parse variation list')
219 self
.known_variations
.add (line
.strip())
221 return num_variations
223 # Parse from the first line after 'Running target ...' to the end
224 # of the run's summary.
225 def parse_run (self
, filename
, file, tool
, variation
, num_variations
):
231 # If this is the first run for this variation, add any text before
232 # the first harness to the header.
233 if not variation
.header
:
234 segment
= Segment (filename
, file.tell())
235 variation
.header
= segment
237 # Parse up until the first line of the summary.
238 if num_variations
== 1:
239 end
= '\t\t=== ' + tool
.name
+ ' Summary ===\n'
241 end
= ('\t\t=== ' + tool
.name
+ ' Summary for '
242 + variation
.name
+ ' ===\n')
244 line
= file.readline()
246 self
.fatal (filename
, 'no recognised summary line')
250 # Look for the start of a new harness.
251 if line
.startswith ('Running ') and line
.endswith (' ...\n'):
252 # Close off the current harness segment, if any.
254 segment
.lines
-= final_using
255 harness
.add_segment (first_key
, segment
)
256 name
= line
[len ('Running '):-len(' ...\n')]
257 harness
= variation
.get_harness (name
)
258 segment
= Segment (filename
, file.tell())
263 # Record test results. Associate the first test result with
264 # the harness segment, so that if a run for a particular harness
265 # has been split up, we can reassemble the individual segments
266 # in a sensible order.
267 match
= self
.result_re
.match (line
)
270 self
.fatal (filename
, 'saw test result before harness name')
271 name
= match
.group (2)
272 # Ugly hack to get the right order for gfortran.
273 if name
.startswith ('gfortran.dg/g77/'):
275 key
= (name
, len (harness
.results
))
276 harness
.results
.append ((key
, line
))
277 if not first_key
and sort_logs
:
280 # 'Using ...' lines are only interesting in a header. Splitting
281 # the test up into parallel runs leads to more 'Using ...' lines
282 # than there would be in a single log.
283 if line
.startswith ('Using '):
288 # Add other text to the current segment, if any.
292 # Close off the final harness segment, if any.
294 segment
.lines
-= final_using
295 harness
.add_segment (first_key
, segment
)
297 # Parse the rest of the summary (the '# of ' lines).
298 if len (variation
.counts
) == 0:
299 variation
.counts
= self
.zero_counts()
302 line
= file.readline()
307 if not line
.startswith ('# '):
311 for i
in range (len (self
.count_names
)):
312 if line
.startswith (self
.count_names
[i
]):
313 count
= line
[len (self
.count_names
[i
]):-1].strip()
314 variation
.counts
[i
] += self
.parse_int (filename
, count
)
318 self
.fatal (filename
, 'unknown test result: ' + line
[:-1])
320 # Parse an acats run, which uses a different format from dejagnu.
321 # We have just skipped over '=== acats configuration ==='.
322 def parse_acats_run (self
, filename
, file):
323 # Parse the preamble, which describes the configuration and logs
324 # the creation of support files.
325 record
= (self
.acats_premable
== '')
327 self
.acats_premable
= '\t\t=== acats configuration ===\n'
329 line
= file.readline()
331 self
.fatal (filename
, 'could not parse acats preamble')
332 if line
== '\t\t=== acats tests ===\n':
335 self
.acats_premable
+= line
337 # Parse the test results themselves, using a dummy variation name.
338 tool
= self
.get_tool ('acats')
339 variation
= tool
.get_variation ('none')
340 self
.parse_run (filename
, file, tool
, variation
, 1)
342 # Parse the failure list.
345 line
= file.readline()
346 if line
.startswith ('*** FAILURES: '):
347 self
.acats_failures
.append (line
[len ('*** FAILURES: '):-1])
352 # Parse the final summary at the end of a log in order to capture
353 # the version output that follows it.
354 def parse_final_summary (self
, filename
, file):
355 record
= (self
.version_output
== '')
357 line
= file.readline()
360 if line
.startswith ('# of '):
363 self
.version_output
+= line
367 # Parse a .log or .sum file.
368 def parse_file (self
, filename
, file):
373 line
= file.readline()
377 # Parse the list of variations, which comes before the test
379 if line
.startswith ('Schedule of variations:'):
380 num_variations
= self
.parse_variations (filename
, file)
383 # Parse a testsuite run for one tool/variation combination.
384 if line
.startswith ('Running target '):
385 name
= line
[len ('Running target '):-1]
387 self
.fatal (filename
, 'could not parse tool name')
388 if name
not in self
.known_variations
:
389 self
.fatal (filename
, 'unknown target: ' + name
)
390 self
.parse_run (filename
, file, tool
,
391 tool
.get_variation (name
),
393 # If there is only one variation then there is no separate
394 # summary for it. Record any following version output.
395 if num_variations
== 1:
396 self
.parse_final_summary (filename
, file)
399 # Parse the start line. In the case where several files are being
400 # parsed, pick the one with the earliest time.
401 match
= self
.test_run_re
.match (line
)
403 time
= self
.parse_time (match
.group (2))
404 if not self
.start_line
or self
.start_line
[0] > time
:
405 self
.start_line
= (time
, line
)
408 # Parse the form used for native testing.
409 if line
.startswith ('Native configuration is '):
410 self
.native_line
= line
413 # Parse the target triplet.
414 if line
.startswith ('Target is '):
415 self
.target_line
= line
418 # Parse the host triplet.
419 if line
.startswith ('Host is '):
420 self
.host_line
= line
423 # Parse the acats premable.
424 if line
== '\t\t=== acats configuration ===\n':
425 self
.parse_acats_run (filename
, file)
428 # Parse the tool name.
429 match
= self
.tool_re
.match (line
)
431 tool
= self
.get_tool (match
.group (1))
434 # Skip over the final summary (which we instead create from
435 # individual runs) and parse the version output.
436 if tool
and line
== '\t\t=== ' + tool
.name
+ ' Summary ===\n':
437 if file.readline() != '\n':
438 self
.fatal (filename
, 'expected blank line after summary')
439 self
.parse_final_summary (filename
, file)
442 # Parse the completion line. In the case where several files
443 # are being parsed, pick the one with the latest time.
444 match
= self
.completed_re
.match (line
)
446 time
= self
.parse_time (match
.group (1))
447 if not self
.end_line
or self
.end_line
[0] < time
:
448 self
.end_line
= (time
, line
)
451 # Sanity check to make sure that important text doesn't get
452 # dropped accidentally.
453 if strict
and line
.strip() != '':
454 self
.fatal (filename
, 'unrecognised line: ' + line
[:-1])
456 # Output a segment of text.
457 def output_segment (self
, segment
):
458 with
open (segment
.filename
, 'r') as file:
459 file.seek (segment
.start
)
460 for i
in range (segment
.lines
):
461 sys
.stdout
.write (file.readline())
463 # Output a summary giving the number of times each type of result has
465 def output_summary (self
, tool
, counts
):
466 for i
in range (len (self
.count_names
)):
467 name
= self
.count_names
[i
]
468 # dejagnu only prints result types that were seen at least once,
469 # but acats always prints a number of unexpected failures.
471 or (tool
.name
== 'acats'
472 and name
.startswith ('# of unexpected failures'))):
473 sys
.stdout
.write ('%s%d\n' % (name
, counts
[i
]))
475 # Output unified .log or .sum information for a particular variation,
476 # with a summary at the end.
477 def output_variation (self
, tool
, variation
):
478 self
.output_segment (variation
.header
)
479 for harness
in sorted (variation
.harnesses
.values()):
480 sys
.stdout
.write ('Running ' + harness
.name
+ ' ...\n')
482 # Keep the original test result order if there was only
483 # one segment for this harness. This is needed for
484 # unsorted.exp, which has unusual test names. Otherwise
485 # sort the tests by test filename. If there are several
486 # subtests for the same test filename (such as 'compilation',
487 # 'test for excess errors', etc.) then keep the subtests
488 # in the original order.
489 if len (harness
.segments
) > 1:
490 harness
.results
.sort()
491 for (key
, line
) in harness
.results
:
492 sys
.stdout
.write (line
)
494 # Rearrange the log segments into test order (but without
495 # rearranging text within those segments).
496 for key
in sorted (harness
.segments
.keys()):
497 self
.output_segment (harness
.segments
[key
])
498 for segment
in harness
.empty
:
499 self
.output_segment (segment
)
500 if len (self
.variations
) > 1:
501 sys
.stdout
.write ('\t\t=== ' + tool
.name
+ ' Summary for '
502 + variation
.name
+ ' ===\n\n')
503 self
.output_summary (tool
, variation
.counts
)
505 # Output unified .log or .sum information for a particular tool,
506 # with a summary at the end.
507 def output_tool (self
, tool
):
508 counts
= self
.zero_counts()
509 if tool
.name
== 'acats':
510 # acats doesn't use variations, so just output everything.
511 # It also has a different approach to whitespace.
512 sys
.stdout
.write ('\t\t=== ' + tool
.name
+ ' tests ===\n')
513 for variation
in tool
.variations
.values():
514 self
.output_variation (tool
, variation
)
515 self
.accumulate_counts (counts
, variation
.counts
)
516 sys
.stdout
.write ('\t\t=== ' + tool
.name
+ ' Summary ===\n')
518 # Output the results in the usual dejagnu runtest format.
519 sys
.stdout
.write ('\n\t\t=== ' + tool
.name
+ ' tests ===\n\n'
520 'Schedule of variations:\n')
521 for name
in self
.variations
:
522 if name
in tool
.variations
:
523 sys
.stdout
.write (' ' + name
+ '\n')
524 sys
.stdout
.write ('\n')
525 for name
in self
.variations
:
526 if name
in tool
.variations
:
527 variation
= tool
.variations
[name
]
528 sys
.stdout
.write ('Running target '
529 + variation
.name
+ '\n')
530 self
.output_variation (tool
, variation
)
531 self
.accumulate_counts (counts
, variation
.counts
)
532 sys
.stdout
.write ('\n\t\t=== ' + tool
.name
+ ' Summary ===\n\n')
533 self
.output_summary (tool
, counts
)
538 # Parse the input files.
539 for filename
in self
.files
:
540 with
open (filename
, 'r') as file:
541 self
.parse_file (filename
, file)
543 # Decide what to output.
544 if len (self
.variations
) == 0:
545 self
.variations
= sorted (self
.known_variations
)
547 for name
in self
.variations
:
548 if name
not in self
.known_variations
:
549 self
.fatal (None, 'no results for ' + name
)
550 if len (self
.tools
) == 0:
551 self
.tools
= sorted (self
.runs
.keys())
555 sys
.stdout
.write (self
.start_line
[1])
556 sys
.stdout
.write (self
.native_line
)
557 sys
.stdout
.write (self
.target_line
)
558 sys
.stdout
.write (self
.host_line
)
559 sys
.stdout
.write (self
.acats_premable
)
561 # Output the main body.
562 for name
in self
.tools
:
563 if name
not in self
.runs
:
564 self
.fatal (None, 'no results for ' + name
)
565 self
.output_tool (self
.runs
[name
])
568 if len (self
.acats_failures
) > 0:
569 sys
.stdout
.write ('*** FAILURES: '
570 + ' '.join (self
.acats_failures
) + '\n')
571 sys
.stdout
.write (self
.version_output
)
573 sys
.stdout
.write (self
.end_line
[1])
575 self
.fatal (e
.filename
, e
.strerror
)
This page took 0.064719 seconds and 6 git commands to generate.