Porting relevant legacy conversion code from performance_lp to src side
BUG=chromium:633253
CQ_INCLUDE_TRYBOTS=master.tryserver.blink:linux_precise_blink_rel
Review-Url: https://codereview.chromium.org/2479543002
Cr-Commit-Position: refs/heads/master@{#429997}
diff --git a/cc/BUILD.gn b/cc/BUILD.gn
index 2f48ba0..36e94cd 100644
--- a/cc/BUILD.gn
+++ b/cc/BUILD.gn
@@ -1095,5 +1095,6 @@
"//testing/scripts/common.py",
"//testing/xvfb.py",
"//testing/scripts/run_gtest_perf_test.py",
+ "//tools/perf/generate_legacy_perf_dashboard_json.py",
]
}
diff --git a/testing/scripts/run_gtest_perf_test.py b/testing/scripts/run_gtest_perf_test.py
index ba4746ce..ac43ec7 100755
--- a/testing/scripts/run_gtest_perf_test.py
+++ b/testing/scripts/run_gtest_perf_test.py
@@ -29,6 +29,18 @@
import common
+
+def GetChromiumSrcDir():
+ return os.path.abspath(
+ os.path.join(os.path.abspath(__file__), '..', '..', '..'))
+
+def GetPerfDir():
+ return os.path.join(GetChromiumSrcDir(), 'tools', 'perf')
+# Add src/tools/perf where generate_legacy_perf_dashboard_json.py lives
+sys.path.append(GetPerfDir())
+
+import generate_legacy_perf_dashboard_json
+
# Add src/testing/ into sys.path for importing xvfb.
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
import xvfb
@@ -78,24 +90,28 @@
executable = '.\%s.exe' % executable
else:
executable = './%s' % executable
+ with common.temporary_file() as tempfile_path:
+ valid = (common.run_command_with_output([executable],
+ env=env, stdoutfile=tempfile_path) == 0)
- rc = common.run_command_with_output([executable] + [
- '--write-abbreviated-json-results-to', args.isolated_script_test_output,
- ], env=env, stdoutfile=args.isolated_script_test_chartjson_output)
-
- # Now get the correct json format from the stdout to write to the
- # perf results file
+ # Now get the correct json format from the stdout to write to the
+ # perf results file
+ results_processor = (
+ generate_legacy_perf_dashboard_json.LegacyResultsProcessor())
+ charts = results_processor.GenerateJsonResults(tempfile_path)
+ # Write the returned encoded json to a the charts output file
+ with open(args.isolated_script_test_chartjson_output, 'w') as f:
+ f.write(charts)
except Exception:
traceback.print_exc()
valid = False
- if not valid:
- failures = ['(entire test suite)']
- with open(args.isolated_script_test_output, 'w') as fp:
- json.dump({
- 'valid': valid,
- 'failures': failures,
- }, fp)
+ failures = [] if valid else ['(entire test suite)']
+ with open(args.isolated_script_test_output, 'w') as fp:
+ json.dump({
+ 'valid': valid,
+ 'failures': failures,
+ }, fp)
return rc
diff --git a/tools/perf/generate_legacy_perf_dashboard_json.py b/tools/perf/generate_legacy_perf_dashboard_json.py
new file mode 100755
index 0000000..69dc8ba
--- /dev/null
+++ b/tools/perf/generate_legacy_perf_dashboard_json.py
@@ -0,0 +1,257 @@
+#!/usr/bin/env python
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+""" Generates legacy perf dashboard json from non-telemetry based perf tests.
+Taken from chromium/build/scripts/slave/performance_log_processory.py
+(https://goo.gl/03SQRk)
+"""
+
+import collections
+import json
+import math
+import logging
+import re
+
+
+class LegacyResultsProcessor(object):
+ """Class for any log processor expecting standard data to be graphed.
+
+ The log will be parsed looking for any lines of the forms:
+ <*>RESULT <graph_name>: <trace_name>= <value> <units>
+ or
+ <*>RESULT <graph_name>: <trace_name>= [<value>,value,value,...] <units>
+ or
+ <*>RESULT <graph_name>: <trace_name>= {<mean>, <std deviation>} <units>
+
+ For example,
+ *RESULT vm_final_browser: kb
+ RESULT startup: ref= [167.00,148.00,146.00,142.00] ms
+ RESULT TabCapturePerformance_foo: Capture= {30.7, 1.45} ms
+
+ The leading * is optional; it indicates that the data from that line should
+ be considered "important", which may mean for example that it's graphed by
+ default.
+
+ If multiple values are given in [], their mean and (sample) standard
+ deviation will be written; if only one value is given, that will be written.
+ A trailing comma is permitted in the list of values.
+
+ NOTE: All lines except for RESULT lines are ignored, including the Avg and
+ Stddev lines output by Telemetry!
+
+ Any of the <fields> except <value> may be empty, in which case the
+ not-terribly-useful defaults will be used. The <graph_name> and <trace_name>
+ should not contain any spaces, colons (:) nor equals-signs (=). Furthermore,
+ the <trace_name> will be used on the waterfall display, so it should be kept
+ short. If the trace_name ends with '_ref', it will be interpreted as a
+ reference value, and shown alongside the corresponding main value on the
+ waterfall.
+
+ Semantic note: The terms graph and chart are used interchangeably here.
+ """
+
+ RESULTS_REGEX = re.compile(r'(?P<IMPORTANT>\*)?RESULT '
+ r'(?P<GRAPH>[^:]*): (?P<TRACE>[^=]*)= '
+ r'(?P<VALUE>[\{\[]?[-\d\., ]+[\}\]]?)('
+ r' ?(?P<UNITS>.+))?')
+ # TODO(eyaich): Determine if this format is still used by any perf tests
+ HISTOGRAM_REGEX = re.compile(r'(?P<IMPORTANT>\*)?HISTOGRAM '
+ r'(?P<GRAPH>[^:]*): (?P<TRACE>[^=]*)= '
+ r'(?P<VALUE_JSON>{.*})(?P<UNITS>.+)?')
+
+ def __init__(self):
+ # A dict of Graph objects, by name.
+ self._graphs = {}
+ # A dict mapping output file names to lists of lines in a file.
+ self._output = {}
+ self._percentiles = [.1, .25, .5, .75, .90, .95, .99]
+
+ class Trace(object):
+ """Encapsulates data for one trace. Here, this means one point."""
+
+ def __init__(self):
+ self.important = False
+ self.value = 0.0
+ self.stddev = 0.0
+
+ def __str__(self):
+ result = _FormatHumanReadable(self.value)
+ if self.stddev:
+ result += '+/-%s' % _FormatHumanReadable(self.stddev)
+ return result
+
+ class Graph(object):
+ """Encapsulates a set of points that should appear on the same graph."""
+
+ def __init__(self):
+ self.units = None
+ self.traces = {}
+
+ def IsImportant(self):
+ """A graph is considered important if any of its traces is important."""
+ for trace in self.traces.itervalues():
+ if trace.important:
+ return True
+ return False
+
+ def BuildTracesDict(self):
+ """Returns a dictionary mapping trace names to [value, stddev]."""
+ traces_dict = {}
+ for name, trace in self.traces.items():
+ traces_dict[name] = [str(trace.value), str(trace.stddev)]
+ return traces_dict
+
+
+ def GenerateJsonResults(self, filename):
+ # Iterate through the file and process each output line
+ with open(filename) as f:
+ for line in f.readlines():
+ self.ProcessLine(line)
+ # After all results have been seen, generate the graph json data
+ return self.GenerateGraphJson()
+
+
+ def _PrependLog(self, filename, data):
+ """Prepends some data to an output file."""
+ self._output[filename] = data + self._output.get(filename, [])
+
+
+ def ProcessLine(self, line):
+ """Processes one result line, and updates the state accordingly."""
+ results_match = self.RESULTS_REGEX.search(line)
+ histogram_match = self.HISTOGRAM_REGEX.search(line)
+ if results_match:
+ self._ProcessResultLine(results_match)
+ elif histogram_match:
+ raise Exception("Error: Histogram results parsing not supported yet")
+
+
+ def _ProcessResultLine(self, line_match):
+ """Processes a line that matches the standard RESULT line format.
+
+ Args:
+ line_match: A MatchObject as returned by re.search.
+ """
+ match_dict = line_match.groupdict()
+ graph_name = match_dict['GRAPH'].strip()
+ trace_name = match_dict['TRACE'].strip()
+
+ graph = self._graphs.get(graph_name, self.Graph())
+ graph.units = match_dict['UNITS'] or ''
+ trace = graph.traces.get(trace_name, self.Trace())
+ trace.value = match_dict['VALUE']
+ trace.important = match_dict['IMPORTANT'] or False
+
+ # Compute the mean and standard deviation for a list or a histogram,
+ # or the numerical value of a scalar value.
+ if trace.value.startswith('['):
+ try:
+ value_list = [float(x) for x in trace.value.strip('[],').split(',')]
+ except ValueError:
+ # Report, but ignore, corrupted data lines. (Lines that are so badly
+ # broken that they don't even match the RESULTS_REGEX won't be
+ # detected.)
+ logging.warning("Bad test output: '%s'" % trace.value.strip())
+ return
+ trace.value, trace.stddev, filedata = self._CalculateStatistics(
+ value_list, trace_name)
+ assert filedata is not None
+ for filename in filedata:
+ self._PrependLog(filename, filedata[filename])
+ elif trace.value.startswith('{'):
+ stripped = trace.value.strip('{},')
+ try:
+ trace.value, trace.stddev = [float(x) for x in stripped.split(',')]
+ except ValueError:
+ logging.warning("Bad test output: '%s'" % trace.value.strip())
+ return
+ else:
+ try:
+ trace.value = float(trace.value)
+ except ValueError:
+ logging.warning("Bad test output: '%s'" % trace.value.strip())
+ return
+
+ graph.traces[trace_name] = trace
+ self._graphs[graph_name] = graph
+
+
+ def GenerateGraphJson(self):
+ """Writes graph json for each graph seen.
+ """
+ charts = {}
+ for graph_name, graph in self._graphs.iteritems():
+ graph_dict = collections.OrderedDict([
+ ('traces', graph.BuildTracesDict()),
+ ('units', str(graph.units)),
+ ])
+
+ # Include a sorted list of important trace names if there are any.
+ important = [t for t in graph.traces.keys() if graph.traces[t].important]
+ if important:
+ graph_dict['important'] = sorted(important)
+
+ charts[graph_name] = graph_dict
+ return json.dumps(charts)
+
+
+ # _CalculateStatistics needs to be a member function.
+ # pylint: disable=R0201
+ # Unused argument value_list.
+ # pylint: disable=W0613
+ def _CalculateStatistics(self, value_list, trace_name):
+ """Returns a tuple with some statistics based on the given value list.
+
+ This method may be overridden by subclasses wanting a different standard
+ deviation calcuation (or some other sort of error value entirely).
+
+ Args:
+ value_list: the list of values to use in the calculation
+ trace_name: the trace that produced the data (not used in the base
+ implementation, but subclasses may use it)
+
+ Returns:
+ A 3-tuple - mean, standard deviation, and a dict which is either
+ empty or contains information about some file contents.
+ """
+ n = len(value_list)
+ if n == 0:
+ return 0.0, 0.0, {}
+ mean = float(sum(value_list)) / n
+ variance = sum([(element - mean)**2 for element in value_list]) / n
+ stddev = math.sqrt(variance)
+
+ return mean, stddev, {}
+
+
+def _FormatHumanReadable(number):
+ """Formats a float into three significant figures, using metric suffixes.
+
+ Only m, k, and M prefixes (for 1/1000, 1000, and 1,000,000) are used.
+ Examples:
+ 0.0387 => 38.7m
+ 1.1234 => 1.12
+ 10866 => 10.8k
+ 682851200 => 683M
+ """
+ metric_prefixes = {-3: 'm', 0: '', 3: 'k', 6: 'M'}
+ scientific = '%.2e' % float(number) # 6.83e+005
+ e_idx = scientific.find('e') # 4, or 5 if negative
+ digits = float(scientific[:e_idx]) # 6.83
+ exponent = int(scientific[e_idx + 1:]) # int('+005') = 5
+ while exponent % 3:
+ digits *= 10
+ exponent -= 1
+ while exponent > 6:
+ digits *= 10
+ exponent -= 1
+ while exponent < -3:
+ digits /= 10
+ exponent += 1
+ if digits >= 100:
+ # Don't append a meaningless '.0' to an integer number.
+ digits = int(digits)
+ # Exponent is now divisible by 3, between -3 and 6 inclusive: (-3, 0, 3, 6).
+ return '%s%s' % (digits, metric_prefixes[exponent])
diff --git a/tools/perf/generate_legacy_perf_dashboard_json_unittest.py b/tools/perf/generate_legacy_perf_dashboard_json_unittest.py
new file mode 100755
index 0000000..aa65bff
--- /dev/null
+++ b/tools/perf/generate_legacy_perf_dashboard_json_unittest.py
@@ -0,0 +1,101 @@
+#!/usr/bin/env python
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+import json
+import os
+import unittest
+
+import generate_legacy_perf_dashboard_json
+
+class LegacyResultsProcessorUnittest(unittest.TestCase):
+ def setUp(self):
+ """Set up for all test method of each test method below."""
+ super(LegacyResultsProcessorUnittest, self).setUp()
+ self.data_directory = os.path.join(os.path.dirname(
+ os.path.abspath(__file__)), 'testdata')
+
+ def _ConstructDefaultProcessor(self):
+ """Creates a LegacyResultsProcessor instance.
+
+ Returns:
+ An instance of LegacyResultsProcessor class
+ """
+ return generate_legacy_perf_dashboard_json.LegacyResultsProcessor()
+
+ def _ProcessLog(self, log_processor, logfile): # pylint: disable=R0201
+ """Reads in a input log file and processes it.
+
+ This changes the state of the log processor object; the output is stored
+ in the object and can be gotten using the PerformanceLogs() method.
+
+ Args:
+ log_processor: An PerformanceLogProcessor instance.
+ logfile: File name of an input performance results log file.
+ """
+ for line in open(os.path.join(self.data_directory, logfile)):
+ log_processor.ProcessLine(line)
+
+ def _CheckFileExistsWithData(self, logs, graph):
+ """Asserts that |graph| exists in the |logs| dict and is non-empty."""
+ self.assertTrue(graph in logs, 'File %s was not output.' % graph)
+ self.assertTrue(logs[graph], 'File %s did not contain data.' % graph)
+
+ def _ConstructParseAndCheckLogfiles(self, inputfiles, graphs):
+ """Uses a log processor to process the given input files.
+
+ Args:
+ inputfiles: A list of input performance results log file names.
+ logfiles: List of expected output ".dat" file names.
+
+ Returns:
+ A dictionary mapping output file name to output file lines.
+ """
+ parser = self._ConstructDefaultProcessor()
+ for inputfile in inputfiles:
+ self._ProcessLog(parser, inputfile)
+
+ logs = json.loads(parser.GenerateGraphJson())
+ for graph in graphs:
+ self._CheckFileExistsWithData(logs, graph)
+
+ return logs
+
+ def _ConstructParseAndCheckJSON(
+ self, inputfiles, logfiles, graphs):
+ """Processes input with a log processor and checks against expectations.
+
+ Args:
+ inputfiles: A list of input performance result log file names.
+ logfiles: A list of expected output ".dat" file names.
+ subdir: Subdirectory containing expected output files.
+ log_processor_class: A log processor class.
+ """
+ logs = self._ConstructParseAndCheckLogfiles(inputfiles, graphs)
+ index = 0
+ for filename in logfiles:
+ graph_name = graphs[index]
+ actual = logs[graph_name]
+ path = os.path.join(self.data_directory, filename)
+ expected = json.load(open(path))
+ self.assertEqual(expected, actual, 'JSON data in %s did not match '
+ 'expectations.' % filename)
+
+ index += 1
+
+
+ def testSummary(self):
+ graphs = ['commit_charge',
+ 'ws_final_total', 'vm_final_browser', 'vm_final_total',
+ 'ws_final_browser', 'processes', 'artificial_graph']
+ # Tests the output of "summary" files, which contain per-graph data.
+ input_files = ['graphing_processor.log']
+ output_files = ['%s-summary.dat' % graph for graph in graphs]
+
+ self._ConstructParseAndCheckJSON(input_files, output_files, graphs)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/tools/perf/testdata/artificial_graph-summary.dat b/tools/perf/testdata/artificial_graph-summary.dat
new file mode 100644
index 0000000..7ef13d59
--- /dev/null
+++ b/tools/perf/testdata/artificial_graph-summary.dat
@@ -0,0 +1 @@
+{"traces": {"trace_with_one_sample": ["177.0", "0.0"], "trace_with_one_sample_comma": ["177.0", "0.0"], "trace_with_three_samples": ["140.0", "43.2049379894"], "trace_with_three_samples_comma": ["140.0", "43.2049379894"]}, "units": "you-nits"}
diff --git a/tools/perf/testdata/commit_charge-summary.dat b/tools/perf/testdata/commit_charge-summary.dat
new file mode 100644
index 0000000..225b00e
--- /dev/null
+++ b/tools/perf/testdata/commit_charge-summary.dat
@@ -0,0 +1 @@
+{"traces": {"12t_cc": ["50200.0", "0.0"], "1t_cc": ["22536.0", "0.0"], "1t_cc_ref": ["22536.0", "0.0"], "5t_cc": ["49412.0", "0.0"]}, "important": ["12t_cc", "1t_cc", "1t_cc_ref", "5t_cc"], "units": "kb"}
diff --git a/tools/perf/testdata/graphing_processor.log b/tools/perf/testdata/graphing_processor.log
new file mode 100644
index 0000000..868c725fc
--- /dev/null
+++ b/tools/perf/testdata/graphing_processor.log
@@ -0,0 +1,49 @@
+[==========] Running 4 tests from 1 test case.
+[----------] Global test environment set-up.
+[----------] 4 tests from MemoryTest
+[ RUN ] MemoryTest.SingleTabTest
+
+RESULT vm_final_browser: 1t_vm_b= 8712 kb
+RESULT ws_final_browser: 1t_ws_b= 16892 kb
+RESULT vm_final_total: 1t_vm= 29696 kb
+*RESULT ws_final_total: 1t_ws= 41960 kb
+RESULT processes: 1t_proc= 2
+*RESULT commit_charge: 1t_cc= 22536 kb
+[ OK ] MemoryTest.SingleTabTest
+[ RUN ] MemoryTest.SingleTabTestRef
+
+RESULT vm_final_browser: 1t_vm_b_ref= 5712 kb
+RESULT ws_final_browser: 1t_ws_b_ref= 26892 kb
+RESULT vm_final_total: 1t_vm_ref= 9696 kb
+*RESULT ws_final_total: 1t_ws_ref= 41960 kb
+RESULT processes: 1t_proc_ref= 1
+*RESULT commit_charge: 1t_cc_ref= 22536 kb
+[ OK ] MemoryTest.SingleTabTestRef
+[ RUN ] MemoryTest.FiveTabTest
+
+RESULT vm_final_browser: 5t_vm_b= 10404 kb
+RESULT ws_final_browser: 5t_ws_b= 18504 kb
+RESULT vm_final_total: 5t_vm= 54936 kb
+*RESULT ws_final_total: 5t_ws= 63276 kb
+RESULT processes: 5t_proc= 3
+*RESULT commit_charge: 5t_cc= 49412 kb
+[ OK ] MemoryTest.FiveTabTest
+[ RUN ] MemoryTest.TwelveTabTest
+
+RESULT vm_final_browser: 12t_vm_b= 10556 kb
+RESULT ws_final_browser: 12t_ws_b= 18612 kb
+RESULT vm_final_total: 12t_vm= 55740 kb
+*RESULT ws_final_total: 12t_ws= 63984 kb
+RESULT processes: 12t_proc= 3
+*RESULT commit_charge: 12t_cc= 50200 kb
+[ OK ] MemoryTest.TwelveTabTest
+[----------] Global test environment tear-down
+[==========] 4 tests from 1 test case ran.
+[ PASSED ] 4 tests.
+
+# Artificial log output to test the [] syntax.
+
+RESULT artificial_graph: trace_with_one_sample= [177.0] you-nits
+RESULT artificial_graph: trace_with_one_sample_comma= [177.0,] you-nits
+RESULT artificial_graph: trace_with_three_samples= [100.0,120.0,200.0] you-nits
+RESULT artificial_graph: trace_with_three_samples_comma= [100.0,120.0,200.0,] you-nits
diff --git a/tools/perf/testdata/processes-summary.dat b/tools/perf/testdata/processes-summary.dat
new file mode 100644
index 0000000..91ce5f3e
--- /dev/null
+++ b/tools/perf/testdata/processes-summary.dat
@@ -0,0 +1 @@
+{"traces": {"1t_proc": ["2.0", "0.0"], "1t_proc_ref": ["1.0", "0.0"], "12t_proc": ["3.0", "0.0"], "5t_proc": ["3.0", "0.0"]}, "units": ""}
diff --git a/tools/perf/testdata/vm_final_browser-summary.dat b/tools/perf/testdata/vm_final_browser-summary.dat
new file mode 100644
index 0000000..396354c
--- /dev/null
+++ b/tools/perf/testdata/vm_final_browser-summary.dat
@@ -0,0 +1 @@
+{"traces": {"1t_vm_b": ["8712.0", "0.0"], "1t_vm_b_ref": ["5712.0", "0.0"], "12t_vm_b": ["10556.0", "0.0"], "5t_vm_b": ["10404.0", "0.0"]}, "units": "kb"}
diff --git a/tools/perf/testdata/vm_final_total-summary.dat b/tools/perf/testdata/vm_final_total-summary.dat
new file mode 100644
index 0000000..e8d7372
--- /dev/null
+++ b/tools/perf/testdata/vm_final_total-summary.dat
@@ -0,0 +1 @@
+{"traces": {"1t_vm": ["29696.0", "0.0"], "1t_vm_ref": ["9696.0", "0.0"], "12t_vm": ["55740.0", "0.0"], "5t_vm": ["54936.0", "0.0"]}, "units": "kb"}
diff --git a/tools/perf/testdata/ws_final_browser-summary.dat b/tools/perf/testdata/ws_final_browser-summary.dat
new file mode 100644
index 0000000..4633efb
--- /dev/null
+++ b/tools/perf/testdata/ws_final_browser-summary.dat
@@ -0,0 +1 @@
+{"traces": {"1t_ws_b": ["16892.0", "0.0"], "1t_ws_b_ref": ["26892.0", "0.0"], "12t_ws_b": ["18612.0", "0.0"], "5t_ws_b": ["18504.0", "0.0"]}, "units": "kb"}
diff --git a/tools/perf/testdata/ws_final_total-summary.dat b/tools/perf/testdata/ws_final_total-summary.dat
new file mode 100644
index 0000000..dc7970a4
--- /dev/null
+++ b/tools/perf/testdata/ws_final_total-summary.dat
@@ -0,0 +1 @@
+{"traces": {"12t_ws": ["63984.0", "0.0"], "1t_ws": ["41960.0", "0.0"], "1t_ws_ref": ["41960.0", "0.0"], "5t_ws": ["63276.0", "0.0"]}, "important": ["12t_ws", "1t_ws", "1t_ws_ref", "5t_ws"], "units": "kb"}