[go: nahoru, domu]
Google Git
Sign in
chromium / chromium / src / 6208ddc049c2f7810c3cee20cf15be526dec61e1 / . / tools / cygprofile / orderfile_generator_backend.py
blob: 8a030e608a9ede73273a7e961de92fb4c6f83bca [file] [log] [blame]
Ari Chivukulaa52f8ba2021-08-10 22:30:39[diff] [blame]1#!/usr/bin/env vpython3
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]2# Copyright (c) 2013 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6""" A utility to generate an up-to-date orderfile.
7
8The orderfile is used by the linker to order text sections such that the
9sections are placed consecutively in the order specified. This allows us
10to page in less code during start-up.
11
12Example usage:
Egor Paskoa7ac67a42020-09-09 12:06:58[diff] [blame]13 tools/cygprofile/orderfile_generator_backend.py --use-goma --target-arch=arm
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]14"""
15
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]16
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]17import argparse
Egor Paskoa7ac67a42020-09-09 12:06:58[diff] [blame]18import csv
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]19import hashlib
20import json
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]21import glob
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]22import logging
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]23import os
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]24import shutil
25import subprocess
26import sys
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]27import tempfile
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]28import time
29
Matthew Cary91df9792018-11-30 14:35:15[diff] [blame]30import cluster
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]31import cyglog_to_orderfile
Benoit Lizefefbb27c2018-01-17 13:54:18[diff] [blame]32import patch_orderfile
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]33import process_profiles
Egor Pasko3bc0b932018-04-03 10:08:27[diff] [blame]34import profile_android_startup
Benoit Lizefefbb27c2018-01-17 13:54:18[diff] [blame]35import symbol_extractor
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]36
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]37_SRC_PATH = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
38sys.path.append(os.path.join(_SRC_PATH, 'third_party', 'catapult', 'devil'))
39from devil.android import device_utils
40from devil.android.sdk import version_codes
41
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]42
43_SRC_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)),
44 os.pardir, os.pardir)
45sys.path.append(os.path.join(_SRC_PATH, 'build', 'android'))
46import devil_chromium
47from pylib import constants
48
49
50# Needs to happen early for GetBuildType()/GetOutDirectory() to work correctly
51constants.SetBuildType('Release')
52
53
Matthew Cary53f74ba2019-01-24 10:07:18[diff] [blame]54# Architecture specific GN args. Trying to build an orderfile for an
55# architecture not listed here will eventually throw.
56_ARCH_GN_ARGS = {
Benoit Lizef5581722021-08-26 09:48:20[diff] [blame]57 'arm': ['target_cpu = "arm"'],
58 'arm64': ['target_cpu = "arm64"', 'android_64bit_browser = true'],
59 'x86': ['target_cpu = "x86"'],
Matthew Cary53f74ba2019-01-24 10:07:18[diff] [blame]60}
61
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]62class CommandError(Exception):
63 """Indicates that a dispatched shell command exited with a non-zero status."""
64
65 def __init__(self, value):
66 super(CommandError, self).__init__()
67 self.value = value
68
69 def __str__(self):
70 return repr(self.value)
71
72
73def _GenerateHash(file_path):
74 """Calculates and returns the hash of the file at file_path."""
75 sha1 = hashlib.sha1()
76 with open(file_path, 'rb') as f:
77 while True:
78 # Read in 1mb chunks, so it doesn't all have to be loaded into memory.
79 chunk = f.read(1024 * 1024)
80 if not chunk:
81 break
82 sha1.update(chunk)
83 return sha1.hexdigest()
84
85
86def _GetFileExtension(file_name):
87 """Calculates the file extension from a file name.
88
89 Args:
90 file_name: The source file name.
91 Returns:
92 The part of file_name after the dot (.) or None if the file has no
93 extension.
94 Examples: /home/user/foo.bar -> bar
95 /home/user.name/foo -> None
96 /home/user/.foo -> None
97 /home/user/foo.bar.baz -> baz
98 """
99 file_name_parts = os.path.basename(file_name).split('.')
100 if len(file_name_parts) > 1:
101 return file_name_parts[-1]
102 else:
103 return None
104
105
106def _StashOutputDirectory(buildpath):
107 """Takes the output directory and stashes it in the default output directory.
108
109 This allows it to be used for incremental builds next time (after unstashing)
110 by keeping it in a place that isn't deleted normally, while also ensuring
111 that it is properly clobbered when appropriate.
112
113 This is a dirty hack to deal with the needs of clobbering while also handling
114 incremental builds and the hardcoded relative paths used in some of the
115 project files.
116
117 Args:
118 buildpath: The path where the building happens. If this corresponds to the
119 default output directory, no action is taken.
120 """
121 if os.path.abspath(buildpath) == os.path.abspath(os.path.dirname(
122 constants.GetOutDirectory())):
123 return
124 name = os.path.basename(buildpath)
125 stashpath = os.path.join(constants.GetOutDirectory(), name)
126 if not os.path.exists(buildpath):
127 return
128 if os.path.exists(stashpath):
129 shutil.rmtree(stashpath, ignore_errors=True)
130 shutil.move(buildpath, stashpath)
131
132
133def _UnstashOutputDirectory(buildpath):
134 """Inverse of _StashOutputDirectory.
135
136 Moves the output directory stashed within the default output directory
137 (out/Release) to the position where the builds can actually happen.
138
139 This is a dirty hack to deal with the needs of clobbering while also handling
140 incremental builds and the hardcoded relative paths used in some of the
141 project files.
142
143 Args:
144 buildpath: The path where the building happens. If this corresponds to the
145 default output directory, no action is taken.
146 """
147 if os.path.abspath(buildpath) == os.path.abspath(os.path.dirname(
148 constants.GetOutDirectory())):
149 return
150 name = os.path.basename(buildpath)
151 stashpath = os.path.join(constants.GetOutDirectory(), name)
152 if not os.path.exists(stashpath):
153 return
154 if os.path.exists(buildpath):
155 shutil.rmtree(buildpath, ignore_errors=True)
156 shutil.move(stashpath, buildpath)
157
158
159class StepRecorder(object):
160 """Records steps and timings."""
161
162 def __init__(self, buildbot):
163 self.timings = []
164 self._previous_step = ('', 0.0)
165 self._buildbot = buildbot
166 self._error_recorded = False
167
168 def BeginStep(self, name):
169 """Marks a beginning of the next step in the script.
170
171 On buildbot, this prints a specially formatted name that will show up
172 in the waterfall. Otherwise, just prints the step name.
173
174 Args:
175 name: The name of the step.
176 """
177 self.EndStep()
178 self._previous_step = (name, time.time())
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]179 print('Running step: ', name)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]180
181 def EndStep(self):
182 """Records successful completion of the current step.
183
184 This is optional if the step is immediately followed by another BeginStep.
185 """
186 if self._previous_step[0]:
187 elapsed = time.time() - self._previous_step[1]
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]188 print('Step %s took %f seconds' % (self._previous_step[0], elapsed))
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]189 self.timings.append((self._previous_step[0], elapsed))
190
191 self._previous_step = ('', 0.0)
192
193 def FailStep(self, message=None):
194 """Marks that a particular step has failed.
195
196 On buildbot, this will mark the current step as failed on the waterfall.
197 Otherwise we will just print an optional failure message.
198
199 Args:
200 message: An optional explanation as to why the step failed.
201 """
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]202 print('STEP FAILED!!')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]203 if message:
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]204 print(message)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]205 self._error_recorded = True
206 self.EndStep()
207
208 def ErrorRecorded(self):
209 """True if FailStep has been called."""
210 return self._error_recorded
211
212 def RunCommand(self, cmd, cwd=constants.DIR_SOURCE_ROOT, raise_on_error=True,
213 stdout=None):
214 """Execute a shell command.
215
216 Args:
217 cmd: A list of command strings.
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]218 cwd: Directory in which the command should be executed, defaults to build
219 root of script's location if not specified.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]220 raise_on_error: If true will raise a CommandError if the call doesn't
221 succeed and mark the step as failed.
222 stdout: A file to redirect stdout for the command to.
223
224 Returns:
225 The process's return code.
226
227 Raises:
228 CommandError: An error executing the specified command.
229 """
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]230 print('Executing %s in %s' % (' '.join(cmd), cwd))
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]231 process = subprocess.Popen(cmd, stdout=stdout, cwd=cwd, env=os.environ)
232 process.wait()
233 if raise_on_error and process.returncode != 0:
234 self.FailStep()
235 raise CommandError('Exception executing command %s' % ' '.join(cmd))
236 return process.returncode
237
238
239class ClankCompiler(object):
240 """Handles compilation of clank."""
241
Christopher Grant073637472019-07-05 13:34:57[diff] [blame]242 def __init__(self, out_dir, step_recorder, arch, use_goma, goma_dir,
243 system_health_profiling, monochrome, public, orderfile_location):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]244 self._out_dir = out_dir
245 self._step_recorder = step_recorder
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]246 self._arch = arch
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]247 self._use_goma = use_goma
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]248 self._goma_dir = goma_dir
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]249 self._system_health_profiling = system_health_profiling
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]250 self._public = public
251 self._orderfile_location = orderfile_location
Matthew Caryd6bfcb72018-09-14 11:27:46[diff] [blame]252 if monochrome:
253 self._apk = 'Monochrome.apk'
Matthew Carye1b00062018-09-19 14:27:44[diff] [blame]254 self._apk_target = 'monochrome_apk'
Matthew Caryd6bfcb72018-09-14 11:27:46[diff] [blame]255 self._libname = 'libmonochrome'
Matthew Cary2216fef2019-06-17 13:35:59[diff] [blame]256 self._libchrome_target = 'libmonochrome'
Matthew Caryd6bfcb72018-09-14 11:27:46[diff] [blame]257 else:
258 self._apk = 'Chrome.apk'
Matthew Carye1b00062018-09-19 14:27:44[diff] [blame]259 self._apk_target = 'chrome_apk'
Matthew Caryd6bfcb72018-09-14 11:27:46[diff] [blame]260 self._libname = 'libchrome'
261 self._libchrome_target = 'libchrome'
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]262 if public:
263 self._apk = self._apk.replace('.apk', 'Public.apk')
264 self._apk_target = self._apk_target.replace('_apk', '_public_apk')
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]265
266 self.obj_dir = os.path.join(self._out_dir, 'Release', 'obj')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]267 self.lib_chrome_so = os.path.join(
Matthew Caryd6bfcb72018-09-14 11:27:46[diff] [blame]268 self._out_dir, 'Release', 'lib.unstripped',
269 '{}.so'.format(self._libname))
270 self.chrome_apk = os.path.join(self._out_dir, 'Release', 'apks', self._apk)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]271
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]272 def Build(self, instrumented, use_call_graph, target):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]273 """Builds the provided ninja target with or without order_profiling on.
274
275 Args:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]276 instrumented: (bool) Whether we want to build an instrumented binary.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]277 use_call_graph: (bool) Whether to use the call graph instrumentation.
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]278 target: (str) The name of the ninja target to build.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]279 """
280 self._step_recorder.BeginStep('Compile %s' % target)
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]281 assert not use_call_graph or instrumented, ('You can not enable call graph '
282 'without instrumentation!')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]283
284 # Set the "Release Official" flavor, the parts affecting performance.
285 args = [
Andrew Grieve0e8790c2020-09-03 17:27:32[diff] [blame]286 'enable_resource_allowlist_generation=false',
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]287 'is_chrome_branded=' + str(not self._public).lower(),
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]288 'is_debug=false',
289 'is_official_build=true',
Egor Pasko1d13d532020-01-14 21:25:19[diff] [blame]290 'symbol_level=1', # to fit 30 GiB RAM on the bot when LLD is running
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]291 'target_os="android"',
292 'use_goma=' + str(self._use_goma).lower(),
293 'use_order_profiling=' + str(instrumented).lower(),
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]294 'use_call_graph=' + str(use_call_graph).lower(),
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]295 ]
Matthew Cary53f74ba2019-01-24 10:07:18[diff] [blame]296 args += _ARCH_GN_ARGS[self._arch]
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]297 if self._goma_dir:
298 args += ['goma_dir="%s"' % self._goma_dir]
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]299 if self._system_health_profiling:
300 args += ['devtools_instrumentation_dumping = ' +
301 str(instrumented).lower()]
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]302
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]303 if self._public and os.path.exists(self._orderfile_location):
304 # GN needs the orderfile path to be source-absolute.
305 src_abs_orderfile = os.path.relpath(self._orderfile_location,
306 constants.DIR_SOURCE_ROOT)
307 args += ['chrome_orderfile="//{}"'.format(src_abs_orderfile)]
308
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]309 self._step_recorder.RunCommand(
310 ['gn', 'gen', os.path.join(self._out_dir, 'Release'),
311 '--args=' + ' '.join(args)])
312
313 self._step_recorder.RunCommand(
Christopher Grant073637472019-07-05 13:34:57[diff] [blame]314 ['autoninja', '-C',
315 os.path.join(self._out_dir, 'Release'), target])
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]316
Christopher Grant10221762019-07-05 12:10:04[diff] [blame]317 def ForceRelink(self):
318 """Forces libchrome.so or libmonochrome.so to be re-linked.
319
320 With partitioned libraries enabled, deleting these library files does not
321 guarantee they'll be recreated by the linker (they may simply be
322 re-extracted from a combined library). To be safe, touch a source file
323 instead. See http://crbug.com/972701 for more explanation.
324 """
325 file_to_touch = os.path.join(constants.DIR_SOURCE_ROOT, 'chrome', 'browser',
326 'chrome_browser_main_android.cc')
327 assert os.path.exists(file_to_touch)
328 self._step_recorder.RunCommand(['touch', file_to_touch])
329
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]330 def CompileChromeApk(self, instrumented, use_call_graph, force_relink=False):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]331 """Builds a Chrome.apk either with or without order_profiling on.
332
333 Args:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]334 instrumented: (bool) Whether to build an instrumented apk.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]335 use_call_graph: (bool) Whether to use the call graph instrumentation.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]336 force_relink: Whether libchromeview.so should be re-created.
337 """
338 if force_relink:
Christopher Grant10221762019-07-05 12:10:04[diff] [blame]339 self.ForceRelink()
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]340 self.Build(instrumented, use_call_graph, self._apk_target)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]341
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]342 def CompileLibchrome(self, instrumented, use_call_graph, force_relink=False):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]343 """Builds a libchrome.so either with or without order_profiling on.
344
345 Args:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]346 instrumented: (bool) Whether to build an instrumented apk.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]347 use_call_graph: (bool) Whether to use the call graph instrumentation.
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]348 force_relink: (bool) Whether libchrome.so should be re-created.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]349 """
350 if force_relink:
Christopher Grant10221762019-07-05 12:10:04[diff] [blame]351 self.ForceRelink()
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]352 self.Build(instrumented, use_call_graph, self._libchrome_target)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]353
354
355class OrderfileUpdater(object):
356 """Handles uploading and committing a new orderfile in the repository.
357
358 Only used for testing or on a bot.
359 """
360
361 _CLOUD_STORAGE_BUCKET_FOR_DEBUG = None
362 _CLOUD_STORAGE_BUCKET = None
363 _UPLOAD_TO_CLOUD_COMMAND = 'upload_to_google_storage.py'
364
Egor Pasko0c533c6682019-11-26 21:16:32[diff] [blame]365 def __init__(self, repository_root, step_recorder):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]366 """Constructor.
367
368 Args:
369 repository_root: (str) Root of the target repository.
370 step_recorder: (StepRecorder) Step recorder, for logging.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]371 """
372 self._repository_root = repository_root
373 self._step_recorder = step_recorder
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]374
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]375 def CommitStashedFileHashes(self, files):
376 """Commits unpatched and patched orderfiles hashes if changed.
377
378 The files are committed only if their associated sha1 hash files match, and
379 are modified in git. In normal operations the hash files are changed only
380 when a file is uploaded to cloud storage. If the hash file is not modified
381 in git, the file is skipped.
382
383 Args:
384 files: [str or None] specifies file paths. None items are ignored.
385
386 Raises:
387 Exception if the hash file does not match the file.
388 NotImplementedError when the commit logic hasn't been overridden.
389 """
Ari Chivukulaa52f8ba2021-08-10 22:30:39[diff] [blame]390 files_to_commit = [_f for _f in files if _f]
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]391 if files_to_commit:
392 self._CommitStashedFiles(files_to_commit)
393
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]394 def UploadToCloudStorage(self, filename, use_debug_location):
395 """Uploads a file to cloud storage.
396
397 Args:
398 filename: (str) File to upload.
399 use_debug_location: (bool) Whether to use the debug location.
400 """
401 bucket = (self._CLOUD_STORAGE_BUCKET_FOR_DEBUG if use_debug_location
402 else self._CLOUD_STORAGE_BUCKET)
403 extension = _GetFileExtension(filename)
404 cmd = [self._UPLOAD_TO_CLOUD_COMMAND, '--bucket', bucket]
405 if extension:
406 cmd.extend(['-z', extension])
407 cmd.append(filename)
408 self._step_recorder.RunCommand(cmd)
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]409 print('Download: https://sandbox.google.com/storage/%s/%s' %
410 (bucket, _GenerateHash(filename)))
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]411
412 def _GetHashFilePathAndContents(self, filename):
413 """Gets the name and content of the hash file created from uploading the
414 given file.
415
416 Args:
417 filename: (str) The file that was uploaded to cloud storage.
418
419 Returns:
420 A tuple of the hash file name, relative to the reository root, and the
421 content, which should be the sha1 hash of the file
422 ('base_file.sha1', hash)
423 """
424 abs_hash_filename = filename + '.sha1'
425 rel_hash_filename = os.path.relpath(
426 abs_hash_filename, self._repository_root)
427 with open(abs_hash_filename, 'r') as f:
428 return (rel_hash_filename, f.read())
429
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]430 def _GitStash(self):
431 """Git stash the current clank tree.
432
433 Raises:
434 NotImplementedError when the stash logic hasn't been overridden.
435 """
436 raise NotImplementedError
437
438 def _CommitStashedFiles(self, expected_files_in_stash):
439 """Commits stashed files.
440
441 The local repository is updated and then the files to commit are taken from
442 modified files from the git stash. The modified files should be a subset of
443 |expected_files_in_stash|. If there are unexpected modified files, this
444 function may raise. This is meant to be paired with _GitStash().
445
446 Args:
447 expected_files_in_stash: [str] paths to a possible superset of files
448 expected to be stashed & committed.
449
450 Raises:
451 NotImplementedError when the commit logic hasn't been overridden.
452 """
453 raise NotImplementedError
454
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]455
456class OrderfileGenerator(object):
457 """A utility for generating a new orderfile for Clank.
458
459 Builds an instrumented binary, profiles a run of the application, and
460 generates an updated orderfile.
461 """
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]462 _CHECK_ORDERFILE_SCRIPT = os.path.join(
463 constants.DIR_SOURCE_ROOT, 'tools', 'cygprofile', 'check_orderfile.py')
464 _BUILD_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(
465 constants.GetOutDirectory()))) # Normally /path/to/src
466
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]467 # Previous orderfile_generator debug files would be overwritten.
468 _DIRECTORY_FOR_DEBUG_FILES = '/tmp/orderfile_generator_debug_files'
469
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]470 def _PrepareOrderfilePaths(self):
471 if self._options.public:
472 self._clank_dir = os.path.join(constants.DIR_SOURCE_ROOT,
473 '')
474 if not os.path.exists(os.path.join(self._clank_dir, 'orderfiles')):
475 os.makedirs(os.path.join(self._clank_dir, 'orderfiles'))
476 else:
477 self._clank_dir = os.path.join(constants.DIR_SOURCE_ROOT,
478 'clank')
479
480 self._unpatched_orderfile_filename = os.path.join(
481 self._clank_dir, 'orderfiles', 'unpatched_orderfile.%s')
482 self._path_to_orderfile = os.path.join(
483 self._clank_dir, 'orderfiles', 'orderfile.%s.out')
484
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]485 def _GetPathToOrderfile(self):
486 """Gets the path to the architecture-specific orderfile."""
Benoit Lize6208ddc2021-08-30 12:26:29[diff] [blame^]487 # Build GN files use the ".arm" orderfile irrespective of the actual
488 # architecture. Fake it, otherwise the orderfile we generate here is not
489 # going to be picked up by builds.
490 orderfile_fake_arch = 'arm'
491 return self._path_to_orderfile % orderfile_fake_arch
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]492
493 def _GetUnpatchedOrderfileFilename(self):
494 """Gets the path to the architecture-specific unpatched orderfile."""
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]495 return self._unpatched_orderfile_filename % self._options.arch
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]496
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]497 def _SetDevice(self):
498 """ Selects the device to be used by the script.
499
500 Returns:
501 (Device with given serial ID) : if the --device flag is set.
502 (Device running Android[K,L]) : if --use-legacy-chrome-apk flag is set or
503 no device running Android N+ was found.
504 (Device running Android N+) : Otherwise.
505
506 Raises Error:
507 If no device meeting the requirements has been found.
508 """
509 devices = None
510 if self._options.device:
511 devices = [device_utils.DeviceUtils(self._options.device)]
512 else:
513 devices = device_utils.DeviceUtils.HealthyDevices()
514
515 assert devices, 'Expected at least one connected device'
516
517 if self._options.use_legacy_chrome_apk:
518 self._monochrome = False
519 for device in devices:
520 device_version = device.build_version_sdk
521 if (device_version >= version_codes.KITKAT
522 and device_version <= version_codes.LOLLIPOP_MR1):
523 return device
524
525 assert not self._options.use_legacy_chrome_apk, \
526 'No device found running suitable android version for Chrome.apk.'
527
528 preferred_device = None
529 for device in devices:
530 if device.build_version_sdk >= version_codes.NOUGAT:
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]531 preferred_device = device
532 break
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]533
534 self._monochrome = preferred_device is not None
535
536 return preferred_device if preferred_device else devices[0]
537
538
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]539 def __init__(self, options, orderfile_updater_class):
540 self._options = options
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]541 self._instrumented_out_dir = os.path.join(
542 self._BUILD_ROOT, self._options.arch + '_instrumented_out')
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]543 if self._options.use_call_graph:
Christopher Grantdfe1bac2019-07-05 13:34:10[diff] [blame]544 self._instrumented_out_dir += '_call_graph'
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]545
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]546 self._uninstrumented_out_dir = os.path.join(
547 self._BUILD_ROOT, self._options.arch + '_uninstrumented_out')
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]548 self._no_orderfile_out_dir = os.path.join(
549 self._BUILD_ROOT, self._options.arch + '_no_orderfile_out')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]550
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]551 self._PrepareOrderfilePaths()
552
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]553 if options.profile:
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]554 output_directory = os.path.join(self._instrumented_out_dir, 'Release')
Matthew Caryb8daed942018-06-11 10:58:08[diff] [blame]555 host_profile_dir = os.path.join(output_directory, 'profile_data')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]556 urls = [profile_android_startup.AndroidProfileTool.TEST_URL]
557 use_wpr = True
558 simulate_user = False
Benoit L96466812018-03-06 15:58:37[diff] [blame]559 urls = options.urls
560 use_wpr = not options.no_wpr
561 simulate_user = options.simulate_user
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]562 device = self._SetDevice()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]563 self._profiler = profile_android_startup.AndroidProfileTool(
Matthew Cary453ff1452018-07-18 12:19:35[diff] [blame]564 output_directory, host_profile_dir, use_wpr, urls, simulate_user,
Matthew Cary1ea82212019-03-20 12:10:27[diff] [blame]565 device, debug=self._options.streamline_for_debugging)
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]566 if options.pregenerated_profiles:
567 self._profiler.SetPregeneratedProfiles(
568 glob.glob(options.pregenerated_profiles))
569 else:
570 assert not options.pregenerated_profiles, (
571 '--pregenerated-profiles cannot be used with --skip-profile')
572 assert not options.profile_save_dir, (
573 '--profile-save-dir cannot be used with --skip-profile')
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]574 self._monochrome = not self._options.use_legacy_chrome_apk
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]575
Matthew Caryf949bba2019-02-04 13:39:23[diff] [blame]576 # Outlined function handling enabled by default for all architectures.
577 self._order_outlined_functions = not options.noorder_outlined_functions
578
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]579 self._output_data = {}
580 self._step_recorder = StepRecorder(options.buildbot)
581 self._compiler = None
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]582 if orderfile_updater_class is None:
Egor Paskod80bfad2020-09-10 21:53:06[diff] [blame]583 orderfile_updater_class = OrderfileUpdater
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]584 assert issubclass(orderfile_updater_class, OrderfileUpdater)
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]585 self._orderfile_updater = orderfile_updater_class(self._clank_dir,
Egor Pasko0c533c6682019-11-26 21:16:32[diff] [blame]586 self._step_recorder)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]587 assert os.path.isdir(constants.DIR_SOURCE_ROOT), 'No src directory found'
Benoit Lizefefbb27c2018-01-17 13:54:18[diff] [blame]588 symbol_extractor.SetArchitecture(options.arch)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]589
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]590 @staticmethod
591 def _RemoveBlanks(src_file, dest_file):
592 """A utility to remove blank lines from a file.
593
594 Args:
595 src_file: The name of the file to remove the blanks from.
596 dest_file: The name of the file to write the output without blanks.
597 """
598 assert src_file != dest_file, 'Source and destination need to be distinct'
599
600 try:
601 src = open(src_file, 'r')
602 dest = open(dest_file, 'w')
603 for line in src:
604 if line and not line.isspace():
605 dest.write(line)
606 finally:
607 src.close()
608 dest.close()
609
610 def _GenerateAndProcessProfile(self):
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]611 """Invokes a script to merge the per-thread traces into one file.
612
613 The produced list of offsets is saved in
614 self._GetUnpatchedOrderfileFilename().
615 """
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]616 self._step_recorder.BeginStep('Generate Profile Data')
617 files = []
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]618 logging.getLogger().setLevel(logging.DEBUG)
Christopher Grantdfe1bac2019-07-05 13:34:10[diff] [blame]619
620 if self._options.profile_save_dir:
621 # The directory must not preexist, to ensure purity of data. Check
622 # before profiling to save time.
623 if os.path.exists(self._options.profile_save_dir):
624 raise Exception('Profile save directory must not pre-exist')
625 os.makedirs(self._options.profile_save_dir)
626
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]627 if self._options.system_health_orderfile:
628 files = self._profiler.CollectSystemHealthProfile(
629 self._compiler.chrome_apk)
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]630 self._MaybeSaveProfile(files)
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]631 try:
632 self._ProcessPhasedOrderfile(files)
633 except Exception:
634 for f in files:
635 self._SaveForDebugging(f)
Matthew Cary8b1416232018-08-10 19:12:22[diff] [blame]636 self._SaveForDebugging(self._compiler.lib_chrome_so)
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]637 raise
638 finally:
639 self._profiler.Cleanup()
640 else:
641 self._CollectLegacyProfile()
642 logging.getLogger().setLevel(logging.INFO)
643
644 def _ProcessPhasedOrderfile(self, files):
645 """Process the phased orderfiles produced by system health benchmarks.
646
647 The offsets will be placed in _GetUnpatchedOrderfileFilename().
648
649 Args:
650 file: Profile files pulled locally.
651 """
652 self._step_recorder.BeginStep('Process Phased Orderfile')
653 profiles = process_profiles.ProfileManager(files)
654 processor = process_profiles.SymbolOffsetProcessor(
655 self._compiler.lib_chrome_so)
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]656 ordered_symbols = cluster.ClusterOffsets(profiles, processor,
657 call_graph=self._options.use_call_graph)
Matthew Cary8b1416232018-08-10 19:12:22[diff] [blame]658 if not ordered_symbols:
659 raise Exception('Failed to get ordered symbols')
Matthew Caryda9db932019-03-11 15:28:17[diff] [blame]660 for sym in ordered_symbols:
661 assert not sym.startswith('OUTLINED_FUNCTION_'), (
662 'Outlined function found in instrumented function, very likely '
663 'something has gone very wrong!')
Matthew Cary91df9792018-11-30 14:35:15[diff] [blame]664 self._output_data['offsets_kib'] = processor.SymbolsSize(
665 ordered_symbols) / 1024
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]666 with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile:
Matthew Cary8b1416232018-08-10 19:12:22[diff] [blame]667 orderfile.write('\n'.join(ordered_symbols))
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]668
669 def _CollectLegacyProfile(self):
Matthew Cary2df8d452018-09-19 07:50:20[diff] [blame]670 files = []
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]671 try:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]672 files = self._profiler.CollectProfile(
673 self._compiler.chrome_apk,
674 constants.PACKAGE_INFO['chrome'])
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]675 self._MaybeSaveProfile(files)
Matthew Caryb8daed942018-06-11 10:58:08[diff] [blame]676 self._step_recorder.BeginStep('Process profile')
Benoit L96466812018-03-06 15:58:37[diff] [blame]677 assert os.path.exists(self._compiler.lib_chrome_so)
678 offsets = process_profiles.GetReachedOffsetsFromDumpFiles(
679 files, self._compiler.lib_chrome_so)
680 if not offsets:
681 raise Exception('No profiler offsets found in {}'.format(
682 '\n'.join(files)))
Matthew Cary8b1416232018-08-10 19:12:22[diff] [blame]683 processor = process_profiles.SymbolOffsetProcessor(
684 self._compiler.lib_chrome_so)
685 ordered_symbols = processor.GetOrderedSymbols(offsets)
686 if not ordered_symbols:
687 raise Exception('No symbol names from offsets found in {}'.format(
688 '\n'.join(files)))
689 with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile:
690 orderfile.write('\n'.join(ordered_symbols))
Matthew Cary0f1f681a2018-01-22 10:40:51[diff] [blame]691 except Exception:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]692 for f in files:
693 self._SaveForDebugging(f)
694 raise
695 finally:
696 self._profiler.Cleanup()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]697
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]698 def _MaybeSaveProfile(self, files):
699 if self._options.profile_save_dir:
700 logging.info('Saving profiles to %s', self._options.profile_save_dir)
701 for f in files:
702 shutil.copy(f, self._options.profile_save_dir)
703 logging.info('Saved profile %s', f)
704
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]705 def _PatchOrderfile(self):
706 """Patches the orderfile using clean version of libchrome.so."""
707 self._step_recorder.BeginStep('Patch Orderfile')
Benoit Lizefefbb27c2018-01-17 13:54:18[diff] [blame]708 patch_orderfile.GeneratePatchedOrderfile(
709 self._GetUnpatchedOrderfileFilename(), self._compiler.lib_chrome_so,
Matthew Caryf949bba2019-02-04 13:39:23[diff] [blame]710 self._GetPathToOrderfile(), self._order_outlined_functions)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]711
712 def _VerifySymbolOrder(self):
713 self._step_recorder.BeginStep('Verify Symbol Order')
714 return_code = self._step_recorder.RunCommand(
715 [self._CHECK_ORDERFILE_SCRIPT, self._compiler.lib_chrome_so,
716 self._GetPathToOrderfile(),
717 '--target-arch=' + self._options.arch],
718 constants.DIR_SOURCE_ROOT,
719 raise_on_error=False)
720 if return_code:
721 self._step_recorder.FailStep('Orderfile check returned %d.' % return_code)
722
723 def _RecordHash(self, file_name):
724 """Records the hash of the file into the output_data dictionary."""
725 self._output_data[os.path.basename(file_name) + '.sha1'] = _GenerateHash(
726 file_name)
727
728 def _SaveFileLocally(self, file_name, file_sha1):
729 """Saves the file to a temporary location and prints the sha1sum."""
730 if not os.path.exists(self._DIRECTORY_FOR_DEBUG_FILES):
731 os.makedirs(self._DIRECTORY_FOR_DEBUG_FILES)
732 shutil.copy(file_name, self._DIRECTORY_FOR_DEBUG_FILES)
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]733 print('File: %s, saved in: %s, sha1sum: %s' %
734 (file_name, self._DIRECTORY_FOR_DEBUG_FILES, file_sha1))
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]735
736 def _SaveForDebugging(self, filename):
737 """Uploads the file to cloud storage or saves to a temporary location."""
738 file_sha1 = _GenerateHash(filename)
739 if not self._options.buildbot:
740 self._SaveFileLocally(filename, file_sha1)
741 else:
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]742 print('Uploading file for debugging: ' + filename)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]743 self._orderfile_updater.UploadToCloudStorage(
744 filename, use_debug_location=True)
745
746 def _SaveForDebuggingWithOverwrite(self, file_name):
747 """Uploads and overwrites the file in cloud storage or copies locally.
748
749 Should be used for large binaries like lib_chrome_so.
750
751 Args:
752 file_name: (str) File to upload.
753 """
754 file_sha1 = _GenerateHash(file_name)
755 if not self._options.buildbot:
756 self._SaveFileLocally(file_name, file_sha1)
757 else:
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]758 print('Uploading file for debugging: %s, sha1sum: %s' % (file_name,
759 file_sha1))
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]760 upload_location = '%s/%s' % (
761 self._CLOUD_STORAGE_BUCKET_FOR_DEBUG, os.path.basename(file_name))
762 self._step_recorder.RunCommand([
763 'gsutil.py', 'cp', file_name, 'gs://' + upload_location])
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]764 print('Uploaded to: https://sandbox.google.com/storage/' +
765 upload_location)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]766
767 def _MaybeArchiveOrderfile(self, filename):
768 """In buildbot configuration, uploads the generated orderfile to
769 Google Cloud Storage.
770
771 Args:
772 filename: (str) Orderfile to upload.
773 """
Matthew Cary91df9792018-11-30 14:35:15[diff] [blame]774 # First compute hashes so that we can download them later if we need to.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]775 self._step_recorder.BeginStep('Compute hash for ' + filename)
776 self._RecordHash(filename)
777 if self._options.buildbot:
778 self._step_recorder.BeginStep('Archive ' + filename)
779 self._orderfile_updater.UploadToCloudStorage(
780 filename, use_debug_location=False)
781
Egor Paskobce64d012018-11-20 12:02:37[diff] [blame]782 def UploadReadyOrderfiles(self):
783 self._step_recorder.BeginStep('Upload Ready Orderfiles')
784 for file_name in [self._GetUnpatchedOrderfileFilename(),
785 self._GetPathToOrderfile()]:
786 self._orderfile_updater.UploadToCloudStorage(
787 file_name, use_debug_location=False)
788
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]789 def _NativeCodeMemoryBenchmark(self, apk):
790 """Runs system_health.memory_mobile to assess native code memory footprint.
791
792 Args:
793 apk: (str) Path to the apk.
794
795 Returns:
796 results: ([int]) Values of native code memory footprint in bytes from the
797 benchmark results.
798 """
799 self._step_recorder.BeginStep("Running orderfile.memory_mobile")
800 try:
801 out_dir = tempfile.mkdtemp()
802 self._profiler._RunCommand(['tools/perf/run_benchmark',
803 '--device={}'.format(
804 self._profiler._device.serial),
805 '--browser=exact',
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]806 '--output-format=csv',
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]807 '--output-dir={}'.format(out_dir),
808 '--reset-results',
809 '--browser-executable={}'.format(apk),
810 'orderfile.memory_mobile'])
811
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]812 out_file_path = os.path.join(out_dir, 'results.csv')
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]813 if not os.path.exists(out_file_path):
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]814 raise Exception('Results file not found!')
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]815
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]816 results = {}
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]817 with open(out_file_path, 'r') as f:
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]818 reader = csv.DictReader(f)
819 for row in reader:
820 if not row['name'].endswith('NativeCodeResidentMemory'):
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]821 continue
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]822 # Note: NativeCodeResidentMemory records a single sample from each
823 # story run, so this average (reported as 'avg') is exactly the value
824 # of that one sample. Each story is run multiple times, so this loop
825 # will accumulate into a list all values for all runs of each story.
826 results.setdefault(row['name'], {}).setdefault(
827 row['stories'], []).append(row['avg'])
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]828
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]829 if not results:
830 raise Exception('Could not find relevant results')
831
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]832 return results
833
834 except Exception as e:
835 return 'Error: ' + str(e)
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]836
837 finally:
838 shutil.rmtree(out_dir)
839
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]840
841 def _PerformanceBenchmark(self, apk):
842 """Runs Speedometer2.0 to assess performance.
843
844 Args:
845 apk: (str) Path to the apk.
846
847 Returns:
848 results: ([float]) Speedometer2.0 results samples in milliseconds.
849 """
850 self._step_recorder.BeginStep("Running Speedometer2.0.")
851 try:
852 out_dir = tempfile.mkdtemp()
853 self._profiler._RunCommand(['tools/perf/run_benchmark',
854 '--device={}'.format(
855 self._profiler._device.serial),
856 '--browser=exact',
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]857 '--output-format=histograms',
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]858 '--output-dir={}'.format(out_dir),
859 '--reset-results',
860 '--browser-executable={}'.format(apk),
861 'speedometer2'])
862
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]863 out_file_path = os.path.join(out_dir, 'histograms.json')
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]864 if not os.path.exists(out_file_path):
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]865 raise Exception('Results file not found!')
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]866
867 with open(out_file_path, 'r') as f:
868 results = json.load(f)
869
870 if not results:
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]871 raise Exception('Results file is empty.')
872
873 for el in results:
874 if 'name' in el and el['name'] == 'Total' and 'sampleValues' in el:
875 return el['sampleValues']
876
877 raise Exception('Unexpected results format.')
878
879 except Exception as e:
880 return 'Error: ' + str(e)
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]881
882 finally:
883 shutil.rmtree(out_dir)
884
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]885
886 def RunBenchmark(self, out_directory, no_orderfile=False):
887 """Builds chrome apk and runs performance and memory benchmarks.
888
889 Builds a non-instrumented version of chrome.
890 Installs chrome apk on the device.
891 Runs Speedometer2.0 benchmark to assess performance.
892 Runs system_health.memory_mobile to evaluate memory footprint.
893
894 Args:
895 out_directory: (str) Path to out directory for this build.
896 no_orderfile: (bool) True if chrome to be built without orderfile.
897
898 Returns:
899 benchmark_results: (dict) Results extracted from benchmarks.
900 """
Benoit Lize74e82b32021-08-26 14:22:01[diff] [blame]901 benchmark_results = {}
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]902 try:
903 _UnstashOutputDirectory(out_directory)
Christopher Grant073637472019-07-05 13:34:57[diff] [blame]904 self._compiler = ClankCompiler(out_directory, self._step_recorder,
905 self._options.arch, self._options.use_goma,
906 self._options.goma_dir,
907 self._options.system_health_orderfile,
908 self._monochrome, self._options.public,
909 self._GetPathToOrderfile())
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]910
911 if no_orderfile:
912 orderfile_path = self._GetPathToOrderfile()
913 backup_orderfile = orderfile_path + '.backup'
914 shutil.move(orderfile_path, backup_orderfile)
915 open(orderfile_path, 'w').close()
916
917 # Build APK to be installed on the device.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]918 self._compiler.CompileChromeApk(instrumented=False,
919 use_call_graph=False,
920 force_relink=True)
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]921 benchmark_results['Speedometer2.0'] = self._PerformanceBenchmark(
922 self._compiler.chrome_apk)
923 benchmark_results['orderfile.memory_mobile'] = (
924 self._NativeCodeMemoryBenchmark(self._compiler.chrome_apk))
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]925
926 except Exception as e:
927 benchmark_results['Error'] = str(e)
928
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]929 finally:
930 if no_orderfile and os.path.exists(backup_orderfile):
931 shutil.move(backup_orderfile, orderfile_path)
932 _StashOutputDirectory(out_directory)
933
934 return benchmark_results
935
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]936 def Generate(self):
937 """Generates and maybe upload an order."""
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]938 assert (bool(self._options.profile) ^
939 bool(self._options.manual_symbol_offsets))
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]940 if self._options.system_health_orderfile and not self._options.profile:
941 raise AssertionError('--system_health_orderfile must be not be used '
942 'with --skip-profile')
943 if (self._options.manual_symbol_offsets and
944 not self._options.system_health_orderfile):
945 raise AssertionError('--manual-symbol-offsets must be used with '
946 '--system_health_orderfile.')
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]947
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]948 if self._options.profile:
949 try:
950 _UnstashOutputDirectory(self._instrumented_out_dir)
951 self._compiler = ClankCompiler(
Christopher Grant073637472019-07-05 13:34:57[diff] [blame]952 self._instrumented_out_dir, self._step_recorder, self._options.arch,
953 self._options.use_goma, self._options.goma_dir,
954 self._options.system_health_orderfile, self._monochrome,
955 self._options.public, self._GetPathToOrderfile())
Matthew Carya2bea452018-11-13 10:21:07[diff] [blame]956 if not self._options.pregenerated_profiles:
957 # If there are pregenerated profiles, the instrumented build should
958 # not be changed to avoid invalidating the pregenerated profile
959 # offsets.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]960 self._compiler.CompileChromeApk(instrumented=True,
961 use_call_graph=
962 self._options.use_call_graph)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]963 self._GenerateAndProcessProfile()
964 self._MaybeArchiveOrderfile(self._GetUnpatchedOrderfileFilename())
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]965 finally:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]966 _StashOutputDirectory(self._instrumented_out_dir)
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]967 elif self._options.manual_symbol_offsets:
968 assert self._options.manual_libname
969 assert self._options.manual_objdir
Ari Chivukulaa52f8ba2021-08-10 22:30:39[diff] [blame]970 with open(self._options.manual_symbol_offsets) as f:
971 symbol_offsets = [int(x) for x in f]
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]972 processor = process_profiles.SymbolOffsetProcessor(
Matthew Caryb46ad282018-11-23 14:43:57[diff] [blame]973 self._compiler.manual_libname)
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]974 generator = cyglog_to_orderfile.OffsetOrderfileGenerator(
975 processor, cyglog_to_orderfile.ObjectFileProcessor(
976 self._options.manual_objdir))
977 ordered_sections = generator.GetOrderedSections(symbol_offsets)
978 if not ordered_sections: # Either None or empty is a problem.
979 raise Exception('Failed to get ordered sections')
980 with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile:
981 orderfile.write('\n'.join(ordered_sections))
982
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]983 if self._options.patch:
984 if self._options.profile:
985 self._RemoveBlanks(self._GetUnpatchedOrderfileFilename(),
986 self._GetPathToOrderfile())
987 try:
988 _UnstashOutputDirectory(self._uninstrumented_out_dir)
989 self._compiler = ClankCompiler(
990 self._uninstrumented_out_dir, self._step_recorder,
Christopher Grant073637472019-07-05 13:34:57[diff] [blame]991 self._options.arch, self._options.use_goma, self._options.goma_dir,
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]992 self._options.system_health_orderfile, self._monochrome,
993 self._options.public, self._GetPathToOrderfile())
994
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]995 self._compiler.CompileLibchrome(instrumented=False,
996 use_call_graph=False)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]997 self._PatchOrderfile()
998 # Because identical code folding is a bit different with and without
999 # the orderfile build, we need to re-patch the orderfile with code
1000 # folding as close to the final version as possible.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]1001 self._compiler.CompileLibchrome(instrumented=False,
1002 use_call_graph=False, force_relink=True)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1003 self._PatchOrderfile()
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]1004 self._compiler.CompileLibchrome(instrumented=False,
1005 use_call_graph=False, force_relink=True)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1006 self._VerifySymbolOrder()
1007 self._MaybeArchiveOrderfile(self._GetPathToOrderfile())
1008 finally:
1009 _StashOutputDirectory(self._uninstrumented_out_dir)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1010
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]1011 if self._options.benchmark:
1012 self._output_data['orderfile_benchmark_results'] = self.RunBenchmark(
1013 self._uninstrumented_out_dir)
1014 self._output_data['no_orderfile_benchmark_results'] = self.RunBenchmark(
1015 self._no_orderfile_out_dir, no_orderfile=True)
1016
Egor Paskod80bfad2020-09-10 21:53:06[diff] [blame]1017 if self._options.buildbot:
1018 self._orderfile_updater._GitStash()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1019 self._step_recorder.EndStep()
1020 return not self._step_recorder.ErrorRecorded()
1021
1022 def GetReportingData(self):
1023 """Get a dictionary of reporting data (timings, output hashes)"""
1024 self._output_data['timings'] = self._step_recorder.timings
1025 return self._output_data
1026
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]1027 def CommitStashedOrderfileHashes(self):
1028 """Commit any orderfile hash files in the current checkout.
1029
1030 Only possible if running on the buildbot.
1031
1032 Returns: true on success.
1033 """
Egor Pasko7ff04122019-11-25 15:47:18[diff] [blame]1034 if not self._options.buildbot:
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]1035 logging.error('Trying to commit when not running on the buildbot')
1036 return False
1037 self._orderfile_updater._CommitStashedFiles([
1038 filename + '.sha1'
1039 for filename in (self._GetUnpatchedOrderfileFilename(),
1040 self._GetPathToOrderfile())])
1041 return True
1042
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1043
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1044def CreateArgumentParser():
1045 """Creates and returns the argument parser."""
1046 parser = argparse.ArgumentParser()
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]1047 parser.add_argument('--no-benchmark', action='store_false', dest='benchmark',
1048 default=True, help='Disables running benchmarks.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1049 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1050 '--buildbot', action='store_true',
1051 help='If true, the script expects to be run on a buildbot')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1052 parser.add_argument(
Matthew Cary453ff1452018-07-18 12:19:35[diff] [blame]1053 '--device', default=None, type=str,
1054 help='Device serial number on which to run profiling.')
1055 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1056 '--verify', action='store_true',
1057 help='If true, the script only verifies the current orderfile')
Benoit Lizef5581722021-08-26 09:48:20[diff] [blame]1058 parser.add_argument('--target-arch',
1059 action='store',
1060 dest='arch',
Stephen Martinis71b391b52018-12-04 15:08:04[diff] [blame]1061 default='arm',
Benoit Lizef5581722021-08-26 09:48:20[diff] [blame]1062 choices=list(_ARCH_GN_ARGS.keys()),
Matthew Cary53f74ba2019-01-24 10:07:18[diff] [blame]1063 help='The target architecture for which to build.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1064 parser.add_argument('--output-json', action='store', dest='json_file',
1065 help='Location to save stats in json format')
1066 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1067 '--skip-profile', action='store_false', dest='profile', default=True,
1068 help='Don\'t generate a profile on the device. Only patch from the '
1069 'existing profile.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1070 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1071 '--skip-patch', action='store_false', dest='patch', default=True,
1072 help='Only generate the raw (unpatched) orderfile, don\'t patch it.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1073 parser.add_argument('--goma-dir', help='GOMA directory.')
1074 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1075 '--use-goma', action='store_true', help='Enable GOMA.', default=False)
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1076 parser.add_argument('--adb-path', help='Path to the adb binary.')
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]1077
Egor Paskod80bfad2020-09-10 21:53:06[diff] [blame]1078 parser.add_argument('--public',
1079 action='store_true',
1080 help='Build non-internal APK and change the orderfile '
1081 'location. Required if your checkout is non-internal.',
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]1082 default=False)
Matthew Cary04f41032018-12-10 15:55:27[diff] [blame]1083 parser.add_argument('--nosystem-health-orderfile', action='store_false',
Benoit Lcba421e2019-04-26 15:28:26[diff] [blame]1084 dest='system_health_orderfile', default=True,
Matthew Caryc262f5d2018-09-19 14:36:28[diff] [blame]1085 help=('Create an orderfile based on an about:blank '
1086 'startup benchmark instead of system health '
1087 'benchmarks.'))
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]1088 parser.add_argument(
1089 '--use-legacy-chrome-apk', action='store_true', default=False,
1090 help=('Compile and instrument chrome for [L, K] devices.'))
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]1091 parser.add_argument('--manual-symbol-offsets', default=None, type=str,
1092 help=('File of list of ordered symbol offsets generated '
1093 'by manual profiling. Must set other --manual* '
1094 'flags if this is used, and must --skip-profile.'))
1095 parser.add_argument('--manual-libname', default=None, type=str,
1096 help=('Library filename corresponding to '
1097 '--manual-symbol-offsets.'))
1098 parser.add_argument('--manual-objdir', default=None, type=str,
1099 help=('Root of object file directory corresponding to '
1100 '--manual-symbol-offsets.'))
Matthew Caryf949bba2019-02-04 13:39:23[diff] [blame]1101 parser.add_argument('--noorder-outlined-functions', action='store_true',
1102 help='Disable outlined functions in the orderfile.')
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]1103 parser.add_argument('--pregenerated-profiles', default=None, type=str,
1104 help=('Pregenerated profiles to use instead of running '
1105 'profile step. Cannot be used with '
1106 '--skip-profiles.'))
1107 parser.add_argument('--profile-save-dir', default=None, type=str,
1108 help=('Directory to save any profiles created. These can '
1109 'be used with --pregenerated-profiles. Cannot be '
1110 'used with --skip-profiles.'))
Egor Paskobce64d012018-11-20 12:02:37[diff] [blame]1111 parser.add_argument('--upload-ready-orderfiles', action='store_true',
1112 help=('Skip orderfile generation and manually upload '
1113 'orderfiles (both patched and unpatched) from '
1114 'their normal location in the tree to the cloud '
1115 'storage. DANGEROUS! USE WITH CARE!'))
Matthew Cary1ea82212019-03-20 12:10:27[diff] [blame]1116 parser.add_argument('--streamline-for-debugging', action='store_true',
1117 help=('Streamline where possible the run for faster '
1118 'iteration while debugging. The orderfile '
1119 'generated will be valid and nontrivial, but '
1120 'may not be based on a representative profile '
1121 'or other such considerations. Use with caution.'))
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]1122 parser.add_argument('--commit-hashes', action='store_true',
1123 help=('Commit any orderfile hash files in the current '
1124 'checkout; performs no other action'))
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]1125 parser.add_argument('--use-call-graph', action='store_true', default=False,
1126 help='Use call graph instrumentation.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1127 profile_android_startup.AddProfileCollectionArguments(parser)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1128 return parser
1129
1130
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]1131def CreateOrderfile(options, orderfile_updater_class=None):
Christopher Grantdfe1bac2019-07-05 13:34:10[diff] [blame]1132 """Creates an orderfile.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1133
1134 Args:
1135 options: As returned from optparse.OptionParser.parse_args()
1136 orderfile_updater_class: (OrderfileUpdater) subclass of OrderfileUpdater.
1137
1138 Returns:
1139 True iff success.
1140 """
1141 logging.basicConfig(level=logging.INFO)
1142 devil_chromium.Initialize(adb_path=options.adb_path)
1143
Benoit Lize6208ddc2021-08-30 12:26:29[diff] [blame^]1144 # Since we generate a ".arm" orderfile irrespective of the architecture (see
1145 # comment in _GetPathToOrderfile()), make sure that we don't commit it.
1146 if options.arch != 'arm':
1147 assert not options.buildbot, (
1148 'ARM is the only supported architecture on bots')
1149 assert not options.upload_ready_orderfiles, (
1150 'ARM is the only supported architecture on bots')
1151
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]1152 generator = OrderfileGenerator(options, orderfile_updater_class)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1153 try:
1154 if options.verify:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]1155 generator._VerifySymbolOrder()
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]1156 elif options.commit_hashes:
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]1157 return generator.CommitStashedOrderfileHashes()
Egor Paskobce64d012018-11-20 12:02:37[diff] [blame]1158 elif options.upload_ready_orderfiles:
1159 return generator.UploadReadyOrderfiles()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1160 else:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]1161 return generator.Generate()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1162 finally:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]1163 json_output = json.dumps(generator.GetReportingData(),
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1164 indent=2) + '\n'
1165 if options.json_file:
1166 with open(options.json_file, 'w') as f:
1167 f.write(json_output)
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]1168 print(json_output)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1169 return False
1170
1171
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1172def main():
1173 parser = CreateArgumentParser()
1174 options = parser.parse_args()
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]1175 return 0 if CreateOrderfile(options) else 1
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1176
1177
1178if __name__ == '__main__':
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1179 sys.exit(main())