[go: nahoru, domu]
Google Git
Sign in
chromium / chromium / src / ee76092107a4b16882fbf919a734a596a86cbcc8 / . / tools / cygprofile / orderfile_generator_backend.py
blob: a21921428fc25399caf2ad5b37dd7c5f9547f77e [file] [log] [blame]
Ari Chivukulaa52f8ba2021-08-10 22:30:39[diff] [blame]1#!/usr/bin/env vpython3
Avi Drissmandfd880852022-09-15 20:11:09[diff] [blame]2# Copyright 2013 The Chromium Authors
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6""" A utility to generate an up-to-date orderfile.
7
8The orderfile is used by the linker to order text sections such that the
9sections are placed consecutively in the order specified. This allows us
10to page in less code during start-up.
11
12Example usage:
Egor Paskoa7ac67a42020-09-09 12:06:58[diff] [blame]13 tools/cygprofile/orderfile_generator_backend.py --use-goma --target-arch=arm
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]14"""
15
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]16
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]17import argparse
Egor Paskoa7ac67a42020-09-09 12:06:58[diff] [blame]18import csv
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]19import hashlib
20import json
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]21import glob
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]22import logging
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]23import os
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]24import shutil
25import subprocess
26import sys
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]27import tempfile
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]28import time
29
Matthew Cary91df9792018-11-30 14:35:15[diff] [blame]30import cluster
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]31import cyglog_to_orderfile
Benoit Lizefefbb27c2018-01-17 13:54:18[diff] [blame]32import patch_orderfile
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]33import process_profiles
Egor Pasko3bc0b932018-04-03 10:08:27[diff] [blame]34import profile_android_startup
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]35
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]36_SRC_PATH = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
37sys.path.append(os.path.join(_SRC_PATH, 'third_party', 'catapult', 'devil'))
38from devil.android import device_utils
39from devil.android.sdk import version_codes
40
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]41
42_SRC_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)),
43 os.pardir, os.pardir)
44sys.path.append(os.path.join(_SRC_PATH, 'build', 'android'))
45import devil_chromium
46from pylib import constants
47
48
49# Needs to happen early for GetBuildType()/GetOutDirectory() to work correctly
50constants.SetBuildType('Release')
51
52
Matthew Cary53f74ba2019-01-24 10:07:18[diff] [blame]53# Architecture specific GN args. Trying to build an orderfile for an
54# architecture not listed here will eventually throw.
55_ARCH_GN_ARGS = {
Egor Paskoee760922024-01-11 19:36:31[diff] [blame^]56 'arm': ['target_cpu="arm"'],
57 'arm64': [
58 'target_cpu="arm64"', 'android_64bit_browser=true',
59 'is_high_end_android=false'
60 ],
61 'x86': ['target_cpu="x86"'],
Matthew Cary53f74ba2019-01-24 10:07:18[diff] [blame]62}
63
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]64class CommandError(Exception):
65 """Indicates that a dispatched shell command exited with a non-zero status."""
66
67 def __init__(self, value):
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]68 super().__init__()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]69 self.value = value
70
71 def __str__(self):
72 return repr(self.value)
73
74
75def _GenerateHash(file_path):
76 """Calculates and returns the hash of the file at file_path."""
77 sha1 = hashlib.sha1()
78 with open(file_path, 'rb') as f:
79 while True:
80 # Read in 1mb chunks, so it doesn't all have to be loaded into memory.
81 chunk = f.read(1024 * 1024)
82 if not chunk:
83 break
84 sha1.update(chunk)
85 return sha1.hexdigest()
86
87
88def _GetFileExtension(file_name):
89 """Calculates the file extension from a file name.
90
91 Args:
92 file_name: The source file name.
93 Returns:
94 The part of file_name after the dot (.) or None if the file has no
95 extension.
96 Examples: /home/user/foo.bar -> bar
97 /home/user.name/foo -> None
98 /home/user/.foo -> None
99 /home/user/foo.bar.baz -> baz
100 """
101 file_name_parts = os.path.basename(file_name).split('.')
102 if len(file_name_parts) > 1:
103 return file_name_parts[-1]
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]104 return None
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]105
106
107def _StashOutputDirectory(buildpath):
108 """Takes the output directory and stashes it in the default output directory.
109
110 This allows it to be used for incremental builds next time (after unstashing)
111 by keeping it in a place that isn't deleted normally, while also ensuring
112 that it is properly clobbered when appropriate.
113
114 This is a dirty hack to deal with the needs of clobbering while also handling
115 incremental builds and the hardcoded relative paths used in some of the
116 project files.
117
118 Args:
119 buildpath: The path where the building happens. If this corresponds to the
120 default output directory, no action is taken.
121 """
122 if os.path.abspath(buildpath) == os.path.abspath(os.path.dirname(
123 constants.GetOutDirectory())):
124 return
125 name = os.path.basename(buildpath)
126 stashpath = os.path.join(constants.GetOutDirectory(), name)
127 if not os.path.exists(buildpath):
128 return
129 if os.path.exists(stashpath):
130 shutil.rmtree(stashpath, ignore_errors=True)
131 shutil.move(buildpath, stashpath)
132
133
134def _UnstashOutputDirectory(buildpath):
135 """Inverse of _StashOutputDirectory.
136
137 Moves the output directory stashed within the default output directory
138 (out/Release) to the position where the builds can actually happen.
139
140 This is a dirty hack to deal with the needs of clobbering while also handling
141 incremental builds and the hardcoded relative paths used in some of the
142 project files.
143
144 Args:
145 buildpath: The path where the building happens. If this corresponds to the
146 default output directory, no action is taken.
147 """
148 if os.path.abspath(buildpath) == os.path.abspath(os.path.dirname(
149 constants.GetOutDirectory())):
150 return
151 name = os.path.basename(buildpath)
152 stashpath = os.path.join(constants.GetOutDirectory(), name)
153 if not os.path.exists(stashpath):
154 return
155 if os.path.exists(buildpath):
156 shutil.rmtree(buildpath, ignore_errors=True)
157 shutil.move(stashpath, buildpath)
158
159
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]160class StepRecorder:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]161 """Records steps and timings."""
162
163 def __init__(self, buildbot):
164 self.timings = []
165 self._previous_step = ('', 0.0)
166 self._buildbot = buildbot
167 self._error_recorded = False
168
169 def BeginStep(self, name):
170 """Marks a beginning of the next step in the script.
171
172 On buildbot, this prints a specially formatted name that will show up
173 in the waterfall. Otherwise, just prints the step name.
174
175 Args:
176 name: The name of the step.
177 """
178 self.EndStep()
179 self._previous_step = (name, time.time())
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]180 print('Running step: ', name)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]181
182 def EndStep(self):
183 """Records successful completion of the current step.
184
185 This is optional if the step is immediately followed by another BeginStep.
186 """
187 if self._previous_step[0]:
188 elapsed = time.time() - self._previous_step[1]
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]189 print('Step %s took %f seconds' % (self._previous_step[0], elapsed))
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]190 self.timings.append((self._previous_step[0], elapsed))
191
192 self._previous_step = ('', 0.0)
193
194 def FailStep(self, message=None):
195 """Marks that a particular step has failed.
196
197 On buildbot, this will mark the current step as failed on the waterfall.
198 Otherwise we will just print an optional failure message.
199
200 Args:
201 message: An optional explanation as to why the step failed.
202 """
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]203 print('STEP FAILED!!')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]204 if message:
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]205 print(message)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]206 self._error_recorded = True
207 self.EndStep()
208
209 def ErrorRecorded(self):
210 """True if FailStep has been called."""
211 return self._error_recorded
212
213 def RunCommand(self, cmd, cwd=constants.DIR_SOURCE_ROOT, raise_on_error=True,
214 stdout=None):
215 """Execute a shell command.
216
217 Args:
218 cmd: A list of command strings.
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]219 cwd: Directory in which the command should be executed, defaults to build
220 root of script's location if not specified.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]221 raise_on_error: If true will raise a CommandError if the call doesn't
222 succeed and mark the step as failed.
223 stdout: A file to redirect stdout for the command to.
224
225 Returns:
226 The process's return code.
227
228 Raises:
229 CommandError: An error executing the specified command.
230 """
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]231 print('Executing %s in %s' % (' '.join(cmd), cwd))
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]232 process = subprocess.Popen(cmd, stdout=stdout, cwd=cwd, env=os.environ)
233 process.wait()
234 if raise_on_error and process.returncode != 0:
235 self.FailStep()
236 raise CommandError('Exception executing command %s' % ' '.join(cmd))
237 return process.returncode
238
239
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]240class ClankCompiler:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]241 """Handles compilation of clank."""
242
Christopher Grant073637472019-07-05 13:34:57[diff] [blame]243 def __init__(self, out_dir, step_recorder, arch, use_goma, goma_dir,
Fumitoshi Ukaibc233ba2023-02-13 11:47:31[diff] [blame]244 use_remoteexec, ninja_command, system_health_profiling,
245 monochrome, public, orderfile_location):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]246 self._out_dir = out_dir
247 self._step_recorder = step_recorder
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]248 self._arch = arch
Fumitoshi Ukaibd2755b2022-10-28 13:15:27[diff] [blame]249 # TODO(b/236070141): remove goma config.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]250 self._use_goma = use_goma
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]251 self._goma_dir = goma_dir
Fumitoshi Ukaibd2755b2022-10-28 13:15:27[diff] [blame]252 self._use_remoteexec = use_remoteexec
Fumitoshi Ukaibc233ba2023-02-13 11:47:31[diff] [blame]253 self._ninja_command = ninja_command
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]254 self._system_health_profiling = system_health_profiling
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]255 self._public = public
256 self._orderfile_location = orderfile_location
Matthew Caryd6bfcb72018-09-14 11:27:46[diff] [blame]257 if monochrome:
258 self._apk = 'Monochrome.apk'
Matthew Carye1b00062018-09-19 14:27:44[diff] [blame]259 self._apk_target = 'monochrome_apk'
Matthew Caryd6bfcb72018-09-14 11:27:46[diff] [blame]260 self._libname = 'libmonochrome'
Matthew Cary2216fef2019-06-17 13:35:59[diff] [blame]261 self._libchrome_target = 'libmonochrome'
Matthew Caryd6bfcb72018-09-14 11:27:46[diff] [blame]262 else:
263 self._apk = 'Chrome.apk'
Matthew Carye1b00062018-09-19 14:27:44[diff] [blame]264 self._apk_target = 'chrome_apk'
Matthew Caryd6bfcb72018-09-14 11:27:46[diff] [blame]265 self._libname = 'libchrome'
266 self._libchrome_target = 'libchrome'
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]267 if public:
268 self._apk = self._apk.replace('.apk', 'Public.apk')
269 self._apk_target = self._apk_target.replace('_apk', '_public_apk')
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]270
271 self.obj_dir = os.path.join(self._out_dir, 'Release', 'obj')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]272 self.lib_chrome_so = os.path.join(
Matthew Caryd6bfcb72018-09-14 11:27:46[diff] [blame]273 self._out_dir, 'Release', 'lib.unstripped',
274 '{}.so'.format(self._libname))
275 self.chrome_apk = os.path.join(self._out_dir, 'Release', 'apks', self._apk)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]276
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]277 def Build(self, instrumented, use_call_graph, target):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]278 """Builds the provided ninja target with or without order_profiling on.
279
280 Args:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]281 instrumented: (bool) Whether we want to build an instrumented binary.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]282 use_call_graph: (bool) Whether to use the call graph instrumentation.
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]283 target: (str) The name of the ninja target to build.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]284 """
285 self._step_recorder.BeginStep('Compile %s' % target)
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]286 assert not use_call_graph or instrumented, ('You can not enable call graph '
287 'without instrumentation!')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]288
289 # Set the "Release Official" flavor, the parts affecting performance.
290 args = [
Andrew Grieve0e8790c2020-09-03 17:27:32[diff] [blame]291 'enable_resource_allowlist_generation=false',
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]292 'is_chrome_branded=' + str(not self._public).lower(),
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]293 'is_debug=false',
294 'is_official_build=true',
Egor Pasko1d13d532020-01-14 21:25:19[diff] [blame]295 'symbol_level=1', # to fit 30 GiB RAM on the bot when LLD is running
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]296 'target_os="android"',
297 'use_goma=' + str(self._use_goma).lower(),
Fumitoshi Ukaibd2755b2022-10-28 13:15:27[diff] [blame]298 'use_remoteexec=' + str(self._use_remoteexec).lower(),
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]299 'use_order_profiling=' + str(instrumented).lower(),
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]300 'use_call_graph=' + str(use_call_graph).lower(),
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]301 ]
Matthew Cary53f74ba2019-01-24 10:07:18[diff] [blame]302 args += _ARCH_GN_ARGS[self._arch]
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]303 if self._goma_dir:
304 args += ['goma_dir="%s"' % self._goma_dir]
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]305 if self._system_health_profiling:
306 args += ['devtools_instrumentation_dumping = ' +
307 str(instrumented).lower()]
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]308
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]309 if self._public and os.path.exists(self._orderfile_location):
310 # GN needs the orderfile path to be source-absolute.
311 src_abs_orderfile = os.path.relpath(self._orderfile_location,
312 constants.DIR_SOURCE_ROOT)
313 args += ['chrome_orderfile="//{}"'.format(src_abs_orderfile)]
314
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]315 self._step_recorder.RunCommand(
316 ['gn', 'gen', os.path.join(self._out_dir, 'Release'),
317 '--args=' + ' '.join(args)])
318
319 self._step_recorder.RunCommand(
Fumitoshi Ukaibc233ba2023-02-13 11:47:31[diff] [blame]320 self._ninja_command + [os.path.join(self._out_dir, 'Release'), target])
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]321
Christopher Grant10221762019-07-05 12:10:04[diff] [blame]322 def ForceRelink(self):
323 """Forces libchrome.so or libmonochrome.so to be re-linked.
324
325 With partitioned libraries enabled, deleting these library files does not
326 guarantee they'll be recreated by the linker (they may simply be
327 re-extracted from a combined library). To be safe, touch a source file
328 instead. See http://crbug.com/972701 for more explanation.
329 """
330 file_to_touch = os.path.join(constants.DIR_SOURCE_ROOT, 'chrome', 'browser',
331 'chrome_browser_main_android.cc')
332 assert os.path.exists(file_to_touch)
333 self._step_recorder.RunCommand(['touch', file_to_touch])
334
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]335 def CompileChromeApk(self, instrumented, use_call_graph, force_relink=False):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]336 """Builds a Chrome.apk either with or without order_profiling on.
337
338 Args:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]339 instrumented: (bool) Whether to build an instrumented apk.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]340 use_call_graph: (bool) Whether to use the call graph instrumentation.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]341 force_relink: Whether libchromeview.so should be re-created.
342 """
343 if force_relink:
Christopher Grant10221762019-07-05 12:10:04[diff] [blame]344 self.ForceRelink()
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]345 self.Build(instrumented, use_call_graph, self._apk_target)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]346
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]347 def CompileLibchrome(self, instrumented, use_call_graph, force_relink=False):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]348 """Builds a libchrome.so either with or without order_profiling on.
349
350 Args:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]351 instrumented: (bool) Whether to build an instrumented apk.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]352 use_call_graph: (bool) Whether to use the call graph instrumentation.
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]353 force_relink: (bool) Whether libchrome.so should be re-created.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]354 """
355 if force_relink:
Christopher Grant10221762019-07-05 12:10:04[diff] [blame]356 self.ForceRelink()
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]357 self.Build(instrumented, use_call_graph, self._libchrome_target)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]358
359
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]360class OrderfileUpdater:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]361 """Handles uploading and committing a new orderfile in the repository.
362
363 Only used for testing or on a bot.
364 """
365
366 _CLOUD_STORAGE_BUCKET_FOR_DEBUG = None
367 _CLOUD_STORAGE_BUCKET = None
368 _UPLOAD_TO_CLOUD_COMMAND = 'upload_to_google_storage.py'
369
Egor Pasko0c533c6682019-11-26 21:16:32[diff] [blame]370 def __init__(self, repository_root, step_recorder):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]371 """Constructor.
372
373 Args:
374 repository_root: (str) Root of the target repository.
375 step_recorder: (StepRecorder) Step recorder, for logging.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]376 """
377 self._repository_root = repository_root
378 self._step_recorder = step_recorder
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]379
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]380 def CommitStashedFileHashes(self, files):
381 """Commits unpatched and patched orderfiles hashes if changed.
382
383 The files are committed only if their associated sha1 hash files match, and
384 are modified in git. In normal operations the hash files are changed only
385 when a file is uploaded to cloud storage. If the hash file is not modified
386 in git, the file is skipped.
387
388 Args:
389 files: [str or None] specifies file paths. None items are ignored.
390
391 Raises:
392 Exception if the hash file does not match the file.
393 NotImplementedError when the commit logic hasn't been overridden.
394 """
Ari Chivukulaa52f8ba2021-08-10 22:30:39[diff] [blame]395 files_to_commit = [_f for _f in files if _f]
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]396 if files_to_commit:
397 self._CommitStashedFiles(files_to_commit)
398
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]399 def UploadToCloudStorage(self, filename, use_debug_location):
400 """Uploads a file to cloud storage.
401
402 Args:
403 filename: (str) File to upload.
404 use_debug_location: (bool) Whether to use the debug location.
405 """
406 bucket = (self._CLOUD_STORAGE_BUCKET_FOR_DEBUG if use_debug_location
407 else self._CLOUD_STORAGE_BUCKET)
408 extension = _GetFileExtension(filename)
409 cmd = [self._UPLOAD_TO_CLOUD_COMMAND, '--bucket', bucket]
410 if extension:
411 cmd.extend(['-z', extension])
412 cmd.append(filename)
413 self._step_recorder.RunCommand(cmd)
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]414 print('Download: https://sandbox.google.com/storage/%s/%s' %
415 (bucket, _GenerateHash(filename)))
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]416
417 def _GetHashFilePathAndContents(self, filename):
418 """Gets the name and content of the hash file created from uploading the
419 given file.
420
421 Args:
422 filename: (str) The file that was uploaded to cloud storage.
423
424 Returns:
425 A tuple of the hash file name, relative to the reository root, and the
426 content, which should be the sha1 hash of the file
427 ('base_file.sha1', hash)
428 """
429 abs_hash_filename = filename + '.sha1'
430 rel_hash_filename = os.path.relpath(
431 abs_hash_filename, self._repository_root)
432 with open(abs_hash_filename, 'r') as f:
433 return (rel_hash_filename, f.read())
434
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]435 def _GitStash(self):
436 """Git stash the current clank tree.
437
438 Raises:
439 NotImplementedError when the stash logic hasn't been overridden.
440 """
441 raise NotImplementedError
442
443 def _CommitStashedFiles(self, expected_files_in_stash):
444 """Commits stashed files.
445
446 The local repository is updated and then the files to commit are taken from
447 modified files from the git stash. The modified files should be a subset of
448 |expected_files_in_stash|. If there are unexpected modified files, this
449 function may raise. This is meant to be paired with _GitStash().
450
451 Args:
452 expected_files_in_stash: [str] paths to a possible superset of files
453 expected to be stashed & committed.
454
455 Raises:
456 NotImplementedError when the commit logic hasn't been overridden.
457 """
458 raise NotImplementedError
459
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]460
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]461class OrderfileGenerator:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]462 """A utility for generating a new orderfile for Clank.
463
464 Builds an instrumented binary, profiles a run of the application, and
465 generates an updated orderfile.
466 """
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]467 _CHECK_ORDERFILE_SCRIPT = os.path.join(
468 constants.DIR_SOURCE_ROOT, 'tools', 'cygprofile', 'check_orderfile.py')
469 _BUILD_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(
470 constants.GetOutDirectory()))) # Normally /path/to/src
471
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]472 # Previous orderfile_generator debug files would be overwritten.
473 _DIRECTORY_FOR_DEBUG_FILES = '/tmp/orderfile_generator_debug_files'
474
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]475 _CLOUD_STORAGE_BUCKET_FOR_DEBUG = None
476
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]477 def _PrepareOrderfilePaths(self):
478 if self._options.public:
479 self._clank_dir = os.path.join(constants.DIR_SOURCE_ROOT,
480 '')
481 if not os.path.exists(os.path.join(self._clank_dir, 'orderfiles')):
482 os.makedirs(os.path.join(self._clank_dir, 'orderfiles'))
483 else:
484 self._clank_dir = os.path.join(constants.DIR_SOURCE_ROOT,
485 'clank')
486
487 self._unpatched_orderfile_filename = os.path.join(
488 self._clank_dir, 'orderfiles', 'unpatched_orderfile.%s')
489 self._path_to_orderfile = os.path.join(
490 self._clank_dir, 'orderfiles', 'orderfile.%s.out')
491
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]492 def _GetPathToOrderfile(self):
493 """Gets the path to the architecture-specific orderfile."""
Benoit Lize6208ddc2021-08-30 12:26:29[diff] [blame]494 # Build GN files use the ".arm" orderfile irrespective of the actual
495 # architecture. Fake it, otherwise the orderfile we generate here is not
496 # going to be picked up by builds.
497 orderfile_fake_arch = 'arm'
498 return self._path_to_orderfile % orderfile_fake_arch
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]499
500 def _GetUnpatchedOrderfileFilename(self):
501 """Gets the path to the architecture-specific unpatched orderfile."""
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]502 return self._unpatched_orderfile_filename % self._options.arch
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]503
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]504 def _SetDevice(self):
505 """ Selects the device to be used by the script.
506
507 Returns:
508 (Device with given serial ID) : if the --device flag is set.
509 (Device running Android[K,L]) : if --use-legacy-chrome-apk flag is set or
510 no device running Android N+ was found.
511 (Device running Android N+) : Otherwise.
512
513 Raises Error:
514 If no device meeting the requirements has been found.
515 """
516 devices = None
517 if self._options.device:
518 devices = [device_utils.DeviceUtils(self._options.device)]
519 else:
520 devices = device_utils.DeviceUtils.HealthyDevices()
521
522 assert devices, 'Expected at least one connected device'
523
524 if self._options.use_legacy_chrome_apk:
525 self._monochrome = False
526 for device in devices:
527 device_version = device.build_version_sdk
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]528 if (version_codes.KITKAT <= device_version <=
529 version_codes.LOLLIPOP_MR1):
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]530 return device
531
532 assert not self._options.use_legacy_chrome_apk, \
533 'No device found running suitable android version for Chrome.apk.'
534
535 preferred_device = None
536 for device in devices:
537 if device.build_version_sdk >= version_codes.NOUGAT:
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]538 preferred_device = device
539 break
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]540
541 self._monochrome = preferred_device is not None
542
543 return preferred_device if preferred_device else devices[0]
544
545
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]546 def __init__(self, options, orderfile_updater_class):
547 self._options = options
Fumitoshi Ukaibc233ba2023-02-13 11:47:31[diff] [blame]548 self._ninja_command = ['autoninja']
549 if self._options.ninja_path:
550 self._ninja_command = [self._options.ninja_path]
551 if self._options.ninja_j:
552 self._ninja_command += ['-j', self._options.ninja_j]
553 self._ninja_command += ['-C']
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]554 self._instrumented_out_dir = os.path.join(
555 self._BUILD_ROOT, self._options.arch + '_instrumented_out')
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]556 if self._options.use_call_graph:
Christopher Grantdfe1bac2019-07-05 13:34:10[diff] [blame]557 self._instrumented_out_dir += '_call_graph'
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]558
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]559 self._uninstrumented_out_dir = os.path.join(
560 self._BUILD_ROOT, self._options.arch + '_uninstrumented_out')
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]561 self._no_orderfile_out_dir = os.path.join(
562 self._BUILD_ROOT, self._options.arch + '_no_orderfile_out')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]563
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]564 self._PrepareOrderfilePaths()
565
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]566 if options.profile:
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]567 output_directory = os.path.join(self._instrumented_out_dir, 'Release')
Matthew Caryb8daed942018-06-11 10:58:08[diff] [blame]568 host_profile_dir = os.path.join(output_directory, 'profile_data')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]569 urls = [profile_android_startup.AndroidProfileTool.TEST_URL]
570 use_wpr = True
571 simulate_user = False
Benoit L96466812018-03-06 15:58:37[diff] [blame]572 urls = options.urls
573 use_wpr = not options.no_wpr
574 simulate_user = options.simulate_user
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]575 device = self._SetDevice()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]576 self._profiler = profile_android_startup.AndroidProfileTool(
Matthew Cary453ff1452018-07-18 12:19:35[diff] [blame]577 output_directory, host_profile_dir, use_wpr, urls, simulate_user,
Matthew Cary1ea82212019-03-20 12:10:27[diff] [blame]578 device, debug=self._options.streamline_for_debugging)
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]579 if options.pregenerated_profiles:
580 self._profiler.SetPregeneratedProfiles(
581 glob.glob(options.pregenerated_profiles))
582 else:
583 assert not options.pregenerated_profiles, (
584 '--pregenerated-profiles cannot be used with --skip-profile')
585 assert not options.profile_save_dir, (
586 '--profile-save-dir cannot be used with --skip-profile')
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]587 self._monochrome = not self._options.use_legacy_chrome_apk
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]588
Matthew Caryf949bba2019-02-04 13:39:23[diff] [blame]589 # Outlined function handling enabled by default for all architectures.
590 self._order_outlined_functions = not options.noorder_outlined_functions
591
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]592 self._output_data = {}
593 self._step_recorder = StepRecorder(options.buildbot)
594 self._compiler = None
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]595 if orderfile_updater_class is None:
Egor Paskod80bfad2020-09-10 21:53:06[diff] [blame]596 orderfile_updater_class = OrderfileUpdater
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]597 assert issubclass(orderfile_updater_class, OrderfileUpdater)
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]598 self._orderfile_updater = orderfile_updater_class(self._clank_dir,
Egor Pasko0c533c6682019-11-26 21:16:32[diff] [blame]599 self._step_recorder)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]600 assert os.path.isdir(constants.DIR_SOURCE_ROOT), 'No src directory found'
601
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]602 @staticmethod
603 def _RemoveBlanks(src_file, dest_file):
604 """A utility to remove blank lines from a file.
605
606 Args:
607 src_file: The name of the file to remove the blanks from.
608 dest_file: The name of the file to write the output without blanks.
609 """
610 assert src_file != dest_file, 'Source and destination need to be distinct'
611
612 try:
613 src = open(src_file, 'r')
614 dest = open(dest_file, 'w')
615 for line in src:
616 if line and not line.isspace():
617 dest.write(line)
618 finally:
619 src.close()
620 dest.close()
621
622 def _GenerateAndProcessProfile(self):
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]623 """Invokes a script to merge the per-thread traces into one file.
624
625 The produced list of offsets is saved in
626 self._GetUnpatchedOrderfileFilename().
627 """
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]628 self._step_recorder.BeginStep('Generate Profile Data')
629 files = []
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]630 logging.getLogger().setLevel(logging.DEBUG)
Christopher Grantdfe1bac2019-07-05 13:34:10[diff] [blame]631
632 if self._options.profile_save_dir:
633 # The directory must not preexist, to ensure purity of data. Check
634 # before profiling to save time.
635 if os.path.exists(self._options.profile_save_dir):
636 raise Exception('Profile save directory must not pre-exist')
637 os.makedirs(self._options.profile_save_dir)
638
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]639 if self._options.system_health_orderfile:
640 files = self._profiler.CollectSystemHealthProfile(
641 self._compiler.chrome_apk)
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]642 self._MaybeSaveProfile(files)
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]643 try:
644 self._ProcessPhasedOrderfile(files)
645 except Exception:
646 for f in files:
647 self._SaveForDebugging(f)
Matthew Cary8b1416232018-08-10 19:12:22[diff] [blame]648 self._SaveForDebugging(self._compiler.lib_chrome_so)
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]649 raise
650 finally:
651 self._profiler.Cleanup()
652 else:
653 self._CollectLegacyProfile()
654 logging.getLogger().setLevel(logging.INFO)
655
656 def _ProcessPhasedOrderfile(self, files):
657 """Process the phased orderfiles produced by system health benchmarks.
658
659 The offsets will be placed in _GetUnpatchedOrderfileFilename().
660
661 Args:
662 file: Profile files pulled locally.
663 """
664 self._step_recorder.BeginStep('Process Phased Orderfile')
665 profiles = process_profiles.ProfileManager(files)
666 processor = process_profiles.SymbolOffsetProcessor(
667 self._compiler.lib_chrome_so)
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]668 ordered_symbols = cluster.ClusterOffsets(profiles, processor,
669 call_graph=self._options.use_call_graph)
Matthew Cary8b1416232018-08-10 19:12:22[diff] [blame]670 if not ordered_symbols:
671 raise Exception('Failed to get ordered symbols')
Matthew Caryda9db932019-03-11 15:28:17[diff] [blame]672 for sym in ordered_symbols:
673 assert not sym.startswith('OUTLINED_FUNCTION_'), (
674 'Outlined function found in instrumented function, very likely '
675 'something has gone very wrong!')
Matthew Cary91df9792018-11-30 14:35:15[diff] [blame]676 self._output_data['offsets_kib'] = processor.SymbolsSize(
677 ordered_symbols) / 1024
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]678 with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile:
Matthew Cary8b1416232018-08-10 19:12:22[diff] [blame]679 orderfile.write('\n'.join(ordered_symbols))
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]680
681 def _CollectLegacyProfile(self):
Matthew Cary2df8d452018-09-19 07:50:20[diff] [blame]682 files = []
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]683 try:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]684 files = self._profiler.CollectProfile(
685 self._compiler.chrome_apk,
686 constants.PACKAGE_INFO['chrome'])
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]687 self._MaybeSaveProfile(files)
Matthew Caryb8daed942018-06-11 10:58:08[diff] [blame]688 self._step_recorder.BeginStep('Process profile')
Benoit L96466812018-03-06 15:58:37[diff] [blame]689 assert os.path.exists(self._compiler.lib_chrome_so)
690 offsets = process_profiles.GetReachedOffsetsFromDumpFiles(
691 files, self._compiler.lib_chrome_so)
692 if not offsets:
693 raise Exception('No profiler offsets found in {}'.format(
694 '\n'.join(files)))
Matthew Cary8b1416232018-08-10 19:12:22[diff] [blame]695 processor = process_profiles.SymbolOffsetProcessor(
696 self._compiler.lib_chrome_so)
697 ordered_symbols = processor.GetOrderedSymbols(offsets)
698 if not ordered_symbols:
699 raise Exception('No symbol names from offsets found in {}'.format(
700 '\n'.join(files)))
701 with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile:
702 orderfile.write('\n'.join(ordered_symbols))
Matthew Cary0f1f681a2018-01-22 10:40:51[diff] [blame]703 except Exception:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]704 for f in files:
705 self._SaveForDebugging(f)
706 raise
707 finally:
708 self._profiler.Cleanup()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]709
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]710 def _MaybeSaveProfile(self, files):
711 if self._options.profile_save_dir:
712 logging.info('Saving profiles to %s', self._options.profile_save_dir)
713 for f in files:
714 shutil.copy(f, self._options.profile_save_dir)
715 logging.info('Saved profile %s', f)
716
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]717 def _PatchOrderfile(self):
718 """Patches the orderfile using clean version of libchrome.so."""
719 self._step_recorder.BeginStep('Patch Orderfile')
Benoit Lizefefbb27c2018-01-17 13:54:18[diff] [blame]720 patch_orderfile.GeneratePatchedOrderfile(
721 self._GetUnpatchedOrderfileFilename(), self._compiler.lib_chrome_so,
Matthew Caryf949bba2019-02-04 13:39:23[diff] [blame]722 self._GetPathToOrderfile(), self._order_outlined_functions)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]723
724 def _VerifySymbolOrder(self):
725 self._step_recorder.BeginStep('Verify Symbol Order')
Andrew Grievec45bb952021-11-02 18:33:27[diff] [blame]726 return_code = self._step_recorder.RunCommand([
727 self._CHECK_ORDERFILE_SCRIPT, self._compiler.lib_chrome_so,
728 self._GetPathToOrderfile()
729 ],
730 constants.DIR_SOURCE_ROOT,
731 raise_on_error=False)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]732 if return_code:
733 self._step_recorder.FailStep('Orderfile check returned %d.' % return_code)
734
735 def _RecordHash(self, file_name):
736 """Records the hash of the file into the output_data dictionary."""
737 self._output_data[os.path.basename(file_name) + '.sha1'] = _GenerateHash(
738 file_name)
739
740 def _SaveFileLocally(self, file_name, file_sha1):
741 """Saves the file to a temporary location and prints the sha1sum."""
742 if not os.path.exists(self._DIRECTORY_FOR_DEBUG_FILES):
743 os.makedirs(self._DIRECTORY_FOR_DEBUG_FILES)
744 shutil.copy(file_name, self._DIRECTORY_FOR_DEBUG_FILES)
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]745 print('File: %s, saved in: %s, sha1sum: %s' %
746 (file_name, self._DIRECTORY_FOR_DEBUG_FILES, file_sha1))
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]747
748 def _SaveForDebugging(self, filename):
749 """Uploads the file to cloud storage or saves to a temporary location."""
750 file_sha1 = _GenerateHash(filename)
751 if not self._options.buildbot:
752 self._SaveFileLocally(filename, file_sha1)
753 else:
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]754 print('Uploading file for debugging: ' + filename)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]755 self._orderfile_updater.UploadToCloudStorage(
756 filename, use_debug_location=True)
757
758 def _SaveForDebuggingWithOverwrite(self, file_name):
759 """Uploads and overwrites the file in cloud storage or copies locally.
760
761 Should be used for large binaries like lib_chrome_so.
762
763 Args:
764 file_name: (str) File to upload.
765 """
766 file_sha1 = _GenerateHash(file_name)
767 if not self._options.buildbot:
768 self._SaveFileLocally(file_name, file_sha1)
769 else:
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]770 print('Uploading file for debugging: %s, sha1sum: %s' % (file_name,
771 file_sha1))
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]772 upload_location = '%s/%s' % (
773 self._CLOUD_STORAGE_BUCKET_FOR_DEBUG, os.path.basename(file_name))
774 self._step_recorder.RunCommand([
775 'gsutil.py', 'cp', file_name, 'gs://' + upload_location])
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]776 print('Uploaded to: https://sandbox.google.com/storage/' +
777 upload_location)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]778
779 def _MaybeArchiveOrderfile(self, filename):
780 """In buildbot configuration, uploads the generated orderfile to
781 Google Cloud Storage.
782
783 Args:
784 filename: (str) Orderfile to upload.
785 """
Matthew Cary91df9792018-11-30 14:35:15[diff] [blame]786 # First compute hashes so that we can download them later if we need to.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]787 self._step_recorder.BeginStep('Compute hash for ' + filename)
788 self._RecordHash(filename)
789 if self._options.buildbot:
790 self._step_recorder.BeginStep('Archive ' + filename)
791 self._orderfile_updater.UploadToCloudStorage(
792 filename, use_debug_location=False)
793
Egor Paskobce64d012018-11-20 12:02:37[diff] [blame]794 def UploadReadyOrderfiles(self):
795 self._step_recorder.BeginStep('Upload Ready Orderfiles')
796 for file_name in [self._GetUnpatchedOrderfileFilename(),
797 self._GetPathToOrderfile()]:
798 self._orderfile_updater.UploadToCloudStorage(
799 file_name, use_debug_location=False)
800
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]801 def _NativeCodeMemoryBenchmark(self, apk):
802 """Runs system_health.memory_mobile to assess native code memory footprint.
803
804 Args:
805 apk: (str) Path to the apk.
806
807 Returns:
808 results: ([int]) Values of native code memory footprint in bytes from the
809 benchmark results.
810 """
811 self._step_recorder.BeginStep("Running orderfile.memory_mobile")
812 try:
813 out_dir = tempfile.mkdtemp()
814 self._profiler._RunCommand(['tools/perf/run_benchmark',
815 '--device={}'.format(
816 self._profiler._device.serial),
817 '--browser=exact',
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]818 '--output-format=csv',
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]819 '--output-dir={}'.format(out_dir),
820 '--reset-results',
821 '--browser-executable={}'.format(apk),
822 'orderfile.memory_mobile'])
823
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]824 out_file_path = os.path.join(out_dir, 'results.csv')
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]825 if not os.path.exists(out_file_path):
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]826 raise Exception('Results file not found!')
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]827
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]828 results = {}
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]829 with open(out_file_path, 'r') as f:
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]830 reader = csv.DictReader(f)
831 for row in reader:
832 if not row['name'].endswith('NativeCodeResidentMemory'):
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]833 continue
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]834 # Note: NativeCodeResidentMemory records a single sample from each
835 # story run, so this average (reported as 'avg') is exactly the value
836 # of that one sample. Each story is run multiple times, so this loop
837 # will accumulate into a list all values for all runs of each story.
838 results.setdefault(row['name'], {}).setdefault(
839 row['stories'], []).append(row['avg'])
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]840
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]841 if not results:
842 raise Exception('Could not find relevant results')
843
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]844 return results
845
846 except Exception as e:
847 return 'Error: ' + str(e)
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]848
849 finally:
850 shutil.rmtree(out_dir)
851
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]852
853 def _PerformanceBenchmark(self, apk):
854 """Runs Speedometer2.0 to assess performance.
855
856 Args:
857 apk: (str) Path to the apk.
858
859 Returns:
860 results: ([float]) Speedometer2.0 results samples in milliseconds.
861 """
862 self._step_recorder.BeginStep("Running Speedometer2.0.")
863 try:
864 out_dir = tempfile.mkdtemp()
865 self._profiler._RunCommand(['tools/perf/run_benchmark',
866 '--device={}'.format(
867 self._profiler._device.serial),
868 '--browser=exact',
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]869 '--output-format=histograms',
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]870 '--output-dir={}'.format(out_dir),
871 '--reset-results',
872 '--browser-executable={}'.format(apk),
873 'speedometer2'])
874
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]875 out_file_path = os.path.join(out_dir, 'histograms.json')
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]876 if not os.path.exists(out_file_path):
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]877 raise Exception('Results file not found!')
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]878
879 with open(out_file_path, 'r') as f:
880 results = json.load(f)
881
882 if not results:
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]883 raise Exception('Results file is empty.')
884
885 for el in results:
886 if 'name' in el and el['name'] == 'Total' and 'sampleValues' in el:
887 return el['sampleValues']
888
889 raise Exception('Unexpected results format.')
890
891 except Exception as e:
892 return 'Error: ' + str(e)
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]893
894 finally:
895 shutil.rmtree(out_dir)
896
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]897
898 def RunBenchmark(self, out_directory, no_orderfile=False):
899 """Builds chrome apk and runs performance and memory benchmarks.
900
901 Builds a non-instrumented version of chrome.
902 Installs chrome apk on the device.
903 Runs Speedometer2.0 benchmark to assess performance.
904 Runs system_health.memory_mobile to evaluate memory footprint.
905
906 Args:
907 out_directory: (str) Path to out directory for this build.
908 no_orderfile: (bool) True if chrome to be built without orderfile.
909
910 Returns:
911 benchmark_results: (dict) Results extracted from benchmarks.
912 """
Benoit Lize74e82b32021-08-26 14:22:01[diff] [blame]913 benchmark_results = {}
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]914 try:
915 _UnstashOutputDirectory(out_directory)
Christopher Grant073637472019-07-05 13:34:57[diff] [blame]916 self._compiler = ClankCompiler(out_directory, self._step_recorder,
917 self._options.arch, self._options.use_goma,
918 self._options.goma_dir,
Fumitoshi Ukaibd2755b2022-10-28 13:15:27[diff] [blame]919 self._options.use_remoteexec,
Fumitoshi Ukaibc233ba2023-02-13 11:47:31[diff] [blame]920 self._ninja_command,
Christopher Grant073637472019-07-05 13:34:57[diff] [blame]921 self._options.system_health_orderfile,
922 self._monochrome, self._options.public,
923 self._GetPathToOrderfile())
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]924
925 if no_orderfile:
926 orderfile_path = self._GetPathToOrderfile()
927 backup_orderfile = orderfile_path + '.backup'
928 shutil.move(orderfile_path, backup_orderfile)
929 open(orderfile_path, 'w').close()
930
931 # Build APK to be installed on the device.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]932 self._compiler.CompileChromeApk(instrumented=False,
933 use_call_graph=False,
934 force_relink=True)
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]935 benchmark_results['Speedometer2.0'] = self._PerformanceBenchmark(
936 self._compiler.chrome_apk)
937 benchmark_results['orderfile.memory_mobile'] = (
938 self._NativeCodeMemoryBenchmark(self._compiler.chrome_apk))
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]939
940 except Exception as e:
941 benchmark_results['Error'] = str(e)
942
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]943 finally:
944 if no_orderfile and os.path.exists(backup_orderfile):
945 shutil.move(backup_orderfile, orderfile_path)
946 _StashOutputDirectory(out_directory)
947
948 return benchmark_results
949
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]950 def Generate(self):
951 """Generates and maybe upload an order."""
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]952 assert (bool(self._options.profile) ^
953 bool(self._options.manual_symbol_offsets))
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]954 if self._options.system_health_orderfile and not self._options.profile:
955 raise AssertionError('--system_health_orderfile must be not be used '
956 'with --skip-profile')
957 if (self._options.manual_symbol_offsets and
958 not self._options.system_health_orderfile):
959 raise AssertionError('--manual-symbol-offsets must be used with '
960 '--system_health_orderfile.')
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]961
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]962 if self._options.profile:
963 try:
964 _UnstashOutputDirectory(self._instrumented_out_dir)
965 self._compiler = ClankCompiler(
Christopher Grant073637472019-07-05 13:34:57[diff] [blame]966 self._instrumented_out_dir, self._step_recorder, self._options.arch,
967 self._options.use_goma, self._options.goma_dir,
Fumitoshi Ukaibc233ba2023-02-13 11:47:31[diff] [blame]968 self._options.use_remoteexec, self._ninja_command,
969 self._options.system_health_orderfile, self._monochrome,
970 self._options.public, self._GetPathToOrderfile())
Matthew Carya2bea452018-11-13 10:21:07[diff] [blame]971 if not self._options.pregenerated_profiles:
972 # If there are pregenerated profiles, the instrumented build should
973 # not be changed to avoid invalidating the pregenerated profile
974 # offsets.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]975 self._compiler.CompileChromeApk(instrumented=True,
976 use_call_graph=
977 self._options.use_call_graph)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]978 self._GenerateAndProcessProfile()
979 self._MaybeArchiveOrderfile(self._GetUnpatchedOrderfileFilename())
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]980 finally:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]981 _StashOutputDirectory(self._instrumented_out_dir)
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]982 elif self._options.manual_symbol_offsets:
983 assert self._options.manual_libname
984 assert self._options.manual_objdir
Ari Chivukulaa52f8ba2021-08-10 22:30:39[diff] [blame]985 with open(self._options.manual_symbol_offsets) as f:
986 symbol_offsets = [int(x) for x in f]
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]987 processor = process_profiles.SymbolOffsetProcessor(
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]988 self._options.manual_libname)
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]989 generator = cyglog_to_orderfile.OffsetOrderfileGenerator(
990 processor, cyglog_to_orderfile.ObjectFileProcessor(
991 self._options.manual_objdir))
992 ordered_sections = generator.GetOrderedSections(symbol_offsets)
993 if not ordered_sections: # Either None or empty is a problem.
994 raise Exception('Failed to get ordered sections')
995 with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile:
996 orderfile.write('\n'.join(ordered_sections))
997
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]998 if self._options.patch:
999 if self._options.profile:
1000 self._RemoveBlanks(self._GetUnpatchedOrderfileFilename(),
1001 self._GetPathToOrderfile())
1002 try:
1003 _UnstashOutputDirectory(self._uninstrumented_out_dir)
1004 self._compiler = ClankCompiler(
1005 self._uninstrumented_out_dir, self._step_recorder,
Christopher Grant073637472019-07-05 13:34:57[diff] [blame]1006 self._options.arch, self._options.use_goma, self._options.goma_dir,
Fumitoshi Ukaibc233ba2023-02-13 11:47:31[diff] [blame]1007 self._options.use_remoteexec, self._ninja_command,
1008 self._options.system_health_orderfile, self._monochrome,
1009 self._options.public, self._GetPathToOrderfile())
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]1010
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]1011 self._compiler.CompileLibchrome(instrumented=False,
1012 use_call_graph=False)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1013 self._PatchOrderfile()
1014 # Because identical code folding is a bit different with and without
1015 # the orderfile build, we need to re-patch the orderfile with code
1016 # folding as close to the final version as possible.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]1017 self._compiler.CompileLibchrome(instrumented=False,
1018 use_call_graph=False, force_relink=True)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1019 self._PatchOrderfile()
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]1020 self._compiler.CompileLibchrome(instrumented=False,
1021 use_call_graph=False, force_relink=True)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1022 self._VerifySymbolOrder()
1023 self._MaybeArchiveOrderfile(self._GetPathToOrderfile())
1024 finally:
1025 _StashOutputDirectory(self._uninstrumented_out_dir)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1026
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]1027 if self._options.benchmark:
1028 self._output_data['orderfile_benchmark_results'] = self.RunBenchmark(
1029 self._uninstrumented_out_dir)
1030 self._output_data['no_orderfile_benchmark_results'] = self.RunBenchmark(
1031 self._no_orderfile_out_dir, no_orderfile=True)
1032
Egor Paskod80bfad2020-09-10 21:53:06[diff] [blame]1033 if self._options.buildbot:
1034 self._orderfile_updater._GitStash()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1035 self._step_recorder.EndStep()
1036 return not self._step_recorder.ErrorRecorded()
1037
1038 def GetReportingData(self):
1039 """Get a dictionary of reporting data (timings, output hashes)"""
1040 self._output_data['timings'] = self._step_recorder.timings
1041 return self._output_data
1042
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]1043 def CommitStashedOrderfileHashes(self):
1044 """Commit any orderfile hash files in the current checkout.
1045
1046 Only possible if running on the buildbot.
1047
1048 Returns: true on success.
1049 """
Egor Pasko7ff04122019-11-25 15:47:18[diff] [blame]1050 if not self._options.buildbot:
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]1051 logging.error('Trying to commit when not running on the buildbot')
1052 return False
1053 self._orderfile_updater._CommitStashedFiles([
1054 filename + '.sha1'
1055 for filename in (self._GetUnpatchedOrderfileFilename(),
1056 self._GetPathToOrderfile())])
1057 return True
1058
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1059
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1060def CreateArgumentParser():
1061 """Creates and returns the argument parser."""
1062 parser = argparse.ArgumentParser()
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]1063 parser.add_argument('--no-benchmark', action='store_false', dest='benchmark',
1064 default=True, help='Disables running benchmarks.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1065 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1066 '--buildbot', action='store_true',
1067 help='If true, the script expects to be run on a buildbot')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1068 parser.add_argument(
Matthew Cary453ff1452018-07-18 12:19:35[diff] [blame]1069 '--device', default=None, type=str,
1070 help='Device serial number on which to run profiling.')
1071 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1072 '--verify', action='store_true',
1073 help='If true, the script only verifies the current orderfile')
Benoit Lizef5581722021-08-26 09:48:20[diff] [blame]1074 parser.add_argument('--target-arch',
1075 action='store',
1076 dest='arch',
Stephen Martinis71b391b52018-12-04 15:08:04[diff] [blame]1077 default='arm',
Benoit Lizef5581722021-08-26 09:48:20[diff] [blame]1078 choices=list(_ARCH_GN_ARGS.keys()),
Matthew Cary53f74ba2019-01-24 10:07:18[diff] [blame]1079 help='The target architecture for which to build.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1080 parser.add_argument('--output-json', action='store', dest='json_file',
1081 help='Location to save stats in json format')
1082 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1083 '--skip-profile', action='store_false', dest='profile', default=True,
1084 help='Don\'t generate a profile on the device. Only patch from the '
1085 'existing profile.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1086 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1087 '--skip-patch', action='store_false', dest='patch', default=True,
1088 help='Only generate the raw (unpatched) orderfile, don\'t patch it.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1089 parser.add_argument('--goma-dir', help='GOMA directory.')
1090 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1091 '--use-goma', action='store_true', help='Enable GOMA.', default=False)
Fumitoshi Ukaibd2755b2022-10-28 13:15:27[diff] [blame]1092 parser.add_argument('--use-remoteexec',
1093 action='store_true',
Fumitoshi Ukaibc233ba2023-02-13 11:47:31[diff] [blame]1094 help='Enable remoteexec. see //build/toolchain/rbe.gni.',
Fumitoshi Ukaibd2755b2022-10-28 13:15:27[diff] [blame]1095 default=False)
Fumitoshi Ukaibc233ba2023-02-13 11:47:31[diff] [blame]1096 parser.add_argument('--ninja-path',
1097 help='Path to the ninja binary. If given, use this'
1098 'instead of autoninja.')
1099 parser.add_argument('--ninja-j',
1100 help='-j value passed to ninja.'
1101 'pass -j to ninja. no need to set this when '
1102 '--ninja-path is not specified.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1103 parser.add_argument('--adb-path', help='Path to the adb binary.')
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]1104
Egor Paskod80bfad2020-09-10 21:53:06[diff] [blame]1105 parser.add_argument('--public',
1106 action='store_true',
1107 help='Build non-internal APK and change the orderfile '
1108 'location. Required if your checkout is non-internal.',
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]1109 default=False)
Matthew Cary04f41032018-12-10 15:55:27[diff] [blame]1110 parser.add_argument('--nosystem-health-orderfile', action='store_false',
Benoit Lcba421e2019-04-26 15:28:26[diff] [blame]1111 dest='system_health_orderfile', default=True,
Matthew Caryc262f5d2018-09-19 14:36:28[diff] [blame]1112 help=('Create an orderfile based on an about:blank '
1113 'startup benchmark instead of system health '
1114 'benchmarks.'))
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]1115 parser.add_argument(
1116 '--use-legacy-chrome-apk', action='store_true', default=False,
1117 help=('Compile and instrument chrome for [L, K] devices.'))
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]1118 parser.add_argument('--manual-symbol-offsets', default=None, type=str,
1119 help=('File of list of ordered symbol offsets generated '
1120 'by manual profiling. Must set other --manual* '
1121 'flags if this is used, and must --skip-profile.'))
1122 parser.add_argument('--manual-libname', default=None, type=str,
1123 help=('Library filename corresponding to '
1124 '--manual-symbol-offsets.'))
1125 parser.add_argument('--manual-objdir', default=None, type=str,
1126 help=('Root of object file directory corresponding to '
1127 '--manual-symbol-offsets.'))
Matthew Caryf949bba2019-02-04 13:39:23[diff] [blame]1128 parser.add_argument('--noorder-outlined-functions', action='store_true',
1129 help='Disable outlined functions in the orderfile.')
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]1130 parser.add_argument('--pregenerated-profiles', default=None, type=str,
1131 help=('Pregenerated profiles to use instead of running '
1132 'profile step. Cannot be used with '
1133 '--skip-profiles.'))
1134 parser.add_argument('--profile-save-dir', default=None, type=str,
1135 help=('Directory to save any profiles created. These can '
1136 'be used with --pregenerated-profiles. Cannot be '
1137 'used with --skip-profiles.'))
Egor Paskobce64d012018-11-20 12:02:37[diff] [blame]1138 parser.add_argument('--upload-ready-orderfiles', action='store_true',
1139 help=('Skip orderfile generation and manually upload '
1140 'orderfiles (both patched and unpatched) from '
1141 'their normal location in the tree to the cloud '
1142 'storage. DANGEROUS! USE WITH CARE!'))
Matthew Cary1ea82212019-03-20 12:10:27[diff] [blame]1143 parser.add_argument('--streamline-for-debugging', action='store_true',
1144 help=('Streamline where possible the run for faster '
1145 'iteration while debugging. The orderfile '
1146 'generated will be valid and nontrivial, but '
1147 'may not be based on a representative profile '
1148 'or other such considerations. Use with caution.'))
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]1149 parser.add_argument('--commit-hashes', action='store_true',
1150 help=('Commit any orderfile hash files in the current '
1151 'checkout; performs no other action'))
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]1152 parser.add_argument('--use-call-graph', action='store_true', default=False,
1153 help='Use call graph instrumentation.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1154 profile_android_startup.AddProfileCollectionArguments(parser)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1155 return parser
1156
1157
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]1158def CreateOrderfile(options, orderfile_updater_class=None):
Christopher Grantdfe1bac2019-07-05 13:34:10[diff] [blame]1159 """Creates an orderfile.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1160
1161 Args:
1162 options: As returned from optparse.OptionParser.parse_args()
1163 orderfile_updater_class: (OrderfileUpdater) subclass of OrderfileUpdater.
1164
1165 Returns:
1166 True iff success.
1167 """
1168 logging.basicConfig(level=logging.INFO)
1169 devil_chromium.Initialize(adb_path=options.adb_path)
1170
Benoit Lize6208ddc2021-08-30 12:26:29[diff] [blame]1171 # Since we generate a ".arm" orderfile irrespective of the architecture (see
1172 # comment in _GetPathToOrderfile()), make sure that we don't commit it.
1173 if options.arch != 'arm':
1174 assert not options.buildbot, (
1175 'ARM is the only supported architecture on bots')
1176 assert not options.upload_ready_orderfiles, (
1177 'ARM is the only supported architecture on bots')
1178
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]1179 generator = OrderfileGenerator(options, orderfile_updater_class)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1180 try:
1181 if options.verify:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]1182 generator._VerifySymbolOrder()
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]1183 elif options.commit_hashes:
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]1184 return generator.CommitStashedOrderfileHashes()
Egor Paskobce64d012018-11-20 12:02:37[diff] [blame]1185 elif options.upload_ready_orderfiles:
1186 return generator.UploadReadyOrderfiles()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1187 else:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]1188 return generator.Generate()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1189 finally:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]1190 json_output = json.dumps(generator.GetReportingData(),
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1191 indent=2) + '\n'
1192 if options.json_file:
1193 with open(options.json_file, 'w') as f:
1194 f.write(json_output)
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]1195 print(json_output)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1196 return False
1197
1198
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1199def main():
1200 parser = CreateArgumentParser()
1201 options = parser.parse_args()
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]1202 return 0 if CreateOrderfile(options) else 1
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1203
1204
1205if __name__ == '__main__':
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1206 sys.exit(main())