[go: nahoru, domu]
Google Git
Sign in
chromium / chromium / src / 61a838d39ee5776d440ccd8b80cb17991aa6ddf4 / . / tools / cygprofile / orderfile_generator_backend.py
blob: b76bde9b15244b696b336dbda66b4f4bb39b8a18 [file] [log] [blame]
Ari Chivukulaa52f8ba2021-08-10 22:30:39[diff] [blame]1#!/usr/bin/env vpython3
Avi Drissmandfd880852022-09-15 20:11:09[diff] [blame]2# Copyright 2013 The Chromium Authors
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6""" A utility to generate an up-to-date orderfile.
7
8The orderfile is used by the linker to order text sections such that the
9sections are placed consecutively in the order specified. This allows us
10to page in less code during start-up.
11
12Example usage:
Egor Paskoa7ac67a42020-09-09 12:06:58[diff] [blame]13 tools/cygprofile/orderfile_generator_backend.py --use-goma --target-arch=arm
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]14"""
15
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]16
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]17import argparse
Egor Paskoa7ac67a42020-09-09 12:06:58[diff] [blame]18import csv
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]19import hashlib
20import json
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]21import glob
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]22import logging
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]23import os
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]24import shutil
25import subprocess
26import sys
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]27import tempfile
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]28import time
29
Matthew Cary91df9792018-11-30 14:35:15[diff] [blame]30import cluster
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]31import cyglog_to_orderfile
Benoit Lizefefbb27c2018-01-17 13:54:18[diff] [blame]32import patch_orderfile
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]33import process_profiles
Egor Pasko3bc0b932018-04-03 10:08:27[diff] [blame]34import profile_android_startup
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]35
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]36_SRC_PATH = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
37sys.path.append(os.path.join(_SRC_PATH, 'third_party', 'catapult', 'devil'))
38from devil.android import device_utils
39from devil.android.sdk import version_codes
40
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]41
42_SRC_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)),
43 os.pardir, os.pardir)
44sys.path.append(os.path.join(_SRC_PATH, 'build', 'android'))
45import devil_chromium
46from pylib import constants
47
48
49# Needs to happen early for GetBuildType()/GetOutDirectory() to work correctly
50constants.SetBuildType('Release')
51
52
Matthew Cary53f74ba2019-01-24 10:07:18[diff] [blame]53# Architecture specific GN args. Trying to build an orderfile for an
54# architecture not listed here will eventually throw.
55_ARCH_GN_ARGS = {
Egor Paskoee760922024-01-11 19:36:31[diff] [blame]56 'arm': ['target_cpu="arm"'],
57 'arm64': [
58 'target_cpu="arm64"', 'android_64bit_browser=true',
Peter Wen503e6c02024-01-18 17:24:45[diff] [blame]59 'is_high_end_android=true'
Egor Paskoee760922024-01-11 19:36:31[diff] [blame]60 ],
61 'x86': ['target_cpu="x86"'],
Matthew Cary53f74ba2019-01-24 10:07:18[diff] [blame]62}
63
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]64class CommandError(Exception):
65 """Indicates that a dispatched shell command exited with a non-zero status."""
66
67 def __init__(self, value):
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]68 super().__init__()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]69 self.value = value
70
71 def __str__(self):
72 return repr(self.value)
73
74
75def _GenerateHash(file_path):
76 """Calculates and returns the hash of the file at file_path."""
77 sha1 = hashlib.sha1()
78 with open(file_path, 'rb') as f:
79 while True:
80 # Read in 1mb chunks, so it doesn't all have to be loaded into memory.
81 chunk = f.read(1024 * 1024)
82 if not chunk:
83 break
84 sha1.update(chunk)
85 return sha1.hexdigest()
86
87
88def _GetFileExtension(file_name):
89 """Calculates the file extension from a file name.
90
91 Args:
92 file_name: The source file name.
93 Returns:
94 The part of file_name after the dot (.) or None if the file has no
95 extension.
96 Examples: /home/user/foo.bar -> bar
97 /home/user.name/foo -> None
98 /home/user/.foo -> None
99 /home/user/foo.bar.baz -> baz
100 """
101 file_name_parts = os.path.basename(file_name).split('.')
102 if len(file_name_parts) > 1:
103 return file_name_parts[-1]
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]104 return None
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]105
106
107def _StashOutputDirectory(buildpath):
108 """Takes the output directory and stashes it in the default output directory.
109
110 This allows it to be used for incremental builds next time (after unstashing)
111 by keeping it in a place that isn't deleted normally, while also ensuring
112 that it is properly clobbered when appropriate.
113
114 This is a dirty hack to deal with the needs of clobbering while also handling
115 incremental builds and the hardcoded relative paths used in some of the
116 project files.
117
118 Args:
119 buildpath: The path where the building happens. If this corresponds to the
120 default output directory, no action is taken.
121 """
122 if os.path.abspath(buildpath) == os.path.abspath(os.path.dirname(
123 constants.GetOutDirectory())):
124 return
125 name = os.path.basename(buildpath)
126 stashpath = os.path.join(constants.GetOutDirectory(), name)
127 if not os.path.exists(buildpath):
128 return
129 if os.path.exists(stashpath):
130 shutil.rmtree(stashpath, ignore_errors=True)
131 shutil.move(buildpath, stashpath)
132
133
134def _UnstashOutputDirectory(buildpath):
135 """Inverse of _StashOutputDirectory.
136
137 Moves the output directory stashed within the default output directory
138 (out/Release) to the position where the builds can actually happen.
139
140 This is a dirty hack to deal with the needs of clobbering while also handling
141 incremental builds and the hardcoded relative paths used in some of the
142 project files.
143
144 Args:
145 buildpath: The path where the building happens. If this corresponds to the
146 default output directory, no action is taken.
147 """
148 if os.path.abspath(buildpath) == os.path.abspath(os.path.dirname(
149 constants.GetOutDirectory())):
150 return
151 name = os.path.basename(buildpath)
152 stashpath = os.path.join(constants.GetOutDirectory(), name)
153 if not os.path.exists(stashpath):
154 return
155 if os.path.exists(buildpath):
156 shutil.rmtree(buildpath, ignore_errors=True)
157 shutil.move(stashpath, buildpath)
158
159
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]160class StepRecorder:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]161 """Records steps and timings."""
162
163 def __init__(self, buildbot):
164 self.timings = []
165 self._previous_step = ('', 0.0)
166 self._buildbot = buildbot
167 self._error_recorded = False
168
169 def BeginStep(self, name):
170 """Marks a beginning of the next step in the script.
171
172 On buildbot, this prints a specially formatted name that will show up
173 in the waterfall. Otherwise, just prints the step name.
174
175 Args:
176 name: The name of the step.
177 """
178 self.EndStep()
179 self._previous_step = (name, time.time())
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]180 print('Running step: ', name)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]181
182 def EndStep(self):
183 """Records successful completion of the current step.
184
185 This is optional if the step is immediately followed by another BeginStep.
186 """
187 if self._previous_step[0]:
188 elapsed = time.time() - self._previous_step[1]
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]189 print('Step %s took %f seconds' % (self._previous_step[0], elapsed))
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]190 self.timings.append((self._previous_step[0], elapsed))
191
192 self._previous_step = ('', 0.0)
193
194 def FailStep(self, message=None):
195 """Marks that a particular step has failed.
196
197 On buildbot, this will mark the current step as failed on the waterfall.
198 Otherwise we will just print an optional failure message.
199
200 Args:
201 message: An optional explanation as to why the step failed.
202 """
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]203 print('STEP FAILED!!')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]204 if message:
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]205 print(message)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]206 self._error_recorded = True
207 self.EndStep()
208
209 def ErrorRecorded(self):
210 """True if FailStep has been called."""
211 return self._error_recorded
212
213 def RunCommand(self, cmd, cwd=constants.DIR_SOURCE_ROOT, raise_on_error=True,
214 stdout=None):
215 """Execute a shell command.
216
217 Args:
218 cmd: A list of command strings.
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]219 cwd: Directory in which the command should be executed, defaults to build
220 root of script's location if not specified.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]221 raise_on_error: If true will raise a CommandError if the call doesn't
222 succeed and mark the step as failed.
223 stdout: A file to redirect stdout for the command to.
224
225 Returns:
226 The process's return code.
227
228 Raises:
229 CommandError: An error executing the specified command.
230 """
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]231 print('Executing %s in %s' % (' '.join(cmd), cwd))
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]232 process = subprocess.Popen(cmd, stdout=stdout, cwd=cwd, env=os.environ)
233 process.wait()
234 if raise_on_error and process.returncode != 0:
235 self.FailStep()
236 raise CommandError('Exception executing command %s' % ' '.join(cmd))
237 return process.returncode
238
239
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]240class ClankCompiler:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]241 """Handles compilation of clank."""
242
Christopher Grant073637472019-07-05 13:34:57[diff] [blame]243 def __init__(self, out_dir, step_recorder, arch, use_goma, goma_dir,
Fumitoshi Ukaibc233ba2023-02-13 11:47:31[diff] [blame]244 use_remoteexec, ninja_command, system_health_profiling,
245 monochrome, public, orderfile_location):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]246 self._out_dir = out_dir
247 self._step_recorder = step_recorder
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]248 self._arch = arch
Fumitoshi Ukaibd2755b2022-10-28 13:15:27[diff] [blame]249 # TODO(b/236070141): remove goma config.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]250 self._use_goma = use_goma
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]251 self._goma_dir = goma_dir
Fumitoshi Ukaibd2755b2022-10-28 13:15:27[diff] [blame]252 self._use_remoteexec = use_remoteexec
Fumitoshi Ukaibc233ba2023-02-13 11:47:31[diff] [blame]253 self._ninja_command = ninja_command
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]254 self._system_health_profiling = system_health_profiling
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]255 self._public = public
256 self._orderfile_location = orderfile_location
Matthew Caryd6bfcb72018-09-14 11:27:46[diff] [blame]257 if monochrome:
258 self._apk = 'Monochrome.apk'
Matthew Carye1b00062018-09-19 14:27:44[diff] [blame]259 self._apk_target = 'monochrome_apk'
Matthew Caryd6bfcb72018-09-14 11:27:46[diff] [blame]260 self._libname = 'libmonochrome'
Matthew Cary2216fef2019-06-17 13:35:59[diff] [blame]261 self._libchrome_target = 'libmonochrome'
Matthew Caryd6bfcb72018-09-14 11:27:46[diff] [blame]262 else:
263 self._apk = 'Chrome.apk'
Matthew Carye1b00062018-09-19 14:27:44[diff] [blame]264 self._apk_target = 'chrome_apk'
Matthew Caryd6bfcb72018-09-14 11:27:46[diff] [blame]265 self._libname = 'libchrome'
266 self._libchrome_target = 'libchrome'
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]267 if public:
268 self._apk = self._apk.replace('.apk', 'Public.apk')
269 self._apk_target = self._apk_target.replace('_apk', '_public_apk')
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]270
271 self.obj_dir = os.path.join(self._out_dir, 'Release', 'obj')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]272 self.lib_chrome_so = os.path.join(
Matthew Caryd6bfcb72018-09-14 11:27:46[diff] [blame]273 self._out_dir, 'Release', 'lib.unstripped',
274 '{}.so'.format(self._libname))
275 self.chrome_apk = os.path.join(self._out_dir, 'Release', 'apks', self._apk)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]276
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]277 def Build(self, instrumented, use_call_graph, target):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]278 """Builds the provided ninja target with or without order_profiling on.
279
280 Args:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]281 instrumented: (bool) Whether we want to build an instrumented binary.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]282 use_call_graph: (bool) Whether to use the call graph instrumentation.
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]283 target: (str) The name of the ninja target to build.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]284 """
285 self._step_recorder.BeginStep('Compile %s' % target)
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]286 assert not use_call_graph or instrumented, ('You can not enable call graph '
287 'without instrumentation!')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]288
289 # Set the "Release Official" flavor, the parts affecting performance.
290 args = [
Andrew Grieve0e8790c2020-09-03 17:27:32[diff] [blame]291 'enable_resource_allowlist_generation=false',
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]292 'is_chrome_branded=' + str(not self._public).lower(),
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]293 'is_debug=false',
294 'is_official_build=true',
Egor Pasko1d13d532020-01-14 21:25:19[diff] [blame]295 'symbol_level=1', # to fit 30 GiB RAM on the bot when LLD is running
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]296 'target_os="android"',
297 'use_goma=' + str(self._use_goma).lower(),
Fumitoshi Ukaibd2755b2022-10-28 13:15:27[diff] [blame]298 'use_remoteexec=' + str(self._use_remoteexec).lower(),
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]299 'use_order_profiling=' + str(instrumented).lower(),
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]300 'use_call_graph=' + str(use_call_graph).lower(),
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]301 ]
Matthew Cary53f74ba2019-01-24 10:07:18[diff] [blame]302 args += _ARCH_GN_ARGS[self._arch]
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]303 if self._goma_dir:
304 args += ['goma_dir="%s"' % self._goma_dir]
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]305 if self._system_health_profiling:
306 args += ['devtools_instrumentation_dumping = ' +
307 str(instrumented).lower()]
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]308
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]309 if self._public and os.path.exists(self._orderfile_location):
310 # GN needs the orderfile path to be source-absolute.
311 src_abs_orderfile = os.path.relpath(self._orderfile_location,
312 constants.DIR_SOURCE_ROOT)
313 args += ['chrome_orderfile="//{}"'.format(src_abs_orderfile)]
314
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]315 self._step_recorder.RunCommand(
316 ['gn', 'gen', os.path.join(self._out_dir, 'Release'),
317 '--args=' + ' '.join(args)])
318
319 self._step_recorder.RunCommand(
Fumitoshi Ukaibc233ba2023-02-13 11:47:31[diff] [blame]320 self._ninja_command + [os.path.join(self._out_dir, 'Release'), target])
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]321
Christopher Grant10221762019-07-05 12:10:04[diff] [blame]322 def ForceRelink(self):
323 """Forces libchrome.so or libmonochrome.so to be re-linked.
324
325 With partitioned libraries enabled, deleting these library files does not
326 guarantee they'll be recreated by the linker (they may simply be
327 re-extracted from a combined library). To be safe, touch a source file
328 instead. See http://crbug.com/972701 for more explanation.
329 """
330 file_to_touch = os.path.join(constants.DIR_SOURCE_ROOT, 'chrome', 'browser',
331 'chrome_browser_main_android.cc')
332 assert os.path.exists(file_to_touch)
333 self._step_recorder.RunCommand(['touch', file_to_touch])
334
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]335 def CompileChromeApk(self, instrumented, use_call_graph, force_relink=False):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]336 """Builds a Chrome.apk either with or without order_profiling on.
337
338 Args:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]339 instrumented: (bool) Whether to build an instrumented apk.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]340 use_call_graph: (bool) Whether to use the call graph instrumentation.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]341 force_relink: Whether libchromeview.so should be re-created.
342 """
343 if force_relink:
Christopher Grant10221762019-07-05 12:10:04[diff] [blame]344 self.ForceRelink()
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]345 self.Build(instrumented, use_call_graph, self._apk_target)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]346
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]347 def CompileLibchrome(self, instrumented, use_call_graph, force_relink=False):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]348 """Builds a libchrome.so either with or without order_profiling on.
349
350 Args:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]351 instrumented: (bool) Whether to build an instrumented apk.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]352 use_call_graph: (bool) Whether to use the call graph instrumentation.
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]353 force_relink: (bool) Whether libchrome.so should be re-created.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]354 """
355 if force_relink:
Christopher Grant10221762019-07-05 12:10:04[diff] [blame]356 self.ForceRelink()
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]357 self.Build(instrumented, use_call_graph, self._libchrome_target)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]358
359
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]360class OrderfileUpdater:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]361 """Handles uploading and committing a new orderfile in the repository.
362
363 Only used for testing or on a bot.
364 """
365
366 _CLOUD_STORAGE_BUCKET_FOR_DEBUG = None
367 _CLOUD_STORAGE_BUCKET = None
368 _UPLOAD_TO_CLOUD_COMMAND = 'upload_to_google_storage.py'
369
Egor Pasko0c533c6682019-11-26 21:16:32[diff] [blame]370 def __init__(self, repository_root, step_recorder):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]371 """Constructor.
372
373 Args:
374 repository_root: (str) Root of the target repository.
375 step_recorder: (StepRecorder) Step recorder, for logging.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]376 """
377 self._repository_root = repository_root
378 self._step_recorder = step_recorder
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]379
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]380 def CommitStashedFileHashes(self, files):
381 """Commits unpatched and patched orderfiles hashes if changed.
382
383 The files are committed only if their associated sha1 hash files match, and
384 are modified in git. In normal operations the hash files are changed only
385 when a file is uploaded to cloud storage. If the hash file is not modified
386 in git, the file is skipped.
387
388 Args:
389 files: [str or None] specifies file paths. None items are ignored.
390
391 Raises:
392 Exception if the hash file does not match the file.
393 NotImplementedError when the commit logic hasn't been overridden.
394 """
Ari Chivukulaa52f8ba2021-08-10 22:30:39[diff] [blame]395 files_to_commit = [_f for _f in files if _f]
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]396 if files_to_commit:
397 self._CommitStashedFiles(files_to_commit)
398
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]399 def UploadToCloudStorage(self, filename, use_debug_location):
400 """Uploads a file to cloud storage.
401
402 Args:
403 filename: (str) File to upload.
404 use_debug_location: (bool) Whether to use the debug location.
405 """
406 bucket = (self._CLOUD_STORAGE_BUCKET_FOR_DEBUG if use_debug_location
407 else self._CLOUD_STORAGE_BUCKET)
408 extension = _GetFileExtension(filename)
409 cmd = [self._UPLOAD_TO_CLOUD_COMMAND, '--bucket', bucket]
410 if extension:
411 cmd.extend(['-z', extension])
412 cmd.append(filename)
413 self._step_recorder.RunCommand(cmd)
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]414 print('Download: https://sandbox.google.com/storage/%s/%s' %
415 (bucket, _GenerateHash(filename)))
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]416
417 def _GetHashFilePathAndContents(self, filename):
418 """Gets the name and content of the hash file created from uploading the
419 given file.
420
421 Args:
422 filename: (str) The file that was uploaded to cloud storage.
423
424 Returns:
425 A tuple of the hash file name, relative to the reository root, and the
426 content, which should be the sha1 hash of the file
427 ('base_file.sha1', hash)
428 """
429 abs_hash_filename = filename + '.sha1'
430 rel_hash_filename = os.path.relpath(
431 abs_hash_filename, self._repository_root)
432 with open(abs_hash_filename, 'r') as f:
433 return (rel_hash_filename, f.read())
434
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]435 def _GitStash(self):
436 """Git stash the current clank tree.
437
438 Raises:
439 NotImplementedError when the stash logic hasn't been overridden.
440 """
441 raise NotImplementedError
442
443 def _CommitStashedFiles(self, expected_files_in_stash):
444 """Commits stashed files.
445
446 The local repository is updated and then the files to commit are taken from
447 modified files from the git stash. The modified files should be a subset of
448 |expected_files_in_stash|. If there are unexpected modified files, this
449 function may raise. This is meant to be paired with _GitStash().
450
451 Args:
452 expected_files_in_stash: [str] paths to a possible superset of files
453 expected to be stashed & committed.
454
455 Raises:
456 NotImplementedError when the commit logic hasn't been overridden.
457 """
458 raise NotImplementedError
459
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]460
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]461class OrderfileGenerator:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]462 """A utility for generating a new orderfile for Clank.
463
464 Builds an instrumented binary, profiles a run of the application, and
465 generates an updated orderfile.
466 """
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]467 _CHECK_ORDERFILE_SCRIPT = os.path.join(
468 constants.DIR_SOURCE_ROOT, 'tools', 'cygprofile', 'check_orderfile.py')
469 _BUILD_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(
470 constants.GetOutDirectory()))) # Normally /path/to/src
471
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]472 # Previous orderfile_generator debug files would be overwritten.
473 _DIRECTORY_FOR_DEBUG_FILES = '/tmp/orderfile_generator_debug_files'
474
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]475 _CLOUD_STORAGE_BUCKET_FOR_DEBUG = None
476
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]477 def _PrepareOrderfilePaths(self):
478 if self._options.public:
479 self._clank_dir = os.path.join(constants.DIR_SOURCE_ROOT,
480 '')
481 if not os.path.exists(os.path.join(self._clank_dir, 'orderfiles')):
482 os.makedirs(os.path.join(self._clank_dir, 'orderfiles'))
483 else:
484 self._clank_dir = os.path.join(constants.DIR_SOURCE_ROOT,
485 'clank')
486
487 self._unpatched_orderfile_filename = os.path.join(
488 self._clank_dir, 'orderfiles', 'unpatched_orderfile.%s')
489 self._path_to_orderfile = os.path.join(
490 self._clank_dir, 'orderfiles', 'orderfile.%s.out')
491
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]492 def _GetPathToOrderfile(self):
493 """Gets the path to the architecture-specific orderfile."""
Peter Wen61a838d2024-01-29 17:13:42[diff] [blame^]494 # TODO(https://crbug.com/1517659): We are testing if arm64 can improve perf
495 # while not regressing arm32 memory or perf by too much. For now we are
496 # keeping the fake arch as 'arm' to avoid needing to change the path. In
497 # the future we should consider either generating multiple orderfiles,
498 # one per architecture, or remove the fake arch as it would no longer be
499 # accurate.
Benoit Lize6208ddc2021-08-30 12:26:29[diff] [blame]500 # Build GN files use the ".arm" orderfile irrespective of the actual
501 # architecture. Fake it, otherwise the orderfile we generate here is not
502 # going to be picked up by builds.
503 orderfile_fake_arch = 'arm'
504 return self._path_to_orderfile % orderfile_fake_arch
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]505
506 def _GetUnpatchedOrderfileFilename(self):
507 """Gets the path to the architecture-specific unpatched orderfile."""
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]508 return self._unpatched_orderfile_filename % self._options.arch
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]509
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]510 def _SetDevice(self):
511 """ Selects the device to be used by the script.
512
513 Returns:
514 (Device with given serial ID) : if the --device flag is set.
515 (Device running Android[K,L]) : if --use-legacy-chrome-apk flag is set or
516 no device running Android N+ was found.
517 (Device running Android N+) : Otherwise.
518
519 Raises Error:
520 If no device meeting the requirements has been found.
521 """
522 devices = None
523 if self._options.device:
524 devices = [device_utils.DeviceUtils(self._options.device)]
525 else:
526 devices = device_utils.DeviceUtils.HealthyDevices()
527
528 assert devices, 'Expected at least one connected device'
529
530 if self._options.use_legacy_chrome_apk:
531 self._monochrome = False
532 for device in devices:
533 device_version = device.build_version_sdk
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]534 if (version_codes.KITKAT <= device_version <=
535 version_codes.LOLLIPOP_MR1):
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]536 return device
537
538 assert not self._options.use_legacy_chrome_apk, \
539 'No device found running suitable android version for Chrome.apk.'
540
541 preferred_device = None
542 for device in devices:
543 if device.build_version_sdk >= version_codes.NOUGAT:
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]544 preferred_device = device
545 break
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]546
547 self._monochrome = preferred_device is not None
548
549 return preferred_device if preferred_device else devices[0]
550
551
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]552 def __init__(self, options, orderfile_updater_class):
553 self._options = options
Fumitoshi Ukaibc233ba2023-02-13 11:47:31[diff] [blame]554 self._ninja_command = ['autoninja']
555 if self._options.ninja_path:
556 self._ninja_command = [self._options.ninja_path]
557 if self._options.ninja_j:
558 self._ninja_command += ['-j', self._options.ninja_j]
559 self._ninja_command += ['-C']
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]560 self._instrumented_out_dir = os.path.join(
561 self._BUILD_ROOT, self._options.arch + '_instrumented_out')
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]562 if self._options.use_call_graph:
Christopher Grantdfe1bac2019-07-05 13:34:10[diff] [blame]563 self._instrumented_out_dir += '_call_graph'
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]564
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]565 self._uninstrumented_out_dir = os.path.join(
566 self._BUILD_ROOT, self._options.arch + '_uninstrumented_out')
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]567 self._no_orderfile_out_dir = os.path.join(
568 self._BUILD_ROOT, self._options.arch + '_no_orderfile_out')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]569
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]570 self._PrepareOrderfilePaths()
571
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]572 if options.profile:
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]573 output_directory = os.path.join(self._instrumented_out_dir, 'Release')
Matthew Caryb8daed942018-06-11 10:58:08[diff] [blame]574 host_profile_dir = os.path.join(output_directory, 'profile_data')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]575 urls = [profile_android_startup.AndroidProfileTool.TEST_URL]
576 use_wpr = True
577 simulate_user = False
Benoit L96466812018-03-06 15:58:37[diff] [blame]578 urls = options.urls
579 use_wpr = not options.no_wpr
580 simulate_user = options.simulate_user
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]581 device = self._SetDevice()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]582 self._profiler = profile_android_startup.AndroidProfileTool(
Matthew Cary453ff1452018-07-18 12:19:35[diff] [blame]583 output_directory, host_profile_dir, use_wpr, urls, simulate_user,
Matthew Cary1ea82212019-03-20 12:10:27[diff] [blame]584 device, debug=self._options.streamline_for_debugging)
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]585 if options.pregenerated_profiles:
586 self._profiler.SetPregeneratedProfiles(
587 glob.glob(options.pregenerated_profiles))
588 else:
589 assert not options.pregenerated_profiles, (
590 '--pregenerated-profiles cannot be used with --skip-profile')
591 assert not options.profile_save_dir, (
592 '--profile-save-dir cannot be used with --skip-profile')
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]593 self._monochrome = not self._options.use_legacy_chrome_apk
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]594
Matthew Caryf949bba2019-02-04 13:39:23[diff] [blame]595 # Outlined function handling enabled by default for all architectures.
596 self._order_outlined_functions = not options.noorder_outlined_functions
597
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]598 self._output_data = {}
599 self._step_recorder = StepRecorder(options.buildbot)
600 self._compiler = None
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]601 if orderfile_updater_class is None:
Egor Paskod80bfad2020-09-10 21:53:06[diff] [blame]602 orderfile_updater_class = OrderfileUpdater
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]603 assert issubclass(orderfile_updater_class, OrderfileUpdater)
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]604 self._orderfile_updater = orderfile_updater_class(self._clank_dir,
Egor Pasko0c533c6682019-11-26 21:16:32[diff] [blame]605 self._step_recorder)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]606 assert os.path.isdir(constants.DIR_SOURCE_ROOT), 'No src directory found'
607
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]608 @staticmethod
609 def _RemoveBlanks(src_file, dest_file):
610 """A utility to remove blank lines from a file.
611
612 Args:
613 src_file: The name of the file to remove the blanks from.
614 dest_file: The name of the file to write the output without blanks.
615 """
616 assert src_file != dest_file, 'Source and destination need to be distinct'
617
618 try:
619 src = open(src_file, 'r')
620 dest = open(dest_file, 'w')
621 for line in src:
622 if line and not line.isspace():
623 dest.write(line)
624 finally:
625 src.close()
626 dest.close()
627
628 def _GenerateAndProcessProfile(self):
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]629 """Invokes a script to merge the per-thread traces into one file.
630
631 The produced list of offsets is saved in
632 self._GetUnpatchedOrderfileFilename().
633 """
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]634 self._step_recorder.BeginStep('Generate Profile Data')
635 files = []
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]636 logging.getLogger().setLevel(logging.DEBUG)
Christopher Grantdfe1bac2019-07-05 13:34:10[diff] [blame]637
638 if self._options.profile_save_dir:
639 # The directory must not preexist, to ensure purity of data. Check
640 # before profiling to save time.
641 if os.path.exists(self._options.profile_save_dir):
642 raise Exception('Profile save directory must not pre-exist')
643 os.makedirs(self._options.profile_save_dir)
644
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]645 if self._options.system_health_orderfile:
646 files = self._profiler.CollectSystemHealthProfile(
647 self._compiler.chrome_apk)
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]648 self._MaybeSaveProfile(files)
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]649 try:
650 self._ProcessPhasedOrderfile(files)
651 except Exception:
652 for f in files:
653 self._SaveForDebugging(f)
Matthew Cary8b1416232018-08-10 19:12:22[diff] [blame]654 self._SaveForDebugging(self._compiler.lib_chrome_so)
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]655 raise
656 finally:
657 self._profiler.Cleanup()
658 else:
659 self._CollectLegacyProfile()
660 logging.getLogger().setLevel(logging.INFO)
661
662 def _ProcessPhasedOrderfile(self, files):
663 """Process the phased orderfiles produced by system health benchmarks.
664
665 The offsets will be placed in _GetUnpatchedOrderfileFilename().
666
667 Args:
668 file: Profile files pulled locally.
669 """
670 self._step_recorder.BeginStep('Process Phased Orderfile')
671 profiles = process_profiles.ProfileManager(files)
672 processor = process_profiles.SymbolOffsetProcessor(
673 self._compiler.lib_chrome_so)
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]674 ordered_symbols = cluster.ClusterOffsets(profiles, processor,
675 call_graph=self._options.use_call_graph)
Matthew Cary8b1416232018-08-10 19:12:22[diff] [blame]676 if not ordered_symbols:
677 raise Exception('Failed to get ordered symbols')
Matthew Caryda9db932019-03-11 15:28:17[diff] [blame]678 for sym in ordered_symbols:
679 assert not sym.startswith('OUTLINED_FUNCTION_'), (
680 'Outlined function found in instrumented function, very likely '
681 'something has gone very wrong!')
Matthew Cary91df9792018-11-30 14:35:15[diff] [blame]682 self._output_data['offsets_kib'] = processor.SymbolsSize(
683 ordered_symbols) / 1024
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]684 with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile:
Matthew Cary8b1416232018-08-10 19:12:22[diff] [blame]685 orderfile.write('\n'.join(ordered_symbols))
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]686
687 def _CollectLegacyProfile(self):
Matthew Cary2df8d452018-09-19 07:50:20[diff] [blame]688 files = []
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]689 try:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]690 files = self._profiler.CollectProfile(
691 self._compiler.chrome_apk,
692 constants.PACKAGE_INFO['chrome'])
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]693 self._MaybeSaveProfile(files)
Matthew Caryb8daed942018-06-11 10:58:08[diff] [blame]694 self._step_recorder.BeginStep('Process profile')
Benoit L96466812018-03-06 15:58:37[diff] [blame]695 assert os.path.exists(self._compiler.lib_chrome_so)
696 offsets = process_profiles.GetReachedOffsetsFromDumpFiles(
697 files, self._compiler.lib_chrome_so)
698 if not offsets:
699 raise Exception('No profiler offsets found in {}'.format(
700 '\n'.join(files)))
Matthew Cary8b1416232018-08-10 19:12:22[diff] [blame]701 processor = process_profiles.SymbolOffsetProcessor(
702 self._compiler.lib_chrome_so)
703 ordered_symbols = processor.GetOrderedSymbols(offsets)
704 if not ordered_symbols:
705 raise Exception('No symbol names from offsets found in {}'.format(
706 '\n'.join(files)))
707 with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile:
708 orderfile.write('\n'.join(ordered_symbols))
Matthew Cary0f1f681a2018-01-22 10:40:51[diff] [blame]709 except Exception:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]710 for f in files:
711 self._SaveForDebugging(f)
712 raise
713 finally:
714 self._profiler.Cleanup()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]715
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]716 def _MaybeSaveProfile(self, files):
717 if self._options.profile_save_dir:
718 logging.info('Saving profiles to %s', self._options.profile_save_dir)
719 for f in files:
720 shutil.copy(f, self._options.profile_save_dir)
721 logging.info('Saved profile %s', f)
722
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]723 def _PatchOrderfile(self):
724 """Patches the orderfile using clean version of libchrome.so."""
725 self._step_recorder.BeginStep('Patch Orderfile')
Benoit Lizefefbb27c2018-01-17 13:54:18[diff] [blame]726 patch_orderfile.GeneratePatchedOrderfile(
727 self._GetUnpatchedOrderfileFilename(), self._compiler.lib_chrome_so,
Matthew Caryf949bba2019-02-04 13:39:23[diff] [blame]728 self._GetPathToOrderfile(), self._order_outlined_functions)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]729
730 def _VerifySymbolOrder(self):
731 self._step_recorder.BeginStep('Verify Symbol Order')
Andrew Grievec45bb952021-11-02 18:33:27[diff] [blame]732 return_code = self._step_recorder.RunCommand([
733 self._CHECK_ORDERFILE_SCRIPT, self._compiler.lib_chrome_so,
734 self._GetPathToOrderfile()
735 ],
736 constants.DIR_SOURCE_ROOT,
737 raise_on_error=False)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]738 if return_code:
739 self._step_recorder.FailStep('Orderfile check returned %d.' % return_code)
740
741 def _RecordHash(self, file_name):
742 """Records the hash of the file into the output_data dictionary."""
743 self._output_data[os.path.basename(file_name) + '.sha1'] = _GenerateHash(
744 file_name)
745
746 def _SaveFileLocally(self, file_name, file_sha1):
747 """Saves the file to a temporary location and prints the sha1sum."""
748 if not os.path.exists(self._DIRECTORY_FOR_DEBUG_FILES):
749 os.makedirs(self._DIRECTORY_FOR_DEBUG_FILES)
750 shutil.copy(file_name, self._DIRECTORY_FOR_DEBUG_FILES)
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]751 print('File: %s, saved in: %s, sha1sum: %s' %
752 (file_name, self._DIRECTORY_FOR_DEBUG_FILES, file_sha1))
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]753
754 def _SaveForDebugging(self, filename):
755 """Uploads the file to cloud storage or saves to a temporary location."""
756 file_sha1 = _GenerateHash(filename)
757 if not self._options.buildbot:
758 self._SaveFileLocally(filename, file_sha1)
759 else:
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]760 print('Uploading file for debugging: ' + filename)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]761 self._orderfile_updater.UploadToCloudStorage(
762 filename, use_debug_location=True)
763
764 def _SaveForDebuggingWithOverwrite(self, file_name):
765 """Uploads and overwrites the file in cloud storage or copies locally.
766
767 Should be used for large binaries like lib_chrome_so.
768
769 Args:
770 file_name: (str) File to upload.
771 """
772 file_sha1 = _GenerateHash(file_name)
773 if not self._options.buildbot:
774 self._SaveFileLocally(file_name, file_sha1)
775 else:
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]776 print('Uploading file for debugging: %s, sha1sum: %s' % (file_name,
777 file_sha1))
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]778 upload_location = '%s/%s' % (
779 self._CLOUD_STORAGE_BUCKET_FOR_DEBUG, os.path.basename(file_name))
780 self._step_recorder.RunCommand([
781 'gsutil.py', 'cp', file_name, 'gs://' + upload_location])
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]782 print('Uploaded to: https://sandbox.google.com/storage/' +
783 upload_location)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]784
785 def _MaybeArchiveOrderfile(self, filename):
786 """In buildbot configuration, uploads the generated orderfile to
787 Google Cloud Storage.
788
789 Args:
790 filename: (str) Orderfile to upload.
791 """
Matthew Cary91df9792018-11-30 14:35:15[diff] [blame]792 # First compute hashes so that we can download them later if we need to.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]793 self._step_recorder.BeginStep('Compute hash for ' + filename)
794 self._RecordHash(filename)
795 if self._options.buildbot:
796 self._step_recorder.BeginStep('Archive ' + filename)
797 self._orderfile_updater.UploadToCloudStorage(
798 filename, use_debug_location=False)
799
Egor Paskobce64d012018-11-20 12:02:37[diff] [blame]800 def UploadReadyOrderfiles(self):
801 self._step_recorder.BeginStep('Upload Ready Orderfiles')
802 for file_name in [self._GetUnpatchedOrderfileFilename(),
803 self._GetPathToOrderfile()]:
804 self._orderfile_updater.UploadToCloudStorage(
805 file_name, use_debug_location=False)
806
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]807 def _NativeCodeMemoryBenchmark(self, apk):
808 """Runs system_health.memory_mobile to assess native code memory footprint.
809
810 Args:
811 apk: (str) Path to the apk.
812
813 Returns:
814 results: ([int]) Values of native code memory footprint in bytes from the
815 benchmark results.
816 """
817 self._step_recorder.BeginStep("Running orderfile.memory_mobile")
818 try:
819 out_dir = tempfile.mkdtemp()
820 self._profiler._RunCommand(['tools/perf/run_benchmark',
821 '--device={}'.format(
822 self._profiler._device.serial),
823 '--browser=exact',
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]824 '--output-format=csv',
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]825 '--output-dir={}'.format(out_dir),
826 '--reset-results',
827 '--browser-executable={}'.format(apk),
828 'orderfile.memory_mobile'])
829
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]830 out_file_path = os.path.join(out_dir, 'results.csv')
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]831 if not os.path.exists(out_file_path):
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]832 raise Exception('Results file not found!')
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]833
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]834 results = {}
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]835 with open(out_file_path, 'r') as f:
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]836 reader = csv.DictReader(f)
837 for row in reader:
838 if not row['name'].endswith('NativeCodeResidentMemory'):
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]839 continue
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]840 # Note: NativeCodeResidentMemory records a single sample from each
841 # story run, so this average (reported as 'avg') is exactly the value
842 # of that one sample. Each story is run multiple times, so this loop
843 # will accumulate into a list all values for all runs of each story.
844 results.setdefault(row['name'], {}).setdefault(
845 row['stories'], []).append(row['avg'])
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]846
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]847 if not results:
848 raise Exception('Could not find relevant results')
849
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]850 return results
851
852 except Exception as e:
853 return 'Error: ' + str(e)
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]854
855 finally:
856 shutil.rmtree(out_dir)
857
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]858
859 def _PerformanceBenchmark(self, apk):
860 """Runs Speedometer2.0 to assess performance.
861
862 Args:
863 apk: (str) Path to the apk.
864
865 Returns:
866 results: ([float]) Speedometer2.0 results samples in milliseconds.
867 """
868 self._step_recorder.BeginStep("Running Speedometer2.0.")
869 try:
870 out_dir = tempfile.mkdtemp()
871 self._profiler._RunCommand(['tools/perf/run_benchmark',
872 '--device={}'.format(
873 self._profiler._device.serial),
874 '--browser=exact',
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]875 '--output-format=histograms',
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]876 '--output-dir={}'.format(out_dir),
877 '--reset-results',
878 '--browser-executable={}'.format(apk),
879 'speedometer2'])
880
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]881 out_file_path = os.path.join(out_dir, 'histograms.json')
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]882 if not os.path.exists(out_file_path):
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]883 raise Exception('Results file not found!')
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]884
885 with open(out_file_path, 'r') as f:
886 results = json.load(f)
887
888 if not results:
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]889 raise Exception('Results file is empty.')
890
891 for el in results:
892 if 'name' in el and el['name'] == 'Total' and 'sampleValues' in el:
893 return el['sampleValues']
894
895 raise Exception('Unexpected results format.')
896
897 except Exception as e:
898 return 'Error: ' + str(e)
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]899
900 finally:
901 shutil.rmtree(out_dir)
902
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]903
904 def RunBenchmark(self, out_directory, no_orderfile=False):
905 """Builds chrome apk and runs performance and memory benchmarks.
906
907 Builds a non-instrumented version of chrome.
908 Installs chrome apk on the device.
909 Runs Speedometer2.0 benchmark to assess performance.
910 Runs system_health.memory_mobile to evaluate memory footprint.
911
912 Args:
913 out_directory: (str) Path to out directory for this build.
914 no_orderfile: (bool) True if chrome to be built without orderfile.
915
916 Returns:
917 benchmark_results: (dict) Results extracted from benchmarks.
918 """
Benoit Lize74e82b32021-08-26 14:22:01[diff] [blame]919 benchmark_results = {}
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]920 try:
921 _UnstashOutputDirectory(out_directory)
Christopher Grant073637472019-07-05 13:34:57[diff] [blame]922 self._compiler = ClankCompiler(out_directory, self._step_recorder,
923 self._options.arch, self._options.use_goma,
924 self._options.goma_dir,
Fumitoshi Ukaibd2755b2022-10-28 13:15:27[diff] [blame]925 self._options.use_remoteexec,
Fumitoshi Ukaibc233ba2023-02-13 11:47:31[diff] [blame]926 self._ninja_command,
Christopher Grant073637472019-07-05 13:34:57[diff] [blame]927 self._options.system_health_orderfile,
928 self._monochrome, self._options.public,
929 self._GetPathToOrderfile())
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]930
931 if no_orderfile:
932 orderfile_path = self._GetPathToOrderfile()
933 backup_orderfile = orderfile_path + '.backup'
934 shutil.move(orderfile_path, backup_orderfile)
935 open(orderfile_path, 'w').close()
936
937 # Build APK to be installed on the device.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]938 self._compiler.CompileChromeApk(instrumented=False,
939 use_call_graph=False,
940 force_relink=True)
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]941 benchmark_results['Speedometer2.0'] = self._PerformanceBenchmark(
942 self._compiler.chrome_apk)
943 benchmark_results['orderfile.memory_mobile'] = (
944 self._NativeCodeMemoryBenchmark(self._compiler.chrome_apk))
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]945
946 except Exception as e:
947 benchmark_results['Error'] = str(e)
948
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]949 finally:
950 if no_orderfile and os.path.exists(backup_orderfile):
951 shutil.move(backup_orderfile, orderfile_path)
952 _StashOutputDirectory(out_directory)
953
954 return benchmark_results
955
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]956 def Generate(self):
957 """Generates and maybe upload an order."""
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]958 assert (bool(self._options.profile) ^
959 bool(self._options.manual_symbol_offsets))
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]960 if self._options.system_health_orderfile and not self._options.profile:
961 raise AssertionError('--system_health_orderfile must be not be used '
962 'with --skip-profile')
963 if (self._options.manual_symbol_offsets and
964 not self._options.system_health_orderfile):
965 raise AssertionError('--manual-symbol-offsets must be used with '
966 '--system_health_orderfile.')
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]967
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]968 if self._options.profile:
969 try:
970 _UnstashOutputDirectory(self._instrumented_out_dir)
971 self._compiler = ClankCompiler(
Christopher Grant073637472019-07-05 13:34:57[diff] [blame]972 self._instrumented_out_dir, self._step_recorder, self._options.arch,
973 self._options.use_goma, self._options.goma_dir,
Fumitoshi Ukaibc233ba2023-02-13 11:47:31[diff] [blame]974 self._options.use_remoteexec, self._ninja_command,
975 self._options.system_health_orderfile, self._monochrome,
976 self._options.public, self._GetPathToOrderfile())
Matthew Carya2bea452018-11-13 10:21:07[diff] [blame]977 if not self._options.pregenerated_profiles:
978 # If there are pregenerated profiles, the instrumented build should
979 # not be changed to avoid invalidating the pregenerated profile
980 # offsets.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]981 self._compiler.CompileChromeApk(instrumented=True,
982 use_call_graph=
983 self._options.use_call_graph)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]984 self._GenerateAndProcessProfile()
985 self._MaybeArchiveOrderfile(self._GetUnpatchedOrderfileFilename())
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]986 finally:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]987 _StashOutputDirectory(self._instrumented_out_dir)
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]988 elif self._options.manual_symbol_offsets:
989 assert self._options.manual_libname
990 assert self._options.manual_objdir
Ari Chivukulaa52f8ba2021-08-10 22:30:39[diff] [blame]991 with open(self._options.manual_symbol_offsets) as f:
992 symbol_offsets = [int(x) for x in f]
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]993 processor = process_profiles.SymbolOffsetProcessor(
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]994 self._options.manual_libname)
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]995 generator = cyglog_to_orderfile.OffsetOrderfileGenerator(
996 processor, cyglog_to_orderfile.ObjectFileProcessor(
997 self._options.manual_objdir))
998 ordered_sections = generator.GetOrderedSections(symbol_offsets)
999 if not ordered_sections: # Either None or empty is a problem.
1000 raise Exception('Failed to get ordered sections')
1001 with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile:
1002 orderfile.write('\n'.join(ordered_sections))
1003
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1004 if self._options.patch:
1005 if self._options.profile:
1006 self._RemoveBlanks(self._GetUnpatchedOrderfileFilename(),
1007 self._GetPathToOrderfile())
1008 try:
1009 _UnstashOutputDirectory(self._uninstrumented_out_dir)
1010 self._compiler = ClankCompiler(
1011 self._uninstrumented_out_dir, self._step_recorder,
Christopher Grant073637472019-07-05 13:34:57[diff] [blame]1012 self._options.arch, self._options.use_goma, self._options.goma_dir,
Fumitoshi Ukaibc233ba2023-02-13 11:47:31[diff] [blame]1013 self._options.use_remoteexec, self._ninja_command,
1014 self._options.system_health_orderfile, self._monochrome,
1015 self._options.public, self._GetPathToOrderfile())
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]1016
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]1017 self._compiler.CompileLibchrome(instrumented=False,
1018 use_call_graph=False)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1019 self._PatchOrderfile()
1020 # Because identical code folding is a bit different with and without
1021 # the orderfile build, we need to re-patch the orderfile with code
1022 # folding as close to the final version as possible.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]1023 self._compiler.CompileLibchrome(instrumented=False,
1024 use_call_graph=False, force_relink=True)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1025 self._PatchOrderfile()
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]1026 self._compiler.CompileLibchrome(instrumented=False,
1027 use_call_graph=False, force_relink=True)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1028 self._VerifySymbolOrder()
1029 self._MaybeArchiveOrderfile(self._GetPathToOrderfile())
1030 finally:
1031 _StashOutputDirectory(self._uninstrumented_out_dir)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1032
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]1033 if self._options.benchmark:
1034 self._output_data['orderfile_benchmark_results'] = self.RunBenchmark(
1035 self._uninstrumented_out_dir)
1036 self._output_data['no_orderfile_benchmark_results'] = self.RunBenchmark(
1037 self._no_orderfile_out_dir, no_orderfile=True)
1038
Egor Paskod80bfad2020-09-10 21:53:06[diff] [blame]1039 if self._options.buildbot:
1040 self._orderfile_updater._GitStash()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1041 self._step_recorder.EndStep()
1042 return not self._step_recorder.ErrorRecorded()
1043
1044 def GetReportingData(self):
1045 """Get a dictionary of reporting data (timings, output hashes)"""
1046 self._output_data['timings'] = self._step_recorder.timings
1047 return self._output_data
1048
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]1049 def CommitStashedOrderfileHashes(self):
1050 """Commit any orderfile hash files in the current checkout.
1051
1052 Only possible if running on the buildbot.
1053
1054 Returns: true on success.
1055 """
Egor Pasko7ff04122019-11-25 15:47:18[diff] [blame]1056 if not self._options.buildbot:
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]1057 logging.error('Trying to commit when not running on the buildbot')
1058 return False
1059 self._orderfile_updater._CommitStashedFiles([
1060 filename + '.sha1'
1061 for filename in (self._GetUnpatchedOrderfileFilename(),
1062 self._GetPathToOrderfile())])
1063 return True
1064
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1065
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1066def CreateArgumentParser():
1067 """Creates and returns the argument parser."""
1068 parser = argparse.ArgumentParser()
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]1069 parser.add_argument('--no-benchmark', action='store_false', dest='benchmark',
1070 default=True, help='Disables running benchmarks.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1071 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1072 '--buildbot', action='store_true',
1073 help='If true, the script expects to be run on a buildbot')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1074 parser.add_argument(
Matthew Cary453ff1452018-07-18 12:19:35[diff] [blame]1075 '--device', default=None, type=str,
1076 help='Device serial number on which to run profiling.')
1077 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1078 '--verify', action='store_true',
1079 help='If true, the script only verifies the current orderfile')
Benoit Lizef5581722021-08-26 09:48:20[diff] [blame]1080 parser.add_argument('--target-arch',
1081 action='store',
1082 dest='arch',
Stephen Martinis71b391b52018-12-04 15:08:04[diff] [blame]1083 default='arm',
Benoit Lizef5581722021-08-26 09:48:20[diff] [blame]1084 choices=list(_ARCH_GN_ARGS.keys()),
Matthew Cary53f74ba2019-01-24 10:07:18[diff] [blame]1085 help='The target architecture for which to build.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1086 parser.add_argument('--output-json', action='store', dest='json_file',
1087 help='Location to save stats in json format')
1088 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1089 '--skip-profile', action='store_false', dest='profile', default=True,
1090 help='Don\'t generate a profile on the device. Only patch from the '
1091 'existing profile.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1092 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1093 '--skip-patch', action='store_false', dest='patch', default=True,
1094 help='Only generate the raw (unpatched) orderfile, don\'t patch it.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1095 parser.add_argument('--goma-dir', help='GOMA directory.')
1096 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1097 '--use-goma', action='store_true', help='Enable GOMA.', default=False)
Fumitoshi Ukaibd2755b2022-10-28 13:15:27[diff] [blame]1098 parser.add_argument('--use-remoteexec',
1099 action='store_true',
Fumitoshi Ukaibc233ba2023-02-13 11:47:31[diff] [blame]1100 help='Enable remoteexec. see //build/toolchain/rbe.gni.',
Fumitoshi Ukaibd2755b2022-10-28 13:15:27[diff] [blame]1101 default=False)
Fumitoshi Ukaibc233ba2023-02-13 11:47:31[diff] [blame]1102 parser.add_argument('--ninja-path',
1103 help='Path to the ninja binary. If given, use this'
1104 'instead of autoninja.')
1105 parser.add_argument('--ninja-j',
1106 help='-j value passed to ninja.'
1107 'pass -j to ninja. no need to set this when '
1108 '--ninja-path is not specified.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1109 parser.add_argument('--adb-path', help='Path to the adb binary.')
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]1110
Egor Paskod80bfad2020-09-10 21:53:06[diff] [blame]1111 parser.add_argument('--public',
1112 action='store_true',
1113 help='Build non-internal APK and change the orderfile '
1114 'location. Required if your checkout is non-internal.',
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]1115 default=False)
Matthew Cary04f41032018-12-10 15:55:27[diff] [blame]1116 parser.add_argument('--nosystem-health-orderfile', action='store_false',
Benoit Lcba421e2019-04-26 15:28:26[diff] [blame]1117 dest='system_health_orderfile', default=True,
Matthew Caryc262f5d2018-09-19 14:36:28[diff] [blame]1118 help=('Create an orderfile based on an about:blank '
1119 'startup benchmark instead of system health '
1120 'benchmarks.'))
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]1121 parser.add_argument(
1122 '--use-legacy-chrome-apk', action='store_true', default=False,
1123 help=('Compile and instrument chrome for [L, K] devices.'))
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]1124 parser.add_argument('--manual-symbol-offsets', default=None, type=str,
1125 help=('File of list of ordered symbol offsets generated '
1126 'by manual profiling. Must set other --manual* '
1127 'flags if this is used, and must --skip-profile.'))
1128 parser.add_argument('--manual-libname', default=None, type=str,
1129 help=('Library filename corresponding to '
1130 '--manual-symbol-offsets.'))
1131 parser.add_argument('--manual-objdir', default=None, type=str,
1132 help=('Root of object file directory corresponding to '
1133 '--manual-symbol-offsets.'))
Matthew Caryf949bba2019-02-04 13:39:23[diff] [blame]1134 parser.add_argument('--noorder-outlined-functions', action='store_true',
1135 help='Disable outlined functions in the orderfile.')
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]1136 parser.add_argument('--pregenerated-profiles', default=None, type=str,
1137 help=('Pregenerated profiles to use instead of running '
1138 'profile step. Cannot be used with '
1139 '--skip-profiles.'))
1140 parser.add_argument('--profile-save-dir', default=None, type=str,
1141 help=('Directory to save any profiles created. These can '
1142 'be used with --pregenerated-profiles. Cannot be '
1143 'used with --skip-profiles.'))
Egor Paskobce64d012018-11-20 12:02:37[diff] [blame]1144 parser.add_argument('--upload-ready-orderfiles', action='store_true',
1145 help=('Skip orderfile generation and manually upload '
1146 'orderfiles (both patched and unpatched) from '
1147 'their normal location in the tree to the cloud '
1148 'storage. DANGEROUS! USE WITH CARE!'))
Matthew Cary1ea82212019-03-20 12:10:27[diff] [blame]1149 parser.add_argument('--streamline-for-debugging', action='store_true',
1150 help=('Streamline where possible the run for faster '
1151 'iteration while debugging. The orderfile '
1152 'generated will be valid and nontrivial, but '
1153 'may not be based on a representative profile '
1154 'or other such considerations. Use with caution.'))
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]1155 parser.add_argument('--commit-hashes', action='store_true',
1156 help=('Commit any orderfile hash files in the current '
1157 'checkout; performs no other action'))
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]1158 parser.add_argument('--use-call-graph', action='store_true', default=False,
1159 help='Use call graph instrumentation.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1160 profile_android_startup.AddProfileCollectionArguments(parser)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1161 return parser
1162
1163
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]1164def CreateOrderfile(options, orderfile_updater_class=None):
Christopher Grantdfe1bac2019-07-05 13:34:10[diff] [blame]1165 """Creates an orderfile.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1166
1167 Args:
1168 options: As returned from optparse.OptionParser.parse_args()
1169 orderfile_updater_class: (OrderfileUpdater) subclass of OrderfileUpdater.
1170
1171 Returns:
1172 True iff success.
1173 """
1174 logging.basicConfig(level=logging.INFO)
1175 devil_chromium.Initialize(adb_path=options.adb_path)
1176
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]1177 generator = OrderfileGenerator(options, orderfile_updater_class)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1178 try:
1179 if options.verify:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]1180 generator._VerifySymbolOrder()
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]1181 elif options.commit_hashes:
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]1182 return generator.CommitStashedOrderfileHashes()
Egor Paskobce64d012018-11-20 12:02:37[diff] [blame]1183 elif options.upload_ready_orderfiles:
1184 return generator.UploadReadyOrderfiles()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1185 else:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]1186 return generator.Generate()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1187 finally:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]1188 json_output = json.dumps(generator.GetReportingData(),
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1189 indent=2) + '\n'
1190 if options.json_file:
1191 with open(options.json_file, 'w') as f:
1192 f.write(json_output)
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]1193 print(json_output)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1194 return False
1195
1196
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1197def main():
1198 parser = CreateArgumentParser()
1199 options = parser.parse_args()
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]1200 return 0 if CreateOrderfile(options) else 1
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1201
1202
1203if __name__ == '__main__':
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1204 sys.exit(main())