[go: nahoru, domu]
Google Git
Sign in
chromium / chromium / src / bc233bac5d89653c39f72e7f5c0b664b387ea2cf / . / tools / cygprofile / orderfile_generator_backend.py
blob: 848ca53f174d35cc69fc5b922d275ab3f35fba9c [file] [log] [blame]
Ari Chivukulaa52f8ba2021-08-10 22:30:39[diff] [blame]1#!/usr/bin/env vpython3
Avi Drissmandfd880852022-09-15 20:11:09[diff] [blame]2# Copyright 2013 The Chromium Authors
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6""" A utility to generate an up-to-date orderfile.
7
8The orderfile is used by the linker to order text sections such that the
9sections are placed consecutively in the order specified. This allows us
10to page in less code during start-up.
11
12Example usage:
Egor Paskoa7ac67a42020-09-09 12:06:58[diff] [blame]13 tools/cygprofile/orderfile_generator_backend.py --use-goma --target-arch=arm
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]14"""
15
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]16
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]17import argparse
Egor Paskoa7ac67a42020-09-09 12:06:58[diff] [blame]18import csv
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]19import hashlib
20import json
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]21import glob
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]22import logging
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]23import os
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]24import shutil
25import subprocess
26import sys
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]27import tempfile
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]28import time
29
Matthew Cary91df9792018-11-30 14:35:15[diff] [blame]30import cluster
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]31import cyglog_to_orderfile
Benoit Lizefefbb27c2018-01-17 13:54:18[diff] [blame]32import patch_orderfile
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]33import process_profiles
Egor Pasko3bc0b932018-04-03 10:08:27[diff] [blame]34import profile_android_startup
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]35
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]36_SRC_PATH = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
37sys.path.append(os.path.join(_SRC_PATH, 'third_party', 'catapult', 'devil'))
38from devil.android import device_utils
39from devil.android.sdk import version_codes
40
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]41
42_SRC_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)),
43 os.pardir, os.pardir)
44sys.path.append(os.path.join(_SRC_PATH, 'build', 'android'))
45import devil_chromium
46from pylib import constants
47
48
49# Needs to happen early for GetBuildType()/GetOutDirectory() to work correctly
50constants.SetBuildType('Release')
51
52
Matthew Cary53f74ba2019-01-24 10:07:18[diff] [blame]53# Architecture specific GN args. Trying to build an orderfile for an
54# architecture not listed here will eventually throw.
55_ARCH_GN_ARGS = {
Benoit Lizef5581722021-08-26 09:48:20[diff] [blame]56 'arm': ['target_cpu = "arm"'],
57 'arm64': ['target_cpu = "arm64"', 'android_64bit_browser = true'],
58 'x86': ['target_cpu = "x86"'],
Matthew Cary53f74ba2019-01-24 10:07:18[diff] [blame]59}
60
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]61class CommandError(Exception):
62 """Indicates that a dispatched shell command exited with a non-zero status."""
63
64 def __init__(self, value):
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]65 super().__init__()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]66 self.value = value
67
68 def __str__(self):
69 return repr(self.value)
70
71
72def _GenerateHash(file_path):
73 """Calculates and returns the hash of the file at file_path."""
74 sha1 = hashlib.sha1()
75 with open(file_path, 'rb') as f:
76 while True:
77 # Read in 1mb chunks, so it doesn't all have to be loaded into memory.
78 chunk = f.read(1024 * 1024)
79 if not chunk:
80 break
81 sha1.update(chunk)
82 return sha1.hexdigest()
83
84
85def _GetFileExtension(file_name):
86 """Calculates the file extension from a file name.
87
88 Args:
89 file_name: The source file name.
90 Returns:
91 The part of file_name after the dot (.) or None if the file has no
92 extension.
93 Examples: /home/user/foo.bar -> bar
94 /home/user.name/foo -> None
95 /home/user/.foo -> None
96 /home/user/foo.bar.baz -> baz
97 """
98 file_name_parts = os.path.basename(file_name).split('.')
99 if len(file_name_parts) > 1:
100 return file_name_parts[-1]
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]101 return None
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]102
103
104def _StashOutputDirectory(buildpath):
105 """Takes the output directory and stashes it in the default output directory.
106
107 This allows it to be used for incremental builds next time (after unstashing)
108 by keeping it in a place that isn't deleted normally, while also ensuring
109 that it is properly clobbered when appropriate.
110
111 This is a dirty hack to deal with the needs of clobbering while also handling
112 incremental builds and the hardcoded relative paths used in some of the
113 project files.
114
115 Args:
116 buildpath: The path where the building happens. If this corresponds to the
117 default output directory, no action is taken.
118 """
119 if os.path.abspath(buildpath) == os.path.abspath(os.path.dirname(
120 constants.GetOutDirectory())):
121 return
122 name = os.path.basename(buildpath)
123 stashpath = os.path.join(constants.GetOutDirectory(), name)
124 if not os.path.exists(buildpath):
125 return
126 if os.path.exists(stashpath):
127 shutil.rmtree(stashpath, ignore_errors=True)
128 shutil.move(buildpath, stashpath)
129
130
131def _UnstashOutputDirectory(buildpath):
132 """Inverse of _StashOutputDirectory.
133
134 Moves the output directory stashed within the default output directory
135 (out/Release) to the position where the builds can actually happen.
136
137 This is a dirty hack to deal with the needs of clobbering while also handling
138 incremental builds and the hardcoded relative paths used in some of the
139 project files.
140
141 Args:
142 buildpath: The path where the building happens. If this corresponds to the
143 default output directory, no action is taken.
144 """
145 if os.path.abspath(buildpath) == os.path.abspath(os.path.dirname(
146 constants.GetOutDirectory())):
147 return
148 name = os.path.basename(buildpath)
149 stashpath = os.path.join(constants.GetOutDirectory(), name)
150 if not os.path.exists(stashpath):
151 return
152 if os.path.exists(buildpath):
153 shutil.rmtree(buildpath, ignore_errors=True)
154 shutil.move(stashpath, buildpath)
155
156
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]157class StepRecorder:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]158 """Records steps and timings."""
159
160 def __init__(self, buildbot):
161 self.timings = []
162 self._previous_step = ('', 0.0)
163 self._buildbot = buildbot
164 self._error_recorded = False
165
166 def BeginStep(self, name):
167 """Marks a beginning of the next step in the script.
168
169 On buildbot, this prints a specially formatted name that will show up
170 in the waterfall. Otherwise, just prints the step name.
171
172 Args:
173 name: The name of the step.
174 """
175 self.EndStep()
176 self._previous_step = (name, time.time())
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]177 print('Running step: ', name)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]178
179 def EndStep(self):
180 """Records successful completion of the current step.
181
182 This is optional if the step is immediately followed by another BeginStep.
183 """
184 if self._previous_step[0]:
185 elapsed = time.time() - self._previous_step[1]
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]186 print('Step %s took %f seconds' % (self._previous_step[0], elapsed))
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]187 self.timings.append((self._previous_step[0], elapsed))
188
189 self._previous_step = ('', 0.0)
190
191 def FailStep(self, message=None):
192 """Marks that a particular step has failed.
193
194 On buildbot, this will mark the current step as failed on the waterfall.
195 Otherwise we will just print an optional failure message.
196
197 Args:
198 message: An optional explanation as to why the step failed.
199 """
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]200 print('STEP FAILED!!')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]201 if message:
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]202 print(message)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]203 self._error_recorded = True
204 self.EndStep()
205
206 def ErrorRecorded(self):
207 """True if FailStep has been called."""
208 return self._error_recorded
209
210 def RunCommand(self, cmd, cwd=constants.DIR_SOURCE_ROOT, raise_on_error=True,
211 stdout=None):
212 """Execute a shell command.
213
214 Args:
215 cmd: A list of command strings.
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]216 cwd: Directory in which the command should be executed, defaults to build
217 root of script's location if not specified.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]218 raise_on_error: If true will raise a CommandError if the call doesn't
219 succeed and mark the step as failed.
220 stdout: A file to redirect stdout for the command to.
221
222 Returns:
223 The process's return code.
224
225 Raises:
226 CommandError: An error executing the specified command.
227 """
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]228 print('Executing %s in %s' % (' '.join(cmd), cwd))
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]229 process = subprocess.Popen(cmd, stdout=stdout, cwd=cwd, env=os.environ)
230 process.wait()
231 if raise_on_error and process.returncode != 0:
232 self.FailStep()
233 raise CommandError('Exception executing command %s' % ' '.join(cmd))
234 return process.returncode
235
236
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]237class ClankCompiler:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]238 """Handles compilation of clank."""
239
Christopher Grant073637472019-07-05 13:34:57[diff] [blame]240 def __init__(self, out_dir, step_recorder, arch, use_goma, goma_dir,
Fumitoshi Ukaibc233ba2023-02-13 11:47:31[diff] [blame^]241 use_remoteexec, ninja_command, system_health_profiling,
242 monochrome, public, orderfile_location):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]243 self._out_dir = out_dir
244 self._step_recorder = step_recorder
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]245 self._arch = arch
Fumitoshi Ukaibd2755b2022-10-28 13:15:27[diff] [blame]246 # TODO(b/236070141): remove goma config.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]247 self._use_goma = use_goma
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]248 self._goma_dir = goma_dir
Fumitoshi Ukaibd2755b2022-10-28 13:15:27[diff] [blame]249 self._use_remoteexec = use_remoteexec
Fumitoshi Ukaibc233ba2023-02-13 11:47:31[diff] [blame^]250 self._ninja_command = ninja_command
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]251 self._system_health_profiling = system_health_profiling
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]252 self._public = public
253 self._orderfile_location = orderfile_location
Matthew Caryd6bfcb72018-09-14 11:27:46[diff] [blame]254 if monochrome:
255 self._apk = 'Monochrome.apk'
Matthew Carye1b00062018-09-19 14:27:44[diff] [blame]256 self._apk_target = 'monochrome_apk'
Matthew Caryd6bfcb72018-09-14 11:27:46[diff] [blame]257 self._libname = 'libmonochrome'
Matthew Cary2216fef2019-06-17 13:35:59[diff] [blame]258 self._libchrome_target = 'libmonochrome'
Matthew Caryd6bfcb72018-09-14 11:27:46[diff] [blame]259 else:
260 self._apk = 'Chrome.apk'
Matthew Carye1b00062018-09-19 14:27:44[diff] [blame]261 self._apk_target = 'chrome_apk'
Matthew Caryd6bfcb72018-09-14 11:27:46[diff] [blame]262 self._libname = 'libchrome'
263 self._libchrome_target = 'libchrome'
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]264 if public:
265 self._apk = self._apk.replace('.apk', 'Public.apk')
266 self._apk_target = self._apk_target.replace('_apk', '_public_apk')
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]267
268 self.obj_dir = os.path.join(self._out_dir, 'Release', 'obj')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]269 self.lib_chrome_so = os.path.join(
Matthew Caryd6bfcb72018-09-14 11:27:46[diff] [blame]270 self._out_dir, 'Release', 'lib.unstripped',
271 '{}.so'.format(self._libname))
272 self.chrome_apk = os.path.join(self._out_dir, 'Release', 'apks', self._apk)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]273
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]274 def Build(self, instrumented, use_call_graph, target):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]275 """Builds the provided ninja target with or without order_profiling on.
276
277 Args:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]278 instrumented: (bool) Whether we want to build an instrumented binary.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]279 use_call_graph: (bool) Whether to use the call graph instrumentation.
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]280 target: (str) The name of the ninja target to build.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]281 """
282 self._step_recorder.BeginStep('Compile %s' % target)
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]283 assert not use_call_graph or instrumented, ('You can not enable call graph '
284 'without instrumentation!')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]285
286 # Set the "Release Official" flavor, the parts affecting performance.
287 args = [
Andrew Grieve0e8790c2020-09-03 17:27:32[diff] [blame]288 'enable_resource_allowlist_generation=false',
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]289 'is_chrome_branded=' + str(not self._public).lower(),
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]290 'is_debug=false',
291 'is_official_build=true',
Egor Pasko1d13d532020-01-14 21:25:19[diff] [blame]292 'symbol_level=1', # to fit 30 GiB RAM on the bot when LLD is running
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]293 'target_os="android"',
294 'use_goma=' + str(self._use_goma).lower(),
Fumitoshi Ukaibd2755b2022-10-28 13:15:27[diff] [blame]295 'use_remoteexec=' + str(self._use_remoteexec).lower(),
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]296 'use_order_profiling=' + str(instrumented).lower(),
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]297 'use_call_graph=' + str(use_call_graph).lower(),
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]298 ]
Matthew Cary53f74ba2019-01-24 10:07:18[diff] [blame]299 args += _ARCH_GN_ARGS[self._arch]
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]300 if self._goma_dir:
301 args += ['goma_dir="%s"' % self._goma_dir]
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]302 if self._system_health_profiling:
303 args += ['devtools_instrumentation_dumping = ' +
304 str(instrumented).lower()]
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]305
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]306 if self._public and os.path.exists(self._orderfile_location):
307 # GN needs the orderfile path to be source-absolute.
308 src_abs_orderfile = os.path.relpath(self._orderfile_location,
309 constants.DIR_SOURCE_ROOT)
310 args += ['chrome_orderfile="//{}"'.format(src_abs_orderfile)]
311
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]312 self._step_recorder.RunCommand(
313 ['gn', 'gen', os.path.join(self._out_dir, 'Release'),
314 '--args=' + ' '.join(args)])
315
316 self._step_recorder.RunCommand(
Fumitoshi Ukaibc233ba2023-02-13 11:47:31[diff] [blame^]317 self._ninja_command + [os.path.join(self._out_dir, 'Release'), target])
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]318
Christopher Grant10221762019-07-05 12:10:04[diff] [blame]319 def ForceRelink(self):
320 """Forces libchrome.so or libmonochrome.so to be re-linked.
321
322 With partitioned libraries enabled, deleting these library files does not
323 guarantee they'll be recreated by the linker (they may simply be
324 re-extracted from a combined library). To be safe, touch a source file
325 instead. See http://crbug.com/972701 for more explanation.
326 """
327 file_to_touch = os.path.join(constants.DIR_SOURCE_ROOT, 'chrome', 'browser',
328 'chrome_browser_main_android.cc')
329 assert os.path.exists(file_to_touch)
330 self._step_recorder.RunCommand(['touch', file_to_touch])
331
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]332 def CompileChromeApk(self, instrumented, use_call_graph, force_relink=False):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]333 """Builds a Chrome.apk either with or without order_profiling on.
334
335 Args:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]336 instrumented: (bool) Whether to build an instrumented apk.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]337 use_call_graph: (bool) Whether to use the call graph instrumentation.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]338 force_relink: Whether libchromeview.so should be re-created.
339 """
340 if force_relink:
Christopher Grant10221762019-07-05 12:10:04[diff] [blame]341 self.ForceRelink()
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]342 self.Build(instrumented, use_call_graph, self._apk_target)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]343
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]344 def CompileLibchrome(self, instrumented, use_call_graph, force_relink=False):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]345 """Builds a libchrome.so either with or without order_profiling on.
346
347 Args:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]348 instrumented: (bool) Whether to build an instrumented apk.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]349 use_call_graph: (bool) Whether to use the call graph instrumentation.
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]350 force_relink: (bool) Whether libchrome.so should be re-created.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]351 """
352 if force_relink:
Christopher Grant10221762019-07-05 12:10:04[diff] [blame]353 self.ForceRelink()
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]354 self.Build(instrumented, use_call_graph, self._libchrome_target)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]355
356
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]357class OrderfileUpdater:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]358 """Handles uploading and committing a new orderfile in the repository.
359
360 Only used for testing or on a bot.
361 """
362
363 _CLOUD_STORAGE_BUCKET_FOR_DEBUG = None
364 _CLOUD_STORAGE_BUCKET = None
365 _UPLOAD_TO_CLOUD_COMMAND = 'upload_to_google_storage.py'
366
Egor Pasko0c533c6682019-11-26 21:16:32[diff] [blame]367 def __init__(self, repository_root, step_recorder):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]368 """Constructor.
369
370 Args:
371 repository_root: (str) Root of the target repository.
372 step_recorder: (StepRecorder) Step recorder, for logging.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]373 """
374 self._repository_root = repository_root
375 self._step_recorder = step_recorder
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]376
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]377 def CommitStashedFileHashes(self, files):
378 """Commits unpatched and patched orderfiles hashes if changed.
379
380 The files are committed only if their associated sha1 hash files match, and
381 are modified in git. In normal operations the hash files are changed only
382 when a file is uploaded to cloud storage. If the hash file is not modified
383 in git, the file is skipped.
384
385 Args:
386 files: [str or None] specifies file paths. None items are ignored.
387
388 Raises:
389 Exception if the hash file does not match the file.
390 NotImplementedError when the commit logic hasn't been overridden.
391 """
Ari Chivukulaa52f8ba2021-08-10 22:30:39[diff] [blame]392 files_to_commit = [_f for _f in files if _f]
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]393 if files_to_commit:
394 self._CommitStashedFiles(files_to_commit)
395
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]396 def UploadToCloudStorage(self, filename, use_debug_location):
397 """Uploads a file to cloud storage.
398
399 Args:
400 filename: (str) File to upload.
401 use_debug_location: (bool) Whether to use the debug location.
402 """
403 bucket = (self._CLOUD_STORAGE_BUCKET_FOR_DEBUG if use_debug_location
404 else self._CLOUD_STORAGE_BUCKET)
405 extension = _GetFileExtension(filename)
406 cmd = [self._UPLOAD_TO_CLOUD_COMMAND, '--bucket', bucket]
407 if extension:
408 cmd.extend(['-z', extension])
409 cmd.append(filename)
410 self._step_recorder.RunCommand(cmd)
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]411 print('Download: https://sandbox.google.com/storage/%s/%s' %
412 (bucket, _GenerateHash(filename)))
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]413
414 def _GetHashFilePathAndContents(self, filename):
415 """Gets the name and content of the hash file created from uploading the
416 given file.
417
418 Args:
419 filename: (str) The file that was uploaded to cloud storage.
420
421 Returns:
422 A tuple of the hash file name, relative to the reository root, and the
423 content, which should be the sha1 hash of the file
424 ('base_file.sha1', hash)
425 """
426 abs_hash_filename = filename + '.sha1'
427 rel_hash_filename = os.path.relpath(
428 abs_hash_filename, self._repository_root)
429 with open(abs_hash_filename, 'r') as f:
430 return (rel_hash_filename, f.read())
431
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]432 def _GitStash(self):
433 """Git stash the current clank tree.
434
435 Raises:
436 NotImplementedError when the stash logic hasn't been overridden.
437 """
438 raise NotImplementedError
439
440 def _CommitStashedFiles(self, expected_files_in_stash):
441 """Commits stashed files.
442
443 The local repository is updated and then the files to commit are taken from
444 modified files from the git stash. The modified files should be a subset of
445 |expected_files_in_stash|. If there are unexpected modified files, this
446 function may raise. This is meant to be paired with _GitStash().
447
448 Args:
449 expected_files_in_stash: [str] paths to a possible superset of files
450 expected to be stashed & committed.
451
452 Raises:
453 NotImplementedError when the commit logic hasn't been overridden.
454 """
455 raise NotImplementedError
456
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]457
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]458class OrderfileGenerator:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]459 """A utility for generating a new orderfile for Clank.
460
461 Builds an instrumented binary, profiles a run of the application, and
462 generates an updated orderfile.
463 """
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]464 _CHECK_ORDERFILE_SCRIPT = os.path.join(
465 constants.DIR_SOURCE_ROOT, 'tools', 'cygprofile', 'check_orderfile.py')
466 _BUILD_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(
467 constants.GetOutDirectory()))) # Normally /path/to/src
468
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]469 # Previous orderfile_generator debug files would be overwritten.
470 _DIRECTORY_FOR_DEBUG_FILES = '/tmp/orderfile_generator_debug_files'
471
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]472 _CLOUD_STORAGE_BUCKET_FOR_DEBUG = None
473
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]474 def _PrepareOrderfilePaths(self):
475 if self._options.public:
476 self._clank_dir = os.path.join(constants.DIR_SOURCE_ROOT,
477 '')
478 if not os.path.exists(os.path.join(self._clank_dir, 'orderfiles')):
479 os.makedirs(os.path.join(self._clank_dir, 'orderfiles'))
480 else:
481 self._clank_dir = os.path.join(constants.DIR_SOURCE_ROOT,
482 'clank')
483
484 self._unpatched_orderfile_filename = os.path.join(
485 self._clank_dir, 'orderfiles', 'unpatched_orderfile.%s')
486 self._path_to_orderfile = os.path.join(
487 self._clank_dir, 'orderfiles', 'orderfile.%s.out')
488
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]489 def _GetPathToOrderfile(self):
490 """Gets the path to the architecture-specific orderfile."""
Benoit Lize6208ddc2021-08-30 12:26:29[diff] [blame]491 # Build GN files use the ".arm" orderfile irrespective of the actual
492 # architecture. Fake it, otherwise the orderfile we generate here is not
493 # going to be picked up by builds.
494 orderfile_fake_arch = 'arm'
495 return self._path_to_orderfile % orderfile_fake_arch
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]496
497 def _GetUnpatchedOrderfileFilename(self):
498 """Gets the path to the architecture-specific unpatched orderfile."""
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]499 return self._unpatched_orderfile_filename % self._options.arch
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]500
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]501 def _SetDevice(self):
502 """ Selects the device to be used by the script.
503
504 Returns:
505 (Device with given serial ID) : if the --device flag is set.
506 (Device running Android[K,L]) : if --use-legacy-chrome-apk flag is set or
507 no device running Android N+ was found.
508 (Device running Android N+) : Otherwise.
509
510 Raises Error:
511 If no device meeting the requirements has been found.
512 """
513 devices = None
514 if self._options.device:
515 devices = [device_utils.DeviceUtils(self._options.device)]
516 else:
517 devices = device_utils.DeviceUtils.HealthyDevices()
518
519 assert devices, 'Expected at least one connected device'
520
521 if self._options.use_legacy_chrome_apk:
522 self._monochrome = False
523 for device in devices:
524 device_version = device.build_version_sdk
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]525 if (version_codes.KITKAT <= device_version <=
526 version_codes.LOLLIPOP_MR1):
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]527 return device
528
529 assert not self._options.use_legacy_chrome_apk, \
530 'No device found running suitable android version for Chrome.apk.'
531
532 preferred_device = None
533 for device in devices:
534 if device.build_version_sdk >= version_codes.NOUGAT:
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]535 preferred_device = device
536 break
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]537
538 self._monochrome = preferred_device is not None
539
540 return preferred_device if preferred_device else devices[0]
541
542
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]543 def __init__(self, options, orderfile_updater_class):
544 self._options = options
Fumitoshi Ukaibc233ba2023-02-13 11:47:31[diff] [blame^]545 self._ninja_command = ['autoninja']
546 if self._options.ninja_path:
547 self._ninja_command = [self._options.ninja_path]
548 if self._options.ninja_j:
549 self._ninja_command += ['-j', self._options.ninja_j]
550 self._ninja_command += ['-C']
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]551 self._instrumented_out_dir = os.path.join(
552 self._BUILD_ROOT, self._options.arch + '_instrumented_out')
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]553 if self._options.use_call_graph:
Christopher Grantdfe1bac2019-07-05 13:34:10[diff] [blame]554 self._instrumented_out_dir += '_call_graph'
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]555
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]556 self._uninstrumented_out_dir = os.path.join(
557 self._BUILD_ROOT, self._options.arch + '_uninstrumented_out')
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]558 self._no_orderfile_out_dir = os.path.join(
559 self._BUILD_ROOT, self._options.arch + '_no_orderfile_out')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]560
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]561 self._PrepareOrderfilePaths()
562
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]563 if options.profile:
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]564 output_directory = os.path.join(self._instrumented_out_dir, 'Release')
Matthew Caryb8daed942018-06-11 10:58:08[diff] [blame]565 host_profile_dir = os.path.join(output_directory, 'profile_data')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]566 urls = [profile_android_startup.AndroidProfileTool.TEST_URL]
567 use_wpr = True
568 simulate_user = False
Benoit L96466812018-03-06 15:58:37[diff] [blame]569 urls = options.urls
570 use_wpr = not options.no_wpr
571 simulate_user = options.simulate_user
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]572 device = self._SetDevice()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]573 self._profiler = profile_android_startup.AndroidProfileTool(
Matthew Cary453ff1452018-07-18 12:19:35[diff] [blame]574 output_directory, host_profile_dir, use_wpr, urls, simulate_user,
Matthew Cary1ea82212019-03-20 12:10:27[diff] [blame]575 device, debug=self._options.streamline_for_debugging)
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]576 if options.pregenerated_profiles:
577 self._profiler.SetPregeneratedProfiles(
578 glob.glob(options.pregenerated_profiles))
579 else:
580 assert not options.pregenerated_profiles, (
581 '--pregenerated-profiles cannot be used with --skip-profile')
582 assert not options.profile_save_dir, (
583 '--profile-save-dir cannot be used with --skip-profile')
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]584 self._monochrome = not self._options.use_legacy_chrome_apk
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]585
Matthew Caryf949bba2019-02-04 13:39:23[diff] [blame]586 # Outlined function handling enabled by default for all architectures.
587 self._order_outlined_functions = not options.noorder_outlined_functions
588
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]589 self._output_data = {}
590 self._step_recorder = StepRecorder(options.buildbot)
591 self._compiler = None
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]592 if orderfile_updater_class is None:
Egor Paskod80bfad2020-09-10 21:53:06[diff] [blame]593 orderfile_updater_class = OrderfileUpdater
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]594 assert issubclass(orderfile_updater_class, OrderfileUpdater)
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]595 self._orderfile_updater = orderfile_updater_class(self._clank_dir,
Egor Pasko0c533c6682019-11-26 21:16:32[diff] [blame]596 self._step_recorder)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]597 assert os.path.isdir(constants.DIR_SOURCE_ROOT), 'No src directory found'
598
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]599 @staticmethod
600 def _RemoveBlanks(src_file, dest_file):
601 """A utility to remove blank lines from a file.
602
603 Args:
604 src_file: The name of the file to remove the blanks from.
605 dest_file: The name of the file to write the output without blanks.
606 """
607 assert src_file != dest_file, 'Source and destination need to be distinct'
608
609 try:
610 src = open(src_file, 'r')
611 dest = open(dest_file, 'w')
612 for line in src:
613 if line and not line.isspace():
614 dest.write(line)
615 finally:
616 src.close()
617 dest.close()
618
619 def _GenerateAndProcessProfile(self):
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]620 """Invokes a script to merge the per-thread traces into one file.
621
622 The produced list of offsets is saved in
623 self._GetUnpatchedOrderfileFilename().
624 """
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]625 self._step_recorder.BeginStep('Generate Profile Data')
626 files = []
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]627 logging.getLogger().setLevel(logging.DEBUG)
Christopher Grantdfe1bac2019-07-05 13:34:10[diff] [blame]628
629 if self._options.profile_save_dir:
630 # The directory must not preexist, to ensure purity of data. Check
631 # before profiling to save time.
632 if os.path.exists(self._options.profile_save_dir):
633 raise Exception('Profile save directory must not pre-exist')
634 os.makedirs(self._options.profile_save_dir)
635
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]636 if self._options.system_health_orderfile:
637 files = self._profiler.CollectSystemHealthProfile(
638 self._compiler.chrome_apk)
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]639 self._MaybeSaveProfile(files)
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]640 try:
641 self._ProcessPhasedOrderfile(files)
642 except Exception:
643 for f in files:
644 self._SaveForDebugging(f)
Matthew Cary8b1416232018-08-10 19:12:22[diff] [blame]645 self._SaveForDebugging(self._compiler.lib_chrome_so)
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]646 raise
647 finally:
648 self._profiler.Cleanup()
649 else:
650 self._CollectLegacyProfile()
651 logging.getLogger().setLevel(logging.INFO)
652
653 def _ProcessPhasedOrderfile(self, files):
654 """Process the phased orderfiles produced by system health benchmarks.
655
656 The offsets will be placed in _GetUnpatchedOrderfileFilename().
657
658 Args:
659 file: Profile files pulled locally.
660 """
661 self._step_recorder.BeginStep('Process Phased Orderfile')
662 profiles = process_profiles.ProfileManager(files)
663 processor = process_profiles.SymbolOffsetProcessor(
664 self._compiler.lib_chrome_so)
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]665 ordered_symbols = cluster.ClusterOffsets(profiles, processor,
666 call_graph=self._options.use_call_graph)
Matthew Cary8b1416232018-08-10 19:12:22[diff] [blame]667 if not ordered_symbols:
668 raise Exception('Failed to get ordered symbols')
Matthew Caryda9db932019-03-11 15:28:17[diff] [blame]669 for sym in ordered_symbols:
670 assert not sym.startswith('OUTLINED_FUNCTION_'), (
671 'Outlined function found in instrumented function, very likely '
672 'something has gone very wrong!')
Matthew Cary91df9792018-11-30 14:35:15[diff] [blame]673 self._output_data['offsets_kib'] = processor.SymbolsSize(
674 ordered_symbols) / 1024
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]675 with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile:
Matthew Cary8b1416232018-08-10 19:12:22[diff] [blame]676 orderfile.write('\n'.join(ordered_symbols))
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]677
678 def _CollectLegacyProfile(self):
Matthew Cary2df8d452018-09-19 07:50:20[diff] [blame]679 files = []
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]680 try:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]681 files = self._profiler.CollectProfile(
682 self._compiler.chrome_apk,
683 constants.PACKAGE_INFO['chrome'])
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]684 self._MaybeSaveProfile(files)
Matthew Caryb8daed942018-06-11 10:58:08[diff] [blame]685 self._step_recorder.BeginStep('Process profile')
Benoit L96466812018-03-06 15:58:37[diff] [blame]686 assert os.path.exists(self._compiler.lib_chrome_so)
687 offsets = process_profiles.GetReachedOffsetsFromDumpFiles(
688 files, self._compiler.lib_chrome_so)
689 if not offsets:
690 raise Exception('No profiler offsets found in {}'.format(
691 '\n'.join(files)))
Matthew Cary8b1416232018-08-10 19:12:22[diff] [blame]692 processor = process_profiles.SymbolOffsetProcessor(
693 self._compiler.lib_chrome_so)
694 ordered_symbols = processor.GetOrderedSymbols(offsets)
695 if not ordered_symbols:
696 raise Exception('No symbol names from offsets found in {}'.format(
697 '\n'.join(files)))
698 with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile:
699 orderfile.write('\n'.join(ordered_symbols))
Matthew Cary0f1f681a2018-01-22 10:40:51[diff] [blame]700 except Exception:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]701 for f in files:
702 self._SaveForDebugging(f)
703 raise
704 finally:
705 self._profiler.Cleanup()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]706
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]707 def _MaybeSaveProfile(self, files):
708 if self._options.profile_save_dir:
709 logging.info('Saving profiles to %s', self._options.profile_save_dir)
710 for f in files:
711 shutil.copy(f, self._options.profile_save_dir)
712 logging.info('Saved profile %s', f)
713
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]714 def _PatchOrderfile(self):
715 """Patches the orderfile using clean version of libchrome.so."""
716 self._step_recorder.BeginStep('Patch Orderfile')
Benoit Lizefefbb27c2018-01-17 13:54:18[diff] [blame]717 patch_orderfile.GeneratePatchedOrderfile(
718 self._GetUnpatchedOrderfileFilename(), self._compiler.lib_chrome_so,
Matthew Caryf949bba2019-02-04 13:39:23[diff] [blame]719 self._GetPathToOrderfile(), self._order_outlined_functions)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]720
721 def _VerifySymbolOrder(self):
722 self._step_recorder.BeginStep('Verify Symbol Order')
Andrew Grievec45bb952021-11-02 18:33:27[diff] [blame]723 return_code = self._step_recorder.RunCommand([
724 self._CHECK_ORDERFILE_SCRIPT, self._compiler.lib_chrome_so,
725 self._GetPathToOrderfile()
726 ],
727 constants.DIR_SOURCE_ROOT,
728 raise_on_error=False)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]729 if return_code:
730 self._step_recorder.FailStep('Orderfile check returned %d.' % return_code)
731
732 def _RecordHash(self, file_name):
733 """Records the hash of the file into the output_data dictionary."""
734 self._output_data[os.path.basename(file_name) + '.sha1'] = _GenerateHash(
735 file_name)
736
737 def _SaveFileLocally(self, file_name, file_sha1):
738 """Saves the file to a temporary location and prints the sha1sum."""
739 if not os.path.exists(self._DIRECTORY_FOR_DEBUG_FILES):
740 os.makedirs(self._DIRECTORY_FOR_DEBUG_FILES)
741 shutil.copy(file_name, self._DIRECTORY_FOR_DEBUG_FILES)
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]742 print('File: %s, saved in: %s, sha1sum: %s' %
743 (file_name, self._DIRECTORY_FOR_DEBUG_FILES, file_sha1))
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]744
745 def _SaveForDebugging(self, filename):
746 """Uploads the file to cloud storage or saves to a temporary location."""
747 file_sha1 = _GenerateHash(filename)
748 if not self._options.buildbot:
749 self._SaveFileLocally(filename, file_sha1)
750 else:
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]751 print('Uploading file for debugging: ' + filename)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]752 self._orderfile_updater.UploadToCloudStorage(
753 filename, use_debug_location=True)
754
755 def _SaveForDebuggingWithOverwrite(self, file_name):
756 """Uploads and overwrites the file in cloud storage or copies locally.
757
758 Should be used for large binaries like lib_chrome_so.
759
760 Args:
761 file_name: (str) File to upload.
762 """
763 file_sha1 = _GenerateHash(file_name)
764 if not self._options.buildbot:
765 self._SaveFileLocally(file_name, file_sha1)
766 else:
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]767 print('Uploading file for debugging: %s, sha1sum: %s' % (file_name,
768 file_sha1))
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]769 upload_location = '%s/%s' % (
770 self._CLOUD_STORAGE_BUCKET_FOR_DEBUG, os.path.basename(file_name))
771 self._step_recorder.RunCommand([
772 'gsutil.py', 'cp', file_name, 'gs://' + upload_location])
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]773 print('Uploaded to: https://sandbox.google.com/storage/' +
774 upload_location)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]775
776 def _MaybeArchiveOrderfile(self, filename):
777 """In buildbot configuration, uploads the generated orderfile to
778 Google Cloud Storage.
779
780 Args:
781 filename: (str) Orderfile to upload.
782 """
Matthew Cary91df9792018-11-30 14:35:15[diff] [blame]783 # First compute hashes so that we can download them later if we need to.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]784 self._step_recorder.BeginStep('Compute hash for ' + filename)
785 self._RecordHash(filename)
786 if self._options.buildbot:
787 self._step_recorder.BeginStep('Archive ' + filename)
788 self._orderfile_updater.UploadToCloudStorage(
789 filename, use_debug_location=False)
790
Egor Paskobce64d012018-11-20 12:02:37[diff] [blame]791 def UploadReadyOrderfiles(self):
792 self._step_recorder.BeginStep('Upload Ready Orderfiles')
793 for file_name in [self._GetUnpatchedOrderfileFilename(),
794 self._GetPathToOrderfile()]:
795 self._orderfile_updater.UploadToCloudStorage(
796 file_name, use_debug_location=False)
797
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]798 def _NativeCodeMemoryBenchmark(self, apk):
799 """Runs system_health.memory_mobile to assess native code memory footprint.
800
801 Args:
802 apk: (str) Path to the apk.
803
804 Returns:
805 results: ([int]) Values of native code memory footprint in bytes from the
806 benchmark results.
807 """
808 self._step_recorder.BeginStep("Running orderfile.memory_mobile")
809 try:
810 out_dir = tempfile.mkdtemp()
811 self._profiler._RunCommand(['tools/perf/run_benchmark',
812 '--device={}'.format(
813 self._profiler._device.serial),
814 '--browser=exact',
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]815 '--output-format=csv',
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]816 '--output-dir={}'.format(out_dir),
817 '--reset-results',
818 '--browser-executable={}'.format(apk),
819 'orderfile.memory_mobile'])
820
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]821 out_file_path = os.path.join(out_dir, 'results.csv')
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]822 if not os.path.exists(out_file_path):
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]823 raise Exception('Results file not found!')
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]824
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]825 results = {}
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]826 with open(out_file_path, 'r') as f:
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]827 reader = csv.DictReader(f)
828 for row in reader:
829 if not row['name'].endswith('NativeCodeResidentMemory'):
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]830 continue
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]831 # Note: NativeCodeResidentMemory records a single sample from each
832 # story run, so this average (reported as 'avg') is exactly the value
833 # of that one sample. Each story is run multiple times, so this loop
834 # will accumulate into a list all values for all runs of each story.
835 results.setdefault(row['name'], {}).setdefault(
836 row['stories'], []).append(row['avg'])
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]837
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]838 if not results:
839 raise Exception('Could not find relevant results')
840
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]841 return results
842
843 except Exception as e:
844 return 'Error: ' + str(e)
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]845
846 finally:
847 shutil.rmtree(out_dir)
848
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]849
850 def _PerformanceBenchmark(self, apk):
851 """Runs Speedometer2.0 to assess performance.
852
853 Args:
854 apk: (str) Path to the apk.
855
856 Returns:
857 results: ([float]) Speedometer2.0 results samples in milliseconds.
858 """
859 self._step_recorder.BeginStep("Running Speedometer2.0.")
860 try:
861 out_dir = tempfile.mkdtemp()
862 self._profiler._RunCommand(['tools/perf/run_benchmark',
863 '--device={}'.format(
864 self._profiler._device.serial),
865 '--browser=exact',
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]866 '--output-format=histograms',
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]867 '--output-dir={}'.format(out_dir),
868 '--reset-results',
869 '--browser-executable={}'.format(apk),
870 'speedometer2'])
871
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]872 out_file_path = os.path.join(out_dir, 'histograms.json')
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]873 if not os.path.exists(out_file_path):
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]874 raise Exception('Results file not found!')
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]875
876 with open(out_file_path, 'r') as f:
877 results = json.load(f)
878
879 if not results:
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]880 raise Exception('Results file is empty.')
881
882 for el in results:
883 if 'name' in el and el['name'] == 'Total' and 'sampleValues' in el:
884 return el['sampleValues']
885
886 raise Exception('Unexpected results format.')
887
888 except Exception as e:
889 return 'Error: ' + str(e)
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]890
891 finally:
892 shutil.rmtree(out_dir)
893
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]894
895 def RunBenchmark(self, out_directory, no_orderfile=False):
896 """Builds chrome apk and runs performance and memory benchmarks.
897
898 Builds a non-instrumented version of chrome.
899 Installs chrome apk on the device.
900 Runs Speedometer2.0 benchmark to assess performance.
901 Runs system_health.memory_mobile to evaluate memory footprint.
902
903 Args:
904 out_directory: (str) Path to out directory for this build.
905 no_orderfile: (bool) True if chrome to be built without orderfile.
906
907 Returns:
908 benchmark_results: (dict) Results extracted from benchmarks.
909 """
Benoit Lize74e82b32021-08-26 14:22:01[diff] [blame]910 benchmark_results = {}
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]911 try:
912 _UnstashOutputDirectory(out_directory)
Christopher Grant073637472019-07-05 13:34:57[diff] [blame]913 self._compiler = ClankCompiler(out_directory, self._step_recorder,
914 self._options.arch, self._options.use_goma,
915 self._options.goma_dir,
Fumitoshi Ukaibd2755b2022-10-28 13:15:27[diff] [blame]916 self._options.use_remoteexec,
Fumitoshi Ukaibc233ba2023-02-13 11:47:31[diff] [blame^]917 self._ninja_command,
Christopher Grant073637472019-07-05 13:34:57[diff] [blame]918 self._options.system_health_orderfile,
919 self._monochrome, self._options.public,
920 self._GetPathToOrderfile())
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]921
922 if no_orderfile:
923 orderfile_path = self._GetPathToOrderfile()
924 backup_orderfile = orderfile_path + '.backup'
925 shutil.move(orderfile_path, backup_orderfile)
926 open(orderfile_path, 'w').close()
927
928 # Build APK to be installed on the device.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]929 self._compiler.CompileChromeApk(instrumented=False,
930 use_call_graph=False,
931 force_relink=True)
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]932 benchmark_results['Speedometer2.0'] = self._PerformanceBenchmark(
933 self._compiler.chrome_apk)
934 benchmark_results['orderfile.memory_mobile'] = (
935 self._NativeCodeMemoryBenchmark(self._compiler.chrome_apk))
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]936
937 except Exception as e:
938 benchmark_results['Error'] = str(e)
939
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]940 finally:
941 if no_orderfile and os.path.exists(backup_orderfile):
942 shutil.move(backup_orderfile, orderfile_path)
943 _StashOutputDirectory(out_directory)
944
945 return benchmark_results
946
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]947 def Generate(self):
948 """Generates and maybe upload an order."""
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]949 assert (bool(self._options.profile) ^
950 bool(self._options.manual_symbol_offsets))
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]951 if self._options.system_health_orderfile and not self._options.profile:
952 raise AssertionError('--system_health_orderfile must be not be used '
953 'with --skip-profile')
954 if (self._options.manual_symbol_offsets and
955 not self._options.system_health_orderfile):
956 raise AssertionError('--manual-symbol-offsets must be used with '
957 '--system_health_orderfile.')
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]958
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]959 if self._options.profile:
960 try:
961 _UnstashOutputDirectory(self._instrumented_out_dir)
962 self._compiler = ClankCompiler(
Christopher Grant073637472019-07-05 13:34:57[diff] [blame]963 self._instrumented_out_dir, self._step_recorder, self._options.arch,
964 self._options.use_goma, self._options.goma_dir,
Fumitoshi Ukaibc233ba2023-02-13 11:47:31[diff] [blame^]965 self._options.use_remoteexec, self._ninja_command,
966 self._options.system_health_orderfile, self._monochrome,
967 self._options.public, self._GetPathToOrderfile())
Matthew Carya2bea452018-11-13 10:21:07[diff] [blame]968 if not self._options.pregenerated_profiles:
969 # If there are pregenerated profiles, the instrumented build should
970 # not be changed to avoid invalidating the pregenerated profile
971 # offsets.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]972 self._compiler.CompileChromeApk(instrumented=True,
973 use_call_graph=
974 self._options.use_call_graph)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]975 self._GenerateAndProcessProfile()
976 self._MaybeArchiveOrderfile(self._GetUnpatchedOrderfileFilename())
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]977 finally:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]978 _StashOutputDirectory(self._instrumented_out_dir)
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]979 elif self._options.manual_symbol_offsets:
980 assert self._options.manual_libname
981 assert self._options.manual_objdir
Ari Chivukulaa52f8ba2021-08-10 22:30:39[diff] [blame]982 with open(self._options.manual_symbol_offsets) as f:
983 symbol_offsets = [int(x) for x in f]
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]984 processor = process_profiles.SymbolOffsetProcessor(
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]985 self._options.manual_libname)
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]986 generator = cyglog_to_orderfile.OffsetOrderfileGenerator(
987 processor, cyglog_to_orderfile.ObjectFileProcessor(
988 self._options.manual_objdir))
989 ordered_sections = generator.GetOrderedSections(symbol_offsets)
990 if not ordered_sections: # Either None or empty is a problem.
991 raise Exception('Failed to get ordered sections')
992 with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile:
993 orderfile.write('\n'.join(ordered_sections))
994
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]995 if self._options.patch:
996 if self._options.profile:
997 self._RemoveBlanks(self._GetUnpatchedOrderfileFilename(),
998 self._GetPathToOrderfile())
999 try:
1000 _UnstashOutputDirectory(self._uninstrumented_out_dir)
1001 self._compiler = ClankCompiler(
1002 self._uninstrumented_out_dir, self._step_recorder,
Christopher Grant073637472019-07-05 13:34:57[diff] [blame]1003 self._options.arch, self._options.use_goma, self._options.goma_dir,
Fumitoshi Ukaibc233ba2023-02-13 11:47:31[diff] [blame^]1004 self._options.use_remoteexec, self._ninja_command,
1005 self._options.system_health_orderfile, self._monochrome,
1006 self._options.public, self._GetPathToOrderfile())
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]1007
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]1008 self._compiler.CompileLibchrome(instrumented=False,
1009 use_call_graph=False)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1010 self._PatchOrderfile()
1011 # Because identical code folding is a bit different with and without
1012 # the orderfile build, we need to re-patch the orderfile with code
1013 # folding as close to the final version as possible.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]1014 self._compiler.CompileLibchrome(instrumented=False,
1015 use_call_graph=False, force_relink=True)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1016 self._PatchOrderfile()
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]1017 self._compiler.CompileLibchrome(instrumented=False,
1018 use_call_graph=False, force_relink=True)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1019 self._VerifySymbolOrder()
1020 self._MaybeArchiveOrderfile(self._GetPathToOrderfile())
1021 finally:
1022 _StashOutputDirectory(self._uninstrumented_out_dir)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1023
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]1024 if self._options.benchmark:
1025 self._output_data['orderfile_benchmark_results'] = self.RunBenchmark(
1026 self._uninstrumented_out_dir)
1027 self._output_data['no_orderfile_benchmark_results'] = self.RunBenchmark(
1028 self._no_orderfile_out_dir, no_orderfile=True)
1029
Egor Paskod80bfad2020-09-10 21:53:06[diff] [blame]1030 if self._options.buildbot:
1031 self._orderfile_updater._GitStash()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1032 self._step_recorder.EndStep()
1033 return not self._step_recorder.ErrorRecorded()
1034
1035 def GetReportingData(self):
1036 """Get a dictionary of reporting data (timings, output hashes)"""
1037 self._output_data['timings'] = self._step_recorder.timings
1038 return self._output_data
1039
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]1040 def CommitStashedOrderfileHashes(self):
1041 """Commit any orderfile hash files in the current checkout.
1042
1043 Only possible if running on the buildbot.
1044
1045 Returns: true on success.
1046 """
Egor Pasko7ff04122019-11-25 15:47:18[diff] [blame]1047 if not self._options.buildbot:
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]1048 logging.error('Trying to commit when not running on the buildbot')
1049 return False
1050 self._orderfile_updater._CommitStashedFiles([
1051 filename + '.sha1'
1052 for filename in (self._GetUnpatchedOrderfileFilename(),
1053 self._GetPathToOrderfile())])
1054 return True
1055
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1056
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1057def CreateArgumentParser():
1058 """Creates and returns the argument parser."""
1059 parser = argparse.ArgumentParser()
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]1060 parser.add_argument('--no-benchmark', action='store_false', dest='benchmark',
1061 default=True, help='Disables running benchmarks.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1062 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1063 '--buildbot', action='store_true',
1064 help='If true, the script expects to be run on a buildbot')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1065 parser.add_argument(
Matthew Cary453ff1452018-07-18 12:19:35[diff] [blame]1066 '--device', default=None, type=str,
1067 help='Device serial number on which to run profiling.')
1068 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1069 '--verify', action='store_true',
1070 help='If true, the script only verifies the current orderfile')
Benoit Lizef5581722021-08-26 09:48:20[diff] [blame]1071 parser.add_argument('--target-arch',
1072 action='store',
1073 dest='arch',
Stephen Martinis71b391b52018-12-04 15:08:04[diff] [blame]1074 default='arm',
Benoit Lizef5581722021-08-26 09:48:20[diff] [blame]1075 choices=list(_ARCH_GN_ARGS.keys()),
Matthew Cary53f74ba2019-01-24 10:07:18[diff] [blame]1076 help='The target architecture for which to build.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1077 parser.add_argument('--output-json', action='store', dest='json_file',
1078 help='Location to save stats in json format')
1079 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1080 '--skip-profile', action='store_false', dest='profile', default=True,
1081 help='Don\'t generate a profile on the device. Only patch from the '
1082 'existing profile.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1083 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1084 '--skip-patch', action='store_false', dest='patch', default=True,
1085 help='Only generate the raw (unpatched) orderfile, don\'t patch it.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1086 parser.add_argument('--goma-dir', help='GOMA directory.')
1087 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1088 '--use-goma', action='store_true', help='Enable GOMA.', default=False)
Fumitoshi Ukaibd2755b2022-10-28 13:15:27[diff] [blame]1089 parser.add_argument('--use-remoteexec',
1090 action='store_true',
Fumitoshi Ukaibc233ba2023-02-13 11:47:31[diff] [blame^]1091 help='Enable remoteexec. see //build/toolchain/rbe.gni.',
Fumitoshi Ukaibd2755b2022-10-28 13:15:27[diff] [blame]1092 default=False)
Fumitoshi Ukaibc233ba2023-02-13 11:47:31[diff] [blame^]1093 parser.add_argument('--ninja-path',
1094 help='Path to the ninja binary. If given, use this'
1095 'instead of autoninja.')
1096 parser.add_argument('--ninja-j',
1097 help='-j value passed to ninja.'
1098 'pass -j to ninja. no need to set this when '
1099 '--ninja-path is not specified.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1100 parser.add_argument('--adb-path', help='Path to the adb binary.')
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]1101
Egor Paskod80bfad2020-09-10 21:53:06[diff] [blame]1102 parser.add_argument('--public',
1103 action='store_true',
1104 help='Build non-internal APK and change the orderfile '
1105 'location. Required if your checkout is non-internal.',
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]1106 default=False)
Matthew Cary04f41032018-12-10 15:55:27[diff] [blame]1107 parser.add_argument('--nosystem-health-orderfile', action='store_false',
Benoit Lcba421e2019-04-26 15:28:26[diff] [blame]1108 dest='system_health_orderfile', default=True,
Matthew Caryc262f5d2018-09-19 14:36:28[diff] [blame]1109 help=('Create an orderfile based on an about:blank '
1110 'startup benchmark instead of system health '
1111 'benchmarks.'))
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]1112 parser.add_argument(
1113 '--use-legacy-chrome-apk', action='store_true', default=False,
1114 help=('Compile and instrument chrome for [L, K] devices.'))
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]1115 parser.add_argument('--manual-symbol-offsets', default=None, type=str,
1116 help=('File of list of ordered symbol offsets generated '
1117 'by manual profiling. Must set other --manual* '
1118 'flags if this is used, and must --skip-profile.'))
1119 parser.add_argument('--manual-libname', default=None, type=str,
1120 help=('Library filename corresponding to '
1121 '--manual-symbol-offsets.'))
1122 parser.add_argument('--manual-objdir', default=None, type=str,
1123 help=('Root of object file directory corresponding to '
1124 '--manual-symbol-offsets.'))
Matthew Caryf949bba2019-02-04 13:39:23[diff] [blame]1125 parser.add_argument('--noorder-outlined-functions', action='store_true',
1126 help='Disable outlined functions in the orderfile.')
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]1127 parser.add_argument('--pregenerated-profiles', default=None, type=str,
1128 help=('Pregenerated profiles to use instead of running '
1129 'profile step. Cannot be used with '
1130 '--skip-profiles.'))
1131 parser.add_argument('--profile-save-dir', default=None, type=str,
1132 help=('Directory to save any profiles created. These can '
1133 'be used with --pregenerated-profiles. Cannot be '
1134 'used with --skip-profiles.'))
Egor Paskobce64d012018-11-20 12:02:37[diff] [blame]1135 parser.add_argument('--upload-ready-orderfiles', action='store_true',
1136 help=('Skip orderfile generation and manually upload '
1137 'orderfiles (both patched and unpatched) from '
1138 'their normal location in the tree to the cloud '
1139 'storage. DANGEROUS! USE WITH CARE!'))
Matthew Cary1ea82212019-03-20 12:10:27[diff] [blame]1140 parser.add_argument('--streamline-for-debugging', action='store_true',
1141 help=('Streamline where possible the run for faster '
1142 'iteration while debugging. The orderfile '
1143 'generated will be valid and nontrivial, but '
1144 'may not be based on a representative profile '
1145 'or other such considerations. Use with caution.'))
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]1146 parser.add_argument('--commit-hashes', action='store_true',
1147 help=('Commit any orderfile hash files in the current '
1148 'checkout; performs no other action'))
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]1149 parser.add_argument('--use-call-graph', action='store_true', default=False,
1150 help='Use call graph instrumentation.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1151 profile_android_startup.AddProfileCollectionArguments(parser)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1152 return parser
1153
1154
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]1155def CreateOrderfile(options, orderfile_updater_class=None):
Christopher Grantdfe1bac2019-07-05 13:34:10[diff] [blame]1156 """Creates an orderfile.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1157
1158 Args:
1159 options: As returned from optparse.OptionParser.parse_args()
1160 orderfile_updater_class: (OrderfileUpdater) subclass of OrderfileUpdater.
1161
1162 Returns:
1163 True iff success.
1164 """
1165 logging.basicConfig(level=logging.INFO)
1166 devil_chromium.Initialize(adb_path=options.adb_path)
1167
Benoit Lize6208ddc2021-08-30 12:26:29[diff] [blame]1168 # Since we generate a ".arm" orderfile irrespective of the architecture (see
1169 # comment in _GetPathToOrderfile()), make sure that we don't commit it.
1170 if options.arch != 'arm':
1171 assert not options.buildbot, (
1172 'ARM is the only supported architecture on bots')
1173 assert not options.upload_ready_orderfiles, (
1174 'ARM is the only supported architecture on bots')
1175
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]1176 generator = OrderfileGenerator(options, orderfile_updater_class)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1177 try:
1178 if options.verify:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]1179 generator._VerifySymbolOrder()
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]1180 elif options.commit_hashes:
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]1181 return generator.CommitStashedOrderfileHashes()
Egor Paskobce64d012018-11-20 12:02:37[diff] [blame]1182 elif options.upload_ready_orderfiles:
1183 return generator.UploadReadyOrderfiles()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1184 else:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]1185 return generator.Generate()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1186 finally:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]1187 json_output = json.dumps(generator.GetReportingData(),
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1188 indent=2) + '\n'
1189 if options.json_file:
1190 with open(options.json_file, 'w') as f:
1191 f.write(json_output)
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]1192 print(json_output)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1193 return False
1194
1195
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1196def main():
1197 parser = CreateArgumentParser()
1198 options = parser.parse_args()
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]1199 return 0 if CreateOrderfile(options) else 1
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1200
1201
1202if __name__ == '__main__':
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1203 sys.exit(main())