[go: nahoru, domu]
Google Git
Sign in
chromium / chromium / src / bd2755b885a4389bb2bd10f87c131bde309c694e / . / tools / cygprofile / orderfile_generator_backend.py
blob: 17abc57a6a02adb0eb46a873f36c96ba7c780e66 [file] [log] [blame]
Ari Chivukulaa52f8ba2021-08-10 22:30:39[diff] [blame]1#!/usr/bin/env vpython3
Avi Drissmandfd880852022-09-15 20:11:09[diff] [blame]2# Copyright 2013 The Chromium Authors
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6""" A utility to generate an up-to-date orderfile.
7
8The orderfile is used by the linker to order text sections such that the
9sections are placed consecutively in the order specified. This allows us
10to page in less code during start-up.
11
12Example usage:
Egor Paskoa7ac67a42020-09-09 12:06:58[diff] [blame]13 tools/cygprofile/orderfile_generator_backend.py --use-goma --target-arch=arm
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]14"""
15
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]16
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]17import argparse
Egor Paskoa7ac67a42020-09-09 12:06:58[diff] [blame]18import csv
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]19import hashlib
20import json
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]21import glob
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]22import logging
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]23import os
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]24import shutil
25import subprocess
26import sys
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]27import tempfile
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]28import time
29
Matthew Cary91df9792018-11-30 14:35:15[diff] [blame]30import cluster
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]31import cyglog_to_orderfile
Benoit Lizefefbb27c2018-01-17 13:54:18[diff] [blame]32import patch_orderfile
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]33import process_profiles
Egor Pasko3bc0b932018-04-03 10:08:27[diff] [blame]34import profile_android_startup
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]35
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]36_SRC_PATH = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
37sys.path.append(os.path.join(_SRC_PATH, 'third_party', 'catapult', 'devil'))
38from devil.android import device_utils
39from devil.android.sdk import version_codes
40
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]41
42_SRC_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)),
43 os.pardir, os.pardir)
44sys.path.append(os.path.join(_SRC_PATH, 'build', 'android'))
45import devil_chromium
46from pylib import constants
47
48
49# Needs to happen early for GetBuildType()/GetOutDirectory() to work correctly
50constants.SetBuildType('Release')
51
52
Matthew Cary53f74ba2019-01-24 10:07:18[diff] [blame]53# Architecture specific GN args. Trying to build an orderfile for an
54# architecture not listed here will eventually throw.
55_ARCH_GN_ARGS = {
Benoit Lizef5581722021-08-26 09:48:20[diff] [blame]56 'arm': ['target_cpu = "arm"'],
57 'arm64': ['target_cpu = "arm64"', 'android_64bit_browser = true'],
58 'x86': ['target_cpu = "x86"'],
Matthew Cary53f74ba2019-01-24 10:07:18[diff] [blame]59}
60
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]61class CommandError(Exception):
62 """Indicates that a dispatched shell command exited with a non-zero status."""
63
64 def __init__(self, value):
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]65 super().__init__()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]66 self.value = value
67
68 def __str__(self):
69 return repr(self.value)
70
71
72def _GenerateHash(file_path):
73 """Calculates and returns the hash of the file at file_path."""
74 sha1 = hashlib.sha1()
75 with open(file_path, 'rb') as f:
76 while True:
77 # Read in 1mb chunks, so it doesn't all have to be loaded into memory.
78 chunk = f.read(1024 * 1024)
79 if not chunk:
80 break
81 sha1.update(chunk)
82 return sha1.hexdigest()
83
84
85def _GetFileExtension(file_name):
86 """Calculates the file extension from a file name.
87
88 Args:
89 file_name: The source file name.
90 Returns:
91 The part of file_name after the dot (.) or None if the file has no
92 extension.
93 Examples: /home/user/foo.bar -> bar
94 /home/user.name/foo -> None
95 /home/user/.foo -> None
96 /home/user/foo.bar.baz -> baz
97 """
98 file_name_parts = os.path.basename(file_name).split('.')
99 if len(file_name_parts) > 1:
100 return file_name_parts[-1]
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]101 return None
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]102
103
104def _StashOutputDirectory(buildpath):
105 """Takes the output directory and stashes it in the default output directory.
106
107 This allows it to be used for incremental builds next time (after unstashing)
108 by keeping it in a place that isn't deleted normally, while also ensuring
109 that it is properly clobbered when appropriate.
110
111 This is a dirty hack to deal with the needs of clobbering while also handling
112 incremental builds and the hardcoded relative paths used in some of the
113 project files.
114
115 Args:
116 buildpath: The path where the building happens. If this corresponds to the
117 default output directory, no action is taken.
118 """
119 if os.path.abspath(buildpath) == os.path.abspath(os.path.dirname(
120 constants.GetOutDirectory())):
121 return
122 name = os.path.basename(buildpath)
123 stashpath = os.path.join(constants.GetOutDirectory(), name)
124 if not os.path.exists(buildpath):
125 return
126 if os.path.exists(stashpath):
127 shutil.rmtree(stashpath, ignore_errors=True)
128 shutil.move(buildpath, stashpath)
129
130
131def _UnstashOutputDirectory(buildpath):
132 """Inverse of _StashOutputDirectory.
133
134 Moves the output directory stashed within the default output directory
135 (out/Release) to the position where the builds can actually happen.
136
137 This is a dirty hack to deal with the needs of clobbering while also handling
138 incremental builds and the hardcoded relative paths used in some of the
139 project files.
140
141 Args:
142 buildpath: The path where the building happens. If this corresponds to the
143 default output directory, no action is taken.
144 """
145 if os.path.abspath(buildpath) == os.path.abspath(os.path.dirname(
146 constants.GetOutDirectory())):
147 return
148 name = os.path.basename(buildpath)
149 stashpath = os.path.join(constants.GetOutDirectory(), name)
150 if not os.path.exists(stashpath):
151 return
152 if os.path.exists(buildpath):
153 shutil.rmtree(buildpath, ignore_errors=True)
154 shutil.move(stashpath, buildpath)
155
156
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]157class StepRecorder:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]158 """Records steps and timings."""
159
160 def __init__(self, buildbot):
161 self.timings = []
162 self._previous_step = ('', 0.0)
163 self._buildbot = buildbot
164 self._error_recorded = False
165
166 def BeginStep(self, name):
167 """Marks a beginning of the next step in the script.
168
169 On buildbot, this prints a specially formatted name that will show up
170 in the waterfall. Otherwise, just prints the step name.
171
172 Args:
173 name: The name of the step.
174 """
175 self.EndStep()
176 self._previous_step = (name, time.time())
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]177 print('Running step: ', name)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]178
179 def EndStep(self):
180 """Records successful completion of the current step.
181
182 This is optional if the step is immediately followed by another BeginStep.
183 """
184 if self._previous_step[0]:
185 elapsed = time.time() - self._previous_step[1]
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]186 print('Step %s took %f seconds' % (self._previous_step[0], elapsed))
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]187 self.timings.append((self._previous_step[0], elapsed))
188
189 self._previous_step = ('', 0.0)
190
191 def FailStep(self, message=None):
192 """Marks that a particular step has failed.
193
194 On buildbot, this will mark the current step as failed on the waterfall.
195 Otherwise we will just print an optional failure message.
196
197 Args:
198 message: An optional explanation as to why the step failed.
199 """
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]200 print('STEP FAILED!!')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]201 if message:
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]202 print(message)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]203 self._error_recorded = True
204 self.EndStep()
205
206 def ErrorRecorded(self):
207 """True if FailStep has been called."""
208 return self._error_recorded
209
210 def RunCommand(self, cmd, cwd=constants.DIR_SOURCE_ROOT, raise_on_error=True,
211 stdout=None):
212 """Execute a shell command.
213
214 Args:
215 cmd: A list of command strings.
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]216 cwd: Directory in which the command should be executed, defaults to build
217 root of script's location if not specified.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]218 raise_on_error: If true will raise a CommandError if the call doesn't
219 succeed and mark the step as failed.
220 stdout: A file to redirect stdout for the command to.
221
222 Returns:
223 The process's return code.
224
225 Raises:
226 CommandError: An error executing the specified command.
227 """
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]228 print('Executing %s in %s' % (' '.join(cmd), cwd))
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]229 process = subprocess.Popen(cmd, stdout=stdout, cwd=cwd, env=os.environ)
230 process.wait()
231 if raise_on_error and process.returncode != 0:
232 self.FailStep()
233 raise CommandError('Exception executing command %s' % ' '.join(cmd))
234 return process.returncode
235
236
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]237class ClankCompiler:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]238 """Handles compilation of clank."""
239
Christopher Grant073637472019-07-05 13:34:57[diff] [blame]240 def __init__(self, out_dir, step_recorder, arch, use_goma, goma_dir,
Fumitoshi Ukaibd2755b2022-10-28 13:15:27[diff] [blame^]241 use_remoteexec, system_health_profiling, monochrome, public,
242 orderfile_location):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]243 self._out_dir = out_dir
244 self._step_recorder = step_recorder
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]245 self._arch = arch
Fumitoshi Ukaibd2755b2022-10-28 13:15:27[diff] [blame^]246 # TODO(b/236070141): remove goma config.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]247 self._use_goma = use_goma
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]248 self._goma_dir = goma_dir
Fumitoshi Ukaibd2755b2022-10-28 13:15:27[diff] [blame^]249 self._use_remoteexec = use_remoteexec
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]250 self._system_health_profiling = system_health_profiling
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]251 self._public = public
252 self._orderfile_location = orderfile_location
Matthew Caryd6bfcb72018-09-14 11:27:46[diff] [blame]253 if monochrome:
254 self._apk = 'Monochrome.apk'
Matthew Carye1b00062018-09-19 14:27:44[diff] [blame]255 self._apk_target = 'monochrome_apk'
Matthew Caryd6bfcb72018-09-14 11:27:46[diff] [blame]256 self._libname = 'libmonochrome'
Matthew Cary2216fef2019-06-17 13:35:59[diff] [blame]257 self._libchrome_target = 'libmonochrome'
Matthew Caryd6bfcb72018-09-14 11:27:46[diff] [blame]258 else:
259 self._apk = 'Chrome.apk'
Matthew Carye1b00062018-09-19 14:27:44[diff] [blame]260 self._apk_target = 'chrome_apk'
Matthew Caryd6bfcb72018-09-14 11:27:46[diff] [blame]261 self._libname = 'libchrome'
262 self._libchrome_target = 'libchrome'
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]263 if public:
264 self._apk = self._apk.replace('.apk', 'Public.apk')
265 self._apk_target = self._apk_target.replace('_apk', '_public_apk')
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]266
267 self.obj_dir = os.path.join(self._out_dir, 'Release', 'obj')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]268 self.lib_chrome_so = os.path.join(
Matthew Caryd6bfcb72018-09-14 11:27:46[diff] [blame]269 self._out_dir, 'Release', 'lib.unstripped',
270 '{}.so'.format(self._libname))
271 self.chrome_apk = os.path.join(self._out_dir, 'Release', 'apks', self._apk)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]272
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]273 def Build(self, instrumented, use_call_graph, target):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]274 """Builds the provided ninja target with or without order_profiling on.
275
276 Args:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]277 instrumented: (bool) Whether we want to build an instrumented binary.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]278 use_call_graph: (bool) Whether to use the call graph instrumentation.
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]279 target: (str) The name of the ninja target to build.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]280 """
281 self._step_recorder.BeginStep('Compile %s' % target)
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]282 assert not use_call_graph or instrumented, ('You can not enable call graph '
283 'without instrumentation!')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]284
285 # Set the "Release Official" flavor, the parts affecting performance.
286 args = [
Andrew Grieve0e8790c2020-09-03 17:27:32[diff] [blame]287 'enable_resource_allowlist_generation=false',
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]288 'is_chrome_branded=' + str(not self._public).lower(),
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]289 'is_debug=false',
290 'is_official_build=true',
Egor Pasko1d13d532020-01-14 21:25:19[diff] [blame]291 'symbol_level=1', # to fit 30 GiB RAM on the bot when LLD is running
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]292 'target_os="android"',
293 'use_goma=' + str(self._use_goma).lower(),
Fumitoshi Ukaibd2755b2022-10-28 13:15:27[diff] [blame^]294 'use_remoteexec=' + str(self._use_remoteexec).lower(),
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]295 'use_order_profiling=' + str(instrumented).lower(),
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]296 'use_call_graph=' + str(use_call_graph).lower(),
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]297 ]
Matthew Cary53f74ba2019-01-24 10:07:18[diff] [blame]298 args += _ARCH_GN_ARGS[self._arch]
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]299 if self._goma_dir:
300 args += ['goma_dir="%s"' % self._goma_dir]
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]301 if self._system_health_profiling:
302 args += ['devtools_instrumentation_dumping = ' +
303 str(instrumented).lower()]
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]304
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]305 if self._public and os.path.exists(self._orderfile_location):
306 # GN needs the orderfile path to be source-absolute.
307 src_abs_orderfile = os.path.relpath(self._orderfile_location,
308 constants.DIR_SOURCE_ROOT)
309 args += ['chrome_orderfile="//{}"'.format(src_abs_orderfile)]
310
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]311 self._step_recorder.RunCommand(
312 ['gn', 'gen', os.path.join(self._out_dir, 'Release'),
313 '--args=' + ' '.join(args)])
314
315 self._step_recorder.RunCommand(
Christopher Grant073637472019-07-05 13:34:57[diff] [blame]316 ['autoninja', '-C',
317 os.path.join(self._out_dir, 'Release'), target])
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]318
Christopher Grant10221762019-07-05 12:10:04[diff] [blame]319 def ForceRelink(self):
320 """Forces libchrome.so or libmonochrome.so to be re-linked.
321
322 With partitioned libraries enabled, deleting these library files does not
323 guarantee they'll be recreated by the linker (they may simply be
324 re-extracted from a combined library). To be safe, touch a source file
325 instead. See http://crbug.com/972701 for more explanation.
326 """
327 file_to_touch = os.path.join(constants.DIR_SOURCE_ROOT, 'chrome', 'browser',
328 'chrome_browser_main_android.cc')
329 assert os.path.exists(file_to_touch)
330 self._step_recorder.RunCommand(['touch', file_to_touch])
331
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]332 def CompileChromeApk(self, instrumented, use_call_graph, force_relink=False):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]333 """Builds a Chrome.apk either with or without order_profiling on.
334
335 Args:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]336 instrumented: (bool) Whether to build an instrumented apk.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]337 use_call_graph: (bool) Whether to use the call graph instrumentation.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]338 force_relink: Whether libchromeview.so should be re-created.
339 """
340 if force_relink:
Christopher Grant10221762019-07-05 12:10:04[diff] [blame]341 self.ForceRelink()
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]342 self.Build(instrumented, use_call_graph, self._apk_target)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]343
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]344 def CompileLibchrome(self, instrumented, use_call_graph, force_relink=False):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]345 """Builds a libchrome.so either with or without order_profiling on.
346
347 Args:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]348 instrumented: (bool) Whether to build an instrumented apk.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]349 use_call_graph: (bool) Whether to use the call graph instrumentation.
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]350 force_relink: (bool) Whether libchrome.so should be re-created.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]351 """
352 if force_relink:
Christopher Grant10221762019-07-05 12:10:04[diff] [blame]353 self.ForceRelink()
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]354 self.Build(instrumented, use_call_graph, self._libchrome_target)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]355
356
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]357class OrderfileUpdater:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]358 """Handles uploading and committing a new orderfile in the repository.
359
360 Only used for testing or on a bot.
361 """
362
363 _CLOUD_STORAGE_BUCKET_FOR_DEBUG = None
364 _CLOUD_STORAGE_BUCKET = None
365 _UPLOAD_TO_CLOUD_COMMAND = 'upload_to_google_storage.py'
366
Egor Pasko0c533c6682019-11-26 21:16:32[diff] [blame]367 def __init__(self, repository_root, step_recorder):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]368 """Constructor.
369
370 Args:
371 repository_root: (str) Root of the target repository.
372 step_recorder: (StepRecorder) Step recorder, for logging.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]373 """
374 self._repository_root = repository_root
375 self._step_recorder = step_recorder
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]376
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]377 def CommitStashedFileHashes(self, files):
378 """Commits unpatched and patched orderfiles hashes if changed.
379
380 The files are committed only if their associated sha1 hash files match, and
381 are modified in git. In normal operations the hash files are changed only
382 when a file is uploaded to cloud storage. If the hash file is not modified
383 in git, the file is skipped.
384
385 Args:
386 files: [str or None] specifies file paths. None items are ignored.
387
388 Raises:
389 Exception if the hash file does not match the file.
390 NotImplementedError when the commit logic hasn't been overridden.
391 """
Ari Chivukulaa52f8ba2021-08-10 22:30:39[diff] [blame]392 files_to_commit = [_f for _f in files if _f]
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]393 if files_to_commit:
394 self._CommitStashedFiles(files_to_commit)
395
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]396 def UploadToCloudStorage(self, filename, use_debug_location):
397 """Uploads a file to cloud storage.
398
399 Args:
400 filename: (str) File to upload.
401 use_debug_location: (bool) Whether to use the debug location.
402 """
403 bucket = (self._CLOUD_STORAGE_BUCKET_FOR_DEBUG if use_debug_location
404 else self._CLOUD_STORAGE_BUCKET)
405 extension = _GetFileExtension(filename)
406 cmd = [self._UPLOAD_TO_CLOUD_COMMAND, '--bucket', bucket]
407 if extension:
408 cmd.extend(['-z', extension])
409 cmd.append(filename)
410 self._step_recorder.RunCommand(cmd)
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]411 print('Download: https://sandbox.google.com/storage/%s/%s' %
412 (bucket, _GenerateHash(filename)))
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]413
414 def _GetHashFilePathAndContents(self, filename):
415 """Gets the name and content of the hash file created from uploading the
416 given file.
417
418 Args:
419 filename: (str) The file that was uploaded to cloud storage.
420
421 Returns:
422 A tuple of the hash file name, relative to the reository root, and the
423 content, which should be the sha1 hash of the file
424 ('base_file.sha1', hash)
425 """
426 abs_hash_filename = filename + '.sha1'
427 rel_hash_filename = os.path.relpath(
428 abs_hash_filename, self._repository_root)
429 with open(abs_hash_filename, 'r') as f:
430 return (rel_hash_filename, f.read())
431
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]432 def _GitStash(self):
433 """Git stash the current clank tree.
434
435 Raises:
436 NotImplementedError when the stash logic hasn't been overridden.
437 """
438 raise NotImplementedError
439
440 def _CommitStashedFiles(self, expected_files_in_stash):
441 """Commits stashed files.
442
443 The local repository is updated and then the files to commit are taken from
444 modified files from the git stash. The modified files should be a subset of
445 |expected_files_in_stash|. If there are unexpected modified files, this
446 function may raise. This is meant to be paired with _GitStash().
447
448 Args:
449 expected_files_in_stash: [str] paths to a possible superset of files
450 expected to be stashed & committed.
451
452 Raises:
453 NotImplementedError when the commit logic hasn't been overridden.
454 """
455 raise NotImplementedError
456
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]457
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]458class OrderfileGenerator:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]459 """A utility for generating a new orderfile for Clank.
460
461 Builds an instrumented binary, profiles a run of the application, and
462 generates an updated orderfile.
463 """
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]464 _CHECK_ORDERFILE_SCRIPT = os.path.join(
465 constants.DIR_SOURCE_ROOT, 'tools', 'cygprofile', 'check_orderfile.py')
466 _BUILD_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(
467 constants.GetOutDirectory()))) # Normally /path/to/src
468
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]469 # Previous orderfile_generator debug files would be overwritten.
470 _DIRECTORY_FOR_DEBUG_FILES = '/tmp/orderfile_generator_debug_files'
471
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]472 _CLOUD_STORAGE_BUCKET_FOR_DEBUG = None
473
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]474 def _PrepareOrderfilePaths(self):
475 if self._options.public:
476 self._clank_dir = os.path.join(constants.DIR_SOURCE_ROOT,
477 '')
478 if not os.path.exists(os.path.join(self._clank_dir, 'orderfiles')):
479 os.makedirs(os.path.join(self._clank_dir, 'orderfiles'))
480 else:
481 self._clank_dir = os.path.join(constants.DIR_SOURCE_ROOT,
482 'clank')
483
484 self._unpatched_orderfile_filename = os.path.join(
485 self._clank_dir, 'orderfiles', 'unpatched_orderfile.%s')
486 self._path_to_orderfile = os.path.join(
487 self._clank_dir, 'orderfiles', 'orderfile.%s.out')
488
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]489 def _GetPathToOrderfile(self):
490 """Gets the path to the architecture-specific orderfile."""
Benoit Lize6208ddc2021-08-30 12:26:29[diff] [blame]491 # Build GN files use the ".arm" orderfile irrespective of the actual
492 # architecture. Fake it, otherwise the orderfile we generate here is not
493 # going to be picked up by builds.
494 orderfile_fake_arch = 'arm'
495 return self._path_to_orderfile % orderfile_fake_arch
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]496
497 def _GetUnpatchedOrderfileFilename(self):
498 """Gets the path to the architecture-specific unpatched orderfile."""
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]499 return self._unpatched_orderfile_filename % self._options.arch
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]500
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]501 def _SetDevice(self):
502 """ Selects the device to be used by the script.
503
504 Returns:
505 (Device with given serial ID) : if the --device flag is set.
506 (Device running Android[K,L]) : if --use-legacy-chrome-apk flag is set or
507 no device running Android N+ was found.
508 (Device running Android N+) : Otherwise.
509
510 Raises Error:
511 If no device meeting the requirements has been found.
512 """
513 devices = None
514 if self._options.device:
515 devices = [device_utils.DeviceUtils(self._options.device)]
516 else:
517 devices = device_utils.DeviceUtils.HealthyDevices()
518
519 assert devices, 'Expected at least one connected device'
520
521 if self._options.use_legacy_chrome_apk:
522 self._monochrome = False
523 for device in devices:
524 device_version = device.build_version_sdk
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]525 if (version_codes.KITKAT <= device_version <=
526 version_codes.LOLLIPOP_MR1):
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]527 return device
528
529 assert not self._options.use_legacy_chrome_apk, \
530 'No device found running suitable android version for Chrome.apk.'
531
532 preferred_device = None
533 for device in devices:
534 if device.build_version_sdk >= version_codes.NOUGAT:
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]535 preferred_device = device
536 break
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]537
538 self._monochrome = preferred_device is not None
539
540 return preferred_device if preferred_device else devices[0]
541
542
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]543 def __init__(self, options, orderfile_updater_class):
544 self._options = options
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]545 self._instrumented_out_dir = os.path.join(
546 self._BUILD_ROOT, self._options.arch + '_instrumented_out')
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]547 if self._options.use_call_graph:
Christopher Grantdfe1bac2019-07-05 13:34:10[diff] [blame]548 self._instrumented_out_dir += '_call_graph'
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]549
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]550 self._uninstrumented_out_dir = os.path.join(
551 self._BUILD_ROOT, self._options.arch + '_uninstrumented_out')
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]552 self._no_orderfile_out_dir = os.path.join(
553 self._BUILD_ROOT, self._options.arch + '_no_orderfile_out')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]554
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]555 self._PrepareOrderfilePaths()
556
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]557 if options.profile:
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]558 output_directory = os.path.join(self._instrumented_out_dir, 'Release')
Matthew Caryb8daed942018-06-11 10:58:08[diff] [blame]559 host_profile_dir = os.path.join(output_directory, 'profile_data')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]560 urls = [profile_android_startup.AndroidProfileTool.TEST_URL]
561 use_wpr = True
562 simulate_user = False
Benoit L96466812018-03-06 15:58:37[diff] [blame]563 urls = options.urls
564 use_wpr = not options.no_wpr
565 simulate_user = options.simulate_user
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]566 device = self._SetDevice()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]567 self._profiler = profile_android_startup.AndroidProfileTool(
Matthew Cary453ff1452018-07-18 12:19:35[diff] [blame]568 output_directory, host_profile_dir, use_wpr, urls, simulate_user,
Matthew Cary1ea82212019-03-20 12:10:27[diff] [blame]569 device, debug=self._options.streamline_for_debugging)
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]570 if options.pregenerated_profiles:
571 self._profiler.SetPregeneratedProfiles(
572 glob.glob(options.pregenerated_profiles))
573 else:
574 assert not options.pregenerated_profiles, (
575 '--pregenerated-profiles cannot be used with --skip-profile')
576 assert not options.profile_save_dir, (
577 '--profile-save-dir cannot be used with --skip-profile')
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]578 self._monochrome = not self._options.use_legacy_chrome_apk
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]579
Matthew Caryf949bba2019-02-04 13:39:23[diff] [blame]580 # Outlined function handling enabled by default for all architectures.
581 self._order_outlined_functions = not options.noorder_outlined_functions
582
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]583 self._output_data = {}
584 self._step_recorder = StepRecorder(options.buildbot)
585 self._compiler = None
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]586 if orderfile_updater_class is None:
Egor Paskod80bfad2020-09-10 21:53:06[diff] [blame]587 orderfile_updater_class = OrderfileUpdater
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]588 assert issubclass(orderfile_updater_class, OrderfileUpdater)
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]589 self._orderfile_updater = orderfile_updater_class(self._clank_dir,
Egor Pasko0c533c6682019-11-26 21:16:32[diff] [blame]590 self._step_recorder)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]591 assert os.path.isdir(constants.DIR_SOURCE_ROOT), 'No src directory found'
592
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]593 @staticmethod
594 def _RemoveBlanks(src_file, dest_file):
595 """A utility to remove blank lines from a file.
596
597 Args:
598 src_file: The name of the file to remove the blanks from.
599 dest_file: The name of the file to write the output without blanks.
600 """
601 assert src_file != dest_file, 'Source and destination need to be distinct'
602
603 try:
604 src = open(src_file, 'r')
605 dest = open(dest_file, 'w')
606 for line in src:
607 if line and not line.isspace():
608 dest.write(line)
609 finally:
610 src.close()
611 dest.close()
612
613 def _GenerateAndProcessProfile(self):
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]614 """Invokes a script to merge the per-thread traces into one file.
615
616 The produced list of offsets is saved in
617 self._GetUnpatchedOrderfileFilename().
618 """
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]619 self._step_recorder.BeginStep('Generate Profile Data')
620 files = []
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]621 logging.getLogger().setLevel(logging.DEBUG)
Christopher Grantdfe1bac2019-07-05 13:34:10[diff] [blame]622
623 if self._options.profile_save_dir:
624 # The directory must not preexist, to ensure purity of data. Check
625 # before profiling to save time.
626 if os.path.exists(self._options.profile_save_dir):
627 raise Exception('Profile save directory must not pre-exist')
628 os.makedirs(self._options.profile_save_dir)
629
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]630 if self._options.system_health_orderfile:
631 files = self._profiler.CollectSystemHealthProfile(
632 self._compiler.chrome_apk)
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]633 self._MaybeSaveProfile(files)
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]634 try:
635 self._ProcessPhasedOrderfile(files)
636 except Exception:
637 for f in files:
638 self._SaveForDebugging(f)
Matthew Cary8b1416232018-08-10 19:12:22[diff] [blame]639 self._SaveForDebugging(self._compiler.lib_chrome_so)
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]640 raise
641 finally:
642 self._profiler.Cleanup()
643 else:
644 self._CollectLegacyProfile()
645 logging.getLogger().setLevel(logging.INFO)
646
647 def _ProcessPhasedOrderfile(self, files):
648 """Process the phased orderfiles produced by system health benchmarks.
649
650 The offsets will be placed in _GetUnpatchedOrderfileFilename().
651
652 Args:
653 file: Profile files pulled locally.
654 """
655 self._step_recorder.BeginStep('Process Phased Orderfile')
656 profiles = process_profiles.ProfileManager(files)
657 processor = process_profiles.SymbolOffsetProcessor(
658 self._compiler.lib_chrome_so)
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]659 ordered_symbols = cluster.ClusterOffsets(profiles, processor,
660 call_graph=self._options.use_call_graph)
Matthew Cary8b1416232018-08-10 19:12:22[diff] [blame]661 if not ordered_symbols:
662 raise Exception('Failed to get ordered symbols')
Matthew Caryda9db932019-03-11 15:28:17[diff] [blame]663 for sym in ordered_symbols:
664 assert not sym.startswith('OUTLINED_FUNCTION_'), (
665 'Outlined function found in instrumented function, very likely '
666 'something has gone very wrong!')
Matthew Cary91df9792018-11-30 14:35:15[diff] [blame]667 self._output_data['offsets_kib'] = processor.SymbolsSize(
668 ordered_symbols) / 1024
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]669 with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile:
Matthew Cary8b1416232018-08-10 19:12:22[diff] [blame]670 orderfile.write('\n'.join(ordered_symbols))
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]671
672 def _CollectLegacyProfile(self):
Matthew Cary2df8d452018-09-19 07:50:20[diff] [blame]673 files = []
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]674 try:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]675 files = self._profiler.CollectProfile(
676 self._compiler.chrome_apk,
677 constants.PACKAGE_INFO['chrome'])
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]678 self._MaybeSaveProfile(files)
Matthew Caryb8daed942018-06-11 10:58:08[diff] [blame]679 self._step_recorder.BeginStep('Process profile')
Benoit L96466812018-03-06 15:58:37[diff] [blame]680 assert os.path.exists(self._compiler.lib_chrome_so)
681 offsets = process_profiles.GetReachedOffsetsFromDumpFiles(
682 files, self._compiler.lib_chrome_so)
683 if not offsets:
684 raise Exception('No profiler offsets found in {}'.format(
685 '\n'.join(files)))
Matthew Cary8b1416232018-08-10 19:12:22[diff] [blame]686 processor = process_profiles.SymbolOffsetProcessor(
687 self._compiler.lib_chrome_so)
688 ordered_symbols = processor.GetOrderedSymbols(offsets)
689 if not ordered_symbols:
690 raise Exception('No symbol names from offsets found in {}'.format(
691 '\n'.join(files)))
692 with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile:
693 orderfile.write('\n'.join(ordered_symbols))
Matthew Cary0f1f681a2018-01-22 10:40:51[diff] [blame]694 except Exception:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]695 for f in files:
696 self._SaveForDebugging(f)
697 raise
698 finally:
699 self._profiler.Cleanup()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]700
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]701 def _MaybeSaveProfile(self, files):
702 if self._options.profile_save_dir:
703 logging.info('Saving profiles to %s', self._options.profile_save_dir)
704 for f in files:
705 shutil.copy(f, self._options.profile_save_dir)
706 logging.info('Saved profile %s', f)
707
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]708 def _PatchOrderfile(self):
709 """Patches the orderfile using clean version of libchrome.so."""
710 self._step_recorder.BeginStep('Patch Orderfile')
Benoit Lizefefbb27c2018-01-17 13:54:18[diff] [blame]711 patch_orderfile.GeneratePatchedOrderfile(
712 self._GetUnpatchedOrderfileFilename(), self._compiler.lib_chrome_so,
Matthew Caryf949bba2019-02-04 13:39:23[diff] [blame]713 self._GetPathToOrderfile(), self._order_outlined_functions)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]714
715 def _VerifySymbolOrder(self):
716 self._step_recorder.BeginStep('Verify Symbol Order')
Andrew Grievec45bb952021-11-02 18:33:27[diff] [blame]717 return_code = self._step_recorder.RunCommand([
718 self._CHECK_ORDERFILE_SCRIPT, self._compiler.lib_chrome_so,
719 self._GetPathToOrderfile()
720 ],
721 constants.DIR_SOURCE_ROOT,
722 raise_on_error=False)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]723 if return_code:
724 self._step_recorder.FailStep('Orderfile check returned %d.' % return_code)
725
726 def _RecordHash(self, file_name):
727 """Records the hash of the file into the output_data dictionary."""
728 self._output_data[os.path.basename(file_name) + '.sha1'] = _GenerateHash(
729 file_name)
730
731 def _SaveFileLocally(self, file_name, file_sha1):
732 """Saves the file to a temporary location and prints the sha1sum."""
733 if not os.path.exists(self._DIRECTORY_FOR_DEBUG_FILES):
734 os.makedirs(self._DIRECTORY_FOR_DEBUG_FILES)
735 shutil.copy(file_name, self._DIRECTORY_FOR_DEBUG_FILES)
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]736 print('File: %s, saved in: %s, sha1sum: %s' %
737 (file_name, self._DIRECTORY_FOR_DEBUG_FILES, file_sha1))
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]738
739 def _SaveForDebugging(self, filename):
740 """Uploads the file to cloud storage or saves to a temporary location."""
741 file_sha1 = _GenerateHash(filename)
742 if not self._options.buildbot:
743 self._SaveFileLocally(filename, file_sha1)
744 else:
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]745 print('Uploading file for debugging: ' + filename)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]746 self._orderfile_updater.UploadToCloudStorage(
747 filename, use_debug_location=True)
748
749 def _SaveForDebuggingWithOverwrite(self, file_name):
750 """Uploads and overwrites the file in cloud storage or copies locally.
751
752 Should be used for large binaries like lib_chrome_so.
753
754 Args:
755 file_name: (str) File to upload.
756 """
757 file_sha1 = _GenerateHash(file_name)
758 if not self._options.buildbot:
759 self._SaveFileLocally(file_name, file_sha1)
760 else:
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]761 print('Uploading file for debugging: %s, sha1sum: %s' % (file_name,
762 file_sha1))
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]763 upload_location = '%s/%s' % (
764 self._CLOUD_STORAGE_BUCKET_FOR_DEBUG, os.path.basename(file_name))
765 self._step_recorder.RunCommand([
766 'gsutil.py', 'cp', file_name, 'gs://' + upload_location])
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]767 print('Uploaded to: https://sandbox.google.com/storage/' +
768 upload_location)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]769
770 def _MaybeArchiveOrderfile(self, filename):
771 """In buildbot configuration, uploads the generated orderfile to
772 Google Cloud Storage.
773
774 Args:
775 filename: (str) Orderfile to upload.
776 """
Matthew Cary91df9792018-11-30 14:35:15[diff] [blame]777 # First compute hashes so that we can download them later if we need to.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]778 self._step_recorder.BeginStep('Compute hash for ' + filename)
779 self._RecordHash(filename)
780 if self._options.buildbot:
781 self._step_recorder.BeginStep('Archive ' + filename)
782 self._orderfile_updater.UploadToCloudStorage(
783 filename, use_debug_location=False)
784
Egor Paskobce64d012018-11-20 12:02:37[diff] [blame]785 def UploadReadyOrderfiles(self):
786 self._step_recorder.BeginStep('Upload Ready Orderfiles')
787 for file_name in [self._GetUnpatchedOrderfileFilename(),
788 self._GetPathToOrderfile()]:
789 self._orderfile_updater.UploadToCloudStorage(
790 file_name, use_debug_location=False)
791
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]792 def _NativeCodeMemoryBenchmark(self, apk):
793 """Runs system_health.memory_mobile to assess native code memory footprint.
794
795 Args:
796 apk: (str) Path to the apk.
797
798 Returns:
799 results: ([int]) Values of native code memory footprint in bytes from the
800 benchmark results.
801 """
802 self._step_recorder.BeginStep("Running orderfile.memory_mobile")
803 try:
804 out_dir = tempfile.mkdtemp()
805 self._profiler._RunCommand(['tools/perf/run_benchmark',
806 '--device={}'.format(
807 self._profiler._device.serial),
808 '--browser=exact',
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]809 '--output-format=csv',
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]810 '--output-dir={}'.format(out_dir),
811 '--reset-results',
812 '--browser-executable={}'.format(apk),
813 'orderfile.memory_mobile'])
814
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]815 out_file_path = os.path.join(out_dir, 'results.csv')
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]816 if not os.path.exists(out_file_path):
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]817 raise Exception('Results file not found!')
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]818
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]819 results = {}
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]820 with open(out_file_path, 'r') as f:
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]821 reader = csv.DictReader(f)
822 for row in reader:
823 if not row['name'].endswith('NativeCodeResidentMemory'):
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]824 continue
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]825 # Note: NativeCodeResidentMemory records a single sample from each
826 # story run, so this average (reported as 'avg') is exactly the value
827 # of that one sample. Each story is run multiple times, so this loop
828 # will accumulate into a list all values for all runs of each story.
829 results.setdefault(row['name'], {}).setdefault(
830 row['stories'], []).append(row['avg'])
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]831
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]832 if not results:
833 raise Exception('Could not find relevant results')
834
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]835 return results
836
837 except Exception as e:
838 return 'Error: ' + str(e)
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]839
840 finally:
841 shutil.rmtree(out_dir)
842
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]843
844 def _PerformanceBenchmark(self, apk):
845 """Runs Speedometer2.0 to assess performance.
846
847 Args:
848 apk: (str) Path to the apk.
849
850 Returns:
851 results: ([float]) Speedometer2.0 results samples in milliseconds.
852 """
853 self._step_recorder.BeginStep("Running Speedometer2.0.")
854 try:
855 out_dir = tempfile.mkdtemp()
856 self._profiler._RunCommand(['tools/perf/run_benchmark',
857 '--device={}'.format(
858 self._profiler._device.serial),
859 '--browser=exact',
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]860 '--output-format=histograms',
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]861 '--output-dir={}'.format(out_dir),
862 '--reset-results',
863 '--browser-executable={}'.format(apk),
864 'speedometer2'])
865
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]866 out_file_path = os.path.join(out_dir, 'histograms.json')
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]867 if not os.path.exists(out_file_path):
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]868 raise Exception('Results file not found!')
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]869
870 with open(out_file_path, 'r') as f:
871 results = json.load(f)
872
873 if not results:
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]874 raise Exception('Results file is empty.')
875
876 for el in results:
877 if 'name' in el and el['name'] == 'Total' and 'sampleValues' in el:
878 return el['sampleValues']
879
880 raise Exception('Unexpected results format.')
881
882 except Exception as e:
883 return 'Error: ' + str(e)
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]884
885 finally:
886 shutil.rmtree(out_dir)
887
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]888
889 def RunBenchmark(self, out_directory, no_orderfile=False):
890 """Builds chrome apk and runs performance and memory benchmarks.
891
892 Builds a non-instrumented version of chrome.
893 Installs chrome apk on the device.
894 Runs Speedometer2.0 benchmark to assess performance.
895 Runs system_health.memory_mobile to evaluate memory footprint.
896
897 Args:
898 out_directory: (str) Path to out directory for this build.
899 no_orderfile: (bool) True if chrome to be built without orderfile.
900
901 Returns:
902 benchmark_results: (dict) Results extracted from benchmarks.
903 """
Benoit Lize74e82b32021-08-26 14:22:01[diff] [blame]904 benchmark_results = {}
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]905 try:
906 _UnstashOutputDirectory(out_directory)
Christopher Grant073637472019-07-05 13:34:57[diff] [blame]907 self._compiler = ClankCompiler(out_directory, self._step_recorder,
908 self._options.arch, self._options.use_goma,
909 self._options.goma_dir,
Fumitoshi Ukaibd2755b2022-10-28 13:15:27[diff] [blame^]910 self._options.use_remoteexec,
Christopher Grant073637472019-07-05 13:34:57[diff] [blame]911 self._options.system_health_orderfile,
912 self._monochrome, self._options.public,
913 self._GetPathToOrderfile())
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]914
915 if no_orderfile:
916 orderfile_path = self._GetPathToOrderfile()
917 backup_orderfile = orderfile_path + '.backup'
918 shutil.move(orderfile_path, backup_orderfile)
919 open(orderfile_path, 'w').close()
920
921 # Build APK to be installed on the device.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]922 self._compiler.CompileChromeApk(instrumented=False,
923 use_call_graph=False,
924 force_relink=True)
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]925 benchmark_results['Speedometer2.0'] = self._PerformanceBenchmark(
926 self._compiler.chrome_apk)
927 benchmark_results['orderfile.memory_mobile'] = (
928 self._NativeCodeMemoryBenchmark(self._compiler.chrome_apk))
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]929
930 except Exception as e:
931 benchmark_results['Error'] = str(e)
932
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]933 finally:
934 if no_orderfile and os.path.exists(backup_orderfile):
935 shutil.move(backup_orderfile, orderfile_path)
936 _StashOutputDirectory(out_directory)
937
938 return benchmark_results
939
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]940 def Generate(self):
941 """Generates and maybe upload an order."""
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]942 assert (bool(self._options.profile) ^
943 bool(self._options.manual_symbol_offsets))
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]944 if self._options.system_health_orderfile and not self._options.profile:
945 raise AssertionError('--system_health_orderfile must be not be used '
946 'with --skip-profile')
947 if (self._options.manual_symbol_offsets and
948 not self._options.system_health_orderfile):
949 raise AssertionError('--manual-symbol-offsets must be used with '
950 '--system_health_orderfile.')
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]951
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]952 if self._options.profile:
953 try:
954 _UnstashOutputDirectory(self._instrumented_out_dir)
955 self._compiler = ClankCompiler(
Christopher Grant073637472019-07-05 13:34:57[diff] [blame]956 self._instrumented_out_dir, self._step_recorder, self._options.arch,
957 self._options.use_goma, self._options.goma_dir,
Fumitoshi Ukaibd2755b2022-10-28 13:15:27[diff] [blame^]958 self._options.use_remoteexec, self._options.system_health_orderfile,
959 self._monochrome, self._options.public, self._GetPathToOrderfile())
Matthew Carya2bea452018-11-13 10:21:07[diff] [blame]960 if not self._options.pregenerated_profiles:
961 # If there are pregenerated profiles, the instrumented build should
962 # not be changed to avoid invalidating the pregenerated profile
963 # offsets.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]964 self._compiler.CompileChromeApk(instrumented=True,
965 use_call_graph=
966 self._options.use_call_graph)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]967 self._GenerateAndProcessProfile()
968 self._MaybeArchiveOrderfile(self._GetUnpatchedOrderfileFilename())
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]969 finally:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]970 _StashOutputDirectory(self._instrumented_out_dir)
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]971 elif self._options.manual_symbol_offsets:
972 assert self._options.manual_libname
973 assert self._options.manual_objdir
Ari Chivukulaa52f8ba2021-08-10 22:30:39[diff] [blame]974 with open(self._options.manual_symbol_offsets) as f:
975 symbol_offsets = [int(x) for x in f]
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]976 processor = process_profiles.SymbolOffsetProcessor(
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]977 self._options.manual_libname)
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]978 generator = cyglog_to_orderfile.OffsetOrderfileGenerator(
979 processor, cyglog_to_orderfile.ObjectFileProcessor(
980 self._options.manual_objdir))
981 ordered_sections = generator.GetOrderedSections(symbol_offsets)
982 if not ordered_sections: # Either None or empty is a problem.
983 raise Exception('Failed to get ordered sections')
984 with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile:
985 orderfile.write('\n'.join(ordered_sections))
986
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]987 if self._options.patch:
988 if self._options.profile:
989 self._RemoveBlanks(self._GetUnpatchedOrderfileFilename(),
990 self._GetPathToOrderfile())
991 try:
992 _UnstashOutputDirectory(self._uninstrumented_out_dir)
993 self._compiler = ClankCompiler(
994 self._uninstrumented_out_dir, self._step_recorder,
Christopher Grant073637472019-07-05 13:34:57[diff] [blame]995 self._options.arch, self._options.use_goma, self._options.goma_dir,
Fumitoshi Ukaibd2755b2022-10-28 13:15:27[diff] [blame^]996 self._options.use_remoteexec, self._options.system_health_orderfile,
997 self._monochrome, self._options.public, self._GetPathToOrderfile())
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]998
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]999 self._compiler.CompileLibchrome(instrumented=False,
1000 use_call_graph=False)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1001 self._PatchOrderfile()
1002 # Because identical code folding is a bit different with and without
1003 # the orderfile build, we need to re-patch the orderfile with code
1004 # folding as close to the final version as possible.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]1005 self._compiler.CompileLibchrome(instrumented=False,
1006 use_call_graph=False, force_relink=True)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1007 self._PatchOrderfile()
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]1008 self._compiler.CompileLibchrome(instrumented=False,
1009 use_call_graph=False, force_relink=True)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1010 self._VerifySymbolOrder()
1011 self._MaybeArchiveOrderfile(self._GetPathToOrderfile())
1012 finally:
1013 _StashOutputDirectory(self._uninstrumented_out_dir)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1014
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]1015 if self._options.benchmark:
1016 self._output_data['orderfile_benchmark_results'] = self.RunBenchmark(
1017 self._uninstrumented_out_dir)
1018 self._output_data['no_orderfile_benchmark_results'] = self.RunBenchmark(
1019 self._no_orderfile_out_dir, no_orderfile=True)
1020
Egor Paskod80bfad2020-09-10 21:53:06[diff] [blame]1021 if self._options.buildbot:
1022 self._orderfile_updater._GitStash()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1023 self._step_recorder.EndStep()
1024 return not self._step_recorder.ErrorRecorded()
1025
1026 def GetReportingData(self):
1027 """Get a dictionary of reporting data (timings, output hashes)"""
1028 self._output_data['timings'] = self._step_recorder.timings
1029 return self._output_data
1030
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]1031 def CommitStashedOrderfileHashes(self):
1032 """Commit any orderfile hash files in the current checkout.
1033
1034 Only possible if running on the buildbot.
1035
1036 Returns: true on success.
1037 """
Egor Pasko7ff04122019-11-25 15:47:18[diff] [blame]1038 if not self._options.buildbot:
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]1039 logging.error('Trying to commit when not running on the buildbot')
1040 return False
1041 self._orderfile_updater._CommitStashedFiles([
1042 filename + '.sha1'
1043 for filename in (self._GetUnpatchedOrderfileFilename(),
1044 self._GetPathToOrderfile())])
1045 return True
1046
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1047
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1048def CreateArgumentParser():
1049 """Creates and returns the argument parser."""
1050 parser = argparse.ArgumentParser()
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]1051 parser.add_argument('--no-benchmark', action='store_false', dest='benchmark',
1052 default=True, help='Disables running benchmarks.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1053 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1054 '--buildbot', action='store_true',
1055 help='If true, the script expects to be run on a buildbot')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1056 parser.add_argument(
Matthew Cary453ff1452018-07-18 12:19:35[diff] [blame]1057 '--device', default=None, type=str,
1058 help='Device serial number on which to run profiling.')
1059 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1060 '--verify', action='store_true',
1061 help='If true, the script only verifies the current orderfile')
Benoit Lizef5581722021-08-26 09:48:20[diff] [blame]1062 parser.add_argument('--target-arch',
1063 action='store',
1064 dest='arch',
Stephen Martinis71b391b52018-12-04 15:08:04[diff] [blame]1065 default='arm',
Benoit Lizef5581722021-08-26 09:48:20[diff] [blame]1066 choices=list(_ARCH_GN_ARGS.keys()),
Matthew Cary53f74ba2019-01-24 10:07:18[diff] [blame]1067 help='The target architecture for which to build.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1068 parser.add_argument('--output-json', action='store', dest='json_file',
1069 help='Location to save stats in json format')
1070 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1071 '--skip-profile', action='store_false', dest='profile', default=True,
1072 help='Don\'t generate a profile on the device. Only patch from the '
1073 'existing profile.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1074 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1075 '--skip-patch', action='store_false', dest='patch', default=True,
1076 help='Only generate the raw (unpatched) orderfile, don\'t patch it.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1077 parser.add_argument('--goma-dir', help='GOMA directory.')
1078 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1079 '--use-goma', action='store_true', help='Enable GOMA.', default=False)
Fumitoshi Ukaibd2755b2022-10-28 13:15:27[diff] [blame^]1080 parser.add_argument('--use-remoteexec',
1081 action='store_true',
1082 help='Enable remoteexec.',
1083 default=False)
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1084 parser.add_argument('--adb-path', help='Path to the adb binary.')
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]1085
Egor Paskod80bfad2020-09-10 21:53:06[diff] [blame]1086 parser.add_argument('--public',
1087 action='store_true',
1088 help='Build non-internal APK and change the orderfile '
1089 'location. Required if your checkout is non-internal.',
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]1090 default=False)
Matthew Cary04f41032018-12-10 15:55:27[diff] [blame]1091 parser.add_argument('--nosystem-health-orderfile', action='store_false',
Benoit Lcba421e2019-04-26 15:28:26[diff] [blame]1092 dest='system_health_orderfile', default=True,
Matthew Caryc262f5d2018-09-19 14:36:28[diff] [blame]1093 help=('Create an orderfile based on an about:blank '
1094 'startup benchmark instead of system health '
1095 'benchmarks.'))
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]1096 parser.add_argument(
1097 '--use-legacy-chrome-apk', action='store_true', default=False,
1098 help=('Compile and instrument chrome for [L, K] devices.'))
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]1099 parser.add_argument('--manual-symbol-offsets', default=None, type=str,
1100 help=('File of list of ordered symbol offsets generated '
1101 'by manual profiling. Must set other --manual* '
1102 'flags if this is used, and must --skip-profile.'))
1103 parser.add_argument('--manual-libname', default=None, type=str,
1104 help=('Library filename corresponding to '
1105 '--manual-symbol-offsets.'))
1106 parser.add_argument('--manual-objdir', default=None, type=str,
1107 help=('Root of object file directory corresponding to '
1108 '--manual-symbol-offsets.'))
Matthew Caryf949bba2019-02-04 13:39:23[diff] [blame]1109 parser.add_argument('--noorder-outlined-functions', action='store_true',
1110 help='Disable outlined functions in the orderfile.')
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]1111 parser.add_argument('--pregenerated-profiles', default=None, type=str,
1112 help=('Pregenerated profiles to use instead of running '
1113 'profile step. Cannot be used with '
1114 '--skip-profiles.'))
1115 parser.add_argument('--profile-save-dir', default=None, type=str,
1116 help=('Directory to save any profiles created. These can '
1117 'be used with --pregenerated-profiles. Cannot be '
1118 'used with --skip-profiles.'))
Egor Paskobce64d012018-11-20 12:02:37[diff] [blame]1119 parser.add_argument('--upload-ready-orderfiles', action='store_true',
1120 help=('Skip orderfile generation and manually upload '
1121 'orderfiles (both patched and unpatched) from '
1122 'their normal location in the tree to the cloud '
1123 'storage. DANGEROUS! USE WITH CARE!'))
Matthew Cary1ea82212019-03-20 12:10:27[diff] [blame]1124 parser.add_argument('--streamline-for-debugging', action='store_true',
1125 help=('Streamline where possible the run for faster '
1126 'iteration while debugging. The orderfile '
1127 'generated will be valid and nontrivial, but '
1128 'may not be based on a representative profile '
1129 'or other such considerations. Use with caution.'))
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]1130 parser.add_argument('--commit-hashes', action='store_true',
1131 help=('Commit any orderfile hash files in the current '
1132 'checkout; performs no other action'))
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]1133 parser.add_argument('--use-call-graph', action='store_true', default=False,
1134 help='Use call graph instrumentation.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1135 profile_android_startup.AddProfileCollectionArguments(parser)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1136 return parser
1137
1138
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]1139def CreateOrderfile(options, orderfile_updater_class=None):
Christopher Grantdfe1bac2019-07-05 13:34:10[diff] [blame]1140 """Creates an orderfile.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1141
1142 Args:
1143 options: As returned from optparse.OptionParser.parse_args()
1144 orderfile_updater_class: (OrderfileUpdater) subclass of OrderfileUpdater.
1145
1146 Returns:
1147 True iff success.
1148 """
1149 logging.basicConfig(level=logging.INFO)
1150 devil_chromium.Initialize(adb_path=options.adb_path)
1151
Benoit Lize6208ddc2021-08-30 12:26:29[diff] [blame]1152 # Since we generate a ".arm" orderfile irrespective of the architecture (see
1153 # comment in _GetPathToOrderfile()), make sure that we don't commit it.
1154 if options.arch != 'arm':
1155 assert not options.buildbot, (
1156 'ARM is the only supported architecture on bots')
1157 assert not options.upload_ready_orderfiles, (
1158 'ARM is the only supported architecture on bots')
1159
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]1160 generator = OrderfileGenerator(options, orderfile_updater_class)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1161 try:
1162 if options.verify:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]1163 generator._VerifySymbolOrder()
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]1164 elif options.commit_hashes:
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]1165 return generator.CommitStashedOrderfileHashes()
Egor Paskobce64d012018-11-20 12:02:37[diff] [blame]1166 elif options.upload_ready_orderfiles:
1167 return generator.UploadReadyOrderfiles()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1168 else:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]1169 return generator.Generate()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1170 finally:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]1171 json_output = json.dumps(generator.GetReportingData(),
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1172 indent=2) + '\n'
1173 if options.json_file:
1174 with open(options.json_file, 'w') as f:
1175 f.write(json_output)
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]1176 print(json_output)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1177 return False
1178
1179
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1180def main():
1181 parser = CreateArgumentParser()
1182 options = parser.parse_args()
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]1183 return 0 if CreateOrderfile(options) else 1
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1184
1185
1186if __name__ == '__main__':
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1187 sys.exit(main())