[go: nahoru, domu]
Google Git
Sign in
chromium / chromium / src / 55e889bfb66922e1cd33d518488acb66152cc0cf / . / tools / cygprofile / orderfile_generator_backend.py
blob: 235c33fdbc5abc84673c3379bde6bb4f4cc13fda [file] [log] [blame]
Ari Chivukulaa52f8ba2021-08-10 22:30:39[diff] [blame]1#!/usr/bin/env vpython3
Avi Drissmandfd880852022-09-15 20:11:09[diff] [blame]2# Copyright 2013 The Chromium Authors
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6""" A utility to generate an up-to-date orderfile.
7
8The orderfile is used by the linker to order text sections such that the
9sections are placed consecutively in the order specified. This allows us
10to page in less code during start-up.
11
12Example usage:
Egor Paskoa7ac67a42020-09-09 12:06:58[diff] [blame]13 tools/cygprofile/orderfile_generator_backend.py --use-goma --target-arch=arm
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]14"""
15
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]16
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]17import argparse
Egor Paskoa7ac67a42020-09-09 12:06:58[diff] [blame]18import csv
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]19import hashlib
20import json
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]21import glob
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]22import logging
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]23import os
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]24import shutil
25import subprocess
26import sys
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]27import tempfile
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]28import time
29
Matthew Cary91df9792018-11-30 14:35:15[diff] [blame]30import cluster
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]31import cyglog_to_orderfile
Benoit Lizefefbb27c2018-01-17 13:54:18[diff] [blame]32import patch_orderfile
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]33import process_profiles
Egor Pasko3bc0b932018-04-03 10:08:27[diff] [blame]34import profile_android_startup
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]35
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]36_SRC_PATH = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
37sys.path.append(os.path.join(_SRC_PATH, 'third_party', 'catapult', 'devil'))
38from devil.android import device_utils
39from devil.android.sdk import version_codes
40
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]41
42_SRC_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)),
43 os.pardir, os.pardir)
44sys.path.append(os.path.join(_SRC_PATH, 'build', 'android'))
45import devil_chromium
46from pylib import constants
47
48
49# Needs to happen early for GetBuildType()/GetOutDirectory() to work correctly
50constants.SetBuildType('Release')
51
52
Matthew Cary53f74ba2019-01-24 10:07:18[diff] [blame]53# Architecture specific GN args. Trying to build an orderfile for an
54# architecture not listed here will eventually throw.
55_ARCH_GN_ARGS = {
Egor Paskoee760922024-01-11 19:36:31[diff] [blame]56 'arm': ['target_cpu="arm"'],
Peter Wen67a2cba2024-02-12 16:20:03[diff] [blame]57 'arm64': ['target_cpu="arm64"'],
Egor Paskoee760922024-01-11 19:36:31[diff] [blame]58 'x86': ['target_cpu="x86"'],
Peter Wen67a2cba2024-02-12 16:20:03[diff] [blame]59 'x64': ['target_cpu="x64"'],
Matthew Cary53f74ba2019-01-24 10:07:18[diff] [blame]60}
61
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]62class CommandError(Exception):
63 """Indicates that a dispatched shell command exited with a non-zero status."""
64
65 def __init__(self, value):
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]66 super().__init__()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]67 self.value = value
68
69 def __str__(self):
70 return repr(self.value)
71
72
73def _GenerateHash(file_path):
74 """Calculates and returns the hash of the file at file_path."""
75 sha1 = hashlib.sha1()
76 with open(file_path, 'rb') as f:
77 while True:
78 # Read in 1mb chunks, so it doesn't all have to be loaded into memory.
79 chunk = f.read(1024 * 1024)
80 if not chunk:
81 break
82 sha1.update(chunk)
83 return sha1.hexdigest()
84
85
86def _GetFileExtension(file_name):
87 """Calculates the file extension from a file name.
88
89 Args:
90 file_name: The source file name.
91 Returns:
92 The part of file_name after the dot (.) or None if the file has no
93 extension.
94 Examples: /home/user/foo.bar -> bar
95 /home/user.name/foo -> None
96 /home/user/.foo -> None
97 /home/user/foo.bar.baz -> baz
98 """
99 file_name_parts = os.path.basename(file_name).split('.')
100 if len(file_name_parts) > 1:
101 return file_name_parts[-1]
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]102 return None
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]103
104
105def _StashOutputDirectory(buildpath):
106 """Takes the output directory and stashes it in the default output directory.
107
108 This allows it to be used for incremental builds next time (after unstashing)
109 by keeping it in a place that isn't deleted normally, while also ensuring
110 that it is properly clobbered when appropriate.
111
112 This is a dirty hack to deal with the needs of clobbering while also handling
113 incremental builds and the hardcoded relative paths used in some of the
114 project files.
115
116 Args:
117 buildpath: The path where the building happens. If this corresponds to the
118 default output directory, no action is taken.
119 """
120 if os.path.abspath(buildpath) == os.path.abspath(os.path.dirname(
121 constants.GetOutDirectory())):
122 return
123 name = os.path.basename(buildpath)
124 stashpath = os.path.join(constants.GetOutDirectory(), name)
125 if not os.path.exists(buildpath):
126 return
127 if os.path.exists(stashpath):
128 shutil.rmtree(stashpath, ignore_errors=True)
129 shutil.move(buildpath, stashpath)
130
131
132def _UnstashOutputDirectory(buildpath):
133 """Inverse of _StashOutputDirectory.
134
135 Moves the output directory stashed within the default output directory
136 (out/Release) to the position where the builds can actually happen.
137
138 This is a dirty hack to deal with the needs of clobbering while also handling
139 incremental builds and the hardcoded relative paths used in some of the
140 project files.
141
142 Args:
143 buildpath: The path where the building happens. If this corresponds to the
144 default output directory, no action is taken.
145 """
146 if os.path.abspath(buildpath) == os.path.abspath(os.path.dirname(
147 constants.GetOutDirectory())):
148 return
149 name = os.path.basename(buildpath)
150 stashpath = os.path.join(constants.GetOutDirectory(), name)
151 if not os.path.exists(stashpath):
152 return
153 if os.path.exists(buildpath):
154 shutil.rmtree(buildpath, ignore_errors=True)
155 shutil.move(stashpath, buildpath)
156
157
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]158class StepRecorder:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]159 """Records steps and timings."""
160
161 def __init__(self, buildbot):
162 self.timings = []
163 self._previous_step = ('', 0.0)
164 self._buildbot = buildbot
165 self._error_recorded = False
166
167 def BeginStep(self, name):
168 """Marks a beginning of the next step in the script.
169
170 On buildbot, this prints a specially formatted name that will show up
171 in the waterfall. Otherwise, just prints the step name.
172
173 Args:
174 name: The name of the step.
175 """
176 self.EndStep()
177 self._previous_step = (name, time.time())
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]178 print('Running step: ', name)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]179
180 def EndStep(self):
181 """Records successful completion of the current step.
182
183 This is optional if the step is immediately followed by another BeginStep.
184 """
185 if self._previous_step[0]:
186 elapsed = time.time() - self._previous_step[1]
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]187 print('Step %s took %f seconds' % (self._previous_step[0], elapsed))
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]188 self.timings.append((self._previous_step[0], elapsed))
189
190 self._previous_step = ('', 0.0)
191
192 def FailStep(self, message=None):
193 """Marks that a particular step has failed.
194
195 On buildbot, this will mark the current step as failed on the waterfall.
196 Otherwise we will just print an optional failure message.
197
198 Args:
199 message: An optional explanation as to why the step failed.
200 """
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]201 print('STEP FAILED!!')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]202 if message:
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]203 print(message)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]204 self._error_recorded = True
205 self.EndStep()
206
207 def ErrorRecorded(self):
208 """True if FailStep has been called."""
209 return self._error_recorded
210
211 def RunCommand(self, cmd, cwd=constants.DIR_SOURCE_ROOT, raise_on_error=True,
212 stdout=None):
213 """Execute a shell command.
214
215 Args:
216 cmd: A list of command strings.
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]217 cwd: Directory in which the command should be executed, defaults to build
218 root of script's location if not specified.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]219 raise_on_error: If true will raise a CommandError if the call doesn't
220 succeed and mark the step as failed.
221 stdout: A file to redirect stdout for the command to.
222
223 Returns:
224 The process's return code.
225
226 Raises:
227 CommandError: An error executing the specified command.
228 """
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]229 print('Executing %s in %s' % (' '.join(cmd), cwd))
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]230 process = subprocess.Popen(cmd, stdout=stdout, cwd=cwd, env=os.environ)
231 process.wait()
232 if raise_on_error and process.returncode != 0:
233 self.FailStep()
234 raise CommandError('Exception executing command %s' % ' '.join(cmd))
235 return process.returncode
236
237
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]238class ClankCompiler:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]239 """Handles compilation of clank."""
240
Christopher Grant073637472019-07-05 13:34:57[diff] [blame]241 def __init__(self, out_dir, step_recorder, arch, use_goma, goma_dir,
Fumitoshi Ukaibc233ba2023-02-13 11:47:31[diff] [blame]242 use_remoteexec, ninja_command, system_health_profiling,
243 monochrome, public, orderfile_location):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]244 self._out_dir = out_dir
245 self._step_recorder = step_recorder
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]246 self._arch = arch
Fumitoshi Ukaibd2755b2022-10-28 13:15:27[diff] [blame]247 # TODO(b/236070141): remove goma config.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]248 self._use_goma = use_goma
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]249 self._goma_dir = goma_dir
Fumitoshi Ukaibd2755b2022-10-28 13:15:27[diff] [blame]250 self._use_remoteexec = use_remoteexec
Fumitoshi Ukaibc233ba2023-02-13 11:47:31[diff] [blame]251 self._ninja_command = ninja_command
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]252 self._system_health_profiling = system_health_profiling
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]253 self._public = public
254 self._orderfile_location = orderfile_location
Matthew Caryd6bfcb72018-09-14 11:27:46[diff] [blame]255 if monochrome:
256 self._apk = 'Monochrome.apk'
Matthew Carye1b00062018-09-19 14:27:44[diff] [blame]257 self._apk_target = 'monochrome_apk'
Peter Wen67a2cba2024-02-12 16:20:03[diff] [blame]258 if '64' in self._arch:
259 # Monochrome has a _64 suffix for arm64 and x64 builds.
260 self._libname = 'libmonochrome_64'
261 self._libchrome_target = 'libmonochrome_64'
262 else:
263 self._libname = 'libmonochrome'
264 self._libchrome_target = 'libmonochrome'
Matthew Caryd6bfcb72018-09-14 11:27:46[diff] [blame]265 else:
266 self._apk = 'Chrome.apk'
Matthew Carye1b00062018-09-19 14:27:44[diff] [blame]267 self._apk_target = 'chrome_apk'
Matthew Caryd6bfcb72018-09-14 11:27:46[diff] [blame]268 self._libname = 'libchrome'
269 self._libchrome_target = 'libchrome'
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]270 if public:
271 self._apk = self._apk.replace('.apk', 'Public.apk')
272 self._apk_target = self._apk_target.replace('_apk', '_public_apk')
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]273
274 self.obj_dir = os.path.join(self._out_dir, 'Release', 'obj')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]275 self.lib_chrome_so = os.path.join(
Matthew Caryd6bfcb72018-09-14 11:27:46[diff] [blame]276 self._out_dir, 'Release', 'lib.unstripped',
277 '{}.so'.format(self._libname))
278 self.chrome_apk = os.path.join(self._out_dir, 'Release', 'apks', self._apk)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]279
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]280 def Build(self, instrumented, use_call_graph, target):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]281 """Builds the provided ninja target with or without order_profiling on.
282
283 Args:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]284 instrumented: (bool) Whether we want to build an instrumented binary.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]285 use_call_graph: (bool) Whether to use the call graph instrumentation.
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]286 target: (str) The name of the ninja target to build.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]287 """
288 self._step_recorder.BeginStep('Compile %s' % target)
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]289 assert not use_call_graph or instrumented, ('You can not enable call graph '
290 'without instrumentation!')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]291
292 # Set the "Release Official" flavor, the parts affecting performance.
293 args = [
Andrew Grieve0e8790c2020-09-03 17:27:32[diff] [blame]294 'enable_resource_allowlist_generation=false',
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]295 'is_chrome_branded=' + str(not self._public).lower(),
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]296 'is_debug=false',
297 'is_official_build=true',
Egor Pasko1d13d532020-01-14 21:25:19[diff] [blame]298 'symbol_level=1', # to fit 30 GiB RAM on the bot when LLD is running
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]299 'target_os="android"',
300 'use_goma=' + str(self._use_goma).lower(),
Fumitoshi Ukaibd2755b2022-10-28 13:15:27[diff] [blame]301 'use_remoteexec=' + str(self._use_remoteexec).lower(),
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]302 'use_order_profiling=' + str(instrumented).lower(),
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]303 'use_call_graph=' + str(use_call_graph).lower(),
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]304 ]
Matthew Cary53f74ba2019-01-24 10:07:18[diff] [blame]305 args += _ARCH_GN_ARGS[self._arch]
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]306 if self._goma_dir:
307 args += ['goma_dir="%s"' % self._goma_dir]
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]308 if self._system_health_profiling:
309 args += ['devtools_instrumentation_dumping = ' +
310 str(instrumented).lower()]
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]311
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]312 if self._public and os.path.exists(self._orderfile_location):
313 # GN needs the orderfile path to be source-absolute.
314 src_abs_orderfile = os.path.relpath(self._orderfile_location,
315 constants.DIR_SOURCE_ROOT)
316 args += ['chrome_orderfile="//{}"'.format(src_abs_orderfile)]
317
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]318 self._step_recorder.RunCommand(
319 ['gn', 'gen', os.path.join(self._out_dir, 'Release'),
320 '--args=' + ' '.join(args)])
321
322 self._step_recorder.RunCommand(
Fumitoshi Ukaibc233ba2023-02-13 11:47:31[diff] [blame]323 self._ninja_command + [os.path.join(self._out_dir, 'Release'), target])
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]324
Christopher Grant10221762019-07-05 12:10:04[diff] [blame]325 def ForceRelink(self):
326 """Forces libchrome.so or libmonochrome.so to be re-linked.
327
328 With partitioned libraries enabled, deleting these library files does not
329 guarantee they'll be recreated by the linker (they may simply be
330 re-extracted from a combined library). To be safe, touch a source file
331 instead. See http://crbug.com/972701 for more explanation.
332 """
333 file_to_touch = os.path.join(constants.DIR_SOURCE_ROOT, 'chrome', 'browser',
334 'chrome_browser_main_android.cc')
335 assert os.path.exists(file_to_touch)
336 self._step_recorder.RunCommand(['touch', file_to_touch])
337
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]338 def CompileChromeApk(self, instrumented, use_call_graph, force_relink=False):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]339 """Builds a Chrome.apk either with or without order_profiling on.
340
341 Args:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]342 instrumented: (bool) Whether to build an instrumented apk.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]343 use_call_graph: (bool) Whether to use the call graph instrumentation.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]344 force_relink: Whether libchromeview.so should be re-created.
345 """
346 if force_relink:
Christopher Grant10221762019-07-05 12:10:04[diff] [blame]347 self.ForceRelink()
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]348 self.Build(instrumented, use_call_graph, self._apk_target)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]349
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]350 def CompileLibchrome(self, instrumented, use_call_graph, force_relink=False):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]351 """Builds a libchrome.so either with or without order_profiling on.
352
353 Args:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]354 instrumented: (bool) Whether to build an instrumented apk.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]355 use_call_graph: (bool) Whether to use the call graph instrumentation.
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]356 force_relink: (bool) Whether libchrome.so should be re-created.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]357 """
358 if force_relink:
Christopher Grant10221762019-07-05 12:10:04[diff] [blame]359 self.ForceRelink()
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]360 self.Build(instrumented, use_call_graph, self._libchrome_target)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]361
362
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]363class OrderfileUpdater:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]364 """Handles uploading and committing a new orderfile in the repository.
365
366 Only used for testing or on a bot.
367 """
368
369 _CLOUD_STORAGE_BUCKET_FOR_DEBUG = None
370 _CLOUD_STORAGE_BUCKET = None
371 _UPLOAD_TO_CLOUD_COMMAND = 'upload_to_google_storage.py'
372
Egor Pasko0c533c6682019-11-26 21:16:32[diff] [blame]373 def __init__(self, repository_root, step_recorder):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]374 """Constructor.
375
376 Args:
377 repository_root: (str) Root of the target repository.
378 step_recorder: (StepRecorder) Step recorder, for logging.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]379 """
380 self._repository_root = repository_root
381 self._step_recorder = step_recorder
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]382
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]383 def CommitStashedFileHashes(self, files):
384 """Commits unpatched and patched orderfiles hashes if changed.
385
386 The files are committed only if their associated sha1 hash files match, and
387 are modified in git. In normal operations the hash files are changed only
388 when a file is uploaded to cloud storage. If the hash file is not modified
389 in git, the file is skipped.
390
391 Args:
392 files: [str or None] specifies file paths. None items are ignored.
393
394 Raises:
395 Exception if the hash file does not match the file.
396 NotImplementedError when the commit logic hasn't been overridden.
397 """
Ari Chivukulaa52f8ba2021-08-10 22:30:39[diff] [blame]398 files_to_commit = [_f for _f in files if _f]
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]399 if files_to_commit:
400 self._CommitStashedFiles(files_to_commit)
401
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]402 def UploadToCloudStorage(self, filename, use_debug_location):
403 """Uploads a file to cloud storage.
404
405 Args:
406 filename: (str) File to upload.
407 use_debug_location: (bool) Whether to use the debug location.
408 """
409 bucket = (self._CLOUD_STORAGE_BUCKET_FOR_DEBUG if use_debug_location
410 else self._CLOUD_STORAGE_BUCKET)
411 extension = _GetFileExtension(filename)
412 cmd = [self._UPLOAD_TO_CLOUD_COMMAND, '--bucket', bucket]
413 if extension:
414 cmd.extend(['-z', extension])
415 cmd.append(filename)
416 self._step_recorder.RunCommand(cmd)
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]417 print('Download: https://sandbox.google.com/storage/%s/%s' %
418 (bucket, _GenerateHash(filename)))
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]419
420 def _GetHashFilePathAndContents(self, filename):
421 """Gets the name and content of the hash file created from uploading the
422 given file.
423
424 Args:
425 filename: (str) The file that was uploaded to cloud storage.
426
427 Returns:
428 A tuple of the hash file name, relative to the reository root, and the
429 content, which should be the sha1 hash of the file
430 ('base_file.sha1', hash)
431 """
432 abs_hash_filename = filename + '.sha1'
433 rel_hash_filename = os.path.relpath(
434 abs_hash_filename, self._repository_root)
435 with open(abs_hash_filename, 'r') as f:
436 return (rel_hash_filename, f.read())
437
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]438 def _GitStash(self):
439 """Git stash the current clank tree.
440
441 Raises:
442 NotImplementedError when the stash logic hasn't been overridden.
443 """
444 raise NotImplementedError
445
446 def _CommitStashedFiles(self, expected_files_in_stash):
447 """Commits stashed files.
448
449 The local repository is updated and then the files to commit are taken from
450 modified files from the git stash. The modified files should be a subset of
451 |expected_files_in_stash|. If there are unexpected modified files, this
452 function may raise. This is meant to be paired with _GitStash().
453
454 Args:
455 expected_files_in_stash: [str] paths to a possible superset of files
456 expected to be stashed & committed.
457
458 Raises:
459 NotImplementedError when the commit logic hasn't been overridden.
460 """
461 raise NotImplementedError
462
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]463
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]464class OrderfileGenerator:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]465 """A utility for generating a new orderfile for Clank.
466
467 Builds an instrumented binary, profiles a run of the application, and
468 generates an updated orderfile.
469 """
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]470 _CHECK_ORDERFILE_SCRIPT = os.path.join(
471 constants.DIR_SOURCE_ROOT, 'tools', 'cygprofile', 'check_orderfile.py')
472 _BUILD_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(
473 constants.GetOutDirectory()))) # Normally /path/to/src
474
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]475 # Previous orderfile_generator debug files would be overwritten.
476 _DIRECTORY_FOR_DEBUG_FILES = '/tmp/orderfile_generator_debug_files'
477
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]478 _CLOUD_STORAGE_BUCKET_FOR_DEBUG = None
479
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]480 def _PrepareOrderfilePaths(self):
481 if self._options.public:
482 self._clank_dir = os.path.join(constants.DIR_SOURCE_ROOT,
483 '')
484 if not os.path.exists(os.path.join(self._clank_dir, 'orderfiles')):
485 os.makedirs(os.path.join(self._clank_dir, 'orderfiles'))
486 else:
487 self._clank_dir = os.path.join(constants.DIR_SOURCE_ROOT,
488 'clank')
489
490 self._unpatched_orderfile_filename = os.path.join(
491 self._clank_dir, 'orderfiles', 'unpatched_orderfile.%s')
492 self._path_to_orderfile = os.path.join(
493 self._clank_dir, 'orderfiles', 'orderfile.%s.out')
494
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]495 def _GetPathToOrderfile(self):
496 """Gets the path to the architecture-specific orderfile."""
Peter Wen61a838d2024-01-29 17:13:42[diff] [blame]497 # TODO(https://crbug.com/1517659): We are testing if arm64 can improve perf
498 # while not regressing arm32 memory or perf by too much. For now we are
499 # keeping the fake arch as 'arm' to avoid needing to change the path. In
500 # the future we should consider either generating multiple orderfiles,
501 # one per architecture, or remove the fake arch as it would no longer be
502 # accurate.
Benoit Lize6208ddc2021-08-30 12:26:29[diff] [blame]503 # Build GN files use the ".arm" orderfile irrespective of the actual
504 # architecture. Fake it, otherwise the orderfile we generate here is not
505 # going to be picked up by builds.
506 orderfile_fake_arch = 'arm'
507 return self._path_to_orderfile % orderfile_fake_arch
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]508
509 def _GetUnpatchedOrderfileFilename(self):
510 """Gets the path to the architecture-specific unpatched orderfile."""
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]511 return self._unpatched_orderfile_filename % self._options.arch
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]512
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]513 def _SetDevice(self):
514 """ Selects the device to be used by the script.
515
516 Returns:
517 (Device with given serial ID) : if the --device flag is set.
518 (Device running Android[K,L]) : if --use-legacy-chrome-apk flag is set or
519 no device running Android N+ was found.
520 (Device running Android N+) : Otherwise.
521
522 Raises Error:
523 If no device meeting the requirements has been found.
524 """
525 devices = None
526 if self._options.device:
527 devices = [device_utils.DeviceUtils(self._options.device)]
528 else:
529 devices = device_utils.DeviceUtils.HealthyDevices()
530
531 assert devices, 'Expected at least one connected device'
532
533 if self._options.use_legacy_chrome_apk:
534 self._monochrome = False
535 for device in devices:
536 device_version = device.build_version_sdk
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]537 if (version_codes.KITKAT <= device_version <=
538 version_codes.LOLLIPOP_MR1):
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]539 return device
540
541 assert not self._options.use_legacy_chrome_apk, \
542 'No device found running suitable android version for Chrome.apk.'
543
544 preferred_device = None
545 for device in devices:
546 if device.build_version_sdk >= version_codes.NOUGAT:
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]547 preferred_device = device
548 break
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]549
550 self._monochrome = preferred_device is not None
551
552 return preferred_device if preferred_device else devices[0]
553
554
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]555 def __init__(self, options, orderfile_updater_class):
556 self._options = options
Fumitoshi Ukaibc233ba2023-02-13 11:47:31[diff] [blame]557 self._ninja_command = ['autoninja']
558 if self._options.ninja_path:
559 self._ninja_command = [self._options.ninja_path]
560 if self._options.ninja_j:
561 self._ninja_command += ['-j', self._options.ninja_j]
562 self._ninja_command += ['-C']
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]563 self._instrumented_out_dir = os.path.join(
564 self._BUILD_ROOT, self._options.arch + '_instrumented_out')
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]565 if self._options.use_call_graph:
Christopher Grantdfe1bac2019-07-05 13:34:10[diff] [blame]566 self._instrumented_out_dir += '_call_graph'
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]567
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]568 self._uninstrumented_out_dir = os.path.join(
569 self._BUILD_ROOT, self._options.arch + '_uninstrumented_out')
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]570 self._no_orderfile_out_dir = os.path.join(
571 self._BUILD_ROOT, self._options.arch + '_no_orderfile_out')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]572
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]573 self._PrepareOrderfilePaths()
574
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]575 if options.profile:
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]576 output_directory = os.path.join(self._instrumented_out_dir, 'Release')
Matthew Caryb8daed942018-06-11 10:58:08[diff] [blame]577 host_profile_dir = os.path.join(output_directory, 'profile_data')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]578 urls = [profile_android_startup.AndroidProfileTool.TEST_URL]
579 use_wpr = True
580 simulate_user = False
Benoit L96466812018-03-06 15:58:37[diff] [blame]581 urls = options.urls
582 use_wpr = not options.no_wpr
583 simulate_user = options.simulate_user
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]584 device = self._SetDevice()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]585 self._profiler = profile_android_startup.AndroidProfileTool(
Matthew Cary453ff1452018-07-18 12:19:35[diff] [blame]586 output_directory, host_profile_dir, use_wpr, urls, simulate_user,
Matthew Cary1ea82212019-03-20 12:10:27[diff] [blame]587 device, debug=self._options.streamline_for_debugging)
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]588 if options.pregenerated_profiles:
589 self._profiler.SetPregeneratedProfiles(
590 glob.glob(options.pregenerated_profiles))
591 else:
592 assert not options.pregenerated_profiles, (
593 '--pregenerated-profiles cannot be used with --skip-profile')
594 assert not options.profile_save_dir, (
595 '--profile-save-dir cannot be used with --skip-profile')
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]596 self._monochrome = not self._options.use_legacy_chrome_apk
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]597
Matthew Caryf949bba2019-02-04 13:39:23[diff] [blame]598 # Outlined function handling enabled by default for all architectures.
599 self._order_outlined_functions = not options.noorder_outlined_functions
600
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]601 self._output_data = {}
602 self._step_recorder = StepRecorder(options.buildbot)
603 self._compiler = None
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]604 if orderfile_updater_class is None:
Egor Paskod80bfad2020-09-10 21:53:06[diff] [blame]605 orderfile_updater_class = OrderfileUpdater
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]606 assert issubclass(orderfile_updater_class, OrderfileUpdater)
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]607 self._orderfile_updater = orderfile_updater_class(self._clank_dir,
Egor Pasko0c533c6682019-11-26 21:16:32[diff] [blame]608 self._step_recorder)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]609 assert os.path.isdir(constants.DIR_SOURCE_ROOT), 'No src directory found'
610
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]611 @staticmethod
612 def _RemoveBlanks(src_file, dest_file):
613 """A utility to remove blank lines from a file.
614
615 Args:
616 src_file: The name of the file to remove the blanks from.
617 dest_file: The name of the file to write the output without blanks.
618 """
619 assert src_file != dest_file, 'Source and destination need to be distinct'
620
621 try:
622 src = open(src_file, 'r')
623 dest = open(dest_file, 'w')
624 for line in src:
625 if line and not line.isspace():
626 dest.write(line)
627 finally:
628 src.close()
629 dest.close()
630
631 def _GenerateAndProcessProfile(self):
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]632 """Invokes a script to merge the per-thread traces into one file.
633
634 The produced list of offsets is saved in
635 self._GetUnpatchedOrderfileFilename().
636 """
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]637 self._step_recorder.BeginStep('Generate Profile Data')
638 files = []
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]639 logging.getLogger().setLevel(logging.DEBUG)
Christopher Grantdfe1bac2019-07-05 13:34:10[diff] [blame]640
641 if self._options.profile_save_dir:
642 # The directory must not preexist, to ensure purity of data. Check
643 # before profiling to save time.
644 if os.path.exists(self._options.profile_save_dir):
645 raise Exception('Profile save directory must not pre-exist')
646 os.makedirs(self._options.profile_save_dir)
647
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]648 if self._options.system_health_orderfile:
649 files = self._profiler.CollectSystemHealthProfile(
650 self._compiler.chrome_apk)
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]651 self._MaybeSaveProfile(files)
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]652 try:
653 self._ProcessPhasedOrderfile(files)
654 except Exception:
655 for f in files:
656 self._SaveForDebugging(f)
Matthew Cary8b1416232018-08-10 19:12:22[diff] [blame]657 self._SaveForDebugging(self._compiler.lib_chrome_so)
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]658 raise
659 finally:
660 self._profiler.Cleanup()
661 else:
662 self._CollectLegacyProfile()
663 logging.getLogger().setLevel(logging.INFO)
664
665 def _ProcessPhasedOrderfile(self, files):
666 """Process the phased orderfiles produced by system health benchmarks.
667
668 The offsets will be placed in _GetUnpatchedOrderfileFilename().
669
670 Args:
671 file: Profile files pulled locally.
672 """
673 self._step_recorder.BeginStep('Process Phased Orderfile')
674 profiles = process_profiles.ProfileManager(files)
675 processor = process_profiles.SymbolOffsetProcessor(
676 self._compiler.lib_chrome_so)
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]677 ordered_symbols = cluster.ClusterOffsets(profiles, processor,
678 call_graph=self._options.use_call_graph)
Matthew Cary8b1416232018-08-10 19:12:22[diff] [blame]679 if not ordered_symbols:
680 raise Exception('Failed to get ordered symbols')
Matthew Caryda9db932019-03-11 15:28:17[diff] [blame]681 for sym in ordered_symbols:
682 assert not sym.startswith('OUTLINED_FUNCTION_'), (
683 'Outlined function found in instrumented function, very likely '
684 'something has gone very wrong!')
Matthew Cary91df9792018-11-30 14:35:15[diff] [blame]685 self._output_data['offsets_kib'] = processor.SymbolsSize(
686 ordered_symbols) / 1024
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]687 with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile:
Matthew Cary8b1416232018-08-10 19:12:22[diff] [blame]688 orderfile.write('\n'.join(ordered_symbols))
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]689
690 def _CollectLegacyProfile(self):
Matthew Cary2df8d452018-09-19 07:50:20[diff] [blame]691 files = []
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]692 try:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]693 files = self._profiler.CollectProfile(
694 self._compiler.chrome_apk,
695 constants.PACKAGE_INFO['chrome'])
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]696 self._MaybeSaveProfile(files)
Matthew Caryb8daed942018-06-11 10:58:08[diff] [blame]697 self._step_recorder.BeginStep('Process profile')
Benoit L96466812018-03-06 15:58:37[diff] [blame]698 assert os.path.exists(self._compiler.lib_chrome_so)
699 offsets = process_profiles.GetReachedOffsetsFromDumpFiles(
700 files, self._compiler.lib_chrome_so)
701 if not offsets:
702 raise Exception('No profiler offsets found in {}'.format(
703 '\n'.join(files)))
Matthew Cary8b1416232018-08-10 19:12:22[diff] [blame]704 processor = process_profiles.SymbolOffsetProcessor(
705 self._compiler.lib_chrome_so)
706 ordered_symbols = processor.GetOrderedSymbols(offsets)
707 if not ordered_symbols:
708 raise Exception('No symbol names from offsets found in {}'.format(
709 '\n'.join(files)))
710 with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile:
711 orderfile.write('\n'.join(ordered_symbols))
Matthew Cary0f1f681a2018-01-22 10:40:51[diff] [blame]712 except Exception:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]713 for f in files:
714 self._SaveForDebugging(f)
715 raise
716 finally:
717 self._profiler.Cleanup()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]718
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]719 def _MaybeSaveProfile(self, files):
720 if self._options.profile_save_dir:
721 logging.info('Saving profiles to %s', self._options.profile_save_dir)
722 for f in files:
723 shutil.copy(f, self._options.profile_save_dir)
724 logging.info('Saved profile %s', f)
725
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]726 def _PatchOrderfile(self):
727 """Patches the orderfile using clean version of libchrome.so."""
728 self._step_recorder.BeginStep('Patch Orderfile')
Benoit Lizefefbb27c2018-01-17 13:54:18[diff] [blame]729 patch_orderfile.GeneratePatchedOrderfile(
730 self._GetUnpatchedOrderfileFilename(), self._compiler.lib_chrome_so,
Matthew Caryf949bba2019-02-04 13:39:23[diff] [blame]731 self._GetPathToOrderfile(), self._order_outlined_functions)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]732
733 def _VerifySymbolOrder(self):
734 self._step_recorder.BeginStep('Verify Symbol Order')
Andrew Grievec45bb952021-11-02 18:33:27[diff] [blame]735 return_code = self._step_recorder.RunCommand([
736 self._CHECK_ORDERFILE_SCRIPT, self._compiler.lib_chrome_so,
737 self._GetPathToOrderfile()
738 ],
739 constants.DIR_SOURCE_ROOT,
740 raise_on_error=False)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]741 if return_code:
742 self._step_recorder.FailStep('Orderfile check returned %d.' % return_code)
743
744 def _RecordHash(self, file_name):
745 """Records the hash of the file into the output_data dictionary."""
746 self._output_data[os.path.basename(file_name) + '.sha1'] = _GenerateHash(
747 file_name)
748
749 def _SaveFileLocally(self, file_name, file_sha1):
750 """Saves the file to a temporary location and prints the sha1sum."""
751 if not os.path.exists(self._DIRECTORY_FOR_DEBUG_FILES):
752 os.makedirs(self._DIRECTORY_FOR_DEBUG_FILES)
753 shutil.copy(file_name, self._DIRECTORY_FOR_DEBUG_FILES)
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]754 print('File: %s, saved in: %s, sha1sum: %s' %
755 (file_name, self._DIRECTORY_FOR_DEBUG_FILES, file_sha1))
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]756
757 def _SaveForDebugging(self, filename):
758 """Uploads the file to cloud storage or saves to a temporary location."""
759 file_sha1 = _GenerateHash(filename)
760 if not self._options.buildbot:
761 self._SaveFileLocally(filename, file_sha1)
762 else:
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]763 print('Uploading file for debugging: ' + filename)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]764 self._orderfile_updater.UploadToCloudStorage(
765 filename, use_debug_location=True)
766
767 def _SaveForDebuggingWithOverwrite(self, file_name):
768 """Uploads and overwrites the file in cloud storage or copies locally.
769
770 Should be used for large binaries like lib_chrome_so.
771
772 Args:
773 file_name: (str) File to upload.
774 """
775 file_sha1 = _GenerateHash(file_name)
776 if not self._options.buildbot:
777 self._SaveFileLocally(file_name, file_sha1)
778 else:
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]779 print('Uploading file for debugging: %s, sha1sum: %s' % (file_name,
780 file_sha1))
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]781 upload_location = '%s/%s' % (
782 self._CLOUD_STORAGE_BUCKET_FOR_DEBUG, os.path.basename(file_name))
783 self._step_recorder.RunCommand([
784 'gsutil.py', 'cp', file_name, 'gs://' + upload_location])
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]785 print('Uploaded to: https://sandbox.google.com/storage/' +
786 upload_location)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]787
788 def _MaybeArchiveOrderfile(self, filename):
789 """In buildbot configuration, uploads the generated orderfile to
790 Google Cloud Storage.
791
792 Args:
793 filename: (str) Orderfile to upload.
794 """
Matthew Cary91df9792018-11-30 14:35:15[diff] [blame]795 # First compute hashes so that we can download them later if we need to.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]796 self._step_recorder.BeginStep('Compute hash for ' + filename)
797 self._RecordHash(filename)
798 if self._options.buildbot:
799 self._step_recorder.BeginStep('Archive ' + filename)
800 self._orderfile_updater.UploadToCloudStorage(
801 filename, use_debug_location=False)
802
Egor Paskobce64d012018-11-20 12:02:37[diff] [blame]803 def UploadReadyOrderfiles(self):
804 self._step_recorder.BeginStep('Upload Ready Orderfiles')
805 for file_name in [self._GetUnpatchedOrderfileFilename(),
806 self._GetPathToOrderfile()]:
807 self._orderfile_updater.UploadToCloudStorage(
808 file_name, use_debug_location=False)
809
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]810 def _NativeCodeMemoryBenchmark(self, apk):
811 """Runs system_health.memory_mobile to assess native code memory footprint.
812
813 Args:
814 apk: (str) Path to the apk.
815
816 Returns:
817 results: ([int]) Values of native code memory footprint in bytes from the
818 benchmark results.
819 """
820 self._step_recorder.BeginStep("Running orderfile.memory_mobile")
821 try:
822 out_dir = tempfile.mkdtemp()
823 self._profiler._RunCommand(['tools/perf/run_benchmark',
824 '--device={}'.format(
825 self._profiler._device.serial),
826 '--browser=exact',
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]827 '--output-format=csv',
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]828 '--output-dir={}'.format(out_dir),
829 '--reset-results',
830 '--browser-executable={}'.format(apk),
831 'orderfile.memory_mobile'])
832
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]833 out_file_path = os.path.join(out_dir, 'results.csv')
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]834 if not os.path.exists(out_file_path):
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]835 raise Exception('Results file not found!')
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]836
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]837 results = {}
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]838 with open(out_file_path, 'r') as f:
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]839 reader = csv.DictReader(f)
840 for row in reader:
841 if not row['name'].endswith('NativeCodeResidentMemory'):
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]842 continue
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]843 # Note: NativeCodeResidentMemory records a single sample from each
844 # story run, so this average (reported as 'avg') is exactly the value
845 # of that one sample. Each story is run multiple times, so this loop
846 # will accumulate into a list all values for all runs of each story.
847 results.setdefault(row['name'], {}).setdefault(
848 row['stories'], []).append(row['avg'])
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]849
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]850 if not results:
851 raise Exception('Could not find relevant results')
852
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]853 return results
854
855 except Exception as e:
856 return 'Error: ' + str(e)
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]857
858 finally:
859 shutil.rmtree(out_dir)
860
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]861
862 def _PerformanceBenchmark(self, apk):
863 """Runs Speedometer2.0 to assess performance.
864
865 Args:
866 apk: (str) Path to the apk.
867
868 Returns:
869 results: ([float]) Speedometer2.0 results samples in milliseconds.
870 """
871 self._step_recorder.BeginStep("Running Speedometer2.0.")
872 try:
873 out_dir = tempfile.mkdtemp()
874 self._profiler._RunCommand(['tools/perf/run_benchmark',
875 '--device={}'.format(
876 self._profiler._device.serial),
877 '--browser=exact',
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]878 '--output-format=histograms',
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]879 '--output-dir={}'.format(out_dir),
880 '--reset-results',
881 '--browser-executable={}'.format(apk),
882 'speedometer2'])
883
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]884 out_file_path = os.path.join(out_dir, 'histograms.json')
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]885 if not os.path.exists(out_file_path):
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]886 raise Exception('Results file not found!')
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]887
888 with open(out_file_path, 'r') as f:
889 results = json.load(f)
890
891 if not results:
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]892 raise Exception('Results file is empty.')
893
894 for el in results:
895 if 'name' in el and el['name'] == 'Total' and 'sampleValues' in el:
896 return el['sampleValues']
897
898 raise Exception('Unexpected results format.')
899
900 except Exception as e:
901 return 'Error: ' + str(e)
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]902
903 finally:
904 shutil.rmtree(out_dir)
905
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]906
907 def RunBenchmark(self, out_directory, no_orderfile=False):
908 """Builds chrome apk and runs performance and memory benchmarks.
909
910 Builds a non-instrumented version of chrome.
911 Installs chrome apk on the device.
912 Runs Speedometer2.0 benchmark to assess performance.
913 Runs system_health.memory_mobile to evaluate memory footprint.
914
915 Args:
916 out_directory: (str) Path to out directory for this build.
917 no_orderfile: (bool) True if chrome to be built without orderfile.
918
919 Returns:
920 benchmark_results: (dict) Results extracted from benchmarks.
921 """
Benoit Lize74e82b32021-08-26 14:22:01[diff] [blame]922 benchmark_results = {}
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]923 try:
924 _UnstashOutputDirectory(out_directory)
Christopher Grant073637472019-07-05 13:34:57[diff] [blame]925 self._compiler = ClankCompiler(out_directory, self._step_recorder,
926 self._options.arch, self._options.use_goma,
927 self._options.goma_dir,
Fumitoshi Ukaibd2755b2022-10-28 13:15:27[diff] [blame]928 self._options.use_remoteexec,
Fumitoshi Ukaibc233ba2023-02-13 11:47:31[diff] [blame]929 self._ninja_command,
Christopher Grant073637472019-07-05 13:34:57[diff] [blame]930 self._options.system_health_orderfile,
931 self._monochrome, self._options.public,
932 self._GetPathToOrderfile())
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]933
934 if no_orderfile:
935 orderfile_path = self._GetPathToOrderfile()
936 backup_orderfile = orderfile_path + '.backup'
937 shutil.move(orderfile_path, backup_orderfile)
938 open(orderfile_path, 'w').close()
939
940 # Build APK to be installed on the device.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]941 self._compiler.CompileChromeApk(instrumented=False,
942 use_call_graph=False,
943 force_relink=True)
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]944 benchmark_results['Speedometer2.0'] = self._PerformanceBenchmark(
945 self._compiler.chrome_apk)
946 benchmark_results['orderfile.memory_mobile'] = (
947 self._NativeCodeMemoryBenchmark(self._compiler.chrome_apk))
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]948
949 except Exception as e:
950 benchmark_results['Error'] = str(e)
951
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]952 finally:
953 if no_orderfile and os.path.exists(backup_orderfile):
954 shutil.move(backup_orderfile, orderfile_path)
955 _StashOutputDirectory(out_directory)
956
957 return benchmark_results
958
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]959 def Generate(self):
960 """Generates and maybe upload an order."""
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]961 assert (bool(self._options.profile) ^
962 bool(self._options.manual_symbol_offsets))
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]963 if self._options.system_health_orderfile and not self._options.profile:
964 raise AssertionError('--system_health_orderfile must be not be used '
965 'with --skip-profile')
966 if (self._options.manual_symbol_offsets and
967 not self._options.system_health_orderfile):
968 raise AssertionError('--manual-symbol-offsets must be used with '
969 '--system_health_orderfile.')
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]970
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]971 if self._options.profile:
972 try:
973 _UnstashOutputDirectory(self._instrumented_out_dir)
974 self._compiler = ClankCompiler(
Christopher Grant073637472019-07-05 13:34:57[diff] [blame]975 self._instrumented_out_dir, self._step_recorder, self._options.arch,
976 self._options.use_goma, self._options.goma_dir,
Fumitoshi Ukaibc233ba2023-02-13 11:47:31[diff] [blame]977 self._options.use_remoteexec, self._ninja_command,
978 self._options.system_health_orderfile, self._monochrome,
979 self._options.public, self._GetPathToOrderfile())
Matthew Carya2bea452018-11-13 10:21:07[diff] [blame]980 if not self._options.pregenerated_profiles:
981 # If there are pregenerated profiles, the instrumented build should
982 # not be changed to avoid invalidating the pregenerated profile
983 # offsets.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]984 self._compiler.CompileChromeApk(instrumented=True,
985 use_call_graph=
986 self._options.use_call_graph)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]987 self._GenerateAndProcessProfile()
988 self._MaybeArchiveOrderfile(self._GetUnpatchedOrderfileFilename())
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]989 finally:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]990 _StashOutputDirectory(self._instrumented_out_dir)
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]991 elif self._options.manual_symbol_offsets:
992 assert self._options.manual_libname
993 assert self._options.manual_objdir
Ari Chivukulaa52f8ba2021-08-10 22:30:39[diff] [blame]994 with open(self._options.manual_symbol_offsets) as f:
995 symbol_offsets = [int(x) for x in f]
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]996 processor = process_profiles.SymbolOffsetProcessor(
Jesse McKennac0b694b72022-06-17 17:46:14[diff] [blame]997 self._options.manual_libname)
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]998 generator = cyglog_to_orderfile.OffsetOrderfileGenerator(
999 processor, cyglog_to_orderfile.ObjectFileProcessor(
1000 self._options.manual_objdir))
1001 ordered_sections = generator.GetOrderedSections(symbol_offsets)
1002 if not ordered_sections: # Either None or empty is a problem.
1003 raise Exception('Failed to get ordered sections')
1004 with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile:
1005 orderfile.write('\n'.join(ordered_sections))
1006
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1007 if self._options.patch:
1008 if self._options.profile:
1009 self._RemoveBlanks(self._GetUnpatchedOrderfileFilename(),
1010 self._GetPathToOrderfile())
1011 try:
1012 _UnstashOutputDirectory(self._uninstrumented_out_dir)
1013 self._compiler = ClankCompiler(
1014 self._uninstrumented_out_dir, self._step_recorder,
Christopher Grant073637472019-07-05 13:34:57[diff] [blame]1015 self._options.arch, self._options.use_goma, self._options.goma_dir,
Fumitoshi Ukaibc233ba2023-02-13 11:47:31[diff] [blame]1016 self._options.use_remoteexec, self._ninja_command,
1017 self._options.system_health_orderfile, self._monochrome,
1018 self._options.public, self._GetPathToOrderfile())
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]1019
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]1020 self._compiler.CompileLibchrome(instrumented=False,
1021 use_call_graph=False)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1022 self._PatchOrderfile()
1023 # Because identical code folding is a bit different with and without
1024 # the orderfile build, we need to re-patch the orderfile with code
1025 # folding as close to the final version as possible.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]1026 self._compiler.CompileLibchrome(instrumented=False,
1027 use_call_graph=False, force_relink=True)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1028 self._PatchOrderfile()
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]1029 self._compiler.CompileLibchrome(instrumented=False,
1030 use_call_graph=False, force_relink=True)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1031 self._VerifySymbolOrder()
1032 self._MaybeArchiveOrderfile(self._GetPathToOrderfile())
1033 finally:
1034 _StashOutputDirectory(self._uninstrumented_out_dir)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1035
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]1036 if self._options.benchmark:
1037 self._output_data['orderfile_benchmark_results'] = self.RunBenchmark(
1038 self._uninstrumented_out_dir)
1039 self._output_data['no_orderfile_benchmark_results'] = self.RunBenchmark(
1040 self._no_orderfile_out_dir, no_orderfile=True)
1041
Egor Paskod80bfad2020-09-10 21:53:06[diff] [blame]1042 if self._options.buildbot:
1043 self._orderfile_updater._GitStash()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1044 self._step_recorder.EndStep()
1045 return not self._step_recorder.ErrorRecorded()
1046
1047 def GetReportingData(self):
1048 """Get a dictionary of reporting data (timings, output hashes)"""
1049 self._output_data['timings'] = self._step_recorder.timings
1050 return self._output_data
1051
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]1052 def CommitStashedOrderfileHashes(self):
1053 """Commit any orderfile hash files in the current checkout.
1054
1055 Only possible if running on the buildbot.
1056
1057 Returns: true on success.
1058 """
Egor Pasko7ff04122019-11-25 15:47:18[diff] [blame]1059 if not self._options.buildbot:
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]1060 logging.error('Trying to commit when not running on the buildbot')
1061 return False
1062 self._orderfile_updater._CommitStashedFiles([
1063 filename + '.sha1'
1064 for filename in (self._GetUnpatchedOrderfileFilename(),
1065 self._GetPathToOrderfile())])
1066 return True
1067
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1068
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1069def CreateArgumentParser():
1070 """Creates and returns the argument parser."""
1071 parser = argparse.ArgumentParser()
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]1072 parser.add_argument('--no-benchmark', action='store_false', dest='benchmark',
1073 default=True, help='Disables running benchmarks.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1074 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1075 '--buildbot', action='store_true',
1076 help='If true, the script expects to be run on a buildbot')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1077 parser.add_argument(
Matthew Cary453ff1452018-07-18 12:19:35[diff] [blame]1078 '--device', default=None, type=str,
1079 help='Device serial number on which to run profiling.')
1080 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1081 '--verify', action='store_true',
1082 help='If true, the script only verifies the current orderfile')
Benoit Lizef5581722021-08-26 09:48:20[diff] [blame]1083 parser.add_argument('--target-arch',
1084 action='store',
1085 dest='arch',
Stephen Martinis71b391b52018-12-04 15:08:04[diff] [blame]1086 default='arm',
Benoit Lizef5581722021-08-26 09:48:20[diff] [blame]1087 choices=list(_ARCH_GN_ARGS.keys()),
Matthew Cary53f74ba2019-01-24 10:07:18[diff] [blame]1088 help='The target architecture for which to build.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1089 parser.add_argument('--output-json', action='store', dest='json_file',
1090 help='Location to save stats in json format')
1091 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1092 '--skip-profile', action='store_false', dest='profile', default=True,
1093 help='Don\'t generate a profile on the device. Only patch from the '
1094 'existing profile.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1095 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1096 '--skip-patch', action='store_false', dest='patch', default=True,
1097 help='Only generate the raw (unpatched) orderfile, don\'t patch it.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1098 parser.add_argument('--goma-dir', help='GOMA directory.')
1099 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1100 '--use-goma', action='store_true', help='Enable GOMA.', default=False)
Fumitoshi Ukaibd2755b2022-10-28 13:15:27[diff] [blame]1101 parser.add_argument('--use-remoteexec',
1102 action='store_true',
Fumitoshi Ukaibc233ba2023-02-13 11:47:31[diff] [blame]1103 help='Enable remoteexec. see //build/toolchain/rbe.gni.',
Fumitoshi Ukaibd2755b2022-10-28 13:15:27[diff] [blame]1104 default=False)
Fumitoshi Ukaibc233ba2023-02-13 11:47:31[diff] [blame]1105 parser.add_argument('--ninja-path',
1106 help='Path to the ninja binary. If given, use this'
1107 'instead of autoninja.')
1108 parser.add_argument('--ninja-j',
1109 help='-j value passed to ninja.'
1110 'pass -j to ninja. no need to set this when '
1111 '--ninja-path is not specified.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1112 parser.add_argument('--adb-path', help='Path to the adb binary.')
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]1113
Egor Paskod80bfad2020-09-10 21:53:06[diff] [blame]1114 parser.add_argument('--public',
1115 action='store_true',
1116 help='Build non-internal APK and change the orderfile '
1117 'location. Required if your checkout is non-internal.',
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]1118 default=False)
Matthew Cary04f41032018-12-10 15:55:27[diff] [blame]1119 parser.add_argument('--nosystem-health-orderfile', action='store_false',
Benoit Lcba421e2019-04-26 15:28:26[diff] [blame]1120 dest='system_health_orderfile', default=True,
Matthew Caryc262f5d2018-09-19 14:36:28[diff] [blame]1121 help=('Create an orderfile based on an about:blank '
1122 'startup benchmark instead of system health '
1123 'benchmarks.'))
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]1124 parser.add_argument(
1125 '--use-legacy-chrome-apk', action='store_true', default=False,
1126 help=('Compile and instrument chrome for [L, K] devices.'))
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]1127 parser.add_argument('--manual-symbol-offsets', default=None, type=str,
1128 help=('File of list of ordered symbol offsets generated '
1129 'by manual profiling. Must set other --manual* '
1130 'flags if this is used, and must --skip-profile.'))
1131 parser.add_argument('--manual-libname', default=None, type=str,
1132 help=('Library filename corresponding to '
1133 '--manual-symbol-offsets.'))
1134 parser.add_argument('--manual-objdir', default=None, type=str,
1135 help=('Root of object file directory corresponding to '
1136 '--manual-symbol-offsets.'))
Matthew Caryf949bba2019-02-04 13:39:23[diff] [blame]1137 parser.add_argument('--noorder-outlined-functions', action='store_true',
1138 help='Disable outlined functions in the orderfile.')
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]1139 parser.add_argument('--pregenerated-profiles', default=None, type=str,
1140 help=('Pregenerated profiles to use instead of running '
1141 'profile step. Cannot be used with '
1142 '--skip-profiles.'))
1143 parser.add_argument('--profile-save-dir', default=None, type=str,
1144 help=('Directory to save any profiles created. These can '
1145 'be used with --pregenerated-profiles. Cannot be '
1146 'used with --skip-profiles.'))
Egor Paskobce64d012018-11-20 12:02:37[diff] [blame]1147 parser.add_argument('--upload-ready-orderfiles', action='store_true',
1148 help=('Skip orderfile generation and manually upload '
1149 'orderfiles (both patched and unpatched) from '
1150 'their normal location in the tree to the cloud '
1151 'storage. DANGEROUS! USE WITH CARE!'))
Matthew Cary1ea82212019-03-20 12:10:27[diff] [blame]1152 parser.add_argument('--streamline-for-debugging', action='store_true',
1153 help=('Streamline where possible the run for faster '
1154 'iteration while debugging. The orderfile '
1155 'generated will be valid and nontrivial, but '
1156 'may not be based on a representative profile '
1157 'or other such considerations. Use with caution.'))
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]1158 parser.add_argument('--commit-hashes', action='store_true',
1159 help=('Commit any orderfile hash files in the current '
1160 'checkout; performs no other action'))
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]1161 parser.add_argument('--use-call-graph', action='store_true', default=False,
1162 help='Use call graph instrumentation.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1163 profile_android_startup.AddProfileCollectionArguments(parser)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1164 return parser
1165
1166
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]1167def CreateOrderfile(options, orderfile_updater_class=None):
Christopher Grantdfe1bac2019-07-05 13:34:10[diff] [blame]1168 """Creates an orderfile.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1169
1170 Args:
1171 options: As returned from optparse.OptionParser.parse_args()
1172 orderfile_updater_class: (OrderfileUpdater) subclass of OrderfileUpdater.
1173
1174 Returns:
1175 True iff success.
1176 """
1177 logging.basicConfig(level=logging.INFO)
1178 devil_chromium.Initialize(adb_path=options.adb_path)
1179
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]1180 generator = OrderfileGenerator(options, orderfile_updater_class)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1181 try:
1182 if options.verify:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]1183 generator._VerifySymbolOrder()
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]1184 elif options.commit_hashes:
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]1185 return generator.CommitStashedOrderfileHashes()
Egor Paskobce64d012018-11-20 12:02:37[diff] [blame]1186 elif options.upload_ready_orderfiles:
1187 return generator.UploadReadyOrderfiles()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1188 else:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]1189 return generator.Generate()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1190 finally:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]1191 json_output = json.dumps(generator.GetReportingData(),
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1192 indent=2) + '\n'
1193 if options.json_file:
1194 with open(options.json_file, 'w') as f:
1195 f.write(json_output)
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]1196 print(json_output)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1197 return False
1198
1199
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1200def main():
1201 parser = CreateArgumentParser()
1202 options = parser.parse_args()
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]1203 return 0 if CreateOrderfile(options) else 1
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1204
1205
1206if __name__ == '__main__':
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1207 sys.exit(main())