[go: nahoru, domu]
Google Git
Sign in
chromium / chromium / src / a7ac67a4ea326996c66f655d5a1bc9ec5db5872f / . / tools / cygprofile / orderfile_generator_backend.py
blob: a0d536e88ebae63ed1a1c6d78603d788504f9675 [file] [log] [blame]
Egor Pasko0462e852d2018-03-29 15:52:09[diff] [blame]1#!/usr/bin/env vpython
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]2# Copyright (c) 2013 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6""" A utility to generate an up-to-date orderfile.
7
8The orderfile is used by the linker to order text sections such that the
9sections are placed consecutively in the order specified. This allows us
10to page in less code during start-up.
11
12Example usage:
Egor Paskoa7ac67a42020-09-09 12:06:58[diff] [blame^]13 tools/cygprofile/orderfile_generator_backend.py --use-goma --target-arch=arm
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]14"""
15
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]16from __future__ import print_function
17
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]18import argparse
Egor Paskoa7ac67a42020-09-09 12:06:58[diff] [blame^]19import csv
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]20import hashlib
21import json
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]22import glob
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]23import logging
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]24import os
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]25import shutil
26import subprocess
27import sys
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]28import tempfile
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]29import time
30
Matthew Cary91df9792018-11-30 14:35:15[diff] [blame]31import cluster
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]32import cyglog_to_orderfile
Benoit Lizefefbb27c2018-01-17 13:54:18[diff] [blame]33import patch_orderfile
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]34import process_profiles
Egor Pasko3bc0b932018-04-03 10:08:27[diff] [blame]35import profile_android_startup
Benoit Lizefefbb27c2018-01-17 13:54:18[diff] [blame]36import symbol_extractor
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]37
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]38_SRC_PATH = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
39sys.path.append(os.path.join(_SRC_PATH, 'third_party', 'catapult', 'devil'))
40from devil.android import device_utils
41from devil.android.sdk import version_codes
42
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]43
44_SRC_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)),
45 os.pardir, os.pardir)
46sys.path.append(os.path.join(_SRC_PATH, 'build', 'android'))
47import devil_chromium
48from pylib import constants
49
50
51# Needs to happen early for GetBuildType()/GetOutDirectory() to work correctly
52constants.SetBuildType('Release')
53
54
Matthew Cary53f74ba2019-01-24 10:07:18[diff] [blame]55# Architecture specific GN args. Trying to build an orderfile for an
56# architecture not listed here will eventually throw.
57_ARCH_GN_ARGS = {
58 'arm': [ 'target_cpu = "arm"' ],
59 'arm64': [ 'target_cpu = "arm64"',
60 'android_64bit_browser = true'],
61}
62
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]63class CommandError(Exception):
64 """Indicates that a dispatched shell command exited with a non-zero status."""
65
66 def __init__(self, value):
67 super(CommandError, self).__init__()
68 self.value = value
69
70 def __str__(self):
71 return repr(self.value)
72
73
74def _GenerateHash(file_path):
75 """Calculates and returns the hash of the file at file_path."""
76 sha1 = hashlib.sha1()
77 with open(file_path, 'rb') as f:
78 while True:
79 # Read in 1mb chunks, so it doesn't all have to be loaded into memory.
80 chunk = f.read(1024 * 1024)
81 if not chunk:
82 break
83 sha1.update(chunk)
84 return sha1.hexdigest()
85
86
87def _GetFileExtension(file_name):
88 """Calculates the file extension from a file name.
89
90 Args:
91 file_name: The source file name.
92 Returns:
93 The part of file_name after the dot (.) or None if the file has no
94 extension.
95 Examples: /home/user/foo.bar -> bar
96 /home/user.name/foo -> None
97 /home/user/.foo -> None
98 /home/user/foo.bar.baz -> baz
99 """
100 file_name_parts = os.path.basename(file_name).split('.')
101 if len(file_name_parts) > 1:
102 return file_name_parts[-1]
103 else:
104 return None
105
106
107def _StashOutputDirectory(buildpath):
108 """Takes the output directory and stashes it in the default output directory.
109
110 This allows it to be used for incremental builds next time (after unstashing)
111 by keeping it in a place that isn't deleted normally, while also ensuring
112 that it is properly clobbered when appropriate.
113
114 This is a dirty hack to deal with the needs of clobbering while also handling
115 incremental builds and the hardcoded relative paths used in some of the
116 project files.
117
118 Args:
119 buildpath: The path where the building happens. If this corresponds to the
120 default output directory, no action is taken.
121 """
122 if os.path.abspath(buildpath) == os.path.abspath(os.path.dirname(
123 constants.GetOutDirectory())):
124 return
125 name = os.path.basename(buildpath)
126 stashpath = os.path.join(constants.GetOutDirectory(), name)
127 if not os.path.exists(buildpath):
128 return
129 if os.path.exists(stashpath):
130 shutil.rmtree(stashpath, ignore_errors=True)
131 shutil.move(buildpath, stashpath)
132
133
134def _UnstashOutputDirectory(buildpath):
135 """Inverse of _StashOutputDirectory.
136
137 Moves the output directory stashed within the default output directory
138 (out/Release) to the position where the builds can actually happen.
139
140 This is a dirty hack to deal with the needs of clobbering while also handling
141 incremental builds and the hardcoded relative paths used in some of the
142 project files.
143
144 Args:
145 buildpath: The path where the building happens. If this corresponds to the
146 default output directory, no action is taken.
147 """
148 if os.path.abspath(buildpath) == os.path.abspath(os.path.dirname(
149 constants.GetOutDirectory())):
150 return
151 name = os.path.basename(buildpath)
152 stashpath = os.path.join(constants.GetOutDirectory(), name)
153 if not os.path.exists(stashpath):
154 return
155 if os.path.exists(buildpath):
156 shutil.rmtree(buildpath, ignore_errors=True)
157 shutil.move(stashpath, buildpath)
158
159
160class StepRecorder(object):
161 """Records steps and timings."""
162
163 def __init__(self, buildbot):
164 self.timings = []
165 self._previous_step = ('', 0.0)
166 self._buildbot = buildbot
167 self._error_recorded = False
168
169 def BeginStep(self, name):
170 """Marks a beginning of the next step in the script.
171
172 On buildbot, this prints a specially formatted name that will show up
173 in the waterfall. Otherwise, just prints the step name.
174
175 Args:
176 name: The name of the step.
177 """
178 self.EndStep()
179 self._previous_step = (name, time.time())
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]180 print('Running step: ', name)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]181
182 def EndStep(self):
183 """Records successful completion of the current step.
184
185 This is optional if the step is immediately followed by another BeginStep.
186 """
187 if self._previous_step[0]:
188 elapsed = time.time() - self._previous_step[1]
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]189 print('Step %s took %f seconds' % (self._previous_step[0], elapsed))
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]190 self.timings.append((self._previous_step[0], elapsed))
191
192 self._previous_step = ('', 0.0)
193
194 def FailStep(self, message=None):
195 """Marks that a particular step has failed.
196
197 On buildbot, this will mark the current step as failed on the waterfall.
198 Otherwise we will just print an optional failure message.
199
200 Args:
201 message: An optional explanation as to why the step failed.
202 """
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]203 print('STEP FAILED!!')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]204 if message:
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]205 print(message)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]206 self._error_recorded = True
207 self.EndStep()
208
209 def ErrorRecorded(self):
210 """True if FailStep has been called."""
211 return self._error_recorded
212
213 def RunCommand(self, cmd, cwd=constants.DIR_SOURCE_ROOT, raise_on_error=True,
214 stdout=None):
215 """Execute a shell command.
216
217 Args:
218 cmd: A list of command strings.
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]219 cwd: Directory in which the command should be executed, defaults to build
220 root of script's location if not specified.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]221 raise_on_error: If true will raise a CommandError if the call doesn't
222 succeed and mark the step as failed.
223 stdout: A file to redirect stdout for the command to.
224
225 Returns:
226 The process's return code.
227
228 Raises:
229 CommandError: An error executing the specified command.
230 """
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]231 print('Executing %s in %s' % (' '.join(cmd), cwd))
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]232 process = subprocess.Popen(cmd, stdout=stdout, cwd=cwd, env=os.environ)
233 process.wait()
234 if raise_on_error and process.returncode != 0:
235 self.FailStep()
236 raise CommandError('Exception executing command %s' % ' '.join(cmd))
237 return process.returncode
238
239
240class ClankCompiler(object):
241 """Handles compilation of clank."""
242
Christopher Grant073637472019-07-05 13:34:57[diff] [blame]243 def __init__(self, out_dir, step_recorder, arch, use_goma, goma_dir,
244 system_health_profiling, monochrome, public, orderfile_location):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]245 self._out_dir = out_dir
246 self._step_recorder = step_recorder
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]247 self._arch = arch
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]248 self._use_goma = use_goma
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]249 self._goma_dir = goma_dir
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]250 self._system_health_profiling = system_health_profiling
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]251 self._public = public
252 self._orderfile_location = orderfile_location
Matthew Caryd6bfcb72018-09-14 11:27:46[diff] [blame]253 if monochrome:
254 self._apk = 'Monochrome.apk'
Matthew Carye1b00062018-09-19 14:27:44[diff] [blame]255 self._apk_target = 'monochrome_apk'
Matthew Caryd6bfcb72018-09-14 11:27:46[diff] [blame]256 self._libname = 'libmonochrome'
Matthew Cary2216fef2019-06-17 13:35:59[diff] [blame]257 self._libchrome_target = 'libmonochrome'
Matthew Caryd6bfcb72018-09-14 11:27:46[diff] [blame]258 else:
259 self._apk = 'Chrome.apk'
Matthew Carye1b00062018-09-19 14:27:44[diff] [blame]260 self._apk_target = 'chrome_apk'
Matthew Caryd6bfcb72018-09-14 11:27:46[diff] [blame]261 self._libname = 'libchrome'
262 self._libchrome_target = 'libchrome'
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]263 if public:
264 self._apk = self._apk.replace('.apk', 'Public.apk')
265 self._apk_target = self._apk_target.replace('_apk', '_public_apk')
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]266
267 self.obj_dir = os.path.join(self._out_dir, 'Release', 'obj')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]268 self.lib_chrome_so = os.path.join(
Matthew Caryd6bfcb72018-09-14 11:27:46[diff] [blame]269 self._out_dir, 'Release', 'lib.unstripped',
270 '{}.so'.format(self._libname))
271 self.chrome_apk = os.path.join(self._out_dir, 'Release', 'apks', self._apk)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]272
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]273 def Build(self, instrumented, use_call_graph, target):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]274 """Builds the provided ninja target with or without order_profiling on.
275
276 Args:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]277 instrumented: (bool) Whether we want to build an instrumented binary.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]278 use_call_graph: (bool) Whether to use the call graph instrumentation.
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]279 target: (str) The name of the ninja target to build.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]280 """
281 self._step_recorder.BeginStep('Compile %s' % target)
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]282 assert not use_call_graph or instrumented, ('You can not enable call graph '
283 'without instrumentation!')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]284
285 # Set the "Release Official" flavor, the parts affecting performance.
286 args = [
Andrew Grieve0e8790c2020-09-03 17:27:32[diff] [blame]287 'enable_resource_allowlist_generation=false',
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]288 'is_chrome_branded=' + str(not self._public).lower(),
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]289 'is_debug=false',
290 'is_official_build=true',
Egor Pasko1d13d532020-01-14 21:25:19[diff] [blame]291 'symbol_level=1', # to fit 30 GiB RAM on the bot when LLD is running
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]292 'target_os="android"',
293 'use_goma=' + str(self._use_goma).lower(),
294 'use_order_profiling=' + str(instrumented).lower(),
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]295 'use_call_graph=' + str(use_call_graph).lower(),
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]296 ]
Matthew Cary53f74ba2019-01-24 10:07:18[diff] [blame]297 args += _ARCH_GN_ARGS[self._arch]
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]298 if self._goma_dir:
299 args += ['goma_dir="%s"' % self._goma_dir]
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]300 if self._system_health_profiling:
301 args += ['devtools_instrumentation_dumping = ' +
302 str(instrumented).lower()]
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]303
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]304 if self._public and os.path.exists(self._orderfile_location):
305 # GN needs the orderfile path to be source-absolute.
306 src_abs_orderfile = os.path.relpath(self._orderfile_location,
307 constants.DIR_SOURCE_ROOT)
308 args += ['chrome_orderfile="//{}"'.format(src_abs_orderfile)]
309
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]310 self._step_recorder.RunCommand(
311 ['gn', 'gen', os.path.join(self._out_dir, 'Release'),
312 '--args=' + ' '.join(args)])
313
314 self._step_recorder.RunCommand(
Christopher Grant073637472019-07-05 13:34:57[diff] [blame]315 ['autoninja', '-C',
316 os.path.join(self._out_dir, 'Release'), target])
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]317
Christopher Grant10221762019-07-05 12:10:04[diff] [blame]318 def ForceRelink(self):
319 """Forces libchrome.so or libmonochrome.so to be re-linked.
320
321 With partitioned libraries enabled, deleting these library files does not
322 guarantee they'll be recreated by the linker (they may simply be
323 re-extracted from a combined library). To be safe, touch a source file
324 instead. See http://crbug.com/972701 for more explanation.
325 """
326 file_to_touch = os.path.join(constants.DIR_SOURCE_ROOT, 'chrome', 'browser',
327 'chrome_browser_main_android.cc')
328 assert os.path.exists(file_to_touch)
329 self._step_recorder.RunCommand(['touch', file_to_touch])
330
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]331 def CompileChromeApk(self, instrumented, use_call_graph, force_relink=False):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]332 """Builds a Chrome.apk either with or without order_profiling on.
333
334 Args:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]335 instrumented: (bool) Whether to build an instrumented apk.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]336 use_call_graph: (bool) Whether to use the call graph instrumentation.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]337 force_relink: Whether libchromeview.so should be re-created.
338 """
339 if force_relink:
Christopher Grant10221762019-07-05 12:10:04[diff] [blame]340 self.ForceRelink()
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]341 self.Build(instrumented, use_call_graph, self._apk_target)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]342
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]343 def CompileLibchrome(self, instrumented, use_call_graph, force_relink=False):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]344 """Builds a libchrome.so either with or without order_profiling on.
345
346 Args:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]347 instrumented: (bool) Whether to build an instrumented apk.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]348 use_call_graph: (bool) Whether to use the call graph instrumentation.
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]349 force_relink: (bool) Whether libchrome.so should be re-created.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]350 """
351 if force_relink:
Christopher Grant10221762019-07-05 12:10:04[diff] [blame]352 self.ForceRelink()
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]353 self.Build(instrumented, use_call_graph, self._libchrome_target)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]354
355
356class OrderfileUpdater(object):
357 """Handles uploading and committing a new orderfile in the repository.
358
359 Only used for testing or on a bot.
360 """
361
362 _CLOUD_STORAGE_BUCKET_FOR_DEBUG = None
363 _CLOUD_STORAGE_BUCKET = None
364 _UPLOAD_TO_CLOUD_COMMAND = 'upload_to_google_storage.py'
365
Egor Pasko0c533c6682019-11-26 21:16:32[diff] [blame]366 def __init__(self, repository_root, step_recorder):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]367 """Constructor.
368
369 Args:
370 repository_root: (str) Root of the target repository.
371 step_recorder: (StepRecorder) Step recorder, for logging.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]372 """
373 self._repository_root = repository_root
374 self._step_recorder = step_recorder
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]375
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]376 def CommitStashedFileHashes(self, files):
377 """Commits unpatched and patched orderfiles hashes if changed.
378
379 The files are committed only if their associated sha1 hash files match, and
380 are modified in git. In normal operations the hash files are changed only
381 when a file is uploaded to cloud storage. If the hash file is not modified
382 in git, the file is skipped.
383
384 Args:
385 files: [str or None] specifies file paths. None items are ignored.
386
387 Raises:
388 Exception if the hash file does not match the file.
389 NotImplementedError when the commit logic hasn't been overridden.
390 """
391 files_to_commit = list(filter(None, files))
392 if files_to_commit:
393 self._CommitStashedFiles(files_to_commit)
394
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]395 def UploadToCloudStorage(self, filename, use_debug_location):
396 """Uploads a file to cloud storage.
397
398 Args:
399 filename: (str) File to upload.
400 use_debug_location: (bool) Whether to use the debug location.
401 """
402 bucket = (self._CLOUD_STORAGE_BUCKET_FOR_DEBUG if use_debug_location
403 else self._CLOUD_STORAGE_BUCKET)
404 extension = _GetFileExtension(filename)
405 cmd = [self._UPLOAD_TO_CLOUD_COMMAND, '--bucket', bucket]
406 if extension:
407 cmd.extend(['-z', extension])
408 cmd.append(filename)
409 self._step_recorder.RunCommand(cmd)
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]410 print('Download: https://sandbox.google.com/storage/%s/%s' %
411 (bucket, _GenerateHash(filename)))
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]412
413 def _GetHashFilePathAndContents(self, filename):
414 """Gets the name and content of the hash file created from uploading the
415 given file.
416
417 Args:
418 filename: (str) The file that was uploaded to cloud storage.
419
420 Returns:
421 A tuple of the hash file name, relative to the reository root, and the
422 content, which should be the sha1 hash of the file
423 ('base_file.sha1', hash)
424 """
425 abs_hash_filename = filename + '.sha1'
426 rel_hash_filename = os.path.relpath(
427 abs_hash_filename, self._repository_root)
428 with open(abs_hash_filename, 'r') as f:
429 return (rel_hash_filename, f.read())
430
431 def _CommitFiles(self, files_to_commit, commit_message_lines):
432 """Commits a list of files, with a given message."""
433 raise NotImplementedError
434
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]435 def _GitStash(self):
436 """Git stash the current clank tree.
437
438 Raises:
439 NotImplementedError when the stash logic hasn't been overridden.
440 """
441 raise NotImplementedError
442
443 def _CommitStashedFiles(self, expected_files_in_stash):
444 """Commits stashed files.
445
446 The local repository is updated and then the files to commit are taken from
447 modified files from the git stash. The modified files should be a subset of
448 |expected_files_in_stash|. If there are unexpected modified files, this
449 function may raise. This is meant to be paired with _GitStash().
450
451 Args:
452 expected_files_in_stash: [str] paths to a possible superset of files
453 expected to be stashed & committed.
454
455 Raises:
456 NotImplementedError when the commit logic hasn't been overridden.
457 """
458 raise NotImplementedError
459
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]460
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]461class OrderfileNoopUpdater(OrderfileUpdater):
462 def CommitFileHashes(self, unpatched_orderfile_filename, orderfile_filename):
463 # pylint: disable=unused-argument
464 return
465
466 def UploadToCloudStorage(self, filename, use_debug_location):
467 # pylint: disable=unused-argument
468 return
469
470 def _CommitFiles(self, files_to_commit, commit_message_lines):
471 raise NotImplementedError
472
473
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]474class OrderfileGenerator(object):
475 """A utility for generating a new orderfile for Clank.
476
477 Builds an instrumented binary, profiles a run of the application, and
478 generates an updated orderfile.
479 """
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]480 _CHECK_ORDERFILE_SCRIPT = os.path.join(
481 constants.DIR_SOURCE_ROOT, 'tools', 'cygprofile', 'check_orderfile.py')
482 _BUILD_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(
483 constants.GetOutDirectory()))) # Normally /path/to/src
484
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]485 # Previous orderfile_generator debug files would be overwritten.
486 _DIRECTORY_FOR_DEBUG_FILES = '/tmp/orderfile_generator_debug_files'
487
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]488 def _PrepareOrderfilePaths(self):
489 if self._options.public:
490 self._clank_dir = os.path.join(constants.DIR_SOURCE_ROOT,
491 '')
492 if not os.path.exists(os.path.join(self._clank_dir, 'orderfiles')):
493 os.makedirs(os.path.join(self._clank_dir, 'orderfiles'))
494 else:
495 self._clank_dir = os.path.join(constants.DIR_SOURCE_ROOT,
496 'clank')
497
498 self._unpatched_orderfile_filename = os.path.join(
499 self._clank_dir, 'orderfiles', 'unpatched_orderfile.%s')
500 self._path_to_orderfile = os.path.join(
501 self._clank_dir, 'orderfiles', 'orderfile.%s.out')
502
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]503 def _GetPathToOrderfile(self):
504 """Gets the path to the architecture-specific orderfile."""
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]505 return self._path_to_orderfile % self._options.arch
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]506
507 def _GetUnpatchedOrderfileFilename(self):
508 """Gets the path to the architecture-specific unpatched orderfile."""
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]509 return self._unpatched_orderfile_filename % self._options.arch
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]510
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]511 def _SetDevice(self):
512 """ Selects the device to be used by the script.
513
514 Returns:
515 (Device with given serial ID) : if the --device flag is set.
516 (Device running Android[K,L]) : if --use-legacy-chrome-apk flag is set or
517 no device running Android N+ was found.
518 (Device running Android N+) : Otherwise.
519
520 Raises Error:
521 If no device meeting the requirements has been found.
522 """
523 devices = None
524 if self._options.device:
525 devices = [device_utils.DeviceUtils(self._options.device)]
526 else:
527 devices = device_utils.DeviceUtils.HealthyDevices()
528
529 assert devices, 'Expected at least one connected device'
530
531 if self._options.use_legacy_chrome_apk:
532 self._monochrome = False
533 for device in devices:
534 device_version = device.build_version_sdk
535 if (device_version >= version_codes.KITKAT
536 and device_version <= version_codes.LOLLIPOP_MR1):
537 return device
538
539 assert not self._options.use_legacy_chrome_apk, \
540 'No device found running suitable android version for Chrome.apk.'
541
542 preferred_device = None
543 for device in devices:
544 if device.build_version_sdk >= version_codes.NOUGAT:
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]545 preferred_device = device
546 break
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]547
548 self._monochrome = preferred_device is not None
549
550 return preferred_device if preferred_device else devices[0]
551
552
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]553 def __init__(self, options, orderfile_updater_class):
554 self._options = options
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]555 self._instrumented_out_dir = os.path.join(
556 self._BUILD_ROOT, self._options.arch + '_instrumented_out')
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]557 if self._options.use_call_graph:
Christopher Grantdfe1bac2019-07-05 13:34:10[diff] [blame]558 self._instrumented_out_dir += '_call_graph'
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]559
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]560 self._uninstrumented_out_dir = os.path.join(
561 self._BUILD_ROOT, self._options.arch + '_uninstrumented_out')
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]562 self._no_orderfile_out_dir = os.path.join(
563 self._BUILD_ROOT, self._options.arch + '_no_orderfile_out')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]564
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]565 self._PrepareOrderfilePaths()
566
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]567 if options.profile:
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]568 output_directory = os.path.join(self._instrumented_out_dir, 'Release')
Matthew Caryb8daed942018-06-11 10:58:08[diff] [blame]569 host_profile_dir = os.path.join(output_directory, 'profile_data')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]570 urls = [profile_android_startup.AndroidProfileTool.TEST_URL]
571 use_wpr = True
572 simulate_user = False
Benoit L96466812018-03-06 15:58:37[diff] [blame]573 urls = options.urls
574 use_wpr = not options.no_wpr
575 simulate_user = options.simulate_user
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]576 device = self._SetDevice()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]577 self._profiler = profile_android_startup.AndroidProfileTool(
Matthew Cary453ff1452018-07-18 12:19:35[diff] [blame]578 output_directory, host_profile_dir, use_wpr, urls, simulate_user,
Matthew Cary1ea82212019-03-20 12:10:27[diff] [blame]579 device, debug=self._options.streamline_for_debugging)
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]580 if options.pregenerated_profiles:
581 self._profiler.SetPregeneratedProfiles(
582 glob.glob(options.pregenerated_profiles))
583 else:
584 assert not options.pregenerated_profiles, (
585 '--pregenerated-profiles cannot be used with --skip-profile')
586 assert not options.profile_save_dir, (
587 '--profile-save-dir cannot be used with --skip-profile')
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]588 self._monochrome = not self._options.use_legacy_chrome_apk
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]589
Matthew Caryf949bba2019-02-04 13:39:23[diff] [blame]590 # Outlined function handling enabled by default for all architectures.
591 self._order_outlined_functions = not options.noorder_outlined_functions
592
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]593 self._output_data = {}
594 self._step_recorder = StepRecorder(options.buildbot)
595 self._compiler = None
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]596 if orderfile_updater_class is None:
597 if options.public:
598 orderfile_updater_class = OrderfileNoopUpdater
599 else:
600 orderfile_updater_class = OrderfileUpdater
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]601 assert issubclass(orderfile_updater_class, OrderfileUpdater)
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]602 self._orderfile_updater = orderfile_updater_class(self._clank_dir,
Egor Pasko0c533c6682019-11-26 21:16:32[diff] [blame]603 self._step_recorder)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]604 assert os.path.isdir(constants.DIR_SOURCE_ROOT), 'No src directory found'
Benoit Lizefefbb27c2018-01-17 13:54:18[diff] [blame]605 symbol_extractor.SetArchitecture(options.arch)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]606
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]607 @staticmethod
608 def _RemoveBlanks(src_file, dest_file):
609 """A utility to remove blank lines from a file.
610
611 Args:
612 src_file: The name of the file to remove the blanks from.
613 dest_file: The name of the file to write the output without blanks.
614 """
615 assert src_file != dest_file, 'Source and destination need to be distinct'
616
617 try:
618 src = open(src_file, 'r')
619 dest = open(dest_file, 'w')
620 for line in src:
621 if line and not line.isspace():
622 dest.write(line)
623 finally:
624 src.close()
625 dest.close()
626
627 def _GenerateAndProcessProfile(self):
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]628 """Invokes a script to merge the per-thread traces into one file.
629
630 The produced list of offsets is saved in
631 self._GetUnpatchedOrderfileFilename().
632 """
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]633 self._step_recorder.BeginStep('Generate Profile Data')
634 files = []
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]635 logging.getLogger().setLevel(logging.DEBUG)
Christopher Grantdfe1bac2019-07-05 13:34:10[diff] [blame]636
637 if self._options.profile_save_dir:
638 # The directory must not preexist, to ensure purity of data. Check
639 # before profiling to save time.
640 if os.path.exists(self._options.profile_save_dir):
641 raise Exception('Profile save directory must not pre-exist')
642 os.makedirs(self._options.profile_save_dir)
643
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]644 if self._options.system_health_orderfile:
645 files = self._profiler.CollectSystemHealthProfile(
646 self._compiler.chrome_apk)
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]647 self._MaybeSaveProfile(files)
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]648 try:
649 self._ProcessPhasedOrderfile(files)
650 except Exception:
651 for f in files:
652 self._SaveForDebugging(f)
Matthew Cary8b1416232018-08-10 19:12:22[diff] [blame]653 self._SaveForDebugging(self._compiler.lib_chrome_so)
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]654 raise
655 finally:
656 self._profiler.Cleanup()
657 else:
658 self._CollectLegacyProfile()
659 logging.getLogger().setLevel(logging.INFO)
660
661 def _ProcessPhasedOrderfile(self, files):
662 """Process the phased orderfiles produced by system health benchmarks.
663
664 The offsets will be placed in _GetUnpatchedOrderfileFilename().
665
666 Args:
667 file: Profile files pulled locally.
668 """
669 self._step_recorder.BeginStep('Process Phased Orderfile')
670 profiles = process_profiles.ProfileManager(files)
671 processor = process_profiles.SymbolOffsetProcessor(
672 self._compiler.lib_chrome_so)
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]673 ordered_symbols = cluster.ClusterOffsets(profiles, processor,
674 call_graph=self._options.use_call_graph)
Matthew Cary8b1416232018-08-10 19:12:22[diff] [blame]675 if not ordered_symbols:
676 raise Exception('Failed to get ordered symbols')
Matthew Caryda9db932019-03-11 15:28:17[diff] [blame]677 for sym in ordered_symbols:
678 assert not sym.startswith('OUTLINED_FUNCTION_'), (
679 'Outlined function found in instrumented function, very likely '
680 'something has gone very wrong!')
Matthew Cary91df9792018-11-30 14:35:15[diff] [blame]681 self._output_data['offsets_kib'] = processor.SymbolsSize(
682 ordered_symbols) / 1024
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]683 with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile:
Matthew Cary8b1416232018-08-10 19:12:22[diff] [blame]684 orderfile.write('\n'.join(ordered_symbols))
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]685
686 def _CollectLegacyProfile(self):
Matthew Cary2df8d452018-09-19 07:50:20[diff] [blame]687 files = []
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]688 try:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]689 files = self._profiler.CollectProfile(
690 self._compiler.chrome_apk,
691 constants.PACKAGE_INFO['chrome'])
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]692 self._MaybeSaveProfile(files)
Matthew Caryb8daed942018-06-11 10:58:08[diff] [blame]693 self._step_recorder.BeginStep('Process profile')
Benoit L96466812018-03-06 15:58:37[diff] [blame]694 assert os.path.exists(self._compiler.lib_chrome_so)
695 offsets = process_profiles.GetReachedOffsetsFromDumpFiles(
696 files, self._compiler.lib_chrome_so)
697 if not offsets:
698 raise Exception('No profiler offsets found in {}'.format(
699 '\n'.join(files)))
Matthew Cary8b1416232018-08-10 19:12:22[diff] [blame]700 processor = process_profiles.SymbolOffsetProcessor(
701 self._compiler.lib_chrome_so)
702 ordered_symbols = processor.GetOrderedSymbols(offsets)
703 if not ordered_symbols:
704 raise Exception('No symbol names from offsets found in {}'.format(
705 '\n'.join(files)))
706 with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile:
707 orderfile.write('\n'.join(ordered_symbols))
Matthew Cary0f1f681a2018-01-22 10:40:51[diff] [blame]708 except Exception:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]709 for f in files:
710 self._SaveForDebugging(f)
711 raise
712 finally:
713 self._profiler.Cleanup()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]714
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]715 def _MaybeSaveProfile(self, files):
716 if self._options.profile_save_dir:
717 logging.info('Saving profiles to %s', self._options.profile_save_dir)
718 for f in files:
719 shutil.copy(f, self._options.profile_save_dir)
720 logging.info('Saved profile %s', f)
721
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]722 def _PatchOrderfile(self):
723 """Patches the orderfile using clean version of libchrome.so."""
724 self._step_recorder.BeginStep('Patch Orderfile')
Benoit Lizefefbb27c2018-01-17 13:54:18[diff] [blame]725 patch_orderfile.GeneratePatchedOrderfile(
726 self._GetUnpatchedOrderfileFilename(), self._compiler.lib_chrome_so,
Matthew Caryf949bba2019-02-04 13:39:23[diff] [blame]727 self._GetPathToOrderfile(), self._order_outlined_functions)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]728
729 def _VerifySymbolOrder(self):
730 self._step_recorder.BeginStep('Verify Symbol Order')
731 return_code = self._step_recorder.RunCommand(
732 [self._CHECK_ORDERFILE_SCRIPT, self._compiler.lib_chrome_so,
733 self._GetPathToOrderfile(),
734 '--target-arch=' + self._options.arch],
735 constants.DIR_SOURCE_ROOT,
736 raise_on_error=False)
737 if return_code:
738 self._step_recorder.FailStep('Orderfile check returned %d.' % return_code)
739
740 def _RecordHash(self, file_name):
741 """Records the hash of the file into the output_data dictionary."""
742 self._output_data[os.path.basename(file_name) + '.sha1'] = _GenerateHash(
743 file_name)
744
745 def _SaveFileLocally(self, file_name, file_sha1):
746 """Saves the file to a temporary location and prints the sha1sum."""
747 if not os.path.exists(self._DIRECTORY_FOR_DEBUG_FILES):
748 os.makedirs(self._DIRECTORY_FOR_DEBUG_FILES)
749 shutil.copy(file_name, self._DIRECTORY_FOR_DEBUG_FILES)
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]750 print('File: %s, saved in: %s, sha1sum: %s' %
751 (file_name, self._DIRECTORY_FOR_DEBUG_FILES, file_sha1))
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]752
753 def _SaveForDebugging(self, filename):
754 """Uploads the file to cloud storage or saves to a temporary location."""
755 file_sha1 = _GenerateHash(filename)
756 if not self._options.buildbot:
757 self._SaveFileLocally(filename, file_sha1)
758 else:
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]759 print('Uploading file for debugging: ' + filename)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]760 self._orderfile_updater.UploadToCloudStorage(
761 filename, use_debug_location=True)
762
763 def _SaveForDebuggingWithOverwrite(self, file_name):
764 """Uploads and overwrites the file in cloud storage or copies locally.
765
766 Should be used for large binaries like lib_chrome_so.
767
768 Args:
769 file_name: (str) File to upload.
770 """
771 file_sha1 = _GenerateHash(file_name)
772 if not self._options.buildbot:
773 self._SaveFileLocally(file_name, file_sha1)
774 else:
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]775 print('Uploading file for debugging: %s, sha1sum: %s' % (file_name,
776 file_sha1))
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]777 upload_location = '%s/%s' % (
778 self._CLOUD_STORAGE_BUCKET_FOR_DEBUG, os.path.basename(file_name))
779 self._step_recorder.RunCommand([
780 'gsutil.py', 'cp', file_name, 'gs://' + upload_location])
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]781 print('Uploaded to: https://sandbox.google.com/storage/' +
782 upload_location)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]783
784 def _MaybeArchiveOrderfile(self, filename):
785 """In buildbot configuration, uploads the generated orderfile to
786 Google Cloud Storage.
787
788 Args:
789 filename: (str) Orderfile to upload.
790 """
Matthew Cary91df9792018-11-30 14:35:15[diff] [blame]791 # First compute hashes so that we can download them later if we need to.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]792 self._step_recorder.BeginStep('Compute hash for ' + filename)
793 self._RecordHash(filename)
794 if self._options.buildbot:
795 self._step_recorder.BeginStep('Archive ' + filename)
796 self._orderfile_updater.UploadToCloudStorage(
797 filename, use_debug_location=False)
798
Egor Paskobce64d012018-11-20 12:02:37[diff] [blame]799 def UploadReadyOrderfiles(self):
800 self._step_recorder.BeginStep('Upload Ready Orderfiles')
801 for file_name in [self._GetUnpatchedOrderfileFilename(),
802 self._GetPathToOrderfile()]:
803 self._orderfile_updater.UploadToCloudStorage(
804 file_name, use_debug_location=False)
805
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]806 def _NativeCodeMemoryBenchmark(self, apk):
807 """Runs system_health.memory_mobile to assess native code memory footprint.
808
809 Args:
810 apk: (str) Path to the apk.
811
812 Returns:
813 results: ([int]) Values of native code memory footprint in bytes from the
814 benchmark results.
815 """
816 self._step_recorder.BeginStep("Running orderfile.memory_mobile")
817 try:
818 out_dir = tempfile.mkdtemp()
819 self._profiler._RunCommand(['tools/perf/run_benchmark',
820 '--device={}'.format(
821 self._profiler._device.serial),
822 '--browser=exact',
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]823 '--output-format=csv',
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]824 '--output-dir={}'.format(out_dir),
825 '--reset-results',
826 '--browser-executable={}'.format(apk),
827 'orderfile.memory_mobile'])
828
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]829 out_file_path = os.path.join(out_dir, 'results.csv')
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]830 if not os.path.exists(out_file_path):
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]831 raise Exception('Results file not found!')
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]832
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]833 results = {}
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]834 with open(out_file_path, 'r') as f:
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]835 reader = csv.DictReader(f)
836 for row in reader:
837 if not row['name'].endswith('NativeCodeResidentMemory'):
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]838 continue
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]839 # Note: NativeCodeResidentMemory records a single sample from each
840 # story run, so this average (reported as 'avg') is exactly the value
841 # of that one sample. Each story is run multiple times, so this loop
842 # will accumulate into a list all values for all runs of each story.
843 results.setdefault(row['name'], {}).setdefault(
844 row['stories'], []).append(row['avg'])
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]845
Juan Antonio Navarro Perez5b0867f72019-10-21 15:21:54[diff] [blame]846 if not results:
847 raise Exception('Could not find relevant results')
848
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]849 return results
850
851 except Exception as e:
852 return 'Error: ' + str(e)
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]853
854 finally:
855 shutil.rmtree(out_dir)
856
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]857
858 def _PerformanceBenchmark(self, apk):
859 """Runs Speedometer2.0 to assess performance.
860
861 Args:
862 apk: (str) Path to the apk.
863
864 Returns:
865 results: ([float]) Speedometer2.0 results samples in milliseconds.
866 """
867 self._step_recorder.BeginStep("Running Speedometer2.0.")
868 try:
869 out_dir = tempfile.mkdtemp()
870 self._profiler._RunCommand(['tools/perf/run_benchmark',
871 '--device={}'.format(
872 self._profiler._device.serial),
873 '--browser=exact',
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]874 '--output-format=histograms',
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]875 '--output-dir={}'.format(out_dir),
876 '--reset-results',
877 '--browser-executable={}'.format(apk),
878 'speedometer2'])
879
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]880 out_file_path = os.path.join(out_dir, 'histograms.json')
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]881 if not os.path.exists(out_file_path):
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]882 raise Exception('Results file not found!')
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]883
884 with open(out_file_path, 'r') as f:
885 results = json.load(f)
886
887 if not results:
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]888 raise Exception('Results file is empty.')
889
890 for el in results:
891 if 'name' in el and el['name'] == 'Total' and 'sampleValues' in el:
892 return el['sampleValues']
893
894 raise Exception('Unexpected results format.')
895
896 except Exception as e:
897 return 'Error: ' + str(e)
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]898
899 finally:
900 shutil.rmtree(out_dir)
901
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]902
903 def RunBenchmark(self, out_directory, no_orderfile=False):
904 """Builds chrome apk and runs performance and memory benchmarks.
905
906 Builds a non-instrumented version of chrome.
907 Installs chrome apk on the device.
908 Runs Speedometer2.0 benchmark to assess performance.
909 Runs system_health.memory_mobile to evaluate memory footprint.
910
911 Args:
912 out_directory: (str) Path to out directory for this build.
913 no_orderfile: (bool) True if chrome to be built without orderfile.
914
915 Returns:
916 benchmark_results: (dict) Results extracted from benchmarks.
917 """
918 try:
919 _UnstashOutputDirectory(out_directory)
Christopher Grant073637472019-07-05 13:34:57[diff] [blame]920 self._compiler = ClankCompiler(out_directory, self._step_recorder,
921 self._options.arch, self._options.use_goma,
922 self._options.goma_dir,
923 self._options.system_health_orderfile,
924 self._monochrome, self._options.public,
925 self._GetPathToOrderfile())
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]926
927 if no_orderfile:
928 orderfile_path = self._GetPathToOrderfile()
929 backup_orderfile = orderfile_path + '.backup'
930 shutil.move(orderfile_path, backup_orderfile)
931 open(orderfile_path, 'w').close()
932
933 # Build APK to be installed on the device.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]934 self._compiler.CompileChromeApk(instrumented=False,
935 use_call_graph=False,
936 force_relink=True)
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]937 benchmark_results = dict()
938 benchmark_results['Speedometer2.0'] = self._PerformanceBenchmark(
939 self._compiler.chrome_apk)
940 benchmark_results['orderfile.memory_mobile'] = (
941 self._NativeCodeMemoryBenchmark(self._compiler.chrome_apk))
Monica Basta8ec87fd2019-05-13 12:12:35[diff] [blame]942
943 except Exception as e:
944 benchmark_results['Error'] = str(e)
945
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]946 finally:
947 if no_orderfile and os.path.exists(backup_orderfile):
948 shutil.move(backup_orderfile, orderfile_path)
949 _StashOutputDirectory(out_directory)
950
951 return benchmark_results
952
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]953 def Generate(self):
954 """Generates and maybe upload an order."""
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]955 assert (bool(self._options.profile) ^
956 bool(self._options.manual_symbol_offsets))
Matthew Cary78aae162018-08-10 17:16:30[diff] [blame]957 if self._options.system_health_orderfile and not self._options.profile:
958 raise AssertionError('--system_health_orderfile must be not be used '
959 'with --skip-profile')
960 if (self._options.manual_symbol_offsets and
961 not self._options.system_health_orderfile):
962 raise AssertionError('--manual-symbol-offsets must be used with '
963 '--system_health_orderfile.')
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]964
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]965 if self._options.profile:
966 try:
967 _UnstashOutputDirectory(self._instrumented_out_dir)
968 self._compiler = ClankCompiler(
Christopher Grant073637472019-07-05 13:34:57[diff] [blame]969 self._instrumented_out_dir, self._step_recorder, self._options.arch,
970 self._options.use_goma, self._options.goma_dir,
971 self._options.system_health_orderfile, self._monochrome,
972 self._options.public, self._GetPathToOrderfile())
Matthew Carya2bea452018-11-13 10:21:07[diff] [blame]973 if not self._options.pregenerated_profiles:
974 # If there are pregenerated profiles, the instrumented build should
975 # not be changed to avoid invalidating the pregenerated profile
976 # offsets.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]977 self._compiler.CompileChromeApk(instrumented=True,
978 use_call_graph=
979 self._options.use_call_graph)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]980 self._GenerateAndProcessProfile()
981 self._MaybeArchiveOrderfile(self._GetUnpatchedOrderfileFilename())
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]982 finally:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]983 _StashOutputDirectory(self._instrumented_out_dir)
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]984 elif self._options.manual_symbol_offsets:
985 assert self._options.manual_libname
986 assert self._options.manual_objdir
987 with file(self._options.manual_symbol_offsets) as f:
988 symbol_offsets = [int(x) for x in f.xreadlines()]
989 processor = process_profiles.SymbolOffsetProcessor(
Matthew Caryb46ad282018-11-23 14:43:57[diff] [blame]990 self._compiler.manual_libname)
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]991 generator = cyglog_to_orderfile.OffsetOrderfileGenerator(
992 processor, cyglog_to_orderfile.ObjectFileProcessor(
993 self._options.manual_objdir))
994 ordered_sections = generator.GetOrderedSections(symbol_offsets)
995 if not ordered_sections: # Either None or empty is a problem.
996 raise Exception('Failed to get ordered sections')
997 with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile:
998 orderfile.write('\n'.join(ordered_sections))
999
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1000 if self._options.patch:
1001 if self._options.profile:
1002 self._RemoveBlanks(self._GetUnpatchedOrderfileFilename(),
1003 self._GetPathToOrderfile())
1004 try:
1005 _UnstashOutputDirectory(self._uninstrumented_out_dir)
1006 self._compiler = ClankCompiler(
1007 self._uninstrumented_out_dir, self._step_recorder,
Christopher Grant073637472019-07-05 13:34:57[diff] [blame]1008 self._options.arch, self._options.use_goma, self._options.goma_dir,
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]1009 self._options.system_health_orderfile, self._monochrome,
1010 self._options.public, self._GetPathToOrderfile())
1011
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]1012 self._compiler.CompileLibchrome(instrumented=False,
1013 use_call_graph=False)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1014 self._PatchOrderfile()
1015 # Because identical code folding is a bit different with and without
1016 # the orderfile build, we need to re-patch the orderfile with code
1017 # folding as close to the final version as possible.
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]1018 self._compiler.CompileLibchrome(instrumented=False,
1019 use_call_graph=False, force_relink=True)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1020 self._PatchOrderfile()
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]1021 self._compiler.CompileLibchrome(instrumented=False,
1022 use_call_graph=False, force_relink=True)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1023 self._VerifySymbolOrder()
1024 self._MaybeArchiveOrderfile(self._GetPathToOrderfile())
1025 finally:
1026 _StashOutputDirectory(self._uninstrumented_out_dir)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1027
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]1028 if self._options.benchmark:
1029 self._output_data['orderfile_benchmark_results'] = self.RunBenchmark(
1030 self._uninstrumented_out_dir)
1031 self._output_data['no_orderfile_benchmark_results'] = self.RunBenchmark(
1032 self._no_orderfile_out_dir, no_orderfile=True)
1033
Egor Pasko0c533c6682019-11-26 21:16:32[diff] [blame]1034 self._orderfile_updater._GitStash()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1035 self._step_recorder.EndStep()
1036 return not self._step_recorder.ErrorRecorded()
1037
1038 def GetReportingData(self):
1039 """Get a dictionary of reporting data (timings, output hashes)"""
1040 self._output_data['timings'] = self._step_recorder.timings
1041 return self._output_data
1042
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]1043 def CommitStashedOrderfileHashes(self):
1044 """Commit any orderfile hash files in the current checkout.
1045
1046 Only possible if running on the buildbot.
1047
1048 Returns: true on success.
1049 """
Egor Pasko7ff04122019-11-25 15:47:18[diff] [blame]1050 if not self._options.buildbot:
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]1051 logging.error('Trying to commit when not running on the buildbot')
1052 return False
1053 self._orderfile_updater._CommitStashedFiles([
1054 filename + '.sha1'
1055 for filename in (self._GetUnpatchedOrderfileFilename(),
1056 self._GetPathToOrderfile())])
1057 return True
1058
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1059
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1060def CreateArgumentParser():
1061 """Creates and returns the argument parser."""
1062 parser = argparse.ArgumentParser()
Monica Salama90b8f5b2019-04-25 11:10:38[diff] [blame]1063 parser.add_argument('--no-benchmark', action='store_false', dest='benchmark',
1064 default=True, help='Disables running benchmarks.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1065 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1066 '--buildbot', action='store_true',
1067 help='If true, the script expects to be run on a buildbot')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1068 parser.add_argument(
Matthew Cary453ff1452018-07-18 12:19:35[diff] [blame]1069 '--device', default=None, type=str,
1070 help='Device serial number on which to run profiling.')
1071 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1072 '--verify', action='store_true',
1073 help='If true, the script only verifies the current orderfile')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1074 parser.add_argument('--target-arch', action='store', dest='arch',
Stephen Martinis71b391b52018-12-04 15:08:04[diff] [blame]1075 default='arm',
Matthew Cary53f74ba2019-01-24 10:07:18[diff] [blame]1076 choices=['arm', 'arm64'],
1077 help='The target architecture for which to build.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1078 parser.add_argument('--output-json', action='store', dest='json_file',
1079 help='Location to save stats in json format')
1080 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1081 '--skip-profile', action='store_false', dest='profile', default=True,
1082 help='Don\'t generate a profile on the device. Only patch from the '
1083 'existing profile.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1084 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1085 '--skip-patch', action='store_false', dest='patch', default=True,
1086 help='Only generate the raw (unpatched) orderfile, don\'t patch it.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1087 parser.add_argument('--goma-dir', help='GOMA directory.')
1088 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1089 '--use-goma', action='store_true', help='Enable GOMA.', default=False)
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1090 parser.add_argument('--adb-path', help='Path to the adb binary.')
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]1091
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]1092 parser.add_argument('--public', action='store_true',
1093 help='Required if your checkout is non-internal.',
1094 default=False)
Matthew Cary04f41032018-12-10 15:55:27[diff] [blame]1095 parser.add_argument('--nosystem-health-orderfile', action='store_false',
Benoit Lcba421e2019-04-26 15:28:26[diff] [blame]1096 dest='system_health_orderfile', default=True,
Matthew Caryc262f5d2018-09-19 14:36:28[diff] [blame]1097 help=('Create an orderfile based on an about:blank '
1098 'startup benchmark instead of system health '
1099 'benchmarks.'))
Monica Salamaeea2d942019-03-11 12:36:18[diff] [blame]1100 parser.add_argument(
1101 '--use-legacy-chrome-apk', action='store_true', default=False,
1102 help=('Compile and instrument chrome for [L, K] devices.'))
Matthew Carye8400642018-06-14 15:43:02[diff] [blame]1103 parser.add_argument('--manual-symbol-offsets', default=None, type=str,
1104 help=('File of list of ordered symbol offsets generated '
1105 'by manual profiling. Must set other --manual* '
1106 'flags if this is used, and must --skip-profile.'))
1107 parser.add_argument('--manual-libname', default=None, type=str,
1108 help=('Library filename corresponding to '
1109 '--manual-symbol-offsets.'))
1110 parser.add_argument('--manual-objdir', default=None, type=str,
1111 help=('Root of object file directory corresponding to '
1112 '--manual-symbol-offsets.'))
Matthew Caryf949bba2019-02-04 13:39:23[diff] [blame]1113 parser.add_argument('--noorder-outlined-functions', action='store_true',
1114 help='Disable outlined functions in the orderfile.')
Matthew Cary69e9e422018-08-10 18:30:06[diff] [blame]1115 parser.add_argument('--pregenerated-profiles', default=None, type=str,
1116 help=('Pregenerated profiles to use instead of running '
1117 'profile step. Cannot be used with '
1118 '--skip-profiles.'))
1119 parser.add_argument('--profile-save-dir', default=None, type=str,
1120 help=('Directory to save any profiles created. These can '
1121 'be used with --pregenerated-profiles. Cannot be '
1122 'used with --skip-profiles.'))
Egor Paskobce64d012018-11-20 12:02:37[diff] [blame]1123 parser.add_argument('--upload-ready-orderfiles', action='store_true',
1124 help=('Skip orderfile generation and manually upload '
1125 'orderfiles (both patched and unpatched) from '
1126 'their normal location in the tree to the cloud '
1127 'storage. DANGEROUS! USE WITH CARE!'))
Matthew Cary1ea82212019-03-20 12:10:27[diff] [blame]1128 parser.add_argument('--streamline-for-debugging', action='store_true',
1129 help=('Streamline where possible the run for faster '
1130 'iteration while debugging. The orderfile '
1131 'generated will be valid and nontrivial, but '
1132 'may not be based on a representative profile '
1133 'or other such considerations. Use with caution.'))
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]1134 parser.add_argument('--commit-hashes', action='store_true',
1135 help=('Commit any orderfile hash files in the current '
1136 'checkout; performs no other action'))
Monica Basta99c101f2019-05-21 13:50:05[diff] [blame]1137 parser.add_argument('--use-call-graph', action='store_true', default=False,
1138 help='Use call graph instrumentation.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1139 profile_android_startup.AddProfileCollectionArguments(parser)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1140 return parser
1141
1142
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]1143def CreateOrderfile(options, orderfile_updater_class=None):
Christopher Grantdfe1bac2019-07-05 13:34:10[diff] [blame]1144 """Creates an orderfile.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1145
1146 Args:
1147 options: As returned from optparse.OptionParser.parse_args()
1148 orderfile_updater_class: (OrderfileUpdater) subclass of OrderfileUpdater.
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]1149 Use to explicitly set an OrderfileUpdater class,
1150 the defaults are OrderfileUpdater, or
1151 OrderfileNoopUpdater if --public is set.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1152
1153 Returns:
1154 True iff success.
1155 """
1156 logging.basicConfig(level=logging.INFO)
1157 devil_chromium.Initialize(adb_path=options.adb_path)
1158
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]1159 generator = OrderfileGenerator(options, orderfile_updater_class)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1160 try:
1161 if options.verify:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]1162 generator._VerifySymbolOrder()
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]1163 elif options.commit_hashes:
Matthew Cary86a226e2019-03-19 12:17:44[diff] [blame]1164 return generator.CommitStashedOrderfileHashes()
Egor Paskobce64d012018-11-20 12:02:37[diff] [blame]1165 elif options.upload_ready_orderfiles:
1166 return generator.UploadReadyOrderfiles()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1167 else:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]1168 return generator.Generate()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1169 finally:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]1170 json_output = json.dumps(generator.GetReportingData(),
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1171 indent=2) + '\n'
1172 if options.json_file:
1173 with open(options.json_file, 'w') as f:
1174 f.write(json_output)
Raul Tambre48f176622019-09-23 10:05:24[diff] [blame]1175 print(json_output)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1176 return False
1177
1178
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1179def main():
1180 parser = CreateArgumentParser()
1181 options = parser.parse_args()
Stephen Kylef11339f2019-03-25 09:00:47[diff] [blame]1182 return 0 if CreateOrderfile(options) else 1
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1183
1184
1185if __name__ == '__main__':
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]1186 sys.exit(main())