[go: nahoru, domu]
Google Git
Sign in
chromium / chromium / src / e8400646d71d2397a3ce6b4ca6ca918b9ee4f6d7 / . / tools / cygprofile / orderfile_generator_backend.py
blob: be4d52dd1075e1724edf765d1deae51ab17a8f55 [file] [log] [blame]
Egor Pasko0462e852d2018-03-29 15:52:09[diff] [blame]1#!/usr/bin/env vpython
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]2# Copyright (c) 2013 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6""" A utility to generate an up-to-date orderfile.
7
8The orderfile is used by the linker to order text sections such that the
9sections are placed consecutively in the order specified. This allows us
10to page in less code during start-up.
11
12Example usage:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]13 tools/cygprofile/orderfile_generator_backend.py -l 20 -j 1000 --use-goma \
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]14 --target-arch=arm
15"""
16
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]17import argparse
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]18import hashlib
19import json
20import logging
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]21import os
22import re
23import shutil
24import subprocess
25import sys
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]26import tempfile
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]27import time
28
Matthew Carye8400642018-06-14 15:43:02[diff] [blame^]29import cyglog_to_orderfile
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]30import cygprofile_utils
Benoit Lizefefbb27c2018-01-17 13:54:18[diff] [blame]31import patch_orderfile
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]32import process_profiles
Egor Pasko3bc0b932018-04-03 10:08:27[diff] [blame]33import profile_android_startup
Benoit Lizefefbb27c2018-01-17 13:54:18[diff] [blame]34import symbol_extractor
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]35
36
37_SRC_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)),
38 os.pardir, os.pardir)
39sys.path.append(os.path.join(_SRC_PATH, 'build', 'android'))
40import devil_chromium
41from pylib import constants
42
43
44# Needs to happen early for GetBuildType()/GetOutDirectory() to work correctly
45constants.SetBuildType('Release')
46
47
48class CommandError(Exception):
49 """Indicates that a dispatched shell command exited with a non-zero status."""
50
51 def __init__(self, value):
52 super(CommandError, self).__init__()
53 self.value = value
54
55 def __str__(self):
56 return repr(self.value)
57
58
59def _GenerateHash(file_path):
60 """Calculates and returns the hash of the file at file_path."""
61 sha1 = hashlib.sha1()
62 with open(file_path, 'rb') as f:
63 while True:
64 # Read in 1mb chunks, so it doesn't all have to be loaded into memory.
65 chunk = f.read(1024 * 1024)
66 if not chunk:
67 break
68 sha1.update(chunk)
69 return sha1.hexdigest()
70
71
72def _GetFileExtension(file_name):
73 """Calculates the file extension from a file name.
74
75 Args:
76 file_name: The source file name.
77 Returns:
78 The part of file_name after the dot (.) or None if the file has no
79 extension.
80 Examples: /home/user/foo.bar -> bar
81 /home/user.name/foo -> None
82 /home/user/.foo -> None
83 /home/user/foo.bar.baz -> baz
84 """
85 file_name_parts = os.path.basename(file_name).split('.')
86 if len(file_name_parts) > 1:
87 return file_name_parts[-1]
88 else:
89 return None
90
91
92def _StashOutputDirectory(buildpath):
93 """Takes the output directory and stashes it in the default output directory.
94
95 This allows it to be used for incremental builds next time (after unstashing)
96 by keeping it in a place that isn't deleted normally, while also ensuring
97 that it is properly clobbered when appropriate.
98
99 This is a dirty hack to deal with the needs of clobbering while also handling
100 incremental builds and the hardcoded relative paths used in some of the
101 project files.
102
103 Args:
104 buildpath: The path where the building happens. If this corresponds to the
105 default output directory, no action is taken.
106 """
107 if os.path.abspath(buildpath) == os.path.abspath(os.path.dirname(
108 constants.GetOutDirectory())):
109 return
110 name = os.path.basename(buildpath)
111 stashpath = os.path.join(constants.GetOutDirectory(), name)
112 if not os.path.exists(buildpath):
113 return
114 if os.path.exists(stashpath):
115 shutil.rmtree(stashpath, ignore_errors=True)
116 shutil.move(buildpath, stashpath)
117
118
119def _UnstashOutputDirectory(buildpath):
120 """Inverse of _StashOutputDirectory.
121
122 Moves the output directory stashed within the default output directory
123 (out/Release) to the position where the builds can actually happen.
124
125 This is a dirty hack to deal with the needs of clobbering while also handling
126 incremental builds and the hardcoded relative paths used in some of the
127 project files.
128
129 Args:
130 buildpath: The path where the building happens. If this corresponds to the
131 default output directory, no action is taken.
132 """
133 if os.path.abspath(buildpath) == os.path.abspath(os.path.dirname(
134 constants.GetOutDirectory())):
135 return
136 name = os.path.basename(buildpath)
137 stashpath = os.path.join(constants.GetOutDirectory(), name)
138 if not os.path.exists(stashpath):
139 return
140 if os.path.exists(buildpath):
141 shutil.rmtree(buildpath, ignore_errors=True)
142 shutil.move(stashpath, buildpath)
143
144
145class StepRecorder(object):
146 """Records steps and timings."""
147
148 def __init__(self, buildbot):
149 self.timings = []
150 self._previous_step = ('', 0.0)
151 self._buildbot = buildbot
152 self._error_recorded = False
153
154 def BeginStep(self, name):
155 """Marks a beginning of the next step in the script.
156
157 On buildbot, this prints a specially formatted name that will show up
158 in the waterfall. Otherwise, just prints the step name.
159
160 Args:
161 name: The name of the step.
162 """
163 self.EndStep()
164 self._previous_step = (name, time.time())
165 print 'Running step: ', name
166
167 def EndStep(self):
168 """Records successful completion of the current step.
169
170 This is optional if the step is immediately followed by another BeginStep.
171 """
172 if self._previous_step[0]:
173 elapsed = time.time() - self._previous_step[1]
174 print 'Step %s took %f seconds' % (self._previous_step[0], elapsed)
175 self.timings.append((self._previous_step[0], elapsed))
176
177 self._previous_step = ('', 0.0)
178
179 def FailStep(self, message=None):
180 """Marks that a particular step has failed.
181
182 On buildbot, this will mark the current step as failed on the waterfall.
183 Otherwise we will just print an optional failure message.
184
185 Args:
186 message: An optional explanation as to why the step failed.
187 """
188 print 'STEP FAILED!!'
189 if message:
190 print message
191 self._error_recorded = True
192 self.EndStep()
193
194 def ErrorRecorded(self):
195 """True if FailStep has been called."""
196 return self._error_recorded
197
198 def RunCommand(self, cmd, cwd=constants.DIR_SOURCE_ROOT, raise_on_error=True,
199 stdout=None):
200 """Execute a shell command.
201
202 Args:
203 cmd: A list of command strings.
204 cwd: Directory in which the command should be executed, defaults to script
205 location if not specified.
206 raise_on_error: If true will raise a CommandError if the call doesn't
207 succeed and mark the step as failed.
208 stdout: A file to redirect stdout for the command to.
209
210 Returns:
211 The process's return code.
212
213 Raises:
214 CommandError: An error executing the specified command.
215 """
216 print 'Executing %s in %s' % (' '.join(cmd), cwd)
217 process = subprocess.Popen(cmd, stdout=stdout, cwd=cwd, env=os.environ)
218 process.wait()
219 if raise_on_error and process.returncode != 0:
220 self.FailStep()
221 raise CommandError('Exception executing command %s' % ' '.join(cmd))
222 return process.returncode
223
224
225class ClankCompiler(object):
226 """Handles compilation of clank."""
227
228 def __init__(self, out_dir, step_recorder, arch, jobs, max_load, use_goma,
Benoit L96466812018-03-06 15:58:37[diff] [blame]229 goma_dir):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]230 self._out_dir = out_dir
231 self._step_recorder = step_recorder
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]232 self._arch = arch
233 self._jobs = jobs
234 self._max_load = max_load
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]235 self._use_goma = use_goma
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]236 self._goma_dir = goma_dir
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]237 lib_chrome_so_dir = 'lib.unstripped'
238 self.lib_chrome_so = os.path.join(
239 self._out_dir, 'Release', lib_chrome_so_dir, 'libchrome.so')
240 self.chrome_apk = os.path.join(
241 self._out_dir, 'Release', 'apks', 'Chrome.apk')
242
243 def Build(self, instrumented, target):
244 """Builds the provided ninja target with or without order_profiling on.
245
246 Args:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]247 instrumented: (bool) Whether we want to build an instrumented binary.
248 target: (str) The name of the ninja target to build.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]249 """
250 self._step_recorder.BeginStep('Compile %s' % target)
251
252 # Set the "Release Official" flavor, the parts affecting performance.
253 args = [
254 'is_chrome_branded=true',
255 'is_debug=false',
256 'is_official_build=true',
257 # We have to build with no symbols if profiling and minimal symbols
258 # otherwise for libchrome.so to fit under the 4 GB limit.
259 # crbug.com/574476
260 'symbol_level=' + ('0' if instrumented else '1'),
261 'target_cpu="' + self._arch + '"',
262 'target_os="android"',
263 'use_goma=' + str(self._use_goma).lower(),
264 'use_order_profiling=' + str(instrumented).lower(),
265 ]
266 if self._goma_dir:
267 args += ['goma_dir="%s"' % self._goma_dir]
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]268
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]269 self._step_recorder.RunCommand(
270 ['gn', 'gen', os.path.join(self._out_dir, 'Release'),
271 '--args=' + ' '.join(args)])
272
273 self._step_recorder.RunCommand(
274 ['ninja', '-C', os.path.join(self._out_dir, 'Release'),
275 '-j' + str(self._jobs), '-l' + str(self._max_load), target])
276
277 def CompileChromeApk(self, instrumented, force_relink=False):
278 """Builds a Chrome.apk either with or without order_profiling on.
279
280 Args:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]281 instrumented: (bool) Whether to build an instrumented apk.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]282 force_relink: Whether libchromeview.so should be re-created.
283 """
284 if force_relink:
285 self._step_recorder.RunCommand(['rm', '-rf', self.lib_chrome_so])
286 self.Build(instrumented, 'chrome_apk')
287
288 def CompileLibchrome(self, instrumented, force_relink=False):
289 """Builds a libchrome.so either with or without order_profiling on.
290
291 Args:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]292 instrumented: (bool) Whether to build an instrumented apk.
293 force_relink: (bool) Whether libchrome.so should be re-created.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]294 """
295 if force_relink:
296 self._step_recorder.RunCommand(['rm', '-rf', self.lib_chrome_so])
297 self.Build(instrumented, 'libchrome')
298
299
300class OrderfileUpdater(object):
301 """Handles uploading and committing a new orderfile in the repository.
302
303 Only used for testing or on a bot.
304 """
305
306 _CLOUD_STORAGE_BUCKET_FOR_DEBUG = None
307 _CLOUD_STORAGE_BUCKET = None
308 _UPLOAD_TO_CLOUD_COMMAND = 'upload_to_google_storage.py'
309
310 def __init__(self, repository_root, step_recorder, branch, netrc):
311 """Constructor.
312
313 Args:
314 repository_root: (str) Root of the target repository.
315 step_recorder: (StepRecorder) Step recorder, for logging.
316 branch: (str) Branch to commit to.
317 netrc: (str) Path to the .netrc file to use.
318 """
319 self._repository_root = repository_root
320 self._step_recorder = step_recorder
321 self._branch = branch
322 self._netrc = netrc
323
324 def CommitFileHashes(self, unpatched_orderfile_filename, orderfile_filename):
325 """Commits unpatched and patched orderfiles hashes, if provided.
326
327 Files must have been successfilly uploaded to cloud storage first.
328
329 Args:
330 unpatched_orderfile_filename: (str or None) Unpatched orderfile path.
331 orderfile_filename: (str or None) Orderfile path.
332
333 Raises:
334 NotImplementedError when the commit logic hasn't been overriden.
335 """
336 files_to_commit = []
337 commit_message_lines = ['Update Orderfile.']
338 for filename in [unpatched_orderfile_filename, orderfile_filename]:
339 if not filename:
340 continue
341 (relative_path, sha1) = self._GetHashFilePathAndContents(filename)
342 commit_message_lines.append('Profile: %s: %s' % (
343 os.path.basename(relative_path), sha1))
344 files_to_commit.append(relative_path)
345 if files_to_commit:
346 self._CommitFiles(files_to_commit, commit_message_lines)
347
348 def UploadToCloudStorage(self, filename, use_debug_location):
349 """Uploads a file to cloud storage.
350
351 Args:
352 filename: (str) File to upload.
353 use_debug_location: (bool) Whether to use the debug location.
354 """
355 bucket = (self._CLOUD_STORAGE_BUCKET_FOR_DEBUG if use_debug_location
356 else self._CLOUD_STORAGE_BUCKET)
357 extension = _GetFileExtension(filename)
358 cmd = [self._UPLOAD_TO_CLOUD_COMMAND, '--bucket', bucket]
359 if extension:
360 cmd.extend(['-z', extension])
361 cmd.append(filename)
362 self._step_recorder.RunCommand(cmd)
363 print 'Download: https://sandbox.google.com/storage/%s/%s' % (
364 bucket, _GenerateHash(filename))
365
366 def _GetHashFilePathAndContents(self, filename):
367 """Gets the name and content of the hash file created from uploading the
368 given file.
369
370 Args:
371 filename: (str) The file that was uploaded to cloud storage.
372
373 Returns:
374 A tuple of the hash file name, relative to the reository root, and the
375 content, which should be the sha1 hash of the file
376 ('base_file.sha1', hash)
377 """
378 abs_hash_filename = filename + '.sha1'
379 rel_hash_filename = os.path.relpath(
380 abs_hash_filename, self._repository_root)
381 with open(abs_hash_filename, 'r') as f:
382 return (rel_hash_filename, f.read())
383
384 def _CommitFiles(self, files_to_commit, commit_message_lines):
385 """Commits a list of files, with a given message."""
386 raise NotImplementedError
387
388
389class OrderfileGenerator(object):
390 """A utility for generating a new orderfile for Clank.
391
392 Builds an instrumented binary, profiles a run of the application, and
393 generates an updated orderfile.
394 """
395 _CLANK_REPO = os.path.join(constants.DIR_SOURCE_ROOT, 'clank')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]396 _CYGLOG_TO_ORDERFILE_SCRIPT = os.path.join(
397 constants.DIR_SOURCE_ROOT, 'tools', 'cygprofile',
398 'cyglog_to_orderfile.py')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]399 _CHECK_ORDERFILE_SCRIPT = os.path.join(
400 constants.DIR_SOURCE_ROOT, 'tools', 'cygprofile', 'check_orderfile.py')
401 _BUILD_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(
402 constants.GetOutDirectory()))) # Normally /path/to/src
403
404 _UNPATCHED_ORDERFILE_FILENAME = os.path.join(
405 _CLANK_REPO, 'orderfiles', 'unpatched_orderfile.%s')
406 _MERGED_CYGLOG_FILENAME = os.path.join(
407 constants.GetOutDirectory(), 'merged_cyglog')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]408
409 _PATH_TO_ORDERFILE = os.path.join(_CLANK_REPO, 'orderfiles',
410 'orderfile.%s.out')
411
412 # Previous orderfile_generator debug files would be overwritten.
413 _DIRECTORY_FOR_DEBUG_FILES = '/tmp/orderfile_generator_debug_files'
414
415 def _GetPathToOrderfile(self):
416 """Gets the path to the architecture-specific orderfile."""
417 return self._PATH_TO_ORDERFILE % self._options.arch
418
419 def _GetUnpatchedOrderfileFilename(self):
420 """Gets the path to the architecture-specific unpatched orderfile."""
421 return self._UNPATCHED_ORDERFILE_FILENAME % self._options.arch
422
423 def __init__(self, options, orderfile_updater_class):
424 self._options = options
425
426 self._instrumented_out_dir = os.path.join(
427 self._BUILD_ROOT, self._options.arch + '_instrumented_out')
428 self._uninstrumented_out_dir = os.path.join(
429 self._BUILD_ROOT, self._options.arch + '_uninstrumented_out')
430
431 if options.profile:
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]432 output_directory = os.path.join(self._instrumented_out_dir, 'Release')
Matthew Caryb8daed942018-06-11 10:58:08[diff] [blame]433 host_profile_dir = os.path.join(output_directory, 'profile_data')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]434 urls = [profile_android_startup.AndroidProfileTool.TEST_URL]
435 use_wpr = True
436 simulate_user = False
Benoit L96466812018-03-06 15:58:37[diff] [blame]437 urls = options.urls
438 use_wpr = not options.no_wpr
439 simulate_user = options.simulate_user
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]440 self._profiler = profile_android_startup.AndroidProfileTool(
Matthew Caryb8daed942018-06-11 10:58:08[diff] [blame]441 output_directory, host_profile_dir, use_wpr, urls, simulate_user)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]442
443 self._output_data = {}
444 self._step_recorder = StepRecorder(options.buildbot)
445 self._compiler = None
446 assert issubclass(orderfile_updater_class, OrderfileUpdater)
447 self._orderfile_updater = orderfile_updater_class(self._CLANK_REPO,
448 self._step_recorder,
449 options.branch,
450 options.netrc)
451 assert os.path.isdir(constants.DIR_SOURCE_ROOT), 'No src directory found'
Benoit Lizefefbb27c2018-01-17 13:54:18[diff] [blame]452 symbol_extractor.SetArchitecture(options.arch)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]453
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]454 @staticmethod
455 def _RemoveBlanks(src_file, dest_file):
456 """A utility to remove blank lines from a file.
457
458 Args:
459 src_file: The name of the file to remove the blanks from.
460 dest_file: The name of the file to write the output without blanks.
461 """
462 assert src_file != dest_file, 'Source and destination need to be distinct'
463
464 try:
465 src = open(src_file, 'r')
466 dest = open(dest_file, 'w')
467 for line in src:
468 if line and not line.isspace():
469 dest.write(line)
470 finally:
471 src.close()
472 dest.close()
473
474 def _GenerateAndProcessProfile(self):
475 """Invokes a script to merge the per-thread traces into one file."""
476 self._step_recorder.BeginStep('Generate Profile Data')
477 files = []
478 try:
479 logging.getLogger().setLevel(logging.DEBUG)
480 files = self._profiler.CollectProfile(
481 self._compiler.chrome_apk,
482 constants.PACKAGE_INFO['chrome'])
Matthew Caryb8daed942018-06-11 10:58:08[diff] [blame]483 self._step_recorder.BeginStep('Process profile')
Benoit L96466812018-03-06 15:58:37[diff] [blame]484 assert os.path.exists(self._compiler.lib_chrome_so)
485 offsets = process_profiles.GetReachedOffsetsFromDumpFiles(
486 files, self._compiler.lib_chrome_so)
487 if not offsets:
488 raise Exception('No profiler offsets found in {}'.format(
489 '\n'.join(files)))
490 with open(self._MERGED_CYGLOG_FILENAME, 'w') as f:
491 f.write('\n'.join(map(str, offsets)))
Matthew Cary0f1f681a2018-01-22 10:40:51[diff] [blame]492 except Exception:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]493 for f in files:
494 self._SaveForDebugging(f)
495 raise
496 finally:
497 self._profiler.Cleanup()
498 logging.getLogger().setLevel(logging.INFO)
499
500 try:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]501 command_args = [
502 '--target-arch=' + self._options.arch,
503 '--native-library=' + self._compiler.lib_chrome_so,
504 '--output=' + self._GetUnpatchedOrderfileFilename()]
Benoit L96466812018-03-06 15:58:37[diff] [blame]505 command_args.append('--reached-offsets=' + self._MERGED_CYGLOG_FILENAME)
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]506 self._step_recorder.RunCommand(
507 [self._CYGLOG_TO_ORDERFILE_SCRIPT] + command_args)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]508 except CommandError:
509 self._SaveForDebugging(self._MERGED_CYGLOG_FILENAME)
510 self._SaveForDebuggingWithOverwrite(self._compiler.lib_chrome_so)
511 raise
512
513 def _DeleteTempFiles(self):
514 """Deletes intermediate step output files."""
515 print 'Delete %s' % (
516 self._MERGED_CYGLOG_FILENAME)
517 if os.path.isfile(self._MERGED_CYGLOG_FILENAME):
518 os.unlink(self._MERGED_CYGLOG_FILENAME)
519
520 def _PatchOrderfile(self):
521 """Patches the orderfile using clean version of libchrome.so."""
522 self._step_recorder.BeginStep('Patch Orderfile')
Benoit Lizefefbb27c2018-01-17 13:54:18[diff] [blame]523 patch_orderfile.GeneratePatchedOrderfile(
524 self._GetUnpatchedOrderfileFilename(), self._compiler.lib_chrome_so,
525 self._GetPathToOrderfile())
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]526
527 def _VerifySymbolOrder(self):
528 self._step_recorder.BeginStep('Verify Symbol Order')
529 return_code = self._step_recorder.RunCommand(
530 [self._CHECK_ORDERFILE_SCRIPT, self._compiler.lib_chrome_so,
531 self._GetPathToOrderfile(),
532 '--target-arch=' + self._options.arch],
533 constants.DIR_SOURCE_ROOT,
534 raise_on_error=False)
535 if return_code:
536 self._step_recorder.FailStep('Orderfile check returned %d.' % return_code)
537
538 def _RecordHash(self, file_name):
539 """Records the hash of the file into the output_data dictionary."""
540 self._output_data[os.path.basename(file_name) + '.sha1'] = _GenerateHash(
541 file_name)
542
543 def _SaveFileLocally(self, file_name, file_sha1):
544 """Saves the file to a temporary location and prints the sha1sum."""
545 if not os.path.exists(self._DIRECTORY_FOR_DEBUG_FILES):
546 os.makedirs(self._DIRECTORY_FOR_DEBUG_FILES)
547 shutil.copy(file_name, self._DIRECTORY_FOR_DEBUG_FILES)
548 print 'File: %s, saved in: %s, sha1sum: %s' % (
549 file_name, self._DIRECTORY_FOR_DEBUG_FILES, file_sha1)
550
551 def _SaveForDebugging(self, filename):
552 """Uploads the file to cloud storage or saves to a temporary location."""
553 file_sha1 = _GenerateHash(filename)
554 if not self._options.buildbot:
555 self._SaveFileLocally(filename, file_sha1)
556 else:
557 print 'Uploading file for debugging: ' + filename
558 self._orderfile_updater.UploadToCloudStorage(
559 filename, use_debug_location=True)
560
561 def _SaveForDebuggingWithOverwrite(self, file_name):
562 """Uploads and overwrites the file in cloud storage or copies locally.
563
564 Should be used for large binaries like lib_chrome_so.
565
566 Args:
567 file_name: (str) File to upload.
568 """
569 file_sha1 = _GenerateHash(file_name)
570 if not self._options.buildbot:
571 self._SaveFileLocally(file_name, file_sha1)
572 else:
573 print 'Uploading file for debugging: %s, sha1sum: %s' % (
574 file_name, file_sha1)
575 upload_location = '%s/%s' % (
576 self._CLOUD_STORAGE_BUCKET_FOR_DEBUG, os.path.basename(file_name))
577 self._step_recorder.RunCommand([
578 'gsutil.py', 'cp', file_name, 'gs://' + upload_location])
579 print ('Uploaded to: https://sandbox.google.com/storage/' +
580 upload_location)
581
582 def _MaybeArchiveOrderfile(self, filename):
583 """In buildbot configuration, uploads the generated orderfile to
584 Google Cloud Storage.
585
586 Args:
587 filename: (str) Orderfile to upload.
588 """
589 # First compute hashes so that we can download them later if we need to
590 self._step_recorder.BeginStep('Compute hash for ' + filename)
591 self._RecordHash(filename)
592 if self._options.buildbot:
593 self._step_recorder.BeginStep('Archive ' + filename)
594 self._orderfile_updater.UploadToCloudStorage(
595 filename, use_debug_location=False)
596
597 def _GetHashFilePathAndContents(self, base_file):
598 """Gets the name and content of the hash file created from uploading the
599 given file.
600
601 Args:
602 base_file: The file that was uploaded to cloud storage.
603
604 Returns:
605 A tuple of the hash file name, relative to the clank repo path, and the
606 content, which should be the sha1 hash of the file
607 ('base_file.sha1', hash)
608 """
609 abs_file_name = base_file + '.sha1'
610 rel_file_name = os.path.relpath(abs_file_name, self._CLANK_REPO)
611 with open(abs_file_name, 'r') as f:
612 return (rel_file_name, f.read())
613
614 def Generate(self):
615 """Generates and maybe upload an order."""
616 profile_uploaded = False
617 orderfile_uploaded = False
618
Matthew Carye8400642018-06-14 15:43:02[diff] [blame^]619 assert (bool(self._options.profile) ^
620 bool(self._options.manual_symbol_offsets))
621
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]622 if self._options.profile:
623 try:
624 _UnstashOutputDirectory(self._instrumented_out_dir)
625 self._compiler = ClankCompiler(
626 self._instrumented_out_dir,
627 self._step_recorder, self._options.arch, self._options.jobs,
628 self._options.max_load, self._options.use_goma,
Benoit L96466812018-03-06 15:58:37[diff] [blame]629 self._options.goma_dir)
Benoit L96466812018-03-06 15:58:37[diff] [blame]630 self._compiler.CompileChromeApk(True)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]631 self._GenerateAndProcessProfile()
632 self._MaybeArchiveOrderfile(self._GetUnpatchedOrderfileFilename())
633 profile_uploaded = True
634 finally:
635 self._DeleteTempFiles()
636 _StashOutputDirectory(self._instrumented_out_dir)
Matthew Carye8400642018-06-14 15:43:02[diff] [blame^]637 elif self._options.manual_symbol_offsets:
638 assert self._options.manual_libname
639 assert self._options.manual_objdir
640 with file(self._options.manual_symbol_offsets) as f:
641 symbol_offsets = [int(x) for x in f.xreadlines()]
642 processor = process_profiles.SymbolOffsetProcessor(
643 self._options.manual_libname)
644 generator = cyglog_to_orderfile.OffsetOrderfileGenerator(
645 processor, cyglog_to_orderfile.ObjectFileProcessor(
646 self._options.manual_objdir))
647 ordered_sections = generator.GetOrderedSections(symbol_offsets)
648 if not ordered_sections: # Either None or empty is a problem.
649 raise Exception('Failed to get ordered sections')
650 with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile:
651 orderfile.write('\n'.join(ordered_sections))
652
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]653 if self._options.patch:
654 if self._options.profile:
655 self._RemoveBlanks(self._GetUnpatchedOrderfileFilename(),
656 self._GetPathToOrderfile())
657 try:
658 _UnstashOutputDirectory(self._uninstrumented_out_dir)
659 self._compiler = ClankCompiler(
660 self._uninstrumented_out_dir, self._step_recorder,
661 self._options.arch, self._options.jobs, self._options.max_load,
Benoit L96466812018-03-06 15:58:37[diff] [blame]662 self._options.use_goma, self._options.goma_dir)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]663 self._compiler.CompileLibchrome(False)
664 self._PatchOrderfile()
665 # Because identical code folding is a bit different with and without
666 # the orderfile build, we need to re-patch the orderfile with code
667 # folding as close to the final version as possible.
668 self._compiler.CompileLibchrome(False, force_relink=True)
669 self._PatchOrderfile()
670 self._compiler.CompileLibchrome(False, force_relink=True)
671 self._VerifySymbolOrder()
672 self._MaybeArchiveOrderfile(self._GetPathToOrderfile())
673 finally:
674 _StashOutputDirectory(self._uninstrumented_out_dir)
675 orderfile_uploaded = True
676
677 if (self._options.buildbot and self._options.netrc
678 and not self._step_recorder.ErrorRecorded()):
679 unpatched_orderfile_filename = (
680 self._GetUnpatchedOrderfileFilename() if profile_uploaded else None)
681 orderfile_filename = (
Benoit Lized913be82017-10-24 13:38:27[diff] [blame]682 self._GetPathToOrderfile() if orderfile_uploaded else None)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]683 self._orderfile_updater.CommitFileHashes(
684 unpatched_orderfile_filename, orderfile_filename)
685
686 self._step_recorder.EndStep()
687 return not self._step_recorder.ErrorRecorded()
688
689 def GetReportingData(self):
690 """Get a dictionary of reporting data (timings, output hashes)"""
691 self._output_data['timings'] = self._step_recorder.timings
692 return self._output_data
693
694
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]695def CreateArgumentParser():
696 """Creates and returns the argument parser."""
697 parser = argparse.ArgumentParser()
698 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]699 '--buildbot', action='store_true',
700 help='If true, the script expects to be run on a buildbot')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]701 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]702 '--verify', action='store_true',
703 help='If true, the script only verifies the current orderfile')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]704 parser.add_argument('--target-arch', action='store', dest='arch',
705 default=cygprofile_utils.DetectArchitecture(),
706 choices=['arm', 'arm64', 'x86', 'x86_64', 'x64', 'mips'],
707 help='The target architecture for which to build')
708 parser.add_argument('--output-json', action='store', dest='json_file',
709 help='Location to save stats in json format')
710 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]711 '--skip-profile', action='store_false', dest='profile', default=True,
712 help='Don\'t generate a profile on the device. Only patch from the '
713 'existing profile.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]714 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]715 '--skip-patch', action='store_false', dest='patch', default=True,
716 help='Only generate the raw (unpatched) orderfile, don\'t patch it.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]717 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]718 '--netrc', action='store',
719 help='A custom .netrc file to use for git checkin. Only used on bots.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]720 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]721 '--branch', action='store', default='master',
722 help='When running on buildbot with a netrc, the branch orderfile '
723 'hashes get checked into.')
724 # Note: -j50 was causing issues on the bot.
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]725 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]726 '-j', '--jobs', action='store', default=20,
727 help='Number of jobs to use for compilation.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]728 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]729 '-l', '--max-load', action='store', default=4, help='Max cpu load.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]730 parser.add_argument('--goma-dir', help='GOMA directory.')
731 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]732 '--use-goma', action='store_true', help='Enable GOMA.', default=False)
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]733 parser.add_argument('--adb-path', help='Path to the adb binary.')
Matthew Carye8400642018-06-14 15:43:02[diff] [blame^]734
735 parser.add_argument('--manual-symbol-offsets', default=None, type=str,
736 help=('File of list of ordered symbol offsets generated '
737 'by manual profiling. Must set other --manual* '
738 'flags if this is used, and must --skip-profile.'))
739 parser.add_argument('--manual-libname', default=None, type=str,
740 help=('Library filename corresponding to '
741 '--manual-symbol-offsets.'))
742 parser.add_argument('--manual-objdir', default=None, type=str,
743 help=('Root of object file directory corresponding to '
744 '--manual-symbol-offsets.'))
745
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]746 profile_android_startup.AddProfileCollectionArguments(parser)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]747 return parser
748
749
750def CreateOrderfile(options, orderfile_updater_class):
751 """Creates an oderfile.
752
753 Args:
754 options: As returned from optparse.OptionParser.parse_args()
755 orderfile_updater_class: (OrderfileUpdater) subclass of OrderfileUpdater.
756
757 Returns:
758 True iff success.
759 """
760 logging.basicConfig(level=logging.INFO)
761 devil_chromium.Initialize(adb_path=options.adb_path)
762
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]763 generator = OrderfileGenerator(options, orderfile_updater_class)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]764 try:
765 if options.verify:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]766 generator._VerifySymbolOrder()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]767 else:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]768 return generator.Generate()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]769 finally:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]770 json_output = json.dumps(generator.GetReportingData(),
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]771 indent=2) + '\n'
772 if options.json_file:
773 with open(options.json_file, 'w') as f:
774 f.write(json_output)
775 print json_output
776 return False
777
778
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]779def main():
780 parser = CreateArgumentParser()
781 options = parser.parse_args()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]782 return 0 if CreateOrderfile(options, OrderfileUpdater) else 1
783
784
785if __name__ == '__main__':
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]786 sys.exit(main())