[go: nahoru, domu]
Google Git
Sign in
chromium / chromium / src / b8daed94b89de7ec0fa503397241f0dd83ce3695 / . / tools / cygprofile / orderfile_generator_backend.py
blob: 129cc7cdbfd82ba0187872128ef27852510d71f6 [file] [log] [blame]
Egor Pasko0462e852d2018-03-29 15:52:09[diff] [blame]1#!/usr/bin/env vpython
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]2# Copyright (c) 2013 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6""" A utility to generate an up-to-date orderfile.
7
8The orderfile is used by the linker to order text sections such that the
9sections are placed consecutively in the order specified. This allows us
10to page in less code during start-up.
11
12Example usage:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]13 tools/cygprofile/orderfile_generator_backend.py -l 20 -j 1000 --use-goma \
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]14 --target-arch=arm
15"""
16
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]17import argparse
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]18import hashlib
19import json
20import logging
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]21import os
22import re
23import shutil
24import subprocess
25import sys
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]26import tempfile
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]27import time
28
29import cygprofile_utils
Benoit Lizefefbb27c2018-01-17 13:54:18[diff] [blame]30import patch_orderfile
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]31import process_profiles
Egor Pasko3bc0b932018-04-03 10:08:27[diff] [blame]32import profile_android_startup
Benoit Lizefefbb27c2018-01-17 13:54:18[diff] [blame]33import symbol_extractor
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]34
35
36_SRC_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)),
37 os.pardir, os.pardir)
38sys.path.append(os.path.join(_SRC_PATH, 'build', 'android'))
39import devil_chromium
40from pylib import constants
41
42
43# Needs to happen early for GetBuildType()/GetOutDirectory() to work correctly
44constants.SetBuildType('Release')
45
46
47class CommandError(Exception):
48 """Indicates that a dispatched shell command exited with a non-zero status."""
49
50 def __init__(self, value):
51 super(CommandError, self).__init__()
52 self.value = value
53
54 def __str__(self):
55 return repr(self.value)
56
57
58def _GenerateHash(file_path):
59 """Calculates and returns the hash of the file at file_path."""
60 sha1 = hashlib.sha1()
61 with open(file_path, 'rb') as f:
62 while True:
63 # Read in 1mb chunks, so it doesn't all have to be loaded into memory.
64 chunk = f.read(1024 * 1024)
65 if not chunk:
66 break
67 sha1.update(chunk)
68 return sha1.hexdigest()
69
70
71def _GetFileExtension(file_name):
72 """Calculates the file extension from a file name.
73
74 Args:
75 file_name: The source file name.
76 Returns:
77 The part of file_name after the dot (.) or None if the file has no
78 extension.
79 Examples: /home/user/foo.bar -> bar
80 /home/user.name/foo -> None
81 /home/user/.foo -> None
82 /home/user/foo.bar.baz -> baz
83 """
84 file_name_parts = os.path.basename(file_name).split('.')
85 if len(file_name_parts) > 1:
86 return file_name_parts[-1]
87 else:
88 return None
89
90
91def _StashOutputDirectory(buildpath):
92 """Takes the output directory and stashes it in the default output directory.
93
94 This allows it to be used for incremental builds next time (after unstashing)
95 by keeping it in a place that isn't deleted normally, while also ensuring
96 that it is properly clobbered when appropriate.
97
98 This is a dirty hack to deal with the needs of clobbering while also handling
99 incremental builds and the hardcoded relative paths used in some of the
100 project files.
101
102 Args:
103 buildpath: The path where the building happens. If this corresponds to the
104 default output directory, no action is taken.
105 """
106 if os.path.abspath(buildpath) == os.path.abspath(os.path.dirname(
107 constants.GetOutDirectory())):
108 return
109 name = os.path.basename(buildpath)
110 stashpath = os.path.join(constants.GetOutDirectory(), name)
111 if not os.path.exists(buildpath):
112 return
113 if os.path.exists(stashpath):
114 shutil.rmtree(stashpath, ignore_errors=True)
115 shutil.move(buildpath, stashpath)
116
117
118def _UnstashOutputDirectory(buildpath):
119 """Inverse of _StashOutputDirectory.
120
121 Moves the output directory stashed within the default output directory
122 (out/Release) to the position where the builds can actually happen.
123
124 This is a dirty hack to deal with the needs of clobbering while also handling
125 incremental builds and the hardcoded relative paths used in some of the
126 project files.
127
128 Args:
129 buildpath: The path where the building happens. If this corresponds to the
130 default output directory, no action is taken.
131 """
132 if os.path.abspath(buildpath) == os.path.abspath(os.path.dirname(
133 constants.GetOutDirectory())):
134 return
135 name = os.path.basename(buildpath)
136 stashpath = os.path.join(constants.GetOutDirectory(), name)
137 if not os.path.exists(stashpath):
138 return
139 if os.path.exists(buildpath):
140 shutil.rmtree(buildpath, ignore_errors=True)
141 shutil.move(stashpath, buildpath)
142
143
144class StepRecorder(object):
145 """Records steps and timings."""
146
147 def __init__(self, buildbot):
148 self.timings = []
149 self._previous_step = ('', 0.0)
150 self._buildbot = buildbot
151 self._error_recorded = False
152
153 def BeginStep(self, name):
154 """Marks a beginning of the next step in the script.
155
156 On buildbot, this prints a specially formatted name that will show up
157 in the waterfall. Otherwise, just prints the step name.
158
159 Args:
160 name: The name of the step.
161 """
162 self.EndStep()
163 self._previous_step = (name, time.time())
164 print 'Running step: ', name
165
166 def EndStep(self):
167 """Records successful completion of the current step.
168
169 This is optional if the step is immediately followed by another BeginStep.
170 """
171 if self._previous_step[0]:
172 elapsed = time.time() - self._previous_step[1]
173 print 'Step %s took %f seconds' % (self._previous_step[0], elapsed)
174 self.timings.append((self._previous_step[0], elapsed))
175
176 self._previous_step = ('', 0.0)
177
178 def FailStep(self, message=None):
179 """Marks that a particular step has failed.
180
181 On buildbot, this will mark the current step as failed on the waterfall.
182 Otherwise we will just print an optional failure message.
183
184 Args:
185 message: An optional explanation as to why the step failed.
186 """
187 print 'STEP FAILED!!'
188 if message:
189 print message
190 self._error_recorded = True
191 self.EndStep()
192
193 def ErrorRecorded(self):
194 """True if FailStep has been called."""
195 return self._error_recorded
196
197 def RunCommand(self, cmd, cwd=constants.DIR_SOURCE_ROOT, raise_on_error=True,
198 stdout=None):
199 """Execute a shell command.
200
201 Args:
202 cmd: A list of command strings.
203 cwd: Directory in which the command should be executed, defaults to script
204 location if not specified.
205 raise_on_error: If true will raise a CommandError if the call doesn't
206 succeed and mark the step as failed.
207 stdout: A file to redirect stdout for the command to.
208
209 Returns:
210 The process's return code.
211
212 Raises:
213 CommandError: An error executing the specified command.
214 """
215 print 'Executing %s in %s' % (' '.join(cmd), cwd)
216 process = subprocess.Popen(cmd, stdout=stdout, cwd=cwd, env=os.environ)
217 process.wait()
218 if raise_on_error and process.returncode != 0:
219 self.FailStep()
220 raise CommandError('Exception executing command %s' % ' '.join(cmd))
221 return process.returncode
222
223
224class ClankCompiler(object):
225 """Handles compilation of clank."""
226
227 def __init__(self, out_dir, step_recorder, arch, jobs, max_load, use_goma,
Benoit L96466812018-03-06 15:58:37[diff] [blame]228 goma_dir):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]229 self._out_dir = out_dir
230 self._step_recorder = step_recorder
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]231 self._arch = arch
232 self._jobs = jobs
233 self._max_load = max_load
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]234 self._use_goma = use_goma
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]235 self._goma_dir = goma_dir
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]236 lib_chrome_so_dir = 'lib.unstripped'
237 self.lib_chrome_so = os.path.join(
238 self._out_dir, 'Release', lib_chrome_so_dir, 'libchrome.so')
239 self.chrome_apk = os.path.join(
240 self._out_dir, 'Release', 'apks', 'Chrome.apk')
241
242 def Build(self, instrumented, target):
243 """Builds the provided ninja target with or without order_profiling on.
244
245 Args:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]246 instrumented: (bool) Whether we want to build an instrumented binary.
247 target: (str) The name of the ninja target to build.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]248 """
249 self._step_recorder.BeginStep('Compile %s' % target)
250
251 # Set the "Release Official" flavor, the parts affecting performance.
252 args = [
253 'is_chrome_branded=true',
254 'is_debug=false',
255 'is_official_build=true',
256 # We have to build with no symbols if profiling and minimal symbols
257 # otherwise for libchrome.so to fit under the 4 GB limit.
258 # crbug.com/574476
259 'symbol_level=' + ('0' if instrumented else '1'),
260 'target_cpu="' + self._arch + '"',
261 'target_os="android"',
262 'use_goma=' + str(self._use_goma).lower(),
263 'use_order_profiling=' + str(instrumented).lower(),
264 ]
265 if self._goma_dir:
266 args += ['goma_dir="%s"' % self._goma_dir]
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]267
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]268 self._step_recorder.RunCommand(
269 ['gn', 'gen', os.path.join(self._out_dir, 'Release'),
270 '--args=' + ' '.join(args)])
271
272 self._step_recorder.RunCommand(
273 ['ninja', '-C', os.path.join(self._out_dir, 'Release'),
274 '-j' + str(self._jobs), '-l' + str(self._max_load), target])
275
276 def CompileChromeApk(self, instrumented, force_relink=False):
277 """Builds a Chrome.apk either with or without order_profiling on.
278
279 Args:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]280 instrumented: (bool) Whether to build an instrumented apk.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]281 force_relink: Whether libchromeview.so should be re-created.
282 """
283 if force_relink:
284 self._step_recorder.RunCommand(['rm', '-rf', self.lib_chrome_so])
285 self.Build(instrumented, 'chrome_apk')
286
287 def CompileLibchrome(self, instrumented, force_relink=False):
288 """Builds a libchrome.so either with or without order_profiling on.
289
290 Args:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]291 instrumented: (bool) Whether to build an instrumented apk.
292 force_relink: (bool) Whether libchrome.so should be re-created.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]293 """
294 if force_relink:
295 self._step_recorder.RunCommand(['rm', '-rf', self.lib_chrome_so])
296 self.Build(instrumented, 'libchrome')
297
298
299class OrderfileUpdater(object):
300 """Handles uploading and committing a new orderfile in the repository.
301
302 Only used for testing or on a bot.
303 """
304
305 _CLOUD_STORAGE_BUCKET_FOR_DEBUG = None
306 _CLOUD_STORAGE_BUCKET = None
307 _UPLOAD_TO_CLOUD_COMMAND = 'upload_to_google_storage.py'
308
309 def __init__(self, repository_root, step_recorder, branch, netrc):
310 """Constructor.
311
312 Args:
313 repository_root: (str) Root of the target repository.
314 step_recorder: (StepRecorder) Step recorder, for logging.
315 branch: (str) Branch to commit to.
316 netrc: (str) Path to the .netrc file to use.
317 """
318 self._repository_root = repository_root
319 self._step_recorder = step_recorder
320 self._branch = branch
321 self._netrc = netrc
322
323 def CommitFileHashes(self, unpatched_orderfile_filename, orderfile_filename):
324 """Commits unpatched and patched orderfiles hashes, if provided.
325
326 Files must have been successfilly uploaded to cloud storage first.
327
328 Args:
329 unpatched_orderfile_filename: (str or None) Unpatched orderfile path.
330 orderfile_filename: (str or None) Orderfile path.
331
332 Raises:
333 NotImplementedError when the commit logic hasn't been overriden.
334 """
335 files_to_commit = []
336 commit_message_lines = ['Update Orderfile.']
337 for filename in [unpatched_orderfile_filename, orderfile_filename]:
338 if not filename:
339 continue
340 (relative_path, sha1) = self._GetHashFilePathAndContents(filename)
341 commit_message_lines.append('Profile: %s: %s' % (
342 os.path.basename(relative_path), sha1))
343 files_to_commit.append(relative_path)
344 if files_to_commit:
345 self._CommitFiles(files_to_commit, commit_message_lines)
346
347 def UploadToCloudStorage(self, filename, use_debug_location):
348 """Uploads a file to cloud storage.
349
350 Args:
351 filename: (str) File to upload.
352 use_debug_location: (bool) Whether to use the debug location.
353 """
354 bucket = (self._CLOUD_STORAGE_BUCKET_FOR_DEBUG if use_debug_location
355 else self._CLOUD_STORAGE_BUCKET)
356 extension = _GetFileExtension(filename)
357 cmd = [self._UPLOAD_TO_CLOUD_COMMAND, '--bucket', bucket]
358 if extension:
359 cmd.extend(['-z', extension])
360 cmd.append(filename)
361 self._step_recorder.RunCommand(cmd)
362 print 'Download: https://sandbox.google.com/storage/%s/%s' % (
363 bucket, _GenerateHash(filename))
364
365 def _GetHashFilePathAndContents(self, filename):
366 """Gets the name and content of the hash file created from uploading the
367 given file.
368
369 Args:
370 filename: (str) The file that was uploaded to cloud storage.
371
372 Returns:
373 A tuple of the hash file name, relative to the reository root, and the
374 content, which should be the sha1 hash of the file
375 ('base_file.sha1', hash)
376 """
377 abs_hash_filename = filename + '.sha1'
378 rel_hash_filename = os.path.relpath(
379 abs_hash_filename, self._repository_root)
380 with open(abs_hash_filename, 'r') as f:
381 return (rel_hash_filename, f.read())
382
383 def _CommitFiles(self, files_to_commit, commit_message_lines):
384 """Commits a list of files, with a given message."""
385 raise NotImplementedError
386
387
388class OrderfileGenerator(object):
389 """A utility for generating a new orderfile for Clank.
390
391 Builds an instrumented binary, profiles a run of the application, and
392 generates an updated orderfile.
393 """
394 _CLANK_REPO = os.path.join(constants.DIR_SOURCE_ROOT, 'clank')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]395 _CYGLOG_TO_ORDERFILE_SCRIPT = os.path.join(
396 constants.DIR_SOURCE_ROOT, 'tools', 'cygprofile',
397 'cyglog_to_orderfile.py')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]398 _CHECK_ORDERFILE_SCRIPT = os.path.join(
399 constants.DIR_SOURCE_ROOT, 'tools', 'cygprofile', 'check_orderfile.py')
400 _BUILD_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(
401 constants.GetOutDirectory()))) # Normally /path/to/src
402
403 _UNPATCHED_ORDERFILE_FILENAME = os.path.join(
404 _CLANK_REPO, 'orderfiles', 'unpatched_orderfile.%s')
405 _MERGED_CYGLOG_FILENAME = os.path.join(
406 constants.GetOutDirectory(), 'merged_cyglog')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]407
408 _PATH_TO_ORDERFILE = os.path.join(_CLANK_REPO, 'orderfiles',
409 'orderfile.%s.out')
410
411 # Previous orderfile_generator debug files would be overwritten.
412 _DIRECTORY_FOR_DEBUG_FILES = '/tmp/orderfile_generator_debug_files'
413
414 def _GetPathToOrderfile(self):
415 """Gets the path to the architecture-specific orderfile."""
416 return self._PATH_TO_ORDERFILE % self._options.arch
417
418 def _GetUnpatchedOrderfileFilename(self):
419 """Gets the path to the architecture-specific unpatched orderfile."""
420 return self._UNPATCHED_ORDERFILE_FILENAME % self._options.arch
421
422 def __init__(self, options, orderfile_updater_class):
423 self._options = options
424
425 self._instrumented_out_dir = os.path.join(
426 self._BUILD_ROOT, self._options.arch + '_instrumented_out')
427 self._uninstrumented_out_dir = os.path.join(
428 self._BUILD_ROOT, self._options.arch + '_uninstrumented_out')
429
430 if options.profile:
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]431 output_directory = os.path.join(self._instrumented_out_dir, 'Release')
Matthew Caryb8daed942018-06-11 10:58:08[diff] [blame^]432 host_profile_dir = os.path.join(output_directory, 'profile_data')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]433 urls = [profile_android_startup.AndroidProfileTool.TEST_URL]
434 use_wpr = True
435 simulate_user = False
Benoit L96466812018-03-06 15:58:37[diff] [blame]436 urls = options.urls
437 use_wpr = not options.no_wpr
438 simulate_user = options.simulate_user
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]439 self._profiler = profile_android_startup.AndroidProfileTool(
Matthew Caryb8daed942018-06-11 10:58:08[diff] [blame^]440 output_directory, host_profile_dir, use_wpr, urls, simulate_user)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]441
442 self._output_data = {}
443 self._step_recorder = StepRecorder(options.buildbot)
444 self._compiler = None
445 assert issubclass(orderfile_updater_class, OrderfileUpdater)
446 self._orderfile_updater = orderfile_updater_class(self._CLANK_REPO,
447 self._step_recorder,
448 options.branch,
449 options.netrc)
450 assert os.path.isdir(constants.DIR_SOURCE_ROOT), 'No src directory found'
Benoit Lizefefbb27c2018-01-17 13:54:18[diff] [blame]451 symbol_extractor.SetArchitecture(options.arch)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]452
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]453 @staticmethod
454 def _RemoveBlanks(src_file, dest_file):
455 """A utility to remove blank lines from a file.
456
457 Args:
458 src_file: The name of the file to remove the blanks from.
459 dest_file: The name of the file to write the output without blanks.
460 """
461 assert src_file != dest_file, 'Source and destination need to be distinct'
462
463 try:
464 src = open(src_file, 'r')
465 dest = open(dest_file, 'w')
466 for line in src:
467 if line and not line.isspace():
468 dest.write(line)
469 finally:
470 src.close()
471 dest.close()
472
473 def _GenerateAndProcessProfile(self):
474 """Invokes a script to merge the per-thread traces into one file."""
475 self._step_recorder.BeginStep('Generate Profile Data')
476 files = []
477 try:
478 logging.getLogger().setLevel(logging.DEBUG)
479 files = self._profiler.CollectProfile(
480 self._compiler.chrome_apk,
481 constants.PACKAGE_INFO['chrome'])
Matthew Caryb8daed942018-06-11 10:58:08[diff] [blame^]482 self._step_recorder.BeginStep('Process profile')
Benoit L96466812018-03-06 15:58:37[diff] [blame]483 assert os.path.exists(self._compiler.lib_chrome_so)
484 offsets = process_profiles.GetReachedOffsetsFromDumpFiles(
485 files, self._compiler.lib_chrome_so)
486 if not offsets:
487 raise Exception('No profiler offsets found in {}'.format(
488 '\n'.join(files)))
489 with open(self._MERGED_CYGLOG_FILENAME, 'w') as f:
490 f.write('\n'.join(map(str, offsets)))
Matthew Cary0f1f681a2018-01-22 10:40:51[diff] [blame]491 except Exception:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]492 for f in files:
493 self._SaveForDebugging(f)
494 raise
495 finally:
496 self._profiler.Cleanup()
497 logging.getLogger().setLevel(logging.INFO)
498
499 try:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]500 command_args = [
501 '--target-arch=' + self._options.arch,
502 '--native-library=' + self._compiler.lib_chrome_so,
503 '--output=' + self._GetUnpatchedOrderfileFilename()]
Benoit L96466812018-03-06 15:58:37[diff] [blame]504 command_args.append('--reached-offsets=' + self._MERGED_CYGLOG_FILENAME)
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]505 self._step_recorder.RunCommand(
506 [self._CYGLOG_TO_ORDERFILE_SCRIPT] + command_args)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]507 except CommandError:
508 self._SaveForDebugging(self._MERGED_CYGLOG_FILENAME)
509 self._SaveForDebuggingWithOverwrite(self._compiler.lib_chrome_so)
510 raise
511
512 def _DeleteTempFiles(self):
513 """Deletes intermediate step output files."""
514 print 'Delete %s' % (
515 self._MERGED_CYGLOG_FILENAME)
516 if os.path.isfile(self._MERGED_CYGLOG_FILENAME):
517 os.unlink(self._MERGED_CYGLOG_FILENAME)
518
519 def _PatchOrderfile(self):
520 """Patches the orderfile using clean version of libchrome.so."""
521 self._step_recorder.BeginStep('Patch Orderfile')
Benoit Lizefefbb27c2018-01-17 13:54:18[diff] [blame]522 patch_orderfile.GeneratePatchedOrderfile(
523 self._GetUnpatchedOrderfileFilename(), self._compiler.lib_chrome_so,
524 self._GetPathToOrderfile())
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]525
526 def _VerifySymbolOrder(self):
527 self._step_recorder.BeginStep('Verify Symbol Order')
528 return_code = self._step_recorder.RunCommand(
529 [self._CHECK_ORDERFILE_SCRIPT, self._compiler.lib_chrome_so,
530 self._GetPathToOrderfile(),
531 '--target-arch=' + self._options.arch],
532 constants.DIR_SOURCE_ROOT,
533 raise_on_error=False)
534 if return_code:
535 self._step_recorder.FailStep('Orderfile check returned %d.' % return_code)
536
537 def _RecordHash(self, file_name):
538 """Records the hash of the file into the output_data dictionary."""
539 self._output_data[os.path.basename(file_name) + '.sha1'] = _GenerateHash(
540 file_name)
541
542 def _SaveFileLocally(self, file_name, file_sha1):
543 """Saves the file to a temporary location and prints the sha1sum."""
544 if not os.path.exists(self._DIRECTORY_FOR_DEBUG_FILES):
545 os.makedirs(self._DIRECTORY_FOR_DEBUG_FILES)
546 shutil.copy(file_name, self._DIRECTORY_FOR_DEBUG_FILES)
547 print 'File: %s, saved in: %s, sha1sum: %s' % (
548 file_name, self._DIRECTORY_FOR_DEBUG_FILES, file_sha1)
549
550 def _SaveForDebugging(self, filename):
551 """Uploads the file to cloud storage or saves to a temporary location."""
552 file_sha1 = _GenerateHash(filename)
553 if not self._options.buildbot:
554 self._SaveFileLocally(filename, file_sha1)
555 else:
556 print 'Uploading file for debugging: ' + filename
557 self._orderfile_updater.UploadToCloudStorage(
558 filename, use_debug_location=True)
559
560 def _SaveForDebuggingWithOverwrite(self, file_name):
561 """Uploads and overwrites the file in cloud storage or copies locally.
562
563 Should be used for large binaries like lib_chrome_so.
564
565 Args:
566 file_name: (str) File to upload.
567 """
568 file_sha1 = _GenerateHash(file_name)
569 if not self._options.buildbot:
570 self._SaveFileLocally(file_name, file_sha1)
571 else:
572 print 'Uploading file for debugging: %s, sha1sum: %s' % (
573 file_name, file_sha1)
574 upload_location = '%s/%s' % (
575 self._CLOUD_STORAGE_BUCKET_FOR_DEBUG, os.path.basename(file_name))
576 self._step_recorder.RunCommand([
577 'gsutil.py', 'cp', file_name, 'gs://' + upload_location])
578 print ('Uploaded to: https://sandbox.google.com/storage/' +
579 upload_location)
580
581 def _MaybeArchiveOrderfile(self, filename):
582 """In buildbot configuration, uploads the generated orderfile to
583 Google Cloud Storage.
584
585 Args:
586 filename: (str) Orderfile to upload.
587 """
588 # First compute hashes so that we can download them later if we need to
589 self._step_recorder.BeginStep('Compute hash for ' + filename)
590 self._RecordHash(filename)
591 if self._options.buildbot:
592 self._step_recorder.BeginStep('Archive ' + filename)
593 self._orderfile_updater.UploadToCloudStorage(
594 filename, use_debug_location=False)
595
596 def _GetHashFilePathAndContents(self, base_file):
597 """Gets the name and content of the hash file created from uploading the
598 given file.
599
600 Args:
601 base_file: The file that was uploaded to cloud storage.
602
603 Returns:
604 A tuple of the hash file name, relative to the clank repo path, and the
605 content, which should be the sha1 hash of the file
606 ('base_file.sha1', hash)
607 """
608 abs_file_name = base_file + '.sha1'
609 rel_file_name = os.path.relpath(abs_file_name, self._CLANK_REPO)
610 with open(abs_file_name, 'r') as f:
611 return (rel_file_name, f.read())
612
613 def Generate(self):
614 """Generates and maybe upload an order."""
615 profile_uploaded = False
616 orderfile_uploaded = False
617
618 if self._options.profile:
619 try:
620 _UnstashOutputDirectory(self._instrumented_out_dir)
621 self._compiler = ClankCompiler(
622 self._instrumented_out_dir,
623 self._step_recorder, self._options.arch, self._options.jobs,
624 self._options.max_load, self._options.use_goma,
Benoit L96466812018-03-06 15:58:37[diff] [blame]625 self._options.goma_dir)
Benoit L96466812018-03-06 15:58:37[diff] [blame]626 self._compiler.CompileChromeApk(True)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]627 self._GenerateAndProcessProfile()
628 self._MaybeArchiveOrderfile(self._GetUnpatchedOrderfileFilename())
629 profile_uploaded = True
630 finally:
631 self._DeleteTempFiles()
632 _StashOutputDirectory(self._instrumented_out_dir)
633 if self._options.patch:
634 if self._options.profile:
635 self._RemoveBlanks(self._GetUnpatchedOrderfileFilename(),
636 self._GetPathToOrderfile())
637 try:
638 _UnstashOutputDirectory(self._uninstrumented_out_dir)
639 self._compiler = ClankCompiler(
640 self._uninstrumented_out_dir, self._step_recorder,
641 self._options.arch, self._options.jobs, self._options.max_load,
Benoit L96466812018-03-06 15:58:37[diff] [blame]642 self._options.use_goma, self._options.goma_dir)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]643 self._compiler.CompileLibchrome(False)
644 self._PatchOrderfile()
645 # Because identical code folding is a bit different with and without
646 # the orderfile build, we need to re-patch the orderfile with code
647 # folding as close to the final version as possible.
648 self._compiler.CompileLibchrome(False, force_relink=True)
649 self._PatchOrderfile()
650 self._compiler.CompileLibchrome(False, force_relink=True)
651 self._VerifySymbolOrder()
652 self._MaybeArchiveOrderfile(self._GetPathToOrderfile())
653 finally:
654 _StashOutputDirectory(self._uninstrumented_out_dir)
655 orderfile_uploaded = True
656
657 if (self._options.buildbot and self._options.netrc
658 and not self._step_recorder.ErrorRecorded()):
659 unpatched_orderfile_filename = (
660 self._GetUnpatchedOrderfileFilename() if profile_uploaded else None)
661 orderfile_filename = (
Benoit Lized913be82017-10-24 13:38:27[diff] [blame]662 self._GetPathToOrderfile() if orderfile_uploaded else None)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]663 self._orderfile_updater.CommitFileHashes(
664 unpatched_orderfile_filename, orderfile_filename)
665
666 self._step_recorder.EndStep()
667 return not self._step_recorder.ErrorRecorded()
668
669 def GetReportingData(self):
670 """Get a dictionary of reporting data (timings, output hashes)"""
671 self._output_data['timings'] = self._step_recorder.timings
672 return self._output_data
673
674
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]675def CreateArgumentParser():
676 """Creates and returns the argument parser."""
677 parser = argparse.ArgumentParser()
678 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]679 '--buildbot', action='store_true',
680 help='If true, the script expects to be run on a buildbot')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]681 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]682 '--verify', action='store_true',
683 help='If true, the script only verifies the current orderfile')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]684 parser.add_argument('--target-arch', action='store', dest='arch',
685 default=cygprofile_utils.DetectArchitecture(),
686 choices=['arm', 'arm64', 'x86', 'x86_64', 'x64', 'mips'],
687 help='The target architecture for which to build')
688 parser.add_argument('--output-json', action='store', dest='json_file',
689 help='Location to save stats in json format')
690 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]691 '--skip-profile', action='store_false', dest='profile', default=True,
692 help='Don\'t generate a profile on the device. Only patch from the '
693 'existing profile.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]694 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]695 '--skip-patch', action='store_false', dest='patch', default=True,
696 help='Only generate the raw (unpatched) orderfile, don\'t patch it.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]697 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]698 '--netrc', action='store',
699 help='A custom .netrc file to use for git checkin. Only used on bots.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]700 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]701 '--branch', action='store', default='master',
702 help='When running on buildbot with a netrc, the branch orderfile '
703 'hashes get checked into.')
704 # Note: -j50 was causing issues on the bot.
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]705 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]706 '-j', '--jobs', action='store', default=20,
707 help='Number of jobs to use for compilation.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]708 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]709 '-l', '--max-load', action='store', default=4, help='Max cpu load.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]710 parser.add_argument('--goma-dir', help='GOMA directory.')
711 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]712 '--use-goma', action='store_true', help='Enable GOMA.', default=False)
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]713 parser.add_argument('--adb-path', help='Path to the adb binary.')
714 profile_android_startup.AddProfileCollectionArguments(parser)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]715 return parser
716
717
718def CreateOrderfile(options, orderfile_updater_class):
719 """Creates an oderfile.
720
721 Args:
722 options: As returned from optparse.OptionParser.parse_args()
723 orderfile_updater_class: (OrderfileUpdater) subclass of OrderfileUpdater.
724
725 Returns:
726 True iff success.
727 """
728 logging.basicConfig(level=logging.INFO)
729 devil_chromium.Initialize(adb_path=options.adb_path)
730
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]731 generator = OrderfileGenerator(options, orderfile_updater_class)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]732 try:
733 if options.verify:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]734 generator._VerifySymbolOrder()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]735 else:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]736 return generator.Generate()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]737 finally:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]738 json_output = json.dumps(generator.GetReportingData(),
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]739 indent=2) + '\n'
740 if options.json_file:
741 with open(options.json_file, 'w') as f:
742 f.write(json_output)
743 print json_output
744 return False
745
746
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]747def main():
748 parser = CreateArgumentParser()
749 options = parser.parse_args()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]750 return 0 if CreateOrderfile(options, OrderfileUpdater) else 1
751
752
753if __name__ == '__main__':
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]754 sys.exit(main())