[go: nahoru, domu]
Google Git
Sign in
chromium / chromium / src / a1b64f89a0c1d18c384818b4fa227c407029cf72 / . / tools / cygprofile / orderfile_generator_backend.py
blob: 0a33f756b544eec8e58f3204288591eb383088a5 [file] [log] [blame]
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1#!/usr/bin/env python
2# Copyright (c) 2013 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6""" A utility to generate an up-to-date orderfile.
7
8The orderfile is used by the linker to order text sections such that the
9sections are placed consecutively in the order specified. This allows us
10to page in less code during start-up.
11
12Example usage:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]13 tools/cygprofile/orderfile_generator_backend.py -l 20 -j 1000 --use-goma \
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]14 --target-arch=arm
15"""
16
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame^]17import argparse
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]18import hashlib
19import json
20import logging
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]21import os
22import re
23import shutil
24import subprocess
25import sys
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]26import tempfile
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]27import time
28
29import cygprofile_utils
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]30import process_profiles
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]31import profile_android_startup
32
33
34_SRC_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)),
35 os.pardir, os.pardir)
36sys.path.append(os.path.join(_SRC_PATH, 'build', 'android'))
37import devil_chromium
38from pylib import constants
39
40
41# Needs to happen early for GetBuildType()/GetOutDirectory() to work correctly
42constants.SetBuildType('Release')
43
44
45class CommandError(Exception):
46 """Indicates that a dispatched shell command exited with a non-zero status."""
47
48 def __init__(self, value):
49 super(CommandError, self).__init__()
50 self.value = value
51
52 def __str__(self):
53 return repr(self.value)
54
55
56def _GenerateHash(file_path):
57 """Calculates and returns the hash of the file at file_path."""
58 sha1 = hashlib.sha1()
59 with open(file_path, 'rb') as f:
60 while True:
61 # Read in 1mb chunks, so it doesn't all have to be loaded into memory.
62 chunk = f.read(1024 * 1024)
63 if not chunk:
64 break
65 sha1.update(chunk)
66 return sha1.hexdigest()
67
68
69def _GetFileExtension(file_name):
70 """Calculates the file extension from a file name.
71
72 Args:
73 file_name: The source file name.
74 Returns:
75 The part of file_name after the dot (.) or None if the file has no
76 extension.
77 Examples: /home/user/foo.bar -> bar
78 /home/user.name/foo -> None
79 /home/user/.foo -> None
80 /home/user/foo.bar.baz -> baz
81 """
82 file_name_parts = os.path.basename(file_name).split('.')
83 if len(file_name_parts) > 1:
84 return file_name_parts[-1]
85 else:
86 return None
87
88
89def _StashOutputDirectory(buildpath):
90 """Takes the output directory and stashes it in the default output directory.
91
92 This allows it to be used for incremental builds next time (after unstashing)
93 by keeping it in a place that isn't deleted normally, while also ensuring
94 that it is properly clobbered when appropriate.
95
96 This is a dirty hack to deal with the needs of clobbering while also handling
97 incremental builds and the hardcoded relative paths used in some of the
98 project files.
99
100 Args:
101 buildpath: The path where the building happens. If this corresponds to the
102 default output directory, no action is taken.
103 """
104 if os.path.abspath(buildpath) == os.path.abspath(os.path.dirname(
105 constants.GetOutDirectory())):
106 return
107 name = os.path.basename(buildpath)
108 stashpath = os.path.join(constants.GetOutDirectory(), name)
109 if not os.path.exists(buildpath):
110 return
111 if os.path.exists(stashpath):
112 shutil.rmtree(stashpath, ignore_errors=True)
113 shutil.move(buildpath, stashpath)
114
115
116def _UnstashOutputDirectory(buildpath):
117 """Inverse of _StashOutputDirectory.
118
119 Moves the output directory stashed within the default output directory
120 (out/Release) to the position where the builds can actually happen.
121
122 This is a dirty hack to deal with the needs of clobbering while also handling
123 incremental builds and the hardcoded relative paths used in some of the
124 project files.
125
126 Args:
127 buildpath: The path where the building happens. If this corresponds to the
128 default output directory, no action is taken.
129 """
130 if os.path.abspath(buildpath) == os.path.abspath(os.path.dirname(
131 constants.GetOutDirectory())):
132 return
133 name = os.path.basename(buildpath)
134 stashpath = os.path.join(constants.GetOutDirectory(), name)
135 if not os.path.exists(stashpath):
136 return
137 if os.path.exists(buildpath):
138 shutil.rmtree(buildpath, ignore_errors=True)
139 shutil.move(stashpath, buildpath)
140
141
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]142def _EnsureOrderfileStartsWithAnchorSection(filename):
143 """Ensures that the orderfile starts with the right anchor symbol.
144
145 This changes the orderfile, if required.
146
147 Args:
148 filename: (str) Path to the orderfile.
149 """
150 anchor_section = '.text.dummy_function_to_anchor_text'
151 with open(filename, 'r') as f:
152 if f.readline().strip() == anchor_section:
153 return
154 try:
155 f = tempfile.NamedTemporaryFile(dir=os.path.dirname(filename), delete=False)
156 f.write(anchor_section + '\n')
157 with open(filename, 'r') as orderfile_file:
158 for line in orderfile_file:
159 f.write(line + '\n')
160 f.close()
161 os.rename(f.name, filename)
162 finally:
163 if os.path.exists(f.name):
164 os.remove(f.name)
165
166
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]167class StepRecorder(object):
168 """Records steps and timings."""
169
170 def __init__(self, buildbot):
171 self.timings = []
172 self._previous_step = ('', 0.0)
173 self._buildbot = buildbot
174 self._error_recorded = False
175
176 def BeginStep(self, name):
177 """Marks a beginning of the next step in the script.
178
179 On buildbot, this prints a specially formatted name that will show up
180 in the waterfall. Otherwise, just prints the step name.
181
182 Args:
183 name: The name of the step.
184 """
185 self.EndStep()
186 self._previous_step = (name, time.time())
187 print 'Running step: ', name
188
189 def EndStep(self):
190 """Records successful completion of the current step.
191
192 This is optional if the step is immediately followed by another BeginStep.
193 """
194 if self._previous_step[0]:
195 elapsed = time.time() - self._previous_step[1]
196 print 'Step %s took %f seconds' % (self._previous_step[0], elapsed)
197 self.timings.append((self._previous_step[0], elapsed))
198
199 self._previous_step = ('', 0.0)
200
201 def FailStep(self, message=None):
202 """Marks that a particular step has failed.
203
204 On buildbot, this will mark the current step as failed on the waterfall.
205 Otherwise we will just print an optional failure message.
206
207 Args:
208 message: An optional explanation as to why the step failed.
209 """
210 print 'STEP FAILED!!'
211 if message:
212 print message
213 self._error_recorded = True
214 self.EndStep()
215
216 def ErrorRecorded(self):
217 """True if FailStep has been called."""
218 return self._error_recorded
219
220 def RunCommand(self, cmd, cwd=constants.DIR_SOURCE_ROOT, raise_on_error=True,
221 stdout=None):
222 """Execute a shell command.
223
224 Args:
225 cmd: A list of command strings.
226 cwd: Directory in which the command should be executed, defaults to script
227 location if not specified.
228 raise_on_error: If true will raise a CommandError if the call doesn't
229 succeed and mark the step as failed.
230 stdout: A file to redirect stdout for the command to.
231
232 Returns:
233 The process's return code.
234
235 Raises:
236 CommandError: An error executing the specified command.
237 """
238 print 'Executing %s in %s' % (' '.join(cmd), cwd)
239 process = subprocess.Popen(cmd, stdout=stdout, cwd=cwd, env=os.environ)
240 process.wait()
241 if raise_on_error and process.returncode != 0:
242 self.FailStep()
243 raise CommandError('Exception executing command %s' % ' '.join(cmd))
244 return process.returncode
245
246
247class ClankCompiler(object):
248 """Handles compilation of clank."""
249
250 def __init__(self, out_dir, step_recorder, arch, jobs, max_load, use_goma,
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]251 goma_dir, lightweight_instrumentation):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]252 self._out_dir = out_dir
253 self._step_recorder = step_recorder
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]254 self._arch = arch
255 self._jobs = jobs
256 self._max_load = max_load
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]257 self._use_goma = use_goma
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]258 self._goma_dir = goma_dir
259 self._lightweight_instrumentation = lightweight_instrumentation
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]260 lib_chrome_so_dir = 'lib.unstripped'
261 self.lib_chrome_so = os.path.join(
262 self._out_dir, 'Release', lib_chrome_so_dir, 'libchrome.so')
263 self.chrome_apk = os.path.join(
264 self._out_dir, 'Release', 'apks', 'Chrome.apk')
265
266 def Build(self, instrumented, target):
267 """Builds the provided ninja target with or without order_profiling on.
268
269 Args:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]270 instrumented: (bool) Whether we want to build an instrumented binary.
271 target: (str) The name of the ninja target to build.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]272 """
273 self._step_recorder.BeginStep('Compile %s' % target)
274
275 # Set the "Release Official" flavor, the parts affecting performance.
276 args = [
277 'is_chrome_branded=true',
278 'is_debug=false',
279 'is_official_build=true',
280 # We have to build with no symbols if profiling and minimal symbols
281 # otherwise for libchrome.so to fit under the 4 GB limit.
282 # crbug.com/574476
283 'symbol_level=' + ('0' if instrumented else '1'),
284 'target_cpu="' + self._arch + '"',
285 'target_os="android"',
286 'use_goma=' + str(self._use_goma).lower(),
287 'use_order_profiling=' + str(instrumented).lower(),
288 ]
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]289 if instrumented and self._lightweight_instrumentation:
290 args.append('use_lightweight_order_profiling=true')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]291 if self._goma_dir:
292 args += ['goma_dir="%s"' % self._goma_dir]
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]293
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]294 self._step_recorder.RunCommand(
295 ['gn', 'gen', os.path.join(self._out_dir, 'Release'),
296 '--args=' + ' '.join(args)])
297
298 self._step_recorder.RunCommand(
299 ['ninja', '-C', os.path.join(self._out_dir, 'Release'),
300 '-j' + str(self._jobs), '-l' + str(self._max_load), target])
301
302 def CompileChromeApk(self, instrumented, force_relink=False):
303 """Builds a Chrome.apk either with or without order_profiling on.
304
305 Args:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]306 instrumented: (bool) Whether to build an instrumented apk.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]307 force_relink: Whether libchromeview.so should be re-created.
308 """
309 if force_relink:
310 self._step_recorder.RunCommand(['rm', '-rf', self.lib_chrome_so])
311 self.Build(instrumented, 'chrome_apk')
312
313 def CompileLibchrome(self, instrumented, force_relink=False):
314 """Builds a libchrome.so either with or without order_profiling on.
315
316 Args:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]317 instrumented: (bool) Whether to build an instrumented apk.
318 force_relink: (bool) Whether libchrome.so should be re-created.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]319 """
320 if force_relink:
321 self._step_recorder.RunCommand(['rm', '-rf', self.lib_chrome_so])
322 self.Build(instrumented, 'libchrome')
323
324
325class OrderfileUpdater(object):
326 """Handles uploading and committing a new orderfile in the repository.
327
328 Only used for testing or on a bot.
329 """
330
331 _CLOUD_STORAGE_BUCKET_FOR_DEBUG = None
332 _CLOUD_STORAGE_BUCKET = None
333 _UPLOAD_TO_CLOUD_COMMAND = 'upload_to_google_storage.py'
334
335 def __init__(self, repository_root, step_recorder, branch, netrc):
336 """Constructor.
337
338 Args:
339 repository_root: (str) Root of the target repository.
340 step_recorder: (StepRecorder) Step recorder, for logging.
341 branch: (str) Branch to commit to.
342 netrc: (str) Path to the .netrc file to use.
343 """
344 self._repository_root = repository_root
345 self._step_recorder = step_recorder
346 self._branch = branch
347 self._netrc = netrc
348
349 def CommitFileHashes(self, unpatched_orderfile_filename, orderfile_filename):
350 """Commits unpatched and patched orderfiles hashes, if provided.
351
352 Files must have been successfilly uploaded to cloud storage first.
353
354 Args:
355 unpatched_orderfile_filename: (str or None) Unpatched orderfile path.
356 orderfile_filename: (str or None) Orderfile path.
357
358 Raises:
359 NotImplementedError when the commit logic hasn't been overriden.
360 """
361 files_to_commit = []
362 commit_message_lines = ['Update Orderfile.']
363 for filename in [unpatched_orderfile_filename, orderfile_filename]:
364 if not filename:
365 continue
366 (relative_path, sha1) = self._GetHashFilePathAndContents(filename)
367 commit_message_lines.append('Profile: %s: %s' % (
368 os.path.basename(relative_path), sha1))
369 files_to_commit.append(relative_path)
370 if files_to_commit:
371 self._CommitFiles(files_to_commit, commit_message_lines)
372
373 def UploadToCloudStorage(self, filename, use_debug_location):
374 """Uploads a file to cloud storage.
375
376 Args:
377 filename: (str) File to upload.
378 use_debug_location: (bool) Whether to use the debug location.
379 """
380 bucket = (self._CLOUD_STORAGE_BUCKET_FOR_DEBUG if use_debug_location
381 else self._CLOUD_STORAGE_BUCKET)
382 extension = _GetFileExtension(filename)
383 cmd = [self._UPLOAD_TO_CLOUD_COMMAND, '--bucket', bucket]
384 if extension:
385 cmd.extend(['-z', extension])
386 cmd.append(filename)
387 self._step_recorder.RunCommand(cmd)
388 print 'Download: https://sandbox.google.com/storage/%s/%s' % (
389 bucket, _GenerateHash(filename))
390
391 def _GetHashFilePathAndContents(self, filename):
392 """Gets the name and content of the hash file created from uploading the
393 given file.
394
395 Args:
396 filename: (str) The file that was uploaded to cloud storage.
397
398 Returns:
399 A tuple of the hash file name, relative to the reository root, and the
400 content, which should be the sha1 hash of the file
401 ('base_file.sha1', hash)
402 """
403 abs_hash_filename = filename + '.sha1'
404 rel_hash_filename = os.path.relpath(
405 abs_hash_filename, self._repository_root)
406 with open(abs_hash_filename, 'r') as f:
407 return (rel_hash_filename, f.read())
408
409 def _CommitFiles(self, files_to_commit, commit_message_lines):
410 """Commits a list of files, with a given message."""
411 raise NotImplementedError
412
413
414class OrderfileGenerator(object):
415 """A utility for generating a new orderfile for Clank.
416
417 Builds an instrumented binary, profiles a run of the application, and
418 generates an updated orderfile.
419 """
420 _CLANK_REPO = os.path.join(constants.DIR_SOURCE_ROOT, 'clank')
421 _MERGE_TRACES_SCRIPT = os.path.join(
422 constants.DIR_SOURCE_ROOT, 'tools', 'cygprofile', 'mergetraces.py')
423 _CYGLOG_TO_ORDERFILE_SCRIPT = os.path.join(
424 constants.DIR_SOURCE_ROOT, 'tools', 'cygprofile',
425 'cyglog_to_orderfile.py')
426 _PATCH_ORDERFILE_SCRIPT = os.path.join(
427 constants.DIR_SOURCE_ROOT, 'tools', 'cygprofile', 'patch_orderfile.py')
428 _CHECK_ORDERFILE_SCRIPT = os.path.join(
429 constants.DIR_SOURCE_ROOT, 'tools', 'cygprofile', 'check_orderfile.py')
430 _BUILD_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(
431 constants.GetOutDirectory()))) # Normally /path/to/src
432
433 _UNPATCHED_ORDERFILE_FILENAME = os.path.join(
434 _CLANK_REPO, 'orderfiles', 'unpatched_orderfile.%s')
435 _MERGED_CYGLOG_FILENAME = os.path.join(
436 constants.GetOutDirectory(), 'merged_cyglog')
437 _TEMP_ORDERFILE_FILENAME = os.path.join(
438 constants.GetOutDirectory(), 'tmp_orderfile')
439
440 _PATH_TO_ORDERFILE = os.path.join(_CLANK_REPO, 'orderfiles',
441 'orderfile.%s.out')
442
443 # Previous orderfile_generator debug files would be overwritten.
444 _DIRECTORY_FOR_DEBUG_FILES = '/tmp/orderfile_generator_debug_files'
445
446 def _GetPathToOrderfile(self):
447 """Gets the path to the architecture-specific orderfile."""
448 return self._PATH_TO_ORDERFILE % self._options.arch
449
450 def _GetUnpatchedOrderfileFilename(self):
451 """Gets the path to the architecture-specific unpatched orderfile."""
452 return self._UNPATCHED_ORDERFILE_FILENAME % self._options.arch
453
454 def __init__(self, options, orderfile_updater_class):
455 self._options = options
456
457 self._instrumented_out_dir = os.path.join(
458 self._BUILD_ROOT, self._options.arch + '_instrumented_out')
459 self._uninstrumented_out_dir = os.path.join(
460 self._BUILD_ROOT, self._options.arch + '_uninstrumented_out')
461
462 if options.profile:
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame^]463 output_directory = os.path.join(self._instrumented_out_dir, 'Release')
464 host_cyglog_dir = os.path.join(output_directory, 'cyglog_data')
465 # Only override the defaults when using lightweight instrumentation,
466 # as the regular profiling code is likely too slow for these.
467 urls = [profile_android_startup.AndroidProfileTool.TEST_URL]
468 use_wpr = True
469 simulate_user = False
470 if options.simulate_user and not options.lightweight_instrumentation:
471 logging.error(
472 '--simulate-user required --lightweight-instrumentation, ignoring.')
473 if options.lightweight_instrumentation:
474 urls = options.urls
475 use_wpr = not options.no_wpr
476 simulate_user = options.simulate_user
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]477 self._profiler = profile_android_startup.AndroidProfileTool(
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame^]478 output_directory, host_cyglog_dir, use_wpr, urls, simulate_user)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]479
480 self._output_data = {}
481 self._step_recorder = StepRecorder(options.buildbot)
482 self._compiler = None
483 assert issubclass(orderfile_updater_class, OrderfileUpdater)
484 self._orderfile_updater = orderfile_updater_class(self._CLANK_REPO,
485 self._step_recorder,
486 options.branch,
487 options.netrc)
488 assert os.path.isdir(constants.DIR_SOURCE_ROOT), 'No src directory found'
489
490 def _RunCygprofileUnitTests(self):
491 """Builds, deploys and runs cygprofile_unittests."""
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]492 # There an no unittests (yet) for the lightweight instrumentation.
493 # TODO(lizeb): Fix this.
494 if self._options.lightweight_instrumentation:
495 return
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]496 tools_compiler = ClankCompiler(
497 os.path.dirname(constants.GetOutDirectory()),
498 self._step_recorder, self._options.arch, self._options.jobs,
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]499 self._options.max_load, self._options.use_goma, self._options.goma_dir,
500 self._options.lightweight_instrumentation)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]501 tools_compiler.Build(instrumented=False, target='android_tools')
502 self._compiler.Build(instrumented=True, target='cygprofile_unittests')
503
504 self._step_recorder.BeginStep('Deploy and run cygprofile_unittests')
505 exit_code = self._profiler.RunCygprofileTests()
506
507 if exit_code != 0:
508 self._step_recorder.FailStep(
509 'cygprofile_unittests exited with non-0 status: %d' % exit_code)
510
511 @staticmethod
512 def _RemoveBlanks(src_file, dest_file):
513 """A utility to remove blank lines from a file.
514
515 Args:
516 src_file: The name of the file to remove the blanks from.
517 dest_file: The name of the file to write the output without blanks.
518 """
519 assert src_file != dest_file, 'Source and destination need to be distinct'
520
521 try:
522 src = open(src_file, 'r')
523 dest = open(dest_file, 'w')
524 for line in src:
525 if line and not line.isspace():
526 dest.write(line)
527 finally:
528 src.close()
529 dest.close()
530
531 def _GenerateAndProcessProfile(self):
532 """Invokes a script to merge the per-thread traces into one file."""
533 self._step_recorder.BeginStep('Generate Profile Data')
534 files = []
535 try:
536 logging.getLogger().setLevel(logging.DEBUG)
537 files = self._profiler.CollectProfile(
538 self._compiler.chrome_apk,
539 constants.PACKAGE_INFO['chrome'])
540 self._step_recorder.BeginStep('Process cyglog')
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]541 if self._options.lightweight_instrumentation:
542 assert os.path.exists(self._compiler.lib_chrome_so)
543 process_profiles.GetReachedSymbolsFromDumpsAndMaybeWriteOffsets(
544 files, self._compiler.lib_chrome_so, self._MERGED_CYGLOG_FILENAME)
545 else:
546 with open(self._MERGED_CYGLOG_FILENAME, 'w') as merged_cyglog:
547 self._step_recorder.RunCommand([self._MERGE_TRACES_SCRIPT] + files,
548 constants.DIR_SOURCE_ROOT,
549 stdout=merged_cyglog)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]550 except CommandError:
551 for f in files:
552 self._SaveForDebugging(f)
553 raise
554 finally:
555 self._profiler.Cleanup()
556 logging.getLogger().setLevel(logging.INFO)
557
558 try:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]559 command_args = [
560 '--target-arch=' + self._options.arch,
561 '--native-library=' + self._compiler.lib_chrome_so,
562 '--output=' + self._GetUnpatchedOrderfileFilename()]
563 if self._options.lightweight_instrumentation:
564 command_args.append('--reached-offsets=' + self._MERGED_CYGLOG_FILENAME)
565 else:
566 command_args.append('--merged-cyglog=' + self._MERGED_CYGLOG_FILENAME)
567 self._step_recorder.RunCommand(
568 [self._CYGLOG_TO_ORDERFILE_SCRIPT] + command_args)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]569 except CommandError:
570 self._SaveForDebugging(self._MERGED_CYGLOG_FILENAME)
571 self._SaveForDebuggingWithOverwrite(self._compiler.lib_chrome_so)
572 raise
573
574 def _DeleteTempFiles(self):
575 """Deletes intermediate step output files."""
576 print 'Delete %s' % (
577 self._MERGED_CYGLOG_FILENAME)
578 if os.path.isfile(self._MERGED_CYGLOG_FILENAME):
579 os.unlink(self._MERGED_CYGLOG_FILENAME)
580
581 def _PatchOrderfile(self):
582 """Patches the orderfile using clean version of libchrome.so."""
583 self._step_recorder.BeginStep('Patch Orderfile')
584 try:
585 tmp_out = open(self._TEMP_ORDERFILE_FILENAME, 'w')
586 self._step_recorder.RunCommand([self._PATCH_ORDERFILE_SCRIPT,
587 self._GetUnpatchedOrderfileFilename(),
588 self._compiler.lib_chrome_so,
589 '--target-arch=' + self._options.arch],
590 constants.DIR_SOURCE_ROOT, stdout=tmp_out)
591 tmp_out.close()
592
593 self._RemoveBlanks(self._TEMP_ORDERFILE_FILENAME,
594 self._GetPathToOrderfile())
595 except CommandError:
596 self._SaveForDebugging(self._GetUnpatchedOrderfileFilename())
597 self._SaveForDebuggingWithOverwrite(self._compiler.lib_chrome_so)
598 raise
599 finally:
600 tmp_out.close()
601 if os.path.isfile(self._TEMP_ORDERFILE_FILENAME):
602 os.unlink(self._TEMP_ORDERFILE_FILENAME)
603
604 def _VerifySymbolOrder(self):
605 self._step_recorder.BeginStep('Verify Symbol Order')
606 return_code = self._step_recorder.RunCommand(
607 [self._CHECK_ORDERFILE_SCRIPT, self._compiler.lib_chrome_so,
608 self._GetPathToOrderfile(),
609 '--target-arch=' + self._options.arch],
610 constants.DIR_SOURCE_ROOT,
611 raise_on_error=False)
612 if return_code:
613 self._step_recorder.FailStep('Orderfile check returned %d.' % return_code)
614
615 def _RecordHash(self, file_name):
616 """Records the hash of the file into the output_data dictionary."""
617 self._output_data[os.path.basename(file_name) + '.sha1'] = _GenerateHash(
618 file_name)
619
620 def _SaveFileLocally(self, file_name, file_sha1):
621 """Saves the file to a temporary location and prints the sha1sum."""
622 if not os.path.exists(self._DIRECTORY_FOR_DEBUG_FILES):
623 os.makedirs(self._DIRECTORY_FOR_DEBUG_FILES)
624 shutil.copy(file_name, self._DIRECTORY_FOR_DEBUG_FILES)
625 print 'File: %s, saved in: %s, sha1sum: %s' % (
626 file_name, self._DIRECTORY_FOR_DEBUG_FILES, file_sha1)
627
628 def _SaveForDebugging(self, filename):
629 """Uploads the file to cloud storage or saves to a temporary location."""
630 file_sha1 = _GenerateHash(filename)
631 if not self._options.buildbot:
632 self._SaveFileLocally(filename, file_sha1)
633 else:
634 print 'Uploading file for debugging: ' + filename
635 self._orderfile_updater.UploadToCloudStorage(
636 filename, use_debug_location=True)
637
638 def _SaveForDebuggingWithOverwrite(self, file_name):
639 """Uploads and overwrites the file in cloud storage or copies locally.
640
641 Should be used for large binaries like lib_chrome_so.
642
643 Args:
644 file_name: (str) File to upload.
645 """
646 file_sha1 = _GenerateHash(file_name)
647 if not self._options.buildbot:
648 self._SaveFileLocally(file_name, file_sha1)
649 else:
650 print 'Uploading file for debugging: %s, sha1sum: %s' % (
651 file_name, file_sha1)
652 upload_location = '%s/%s' % (
653 self._CLOUD_STORAGE_BUCKET_FOR_DEBUG, os.path.basename(file_name))
654 self._step_recorder.RunCommand([
655 'gsutil.py', 'cp', file_name, 'gs://' + upload_location])
656 print ('Uploaded to: https://sandbox.google.com/storage/' +
657 upload_location)
658
659 def _MaybeArchiveOrderfile(self, filename):
660 """In buildbot configuration, uploads the generated orderfile to
661 Google Cloud Storage.
662
663 Args:
664 filename: (str) Orderfile to upload.
665 """
666 # First compute hashes so that we can download them later if we need to
667 self._step_recorder.BeginStep('Compute hash for ' + filename)
668 self._RecordHash(filename)
669 if self._options.buildbot:
670 self._step_recorder.BeginStep('Archive ' + filename)
671 self._orderfile_updater.UploadToCloudStorage(
672 filename, use_debug_location=False)
673
674 def _GetHashFilePathAndContents(self, base_file):
675 """Gets the name and content of the hash file created from uploading the
676 given file.
677
678 Args:
679 base_file: The file that was uploaded to cloud storage.
680
681 Returns:
682 A tuple of the hash file name, relative to the clank repo path, and the
683 content, which should be the sha1 hash of the file
684 ('base_file.sha1', hash)
685 """
686 abs_file_name = base_file + '.sha1'
687 rel_file_name = os.path.relpath(abs_file_name, self._CLANK_REPO)
688 with open(abs_file_name, 'r') as f:
689 return (rel_file_name, f.read())
690
691 def Generate(self):
692 """Generates and maybe upload an order."""
693 profile_uploaded = False
694 orderfile_uploaded = False
695
696 if self._options.profile:
697 try:
698 _UnstashOutputDirectory(self._instrumented_out_dir)
699 self._compiler = ClankCompiler(
700 self._instrumented_out_dir,
701 self._step_recorder, self._options.arch, self._options.jobs,
702 self._options.max_load, self._options.use_goma,
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]703 self._options.goma_dir,
704 self._options.lightweight_instrumentation)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]705 self._RunCygprofileUnitTests()
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]706 if self._options.lightweight_instrumentation:
707 _EnsureOrderfileStartsWithAnchorSection(self._GetPathToOrderfile())
708 self._compiler.CompileChromeApk(
709 True, self._options.lightweight_instrumentation)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]710 self._GenerateAndProcessProfile()
711 self._MaybeArchiveOrderfile(self._GetUnpatchedOrderfileFilename())
712 profile_uploaded = True
713 finally:
714 self._DeleteTempFiles()
715 _StashOutputDirectory(self._instrumented_out_dir)
716 if self._options.patch:
717 if self._options.profile:
718 self._RemoveBlanks(self._GetUnpatchedOrderfileFilename(),
719 self._GetPathToOrderfile())
720 try:
721 _UnstashOutputDirectory(self._uninstrumented_out_dir)
722 self._compiler = ClankCompiler(
723 self._uninstrumented_out_dir, self._step_recorder,
724 self._options.arch, self._options.jobs, self._options.max_load,
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]725 self._options.use_goma, self._options.goma_dir,
726 self._options.lightweight_instrumentation)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]727 self._compiler.CompileLibchrome(False)
728 self._PatchOrderfile()
729 # Because identical code folding is a bit different with and without
730 # the orderfile build, we need to re-patch the orderfile with code
731 # folding as close to the final version as possible.
732 self._compiler.CompileLibchrome(False, force_relink=True)
733 self._PatchOrderfile()
734 self._compiler.CompileLibchrome(False, force_relink=True)
735 self._VerifySymbolOrder()
736 self._MaybeArchiveOrderfile(self._GetPathToOrderfile())
737 finally:
738 _StashOutputDirectory(self._uninstrumented_out_dir)
739 orderfile_uploaded = True
740
741 if (self._options.buildbot and self._options.netrc
742 and not self._step_recorder.ErrorRecorded()):
743 unpatched_orderfile_filename = (
744 self._GetUnpatchedOrderfileFilename() if profile_uploaded else None)
745 orderfile_filename = (
Benoit Lized913be82017-10-24 13:38:27[diff] [blame]746 self._GetPathToOrderfile() if orderfile_uploaded else None)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]747 self._orderfile_updater.CommitFileHashes(
748 unpatched_orderfile_filename, orderfile_filename)
749
750 self._step_recorder.EndStep()
751 return not self._step_recorder.ErrorRecorded()
752
753 def GetReportingData(self):
754 """Get a dictionary of reporting data (timings, output hashes)"""
755 self._output_data['timings'] = self._step_recorder.timings
756 return self._output_data
757
758
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame^]759def CreateArgumentParser():
760 """Creates and returns the argument parser."""
761 parser = argparse.ArgumentParser()
762 parser.add_argument(
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]763 '--lightweight-instrumentation', action='store_true', default=False,
764 help='Use the lightweight instrumentation path')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame^]765 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]766 '--buildbot', action='store_true',
767 help='If true, the script expects to be run on a buildbot')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame^]768 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]769 '--verify', action='store_true',
770 help='If true, the script only verifies the current orderfile')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame^]771 parser.add_argument('--target-arch', action='store', dest='arch',
772 default=cygprofile_utils.DetectArchitecture(),
773 choices=['arm', 'arm64', 'x86', 'x86_64', 'x64', 'mips'],
774 help='The target architecture for which to build')
775 parser.add_argument('--output-json', action='store', dest='json_file',
776 help='Location to save stats in json format')
777 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]778 '--skip-profile', action='store_false', dest='profile', default=True,
779 help='Don\'t generate a profile on the device. Only patch from the '
780 'existing profile.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame^]781 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]782 '--skip-patch', action='store_false', dest='patch', default=True,
783 help='Only generate the raw (unpatched) orderfile, don\'t patch it.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame^]784 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]785 '--netrc', action='store',
786 help='A custom .netrc file to use for git checkin. Only used on bots.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame^]787 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]788 '--branch', action='store', default='master',
789 help='When running on buildbot with a netrc, the branch orderfile '
790 'hashes get checked into.')
791 # Note: -j50 was causing issues on the bot.
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame^]792 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]793 '-j', '--jobs', action='store', default=20,
794 help='Number of jobs to use for compilation.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame^]795 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]796 '-l', '--max-load', action='store', default=4, help='Max cpu load.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame^]797 parser.add_argument('--goma-dir', help='GOMA directory.')
798 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]799 '--use-goma', action='store_true', help='Enable GOMA.', default=False)
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame^]800 parser.add_argument('--adb-path', help='Path to the adb binary.')
801 profile_android_startup.AddProfileCollectionArguments(parser)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]802 return parser
803
804
805def CreateOrderfile(options, orderfile_updater_class):
806 """Creates an oderfile.
807
808 Args:
809 options: As returned from optparse.OptionParser.parse_args()
810 orderfile_updater_class: (OrderfileUpdater) subclass of OrderfileUpdater.
811
812 Returns:
813 True iff success.
814 """
815 logging.basicConfig(level=logging.INFO)
816 devil_chromium.Initialize(adb_path=options.adb_path)
817
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]818 generator = OrderfileGenerator(options, orderfile_updater_class)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]819 try:
820 if options.verify:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]821 generator._VerifySymbolOrder()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]822 else:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]823 return generator.Generate()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]824 finally:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]825 json_output = json.dumps(generator.GetReportingData(),
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]826 indent=2) + '\n'
827 if options.json_file:
828 with open(options.json_file, 'w') as f:
829 f.write(json_output)
830 print json_output
831 return False
832
833
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame^]834def main():
835 parser = CreateArgumentParser()
836 options = parser.parse_args()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]837 return 0 if CreateOrderfile(options, OrderfileUpdater) else 1
838
839
840if __name__ == '__main__':
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame^]841 sys.exit(main())