[go: nahoru, domu]
Google Git
Sign in
chromium / chromium / src / a87e5bcebf6bc0b3a0a8fa3adb692f7125d72f82 / . / tools / cygprofile / orderfile_generator_backend.py
blob: 23ddbdc29b62c9af3fca38de0e72b68e40c1c728 [file] [log] [blame]
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1#!/usr/bin/env python
2# Copyright (c) 2013 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6""" A utility to generate an up-to-date orderfile.
7
8The orderfile is used by the linker to order text sections such that the
9sections are placed consecutively in the order specified. This allows us
10to page in less code during start-up.
11
12Example usage:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]13 tools/cygprofile/orderfile_generator_backend.py -l 20 -j 1000 --use-goma \
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]14 --target-arch=arm
15"""
16
17import hashlib
18import json
19import logging
20import optparse
21import os
22import re
23import shutil
24import subprocess
25import sys
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame^]26import tempfile
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]27import time
28
29import cygprofile_utils
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame^]30import process_profiles
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]31import profile_android_startup
32
33
34_SRC_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)),
35 os.pardir, os.pardir)
36sys.path.append(os.path.join(_SRC_PATH, 'build', 'android'))
37import devil_chromium
38from pylib import constants
39
40
41# Needs to happen early for GetBuildType()/GetOutDirectory() to work correctly
42constants.SetBuildType('Release')
43
44
45class CommandError(Exception):
46 """Indicates that a dispatched shell command exited with a non-zero status."""
47
48 def __init__(self, value):
49 super(CommandError, self).__init__()
50 self.value = value
51
52 def __str__(self):
53 return repr(self.value)
54
55
56def _GenerateHash(file_path):
57 """Calculates and returns the hash of the file at file_path."""
58 sha1 = hashlib.sha1()
59 with open(file_path, 'rb') as f:
60 while True:
61 # Read in 1mb chunks, so it doesn't all have to be loaded into memory.
62 chunk = f.read(1024 * 1024)
63 if not chunk:
64 break
65 sha1.update(chunk)
66 return sha1.hexdigest()
67
68
69def _GetFileExtension(file_name):
70 """Calculates the file extension from a file name.
71
72 Args:
73 file_name: The source file name.
74 Returns:
75 The part of file_name after the dot (.) or None if the file has no
76 extension.
77 Examples: /home/user/foo.bar -> bar
78 /home/user.name/foo -> None
79 /home/user/.foo -> None
80 /home/user/foo.bar.baz -> baz
81 """
82 file_name_parts = os.path.basename(file_name).split('.')
83 if len(file_name_parts) > 1:
84 return file_name_parts[-1]
85 else:
86 return None
87
88
89def _StashOutputDirectory(buildpath):
90 """Takes the output directory and stashes it in the default output directory.
91
92 This allows it to be used for incremental builds next time (after unstashing)
93 by keeping it in a place that isn't deleted normally, while also ensuring
94 that it is properly clobbered when appropriate.
95
96 This is a dirty hack to deal with the needs of clobbering while also handling
97 incremental builds and the hardcoded relative paths used in some of the
98 project files.
99
100 Args:
101 buildpath: The path where the building happens. If this corresponds to the
102 default output directory, no action is taken.
103 """
104 if os.path.abspath(buildpath) == os.path.abspath(os.path.dirname(
105 constants.GetOutDirectory())):
106 return
107 name = os.path.basename(buildpath)
108 stashpath = os.path.join(constants.GetOutDirectory(), name)
109 if not os.path.exists(buildpath):
110 return
111 if os.path.exists(stashpath):
112 shutil.rmtree(stashpath, ignore_errors=True)
113 shutil.move(buildpath, stashpath)
114
115
116def _UnstashOutputDirectory(buildpath):
117 """Inverse of _StashOutputDirectory.
118
119 Moves the output directory stashed within the default output directory
120 (out/Release) to the position where the builds can actually happen.
121
122 This is a dirty hack to deal with the needs of clobbering while also handling
123 incremental builds and the hardcoded relative paths used in some of the
124 project files.
125
126 Args:
127 buildpath: The path where the building happens. If this corresponds to the
128 default output directory, no action is taken.
129 """
130 if os.path.abspath(buildpath) == os.path.abspath(os.path.dirname(
131 constants.GetOutDirectory())):
132 return
133 name = os.path.basename(buildpath)
134 stashpath = os.path.join(constants.GetOutDirectory(), name)
135 if not os.path.exists(stashpath):
136 return
137 if os.path.exists(buildpath):
138 shutil.rmtree(buildpath, ignore_errors=True)
139 shutil.move(stashpath, buildpath)
140
141
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame^]142def _EnsureOrderfileStartsWithAnchorSection(filename):
143 """Ensures that the orderfile starts with the right anchor symbol.
144
145 This changes the orderfile, if required.
146
147 Args:
148 filename: (str) Path to the orderfile.
149 """
150 anchor_section = '.text.dummy_function_to_anchor_text'
151 with open(filename, 'r') as f:
152 if f.readline().strip() == anchor_section:
153 return
154 try:
155 f = tempfile.NamedTemporaryFile(dir=os.path.dirname(filename), delete=False)
156 f.write(anchor_section + '\n')
157 with open(filename, 'r') as orderfile_file:
158 for line in orderfile_file:
159 f.write(line + '\n')
160 f.close()
161 os.rename(f.name, filename)
162 finally:
163 if os.path.exists(f.name):
164 os.remove(f.name)
165
166
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]167class StepRecorder(object):
168 """Records steps and timings."""
169
170 def __init__(self, buildbot):
171 self.timings = []
172 self._previous_step = ('', 0.0)
173 self._buildbot = buildbot
174 self._error_recorded = False
175
176 def BeginStep(self, name):
177 """Marks a beginning of the next step in the script.
178
179 On buildbot, this prints a specially formatted name that will show up
180 in the waterfall. Otherwise, just prints the step name.
181
182 Args:
183 name: The name of the step.
184 """
185 self.EndStep()
186 self._previous_step = (name, time.time())
187 print 'Running step: ', name
188
189 def EndStep(self):
190 """Records successful completion of the current step.
191
192 This is optional if the step is immediately followed by another BeginStep.
193 """
194 if self._previous_step[0]:
195 elapsed = time.time() - self._previous_step[1]
196 print 'Step %s took %f seconds' % (self._previous_step[0], elapsed)
197 self.timings.append((self._previous_step[0], elapsed))
198
199 self._previous_step = ('', 0.0)
200
201 def FailStep(self, message=None):
202 """Marks that a particular step has failed.
203
204 On buildbot, this will mark the current step as failed on the waterfall.
205 Otherwise we will just print an optional failure message.
206
207 Args:
208 message: An optional explanation as to why the step failed.
209 """
210 print 'STEP FAILED!!'
211 if message:
212 print message
213 self._error_recorded = True
214 self.EndStep()
215
216 def ErrorRecorded(self):
217 """True if FailStep has been called."""
218 return self._error_recorded
219
220 def RunCommand(self, cmd, cwd=constants.DIR_SOURCE_ROOT, raise_on_error=True,
221 stdout=None):
222 """Execute a shell command.
223
224 Args:
225 cmd: A list of command strings.
226 cwd: Directory in which the command should be executed, defaults to script
227 location if not specified.
228 raise_on_error: If true will raise a CommandError if the call doesn't
229 succeed and mark the step as failed.
230 stdout: A file to redirect stdout for the command to.
231
232 Returns:
233 The process's return code.
234
235 Raises:
236 CommandError: An error executing the specified command.
237 """
238 print 'Executing %s in %s' % (' '.join(cmd), cwd)
239 process = subprocess.Popen(cmd, stdout=stdout, cwd=cwd, env=os.environ)
240 process.wait()
241 if raise_on_error and process.returncode != 0:
242 self.FailStep()
243 raise CommandError('Exception executing command %s' % ' '.join(cmd))
244 return process.returncode
245
246
247class ClankCompiler(object):
248 """Handles compilation of clank."""
249
250 def __init__(self, out_dir, step_recorder, arch, jobs, max_load, use_goma,
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame^]251 goma_dir, lightweight_instrumentation):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]252 self._out_dir = out_dir
253 self._step_recorder = step_recorder
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame^]254 self._arch = arch
255 self._jobs = jobs
256 self._max_load = max_load
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]257 self._use_goma = use_goma
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame^]258 self._goma_dir = goma_dir
259 self._lightweight_instrumentation = lightweight_instrumentation
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]260 lib_chrome_so_dir = 'lib.unstripped'
261 self.lib_chrome_so = os.path.join(
262 self._out_dir, 'Release', lib_chrome_so_dir, 'libchrome.so')
263 self.chrome_apk = os.path.join(
264 self._out_dir, 'Release', 'apks', 'Chrome.apk')
265
266 def Build(self, instrumented, target):
267 """Builds the provided ninja target with or without order_profiling on.
268
269 Args:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame^]270 instrumented: (bool) Whether we want to build an instrumented binary.
271 target: (str) The name of the ninja target to build.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]272 """
273 self._step_recorder.BeginStep('Compile %s' % target)
274
275 # Set the "Release Official" flavor, the parts affecting performance.
276 args = [
277 'is_chrome_branded=true',
278 'is_debug=false',
279 'is_official_build=true',
280 # We have to build with no symbols if profiling and minimal symbols
281 # otherwise for libchrome.so to fit under the 4 GB limit.
282 # crbug.com/574476
283 'symbol_level=' + ('0' if instrumented else '1'),
284 'target_cpu="' + self._arch + '"',
285 'target_os="android"',
286 'use_goma=' + str(self._use_goma).lower(),
287 'use_order_profiling=' + str(instrumented).lower(),
288 ]
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame^]289 if instrumented and self._lightweight_instrumentation:
290 args.append('use_lightweight_order_profiling=true')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]291 if self._goma_dir:
292 args += ['goma_dir="%s"' % self._goma_dir]
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame^]293
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]294 self._step_recorder.RunCommand(
295 ['gn', 'gen', os.path.join(self._out_dir, 'Release'),
296 '--args=' + ' '.join(args)])
297
298 self._step_recorder.RunCommand(
299 ['ninja', '-C', os.path.join(self._out_dir, 'Release'),
300 '-j' + str(self._jobs), '-l' + str(self._max_load), target])
301
302 def CompileChromeApk(self, instrumented, force_relink=False):
303 """Builds a Chrome.apk either with or without order_profiling on.
304
305 Args:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame^]306 instrumented: (bool) Whether to build an instrumented apk.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]307 force_relink: Whether libchromeview.so should be re-created.
308 """
309 if force_relink:
310 self._step_recorder.RunCommand(['rm', '-rf', self.lib_chrome_so])
311 self.Build(instrumented, 'chrome_apk')
312
313 def CompileLibchrome(self, instrumented, force_relink=False):
314 """Builds a libchrome.so either with or without order_profiling on.
315
316 Args:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame^]317 instrumented: (bool) Whether to build an instrumented apk.
318 force_relink: (bool) Whether libchrome.so should be re-created.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]319 """
320 if force_relink:
321 self._step_recorder.RunCommand(['rm', '-rf', self.lib_chrome_so])
322 self.Build(instrumented, 'libchrome')
323
324
325class OrderfileUpdater(object):
326 """Handles uploading and committing a new orderfile in the repository.
327
328 Only used for testing or on a bot.
329 """
330
331 _CLOUD_STORAGE_BUCKET_FOR_DEBUG = None
332 _CLOUD_STORAGE_BUCKET = None
333 _UPLOAD_TO_CLOUD_COMMAND = 'upload_to_google_storage.py'
334
335 def __init__(self, repository_root, step_recorder, branch, netrc):
336 """Constructor.
337
338 Args:
339 repository_root: (str) Root of the target repository.
340 step_recorder: (StepRecorder) Step recorder, for logging.
341 branch: (str) Branch to commit to.
342 netrc: (str) Path to the .netrc file to use.
343 """
344 self._repository_root = repository_root
345 self._step_recorder = step_recorder
346 self._branch = branch
347 self._netrc = netrc
348
349 def CommitFileHashes(self, unpatched_orderfile_filename, orderfile_filename):
350 """Commits unpatched and patched orderfiles hashes, if provided.
351
352 Files must have been successfilly uploaded to cloud storage first.
353
354 Args:
355 unpatched_orderfile_filename: (str or None) Unpatched orderfile path.
356 orderfile_filename: (str or None) Orderfile path.
357
358 Raises:
359 NotImplementedError when the commit logic hasn't been overriden.
360 """
361 files_to_commit = []
362 commit_message_lines = ['Update Orderfile.']
363 for filename in [unpatched_orderfile_filename, orderfile_filename]:
364 if not filename:
365 continue
366 (relative_path, sha1) = self._GetHashFilePathAndContents(filename)
367 commit_message_lines.append('Profile: %s: %s' % (
368 os.path.basename(relative_path), sha1))
369 files_to_commit.append(relative_path)
370 if files_to_commit:
371 self._CommitFiles(files_to_commit, commit_message_lines)
372
373 def UploadToCloudStorage(self, filename, use_debug_location):
374 """Uploads a file to cloud storage.
375
376 Args:
377 filename: (str) File to upload.
378 use_debug_location: (bool) Whether to use the debug location.
379 """
380 bucket = (self._CLOUD_STORAGE_BUCKET_FOR_DEBUG if use_debug_location
381 else self._CLOUD_STORAGE_BUCKET)
382 extension = _GetFileExtension(filename)
383 cmd = [self._UPLOAD_TO_CLOUD_COMMAND, '--bucket', bucket]
384 if extension:
385 cmd.extend(['-z', extension])
386 cmd.append(filename)
387 self._step_recorder.RunCommand(cmd)
388 print 'Download: https://sandbox.google.com/storage/%s/%s' % (
389 bucket, _GenerateHash(filename))
390
391 def _GetHashFilePathAndContents(self, filename):
392 """Gets the name and content of the hash file created from uploading the
393 given file.
394
395 Args:
396 filename: (str) The file that was uploaded to cloud storage.
397
398 Returns:
399 A tuple of the hash file name, relative to the reository root, and the
400 content, which should be the sha1 hash of the file
401 ('base_file.sha1', hash)
402 """
403 abs_hash_filename = filename + '.sha1'
404 rel_hash_filename = os.path.relpath(
405 abs_hash_filename, self._repository_root)
406 with open(abs_hash_filename, 'r') as f:
407 return (rel_hash_filename, f.read())
408
409 def _CommitFiles(self, files_to_commit, commit_message_lines):
410 """Commits a list of files, with a given message."""
411 raise NotImplementedError
412
413
414class OrderfileGenerator(object):
415 """A utility for generating a new orderfile for Clank.
416
417 Builds an instrumented binary, profiles a run of the application, and
418 generates an updated orderfile.
419 """
420 _CLANK_REPO = os.path.join(constants.DIR_SOURCE_ROOT, 'clank')
421 _MERGE_TRACES_SCRIPT = os.path.join(
422 constants.DIR_SOURCE_ROOT, 'tools', 'cygprofile', 'mergetraces.py')
423 _CYGLOG_TO_ORDERFILE_SCRIPT = os.path.join(
424 constants.DIR_SOURCE_ROOT, 'tools', 'cygprofile',
425 'cyglog_to_orderfile.py')
426 _PATCH_ORDERFILE_SCRIPT = os.path.join(
427 constants.DIR_SOURCE_ROOT, 'tools', 'cygprofile', 'patch_orderfile.py')
428 _CHECK_ORDERFILE_SCRIPT = os.path.join(
429 constants.DIR_SOURCE_ROOT, 'tools', 'cygprofile', 'check_orderfile.py')
430 _BUILD_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(
431 constants.GetOutDirectory()))) # Normally /path/to/src
432
433 _UNPATCHED_ORDERFILE_FILENAME = os.path.join(
434 _CLANK_REPO, 'orderfiles', 'unpatched_orderfile.%s')
435 _MERGED_CYGLOG_FILENAME = os.path.join(
436 constants.GetOutDirectory(), 'merged_cyglog')
437 _TEMP_ORDERFILE_FILENAME = os.path.join(
438 constants.GetOutDirectory(), 'tmp_orderfile')
439
440 _PATH_TO_ORDERFILE = os.path.join(_CLANK_REPO, 'orderfiles',
441 'orderfile.%s.out')
442
443 # Previous orderfile_generator debug files would be overwritten.
444 _DIRECTORY_FOR_DEBUG_FILES = '/tmp/orderfile_generator_debug_files'
445
446 def _GetPathToOrderfile(self):
447 """Gets the path to the architecture-specific orderfile."""
448 return self._PATH_TO_ORDERFILE % self._options.arch
449
450 def _GetUnpatchedOrderfileFilename(self):
451 """Gets the path to the architecture-specific unpatched orderfile."""
452 return self._UNPATCHED_ORDERFILE_FILENAME % self._options.arch
453
454 def __init__(self, options, orderfile_updater_class):
455 self._options = options
456
457 self._instrumented_out_dir = os.path.join(
458 self._BUILD_ROOT, self._options.arch + '_instrumented_out')
459 self._uninstrumented_out_dir = os.path.join(
460 self._BUILD_ROOT, self._options.arch + '_uninstrumented_out')
461
462 if options.profile:
463 self._profiler = profile_android_startup.AndroidProfileTool(
464 os.path.join(self._instrumented_out_dir, 'Release'))
465
466 self._output_data = {}
467 self._step_recorder = StepRecorder(options.buildbot)
468 self._compiler = None
469 assert issubclass(orderfile_updater_class, OrderfileUpdater)
470 self._orderfile_updater = orderfile_updater_class(self._CLANK_REPO,
471 self._step_recorder,
472 options.branch,
473 options.netrc)
474 assert os.path.isdir(constants.DIR_SOURCE_ROOT), 'No src directory found'
475
476 def _RunCygprofileUnitTests(self):
477 """Builds, deploys and runs cygprofile_unittests."""
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame^]478 # There an no unittests (yet) for the lightweight instrumentation.
479 # TODO(lizeb): Fix this.
480 if self._options.lightweight_instrumentation:
481 return
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]482 tools_compiler = ClankCompiler(
483 os.path.dirname(constants.GetOutDirectory()),
484 self._step_recorder, self._options.arch, self._options.jobs,
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame^]485 self._options.max_load, self._options.use_goma, self._options.goma_dir,
486 self._options.lightweight_instrumentation)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]487 tools_compiler.Build(instrumented=False, target='android_tools')
488 self._compiler.Build(instrumented=True, target='cygprofile_unittests')
489
490 self._step_recorder.BeginStep('Deploy and run cygprofile_unittests')
491 exit_code = self._profiler.RunCygprofileTests()
492
493 if exit_code != 0:
494 self._step_recorder.FailStep(
495 'cygprofile_unittests exited with non-0 status: %d' % exit_code)
496
497 @staticmethod
498 def _RemoveBlanks(src_file, dest_file):
499 """A utility to remove blank lines from a file.
500
501 Args:
502 src_file: The name of the file to remove the blanks from.
503 dest_file: The name of the file to write the output without blanks.
504 """
505 assert src_file != dest_file, 'Source and destination need to be distinct'
506
507 try:
508 src = open(src_file, 'r')
509 dest = open(dest_file, 'w')
510 for line in src:
511 if line and not line.isspace():
512 dest.write(line)
513 finally:
514 src.close()
515 dest.close()
516
517 def _GenerateAndProcessProfile(self):
518 """Invokes a script to merge the per-thread traces into one file."""
519 self._step_recorder.BeginStep('Generate Profile Data')
520 files = []
521 try:
522 logging.getLogger().setLevel(logging.DEBUG)
523 files = self._profiler.CollectProfile(
524 self._compiler.chrome_apk,
525 constants.PACKAGE_INFO['chrome'])
526 self._step_recorder.BeginStep('Process cyglog')
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame^]527 if self._options.lightweight_instrumentation:
528 assert os.path.exists(self._compiler.lib_chrome_so)
529 process_profiles.GetReachedSymbolsFromDumpsAndMaybeWriteOffsets(
530 files, self._compiler.lib_chrome_so, self._MERGED_CYGLOG_FILENAME)
531 else:
532 with open(self._MERGED_CYGLOG_FILENAME, 'w') as merged_cyglog:
533 self._step_recorder.RunCommand([self._MERGE_TRACES_SCRIPT] + files,
534 constants.DIR_SOURCE_ROOT,
535 stdout=merged_cyglog)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]536 except CommandError:
537 for f in files:
538 self._SaveForDebugging(f)
539 raise
540 finally:
541 self._profiler.Cleanup()
542 logging.getLogger().setLevel(logging.INFO)
543
544 try:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame^]545 command_args = [
546 '--target-arch=' + self._options.arch,
547 '--native-library=' + self._compiler.lib_chrome_so,
548 '--output=' + self._GetUnpatchedOrderfileFilename()]
549 if self._options.lightweight_instrumentation:
550 command_args.append('--reached-offsets=' + self._MERGED_CYGLOG_FILENAME)
551 else:
552 command_args.append('--merged-cyglog=' + self._MERGED_CYGLOG_FILENAME)
553 self._step_recorder.RunCommand(
554 [self._CYGLOG_TO_ORDERFILE_SCRIPT] + command_args)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]555 except CommandError:
556 self._SaveForDebugging(self._MERGED_CYGLOG_FILENAME)
557 self._SaveForDebuggingWithOverwrite(self._compiler.lib_chrome_so)
558 raise
559
560 def _DeleteTempFiles(self):
561 """Deletes intermediate step output files."""
562 print 'Delete %s' % (
563 self._MERGED_CYGLOG_FILENAME)
564 if os.path.isfile(self._MERGED_CYGLOG_FILENAME):
565 os.unlink(self._MERGED_CYGLOG_FILENAME)
566
567 def _PatchOrderfile(self):
568 """Patches the orderfile using clean version of libchrome.so."""
569 self._step_recorder.BeginStep('Patch Orderfile')
570 try:
571 tmp_out = open(self._TEMP_ORDERFILE_FILENAME, 'w')
572 self._step_recorder.RunCommand([self._PATCH_ORDERFILE_SCRIPT,
573 self._GetUnpatchedOrderfileFilename(),
574 self._compiler.lib_chrome_so,
575 '--target-arch=' + self._options.arch],
576 constants.DIR_SOURCE_ROOT, stdout=tmp_out)
577 tmp_out.close()
578
579 self._RemoveBlanks(self._TEMP_ORDERFILE_FILENAME,
580 self._GetPathToOrderfile())
581 except CommandError:
582 self._SaveForDebugging(self._GetUnpatchedOrderfileFilename())
583 self._SaveForDebuggingWithOverwrite(self._compiler.lib_chrome_so)
584 raise
585 finally:
586 tmp_out.close()
587 if os.path.isfile(self._TEMP_ORDERFILE_FILENAME):
588 os.unlink(self._TEMP_ORDERFILE_FILENAME)
589
590 def _VerifySymbolOrder(self):
591 self._step_recorder.BeginStep('Verify Symbol Order')
592 return_code = self._step_recorder.RunCommand(
593 [self._CHECK_ORDERFILE_SCRIPT, self._compiler.lib_chrome_so,
594 self._GetPathToOrderfile(),
595 '--target-arch=' + self._options.arch],
596 constants.DIR_SOURCE_ROOT,
597 raise_on_error=False)
598 if return_code:
599 self._step_recorder.FailStep('Orderfile check returned %d.' % return_code)
600
601 def _RecordHash(self, file_name):
602 """Records the hash of the file into the output_data dictionary."""
603 self._output_data[os.path.basename(file_name) + '.sha1'] = _GenerateHash(
604 file_name)
605
606 def _SaveFileLocally(self, file_name, file_sha1):
607 """Saves the file to a temporary location and prints the sha1sum."""
608 if not os.path.exists(self._DIRECTORY_FOR_DEBUG_FILES):
609 os.makedirs(self._DIRECTORY_FOR_DEBUG_FILES)
610 shutil.copy(file_name, self._DIRECTORY_FOR_DEBUG_FILES)
611 print 'File: %s, saved in: %s, sha1sum: %s' % (
612 file_name, self._DIRECTORY_FOR_DEBUG_FILES, file_sha1)
613
614 def _SaveForDebugging(self, filename):
615 """Uploads the file to cloud storage or saves to a temporary location."""
616 file_sha1 = _GenerateHash(filename)
617 if not self._options.buildbot:
618 self._SaveFileLocally(filename, file_sha1)
619 else:
620 print 'Uploading file for debugging: ' + filename
621 self._orderfile_updater.UploadToCloudStorage(
622 filename, use_debug_location=True)
623
624 def _SaveForDebuggingWithOverwrite(self, file_name):
625 """Uploads and overwrites the file in cloud storage or copies locally.
626
627 Should be used for large binaries like lib_chrome_so.
628
629 Args:
630 file_name: (str) File to upload.
631 """
632 file_sha1 = _GenerateHash(file_name)
633 if not self._options.buildbot:
634 self._SaveFileLocally(file_name, file_sha1)
635 else:
636 print 'Uploading file for debugging: %s, sha1sum: %s' % (
637 file_name, file_sha1)
638 upload_location = '%s/%s' % (
639 self._CLOUD_STORAGE_BUCKET_FOR_DEBUG, os.path.basename(file_name))
640 self._step_recorder.RunCommand([
641 'gsutil.py', 'cp', file_name, 'gs://' + upload_location])
642 print ('Uploaded to: https://sandbox.google.com/storage/' +
643 upload_location)
644
645 def _MaybeArchiveOrderfile(self, filename):
646 """In buildbot configuration, uploads the generated orderfile to
647 Google Cloud Storage.
648
649 Args:
650 filename: (str) Orderfile to upload.
651 """
652 # First compute hashes so that we can download them later if we need to
653 self._step_recorder.BeginStep('Compute hash for ' + filename)
654 self._RecordHash(filename)
655 if self._options.buildbot:
656 self._step_recorder.BeginStep('Archive ' + filename)
657 self._orderfile_updater.UploadToCloudStorage(
658 filename, use_debug_location=False)
659
660 def _GetHashFilePathAndContents(self, base_file):
661 """Gets the name and content of the hash file created from uploading the
662 given file.
663
664 Args:
665 base_file: The file that was uploaded to cloud storage.
666
667 Returns:
668 A tuple of the hash file name, relative to the clank repo path, and the
669 content, which should be the sha1 hash of the file
670 ('base_file.sha1', hash)
671 """
672 abs_file_name = base_file + '.sha1'
673 rel_file_name = os.path.relpath(abs_file_name, self._CLANK_REPO)
674 with open(abs_file_name, 'r') as f:
675 return (rel_file_name, f.read())
676
677 def Generate(self):
678 """Generates and maybe upload an order."""
679 profile_uploaded = False
680 orderfile_uploaded = False
681
682 if self._options.profile:
683 try:
684 _UnstashOutputDirectory(self._instrumented_out_dir)
685 self._compiler = ClankCompiler(
686 self._instrumented_out_dir,
687 self._step_recorder, self._options.arch, self._options.jobs,
688 self._options.max_load, self._options.use_goma,
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame^]689 self._options.goma_dir,
690 self._options.lightweight_instrumentation)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]691 self._RunCygprofileUnitTests()
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame^]692 if self._options.lightweight_instrumentation:
693 _EnsureOrderfileStartsWithAnchorSection(self._GetPathToOrderfile())
694 self._compiler.CompileChromeApk(
695 True, self._options.lightweight_instrumentation)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]696 self._GenerateAndProcessProfile()
697 self._MaybeArchiveOrderfile(self._GetUnpatchedOrderfileFilename())
698 profile_uploaded = True
699 finally:
700 self._DeleteTempFiles()
701 _StashOutputDirectory(self._instrumented_out_dir)
702 if self._options.patch:
703 if self._options.profile:
704 self._RemoveBlanks(self._GetUnpatchedOrderfileFilename(),
705 self._GetPathToOrderfile())
706 try:
707 _UnstashOutputDirectory(self._uninstrumented_out_dir)
708 self._compiler = ClankCompiler(
709 self._uninstrumented_out_dir, self._step_recorder,
710 self._options.arch, self._options.jobs, self._options.max_load,
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame^]711 self._options.use_goma, self._options.goma_dir,
712 self._options.lightweight_instrumentation)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]713 self._compiler.CompileLibchrome(False)
714 self._PatchOrderfile()
715 # Because identical code folding is a bit different with and without
716 # the orderfile build, we need to re-patch the orderfile with code
717 # folding as close to the final version as possible.
718 self._compiler.CompileLibchrome(False, force_relink=True)
719 self._PatchOrderfile()
720 self._compiler.CompileLibchrome(False, force_relink=True)
721 self._VerifySymbolOrder()
722 self._MaybeArchiveOrderfile(self._GetPathToOrderfile())
723 finally:
724 _StashOutputDirectory(self._uninstrumented_out_dir)
725 orderfile_uploaded = True
726
727 if (self._options.buildbot and self._options.netrc
728 and not self._step_recorder.ErrorRecorded()):
729 unpatched_orderfile_filename = (
730 self._GetUnpatchedOrderfileFilename() if profile_uploaded else None)
731 orderfile_filename = (
Benoit Lized913be82017-10-24 13:38:27[diff] [blame]732 self._GetPathToOrderfile() if orderfile_uploaded else None)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]733 self._orderfile_updater.CommitFileHashes(
734 unpatched_orderfile_filename, orderfile_filename)
735
736 self._step_recorder.EndStep()
737 return not self._step_recorder.ErrorRecorded()
738
739 def GetReportingData(self):
740 """Get a dictionary of reporting data (timings, output hashes)"""
741 self._output_data['timings'] = self._step_recorder.timings
742 return self._output_data
743
744
745def CreateOptionParser():
746 parser = optparse.OptionParser()
747 parser.add_option(
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame^]748 '--lightweight-instrumentation', action='store_true', default=False,
749 help='Use the lightweight instrumentation path')
750 parser.add_option(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]751 '--buildbot', action='store_true',
752 help='If true, the script expects to be run on a buildbot')
753 parser.add_option(
754 '--verify', action='store_true',
755 help='If true, the script only verifies the current orderfile')
756 parser.add_option('--target-arch', action='store', dest='arch',
757 default=cygprofile_utils.DetectArchitecture(),
758 choices=['arm', 'arm64', 'x86', 'x86_64', 'x64', 'mips'],
759 help='The target architecture for which to build')
760 parser.add_option('--output-json', action='store', dest='json_file',
761 help='Location to save stats in json format')
762 parser.add_option(
763 '--skip-profile', action='store_false', dest='profile', default=True,
764 help='Don\'t generate a profile on the device. Only patch from the '
765 'existing profile.')
766 parser.add_option(
767 '--skip-patch', action='store_false', dest='patch', default=True,
768 help='Only generate the raw (unpatched) orderfile, don\'t patch it.')
769 parser.add_option(
770 '--netrc', action='store',
771 help='A custom .netrc file to use for git checkin. Only used on bots.')
772 parser.add_option(
773 '--branch', action='store', default='master',
774 help='When running on buildbot with a netrc, the branch orderfile '
775 'hashes get checked into.')
776 # Note: -j50 was causing issues on the bot.
777 parser.add_option(
778 '-j', '--jobs', action='store', default=20,
779 help='Number of jobs to use for compilation.')
780 parser.add_option(
781 '-l', '--max-load', action='store', default=4, help='Max cpu load.')
782 parser.add_option('--goma-dir', help='GOMA directory.')
783 parser.add_option(
784 '--use-goma', action='store_true', help='Enable GOMA.', default=False)
785 parser.add_option('--adb-path', help='Path to the adb binary.')
786 return parser
787
788
789def CreateOrderfile(options, orderfile_updater_class):
790 """Creates an oderfile.
791
792 Args:
793 options: As returned from optparse.OptionParser.parse_args()
794 orderfile_updater_class: (OrderfileUpdater) subclass of OrderfileUpdater.
795
796 Returns:
797 True iff success.
798 """
799 logging.basicConfig(level=logging.INFO)
800 devil_chromium.Initialize(adb_path=options.adb_path)
801
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]802 generator = OrderfileGenerator(options, orderfile_updater_class)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]803 try:
804 if options.verify:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]805 generator._VerifySymbolOrder()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]806 else:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]807 return generator.Generate()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]808 finally:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]809 json_output = json.dumps(generator.GetReportingData(),
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]810 indent=2) + '\n'
811 if options.json_file:
812 with open(options.json_file, 'w') as f:
813 f.write(json_output)
814 print json_output
815 return False
816
817
818def main(argv):
819 parser = CreateOptionParser()
820 options, _ = parser.parse_args(argv)
821 return 0 if CreateOrderfile(options, OrderfileUpdater) else 1
822
823
824if __name__ == '__main__':
825 sys.exit(main(sys.argv))