[go: nahoru, domu]
Google Git
Sign in
chromium / chromium / src / fefbb27ca4d2401bd7fc653823ada3c958af9f7d / . / tools / cygprofile / orderfile_generator_backend.py
blob: 1b44988fcab505060b31164ace5d88664ecda245 [file] [log] [blame]
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1#!/usr/bin/env python
2# Copyright (c) 2013 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6""" A utility to generate an up-to-date orderfile.
7
8The orderfile is used by the linker to order text sections such that the
9sections are placed consecutively in the order specified. This allows us
10to page in less code during start-up.
11
12Example usage:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]13 tools/cygprofile/orderfile_generator_backend.py -l 20 -j 1000 --use-goma \
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]14 --target-arch=arm
15"""
16
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]17import argparse
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]18import hashlib
19import json
20import logging
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]21import os
22import re
23import shutil
24import subprocess
25import sys
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]26import tempfile
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]27import time
28
29import cygprofile_utils
Benoit Lizefefbb27c2018-01-17 13:54:18[diff] [blame^]30import patch_orderfile
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]31import process_profiles
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]32import profile_android_startup
Benoit Lizefefbb27c2018-01-17 13:54:18[diff] [blame^]33import symbol_extractor
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]34
35
36_SRC_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)),
37 os.pardir, os.pardir)
38sys.path.append(os.path.join(_SRC_PATH, 'build', 'android'))
39import devil_chromium
40from pylib import constants
41
42
43# Needs to happen early for GetBuildType()/GetOutDirectory() to work correctly
44constants.SetBuildType('Release')
45
46
47class CommandError(Exception):
48 """Indicates that a dispatched shell command exited with a non-zero status."""
49
50 def __init__(self, value):
51 super(CommandError, self).__init__()
52 self.value = value
53
54 def __str__(self):
55 return repr(self.value)
56
57
58def _GenerateHash(file_path):
59 """Calculates and returns the hash of the file at file_path."""
60 sha1 = hashlib.sha1()
61 with open(file_path, 'rb') as f:
62 while True:
63 # Read in 1mb chunks, so it doesn't all have to be loaded into memory.
64 chunk = f.read(1024 * 1024)
65 if not chunk:
66 break
67 sha1.update(chunk)
68 return sha1.hexdigest()
69
70
71def _GetFileExtension(file_name):
72 """Calculates the file extension from a file name.
73
74 Args:
75 file_name: The source file name.
76 Returns:
77 The part of file_name after the dot (.) or None if the file has no
78 extension.
79 Examples: /home/user/foo.bar -> bar
80 /home/user.name/foo -> None
81 /home/user/.foo -> None
82 /home/user/foo.bar.baz -> baz
83 """
84 file_name_parts = os.path.basename(file_name).split('.')
85 if len(file_name_parts) > 1:
86 return file_name_parts[-1]
87 else:
88 return None
89
90
91def _StashOutputDirectory(buildpath):
92 """Takes the output directory and stashes it in the default output directory.
93
94 This allows it to be used for incremental builds next time (after unstashing)
95 by keeping it in a place that isn't deleted normally, while also ensuring
96 that it is properly clobbered when appropriate.
97
98 This is a dirty hack to deal with the needs of clobbering while also handling
99 incremental builds and the hardcoded relative paths used in some of the
100 project files.
101
102 Args:
103 buildpath: The path where the building happens. If this corresponds to the
104 default output directory, no action is taken.
105 """
106 if os.path.abspath(buildpath) == os.path.abspath(os.path.dirname(
107 constants.GetOutDirectory())):
108 return
109 name = os.path.basename(buildpath)
110 stashpath = os.path.join(constants.GetOutDirectory(), name)
111 if not os.path.exists(buildpath):
112 return
113 if os.path.exists(stashpath):
114 shutil.rmtree(stashpath, ignore_errors=True)
115 shutil.move(buildpath, stashpath)
116
117
118def _UnstashOutputDirectory(buildpath):
119 """Inverse of _StashOutputDirectory.
120
121 Moves the output directory stashed within the default output directory
122 (out/Release) to the position where the builds can actually happen.
123
124 This is a dirty hack to deal with the needs of clobbering while also handling
125 incremental builds and the hardcoded relative paths used in some of the
126 project files.
127
128 Args:
129 buildpath: The path where the building happens. If this corresponds to the
130 default output directory, no action is taken.
131 """
132 if os.path.abspath(buildpath) == os.path.abspath(os.path.dirname(
133 constants.GetOutDirectory())):
134 return
135 name = os.path.basename(buildpath)
136 stashpath = os.path.join(constants.GetOutDirectory(), name)
137 if not os.path.exists(stashpath):
138 return
139 if os.path.exists(buildpath):
140 shutil.rmtree(buildpath, ignore_errors=True)
141 shutil.move(stashpath, buildpath)
142
143
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]144def _EnsureOrderfileStartsWithAnchorSection(filename):
145 """Ensures that the orderfile starts with the right anchor symbol.
146
147 This changes the orderfile, if required.
148
149 Args:
150 filename: (str) Path to the orderfile.
151 """
152 anchor_section = '.text.dummy_function_to_anchor_text'
153 with open(filename, 'r') as f:
154 if f.readline().strip() == anchor_section:
155 return
156 try:
157 f = tempfile.NamedTemporaryFile(dir=os.path.dirname(filename), delete=False)
158 f.write(anchor_section + '\n')
159 with open(filename, 'r') as orderfile_file:
160 for line in orderfile_file:
161 f.write(line + '\n')
162 f.close()
163 os.rename(f.name, filename)
164 finally:
165 if os.path.exists(f.name):
166 os.remove(f.name)
167
168
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]169class StepRecorder(object):
170 """Records steps and timings."""
171
172 def __init__(self, buildbot):
173 self.timings = []
174 self._previous_step = ('', 0.0)
175 self._buildbot = buildbot
176 self._error_recorded = False
177
178 def BeginStep(self, name):
179 """Marks a beginning of the next step in the script.
180
181 On buildbot, this prints a specially formatted name that will show up
182 in the waterfall. Otherwise, just prints the step name.
183
184 Args:
185 name: The name of the step.
186 """
187 self.EndStep()
188 self._previous_step = (name, time.time())
189 print 'Running step: ', name
190
191 def EndStep(self):
192 """Records successful completion of the current step.
193
194 This is optional if the step is immediately followed by another BeginStep.
195 """
196 if self._previous_step[0]:
197 elapsed = time.time() - self._previous_step[1]
198 print 'Step %s took %f seconds' % (self._previous_step[0], elapsed)
199 self.timings.append((self._previous_step[0], elapsed))
200
201 self._previous_step = ('', 0.0)
202
203 def FailStep(self, message=None):
204 """Marks that a particular step has failed.
205
206 On buildbot, this will mark the current step as failed on the waterfall.
207 Otherwise we will just print an optional failure message.
208
209 Args:
210 message: An optional explanation as to why the step failed.
211 """
212 print 'STEP FAILED!!'
213 if message:
214 print message
215 self._error_recorded = True
216 self.EndStep()
217
218 def ErrorRecorded(self):
219 """True if FailStep has been called."""
220 return self._error_recorded
221
222 def RunCommand(self, cmd, cwd=constants.DIR_SOURCE_ROOT, raise_on_error=True,
223 stdout=None):
224 """Execute a shell command.
225
226 Args:
227 cmd: A list of command strings.
228 cwd: Directory in which the command should be executed, defaults to script
229 location if not specified.
230 raise_on_error: If true will raise a CommandError if the call doesn't
231 succeed and mark the step as failed.
232 stdout: A file to redirect stdout for the command to.
233
234 Returns:
235 The process's return code.
236
237 Raises:
238 CommandError: An error executing the specified command.
239 """
240 print 'Executing %s in %s' % (' '.join(cmd), cwd)
241 process = subprocess.Popen(cmd, stdout=stdout, cwd=cwd, env=os.environ)
242 process.wait()
243 if raise_on_error and process.returncode != 0:
244 self.FailStep()
245 raise CommandError('Exception executing command %s' % ' '.join(cmd))
246 return process.returncode
247
248
249class ClankCompiler(object):
250 """Handles compilation of clank."""
251
252 def __init__(self, out_dir, step_recorder, arch, jobs, max_load, use_goma,
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]253 goma_dir, lightweight_instrumentation):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]254 self._out_dir = out_dir
255 self._step_recorder = step_recorder
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]256 self._arch = arch
257 self._jobs = jobs
258 self._max_load = max_load
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]259 self._use_goma = use_goma
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]260 self._goma_dir = goma_dir
261 self._lightweight_instrumentation = lightweight_instrumentation
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]262 lib_chrome_so_dir = 'lib.unstripped'
263 self.lib_chrome_so = os.path.join(
264 self._out_dir, 'Release', lib_chrome_so_dir, 'libchrome.so')
265 self.chrome_apk = os.path.join(
266 self._out_dir, 'Release', 'apks', 'Chrome.apk')
267
268 def Build(self, instrumented, target):
269 """Builds the provided ninja target with or without order_profiling on.
270
271 Args:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]272 instrumented: (bool) Whether we want to build an instrumented binary.
273 target: (str) The name of the ninja target to build.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]274 """
275 self._step_recorder.BeginStep('Compile %s' % target)
276
277 # Set the "Release Official" flavor, the parts affecting performance.
278 args = [
279 'is_chrome_branded=true',
280 'is_debug=false',
281 'is_official_build=true',
282 # We have to build with no symbols if profiling and minimal symbols
283 # otherwise for libchrome.so to fit under the 4 GB limit.
284 # crbug.com/574476
285 'symbol_level=' + ('0' if instrumented else '1'),
286 'target_cpu="' + self._arch + '"',
287 'target_os="android"',
288 'use_goma=' + str(self._use_goma).lower(),
289 'use_order_profiling=' + str(instrumented).lower(),
290 ]
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]291 if instrumented and self._lightweight_instrumentation:
292 args.append('use_lightweight_order_profiling=true')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]293 if self._goma_dir:
294 args += ['goma_dir="%s"' % self._goma_dir]
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]295
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]296 self._step_recorder.RunCommand(
297 ['gn', 'gen', os.path.join(self._out_dir, 'Release'),
298 '--args=' + ' '.join(args)])
299
300 self._step_recorder.RunCommand(
301 ['ninja', '-C', os.path.join(self._out_dir, 'Release'),
302 '-j' + str(self._jobs), '-l' + str(self._max_load), target])
303
304 def CompileChromeApk(self, instrumented, force_relink=False):
305 """Builds a Chrome.apk either with or without order_profiling on.
306
307 Args:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]308 instrumented: (bool) Whether to build an instrumented apk.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]309 force_relink: Whether libchromeview.so should be re-created.
310 """
311 if force_relink:
312 self._step_recorder.RunCommand(['rm', '-rf', self.lib_chrome_so])
313 self.Build(instrumented, 'chrome_apk')
314
315 def CompileLibchrome(self, instrumented, force_relink=False):
316 """Builds a libchrome.so either with or without order_profiling on.
317
318 Args:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]319 instrumented: (bool) Whether to build an instrumented apk.
320 force_relink: (bool) Whether libchrome.so should be re-created.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]321 """
322 if force_relink:
323 self._step_recorder.RunCommand(['rm', '-rf', self.lib_chrome_so])
324 self.Build(instrumented, 'libchrome')
325
326
327class OrderfileUpdater(object):
328 """Handles uploading and committing a new orderfile in the repository.
329
330 Only used for testing or on a bot.
331 """
332
333 _CLOUD_STORAGE_BUCKET_FOR_DEBUG = None
334 _CLOUD_STORAGE_BUCKET = None
335 _UPLOAD_TO_CLOUD_COMMAND = 'upload_to_google_storage.py'
336
337 def __init__(self, repository_root, step_recorder, branch, netrc):
338 """Constructor.
339
340 Args:
341 repository_root: (str) Root of the target repository.
342 step_recorder: (StepRecorder) Step recorder, for logging.
343 branch: (str) Branch to commit to.
344 netrc: (str) Path to the .netrc file to use.
345 """
346 self._repository_root = repository_root
347 self._step_recorder = step_recorder
348 self._branch = branch
349 self._netrc = netrc
350
351 def CommitFileHashes(self, unpatched_orderfile_filename, orderfile_filename):
352 """Commits unpatched and patched orderfiles hashes, if provided.
353
354 Files must have been successfilly uploaded to cloud storage first.
355
356 Args:
357 unpatched_orderfile_filename: (str or None) Unpatched orderfile path.
358 orderfile_filename: (str or None) Orderfile path.
359
360 Raises:
361 NotImplementedError when the commit logic hasn't been overriden.
362 """
363 files_to_commit = []
364 commit_message_lines = ['Update Orderfile.']
365 for filename in [unpatched_orderfile_filename, orderfile_filename]:
366 if not filename:
367 continue
368 (relative_path, sha1) = self._GetHashFilePathAndContents(filename)
369 commit_message_lines.append('Profile: %s: %s' % (
370 os.path.basename(relative_path), sha1))
371 files_to_commit.append(relative_path)
372 if files_to_commit:
373 self._CommitFiles(files_to_commit, commit_message_lines)
374
375 def UploadToCloudStorage(self, filename, use_debug_location):
376 """Uploads a file to cloud storage.
377
378 Args:
379 filename: (str) File to upload.
380 use_debug_location: (bool) Whether to use the debug location.
381 """
382 bucket = (self._CLOUD_STORAGE_BUCKET_FOR_DEBUG if use_debug_location
383 else self._CLOUD_STORAGE_BUCKET)
384 extension = _GetFileExtension(filename)
385 cmd = [self._UPLOAD_TO_CLOUD_COMMAND, '--bucket', bucket]
386 if extension:
387 cmd.extend(['-z', extension])
388 cmd.append(filename)
389 self._step_recorder.RunCommand(cmd)
390 print 'Download: https://sandbox.google.com/storage/%s/%s' % (
391 bucket, _GenerateHash(filename))
392
393 def _GetHashFilePathAndContents(self, filename):
394 """Gets the name and content of the hash file created from uploading the
395 given file.
396
397 Args:
398 filename: (str) The file that was uploaded to cloud storage.
399
400 Returns:
401 A tuple of the hash file name, relative to the reository root, and the
402 content, which should be the sha1 hash of the file
403 ('base_file.sha1', hash)
404 """
405 abs_hash_filename = filename + '.sha1'
406 rel_hash_filename = os.path.relpath(
407 abs_hash_filename, self._repository_root)
408 with open(abs_hash_filename, 'r') as f:
409 return (rel_hash_filename, f.read())
410
411 def _CommitFiles(self, files_to_commit, commit_message_lines):
412 """Commits a list of files, with a given message."""
413 raise NotImplementedError
414
415
416class OrderfileGenerator(object):
417 """A utility for generating a new orderfile for Clank.
418
419 Builds an instrumented binary, profiles a run of the application, and
420 generates an updated orderfile.
421 """
422 _CLANK_REPO = os.path.join(constants.DIR_SOURCE_ROOT, 'clank')
423 _MERGE_TRACES_SCRIPT = os.path.join(
424 constants.DIR_SOURCE_ROOT, 'tools', 'cygprofile', 'mergetraces.py')
425 _CYGLOG_TO_ORDERFILE_SCRIPT = os.path.join(
426 constants.DIR_SOURCE_ROOT, 'tools', 'cygprofile',
427 'cyglog_to_orderfile.py')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]428 _CHECK_ORDERFILE_SCRIPT = os.path.join(
429 constants.DIR_SOURCE_ROOT, 'tools', 'cygprofile', 'check_orderfile.py')
430 _BUILD_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(
431 constants.GetOutDirectory()))) # Normally /path/to/src
432
433 _UNPATCHED_ORDERFILE_FILENAME = os.path.join(
434 _CLANK_REPO, 'orderfiles', 'unpatched_orderfile.%s')
435 _MERGED_CYGLOG_FILENAME = os.path.join(
436 constants.GetOutDirectory(), 'merged_cyglog')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]437
438 _PATH_TO_ORDERFILE = os.path.join(_CLANK_REPO, 'orderfiles',
439 'orderfile.%s.out')
440
441 # Previous orderfile_generator debug files would be overwritten.
442 _DIRECTORY_FOR_DEBUG_FILES = '/tmp/orderfile_generator_debug_files'
443
444 def _GetPathToOrderfile(self):
445 """Gets the path to the architecture-specific orderfile."""
446 return self._PATH_TO_ORDERFILE % self._options.arch
447
448 def _GetUnpatchedOrderfileFilename(self):
449 """Gets the path to the architecture-specific unpatched orderfile."""
450 return self._UNPATCHED_ORDERFILE_FILENAME % self._options.arch
451
452 def __init__(self, options, orderfile_updater_class):
453 self._options = options
454
455 self._instrumented_out_dir = os.path.join(
456 self._BUILD_ROOT, self._options.arch + '_instrumented_out')
457 self._uninstrumented_out_dir = os.path.join(
458 self._BUILD_ROOT, self._options.arch + '_uninstrumented_out')
459
460 if options.profile:
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]461 output_directory = os.path.join(self._instrumented_out_dir, 'Release')
462 host_cyglog_dir = os.path.join(output_directory, 'cyglog_data')
463 # Only override the defaults when using lightweight instrumentation,
464 # as the regular profiling code is likely too slow for these.
465 urls = [profile_android_startup.AndroidProfileTool.TEST_URL]
466 use_wpr = True
467 simulate_user = False
468 if options.simulate_user and not options.lightweight_instrumentation:
469 logging.error(
470 '--simulate-user required --lightweight-instrumentation, ignoring.')
471 if options.lightweight_instrumentation:
472 urls = options.urls
473 use_wpr = not options.no_wpr
474 simulate_user = options.simulate_user
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]475 self._profiler = profile_android_startup.AndroidProfileTool(
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]476 output_directory, host_cyglog_dir, use_wpr, urls, simulate_user)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]477
478 self._output_data = {}
479 self._step_recorder = StepRecorder(options.buildbot)
480 self._compiler = None
481 assert issubclass(orderfile_updater_class, OrderfileUpdater)
482 self._orderfile_updater = orderfile_updater_class(self._CLANK_REPO,
483 self._step_recorder,
484 options.branch,
485 options.netrc)
486 assert os.path.isdir(constants.DIR_SOURCE_ROOT), 'No src directory found'
Benoit Lizefefbb27c2018-01-17 13:54:18[diff] [blame^]487 symbol_extractor.SetArchitecture(options.arch)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]488
489 def _RunCygprofileUnitTests(self):
490 """Builds, deploys and runs cygprofile_unittests."""
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]491 # There an no unittests (yet) for the lightweight instrumentation.
492 # TODO(lizeb): Fix this.
493 if self._options.lightweight_instrumentation:
494 return
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]495 tools_compiler = ClankCompiler(
496 os.path.dirname(constants.GetOutDirectory()),
497 self._step_recorder, self._options.arch, self._options.jobs,
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]498 self._options.max_load, self._options.use_goma, self._options.goma_dir,
499 self._options.lightweight_instrumentation)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]500 tools_compiler.Build(instrumented=False, target='android_tools')
501 self._compiler.Build(instrumented=True, target='cygprofile_unittests')
502
503 self._step_recorder.BeginStep('Deploy and run cygprofile_unittests')
504 exit_code = self._profiler.RunCygprofileTests()
505
506 if exit_code != 0:
507 self._step_recorder.FailStep(
508 'cygprofile_unittests exited with non-0 status: %d' % exit_code)
509
510 @staticmethod
511 def _RemoveBlanks(src_file, dest_file):
512 """A utility to remove blank lines from a file.
513
514 Args:
515 src_file: The name of the file to remove the blanks from.
516 dest_file: The name of the file to write the output without blanks.
517 """
518 assert src_file != dest_file, 'Source and destination need to be distinct'
519
520 try:
521 src = open(src_file, 'r')
522 dest = open(dest_file, 'w')
523 for line in src:
524 if line and not line.isspace():
525 dest.write(line)
526 finally:
527 src.close()
528 dest.close()
529
530 def _GenerateAndProcessProfile(self):
531 """Invokes a script to merge the per-thread traces into one file."""
532 self._step_recorder.BeginStep('Generate Profile Data')
533 files = []
534 try:
535 logging.getLogger().setLevel(logging.DEBUG)
536 files = self._profiler.CollectProfile(
537 self._compiler.chrome_apk,
538 constants.PACKAGE_INFO['chrome'])
539 self._step_recorder.BeginStep('Process cyglog')
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]540 if self._options.lightweight_instrumentation:
541 assert os.path.exists(self._compiler.lib_chrome_so)
542 process_profiles.GetReachedSymbolsFromDumpsAndMaybeWriteOffsets(
543 files, self._compiler.lib_chrome_so, self._MERGED_CYGLOG_FILENAME)
544 else:
545 with open(self._MERGED_CYGLOG_FILENAME, 'w') as merged_cyglog:
546 self._step_recorder.RunCommand([self._MERGE_TRACES_SCRIPT] + files,
547 constants.DIR_SOURCE_ROOT,
548 stdout=merged_cyglog)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]549 except CommandError:
550 for f in files:
551 self._SaveForDebugging(f)
552 raise
553 finally:
554 self._profiler.Cleanup()
555 logging.getLogger().setLevel(logging.INFO)
556
557 try:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]558 command_args = [
559 '--target-arch=' + self._options.arch,
560 '--native-library=' + self._compiler.lib_chrome_so,
561 '--output=' + self._GetUnpatchedOrderfileFilename()]
562 if self._options.lightweight_instrumentation:
563 command_args.append('--reached-offsets=' + self._MERGED_CYGLOG_FILENAME)
564 else:
565 command_args.append('--merged-cyglog=' + self._MERGED_CYGLOG_FILENAME)
566 self._step_recorder.RunCommand(
567 [self._CYGLOG_TO_ORDERFILE_SCRIPT] + command_args)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]568 except CommandError:
569 self._SaveForDebugging(self._MERGED_CYGLOG_FILENAME)
570 self._SaveForDebuggingWithOverwrite(self._compiler.lib_chrome_so)
571 raise
572
573 def _DeleteTempFiles(self):
574 """Deletes intermediate step output files."""
575 print 'Delete %s' % (
576 self._MERGED_CYGLOG_FILENAME)
577 if os.path.isfile(self._MERGED_CYGLOG_FILENAME):
578 os.unlink(self._MERGED_CYGLOG_FILENAME)
579
580 def _PatchOrderfile(self):
581 """Patches the orderfile using clean version of libchrome.so."""
582 self._step_recorder.BeginStep('Patch Orderfile')
Benoit Lizefefbb27c2018-01-17 13:54:18[diff] [blame^]583 patch_orderfile.GeneratePatchedOrderfile(
584 self._GetUnpatchedOrderfileFilename(), self._compiler.lib_chrome_so,
585 self._GetPathToOrderfile())
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]586
587 def _VerifySymbolOrder(self):
588 self._step_recorder.BeginStep('Verify Symbol Order')
589 return_code = self._step_recorder.RunCommand(
590 [self._CHECK_ORDERFILE_SCRIPT, self._compiler.lib_chrome_so,
591 self._GetPathToOrderfile(),
592 '--target-arch=' + self._options.arch],
593 constants.DIR_SOURCE_ROOT,
594 raise_on_error=False)
595 if return_code:
596 self._step_recorder.FailStep('Orderfile check returned %d.' % return_code)
597
598 def _RecordHash(self, file_name):
599 """Records the hash of the file into the output_data dictionary."""
600 self._output_data[os.path.basename(file_name) + '.sha1'] = _GenerateHash(
601 file_name)
602
603 def _SaveFileLocally(self, file_name, file_sha1):
604 """Saves the file to a temporary location and prints the sha1sum."""
605 if not os.path.exists(self._DIRECTORY_FOR_DEBUG_FILES):
606 os.makedirs(self._DIRECTORY_FOR_DEBUG_FILES)
607 shutil.copy(file_name, self._DIRECTORY_FOR_DEBUG_FILES)
608 print 'File: %s, saved in: %s, sha1sum: %s' % (
609 file_name, self._DIRECTORY_FOR_DEBUG_FILES, file_sha1)
610
611 def _SaveForDebugging(self, filename):
612 """Uploads the file to cloud storage or saves to a temporary location."""
613 file_sha1 = _GenerateHash(filename)
614 if not self._options.buildbot:
615 self._SaveFileLocally(filename, file_sha1)
616 else:
617 print 'Uploading file for debugging: ' + filename
618 self._orderfile_updater.UploadToCloudStorage(
619 filename, use_debug_location=True)
620
621 def _SaveForDebuggingWithOverwrite(self, file_name):
622 """Uploads and overwrites the file in cloud storage or copies locally.
623
624 Should be used for large binaries like lib_chrome_so.
625
626 Args:
627 file_name: (str) File to upload.
628 """
629 file_sha1 = _GenerateHash(file_name)
630 if not self._options.buildbot:
631 self._SaveFileLocally(file_name, file_sha1)
632 else:
633 print 'Uploading file for debugging: %s, sha1sum: %s' % (
634 file_name, file_sha1)
635 upload_location = '%s/%s' % (
636 self._CLOUD_STORAGE_BUCKET_FOR_DEBUG, os.path.basename(file_name))
637 self._step_recorder.RunCommand([
638 'gsutil.py', 'cp', file_name, 'gs://' + upload_location])
639 print ('Uploaded to: https://sandbox.google.com/storage/' +
640 upload_location)
641
642 def _MaybeArchiveOrderfile(self, filename):
643 """In buildbot configuration, uploads the generated orderfile to
644 Google Cloud Storage.
645
646 Args:
647 filename: (str) Orderfile to upload.
648 """
649 # First compute hashes so that we can download them later if we need to
650 self._step_recorder.BeginStep('Compute hash for ' + filename)
651 self._RecordHash(filename)
652 if self._options.buildbot:
653 self._step_recorder.BeginStep('Archive ' + filename)
654 self._orderfile_updater.UploadToCloudStorage(
655 filename, use_debug_location=False)
656
657 def _GetHashFilePathAndContents(self, base_file):
658 """Gets the name and content of the hash file created from uploading the
659 given file.
660
661 Args:
662 base_file: The file that was uploaded to cloud storage.
663
664 Returns:
665 A tuple of the hash file name, relative to the clank repo path, and the
666 content, which should be the sha1 hash of the file
667 ('base_file.sha1', hash)
668 """
669 abs_file_name = base_file + '.sha1'
670 rel_file_name = os.path.relpath(abs_file_name, self._CLANK_REPO)
671 with open(abs_file_name, 'r') as f:
672 return (rel_file_name, f.read())
673
674 def Generate(self):
675 """Generates and maybe upload an order."""
676 profile_uploaded = False
677 orderfile_uploaded = False
678
679 if self._options.profile:
680 try:
681 _UnstashOutputDirectory(self._instrumented_out_dir)
682 self._compiler = ClankCompiler(
683 self._instrumented_out_dir,
684 self._step_recorder, self._options.arch, self._options.jobs,
685 self._options.max_load, self._options.use_goma,
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]686 self._options.goma_dir,
687 self._options.lightweight_instrumentation)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]688 self._RunCygprofileUnitTests()
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]689 if self._options.lightweight_instrumentation:
690 _EnsureOrderfileStartsWithAnchorSection(self._GetPathToOrderfile())
691 self._compiler.CompileChromeApk(
692 True, self._options.lightweight_instrumentation)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]693 self._GenerateAndProcessProfile()
694 self._MaybeArchiveOrderfile(self._GetUnpatchedOrderfileFilename())
695 profile_uploaded = True
696 finally:
697 self._DeleteTempFiles()
698 _StashOutputDirectory(self._instrumented_out_dir)
699 if self._options.patch:
700 if self._options.profile:
701 self._RemoveBlanks(self._GetUnpatchedOrderfileFilename(),
702 self._GetPathToOrderfile())
703 try:
704 _UnstashOutputDirectory(self._uninstrumented_out_dir)
705 self._compiler = ClankCompiler(
706 self._uninstrumented_out_dir, self._step_recorder,
707 self._options.arch, self._options.jobs, self._options.max_load,
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]708 self._options.use_goma, self._options.goma_dir,
709 self._options.lightweight_instrumentation)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]710 self._compiler.CompileLibchrome(False)
711 self._PatchOrderfile()
712 # Because identical code folding is a bit different with and without
713 # the orderfile build, we need to re-patch the orderfile with code
714 # folding as close to the final version as possible.
715 self._compiler.CompileLibchrome(False, force_relink=True)
716 self._PatchOrderfile()
717 self._compiler.CompileLibchrome(False, force_relink=True)
718 self._VerifySymbolOrder()
719 self._MaybeArchiveOrderfile(self._GetPathToOrderfile())
720 finally:
721 _StashOutputDirectory(self._uninstrumented_out_dir)
722 orderfile_uploaded = True
723
724 if (self._options.buildbot and self._options.netrc
725 and not self._step_recorder.ErrorRecorded()):
726 unpatched_orderfile_filename = (
727 self._GetUnpatchedOrderfileFilename() if profile_uploaded else None)
728 orderfile_filename = (
Benoit Lized913be82017-10-24 13:38:27[diff] [blame]729 self._GetPathToOrderfile() if orderfile_uploaded else None)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]730 self._orderfile_updater.CommitFileHashes(
731 unpatched_orderfile_filename, orderfile_filename)
732
733 self._step_recorder.EndStep()
734 return not self._step_recorder.ErrorRecorded()
735
736 def GetReportingData(self):
737 """Get a dictionary of reporting data (timings, output hashes)"""
738 self._output_data['timings'] = self._step_recorder.timings
739 return self._output_data
740
741
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]742def CreateArgumentParser():
743 """Creates and returns the argument parser."""
744 parser = argparse.ArgumentParser()
745 parser.add_argument(
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]746 '--lightweight-instrumentation', action='store_true', default=False,
747 help='Use the lightweight instrumentation path')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]748 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]749 '--buildbot', action='store_true',
750 help='If true, the script expects to be run on a buildbot')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]751 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]752 '--verify', action='store_true',
753 help='If true, the script only verifies the current orderfile')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]754 parser.add_argument('--target-arch', action='store', dest='arch',
755 default=cygprofile_utils.DetectArchitecture(),
756 choices=['arm', 'arm64', 'x86', 'x86_64', 'x64', 'mips'],
757 help='The target architecture for which to build')
758 parser.add_argument('--output-json', action='store', dest='json_file',
759 help='Location to save stats in json format')
760 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]761 '--skip-profile', action='store_false', dest='profile', default=True,
762 help='Don\'t generate a profile on the device. Only patch from the '
763 'existing profile.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]764 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]765 '--skip-patch', action='store_false', dest='patch', default=True,
766 help='Only generate the raw (unpatched) orderfile, don\'t patch it.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]767 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]768 '--netrc', action='store',
769 help='A custom .netrc file to use for git checkin. Only used on bots.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]770 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]771 '--branch', action='store', default='master',
772 help='When running on buildbot with a netrc, the branch orderfile '
773 'hashes get checked into.')
774 # Note: -j50 was causing issues on the bot.
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]775 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]776 '-j', '--jobs', action='store', default=20,
777 help='Number of jobs to use for compilation.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]778 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]779 '-l', '--max-load', action='store', default=4, help='Max cpu load.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]780 parser.add_argument('--goma-dir', help='GOMA directory.')
781 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]782 '--use-goma', action='store_true', help='Enable GOMA.', default=False)
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]783 parser.add_argument('--adb-path', help='Path to the adb binary.')
784 profile_android_startup.AddProfileCollectionArguments(parser)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]785 return parser
786
787
788def CreateOrderfile(options, orderfile_updater_class):
789 """Creates an oderfile.
790
791 Args:
792 options: As returned from optparse.OptionParser.parse_args()
793 orderfile_updater_class: (OrderfileUpdater) subclass of OrderfileUpdater.
794
795 Returns:
796 True iff success.
797 """
798 logging.basicConfig(level=logging.INFO)
799 devil_chromium.Initialize(adb_path=options.adb_path)
800
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]801 generator = OrderfileGenerator(options, orderfile_updater_class)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]802 try:
803 if options.verify:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]804 generator._VerifySymbolOrder()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]805 else:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]806 return generator.Generate()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]807 finally:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]808 json_output = json.dumps(generator.GetReportingData(),
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]809 indent=2) + '\n'
810 if options.json_file:
811 with open(options.json_file, 'w') as f:
812 f.write(json_output)
813 print json_output
814 return False
815
816
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]817def main():
818 parser = CreateArgumentParser()
819 options = parser.parse_args()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]820 return 0 if CreateOrderfile(options, OrderfileUpdater) else 1
821
822
823if __name__ == '__main__':
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]824 sys.exit(main())