[go: nahoru, domu]
Google Git
Sign in
chromium / chromium / src / 964668123b69ad0bf8852c8ffaacfadbb6182b15 / . / tools / cygprofile / orderfile_generator_backend.py
blob: 96348a589cc204638ce9bd484374f64fd0d48427 [file] [log] [blame]
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]1#!/usr/bin/env python
2# Copyright (c) 2013 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6""" A utility to generate an up-to-date orderfile.
7
8The orderfile is used by the linker to order text sections such that the
9sections are placed consecutively in the order specified. This allows us
10to page in less code during start-up.
11
12Example usage:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]13 tools/cygprofile/orderfile_generator_backend.py -l 20 -j 1000 --use-goma \
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]14 --target-arch=arm
15"""
16
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]17import argparse
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]18import hashlib
19import json
20import logging
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]21import os
22import re
23import shutil
24import subprocess
25import sys
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]26import tempfile
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]27import time
28
29import cygprofile_utils
Benoit Lizefefbb27c2018-01-17 13:54:18[diff] [blame]30import patch_orderfile
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]31import process_profiles
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]32import profile_android_startup
Benoit Lizefefbb27c2018-01-17 13:54:18[diff] [blame]33import symbol_extractor
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]34
35
36_SRC_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)),
37 os.pardir, os.pardir)
38sys.path.append(os.path.join(_SRC_PATH, 'build', 'android'))
39import devil_chromium
40from pylib import constants
41
42
43# Needs to happen early for GetBuildType()/GetOutDirectory() to work correctly
44constants.SetBuildType('Release')
45
46
47class CommandError(Exception):
48 """Indicates that a dispatched shell command exited with a non-zero status."""
49
50 def __init__(self, value):
51 super(CommandError, self).__init__()
52 self.value = value
53
54 def __str__(self):
55 return repr(self.value)
56
57
58def _GenerateHash(file_path):
59 """Calculates and returns the hash of the file at file_path."""
60 sha1 = hashlib.sha1()
61 with open(file_path, 'rb') as f:
62 while True:
63 # Read in 1mb chunks, so it doesn't all have to be loaded into memory.
64 chunk = f.read(1024 * 1024)
65 if not chunk:
66 break
67 sha1.update(chunk)
68 return sha1.hexdigest()
69
70
71def _GetFileExtension(file_name):
72 """Calculates the file extension from a file name.
73
74 Args:
75 file_name: The source file name.
76 Returns:
77 The part of file_name after the dot (.) or None if the file has no
78 extension.
79 Examples: /home/user/foo.bar -> bar
80 /home/user.name/foo -> None
81 /home/user/.foo -> None
82 /home/user/foo.bar.baz -> baz
83 """
84 file_name_parts = os.path.basename(file_name).split('.')
85 if len(file_name_parts) > 1:
86 return file_name_parts[-1]
87 else:
88 return None
89
90
91def _StashOutputDirectory(buildpath):
92 """Takes the output directory and stashes it in the default output directory.
93
94 This allows it to be used for incremental builds next time (after unstashing)
95 by keeping it in a place that isn't deleted normally, while also ensuring
96 that it is properly clobbered when appropriate.
97
98 This is a dirty hack to deal with the needs of clobbering while also handling
99 incremental builds and the hardcoded relative paths used in some of the
100 project files.
101
102 Args:
103 buildpath: The path where the building happens. If this corresponds to the
104 default output directory, no action is taken.
105 """
106 if os.path.abspath(buildpath) == os.path.abspath(os.path.dirname(
107 constants.GetOutDirectory())):
108 return
109 name = os.path.basename(buildpath)
110 stashpath = os.path.join(constants.GetOutDirectory(), name)
111 if not os.path.exists(buildpath):
112 return
113 if os.path.exists(stashpath):
114 shutil.rmtree(stashpath, ignore_errors=True)
115 shutil.move(buildpath, stashpath)
116
117
118def _UnstashOutputDirectory(buildpath):
119 """Inverse of _StashOutputDirectory.
120
121 Moves the output directory stashed within the default output directory
122 (out/Release) to the position where the builds can actually happen.
123
124 This is a dirty hack to deal with the needs of clobbering while also handling
125 incremental builds and the hardcoded relative paths used in some of the
126 project files.
127
128 Args:
129 buildpath: The path where the building happens. If this corresponds to the
130 default output directory, no action is taken.
131 """
132 if os.path.abspath(buildpath) == os.path.abspath(os.path.dirname(
133 constants.GetOutDirectory())):
134 return
135 name = os.path.basename(buildpath)
136 stashpath = os.path.join(constants.GetOutDirectory(), name)
137 if not os.path.exists(stashpath):
138 return
139 if os.path.exists(buildpath):
140 shutil.rmtree(buildpath, ignore_errors=True)
141 shutil.move(stashpath, buildpath)
142
143
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]144def _EnsureOrderfileStartsWithAnchorSection(filename):
145 """Ensures that the orderfile starts with the right anchor symbol.
146
147 This changes the orderfile, if required.
148
149 Args:
150 filename: (str) Path to the orderfile.
151 """
152 anchor_section = '.text.dummy_function_to_anchor_text'
153 with open(filename, 'r') as f:
154 if f.readline().strip() == anchor_section:
155 return
156 try:
157 f = tempfile.NamedTemporaryFile(dir=os.path.dirname(filename), delete=False)
158 f.write(anchor_section + '\n')
159 with open(filename, 'r') as orderfile_file:
160 for line in orderfile_file:
161 f.write(line + '\n')
162 f.close()
163 os.rename(f.name, filename)
164 finally:
165 if os.path.exists(f.name):
166 os.remove(f.name)
167
168
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]169class StepRecorder(object):
170 """Records steps and timings."""
171
172 def __init__(self, buildbot):
173 self.timings = []
174 self._previous_step = ('', 0.0)
175 self._buildbot = buildbot
176 self._error_recorded = False
177
178 def BeginStep(self, name):
179 """Marks a beginning of the next step in the script.
180
181 On buildbot, this prints a specially formatted name that will show up
182 in the waterfall. Otherwise, just prints the step name.
183
184 Args:
185 name: The name of the step.
186 """
187 self.EndStep()
188 self._previous_step = (name, time.time())
189 print 'Running step: ', name
190
191 def EndStep(self):
192 """Records successful completion of the current step.
193
194 This is optional if the step is immediately followed by another BeginStep.
195 """
196 if self._previous_step[0]:
197 elapsed = time.time() - self._previous_step[1]
198 print 'Step %s took %f seconds' % (self._previous_step[0], elapsed)
199 self.timings.append((self._previous_step[0], elapsed))
200
201 self._previous_step = ('', 0.0)
202
203 def FailStep(self, message=None):
204 """Marks that a particular step has failed.
205
206 On buildbot, this will mark the current step as failed on the waterfall.
207 Otherwise we will just print an optional failure message.
208
209 Args:
210 message: An optional explanation as to why the step failed.
211 """
212 print 'STEP FAILED!!'
213 if message:
214 print message
215 self._error_recorded = True
216 self.EndStep()
217
218 def ErrorRecorded(self):
219 """True if FailStep has been called."""
220 return self._error_recorded
221
222 def RunCommand(self, cmd, cwd=constants.DIR_SOURCE_ROOT, raise_on_error=True,
223 stdout=None):
224 """Execute a shell command.
225
226 Args:
227 cmd: A list of command strings.
228 cwd: Directory in which the command should be executed, defaults to script
229 location if not specified.
230 raise_on_error: If true will raise a CommandError if the call doesn't
231 succeed and mark the step as failed.
232 stdout: A file to redirect stdout for the command to.
233
234 Returns:
235 The process's return code.
236
237 Raises:
238 CommandError: An error executing the specified command.
239 """
240 print 'Executing %s in %s' % (' '.join(cmd), cwd)
241 process = subprocess.Popen(cmd, stdout=stdout, cwd=cwd, env=os.environ)
242 process.wait()
243 if raise_on_error and process.returncode != 0:
244 self.FailStep()
245 raise CommandError('Exception executing command %s' % ' '.join(cmd))
246 return process.returncode
247
248
249class ClankCompiler(object):
250 """Handles compilation of clank."""
251
252 def __init__(self, out_dir, step_recorder, arch, jobs, max_load, use_goma,
Benoit L96466812018-03-06 15:58:37[diff] [blame^]253 goma_dir):
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]254 self._out_dir = out_dir
255 self._step_recorder = step_recorder
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]256 self._arch = arch
257 self._jobs = jobs
258 self._max_load = max_load
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]259 self._use_goma = use_goma
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]260 self._goma_dir = goma_dir
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]261 lib_chrome_so_dir = 'lib.unstripped'
262 self.lib_chrome_so = os.path.join(
263 self._out_dir, 'Release', lib_chrome_so_dir, 'libchrome.so')
264 self.chrome_apk = os.path.join(
265 self._out_dir, 'Release', 'apks', 'Chrome.apk')
266
267 def Build(self, instrumented, target):
268 """Builds the provided ninja target with or without order_profiling on.
269
270 Args:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]271 instrumented: (bool) Whether we want to build an instrumented binary.
272 target: (str) The name of the ninja target to build.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]273 """
274 self._step_recorder.BeginStep('Compile %s' % target)
275
276 # Set the "Release Official" flavor, the parts affecting performance.
277 args = [
278 'is_chrome_branded=true',
279 'is_debug=false',
280 'is_official_build=true',
281 # We have to build with no symbols if profiling and minimal symbols
282 # otherwise for libchrome.so to fit under the 4 GB limit.
283 # crbug.com/574476
284 'symbol_level=' + ('0' if instrumented else '1'),
285 'target_cpu="' + self._arch + '"',
286 'target_os="android"',
287 'use_goma=' + str(self._use_goma).lower(),
288 'use_order_profiling=' + str(instrumented).lower(),
289 ]
290 if self._goma_dir:
291 args += ['goma_dir="%s"' % self._goma_dir]
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]292
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]293 self._step_recorder.RunCommand(
294 ['gn', 'gen', os.path.join(self._out_dir, 'Release'),
295 '--args=' + ' '.join(args)])
296
297 self._step_recorder.RunCommand(
298 ['ninja', '-C', os.path.join(self._out_dir, 'Release'),
299 '-j' + str(self._jobs), '-l' + str(self._max_load), target])
300
301 def CompileChromeApk(self, instrumented, force_relink=False):
302 """Builds a Chrome.apk either with or without order_profiling on.
303
304 Args:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]305 instrumented: (bool) Whether to build an instrumented apk.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]306 force_relink: Whether libchromeview.so should be re-created.
307 """
308 if force_relink:
309 self._step_recorder.RunCommand(['rm', '-rf', self.lib_chrome_so])
310 self.Build(instrumented, 'chrome_apk')
311
312 def CompileLibchrome(self, instrumented, force_relink=False):
313 """Builds a libchrome.so either with or without order_profiling on.
314
315 Args:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]316 instrumented: (bool) Whether to build an instrumented apk.
317 force_relink: (bool) Whether libchrome.so should be re-created.
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]318 """
319 if force_relink:
320 self._step_recorder.RunCommand(['rm', '-rf', self.lib_chrome_so])
321 self.Build(instrumented, 'libchrome')
322
323
324class OrderfileUpdater(object):
325 """Handles uploading and committing a new orderfile in the repository.
326
327 Only used for testing or on a bot.
328 """
329
330 _CLOUD_STORAGE_BUCKET_FOR_DEBUG = None
331 _CLOUD_STORAGE_BUCKET = None
332 _UPLOAD_TO_CLOUD_COMMAND = 'upload_to_google_storage.py'
333
334 def __init__(self, repository_root, step_recorder, branch, netrc):
335 """Constructor.
336
337 Args:
338 repository_root: (str) Root of the target repository.
339 step_recorder: (StepRecorder) Step recorder, for logging.
340 branch: (str) Branch to commit to.
341 netrc: (str) Path to the .netrc file to use.
342 """
343 self._repository_root = repository_root
344 self._step_recorder = step_recorder
345 self._branch = branch
346 self._netrc = netrc
347
348 def CommitFileHashes(self, unpatched_orderfile_filename, orderfile_filename):
349 """Commits unpatched and patched orderfiles hashes, if provided.
350
351 Files must have been successfilly uploaded to cloud storage first.
352
353 Args:
354 unpatched_orderfile_filename: (str or None) Unpatched orderfile path.
355 orderfile_filename: (str or None) Orderfile path.
356
357 Raises:
358 NotImplementedError when the commit logic hasn't been overriden.
359 """
360 files_to_commit = []
361 commit_message_lines = ['Update Orderfile.']
362 for filename in [unpatched_orderfile_filename, orderfile_filename]:
363 if not filename:
364 continue
365 (relative_path, sha1) = self._GetHashFilePathAndContents(filename)
366 commit_message_lines.append('Profile: %s: %s' % (
367 os.path.basename(relative_path), sha1))
368 files_to_commit.append(relative_path)
369 if files_to_commit:
370 self._CommitFiles(files_to_commit, commit_message_lines)
371
372 def UploadToCloudStorage(self, filename, use_debug_location):
373 """Uploads a file to cloud storage.
374
375 Args:
376 filename: (str) File to upload.
377 use_debug_location: (bool) Whether to use the debug location.
378 """
379 bucket = (self._CLOUD_STORAGE_BUCKET_FOR_DEBUG if use_debug_location
380 else self._CLOUD_STORAGE_BUCKET)
381 extension = _GetFileExtension(filename)
382 cmd = [self._UPLOAD_TO_CLOUD_COMMAND, '--bucket', bucket]
383 if extension:
384 cmd.extend(['-z', extension])
385 cmd.append(filename)
386 self._step_recorder.RunCommand(cmd)
387 print 'Download: https://sandbox.google.com/storage/%s/%s' % (
388 bucket, _GenerateHash(filename))
389
390 def _GetHashFilePathAndContents(self, filename):
391 """Gets the name and content of the hash file created from uploading the
392 given file.
393
394 Args:
395 filename: (str) The file that was uploaded to cloud storage.
396
397 Returns:
398 A tuple of the hash file name, relative to the reository root, and the
399 content, which should be the sha1 hash of the file
400 ('base_file.sha1', hash)
401 """
402 abs_hash_filename = filename + '.sha1'
403 rel_hash_filename = os.path.relpath(
404 abs_hash_filename, self._repository_root)
405 with open(abs_hash_filename, 'r') as f:
406 return (rel_hash_filename, f.read())
407
408 def _CommitFiles(self, files_to_commit, commit_message_lines):
409 """Commits a list of files, with a given message."""
410 raise NotImplementedError
411
412
413class OrderfileGenerator(object):
414 """A utility for generating a new orderfile for Clank.
415
416 Builds an instrumented binary, profiles a run of the application, and
417 generates an updated orderfile.
418 """
419 _CLANK_REPO = os.path.join(constants.DIR_SOURCE_ROOT, 'clank')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]420 _CYGLOG_TO_ORDERFILE_SCRIPT = os.path.join(
421 constants.DIR_SOURCE_ROOT, 'tools', 'cygprofile',
422 'cyglog_to_orderfile.py')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]423 _CHECK_ORDERFILE_SCRIPT = os.path.join(
424 constants.DIR_SOURCE_ROOT, 'tools', 'cygprofile', 'check_orderfile.py')
425 _BUILD_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(
426 constants.GetOutDirectory()))) # Normally /path/to/src
427
428 _UNPATCHED_ORDERFILE_FILENAME = os.path.join(
429 _CLANK_REPO, 'orderfiles', 'unpatched_orderfile.%s')
430 _MERGED_CYGLOG_FILENAME = os.path.join(
431 constants.GetOutDirectory(), 'merged_cyglog')
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]432
433 _PATH_TO_ORDERFILE = os.path.join(_CLANK_REPO, 'orderfiles',
434 'orderfile.%s.out')
435
436 # Previous orderfile_generator debug files would be overwritten.
437 _DIRECTORY_FOR_DEBUG_FILES = '/tmp/orderfile_generator_debug_files'
438
439 def _GetPathToOrderfile(self):
440 """Gets the path to the architecture-specific orderfile."""
441 return self._PATH_TO_ORDERFILE % self._options.arch
442
443 def _GetUnpatchedOrderfileFilename(self):
444 """Gets the path to the architecture-specific unpatched orderfile."""
445 return self._UNPATCHED_ORDERFILE_FILENAME % self._options.arch
446
447 def __init__(self, options, orderfile_updater_class):
448 self._options = options
449
450 self._instrumented_out_dir = os.path.join(
451 self._BUILD_ROOT, self._options.arch + '_instrumented_out')
452 self._uninstrumented_out_dir = os.path.join(
453 self._BUILD_ROOT, self._options.arch + '_uninstrumented_out')
454
455 if options.profile:
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]456 output_directory = os.path.join(self._instrumented_out_dir, 'Release')
457 host_cyglog_dir = os.path.join(output_directory, 'cyglog_data')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]458 urls = [profile_android_startup.AndroidProfileTool.TEST_URL]
459 use_wpr = True
460 simulate_user = False
Benoit L96466812018-03-06 15:58:37[diff] [blame^]461 urls = options.urls
462 use_wpr = not options.no_wpr
463 simulate_user = options.simulate_user
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]464 self._profiler = profile_android_startup.AndroidProfileTool(
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]465 output_directory, host_cyglog_dir, use_wpr, urls, simulate_user)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]466
467 self._output_data = {}
468 self._step_recorder = StepRecorder(options.buildbot)
469 self._compiler = None
470 assert issubclass(orderfile_updater_class, OrderfileUpdater)
471 self._orderfile_updater = orderfile_updater_class(self._CLANK_REPO,
472 self._step_recorder,
473 options.branch,
474 options.netrc)
475 assert os.path.isdir(constants.DIR_SOURCE_ROOT), 'No src directory found'
Benoit Lizefefbb27c2018-01-17 13:54:18[diff] [blame]476 symbol_extractor.SetArchitecture(options.arch)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]477
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]478 @staticmethod
479 def _RemoveBlanks(src_file, dest_file):
480 """A utility to remove blank lines from a file.
481
482 Args:
483 src_file: The name of the file to remove the blanks from.
484 dest_file: The name of the file to write the output without blanks.
485 """
486 assert src_file != dest_file, 'Source and destination need to be distinct'
487
488 try:
489 src = open(src_file, 'r')
490 dest = open(dest_file, 'w')
491 for line in src:
492 if line and not line.isspace():
493 dest.write(line)
494 finally:
495 src.close()
496 dest.close()
497
498 def _GenerateAndProcessProfile(self):
499 """Invokes a script to merge the per-thread traces into one file."""
500 self._step_recorder.BeginStep('Generate Profile Data')
501 files = []
502 try:
503 logging.getLogger().setLevel(logging.DEBUG)
504 files = self._profiler.CollectProfile(
505 self._compiler.chrome_apk,
506 constants.PACKAGE_INFO['chrome'])
507 self._step_recorder.BeginStep('Process cyglog')
Benoit L96466812018-03-06 15:58:37[diff] [blame^]508 assert os.path.exists(self._compiler.lib_chrome_so)
509 offsets = process_profiles.GetReachedOffsetsFromDumpFiles(
510 files, self._compiler.lib_chrome_so)
511 if not offsets:
512 raise Exception('No profiler offsets found in {}'.format(
513 '\n'.join(files)))
514 with open(self._MERGED_CYGLOG_FILENAME, 'w') as f:
515 f.write('\n'.join(map(str, offsets)))
Matthew Cary0f1f681a2018-01-22 10:40:51[diff] [blame]516 except Exception:
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]517 for f in files:
518 self._SaveForDebugging(f)
519 raise
520 finally:
521 self._profiler.Cleanup()
522 logging.getLogger().setLevel(logging.INFO)
523
524 try:
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]525 command_args = [
526 '--target-arch=' + self._options.arch,
527 '--native-library=' + self._compiler.lib_chrome_so,
528 '--output=' + self._GetUnpatchedOrderfileFilename()]
Benoit L96466812018-03-06 15:58:37[diff] [blame^]529 command_args.append('--reached-offsets=' + self._MERGED_CYGLOG_FILENAME)
Benoit Lizea87e5bc2017-11-07 15:12:57[diff] [blame]530 self._step_recorder.RunCommand(
531 [self._CYGLOG_TO_ORDERFILE_SCRIPT] + command_args)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]532 except CommandError:
533 self._SaveForDebugging(self._MERGED_CYGLOG_FILENAME)
534 self._SaveForDebuggingWithOverwrite(self._compiler.lib_chrome_so)
535 raise
536
537 def _DeleteTempFiles(self):
538 """Deletes intermediate step output files."""
539 print 'Delete %s' % (
540 self._MERGED_CYGLOG_FILENAME)
541 if os.path.isfile(self._MERGED_CYGLOG_FILENAME):
542 os.unlink(self._MERGED_CYGLOG_FILENAME)
543
544 def _PatchOrderfile(self):
545 """Patches the orderfile using clean version of libchrome.so."""
546 self._step_recorder.BeginStep('Patch Orderfile')
Benoit Lizefefbb27c2018-01-17 13:54:18[diff] [blame]547 patch_orderfile.GeneratePatchedOrderfile(
548 self._GetUnpatchedOrderfileFilename(), self._compiler.lib_chrome_so,
549 self._GetPathToOrderfile())
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]550
551 def _VerifySymbolOrder(self):
552 self._step_recorder.BeginStep('Verify Symbol Order')
553 return_code = self._step_recorder.RunCommand(
554 [self._CHECK_ORDERFILE_SCRIPT, self._compiler.lib_chrome_so,
555 self._GetPathToOrderfile(),
556 '--target-arch=' + self._options.arch],
557 constants.DIR_SOURCE_ROOT,
558 raise_on_error=False)
559 if return_code:
560 self._step_recorder.FailStep('Orderfile check returned %d.' % return_code)
561
562 def _RecordHash(self, file_name):
563 """Records the hash of the file into the output_data dictionary."""
564 self._output_data[os.path.basename(file_name) + '.sha1'] = _GenerateHash(
565 file_name)
566
567 def _SaveFileLocally(self, file_name, file_sha1):
568 """Saves the file to a temporary location and prints the sha1sum."""
569 if not os.path.exists(self._DIRECTORY_FOR_DEBUG_FILES):
570 os.makedirs(self._DIRECTORY_FOR_DEBUG_FILES)
571 shutil.copy(file_name, self._DIRECTORY_FOR_DEBUG_FILES)
572 print 'File: %s, saved in: %s, sha1sum: %s' % (
573 file_name, self._DIRECTORY_FOR_DEBUG_FILES, file_sha1)
574
575 def _SaveForDebugging(self, filename):
576 """Uploads the file to cloud storage or saves to a temporary location."""
577 file_sha1 = _GenerateHash(filename)
578 if not self._options.buildbot:
579 self._SaveFileLocally(filename, file_sha1)
580 else:
581 print 'Uploading file for debugging: ' + filename
582 self._orderfile_updater.UploadToCloudStorage(
583 filename, use_debug_location=True)
584
585 def _SaveForDebuggingWithOverwrite(self, file_name):
586 """Uploads and overwrites the file in cloud storage or copies locally.
587
588 Should be used for large binaries like lib_chrome_so.
589
590 Args:
591 file_name: (str) File to upload.
592 """
593 file_sha1 = _GenerateHash(file_name)
594 if not self._options.buildbot:
595 self._SaveFileLocally(file_name, file_sha1)
596 else:
597 print 'Uploading file for debugging: %s, sha1sum: %s' % (
598 file_name, file_sha1)
599 upload_location = '%s/%s' % (
600 self._CLOUD_STORAGE_BUCKET_FOR_DEBUG, os.path.basename(file_name))
601 self._step_recorder.RunCommand([
602 'gsutil.py', 'cp', file_name, 'gs://' + upload_location])
603 print ('Uploaded to: https://sandbox.google.com/storage/' +
604 upload_location)
605
606 def _MaybeArchiveOrderfile(self, filename):
607 """In buildbot configuration, uploads the generated orderfile to
608 Google Cloud Storage.
609
610 Args:
611 filename: (str) Orderfile to upload.
612 """
613 # First compute hashes so that we can download them later if we need to
614 self._step_recorder.BeginStep('Compute hash for ' + filename)
615 self._RecordHash(filename)
616 if self._options.buildbot:
617 self._step_recorder.BeginStep('Archive ' + filename)
618 self._orderfile_updater.UploadToCloudStorage(
619 filename, use_debug_location=False)
620
621 def _GetHashFilePathAndContents(self, base_file):
622 """Gets the name and content of the hash file created from uploading the
623 given file.
624
625 Args:
626 base_file: The file that was uploaded to cloud storage.
627
628 Returns:
629 A tuple of the hash file name, relative to the clank repo path, and the
630 content, which should be the sha1 hash of the file
631 ('base_file.sha1', hash)
632 """
633 abs_file_name = base_file + '.sha1'
634 rel_file_name = os.path.relpath(abs_file_name, self._CLANK_REPO)
635 with open(abs_file_name, 'r') as f:
636 return (rel_file_name, f.read())
637
638 def Generate(self):
639 """Generates and maybe upload an order."""
640 profile_uploaded = False
641 orderfile_uploaded = False
642
643 if self._options.profile:
644 try:
645 _UnstashOutputDirectory(self._instrumented_out_dir)
646 self._compiler = ClankCompiler(
647 self._instrumented_out_dir,
648 self._step_recorder, self._options.arch, self._options.jobs,
649 self._options.max_load, self._options.use_goma,
Benoit L96466812018-03-06 15:58:37[diff] [blame^]650 self._options.goma_dir)
651 _EnsureOrderfileStartsWithAnchorSection(self._GetPathToOrderfile())
652 self._compiler.CompileChromeApk(True)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]653 self._GenerateAndProcessProfile()
654 self._MaybeArchiveOrderfile(self._GetUnpatchedOrderfileFilename())
655 profile_uploaded = True
656 finally:
657 self._DeleteTempFiles()
658 _StashOutputDirectory(self._instrumented_out_dir)
659 if self._options.patch:
660 if self._options.profile:
661 self._RemoveBlanks(self._GetUnpatchedOrderfileFilename(),
662 self._GetPathToOrderfile())
663 try:
664 _UnstashOutputDirectory(self._uninstrumented_out_dir)
665 self._compiler = ClankCompiler(
666 self._uninstrumented_out_dir, self._step_recorder,
667 self._options.arch, self._options.jobs, self._options.max_load,
Benoit L96466812018-03-06 15:58:37[diff] [blame^]668 self._options.use_goma, self._options.goma_dir)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]669 self._compiler.CompileLibchrome(False)
670 self._PatchOrderfile()
671 # Because identical code folding is a bit different with and without
672 # the orderfile build, we need to re-patch the orderfile with code
673 # folding as close to the final version as possible.
674 self._compiler.CompileLibchrome(False, force_relink=True)
675 self._PatchOrderfile()
676 self._compiler.CompileLibchrome(False, force_relink=True)
677 self._VerifySymbolOrder()
678 self._MaybeArchiveOrderfile(self._GetPathToOrderfile())
679 finally:
680 _StashOutputDirectory(self._uninstrumented_out_dir)
681 orderfile_uploaded = True
682
683 if (self._options.buildbot and self._options.netrc
684 and not self._step_recorder.ErrorRecorded()):
685 unpatched_orderfile_filename = (
686 self._GetUnpatchedOrderfileFilename() if profile_uploaded else None)
687 orderfile_filename = (
Benoit Lized913be82017-10-24 13:38:27[diff] [blame]688 self._GetPathToOrderfile() if orderfile_uploaded else None)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]689 self._orderfile_updater.CommitFileHashes(
690 unpatched_orderfile_filename, orderfile_filename)
691
692 self._step_recorder.EndStep()
693 return not self._step_recorder.ErrorRecorded()
694
695 def GetReportingData(self):
696 """Get a dictionary of reporting data (timings, output hashes)"""
697 self._output_data['timings'] = self._step_recorder.timings
698 return self._output_data
699
700
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]701def CreateArgumentParser():
702 """Creates and returns the argument parser."""
703 parser = argparse.ArgumentParser()
704 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]705 '--buildbot', action='store_true',
706 help='If true, the script expects to be run on a buildbot')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]707 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]708 '--verify', action='store_true',
709 help='If true, the script only verifies the current orderfile')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]710 parser.add_argument('--target-arch', action='store', dest='arch',
711 default=cygprofile_utils.DetectArchitecture(),
712 choices=['arm', 'arm64', 'x86', 'x86_64', 'x64', 'mips'],
713 help='The target architecture for which to build')
714 parser.add_argument('--output-json', action='store', dest='json_file',
715 help='Location to save stats in json format')
716 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]717 '--skip-profile', action='store_false', dest='profile', default=True,
718 help='Don\'t generate a profile on the device. Only patch from the '
719 'existing profile.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]720 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]721 '--skip-patch', action='store_false', dest='patch', default=True,
722 help='Only generate the raw (unpatched) orderfile, don\'t patch it.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]723 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]724 '--netrc', action='store',
725 help='A custom .netrc file to use for git checkin. Only used on bots.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]726 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]727 '--branch', action='store', default='master',
728 help='When running on buildbot with a netrc, the branch orderfile '
729 'hashes get checked into.')
730 # Note: -j50 was causing issues on the bot.
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]731 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]732 '-j', '--jobs', action='store', default=20,
733 help='Number of jobs to use for compilation.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]734 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]735 '-l', '--max-load', action='store', default=4, help='Max cpu load.')
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]736 parser.add_argument('--goma-dir', help='GOMA directory.')
737 parser.add_argument(
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]738 '--use-goma', action='store_true', help='Enable GOMA.', default=False)
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]739 parser.add_argument('--adb-path', help='Path to the adb binary.')
740 profile_android_startup.AddProfileCollectionArguments(parser)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]741 return parser
742
743
744def CreateOrderfile(options, orderfile_updater_class):
745 """Creates an oderfile.
746
747 Args:
748 options: As returned from optparse.OptionParser.parse_args()
749 orderfile_updater_class: (OrderfileUpdater) subclass of OrderfileUpdater.
750
751 Returns:
752 True iff success.
753 """
754 logging.basicConfig(level=logging.INFO)
755 devil_chromium.Initialize(adb_path=options.adb_path)
756
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]757 generator = OrderfileGenerator(options, orderfile_updater_class)
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]758 try:
759 if options.verify:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]760 generator._VerifySymbolOrder()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]761 else:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]762 return generator.Generate()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]763 finally:
Egor Pasko93e514e2017-10-31 13:32:36[diff] [blame]764 json_output = json.dumps(generator.GetReportingData(),
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]765 indent=2) + '\n'
766 if options.json_file:
767 with open(options.json_file, 'w') as f:
768 f.write(json_output)
769 print json_output
770 return False
771
772
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]773def main():
774 parser = CreateArgumentParser()
775 options = parser.parse_args()
Benoit Lizea3fe2932017-10-20 10:24:52[diff] [blame]776 return 0 if CreateOrderfile(options, OrderfileUpdater) else 1
777
778
779if __name__ == '__main__':
Benoit Lizea1b64f82017-12-07 10:12:50[diff] [blame]780 sys.exit(main())