Blame - tools/cygprofile/orderfile_generator_backend.py - chromium/src

blob: b043fce36506d58eaaaba7fe44b14608107cb780 [file] [log] [blame]

Egor Pasko	0462e852d	2018-03-29 15:52:09	[diff] [blame]	1	#!/usr/bin/env vpython
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	2	# Copyright (c) 2013 The Chromium Authors. All rights reserved.
				3	# Use of this source code is governed by a BSD-style license that can be
				4	# found in the LICENSE file.
				5
				6	""" A utility to generate an up-to-date orderfile.
				7
				8	The orderfile is used by the linker to order text sections such that the
				9	sections are placed consecutively in the order specified. This allows us
				10	to page in less code during start-up.
				11
				12	Example usage:
Egor Pasko	93e514e	2017-10-31 13:32:36	[diff] [blame]	13	tools/cygprofile/orderfile_generator_backend.py -l 20 -j 1000 --use-goma \
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	14	--target-arch=arm
				15	"""
				16
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	17	import argparse
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	18	import hashlib
				19	import json
Matthew Cary	69e9e42	2018-08-10 18:30:06	[diff] [blame]	20	import glob
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	21	import logging
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	22	import os
				23	import re
				24	import shutil
				25	import subprocess
				26	import sys
Benoit Lize	a87e5bc	2017-11-07 15:12:57	[diff] [blame]	27	import tempfile
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	28	import time
				29
Matthew Cary	e840064	2018-06-14 15:43:02	[diff] [blame]	30	import cyglog_to_orderfile
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	31	import cygprofile_utils
Benoit Lize	fefbb27c	2018-01-17 13:54:18	[diff] [blame]	32	import patch_orderfile
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	33	import phased_orderfile
Benoit Lize	a87e5bc	2017-11-07 15:12:57	[diff] [blame]	34	import process_profiles
Egor Pasko	3bc0b93	2018-04-03 10:08:27	[diff] [blame]	35	import profile_android_startup
Benoit Lize	fefbb27c	2018-01-17 13:54:18	[diff] [blame]	36	import symbol_extractor
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	37
				38
				39	_SRC_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)),
				40	os.pardir, os.pardir)
				41	sys.path.append(os.path.join(_SRC_PATH, 'build', 'android'))
				42	import devil_chromium
				43	from pylib import constants
				44
				45
				46	# Needs to happen early for GetBuildType()/GetOutDirectory() to work correctly
				47	constants.SetBuildType('Release')
				48
				49
				50	class CommandError(Exception):
				51	"""Indicates that a dispatched shell command exited with a non-zero status."""
				52
				53	def __init__(self, value):
				54	super(CommandError, self).__init__()
				55	self.value = value
				56
				57	def __str__(self):
				58	return repr(self.value)
				59
				60
				61	def _GenerateHash(file_path):
				62	"""Calculates and returns the hash of the file at file_path."""
				63	sha1 = hashlib.sha1()
				64	with open(file_path, 'rb') as f:
				65	while True:
				66	# Read in 1mb chunks, so it doesn't all have to be loaded into memory.
				67	chunk = f.read(1024 * 1024)
				68	if not chunk:
				69	break
				70	sha1.update(chunk)
				71	return sha1.hexdigest()
				72
				73
				74	def _GetFileExtension(file_name):
				75	"""Calculates the file extension from a file name.
				76
				77	Args:
				78	file_name: The source file name.
				79	Returns:
				80	The part of file_name after the dot (.) or None if the file has no
				81	extension.
				82	Examples: /home/user/foo.bar -> bar
				83	/home/user.name/foo -> None
				84	/home/user/.foo -> None
				85	/home/user/foo.bar.baz -> baz
				86	"""
				87	file_name_parts = os.path.basename(file_name).split('.')
				88	if len(file_name_parts) > 1:
				89	return file_name_parts[-1]
				90	else:
				91	return None
				92
				93
				94	def _StashOutputDirectory(buildpath):
				95	"""Takes the output directory and stashes it in the default output directory.
				96
				97	This allows it to be used for incremental builds next time (after unstashing)
				98	by keeping it in a place that isn't deleted normally, while also ensuring
				99	that it is properly clobbered when appropriate.
				100
				101	This is a dirty hack to deal with the needs of clobbering while also handling
				102	incremental builds and the hardcoded relative paths used in some of the
				103	project files.
				104
				105	Args:
				106	buildpath: The path where the building happens. If this corresponds to the
				107	default output directory, no action is taken.
				108	"""
				109	if os.path.abspath(buildpath) == os.path.abspath(os.path.dirname(
				110	constants.GetOutDirectory())):
				111	return
				112	name = os.path.basename(buildpath)
				113	stashpath = os.path.join(constants.GetOutDirectory(), name)
				114	if not os.path.exists(buildpath):
				115	return
				116	if os.path.exists(stashpath):
				117	shutil.rmtree(stashpath, ignore_errors=True)
				118	shutil.move(buildpath, stashpath)
				119
				120
				121	def _UnstashOutputDirectory(buildpath):
				122	"""Inverse of _StashOutputDirectory.
				123
				124	Moves the output directory stashed within the default output directory
				125	(out/Release) to the position where the builds can actually happen.
				126
				127	This is a dirty hack to deal with the needs of clobbering while also handling
				128	incremental builds and the hardcoded relative paths used in some of the
				129	project files.
				130
				131	Args:
				132	buildpath: The path where the building happens. If this corresponds to the
				133	default output directory, no action is taken.
				134	"""
				135	if os.path.abspath(buildpath) == os.path.abspath(os.path.dirname(
				136	constants.GetOutDirectory())):
				137	return
				138	name = os.path.basename(buildpath)
				139	stashpath = os.path.join(constants.GetOutDirectory(), name)
				140	if not os.path.exists(stashpath):
				141	return
				142	if os.path.exists(buildpath):
				143	shutil.rmtree(buildpath, ignore_errors=True)
				144	shutil.move(stashpath, buildpath)
				145
				146
				147	class StepRecorder(object):
				148	"""Records steps and timings."""
				149
				150	def __init__(self, buildbot):
				151	self.timings = []
				152	self._previous_step = ('', 0.0)
				153	self._buildbot = buildbot
				154	self._error_recorded = False
				155
				156	def BeginStep(self, name):
				157	"""Marks a beginning of the next step in the script.
				158
				159	On buildbot, this prints a specially formatted name that will show up
				160	in the waterfall. Otherwise, just prints the step name.
				161
				162	Args:
				163	name: The name of the step.
				164	"""
				165	self.EndStep()
				166	self._previous_step = (name, time.time())
				167	print 'Running step: ', name
				168
				169	def EndStep(self):
				170	"""Records successful completion of the current step.
				171
				172	This is optional if the step is immediately followed by another BeginStep.
				173	"""
				174	if self._previous_step[0]:
				175	elapsed = time.time() - self._previous_step[1]
				176	print 'Step %s took %f seconds' % (self._previous_step[0], elapsed)
				177	self.timings.append((self._previous_step[0], elapsed))
				178
				179	self._previous_step = ('', 0.0)
				180
				181	def FailStep(self, message=None):
				182	"""Marks that a particular step has failed.
				183
				184	On buildbot, this will mark the current step as failed on the waterfall.
				185	Otherwise we will just print an optional failure message.
				186
				187	Args:
				188	message: An optional explanation as to why the step failed.
				189	"""
				190	print 'STEP FAILED!!'
				191	if message:
				192	print message
				193	self._error_recorded = True
				194	self.EndStep()
				195
				196	def ErrorRecorded(self):
				197	"""True if FailStep has been called."""
				198	return self._error_recorded
				199
				200	def RunCommand(self, cmd, cwd=constants.DIR_SOURCE_ROOT, raise_on_error=True,
				201	stdout=None):
				202	"""Execute a shell command.
				203
				204	Args:
				205	cmd: A list of command strings.
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	206	cwd: Directory in which the command should be executed, defaults to build
				207	root of script's location if not specified.
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	208	raise_on_error: If true will raise a CommandError if the call doesn't
				209	succeed and mark the step as failed.
				210	stdout: A file to redirect stdout for the command to.
				211
				212	Returns:
				213	The process's return code.
				214
				215	Raises:
				216	CommandError: An error executing the specified command.
				217	"""
				218	print 'Executing %s in %s' % (' '.join(cmd), cwd)
				219	process = subprocess.Popen(cmd, stdout=stdout, cwd=cwd, env=os.environ)
				220	process.wait()
				221	if raise_on_error and process.returncode != 0:
				222	self.FailStep()
				223	raise CommandError('Exception executing command %s' % ' '.join(cmd))
				224	return process.returncode
				225
				226
				227	class ClankCompiler(object):
				228	"""Handles compilation of clank."""
				229
				230	def __init__(self, out_dir, step_recorder, arch, jobs, max_load, use_goma,
Matthew Cary	d6bfcb7	2018-09-14 11:27:46	[diff] [blame]	231	goma_dir, system_health_profiling, monochrome):
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	232	self._out_dir = out_dir
				233	self._step_recorder = step_recorder
Benoit Lize	a87e5bc	2017-11-07 15:12:57	[diff] [blame]	234	self._arch = arch
				235	self._jobs = jobs
				236	self._max_load = max_load
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	237	self._use_goma = use_goma
Benoit Lize	a87e5bc	2017-11-07 15:12:57	[diff] [blame]	238	self._goma_dir = goma_dir
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	239	self._system_health_profiling = system_health_profiling
Matthew Cary	d6bfcb7	2018-09-14 11:27:46	[diff] [blame]	240	if monochrome:
				241	self._apk = 'Monochrome.apk'
Matthew Cary	e1b0006	2018-09-19 14:27:44	[diff] [blame]	242	self._apk_target = 'monochrome_apk'
Matthew Cary	d6bfcb7	2018-09-14 11:27:46	[diff] [blame]	243	self._libname = 'libmonochrome'
				244	self._libchrome_target = 'monochrome'
				245	else:
				246	self._apk = 'Chrome.apk'
Matthew Cary	e1b0006	2018-09-19 14:27:44	[diff] [blame]	247	self._apk_target = 'chrome_apk'
Matthew Cary	d6bfcb7	2018-09-14 11:27:46	[diff] [blame]	248	self._libname = 'libchrome'
				249	self._libchrome_target = 'libchrome'
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	250
				251	self.obj_dir = os.path.join(self._out_dir, 'Release', 'obj')
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	252	self.lib_chrome_so = os.path.join(
Matthew Cary	d6bfcb7	2018-09-14 11:27:46	[diff] [blame]	253	self._out_dir, 'Release', 'lib.unstripped',
				254	'{}.so'.format(self._libname))
				255	self.chrome_apk = os.path.join(self._out_dir, 'Release', 'apks', self._apk)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	256
				257	def Build(self, instrumented, target):
				258	"""Builds the provided ninja target with or without order_profiling on.
				259
				260	Args:
Benoit Lize	a87e5bc	2017-11-07 15:12:57	[diff] [blame]	261	instrumented: (bool) Whether we want to build an instrumented binary.
				262	target: (str) The name of the ninja target to build.
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	263	"""
				264	self._step_recorder.BeginStep('Compile %s' % target)
				265
				266	# Set the "Release Official" flavor, the parts affecting performance.
				267	args = [
Peter Collingbourne	1c85d5d2	2018-08-22 18:06:35	[diff] [blame]	268	'enable_resource_whitelist_generation=false',
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	269	'is_chrome_branded=true',
				270	'is_debug=false',
				271	'is_official_build=true',
				272	# We have to build with no symbols if profiling and minimal symbols
				273	# otherwise for libchrome.so to fit under the 4 GB limit.
				274	# crbug.com/574476
				275	'symbol_level=' + ('0' if instrumented else '1'),
				276	'target_cpu="' + self._arch + '"',
				277	'target_os="android"',
				278	'use_goma=' + str(self._use_goma).lower(),
				279	'use_order_profiling=' + str(instrumented).lower(),
				280	]
				281	if self._goma_dir:
				282	args += ['goma_dir="%s"' % self._goma_dir]
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	283	if self._system_health_profiling:
				284	args += ['devtools_instrumentation_dumping = ' +
				285	str(instrumented).lower()]
Benoit Lize	a87e5bc	2017-11-07 15:12:57	[diff] [blame]	286
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	287	self._step_recorder.RunCommand(
				288	['gn', 'gen', os.path.join(self._out_dir, 'Release'),
				289	'--args=' + ' '.join(args)])
				290
				291	self._step_recorder.RunCommand(
				292	['ninja', '-C', os.path.join(self._out_dir, 'Release'),
				293	'-j' + str(self._jobs), '-l' + str(self._max_load), target])
				294
				295	def CompileChromeApk(self, instrumented, force_relink=False):
				296	"""Builds a Chrome.apk either with or without order_profiling on.
				297
				298	Args:
Benoit Lize	a87e5bc	2017-11-07 15:12:57	[diff] [blame]	299	instrumented: (bool) Whether to build an instrumented apk.
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	300	force_relink: Whether libchromeview.so should be re-created.
				301	"""
				302	if force_relink:
				303	self._step_recorder.RunCommand(['rm', '-rf', self.lib_chrome_so])
Matthew Cary	e1b0006	2018-09-19 14:27:44	[diff] [blame]	304	self.Build(instrumented, self._apk_target)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	305
				306	def CompileLibchrome(self, instrumented, force_relink=False):
				307	"""Builds a libchrome.so either with or without order_profiling on.
				308
				309	Args:
Benoit Lize	a87e5bc	2017-11-07 15:12:57	[diff] [blame]	310	instrumented: (bool) Whether to build an instrumented apk.
				311	force_relink: (bool) Whether libchrome.so should be re-created.
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	312	"""
				313	if force_relink:
				314	self._step_recorder.RunCommand(['rm', '-rf', self.lib_chrome_so])
Matthew Cary	d6bfcb7	2018-09-14 11:27:46	[diff] [blame]	315	self.Build(instrumented, self._libchrome_target)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	316
				317
				318	class OrderfileUpdater(object):
				319	"""Handles uploading and committing a new orderfile in the repository.
				320
				321	Only used for testing or on a bot.
				322	"""
				323
				324	_CLOUD_STORAGE_BUCKET_FOR_DEBUG = None
				325	_CLOUD_STORAGE_BUCKET = None
				326	_UPLOAD_TO_CLOUD_COMMAND = 'upload_to_google_storage.py'
				327
				328	def __init__(self, repository_root, step_recorder, branch, netrc):
				329	"""Constructor.
				330
				331	Args:
				332	repository_root: (str) Root of the target repository.
				333	step_recorder: (StepRecorder) Step recorder, for logging.
				334	branch: (str) Branch to commit to.
				335	netrc: (str) Path to the .netrc file to use.
				336	"""
				337	self._repository_root = repository_root
				338	self._step_recorder = step_recorder
				339	self._branch = branch
				340	self._netrc = netrc
				341
				342	def CommitFileHashes(self, unpatched_orderfile_filename, orderfile_filename):
				343	"""Commits unpatched and patched orderfiles hashes, if provided.
				344
				345	Files must have been successfilly uploaded to cloud storage first.
				346
				347	Args:
				348	unpatched_orderfile_filename: (str or None) Unpatched orderfile path.
				349	orderfile_filename: (str or None) Orderfile path.
				350
				351	Raises:
				352	NotImplementedError when the commit logic hasn't been overriden.
				353	"""
				354	files_to_commit = []
				355	commit_message_lines = ['Update Orderfile.']
				356	for filename in [unpatched_orderfile_filename, orderfile_filename]:
				357	if not filename:
				358	continue
				359	(relative_path, sha1) = self._GetHashFilePathAndContents(filename)
				360	commit_message_lines.append('Profile: %s: %s' % (
				361	os.path.basename(relative_path), sha1))
				362	files_to_commit.append(relative_path)
				363	if files_to_commit:
				364	self._CommitFiles(files_to_commit, commit_message_lines)
				365
				366	def UploadToCloudStorage(self, filename, use_debug_location):
				367	"""Uploads a file to cloud storage.
				368
				369	Args:
				370	filename: (str) File to upload.
				371	use_debug_location: (bool) Whether to use the debug location.
				372	"""
				373	bucket = (self._CLOUD_STORAGE_BUCKET_FOR_DEBUG if use_debug_location
				374	else self._CLOUD_STORAGE_BUCKET)
				375	extension = _GetFileExtension(filename)
				376	cmd = [self._UPLOAD_TO_CLOUD_COMMAND, '--bucket', bucket]
				377	if extension:
				378	cmd.extend(['-z', extension])
				379	cmd.append(filename)
				380	self._step_recorder.RunCommand(cmd)
				381	print 'Download: https://sandbox.google.com/storage/%s/%s' % (
				382	bucket, _GenerateHash(filename))
				383
				384	def _GetHashFilePathAndContents(self, filename):
				385	"""Gets the name and content of the hash file created from uploading the
				386	given file.
				387
				388	Args:
				389	filename: (str) The file that was uploaded to cloud storage.
				390
				391	Returns:
				392	A tuple of the hash file name, relative to the reository root, and the
				393	content, which should be the sha1 hash of the file
				394	('base_file.sha1', hash)
				395	"""
				396	abs_hash_filename = filename + '.sha1'
				397	rel_hash_filename = os.path.relpath(
				398	abs_hash_filename, self._repository_root)
				399	with open(abs_hash_filename, 'r') as f:
				400	return (rel_hash_filename, f.read())
				401
				402	def _CommitFiles(self, files_to_commit, commit_message_lines):
				403	"""Commits a list of files, with a given message."""
				404	raise NotImplementedError
				405
				406
				407	class OrderfileGenerator(object):
				408	"""A utility for generating a new orderfile for Clank.
				409
				410	Builds an instrumented binary, profiles a run of the application, and
				411	generates an updated orderfile.
				412	"""
				413	_CLANK_REPO = os.path.join(constants.DIR_SOURCE_ROOT, 'clank')
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	414	_CHECK_ORDERFILE_SCRIPT = os.path.join(
				415	constants.DIR_SOURCE_ROOT, 'tools', 'cygprofile', 'check_orderfile.py')
				416	_BUILD_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(
				417	constants.GetOutDirectory()))) # Normally /path/to/src
				418
				419	_UNPATCHED_ORDERFILE_FILENAME = os.path.join(
				420	_CLANK_REPO, 'orderfiles', 'unpatched_orderfile.%s')
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	421
				422	_PATH_TO_ORDERFILE = os.path.join(_CLANK_REPO, 'orderfiles',
				423	'orderfile.%s.out')
				424
				425	# Previous orderfile_generator debug files would be overwritten.
				426	_DIRECTORY_FOR_DEBUG_FILES = '/tmp/orderfile_generator_debug_files'
				427
				428	def _GetPathToOrderfile(self):
				429	"""Gets the path to the architecture-specific orderfile."""
				430	return self._PATH_TO_ORDERFILE % self._options.arch
				431
				432	def _GetUnpatchedOrderfileFilename(self):
				433	"""Gets the path to the architecture-specific unpatched orderfile."""
				434	return self._UNPATCHED_ORDERFILE_FILENAME % self._options.arch
				435
				436	def __init__(self, options, orderfile_updater_class):
				437	self._options = options
				438
				439	self._instrumented_out_dir = os.path.join(
				440	self._BUILD_ROOT, self._options.arch + '_instrumented_out')
				441	self._uninstrumented_out_dir = os.path.join(
				442	self._BUILD_ROOT, self._options.arch + '_uninstrumented_out')
				443
				444	if options.profile:
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	445	output_directory = os.path.join(self._instrumented_out_dir, 'Release')
Matthew Cary	b8daed94	2018-06-11 10:58:08	[diff] [blame]	446	host_profile_dir = os.path.join(output_directory, 'profile_data')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	447	urls = [profile_android_startup.AndroidProfileTool.TEST_URL]
				448	use_wpr = True
				449	simulate_user = False
Benoit L	9646681	2018-03-06 15:58:37	[diff] [blame]	450	urls = options.urls
				451	use_wpr = not options.no_wpr
				452	simulate_user = options.simulate_user
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	453	self._profiler = profile_android_startup.AndroidProfileTool(
Matthew Cary	453ff145	2018-07-18 12:19:35	[diff] [blame]	454	output_directory, host_profile_dir, use_wpr, urls, simulate_user,
				455	device=options.device)
Matthew Cary	69e9e42	2018-08-10 18:30:06	[diff] [blame]	456	if options.pregenerated_profiles:
				457	self._profiler.SetPregeneratedProfiles(
				458	glob.glob(options.pregenerated_profiles))
				459	else:
				460	assert not options.pregenerated_profiles, (
				461	'--pregenerated-profiles cannot be used with --skip-profile')
				462	assert not options.profile_save_dir, (
				463	'--profile-save-dir cannot be used with --skip-profile')
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	464
				465	self._output_data = {}
				466	self._step_recorder = StepRecorder(options.buildbot)
				467	self._compiler = None
				468	assert issubclass(orderfile_updater_class, OrderfileUpdater)
				469	self._orderfile_updater = orderfile_updater_class(self._CLANK_REPO,
				470	self._step_recorder,
				471	options.branch,
				472	options.netrc)
				473	assert os.path.isdir(constants.DIR_SOURCE_ROOT), 'No src directory found'
Benoit Lize	fefbb27c	2018-01-17 13:54:18	[diff] [blame]	474	symbol_extractor.SetArchitecture(options.arch)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	475
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	476	@staticmethod
				477	def _RemoveBlanks(src_file, dest_file):
				478	"""A utility to remove blank lines from a file.
				479
				480	Args:
				481	src_file: The name of the file to remove the blanks from.
				482	dest_file: The name of the file to write the output without blanks.
				483	"""
				484	assert src_file != dest_file, 'Source and destination need to be distinct'
				485
				486	try:
				487	src = open(src_file, 'r')
				488	dest = open(dest_file, 'w')
				489	for line in src:
				490	if line and not line.isspace():
				491	dest.write(line)
				492	finally:
				493	src.close()
				494	dest.close()
				495
				496	def _GenerateAndProcessProfile(self):
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	497	"""Invokes a script to merge the per-thread traces into one file.
				498
				499	The produced list of offsets is saved in
				500	self._GetUnpatchedOrderfileFilename().
				501	"""
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	502	self._step_recorder.BeginStep('Generate Profile Data')
				503	files = []
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	504	logging.getLogger().setLevel(logging.DEBUG)
				505	if self._options.system_health_orderfile:
				506	files = self._profiler.CollectSystemHealthProfile(
				507	self._compiler.chrome_apk)
Matthew Cary	69e9e42	2018-08-10 18:30:06	[diff] [blame]	508	self._MaybeSaveProfile(files)
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	509	try:
				510	self._ProcessPhasedOrderfile(files)
				511	except Exception:
				512	for f in files:
				513	self._SaveForDebugging(f)
Matthew Cary	8b141623	2018-08-10 19:12:22	[diff] [blame]	514	self._SaveForDebugging(self._compiler.lib_chrome_so)
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	515	raise
				516	finally:
				517	self._profiler.Cleanup()
				518	else:
				519	self._CollectLegacyProfile()
				520	logging.getLogger().setLevel(logging.INFO)
				521
				522	def _ProcessPhasedOrderfile(self, files):
				523	"""Process the phased orderfiles produced by system health benchmarks.
				524
				525	The offsets will be placed in _GetUnpatchedOrderfileFilename().
				526
				527	Args:
				528	file: Profile files pulled locally.
				529	"""
				530	self._step_recorder.BeginStep('Process Phased Orderfile')
				531	profiles = process_profiles.ProfileManager(files)
				532	processor = process_profiles.SymbolOffsetProcessor(
				533	self._compiler.lib_chrome_so)
				534	phaser = phased_orderfile.PhasedAnalyzer(profiles, processor)
				535	if self._options.offsets_for_memory:
				536	profile_offsets = phaser.GetOffsetsForMemoryFootprint()
				537	else:
				538	profile_offsets = phaser.GetOffsetsForStartup()
				539	self._output_data['orderfile_size'] = {
				540	'startup_kib': processor.OffsetsPrimarySize(
				541	profile_offsets.startup) / 1024,
				542	'common_kib': processor.OffsetsPrimarySize(
				543	profile_offsets.common) / 1024,
				544	'interaction_kib': processor.OffsetsPrimarySize(
				545	profile_offsets.interaction) / 1024}
				546
				547	offsets_list = (profile_offsets.startup +
				548	profile_offsets.common +
				549	profile_offsets.interaction)
Matthew Cary	8b141623	2018-08-10 19:12:22	[diff] [blame]	550	ordered_symbols = processor.GetOrderedSymbols(offsets_list)
				551	if not ordered_symbols:
				552	raise Exception('Failed to get ordered symbols')
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	553	with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile:
Matthew Cary	8b141623	2018-08-10 19:12:22	[diff] [blame]	554	orderfile.write('\n'.join(ordered_symbols))
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	555
				556	def _CollectLegacyProfile(self):
Matthew Cary	2df8d45	2018-09-19 07:50:20	[diff] [blame]	557	files = []
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	558	try:
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	559	files = self._profiler.CollectProfile(
				560	self._compiler.chrome_apk,
				561	constants.PACKAGE_INFO['chrome'])
Matthew Cary	69e9e42	2018-08-10 18:30:06	[diff] [blame]	562	self._MaybeSaveProfile(files)
Matthew Cary	b8daed94	2018-06-11 10:58:08	[diff] [blame]	563	self._step_recorder.BeginStep('Process profile')
Benoit L	9646681	2018-03-06 15:58:37	[diff] [blame]	564	assert os.path.exists(self._compiler.lib_chrome_so)
				565	offsets = process_profiles.GetReachedOffsetsFromDumpFiles(
				566	files, self._compiler.lib_chrome_so)
				567	if not offsets:
				568	raise Exception('No profiler offsets found in {}'.format(
				569	'\n'.join(files)))
Matthew Cary	8b141623	2018-08-10 19:12:22	[diff] [blame]	570	processor = process_profiles.SymbolOffsetProcessor(
				571	self._compiler.lib_chrome_so)
				572	ordered_symbols = processor.GetOrderedSymbols(offsets)
				573	if not ordered_symbols:
				574	raise Exception('No symbol names from offsets found in {}'.format(
				575	'\n'.join(files)))
				576	with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile:
				577	orderfile.write('\n'.join(ordered_symbols))
Matthew Cary	0f1f681a	2018-01-22 10:40:51	[diff] [blame]	578	except Exception:
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	579	for f in files:
				580	self._SaveForDebugging(f)
				581	raise
				582	finally:
				583	self._profiler.Cleanup()
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	584
Matthew Cary	69e9e42	2018-08-10 18:30:06	[diff] [blame]	585	def _MaybeSaveProfile(self, files):
				586	if self._options.profile_save_dir:
				587	logging.info('Saving profiles to %s', self._options.profile_save_dir)
				588	for f in files:
				589	shutil.copy(f, self._options.profile_save_dir)
				590	logging.info('Saved profile %s', f)
				591
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	592	def _PatchOrderfile(self):
				593	"""Patches the orderfile using clean version of libchrome.so."""
				594	self._step_recorder.BeginStep('Patch Orderfile')
Benoit Lize	fefbb27c	2018-01-17 13:54:18	[diff] [blame]	595	patch_orderfile.GeneratePatchedOrderfile(
				596	self._GetUnpatchedOrderfileFilename(), self._compiler.lib_chrome_so,
				597	self._GetPathToOrderfile())
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	598
				599	def _VerifySymbolOrder(self):
				600	self._step_recorder.BeginStep('Verify Symbol Order')
				601	return_code = self._step_recorder.RunCommand(
				602	[self._CHECK_ORDERFILE_SCRIPT, self._compiler.lib_chrome_so,
				603	self._GetPathToOrderfile(),
				604	'--target-arch=' + self._options.arch],
				605	constants.DIR_SOURCE_ROOT,
				606	raise_on_error=False)
				607	if return_code:
				608	self._step_recorder.FailStep('Orderfile check returned %d.' % return_code)
				609
				610	def _RecordHash(self, file_name):
				611	"""Records the hash of the file into the output_data dictionary."""
				612	self._output_data[os.path.basename(file_name) + '.sha1'] = _GenerateHash(
				613	file_name)
				614
				615	def _SaveFileLocally(self, file_name, file_sha1):
				616	"""Saves the file to a temporary location and prints the sha1sum."""
				617	if not os.path.exists(self._DIRECTORY_FOR_DEBUG_FILES):
				618	os.makedirs(self._DIRECTORY_FOR_DEBUG_FILES)
				619	shutil.copy(file_name, self._DIRECTORY_FOR_DEBUG_FILES)
				620	print 'File: %s, saved in: %s, sha1sum: %s' % (
				621	file_name, self._DIRECTORY_FOR_DEBUG_FILES, file_sha1)
				622
				623	def _SaveForDebugging(self, filename):
				624	"""Uploads the file to cloud storage or saves to a temporary location."""
				625	file_sha1 = _GenerateHash(filename)
				626	if not self._options.buildbot:
				627	self._SaveFileLocally(filename, file_sha1)
				628	else:
				629	print 'Uploading file for debugging: ' + filename
				630	self._orderfile_updater.UploadToCloudStorage(
				631	filename, use_debug_location=True)
				632
				633	def _SaveForDebuggingWithOverwrite(self, file_name):
				634	"""Uploads and overwrites the file in cloud storage or copies locally.
				635
				636	Should be used for large binaries like lib_chrome_so.
				637
				638	Args:
				639	file_name: (str) File to upload.
				640	"""
				641	file_sha1 = _GenerateHash(file_name)
				642	if not self._options.buildbot:
				643	self._SaveFileLocally(file_name, file_sha1)
				644	else:
				645	print 'Uploading file for debugging: %s, sha1sum: %s' % (
				646	file_name, file_sha1)
				647	upload_location = '%s/%s' % (
				648	self._CLOUD_STORAGE_BUCKET_FOR_DEBUG, os.path.basename(file_name))
				649	self._step_recorder.RunCommand([
				650	'gsutil.py', 'cp', file_name, 'gs://' + upload_location])
				651	print ('Uploaded to: https://sandbox.google.com/storage/' +
				652	upload_location)
				653
				654	def _MaybeArchiveOrderfile(self, filename):
				655	"""In buildbot configuration, uploads the generated orderfile to
				656	Google Cloud Storage.
				657
				658	Args:
				659	filename: (str) Orderfile to upload.
				660	"""
				661	# First compute hashes so that we can download them later if we need to
				662	self._step_recorder.BeginStep('Compute hash for ' + filename)
				663	self._RecordHash(filename)
				664	if self._options.buildbot:
				665	self._step_recorder.BeginStep('Archive ' + filename)
				666	self._orderfile_updater.UploadToCloudStorage(
				667	filename, use_debug_location=False)
				668
				669	def _GetHashFilePathAndContents(self, base_file):
				670	"""Gets the name and content of the hash file created from uploading the
				671	given file.
				672
				673	Args:
				674	base_file: The file that was uploaded to cloud storage.
				675
				676	Returns:
				677	A tuple of the hash file name, relative to the clank repo path, and the
				678	content, which should be the sha1 hash of the file
				679	('base_file.sha1', hash)
				680	"""
				681	abs_file_name = base_file + '.sha1'
				682	rel_file_name = os.path.relpath(abs_file_name, self._CLANK_REPO)
				683	with open(abs_file_name, 'r') as f:
				684	return (rel_file_name, f.read())
				685
				686	def Generate(self):
				687	"""Generates and maybe upload an order."""
				688	profile_uploaded = False
				689	orderfile_uploaded = False
				690
Matthew Cary	e840064	2018-06-14 15:43:02	[diff] [blame]	691	assert (bool(self._options.profile) ^
				692	bool(self._options.manual_symbol_offsets))
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	693	if self._options.system_health_orderfile and not self._options.profile:
				694	raise AssertionError('--system_health_orderfile must be not be used '
				695	'with --skip-profile')
				696	if (self._options.manual_symbol_offsets and
				697	not self._options.system_health_orderfile):
				698	raise AssertionError('--manual-symbol-offsets must be used with '
				699	'--system_health_orderfile.')
Matthew Cary	e840064	2018-06-14 15:43:02	[diff] [blame]	700
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	701	if self._options.profile:
				702	try:
				703	_UnstashOutputDirectory(self._instrumented_out_dir)
				704	self._compiler = ClankCompiler(
				705	self._instrumented_out_dir,
				706	self._step_recorder, self._options.arch, self._options.jobs,
				707	self._options.max_load, self._options.use_goma,
Matthew Cary	d6bfcb7	2018-09-14 11:27:46	[diff] [blame]	708	self._options.goma_dir, self._options.system_health_orderfile,
				709	self._options.monochrome)
Benoit L	9646681	2018-03-06 15:58:37	[diff] [blame]	710	self._compiler.CompileChromeApk(True)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	711	self._GenerateAndProcessProfile()
				712	self._MaybeArchiveOrderfile(self._GetUnpatchedOrderfileFilename())
				713	profile_uploaded = True
				714	finally:
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	715	_StashOutputDirectory(self._instrumented_out_dir)
Matthew Cary	e840064	2018-06-14 15:43:02	[diff] [blame]	716	elif self._options.manual_symbol_offsets:
				717	assert self._options.manual_libname
				718	assert self._options.manual_objdir
				719	with file(self._options.manual_symbol_offsets) as f:
				720	symbol_offsets = [int(x) for x in f.xreadlines()]
				721	processor = process_profiles.SymbolOffsetProcessor(
				722	self._options.manual_libname)
				723	generator = cyglog_to_orderfile.OffsetOrderfileGenerator(
				724	processor, cyglog_to_orderfile.ObjectFileProcessor(
				725	self._options.manual_objdir))
				726	ordered_sections = generator.GetOrderedSections(symbol_offsets)
				727	if not ordered_sections: # Either None or empty is a problem.
				728	raise Exception('Failed to get ordered sections')
				729	with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile:
				730	orderfile.write('\n'.join(ordered_sections))
				731
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	732	if self._options.patch:
				733	if self._options.profile:
				734	self._RemoveBlanks(self._GetUnpatchedOrderfileFilename(),
				735	self._GetPathToOrderfile())
				736	try:
				737	_UnstashOutputDirectory(self._uninstrumented_out_dir)
				738	self._compiler = ClankCompiler(
				739	self._uninstrumented_out_dir, self._step_recorder,
				740	self._options.arch, self._options.jobs, self._options.max_load,
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	741	self._options.use_goma, self._options.goma_dir,
Matthew Cary	d6bfcb7	2018-09-14 11:27:46	[diff] [blame]	742	self._options.system_health_orderfile, self._options.monochrome)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	743	self._compiler.CompileLibchrome(False)
				744	self._PatchOrderfile()
				745	# Because identical code folding is a bit different with and without
				746	# the orderfile build, we need to re-patch the orderfile with code
				747	# folding as close to the final version as possible.
				748	self._compiler.CompileLibchrome(False, force_relink=True)
				749	self._PatchOrderfile()
				750	self._compiler.CompileLibchrome(False, force_relink=True)
				751	self._VerifySymbolOrder()
				752	self._MaybeArchiveOrderfile(self._GetPathToOrderfile())
				753	finally:
				754	_StashOutputDirectory(self._uninstrumented_out_dir)
				755	orderfile_uploaded = True
				756
				757	if (self._options.buildbot and self._options.netrc
				758	and not self._step_recorder.ErrorRecorded()):
				759	unpatched_orderfile_filename = (
				760	self._GetUnpatchedOrderfileFilename() if profile_uploaded else None)
				761	orderfile_filename = (
Benoit Lize	d913be8	2017-10-24 13:38:27	[diff] [blame]	762	self._GetPathToOrderfile() if orderfile_uploaded else None)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	763	self._orderfile_updater.CommitFileHashes(
				764	unpatched_orderfile_filename, orderfile_filename)
				765
				766	self._step_recorder.EndStep()
				767	return not self._step_recorder.ErrorRecorded()
				768
				769	def GetReportingData(self):
				770	"""Get a dictionary of reporting data (timings, output hashes)"""
				771	self._output_data['timings'] = self._step_recorder.timings
				772	return self._output_data
				773
				774
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	775	def CreateArgumentParser():
				776	"""Creates and returns the argument parser."""
				777	parser = argparse.ArgumentParser()
				778	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	779	'--buildbot', action='store_true',
				780	help='If true, the script expects to be run on a buildbot')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	781	parser.add_argument(
Matthew Cary	453ff145	2018-07-18 12:19:35	[diff] [blame]	782	'--device', default=None, type=str,
				783	help='Device serial number on which to run profiling.')
				784	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	785	'--verify', action='store_true',
				786	help='If true, the script only verifies the current orderfile')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	787	parser.add_argument('--target-arch', action='store', dest='arch',
				788	default=cygprofile_utils.DetectArchitecture(),
				789	choices=['arm', 'arm64', 'x86', 'x86_64', 'x64', 'mips'],
				790	help='The target architecture for which to build')
				791	parser.add_argument('--output-json', action='store', dest='json_file',
				792	help='Location to save stats in json format')
				793	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	794	'--skip-profile', action='store_false', dest='profile', default=True,
				795	help='Don\'t generate a profile on the device. Only patch from the '
				796	'existing profile.')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	797	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	798	'--skip-patch', action='store_false', dest='patch', default=True,
				799	help='Only generate the raw (unpatched) orderfile, don\'t patch it.')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	800	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	801	'--netrc', action='store',
				802	help='A custom .netrc file to use for git checkin. Only used on bots.')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	803	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	804	'--branch', action='store', default='master',
				805	help='When running on buildbot with a netrc, the branch orderfile '
				806	'hashes get checked into.')
				807	# Note: -j50 was causing issues on the bot.
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	808	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	809	'-j', '--jobs', action='store', default=20,
				810	help='Number of jobs to use for compilation.')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	811	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	812	'-l', '--max-load', action='store', default=4, help='Max cpu load.')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	813	parser.add_argument('--goma-dir', help='GOMA directory.')
				814	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	815	'--use-goma', action='store_true', help='Enable GOMA.', default=False)
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	816	parser.add_argument('--adb-path', help='Path to the adb binary.')
Matthew Cary	e840064	2018-06-14 15:43:02	[diff] [blame]	817
Matthew Cary	c262f5d	2018-09-19 14:36:28	[diff] [blame^]	818	parser.add_argument('--no-system-health-orderfile', action='store_false',
				819	dest='system_health_orderfile', default=True,
				820	help=('Create an orderfile based on an about:blank '
				821	'startup benchmark instead of system health '
				822	'benchmarks.'))
Matthew Cary	d6bfcb7	2018-09-14 11:27:46	[diff] [blame]	823	parser.add_argument('--monochrome', action='store_true',
				824	help=('Compile and instrument monochrome (for post-N '
				825	'devices).'))
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	826	parser.add_argument('--offsets-for-memory', action='store_true',
				827	help=('Favor memory savings in the orderfile. Used '
				828	'with --system-health-orderfile.'),
				829	default=False)
				830
Matthew Cary	e840064	2018-06-14 15:43:02	[diff] [blame]	831	parser.add_argument('--manual-symbol-offsets', default=None, type=str,
				832	help=('File of list of ordered symbol offsets generated '
				833	'by manual profiling. Must set other --manual* '
				834	'flags if this is used, and must --skip-profile.'))
				835	parser.add_argument('--manual-libname', default=None, type=str,
				836	help=('Library filename corresponding to '
				837	'--manual-symbol-offsets.'))
				838	parser.add_argument('--manual-objdir', default=None, type=str,
				839	help=('Root of object file directory corresponding to '
				840	'--manual-symbol-offsets.'))
Matthew Cary	69e9e42	2018-08-10 18:30:06	[diff] [blame]	841	parser.add_argument('--pregenerated-profiles', default=None, type=str,
				842	help=('Pregenerated profiles to use instead of running '
				843	'profile step. Cannot be used with '
				844	'--skip-profiles.'))
				845	parser.add_argument('--profile-save-dir', default=None, type=str,
				846	help=('Directory to save any profiles created. These can '
				847	'be used with --pregenerated-profiles. Cannot be '
				848	'used with --skip-profiles.'))
Matthew Cary	e840064	2018-06-14 15:43:02	[diff] [blame]	849
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	850	profile_android_startup.AddProfileCollectionArguments(parser)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	851	return parser
				852
				853
				854	def CreateOrderfile(options, orderfile_updater_class):
				855	"""Creates an oderfile.
				856
				857	Args:
				858	options: As returned from optparse.OptionParser.parse_args()
				859	orderfile_updater_class: (OrderfileUpdater) subclass of OrderfileUpdater.
				860
				861	Returns:
				862	True iff success.
				863	"""
				864	logging.basicConfig(level=logging.INFO)
				865	devil_chromium.Initialize(adb_path=options.adb_path)
				866
Egor Pasko	93e514e	2017-10-31 13:32:36	[diff] [blame]	867	generator = OrderfileGenerator(options, orderfile_updater_class)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	868	try:
				869	if options.verify:
Egor Pasko	93e514e	2017-10-31 13:32:36	[diff] [blame]	870	generator._VerifySymbolOrder()
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	871	else:
Egor Pasko	93e514e	2017-10-31 13:32:36	[diff] [blame]	872	return generator.Generate()
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	873	finally:
Egor Pasko	93e514e	2017-10-31 13:32:36	[diff] [blame]	874	json_output = json.dumps(generator.GetReportingData(),
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	875	indent=2) + '\n'
				876	if options.json_file:
				877	with open(options.json_file, 'w') as f:
				878	f.write(json_output)
				879	print json_output
				880	return False
				881
				882
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	883	def main():
				884	parser = CreateArgumentParser()
				885	options = parser.parse_args()
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	886	return 0 if CreateOrderfile(options, OrderfileUpdater) else 1
				887
				888
				889	if __name__ == '__main__':
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	890	sys.exit(main())