Blame - tools/cygprofile/orderfile_generator_backend.py - chromium/src

blob: c4af0b4ad1d2dd79423828a046f634b791a489de [file] [log] [blame]

Egor Pasko	0462e852d	2018-03-29 15:52:09	[diff] [blame]	1	#!/usr/bin/env vpython
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	2	# Copyright (c) 2013 The Chromium Authors. All rights reserved.
				3	# Use of this source code is governed by a BSD-style license that can be
				4	# found in the LICENSE file.
				5
				6	""" A utility to generate an up-to-date orderfile.
				7
				8	The orderfile is used by the linker to order text sections such that the
				9	sections are placed consecutively in the order specified. This allows us
				10	to page in less code during start-up.
				11
				12	Example usage:
Egor Pasko	93e514e	2017-10-31 13:32:36	[diff] [blame]	13	tools/cygprofile/orderfile_generator_backend.py -l 20 -j 1000 --use-goma \
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	14	--target-arch=arm
				15	"""
				16
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	17	import argparse
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	18	import hashlib
				19	import json
Matthew Cary	69e9e42	2018-08-10 18:30:06	[diff] [blame]	20	import glob
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	21	import logging
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	22	import os
				23	import re
				24	import shutil
				25	import subprocess
				26	import sys
Benoit Lize	a87e5bc	2017-11-07 15:12:57	[diff] [blame]	27	import tempfile
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	28	import time
				29
Matthew Cary	e840064	2018-06-14 15:43:02	[diff] [blame]	30	import cyglog_to_orderfile
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	31	import cygprofile_utils
Benoit Lize	fefbb27c	2018-01-17 13:54:18	[diff] [blame]	32	import patch_orderfile
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	33	import phased_orderfile
Benoit Lize	a87e5bc	2017-11-07 15:12:57	[diff] [blame]	34	import process_profiles
Egor Pasko	3bc0b93	2018-04-03 10:08:27	[diff] [blame]	35	import profile_android_startup
Benoit Lize	fefbb27c	2018-01-17 13:54:18	[diff] [blame]	36	import symbol_extractor
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	37
				38
				39	_SRC_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)),
				40	os.pardir, os.pardir)
				41	sys.path.append(os.path.join(_SRC_PATH, 'build', 'android'))
				42	import devil_chromium
				43	from pylib import constants
				44
				45
				46	# Needs to happen early for GetBuildType()/GetOutDirectory() to work correctly
				47	constants.SetBuildType('Release')
				48
				49
				50	class CommandError(Exception):
				51	"""Indicates that a dispatched shell command exited with a non-zero status."""
				52
				53	def __init__(self, value):
				54	super(CommandError, self).__init__()
				55	self.value = value
				56
				57	def __str__(self):
				58	return repr(self.value)
				59
				60
				61	def _GenerateHash(file_path):
				62	"""Calculates and returns the hash of the file at file_path."""
				63	sha1 = hashlib.sha1()
				64	with open(file_path, 'rb') as f:
				65	while True:
				66	# Read in 1mb chunks, so it doesn't all have to be loaded into memory.
				67	chunk = f.read(1024 * 1024)
				68	if not chunk:
				69	break
				70	sha1.update(chunk)
				71	return sha1.hexdigest()
				72
				73
				74	def _GetFileExtension(file_name):
				75	"""Calculates the file extension from a file name.
				76
				77	Args:
				78	file_name: The source file name.
				79	Returns:
				80	The part of file_name after the dot (.) or None if the file has no
				81	extension.
				82	Examples: /home/user/foo.bar -> bar
				83	/home/user.name/foo -> None
				84	/home/user/.foo -> None
				85	/home/user/foo.bar.baz -> baz
				86	"""
				87	file_name_parts = os.path.basename(file_name).split('.')
				88	if len(file_name_parts) > 1:
				89	return file_name_parts[-1]
				90	else:
				91	return None
				92
				93
				94	def _StashOutputDirectory(buildpath):
				95	"""Takes the output directory and stashes it in the default output directory.
				96
				97	This allows it to be used for incremental builds next time (after unstashing)
				98	by keeping it in a place that isn't deleted normally, while also ensuring
				99	that it is properly clobbered when appropriate.
				100
				101	This is a dirty hack to deal with the needs of clobbering while also handling
				102	incremental builds and the hardcoded relative paths used in some of the
				103	project files.
				104
				105	Args:
				106	buildpath: The path where the building happens. If this corresponds to the
				107	default output directory, no action is taken.
				108	"""
				109	if os.path.abspath(buildpath) == os.path.abspath(os.path.dirname(
				110	constants.GetOutDirectory())):
				111	return
				112	name = os.path.basename(buildpath)
				113	stashpath = os.path.join(constants.GetOutDirectory(), name)
				114	if not os.path.exists(buildpath):
				115	return
				116	if os.path.exists(stashpath):
				117	shutil.rmtree(stashpath, ignore_errors=True)
				118	shutil.move(buildpath, stashpath)
				119
				120
				121	def _UnstashOutputDirectory(buildpath):
				122	"""Inverse of _StashOutputDirectory.
				123
				124	Moves the output directory stashed within the default output directory
				125	(out/Release) to the position where the builds can actually happen.
				126
				127	This is a dirty hack to deal with the needs of clobbering while also handling
				128	incremental builds and the hardcoded relative paths used in some of the
				129	project files.
				130
				131	Args:
				132	buildpath: The path where the building happens. If this corresponds to the
				133	default output directory, no action is taken.
				134	"""
				135	if os.path.abspath(buildpath) == os.path.abspath(os.path.dirname(
				136	constants.GetOutDirectory())):
				137	return
				138	name = os.path.basename(buildpath)
				139	stashpath = os.path.join(constants.GetOutDirectory(), name)
				140	if not os.path.exists(stashpath):
				141	return
				142	if os.path.exists(buildpath):
				143	shutil.rmtree(buildpath, ignore_errors=True)
				144	shutil.move(stashpath, buildpath)
				145
				146
				147	class StepRecorder(object):
				148	"""Records steps and timings."""
				149
				150	def __init__(self, buildbot):
				151	self.timings = []
				152	self._previous_step = ('', 0.0)
				153	self._buildbot = buildbot
				154	self._error_recorded = False
				155
				156	def BeginStep(self, name):
				157	"""Marks a beginning of the next step in the script.
				158
				159	On buildbot, this prints a specially formatted name that will show up
				160	in the waterfall. Otherwise, just prints the step name.
				161
				162	Args:
				163	name: The name of the step.
				164	"""
				165	self.EndStep()
				166	self._previous_step = (name, time.time())
				167	print 'Running step: ', name
				168
				169	def EndStep(self):
				170	"""Records successful completion of the current step.
				171
				172	This is optional if the step is immediately followed by another BeginStep.
				173	"""
				174	if self._previous_step[0]:
				175	elapsed = time.time() - self._previous_step[1]
				176	print 'Step %s took %f seconds' % (self._previous_step[0], elapsed)
				177	self.timings.append((self._previous_step[0], elapsed))
				178
				179	self._previous_step = ('', 0.0)
				180
				181	def FailStep(self, message=None):
				182	"""Marks that a particular step has failed.
				183
				184	On buildbot, this will mark the current step as failed on the waterfall.
				185	Otherwise we will just print an optional failure message.
				186
				187	Args:
				188	message: An optional explanation as to why the step failed.
				189	"""
				190	print 'STEP FAILED!!'
				191	if message:
				192	print message
				193	self._error_recorded = True
				194	self.EndStep()
				195
				196	def ErrorRecorded(self):
				197	"""True if FailStep has been called."""
				198	return self._error_recorded
				199
				200	def RunCommand(self, cmd, cwd=constants.DIR_SOURCE_ROOT, raise_on_error=True,
				201	stdout=None):
				202	"""Execute a shell command.
				203
				204	Args:
				205	cmd: A list of command strings.
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	206	cwd: Directory in which the command should be executed, defaults to build
				207	root of script's location if not specified.
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	208	raise_on_error: If true will raise a CommandError if the call doesn't
				209	succeed and mark the step as failed.
				210	stdout: A file to redirect stdout for the command to.
				211
				212	Returns:
				213	The process's return code.
				214
				215	Raises:
				216	CommandError: An error executing the specified command.
				217	"""
				218	print 'Executing %s in %s' % (' '.join(cmd), cwd)
				219	process = subprocess.Popen(cmd, stdout=stdout, cwd=cwd, env=os.environ)
				220	process.wait()
				221	if raise_on_error and process.returncode != 0:
				222	self.FailStep()
				223	raise CommandError('Exception executing command %s' % ' '.join(cmd))
				224	return process.returncode
				225
				226
				227	class ClankCompiler(object):
				228	"""Handles compilation of clank."""
				229
				230	def __init__(self, out_dir, step_recorder, arch, jobs, max_load, use_goma,
Matthew Cary	d6bfcb7	2018-09-14 11:27:46	[diff] [blame^]	231	goma_dir, system_health_profiling, monochrome):
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	232	self._out_dir = out_dir
				233	self._step_recorder = step_recorder
Benoit Lize	a87e5bc	2017-11-07 15:12:57	[diff] [blame]	234	self._arch = arch
				235	self._jobs = jobs
				236	self._max_load = max_load
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	237	self._use_goma = use_goma
Benoit Lize	a87e5bc	2017-11-07 15:12:57	[diff] [blame]	238	self._goma_dir = goma_dir
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	239	self._system_health_profiling = system_health_profiling
Matthew Cary	d6bfcb7	2018-09-14 11:27:46	[diff] [blame^]	240	if monochrome:
				241	self._apk = 'Monochrome.apk'
				242	self._libname = 'libmonochrome'
				243	self._libchrome_target = 'monochrome'
				244	else:
				245	self._apk = 'Chrome.apk'
				246	self._libname = 'libchrome'
				247	self._libchrome_target = 'libchrome'
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	248
				249	self.obj_dir = os.path.join(self._out_dir, 'Release', 'obj')
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	250	self.lib_chrome_so = os.path.join(
Matthew Cary	d6bfcb7	2018-09-14 11:27:46	[diff] [blame^]	251	self._out_dir, 'Release', 'lib.unstripped',
				252	'{}.so'.format(self._libname))
				253	self.chrome_apk = os.path.join(self._out_dir, 'Release', 'apks', self._apk)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	254
				255	def Build(self, instrumented, target):
				256	"""Builds the provided ninja target with or without order_profiling on.
				257
				258	Args:
Benoit Lize	a87e5bc	2017-11-07 15:12:57	[diff] [blame]	259	instrumented: (bool) Whether we want to build an instrumented binary.
				260	target: (str) The name of the ninja target to build.
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	261	"""
				262	self._step_recorder.BeginStep('Compile %s' % target)
				263
				264	# Set the "Release Official" flavor, the parts affecting performance.
				265	args = [
Peter Collingbourne	1c85d5d2	2018-08-22 18:06:35	[diff] [blame]	266	'enable_resource_whitelist_generation=false',
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	267	'is_chrome_branded=true',
				268	'is_debug=false',
				269	'is_official_build=true',
				270	# We have to build with no symbols if profiling and minimal symbols
				271	# otherwise for libchrome.so to fit under the 4 GB limit.
				272	# crbug.com/574476
				273	'symbol_level=' + ('0' if instrumented else '1'),
				274	'target_cpu="' + self._arch + '"',
				275	'target_os="android"',
				276	'use_goma=' + str(self._use_goma).lower(),
				277	'use_order_profiling=' + str(instrumented).lower(),
				278	]
				279	if self._goma_dir:
				280	args += ['goma_dir="%s"' % self._goma_dir]
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	281	if self._system_health_profiling:
				282	args += ['devtools_instrumentation_dumping = ' +
				283	str(instrumented).lower()]
Benoit Lize	a87e5bc	2017-11-07 15:12:57	[diff] [blame]	284
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	285	self._step_recorder.RunCommand(
				286	['gn', 'gen', os.path.join(self._out_dir, 'Release'),
				287	'--args=' + ' '.join(args)])
				288
				289	self._step_recorder.RunCommand(
				290	['ninja', '-C', os.path.join(self._out_dir, 'Release'),
				291	'-j' + str(self._jobs), '-l' + str(self._max_load), target])
				292
				293	def CompileChromeApk(self, instrumented, force_relink=False):
				294	"""Builds a Chrome.apk either with or without order_profiling on.
				295
				296	Args:
Benoit Lize	a87e5bc	2017-11-07 15:12:57	[diff] [blame]	297	instrumented: (bool) Whether to build an instrumented apk.
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	298	force_relink: Whether libchromeview.so should be re-created.
				299	"""
				300	if force_relink:
				301	self._step_recorder.RunCommand(['rm', '-rf', self.lib_chrome_so])
				302	self.Build(instrumented, 'chrome_apk')
				303
				304	def CompileLibchrome(self, instrumented, force_relink=False):
				305	"""Builds a libchrome.so either with or without order_profiling on.
				306
				307	Args:
Benoit Lize	a87e5bc	2017-11-07 15:12:57	[diff] [blame]	308	instrumented: (bool) Whether to build an instrumented apk.
				309	force_relink: (bool) Whether libchrome.so should be re-created.
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	310	"""
				311	if force_relink:
				312	self._step_recorder.RunCommand(['rm', '-rf', self.lib_chrome_so])
Matthew Cary	d6bfcb7	2018-09-14 11:27:46	[diff] [blame^]	313	self.Build(instrumented, self._libchrome_target)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	314
				315
				316	class OrderfileUpdater(object):
				317	"""Handles uploading and committing a new orderfile in the repository.
				318
				319	Only used for testing or on a bot.
				320	"""
				321
				322	_CLOUD_STORAGE_BUCKET_FOR_DEBUG = None
				323	_CLOUD_STORAGE_BUCKET = None
				324	_UPLOAD_TO_CLOUD_COMMAND = 'upload_to_google_storage.py'
				325
				326	def __init__(self, repository_root, step_recorder, branch, netrc):
				327	"""Constructor.
				328
				329	Args:
				330	repository_root: (str) Root of the target repository.
				331	step_recorder: (StepRecorder) Step recorder, for logging.
				332	branch: (str) Branch to commit to.
				333	netrc: (str) Path to the .netrc file to use.
				334	"""
				335	self._repository_root = repository_root
				336	self._step_recorder = step_recorder
				337	self._branch = branch
				338	self._netrc = netrc
				339
				340	def CommitFileHashes(self, unpatched_orderfile_filename, orderfile_filename):
				341	"""Commits unpatched and patched orderfiles hashes, if provided.
				342
				343	Files must have been successfilly uploaded to cloud storage first.
				344
				345	Args:
				346	unpatched_orderfile_filename: (str or None) Unpatched orderfile path.
				347	orderfile_filename: (str or None) Orderfile path.
				348
				349	Raises:
				350	NotImplementedError when the commit logic hasn't been overriden.
				351	"""
				352	files_to_commit = []
				353	commit_message_lines = ['Update Orderfile.']
				354	for filename in [unpatched_orderfile_filename, orderfile_filename]:
				355	if not filename:
				356	continue
				357	(relative_path, sha1) = self._GetHashFilePathAndContents(filename)
				358	commit_message_lines.append('Profile: %s: %s' % (
				359	os.path.basename(relative_path), sha1))
				360	files_to_commit.append(relative_path)
				361	if files_to_commit:
				362	self._CommitFiles(files_to_commit, commit_message_lines)
				363
				364	def UploadToCloudStorage(self, filename, use_debug_location):
				365	"""Uploads a file to cloud storage.
				366
				367	Args:
				368	filename: (str) File to upload.
				369	use_debug_location: (bool) Whether to use the debug location.
				370	"""
				371	bucket = (self._CLOUD_STORAGE_BUCKET_FOR_DEBUG if use_debug_location
				372	else self._CLOUD_STORAGE_BUCKET)
				373	extension = _GetFileExtension(filename)
				374	cmd = [self._UPLOAD_TO_CLOUD_COMMAND, '--bucket', bucket]
				375	if extension:
				376	cmd.extend(['-z', extension])
				377	cmd.append(filename)
				378	self._step_recorder.RunCommand(cmd)
				379	print 'Download: https://sandbox.google.com/storage/%s/%s' % (
				380	bucket, _GenerateHash(filename))
				381
				382	def _GetHashFilePathAndContents(self, filename):
				383	"""Gets the name and content of the hash file created from uploading the
				384	given file.
				385
				386	Args:
				387	filename: (str) The file that was uploaded to cloud storage.
				388
				389	Returns:
				390	A tuple of the hash file name, relative to the reository root, and the
				391	content, which should be the sha1 hash of the file
				392	('base_file.sha1', hash)
				393	"""
				394	abs_hash_filename = filename + '.sha1'
				395	rel_hash_filename = os.path.relpath(
				396	abs_hash_filename, self._repository_root)
				397	with open(abs_hash_filename, 'r') as f:
				398	return (rel_hash_filename, f.read())
				399
				400	def _CommitFiles(self, files_to_commit, commit_message_lines):
				401	"""Commits a list of files, with a given message."""
				402	raise NotImplementedError
				403
				404
				405	class OrderfileGenerator(object):
				406	"""A utility for generating a new orderfile for Clank.
				407
				408	Builds an instrumented binary, profiles a run of the application, and
				409	generates an updated orderfile.
				410	"""
				411	_CLANK_REPO = os.path.join(constants.DIR_SOURCE_ROOT, 'clank')
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	412	_CHECK_ORDERFILE_SCRIPT = os.path.join(
				413	constants.DIR_SOURCE_ROOT, 'tools', 'cygprofile', 'check_orderfile.py')
				414	_BUILD_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(
				415	constants.GetOutDirectory()))) # Normally /path/to/src
				416
				417	_UNPATCHED_ORDERFILE_FILENAME = os.path.join(
				418	_CLANK_REPO, 'orderfiles', 'unpatched_orderfile.%s')
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	419
				420	_PATH_TO_ORDERFILE = os.path.join(_CLANK_REPO, 'orderfiles',
				421	'orderfile.%s.out')
				422
				423	# Previous orderfile_generator debug files would be overwritten.
				424	_DIRECTORY_FOR_DEBUG_FILES = '/tmp/orderfile_generator_debug_files'
				425
				426	def _GetPathToOrderfile(self):
				427	"""Gets the path to the architecture-specific orderfile."""
				428	return self._PATH_TO_ORDERFILE % self._options.arch
				429
				430	def _GetUnpatchedOrderfileFilename(self):
				431	"""Gets the path to the architecture-specific unpatched orderfile."""
				432	return self._UNPATCHED_ORDERFILE_FILENAME % self._options.arch
				433
				434	def __init__(self, options, orderfile_updater_class):
				435	self._options = options
				436
				437	self._instrumented_out_dir = os.path.join(
				438	self._BUILD_ROOT, self._options.arch + '_instrumented_out')
				439	self._uninstrumented_out_dir = os.path.join(
				440	self._BUILD_ROOT, self._options.arch + '_uninstrumented_out')
				441
				442	if options.profile:
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	443	output_directory = os.path.join(self._instrumented_out_dir, 'Release')
Matthew Cary	b8daed94	2018-06-11 10:58:08	[diff] [blame]	444	host_profile_dir = os.path.join(output_directory, 'profile_data')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	445	urls = [profile_android_startup.AndroidProfileTool.TEST_URL]
				446	use_wpr = True
				447	simulate_user = False
Benoit L	9646681	2018-03-06 15:58:37	[diff] [blame]	448	urls = options.urls
				449	use_wpr = not options.no_wpr
				450	simulate_user = options.simulate_user
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	451	self._profiler = profile_android_startup.AndroidProfileTool(
Matthew Cary	453ff145	2018-07-18 12:19:35	[diff] [blame]	452	output_directory, host_profile_dir, use_wpr, urls, simulate_user,
				453	device=options.device)
Matthew Cary	69e9e42	2018-08-10 18:30:06	[diff] [blame]	454	if options.pregenerated_profiles:
				455	self._profiler.SetPregeneratedProfiles(
				456	glob.glob(options.pregenerated_profiles))
				457	else:
				458	assert not options.pregenerated_profiles, (
				459	'--pregenerated-profiles cannot be used with --skip-profile')
				460	assert not options.profile_save_dir, (
				461	'--profile-save-dir cannot be used with --skip-profile')
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	462
				463	self._output_data = {}
				464	self._step_recorder = StepRecorder(options.buildbot)
				465	self._compiler = None
				466	assert issubclass(orderfile_updater_class, OrderfileUpdater)
				467	self._orderfile_updater = orderfile_updater_class(self._CLANK_REPO,
				468	self._step_recorder,
				469	options.branch,
				470	options.netrc)
				471	assert os.path.isdir(constants.DIR_SOURCE_ROOT), 'No src directory found'
Benoit Lize	fefbb27c	2018-01-17 13:54:18	[diff] [blame]	472	symbol_extractor.SetArchitecture(options.arch)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	473
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	474	@staticmethod
				475	def _RemoveBlanks(src_file, dest_file):
				476	"""A utility to remove blank lines from a file.
				477
				478	Args:
				479	src_file: The name of the file to remove the blanks from.
				480	dest_file: The name of the file to write the output without blanks.
				481	"""
				482	assert src_file != dest_file, 'Source and destination need to be distinct'
				483
				484	try:
				485	src = open(src_file, 'r')
				486	dest = open(dest_file, 'w')
				487	for line in src:
				488	if line and not line.isspace():
				489	dest.write(line)
				490	finally:
				491	src.close()
				492	dest.close()
				493
				494	def _GenerateAndProcessProfile(self):
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	495	"""Invokes a script to merge the per-thread traces into one file.
				496
				497	The produced list of offsets is saved in
				498	self._GetUnpatchedOrderfileFilename().
				499	"""
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	500	self._step_recorder.BeginStep('Generate Profile Data')
				501	files = []
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	502	logging.getLogger().setLevel(logging.DEBUG)
				503	if self._options.system_health_orderfile:
				504	files = self._profiler.CollectSystemHealthProfile(
				505	self._compiler.chrome_apk)
Matthew Cary	69e9e42	2018-08-10 18:30:06	[diff] [blame]	506	self._MaybeSaveProfile(files)
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	507	try:
				508	self._ProcessPhasedOrderfile(files)
				509	except Exception:
				510	for f in files:
				511	self._SaveForDebugging(f)
Matthew Cary	8b141623	2018-08-10 19:12:22	[diff] [blame]	512	self._SaveForDebugging(self._compiler.lib_chrome_so)
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	513	raise
				514	finally:
				515	self._profiler.Cleanup()
				516	else:
				517	self._CollectLegacyProfile()
				518	logging.getLogger().setLevel(logging.INFO)
				519
				520	def _ProcessPhasedOrderfile(self, files):
				521	"""Process the phased orderfiles produced by system health benchmarks.
				522
				523	The offsets will be placed in _GetUnpatchedOrderfileFilename().
				524
				525	Args:
				526	file: Profile files pulled locally.
				527	"""
				528	self._step_recorder.BeginStep('Process Phased Orderfile')
				529	profiles = process_profiles.ProfileManager(files)
				530	processor = process_profiles.SymbolOffsetProcessor(
				531	self._compiler.lib_chrome_so)
				532	phaser = phased_orderfile.PhasedAnalyzer(profiles, processor)
				533	if self._options.offsets_for_memory:
				534	profile_offsets = phaser.GetOffsetsForMemoryFootprint()
				535	else:
				536	profile_offsets = phaser.GetOffsetsForStartup()
				537	self._output_data['orderfile_size'] = {
				538	'startup_kib': processor.OffsetsPrimarySize(
				539	profile_offsets.startup) / 1024,
				540	'common_kib': processor.OffsetsPrimarySize(
				541	profile_offsets.common) / 1024,
				542	'interaction_kib': processor.OffsetsPrimarySize(
				543	profile_offsets.interaction) / 1024}
				544
				545	offsets_list = (profile_offsets.startup +
				546	profile_offsets.common +
				547	profile_offsets.interaction)
Matthew Cary	8b141623	2018-08-10 19:12:22	[diff] [blame]	548	ordered_symbols = processor.GetOrderedSymbols(offsets_list)
				549	if not ordered_symbols:
				550	raise Exception('Failed to get ordered symbols')
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	551	with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile:
Matthew Cary	8b141623	2018-08-10 19:12:22	[diff] [blame]	552	orderfile.write('\n'.join(ordered_symbols))
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	553
				554	def _CollectLegacyProfile(self):
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	555	try:
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	556	files = self._profiler.CollectProfile(
				557	self._compiler.chrome_apk,
				558	constants.PACKAGE_INFO['chrome'])
Matthew Cary	69e9e42	2018-08-10 18:30:06	[diff] [blame]	559	self._MaybeSaveProfile(files)
Matthew Cary	b8daed94	2018-06-11 10:58:08	[diff] [blame]	560	self._step_recorder.BeginStep('Process profile')
Benoit L	9646681	2018-03-06 15:58:37	[diff] [blame]	561	assert os.path.exists(self._compiler.lib_chrome_so)
				562	offsets = process_profiles.GetReachedOffsetsFromDumpFiles(
				563	files, self._compiler.lib_chrome_so)
				564	if not offsets:
				565	raise Exception('No profiler offsets found in {}'.format(
				566	'\n'.join(files)))
Matthew Cary	8b141623	2018-08-10 19:12:22	[diff] [blame]	567	processor = process_profiles.SymbolOffsetProcessor(
				568	self._compiler.lib_chrome_so)
				569	ordered_symbols = processor.GetOrderedSymbols(offsets)
				570	if not ordered_symbols:
				571	raise Exception('No symbol names from offsets found in {}'.format(
				572	'\n'.join(files)))
				573	with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile:
				574	orderfile.write('\n'.join(ordered_symbols))
Matthew Cary	0f1f681a	2018-01-22 10:40:51	[diff] [blame]	575	except Exception:
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	576	for f in files:
				577	self._SaveForDebugging(f)
				578	raise
				579	finally:
				580	self._profiler.Cleanup()
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	581
Matthew Cary	69e9e42	2018-08-10 18:30:06	[diff] [blame]	582	def _MaybeSaveProfile(self, files):
				583	if self._options.profile_save_dir:
				584	logging.info('Saving profiles to %s', self._options.profile_save_dir)
				585	for f in files:
				586	shutil.copy(f, self._options.profile_save_dir)
				587	logging.info('Saved profile %s', f)
				588
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	589	def _PatchOrderfile(self):
				590	"""Patches the orderfile using clean version of libchrome.so."""
				591	self._step_recorder.BeginStep('Patch Orderfile')
Benoit Lize	fefbb27c	2018-01-17 13:54:18	[diff] [blame]	592	patch_orderfile.GeneratePatchedOrderfile(
				593	self._GetUnpatchedOrderfileFilename(), self._compiler.lib_chrome_so,
				594	self._GetPathToOrderfile())
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	595
				596	def _VerifySymbolOrder(self):
				597	self._step_recorder.BeginStep('Verify Symbol Order')
				598	return_code = self._step_recorder.RunCommand(
				599	[self._CHECK_ORDERFILE_SCRIPT, self._compiler.lib_chrome_so,
				600	self._GetPathToOrderfile(),
				601	'--target-arch=' + self._options.arch],
				602	constants.DIR_SOURCE_ROOT,
				603	raise_on_error=False)
				604	if return_code:
				605	self._step_recorder.FailStep('Orderfile check returned %d.' % return_code)
				606
				607	def _RecordHash(self, file_name):
				608	"""Records the hash of the file into the output_data dictionary."""
				609	self._output_data[os.path.basename(file_name) + '.sha1'] = _GenerateHash(
				610	file_name)
				611
				612	def _SaveFileLocally(self, file_name, file_sha1):
				613	"""Saves the file to a temporary location and prints the sha1sum."""
				614	if not os.path.exists(self._DIRECTORY_FOR_DEBUG_FILES):
				615	os.makedirs(self._DIRECTORY_FOR_DEBUG_FILES)
				616	shutil.copy(file_name, self._DIRECTORY_FOR_DEBUG_FILES)
				617	print 'File: %s, saved in: %s, sha1sum: %s' % (
				618	file_name, self._DIRECTORY_FOR_DEBUG_FILES, file_sha1)
				619
				620	def _SaveForDebugging(self, filename):
				621	"""Uploads the file to cloud storage or saves to a temporary location."""
				622	file_sha1 = _GenerateHash(filename)
				623	if not self._options.buildbot:
				624	self._SaveFileLocally(filename, file_sha1)
				625	else:
				626	print 'Uploading file for debugging: ' + filename
				627	self._orderfile_updater.UploadToCloudStorage(
				628	filename, use_debug_location=True)
				629
				630	def _SaveForDebuggingWithOverwrite(self, file_name):
				631	"""Uploads and overwrites the file in cloud storage or copies locally.
				632
				633	Should be used for large binaries like lib_chrome_so.
				634
				635	Args:
				636	file_name: (str) File to upload.
				637	"""
				638	file_sha1 = _GenerateHash(file_name)
				639	if not self._options.buildbot:
				640	self._SaveFileLocally(file_name, file_sha1)
				641	else:
				642	print 'Uploading file for debugging: %s, sha1sum: %s' % (
				643	file_name, file_sha1)
				644	upload_location = '%s/%s' % (
				645	self._CLOUD_STORAGE_BUCKET_FOR_DEBUG, os.path.basename(file_name))
				646	self._step_recorder.RunCommand([
				647	'gsutil.py', 'cp', file_name, 'gs://' + upload_location])
				648	print ('Uploaded to: https://sandbox.google.com/storage/' +
				649	upload_location)
				650
				651	def _MaybeArchiveOrderfile(self, filename):
				652	"""In buildbot configuration, uploads the generated orderfile to
				653	Google Cloud Storage.
				654
				655	Args:
				656	filename: (str) Orderfile to upload.
				657	"""
				658	# First compute hashes so that we can download them later if we need to
				659	self._step_recorder.BeginStep('Compute hash for ' + filename)
				660	self._RecordHash(filename)
				661	if self._options.buildbot:
				662	self._step_recorder.BeginStep('Archive ' + filename)
				663	self._orderfile_updater.UploadToCloudStorage(
				664	filename, use_debug_location=False)
				665
				666	def _GetHashFilePathAndContents(self, base_file):
				667	"""Gets the name and content of the hash file created from uploading the
				668	given file.
				669
				670	Args:
				671	base_file: The file that was uploaded to cloud storage.
				672
				673	Returns:
				674	A tuple of the hash file name, relative to the clank repo path, and the
				675	content, which should be the sha1 hash of the file
				676	('base_file.sha1', hash)
				677	"""
				678	abs_file_name = base_file + '.sha1'
				679	rel_file_name = os.path.relpath(abs_file_name, self._CLANK_REPO)
				680	with open(abs_file_name, 'r') as f:
				681	return (rel_file_name, f.read())
				682
				683	def Generate(self):
				684	"""Generates and maybe upload an order."""
				685	profile_uploaded = False
				686	orderfile_uploaded = False
				687
Matthew Cary	e840064	2018-06-14 15:43:02	[diff] [blame]	688	assert (bool(self._options.profile) ^
				689	bool(self._options.manual_symbol_offsets))
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	690	if self._options.system_health_orderfile and not self._options.profile:
				691	raise AssertionError('--system_health_orderfile must be not be used '
				692	'with --skip-profile')
				693	if (self._options.manual_symbol_offsets and
				694	not self._options.system_health_orderfile):
				695	raise AssertionError('--manual-symbol-offsets must be used with '
				696	'--system_health_orderfile.')
Matthew Cary	e840064	2018-06-14 15:43:02	[diff] [blame]	697
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	698	if self._options.profile:
				699	try:
				700	_UnstashOutputDirectory(self._instrumented_out_dir)
				701	self._compiler = ClankCompiler(
				702	self._instrumented_out_dir,
				703	self._step_recorder, self._options.arch, self._options.jobs,
				704	self._options.max_load, self._options.use_goma,
Matthew Cary	d6bfcb7	2018-09-14 11:27:46	[diff] [blame^]	705	self._options.goma_dir, self._options.system_health_orderfile,
				706	self._options.monochrome)
Benoit L	9646681	2018-03-06 15:58:37	[diff] [blame]	707	self._compiler.CompileChromeApk(True)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	708	self._GenerateAndProcessProfile()
				709	self._MaybeArchiveOrderfile(self._GetUnpatchedOrderfileFilename())
				710	profile_uploaded = True
				711	finally:
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	712	_StashOutputDirectory(self._instrumented_out_dir)
Matthew Cary	e840064	2018-06-14 15:43:02	[diff] [blame]	713	elif self._options.manual_symbol_offsets:
				714	assert self._options.manual_libname
				715	assert self._options.manual_objdir
				716	with file(self._options.manual_symbol_offsets) as f:
				717	symbol_offsets = [int(x) for x in f.xreadlines()]
				718	processor = process_profiles.SymbolOffsetProcessor(
				719	self._options.manual_libname)
				720	generator = cyglog_to_orderfile.OffsetOrderfileGenerator(
				721	processor, cyglog_to_orderfile.ObjectFileProcessor(
				722	self._options.manual_objdir))
				723	ordered_sections = generator.GetOrderedSections(symbol_offsets)
				724	if not ordered_sections: # Either None or empty is a problem.
				725	raise Exception('Failed to get ordered sections')
				726	with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile:
				727	orderfile.write('\n'.join(ordered_sections))
				728
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	729	if self._options.patch:
				730	if self._options.profile:
				731	self._RemoveBlanks(self._GetUnpatchedOrderfileFilename(),
				732	self._GetPathToOrderfile())
				733	try:
				734	_UnstashOutputDirectory(self._uninstrumented_out_dir)
				735	self._compiler = ClankCompiler(
				736	self._uninstrumented_out_dir, self._step_recorder,
				737	self._options.arch, self._options.jobs, self._options.max_load,
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	738	self._options.use_goma, self._options.goma_dir,
Matthew Cary	d6bfcb7	2018-09-14 11:27:46	[diff] [blame^]	739	self._options.system_health_orderfile, self._options.monochrome)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	740	self._compiler.CompileLibchrome(False)
				741	self._PatchOrderfile()
				742	# Because identical code folding is a bit different with and without
				743	# the orderfile build, we need to re-patch the orderfile with code
				744	# folding as close to the final version as possible.
				745	self._compiler.CompileLibchrome(False, force_relink=True)
				746	self._PatchOrderfile()
				747	self._compiler.CompileLibchrome(False, force_relink=True)
				748	self._VerifySymbolOrder()
				749	self._MaybeArchiveOrderfile(self._GetPathToOrderfile())
				750	finally:
				751	_StashOutputDirectory(self._uninstrumented_out_dir)
				752	orderfile_uploaded = True
				753
				754	if (self._options.buildbot and self._options.netrc
				755	and not self._step_recorder.ErrorRecorded()):
				756	unpatched_orderfile_filename = (
				757	self._GetUnpatchedOrderfileFilename() if profile_uploaded else None)
				758	orderfile_filename = (
Benoit Lize	d913be8	2017-10-24 13:38:27	[diff] [blame]	759	self._GetPathToOrderfile() if orderfile_uploaded else None)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	760	self._orderfile_updater.CommitFileHashes(
				761	unpatched_orderfile_filename, orderfile_filename)
				762
				763	self._step_recorder.EndStep()
				764	return not self._step_recorder.ErrorRecorded()
				765
				766	def GetReportingData(self):
				767	"""Get a dictionary of reporting data (timings, output hashes)"""
				768	self._output_data['timings'] = self._step_recorder.timings
				769	return self._output_data
				770
				771
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	772	def CreateArgumentParser():
				773	"""Creates and returns the argument parser."""
				774	parser = argparse.ArgumentParser()
				775	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	776	'--buildbot', action='store_true',
				777	help='If true, the script expects to be run on a buildbot')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	778	parser.add_argument(
Matthew Cary	453ff145	2018-07-18 12:19:35	[diff] [blame]	779	'--device', default=None, type=str,
				780	help='Device serial number on which to run profiling.')
				781	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	782	'--verify', action='store_true',
				783	help='If true, the script only verifies the current orderfile')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	784	parser.add_argument('--target-arch', action='store', dest='arch',
				785	default=cygprofile_utils.DetectArchitecture(),
				786	choices=['arm', 'arm64', 'x86', 'x86_64', 'x64', 'mips'],
				787	help='The target architecture for which to build')
				788	parser.add_argument('--output-json', action='store', dest='json_file',
				789	help='Location to save stats in json format')
				790	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	791	'--skip-profile', action='store_false', dest='profile', default=True,
				792	help='Don\'t generate a profile on the device. Only patch from the '
				793	'existing profile.')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	794	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	795	'--skip-patch', action='store_false', dest='patch', default=True,
				796	help='Only generate the raw (unpatched) orderfile, don\'t patch it.')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	797	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	798	'--netrc', action='store',
				799	help='A custom .netrc file to use for git checkin. Only used on bots.')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	800	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	801	'--branch', action='store', default='master',
				802	help='When running on buildbot with a netrc, the branch orderfile '
				803	'hashes get checked into.')
				804	# Note: -j50 was causing issues on the bot.
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	805	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	806	'-j', '--jobs', action='store', default=20,
				807	help='Number of jobs to use for compilation.')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	808	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	809	'-l', '--max-load', action='store', default=4, help='Max cpu load.')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	810	parser.add_argument('--goma-dir', help='GOMA directory.')
				811	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	812	'--use-goma', action='store_true', help='Enable GOMA.', default=False)
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	813	parser.add_argument('--adb-path', help='Path to the adb binary.')
Matthew Cary	e840064	2018-06-14 15:43:02	[diff] [blame]	814
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	815	parser.add_argument('--system-health-orderfile', action='store_true',
				816	help=('Create an orderfile based on system health '
				817	'benchmarks.'),
				818	default=False)
Matthew Cary	d6bfcb7	2018-09-14 11:27:46	[diff] [blame^]	819	parser.add_argument('--monochrome', action='store_true',
				820	help=('Compile and instrument monochrome (for post-N '
				821	'devices).'))
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	822	parser.add_argument('--offsets-for-memory', action='store_true',
				823	help=('Favor memory savings in the orderfile. Used '
				824	'with --system-health-orderfile.'),
				825	default=False)
				826
Matthew Cary	e840064	2018-06-14 15:43:02	[diff] [blame]	827	parser.add_argument('--manual-symbol-offsets', default=None, type=str,
				828	help=('File of list of ordered symbol offsets generated '
				829	'by manual profiling. Must set other --manual* '
				830	'flags if this is used, and must --skip-profile.'))
				831	parser.add_argument('--manual-libname', default=None, type=str,
				832	help=('Library filename corresponding to '
				833	'--manual-symbol-offsets.'))
				834	parser.add_argument('--manual-objdir', default=None, type=str,
				835	help=('Root of object file directory corresponding to '
				836	'--manual-symbol-offsets.'))
Matthew Cary	69e9e42	2018-08-10 18:30:06	[diff] [blame]	837	parser.add_argument('--pregenerated-profiles', default=None, type=str,
				838	help=('Pregenerated profiles to use instead of running '
				839	'profile step. Cannot be used with '
				840	'--skip-profiles.'))
				841	parser.add_argument('--profile-save-dir', default=None, type=str,
				842	help=('Directory to save any profiles created. These can '
				843	'be used with --pregenerated-profiles. Cannot be '
				844	'used with --skip-profiles.'))
Matthew Cary	e840064	2018-06-14 15:43:02	[diff] [blame]	845
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	846	profile_android_startup.AddProfileCollectionArguments(parser)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	847	return parser
				848
				849
				850	def CreateOrderfile(options, orderfile_updater_class):
				851	"""Creates an oderfile.
				852
				853	Args:
				854	options: As returned from optparse.OptionParser.parse_args()
				855	orderfile_updater_class: (OrderfileUpdater) subclass of OrderfileUpdater.
				856
				857	Returns:
				858	True iff success.
				859	"""
				860	logging.basicConfig(level=logging.INFO)
				861	devil_chromium.Initialize(adb_path=options.adb_path)
				862
Egor Pasko	93e514e	2017-10-31 13:32:36	[diff] [blame]	863	generator = OrderfileGenerator(options, orderfile_updater_class)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	864	try:
				865	if options.verify:
Egor Pasko	93e514e	2017-10-31 13:32:36	[diff] [blame]	866	generator._VerifySymbolOrder()
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	867	else:
Egor Pasko	93e514e	2017-10-31 13:32:36	[diff] [blame]	868	return generator.Generate()
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	869	finally:
Egor Pasko	93e514e	2017-10-31 13:32:36	[diff] [blame]	870	json_output = json.dumps(generator.GetReportingData(),
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	871	indent=2) + '\n'
				872	if options.json_file:
				873	with open(options.json_file, 'w') as f:
				874	f.write(json_output)
				875	print json_output
				876	return False
				877
				878
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	879	def main():
				880	parser = CreateArgumentParser()
				881	options = parser.parse_args()
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	882	return 0 if CreateOrderfile(options, OrderfileUpdater) else 1
				883
				884
				885	if __name__ == '__main__':
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	886	sys.exit(main())