Blame - tools/cygprofile/orderfile_generator_backend.py - chromium/src

blob: c6234c46c87cc5c080d3acb6583527c83a486f92 [file] [log] [blame]

Egor Pasko	0462e852d	2018-03-29 15:52:09	[diff] [blame]	1	#!/usr/bin/env vpython
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	2	# Copyright (c) 2013 The Chromium Authors. All rights reserved.
				3	# Use of this source code is governed by a BSD-style license that can be
				4	# found in the LICENSE file.
				5
				6	""" A utility to generate an up-to-date orderfile.
				7
				8	The orderfile is used by the linker to order text sections such that the
				9	sections are placed consecutively in the order specified. This allows us
				10	to page in less code during start-up.
				11
				12	Example usage:
Egor Pasko	93e514e	2017-10-31 13:32:36	[diff] [blame]	13	tools/cygprofile/orderfile_generator_backend.py -l 20 -j 1000 --use-goma \
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	14	--target-arch=arm
				15	"""
				16
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	17	import argparse
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	18	import hashlib
				19	import json
Matthew Cary	69e9e42	2018-08-10 18:30:06	[diff] [blame]	20	import glob
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	21	import logging
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	22	import os
				23	import re
				24	import shutil
				25	import subprocess
				26	import sys
Benoit Lize	a87e5bc	2017-11-07 15:12:57	[diff] [blame]	27	import tempfile
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	28	import time
				29
Matthew Cary	e840064	2018-06-14 15:43:02	[diff] [blame]	30	import cyglog_to_orderfile
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	31	import cygprofile_utils
Benoit Lize	fefbb27c	2018-01-17 13:54:18	[diff] [blame]	32	import patch_orderfile
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	33	import phased_orderfile
Benoit Lize	a87e5bc	2017-11-07 15:12:57	[diff] [blame]	34	import process_profiles
Egor Pasko	3bc0b93	2018-04-03 10:08:27	[diff] [blame]	35	import profile_android_startup
Benoit Lize	fefbb27c	2018-01-17 13:54:18	[diff] [blame]	36	import symbol_extractor
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	37
				38
				39	_SRC_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)),
				40	os.pardir, os.pardir)
				41	sys.path.append(os.path.join(_SRC_PATH, 'build', 'android'))
				42	import devil_chromium
				43	from pylib import constants
				44
				45
				46	# Needs to happen early for GetBuildType()/GetOutDirectory() to work correctly
				47	constants.SetBuildType('Release')
				48
				49
				50	class CommandError(Exception):
				51	"""Indicates that a dispatched shell command exited with a non-zero status."""
				52
				53	def __init__(self, value):
				54	super(CommandError, self).__init__()
				55	self.value = value
				56
				57	def __str__(self):
				58	return repr(self.value)
				59
				60
				61	def _GenerateHash(file_path):
				62	"""Calculates and returns the hash of the file at file_path."""
				63	sha1 = hashlib.sha1()
				64	with open(file_path, 'rb') as f:
				65	while True:
				66	# Read in 1mb chunks, so it doesn't all have to be loaded into memory.
				67	chunk = f.read(1024 * 1024)
				68	if not chunk:
				69	break
				70	sha1.update(chunk)
				71	return sha1.hexdigest()
				72
				73
				74	def _GetFileExtension(file_name):
				75	"""Calculates the file extension from a file name.
				76
				77	Args:
				78	file_name: The source file name.
				79	Returns:
				80	The part of file_name after the dot (.) or None if the file has no
				81	extension.
				82	Examples: /home/user/foo.bar -> bar
				83	/home/user.name/foo -> None
				84	/home/user/.foo -> None
				85	/home/user/foo.bar.baz -> baz
				86	"""
				87	file_name_parts = os.path.basename(file_name).split('.')
				88	if len(file_name_parts) > 1:
				89	return file_name_parts[-1]
				90	else:
				91	return None
				92
				93
				94	def _StashOutputDirectory(buildpath):
				95	"""Takes the output directory and stashes it in the default output directory.
				96
				97	This allows it to be used for incremental builds next time (after unstashing)
				98	by keeping it in a place that isn't deleted normally, while also ensuring
				99	that it is properly clobbered when appropriate.
				100
				101	This is a dirty hack to deal with the needs of clobbering while also handling
				102	incremental builds and the hardcoded relative paths used in some of the
				103	project files.
				104
				105	Args:
				106	buildpath: The path where the building happens. If this corresponds to the
				107	default output directory, no action is taken.
				108	"""
				109	if os.path.abspath(buildpath) == os.path.abspath(os.path.dirname(
				110	constants.GetOutDirectory())):
				111	return
				112	name = os.path.basename(buildpath)
				113	stashpath = os.path.join(constants.GetOutDirectory(), name)
				114	if not os.path.exists(buildpath):
				115	return
				116	if os.path.exists(stashpath):
				117	shutil.rmtree(stashpath, ignore_errors=True)
				118	shutil.move(buildpath, stashpath)
				119
				120
				121	def _UnstashOutputDirectory(buildpath):
				122	"""Inverse of _StashOutputDirectory.
				123
				124	Moves the output directory stashed within the default output directory
				125	(out/Release) to the position where the builds can actually happen.
				126
				127	This is a dirty hack to deal with the needs of clobbering while also handling
				128	incremental builds and the hardcoded relative paths used in some of the
				129	project files.
				130
				131	Args:
				132	buildpath: The path where the building happens. If this corresponds to the
				133	default output directory, no action is taken.
				134	"""
				135	if os.path.abspath(buildpath) == os.path.abspath(os.path.dirname(
				136	constants.GetOutDirectory())):
				137	return
				138	name = os.path.basename(buildpath)
				139	stashpath = os.path.join(constants.GetOutDirectory(), name)
				140	if not os.path.exists(stashpath):
				141	return
				142	if os.path.exists(buildpath):
				143	shutil.rmtree(buildpath, ignore_errors=True)
				144	shutil.move(stashpath, buildpath)
				145
				146
				147	class StepRecorder(object):
				148	"""Records steps and timings."""
				149
				150	def __init__(self, buildbot):
				151	self.timings = []
				152	self._previous_step = ('', 0.0)
				153	self._buildbot = buildbot
				154	self._error_recorded = False
				155
				156	def BeginStep(self, name):
				157	"""Marks a beginning of the next step in the script.
				158
				159	On buildbot, this prints a specially formatted name that will show up
				160	in the waterfall. Otherwise, just prints the step name.
				161
				162	Args:
				163	name: The name of the step.
				164	"""
				165	self.EndStep()
				166	self._previous_step = (name, time.time())
				167	print 'Running step: ', name
				168
				169	def EndStep(self):
				170	"""Records successful completion of the current step.
				171
				172	This is optional if the step is immediately followed by another BeginStep.
				173	"""
				174	if self._previous_step[0]:
				175	elapsed = time.time() - self._previous_step[1]
				176	print 'Step %s took %f seconds' % (self._previous_step[0], elapsed)
				177	self.timings.append((self._previous_step[0], elapsed))
				178
				179	self._previous_step = ('', 0.0)
				180
				181	def FailStep(self, message=None):
				182	"""Marks that a particular step has failed.
				183
				184	On buildbot, this will mark the current step as failed on the waterfall.
				185	Otherwise we will just print an optional failure message.
				186
				187	Args:
				188	message: An optional explanation as to why the step failed.
				189	"""
				190	print 'STEP FAILED!!'
				191	if message:
				192	print message
				193	self._error_recorded = True
				194	self.EndStep()
				195
				196	def ErrorRecorded(self):
				197	"""True if FailStep has been called."""
				198	return self._error_recorded
				199
				200	def RunCommand(self, cmd, cwd=constants.DIR_SOURCE_ROOT, raise_on_error=True,
				201	stdout=None):
				202	"""Execute a shell command.
				203
				204	Args:
				205	cmd: A list of command strings.
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	206	cwd: Directory in which the command should be executed, defaults to build
				207	root of script's location if not specified.
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	208	raise_on_error: If true will raise a CommandError if the call doesn't
				209	succeed and mark the step as failed.
				210	stdout: A file to redirect stdout for the command to.
				211
				212	Returns:
				213	The process's return code.
				214
				215	Raises:
				216	CommandError: An error executing the specified command.
				217	"""
				218	print 'Executing %s in %s' % (' '.join(cmd), cwd)
				219	process = subprocess.Popen(cmd, stdout=stdout, cwd=cwd, env=os.environ)
				220	process.wait()
				221	if raise_on_error and process.returncode != 0:
				222	self.FailStep()
				223	raise CommandError('Exception executing command %s' % ' '.join(cmd))
				224	return process.returncode
				225
				226
				227	class ClankCompiler(object):
				228	"""Handles compilation of clank."""
				229
				230	def __init__(self, out_dir, step_recorder, arch, jobs, max_load, use_goma,
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	231	goma_dir, system_health_profiling):
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	232	self._out_dir = out_dir
				233	self._step_recorder = step_recorder
Benoit Lize	a87e5bc	2017-11-07 15:12:57	[diff] [blame]	234	self._arch = arch
				235	self._jobs = jobs
				236	self._max_load = max_load
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	237	self._use_goma = use_goma
Benoit Lize	a87e5bc	2017-11-07 15:12:57	[diff] [blame]	238	self._goma_dir = goma_dir
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	239	self._system_health_profiling = system_health_profiling
				240
				241	self.obj_dir = os.path.join(self._out_dir, 'Release', 'obj')
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	242	self.lib_chrome_so = os.path.join(
Matthew Cary	8b141623	2018-08-10 19:12:22	[diff] [blame^]	243	self._out_dir, 'Release', 'lib.unstripped', 'libchrome.so')
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	244	self.chrome_apk = os.path.join(
				245	self._out_dir, 'Release', 'apks', 'Chrome.apk')
				246
				247	def Build(self, instrumented, target):
				248	"""Builds the provided ninja target with or without order_profiling on.
				249
				250	Args:
Benoit Lize	a87e5bc	2017-11-07 15:12:57	[diff] [blame]	251	instrumented: (bool) Whether we want to build an instrumented binary.
				252	target: (str) The name of the ninja target to build.
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	253	"""
				254	self._step_recorder.BeginStep('Compile %s' % target)
				255
				256	# Set the "Release Official" flavor, the parts affecting performance.
				257	args = [
				258	'is_chrome_branded=true',
				259	'is_debug=false',
				260	'is_official_build=true',
				261	# We have to build with no symbols if profiling and minimal symbols
				262	# otherwise for libchrome.so to fit under the 4 GB limit.
				263	# crbug.com/574476
				264	'symbol_level=' + ('0' if instrumented else '1'),
				265	'target_cpu="' + self._arch + '"',
				266	'target_os="android"',
				267	'use_goma=' + str(self._use_goma).lower(),
				268	'use_order_profiling=' + str(instrumented).lower(),
				269	]
				270	if self._goma_dir:
				271	args += ['goma_dir="%s"' % self._goma_dir]
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	272	if self._system_health_profiling:
				273	args += ['devtools_instrumentation_dumping = ' +
				274	str(instrumented).lower()]
Benoit Lize	a87e5bc	2017-11-07 15:12:57	[diff] [blame]	275
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	276	self._step_recorder.RunCommand(
				277	['gn', 'gen', os.path.join(self._out_dir, 'Release'),
				278	'--args=' + ' '.join(args)])
				279
				280	self._step_recorder.RunCommand(
				281	['ninja', '-C', os.path.join(self._out_dir, 'Release'),
				282	'-j' + str(self._jobs), '-l' + str(self._max_load), target])
				283
				284	def CompileChromeApk(self, instrumented, force_relink=False):
				285	"""Builds a Chrome.apk either with or without order_profiling on.
				286
				287	Args:
Benoit Lize	a87e5bc	2017-11-07 15:12:57	[diff] [blame]	288	instrumented: (bool) Whether to build an instrumented apk.
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	289	force_relink: Whether libchromeview.so should be re-created.
				290	"""
				291	if force_relink:
				292	self._step_recorder.RunCommand(['rm', '-rf', self.lib_chrome_so])
				293	self.Build(instrumented, 'chrome_apk')
				294
				295	def CompileLibchrome(self, instrumented, force_relink=False):
				296	"""Builds a libchrome.so either with or without order_profiling on.
				297
				298	Args:
Benoit Lize	a87e5bc	2017-11-07 15:12:57	[diff] [blame]	299	instrumented: (bool) Whether to build an instrumented apk.
				300	force_relink: (bool) Whether libchrome.so should be re-created.
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	301	"""
				302	if force_relink:
				303	self._step_recorder.RunCommand(['rm', '-rf', self.lib_chrome_so])
				304	self.Build(instrumented, 'libchrome')
				305
				306
				307	class OrderfileUpdater(object):
				308	"""Handles uploading and committing a new orderfile in the repository.
				309
				310	Only used for testing or on a bot.
				311	"""
				312
				313	_CLOUD_STORAGE_BUCKET_FOR_DEBUG = None
				314	_CLOUD_STORAGE_BUCKET = None
				315	_UPLOAD_TO_CLOUD_COMMAND = 'upload_to_google_storage.py'
				316
				317	def __init__(self, repository_root, step_recorder, branch, netrc):
				318	"""Constructor.
				319
				320	Args:
				321	repository_root: (str) Root of the target repository.
				322	step_recorder: (StepRecorder) Step recorder, for logging.
				323	branch: (str) Branch to commit to.
				324	netrc: (str) Path to the .netrc file to use.
				325	"""
				326	self._repository_root = repository_root
				327	self._step_recorder = step_recorder
				328	self._branch = branch
				329	self._netrc = netrc
				330
				331	def CommitFileHashes(self, unpatched_orderfile_filename, orderfile_filename):
				332	"""Commits unpatched and patched orderfiles hashes, if provided.
				333
				334	Files must have been successfilly uploaded to cloud storage first.
				335
				336	Args:
				337	unpatched_orderfile_filename: (str or None) Unpatched orderfile path.
				338	orderfile_filename: (str or None) Orderfile path.
				339
				340	Raises:
				341	NotImplementedError when the commit logic hasn't been overriden.
				342	"""
				343	files_to_commit = []
				344	commit_message_lines = ['Update Orderfile.']
				345	for filename in [unpatched_orderfile_filename, orderfile_filename]:
				346	if not filename:
				347	continue
				348	(relative_path, sha1) = self._GetHashFilePathAndContents(filename)
				349	commit_message_lines.append('Profile: %s: %s' % (
				350	os.path.basename(relative_path), sha1))
				351	files_to_commit.append(relative_path)
				352	if files_to_commit:
				353	self._CommitFiles(files_to_commit, commit_message_lines)
				354
				355	def UploadToCloudStorage(self, filename, use_debug_location):
				356	"""Uploads a file to cloud storage.
				357
				358	Args:
				359	filename: (str) File to upload.
				360	use_debug_location: (bool) Whether to use the debug location.
				361	"""
				362	bucket = (self._CLOUD_STORAGE_BUCKET_FOR_DEBUG if use_debug_location
				363	else self._CLOUD_STORAGE_BUCKET)
				364	extension = _GetFileExtension(filename)
				365	cmd = [self._UPLOAD_TO_CLOUD_COMMAND, '--bucket', bucket]
				366	if extension:
				367	cmd.extend(['-z', extension])
				368	cmd.append(filename)
				369	self._step_recorder.RunCommand(cmd)
				370	print 'Download: https://sandbox.google.com/storage/%s/%s' % (
				371	bucket, _GenerateHash(filename))
				372
				373	def _GetHashFilePathAndContents(self, filename):
				374	"""Gets the name and content of the hash file created from uploading the
				375	given file.
				376
				377	Args:
				378	filename: (str) The file that was uploaded to cloud storage.
				379
				380	Returns:
				381	A tuple of the hash file name, relative to the reository root, and the
				382	content, which should be the sha1 hash of the file
				383	('base_file.sha1', hash)
				384	"""
				385	abs_hash_filename = filename + '.sha1'
				386	rel_hash_filename = os.path.relpath(
				387	abs_hash_filename, self._repository_root)
				388	with open(abs_hash_filename, 'r') as f:
				389	return (rel_hash_filename, f.read())
				390
				391	def _CommitFiles(self, files_to_commit, commit_message_lines):
				392	"""Commits a list of files, with a given message."""
				393	raise NotImplementedError
				394
				395
				396	class OrderfileGenerator(object):
				397	"""A utility for generating a new orderfile for Clank.
				398
				399	Builds an instrumented binary, profiles a run of the application, and
				400	generates an updated orderfile.
				401	"""
				402	_CLANK_REPO = os.path.join(constants.DIR_SOURCE_ROOT, 'clank')
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	403	_CHECK_ORDERFILE_SCRIPT = os.path.join(
				404	constants.DIR_SOURCE_ROOT, 'tools', 'cygprofile', 'check_orderfile.py')
				405	_BUILD_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(
				406	constants.GetOutDirectory()))) # Normally /path/to/src
				407
				408	_UNPATCHED_ORDERFILE_FILENAME = os.path.join(
				409	_CLANK_REPO, 'orderfiles', 'unpatched_orderfile.%s')
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	410
				411	_PATH_TO_ORDERFILE = os.path.join(_CLANK_REPO, 'orderfiles',
				412	'orderfile.%s.out')
				413
				414	# Previous orderfile_generator debug files would be overwritten.
				415	_DIRECTORY_FOR_DEBUG_FILES = '/tmp/orderfile_generator_debug_files'
				416
				417	def _GetPathToOrderfile(self):
				418	"""Gets the path to the architecture-specific orderfile."""
				419	return self._PATH_TO_ORDERFILE % self._options.arch
				420
				421	def _GetUnpatchedOrderfileFilename(self):
				422	"""Gets the path to the architecture-specific unpatched orderfile."""
				423	return self._UNPATCHED_ORDERFILE_FILENAME % self._options.arch
				424
				425	def __init__(self, options, orderfile_updater_class):
				426	self._options = options
				427
				428	self._instrumented_out_dir = os.path.join(
				429	self._BUILD_ROOT, self._options.arch + '_instrumented_out')
				430	self._uninstrumented_out_dir = os.path.join(
				431	self._BUILD_ROOT, self._options.arch + '_uninstrumented_out')
				432
				433	if options.profile:
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	434	output_directory = os.path.join(self._instrumented_out_dir, 'Release')
Matthew Cary	b8daed94	2018-06-11 10:58:08	[diff] [blame]	435	host_profile_dir = os.path.join(output_directory, 'profile_data')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	436	urls = [profile_android_startup.AndroidProfileTool.TEST_URL]
				437	use_wpr = True
				438	simulate_user = False
Benoit L	9646681	2018-03-06 15:58:37	[diff] [blame]	439	urls = options.urls
				440	use_wpr = not options.no_wpr
				441	simulate_user = options.simulate_user
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	442	self._profiler = profile_android_startup.AndroidProfileTool(
Matthew Cary	453ff145	2018-07-18 12:19:35	[diff] [blame]	443	output_directory, host_profile_dir, use_wpr, urls, simulate_user,
				444	device=options.device)
Matthew Cary	69e9e42	2018-08-10 18:30:06	[diff] [blame]	445	if options.pregenerated_profiles:
				446	self._profiler.SetPregeneratedProfiles(
				447	glob.glob(options.pregenerated_profiles))
				448	else:
				449	assert not options.pregenerated_profiles, (
				450	'--pregenerated-profiles cannot be used with --skip-profile')
				451	assert not options.profile_save_dir, (
				452	'--profile-save-dir cannot be used with --skip-profile')
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	453
				454	self._output_data = {}
				455	self._step_recorder = StepRecorder(options.buildbot)
				456	self._compiler = None
				457	assert issubclass(orderfile_updater_class, OrderfileUpdater)
				458	self._orderfile_updater = orderfile_updater_class(self._CLANK_REPO,
				459	self._step_recorder,
				460	options.branch,
				461	options.netrc)
				462	assert os.path.isdir(constants.DIR_SOURCE_ROOT), 'No src directory found'
Benoit Lize	fefbb27c	2018-01-17 13:54:18	[diff] [blame]	463	symbol_extractor.SetArchitecture(options.arch)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	464
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	465	@staticmethod
				466	def _RemoveBlanks(src_file, dest_file):
				467	"""A utility to remove blank lines from a file.
				468
				469	Args:
				470	src_file: The name of the file to remove the blanks from.
				471	dest_file: The name of the file to write the output without blanks.
				472	"""
				473	assert src_file != dest_file, 'Source and destination need to be distinct'
				474
				475	try:
				476	src = open(src_file, 'r')
				477	dest = open(dest_file, 'w')
				478	for line in src:
				479	if line and not line.isspace():
				480	dest.write(line)
				481	finally:
				482	src.close()
				483	dest.close()
				484
				485	def _GenerateAndProcessProfile(self):
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	486	"""Invokes a script to merge the per-thread traces into one file.
				487
				488	The produced list of offsets is saved in
				489	self._GetUnpatchedOrderfileFilename().
				490	"""
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	491	self._step_recorder.BeginStep('Generate Profile Data')
				492	files = []
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	493	logging.getLogger().setLevel(logging.DEBUG)
				494	if self._options.system_health_orderfile:
				495	files = self._profiler.CollectSystemHealthProfile(
				496	self._compiler.chrome_apk)
Matthew Cary	69e9e42	2018-08-10 18:30:06	[diff] [blame]	497	self._MaybeSaveProfile(files)
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	498	try:
				499	self._ProcessPhasedOrderfile(files)
				500	except Exception:
				501	for f in files:
				502	self._SaveForDebugging(f)
Matthew Cary	8b141623	2018-08-10 19:12:22	[diff] [blame^]	503	self._SaveForDebugging(self._compiler.lib_chrome_so)
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	504	raise
				505	finally:
				506	self._profiler.Cleanup()
				507	else:
				508	self._CollectLegacyProfile()
				509	logging.getLogger().setLevel(logging.INFO)
				510
				511	def _ProcessPhasedOrderfile(self, files):
				512	"""Process the phased orderfiles produced by system health benchmarks.
				513
				514	The offsets will be placed in _GetUnpatchedOrderfileFilename().
				515
				516	Args:
				517	file: Profile files pulled locally.
				518	"""
				519	self._step_recorder.BeginStep('Process Phased Orderfile')
				520	profiles = process_profiles.ProfileManager(files)
				521	processor = process_profiles.SymbolOffsetProcessor(
				522	self._compiler.lib_chrome_so)
				523	phaser = phased_orderfile.PhasedAnalyzer(profiles, processor)
				524	if self._options.offsets_for_memory:
				525	profile_offsets = phaser.GetOffsetsForMemoryFootprint()
				526	else:
				527	profile_offsets = phaser.GetOffsetsForStartup()
				528	self._output_data['orderfile_size'] = {
				529	'startup_kib': processor.OffsetsPrimarySize(
				530	profile_offsets.startup) / 1024,
				531	'common_kib': processor.OffsetsPrimarySize(
				532	profile_offsets.common) / 1024,
				533	'interaction_kib': processor.OffsetsPrimarySize(
				534	profile_offsets.interaction) / 1024}
				535
				536	offsets_list = (profile_offsets.startup +
				537	profile_offsets.common +
				538	profile_offsets.interaction)
Matthew Cary	8b141623	2018-08-10 19:12:22	[diff] [blame^]	539	ordered_symbols = processor.GetOrderedSymbols(offsets_list)
				540	if not ordered_symbols:
				541	raise Exception('Failed to get ordered symbols')
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	542	with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile:
Matthew Cary	8b141623	2018-08-10 19:12:22	[diff] [blame^]	543	orderfile.write('\n'.join(ordered_symbols))
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	544
				545	def _CollectLegacyProfile(self):
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	546	try:
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	547	files = self._profiler.CollectProfile(
				548	self._compiler.chrome_apk,
				549	constants.PACKAGE_INFO['chrome'])
Matthew Cary	69e9e42	2018-08-10 18:30:06	[diff] [blame]	550	self._MaybeSaveProfile(files)
Matthew Cary	b8daed94	2018-06-11 10:58:08	[diff] [blame]	551	self._step_recorder.BeginStep('Process profile')
Benoit L	9646681	2018-03-06 15:58:37	[diff] [blame]	552	assert os.path.exists(self._compiler.lib_chrome_so)
				553	offsets = process_profiles.GetReachedOffsetsFromDumpFiles(
				554	files, self._compiler.lib_chrome_so)
				555	if not offsets:
				556	raise Exception('No profiler offsets found in {}'.format(
				557	'\n'.join(files)))
Matthew Cary	8b141623	2018-08-10 19:12:22	[diff] [blame^]	558	processor = process_profiles.SymbolOffsetProcessor(
				559	self._compiler.lib_chrome_so)
				560	ordered_symbols = processor.GetOrderedSymbols(offsets)
				561	if not ordered_symbols:
				562	raise Exception('No symbol names from offsets found in {}'.format(
				563	'\n'.join(files)))
				564	with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile:
				565	orderfile.write('\n'.join(ordered_symbols))
Matthew Cary	0f1f681a	2018-01-22 10:40:51	[diff] [blame]	566	except Exception:
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	567	for f in files:
				568	self._SaveForDebugging(f)
				569	raise
				570	finally:
				571	self._profiler.Cleanup()
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	572
Matthew Cary	69e9e42	2018-08-10 18:30:06	[diff] [blame]	573	def _MaybeSaveProfile(self, files):
				574	if self._options.profile_save_dir:
				575	logging.info('Saving profiles to %s', self._options.profile_save_dir)
				576	for f in files:
				577	shutil.copy(f, self._options.profile_save_dir)
				578	logging.info('Saved profile %s', f)
				579
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	580	def _PatchOrderfile(self):
				581	"""Patches the orderfile using clean version of libchrome.so."""
				582	self._step_recorder.BeginStep('Patch Orderfile')
Benoit Lize	fefbb27c	2018-01-17 13:54:18	[diff] [blame]	583	patch_orderfile.GeneratePatchedOrderfile(
				584	self._GetUnpatchedOrderfileFilename(), self._compiler.lib_chrome_so,
				585	self._GetPathToOrderfile())
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	586
				587	def _VerifySymbolOrder(self):
				588	self._step_recorder.BeginStep('Verify Symbol Order')
				589	return_code = self._step_recorder.RunCommand(
				590	[self._CHECK_ORDERFILE_SCRIPT, self._compiler.lib_chrome_so,
				591	self._GetPathToOrderfile(),
				592	'--target-arch=' + self._options.arch],
				593	constants.DIR_SOURCE_ROOT,
				594	raise_on_error=False)
				595	if return_code:
				596	self._step_recorder.FailStep('Orderfile check returned %d.' % return_code)
				597
				598	def _RecordHash(self, file_name):
				599	"""Records the hash of the file into the output_data dictionary."""
				600	self._output_data[os.path.basename(file_name) + '.sha1'] = _GenerateHash(
				601	file_name)
				602
				603	def _SaveFileLocally(self, file_name, file_sha1):
				604	"""Saves the file to a temporary location and prints the sha1sum."""
				605	if not os.path.exists(self._DIRECTORY_FOR_DEBUG_FILES):
				606	os.makedirs(self._DIRECTORY_FOR_DEBUG_FILES)
				607	shutil.copy(file_name, self._DIRECTORY_FOR_DEBUG_FILES)
				608	print 'File: %s, saved in: %s, sha1sum: %s' % (
				609	file_name, self._DIRECTORY_FOR_DEBUG_FILES, file_sha1)
				610
				611	def _SaveForDebugging(self, filename):
				612	"""Uploads the file to cloud storage or saves to a temporary location."""
				613	file_sha1 = _GenerateHash(filename)
				614	if not self._options.buildbot:
				615	self._SaveFileLocally(filename, file_sha1)
				616	else:
				617	print 'Uploading file for debugging: ' + filename
				618	self._orderfile_updater.UploadToCloudStorage(
				619	filename, use_debug_location=True)
				620
				621	def _SaveForDebuggingWithOverwrite(self, file_name):
				622	"""Uploads and overwrites the file in cloud storage or copies locally.
				623
				624	Should be used for large binaries like lib_chrome_so.
				625
				626	Args:
				627	file_name: (str) File to upload.
				628	"""
				629	file_sha1 = _GenerateHash(file_name)
				630	if not self._options.buildbot:
				631	self._SaveFileLocally(file_name, file_sha1)
				632	else:
				633	print 'Uploading file for debugging: %s, sha1sum: %s' % (
				634	file_name, file_sha1)
				635	upload_location = '%s/%s' % (
				636	self._CLOUD_STORAGE_BUCKET_FOR_DEBUG, os.path.basename(file_name))
				637	self._step_recorder.RunCommand([
				638	'gsutil.py', 'cp', file_name, 'gs://' + upload_location])
				639	print ('Uploaded to: https://sandbox.google.com/storage/' +
				640	upload_location)
				641
				642	def _MaybeArchiveOrderfile(self, filename):
				643	"""In buildbot configuration, uploads the generated orderfile to
				644	Google Cloud Storage.
				645
				646	Args:
				647	filename: (str) Orderfile to upload.
				648	"""
				649	# First compute hashes so that we can download them later if we need to
				650	self._step_recorder.BeginStep('Compute hash for ' + filename)
				651	self._RecordHash(filename)
				652	if self._options.buildbot:
				653	self._step_recorder.BeginStep('Archive ' + filename)
				654	self._orderfile_updater.UploadToCloudStorage(
				655	filename, use_debug_location=False)
				656
				657	def _GetHashFilePathAndContents(self, base_file):
				658	"""Gets the name and content of the hash file created from uploading the
				659	given file.
				660
				661	Args:
				662	base_file: The file that was uploaded to cloud storage.
				663
				664	Returns:
				665	A tuple of the hash file name, relative to the clank repo path, and the
				666	content, which should be the sha1 hash of the file
				667	('base_file.sha1', hash)
				668	"""
				669	abs_file_name = base_file + '.sha1'
				670	rel_file_name = os.path.relpath(abs_file_name, self._CLANK_REPO)
				671	with open(abs_file_name, 'r') as f:
				672	return (rel_file_name, f.read())
				673
				674	def Generate(self):
				675	"""Generates and maybe upload an order."""
				676	profile_uploaded = False
				677	orderfile_uploaded = False
				678
Matthew Cary	e840064	2018-06-14 15:43:02	[diff] [blame]	679	assert (bool(self._options.profile) ^
				680	bool(self._options.manual_symbol_offsets))
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	681	if self._options.system_health_orderfile and not self._options.profile:
				682	raise AssertionError('--system_health_orderfile must be not be used '
				683	'with --skip-profile')
				684	if (self._options.manual_symbol_offsets and
				685	not self._options.system_health_orderfile):
				686	raise AssertionError('--manual-symbol-offsets must be used with '
				687	'--system_health_orderfile.')
Matthew Cary	e840064	2018-06-14 15:43:02	[diff] [blame]	688
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	689	if self._options.profile:
				690	try:
				691	_UnstashOutputDirectory(self._instrumented_out_dir)
				692	self._compiler = ClankCompiler(
				693	self._instrumented_out_dir,
				694	self._step_recorder, self._options.arch, self._options.jobs,
				695	self._options.max_load, self._options.use_goma,
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	696	self._options.goma_dir, self._options.system_health_orderfile)
Benoit L	9646681	2018-03-06 15:58:37	[diff] [blame]	697	self._compiler.CompileChromeApk(True)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	698	self._GenerateAndProcessProfile()
				699	self._MaybeArchiveOrderfile(self._GetUnpatchedOrderfileFilename())
				700	profile_uploaded = True
				701	finally:
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	702	_StashOutputDirectory(self._instrumented_out_dir)
Matthew Cary	e840064	2018-06-14 15:43:02	[diff] [blame]	703	elif self._options.manual_symbol_offsets:
				704	assert self._options.manual_libname
				705	assert self._options.manual_objdir
				706	with file(self._options.manual_symbol_offsets) as f:
				707	symbol_offsets = [int(x) for x in f.xreadlines()]
				708	processor = process_profiles.SymbolOffsetProcessor(
				709	self._options.manual_libname)
				710	generator = cyglog_to_orderfile.OffsetOrderfileGenerator(
				711	processor, cyglog_to_orderfile.ObjectFileProcessor(
				712	self._options.manual_objdir))
				713	ordered_sections = generator.GetOrderedSections(symbol_offsets)
				714	if not ordered_sections: # Either None or empty is a problem.
				715	raise Exception('Failed to get ordered sections')
				716	with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile:
				717	orderfile.write('\n'.join(ordered_sections))
				718
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	719	if self._options.patch:
				720	if self._options.profile:
				721	self._RemoveBlanks(self._GetUnpatchedOrderfileFilename(),
				722	self._GetPathToOrderfile())
				723	try:
				724	_UnstashOutputDirectory(self._uninstrumented_out_dir)
				725	self._compiler = ClankCompiler(
				726	self._uninstrumented_out_dir, self._step_recorder,
				727	self._options.arch, self._options.jobs, self._options.max_load,
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	728	self._options.use_goma, self._options.goma_dir,
				729	self._options.system_health_orderfile)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	730	self._compiler.CompileLibchrome(False)
				731	self._PatchOrderfile()
				732	# Because identical code folding is a bit different with and without
				733	# the orderfile build, we need to re-patch the orderfile with code
				734	# folding as close to the final version as possible.
				735	self._compiler.CompileLibchrome(False, force_relink=True)
				736	self._PatchOrderfile()
				737	self._compiler.CompileLibchrome(False, force_relink=True)
				738	self._VerifySymbolOrder()
				739	self._MaybeArchiveOrderfile(self._GetPathToOrderfile())
				740	finally:
				741	_StashOutputDirectory(self._uninstrumented_out_dir)
				742	orderfile_uploaded = True
				743
				744	if (self._options.buildbot and self._options.netrc
				745	and not self._step_recorder.ErrorRecorded()):
				746	unpatched_orderfile_filename = (
				747	self._GetUnpatchedOrderfileFilename() if profile_uploaded else None)
				748	orderfile_filename = (
Benoit Lize	d913be8	2017-10-24 13:38:27	[diff] [blame]	749	self._GetPathToOrderfile() if orderfile_uploaded else None)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	750	self._orderfile_updater.CommitFileHashes(
				751	unpatched_orderfile_filename, orderfile_filename)
				752
				753	self._step_recorder.EndStep()
				754	return not self._step_recorder.ErrorRecorded()
				755
				756	def GetReportingData(self):
				757	"""Get a dictionary of reporting data (timings, output hashes)"""
				758	self._output_data['timings'] = self._step_recorder.timings
				759	return self._output_data
				760
				761
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	762	def CreateArgumentParser():
				763	"""Creates and returns the argument parser."""
				764	parser = argparse.ArgumentParser()
				765	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	766	'--buildbot', action='store_true',
				767	help='If true, the script expects to be run on a buildbot')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	768	parser.add_argument(
Matthew Cary	453ff145	2018-07-18 12:19:35	[diff] [blame]	769	'--device', default=None, type=str,
				770	help='Device serial number on which to run profiling.')
				771	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	772	'--verify', action='store_true',
				773	help='If true, the script only verifies the current orderfile')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	774	parser.add_argument('--target-arch', action='store', dest='arch',
				775	default=cygprofile_utils.DetectArchitecture(),
				776	choices=['arm', 'arm64', 'x86', 'x86_64', 'x64', 'mips'],
				777	help='The target architecture for which to build')
				778	parser.add_argument('--output-json', action='store', dest='json_file',
				779	help='Location to save stats in json format')
				780	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	781	'--skip-profile', action='store_false', dest='profile', default=True,
				782	help='Don\'t generate a profile on the device. Only patch from the '
				783	'existing profile.')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	784	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	785	'--skip-patch', action='store_false', dest='patch', default=True,
				786	help='Only generate the raw (unpatched) orderfile, don\'t patch it.')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	787	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	788	'--netrc', action='store',
				789	help='A custom .netrc file to use for git checkin. Only used on bots.')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	790	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	791	'--branch', action='store', default='master',
				792	help='When running on buildbot with a netrc, the branch orderfile '
				793	'hashes get checked into.')
				794	# Note: -j50 was causing issues on the bot.
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	795	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	796	'-j', '--jobs', action='store', default=20,
				797	help='Number of jobs to use for compilation.')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	798	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	799	'-l', '--max-load', action='store', default=4, help='Max cpu load.')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	800	parser.add_argument('--goma-dir', help='GOMA directory.')
				801	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	802	'--use-goma', action='store_true', help='Enable GOMA.', default=False)
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	803	parser.add_argument('--adb-path', help='Path to the adb binary.')
Matthew Cary	e840064	2018-06-14 15:43:02	[diff] [blame]	804
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	805	parser.add_argument('--system-health-orderfile', action='store_true',
				806	help=('Create an orderfile based on system health '
				807	'benchmarks.'),
				808	default=False)
				809	parser.add_argument('--offsets-for-memory', action='store_true',
				810	help=('Favor memory savings in the orderfile. Used '
				811	'with --system-health-orderfile.'),
				812	default=False)
				813
Matthew Cary	e840064	2018-06-14 15:43:02	[diff] [blame]	814	parser.add_argument('--manual-symbol-offsets', default=None, type=str,
				815	help=('File of list of ordered symbol offsets generated '
				816	'by manual profiling. Must set other --manual* '
				817	'flags if this is used, and must --skip-profile.'))
				818	parser.add_argument('--manual-libname', default=None, type=str,
				819	help=('Library filename corresponding to '
				820	'--manual-symbol-offsets.'))
				821	parser.add_argument('--manual-objdir', default=None, type=str,
				822	help=('Root of object file directory corresponding to '
				823	'--manual-symbol-offsets.'))
Matthew Cary	69e9e42	2018-08-10 18:30:06	[diff] [blame]	824	parser.add_argument('--pregenerated-profiles', default=None, type=str,
				825	help=('Pregenerated profiles to use instead of running '
				826	'profile step. Cannot be used with '
				827	'--skip-profiles.'))
				828	parser.add_argument('--profile-save-dir', default=None, type=str,
				829	help=('Directory to save any profiles created. These can '
				830	'be used with --pregenerated-profiles. Cannot be '
				831	'used with --skip-profiles.'))
Matthew Cary	e840064	2018-06-14 15:43:02	[diff] [blame]	832
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	833	profile_android_startup.AddProfileCollectionArguments(parser)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	834	return parser
				835
				836
				837	def CreateOrderfile(options, orderfile_updater_class):
				838	"""Creates an oderfile.
				839
				840	Args:
				841	options: As returned from optparse.OptionParser.parse_args()
				842	orderfile_updater_class: (OrderfileUpdater) subclass of OrderfileUpdater.
				843
				844	Returns:
				845	True iff success.
				846	"""
				847	logging.basicConfig(level=logging.INFO)
				848	devil_chromium.Initialize(adb_path=options.adb_path)
				849
Egor Pasko	93e514e	2017-10-31 13:32:36	[diff] [blame]	850	generator = OrderfileGenerator(options, orderfile_updater_class)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	851	try:
				852	if options.verify:
Egor Pasko	93e514e	2017-10-31 13:32:36	[diff] [blame]	853	generator._VerifySymbolOrder()
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	854	else:
Egor Pasko	93e514e	2017-10-31 13:32:36	[diff] [blame]	855	return generator.Generate()
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	856	finally:
Egor Pasko	93e514e	2017-10-31 13:32:36	[diff] [blame]	857	json_output = json.dumps(generator.GetReportingData(),
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	858	indent=2) + '\n'
				859	if options.json_file:
				860	with open(options.json_file, 'w') as f:
				861	f.write(json_output)
				862	print json_output
				863	return False
				864
				865
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	866	def main():
				867	parser = CreateArgumentParser()
				868	options = parser.parse_args()
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	869	return 0 if CreateOrderfile(options, OrderfileUpdater) else 1
				870
				871
				872	if __name__ == '__main__':
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	873	sys.exit(main())