Blame - tools/cygprofile/orderfile_generator_backend.py - chromium/src

blob: 412fd7e0a1083547996c23523e7770df4d64cddd [file] [log] [blame]

Egor Pasko	0462e852d	2018-03-29 15:52:09	[diff] [blame]	1	#!/usr/bin/env vpython
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	2	# Copyright (c) 2013 The Chromium Authors. All rights reserved.
				3	# Use of this source code is governed by a BSD-style license that can be
				4	# found in the LICENSE file.
				5
				6	""" A utility to generate an up-to-date orderfile.
				7
				8	The orderfile is used by the linker to order text sections such that the
				9	sections are placed consecutively in the order specified. This allows us
				10	to page in less code during start-up.
				11
				12	Example usage:
Egor Pasko	93e514e	2017-10-31 13:32:36	[diff] [blame]	13	tools/cygprofile/orderfile_generator_backend.py -l 20 -j 1000 --use-goma \
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	14	--target-arch=arm
				15	"""
				16
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	17	import argparse
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	18	import hashlib
				19	import json
Matthew Cary	69e9e42	2018-08-10 18:30:06	[diff] [blame]	20	import glob
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	21	import logging
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	22	import os
				23	import re
				24	import shutil
				25	import subprocess
				26	import sys
Benoit Lize	a87e5bc	2017-11-07 15:12:57	[diff] [blame]	27	import tempfile
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	28	import time
				29
Matthew Cary	91df979	2018-11-30 14:35:15	[diff] [blame]	30	import cluster
Matthew Cary	e840064	2018-06-14 15:43:02	[diff] [blame]	31	import cyglog_to_orderfile
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	32	import cygprofile_utils
Benoit Lize	fefbb27c	2018-01-17 13:54:18	[diff] [blame]	33	import patch_orderfile
Benoit Lize	a87e5bc	2017-11-07 15:12:57	[diff] [blame]	34	import process_profiles
Egor Pasko	3bc0b93	2018-04-03 10:08:27	[diff] [blame]	35	import profile_android_startup
Benoit Lize	fefbb27c	2018-01-17 13:54:18	[diff] [blame]	36	import symbol_extractor
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	37
Monica Salama	eea2d94	2019-03-11 12:36:18	[diff] [blame]	38	_SRC_PATH = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
				39	sys.path.append(os.path.join(_SRC_PATH, 'third_party', 'catapult', 'devil'))
				40	from devil.android import device_utils
				41	from devil.android.sdk import version_codes
				42
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	43
				44	_SRC_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)),
				45	os.pardir, os.pardir)
				46	sys.path.append(os.path.join(_SRC_PATH, 'build', 'android'))
				47	import devil_chromium
				48	from pylib import constants
				49
				50
				51	# Needs to happen early for GetBuildType()/GetOutDirectory() to work correctly
				52	constants.SetBuildType('Release')
				53
				54
Matthew Cary	53f74ba	2019-01-24 10:07:18	[diff] [blame]	55	# Architecture specific GN args. Trying to build an orderfile for an
				56	# architecture not listed here will eventually throw.
				57	_ARCH_GN_ARGS = {
				58	'arm': [ 'target_cpu = "arm"' ],
				59	'arm64': [ 'target_cpu = "arm64"',
				60	'android_64bit_browser = true'],
				61	}
				62
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	63	class CommandError(Exception):
				64	"""Indicates that a dispatched shell command exited with a non-zero status."""
				65
				66	def __init__(self, value):
				67	super(CommandError, self).__init__()
				68	self.value = value
				69
				70	def __str__(self):
				71	return repr(self.value)
				72
				73
				74	def _GenerateHash(file_path):
				75	"""Calculates and returns the hash of the file at file_path."""
				76	sha1 = hashlib.sha1()
				77	with open(file_path, 'rb') as f:
				78	while True:
				79	# Read in 1mb chunks, so it doesn't all have to be loaded into memory.
				80	chunk = f.read(1024 * 1024)
				81	if not chunk:
				82	break
				83	sha1.update(chunk)
				84	return sha1.hexdigest()
				85
				86
				87	def _GetFileExtension(file_name):
				88	"""Calculates the file extension from a file name.
				89
				90	Args:
				91	file_name: The source file name.
				92	Returns:
				93	The part of file_name after the dot (.) or None if the file has no
				94	extension.
				95	Examples: /home/user/foo.bar -> bar
				96	/home/user.name/foo -> None
				97	/home/user/.foo -> None
				98	/home/user/foo.bar.baz -> baz
				99	"""
				100	file_name_parts = os.path.basename(file_name).split('.')
				101	if len(file_name_parts) > 1:
				102	return file_name_parts[-1]
				103	else:
				104	return None
				105
				106
				107	def _StashOutputDirectory(buildpath):
				108	"""Takes the output directory and stashes it in the default output directory.
				109
				110	This allows it to be used for incremental builds next time (after unstashing)
				111	by keeping it in a place that isn't deleted normally, while also ensuring
				112	that it is properly clobbered when appropriate.
				113
				114	This is a dirty hack to deal with the needs of clobbering while also handling
				115	incremental builds and the hardcoded relative paths used in some of the
				116	project files.
				117
				118	Args:
				119	buildpath: The path where the building happens. If this corresponds to the
				120	default output directory, no action is taken.
				121	"""
				122	if os.path.abspath(buildpath) == os.path.abspath(os.path.dirname(
				123	constants.GetOutDirectory())):
				124	return
				125	name = os.path.basename(buildpath)
				126	stashpath = os.path.join(constants.GetOutDirectory(), name)
				127	if not os.path.exists(buildpath):
				128	return
				129	if os.path.exists(stashpath):
				130	shutil.rmtree(stashpath, ignore_errors=True)
				131	shutil.move(buildpath, stashpath)
				132
				133
				134	def _UnstashOutputDirectory(buildpath):
				135	"""Inverse of _StashOutputDirectory.
				136
				137	Moves the output directory stashed within the default output directory
				138	(out/Release) to the position where the builds can actually happen.
				139
				140	This is a dirty hack to deal with the needs of clobbering while also handling
				141	incremental builds and the hardcoded relative paths used in some of the
				142	project files.
				143
				144	Args:
				145	buildpath: The path where the building happens. If this corresponds to the
				146	default output directory, no action is taken.
				147	"""
				148	if os.path.abspath(buildpath) == os.path.abspath(os.path.dirname(
				149	constants.GetOutDirectory())):
				150	return
				151	name = os.path.basename(buildpath)
				152	stashpath = os.path.join(constants.GetOutDirectory(), name)
				153	if not os.path.exists(stashpath):
				154	return
				155	if os.path.exists(buildpath):
				156	shutil.rmtree(buildpath, ignore_errors=True)
				157	shutil.move(stashpath, buildpath)
				158
				159
				160	class StepRecorder(object):
				161	"""Records steps and timings."""
				162
				163	def __init__(self, buildbot):
				164	self.timings = []
				165	self._previous_step = ('', 0.0)
				166	self._buildbot = buildbot
				167	self._error_recorded = False
				168
				169	def BeginStep(self, name):
				170	"""Marks a beginning of the next step in the script.
				171
				172	On buildbot, this prints a specially formatted name that will show up
				173	in the waterfall. Otherwise, just prints the step name.
				174
				175	Args:
				176	name: The name of the step.
				177	"""
				178	self.EndStep()
				179	self._previous_step = (name, time.time())
				180	print 'Running step: ', name
				181
				182	def EndStep(self):
				183	"""Records successful completion of the current step.
				184
				185	This is optional if the step is immediately followed by another BeginStep.
				186	"""
				187	if self._previous_step[0]:
				188	elapsed = time.time() - self._previous_step[1]
				189	print 'Step %s took %f seconds' % (self._previous_step[0], elapsed)
				190	self.timings.append((self._previous_step[0], elapsed))
				191
				192	self._previous_step = ('', 0.0)
				193
				194	def FailStep(self, message=None):
				195	"""Marks that a particular step has failed.
				196
				197	On buildbot, this will mark the current step as failed on the waterfall.
				198	Otherwise we will just print an optional failure message.
				199
				200	Args:
				201	message: An optional explanation as to why the step failed.
				202	"""
				203	print 'STEP FAILED!!'
				204	if message:
				205	print message
				206	self._error_recorded = True
				207	self.EndStep()
				208
				209	def ErrorRecorded(self):
				210	"""True if FailStep has been called."""
				211	return self._error_recorded
				212
				213	def RunCommand(self, cmd, cwd=constants.DIR_SOURCE_ROOT, raise_on_error=True,
				214	stdout=None):
				215	"""Execute a shell command.
				216
				217	Args:
				218	cmd: A list of command strings.
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	219	cwd: Directory in which the command should be executed, defaults to build
				220	root of script's location if not specified.
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	221	raise_on_error: If true will raise a CommandError if the call doesn't
				222	succeed and mark the step as failed.
				223	stdout: A file to redirect stdout for the command to.
				224
				225	Returns:
				226	The process's return code.
				227
				228	Raises:
				229	CommandError: An error executing the specified command.
				230	"""
				231	print 'Executing %s in %s' % (' '.join(cmd), cwd)
				232	process = subprocess.Popen(cmd, stdout=stdout, cwd=cwd, env=os.environ)
				233	process.wait()
				234	if raise_on_error and process.returncode != 0:
				235	self.FailStep()
				236	raise CommandError('Exception executing command %s' % ' '.join(cmd))
				237	return process.returncode
				238
				239
				240	class ClankCompiler(object):
				241	"""Handles compilation of clank."""
				242
				243	def __init__(self, out_dir, step_recorder, arch, jobs, max_load, use_goma,
Matthew Cary	d6bfcb7	2018-09-14 11:27:46	[diff] [blame]	244	goma_dir, system_health_profiling, monochrome):
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	245	self._out_dir = out_dir
				246	self._step_recorder = step_recorder
Benoit Lize	a87e5bc	2017-11-07 15:12:57	[diff] [blame]	247	self._arch = arch
				248	self._jobs = jobs
				249	self._max_load = max_load
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	250	self._use_goma = use_goma
Benoit Lize	a87e5bc	2017-11-07 15:12:57	[diff] [blame]	251	self._goma_dir = goma_dir
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	252	self._system_health_profiling = system_health_profiling
Matthew Cary	d6bfcb7	2018-09-14 11:27:46	[diff] [blame]	253	if monochrome:
				254	self._apk = 'Monochrome.apk'
Matthew Cary	e1b0006	2018-09-19 14:27:44	[diff] [blame]	255	self._apk_target = 'monochrome_apk'
Matthew Cary	d6bfcb7	2018-09-14 11:27:46	[diff] [blame]	256	self._libname = 'libmonochrome'
				257	self._libchrome_target = 'monochrome'
				258	else:
				259	self._apk = 'Chrome.apk'
Matthew Cary	e1b0006	2018-09-19 14:27:44	[diff] [blame]	260	self._apk_target = 'chrome_apk'
Matthew Cary	d6bfcb7	2018-09-14 11:27:46	[diff] [blame]	261	self._libname = 'libchrome'
				262	self._libchrome_target = 'libchrome'
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	263
				264	self.obj_dir = os.path.join(self._out_dir, 'Release', 'obj')
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	265	self.lib_chrome_so = os.path.join(
Matthew Cary	d6bfcb7	2018-09-14 11:27:46	[diff] [blame]	266	self._out_dir, 'Release', 'lib.unstripped',
				267	'{}.so'.format(self._libname))
				268	self.chrome_apk = os.path.join(self._out_dir, 'Release', 'apks', self._apk)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	269
				270	def Build(self, instrumented, target):
				271	"""Builds the provided ninja target with or without order_profiling on.
				272
				273	Args:
Benoit Lize	a87e5bc	2017-11-07 15:12:57	[diff] [blame]	274	instrumented: (bool) Whether we want to build an instrumented binary.
				275	target: (str) The name of the ninja target to build.
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	276	"""
				277	self._step_recorder.BeginStep('Compile %s' % target)
				278
				279	# Set the "Release Official" flavor, the parts affecting performance.
				280	args = [
Peter Collingbourne	1c85d5d2	2018-08-22 18:06:35	[diff] [blame]	281	'enable_resource_whitelist_generation=false',
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	282	'is_chrome_branded=true',
				283	'is_debug=false',
				284	'is_official_build=true',
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	285	'target_os="android"',
				286	'use_goma=' + str(self._use_goma).lower(),
				287	'use_order_profiling=' + str(instrumented).lower(),
				288	]
Matthew Cary	53f74ba	2019-01-24 10:07:18	[diff] [blame]	289	args += _ARCH_GN_ARGS[self._arch]
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	290	if self._goma_dir:
				291	args += ['goma_dir="%s"' % self._goma_dir]
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	292	if self._system_health_profiling:
				293	args += ['devtools_instrumentation_dumping = ' +
				294	str(instrumented).lower()]
Benoit Lize	a87e5bc	2017-11-07 15:12:57	[diff] [blame]	295
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	296	self._step_recorder.RunCommand(
				297	['gn', 'gen', os.path.join(self._out_dir, 'Release'),
				298	'--args=' + ' '.join(args)])
				299
				300	self._step_recorder.RunCommand(
				301	['ninja', '-C', os.path.join(self._out_dir, 'Release'),
				302	'-j' + str(self._jobs), '-l' + str(self._max_load), target])
				303
				304	def CompileChromeApk(self, instrumented, force_relink=False):
				305	"""Builds a Chrome.apk either with or without order_profiling on.
				306
				307	Args:
Benoit Lize	a87e5bc	2017-11-07 15:12:57	[diff] [blame]	308	instrumented: (bool) Whether to build an instrumented apk.
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	309	force_relink: Whether libchromeview.so should be re-created.
				310	"""
				311	if force_relink:
				312	self._step_recorder.RunCommand(['rm', '-rf', self.lib_chrome_so])
Matthew Cary	e1b0006	2018-09-19 14:27:44	[diff] [blame]	313	self.Build(instrumented, self._apk_target)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	314
				315	def CompileLibchrome(self, instrumented, force_relink=False):
				316	"""Builds a libchrome.so either with or without order_profiling on.
				317
				318	Args:
Benoit Lize	a87e5bc	2017-11-07 15:12:57	[diff] [blame]	319	instrumented: (bool) Whether to build an instrumented apk.
				320	force_relink: (bool) Whether libchrome.so should be re-created.
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	321	"""
				322	if force_relink:
				323	self._step_recorder.RunCommand(['rm', '-rf', self.lib_chrome_so])
Matthew Cary	d6bfcb7	2018-09-14 11:27:46	[diff] [blame]	324	self.Build(instrumented, self._libchrome_target)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	325
				326
				327	class OrderfileUpdater(object):
				328	"""Handles uploading and committing a new orderfile in the repository.
				329
				330	Only used for testing or on a bot.
				331	"""
				332
				333	_CLOUD_STORAGE_BUCKET_FOR_DEBUG = None
				334	_CLOUD_STORAGE_BUCKET = None
				335	_UPLOAD_TO_CLOUD_COMMAND = 'upload_to_google_storage.py'
				336
				337	def __init__(self, repository_root, step_recorder, branch, netrc):
				338	"""Constructor.
				339
				340	Args:
				341	repository_root: (str) Root of the target repository.
				342	step_recorder: (StepRecorder) Step recorder, for logging.
				343	branch: (str) Branch to commit to.
				344	netrc: (str) Path to the .netrc file to use.
				345	"""
				346	self._repository_root = repository_root
				347	self._step_recorder = step_recorder
				348	self._branch = branch
				349	self._netrc = netrc
				350
Matthew Cary	86a226e	2019-03-19 12:17:44	[diff] [blame^]	351	def CommitStashedFileHashes(self, files):
				352	"""Commits unpatched and patched orderfiles hashes if changed.
				353
				354	The files are committed only if their associated sha1 hash files match, and
				355	are modified in git. In normal operations the hash files are changed only
				356	when a file is uploaded to cloud storage. If the hash file is not modified
				357	in git, the file is skipped.
				358
				359	Args:
				360	files: [str or None] specifies file paths. None items are ignored.
				361
				362	Raises:
				363	Exception if the hash file does not match the file.
				364	NotImplementedError when the commit logic hasn't been overridden.
				365	"""
				366	files_to_commit = list(filter(None, files))
				367	if files_to_commit:
				368	self._CommitStashedFiles(files_to_commit)
				369
				370	def LegacyCommitFileHashes(self,
				371	unpatched_orderfile_filename,
				372	orderfile_filename):
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	373	"""Commits unpatched and patched orderfiles hashes, if provided.
				374
Matthew Cary	86a226e	2019-03-19 12:17:44	[diff] [blame^]	375	DEPRECATED. Left in place during transition.
				376
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	377	Files must have been successfilly uploaded to cloud storage first.
				378
				379	Args:
				380	unpatched_orderfile_filename: (str or None) Unpatched orderfile path.
				381	orderfile_filename: (str or None) Orderfile path.
				382
				383	Raises:
Matthew Cary	86a226e	2019-03-19 12:17:44	[diff] [blame^]	384	NotImplementedError when the commit logic hasn't been overridden.
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	385	"""
				386	files_to_commit = []
				387	commit_message_lines = ['Update Orderfile.']
				388	for filename in [unpatched_orderfile_filename, orderfile_filename]:
				389	if not filename:
				390	continue
				391	(relative_path, sha1) = self._GetHashFilePathAndContents(filename)
				392	commit_message_lines.append('Profile: %s: %s' % (
				393	os.path.basename(relative_path), sha1))
				394	files_to_commit.append(relative_path)
				395	if files_to_commit:
				396	self._CommitFiles(files_to_commit, commit_message_lines)
				397
				398	def UploadToCloudStorage(self, filename, use_debug_location):
				399	"""Uploads a file to cloud storage.
				400
				401	Args:
				402	filename: (str) File to upload.
				403	use_debug_location: (bool) Whether to use the debug location.
				404	"""
				405	bucket = (self._CLOUD_STORAGE_BUCKET_FOR_DEBUG if use_debug_location
				406	else self._CLOUD_STORAGE_BUCKET)
				407	extension = _GetFileExtension(filename)
				408	cmd = [self._UPLOAD_TO_CLOUD_COMMAND, '--bucket', bucket]
				409	if extension:
				410	cmd.extend(['-z', extension])
				411	cmd.append(filename)
				412	self._step_recorder.RunCommand(cmd)
				413	print 'Download: https://sandbox.google.com/storage/%s/%s' % (
				414	bucket, _GenerateHash(filename))
				415
				416	def _GetHashFilePathAndContents(self, filename):
				417	"""Gets the name and content of the hash file created from uploading the
				418	given file.
				419
				420	Args:
				421	filename: (str) The file that was uploaded to cloud storage.
				422
				423	Returns:
				424	A tuple of the hash file name, relative to the reository root, and the
				425	content, which should be the sha1 hash of the file
				426	('base_file.sha1', hash)
				427	"""
				428	abs_hash_filename = filename + '.sha1'
				429	rel_hash_filename = os.path.relpath(
				430	abs_hash_filename, self._repository_root)
				431	with open(abs_hash_filename, 'r') as f:
				432	return (rel_hash_filename, f.read())
				433
				434	def _CommitFiles(self, files_to_commit, commit_message_lines):
				435	"""Commits a list of files, with a given message."""
				436	raise NotImplementedError
				437
Matthew Cary	86a226e	2019-03-19 12:17:44	[diff] [blame^]	438	def _GitStash(self):
				439	"""Git stash the current clank tree.
				440
				441	Raises:
				442	NotImplementedError when the stash logic hasn't been overridden.
				443	"""
				444	raise NotImplementedError
				445
				446	def _CommitStashedFiles(self, expected_files_in_stash):
				447	"""Commits stashed files.
				448
				449	The local repository is updated and then the files to commit are taken from
				450	modified files from the git stash. The modified files should be a subset of
				451	\|expected_files_in_stash\|. If there are unexpected modified files, this
				452	function may raise. This is meant to be paired with _GitStash().
				453
				454	Args:
				455	expected_files_in_stash: [str] paths to a possible superset of files
				456	expected to be stashed & committed.
				457
				458	Raises:
				459	NotImplementedError when the commit logic hasn't been overridden.
				460	"""
				461	raise NotImplementedError
				462
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	463
				464	class OrderfileGenerator(object):
				465	"""A utility for generating a new orderfile for Clank.
				466
				467	Builds an instrumented binary, profiles a run of the application, and
				468	generates an updated orderfile.
				469	"""
				470	_CLANK_REPO = os.path.join(constants.DIR_SOURCE_ROOT, 'clank')
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	471	_CHECK_ORDERFILE_SCRIPT = os.path.join(
				472	constants.DIR_SOURCE_ROOT, 'tools', 'cygprofile', 'check_orderfile.py')
				473	_BUILD_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(
				474	constants.GetOutDirectory()))) # Normally /path/to/src
				475
				476	_UNPATCHED_ORDERFILE_FILENAME = os.path.join(
				477	_CLANK_REPO, 'orderfiles', 'unpatched_orderfile.%s')
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	478
				479	_PATH_TO_ORDERFILE = os.path.join(_CLANK_REPO, 'orderfiles',
				480	'orderfile.%s.out')
				481
				482	# Previous orderfile_generator debug files would be overwritten.
				483	_DIRECTORY_FOR_DEBUG_FILES = '/tmp/orderfile_generator_debug_files'
				484
				485	def _GetPathToOrderfile(self):
				486	"""Gets the path to the architecture-specific orderfile."""
				487	return self._PATH_TO_ORDERFILE % self._options.arch
				488
				489	def _GetUnpatchedOrderfileFilename(self):
				490	"""Gets the path to the architecture-specific unpatched orderfile."""
				491	return self._UNPATCHED_ORDERFILE_FILENAME % self._options.arch
				492
Monica Salama	eea2d94	2019-03-11 12:36:18	[diff] [blame]	493	def _SetDevice(self):
				494	""" Selects the device to be used by the script.
				495
				496	Returns:
				497	(Device with given serial ID) : if the --device flag is set.
				498	(Device running Android[K,L]) : if --use-legacy-chrome-apk flag is set or
				499	no device running Android N+ was found.
				500	(Device running Android N+) : Otherwise.
				501
				502	Raises Error:
				503	If no device meeting the requirements has been found.
				504	"""
				505	devices = None
				506	if self._options.device:
				507	devices = [device_utils.DeviceUtils(self._options.device)]
				508	else:
				509	devices = device_utils.DeviceUtils.HealthyDevices()
				510
				511	assert devices, 'Expected at least one connected device'
				512
				513	if self._options.use_legacy_chrome_apk:
				514	self._monochrome = False
				515	for device in devices:
				516	device_version = device.build_version_sdk
				517	if (device_version >= version_codes.KITKAT
				518	and device_version <= version_codes.LOLLIPOP_MR1):
				519	return device
				520
				521	assert not self._options.use_legacy_chrome_apk, \
				522	'No device found running suitable android version for Chrome.apk.'
				523
				524	preferred_device = None
				525	for device in devices:
				526	if device.build_version_sdk >= version_codes.NOUGAT:
				527	preferred_device = device
				528	break
				529
				530	self._monochrome = preferred_device is not None
				531
				532	return preferred_device if preferred_device else devices[0]
				533
				534
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	535	def __init__(self, options, orderfile_updater_class):
				536	self._options = options
				537
				538	self._instrumented_out_dir = os.path.join(
				539	self._BUILD_ROOT, self._options.arch + '_instrumented_out')
				540	self._uninstrumented_out_dir = os.path.join(
				541	self._BUILD_ROOT, self._options.arch + '_uninstrumented_out')
				542
				543	if options.profile:
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	544	output_directory = os.path.join(self._instrumented_out_dir, 'Release')
Matthew Cary	b8daed94	2018-06-11 10:58:08	[diff] [blame]	545	host_profile_dir = os.path.join(output_directory, 'profile_data')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	546	urls = [profile_android_startup.AndroidProfileTool.TEST_URL]
				547	use_wpr = True
				548	simulate_user = False
Benoit L	9646681	2018-03-06 15:58:37	[diff] [blame]	549	urls = options.urls
				550	use_wpr = not options.no_wpr
				551	simulate_user = options.simulate_user
Monica Salama	eea2d94	2019-03-11 12:36:18	[diff] [blame]	552	device = self._SetDevice()
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	553	self._profiler = profile_android_startup.AndroidProfileTool(
Matthew Cary	453ff145	2018-07-18 12:19:35	[diff] [blame]	554	output_directory, host_profile_dir, use_wpr, urls, simulate_user,
Monica Salama	eea2d94	2019-03-11 12:36:18	[diff] [blame]	555	device=device)
Matthew Cary	69e9e42	2018-08-10 18:30:06	[diff] [blame]	556	if options.pregenerated_profiles:
				557	self._profiler.SetPregeneratedProfiles(
				558	glob.glob(options.pregenerated_profiles))
				559	else:
				560	assert not options.pregenerated_profiles, (
				561	'--pregenerated-profiles cannot be used with --skip-profile')
				562	assert not options.profile_save_dir, (
				563	'--profile-save-dir cannot be used with --skip-profile')
Monica Salama	eea2d94	2019-03-11 12:36:18	[diff] [blame]	564	self._monochrome = not self._options.use_legacy_chrome_apk
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	565
Matthew Cary	f949bba	2019-02-04 13:39:23	[diff] [blame]	566	# Outlined function handling enabled by default for all architectures.
				567	self._order_outlined_functions = not options.noorder_outlined_functions
				568
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	569	self._output_data = {}
				570	self._step_recorder = StepRecorder(options.buildbot)
				571	self._compiler = None
				572	assert issubclass(orderfile_updater_class, OrderfileUpdater)
				573	self._orderfile_updater = orderfile_updater_class(self._CLANK_REPO,
				574	self._step_recorder,
				575	options.branch,
				576	options.netrc)
				577	assert os.path.isdir(constants.DIR_SOURCE_ROOT), 'No src directory found'
Benoit Lize	fefbb27c	2018-01-17 13:54:18	[diff] [blame]	578	symbol_extractor.SetArchitecture(options.arch)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	579
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	580	@staticmethod
				581	def _RemoveBlanks(src_file, dest_file):
				582	"""A utility to remove blank lines from a file.
				583
				584	Args:
				585	src_file: The name of the file to remove the blanks from.
				586	dest_file: The name of the file to write the output without blanks.
				587	"""
				588	assert src_file != dest_file, 'Source and destination need to be distinct'
				589
				590	try:
				591	src = open(src_file, 'r')
				592	dest = open(dest_file, 'w')
				593	for line in src:
				594	if line and not line.isspace():
				595	dest.write(line)
				596	finally:
				597	src.close()
				598	dest.close()
				599
				600	def _GenerateAndProcessProfile(self):
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	601	"""Invokes a script to merge the per-thread traces into one file.
				602
				603	The produced list of offsets is saved in
				604	self._GetUnpatchedOrderfileFilename().
				605	"""
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	606	self._step_recorder.BeginStep('Generate Profile Data')
				607	files = []
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	608	logging.getLogger().setLevel(logging.DEBUG)
				609	if self._options.system_health_orderfile:
				610	files = self._profiler.CollectSystemHealthProfile(
				611	self._compiler.chrome_apk)
Matthew Cary	69e9e42	2018-08-10 18:30:06	[diff] [blame]	612	self._MaybeSaveProfile(files)
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	613	try:
				614	self._ProcessPhasedOrderfile(files)
				615	except Exception:
				616	for f in files:
				617	self._SaveForDebugging(f)
Matthew Cary	8b141623	2018-08-10 19:12:22	[diff] [blame]	618	self._SaveForDebugging(self._compiler.lib_chrome_so)
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	619	raise
				620	finally:
				621	self._profiler.Cleanup()
				622	else:
				623	self._CollectLegacyProfile()
				624	logging.getLogger().setLevel(logging.INFO)
				625
				626	def _ProcessPhasedOrderfile(self, files):
				627	"""Process the phased orderfiles produced by system health benchmarks.
				628
				629	The offsets will be placed in _GetUnpatchedOrderfileFilename().
				630
				631	Args:
				632	file: Profile files pulled locally.
				633	"""
				634	self._step_recorder.BeginStep('Process Phased Orderfile')
				635	profiles = process_profiles.ProfileManager(files)
				636	processor = process_profiles.SymbolOffsetProcessor(
				637	self._compiler.lib_chrome_so)
Matthew Cary	da9db93	2019-03-11 15:28:17	[diff] [blame]	638	ordered_symbols = cluster.ClusterOffsets(profiles, processor)
Matthew Cary	8b141623	2018-08-10 19:12:22	[diff] [blame]	639	if not ordered_symbols:
				640	raise Exception('Failed to get ordered symbols')
Matthew Cary	da9db93	2019-03-11 15:28:17	[diff] [blame]	641	for sym in ordered_symbols:
				642	assert not sym.startswith('OUTLINED_FUNCTION_'), (
				643	'Outlined function found in instrumented function, very likely '
				644	'something has gone very wrong!')
Matthew Cary	91df979	2018-11-30 14:35:15	[diff] [blame]	645	self._output_data['offsets_kib'] = processor.SymbolsSize(
				646	ordered_symbols) / 1024
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	647	with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile:
Matthew Cary	8b141623	2018-08-10 19:12:22	[diff] [blame]	648	orderfile.write('\n'.join(ordered_symbols))
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	649
				650	def _CollectLegacyProfile(self):
Matthew Cary	2df8d45	2018-09-19 07:50:20	[diff] [blame]	651	files = []
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	652	try:
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	653	files = self._profiler.CollectProfile(
				654	self._compiler.chrome_apk,
				655	constants.PACKAGE_INFO['chrome'])
Matthew Cary	69e9e42	2018-08-10 18:30:06	[diff] [blame]	656	self._MaybeSaveProfile(files)
Matthew Cary	b8daed94	2018-06-11 10:58:08	[diff] [blame]	657	self._step_recorder.BeginStep('Process profile')
Benoit L	9646681	2018-03-06 15:58:37	[diff] [blame]	658	assert os.path.exists(self._compiler.lib_chrome_so)
				659	offsets = process_profiles.GetReachedOffsetsFromDumpFiles(
				660	files, self._compiler.lib_chrome_so)
				661	if not offsets:
				662	raise Exception('No profiler offsets found in {}'.format(
				663	'\n'.join(files)))
Matthew Cary	8b141623	2018-08-10 19:12:22	[diff] [blame]	664	processor = process_profiles.SymbolOffsetProcessor(
				665	self._compiler.lib_chrome_so)
				666	ordered_symbols = processor.GetOrderedSymbols(offsets)
				667	if not ordered_symbols:
				668	raise Exception('No symbol names from offsets found in {}'.format(
				669	'\n'.join(files)))
				670	with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile:
				671	orderfile.write('\n'.join(ordered_symbols))
Matthew Cary	0f1f681a	2018-01-22 10:40:51	[diff] [blame]	672	except Exception:
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	673	for f in files:
				674	self._SaveForDebugging(f)
				675	raise
				676	finally:
				677	self._profiler.Cleanup()
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	678
Matthew Cary	69e9e42	2018-08-10 18:30:06	[diff] [blame]	679	def _MaybeSaveProfile(self, files):
				680	if self._options.profile_save_dir:
				681	logging.info('Saving profiles to %s', self._options.profile_save_dir)
				682	for f in files:
				683	shutil.copy(f, self._options.profile_save_dir)
				684	logging.info('Saved profile %s', f)
				685
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	686	def _PatchOrderfile(self):
				687	"""Patches the orderfile using clean version of libchrome.so."""
				688	self._step_recorder.BeginStep('Patch Orderfile')
Benoit Lize	fefbb27c	2018-01-17 13:54:18	[diff] [blame]	689	patch_orderfile.GeneratePatchedOrderfile(
				690	self._GetUnpatchedOrderfileFilename(), self._compiler.lib_chrome_so,
Matthew Cary	f949bba	2019-02-04 13:39:23	[diff] [blame]	691	self._GetPathToOrderfile(), self._order_outlined_functions)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	692
				693	def _VerifySymbolOrder(self):
				694	self._step_recorder.BeginStep('Verify Symbol Order')
				695	return_code = self._step_recorder.RunCommand(
				696	[self._CHECK_ORDERFILE_SCRIPT, self._compiler.lib_chrome_so,
				697	self._GetPathToOrderfile(),
				698	'--target-arch=' + self._options.arch],
				699	constants.DIR_SOURCE_ROOT,
				700	raise_on_error=False)
				701	if return_code:
				702	self._step_recorder.FailStep('Orderfile check returned %d.' % return_code)
				703
				704	def _RecordHash(self, file_name):
				705	"""Records the hash of the file into the output_data dictionary."""
				706	self._output_data[os.path.basename(file_name) + '.sha1'] = _GenerateHash(
				707	file_name)
				708
				709	def _SaveFileLocally(self, file_name, file_sha1):
				710	"""Saves the file to a temporary location and prints the sha1sum."""
				711	if not os.path.exists(self._DIRECTORY_FOR_DEBUG_FILES):
				712	os.makedirs(self._DIRECTORY_FOR_DEBUG_FILES)
				713	shutil.copy(file_name, self._DIRECTORY_FOR_DEBUG_FILES)
				714	print 'File: %s, saved in: %s, sha1sum: %s' % (
				715	file_name, self._DIRECTORY_FOR_DEBUG_FILES, file_sha1)
				716
				717	def _SaveForDebugging(self, filename):
				718	"""Uploads the file to cloud storage or saves to a temporary location."""
				719	file_sha1 = _GenerateHash(filename)
				720	if not self._options.buildbot:
				721	self._SaveFileLocally(filename, file_sha1)
				722	else:
				723	print 'Uploading file for debugging: ' + filename
				724	self._orderfile_updater.UploadToCloudStorage(
				725	filename, use_debug_location=True)
				726
				727	def _SaveForDebuggingWithOverwrite(self, file_name):
				728	"""Uploads and overwrites the file in cloud storage or copies locally.
				729
				730	Should be used for large binaries like lib_chrome_so.
				731
				732	Args:
				733	file_name: (str) File to upload.
				734	"""
				735	file_sha1 = _GenerateHash(file_name)
				736	if not self._options.buildbot:
				737	self._SaveFileLocally(file_name, file_sha1)
				738	else:
				739	print 'Uploading file for debugging: %s, sha1sum: %s' % (
				740	file_name, file_sha1)
				741	upload_location = '%s/%s' % (
				742	self._CLOUD_STORAGE_BUCKET_FOR_DEBUG, os.path.basename(file_name))
				743	self._step_recorder.RunCommand([
				744	'gsutil.py', 'cp', file_name, 'gs://' + upload_location])
				745	print ('Uploaded to: https://sandbox.google.com/storage/' +
				746	upload_location)
				747
				748	def _MaybeArchiveOrderfile(self, filename):
				749	"""In buildbot configuration, uploads the generated orderfile to
				750	Google Cloud Storage.
				751
				752	Args:
				753	filename: (str) Orderfile to upload.
				754	"""
Matthew Cary	91df979	2018-11-30 14:35:15	[diff] [blame]	755	# First compute hashes so that we can download them later if we need to.
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	756	self._step_recorder.BeginStep('Compute hash for ' + filename)
				757	self._RecordHash(filename)
				758	if self._options.buildbot:
				759	self._step_recorder.BeginStep('Archive ' + filename)
				760	self._orderfile_updater.UploadToCloudStorage(
				761	filename, use_debug_location=False)
				762
Egor Pasko	bce64d01	2018-11-20 12:02:37	[diff] [blame]	763	def UploadReadyOrderfiles(self):
				764	self._step_recorder.BeginStep('Upload Ready Orderfiles')
				765	for file_name in [self._GetUnpatchedOrderfileFilename(),
				766	self._GetPathToOrderfile()]:
				767	self._orderfile_updater.UploadToCloudStorage(
				768	file_name, use_debug_location=False)
				769
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	770	def _GetHashFilePathAndContents(self, base_file):
				771	"""Gets the name and content of the hash file created from uploading the
				772	given file.
				773
				774	Args:
				775	base_file: The file that was uploaded to cloud storage.
				776
				777	Returns:
				778	A tuple of the hash file name, relative to the clank repo path, and the
				779	content, which should be the sha1 hash of the file
				780	('base_file.sha1', hash)
				781	"""
				782	abs_file_name = base_file + '.sha1'
				783	rel_file_name = os.path.relpath(abs_file_name, self._CLANK_REPO)
				784	with open(abs_file_name, 'r') as f:
				785	return (rel_file_name, f.read())
				786
				787	def Generate(self):
				788	"""Generates and maybe upload an order."""
				789	profile_uploaded = False
				790	orderfile_uploaded = False
				791
Matthew Cary	e840064	2018-06-14 15:43:02	[diff] [blame]	792	assert (bool(self._options.profile) ^
				793	bool(self._options.manual_symbol_offsets))
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	794	if self._options.system_health_orderfile and not self._options.profile:
				795	raise AssertionError('--system_health_orderfile must be not be used '
				796	'with --skip-profile')
				797	if (self._options.manual_symbol_offsets and
				798	not self._options.system_health_orderfile):
				799	raise AssertionError('--manual-symbol-offsets must be used with '
				800	'--system_health_orderfile.')
Matthew Cary	e840064	2018-06-14 15:43:02	[diff] [blame]	801
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	802	if self._options.profile:
				803	try:
				804	_UnstashOutputDirectory(self._instrumented_out_dir)
				805	self._compiler = ClankCompiler(
				806	self._instrumented_out_dir,
				807	self._step_recorder, self._options.arch, self._options.jobs,
				808	self._options.max_load, self._options.use_goma,
Matthew Cary	d6bfcb7	2018-09-14 11:27:46	[diff] [blame]	809	self._options.goma_dir, self._options.system_health_orderfile,
Monica Salama	eea2d94	2019-03-11 12:36:18	[diff] [blame]	810	self._monochrome)
Matthew Cary	a2bea45	2018-11-13 10:21:07	[diff] [blame]	811	if not self._options.pregenerated_profiles:
				812	# If there are pregenerated profiles, the instrumented build should
				813	# not be changed to avoid invalidating the pregenerated profile
				814	# offsets.
				815	self._compiler.CompileChromeApk(True)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	816	self._GenerateAndProcessProfile()
				817	self._MaybeArchiveOrderfile(self._GetUnpatchedOrderfileFilename())
				818	profile_uploaded = True
				819	finally:
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	820	_StashOutputDirectory(self._instrumented_out_dir)
Matthew Cary	e840064	2018-06-14 15:43:02	[diff] [blame]	821	elif self._options.manual_symbol_offsets:
				822	assert self._options.manual_libname
				823	assert self._options.manual_objdir
				824	with file(self._options.manual_symbol_offsets) as f:
				825	symbol_offsets = [int(x) for x in f.xreadlines()]
				826	processor = process_profiles.SymbolOffsetProcessor(
Matthew Cary	b46ad28	2018-11-23 14:43:57	[diff] [blame]	827	self._compiler.manual_libname)
Matthew Cary	e840064	2018-06-14 15:43:02	[diff] [blame]	828	generator = cyglog_to_orderfile.OffsetOrderfileGenerator(
				829	processor, cyglog_to_orderfile.ObjectFileProcessor(
				830	self._options.manual_objdir))
				831	ordered_sections = generator.GetOrderedSections(symbol_offsets)
				832	if not ordered_sections: # Either None or empty is a problem.
				833	raise Exception('Failed to get ordered sections')
				834	with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile:
				835	orderfile.write('\n'.join(ordered_sections))
				836
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	837	if self._options.patch:
				838	if self._options.profile:
				839	self._RemoveBlanks(self._GetUnpatchedOrderfileFilename(),
				840	self._GetPathToOrderfile())
				841	try:
				842	_UnstashOutputDirectory(self._uninstrumented_out_dir)
				843	self._compiler = ClankCompiler(
				844	self._uninstrumented_out_dir, self._step_recorder,
				845	self._options.arch, self._options.jobs, self._options.max_load,
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	846	self._options.use_goma, self._options.goma_dir,
Monica Salama	eea2d94	2019-03-11 12:36:18	[diff] [blame]	847	self._options.system_health_orderfile, self._monochrome)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	848	self._compiler.CompileLibchrome(False)
				849	self._PatchOrderfile()
				850	# Because identical code folding is a bit different with and without
				851	# the orderfile build, we need to re-patch the orderfile with code
				852	# folding as close to the final version as possible.
				853	self._compiler.CompileLibchrome(False, force_relink=True)
				854	self._PatchOrderfile()
				855	self._compiler.CompileLibchrome(False, force_relink=True)
				856	self._VerifySymbolOrder()
				857	self._MaybeArchiveOrderfile(self._GetPathToOrderfile())
				858	finally:
				859	_StashOutputDirectory(self._uninstrumented_out_dir)
				860	orderfile_uploaded = True
				861
Matthew Cary	86a226e	2019-03-19 12:17:44	[diff] [blame^]	862	if self._options._new_commit_flow:
				863	self._orderfile_updater._GitStash()
				864	else:
				865	if (self._options.buildbot and self._options.netrc
				866	and not self._step_recorder.ErrorRecorded()):
				867	unpatched_orderfile_filename = (
				868	self._GetUnpatchedOrderfileFilename() if profile_uploaded else None)
				869	orderfile_filename = (
				870	self._GetPathToOrderfile() if orderfile_uploaded else None)
				871	self._orderfile_updater.LegacyCommitFileHashes(
				872	unpatched_orderfile_filename, orderfile_filename)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	873	self._step_recorder.EndStep()
				874	return not self._step_recorder.ErrorRecorded()
				875
				876	def GetReportingData(self):
				877	"""Get a dictionary of reporting data (timings, output hashes)"""
				878	self._output_data['timings'] = self._step_recorder.timings
				879	return self._output_data
				880
Matthew Cary	86a226e	2019-03-19 12:17:44	[diff] [blame^]	881	def CommitStashedOrderfileHashes(self):
				882	"""Commit any orderfile hash files in the current checkout.
				883
				884	Only possible if running on the buildbot.
				885
				886	Returns: true on success.
				887	"""
				888	if not (self._options.buildbot and self._options.netrc):
				889	logging.error('Trying to commit when not running on the buildbot')
				890	return False
				891	self._orderfile_updater._CommitStashedFiles([
				892	filename + '.sha1'
				893	for filename in (self._GetUnpatchedOrderfileFilename(),
				894	self._GetPathToOrderfile())])
				895	return True
				896
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	897
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	898	def CreateArgumentParser():
				899	"""Creates and returns the argument parser."""
				900	parser = argparse.ArgumentParser()
				901	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	902	'--buildbot', action='store_true',
				903	help='If true, the script expects to be run on a buildbot')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	904	parser.add_argument(
Matthew Cary	453ff145	2018-07-18 12:19:35	[diff] [blame]	905	'--device', default=None, type=str,
				906	help='Device serial number on which to run profiling.')
				907	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	908	'--verify', action='store_true',
				909	help='If true, the script only verifies the current orderfile')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	910	parser.add_argument('--target-arch', action='store', dest='arch',
Stephen Martinis	71b391b5	2018-12-04 15:08:04	[diff] [blame]	911	default='arm',
Matthew Cary	53f74ba	2019-01-24 10:07:18	[diff] [blame]	912	choices=['arm', 'arm64'],
				913	help='The target architecture for which to build.')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	914	parser.add_argument('--output-json', action='store', dest='json_file',
				915	help='Location to save stats in json format')
				916	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	917	'--skip-profile', action='store_false', dest='profile', default=True,
				918	help='Don\'t generate a profile on the device. Only patch from the '
				919	'existing profile.')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	920	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	921	'--skip-patch', action='store_false', dest='patch', default=True,
				922	help='Only generate the raw (unpatched) orderfile, don\'t patch it.')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	923	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	924	'--netrc', action='store',
				925	help='A custom .netrc file to use for git checkin. Only used on bots.')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	926	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	927	'--branch', action='store', default='master',
				928	help='When running on buildbot with a netrc, the branch orderfile '
				929	'hashes get checked into.')
				930	# Note: -j50 was causing issues on the bot.
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	931	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	932	'-j', '--jobs', action='store', default=20,
				933	help='Number of jobs to use for compilation.')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	934	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	935	'-l', '--max-load', action='store', default=4, help='Max cpu load.')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	936	parser.add_argument('--goma-dir', help='GOMA directory.')
				937	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	938	'--use-goma', action='store_true', help='Enable GOMA.', default=False)
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	939	parser.add_argument('--adb-path', help='Path to the adb binary.')
Matthew Cary	e840064	2018-06-14 15:43:02	[diff] [blame]	940
Matthew Cary	04f4103	2018-12-10 15:55:27	[diff] [blame]	941	parser.add_argument('--nosystem-health-orderfile', action='store_false',
				942	dest='system_health_orderfile', default=True,
Matthew Cary	c262f5d	2018-09-19 14:36:28	[diff] [blame]	943	help=('Create an orderfile based on an about:blank '
				944	'startup benchmark instead of system health '
				945	'benchmarks.'))
Monica Salama	eea2d94	2019-03-11 12:36:18	[diff] [blame]	946	parser.add_argument(
				947	'--use-legacy-chrome-apk', action='store_true', default=False,
				948	help=('Compile and instrument chrome for [L, K] devices.'))
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	949
Matthew Cary	e840064	2018-06-14 15:43:02	[diff] [blame]	950	parser.add_argument('--manual-symbol-offsets', default=None, type=str,
				951	help=('File of list of ordered symbol offsets generated '
				952	'by manual profiling. Must set other --manual* '
				953	'flags if this is used, and must --skip-profile.'))
				954	parser.add_argument('--manual-libname', default=None, type=str,
				955	help=('Library filename corresponding to '
				956	'--manual-symbol-offsets.'))
				957	parser.add_argument('--manual-objdir', default=None, type=str,
				958	help=('Root of object file directory corresponding to '
				959	'--manual-symbol-offsets.'))
Matthew Cary	f949bba	2019-02-04 13:39:23	[diff] [blame]	960	parser.add_argument('--noorder-outlined-functions', action='store_true',
				961	help='Disable outlined functions in the orderfile.')
Matthew Cary	69e9e42	2018-08-10 18:30:06	[diff] [blame]	962	parser.add_argument('--pregenerated-profiles', default=None, type=str,
				963	help=('Pregenerated profiles to use instead of running '
				964	'profile step. Cannot be used with '
				965	'--skip-profiles.'))
				966	parser.add_argument('--profile-save-dir', default=None, type=str,
				967	help=('Directory to save any profiles created. These can '
				968	'be used with --pregenerated-profiles. Cannot be '
				969	'used with --skip-profiles.'))
Egor Pasko	bce64d01	2018-11-20 12:02:37	[diff] [blame]	970	parser.add_argument('--upload-ready-orderfiles', action='store_true',
				971	help=('Skip orderfile generation and manually upload '
				972	'orderfiles (both patched and unpatched) from '
				973	'their normal location in the tree to the cloud '
				974	'storage. DANGEROUS! USE WITH CARE!'))
Matthew Cary	86a226e	2019-03-19 12:17:44	[diff] [blame^]	975	parser.add_argument('--commit-hashes', action='store_true',
				976	help=('Commit any orderfile hash files in the current '
				977	'checkout; performs no other action'))
				978	parser.add_argument('--new-commit-flow', action='store_true',
				979	help='Use the new two-step commit flow.')
Matthew Cary	e840064	2018-06-14 15:43:02	[diff] [blame]	980
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	981	profile_android_startup.AddProfileCollectionArguments(parser)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	982	return parser
				983
				984
				985	def CreateOrderfile(options, orderfile_updater_class):
				986	"""Creates an oderfile.
				987
				988	Args:
				989	options: As returned from optparse.OptionParser.parse_args()
				990	orderfile_updater_class: (OrderfileUpdater) subclass of OrderfileUpdater.
				991
				992	Returns:
				993	True iff success.
				994	"""
				995	logging.basicConfig(level=logging.INFO)
				996	devil_chromium.Initialize(adb_path=options.adb_path)
				997
Egor Pasko	93e514e	2017-10-31 13:32:36	[diff] [blame]	998	generator = OrderfileGenerator(options, orderfile_updater_class)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	999	try:
				1000	if options.verify:
Egor Pasko	93e514e	2017-10-31 13:32:36	[diff] [blame]	1001	generator._VerifySymbolOrder()
Matthew Cary	86a226e	2019-03-19 12:17:44	[diff] [blame^]	1002	elif options.commit_hashes:
				1003	if not options._new_commit_flow:
				1004	raise Exception('--commit-hashes requries --new-commit-flow')
				1005	return generator.CommitStashedOrderfileHashes()
Egor Pasko	bce64d01	2018-11-20 12:02:37	[diff] [blame]	1006	elif options.upload_ready_orderfiles:
				1007	return generator.UploadReadyOrderfiles()
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	1008	else:
Egor Pasko	93e514e	2017-10-31 13:32:36	[diff] [blame]	1009	return generator.Generate()
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	1010	finally:
Egor Pasko	93e514e	2017-10-31 13:32:36	[diff] [blame]	1011	json_output = json.dumps(generator.GetReportingData(),
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	1012	indent=2) + '\n'
				1013	if options.json_file:
				1014	with open(options.json_file, 'w') as f:
				1015	f.write(json_output)
				1016	print json_output
				1017	return False
				1018
				1019
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	1020	def main():
				1021	parser = CreateArgumentParser()
				1022	options = parser.parse_args()
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	1023	return 0 if CreateOrderfile(options, OrderfileUpdater) else 1
				1024
				1025
				1026	if __name__ == '__main__':
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	1027	sys.exit(main())