Blame - tools/cygprofile/orderfile_generator_backend.py - chromium/src

blob: 6649a8db2e183de6530800c531bb927c54757e3f [file] [log] [blame]

Egor Pasko	0462e852d	2018-03-29 15:52:09	[diff] [blame]	1	#!/usr/bin/env vpython
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	2	# Copyright (c) 2013 The Chromium Authors. All rights reserved.
				3	# Use of this source code is governed by a BSD-style license that can be
				4	# found in the LICENSE file.
				5
				6	""" A utility to generate an up-to-date orderfile.
				7
				8	The orderfile is used by the linker to order text sections such that the
				9	sections are placed consecutively in the order specified. This allows us
				10	to page in less code during start-up.
				11
				12	Example usage:
Egor Pasko	93e514e	2017-10-31 13:32:36	[diff] [blame]	13	tools/cygprofile/orderfile_generator_backend.py -l 20 -j 1000 --use-goma \
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	14	--target-arch=arm
				15	"""
				16
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	17	import argparse
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	18	import hashlib
				19	import json
Matthew Cary	69e9e42	2018-08-10 18:30:06	[diff] [blame]	20	import glob
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	21	import logging
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	22	import os
				23	import re
				24	import shutil
				25	import subprocess
				26	import sys
Benoit Lize	a87e5bc	2017-11-07 15:12:57	[diff] [blame]	27	import tempfile
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	28	import time
				29
Matthew Cary	91df979	2018-11-30 14:35:15	[diff] [blame]	30	import cluster
Matthew Cary	e840064	2018-06-14 15:43:02	[diff] [blame]	31	import cyglog_to_orderfile
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	32	import cygprofile_utils
Benoit Lize	fefbb27c	2018-01-17 13:54:18	[diff] [blame]	33	import patch_orderfile
Benoit Lize	a87e5bc	2017-11-07 15:12:57	[diff] [blame]	34	import process_profiles
Egor Pasko	3bc0b93	2018-04-03 10:08:27	[diff] [blame]	35	import profile_android_startup
Benoit Lize	fefbb27c	2018-01-17 13:54:18	[diff] [blame]	36	import symbol_extractor
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	37
				38
				39	_SRC_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)),
				40	os.pardir, os.pardir)
				41	sys.path.append(os.path.join(_SRC_PATH, 'build', 'android'))
				42	import devil_chromium
				43	from pylib import constants
				44
				45
				46	# Needs to happen early for GetBuildType()/GetOutDirectory() to work correctly
				47	constants.SetBuildType('Release')
				48
				49
Matthew Cary	53f74ba	2019-01-24 10:07:18	[diff] [blame^]	50	# Architecture specific GN args. Trying to build an orderfile for an
				51	# architecture not listed here will eventually throw.
				52	_ARCH_GN_ARGS = {
				53	'arm': [ 'target_cpu = "arm"' ],
				54	'arm64': [ 'target_cpu = "arm64"',
				55	'android_64bit_browser = true'],
				56	}
				57
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	58	class CommandError(Exception):
				59	"""Indicates that a dispatched shell command exited with a non-zero status."""
				60
				61	def __init__(self, value):
				62	super(CommandError, self).__init__()
				63	self.value = value
				64
				65	def __str__(self):
				66	return repr(self.value)
				67
				68
				69	def _GenerateHash(file_path):
				70	"""Calculates and returns the hash of the file at file_path."""
				71	sha1 = hashlib.sha1()
				72	with open(file_path, 'rb') as f:
				73	while True:
				74	# Read in 1mb chunks, so it doesn't all have to be loaded into memory.
				75	chunk = f.read(1024 * 1024)
				76	if not chunk:
				77	break
				78	sha1.update(chunk)
				79	return sha1.hexdigest()
				80
				81
				82	def _GetFileExtension(file_name):
				83	"""Calculates the file extension from a file name.
				84
				85	Args:
				86	file_name: The source file name.
				87	Returns:
				88	The part of file_name after the dot (.) or None if the file has no
				89	extension.
				90	Examples: /home/user/foo.bar -> bar
				91	/home/user.name/foo -> None
				92	/home/user/.foo -> None
				93	/home/user/foo.bar.baz -> baz
				94	"""
				95	file_name_parts = os.path.basename(file_name).split('.')
				96	if len(file_name_parts) > 1:
				97	return file_name_parts[-1]
				98	else:
				99	return None
				100
				101
				102	def _StashOutputDirectory(buildpath):
				103	"""Takes the output directory and stashes it in the default output directory.
				104
				105	This allows it to be used for incremental builds next time (after unstashing)
				106	by keeping it in a place that isn't deleted normally, while also ensuring
				107	that it is properly clobbered when appropriate.
				108
				109	This is a dirty hack to deal with the needs of clobbering while also handling
				110	incremental builds and the hardcoded relative paths used in some of the
				111	project files.
				112
				113	Args:
				114	buildpath: The path where the building happens. If this corresponds to the
				115	default output directory, no action is taken.
				116	"""
				117	if os.path.abspath(buildpath) == os.path.abspath(os.path.dirname(
				118	constants.GetOutDirectory())):
				119	return
				120	name = os.path.basename(buildpath)
				121	stashpath = os.path.join(constants.GetOutDirectory(), name)
				122	if not os.path.exists(buildpath):
				123	return
				124	if os.path.exists(stashpath):
				125	shutil.rmtree(stashpath, ignore_errors=True)
				126	shutil.move(buildpath, stashpath)
				127
				128
				129	def _UnstashOutputDirectory(buildpath):
				130	"""Inverse of _StashOutputDirectory.
				131
				132	Moves the output directory stashed within the default output directory
				133	(out/Release) to the position where the builds can actually happen.
				134
				135	This is a dirty hack to deal with the needs of clobbering while also handling
				136	incremental builds and the hardcoded relative paths used in some of the
				137	project files.
				138
				139	Args:
				140	buildpath: The path where the building happens. If this corresponds to the
				141	default output directory, no action is taken.
				142	"""
				143	if os.path.abspath(buildpath) == os.path.abspath(os.path.dirname(
				144	constants.GetOutDirectory())):
				145	return
				146	name = os.path.basename(buildpath)
				147	stashpath = os.path.join(constants.GetOutDirectory(), name)
				148	if not os.path.exists(stashpath):
				149	return
				150	if os.path.exists(buildpath):
				151	shutil.rmtree(buildpath, ignore_errors=True)
				152	shutil.move(stashpath, buildpath)
				153
				154
				155	class StepRecorder(object):
				156	"""Records steps and timings."""
				157
				158	def __init__(self, buildbot):
				159	self.timings = []
				160	self._previous_step = ('', 0.0)
				161	self._buildbot = buildbot
				162	self._error_recorded = False
				163
				164	def BeginStep(self, name):
				165	"""Marks a beginning of the next step in the script.
				166
				167	On buildbot, this prints a specially formatted name that will show up
				168	in the waterfall. Otherwise, just prints the step name.
				169
				170	Args:
				171	name: The name of the step.
				172	"""
				173	self.EndStep()
				174	self._previous_step = (name, time.time())
				175	print 'Running step: ', name
				176
				177	def EndStep(self):
				178	"""Records successful completion of the current step.
				179
				180	This is optional if the step is immediately followed by another BeginStep.
				181	"""
				182	if self._previous_step[0]:
				183	elapsed = time.time() - self._previous_step[1]
				184	print 'Step %s took %f seconds' % (self._previous_step[0], elapsed)
				185	self.timings.append((self._previous_step[0], elapsed))
				186
				187	self._previous_step = ('', 0.0)
				188
				189	def FailStep(self, message=None):
				190	"""Marks that a particular step has failed.
				191
				192	On buildbot, this will mark the current step as failed on the waterfall.
				193	Otherwise we will just print an optional failure message.
				194
				195	Args:
				196	message: An optional explanation as to why the step failed.
				197	"""
				198	print 'STEP FAILED!!'
				199	if message:
				200	print message
				201	self._error_recorded = True
				202	self.EndStep()
				203
				204	def ErrorRecorded(self):
				205	"""True if FailStep has been called."""
				206	return self._error_recorded
				207
				208	def RunCommand(self, cmd, cwd=constants.DIR_SOURCE_ROOT, raise_on_error=True,
				209	stdout=None):
				210	"""Execute a shell command.
				211
				212	Args:
				213	cmd: A list of command strings.
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	214	cwd: Directory in which the command should be executed, defaults to build
				215	root of script's location if not specified.
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	216	raise_on_error: If true will raise a CommandError if the call doesn't
				217	succeed and mark the step as failed.
				218	stdout: A file to redirect stdout for the command to.
				219
				220	Returns:
				221	The process's return code.
				222
				223	Raises:
				224	CommandError: An error executing the specified command.
				225	"""
				226	print 'Executing %s in %s' % (' '.join(cmd), cwd)
				227	process = subprocess.Popen(cmd, stdout=stdout, cwd=cwd, env=os.environ)
				228	process.wait()
				229	if raise_on_error and process.returncode != 0:
				230	self.FailStep()
				231	raise CommandError('Exception executing command %s' % ' '.join(cmd))
				232	return process.returncode
				233
				234
				235	class ClankCompiler(object):
				236	"""Handles compilation of clank."""
				237
				238	def __init__(self, out_dir, step_recorder, arch, jobs, max_load, use_goma,
Matthew Cary	d6bfcb7	2018-09-14 11:27:46	[diff] [blame]	239	goma_dir, system_health_profiling, monochrome):
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	240	self._out_dir = out_dir
				241	self._step_recorder = step_recorder
Benoit Lize	a87e5bc	2017-11-07 15:12:57	[diff] [blame]	242	self._arch = arch
				243	self._jobs = jobs
				244	self._max_load = max_load
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	245	self._use_goma = use_goma
Benoit Lize	a87e5bc	2017-11-07 15:12:57	[diff] [blame]	246	self._goma_dir = goma_dir
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	247	self._system_health_profiling = system_health_profiling
Matthew Cary	d6bfcb7	2018-09-14 11:27:46	[diff] [blame]	248	if monochrome:
				249	self._apk = 'Monochrome.apk'
Matthew Cary	e1b0006	2018-09-19 14:27:44	[diff] [blame]	250	self._apk_target = 'monochrome_apk'
Matthew Cary	d6bfcb7	2018-09-14 11:27:46	[diff] [blame]	251	self._libname = 'libmonochrome'
				252	self._libchrome_target = 'monochrome'
				253	else:
				254	self._apk = 'Chrome.apk'
Matthew Cary	e1b0006	2018-09-19 14:27:44	[diff] [blame]	255	self._apk_target = 'chrome_apk'
Matthew Cary	d6bfcb7	2018-09-14 11:27:46	[diff] [blame]	256	self._libname = 'libchrome'
				257	self._libchrome_target = 'libchrome'
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	258
				259	self.obj_dir = os.path.join(self._out_dir, 'Release', 'obj')
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	260	self.lib_chrome_so = os.path.join(
Matthew Cary	d6bfcb7	2018-09-14 11:27:46	[diff] [blame]	261	self._out_dir, 'Release', 'lib.unstripped',
				262	'{}.so'.format(self._libname))
				263	self.chrome_apk = os.path.join(self._out_dir, 'Release', 'apks', self._apk)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	264
				265	def Build(self, instrumented, target):
				266	"""Builds the provided ninja target with or without order_profiling on.
				267
				268	Args:
Benoit Lize	a87e5bc	2017-11-07 15:12:57	[diff] [blame]	269	instrumented: (bool) Whether we want to build an instrumented binary.
				270	target: (str) The name of the ninja target to build.
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	271	"""
				272	self._step_recorder.BeginStep('Compile %s' % target)
				273
				274	# Set the "Release Official" flavor, the parts affecting performance.
				275	args = [
Peter Collingbourne	1c85d5d2	2018-08-22 18:06:35	[diff] [blame]	276	'enable_resource_whitelist_generation=false',
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	277	'is_chrome_branded=true',
				278	'is_debug=false',
				279	'is_official_build=true',
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	280	'target_os="android"',
				281	'use_goma=' + str(self._use_goma).lower(),
				282	'use_order_profiling=' + str(instrumented).lower(),
				283	]
Matthew Cary	53f74ba	2019-01-24 10:07:18	[diff] [blame^]	284	args += _ARCH_GN_ARGS[self._arch]
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	285	if self._goma_dir:
				286	args += ['goma_dir="%s"' % self._goma_dir]
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	287	if self._system_health_profiling:
				288	args += ['devtools_instrumentation_dumping = ' +
				289	str(instrumented).lower()]
Benoit Lize	a87e5bc	2017-11-07 15:12:57	[diff] [blame]	290
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	291	self._step_recorder.RunCommand(
				292	['gn', 'gen', os.path.join(self._out_dir, 'Release'),
				293	'--args=' + ' '.join(args)])
				294
				295	self._step_recorder.RunCommand(
				296	['ninja', '-C', os.path.join(self._out_dir, 'Release'),
				297	'-j' + str(self._jobs), '-l' + str(self._max_load), target])
				298
				299	def CompileChromeApk(self, instrumented, force_relink=False):
				300	"""Builds a Chrome.apk either with or without order_profiling on.
				301
				302	Args:
Benoit Lize	a87e5bc	2017-11-07 15:12:57	[diff] [blame]	303	instrumented: (bool) Whether to build an instrumented apk.
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	304	force_relink: Whether libchromeview.so should be re-created.
				305	"""
				306	if force_relink:
				307	self._step_recorder.RunCommand(['rm', '-rf', self.lib_chrome_so])
Matthew Cary	e1b0006	2018-09-19 14:27:44	[diff] [blame]	308	self.Build(instrumented, self._apk_target)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	309
				310	def CompileLibchrome(self, instrumented, force_relink=False):
				311	"""Builds a libchrome.so either with or without order_profiling on.
				312
				313	Args:
Benoit Lize	a87e5bc	2017-11-07 15:12:57	[diff] [blame]	314	instrumented: (bool) Whether to build an instrumented apk.
				315	force_relink: (bool) Whether libchrome.so should be re-created.
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	316	"""
				317	if force_relink:
				318	self._step_recorder.RunCommand(['rm', '-rf', self.lib_chrome_so])
Matthew Cary	d6bfcb7	2018-09-14 11:27:46	[diff] [blame]	319	self.Build(instrumented, self._libchrome_target)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	320
				321
				322	class OrderfileUpdater(object):
				323	"""Handles uploading and committing a new orderfile in the repository.
				324
				325	Only used for testing or on a bot.
				326	"""
				327
				328	_CLOUD_STORAGE_BUCKET_FOR_DEBUG = None
				329	_CLOUD_STORAGE_BUCKET = None
				330	_UPLOAD_TO_CLOUD_COMMAND = 'upload_to_google_storage.py'
				331
				332	def __init__(self, repository_root, step_recorder, branch, netrc):
				333	"""Constructor.
				334
				335	Args:
				336	repository_root: (str) Root of the target repository.
				337	step_recorder: (StepRecorder) Step recorder, for logging.
				338	branch: (str) Branch to commit to.
				339	netrc: (str) Path to the .netrc file to use.
				340	"""
				341	self._repository_root = repository_root
				342	self._step_recorder = step_recorder
				343	self._branch = branch
				344	self._netrc = netrc
				345
				346	def CommitFileHashes(self, unpatched_orderfile_filename, orderfile_filename):
				347	"""Commits unpatched and patched orderfiles hashes, if provided.
				348
				349	Files must have been successfilly uploaded to cloud storage first.
				350
				351	Args:
				352	unpatched_orderfile_filename: (str or None) Unpatched orderfile path.
				353	orderfile_filename: (str or None) Orderfile path.
				354
				355	Raises:
				356	NotImplementedError when the commit logic hasn't been overriden.
				357	"""
				358	files_to_commit = []
				359	commit_message_lines = ['Update Orderfile.']
				360	for filename in [unpatched_orderfile_filename, orderfile_filename]:
				361	if not filename:
				362	continue
				363	(relative_path, sha1) = self._GetHashFilePathAndContents(filename)
				364	commit_message_lines.append('Profile: %s: %s' % (
				365	os.path.basename(relative_path), sha1))
				366	files_to_commit.append(relative_path)
				367	if files_to_commit:
				368	self._CommitFiles(files_to_commit, commit_message_lines)
				369
				370	def UploadToCloudStorage(self, filename, use_debug_location):
				371	"""Uploads a file to cloud storage.
				372
				373	Args:
				374	filename: (str) File to upload.
				375	use_debug_location: (bool) Whether to use the debug location.
				376	"""
				377	bucket = (self._CLOUD_STORAGE_BUCKET_FOR_DEBUG if use_debug_location
				378	else self._CLOUD_STORAGE_BUCKET)
				379	extension = _GetFileExtension(filename)
				380	cmd = [self._UPLOAD_TO_CLOUD_COMMAND, '--bucket', bucket]
				381	if extension:
				382	cmd.extend(['-z', extension])
				383	cmd.append(filename)
				384	self._step_recorder.RunCommand(cmd)
				385	print 'Download: https://sandbox.google.com/storage/%s/%s' % (
				386	bucket, _GenerateHash(filename))
				387
				388	def _GetHashFilePathAndContents(self, filename):
				389	"""Gets the name and content of the hash file created from uploading the
				390	given file.
				391
				392	Args:
				393	filename: (str) The file that was uploaded to cloud storage.
				394
				395	Returns:
				396	A tuple of the hash file name, relative to the reository root, and the
				397	content, which should be the sha1 hash of the file
				398	('base_file.sha1', hash)
				399	"""
				400	abs_hash_filename = filename + '.sha1'
				401	rel_hash_filename = os.path.relpath(
				402	abs_hash_filename, self._repository_root)
				403	with open(abs_hash_filename, 'r') as f:
				404	return (rel_hash_filename, f.read())
				405
				406	def _CommitFiles(self, files_to_commit, commit_message_lines):
				407	"""Commits a list of files, with a given message."""
				408	raise NotImplementedError
				409
				410
				411	class OrderfileGenerator(object):
				412	"""A utility for generating a new orderfile for Clank.
				413
				414	Builds an instrumented binary, profiles a run of the application, and
				415	generates an updated orderfile.
				416	"""
				417	_CLANK_REPO = os.path.join(constants.DIR_SOURCE_ROOT, 'clank')
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	418	_CHECK_ORDERFILE_SCRIPT = os.path.join(
				419	constants.DIR_SOURCE_ROOT, 'tools', 'cygprofile', 'check_orderfile.py')
				420	_BUILD_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(
				421	constants.GetOutDirectory()))) # Normally /path/to/src
				422
				423	_UNPATCHED_ORDERFILE_FILENAME = os.path.join(
				424	_CLANK_REPO, 'orderfiles', 'unpatched_orderfile.%s')
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	425
				426	_PATH_TO_ORDERFILE = os.path.join(_CLANK_REPO, 'orderfiles',
				427	'orderfile.%s.out')
				428
				429	# Previous orderfile_generator debug files would be overwritten.
				430	_DIRECTORY_FOR_DEBUG_FILES = '/tmp/orderfile_generator_debug_files'
				431
				432	def _GetPathToOrderfile(self):
				433	"""Gets the path to the architecture-specific orderfile."""
				434	return self._PATH_TO_ORDERFILE % self._options.arch
				435
				436	def _GetUnpatchedOrderfileFilename(self):
				437	"""Gets the path to the architecture-specific unpatched orderfile."""
				438	return self._UNPATCHED_ORDERFILE_FILENAME % self._options.arch
				439
				440	def __init__(self, options, orderfile_updater_class):
				441	self._options = options
				442
				443	self._instrumented_out_dir = os.path.join(
				444	self._BUILD_ROOT, self._options.arch + '_instrumented_out')
				445	self._uninstrumented_out_dir = os.path.join(
				446	self._BUILD_ROOT, self._options.arch + '_uninstrumented_out')
				447
				448	if options.profile:
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	449	output_directory = os.path.join(self._instrumented_out_dir, 'Release')
Matthew Cary	b8daed94	2018-06-11 10:58:08	[diff] [blame]	450	host_profile_dir = os.path.join(output_directory, 'profile_data')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	451	urls = [profile_android_startup.AndroidProfileTool.TEST_URL]
				452	use_wpr = True
				453	simulate_user = False
Benoit L	9646681	2018-03-06 15:58:37	[diff] [blame]	454	urls = options.urls
				455	use_wpr = not options.no_wpr
				456	simulate_user = options.simulate_user
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	457	self._profiler = profile_android_startup.AndroidProfileTool(
Matthew Cary	453ff145	2018-07-18 12:19:35	[diff] [blame]	458	output_directory, host_profile_dir, use_wpr, urls, simulate_user,
				459	device=options.device)
Matthew Cary	69e9e42	2018-08-10 18:30:06	[diff] [blame]	460	if options.pregenerated_profiles:
				461	self._profiler.SetPregeneratedProfiles(
				462	glob.glob(options.pregenerated_profiles))
				463	else:
				464	assert not options.pregenerated_profiles, (
				465	'--pregenerated-profiles cannot be used with --skip-profile')
				466	assert not options.profile_save_dir, (
				467	'--profile-save-dir cannot be used with --skip-profile')
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	468
				469	self._output_data = {}
				470	self._step_recorder = StepRecorder(options.buildbot)
				471	self._compiler = None
				472	assert issubclass(orderfile_updater_class, OrderfileUpdater)
				473	self._orderfile_updater = orderfile_updater_class(self._CLANK_REPO,
				474	self._step_recorder,
				475	options.branch,
				476	options.netrc)
				477	assert os.path.isdir(constants.DIR_SOURCE_ROOT), 'No src directory found'
Benoit Lize	fefbb27c	2018-01-17 13:54:18	[diff] [blame]	478	symbol_extractor.SetArchitecture(options.arch)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	479
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	480	@staticmethod
				481	def _RemoveBlanks(src_file, dest_file):
				482	"""A utility to remove blank lines from a file.
				483
				484	Args:
				485	src_file: The name of the file to remove the blanks from.
				486	dest_file: The name of the file to write the output without blanks.
				487	"""
				488	assert src_file != dest_file, 'Source and destination need to be distinct'
				489
				490	try:
				491	src = open(src_file, 'r')
				492	dest = open(dest_file, 'w')
				493	for line in src:
				494	if line and not line.isspace():
				495	dest.write(line)
				496	finally:
				497	src.close()
				498	dest.close()
				499
				500	def _GenerateAndProcessProfile(self):
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	501	"""Invokes a script to merge the per-thread traces into one file.
				502
				503	The produced list of offsets is saved in
				504	self._GetUnpatchedOrderfileFilename().
				505	"""
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	506	self._step_recorder.BeginStep('Generate Profile Data')
				507	files = []
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	508	logging.getLogger().setLevel(logging.DEBUG)
				509	if self._options.system_health_orderfile:
				510	files = self._profiler.CollectSystemHealthProfile(
				511	self._compiler.chrome_apk)
Matthew Cary	69e9e42	2018-08-10 18:30:06	[diff] [blame]	512	self._MaybeSaveProfile(files)
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	513	try:
				514	self._ProcessPhasedOrderfile(files)
				515	except Exception:
				516	for f in files:
				517	self._SaveForDebugging(f)
Matthew Cary	8b141623	2018-08-10 19:12:22	[diff] [blame]	518	self._SaveForDebugging(self._compiler.lib_chrome_so)
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	519	raise
				520	finally:
				521	self._profiler.Cleanup()
				522	else:
				523	self._CollectLegacyProfile()
				524	logging.getLogger().setLevel(logging.INFO)
				525
				526	def _ProcessPhasedOrderfile(self, files):
				527	"""Process the phased orderfiles produced by system health benchmarks.
				528
				529	The offsets will be placed in _GetUnpatchedOrderfileFilename().
				530
				531	Args:
				532	file: Profile files pulled locally.
				533	"""
				534	self._step_recorder.BeginStep('Process Phased Orderfile')
				535	profiles = process_profiles.ProfileManager(files)
				536	processor = process_profiles.SymbolOffsetProcessor(
				537	self._compiler.lib_chrome_so)
Matthew Cary	91df979	2018-11-30 14:35:15	[diff] [blame]	538	ordered_symbols= cluster.ClusterOffsets(profiles, processor)
Matthew Cary	8b141623	2018-08-10 19:12:22	[diff] [blame]	539	if not ordered_symbols:
				540	raise Exception('Failed to get ordered symbols')
Matthew Cary	91df979	2018-11-30 14:35:15	[diff] [blame]	541	self._output_data['offsets_kib'] = processor.SymbolsSize(
				542	ordered_symbols) / 1024
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	543	with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile:
Matthew Cary	8b141623	2018-08-10 19:12:22	[diff] [blame]	544	orderfile.write('\n'.join(ordered_symbols))
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	545
				546	def _CollectLegacyProfile(self):
Matthew Cary	2df8d45	2018-09-19 07:50:20	[diff] [blame]	547	files = []
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	548	try:
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	549	files = self._profiler.CollectProfile(
				550	self._compiler.chrome_apk,
				551	constants.PACKAGE_INFO['chrome'])
Matthew Cary	69e9e42	2018-08-10 18:30:06	[diff] [blame]	552	self._MaybeSaveProfile(files)
Matthew Cary	b8daed94	2018-06-11 10:58:08	[diff] [blame]	553	self._step_recorder.BeginStep('Process profile')
Benoit L	9646681	2018-03-06 15:58:37	[diff] [blame]	554	assert os.path.exists(self._compiler.lib_chrome_so)
				555	offsets = process_profiles.GetReachedOffsetsFromDumpFiles(
				556	files, self._compiler.lib_chrome_so)
				557	if not offsets:
				558	raise Exception('No profiler offsets found in {}'.format(
				559	'\n'.join(files)))
Matthew Cary	8b141623	2018-08-10 19:12:22	[diff] [blame]	560	processor = process_profiles.SymbolOffsetProcessor(
				561	self._compiler.lib_chrome_so)
				562	ordered_symbols = processor.GetOrderedSymbols(offsets)
				563	if not ordered_symbols:
				564	raise Exception('No symbol names from offsets found in {}'.format(
				565	'\n'.join(files)))
				566	with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile:
				567	orderfile.write('\n'.join(ordered_symbols))
Matthew Cary	0f1f681a	2018-01-22 10:40:51	[diff] [blame]	568	except Exception:
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	569	for f in files:
				570	self._SaveForDebugging(f)
				571	raise
				572	finally:
				573	self._profiler.Cleanup()
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	574
Matthew Cary	69e9e42	2018-08-10 18:30:06	[diff] [blame]	575	def _MaybeSaveProfile(self, files):
				576	if self._options.profile_save_dir:
				577	logging.info('Saving profiles to %s', self._options.profile_save_dir)
				578	for f in files:
				579	shutil.copy(f, self._options.profile_save_dir)
				580	logging.info('Saved profile %s', f)
				581
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	582	def _PatchOrderfile(self):
				583	"""Patches the orderfile using clean version of libchrome.so."""
				584	self._step_recorder.BeginStep('Patch Orderfile')
Benoit Lize	fefbb27c	2018-01-17 13:54:18	[diff] [blame]	585	patch_orderfile.GeneratePatchedOrderfile(
				586	self._GetUnpatchedOrderfileFilename(), self._compiler.lib_chrome_so,
				587	self._GetPathToOrderfile())
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	588
				589	def _VerifySymbolOrder(self):
				590	self._step_recorder.BeginStep('Verify Symbol Order')
				591	return_code = self._step_recorder.RunCommand(
				592	[self._CHECK_ORDERFILE_SCRIPT, self._compiler.lib_chrome_so,
				593	self._GetPathToOrderfile(),
				594	'--target-arch=' + self._options.arch],
				595	constants.DIR_SOURCE_ROOT,
				596	raise_on_error=False)
				597	if return_code:
				598	self._step_recorder.FailStep('Orderfile check returned %d.' % return_code)
				599
				600	def _RecordHash(self, file_name):
				601	"""Records the hash of the file into the output_data dictionary."""
				602	self._output_data[os.path.basename(file_name) + '.sha1'] = _GenerateHash(
				603	file_name)
				604
				605	def _SaveFileLocally(self, file_name, file_sha1):
				606	"""Saves the file to a temporary location and prints the sha1sum."""
				607	if not os.path.exists(self._DIRECTORY_FOR_DEBUG_FILES):
				608	os.makedirs(self._DIRECTORY_FOR_DEBUG_FILES)
				609	shutil.copy(file_name, self._DIRECTORY_FOR_DEBUG_FILES)
				610	print 'File: %s, saved in: %s, sha1sum: %s' % (
				611	file_name, self._DIRECTORY_FOR_DEBUG_FILES, file_sha1)
				612
				613	def _SaveForDebugging(self, filename):
				614	"""Uploads the file to cloud storage or saves to a temporary location."""
				615	file_sha1 = _GenerateHash(filename)
				616	if not self._options.buildbot:
				617	self._SaveFileLocally(filename, file_sha1)
				618	else:
				619	print 'Uploading file for debugging: ' + filename
				620	self._orderfile_updater.UploadToCloudStorage(
				621	filename, use_debug_location=True)
				622
				623	def _SaveForDebuggingWithOverwrite(self, file_name):
				624	"""Uploads and overwrites the file in cloud storage or copies locally.
				625
				626	Should be used for large binaries like lib_chrome_so.
				627
				628	Args:
				629	file_name: (str) File to upload.
				630	"""
				631	file_sha1 = _GenerateHash(file_name)
				632	if not self._options.buildbot:
				633	self._SaveFileLocally(file_name, file_sha1)
				634	else:
				635	print 'Uploading file for debugging: %s, sha1sum: %s' % (
				636	file_name, file_sha1)
				637	upload_location = '%s/%s' % (
				638	self._CLOUD_STORAGE_BUCKET_FOR_DEBUG, os.path.basename(file_name))
				639	self._step_recorder.RunCommand([
				640	'gsutil.py', 'cp', file_name, 'gs://' + upload_location])
				641	print ('Uploaded to: https://sandbox.google.com/storage/' +
				642	upload_location)
				643
				644	def _MaybeArchiveOrderfile(self, filename):
				645	"""In buildbot configuration, uploads the generated orderfile to
				646	Google Cloud Storage.
				647
				648	Args:
				649	filename: (str) Orderfile to upload.
				650	"""
Matthew Cary	91df979	2018-11-30 14:35:15	[diff] [blame]	651	# First compute hashes so that we can download them later if we need to.
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	652	self._step_recorder.BeginStep('Compute hash for ' + filename)
				653	self._RecordHash(filename)
				654	if self._options.buildbot:
				655	self._step_recorder.BeginStep('Archive ' + filename)
				656	self._orderfile_updater.UploadToCloudStorage(
				657	filename, use_debug_location=False)
				658
Egor Pasko	bce64d01	2018-11-20 12:02:37	[diff] [blame]	659	def UploadReadyOrderfiles(self):
				660	self._step_recorder.BeginStep('Upload Ready Orderfiles')
				661	for file_name in [self._GetUnpatchedOrderfileFilename(),
				662	self._GetPathToOrderfile()]:
				663	self._orderfile_updater.UploadToCloudStorage(
				664	file_name, use_debug_location=False)
				665
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	666	def _GetHashFilePathAndContents(self, base_file):
				667	"""Gets the name and content of the hash file created from uploading the
				668	given file.
				669
				670	Args:
				671	base_file: The file that was uploaded to cloud storage.
				672
				673	Returns:
				674	A tuple of the hash file name, relative to the clank repo path, and the
				675	content, which should be the sha1 hash of the file
				676	('base_file.sha1', hash)
				677	"""
				678	abs_file_name = base_file + '.sha1'
				679	rel_file_name = os.path.relpath(abs_file_name, self._CLANK_REPO)
				680	with open(abs_file_name, 'r') as f:
				681	return (rel_file_name, f.read())
				682
				683	def Generate(self):
				684	"""Generates and maybe upload an order."""
				685	profile_uploaded = False
				686	orderfile_uploaded = False
				687
Matthew Cary	e840064	2018-06-14 15:43:02	[diff] [blame]	688	assert (bool(self._options.profile) ^
				689	bool(self._options.manual_symbol_offsets))
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	690	if self._options.system_health_orderfile and not self._options.profile:
				691	raise AssertionError('--system_health_orderfile must be not be used '
				692	'with --skip-profile')
				693	if (self._options.manual_symbol_offsets and
				694	not self._options.system_health_orderfile):
				695	raise AssertionError('--manual-symbol-offsets must be used with '
				696	'--system_health_orderfile.')
Matthew Cary	e840064	2018-06-14 15:43:02	[diff] [blame]	697
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	698	if self._options.profile:
				699	try:
				700	_UnstashOutputDirectory(self._instrumented_out_dir)
				701	self._compiler = ClankCompiler(
				702	self._instrumented_out_dir,
				703	self._step_recorder, self._options.arch, self._options.jobs,
				704	self._options.max_load, self._options.use_goma,
Matthew Cary	d6bfcb7	2018-09-14 11:27:46	[diff] [blame]	705	self._options.goma_dir, self._options.system_health_orderfile,
				706	self._options.monochrome)
Matthew Cary	a2bea45	2018-11-13 10:21:07	[diff] [blame]	707	if not self._options.pregenerated_profiles:
				708	# If there are pregenerated profiles, the instrumented build should
				709	# not be changed to avoid invalidating the pregenerated profile
				710	# offsets.
				711	self._compiler.CompileChromeApk(True)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	712	self._GenerateAndProcessProfile()
				713	self._MaybeArchiveOrderfile(self._GetUnpatchedOrderfileFilename())
				714	profile_uploaded = True
				715	finally:
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	716	_StashOutputDirectory(self._instrumented_out_dir)
Matthew Cary	e840064	2018-06-14 15:43:02	[diff] [blame]	717	elif self._options.manual_symbol_offsets:
				718	assert self._options.manual_libname
				719	assert self._options.manual_objdir
				720	with file(self._options.manual_symbol_offsets) as f:
				721	symbol_offsets = [int(x) for x in f.xreadlines()]
				722	processor = process_profiles.SymbolOffsetProcessor(
Matthew Cary	b46ad28	2018-11-23 14:43:57	[diff] [blame]	723	self._compiler.manual_libname)
Matthew Cary	e840064	2018-06-14 15:43:02	[diff] [blame]	724	generator = cyglog_to_orderfile.OffsetOrderfileGenerator(
				725	processor, cyglog_to_orderfile.ObjectFileProcessor(
				726	self._options.manual_objdir))
				727	ordered_sections = generator.GetOrderedSections(symbol_offsets)
				728	if not ordered_sections: # Either None or empty is a problem.
				729	raise Exception('Failed to get ordered sections')
				730	with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile:
				731	orderfile.write('\n'.join(ordered_sections))
				732
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	733	if self._options.patch:
				734	if self._options.profile:
				735	self._RemoveBlanks(self._GetUnpatchedOrderfileFilename(),
				736	self._GetPathToOrderfile())
				737	try:
				738	_UnstashOutputDirectory(self._uninstrumented_out_dir)
				739	self._compiler = ClankCompiler(
				740	self._uninstrumented_out_dir, self._step_recorder,
				741	self._options.arch, self._options.jobs, self._options.max_load,
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	742	self._options.use_goma, self._options.goma_dir,
Matthew Cary	d6bfcb7	2018-09-14 11:27:46	[diff] [blame]	743	self._options.system_health_orderfile, self._options.monochrome)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	744	self._compiler.CompileLibchrome(False)
				745	self._PatchOrderfile()
				746	# Because identical code folding is a bit different with and without
				747	# the orderfile build, we need to re-patch the orderfile with code
				748	# folding as close to the final version as possible.
				749	self._compiler.CompileLibchrome(False, force_relink=True)
				750	self._PatchOrderfile()
				751	self._compiler.CompileLibchrome(False, force_relink=True)
				752	self._VerifySymbolOrder()
				753	self._MaybeArchiveOrderfile(self._GetPathToOrderfile())
				754	finally:
				755	_StashOutputDirectory(self._uninstrumented_out_dir)
				756	orderfile_uploaded = True
				757
				758	if (self._options.buildbot and self._options.netrc
				759	and not self._step_recorder.ErrorRecorded()):
				760	unpatched_orderfile_filename = (
				761	self._GetUnpatchedOrderfileFilename() if profile_uploaded else None)
				762	orderfile_filename = (
Benoit Lize	d913be8	2017-10-24 13:38:27	[diff] [blame]	763	self._GetPathToOrderfile() if orderfile_uploaded else None)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	764	self._orderfile_updater.CommitFileHashes(
				765	unpatched_orderfile_filename, orderfile_filename)
				766
				767	self._step_recorder.EndStep()
				768	return not self._step_recorder.ErrorRecorded()
				769
				770	def GetReportingData(self):
				771	"""Get a dictionary of reporting data (timings, output hashes)"""
				772	self._output_data['timings'] = self._step_recorder.timings
				773	return self._output_data
				774
				775
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	776	def CreateArgumentParser():
				777	"""Creates and returns the argument parser."""
				778	parser = argparse.ArgumentParser()
				779	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	780	'--buildbot', action='store_true',
				781	help='If true, the script expects to be run on a buildbot')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	782	parser.add_argument(
Matthew Cary	453ff145	2018-07-18 12:19:35	[diff] [blame]	783	'--device', default=None, type=str,
				784	help='Device serial number on which to run profiling.')
				785	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	786	'--verify', action='store_true',
				787	help='If true, the script only verifies the current orderfile')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	788	parser.add_argument('--target-arch', action='store', dest='arch',
Stephen Martinis	71b391b5	2018-12-04 15:08:04	[diff] [blame]	789	default='arm',
Matthew Cary	53f74ba	2019-01-24 10:07:18	[diff] [blame^]	790	choices=['arm', 'arm64'],
				791	help='The target architecture for which to build.')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	792	parser.add_argument('--output-json', action='store', dest='json_file',
				793	help='Location to save stats in json format')
				794	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	795	'--skip-profile', action='store_false', dest='profile', default=True,
				796	help='Don\'t generate a profile on the device. Only patch from the '
				797	'existing profile.')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	798	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	799	'--skip-patch', action='store_false', dest='patch', default=True,
				800	help='Only generate the raw (unpatched) orderfile, don\'t patch it.')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	801	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	802	'--netrc', action='store',
				803	help='A custom .netrc file to use for git checkin. Only used on bots.')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	804	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	805	'--branch', action='store', default='master',
				806	help='When running on buildbot with a netrc, the branch orderfile '
				807	'hashes get checked into.')
				808	# Note: -j50 was causing issues on the bot.
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	809	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	810	'-j', '--jobs', action='store', default=20,
				811	help='Number of jobs to use for compilation.')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	812	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	813	'-l', '--max-load', action='store', default=4, help='Max cpu load.')
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	814	parser.add_argument('--goma-dir', help='GOMA directory.')
				815	parser.add_argument(
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	816	'--use-goma', action='store_true', help='Enable GOMA.', default=False)
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	817	parser.add_argument('--adb-path', help='Path to the adb binary.')
Matthew Cary	e840064	2018-06-14 15:43:02	[diff] [blame]	818
Matthew Cary	04f4103	2018-12-10 15:55:27	[diff] [blame]	819	parser.add_argument('--nosystem-health-orderfile', action='store_false',
				820	dest='system_health_orderfile', default=True,
Matthew Cary	c262f5d	2018-09-19 14:36:28	[diff] [blame]	821	help=('Create an orderfile based on an about:blank '
				822	'startup benchmark instead of system health '
				823	'benchmarks.'))
Matthew Cary	d6bfcb7	2018-09-14 11:27:46	[diff] [blame]	824	parser.add_argument('--monochrome', action='store_true',
				825	help=('Compile and instrument monochrome (for post-N '
				826	'devices).'))
Matthew Cary	78aae16	2018-08-10 17:16:30	[diff] [blame]	827
Matthew Cary	e840064	2018-06-14 15:43:02	[diff] [blame]	828	parser.add_argument('--manual-symbol-offsets', default=None, type=str,
				829	help=('File of list of ordered symbol offsets generated '
				830	'by manual profiling. Must set other --manual* '
				831	'flags if this is used, and must --skip-profile.'))
				832	parser.add_argument('--manual-libname', default=None, type=str,
				833	help=('Library filename corresponding to '
				834	'--manual-symbol-offsets.'))
				835	parser.add_argument('--manual-objdir', default=None, type=str,
				836	help=('Root of object file directory corresponding to '
				837	'--manual-symbol-offsets.'))
Matthew Cary	69e9e42	2018-08-10 18:30:06	[diff] [blame]	838	parser.add_argument('--pregenerated-profiles', default=None, type=str,
				839	help=('Pregenerated profiles to use instead of running '
				840	'profile step. Cannot be used with '
				841	'--skip-profiles.'))
				842	parser.add_argument('--profile-save-dir', default=None, type=str,
				843	help=('Directory to save any profiles created. These can '
				844	'be used with --pregenerated-profiles. Cannot be '
				845	'used with --skip-profiles.'))
Egor Pasko	bce64d01	2018-11-20 12:02:37	[diff] [blame]	846	parser.add_argument('--upload-ready-orderfiles', action='store_true',
				847	help=('Skip orderfile generation and manually upload '
				848	'orderfiles (both patched and unpatched) from '
				849	'their normal location in the tree to the cloud '
				850	'storage. DANGEROUS! USE WITH CARE!'))
Matthew Cary	e840064	2018-06-14 15:43:02	[diff] [blame]	851
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	852	profile_android_startup.AddProfileCollectionArguments(parser)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	853	return parser
				854
				855
				856	def CreateOrderfile(options, orderfile_updater_class):
				857	"""Creates an oderfile.
				858
				859	Args:
				860	options: As returned from optparse.OptionParser.parse_args()
				861	orderfile_updater_class: (OrderfileUpdater) subclass of OrderfileUpdater.
				862
				863	Returns:
				864	True iff success.
				865	"""
				866	logging.basicConfig(level=logging.INFO)
				867	devil_chromium.Initialize(adb_path=options.adb_path)
				868
Egor Pasko	93e514e	2017-10-31 13:32:36	[diff] [blame]	869	generator = OrderfileGenerator(options, orderfile_updater_class)
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	870	try:
				871	if options.verify:
Egor Pasko	93e514e	2017-10-31 13:32:36	[diff] [blame]	872	generator._VerifySymbolOrder()
Egor Pasko	bce64d01	2018-11-20 12:02:37	[diff] [blame]	873	elif options.upload_ready_orderfiles:
				874	return generator.UploadReadyOrderfiles()
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	875	else:
Egor Pasko	93e514e	2017-10-31 13:32:36	[diff] [blame]	876	return generator.Generate()
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	877	finally:
Egor Pasko	93e514e	2017-10-31 13:32:36	[diff] [blame]	878	json_output = json.dumps(generator.GetReportingData(),
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	879	indent=2) + '\n'
				880	if options.json_file:
				881	with open(options.json_file, 'w') as f:
				882	f.write(json_output)
				883	print json_output
				884	return False
				885
				886
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	887	def main():
				888	parser = CreateArgumentParser()
				889	options = parser.parse_args()
Benoit Lize	a3fe293	2017-10-20 10:24:52	[diff] [blame]	890	return 0 if CreateOrderfile(options, OrderfileUpdater) else 1
				891
				892
				893	if __name__ == '__main__':
Benoit Lize	a1b64f8	2017-12-07 10:12:50	[diff] [blame]	894	sys.exit(main())