Blame - startop/scripts/app_startup/app_startup_runner.py - platform/frameworks/base

2018-09-11 16:37:18 -0700

[diff] [blame]

1

#!/usr/bin/env python3

#

#

# Licensed under the Apache License, Version 2.0 (the "License");

6

# you may not use this file except in compliance with the License.

7

# You may obtain a copy of the License at

8

#

9

# http://www.apache.org/licenses/LICENSE-2.0

10

#

11

# Unless required by applicable law or agreed to in writing, software

12

# distributed under the License is distributed on an "AS IS" BASIS,

13

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

14

# See the License for the specific language governing permissions and

15

# limitations under the License.

#

#

# Measure application start-up time by launching applications under various combinations.

20

# See --help for more details.

#

#

# Sample usage:

# $> ./app_startup_runner.py -p com.google.android.calculator -r warm -r cold -lc 10 -o out.csv

25

# $> ./analyze_metrics.py out.csv

#

#

import argparse

import csv

import itertools

import os

Igor Murashkin

2018-09-11 16:37:18 -0700

[diff] [blame]

33

import sys

34

import tempfile

Yan Wang

2019-07-26 10:53:45 -0700

[diff] [blame]

35

from datetime import timedelta

Yan Wang

2019-07-02 15:46:34 -0700

[diff] [blame]

36

from typing import Any, Callable, Iterable, List, NamedTuple, TextIO, Tuple, \

Yan Wang

2019-07-23 18:09:41 -0700

[diff] [blame]

37

TypeVar, Union, Optional

Yan Wang

2019-07-02 15:46:34 -0700

[diff] [blame]

38

39

# local import

40

DIR = os.path.abspath(os.path.dirname(__file__))

41

sys.path.append(os.path.dirname(DIR))

Yan Wang

2019-07-26 10:53:45 -0700

[diff] [blame]

42

import lib.cmd_utils as cmd_utils

43

import lib.print_utils as print_utils

Yan Wang

2019-07-23 18:09:41 -0700

[diff] [blame]

44

from app_startup.run_app_with_prefetch import PrefetchAppRunner

Yan Wang

2019-07-02 15:46:34 -0700

[diff] [blame]

45

import app_startup.lib.args_utils as args_utils

46

from app_startup.lib.data_frame import DataFrame

Yan Wang

2019-07-26 10:53:45 -0700

[diff] [blame]

47

from app_startup.lib.perfetto_trace_collector import PerfettoTraceCollector

Yan Wang

2019-09-06 16:19:03 -0700

[diff] [blame]

48

from iorap.compiler import CompilerType

49

import iorap.compiler as compiler

Igor Murashkin

2018-09-11 16:37:18 -0700

[diff] [blame]

50

51

# The following command line options participate in the combinatorial generation.

52

# All other arguments have a global effect.

Yan Wang

2019-07-26 10:53:45 -0700

[diff] [blame]

53

_COMBINATORIAL_OPTIONS = ['package', 'readahead', 'compiler_filter',

54

'activity', 'trace_duration']

Yan Wang

2019-07-02 15:46:34 -0700

[diff] [blame]

55

_TRACING_READAHEADS = ['mlock', 'fadvise']

56

_FORWARD_OPTIONS = {'loop_count': '--count'}

57

_RUN_SCRIPT = os.path.join(os.path.dirname(os.path.realpath(__file__)),

58

'run_app_with_prefetch.py')

Igor Murashkin

2018-09-11 16:37:18 -0700

[diff] [blame]

59

Yan Wang

2019-07-02 15:46:34 -0700

[diff] [blame]

60

CollectorPackageInfo = NamedTuple('CollectorPackageInfo',

61

[('package', str), ('compiler_filter', str)])

Yan Wang

2019-07-02 15:46:34 -0700

[diff] [blame]

62

# by 2; systrace starts up slowly.

Igor Murashkin

2018-09-11 16:37:18 -0700

[diff] [blame]

63

Yan Wang

2019-07-02 15:46:34 -0700

[diff] [blame]

64

_UNLOCK_SCREEN_SCRIPT = os.path.join(

65

os.path.dirname(os.path.realpath(__file__)), 'unlock_screen')

Igor Murashkin

2018-09-11 16:37:18 -0700

[diff] [blame]

66

Yan Wang

2019-07-23 18:09:41 -0700

[diff] [blame]

67

RunCommandArgs = NamedTuple('RunCommandArgs',

68

[('package', str),

69

('readahead', str),

70

('activity', Optional[str]),

71

('compiler_filter', Optional[str]),

72

('timeout', Optional[int]),

73

('debug', bool),

74

('simulate', bool),

Yan Wang

2019-07-26 10:53:45 -0700

[diff] [blame]

75

('input', Optional[str]),

76

('trace_duration', Optional[timedelta])])

Yan Wang

2019-07-23 18:09:41 -0700

[diff] [blame]

77

Igor Murashkin

2018-09-11 16:37:18 -0700

[diff] [blame]

78

# This must be the only mutable global variable. All other global variables are constants to avoid magic literals.

79

_debug = False # See -d/--debug flag.

80

_DEBUG_FORCE = None # Ignore -d/--debug if this is not none.

Yan Wang

2019-07-26 10:53:45 -0700

[diff] [blame]

81

_PERFETTO_TRACE_DURATION_MS = 5000 # milliseconds

82

_PERFETTO_TRACE_DURATION = timedelta(milliseconds=_PERFETTO_TRACE_DURATION_MS)

Igor Murashkin

2018-09-11 16:37:18 -0700

[diff] [blame]

83

84

# Type hinting names.

85

T = TypeVar('T')

Yan Wang

2019-07-02 15:46:34 -0700

[diff] [blame]

86

NamedTupleMeta = Callable[

87

..., T] # approximation of a (S : NamedTuple<T> where S() == T) metatype.

Igor Murashkin

2018-09-11 16:37:18 -0700

[diff] [blame]

88

89

def parse_options(argv: List[str] = None):

90

"""Parse command line arguments and return an argparse Namespace object."""

Yan Wang

2019-07-02 15:46:34 -0700

[diff] [blame]

91

parser = argparse.ArgumentParser(description="Run one or more Android "

92

"applications under various "

93

"settings in order to measure "

94

"startup time.")

Igor Murashkin

2018-09-11 16:37:18 -0700

[diff] [blame]

95

# argparse considers args starting with - and -- optional in --help, even though required=True.

96

# by using a named argument group --help will clearly say that it's required instead of optional.

97

required_named = parser.add_argument_group('required named arguments')

Yan Wang

2019-07-02 15:46:34 -0700

[diff] [blame]

98

required_named.add_argument('-p', '--package', action='append',

99

dest='packages',

100

help='package of the application', required=True)

101

required_named.add_argument('-r', '--readahead', action='append',

102

dest='readaheads',

103

help='which readahead mode to use',

104

choices=('warm', 'cold', 'mlock', 'fadvise'),

105

required=True)

Igor Murashkin

2018-09-11 16:37:18 -0700

[diff] [blame]

106

107

# optional arguments

108

# use a group here to get the required arguments to appear 'above' the optional arguments in help.

109

optional_named = parser.add_argument_group('optional named arguments')

Yan Wang

2019-07-02 15:46:34 -0700

[diff] [blame]

110

optional_named.add_argument('-c', '--compiler-filter', action='append',

111

dest='compiler_filters',

112

help='which compiler filter to use. if omitted it does not enforce the app\'s compiler filter',

113

choices=('speed', 'speed-profile', 'quicken'))

114

optional_named.add_argument('-s', '--simulate', dest='simulate',

115

action='store_true',

116

help='Print which commands will run, but don\'t run the apps')

117

optional_named.add_argument('-d', '--debug', dest='debug',

118

action='store_true',

119

help='Add extra debugging output')

120

optional_named.add_argument('-o', '--output', dest='output', action='store',

121

help='Write CSV output to file.')

122

optional_named.add_argument('-t', '--timeout', dest='timeout', action='store',

123

type=int, default=10,

124

help='Timeout after this many seconds when executing a single run.')

125

optional_named.add_argument('-lc', '--loop-count', dest='loop_count',

126

default=1, type=int, action='store',

127

help='How many times to loop a single run.')

128

optional_named.add_argument('-in', '--inodes', dest='inodes', type=str,

129

action='store',

130

help='Path to inodes file (system/extras/pagecache/pagecache.py -d inodes)')

Yan Wang

2019-07-26 10:53:45 -0700

[diff] [blame]

131

optional_named.add_argument('--compiler-trace-duration-ms',

132

dest='trace_duration',

133

type=lambda ms_str: timedelta(milliseconds=int(ms_str)),

134

action='append',

135

help='The trace duration (milliseconds) in '

136

'compilation')

Yan Wang

2019-09-06 16:19:03 -0700

[diff] [blame]

137

optional_named.add_argument('--compiler-type', dest='compiler_type',

138

type=CompilerType, choices=list(CompilerType),

139

default=CompilerType.DEVICE,

140

help='The type of compiler.')

Igor Murashkin

2018-09-11 16:37:18 -0700

[diff] [blame]

141

142

return parser.parse_args(argv)

143

Igor Murashkin

2018-09-11 16:37:18 -0700

[diff] [blame]

144

def key_to_cmdline_flag(key: str) -> str:

145

"""Convert key into a command line flag, e.g. 'foo-bars' -> '--foo-bar' """

146

if key.endswith("s"):

147

key = key[:-1]

148

return "--" + key.replace("_", "-")

149

150

def as_run_command(tpl: NamedTuple) -> List[Union[str, Any]]:

151

"""

152

Convert a named tuple into a command-line compatible arguments list.

153

154

Example: ABC(1, 2, 3) -> ['--a', 1, '--b', 2, '--c', 3]

155

"""

156

args = []

157

for key, value in tpl._asdict().items():

158

if value is None:

159

continue

160

args.append(key_to_cmdline_flag(key))

args.append(value)

return args

Yan Wang

2019-07-26 10:53:45 -0700

[diff] [blame]

164

def run_perfetto_collector(collector_info: CollectorPackageInfo,

165

timeout: int,

166

simulate: bool) -> Tuple[bool, TextIO]:

167

"""Run collector to collect prefetching trace.

Igor Murashkin

2018-09-11 16:37:18 -0700

[diff] [blame]

168

Yan Wang

2019-07-26 10:53:45 -0700

[diff] [blame]

169

Returns:

170

A tuple of whether the collection succeeds and the generated trace file.

171

"""

172

tmp_output_file = tempfile.NamedTemporaryFile()

Igor Murashkin

2018-09-11 16:37:18 -0700

[diff] [blame]

173

Yan Wang

2019-07-26 10:53:45 -0700

[diff] [blame]

174

collector = PerfettoTraceCollector(package=collector_info.package,

175

activity=None,

176

compiler_filter=collector_info.compiler_filter,

177

timeout=timeout,

178

simulate=simulate,

179

trace_duration=_PERFETTO_TRACE_DURATION,

180

save_destination_file_path=tmp_output_file.name)

181

result = collector.run()

182

183

return result is not None, tmp_output_file

Igor Murashkin

2018-09-11 16:37:18 -0700

[diff] [blame]

184

Igor Murashkin

2019-05-13 16:31:25 -0700

[diff] [blame]

185

def parse_run_script_csv_file(csv_file: TextIO) -> DataFrame:

186

"""Parse a CSV file full of integers into a DataFrame."""

187

csv_reader = csv.reader(csv_file)

188

189

try:

190

header_list = next(csv_reader)

191

except StopIteration:

header_list = []

if not header_list:

return None

headers = [i for i in header_list]

198

199

d = {}

200

for row in csv_reader:

header_idx = 0

for i in row:

v = i

if i:

v = int(i)

header_key = headers[header_idx]

209

l = d.get(header_key, [])

l.append(v)

d[header_key] = l

header_idx = header_idx + 1

return DataFrame(d)

Yan Wang

2019-09-06 16:19:03 -0700

[diff] [blame]

217

def build_ri_compiler_argv(inodes_path: str,

Yan Wang

2019-07-26 10:53:45 -0700

[diff] [blame]

218

perfetto_trace_file: str,

Yan Wang

2019-09-06 16:19:03 -0700

[diff] [blame]

219

trace_duration: Optional[timedelta]

220

) -> str:

221

argv = ['-i', inodes_path, '--perfetto-trace',

222

perfetto_trace_file]

Yan Wang

2019-07-26 10:53:45 -0700

[diff] [blame]

223

224

if trace_duration is not None:

225

argv += ['--duration', str(int(trace_duration.total_seconds()

Yan Wang

2019-09-06 16:19:03 -0700

[diff] [blame]

226

* PerfettoTraceCollector.MS_PER_SEC))]

Yan Wang

2019-07-26 10:53:45 -0700

[diff] [blame]

227

228

print_utils.debug_print(argv)

Yan Wang

2019-09-06 16:19:03 -0700

[diff] [blame]

229

return argv

Yan Wang

2019-07-26 10:53:45 -0700

[diff] [blame]

230

231

def execute_run_using_perfetto_trace(collector_info,

232

run_combos: Iterable[RunCommandArgs],

233

simulate: bool,

234

inodes_path: str,

Yan Wang

2019-09-06 16:19:03 -0700

[diff] [blame]

235

timeout: int,

Yan Wang

2019-10-09 16:26:54 -0700

[diff] [blame^]

236

compiler_type: CompilerType,

237

requires_trace_collection: bool) -> DataFrame:

Yan Wang

2019-07-26 10:53:45 -0700

[diff] [blame]

238

""" Executes run based on perfetto trace. """

Yan Wang

2019-10-09 16:26:54 -0700

[diff] [blame^]

239

if requires_trace_collection:

240

passed, perfetto_trace_file = run_perfetto_collector(collector_info,

timeout,

simulate)

if not passed:

raise RuntimeError('Cannot run perfetto collector!')

245

else:

246

perfetto_trace_file = tempfile.NamedTemporaryFile()

Yan Wang

2019-07-26 10:53:45 -0700

[diff] [blame]

247

248

with perfetto_trace_file:

249

for combos in run_combos:

250

if combos.readahead in _TRACING_READAHEADS:

251

if simulate:

252

compiler_trace_file = tempfile.NamedTemporaryFile()

253

else:

Yan Wang

2019-09-06 16:19:03 -0700

[diff] [blame]

254

ri_compiler_argv = build_ri_compiler_argv(inodes_path,

255

perfetto_trace_file.name,

256

combos.trace_duration)

257

compiler_trace_file = compiler.compile(compiler_type,

inodes_path,

ri_compiler_argv,

combos.package,

combos.activity)

Yan Wang

2019-07-26 10:53:45 -0700

[diff] [blame]

263

with compiler_trace_file:

264

combos = combos._replace(input=compiler_trace_file.name)

265

print_utils.debug_print(combos)

266

output = PrefetchAppRunner(**combos._asdict()).run()

267

else:

268

print_utils.debug_print(combos)

269

output = PrefetchAppRunner(**combos._asdict()).run()

270

271

yield DataFrame(dict((x, [y]) for x, y in output)) if output else None

272

Yan Wang

2019-07-02 15:46:34 -0700

[diff] [blame]

273

def execute_run_combos(

Yan Wang

2019-07-23 18:09:41 -0700

[diff] [blame]

274

grouped_run_combos: Iterable[Tuple[CollectorPackageInfo, Iterable[RunCommandArgs]]],

Yan Wang

2019-07-02 15:46:34 -0700

[diff] [blame]

275

simulate: bool,

276

inodes_path: str,

Yan Wang

2019-09-06 16:19:03 -0700

[diff] [blame]

277

timeout: int,

Yan Wang

2019-10-09 16:26:54 -0700

[diff] [blame^]

278

compiler_type: CompilerType,

279

requires_trace_collection: bool):

Igor Murashkin

2018-09-11 16:37:18 -0700

[diff] [blame]

280

# nothing will work if the screen isn't unlocked first.

Yan Wang

2019-07-02 15:46:34 -0700

[diff] [blame]

281

cmd_utils.execute_arbitrary_command([_UNLOCK_SCREEN_SCRIPT],

282

timeout,

283

simulate=simulate,

284

shell=False)

Igor Murashkin

2018-09-11 16:37:18 -0700

[diff] [blame]

285

286

for collector_info, run_combos in grouped_run_combos:

Yan Wang

2019-07-26 10:53:45 -0700

[diff] [blame]

287

yield from execute_run_using_perfetto_trace(collector_info,

288

run_combos,

289

simulate,

290

inodes_path,

Yan Wang

2019-09-06 16:19:03 -0700

[diff] [blame]

291

timeout,

Yan Wang

2019-10-09 16:26:54 -0700

[diff] [blame^]

292

compiler_type,

293

requires_trace_collection)

Igor Murashkin

2018-09-11 16:37:18 -0700

[diff] [blame]

294

Yan Wang

2019-07-02 15:46:34 -0700

[diff] [blame]

295

def gather_results(commands: Iterable[Tuple[DataFrame]],

296

key_list: List[str], value_list: List[Tuple[str, ...]]):

297

print_utils.debug_print("gather_results: key_list = ", key_list)

Igor Murashkin

2018-09-11 16:37:18 -0700

[diff] [blame]

298

stringify_none = lambda s: s is None and "<none>" or s

Yan Wang

2019-07-02 15:46:34 -0700

[diff] [blame]

299

# yield key_list + ["time(ms)"]

300

for (run_result_list, values) in itertools.zip_longest(commands, value_list):

301

print_utils.debug_print("run_result_list = ", run_result_list)

302

print_utils.debug_print("values = ", values)

Igor Murashkin

2018-09-11 16:37:18 -0700

[diff] [blame]

303

Yan Wang

2019-07-02 15:46:34 -0700

[diff] [blame]

304

if not run_result_list:

Igor Murashkin

2018-09-11 16:37:18 -0700

[diff] [blame]

305

continue

Igor Murashkin

2018-09-11 16:37:18 -0700

[diff] [blame]

306

Igor Murashkin

2019-05-13 16:31:25 -0700

[diff] [blame]

307

# RunCommandArgs(package='com.whatever', readahead='warm', compiler_filter=None)

308

# -> {'package':['com.whatever'], 'readahead':['warm'], 'compiler_filter':[None]}

Yan Wang

2019-07-02 15:46:34 -0700

[diff] [blame]

309

values_dict = {}

310

for k, v in values._asdict().items():

311

if not k in key_list:

312

continue

313

values_dict[k] = [stringify_none(v)]

Igor Murashkin

2019-05-13 16:31:25 -0700

[diff] [blame]

314

315

values_df = DataFrame(values_dict)

316

# project 'values_df' to be same number of rows as run_result_list.

317

values_df = values_df.repeat(run_result_list.data_row_len)

318

319

# the results are added as right-hand-side columns onto the existing labels for the table.

320

values_df.merge_data_columns(run_result_list)

321

322

yield values_df

Igor Murashkin

2018-09-11 16:37:18 -0700

[diff] [blame]

323

324

def eval_and_save_to_csv(output, annotated_result_values):

Igor Murashkin

2019-05-13 16:31:25 -0700

[diff] [blame]

325

printed_header = False

326

Igor Murashkin

2018-09-11 16:37:18 -0700

[diff] [blame]

327

csv_writer = csv.writer(output)

328

for row in annotated_result_values:

Igor Murashkin

2019-05-13 16:31:25 -0700

[diff] [blame]

329

if not printed_header:

330

headers = row.headers

331

csv_writer.writerow(headers)

332

printed_header = True

333

# TODO: what about when headers change?

334

335

for data_row in row.data_table:

Yan Wang

2019-07-02 15:46:34 -0700

[diff] [blame]

336

data_row = [d for d in data_row]

Igor Murashkin

2019-05-13 16:31:25 -0700

[diff] [blame]

337

csv_writer.writerow(data_row)

338

Yan Wang

2019-07-02 15:46:34 -0700

[diff] [blame]

339

output.flush() # see the output live.

340

341

def coerce_to_list(opts: dict):

342

"""Tranform values of the dictionary to list.

343

For example:

344

1 -> [1], None -> [None], [1,2,3] -> [1,2,3]

345

[[1],[2]] -> [[1],[2]], {1:1, 2:2} -> [{1:1, 2:2}]

"""

result = {}

for key in opts:

val = opts[key]

result[key] = val if issubclass(type(val), list) else [val]

351

return result

Igor Murashkin

2018-09-11 16:37:18 -0700

[diff] [blame]

def main():

global _debug

opts = parse_options()

357

_debug = opts.debug

358

if _DEBUG_FORCE is not None:

359

_debug = _DEBUG_FORCE

Yan Wang

2019-07-02 15:46:34 -0700

[diff] [blame]

360

361

print_utils.DEBUG = _debug

362

cmd_utils.SIMULATE = opts.simulate

363

364

print_utils.debug_print("parsed options: ", opts)

Igor Murashkin

2018-09-11 16:37:18 -0700

[diff] [blame]

365

366

output_file = opts.output and open(opts.output, 'w') or sys.stdout

367

Yan Wang

2019-07-02 15:46:34 -0700

[diff] [blame]

368

combos = lambda: args_utils.generate_run_combinations(

Yan Wang

2019-07-23 18:09:41 -0700

[diff] [blame]

369

RunCommandArgs,

Yan Wang

2019-07-02 15:46:34 -0700

[diff] [blame]

370

coerce_to_list(vars(opts)),

371

opts.loop_count)

372

print_utils.debug_print_gen("run combinations: ", combos())

Igor Murashkin

2018-09-11 16:37:18 -0700

[diff] [blame]

373

Yan Wang

2019-07-02 15:46:34 -0700

[diff] [blame]

374

grouped_combos = lambda: args_utils.generate_group_run_combinations(combos(),

375

CollectorPackageInfo)

Igor Murashkin

2018-09-11 16:37:18 -0700

[diff] [blame]

376

Yan Wang

2019-07-02 15:46:34 -0700

[diff] [blame]

377

print_utils.debug_print_gen("grouped run combinations: ", grouped_combos())

Yan Wang

2019-10-09 16:26:54 -0700

[diff] [blame^]

378

requires_trace_collection = any(i in _TRACING_READAHEADS for i in opts.readaheads)

Yan Wang

2019-07-02 15:46:34 -0700

[diff] [blame]

379

exec = execute_run_combos(grouped_combos(),

380

opts.simulate,

381

opts.inodes,

Yan Wang

2019-09-06 16:19:03 -0700

[diff] [blame]

382

opts.timeout,

Yan Wang

2019-10-09 16:26:54 -0700

[diff] [blame^]

383

opts.compiler_type,

384

requires_trace_collection)

Yan Wang

2019-07-02 15:46:34 -0700

[diff] [blame]

385

Igor Murashkin

2018-09-11 16:37:18 -0700

[diff] [blame]

386

results = gather_results(exec, _COMBINATORIAL_OPTIONS, combos())

Yan Wang

2019-07-02 15:46:34 -0700

[diff] [blame]

387

Igor Murashkin

2018-09-11 16:37:18 -0700

[diff] [blame]

388

eval_and_save_to_csv(output_file, results)

389

Yan Wang

2019-07-02 15:46:34 -0700

[diff] [blame]

390

return 1

Igor Murashkin