Reland "Add runhooks to download PGO profiles and allow query profile path"
This is a reland of 5f5008fc7536c6f18b6099cb63675882a7b896e9
Original change's description:
> Add runhooks to download PGO profiles and allow query profile path
>
> This CL:
> 1. Adds gclient runhooks to download PGO profiles.
> 2. Allows the build system to query the path to the current profile.
>
> Bug: 1071576
> Change-Id: I6b8954f81b61ef7362f28ca2bf585abd84e1a799
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2165448
> Commit-Queue: Yuke Liao <liaoyuke@chromium.org>
> Reviewed-by: Nico Weber <thakis@chromium.org>
> Cr-Commit-Position: refs/heads/master@{#763123}
TBR=thakis@chromium.org
Bug: 1071576
Change-Id: Ib5ccfc60d12fa3d07c344691c81cc63ff3efe642
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2171523
Reviewed-by: Yuke Liao <liaoyuke@chromium.org>
Reviewed-by: Erik Staab <estaab@chromium.org>
Commit-Queue: Yuke Liao <liaoyuke@chromium.org>
Cr-Commit-Position: refs/heads/master@{#763663}
diff --git a/.gitignore b/.gitignore
index 99067144..4130b92 100644
--- a/.gitignore
+++ b/.gitignore
@@ -85,6 +85,7 @@
/chrome/build/chrome.x86.orderfile
/chrome/build/chrome_child.x64.orderfile
/chrome/build/chrome_child.x86.orderfile
+/chrome/build/pgo_profiles/
/chrome/gl_tests_run.xml
/chrome/gles2_conform_test_run.xml
/chrome/tab_capture_performance_tests_run.xml
diff --git a/DEPS b/DEPS
index 5381f82..fb30c9c 100644
--- a/DEPS
+++ b/DEPS
@@ -100,6 +100,9 @@
# the gn arg 'use_clang_coverage').
'checkout_clang_coverage_tools': False,
+ # Fetch the pgo profiles to optimize official builds.
+ 'checkout_pgo_profiles': False,
+
# Fetch clang-tidy into the same bin/ directory as our clang binary.
'checkout_clang_tidy': False,
@@ -4361,6 +4364,41 @@
],
},
+ # Download PGO profiles.
+ {
+ 'name': 'Fetch PGO profiles for win32',
+ 'pattern': '.',
+ 'condition': 'checkout_pgo_profiles and checkout_win',
+ 'action': [ 'vpython',
+ 'src/tools/update_pgo_profiles.py',
+ '--target=win32',
+ 'update',
+ '--gs-url-base=chromium-optimization-profiles/pgo_profiles',
+ ],
+ },
+ {
+ 'name': 'Fetch PGO profiles for win64',
+ 'pattern': '.',
+ 'condition': 'checkout_pgo_profiles and checkout_win',
+ 'action': [ 'vpython',
+ 'src/tools/update_pgo_profiles.py',
+ '--target=win64',
+ 'update',
+ '--gs-url-base=chromium-optimization-profiles/pgo_profiles',
+ ],
+ },
+ {
+ 'name': 'Fetch PGO profiles for mac',
+ 'pattern': '.',
+ 'condition': 'checkout_pgo_profiles and checkout_mac',
+ 'action': [ 'vpython',
+ 'src/tools/update_pgo_profiles.py',
+ '--target=mac',
+ 'update',
+ '--gs-url-base=chromium-optimization-profiles/pgo_profiles',
+ ],
+ },
+
# Download and initialize "vpython" VirtualEnv environment packages.
{
'name': 'vpython_common',
diff --git a/chrome/build/mac.pgo.txt b/chrome/build/mac.pgo.txt
new file mode 100644
index 0000000..05bbb70
--- /dev/null
+++ b/chrome/build/mac.pgo.txt
@@ -0,0 +1 @@
+chrome-mac-master-d2d91c2fe263909bb31bcfaf0ae882b731320f0d-0000000.profdata
diff --git a/chrome/build/win32.pgo.txt b/chrome/build/win32.pgo.txt
new file mode 100644
index 0000000..4337488
--- /dev/null
+++ b/chrome/build/win32.pgo.txt
@@ -0,0 +1 @@
+chrome-win32-master-67ad3c89d2017131cc9ce664a1580315517550d1-0000000.profdata
diff --git a/chrome/build/win64.pgo.txt b/chrome/build/win64.pgo.txt
new file mode 100644
index 0000000..6a1bb8c
--- /dev/null
+++ b/chrome/build/win64.pgo.txt
@@ -0,0 +1 @@
+chrome-win64-master-cfc722972bb38f406ad47e4c56911f35ab1231f9-0000000.profdata
diff --git a/tools/update_pgo_profiles.py b/tools/update_pgo_profiles.py
new file mode 100755
index 0000000..775604a
--- /dev/null
+++ b/tools/update_pgo_profiles.py
@@ -0,0 +1,139 @@
+#!/usr/bin/env python
+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Downloads pgo profiles for optimizing official Chrome.
+
+This script has the following responsibilities:
+1. Download a requested profile if necessary.
+2. Return a path to the current profile to feed to the build system.
+3. Removed stale profiles (2 days) to save disk spaces because profiles are
+ large (~1GB) and updated frequently (~4 times a day).
+"""
+
+from __future__ import print_function
+
+import argparse
+import os
+import sys
+import time
+
+_SRC_ROOT = os.path.abspath(
+ os.path.join(os.path.dirname(__file__), os.path.pardir))
+sys.path.append(os.path.join(_SRC_ROOT, 'third_party', 'depot_tools'))
+import download_from_google_storage
+
+sys.path.append(os.path.join(_SRC_ROOT, 'build'))
+import gn_helpers
+
+# Absolute path to the directory that stores pgo related state files, which
+# specifcies which profile to update and use.
+_PGO_DIR = os.path.join(_SRC_ROOT, 'chrome', 'build')
+
+# Absolute path to the directory that stores pgo profiles.
+_PGO_PROFILE_DIR = os.path.join(_PGO_DIR, 'pgo_profiles')
+
+
+def _read_profile_name(target):
+ """Read profile name given a target.
+
+ Args:
+ target(str): The target name, such as win32, mac.
+
+ Returns:
+ Name of the profile to update and use, such as:
+ chrome-win32-master-67ad3c89d2017131cc9ce664a1580315517550d1.profdata.
+ """
+ state_file = os.path.join(_PGO_DIR, '%s.pgo.txt' % target)
+ with open(state_file, 'r') as f:
+ profile_name = f.read().strip()
+
+ return profile_name
+
+
+def _remove_unused_profiles(current_profile_name):
+ """Removes unused profiles, except the current one, to save disk space."""
+ days = 2
+ expiration_duration = 60 * 60 * 24 * days
+ for f in os.listdir(_PGO_PROFILE_DIR):
+ if f == current_profile_name:
+ continue
+
+ p = os.path.join(_PGO_PROFILE_DIR, f)
+ age = time.time() - os.path.getmtime(p)
+ if age > expiration_duration:
+ print('Removing profile %s as it hasn\'t been used in the past %d days' %
+ (p, days))
+ os.remove(p)
+
+
+def _update(args):
+ """Update profile if necessary according to the state file.
+
+ Args:
+ args(dict): A dict of cmd arguments, such as target and gs_url_base.
+
+ Raises:
+ RuntimeError: If failed to download profiles from gcs.
+ """
+ profile_name = _read_profile_name(args.target)
+ profile_path = os.path.join(_PGO_PROFILE_DIR, profile_name)
+ if os.path.isfile(profile_path):
+ os.utime(profile_path, None)
+ return
+
+ gsutil = download_from_google_storage.Gsutil(
+ download_from_google_storage.GSUTIL_DEFAULT_PATH)
+ gs_path = 'gs://' + args.gs_url_base.strip('/') + '/' + profile_name
+ code = gsutil.call('cp', gs_path, profile_path)
+ if code != 0:
+ raise RuntimeError('gsutil failed to download "%s"' % gs_path)
+
+ _remove_unused_profiles(profile_name)
+
+
+def _get_profile_path(args):
+ """Returns an absolute path to the current profile.
+
+ Args:
+ args(dict): A dict of cmd arguments, such as target and gs_url_base.
+
+ Raises:
+ RuntimeError: If the current profile is missing.
+ """
+ profile_path = os.path.join(_PGO_PROFILE_DIR, _read_profile_name(args.target))
+ if not os.path.isfile(profile_path):
+ raise RuntimeError('requested profile "%s" doesn\'t exist, please run '
+ '"gclient runhooks" to download it')
+
+ os.utime(profile_path, None)
+ profile_path.rstrip(os.sep)
+ print(gn_helpers.ToGNString(profile_path))
+
+
+def main():
+ parser = argparse.ArgumentParser(
+ description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ parser.add_argument(
+ '--target',
+ required=True,
+ choices=['win32', 'win64', 'mac'],
+ help='Identifier of a specific target platform + architecture.')
+ subparsers = parser.add_subparsers()
+
+ parser_update = subparsers.add_parser('update')
+ parser_update.add_argument(
+ '--gs-url-base',
+ required=True,
+ help='The base GS URL to search for the profile.')
+ parser_update.set_defaults(func=_update)
+
+ parser_get_profile_path = subparsers.add_parser('get_profile_path')
+ parser_get_profile_path.set_defaults(func=_get_profile_path)
+
+ args = parser.parse_args()
+ return args.func(args)
+
+
+if __name__ == '__main__':
+ sys.exit(main())