| #!/usr/bin/env python |
| # Copyright 2015 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| import argparse |
| import collections |
| import logging |
| import os |
| import re |
| import subprocess |
| import sys |
| import time |
| |
| extra_trybots = [ |
| { |
| "mastername": "master.tryserver.chromium.win", |
| "buildernames": ["win_optional_gpu_tests_rel"] |
| }, |
| { |
| "mastername": "master.tryserver.chromium.mac", |
| "buildernames": ["mac_optional_gpu_tests_rel"] |
| }, |
| { |
| "mastername": "master.tryserver.chromium.linux", |
| "buildernames": ["linux_optional_gpu_tests_rel"] |
| }, |
| { |
| "mastername": "master.tryserver.chromium.android", |
| "buildernames": ["android_optional_gpu_tests_rel"] |
| }, |
| ] |
| |
| SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) |
| SRC_DIR = os.path.abspath(os.path.join(SCRIPT_DIR, os.pardir)) |
| sys.path.insert(0, os.path.join(SRC_DIR, 'build')) |
| import find_depot_tools |
| find_depot_tools.add_depot_tools_to_path() |
| import roll_dep_svn |
| from third_party import upload |
| |
| # Avoid depot_tools/third_party/upload.py print verbose messages. |
| upload.verbosity = 0 # Errors only. |
| |
| CHROMIUM_GIT_URL = 'https://chromium.googlesource.com/chromium/src.git' |
| CL_ISSUE_RE = re.compile('^Issue number: ([0-9]+) \((.*)\)$') |
| REVIEW_URL_RE = re.compile('^https?://(.*)/(.*)') |
| ROLL_BRANCH_NAME = 'special_webgl_roll_branch' |
| TRYJOB_STATUS_SLEEP_SECONDS = 30 |
| |
| # Use a shell for subcommands on Windows to get a PATH search. |
| IS_WIN = sys.platform.startswith('win') |
| WEBGL_PATH = os.path.join('third_party', 'webgl', 'src') |
| WEBGL_REVISION_TEXT_FILE = os.path.join( |
| 'content', 'test', 'gpu', 'gpu_tests', 'webgl_conformance_revision.txt') |
| |
| CommitInfo = collections.namedtuple('CommitInfo', ['git_commit', |
| 'git_repo_url']) |
| CLInfo = collections.namedtuple('CLInfo', ['issue', 'url', 'review_server']) |
| |
| |
| def _VarLookup(local_scope): |
| return lambda var_name: local_scope['vars'][var_name] |
| |
| |
| def _PosixPath(path): |
| """Convert a possibly-Windows path to a posix-style path.""" |
| (_, path) = os.path.splitdrive(path) |
| return path.replace(os.sep, '/') |
| |
| def _ParseGitCommitHash(description): |
| for line in description.splitlines(): |
| if line.startswith('commit '): |
| return line.split()[1] |
| logging.error('Failed to parse git commit id from:\n%s\n', description) |
| sys.exit(-1) |
| return None |
| |
| |
| def _ParseDepsFile(filename): |
| with open(filename, 'rb') as f: |
| deps_content = f.read() |
| return _ParseDepsDict(deps_content) |
| |
| |
| def _ParseDepsDict(deps_content): |
| local_scope = {} |
| global_scope = { |
| 'Var': _VarLookup(local_scope), |
| 'deps_os': {}, |
| } |
| exec(deps_content, global_scope, local_scope) |
| return local_scope |
| |
| |
| def _GenerateCLDescriptionCommand(webgl_current, webgl_new, bugs): |
| def GetChangeString(current_hash, new_hash): |
| return '%s..%s' % (current_hash[0:7], new_hash[0:7]); |
| |
| def GetChangeLogURL(git_repo_url, change_string): |
| return '%s/+log/%s' % (git_repo_url, change_string) |
| |
| def GetBugString(bugs): |
| bug_str = 'BUG=' |
| for bug in bugs: |
| bug_str += str(bug) + ',' |
| return bug_str.rstrip(',') |
| |
| change_str = GetChangeString(webgl_current.git_commit, |
| webgl_new.git_commit) |
| changelog_url = GetChangeLogURL(webgl_current.git_repo_url, |
| change_str) |
| if webgl_current.git_commit == webgl_new.git_commit: |
| print 'WARNING: WebGL repository is unchanged; proceeding with no-op roll' |
| |
| def GetExtraTrybotString(): |
| s = '' |
| for t in extra_trybots: |
| if s: |
| s += ';' |
| s += t['mastername'] + ':' + ','.join(t['buildernames']) |
| return s |
| |
| extra_trybot_args = [] |
| if extra_trybots: |
| extra_trybot_string = GetExtraTrybotString() |
| extra_trybot_args = ['-m', 'CQ_INCLUDE_TRYBOTS=' + extra_trybot_string] |
| |
| return [ |
| '-m', 'Roll WebGL ' + change_str, |
| '-m', '%s' % changelog_url, |
| '-m', GetBugString(bugs), |
| '-m', 'TEST=bots', |
| ] + extra_trybot_args |
| |
| |
| class AutoRoller(object): |
| def __init__(self, chromium_src): |
| self._chromium_src = chromium_src |
| |
| def _RunCommand(self, command, working_dir=None, ignore_exit_code=False, |
| extra_env=None): |
| """Runs a command and returns the stdout from that command. |
| |
| If the command fails (exit code != 0), the function will exit the process. |
| """ |
| working_dir = working_dir or self._chromium_src |
| logging.debug('cmd: %s cwd: %s', ' '.join(command), working_dir) |
| env = os.environ.copy() |
| if extra_env: |
| logging.debug('extra env: %s', extra_env) |
| env.update(extra_env) |
| p = subprocess.Popen(command, stdout=subprocess.PIPE, |
| stderr=subprocess.PIPE, shell=IS_WIN, env=env, |
| cwd=working_dir, universal_newlines=True) |
| output = p.stdout.read() |
| p.wait() |
| p.stdout.close() |
| p.stderr.close() |
| |
| if not ignore_exit_code and p.returncode != 0: |
| logging.error('Command failed: %s\n%s', str(command), output) |
| sys.exit(p.returncode) |
| return output |
| |
| def _GetCommitInfo(self, path_below_src, git_hash=None, git_repo_url=None): |
| working_dir = os.path.join(self._chromium_src, path_below_src) |
| self._RunCommand(['git', 'fetch', 'origin'], working_dir=working_dir) |
| revision_range = git_hash or 'origin' |
| ret = self._RunCommand( |
| ['git', '--no-pager', 'log', revision_range, |
| '--no-abbrev-commit', '--pretty=full', '-1'], |
| working_dir=working_dir) |
| return CommitInfo(_ParseGitCommitHash(ret), git_repo_url) |
| |
| def _GetDepsCommitInfo(self, deps_dict, path_below_src): |
| entry = deps_dict['deps'][_PosixPath('src/%s' % path_below_src)] |
| at_index = entry.find('@') |
| git_repo_url = entry[:at_index] |
| git_hash = entry[at_index + 1:] |
| return self._GetCommitInfo(path_below_src, git_hash, git_repo_url) |
| |
| def _GetCLInfo(self): |
| cl_output = self._RunCommand(['git', 'cl', 'issue']) |
| m = CL_ISSUE_RE.match(cl_output.strip()) |
| if not m: |
| logging.error('Cannot find any CL info. Output was:\n%s', cl_output) |
| sys.exit(-1) |
| issue_number = int(m.group(1)) |
| url = m.group(2) |
| |
| # Parse the codereview host from the URL. |
| m = REVIEW_URL_RE.match(url) |
| if not m: |
| logging.error('Cannot parse codereview host from URL: %s', url) |
| sys.exit(-1) |
| review_server = m.group(1) |
| return CLInfo(issue_number, url, review_server) |
| |
| def _GetCurrentBranchName(self): |
| return self._RunCommand( |
| ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).splitlines()[0] |
| |
| def _IsTreeClean(self): |
| lines = self._RunCommand( |
| ['git', 'status', '--porcelain', '-uno']).splitlines() |
| if len(lines) == 0: |
| return True |
| |
| logging.debug('Dirty/unversioned files:\n%s', '\n'.join(lines)) |
| return False |
| |
| def _GetBugList(self, path_below_src, webgl_current, webgl_new): |
| # TODO(kbr): this isn't useful, at least not yet, when run against |
| # the WebGL Github repository. |
| working_dir = os.path.join(self._chromium_src, path_below_src) |
| lines = self._RunCommand( |
| ['git','log', |
| '%s..%s' % (webgl_current.git_commit, webgl_new.git_commit)], |
| working_dir=working_dir).split('\n') |
| bugs = set() |
| for line in lines: |
| line = line.strip() |
| bug_prefix = 'BUG=' |
| if line.startswith(bug_prefix): |
| bugs_strings = line[len(bug_prefix):].split(',') |
| for bug_string in bugs_strings: |
| try: |
| bugs.add(int(bug_string)) |
| except: |
| # skip this, it may be a project specific bug such as |
| # "angleproject:X" or an ill-formed BUG= message |
| pass |
| return bugs |
| |
| def _UpdateReadmeFile(self, readme_path, new_revision): |
| readme = open(os.path.join(self._chromium_src, readme_path), 'r+') |
| txt = readme.read() |
| m = re.sub(re.compile('.*^Revision\: ([0-9]*).*', re.MULTILINE), |
| ('Revision: %s' % new_revision), txt) |
| readme.seek(0) |
| readme.write(m) |
| readme.truncate() |
| |
| def PrepareRoll(self, ignore_checks, run_tryjobs): |
| # TODO(kjellander): use os.path.normcase, os.path.join etc for all paths for |
| # cross platform compatibility. |
| |
| if not ignore_checks: |
| if self._GetCurrentBranchName() != 'master': |
| logging.error('Please checkout the master branch.') |
| return -1 |
| if not self._IsTreeClean(): |
| logging.error('Please make sure you don\'t have any modified files.') |
| return -1 |
| |
| # Always clean up any previous roll. |
| self.Abort() |
| |
| logging.debug('Pulling latest changes') |
| if not ignore_checks: |
| self._RunCommand(['git', 'pull']) |
| |
| self._RunCommand(['git', 'checkout', '-b', ROLL_BRANCH_NAME]) |
| |
| # Modify Chromium's DEPS file. |
| |
| # Parse current hashes. |
| deps_filename = os.path.join(self._chromium_src, 'DEPS') |
| deps = _ParseDepsFile(deps_filename) |
| webgl_current = self._GetDepsCommitInfo(deps, WEBGL_PATH) |
| |
| # Find ToT revisions. |
| webgl_latest = self._GetCommitInfo(WEBGL_PATH) |
| |
| if IS_WIN: |
| # Make sure the roll script doesn't use windows line endings |
| self._RunCommand(['git', 'config', 'core.autocrlf', 'true']) |
| |
| self._UpdateDep(deps_filename, WEBGL_PATH, webgl_latest) |
| self._UpdateWebGLRevTextFile(WEBGL_REVISION_TEXT_FILE, webgl_latest) |
| |
| if self._IsTreeClean(): |
| logging.debug('Tree is clean - no changes detected.') |
| self._DeleteRollBranch() |
| else: |
| bugs = self._GetBugList(WEBGL_PATH, webgl_current, webgl_latest) |
| description = _GenerateCLDescriptionCommand( |
| webgl_current, webgl_latest, bugs) |
| logging.debug('Committing changes locally.') |
| self._RunCommand(['git', 'add', '--update', '.']) |
| self._RunCommand(['git', 'commit'] + description) |
| logging.debug('Uploading changes...') |
| self._RunCommand(['git', 'cl', 'upload'], |
| extra_env={'EDITOR': 'true'}) |
| |
| if run_tryjobs: |
| # Kick off tryjobs. |
| base_try_cmd = ['git', 'cl', 'try'] |
| self._RunCommand(base_try_cmd) |
| |
| cl_info = self._GetCLInfo() |
| print 'Issue: %d URL: %s' % (cl_info.issue, cl_info.url) |
| |
| # Checkout master again. |
| self._RunCommand(['git', 'checkout', 'master']) |
| print 'Roll branch left as ' + ROLL_BRANCH_NAME |
| return 0 |
| |
| def _UpdateDep(self, deps_filename, dep_relative_to_src, commit_info): |
| dep_name = _PosixPath(os.path.join('src', dep_relative_to_src)) |
| |
| # roll_dep_svn.py relies on cwd being the Chromium checkout, so let's |
| # temporarily change the working directory and then change back. |
| cwd = os.getcwd() |
| os.chdir(os.path.dirname(deps_filename)) |
| roll_dep_svn.update_deps(deps_filename, dep_relative_to_src, dep_name, |
| commit_info.git_commit, '') |
| os.chdir(cwd) |
| |
| def _UpdateWebGLRevTextFile(self, txt_filename, commit_info): |
| # Rolling the WebGL conformance tests must cause at least all of |
| # the WebGL tests to run. There are already exclusions in |
| # trybot_analyze_config.json which force all tests to run if |
| # changes under src/content/test/gpu are made. (This rule |
| # typically only takes effect on the GPU bots.) To make sure this |
| # happens all the time, update an autogenerated text file in this |
| # directory. |
| with open(txt_filename, 'w') as fh: |
| print >> fh, '# AUTOGENERATED FILE - DO NOT EDIT' |
| print >> fh, '# SEE roll_webgl_conformance.py' |
| print >> fh, 'Current webgl revision %s' % commit_info.git_commit |
| |
| def _DeleteRollBranch(self): |
| self._RunCommand(['git', 'checkout', 'master']) |
| self._RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME]) |
| logging.debug('Deleted the local roll branch (%s)', ROLL_BRANCH_NAME) |
| |
| |
| def _GetBranches(self): |
| """Returns a tuple of active,branches. |
| |
| The 'active' is the name of the currently active branch and 'branches' is a |
| list of all branches. |
| """ |
| lines = self._RunCommand(['git', 'branch']).split('\n') |
| branches = [] |
| active = '' |
| for l in lines: |
| if '*' in l: |
| # The assumption is that the first char will always be the '*'. |
| active = l[1:].strip() |
| branches.append(active) |
| else: |
| b = l.strip() |
| if b: |
| branches.append(b) |
| return (active, branches) |
| |
| def Abort(self): |
| active_branch, branches = self._GetBranches() |
| if active_branch == ROLL_BRANCH_NAME: |
| active_branch = 'master' |
| if ROLL_BRANCH_NAME in branches: |
| print 'Aborting pending roll.' |
| self._RunCommand(['git', 'checkout', ROLL_BRANCH_NAME]) |
| # Ignore an error here in case an issue wasn't created for some reason. |
| self._RunCommand(['git', 'cl', 'set_close'], ignore_exit_code=True) |
| self._RunCommand(['git', 'checkout', active_branch]) |
| self._RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME]) |
| return 0 |
| |
| |
| def main(): |
| parser = argparse.ArgumentParser( |
| description='Auto-generates a CL containing a WebGL conformance roll.') |
| parser.add_argument('--abort', |
| help=('Aborts a previously prepared roll. ' |
| 'Closes any associated issues and deletes the roll branches'), |
| action='store_true') |
| parser.add_argument('--ignore-checks', action='store_true', default=False, |
| help=('Skips checks for being on the master branch, dirty workspaces and ' |
| 'the updating of the checkout. Will still delete and create local ' |
| 'Git branches.')) |
| parser.add_argument('--run-tryjobs', action='store_true', default=False, |
| help=('Start the dry-run tryjobs for the newly generated CL. Use this ' |
| 'when you have no need to make changes to the WebGL conformance ' |
| 'test expectations in the same CL and want to avoid.')) |
| parser.add_argument('-v', '--verbose', action='store_true', default=False, |
| help='Be extra verbose in printing of log messages.') |
| args = parser.parse_args() |
| |
| if args.verbose: |
| logging.basicConfig(level=logging.DEBUG) |
| else: |
| logging.basicConfig(level=logging.ERROR) |
| |
| autoroller = AutoRoller(SRC_DIR) |
| if args.abort: |
| return autoroller.Abort() |
| else: |
| return autoroller.PrepareRoll(args.ignore_checks, args.run_tryjobs) |
| |
| if __name__ == '__main__': |
| sys.exit(main()) |