mseaborn@chromium.org | e3a24f6 | 2011-04-26 02:01:55 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | # Copyright (c) 2011 The Chromium Authors. All rights reserved. |
| 3 | # Use of this source code is governed by a BSD-style license that can be |
| 4 | # found in the LICENSE file. |
| 5 | |
| 6 | import hashlib |
| 7 | import optparse |
| 8 | import os |
| 9 | import urllib2 |
| 10 | import sys |
| 11 | import time |
| 12 | |
| 13 | |
| 14 | # Print a dot every time this number of bytes is read. |
| 15 | PROGRESS_SPACING = 128 * 1024 |
| 16 | |
| 17 | |
| 18 | def ReadFile(filename): |
| 19 | fh = open(filename, 'r') |
| 20 | try: |
| 21 | return fh.read() |
| 22 | finally: |
| 23 | fh.close() |
| 24 | |
| 25 | |
| 26 | def WriteFile(filename, data): |
| 27 | fh = open(filename, 'w') |
| 28 | try: |
| 29 | fh.write(data) |
| 30 | finally: |
| 31 | fh.close() |
| 32 | |
| 33 | |
| 34 | def HashFile(filename): |
| 35 | hasher = hashlib.sha1() |
| 36 | fh = open(filename, 'rb') |
| 37 | try: |
| 38 | while True: |
| 39 | data = fh.read(4096) |
| 40 | if len(data) == 0: |
| 41 | break |
| 42 | hasher.update(data) |
| 43 | finally: |
| 44 | fh.close() |
| 45 | return hasher.hexdigest() |
| 46 | |
| 47 | |
| 48 | def CopyStream(input_stream, output_stream): |
| 49 | """Copies the contents of input_stream to output_stream. Prints |
| 50 | dots to indicate progress. |
| 51 | """ |
| 52 | bytes_read = 0 |
| 53 | dots_printed = 0 |
| 54 | while True: |
| 55 | data = input_stream.read(4096) |
| 56 | if len(data) == 0: |
| 57 | break |
| 58 | output_stream.write(data) |
| 59 | bytes_read += len(data) |
| 60 | if bytes_read / PROGRESS_SPACING > dots_printed: |
| 61 | sys.stdout.write('.') |
| 62 | sys.stdout.flush() |
| 63 | dots_printed += 1 |
| 64 | |
| 65 | |
| 66 | def RenameWithRetry(old_path, new_path): |
| 67 | # Renames of files that have recently been closed are known to be |
| 68 | # unreliable on Windows, because virus checkers like to keep the |
| 69 | # file open for a little while longer. This tends to happen more |
| 70 | # for files that look like Windows executables, which does not apply |
| 71 | # to our files, but we retry the rename here just in case. |
| 72 | if sys.platform in ('win32', 'cygwin'): |
| 73 | for i in range(5): |
| 74 | try: |
bradnelson@google.com | 31e439b | 2011-05-07 00:11:04 | [diff] [blame] | 75 | if os.path.exists(new_path): |
| 76 | os.remove(new_path) |
mseaborn@chromium.org | e3a24f6 | 2011-04-26 02:01:55 | [diff] [blame] | 77 | os.rename(old_path, new_path) |
bradnelson@google.com | 31e439b | 2011-05-07 00:11:04 | [diff] [blame] | 78 | return |
mseaborn@chromium.org | e3a24f6 | 2011-04-26 02:01:55 | [diff] [blame] | 79 | except Exception, exn: |
| 80 | sys.stdout.write('Rename failed with %r. Retrying...\n' % str(exn)) |
| 81 | sys.stdout.flush() |
| 82 | time.sleep(1) |
bradnelson@google.com | 31e439b | 2011-05-07 00:11:04 | [diff] [blame] | 83 | raise Exception('Unabled to rename irt file') |
mseaborn@chromium.org | e3a24f6 | 2011-04-26 02:01:55 | [diff] [blame] | 84 | else: |
| 85 | os.rename(old_path, new_path) |
| 86 | |
| 87 | |
| 88 | def DownloadFile(dest_path, url): |
| 89 | url_path = '%s.url' % dest_path |
| 90 | temp_path = '%s.temp' % dest_path |
| 91 | if os.path.exists(url_path) and ReadFile(url_path).strip() == url: |
| 92 | # The URL matches that of the file we previously downloaded, so |
| 93 | # there should be nothing to do. |
| 94 | return |
| 95 | sys.stdout.write('Downloading %r to %r\n' % (url, dest_path)) |
| 96 | output_fh = open(temp_path, 'wb') |
| 97 | stream = urllib2.urlopen(url) |
| 98 | CopyStream(stream, output_fh) |
| 99 | output_fh.close() |
| 100 | sys.stdout.write(' done\n') |
| 101 | if os.path.exists(url_path): |
| 102 | os.unlink(url_path) |
| 103 | RenameWithRetry(temp_path, dest_path) |
| 104 | WriteFile(url_path, url + '\n') |
| 105 | stream.close() |
| 106 | |
| 107 | |
| 108 | def DownloadFileWithRetry(dest_path, url): |
| 109 | for i in range(5): |
| 110 | try: |
| 111 | DownloadFile(dest_path, url) |
| 112 | break |
| 113 | except urllib2.HTTPError, exn: |
| 114 | if exn.getcode() == 404: |
| 115 | raise |
| 116 | sys.stdout.write('Download failed with error %r. Retrying...\n' |
| 117 | % str(exn)) |
| 118 | sys.stdout.flush() |
| 119 | time.sleep(1) |
| 120 | |
| 121 | |
mseaborn@chromium.org | 0ee0d69 | 2011-05-10 21:42:58 | [diff] [blame] | 122 | def EvalDepsFile(path): |
| 123 | scope = {'Var': lambda name: scope['vars'][name]} |
| 124 | execfile(path, {}, scope) |
| 125 | return scope |
| 126 | |
| 127 | |
mseaborn@chromium.org | e3a24f6 | 2011-04-26 02:01:55 | [diff] [blame] | 128 | def Main(): |
| 129 | parser = optparse.OptionParser() |
| 130 | parser.add_option( |
| 131 | '--base_url', dest='base_url', |
| 132 | # For a view of this site that includes directory listings, see: |
| 133 | # http://gsdview.appspot.com/nativeclient-archive2/ |
| 134 | # (The trailing slash is required.) |
| 135 | default=('http://commondatastorage.googleapis.com/' |
| 136 | 'nativeclient-archive2/irt'), |
| 137 | help='Base URL from which to download.') |
| 138 | parser.add_option( |
| 139 | '--nacl_revision', dest='nacl_revision', |
| 140 | help='Download an IRT binary that was built from this ' |
| 141 | 'SVN revision of Native Client.') |
| 142 | parser.add_option( |
| 143 | '--file_hash', dest='file_hashes', action='append', nargs=2, default=[], |
| 144 | metavar='ARCH HASH', |
| 145 | help='ARCH gives the name of the architecture (e.g. "x86_32") for ' |
| 146 | 'which to download an IRT binary. ' |
| 147 | 'HASH gives the expected SHA1 hash of the file.') |
| 148 | options, args = parser.parse_args() |
| 149 | if len(args) != 0: |
| 150 | parser.error('Unexpected arguments: %r' % args) |
| 151 | |
mseaborn@chromium.org | 0ee0d69 | 2011-05-10 21:42:58 | [diff] [blame] | 152 | if options.nacl_revision is None and len(options.file_hashes) == 0: |
| 153 | # The script must have been invoked directly with no arguments, |
| 154 | # rather than being invoked by gclient. In this case, read the |
| 155 | # DEPS file ourselves rather than having gclient pass us values |
| 156 | # from DEPS. |
| 157 | deps_data = EvalDepsFile(os.path.join('src', 'DEPS')) |
| 158 | options.nacl_revision = deps_data['vars']['nacl_revision'] |
| 159 | options.file_hashes = [ |
| 160 | ('x86_32', deps_data['vars']['nacl_irt_hash_x86_32']), |
| 161 | ('x86_64', deps_data['vars']['nacl_irt_hash_x86_64']), |
| 162 | ] |
| 163 | |
mseaborn@chromium.org | e3a24f6 | 2011-04-26 02:01:55 | [diff] [blame] | 164 | nacl_dir = os.path.join('src', 'native_client') |
| 165 | if not os.path.exists(nacl_dir): |
| 166 | # If "native_client" is not present, this might be because the |
| 167 | # developer has put '"src/native_client": None' in their |
| 168 | # '.gclient' file, because they don't want to build Chromium with |
| 169 | # Native Client support. So don't create 'src/native_client', |
| 170 | # because that would interfere with checking it out from SVN |
| 171 | # later. |
| 172 | sys.stdout.write( |
| 173 | 'The directory %r does not exist: skipping downloading binaries ' |
mseaborn@chromium.org | c353642 | 2011-05-09 15:35:04 | [diff] [blame] | 174 | 'for Native Client\'s IRT library\n' % nacl_dir) |
mseaborn@chromium.org | e3a24f6 | 2011-04-26 02:01:55 | [diff] [blame] | 175 | return |
| 176 | if len(options.file_hashes) == 0: |
| 177 | sys.stdout.write('No --file_hash arguments given: nothing to update\n') |
| 178 | |
| 179 | new_deps = [] |
| 180 | for arch, expected_hash in options.file_hashes: |
| 181 | url = '%s/r%s/irt_%s.nexe' % (options.base_url, |
| 182 | options.nacl_revision, |
| 183 | arch) |
| 184 | dest_dir = os.path.join(nacl_dir, 'irt_binaries') |
| 185 | if not os.path.exists(dest_dir): |
| 186 | os.makedirs(dest_dir) |
mseaborn@chromium.org | b4e6d2db | 2011-04-28 00:32:50 | [diff] [blame] | 187 | dest_path = os.path.join(dest_dir, 'nacl_irt_%s.nexe' % arch) |
mseaborn@chromium.org | e3a24f6 | 2011-04-26 02:01:55 | [diff] [blame] | 188 | DownloadFileWithRetry(dest_path, url) |
| 189 | downloaded_hash = HashFile(dest_path) |
| 190 | if downloaded_hash != expected_hash: |
| 191 | sys.stdout.write( |
| 192 | 'Hash mismatch: the file downloaded from URL %r had hash %r, ' |
| 193 | 'but we expected %r\n' % (url, downloaded_hash, expected_hash)) |
| 194 | new_deps.append(' "nacl_irt_hash_%s": "%s",\n' |
| 195 | % (arch, downloaded_hash)) |
| 196 | |
| 197 | if len(new_deps) > 0: |
| 198 | sys.stdout.write('\nIf you have changed nacl_revision, the DEPS file ' |
| 199 | 'probably needs to be updated with the following:\n%s\n' |
| 200 | % ''.join(new_deps)) |
| 201 | sys.exit(1) |
| 202 | |
| 203 | |
| 204 | if __name__ == '__main__': |
| 205 | Main() |