forked from flutter/flutter
-
Notifications
You must be signed in to change notification settings - Fork 7
/
unpublish_package.dart
525 lines (482 loc) · 19.3 KB
/
unpublish_package.dart
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
/// This script removes published archives from the cloud storage and the
/// corresponding JSON metadata file that the website uses to determine what
/// releases are available.
///
/// If asked to remove a release that is currently the release on that channel,
/// it will replace that release with the next most recent release on that
/// channel.
import 'dart:async';
import 'dart:convert';
import 'dart:io' hide Platform;
import 'package:args/args.dart';
import 'package:path/path.dart' as path;
import 'package:platform/platform.dart' show LocalPlatform, Platform;
import 'package:process/process.dart';
const String gsBase = 'gs://flutter_infra_release';
const String releaseFolder = '/releases';
const String gsReleaseFolder = '$gsBase$releaseFolder';
const String baseUrl = 'https://storage.googleapis.com/flutter_infra_release';
/// Exception class for when a process fails to run, so we can catch
/// it and provide something more readable than a stack trace.
class UnpublishException implements Exception {
UnpublishException(this.message, [this.result]);
final String message;
final ProcessResult? result;
int get exitCode => result?.exitCode ?? -1;
@override
String toString() {
String output = runtimeType.toString();
if (message != null) {
output += ': $message';
}
final String stderr = result?.stderr as String? ?? '';
if (stderr.isNotEmpty) {
output += ':\n$stderr';
}
return output;
}
}
enum Channel { dev, beta, stable }
String getChannelName(Channel channel) {
switch (channel) {
case Channel.beta:
return 'beta';
case Channel.dev:
return 'dev';
case Channel.stable:
return 'stable';
}
}
Channel fromChannelName(String? name) {
switch (name) {
case 'beta':
return Channel.beta;
case 'dev':
return Channel.dev;
case 'stable':
return Channel.stable;
default:
throw ArgumentError('Invalid channel name.');
}
}
enum PublishedPlatform { linux, macos, windows }
String getPublishedPlatform(PublishedPlatform platform) {
switch (platform) {
case PublishedPlatform.linux:
return 'linux';
case PublishedPlatform.macos:
return 'macos';
case PublishedPlatform.windows:
return 'windows';
}
}
PublishedPlatform fromPublishedPlatform(String name) {
switch (name) {
case 'linux':
return PublishedPlatform.linux;
case 'macos':
return PublishedPlatform.macos;
case 'windows':
return PublishedPlatform.windows;
default:
throw ArgumentError('Invalid published platform name.');
}
}
/// A helper class for classes that want to run a process, optionally have the
/// stderr and stdout reported as the process runs, and capture the stdout
/// properly without dropping any.
class ProcessRunner {
/// Creates a [ProcessRunner].
///
/// The [processManager], [subprocessOutput], and [platform] arguments must
/// not be null.
ProcessRunner({
this.processManager = const LocalProcessManager(),
this.subprocessOutput = true,
this.defaultWorkingDirectory,
this.platform = const LocalPlatform(),
}) : assert(subprocessOutput != null),
assert(processManager != null),
assert(platform != null) {
environment = Map<String, String>.from(platform.environment);
}
/// The platform to use for a starting environment.
final Platform platform;
/// Set [subprocessOutput] to show output as processes run. Stdout from the
/// process will be printed to stdout, and stderr printed to stderr.
final bool subprocessOutput;
/// Set the [processManager] in order to inject a test instance to perform
/// testing.
final ProcessManager processManager;
/// Sets the default directory used when `workingDirectory` is not specified
/// to [runProcess].
final Directory? defaultWorkingDirectory;
/// The environment to run processes with.
late Map<String, String> environment;
/// Run the command and arguments in `commandLine` as a sub-process from
/// `workingDirectory` if set, or the [defaultWorkingDirectory] if not. Uses
/// [Directory.current] if [defaultWorkingDirectory] is not set.
///
/// Set `failOk` if [runProcess] should not throw an exception when the
/// command completes with a non-zero exit code.
Future<String> runProcess(
List<String> commandLine, {
Directory? workingDirectory,
bool failOk = false,
}) async {
workingDirectory ??= defaultWorkingDirectory ?? Directory.current;
if (subprocessOutput) {
stderr.write('Running "${commandLine.join(' ')}" in ${workingDirectory.path}.\n');
}
final List<int> output = <int>[];
final Completer<void> stdoutComplete = Completer<void>();
final Completer<void> stderrComplete = Completer<void>();
late Process process;
Future<int> allComplete() async {
await stderrComplete.future;
await stdoutComplete.future;
return process.exitCode;
}
try {
process = await processManager.start(
commandLine,
workingDirectory: workingDirectory.absolute.path,
environment: environment,
);
process.stdout.listen(
(List<int> event) {
output.addAll(event);
if (subprocessOutput) {
stdout.add(event);
}
},
onDone: () async => stdoutComplete.complete(),
);
if (subprocessOutput) {
process.stderr.listen(
(List<int> event) {
stderr.add(event);
},
onDone: () async => stderrComplete.complete(),
);
} else {
stderrComplete.complete();
}
} on ProcessException catch (e) {
final String message = 'Running "${commandLine.join(' ')}" in ${workingDirectory.path} '
'failed with:\n$e';
throw UnpublishException(message);
} on ArgumentError catch (e) {
final String message = 'Running "${commandLine.join(' ')}" in ${workingDirectory.path} '
'failed with:\n$e';
throw UnpublishException(message);
}
final int exitCode = await allComplete();
if (exitCode != 0 && !failOk) {
final String message = 'Running "${commandLine.join(' ')}" in ${workingDirectory.path} failed';
throw UnpublishException(
message,
ProcessResult(0, exitCode, null, 'returned $exitCode'),
);
}
return utf8.decoder.convert(output).trim();
}
}
class ArchiveUnpublisher {
ArchiveUnpublisher(
this.tempDir,
this.revisionsBeingRemoved,
this.channels,
this.platform, {
this.confirmed = false,
ProcessManager? processManager,
bool subprocessOutput = true,
}) : assert(revisionsBeingRemoved.length == 40),
metadataGsPath = '$gsReleaseFolder/${getMetadataFilename(platform)}',
_processRunner = ProcessRunner(
processManager: processManager ?? const LocalProcessManager(),
subprocessOutput: subprocessOutput,
);
final PublishedPlatform platform;
final String metadataGsPath;
final Set<Channel> channels;
final Set<String> revisionsBeingRemoved;
final bool confirmed;
final Directory tempDir;
final ProcessRunner _processRunner;
static String getMetadataFilename(PublishedPlatform platform) => 'releases_${getPublishedPlatform(platform)}.json';
/// Remove the archive from Google Storage.
Future<void> unpublishArchive() async {
final Map<String, dynamic> jsonData = await _loadMetadata();
final List<Map<String, String>> releases = (jsonData['releases'] as List<dynamic>).map<Map<String, String>>((dynamic entry) {
final Map<String, dynamic> mapEntry = entry as Map<String, dynamic>;
return mapEntry.cast<String, String>();
}).toList();
final Map<Channel, Map<String, String>> paths = await _getArchivePaths(releases);
releases.removeWhere((Map<String, String> value) => revisionsBeingRemoved.contains(value['hash']) && channels.contains(fromChannelName(value['channel'])));
releases.sort((Map<String, String> a, Map<String, String> b) {
final DateTime aDate = DateTime.parse(a['release_date']!);
final DateTime bDate = DateTime.parse(b['release_date']!);
return bDate.compareTo(aDate);
});
jsonData['releases'] = releases;
for (final Channel channel in channels) {
if (!revisionsBeingRemoved.contains((jsonData['current_release'] as Map<String, dynamic>)[getChannelName(channel)])) {
// Don't replace the current release if it's not one of the revisions we're removing.
continue;
}
final Map<String, String> replacementRelease = releases.firstWhere((Map<String, String> value) => value['channel'] == getChannelName(channel));
if (replacementRelease == null) {
throw UnpublishException('Unable to find previous release for channel ${getChannelName(channel)}.');
}
(jsonData['current_release'] as Map<String, dynamic>)[getChannelName(channel)] = replacementRelease['hash'];
print(
'${confirmed ? 'Reverting' : 'Would revert'} current ${getChannelName(channel)} '
'${getPublishedPlatform(platform)} release to ${replacementRelease['hash']} (version ${replacementRelease['version']}).'
);
}
await _cloudRemoveArchive(paths);
await _updateMetadata(jsonData);
}
Future<Map<Channel, Map<String, String>>> _getArchivePaths(List<Map<String, String>> releases) async {
final Set<String> hashes = <String>{};
final Map<Channel, Map<String, String>> paths = <Channel, Map<String, String>>{};
for (final Map<String, String> revision in releases) {
final String hash = revision['hash']!;
final Channel channel = fromChannelName(revision['channel']);
hashes.add(hash);
if (revisionsBeingRemoved.contains(hash) && channels.contains(channel)) {
paths[channel] ??= <String, String>{};
paths[channel]![hash] = revision['archive']!;
}
}
final Set<String> missingRevisions = revisionsBeingRemoved.difference(hashes.intersection(revisionsBeingRemoved));
if (missingRevisions.isNotEmpty) {
final bool plural = missingRevisions.length > 1;
throw UnpublishException('Revision${plural ? 's' : ''} $missingRevisions ${plural ? 'are' : 'is'} not present in the server metadata.');
}
return paths;
}
Future<Map<String, dynamic>> _loadMetadata() async {
final File metadataFile = File(
path.join(tempDir.absolute.path, getMetadataFilename(platform)),
);
// Always run this, even in dry runs.
await _runGsUtil(<String>['cp', metadataGsPath, metadataFile.absolute.path], confirm: true);
final String currentMetadata = metadataFile.readAsStringSync();
if (currentMetadata.isEmpty) {
throw UnpublishException('Empty metadata received from server');
}
Map<String, dynamic> jsonData;
try {
jsonData = json.decode(currentMetadata) as Map<String, dynamic>;
} on FormatException catch (e) {
throw UnpublishException('Unable to parse JSON metadata received from cloud: $e');
}
return jsonData;
}
Future<void> _updateMetadata(Map<String, dynamic> jsonData) async {
// We can't just cat the metadata from the server with 'gsutil cat', because
// Windows wants to echo the commands that execute in gsutil.bat to the
// stdout when we do that. So, we copy the file locally and then read it
// back in.
final File metadataFile = File(
path.join(tempDir.absolute.path, getMetadataFilename(platform)),
);
const JsonEncoder encoder = JsonEncoder.withIndent(' ');
metadataFile.writeAsStringSync(encoder.convert(jsonData));
print('${confirmed ? 'Overwriting' : 'Would overwrite'} $metadataGsPath with contents of ${metadataFile.absolute.path}');
await _cloudReplaceDest(metadataFile.absolute.path, metadataGsPath);
}
Future<String> _runGsUtil(
List<String> args, {
Directory? workingDirectory,
bool failOk = false,
bool confirm = false,
}) async {
final List<String> command = <String>['gsutil', '--', ...args];
if (confirm) {
return _processRunner.runProcess(
command,
workingDirectory: workingDirectory,
failOk: failOk,
);
} else {
print('Would run: ${command.join(' ')}');
return '';
}
}
Future<void> _cloudRemoveArchive(Map<Channel, Map<String, String>> paths) async {
final List<String> files = <String>[];
print('${confirmed ? 'Removing' : 'Would remove'} the following release archives:');
for (final Channel channel in paths.keys) {
final Map<String, String> hashes = paths[channel]!;
for (final String hash in hashes.keys) {
final String file = '$gsReleaseFolder/${hashes[hash]}';
files.add(file);
print(' $file');
}
}
await _runGsUtil(<String>['rm', ...files], failOk: true, confirm: confirmed);
}
Future<String> _cloudReplaceDest(String src, String dest) async {
assert(dest.startsWith('gs:'), '_cloudReplaceDest must have a destination in cloud storage.');
assert(!src.startsWith('gs:'), '_cloudReplaceDest must have a local source file.');
// We often don't have permission to overwrite, but
// we have permission to remove, so that's what we do first.
await _runGsUtil(<String>['rm', dest], failOk: true, confirm: confirmed);
String? mimeType;
if (dest.endsWith('.tar.xz')) {
mimeType = 'application/x-gtar';
}
if (dest.endsWith('.zip')) {
mimeType = 'application/zip';
}
if (dest.endsWith('.json')) {
mimeType = 'application/json';
}
final List<String> args = <String>[
// Use our preferred MIME type for the files we care about
// and let gsutil figure it out for anything else.
if (mimeType != null) ...<String>['-h', 'Content-Type:$mimeType'],
...<String>['cp', src, dest],
];
return _runGsUtil(args, confirm: confirmed);
}
}
void _printBanner(String message) {
final String banner = '*** $message ***';
print('\n');
print('*' * banner.length);
print(banner);
print('*' * banner.length);
print('\n');
}
/// Prepares a flutter git repo to be removed from the published cloud storage.
Future<void> main(List<String> rawArguments) async {
final List<String> allowedChannelValues = Channel.values.map<String>((Channel channel) => getChannelName(channel)).toList();
final List<String> allowedPlatformNames = PublishedPlatform.values.map<String>((PublishedPlatform platform) => getPublishedPlatform(platform)).toList();
final ArgParser argParser = ArgParser();
argParser.addOption(
'temp_dir',
help: 'A location where temporary files may be written. Defaults to a '
'directory in the system temp folder. If a temp_dir is not '
'specified, then by default a generated temporary directory will be '
'created, used, and removed automatically when the script exits.',
);
argParser.addMultiOption('revision',
help: 'The Flutter git repo revisions to remove from the published site. '
'Must be full 40-character hashes. More than one may be specified, '
'either by giving the option more than once, or by giving a comma '
'separated list. Required.');
argParser.addMultiOption(
'channel',
allowed: allowedChannelValues,
help: 'The Flutter channels to remove the archives corresponding to the '
'revisions given with --revision. More than one may be specified, '
'either by giving the option more than once, or by giving a '
'comma separated list. If not specified, then the archives from all '
'channels that a revision appears in will be removed.',
);
argParser.addMultiOption(
'platform',
allowed: allowedPlatformNames,
help: 'The Flutter platforms to remove the archive from. May specify more '
'than one, either by giving the option more than once, or by giving a '
'comma separated list. If not specified, then the archives from all '
'platforms that a revision appears in will be removed.',
);
argParser.addFlag(
'confirm',
help: 'If set, will actually remove the archive from Google Cloud Storage '
'upon successful execution of this script. Published archives will be '
'removed from this directory: $baseUrl$releaseFolder. This option '
'must be set to perform any action on the server, otherwise only a dry '
'run is performed.',
);
argParser.addFlag(
'help',
negatable: false,
help: 'Print help for this command.',
);
final ArgResults parsedArguments = argParser.parse(rawArguments);
if (parsedArguments['help'] as bool) {
print(argParser.usage);
exit(0);
}
void errorExit(String message, {int exitCode = -1}) {
stderr.write('Error: $message\n\n');
stderr.write('${argParser.usage}\n');
exit(exitCode);
}
final List<String> revisions = parsedArguments['revision'] as List<String>;
if (revisions.isEmpty) {
errorExit('Invalid argument: at least one --revision must be specified.');
}
for (final String revision in revisions) {
if (revision.length != 40) {
errorExit('Invalid argument: --revision "$revision" must be the entire hash, not just a prefix.');
}
if (revision.contains(RegExp(r'[^a-fA-F0-9]'))) {
errorExit('Invalid argument: --revision "$revision" contains non-hex characters.');
}
}
final String tempDirArg = parsedArguments['temp_dir'] as String;
Directory tempDir;
bool removeTempDir = false;
if (tempDirArg == null || tempDirArg.isEmpty) {
tempDir = Directory.systemTemp.createTempSync('flutter_package.');
removeTempDir = true;
} else {
tempDir = Directory(tempDirArg);
if (!tempDir.existsSync()) {
errorExit("Temporary directory $tempDirArg doesn't exist.");
}
}
if (!(parsedArguments['confirm'] as bool)) {
_printBanner('This will be just a dry run. To actually perform the changes below, re-run with --confirm argument.');
}
final List<String> channelArg = parsedArguments['channel'] as List<String>;
final List<String> channelOptions = channelArg.isNotEmpty ? channelArg : allowedChannelValues;
final Set<Channel> channels = channelOptions.map<Channel>((String value) => fromChannelName(value)).toSet();
final List<String> platformArg = parsedArguments['platform'] as List<String>;
final List<String> platformOptions = platformArg.isNotEmpty ? platformArg : allowedPlatformNames;
final List<PublishedPlatform> platforms = platformOptions.map<PublishedPlatform>((String value) => fromPublishedPlatform(value)).toList();
int exitCode = 0;
late String message;
late String stack;
try {
for (final PublishedPlatform platform in platforms) {
final ArchiveUnpublisher publisher = ArchiveUnpublisher(
tempDir,
revisions.toSet(),
channels,
platform,
confirmed: parsedArguments['confirm'] as bool,
);
await publisher.unpublishArchive();
}
} on UnpublishException catch (e, s) {
exitCode = e.exitCode;
message = e.message;
stack = s.toString();
} catch (e, s) {
exitCode = -1;
message = e.toString();
stack = s.toString();
} finally {
if (removeTempDir) {
tempDir.deleteSync(recursive: true);
}
if (exitCode != 0) {
errorExit('$message\n$stack', exitCode: exitCode);
}
if (!(parsedArguments['confirm'] as bool)) {
_printBanner('This was just a dry run. To actually perform the above changes, re-run with --confirm argument.');
}
exit(0);
}
}