Skip to content

Commit

Permalink
Always use user-level pub cache (#121802)
Browse files Browse the repository at this point in the history 
Use the pub cache resolved by pub itself.
To add packages to the flutter.zip download they are packaged as tar.gz and added to the pub-cache on first run by using  `pub cache preload`.
  • Loading branch information
@sigurdm
sigurdm authored Mar 13, 2023
1 parent 9ba0d08 commit 1fd84f8
Show file tree
Hide file tree
Showing 8 changed files with 160 additions and 256 deletions.
2 changes: 1 addition & 1 deletion .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ analysis_benchmark.json
.flutter-plugins-dependencies
**/generated_plugin_registrant.dart
.packages
.pub-cache/
.pub-preload-cache/
.pub/
build/
flutter_*.png
Expand Down
2 changes: 0 additions & 2 deletions bin/internal/shared.bat
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,6 @@ SET script_path=%flutter_tools_dir%\bin\flutter_tools.dart
SET dart_sdk_path=%cache_dir%\dart-sdk
SET engine_stamp=%cache_dir%\engine-dart-sdk.stamp
SET engine_version_path=%FLUTTER_ROOT%\bin\internal\engine.version
SET pub_cache_path=%FLUTTER_ROOT%\.pub-cache

SET dart=%dart_sdk_path%\bin\dart.exe

REM Ensure that bin/cache exists.
Expand Down
3 changes: 0 additions & 3 deletions bin/internal/shared.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,9 +149,6 @@ function upgrade_flutter () (
export PUB_SUMMARY_ONLY=1
fi
export PUB_ENVIRONMENT="$PUB_ENVIRONMENT:flutter_install"
if [[ -d "$FLUTTER_ROOT/.pub-cache" ]]; then
export PUB_CACHE="${PUB_CACHE:-"$FLUTTER_ROOT/.pub-cache"}"
fi
pub_upgrade_with_retry

# Move the old snapshot - we can't just overwrite it as the VM might currently have it
Expand Down
108 changes: 104 additions & 4 deletions dev/bots/prepare_package.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,13 @@ import 'dart:io' hide Platform;
import 'dart:typed_data';

import 'package:args/args.dart';
import 'package:convert/convert.dart';
import 'package:crypto/crypto.dart';
import 'package:crypto/src/digest_sink.dart';
import 'package:http/http.dart' as http;
import 'package:path/path.dart' as path;
import 'package:platform/platform.dart' show LocalPlatform, Platform;
import 'package:pool/pool.dart';
import 'package:process/process.dart';

const String gobMirror =
Expand Down Expand Up @@ -189,7 +191,7 @@ class ArchiveCreator {
subprocessOutput: subprocessOutput,
platform: platform,
)..environment['PUB_CACHE'] = path.join(
flutterRoot.absolute.path, '.pub-cache',
tempDir.path, '.pub-cache',
);
final String flutterExecutable = path.join(
flutterRoot.absolute.path,
Expand Down Expand Up @@ -426,6 +428,104 @@ class ArchiveCreator {
await _unzipArchive(gitFile, workingDirectory: minGitPath);
}

/// Downloads an archive of every package that is present in the temporary
/// pub-cache from pub.dev. Stores the archives in
/// $flutterRoot/.pub-preload-cache.
///
/// These archives will be installed in the user-level cache on first
/// following flutter command that accesses the cache.
///
/// Precondition: all packages currently in the PUB_CACHE of [_processRunner]
/// are installed from pub.dev.
Future<void> _downloadPubPackageArchives() async {
final Pool pool = Pool(10); // Number of simultaneous downloads.
final http.Client client = http.Client();
final Directory preloadCache = Directory(path.join(flutterRoot.path, '.pub-preload-cache'));
preloadCache.createSync(recursive: true);
/// Fetch a single package.
Future<void> fetchPackageArchive(String name, String version) async {
await pool.withResource(() async {
stderr.write('Fetching package archive for $name-$version.\n');
int retries = 7;
while (true) {
retries-=1;
try {
final Uri packageListingUrl =
Uri.parse('https://pub.dev/api/packages/$name');
// Fetch the package listing to obtain the package download url.
final http.Response packageListingResponse =
await client.get(packageListingUrl);
if (packageListingResponse.statusCode != 200) {
throw Exception('Downloading $packageListingUrl failed. Status code ${packageListingResponse.statusCode}.');
}
final dynamic decodedPackageListing = json.decode(packageListingResponse.body);
if (decodedPackageListing is! Map) {
throw const FormatException('Package listing should be a map');
}
final dynamic versions = decodedPackageListing['versions'];
if (versions is! List) {
throw const FormatException('.versions should be a list');
}
final Map<String, dynamic> versionDescription = versions.firstWhere(
(dynamic description) {
if (description is! Map) {
throw const FormatException('.versions elements should be maps');
}
return description['version'] == version;
},
orElse: () => throw FormatException('Could not find $name-$version in package listing')
) as Map<String, dynamic>;
final dynamic downloadUrl = versionDescription['archive_url'];
if (downloadUrl is! String) {
throw const FormatException('archive_url should be a string');
}
final dynamic archiveSha256 = versionDescription['archive_sha256'];
if (archiveSha256 is! String) {
throw const FormatException('archive_sha256 should be a string');
}
final http.Request request = http.Request('get', Uri.parse(downloadUrl));
final http.StreamedResponse response = await client.send(request);
if (response.statusCode != 200) {
throw Exception('Downloading ${request.url} failed. Status code ${response.statusCode}.');
}
final File archiveFile = File(
path.join(preloadCache.path, '$name-$version.tar.gz'),
);
await response.stream.pipe(archiveFile.openWrite());
final Stream<List<int>> archiveStream = archiveFile.openRead();
final Digest r = await sha256.bind(archiveStream).first;
if (hex.encode(r.bytes) != archiveSha256) {
throw Exception('Hash mismatch of downloaded archive');
}
} on Exception catch (e) {
stderr.write('Failed downloading $name-$version. $e\n');
if (retries > 0) {
stderr.write('Retrying download of $name-$version...');
// Retry.
continue;
} else {
rethrow;
}
}
break;
}
});
}
final Map<String, dynamic> cacheDescription =
json.decode(await _runFlutter(<String>['pub', 'cache', 'list'])) as Map<String, dynamic>;
final Map<String, dynamic> packages = cacheDescription['packages'] as Map<String, dynamic>;
final List<Future<void>> downloads = <Future<void>>[];
for (final MapEntry<String, dynamic> package in packages.entries) {
final String name = package.key;
final Map<String, dynamic> versions = package.value as Map<String, dynamic>;
for (final String version in versions.keys) {
downloads.add(fetchPackageArchive(name, version));
}
}
await Future.wait(downloads);
client.close();
}

/// Prepare the archive repo so that it has all of the caches warmed up and
/// is configured for the user to begin working.
Future<void> _populateCaches() async {
Expand All @@ -446,7 +546,7 @@ class ArchiveCreator {
workingDirectory: tempDir,
);
}

await _downloadPubPackageArchives();
// Yes, we could just skip all .packages files when constructing
// the archive, but some are checked in, and we don't want to skip
// those.
Expand Down Expand Up @@ -795,8 +895,8 @@ class ArchivePublisher {
}
}

/// Prepares a flutter git repo to be packaged up for distribution.
/// It mainly serves to populate the .pub-cache with any appropriate Dart
/// Prepares a flutter git repo to be packaged up for distribution. It mainly
/// serves to populate the .pub-preload-cache with any appropriate Dart
/// packages, and the flutter cache in bin/cache with the appropriate
/// dependencies and snapshots.
///
Expand Down
5 changes: 5 additions & 0 deletions dev/bots/test/prepare_package_test.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,7 @@ void main() {
'$flutter create --template=app ${createBase}app': null,
'$flutter create --template=package ${createBase}package': null,
'$flutter create --template=plugin ${createBase}plugin': null,
'$flutter pub cache list': <ProcessResult>[ProcessResult(0,0,'{"packages":{}}','')],
'git clean -f -x -- **/.packages': null,
'git clean -f -x -- **/.dart_tool/': null,
if (platform.isMacOS) 'codesign -vvvv --check-notarization ${path.join(tempDir.path, 'flutter', 'bin', 'cache', 'dart-sdk', 'bin', 'dart')}': null,
Expand Down Expand Up @@ -180,6 +181,7 @@ void main() {
'$flutter create --template=app ${createBase}app': null,
'$flutter create --template=package ${createBase}package': null,
'$flutter create --template=plugin ${createBase}plugin': null,
'$flutter pub cache list': <ProcessResult>[ProcessResult(0,0,'{"packages":{}}','')],
'git clean -f -x -- **/.packages': null,
'git clean -f -x -- **/.dart_tool/': null,
if (platform.isMacOS) 'codesign -vvvv --check-notarization ${path.join(tempDir.path, 'flutter', 'bin', 'cache', 'dart-sdk', 'bin', 'dart')}': null,
Expand Down Expand Up @@ -228,6 +230,7 @@ void main() {
'$flutter create --template=app ${createBase}app': null,
'$flutter create --template=package ${createBase}package': null,
'$flutter create --template=plugin ${createBase}plugin': null,
'$flutter pub cache list': <ProcessResult>[ProcessResult(0,0,'{"packages":{}}','')],
'git clean -f -x -- **/.packages': null,
'git clean -f -x -- **/.dart_tool/': null,
if (platform.isMacOS) 'codesign -vvvv --check-notarization ${path.join(tempDir.path, 'flutter', 'bin', 'cache', 'dart-sdk', 'bin', 'dart')}': null,
Expand Down Expand Up @@ -286,6 +289,7 @@ void main() {
'$flutter create --template=app ${createBase}app': null,
'$flutter create --template=package ${createBase}package': null,
'$flutter create --template=plugin ${createBase}plugin': null,
'$flutter pub cache list': <ProcessResult>[ProcessResult(0,0,'{"packages":{}}','')],
'git clean -f -x -- **/.packages': null,
'git clean -f -x -- **/.dart_tool/': null,
if (platform.isWindows) 'attrib -h .git': null,
Expand Down Expand Up @@ -336,6 +340,7 @@ void main() {
'$flutter create --template=app ${createBase}app': null,
'$flutter create --template=package ${createBase}package': null,
'$flutter create --template=plugin ${createBase}plugin': null,
'$flutter pub cache list': <ProcessResult>[ProcessResult(0,0,'{"packages":{}}','')],
'git clean -f -x -- **/.packages': null,
'git clean -f -x -- **/.dart_tool/': null,
if (platform.isMacOS) 'codesign -vvvv --check-notarization $binPath': <ProcessResult>[codesignFailure],
Expand Down
Loading

0 comments on commit 1fd84f8

Please sign in to comment.