diff --git a/CHANGELOG.md b/CHANGELOG.md index ff091eb4d29..0784d362393 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,8 @@ ## Master -### New features +### ✨ New features + - [core] Port line-sort-key and fill-sort-key ([#15839](https://github.com/mapbox/mapbox-gl-native/pull/15839)) The new feature allows to sort line and fill layer features. Similar to `symbol-sort-key`. @@ -27,7 +28,8 @@ This patch introduces batch conversion between LatLng and ScreenCoordinate in Gl-Native core, so for multiple conversions with single point/latLng previously now it can be done with invoking one function call by passing vector of points/latLngs. -### Bug fixes +### 🐞 Bug fixes + - [core] Stable position of labels at tile borders in tile mode ([#16040](https://github.com/mapbox/mapbox-gl-native/pull/16040)) These changes allow to avoid cutting-off labels on tile borders if the variable text placement is enabled. @@ -78,7 +80,8 @@ This fixes rendering by account for the 1px texture padding around icons that were stretched with icon-text-fit. -### Performance improvements +### 🏁 Performance improvements + - [core] Calculate GeoJSON tile geometries in a background thread ([#15953](https://github.com/mapbox/mapbox-gl-native/pull/15953)) Call `mapbox::geojsonvt::GeoJSONVT::getTile()` in a background thread, so that the rendering thread is not blocked. @@ -97,11 +100,22 @@ Before this fix, repeated request for an already obtained image was erroneously treated as pending, and it prevented from the tiles load completion. -### Architectural changes -- [core] Remove Map::cycleDebugOptions ([#16005](https://github.com/mapbox/mapbox-gl-native/pull/16005)) - - This function was mostly used by the Android API, which is no longer necessary. +### 🧩 Architectural changes - [core] Merge style::Layer::set{Layout,Paint}Property ([#15997](https://github.com/mapbox/mapbox-gl-native/pull/15997)) - [core] Use expected.hpp from mapbox-base ([#15898](https://github.com/mapbox/mapbox-gl-native/pull/15898)) + +##### ⚠️ Breaking changes + +- [core] Refactor DefaultFileSource codebase ([#15768](https://github.com/mapbox/mapbox-gl-native/pull/15768)) + - Adds `FileSourceManager` interface that provides access to `FileSource` instances and means of registering / unregistering `FileSource` factories + - Splits `DefaultFileSource` into smaller parts + - Adds `DatabaseFileSource` interface and it's default implementation + - Removes inter-dependencies between concrete `FileSource` classes + - All sources operate on dedicated thread, except `MainResourceLoader` that acts as a dispatcher and works on thread that requested it. + - Removes `ResourceOptions::withCacheOnlyRequestsSupport` method + +- [core] Remove Map::cycleDebugOptions ([#16005](https://github.com/mapbox/mapbox-gl-native/pull/16005)) + + This function was mostly used by the Android API, which is no longer necessary. diff --git a/bin/cache.cpp b/bin/cache.cpp index eee1d61b325..1df782f7523 100644 --- a/bin/cache.cpp +++ b/bin/cache.cpp @@ -1,5 +1,6 @@ -#include +#include #include +#include #include #include @@ -89,9 +90,9 @@ int main(int argc, char* argv[]) { } mbgl::util::RunLoop loop; - mbgl::DefaultFileSource fileSource(args::get(cacheValue), "."); - - fileSource.put(resource, response); - + auto dbfs = mbgl::FileSourceManager::get()->getFileSource( + mbgl::FileSourceType::Database, mbgl::ResourceOptions().withCachePath(args::get(cacheValue))); + dbfs->forward(resource, response, [&loop] { loop.stop(); }); + loop.run(); return 0; } diff --git a/bin/offline.cpp b/bin/offline.cpp index fcc6adc3efc..c18fc31810f 100644 --- a/bin/offline.cpp +++ b/bin/offline.cpp @@ -3,7 +3,9 @@ #include #include -#include +#include +#include +#include #include @@ -161,17 +163,16 @@ int main(int argc, char *argv[]) { util::RunLoop loop; - DefaultFileSource fileSource(output, "."); - std::unique_ptr region; + std::shared_ptr fileSource = + std::static_pointer_cast(FileSourceManager::get()->getFileSource( + FileSourceType::Database, + ResourceOptions().withAccessToken(token).withBaseURL(apiBaseURL).withCachePath(output))); - fileSource.setAccessToken(token); - fileSource.setAPIBaseURL(apiBaseURL); + std::unique_ptr region; if (inputDb && mergePath) { - DefaultFileSource inputSource(*inputDb, "."); - inputSource.setAccessToken(token); - inputSource.setAPIBaseURL(apiBaseURL); - + DatabaseFileSource inputSource(ResourceOptions().withCachePath(*inputDb)); + int retCode = 0; std::cout << "Start Merge" << std::endl; inputSource.mergeOfflineRegions(*mergePath, [&] (mbgl::expected, std::exception_ptr> result) { @@ -193,13 +194,15 @@ int main(int argc, char *argv[]) { class Observer : public OfflineRegionObserver { public: - Observer(OfflineRegion& region_, DefaultFileSource& fileSource_, util::RunLoop& loop_, mbgl::optional mergePath_) + Observer(OfflineRegion& region_, + std::shared_ptr fileSource_, + util::RunLoop& loop_, + mbgl::optional mergePath_) : region(region_), - fileSource(fileSource_), + fileSource(std::move(fileSource_)), loop(loop_), mergePath(std::move(mergePath_)), - start(util::now()) { - } + start(util::now()) {} void statusChanged(OfflineRegionStatus status) override { if (status.downloadState == OfflineRegionDownloadState::Inactive) { @@ -215,14 +218,11 @@ int main(int argc, char *argv[]) { bytesPerSecond = util::toString(status.completedResourceSize / elapsedSeconds); } - std::cout << status.completedResourceCount << " / " << status.requiredResourceCount - << " resources" - << status.completedTileCount << " / " << status.requiredTileCount - << "tiles" - << (status.requiredResourceCountIsPrecise ? "; " : " (indeterminate); ") + std::cout << status.completedResourceCount << " / " << status.requiredResourceCount << " resources | " + << status.completedTileCount << " / " << status.requiredTileCount << " tiles" + << (status.requiredResourceCountIsPrecise ? " | " : " (indeterminate); ") << status.completedResourceSize << " bytes downloaded" - << " (" << bytesPerSecond << " bytes/sec)" - << std::endl; + << " (" << bytesPerSecond << " bytes/sec)" << std::endl; if (status.complete()) { std::cout << "Finished Download" << std::endl; @@ -239,7 +239,7 @@ int main(int argc, char *argv[]) { } OfflineRegion& region; - DefaultFileSource& fileSource; + std::shared_ptr fileSource; util::RunLoop& loop; mbgl::optional mergePath; Timestamp start; @@ -248,24 +248,26 @@ int main(int argc, char *argv[]) { static auto stop = [&] { if (region) { std::cout << "Stopping download... "; - fileSource.setOfflineRegionDownloadState(*region, OfflineRegionDownloadState::Inactive); + fileSource->setOfflineRegionDownloadState(*region, OfflineRegionDownloadState::Inactive); } }; std::signal(SIGINT, [] (int) { stop(); }); - fileSource.createOfflineRegion(definition, metadata, [&] (mbgl::expected region_) { - if (!region_) { - std::cerr << "Error creating region: " << util::toString(region_.error()) << std::endl; - loop.stop(); - exit(1); - } else { - assert(region_); - region = std::make_unique(std::move(*region_)); - fileSource.setOfflineRegionObserver(*region, std::make_unique(*region, fileSource, loop, mergePath)); - fileSource.setOfflineRegionDownloadState(*region, OfflineRegionDownloadState::Active); - } - }); + fileSource->createOfflineRegion( + definition, metadata, [&](mbgl::expected region_) { + if (!region_) { + std::cerr << "Error creating region: " << util::toString(region_.error()) << std::endl; + loop.stop(); + exit(1); + } else { + assert(region_); + region = std::make_unique(std::move(*region_)); + fileSource->setOfflineRegionObserver(*region, + std::make_unique(*region, fileSource, loop, mergePath)); + fileSource->setOfflineRegionDownloadState(*region, OfflineRegionDownloadState::Active); + } + }); loop.run(); return 0; diff --git a/include/mbgl/storage/default_file_source.hpp b/include/mbgl/storage/database_file_source.hpp similarity index 67% rename from include/mbgl/storage/default_file_source.hpp rename to include/mbgl/storage/database_file_source.hpp index fdd430ccadd..8ccb5ce39b7 100644 --- a/include/mbgl/storage/default_file_source.hpp +++ b/include/mbgl/storage/database_file_source.hpp @@ -1,45 +1,128 @@ #pragma once -#include #include #include -#include -#include #include - -#include -#include +#include namespace mbgl { -namespace util { -template class Thread; -} // namespace util +class ResourceOptions; -class ResourceTransform; +// TODO: Split DatabaseFileSource into Ambient cache and Database interfaces. +class DatabaseFileSource : public FileSource { +public: + explicit DatabaseFileSource(const ResourceOptions& options); + ~DatabaseFileSource() override; -// TODO: the callback should include a potential error info when https://github.com/mapbox/mapbox-gl-native/issues/14759 is resolved -using PathChangeCallback = std::function; + // FileSource overrides + std::unique_ptr request(const Resource&, Callback) override; + void forward(const Resource&, const Response&, std::function callback) override; + bool canRequest(const Resource&) const override; + void setProperty(const std::string&, const mapbox::base::Value&) override; -class DefaultFileSource : public FileSource { -public: - DefaultFileSource(const std::string& cachePath, const std::string& assetPath, bool supportCacheOnlyRequests = true); - DefaultFileSource(const std::string& cachePath, std::unique_ptr&& assetFileSource, bool supportCacheOnlyRequests = true); - ~DefaultFileSource() override; + // Methods common to Ambient cache and Offline functionality - bool supportsCacheOnlyRequests() const override; + /* + * Sets path of a database to be used by DatabaseFileSource and invokes provided + * callback when a database path is set. + */ + virtual void setDatabasePath(const std::string&, std::function callback); - void setAPIBaseURL(const std::string&); - std::string getAPIBaseURL(); + /* + * Delete existing database and re-initialize. + * + * When the operation is complete or encounters an error, the given callback will be + * executed on the database thread; it is the responsibility of the SDK bindings + * to re-execute a user-provided callback on the main thread. + */ + virtual void resetDatabase(std::function); - void setAccessToken(const std::string&); - std::string getAccessToken(); + /* + * Packs the existing database file into a minimal amount of disk space. + * + * This operation has a performance impact as it will vacuum the database, + * forcing it to move pages on the filesystem. + * + * When the operation is complete or encounters an error, the given callback will be + * executed on the database thread; it is the responsibility of the SDK bindings + * to re-execute a user-provided callback on the main thread. + */ + virtual void packDatabase(std::function callback); - void setResourceTransform(optional>&&); + /* + * Sets whether packing the database file occurs automatically after an offline + * region is deleted (deleteOfflineRegion()) or the ambient cache is cleared + * (clearAmbientCache()). + * + * By default, packing is enabled. If disabled, disk space will not be freed + * after resources are removed unless packDatabase() is explicitly called. + */ + virtual void runPackDatabaseAutomatically(bool); - void setResourceCachePath(const std::string&, optional>&&); + // Ambient cache - std::unique_ptr request(const Resource&, Callback) override; + /* + * Insert the provided resource into the ambient cache + * + * Consumers of the resource will expect the uncompressed version; the + * OfflineDatabase will determine whether to compress the data on disk. + * This call is asynchronous: the data may not be immediately available + * for in-progress requests, although subsequent requests should have + * access to the cached data. + */ + virtual void put(const Resource&, const Response&); + + /* + * Forces revalidation of the ambient cache. + * + * Forces Mapbox GL Native to revalidate resources stored in the ambient + * cache with the tile server before using them, making sure they + * are the latest version. This is more efficient than cleaning the + * cache because if the resource is considered valid after the server + * lookup, it will not get downloaded again. + * + * Resources overlapping with offline regions will not be affected + * by this call. + */ + virtual void invalidateAmbientCache(std::function); + + /* + * Erase resources from the ambient cache, freeing storage space. + * + * Erases the ambient cache, freeing resources. + * + * Note that this operation can be potentially slow if packing the database + * occurs automatically (see runPackDatabaseAutomatically() and packDatabase()). + * + * Resources overlapping with offline regions will not be affected + * by this call. + */ + virtual void clearAmbientCache(std::function); + + /* + * Sets the maximum size in bytes for the ambient cache. + * + * This call is potentially expensive because it will try + * to trim the data in case the database is larger than the + * size defined. The size of offline regions are not affected + * by this settings, but the ambient cache will always try + * to not exceed the maximum size defined, taking into account + * the current size for the offline regions. + * + * If the maximum size is set to 50 MB and 40 MB are already + * used by offline regions, the cache size will be effectively + * 10 MB. + * + * Setting the size to 0 will disable the cache if there is no + * offline region on the database. + * + * This method should always be called before using the database, + * otherwise the default maximum size will be used. + */ + virtual void setMaximumAmbientCacheSize(uint64_t size, std::function callback); + + // Offline /* * Retrieve all regions in the offline database. @@ -48,7 +131,7 @@ class DefaultFileSource : public FileSource { * callback, which will be executed on the database thread; it is the responsibility * of the SDK bindings to re-execute a user-provided callback on the main thread. */ - void listOfflineRegions(std::function)>); + virtual void listOfflineRegions(std::function)>); /* * Create an offline region in the database. @@ -61,25 +144,25 @@ class DefaultFileSource : public FileSource { * downloading resources, call `setOfflineRegionDownloadState(OfflineRegionDownloadState::Active)`, * optionally registering an `OfflineRegionObserver` beforehand. */ - void createOfflineRegion(const OfflineRegionDefinition& definition, - const OfflineRegionMetadata& metadata, - std::function)>); - + virtual void createOfflineRegion(const OfflineRegionDefinition& definition, + const OfflineRegionMetadata& metadata, + std::function)>); /* * Update an offline region metadata in the database. */ - void updateOfflineMetadata(const int64_t regionID, - const OfflineRegionMetadata& metadata, - std::function)>); + virtual void updateOfflineMetadata(const int64_t regionID, + const OfflineRegionMetadata& metadata, + std::function)>); + /* * Register an observer to be notified when the state of the region changes. */ - void setOfflineRegionObserver(OfflineRegion&, std::unique_ptr); + virtual void setOfflineRegionObserver(OfflineRegion&, std::unique_ptr); /* * Pause or resume downloading of regional resources. */ - void setOfflineRegionDownloadState(OfflineRegion&, OfflineRegionDownloadState); + virtual void setOfflineRegionDownloadState(OfflineRegion&, OfflineRegionDownloadState); /* * Retrieve the current status of the region. The query will be executed @@ -87,9 +170,8 @@ class DefaultFileSource : public FileSource { * executed on the database thread; it is the responsibility of the SDK bindings * to re-execute a user-provided callback on the main thread. */ - void getOfflineRegionStatus( - OfflineRegion&, - std::function)>) const; + virtual void getOfflineRegionStatus(OfflineRegion&, + std::function)>) const; /* * Merge offline regions from a secondary database into the main offline database. @@ -111,8 +193,8 @@ class DefaultFileSource : public FileSource { * Merged regions may not be in a completed status if the secondary database * does not contain all the tiles or resources required by the region definition. */ - void mergeOfflineRegions(const std::string& sideDatabasePath, - std::function)>); + virtual void mergeOfflineRegions(const std::string& sideDatabasePath, + std::function)>); /* * Remove an offline region from the database and perform any resources evictions @@ -132,7 +214,7 @@ class DefaultFileSource : public FileSource { * executed on the database thread; it is the responsibility of the SDK bindings * to re-execute a user-provided callback on the main thread. */ - void deleteOfflineRegion(OfflineRegion&&, std::function); + virtual void deleteOfflineRegion(OfflineRegion, std::function); /* * Invalidate all the tiles from an offline region forcing Mapbox GL to revalidate @@ -140,140 +222,17 @@ class DefaultFileSource : public FileSource { * offline region and downloading it again because if the data on the cache matches * the server, no new data gets transmitted. */ - void invalidateOfflineRegion(OfflineRegion&, std::function); + virtual void invalidateOfflineRegion(OfflineRegion&, std::function); /* * Changing or bypassing this limit without permission from Mapbox is prohibited * by the Mapbox Terms of Service. */ - void setOfflineMapboxTileCountLimit(uint64_t) const; - - /* - * Pause file request activity. - * - * If pause is called then no revalidation or network request activity - * will occur. - */ - void pause(); - - /* - * Resume file request activity. - * - * Calling resume will unpause the file source and process any tasks that - * expired while the file source was paused. - */ - void resume(); - - /* - * Insert the provided resource into the ambient cache - * - * Consumers of the resource will expect the uncompressed version; the - * OfflineDatabase will determine whether to compress the data on disk. - * This call is asynchronous: the data may not be immediately available - * for in-progress requests, although subsequent requests should have - * access to the cached data. - */ - void put(const Resource&, const Response&); - - /* - * Delete existing database and re-initialize. - * - * When the operation is complete or encounters an error, the given callback will be - * executed on the database thread; it is the responsibility of the SDK bindings - * to re-execute a user-provided callback on the main thread. - */ - void resetDatabase(std::function); - - /* - * Packs the existing database file into a minimal amount of disk space. - * - * This operation has a performance impact as it will vacuum the database, - * forcing it to move pages on the filesystem. - * - * When the operation is complete or encounters an error, the given callback will be - * executed on the database thread; it is the responsibility of the SDK bindings - * to re-execute a user-provided callback on the main thread. - */ - void packDatabase(std::function callback); - - /* - * Sets whether packing the database file occurs automatically after an offline - * region is deleted (deleteOfflineRegion()) or the ambient cache is cleared - * (clearAmbientCache()). - * - * By default, packing is enabled. If disabled, disk space will not be freed - * after resources are removed unless packDatabase() is explicitly called. - */ - void runPackDatabaseAutomatically(bool); - - /* - * Forces revalidation of the ambient cache. - * - * Forces Mapbox GL Native to revalidate resources stored in the ambient - * cache with the tile server before using them, making sure they - * are the latest version. This is more efficient than cleaning the - * cache because if the resource is considered valid after the server - * lookup, it will not get downloaded again. - * - * Resources overlapping with offline regions will not be affected - * by this call. - */ - void invalidateAmbientCache(std::function); - - /* - * Erase resources from the ambient cache, freeing storage space. - * - * Erases the ambient cache, freeing resources. - * - * Note that this operation can be potentially slow if packing the database - * occurs automatically (see runPackDatabaseAutomatically() and packDatabase()). - * - * Resources overlapping with offline regions will not be affected - * by this call. - */ - void clearAmbientCache(std::function); - - /* - * Sets the maximum size in bytes for the ambient cache. - * - * This call is potentially expensive because it will try - * to trim the data in case the database is larger than the - * size defined. The size of offline regions are not affected - * by this settings, but the ambient cache will always try - * to not exceed the maximum size defined, taking into account - * the current size for the offline regions. - * - * If the maximum size is set to 50 MB and 40 MB are already - * used by offline regions, the cache size will be effectively - * 10 MB. - * - * Setting the size to 0 will disable the cache if there is no - * offline region on the database. - * - * This method should always be called before using the database, - * otherwise the default maximum size will be used. - */ - void setMaximumAmbientCacheSize(uint64_t size, std::function callback); - - // For testing only. - void setOnlineStatus(bool); - void reopenDatabaseReadOnlyForTesting(); - void setMaximumConcurrentRequests(uint32_t); - - class Impl; + virtual void setOfflineMapboxTileCountLimit(uint64_t) const; private: - // Shared so destruction is done on this thread - const std::shared_ptr assetFileSource; - const std::unique_ptr> impl; - - std::mutex cachedBaseURLMutex; - std::string cachedBaseURL = mbgl::util::API_BASE_URL; - - std::mutex cachedAccessTokenMutex; - std::string cachedAccessToken; - - const bool supportCacheOnlyRequests; + class Impl; + const std::unique_ptr impl; }; } // namespace mbgl diff --git a/include/mbgl/storage/file_source.hpp b/include/mbgl/storage/file_source.hpp index 2270038c49a..f05f1473e92 100644 --- a/include/mbgl/storage/file_source.hpp +++ b/include/mbgl/storage/file_source.hpp @@ -1,19 +1,34 @@ #pragma once -#include #include -#include -#include +#include #include #include namespace mbgl { -class ResourceOptions; -class ResourceTransform; +class AsyncRequest; +class Resource; +// TODO: Rename to ResourceProviderType +enum FileSourceType : uint8_t { + Asset, + // TODO: split to separate types + // - Cache for fast KV store (FASTER, LevelDB, RocksDB) + // - Database for read-only offline use-cases + Database, + FileSystem, + Network, + // Resource loader acts as a proxy and has logic + // for request delegation to Asset, Cache, and other + // file sources. + ResourceLoader +}; + +// TODO: Rename to ResourceProvider to avoid confusion with +// GeoJSONSource, RasterSource, VectorSource, CustomGeometrySource and other *Sources. class FileSource { public: FileSource(const FileSource&) = delete; @@ -29,22 +44,44 @@ class FileSource { // not be executed. virtual std::unique_ptr request(const Resource&, Callback) = 0; + // Allows to forward response from one source to another. + // Optionally, callback can be provided to receive notification for forward + // operation. + virtual void forward(const Resource&, const Response&, std::function = {}) {} + // When a file source supports consulting a local cache only, it must return true. // Cache-only requests are requests that aren't as urgent, but could be useful, e.g. // to cover part of the map while loading. The FileSource should only do cheap actions to // retrieve the data, e.g. load it from a cache, but not from the internet. - virtual bool supportsCacheOnlyRequests() const { - return false; - } + virtual bool supportsCacheOnlyRequests() const { return false; } + + // Checks whether a resource could be requested from this file source. + virtual bool canRequest(const Resource&) const = 0; + + /* + * Pause file request activity. + * + * If pause is called then no revalidation or network request activity + * will occur. + */ + virtual void pause() {} + + /* + * Resume file request activity. + * + * Calling resume will unpause the file source and process any tasks that + * expired while the file source was paused. + */ + virtual void resume() {} - // Singleton for obtaining the shared platform-specific file source. A single instance of a file source is provided - // for each unique combination of a Mapbox API base URL, access token, cache path and platform context. - static std::shared_ptr getSharedFileSource(const ResourceOptions&); + /* + * Generic property setter / getter methods. + */ + virtual void setProperty(const std::string&, const mapbox::base::Value&){}; + virtual mapbox::base::Value getProperty(const std::string&) const { return {}; }; protected: FileSource() = default; - // Factory for creating a platform-specific file source. - static std::shared_ptr createPlatformFileSource(const ResourceOptions&); }; } // namespace mbgl diff --git a/include/mbgl/storage/file_source_manager.hpp b/include/mbgl/storage/file_source_manager.hpp new file mode 100644 index 00000000000..2b2a43cbeca --- /dev/null +++ b/include/mbgl/storage/file_source_manager.hpp @@ -0,0 +1,50 @@ +#pragma once + +#include + +namespace mbgl { + +class ResourceOptions; + +/** + * @brief A singleton class responsible for managing file sources. + * + * The FileSourceManager provides following functionality: + * + * - provides access to file sources of a specific type and configuration + * - caches previously created file sources of a (type, configuration) tuples + * - allows to register and unregister file source factories + */ +class FileSourceManager { +public: + using FileSourceFactory = std::function(const ResourceOptions&)>; + + /** + * @brief A singleton getter. + * + * @return FileSourceManager* + */ + static FileSourceManager* get() noexcept; + + // Returns shared instance of a file source for (type, options) tuple. + // Creates new instance via registered factory if needed. If new instance cannot be + // created, nullptr would be returned. + std::shared_ptr getFileSource(FileSourceType, const ResourceOptions&) noexcept; + + // Registers file source factory for a provided FileSourceType type. If factory for the + // same type was already registered, will unregister previously registered factory. + // Provided factory must not be null. + virtual void registerFileSourceFactory(FileSourceType, FileSourceFactory&&) noexcept; + + // Unregisters file source factory. If there are no registered factories for a FileSourceType + // invocation has no effect. + virtual FileSourceFactory unRegisterFileSourceFactory(FileSourceType) noexcept; + +protected: + FileSourceManager(); + class Impl; + std::unique_ptr impl; + virtual ~FileSourceManager(); +}; + +} // namespace mbgl diff --git a/include/mbgl/storage/online_file_source.hpp b/include/mbgl/storage/online_file_source.hpp index b2e9b43e5d1..8969d218714 100644 --- a/include/mbgl/storage/online_file_source.hpp +++ b/include/mbgl/storage/online_file_source.hpp @@ -1,42 +1,46 @@ #pragma once -#include #include -#include #include namespace mbgl { class ResourceTransform; +// Properties that may be supported by online file sources. + +// Property name to set / get an access token. +// type: std::string +constexpr const char* ACCESS_TOKEN_KEY = "access-token"; + +// Property name to set / get base url. +// type: std::string +constexpr const char* API_BASE_URL_KEY = "api-base-url"; + +// Property name to set / get maximum number of concurrent requests. +// type: unsigned +constexpr const char* MAX_CONCURRENT_REQUESTS_KEY = "max-concurrent-requests"; + class OnlineFileSource : public FileSource { public: OnlineFileSource(); ~OnlineFileSource() override; - void setAPIBaseURL(const std::string& t) { apiBaseURL = t; } - std::string getAPIBaseURL() const { return apiBaseURL; } - - void setAccessToken(const std::string& t) { accessToken = t; } - std::string getAccessToken() const { return accessToken; } - - void setResourceTransform(optional>&&); - + // FileSource overrides std::unique_ptr request(const Resource&, Callback) override; + bool canRequest(const Resource&) const override; + void pause() override; + void resume() override; + void setProperty(const std::string&, const mapbox::base::Value&) override; + mapbox::base::Value getProperty(const std::string&) const override; - void setMaximumConcurrentRequests(uint32_t); - uint32_t getMaximumConcurrentRequests() const; - - // For testing only. - void setOnlineStatus(bool); + // OnlineFileSource interface. + // TODO: Would be nice to drop it to get uniform interface. + virtual void setResourceTransform(ResourceTransform); private: - friend class OnlineFileRequest; - class Impl; const std::unique_ptr impl; - std::string accessToken; - std::string apiBaseURL = mbgl::util::API_BASE_URL; }; } // namespace mbgl diff --git a/include/mbgl/storage/resource.hpp b/include/mbgl/storage/resource.hpp index d00f336669e..b21265ef54c 100644 --- a/include/mbgl/storage/resource.hpp +++ b/include/mbgl/storage/resource.hpp @@ -64,7 +64,7 @@ class Resource { void setPriority(Priority p) { priority = p; } void setUsage(Usage u) { usage = u; } - bool hasLoadingMethod(LoadingMethod method); + bool hasLoadingMethod(LoadingMethod method) const; static Resource style(const std::string& url); static Resource source(const std::string& url); @@ -97,7 +97,7 @@ class Resource { std::shared_ptr priorData; }; -inline bool Resource::hasLoadingMethod(Resource::LoadingMethod method) { +inline bool Resource::hasLoadingMethod(Resource::LoadingMethod method) const { return (loadingMethod & method); } diff --git a/include/mbgl/storage/resource_options.hpp b/include/mbgl/storage/resource_options.hpp index 00dc6e10df5..6d603b8ccaa 100644 --- a/include/mbgl/storage/resource_options.hpp +++ b/include/mbgl/storage/resource_options.hpp @@ -96,21 +96,6 @@ class ResourceOptions final { */ uint64_t maximumCacheSize() const; - /** - * @brief Sets whether to support cache-only requests. - * - * @return Whether or not cache-only requests are supported. - */ - bool supportsCacheOnlyRequests() const; - - /** - * @brief Gets the previously set (or default) support for cache-only requests. - * - * @param cacheOnly Whether or not cache-only requests are supported. - * @return reference to ResourceOptions for chaining options together. - */ - ResourceOptions& withCacheOnlyRequestsSupport(bool cacheOnly); - /** * @brief Sets the platform context. A platform context is usually an object * that assists the creation of a file source. diff --git a/include/mbgl/storage/resource_transform.hpp b/include/mbgl/storage/resource_transform.hpp index b8e3dbac76b..29d19bb4ecc 100644 --- a/include/mbgl/storage/resource_transform.hpp +++ b/include/mbgl/storage/resource_transform.hpp @@ -1,6 +1,5 @@ #pragma once -#include #include #include @@ -8,16 +7,14 @@ namespace mbgl { -class Mailbox; - class ResourceTransform { public: - using TransformCallback = std::function; - using FinishedCallback = std::function; - - ResourceTransform(ActorRef, TransformCallback&&); + using FinishedCallback = std::function; + using TransformCallback = std::function; - void transform(Resource::Kind, const std::string& url, FinishedCallback&&); + ResourceTransform(TransformCallback = {}); + void transform(Resource::Kind, const std::string& url, FinishedCallback); + explicit operator bool() const { return bool(transformCallback); } private: TransformCallback transformCallback; diff --git a/include/mbgl/style/style.hpp b/include/mbgl/style/style.hpp index 4a6a542b881..83d6ad5bb43 100644 --- a/include/mbgl/style/style.hpp +++ b/include/mbgl/style/style.hpp @@ -21,7 +21,7 @@ class Layer; class Style { public: - Style(FileSource&, float pixelRatio); + Style(std::shared_ptr, float pixelRatio); ~Style(); void loadJSON(const std::string&); diff --git a/include/mbgl/util/constants.hpp b/include/mbgl/util/constants.hpp index ffdc4b2b300..56f42ac8940 100644 --- a/include/mbgl/util/constants.hpp +++ b/include/mbgl/util/constants.hpp @@ -60,6 +60,9 @@ constexpr UnitBezier DEFAULT_TRANSITION_EASE = { 0, 0, 0.25, 1 }; constexpr int DEFAULT_RATE_LIMIT_TIMEOUT = 5; constexpr const char* API_BASE_URL = "https://api.mapbox.com"; +constexpr const char* ASSET_PROTOCOL = "asset://"; +constexpr const char* FILE_PROTOCOL = "file://"; +constexpr uint32_t DEFAULT_MAXIMUM_CONCURRENT_REQUESTS = 20; constexpr uint8_t TERRAIN_RGB_MAXZOOM = 15; diff --git a/metrics/next-binary-size/linux-clang8/metrics.json b/metrics/next-binary-size/linux-clang8/metrics.json index ef40eb2ea79..c1e7c842718 100644 --- a/metrics/next-binary-size/linux-clang8/metrics.json +++ b/metrics/next-binary-size/linux-clang8/metrics.json @@ -3,17 +3,17 @@ [ "mbgl-glfw", "/tmp/attach/install/next-linux-clang8-release/bin/mbgl-glfw", - 6380744 + 6442184 ], [ "mbgl-offline", "/tmp/attach/install/next-linux-clang8-release/bin/mbgl-offline", - 5651144 + 5720776 ], [ "mbgl-render", "/tmp/attach/install/next-linux-clang8-release/bin/mbgl-render", - 6282152 + 6343592 ] ] } \ No newline at end of file diff --git a/metrics/next-binary-size/linux-gcc8/metrics.json b/metrics/next-binary-size/linux-gcc8/metrics.json index e7c8cb0d08e..d46a45d2ba6 100644 --- a/metrics/next-binary-size/linux-gcc8/metrics.json +++ b/metrics/next-binary-size/linux-gcc8/metrics.json @@ -3,17 +3,17 @@ [ "mbgl-glfw", "/tmp/attach/install/next-linux-gcc8-release/bin/mbgl-glfw", - 7369032 + 7475528 ], [ "mbgl-offline", "/tmp/attach/install/next-linux-gcc8-release/bin/mbgl-offline", - 6463720 + 6570216 ], [ "mbgl-render", "/tmp/attach/install/next-linux-gcc8-release/bin/mbgl-render", - 7254344 + 7360840 ] ] } \ No newline at end of file diff --git a/metrics/next-binary-size/macos-xcode11/metrics.json b/metrics/next-binary-size/macos-xcode11/metrics.json index f516ca94ffb..27b3e3b4529 100644 --- a/metrics/next-binary-size/macos-xcode11/metrics.json +++ b/metrics/next-binary-size/macos-xcode11/metrics.json @@ -3,17 +3,17 @@ [ "mbgl-glfw", "/tmp/attach/install/next-macos-xcode11-release/bin/mbgl-glfw", - 5529484 + 5562556 ], [ "mbgl-offline", "/tmp/attach/install/next-macos-xcode11-release/bin/mbgl-offline", - 5389808 + 5427032 ], [ "mbgl-render", "/tmp/attach/install/next-macos-xcode11-release/bin/mbgl-render", - 5444188 + 5477244 ] ] -} \ No newline at end of file +} diff --git a/next/CMakeLists.txt b/next/CMakeLists.txt index f0beebdb460..1950a1144ad 100644 --- a/next/CMakeLists.txt +++ b/next/CMakeLists.txt @@ -130,8 +130,9 @@ add_library( ${MBGL_ROOT}/include/mbgl/renderer/renderer_frontend.hpp ${MBGL_ROOT}/include/mbgl/renderer/renderer_observer.hpp ${MBGL_ROOT}/include/mbgl/renderer/renderer_state.hpp - ${MBGL_ROOT}/include/mbgl/storage/default_file_source.hpp + ${MBGL_ROOT}/include/mbgl/storage/database_file_source.hpp ${MBGL_ROOT}/include/mbgl/storage/file_source.hpp + ${MBGL_ROOT}/include/mbgl/storage/file_source_manager.hpp ${MBGL_ROOT}/include/mbgl/storage/network_status.hpp ${MBGL_ROOT}/include/mbgl/storage/offline.hpp ${MBGL_ROOT}/include/mbgl/storage/online_file_source.hpp @@ -596,9 +597,10 @@ add_library( ${MBGL_ROOT}/src/mbgl/sprite/sprite_parser.cpp ${MBGL_ROOT}/src/mbgl/sprite/sprite_parser.hpp ${MBGL_ROOT}/src/mbgl/storage/asset_file_source.hpp - ${MBGL_ROOT}/src/mbgl/storage/file_source.cpp + ${MBGL_ROOT}/src/mbgl/storage/file_source_manager.cpp ${MBGL_ROOT}/src/mbgl/storage/http_file_source.hpp ${MBGL_ROOT}/src/mbgl/storage/local_file_source.hpp + ${MBGL_ROOT}/src/mbgl/storage/main_resource_loader.hpp ${MBGL_ROOT}/src/mbgl/storage/network_status.cpp ${MBGL_ROOT}/src/mbgl/storage/resource.cpp ${MBGL_ROOT}/src/mbgl/storage/resource_options.cpp diff --git a/next/platform/android/android.cmake b/next/platform/android/android.cmake index 429963b87b3..454e11bcd21 100644 --- a/next/platform/android/android.cmake +++ b/next/platform/android/android.cmake @@ -214,10 +214,12 @@ target_sources( ${MBGL_ROOT}/platform/default/src/mbgl/gl/headless_backend.cpp ${MBGL_ROOT}/platform/default/src/mbgl/map/map_snapshotter.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/asset_file_source.cpp - ${MBGL_ROOT}/platform/default/src/mbgl/storage/default_file_source.cpp + ${MBGL_ROOT}/platform/default/src/mbgl/storage/database_file_source.cpp + ${MBGL_ROOT}/platform/default/src/mbgl/storage/file_source_manager.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/file_source_request.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/local_file_request.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/local_file_source.cpp + ${MBGL_ROOT}/platform/default/src/mbgl/storage/main_resource_loader.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/offline.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/offline_database.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/offline_download.cpp @@ -327,7 +329,6 @@ add_library( ${MBGL_ROOT}/platform/default/src/mbgl/text/local_glyph_rasterizer.cpp ${MBGL_ROOT}/platform/android/src/test/collator_test_stub.cpp ${MBGL_ROOT}/platform/android/src/test/number_format_test_stub.cpp - ${MBGL_ROOT}/platform/default/src/mbgl/storage/file_source.cpp ${MBGL_ROOT}/platform/android/src/test/http_file_source_test_stub.cpp ) @@ -377,7 +378,6 @@ add_library( ${MBGL_ROOT}/platform/default/src/mbgl/text/local_glyph_rasterizer.cpp ${MBGL_ROOT}/platform/android/src/test/collator_test_stub.cpp ${MBGL_ROOT}/platform/android/src/test/number_format_test_stub.cpp - ${MBGL_ROOT}/platform/default/src/mbgl/storage/file_source.cpp ${MBGL_ROOT}/platform/android/src/test/http_file_source_test_stub.cpp ) diff --git a/next/platform/ios/ios.cmake b/next/platform/ios/ios.cmake index f2ab071812a..0de0fc1ddc3 100644 --- a/next/platform/ios/ios.cmake +++ b/next/platform/ios/ios.cmake @@ -39,11 +39,12 @@ target_sources( ${MBGL_ROOT}/platform/default/src/mbgl/gl/headless_backend.cpp ${MBGL_ROOT}/platform/default/src/mbgl/map/map_snapshotter.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/asset_file_source.cpp - ${MBGL_ROOT}/platform/default/src/mbgl/storage/default_file_source.cpp - ${MBGL_ROOT}/platform/default/src/mbgl/storage/file_source.cpp + ${MBGL_ROOT}/platform/default/src/mbgl/storage/database_file_source.cpp + ${MBGL_ROOT}/platform/default/src/mbgl/storage/file_source_manager.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/file_source_request.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/local_file_request.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/local_file_source.cpp + ${MBGL_ROOT}/platform/default/src/mbgl/storage/main_resource_loader.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/offline.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/offline_database.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/offline_download.cpp diff --git a/next/platform/linux/linux.cmake b/next/platform/linux/linux.cmake index 6c06a75e619..6f2ac12e49a 100644 --- a/next/platform/linux/linux.cmake +++ b/next/platform/linux/linux.cmake @@ -19,12 +19,13 @@ target_sources( ${MBGL_ROOT}/platform/default/src/mbgl/i18n/number_format.cpp ${MBGL_ROOT}/platform/default/src/mbgl/layermanager/layer_manager.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/asset_file_source.cpp - ${MBGL_ROOT}/platform/default/src/mbgl/storage/default_file_source.cpp - ${MBGL_ROOT}/platform/default/src/mbgl/storage/file_source.cpp + ${MBGL_ROOT}/platform/default/src/mbgl/storage/database_file_source.cpp + ${MBGL_ROOT}/platform/default/src/mbgl/storage/file_source_manager.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/file_source_request.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/http_file_source.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/local_file_request.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/local_file_source.cpp + ${MBGL_ROOT}/platform/default/src/mbgl/storage/main_resource_loader.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/offline.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/offline_database.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/offline_download.cpp diff --git a/next/platform/macos/macos.cmake b/next/platform/macos/macos.cmake index 9a7f538db61..2b28b7e1e51 100644 --- a/next/platform/macos/macos.cmake +++ b/next/platform/macos/macos.cmake @@ -95,11 +95,12 @@ target_sources( ${MBGL_ROOT}/platform/default/src/mbgl/gl/headless_backend.cpp ${MBGL_ROOT}/platform/default/src/mbgl/map/map_snapshotter.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/asset_file_source.cpp - ${MBGL_ROOT}/platform/default/src/mbgl/storage/default_file_source.cpp - ${MBGL_ROOT}/platform/default/src/mbgl/storage/file_source.cpp + ${MBGL_ROOT}/platform/default/src/mbgl/storage/database_file_source.cpp + ${MBGL_ROOT}/platform/default/src/mbgl/storage/file_source_manager.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/file_source_request.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/local_file_request.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/local_file_source.cpp + ${MBGL_ROOT}/platform/default/src/mbgl/storage/main_resource_loader.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/offline.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/offline_database.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/offline_download.cpp diff --git a/next/platform/qt/qt.cmake b/next/platform/qt/qt.cmake index 4d818919479..74cea29efbf 100644 --- a/next/platform/qt/qt.cmake +++ b/next/platform/qt/qt.cmake @@ -32,11 +32,12 @@ target_sources( ${MBGL_ROOT}/platform/default/src/mbgl/i18n/collator.cpp ${MBGL_ROOT}/platform/default/src/mbgl/layermanager/layer_manager.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/asset_file_source.cpp - ${MBGL_ROOT}/platform/default/src/mbgl/storage/default_file_source.cpp - ${MBGL_ROOT}/platform/default/src/mbgl/storage/file_source.cpp + ${MBGL_ROOT}/platform/default/src/mbgl/storage/database_file_source.cpp + ${MBGL_ROOT}/platform/default/src/mbgl/storage/file_source_manager.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/file_source_request.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/local_file_request.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/local_file_source.cpp + ${MBGL_ROOT}/platform/default/src/mbgl/storage/main_resource_loader.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/offline.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/offline_database.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/offline_download.cpp diff --git a/next/test/CMakeLists.txt b/next/test/CMakeLists.txt index 06d8045d0a2..11d108bf3d4 100644 --- a/next/test/CMakeLists.txt +++ b/next/test/CMakeLists.txt @@ -35,10 +35,10 @@ add_library( ${MBGL_ROOT}/test/src/mbgl/test/test.cpp ${MBGL_ROOT}/test/src/mbgl/test/util.cpp ${MBGL_ROOT}/test/storage/asset_file_source.test.cpp - ${MBGL_ROOT}/test/storage/default_file_source.test.cpp ${MBGL_ROOT}/test/storage/headers.test.cpp ${MBGL_ROOT}/test/storage/http_file_source.test.cpp ${MBGL_ROOT}/test/storage/local_file_source.test.cpp + ${MBGL_ROOT}/test/storage/main_resource_loader.test.cpp ${MBGL_ROOT}/test/storage/offline.test.cpp ${MBGL_ROOT}/test/storage/offline_database.test.cpp ${MBGL_ROOT}/test/storage/offline_download.test.cpp diff --git a/platform/android/src/test/http_file_source_test_stub.cpp b/platform/android/src/test/http_file_source_test_stub.cpp index 930a20907a5..48e6a965ff2 100644 --- a/platform/android/src/test/http_file_source_test_stub.cpp +++ b/platform/android/src/test/http_file_source_test_stub.cpp @@ -1,6 +1,6 @@ #include #include - +#include #include namespace mbgl { diff --git a/platform/darwin/filesource-files.json b/platform/darwin/filesource-files.json index 62043a0dcd1..5d72549f733 100644 --- a/platform/darwin/filesource-files.json +++ b/platform/darwin/filesource-files.json @@ -5,6 +5,7 @@ "platform/darwin/src/MGLNetworkConfiguration.m", "platform/darwin/src/http_file_source.mm", "platform/default/src/mbgl/storage/file_source.cpp", + "platform/default/src/mbgl/storage/file_source_manager.cpp", "platform/default/src/mbgl/storage/sqlite3.cpp" ], "public_headers": {}, diff --git a/platform/default/filesource-files.json b/platform/default/filesource-files.json index f61aa6a3352..72e76670b89 100644 --- a/platform/default/filesource-files.json +++ b/platform/default/filesource-files.json @@ -2,19 +2,22 @@ "//": "This file can be edited manually and is the canonical source.", "sources": [ "platform/default/src/mbgl/storage/asset_file_source.cpp", - "platform/default/src/mbgl/storage/default_file_source.cpp", + "platform/default/src/mbgl/storage/database_file_source.cpp", + "platform/default/src/mbgl/storage/file_source_manager.cpp", "platform/default/src/mbgl/storage/file_source_request.cpp", "platform/default/src/mbgl/storage/local_file_request.cpp", "platform/default/src/mbgl/storage/local_file_source.cpp", + "platform/default/src/mbgl/storage/main_resource_loader.cpp", "platform/default/src/mbgl/storage/offline.cpp", "platform/default/src/mbgl/storage/offline_database.cpp", "platform/default/src/mbgl/storage/offline_download.cpp", "platform/default/src/mbgl/storage/online_file_source.cpp" ], "public_headers": { - "mbgl/storage/default_file_source.hpp": "include/mbgl/storage/default_file_source.hpp", + "mbgl/storage/offline_file_source.hpp": "include/mbgl/storage/database_file_source.hpp", "mbgl/storage/offline.hpp": "include/mbgl/storage/offline.hpp", "mbgl/storage/online_file_source.hpp": "include/mbgl/storage/online_file_source.hpp", + "mbgl/storage/file_source_manager.hpp": "include/mbgl/storage/file_source_manager.hpp", "mbgl/storage/file_source_request.hpp": "platform/default/include/mbgl/storage/file_source_request.hpp", "mbgl/storage/local_file_request.hpp": "platform/default/include/mbgl/storage/local_file_request.hpp", "mbgl/storage/merge_sideloaded.hpp": "platform/default/include/mbgl/storage/merge_sideloaded.hpp", @@ -24,6 +27,7 @@ "mbgl/storage/sqlite3.hpp": "platform/default/include/mbgl/storage/sqlite3.hpp" }, "private_headers": { + "mbgl/storage/main_resource_loader.hpp": "src/mbgl/storage/main_resource_loader.hpp", "mbgl/storage/asset_file_source.hpp": "src/mbgl/storage/asset_file_source.hpp", "mbgl/storage/http_file_source.hpp": "src/mbgl/storage/http_file_source.hpp", "mbgl/storage/local_file_source.hpp": "src/mbgl/storage/local_file_source.hpp" diff --git a/platform/default/include/mbgl/storage/offline_download.hpp b/platform/default/include/mbgl/storage/offline_download.hpp index 53b42ae9d14..3a5159470a9 100644 --- a/platform/default/include/mbgl/storage/offline_download.hpp +++ b/platform/default/include/mbgl/storage/offline_download.hpp @@ -1,8 +1,8 @@ #pragma once +#include #include #include -#include #include #include @@ -28,7 +28,7 @@ class Parser; */ class OfflineDownload { public: - OfflineDownload(int64_t id, OfflineRegionDefinition&&, OfflineDatabase& offline, OnlineFileSource& online); + OfflineDownload(int64_t id, OfflineRegionDefinition, OfflineDatabase& offline, FileSource& online); ~OfflineDownload(); void setObserver(std::unique_ptr); @@ -53,7 +53,7 @@ class OfflineDownload { int64_t id; OfflineRegionDefinition definition; OfflineDatabase& offlineDatabase; - OnlineFileSource& onlineFileSource; + FileSource& onlineFileSource; OfflineRegionStatus status; std::unique_ptr observer; diff --git a/platform/default/src/mbgl/storage/asset_file_source.cpp b/platform/default/src/mbgl/storage/asset_file_source.cpp index b14d73045f8..7abd609b199 100644 --- a/platform/default/src/mbgl/storage/asset_file_source.cpp +++ b/platform/default/src/mbgl/storage/asset_file_source.cpp @@ -1,15 +1,17 @@ #include #include #include +#include #include +#include #include #include #include namespace { - -const std::string assetProtocol = "asset://"; - +bool acceptsURL(const std::string& url) { + return 0 == url.rfind(mbgl::util::ASSET_PROTOCOL, 0); +} } // namespace namespace mbgl { @@ -30,7 +32,8 @@ class AssetFileSource::Impl { } // Cut off the protocol and prefix with path. - const auto path = root + "/" + mbgl::util::percentDecode(url.substr(assetProtocol.size())); + const auto path = + root + "/" + mbgl::util::percentDecode(url.substr(std::char_traits::length(util::ASSET_PROTOCOL))); requestLocalFile(path, std::move(req)); } @@ -52,8 +55,16 @@ std::unique_ptr AssetFileSource::request(const Resource& resource, return std::move(req); } -bool AssetFileSource::acceptsURL(const std::string& url) { - return 0 == url.rfind(assetProtocol, 0); +bool AssetFileSource::canRequest(const Resource& resource) const { + return acceptsURL(resource.url); +} + +void AssetFileSource::pause() { + impl->pause(); +} + +void AssetFileSource::resume() { + impl->resume(); } } // namespace mbgl diff --git a/platform/default/src/mbgl/storage/database_file_source.cpp b/platform/default/src/mbgl/storage/database_file_source.cpp new file mode 100644 index 00000000000..ba8ba7b31d0 --- /dev/null +++ b/platform/default/src/mbgl/storage/database_file_source.cpp @@ -0,0 +1,280 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace mbgl { + +// For testing use only +constexpr const char* READ_ONLY_MODE_KEY = "read-only-mode"; + +class DatabaseFileSourceThread { +public: + DatabaseFileSourceThread(std::shared_ptr onlineFileSource_, std::string cachePath) + : db(std::make_unique(cachePath)), onlineFileSource(onlineFileSource_) {} + + void request(const Resource& resource, ActorRef req) { + auto offlineResponse = db->get(resource); + if (!offlineResponse) { + offlineResponse.emplace(); + offlineResponse->noContent = true; + offlineResponse->error = + std::make_unique(Response::Error::Reason::NotFound, "Not found in offline database"); + } else if (!offlineResponse->isUsable()) { + offlineResponse->error = + std::make_unique(Response::Error::Reason::NotFound, "Cached resource is unusable"); + } + req.invoke(&FileSourceRequest::setResponse, *offlineResponse); + } + + void setDatabasePath(const std::string& path, std::function callback) { + db->changePath(path); + if (callback) { + callback(); + } + } + + void forward(const Resource& resource, const Response& response, std::function callback) { + db->put(resource, response); + if (callback) { + callback(); + } + } + + void resetDatabase(std::function callback) { callback(db->resetDatabase()); } + + void packDatabase(std::function callback) { callback(db->pack()); } + + void runPackDatabaseAutomatically(bool autopack) { db->runPackDatabaseAutomatically(autopack); } + + void put(const Resource& resource, const Response& response) { db->put(resource, response); } + + void invalidateAmbientCache(std::function callback) { + callback(db->invalidateAmbientCache()); + } + + void clearAmbientCache(std::function callback) { callback(db->clearAmbientCache()); } + + void setMaximumAmbientCacheSize(uint64_t size, std::function callback) { + callback(db->setMaximumAmbientCacheSize(size)); + } + + void listRegions(std::function)> callback) { + callback(db->listRegions()); + } + + void createRegion(const OfflineRegionDefinition& definition, + const OfflineRegionMetadata& metadata, + std::function)> callback) { + callback(db->createRegion(definition, metadata)); + } + + void mergeOfflineRegions(const std::string& sideDatabasePath, + std::function)> callback) { + callback(db->mergeDatabase(sideDatabasePath)); + } + + void updateMetadata(const int64_t regionID, + const OfflineRegionMetadata& metadata, + std::function)> callback) { + callback(db->updateMetadata(regionID, metadata)); + } + + void getRegionStatus(int64_t regionID, + std::function)> callback) { + if (auto download = getDownload(regionID)) { + callback(download.value()->getStatus()); + } else { + callback(unexpected(download.error())); + } + } + + void deleteRegion(OfflineRegion region, std::function callback) { + downloads.erase(region.getID()); + callback(db->deleteRegion(std::move(region))); + } + + void invalidateRegion(int64_t regionID, std::function callback) { + callback(db->invalidateRegion(regionID)); + } + + void setRegionObserver(int64_t regionID, std::unique_ptr observer) { + if (auto download = getDownload(regionID)) { + download.value()->setObserver(std::move(observer)); + } + } + + void setRegionDownloadState(int64_t regionID, OfflineRegionDownloadState state) { + if (auto download = getDownload(regionID)) { + download.value()->setState(state); + } + } + + void setOfflineMapboxTileCountLimit(uint64_t limit) { db->setOfflineMapboxTileCountLimit(limit); } + + void reopenDatabaseReadOnlyForTesting() { db->reopenDatabaseReadOnlyForTesting(); } + +private: + expected getDownload(int64_t regionID) { + if (!onlineFileSource) { + return unexpected( + std::make_exception_ptr(std::runtime_error("Network file source unavailable."))); + } + + auto it = downloads.find(regionID); + if (it != downloads.end()) { + return it->second.get(); + } + auto definition = db->getRegionDefinition(regionID); + if (!definition) { + return unexpected(definition.error()); + } + auto download = + std::make_unique(regionID, std::move(definition.value()), *db, *onlineFileSource); + return downloads.emplace(regionID, std::move(download)).first->second.get(); + } + + std::unique_ptr db; + std::unordered_map> downloads; + std::shared_ptr onlineFileSource; +}; + +class DatabaseFileSource::Impl { +public: + Impl(std::shared_ptr onlineFileSource, const std::string& cachePath) + : thread(std::make_unique>( + "DatabaseFileSource", std::move(onlineFileSource), cachePath)) {} + + ActorRef actor() const { return thread->actor(); } + +private: + const std::unique_ptr> thread; +}; + +DatabaseFileSource::DatabaseFileSource(const ResourceOptions& options) + : impl(std::make_unique(FileSourceManager::get()->getFileSource(FileSourceType::Network, options), + options.cachePath())) {} + +DatabaseFileSource::~DatabaseFileSource() = default; + +std::unique_ptr DatabaseFileSource::request(const Resource& resource, Callback callback) { + auto req = std::make_unique(std::move(callback)); + impl->actor().invoke(&DatabaseFileSourceThread::request, resource, req->actor()); + return std::move(req); +} + +void DatabaseFileSource::forward(const Resource& res, const Response& response, std::function callback) { + std::function wrapper; + if (callback) { + wrapper = Scheduler::GetCurrent()->bindOnce(std::move(callback)); + } + impl->actor().invoke(&DatabaseFileSourceThread::forward, res, response, std::move(wrapper)); +} + +bool DatabaseFileSource::canRequest(const Resource& resource) const { + return resource.hasLoadingMethod(Resource::LoadingMethod::Cache) && + resource.url.rfind(mbgl::util::ASSET_PROTOCOL, 0) == std::string::npos && + resource.url.rfind(mbgl::util::FILE_PROTOCOL, 0) == std::string::npos; +} + +void DatabaseFileSource::setDatabasePath(const std::string& path, std::function callback) { + impl->actor().invoke(&DatabaseFileSourceThread::setDatabasePath, path, std::move(callback)); +} + +void DatabaseFileSource::resetDatabase(std::function callback) { + impl->actor().invoke(&DatabaseFileSourceThread::resetDatabase, std::move(callback)); +} + +void DatabaseFileSource::packDatabase(std::function callback) { + impl->actor().invoke(&DatabaseFileSourceThread::packDatabase, std::move(callback)); +} + +void DatabaseFileSource::runPackDatabaseAutomatically(bool autopack) { + impl->actor().invoke(&DatabaseFileSourceThread::runPackDatabaseAutomatically, autopack); +} + +void DatabaseFileSource::put(const Resource& resource, const Response& response) { + impl->actor().invoke(&DatabaseFileSourceThread::put, resource, response); +} + +void DatabaseFileSource::invalidateAmbientCache(std::function callback) { + impl->actor().invoke(&DatabaseFileSourceThread::invalidateAmbientCache, std::move(callback)); +} + +void DatabaseFileSource::clearAmbientCache(std::function callback) { + impl->actor().invoke(&DatabaseFileSourceThread::clearAmbientCache, std::move(callback)); +} + +void DatabaseFileSource::setMaximumAmbientCacheSize(uint64_t size, std::function callback) { + impl->actor().invoke(&DatabaseFileSourceThread::setMaximumAmbientCacheSize, size, std::move(callback)); +} + +void DatabaseFileSource::listOfflineRegions( + std::function)> callback) { + impl->actor().invoke(&DatabaseFileSourceThread::listRegions, callback); +} + +void DatabaseFileSource::createOfflineRegion( + const OfflineRegionDefinition& definition, + const OfflineRegionMetadata& metadata, + std::function)> callback) { + impl->actor().invoke(&DatabaseFileSourceThread::createRegion, definition, metadata, callback); +} + +void DatabaseFileSource::mergeOfflineRegions( + const std::string& sideDatabasePath, std::function)> callback) { + impl->actor().invoke(&DatabaseFileSourceThread::mergeOfflineRegions, sideDatabasePath, callback); +} + +void DatabaseFileSource::updateOfflineMetadata( + const int64_t regionID, + const OfflineRegionMetadata& metadata, + std::function)> callback) { + impl->actor().invoke(&DatabaseFileSourceThread::updateMetadata, regionID, metadata, callback); +} + +void DatabaseFileSource::deleteOfflineRegion(OfflineRegion region, std::function callback) { + impl->actor().invoke(&DatabaseFileSourceThread::deleteRegion, std::move(region), callback); +} + +void DatabaseFileSource::invalidateOfflineRegion(OfflineRegion& region, + std::function callback) { + impl->actor().invoke(&DatabaseFileSourceThread::invalidateRegion, region.getID(), callback); +} + +void DatabaseFileSource::setOfflineRegionObserver(OfflineRegion& region, + std::unique_ptr observer) { + impl->actor().invoke(&DatabaseFileSourceThread::setRegionObserver, region.getID(), std::move(observer)); +} + +void DatabaseFileSource::setOfflineRegionDownloadState(OfflineRegion& region, OfflineRegionDownloadState state) { + impl->actor().invoke(&DatabaseFileSourceThread::setRegionDownloadState, region.getID(), state); +} + +void DatabaseFileSource::getOfflineRegionStatus( + OfflineRegion& region, std::function)> callback) const { + impl->actor().invoke(&DatabaseFileSourceThread::getRegionStatus, region.getID(), callback); +} + +void DatabaseFileSource::setOfflineMapboxTileCountLimit(uint64_t limit) const { + impl->actor().invoke(&DatabaseFileSourceThread::setOfflineMapboxTileCountLimit, limit); +} + +void DatabaseFileSource::setProperty(const std::string& key, const mapbox::base::Value& value) { + if (key == READ_ONLY_MODE_KEY && value.getBool()) { + if (*value.getBool()) { + impl->actor().invoke(&DatabaseFileSourceThread::reopenDatabaseReadOnlyForTesting); + } + } else { + std::string message = "Resource provider does not support property " + key; + Log::Error(Event::General, message.c_str()); + } +} + +} // namespace mbgl diff --git a/platform/default/src/mbgl/storage/default_file_source.cpp b/platform/default/src/mbgl/storage/default_file_source.cpp deleted file mode 100644 index 2d96a5a9a2b..00000000000 --- a/platform/default/src/mbgl/storage/default_file_source.cpp +++ /dev/null @@ -1,398 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#include -#include - -namespace mbgl { - -class DefaultFileSource::Impl { -public: - Impl(std::shared_ptr assetFileSource_, std::string cachePath) - : assetFileSource(std::move(assetFileSource_)) - , localFileSource(std::make_unique()) - , offlineDatabase(std::make_unique(std::move(cachePath))) { - } - - void setAPIBaseURL(const std::string& url) { - onlineFileSource.setAPIBaseURL(url); - } - - std::string getAPIBaseURL() const{ - return onlineFileSource.getAPIBaseURL(); - } - - void setAccessToken(const std::string& accessToken) { - onlineFileSource.setAccessToken(accessToken); - } - - std::string getAccessToken() const { - return onlineFileSource.getAccessToken(); - } - - void setResourceTransform(optional>&& transform) { - onlineFileSource.setResourceTransform(std::move(transform)); - } - - void setResourceCachePath(const std::string& path, optional>&& callback) { - offlineDatabase->changePath(path); - if (callback) { - callback->invoke(&PathChangeCallback::operator()); - } - } - - void listRegions(std::function)> callback) { - callback(offlineDatabase->listRegions()); - } - - void createRegion(const OfflineRegionDefinition& definition, - const OfflineRegionMetadata& metadata, - std::function)> callback) { - callback(offlineDatabase->createRegion(definition, metadata)); - } - - void mergeOfflineRegions(const std::string& sideDatabasePath, - std::function)> callback) { - callback(offlineDatabase->mergeDatabase(sideDatabasePath)); - } - - void updateMetadata(const int64_t regionID, - const OfflineRegionMetadata& metadata, - std::function)> callback) { - callback(offlineDatabase->updateMetadata(regionID, metadata)); - } - - void getRegionStatus(int64_t regionID, std::function)> callback) { - if (auto download = getDownload(regionID)) { - callback(download.value()->getStatus()); - } else { - callback(unexpected(download.error())); - } - } - - void deleteRegion(OfflineRegion&& region, std::function callback) { - downloads.erase(region.getID()); - callback(offlineDatabase->deleteRegion(std::move(region))); - } - - void invalidateRegion(int64_t regionID, std::function callback) { - callback(offlineDatabase->invalidateRegion(regionID)); - } - - void setRegionObserver(int64_t regionID, std::unique_ptr observer) { - if (auto download = getDownload(regionID)) { - download.value()->setObserver(std::move(observer)); - } - } - - void setRegionDownloadState(int64_t regionID, OfflineRegionDownloadState state) { - if (auto download = getDownload(regionID)) { - download.value()->setState(state); - } - } - - void request(AsyncRequest* req, Resource resource, ActorRef ref) { - auto callback = [ref] (const Response& res) { - ref.invoke(&FileSourceRequest::setResponse, res); - }; - - if (AssetFileSource::acceptsURL(resource.url)) { - //Asset request - tasks[req] = assetFileSource->request(resource, callback); - } else if (LocalFileSource::acceptsURL(resource.url)) { - //Local file request - tasks[req] = localFileSource->request(resource, callback); - } else { - // Try the offline database - if (resource.hasLoadingMethod(Resource::LoadingMethod::Cache)) { - auto offlineResponse = offlineDatabase->get(resource); - - if (resource.loadingMethod == Resource::LoadingMethod::CacheOnly) { - if (!offlineResponse) { - // Ensure there's always a response that we can send, so the caller knows that - // there's no optional data available in the cache, when it's the only place - // we're supposed to load from. - offlineResponse.emplace(); - offlineResponse->noContent = true; - offlineResponse->error = std::make_unique( - Response::Error::Reason::NotFound, "Not found in offline database"); - } else if (!offlineResponse->isUsable()) { - // Don't return resources the server requested not to show when they're stale. - // Even if we can't directly use the response, we may still use it to send a - // conditional HTTP request, which is why we're saving it above. - offlineResponse->error = std::make_unique( - Response::Error::Reason::NotFound, "Cached resource is unusable"); - } - callback(*offlineResponse); - } else if (offlineResponse) { - // Copy over the fields so that we can use them when making a refresh request. - resource.priorModified = offlineResponse->modified; - resource.priorExpires = offlineResponse->expires; - resource.priorEtag = offlineResponse->etag; - resource.priorData = offlineResponse->data; - - if (offlineResponse->isUsable()) { - callback(*offlineResponse); - // Set the priority of existing resource to low if it's expired but usable. - resource.setPriority(Resource::Priority::Low); - } - } - } - - // Get from the online file source - if (resource.hasLoadingMethod(Resource::LoadingMethod::Network)) { - MBGL_TIMING_START(watch); - tasks[req] = onlineFileSource.request(resource, [=] (Response onlineResponse) { - this->offlineDatabase->put(resource, onlineResponse); - if (resource.kind == Resource::Kind::Tile) { - // onlineResponse.data will be null if data not modified - MBGL_TIMING_FINISH(watch, - " Action: " << "Requesting," << - " URL: " << resource.url.c_str() << - " Size: " << (onlineResponse.data != nullptr ? onlineResponse.data->size() : 0) << "B," << - " Time") - } - callback(onlineResponse); - }); - } - } - } - - void cancel(AsyncRequest* req) { - tasks.erase(req); - } - - void setOfflineMapboxTileCountLimit(uint64_t limit) { - offlineDatabase->setOfflineMapboxTileCountLimit(limit); - } - - void setOnlineStatus(const bool status) { - onlineFileSource.setOnlineStatus(status); - } - - void reopenDatabaseReadOnlyForTesting() { offlineDatabase->reopenDatabaseReadOnlyForTesting(); } - - void setMaximumConcurrentRequests(uint32_t maximumConcurrentRequests_) { - onlineFileSource.setMaximumConcurrentRequests(maximumConcurrentRequests_); - } - - void put(const Resource& resource, const Response& response) { - offlineDatabase->put(resource, response); - } - - void resetDatabase(std::function callback) { - callback(offlineDatabase->resetDatabase()); - } - - void invalidateAmbientCache(std::function callback) { - callback(offlineDatabase->invalidateAmbientCache()); - } - - void clearAmbientCache(std::function callback) { - callback(offlineDatabase->clearAmbientCache()); - } - - void setMaximumAmbientCacheSize(uint64_t size, std::function callback) { - callback(offlineDatabase->setMaximumAmbientCacheSize(size)); - } - - void packDatabase(std::function callback) { callback(offlineDatabase->pack()); } - - void runPackDatabaseAutomatically(bool autopack) { offlineDatabase->runPackDatabaseAutomatically(autopack); } - -private: - expected getDownload(int64_t regionID) { - auto it = downloads.find(regionID); - if (it != downloads.end()) { - return it->second.get(); - } - auto definition = offlineDatabase->getRegionDefinition(regionID); - if (!definition) { - return unexpected(definition.error()); - } - auto download = std::make_unique(regionID, std::move(definition.value()), - *offlineDatabase, onlineFileSource); - return downloads.emplace(regionID, std::move(download)).first->second.get(); - } - - // shared so that destruction is done on the creating thread - const std::shared_ptr assetFileSource; - const std::unique_ptr localFileSource; - std::unique_ptr offlineDatabase; - OnlineFileSource onlineFileSource; - std::unordered_map> tasks; - std::unordered_map> downloads; -}; - -DefaultFileSource::DefaultFileSource(const std::string& cachePath, const std::string& assetPath, bool supportCacheOnlyRequests_) - : DefaultFileSource(cachePath, std::make_unique(assetPath), supportCacheOnlyRequests_) { -} - -DefaultFileSource::DefaultFileSource(const std::string& cachePath, std::unique_ptr&& assetFileSource_, bool supportCacheOnlyRequests_) - : assetFileSource(std::move(assetFileSource_)) - , impl(std::make_unique>("DefaultFileSource", assetFileSource, cachePath)) - , supportCacheOnlyRequests(supportCacheOnlyRequests_) { -} - -DefaultFileSource::~DefaultFileSource() = default; - -bool DefaultFileSource::supportsCacheOnlyRequests() const { - return supportCacheOnlyRequests; -} - -void DefaultFileSource::setAPIBaseURL(const std::string& baseURL) { - impl->actor().invoke(&Impl::setAPIBaseURL, baseURL); - - { - std::lock_guard lock(cachedBaseURLMutex); - cachedBaseURL = baseURL; - } -} - -std::string DefaultFileSource::getAPIBaseURL() { - std::lock_guard lock(cachedBaseURLMutex); - return cachedBaseURL; -} - -void DefaultFileSource::setAccessToken(const std::string& accessToken) { - impl->actor().invoke(&Impl::setAccessToken, accessToken); - - { - std::lock_guard lock(cachedAccessTokenMutex); - cachedAccessToken = accessToken; - } -} - -std::string DefaultFileSource::getAccessToken() { - std::lock_guard lock(cachedAccessTokenMutex); - return cachedAccessToken; -} - -void DefaultFileSource::setResourceTransform(optional>&& transform) { - impl->actor().invoke(&Impl::setResourceTransform, std::move(transform)); -} - -void DefaultFileSource::setResourceCachePath(const std::string& path, optional>&& callback) { - impl->actor().invoke(&Impl::setResourceCachePath, path, std::move(callback)); -} - -std::unique_ptr DefaultFileSource::request(const Resource& resource, Callback callback) { - auto req = std::make_unique(std::move(callback)); - - req->onCancel([fs = impl->actor(), req = req.get()] () { fs.invoke(&Impl::cancel, req); }); - - impl->actor().invoke(&Impl::request, req.get(), resource, req->actor()); - - return std::move(req); -} - -void DefaultFileSource::listOfflineRegions(std::function)> callback) { - impl->actor().invoke(&Impl::listRegions, callback); -} - -void DefaultFileSource::createOfflineRegion(const OfflineRegionDefinition& definition, - const OfflineRegionMetadata& metadata, - std::function)> callback) { - impl->actor().invoke(&Impl::createRegion, definition, metadata, callback); -} - -void DefaultFileSource::mergeOfflineRegions(const std::string& sideDatabasePath, - std::function)> callback) { - impl->actor().invoke(&Impl::mergeOfflineRegions, sideDatabasePath, callback); -} - -void DefaultFileSource::updateOfflineMetadata(const int64_t regionID, - const OfflineRegionMetadata& metadata, - std::function)> callback) { - impl->actor().invoke(&Impl::updateMetadata, regionID, metadata, callback); -} - -void DefaultFileSource::deleteOfflineRegion(OfflineRegion&& region, std::function callback) { - impl->actor().invoke(&Impl::deleteRegion, std::move(region), callback); -} - -void DefaultFileSource::invalidateOfflineRegion(OfflineRegion& region, - std::function callback) { - impl->actor().invoke(&Impl::invalidateRegion, region.getID(), callback); -} - -void DefaultFileSource::setOfflineRegionObserver(OfflineRegion& region, std::unique_ptr observer) { - impl->actor().invoke(&Impl::setRegionObserver, region.getID(), std::move(observer)); -} - -void DefaultFileSource::setOfflineRegionDownloadState(OfflineRegion& region, OfflineRegionDownloadState state) { - impl->actor().invoke(&Impl::setRegionDownloadState, region.getID(), state); -} - -void DefaultFileSource::getOfflineRegionStatus(OfflineRegion& region, std::function)> callback) const { - impl->actor().invoke(&Impl::getRegionStatus, region.getID(), callback); -} - -void DefaultFileSource::setOfflineMapboxTileCountLimit(uint64_t limit) const { - impl->actor().invoke(&Impl::setOfflineMapboxTileCountLimit, limit); -} - -void DefaultFileSource::pause() { - impl->pause(); -} - -void DefaultFileSource::resume() { - impl->resume(); -} - -void DefaultFileSource::put(const Resource& resource, const Response& response) { - impl->actor().invoke(&Impl::put, resource, response); -} - -void DefaultFileSource::resetDatabase(std::function callback) { - impl->actor().invoke(&Impl::resetDatabase, std::move(callback)); -} - -void DefaultFileSource::packDatabase(std::function callback) { - impl->actor().invoke(&Impl::packDatabase, std::move(callback)); -} - -void DefaultFileSource::runPackDatabaseAutomatically(bool autopack) { - impl->actor().invoke(&Impl::runPackDatabaseAutomatically, autopack); -} - -void DefaultFileSource::invalidateAmbientCache(std::function callback) { - impl->actor().invoke(&Impl::invalidateAmbientCache, std::move(callback)); -} - -void DefaultFileSource::clearAmbientCache(std::function callback) { - impl->actor().invoke(&Impl::clearAmbientCache, std::move(callback)); -} - -void DefaultFileSource::setMaximumAmbientCacheSize(uint64_t size, std::function callback) { - impl->actor().invoke(&Impl::setMaximumAmbientCacheSize, size, std::move(callback)); -} - -// For testing only: - -void DefaultFileSource::setOnlineStatus(const bool status) { - impl->actor().invoke(&Impl::setOnlineStatus, status); -} - -void DefaultFileSource::reopenDatabaseReadOnlyForTesting() { - impl->actor().invoke(&Impl::reopenDatabaseReadOnlyForTesting); -} - -void DefaultFileSource::setMaximumConcurrentRequests(uint32_t maximumConcurrentRequests_) { - impl->actor().invoke(&Impl::setMaximumConcurrentRequests, maximumConcurrentRequests_); -} - -} // namespace mbgl diff --git a/platform/default/src/mbgl/storage/file_source.cpp b/platform/default/src/mbgl/storage/file_source.cpp deleted file mode 100644 index 4e800cc8f48..00000000000 --- a/platform/default/src/mbgl/storage/file_source.cpp +++ /dev/null @@ -1,15 +0,0 @@ -#include -#include - -#include - -namespace mbgl { - -std::shared_ptr FileSource::createPlatformFileSource(const ResourceOptions& options) { - auto fileSource = std::make_shared(options.cachePath(), options.assetPath(), options.supportsCacheOnlyRequests()); - fileSource->setAccessToken(options.accessToken()); - fileSource->setAPIBaseURL(options.baseURL()); - return fileSource; -} - -} // namespace mbgl diff --git a/platform/default/src/mbgl/storage/file_source_manager.cpp b/platform/default/src/mbgl/storage/file_source_manager.cpp new file mode 100644 index 00000000000..2981096dace --- /dev/null +++ b/platform/default/src/mbgl/storage/file_source_manager.cpp @@ -0,0 +1,43 @@ +#include +#include +#include +#include +#include +#include +#include + +namespace mbgl { + +class DefaultFileSourceManagerImpl final : public FileSourceManager { +public: + DefaultFileSourceManagerImpl() { + registerFileSourceFactory(FileSourceType::ResourceLoader, [](const ResourceOptions& options) { + return std::make_unique(options); + }); + + registerFileSourceFactory(FileSourceType::Asset, [](const ResourceOptions& options) { + return std::make_unique(options.assetPath()); + }); + + registerFileSourceFactory(FileSourceType::Database, [](const ResourceOptions& options) { + return std::make_unique(options); + }); + + registerFileSourceFactory(FileSourceType::FileSystem, + [](const ResourceOptions&) { return std::make_unique(); }); + + registerFileSourceFactory(FileSourceType::Network, [](const ResourceOptions& options) { + auto networkSource = std::make_unique(); + networkSource->setProperty(ACCESS_TOKEN_KEY, options.accessToken()); + networkSource->setProperty(API_BASE_URL_KEY, options.baseURL()); + return networkSource; + }); + } +}; + +FileSourceManager* FileSourceManager::get() noexcept { + static DefaultFileSourceManagerImpl instance; + return &instance; +} + +} // namespace mbgl diff --git a/platform/default/src/mbgl/storage/local_file_source.cpp b/platform/default/src/mbgl/storage/local_file_source.cpp index ca2eedc7ba0..54f12baf794 100644 --- a/platform/default/src/mbgl/storage/local_file_source.cpp +++ b/platform/default/src/mbgl/storage/local_file_source.cpp @@ -1,15 +1,17 @@ -#include #include #include +#include +#include #include +#include #include #include #include namespace { - -const std::string fileProtocol = "file://"; - +bool acceptsURL(const std::string& url) { + return 0 == url.rfind(mbgl::util::FILE_PROTOCOL, 0); +} } // namespace namespace mbgl { @@ -28,7 +30,7 @@ class LocalFileSource::Impl { } // Cut off the protocol and prefix with path. - const auto path = mbgl::util::percentDecode(url.substr(fileProtocol.size())); + const auto path = mbgl::util::percentDecode(url.substr(std::char_traits::length(util::FILE_PROTOCOL))); requestLocalFile(path, std::move(req)); } }; @@ -47,8 +49,16 @@ std::unique_ptr LocalFileSource::request(const Resource& resource, return std::move(req); } -bool LocalFileSource::acceptsURL(const std::string& url) { - return 0 == url.rfind(fileProtocol, 0); +bool LocalFileSource::canRequest(const Resource& resource) const { + return acceptsURL(resource.url); +} + +void LocalFileSource::pause() { + impl->pause(); +} + +void LocalFileSource::resume() { + impl->resume(); } } // namespace mbgl diff --git a/platform/default/src/mbgl/storage/main_resource_loader.cpp b/platform/default/src/mbgl/storage/main_resource_loader.cpp new file mode 100644 index 00000000000..e39ca8ef47e --- /dev/null +++ b/platform/default/src/mbgl/storage/main_resource_loader.cpp @@ -0,0 +1,223 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace mbgl { + +class ResourceLoaderRequestor { +public: + explicit ResourceLoaderRequestor(MainResourceLoader::Impl& impl_); + void request(AsyncRequest*, Resource, ActorRef); + void cancel(AsyncRequest*); + void pause(); + void resume(); + +private: + MainResourceLoader::Impl& impl; +}; + +class MainResourceLoader::Impl { +public: + Impl(std::shared_ptr assetFileSource_, + std::shared_ptr databaseFileSource_, + std::shared_ptr localFileSource_, + std::shared_ptr onlineFileSource_) + : assetFileSource(std::move(assetFileSource_)), + databaseFileSource(std::move(databaseFileSource_)), + localFileSource(std::move(localFileSource_)), + onlineFileSource(std::move(onlineFileSource_)), + supportsCacheOnlyRequests_(bool(databaseFileSource)), + requestor(std::make_unique>(*Scheduler::GetCurrent(), *this)) {} + + std::unique_ptr request(const Resource& resource, Callback callback) { + auto req = std::make_unique(std::move(callback)); + req->onCancel([actorRef = requestor->self(), req = req.get()]() { + actorRef.invoke(&ResourceLoaderRequestor::cancel, req); + }); + requestor->self().invoke(&ResourceLoaderRequestor::request, req.get(), resource, req->actor()); + return std::move(req); + } + + bool canRequest(const Resource& resource) const { + return (assetFileSource && assetFileSource->canRequest(resource)) || + (localFileSource && localFileSource->canRequest(resource)) || + (databaseFileSource && databaseFileSource->canRequest(resource)) || + (onlineFileSource && onlineFileSource->canRequest(resource)); + } + + bool supportsCacheOnlyRequests() const { return supportsCacheOnlyRequests_; } + + void pause() { requestor->self().invoke(&ResourceLoaderRequestor::pause); } + + void resume() { requestor->self().invoke(&ResourceLoaderRequestor::resume); } + +private: + void request(AsyncRequest* req, Resource resource, ActorRef ref) { + auto callback = [ref](const Response& res) { ref.invoke(&FileSourceRequest::setResponse, res); }; + + auto requestFromNetwork = [=](const Resource& res, + std::unique_ptr parent) -> std::unique_ptr { + if (!onlineFileSource || !onlineFileSource->canRequest(resource)) { + return parent; + } + + // Keep parent request alive while chained request is being processed. + std::shared_ptr parentKeepAlive = std::move(parent); + + MBGL_TIMING_START(watch); + return onlineFileSource->request(res, [=, ptr = parentKeepAlive](Response response) { + if (databaseFileSource) { + databaseFileSource->forward(res, response); + } + if (res.kind == Resource::Kind::Tile) { + // onlineResponse.data will be null if data not modified + MBGL_TIMING_FINISH(watch, + " Action: " + << "Requesting," + << " URL: " << res.url.c_str() << " Size: " + << (response.data != nullptr ? response.data->size() : 0) << "B," + << " Time") + } + callback(response); + }); + }; + + // Initial tasksSize is used to check whether any of + // the sources were able to request a resource. + const std::size_t tasksSize = tasks.size(); + + // Waterfall resource request processing and return early once resource was requested. + if (assetFileSource && assetFileSource->canRequest(resource)) { + // Asset request + tasks[req] = assetFileSource->request(resource, callback); + } else if (localFileSource && localFileSource->canRequest(resource)) { + // Local file request + tasks[req] = localFileSource->request(resource, callback); + } else if (databaseFileSource && databaseFileSource->canRequest(resource)) { + // Try cache only request if needed. + if (resource.loadingMethod == Resource::LoadingMethod::CacheOnly) { + tasks[req] = databaseFileSource->request(resource, callback); + } else { + // Cache request with fallback to network with cache control + tasks[req] = databaseFileSource->request(resource, [=](Response response) { + Resource res = resource; + + // Resource is in the cache + if (!response.noContent) { + if (response.isUsable()) { + callback(response); + // Set the priority of existing resource to low if it's expired but usable. + res.setPriority(Resource::Priority::Low); + } + + // Copy response fields for cache control request + res.priorModified = response.modified; + res.priorExpires = response.expires; + res.priorEtag = response.etag; + res.priorData = response.data; + } + + tasks[req] = requestFromNetwork(res, std::move(tasks[req])); + }); + } + } else if (auto networkReq = requestFromNetwork(resource, nullptr)) { + // Get from the online file source + tasks[req] = std::move(networkReq); + } + + // If no new tasks were added, notify client that request cannot be processed. + if (tasks.size() == tasksSize) { + Response response; + response.noContent = true; + response.error = + std::make_unique(Response::Error::Reason::Other, "Unsupported resource request."); + callback(response); + } + } + + void pauseInternal() { + if (assetFileSource) assetFileSource->pause(); + if (databaseFileSource) databaseFileSource->pause(); + if (localFileSource) localFileSource->pause(); + if (onlineFileSource) onlineFileSource->pause(); + } + + void resumeInternal() { + if (assetFileSource) assetFileSource->resume(); + if (databaseFileSource) databaseFileSource->resume(); + if (localFileSource) localFileSource->resume(); + if (onlineFileSource) onlineFileSource->resume(); + } + + void cancel(AsyncRequest* req) { + assert(req); + tasks.erase(req); + } + +private: + friend class ResourceLoaderRequestor; + const std::shared_ptr assetFileSource; + const std::shared_ptr databaseFileSource; + const std::shared_ptr localFileSource; + const std::shared_ptr onlineFileSource; + const bool supportsCacheOnlyRequests_; + std::unique_ptr> requestor; + std::unordered_map> tasks; +}; + +ResourceLoaderRequestor::ResourceLoaderRequestor(MainResourceLoader::Impl& impl_) : impl(impl_) {} + +void ResourceLoaderRequestor::request(AsyncRequest* req, Resource resource, ActorRef ref) { + assert(req); + impl.request(req, std::move(resource), std::move(ref)); +} + +void ResourceLoaderRequestor::cancel(AsyncRequest* req) { + assert(req); + impl.cancel(req); +} + +void ResourceLoaderRequestor::pause() { + impl.pauseInternal(); +} + +void ResourceLoaderRequestor::resume() { + impl.resumeInternal(); +} + +MainResourceLoader::MainResourceLoader(const ResourceOptions& options) + : impl(std::make_unique(FileSourceManager::get()->getFileSource(FileSourceType::Asset, options), + FileSourceManager::get()->getFileSource(FileSourceType::Database, options), + FileSourceManager::get()->getFileSource(FileSourceType::FileSystem, options), + FileSourceManager::get()->getFileSource(FileSourceType::Network, options))) {} + +MainResourceLoader::~MainResourceLoader() = default; + +bool MainResourceLoader::supportsCacheOnlyRequests() const { + return impl->supportsCacheOnlyRequests(); +} + +std::unique_ptr MainResourceLoader::request(const Resource& resource, Callback callback) { + return impl->request(resource, std::move(callback)); +} + +bool MainResourceLoader::canRequest(const Resource& resource) const { + return impl->canRequest(resource); +} + +void MainResourceLoader::pause() { + impl->pause(); +} + +void MainResourceLoader::resume() { + impl->resume(); +} + +} // namespace mbgl diff --git a/platform/default/src/mbgl/storage/offline_download.cpp b/platform/default/src/mbgl/storage/offline_download.cpp index 98eb1d38842..32fcb4f6253 100644 --- a/platform/default/src/mbgl/storage/offline_download.cpp +++ b/platform/default/src/mbgl/storage/offline_download.cpp @@ -1,4 +1,3 @@ -#include #include #include #include @@ -90,11 +89,11 @@ uint64_t tileCount(const OfflineRegionDefinition& definition, style::SourceType // OfflineDownload OfflineDownload::OfflineDownload(int64_t id_, - OfflineRegionDefinition&& definition_, + OfflineRegionDefinition definition_, OfflineDatabase& offlineDatabase_, - OnlineFileSource& onlineFileSource_) + FileSource& onlineFileSource_) : id(id_), - definition(definition_), + definition(std::move(definition_)), offlineDatabase(offlineDatabase_), onlineFileSource(onlineFileSource_) { setObserver(nullptr); @@ -369,7 +368,13 @@ void OfflineDownload::continueDownload() { if (resourcesToBeMarkedAsUsed.size() >= kMarkBatchSize) markPendingUsedResources(); - while (!resourcesRemaining.empty() && requests.size() < onlineFileSource.getMaximumConcurrentRequests()) { + uint32_t maxConcurrentRequests = util::DEFAULT_MAXIMUM_CONCURRENT_REQUESTS; + auto value = onlineFileSource.getProperty("max-concurrent-requests"); + if (uint64_t* maxRequests = value.getUint()) { + maxConcurrentRequests = static_cast(*maxRequests); + } + + while (!resourcesRemaining.empty() && requests.size() < maxConcurrentRequests) { ensureResource(std::move(resourcesRemaining.front())); resourcesRemaining.pop_front(); } diff --git a/platform/default/src/mbgl/storage/online_file_source.cpp b/platform/default/src/mbgl/storage/online_file_source.cpp index f4225bdf3fe..0f5b438e1d5 100644 --- a/platform/default/src/mbgl/storage/online_file_source.cpp +++ b/platform/default/src/mbgl/storage/online_file_source.cpp @@ -2,52 +2,59 @@ #include #include +#include #include #include #include #include +#include +#include #include -#include #include -#include -#include +#include +#include #include #include +#include #include -#include #include #include #include -#include +#include #include +#include namespace mbgl { -static uint32_t DEFAULT_MAXIMUM_CONCURRENT_REQUESTS = 20; +// For testing only +constexpr const char* ONLINE_STATUS_KEY = "online-status"; -class OnlineFileRequest : public AsyncRequest { -public: - using Callback = std::function; +class OnlineFileSourceThread; - OnlineFileRequest(Resource, Callback, OnlineFileSource::Impl&); - ~OnlineFileRequest() override; +struct OnlineFileRequest { + using Callback = std::function; + + OnlineFileRequest(Resource resource_, Callback callback_, OnlineFileSourceThread& impl_); + ~OnlineFileRequest(); void networkIsReachableAgain(); void schedule(); void schedule(optional expires); void completed(Response); - void setTransformedURL(const std::string&& url); + void setTransformedURL(const std::string& url); ActorRef actor(); + void onCancel(std::function); - OnlineFileSource::Impl& impl; + OnlineFileSourceThread& impl; Resource resource; std::unique_ptr request; util::Timer timer; Callback callback; + std::function cancelCallback = nullptr; std::shared_ptr mailbox; // Counts the number of times a response was already expired when received. We're using @@ -62,101 +69,99 @@ class OnlineFileRequest : public AsyncRequest { optional retryAfter; }; -class OnlineFileSource::Impl { +class OnlineFileSourceThread { public: - Impl() { + OnlineFileSourceThread() { NetworkStatus::Subscribe(&reachability); - setMaximumConcurrentRequests(DEFAULT_MAXIMUM_CONCURRENT_REQUESTS); + setMaximumConcurrentRequests(util::DEFAULT_MAXIMUM_CONCURRENT_REQUESTS); + } + + ~OnlineFileSourceThread() { NetworkStatus::Unsubscribe(&reachability); } + + void request(AsyncRequest* req, Resource resource, ActorRef ref) { + auto callback = [ref](const Response& res) { ref.invoke(&FileSourceRequest::setResponse, res); }; + tasks[req] = std::make_unique(std::move(resource), std::move(callback), *this); } - ~Impl() { - NetworkStatus::Unsubscribe(&reachability); + void cancel(AsyncRequest* req) { + auto it = tasks.find(req); + assert(it != tasks.end()); + remove(it->second.get()); + tasks.erase(it); } - void add(OnlineFileRequest* request) { - allRequests.insert(request); + void add(OnlineFileRequest* req) { + allRequests.insert(req); if (resourceTransform) { // Request the ResourceTransform actor a new url and replace the resource url with the // transformed one before proceeding to schedule the request. - resourceTransform->invoke(&ResourceTransform::transform, - request->resource.kind, - std::move(request->resource.url), - [ref = request->actor()](const std::string&& url) { - ref.invoke(&OnlineFileRequest::setTransformedURL, url); - }); + resourceTransform.transform( + req->resource.kind, req->resource.url, [ref = req->actor()](const std::string& url) { + ref.invoke(&OnlineFileRequest::setTransformedURL, url); + }); } else { - request->schedule(); + req->schedule(); } } - void remove(OnlineFileRequest* request) { - allRequests.erase(request); - if (activeRequests.erase(request)) { + void remove(OnlineFileRequest* req) { + allRequests.erase(req); + if (activeRequests.erase(req)) { activatePendingRequest(); } else { - pendingRequests.remove(request); + pendingRequests.remove(req); } } - void activateOrQueueRequest(OnlineFileRequest* request) { - assert(allRequests.find(request) != allRequests.end()); - assert(activeRequests.find(request) == activeRequests.end()); - assert(!request->request); + void activateOrQueueRequest(OnlineFileRequest* req) { + assert(allRequests.find(req) != allRequests.end()); + assert(activeRequests.find(req) == activeRequests.end()); + assert(!req->request); if (activeRequests.size() >= getMaximumConcurrentRequests()) { - queueRequest(request); + queueRequest(req); } else { - activateRequest(request); + activateRequest(req); } } - void queueRequest(OnlineFileRequest* request) { - pendingRequests.insert(request); - } + void queueRequest(OnlineFileRequest* req) { pendingRequests.insert(req); } - void activateRequest(OnlineFileRequest* request) { + void activateRequest(OnlineFileRequest* req) { auto callback = [=](Response response) { - activeRequests.erase(request); - request->request.reset(); - request->completed(response); + activeRequests.erase(req); + req->request.reset(); + req->completed(response); activatePendingRequest(); }; - activeRequests.insert(request); + activeRequests.insert(req); if (online) { - request->request = httpFileSource.request(request->resource, callback); + req->request = httpFileSource.request(req->resource, callback); } else { Response response; response.error = std::make_unique(Response::Error::Reason::Connection, "Online connectivity is disabled."); callback(response); } - } void activatePendingRequest() { + auto req = pendingRequests.pop(); - auto request = pendingRequests.pop(); - - if (request) { - activateRequest(*request); + if (req) { + activateRequest(*req); } } - bool isPending(OnlineFileRequest* request) { - return pendingRequests.contains(request); - } + bool isPending(OnlineFileRequest* req) { return pendingRequests.contains(req); } - bool isActive(OnlineFileRequest* request) { - return activeRequests.find(request) != activeRequests.end(); - } + bool isActive(OnlineFileRequest* req) { return activeRequests.find(req) != activeRequests.end(); } - void setResourceTransform(optional>&& transform) { - resourceTransform = std::move(transform); - } + void setResourceTransform(ResourceTransform transform) { resourceTransform = std::move(transform); } - void setOnlineStatus(const bool status) { + void setOnlineStatus(bool status) { online = status; if (online) { networkIsReachableAgain(); @@ -171,20 +176,27 @@ class OnlineFileSource::Impl { maximumConcurrentRequests = maximumConcurrentRequests_; } + void setAPIBaseURL(const std::string& t) { apiBaseURL = t; } + std::string getAPIBaseURL() const { return apiBaseURL; } + + void setAccessToken(const std::string& t) { accessToken = t; } + std::string getAccessToken() const { return accessToken; } + private: + friend struct OnlineFileRequest; void networkIsReachableAgain() { // Notify regular priority requests. - for (auto& request : allRequests) { - if (request->resource.priority == Resource::Priority::Regular) { - request->networkIsReachableAgain(); + for (auto& req : allRequests) { + if (req->resource.priority == Resource::Priority::Regular) { + req->networkIsReachableAgain(); } } // Notify low priority requests. - for (auto& request : allRequests) { - if (request->resource.priority == Resource::Priority::Low) { - request->networkIsReachableAgain(); + for (auto& req : allRequests) { + if (req->resource.priority == Resource::Priority::Low) { + req->networkIsReachableAgain(); } } } @@ -231,7 +243,6 @@ class OnlineFileSource::Impl { } } - optional pop() { if (queue.empty()) { return optional(); @@ -252,7 +263,7 @@ class OnlineFileSource::Impl { }; - optional> resourceTransform; + ResourceTransform resourceTransform; /** * The lifetime of a request is: @@ -274,56 +285,99 @@ class OnlineFileSource::Impl { bool online = true; uint32_t maximumConcurrentRequests; HTTPFileSource httpFileSource; - util::AsyncTask reachability { std::bind(&Impl::networkIsReachableAgain, this) }; + util::AsyncTask reachability{std::bind(&OnlineFileSourceThread::networkIsReachableAgain, this)}; + std::string accessToken; + std::string apiBaseURL = mbgl::util::API_BASE_URL; + std::map> tasks; }; -OnlineFileSource::OnlineFileSource() - : impl(std::make_unique()) { -} +class OnlineFileSource::Impl { +public: + Impl() : thread(std::make_unique>("OnlineFileSource")) {} -OnlineFileSource::~OnlineFileSource() = default; + std::unique_ptr request(Callback callback, Resource res) { + auto req = std::make_unique(std::move(callback)); + req->onCancel( + [actorRef = thread->actor(), req = req.get()]() { actorRef.invoke(&OnlineFileSourceThread::cancel, req); }); + thread->actor().invoke(&OnlineFileSourceThread::request, req.get(), std::move(res), req->actor()); + return std::move(req); + } -std::unique_ptr OnlineFileSource::request(const Resource& resource, Callback callback) { - Resource res = resource; + void pause() { thread->pause(); } - switch (resource.kind) { - case Resource::Kind::Unknown: - case Resource::Kind::Image: - break; + void resume() { thread->resume(); } + + void setResourceTransform(ResourceTransform transform) { + thread->actor().invoke(&OnlineFileSourceThread::setResourceTransform, std::move(transform)); + } + + void setOnlineStatus(bool status) { thread->actor().invoke(&OnlineFileSourceThread::setOnlineStatus, status); } - case Resource::Kind::Style: - res.url = mbgl::util::mapbox::normalizeStyleURL(apiBaseURL, resource.url, accessToken); - break; + void setAPIBaseURL(const mapbox::base::Value& value) { + if (auto* baseURL = value.getString()) { + thread->actor().invoke(&OnlineFileSourceThread::setAPIBaseURL, *baseURL); + { + std::lock_guard lock(cachedBaseURLMutex); + cachedBaseURL = *baseURL; + } + } else { + Log::Error(Event::General, "Invalid api-base-url property value type."); + } + } - case Resource::Kind::Source: - res.url = util::mapbox::normalizeSourceURL(apiBaseURL, resource.url, accessToken); - break; + std::string getAPIBaseURL() const { + std::lock_guard lock(cachedBaseURLMutex); + return cachedBaseURL; + } - case Resource::Kind::Glyphs: - res.url = util::mapbox::normalizeGlyphsURL(apiBaseURL, resource.url, accessToken); - break; + void setMaximumConcurrentRequests(const mapbox::base::Value& value) { + if (auto* maximumConcurrentRequests = value.getUint()) { + assert(*maximumConcurrentRequests < std::numeric_limits::max()); + const uint32_t maxConcurretnRequests = static_cast(*maximumConcurrentRequests); + thread->actor().invoke(&OnlineFileSourceThread::setMaximumConcurrentRequests, maxConcurretnRequests); + { + std::lock_guard lock(maximumConcurrentRequestsMutex); + cachedMaximumConcurrentRequests = maxConcurretnRequests; + } + } else { + Log::Error(Event::General, "Invalid max-concurrent-requests property value type."); + } + } - case Resource::Kind::SpriteImage: - case Resource::Kind::SpriteJSON: - res.url = util::mapbox::normalizeSpriteURL(apiBaseURL, resource.url, accessToken); - break; + uint32_t getMaximumConcurrentRequests() const { + std::lock_guard lock(maximumConcurrentRequestsMutex); + return cachedMaximumConcurrentRequests; + } - case Resource::Kind::Tile: - res.url = util::mapbox::normalizeTileURL(apiBaseURL, resource.url, accessToken); - break; + void setAccessToken(const mapbox::base::Value& value) { + if (auto* accessToken = value.getString()) { + thread->actor().invoke(&OnlineFileSourceThread::setAccessToken, *accessToken); + { + std::lock_guard lock(cachedAccessTokenMutex); + cachedAccessToken = *accessToken; + } + } else { + Log::Error(Event::General, "Invalid access-token property value type."); + } } - return std::make_unique(std::move(res), std::move(callback), *impl); -} + std::string getAccessToken() const { + std::lock_guard lock(cachedAccessTokenMutex); + return cachedAccessToken; + } -void OnlineFileSource::setResourceTransform(optional>&& transform) { - impl->setResourceTransform(std::move(transform)); -} +private: + mutable std::mutex cachedAccessTokenMutex; + std::string cachedAccessToken; + mutable std::mutex cachedBaseURLMutex; + std::string cachedBaseURL = util::API_BASE_URL; + mutable std::mutex maximumConcurrentRequestsMutex; + uint32_t cachedMaximumConcurrentRequests = util::DEFAULT_MAXIMUM_CONCURRENT_REQUESTS; + const std::unique_ptr> thread; +}; -OnlineFileRequest::OnlineFileRequest(Resource resource_, Callback callback_, OnlineFileSource::Impl& impl_) - : impl(impl_), - resource(std::move(resource_)), - callback(std::move(callback_)) { +OnlineFileRequest::OnlineFileRequest(Resource resource_, Callback callback_, OnlineFileSourceThread& impl_) + : impl(impl_), resource(std::move(resource_)), callback(std::move(callback_)) { impl.add(this); } @@ -337,12 +391,12 @@ void OnlineFileRequest::schedule() { } OnlineFileRequest::~OnlineFileRequest() { - impl.remove(this); + if (mailbox) { + mailbox->close(); + } } -Timestamp interpolateExpiration(const Timestamp& current, - optional prior, - bool& expired) { +Timestamp interpolateExpiration(const Timestamp& current, optional prior, bool& expired) { auto now = util::now(); if (current > now) { return current; @@ -383,9 +437,8 @@ void OnlineFileRequest::schedule(optional expires) { // If we're not being asked for a forced refresh, calculate a timeout that depends on how many // consecutive errors we've encountered, and on the expiration time, if present. - Duration timeout = std::min( - http::errorRetryTimeout(failedRequestReason, failedRequests, retryAfter), - http::expirationTimeout(expires, expiredRequests)); + Duration timeout = std::min(http::errorRetryTimeout(failedRequestReason, failedRequests, retryAfter), + http::expirationTimeout(expires, expiredRequests)); if (timeout == Duration::max()) { return; @@ -469,7 +522,7 @@ void OnlineFileRequest::networkIsReachableAgain() { } } -void OnlineFileRequest::setTransformedURL(const std::string&& url) { +void OnlineFileRequest::setTransformedURL(const std::string& url) { resource.url = url; schedule(); } @@ -484,19 +537,95 @@ ActorRef OnlineFileRequest::actor() { return ActorRef(*this, mailbox); } -void OnlineFileSource::setMaximumConcurrentRequests(uint32_t maximumConcurrentRequests_) { - impl->setMaximumConcurrentRequests(maximumConcurrentRequests_); +void OnlineFileRequest::onCancel(std::function callback_) { + cancelCallback = std::move(callback_); +} + +OnlineFileSource::OnlineFileSource() : impl(std::make_unique()) {} + +OnlineFileSource::~OnlineFileSource() = default; + +std::unique_ptr OnlineFileSource::request(const Resource& resource, Callback callback) { + Resource res = resource; + + switch (resource.kind) { + case Resource::Kind::Unknown: + case Resource::Kind::Image: + break; + + case Resource::Kind::Style: + res.url = + mbgl::util::mapbox::normalizeStyleURL(impl->getAPIBaseURL(), resource.url, impl->getAccessToken()); + break; + + case Resource::Kind::Source: + res.url = util::mapbox::normalizeSourceURL(impl->getAPIBaseURL(), resource.url, impl->getAccessToken()); + break; + + case Resource::Kind::Glyphs: + res.url = util::mapbox::normalizeGlyphsURL(impl->getAPIBaseURL(), resource.url, impl->getAccessToken()); + break; + + case Resource::Kind::SpriteImage: + case Resource::Kind::SpriteJSON: + res.url = util::mapbox::normalizeSpriteURL(impl->getAPIBaseURL(), resource.url, impl->getAccessToken()); + break; + + case Resource::Kind::Tile: + res.url = util::mapbox::normalizeTileURL(impl->getAPIBaseURL(), resource.url, impl->getAccessToken()); + break; + } + + return impl->request(std::move(callback), std::move(res)); +} + +bool OnlineFileSource::canRequest(const Resource& resource) const { + return resource.hasLoadingMethod(Resource::LoadingMethod::Network) && + resource.url.rfind(mbgl::util::ASSET_PROTOCOL, 0) == std::string::npos && + resource.url.rfind(mbgl::util::FILE_PROTOCOL, 0) == std::string::npos; } -uint32_t OnlineFileSource::getMaximumConcurrentRequests() const { - return impl->getMaximumConcurrentRequests(); +void OnlineFileSource::pause() { + impl->pause(); } +void OnlineFileSource::resume() { + impl->resume(); +} + +void OnlineFileSource::setProperty(const std::string& key, const mapbox::base::Value& value) { + if (key == ACCESS_TOKEN_KEY) { + impl->setAccessToken(value); + } else if (key == API_BASE_URL_KEY) { + impl->setAPIBaseURL(value); + } else if (key == MAX_CONCURRENT_REQUESTS_KEY) { + impl->setMaximumConcurrentRequests(value); + } else if (key == ONLINE_STATUS_KEY) { + // For testing only + if (auto* boolValue = value.getBool()) { + impl->setOnlineStatus(*boolValue); + } + } else { + std::string message = "Resource provider does not support property " + key; + Log::Error(Event::General, message.c_str()); + } +} -// For testing only: +mapbox::base::Value OnlineFileSource::getProperty(const std::string& key) const { + if (key == ACCESS_TOKEN_KEY) { + return impl->getAccessToken(); + } else if (key == API_BASE_URL_KEY) { + return impl->getAPIBaseURL(); + } else if (key == MAX_CONCURRENT_REQUESTS_KEY) { + return impl->getMaximumConcurrentRequests(); + } + std::string message = "Resource provider does not support property " + key; + Log::Error(Event::General, message.c_str()); + return {}; +} -void OnlineFileSource::setOnlineStatus(const bool status) { - impl->setOnlineStatus(status); +void OnlineFileSource::setResourceTransform(ResourceTransform transform) { + impl->setResourceTransform(std::move(transform)); } } // namespace mbgl diff --git a/platform/glfw/main.cpp b/platform/glfw/main.cpp index 8f134804f0f..ded8ee3e1fb 100644 --- a/platform/glfw/main.cpp +++ b/platform/glfw/main.cpp @@ -3,13 +3,14 @@ #include "settings_json.hpp" #include +#include +#include +#include +#include #include #include #include #include -#include -#include -#include #include @@ -106,10 +107,16 @@ int main(int argc, char *argv[]) { mbgl::ResourceOptions resourceOptions; resourceOptions.withCachePath(cacheDB).withAccessToken(token); - auto fileSource = std::static_pointer_cast(mbgl::FileSource::getSharedFileSource(resourceOptions)); + auto onlineFileSource = + mbgl::FileSourceManager::get()->getFileSource(mbgl::FileSourceType::Network, resourceOptions); if (!settings.online) { - fileSource->setOnlineStatus(false); - mbgl::Log::Warning(mbgl::Event::Setup, "Application is offline. Press `O` to toggle online status."); + if (onlineFileSource) { + onlineFileSource->setProperty("online-status", false); + mbgl::Log::Warning(mbgl::Event::Setup, "Application is offline. Press `O` to toggle online status."); + } else { + mbgl::Log::Warning(mbgl::Event::Setup, + "Network resource provider is not available, only local requests are supported."); + } } GLFWRendererFrontend rendererFrontend { std::make_unique(view->getRendererBackend(), view->getPixelRatio()), *view }; @@ -132,9 +139,14 @@ int main(int argc, char *argv[]) { if (testDirValue) view->setTestDirectory(args::get(testDirValue)); - view->setOnlineStatusCallback([&settings, fileSource]() { + view->setOnlineStatusCallback([&settings, onlineFileSource]() { + if (!onlineFileSource) { + mbgl::Log::Warning(mbgl::Event::Setup, + "Cannot change online status. Network resource provider is not available."); + return; + } settings.online = !settings.online; - fileSource->setOnlineStatus(settings.online); + onlineFileSource->setProperty("online-status", settings.online); mbgl::Log::Info(mbgl::Event::Setup, "Application is %s. Press `O` to toggle online status.", settings.online ? "online" : "offline"); }); @@ -152,20 +164,26 @@ int main(int argc, char *argv[]) { mbgl::Log::Info(mbgl::Event::Setup, "Changed style to: %s", newStyle.name); }); - view->setPauseResumeCallback([fileSource] () { + // Resource loader controls top-level request processing and can resume / pause all managed sources simultaneously. + auto resourceLoader = + mbgl::FileSourceManager::get()->getFileSource(mbgl::FileSourceType::ResourceLoader, resourceOptions); + view->setPauseResumeCallback([resourceLoader]() { static bool isPaused = false; if (isPaused) { - fileSource->resume(); + resourceLoader->resume(); } else { - fileSource->pause(); + resourceLoader->pause(); } isPaused = !isPaused; }); - view->setResetCacheCallback([fileSource] () { - fileSource->resetDatabase([](std::exception_ptr ex) { + // Database file source. + auto databaseFileSource = std::static_pointer_cast( + mbgl::FileSourceManager::get()->getFileSource(mbgl::FileSourceType::Database, resourceOptions)); + view->setResetCacheCallback([databaseFileSource]() { + databaseFileSource->resetDatabase([](std::exception_ptr ex) { if (ex) { mbgl::Log::Error(mbgl::Event::Database, "Failed to reset cache:: %s", mbgl::util::toString(ex).c_str()); } diff --git a/platform/linux/filesource-files.json b/platform/linux/filesource-files.json index 448f5f8613a..669a4e612ce 100644 --- a/platform/linux/filesource-files.json +++ b/platform/linux/filesource-files.json @@ -1,7 +1,6 @@ { "//": "This file can be edited manually and is the canonical source.", "sources": [ - "platform/default/src/mbgl/storage/file_source.cpp", "platform/default/src/mbgl/storage/http_file_source.cpp", "platform/default/src/mbgl/storage/sqlite3.cpp" ], diff --git a/render-test/file_source.cpp b/render-test/file_source.cpp index 4d6a800d1c1..f72bc08e376 100644 --- a/render-test/file_source.cpp +++ b/render-test/file_source.cpp @@ -1,6 +1,11 @@ +#include +#include #include +#include #include +#include + #include "file_source.hpp" namespace mbgl { @@ -10,8 +15,14 @@ std::atomic_size_t transferredSize{0}; std::atomic_bool active{false}; std::atomic_bool offline{true}; -ProxyFileSource::ProxyFileSource(const std::string& cachePath, const std::string& assetPath) - : DefaultFileSource(cachePath, assetPath, false) {} +ProxyFileSource::ProxyFileSource(std::shared_ptr defaultResourceLoader_, const ResourceOptions& options) + : defaultResourceLoader(std::move(defaultResourceLoader_)) { + assert(defaultResourceLoader); + if (offline) { + auto dbfs = FileSourceManager::get()->getFileSource(FileSourceType::Database, options); + dbfs->setProperty("read-only-mode", true); + } +} ProxyFileSource::~ProxyFileSource() = default; @@ -43,7 +54,7 @@ std::unique_ptr ProxyFileSource::request(const Resource& resource, } } - return DefaultFileSource::request(transformed, [=](Response response) { + return defaultResourceLoader->request(transformed, [=](Response response) { if (transformed.loadingMethod == Resource::LoadingMethod::CacheOnly && response.noContent) { if (transformed.kind == Resource::Kind::Tile && transformed.tileData) { mbgl::Log::Info(mbgl::Event::Database, @@ -64,19 +75,6 @@ std::unique_ptr ProxyFileSource::request(const Resource& resource, }); } -std::shared_ptr FileSource::createPlatformFileSource(const ResourceOptions& options) { - auto fileSource = std::make_shared(options.cachePath(), options.assetPath()); - - fileSource->setAccessToken(options.accessToken()); - fileSource->setAPIBaseURL(options.baseURL()); - - if (offline) { - fileSource->reopenDatabaseReadOnlyForTesting(); - } - - return fileSource; -} - // static void ProxyFileSource::setOffline(bool status) { offline = status; diff --git a/render-test/file_source.hpp b/render-test/file_source.hpp index 34ba739a228..d0496ab8f61 100644 --- a/render-test/file_source.hpp +++ b/render-test/file_source.hpp @@ -1,15 +1,18 @@ #pragma once -#include +#include namespace mbgl { -class ProxyFileSource : public DefaultFileSource { +class ResourceOptions; + +class ProxyFileSource : public FileSource { public: - ProxyFileSource(const std::string& cachePath, const std::string& assetPath); + ProxyFileSource(std::shared_ptr, const ResourceOptions&); ~ProxyFileSource(); - std::unique_ptr request(const Resource&, Callback); + std::unique_ptr request(const Resource&, Callback) override; + bool canRequest(const Resource&) const override { return true; } /** * @brief Flag to change the networking mode of the file source. @@ -39,6 +42,9 @@ class ProxyFileSource : public DefaultFileSource { * @return size_t */ static size_t getTransferredSize(); + +private: + std::shared_ptr defaultResourceLoader; }; } // namespace mbgl diff --git a/render-test/parser.cpp b/render-test/parser.cpp index 484428976ee..8147852c4f4 100644 --- a/render-test/parser.cpp +++ b/render-test/parser.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include diff --git a/render-test/runner.cpp b/render-test/runner.cpp index 04928680517..cbfb0c34bec 100644 --- a/render-test/runner.cpp +++ b/render-test/runner.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -86,7 +87,26 @@ std::string simpleDiff(const Value& result, const Value& expected) { } TestRunner::TestRunner(Manifest manifest_, UpdateResults updateResults_) - : manifest(std::move(manifest_)), updateResults(updateResults_) {} + : manifest(std::move(manifest_)), updateResults(updateResults_) { + registerProxyFileSource(); +} + +void TestRunner::registerProxyFileSource() { + static std::once_flag registerProxyFlag; + std::call_once(registerProxyFlag, [] { + auto* fileSourceManager = mbgl::FileSourceManager::get(); + + auto resourceLoaderFactory = + fileSourceManager->unRegisterFileSourceFactory(mbgl::FileSourceType::ResourceLoader); + auto factory = [defaultFactory = std::move(resourceLoaderFactory)](const mbgl::ResourceOptions& options) { + assert(defaultFactory); + std::shared_ptr fileSource = defaultFactory(options); + return std::make_unique(std::move(fileSource), options); + }; + + fileSourceManager->registerFileSourceFactory(mbgl::FileSourceType::ResourceLoader, std::move(factory)); + }); +} const Manifest& TestRunner::getManifest() const { return manifest; @@ -654,7 +674,7 @@ uint32_t getImageTileOffset(const std::set& dims, uint32_t dim) { TestRunner::Impl::Impl(const TestMetadata& metadata, const mbgl::ResourceOptions& resourceOptions) : observer(std::make_unique()), frontend(metadata.size, metadata.pixelRatio, swapBehavior(metadata.mapMode)), - fileSource(mbgl::FileSource::getSharedFileSource(resourceOptions)), + fileSource(mbgl::FileSourceManager::get()->getFileSource(mbgl::FileSourceType::ResourceLoader, resourceOptions)), map(frontend, *observer.get(), mbgl::MapOptions() diff --git a/render-test/runner.hpp b/render-test/runner.hpp index e6027e335c4..72320dd34fc 100644 --- a/render-test/runner.hpp +++ b/render-test/runner.hpp @@ -53,6 +53,7 @@ class TestRunner { TestMetadata&); void checkRenderTestResults(mbgl::PremultipliedImage&& image, TestMetadata&); void checkProbingResults(TestMetadata&); + void registerProxyFileSource(); struct Impl { Impl(const TestMetadata&, const mbgl::ResourceOptions&); diff --git a/src/core-files.json b/src/core-files.json index 58efba1ee6f..fcc508fb3a1 100644 --- a/src/core-files.json +++ b/src/core-files.json @@ -153,7 +153,7 @@ "src/mbgl/sprite/sprite_loader.cpp", "src/mbgl/sprite/sprite_loader_worker.cpp", "src/mbgl/sprite/sprite_parser.cpp", - "src/mbgl/storage/file_source.cpp", + "src/mbgl/storage/file_source_manager.cpp", "src/mbgl/storage/network_status.cpp", "src/mbgl/storage/resource.cpp", "src/mbgl/storage/resource_options.cpp", @@ -374,8 +374,9 @@ "mbgl/renderer/renderer_frontend.hpp": "include/mbgl/renderer/renderer_frontend.hpp", "mbgl/renderer/renderer_observer.hpp": "include/mbgl/renderer/renderer_observer.hpp", "mbgl/renderer/renderer_state.hpp": "include/mbgl/renderer/renderer_state.hpp", - "mbgl/storage/default_file_source.hpp": "include/mbgl/storage/default_file_source.hpp", + "mbgl/storage/database_file_source.hpp": "include/mbgl/storage/database_file_source.hpp", "mbgl/storage/file_source.hpp": "include/mbgl/storage/file_source.hpp", + "mbgl/storage/file_source_manager.hpp": "include/mbgl/storage/file_source_manager.hpp", "mbgl/storage/network_status.hpp": "include/mbgl/storage/network_status.hpp", "mbgl/storage/offline.hpp": "include/mbgl/storage/offline.hpp", "mbgl/storage/online_file_source.hpp": "include/mbgl/storage/online_file_source.hpp", @@ -695,6 +696,7 @@ "mbgl/storage/asset_file_source.hpp": "src/mbgl/storage/asset_file_source.hpp", "mbgl/storage/http_file_source.hpp": "src/mbgl/storage/http_file_source.hpp", "mbgl/storage/local_file_source.hpp": "src/mbgl/storage/local_file_source.hpp", + "mbgl/storage/main_resource_loader.hpp": "src/mbgl/storage/main_resource_loader.hpp", "mbgl/style/collection.hpp": "src/mbgl/style/collection.hpp", "mbgl/style/conversion/json.hpp": "src/mbgl/style/conversion/json.hpp", "mbgl/style/conversion/stringify.hpp": "src/mbgl/style/conversion/stringify.hpp", diff --git a/src/mbgl/map/map.cpp b/src/mbgl/map/map.cpp index a994af305ff..061669f5606 100644 --- a/src/mbgl/map/map.cpp +++ b/src/mbgl/map/map.cpp @@ -1,24 +1,24 @@ +#include +#include +#include #include #include -#include #include -#include -#include -#include -#include -#include +#include #include #include -#include +#include +#include #include #include +#include +#include #include -#include #include +#include #include +#include #include -#include -#include #include @@ -30,9 +30,11 @@ Map::Map(RendererFrontend& frontend, MapObserver& observer, const MapOptions& mapOptions, const ResourceOptions& resourceOptions) - : impl(std::make_unique(frontend, observer, - FileSource::getSharedFileSource(resourceOptions), - mapOptions)) {} + : impl(std::make_unique( + frontend, + observer, + FileSourceManager::get() ? FileSourceManager::get()->getFileSource(ResourceLoader, resourceOptions) : nullptr, + mapOptions)) {} Map::Map(std::unique_ptr impl_) : impl(std::move(impl_)) {} diff --git a/src/mbgl/map/map_impl.cpp b/src/mbgl/map/map_impl.cpp index 69c3de97834..bc2a37fe07d 100644 --- a/src/mbgl/map/map_impl.cpp +++ b/src/mbgl/map/map_impl.cpp @@ -11,15 +11,15 @@ Map::Impl::Impl(RendererFrontend& frontend_, MapObserver& observer_, std::shared_ptr fileSource_, const MapOptions& mapOptions) - : observer(observer_), - rendererFrontend(frontend_), - transform(observer, mapOptions.constrainMode(), mapOptions.viewportMode()), - mode(mapOptions.mapMode()), - pixelRatio(mapOptions.pixelRatio()), - crossSourceCollisions(mapOptions.crossSourceCollisions()), - fileSource(std::move(fileSource_)), - style(std::make_unique(*fileSource, pixelRatio)), - annotationManager(*style) { + : observer(observer_), + rendererFrontend(frontend_), + transform(observer, mapOptions.constrainMode(), mapOptions.viewportMode()), + mode(mapOptions.mapMode()), + pixelRatio(mapOptions.pixelRatio()), + crossSourceCollisions(mapOptions.crossSourceCollisions()), + fileSource(std::move(fileSource_)), + style(std::make_unique(fileSource, pixelRatio)), + annotationManager(*style) { transform.setNorthOrientation(mapOptions.northOrientation()); style->impl->setObserver(this); rendererFrontend.setObserver(*this); diff --git a/src/mbgl/sprite/sprite_loader.cpp b/src/mbgl/sprite/sprite_loader.cpp index bfb0c570d65..d4b1cade138 100644 --- a/src/mbgl/sprite/sprite_loader.cpp +++ b/src/mbgl/sprite/sprite_loader.cpp @@ -1,17 +1,18 @@ +#include +#include #include -#include #include +#include #include -#include -#include -#include -#include -#include #include #include #include -#include -#include +#include +#include +#include +#include +#include +#include #include diff --git a/src/mbgl/storage/asset_file_source.hpp b/src/mbgl/storage/asset_file_source.hpp index cc15dbb60bf..6dfd3ce4ad3 100644 --- a/src/mbgl/storage/asset_file_source.hpp +++ b/src/mbgl/storage/asset_file_source.hpp @@ -14,8 +14,9 @@ class AssetFileSource : public FileSource { ~AssetFileSource() override; std::unique_ptr request(const Resource&, Callback) override; - - static bool acceptsURL(const std::string& url); + bool canRequest(const Resource&) const override; + void pause() override; + void resume() override; private: class Impl; diff --git a/src/mbgl/storage/file_source.cpp b/src/mbgl/storage/file_source.cpp deleted file mode 100644 index 5f60a052780..00000000000 --- a/src/mbgl/storage/file_source.cpp +++ /dev/null @@ -1,37 +0,0 @@ -#include -#include -#include - -#include -#include - -namespace mbgl { - -std::shared_ptr FileSource::getSharedFileSource(const ResourceOptions& options) { - static std::mutex mutex; - static std::map> fileSources; - - std::lock_guard lock(mutex); - - // Purge entries no longer in use. - for (auto it = fileSources.begin(); it != fileSources.end();) { - it = it->second.expired() ? fileSources.erase(it) : ++it; - } - - const auto context = reinterpret_cast(options.platformContext()); - const std::string key = options.baseURL() + '|' + options.accessToken() + '|' + options.cachePath() + '|' + util::toString(context); - - std::shared_ptr fileSource; - auto tuple = fileSources.find(key); - if (tuple != fileSources.end()) { - fileSource = tuple->second.lock(); - } - - if (!fileSource) { - fileSources[key] = fileSource = createPlatformFileSource(options); - } - - return fileSource; -} - -} // namespace mbgl diff --git a/src/mbgl/storage/file_source_manager.cpp b/src/mbgl/storage/file_source_manager.cpp new file mode 100644 index 00000000000..6817717f1a1 --- /dev/null +++ b/src/mbgl/storage/file_source_manager.cpp @@ -0,0 +1,71 @@ +#include +#include +#include + +#include +#include +#include + +namespace mbgl { + +class FileSourceManager::Impl { +public: + using Key = std::tuple; + std::map> fileSources; + std::map fileSourceFactories; + std::recursive_mutex mutex; +}; + +FileSourceManager::FileSourceManager() : impl(std::make_unique()) {} + +FileSourceManager::~FileSourceManager() = default; + +std::shared_ptr FileSourceManager::getFileSource(FileSourceType type, + const ResourceOptions& options) noexcept { + std::lock_guard lock(impl->mutex); + + // Remove released file sources. + for (auto it = impl->fileSources.begin(); it != impl->fileSources.end();) { + it = it->second.expired() ? impl->fileSources.erase(it) : ++it; + } + + const auto context = reinterpret_cast(options.platformContext()); + const std::string optionsKey = + options.baseURL() + '|' + options.accessToken() + '|' + options.cachePath() + '|' + util::toString(context); + const auto key = std::tie(type, optionsKey); + + std::shared_ptr fileSource; + auto tuple = impl->fileSources.find(key); + if (tuple != impl->fileSources.end()) { + fileSource = tuple->second.lock(); + } + + if (!fileSource) { + auto it = impl->fileSourceFactories.find(type); + if (it != impl->fileSourceFactories.end()) { + assert(it->second); + impl->fileSources[key] = fileSource = it->second(options); + } + } + + return fileSource; +} + +void FileSourceManager::registerFileSourceFactory(FileSourceType type, FileSourceFactory&& factory) noexcept { + assert(factory); + std::lock_guard lock(impl->mutex); + impl->fileSourceFactories[type] = std::move(factory); +} + +FileSourceManager::FileSourceFactory FileSourceManager::unRegisterFileSourceFactory(FileSourceType type) noexcept { + std::lock_guard lock(impl->mutex); + auto it = impl->fileSourceFactories.find(type); + FileSourceFactory factory; + if (it != impl->fileSourceFactories.end()) { + factory = std::move(it->second); + impl->fileSourceFactories.erase(it); + } + return factory; +} + +} // namespace mbgl diff --git a/src/mbgl/storage/http_file_source.hpp b/src/mbgl/storage/http_file_source.hpp index 09834aa4dc4..693ea3414d3 100644 --- a/src/mbgl/storage/http_file_source.hpp +++ b/src/mbgl/storage/http_file_source.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include namespace mbgl { @@ -10,6 +11,9 @@ class HTTPFileSource : public FileSource { ~HTTPFileSource() override; std::unique_ptr request(const Resource&, Callback) override; + bool canRequest(const Resource& resource) const override { + return resource.hasLoadingMethod(Resource::LoadingMethod::Network); + } class Impl; diff --git a/src/mbgl/storage/local_file_source.hpp b/src/mbgl/storage/local_file_source.hpp index 0f065e0b5f8..39ebc8c4bdc 100644 --- a/src/mbgl/storage/local_file_source.hpp +++ b/src/mbgl/storage/local_file_source.hpp @@ -14,12 +14,12 @@ class LocalFileSource : public FileSource { ~LocalFileSource() override; std::unique_ptr request(const Resource&, Callback) override; - - static bool acceptsURL(const std::string& url); + bool canRequest(const Resource&) const override; + void pause() override; + void resume() override; private: class Impl; - std::unique_ptr> impl; }; diff --git a/src/mbgl/storage/main_resource_loader.hpp b/src/mbgl/storage/main_resource_loader.hpp new file mode 100644 index 00000000000..f78ff9af2e8 --- /dev/null +++ b/src/mbgl/storage/main_resource_loader.hpp @@ -0,0 +1,27 @@ +#pragma once + +#include + +namespace mbgl { + +class ResourceTransform; +class ResourceOptions; + +class MainResourceLoader final : public FileSource { +public: + explicit MainResourceLoader(const ResourceOptions& options); + ~MainResourceLoader() override; + + bool supportsCacheOnlyRequests() const override; + std::unique_ptr request(const Resource&, Callback) override; + bool canRequest(const Resource&) const override; + void pause() override; + void resume() override; + +private: + friend class ResourceLoaderRequestor; + class Impl; + const std::unique_ptr impl; +}; + +} // namespace mbgl diff --git a/src/mbgl/storage/resource_options.cpp b/src/mbgl/storage/resource_options.cpp index 21ecca979a7..c56a22540b0 100644 --- a/src/mbgl/storage/resource_options.cpp +++ b/src/mbgl/storage/resource_options.cpp @@ -10,7 +10,6 @@ class ResourceOptions::Impl { std::string cachePath = ":memory:"; std::string assetPath = "."; uint64_t maximumSize = mbgl::util::DEFAULT_MAX_CACHE_SIZE; - bool supportCacheOnlyRequests = true; void* platformContext = nullptr; }; @@ -69,15 +68,6 @@ uint64_t ResourceOptions::maximumCacheSize() const { return impl_->maximumSize; } -ResourceOptions& ResourceOptions::withCacheOnlyRequestsSupport(bool supportCacheOnlyRequests) { - impl_->supportCacheOnlyRequests = supportCacheOnlyRequests; - return *this; -} - -bool ResourceOptions::supportsCacheOnlyRequests() const { - return impl_->supportCacheOnlyRequests; -} - ResourceOptions& ResourceOptions::withPlatformContext(void* context) { impl_->platformContext = context; return *this; diff --git a/src/mbgl/storage/resource_transform.cpp b/src/mbgl/storage/resource_transform.cpp index 6596551e601..eaf10c93fdd 100644 --- a/src/mbgl/storage/resource_transform.cpp +++ b/src/mbgl/storage/resource_transform.cpp @@ -2,12 +2,12 @@ namespace mbgl { -ResourceTransform::ResourceTransform(ActorRef, TransformCallback&& callback) - : transformCallback(std::move(callback)) { -} +ResourceTransform::ResourceTransform(TransformCallback callback) : transformCallback(std::move(callback)) {} -void ResourceTransform::transform(Resource::Kind kind, const std::string& url, FinishedCallback&& finished) { - finished(transformCallback(kind, url)); +void ResourceTransform::transform(Resource::Kind kind, const std::string& url, FinishedCallback finished) { + assert(finished); + assert(transformCallback); + transformCallback(kind, url, std::move(finished)); } } // namespace mbgl diff --git a/src/mbgl/style/sources/geojson_source.cpp b/src/mbgl/style/sources/geojson_source.cpp index 7767859b38a..d8414c1f847 100644 --- a/src/mbgl/style/sources/geojson_source.cpp +++ b/src/mbgl/style/sources/geojson_source.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include diff --git a/src/mbgl/style/sources/image_source.cpp b/src/mbgl/style/sources/image_source.cpp index 4c18ae5818f..d55f7c9f09c 100644 --- a/src/mbgl/style/sources/image_source.cpp +++ b/src/mbgl/style/sources/image_source.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include diff --git a/src/mbgl/style/sources/raster_source.cpp b/src/mbgl/style/sources/raster_source.cpp index 851f32573ed..f90306945ec 100644 --- a/src/mbgl/style/sources/raster_source.cpp +++ b/src/mbgl/style/sources/raster_source.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include diff --git a/src/mbgl/style/sources/vector_source.cpp b/src/mbgl/style/sources/vector_source.cpp index dc1a45fdffa..510106adb9b 100644 --- a/src/mbgl/style/sources/vector_source.cpp +++ b/src/mbgl/style/sources/vector_source.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include diff --git a/src/mbgl/style/style.cpp b/src/mbgl/style/style.cpp index 783c8500979..8a821e5a5ed 100644 --- a/src/mbgl/style/style.cpp +++ b/src/mbgl/style/style.cpp @@ -8,9 +8,8 @@ namespace mbgl { namespace style { -Style::Style(FileSource& fileSource, float pixelRatio) - : impl(std::make_unique(fileSource, pixelRatio)) { -} +Style::Style(std::shared_ptr fileSource, float pixelRatio) + : impl(std::make_unique(std::move(fileSource), pixelRatio)) {} Style::~Style() = default; diff --git a/src/mbgl/style/style_impl.cpp b/src/mbgl/style/style_impl.cpp index ef1f8436fc6..d5961b5901e 100644 --- a/src/mbgl/style/style_impl.cpp +++ b/src/mbgl/style/style_impl.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -29,8 +30,8 @@ namespace style { static Observer nullObserver; -Style::Impl::Impl(FileSource& fileSource_, float pixelRatio) - : fileSource(fileSource_), +Style::Impl::Impl(std::shared_ptr fileSource_, float pixelRatio) + : fileSource(std::move(fileSource_)), spriteLoader(std::make_unique(pixelRatio)), light(std::make_unique()), observer(&nullObserver) { @@ -49,13 +50,19 @@ void Style::Impl::loadJSON(const std::string& json_) { } void Style::Impl::loadURL(const std::string& url_) { + if (!fileSource) { + observer->onStyleError( + std::make_exception_ptr(util::StyleLoadException("Unable to find resource provider for style url."))); + return; + } + lastError = nullptr; observer->onStyleLoading(); loaded = false; url = url_; - styleRequest = fileSource.request(Resource::style(url), [this](Response res) { + styleRequest = fileSource->request(Resource::style(url), [this](Response res) { // Don't allow a loaded, mutated style to be overwritten with a new version. if (mutated && loaded) { return; @@ -112,7 +119,11 @@ void Style::Impl::parse(const std::string& json_) { setLight(std::make_unique(parser.light)); spriteLoaded = false; - spriteLoader->load(parser.spriteURL, fileSource); + if (fileSource) { + spriteLoader->load(parser.spriteURL, *fileSource); + } else { + onSpriteError(std::make_exception_ptr(std::runtime_error("Unable to find resource provider for sprite url."))); + } glyphURL = parser.glyphURL; loaded = true; @@ -143,7 +154,9 @@ void Style::Impl::addSource(std::unique_ptr source) { source->setObserver(this); auto item = sources.add(std::move(source)); - item->loadDescription(fileSource); + if (fileSource) { + item->loadDescription(*fileSource); + } } std::unique_ptr Style::Impl::removeSource(const std::string& id) { @@ -301,8 +314,8 @@ void Style::Impl::onSourceError(Source& source, std::exception_ptr error) { void Style::Impl::onSourceDescriptionChanged(Source& source) { sources.update(source); observer->onSourceDescriptionChanged(source); - if (!source.loaded) { - source.loadDescription(fileSource); + if (!source.loaded && fileSource) { + source.loadDescription(*fileSource); } } diff --git a/src/mbgl/style/style_impl.hpp b/src/mbgl/style/style_impl.hpp index c4c0a9a4129..ca165e24f04 100644 --- a/src/mbgl/style/style_impl.hpp +++ b/src/mbgl/style/style_impl.hpp @@ -37,7 +37,7 @@ class Style::Impl : public SpriteLoaderObserver, public LightObserver, public util::noncopyable { public: - Impl(FileSource&, float pixelRatio); + Impl(std::shared_ptr, float pixelRatio); ~Impl() override; void loadJSON(const std::string&); @@ -97,7 +97,7 @@ class Style::Impl : public SpriteLoaderObserver, private: void parse(const std::string&); - FileSource& fileSource; + std::shared_ptr fileSource; std::string url; std::string json; diff --git a/src/mbgl/text/glyph_manager.cpp b/src/mbgl/text/glyph_manager.cpp index 35ea1031d53..8caac1be317 100644 --- a/src/mbgl/text/glyph_manager.cpp +++ b/src/mbgl/text/glyph_manager.cpp @@ -1,11 +1,12 @@ -#include -#include -#include #include #include #include -#include +#include +#include +#include +#include #include +#include namespace mbgl { diff --git a/src/mbgl/tile/geometry_tile.cpp b/src/mbgl/tile/geometry_tile.cpp index 7df81fa30f6..67e4459104c 100644 --- a/src/mbgl/tile/geometry_tile.cpp +++ b/src/mbgl/tile/geometry_tile.cpp @@ -262,7 +262,9 @@ void GeometryTile::onGlyphsAvailable(GlyphMap glyphs) { } void GeometryTile::getGlyphs(GlyphDependencies glyphDependencies) { - glyphManager.getGlyphs(*this, std::move(glyphDependencies), *fileSource); + if (fileSource) { + glyphManager.getGlyphs(*this, std::move(glyphDependencies), *fileSource); + } } void GeometryTile::onImagesAvailable(ImageMap images, ImageMap patterns, ImageVersionMap versionMap, uint64_t imageCorrelationID) { diff --git a/src/mbgl/tile/tile_loader_impl.hpp b/src/mbgl/tile/tile_loader_impl.hpp index 7c020210834..51efbb99e98 100644 --- a/src/mbgl/tile/tile_loader_impl.hpp +++ b/src/mbgl/tile/tile_loader_impl.hpp @@ -1,12 +1,19 @@ #pragma once -#include -#include #include +#include +#include +#include #include #include +namespace { +inline std::exception_ptr getCantLoadTileError() { + return std::make_exception_ptr(std::runtime_error("Can't load tile.")); +} +} // namespace + namespace mbgl { template @@ -26,6 +33,11 @@ TileLoader::TileLoader(T& tile_, Resource::LoadingMethod::CacheOnly)), fileSource(parameters.fileSource) { assert(!request); + if (!fileSource) { + tile.setError(getCantLoadTileError()); + return; + } + if (fileSource->supportsCacheOnlyRequests()) { // When supported, the first request is always optional, even if the TileLoader // is marked as required. That way, we can let the first optional request continue @@ -49,6 +61,10 @@ TileLoader::~TileLoader() = default; template void TileLoader::loadFromCache() { assert(!request); + if (!fileSource) { + tile.setError(getCantLoadTileError()); + return; + } resource.loadingMethod = Resource::LoadingMethod::CacheOnly; request = fileSource->request(resource, [this](Response res) { @@ -113,6 +129,10 @@ void TileLoader::loadedData(const Response& res) { template void TileLoader::loadFromNetwork() { assert(!request); + if (!fileSource) { + tile.setError(getCantLoadTileError()); + return; + } // Instead of using Resource::LoadingMethod::All, we're first doing a CacheOnly, and then a // NetworkOnly request. diff --git a/test/map/map.test.cpp b/test/map/map.test.cpp index 3eb01a73837..89be4ad73ec 100644 --- a/test/map/map.test.cpp +++ b/test/map/map.test.cpp @@ -10,7 +10,8 @@ #include #include #include -#include +#include +#include #include #include #include @@ -47,13 +48,17 @@ class MapTest { MapOptions().withMapMode(mode).withSize(frontend.getSize()).withPixelRatio(pixelRatio)) {} template - MapTest(const std::string& cachePath, const std::string& assetPath, - float pixelRatio = 1, MapMode mode = MapMode::Static, - typename std::enable_if::value>::type* = nullptr) - : fileSource(std::make_shared(cachePath, assetPath)) - , frontend(pixelRatio) - , map(frontend, observer, fileSource, - MapOptions().withMapMode(mode).withSize(frontend.getSize()).withPixelRatio(pixelRatio)) {} + MapTest(const std::string& cachePath, + const std::string& assetPath, + float pixelRatio = 1, + MapMode mode = MapMode::Static, + typename std::enable_if::value>::type* = nullptr) + : fileSource(std::make_shared(ResourceOptions().withCachePath(cachePath).withAssetPath(assetPath))), + frontend(pixelRatio), + map(frontend, + observer, + fileSource, + MapOptions().withMapMode(mode).withSize(frontend.getSize()).withPixelRatio(pixelRatio)) {} }; TEST(Map, RendererState) { @@ -295,7 +300,7 @@ TEST(Map, CameraToLatLngBoundsUnwrappedCrossDateLine) { } TEST(Map, Offline) { - MapTest test {":memory:", "."}; + MapTest test{":memory:", "."}; auto expiredItem = [] (const std::string& path) { Response response; @@ -304,19 +309,21 @@ TEST(Map, Offline) { return response; }; - const std::string prefix = "http://127.0.0.1:3000/"; - test.fileSource->put(Resource::style(prefix + "style.json"), expiredItem("style.json")); - test.fileSource->put(Resource::source(prefix + "streets.json"), expiredItem("streets.json")); - test.fileSource->put(Resource::spriteJSON(prefix + "sprite", 1.0), expiredItem("sprite.json")); - test.fileSource->put(Resource::spriteImage(prefix + "sprite", 1.0), expiredItem("sprite.png")); - test.fileSource->put(Resource::tile(prefix + "{z}-{x}-{y}.vector.pbf", 1.0, 0, 0, 0, Tileset::Scheme::XYZ), expiredItem("0-0-0.vector.pbf")); - test.fileSource->put(Resource::glyphs(prefix + "{fontstack}/{range}.pbf", {{"Helvetica"}}, {0, 255}), expiredItem("glyph.pbf")); NetworkStatus::Set(NetworkStatus::Status::Offline); - - test.map.getStyle().loadURL(prefix + "style.json"); + const std::string prefix = "http://127.0.0.1:3000/"; + auto dbfs = FileSourceManager::get()->getFileSource(FileSourceType::Database, ResourceOptions{}); + dbfs->forward(Resource::style(prefix + "style.json"), expiredItem("style.json")); + dbfs->forward(Resource::source(prefix + "streets.json"), expiredItem("streets.json")); + dbfs->forward(Resource::spriteJSON(prefix + "sprite", 1.0), expiredItem("sprite.json")); + dbfs->forward(Resource::spriteImage(prefix + "sprite", 1.0), expiredItem("sprite.png")); + dbfs->forward(Resource::tile(prefix + "{z}-{x}-{y}.vector.pbf", 1.0, 0, 0, 0, Tileset::Scheme::XYZ), + expiredItem("0-0-0.vector.pbf")); + dbfs->forward(Resource::glyphs(prefix + "{fontstack}/{range}.pbf", {{"Helvetica"}}, {0, 255}), + expiredItem("glyph.pbf"), + [&] { test.map.getStyle().loadURL(prefix + "style.json"); }); #if ANDROID - test::checkImage("test/fixtures/map/offline", test.frontend.render(test.map).image, 0.0045, 0.1); + test::checkImage("test/fixtures/map/offline", test.frontend.render(test.map).image, 0.0046, 0.1); #else test::checkImage("test/fixtures/map/offline", test.frontend.render(test.map).image, 0.0015, 0.1); #endif @@ -672,7 +679,7 @@ TEST(Map, WithoutVAOExtension) { return; } - MapTest test { ":memory:", "test/fixtures/api/assets" }; + MapTest test{":memory:", "test/fixtures/api/assets"}; gfx::BackendScope scope { *test.frontend.getBackend() }; static_cast(test.frontend.getBackend()->getContext()).disableVAOExtension = true; @@ -836,7 +843,7 @@ TEST(Map, TEST_DISABLED_ON_CI(ContinuousRendering)) { } TEST(Map, NoContentTiles) { - MapTest test {":memory:", "."}; + MapTest test{":memory:", "."}; using namespace std::chrono_literals; @@ -844,33 +851,32 @@ TEST(Map, NoContentTiles) { Response response; response.noContent = true; response.expires = util::now() + 1h; - test.fileSource->put(Resource::tile("http://example.com/{z}-{x}-{y}.vector.pbf", 1.0, 0, 0, 0, - Tileset::Scheme::XYZ), - response); - - test.map.getStyle().loadJSON(R"STYLE({ - "version": 8, - "name": "Water", - "sources": { - "mapbox": { - "type": "vector", - "tiles": ["http://example.com/{z}-{x}-{y}.vector.pbf"] - } - }, - "layers": [{ - "id": "background", - "type": "background", - "paint": { - "background-color": "red" - } - }, { - "id": "water", - "type": "fill", - "source": "mapbox", - "source-layer": "water" - }] - })STYLE"); - + auto dbfs = FileSourceManager::get()->getFileSource(FileSourceType::Database, ResourceOptions{}); + dbfs->forward( + Resource::tile("http://example.com/{z}-{x}-{y}.vector.pbf", 1.0, 0, 0, 0, Tileset::Scheme::XYZ), response, [&] { + test.map.getStyle().loadJSON(R"STYLE({ + "version": 8, + "name": "Water", + "sources": { + "mapbox": { + "type": "vector", + "tiles": ["http://example.com/{z}-{x}-{y}.vector.pbf"] + } + }, + "layers": [{ + "id": "background", + "type": "background", + "paint": { + "background-color": "red" + } + }, { + "id": "water", + "type": "fill", + "source": "mapbox", + "source-layer": "water" + }] + })STYLE"); + }); test::checkImage("test/fixtures/map/nocontent", test.frontend.render(test.map).image, 0.0015, 0.1); } diff --git a/test/src/mbgl/test/fake_file_source.hpp b/test/src/mbgl/test/fake_file_source.hpp index 8803e9576b2..1faf4b7a18e 100644 --- a/test/src/mbgl/test/fake_file_source.hpp +++ b/test/src/mbgl/test/fake_file_source.hpp @@ -2,6 +2,8 @@ #include #include +#include +#include #include #include @@ -42,6 +44,8 @@ class FakeFileSource : public FileSource { return std::make_unique(resource, callback, requests); } + bool canRequest(const Resource&) const override { return true; } + bool respond(Resource::Kind kind, const Response& response) { auto it = std::find_if(requests.begin(), requests.end(), [&] (FakeFileRequest* fakeRequest) { return fakeRequest->resource.kind == kind; @@ -60,7 +64,7 @@ class FakeFileSource : public FileSource { }; -class FakeOnlineFileSource : public OnlineFileSource, public FakeFileSource { +class FakeOnlineFileSource : public FakeFileSource { public: std::unique_ptr request(const Resource& resource, Callback callback) override { return FakeFileSource::request(resource, callback); @@ -69,7 +73,12 @@ class FakeOnlineFileSource : public OnlineFileSource, public FakeFileSource { bool respond(Resource::Kind kind, const Response& response) { return FakeFileSource::respond(kind, response); } -}; + mapbox::base::Value getProperty(const std::string& property) const override { + return onlineFs.getProperty(property); + } + + OnlineFileSource onlineFs; +}; } // namespace mbgl diff --git a/test/src/mbgl/test/stub_file_source.cpp b/test/src/mbgl/test/stub_file_source.cpp index 0bbff84ff32..8870a45bdca 100644 --- a/test/src/mbgl/test/stub_file_source.cpp +++ b/test/src/mbgl/test/stub_file_source.cpp @@ -1,4 +1,5 @@ #include +#include namespace mbgl { diff --git a/test/src/mbgl/test/stub_file_source.hpp b/test/src/mbgl/test/stub_file_source.hpp index 1135fa9a80a..46bb33d5e23 100644 --- a/test/src/mbgl/test/stub_file_source.hpp +++ b/test/src/mbgl/test/stub_file_source.hpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -19,6 +20,7 @@ class StubFileSource : public FileSource { ~StubFileSource() override; std::unique_ptr request(const Resource&, Callback) override; + bool canRequest(const Resource&) const override { return true; } void remove(AsyncRequest*); using ResponseFunction = std::function (const Resource&)>; @@ -48,15 +50,4 @@ class StubFileSource : public FileSource { util::Timer timer; }; -class StubOnlineFileSource : public StubFileSource, public OnlineFileSource { -public: - - StubOnlineFileSource(ResponseType t = ResponseType::Asynchronous) : StubFileSource(t) {}; - ~StubOnlineFileSource() override = default; - - std::unique_ptr request(const Resource& r, Callback c) override { return StubFileSource::request(r, c); }; - void remove(AsyncRequest* r) { StubFileSource::remove(r); }; -}; - - } // namespace mbgl diff --git a/test/storage/asset_file_source.test.cpp b/test/storage/asset_file_source.test.cpp index 978a41a306c..ac04bc7dc2d 100644 --- a/test/storage/asset_file_source.test.cpp +++ b/test/storage/asset_file_source.test.cpp @@ -1,9 +1,10 @@ +#include #include -#include +#include #include +#include #include #include -#include #include #include @@ -70,12 +71,13 @@ TEST(AssetFileSource, Load) { } TEST(AssetFileSource, AcceptsURL) { - EXPECT_TRUE(AssetFileSource::acceptsURL("asset://empty")); - EXPECT_TRUE(AssetFileSource::acceptsURL("asset:///test")); - EXPECT_FALSE(AssetFileSource::acceptsURL("assds://foo")); - EXPECT_FALSE(AssetFileSource::acceptsURL("asset:")); - EXPECT_FALSE(AssetFileSource::acceptsURL("style.json")); - EXPECT_FALSE(AssetFileSource::acceptsURL("")); + AssetFileSource fs("test/fixtures/storage/assets"); + EXPECT_TRUE(fs.canRequest(Resource::style("asset://empty"))); + EXPECT_TRUE(fs.canRequest(Resource::style("asset:///test"))); + EXPECT_FALSE(fs.canRequest(Resource::style("assds://foo"))); + EXPECT_FALSE(fs.canRequest(Resource::style("asset:"))); + EXPECT_FALSE(fs.canRequest(Resource::style("style.json"))); + EXPECT_FALSE(fs.canRequest(Resource::style(""))); } TEST(AssetFileSource, EmptyFile) { diff --git a/test/storage/http_file_source.test.cpp b/test/storage/http_file_source.test.cpp index 42b4174e696..37476c8e7c4 100644 --- a/test/storage/http_file_source.test.cpp +++ b/test/storage/http_file_source.test.cpp @@ -1,9 +1,10 @@ -#include #include -#include +#include +#include #include -#include +#include #include +#include using namespace mbgl; diff --git a/test/storage/local_file_source.test.cpp b/test/storage/local_file_source.test.cpp index e1756f8e7d0..45c8c54d917 100644 --- a/test/storage/local_file_source.test.cpp +++ b/test/storage/local_file_source.test.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -21,12 +22,13 @@ std::string toAbsoluteURL(const std::string& fileName) { using namespace mbgl; TEST(LocalFileSource, AcceptsURL) { - EXPECT_TRUE(LocalFileSource::acceptsURL("file://empty")); - EXPECT_TRUE(LocalFileSource::acceptsURL("file:///test")); - EXPECT_FALSE(LocalFileSource::acceptsURL("flie://foo")); - EXPECT_FALSE(LocalFileSource::acceptsURL("file:")); - EXPECT_FALSE(LocalFileSource::acceptsURL("style.json")); - EXPECT_FALSE(LocalFileSource::acceptsURL("")); + LocalFileSource fs; + EXPECT_TRUE(fs.canRequest(Resource::style("file://empty"))); + EXPECT_TRUE(fs.canRequest(Resource::style("file:///test"))); + EXPECT_FALSE(fs.canRequest(Resource::style("flie://foo"))); + EXPECT_FALSE(fs.canRequest(Resource::style("file:"))); + EXPECT_FALSE(fs.canRequest(Resource::style("style.json"))); + EXPECT_FALSE(fs.canRequest(Resource::style(""))); } TEST(LocalFileSource, EmptyFile) { diff --git a/test/storage/default_file_source.test.cpp b/test/storage/main_resource_loader.test.cpp similarity index 64% rename from test/storage/default_file_source.test.cpp rename to test/storage/main_resource_loader.test.cpp index 52051ac839b..c5f1a9c707a 100644 --- a/test/storage/default_file_source.test.cpp +++ b/test/storage/main_resource_loader.test.cpp @@ -1,17 +1,21 @@ #include -#include +#include +#include +#include #include +#include +#include #include #include #include using namespace mbgl; -TEST(DefaultFileSource, TEST_REQUIRES_SERVER(CacheResponse)) { +TEST(MainResourceLoader, TEST_REQUIRES_SERVER(CacheResponse)) { util::RunLoop loop; - DefaultFileSource fs(":memory:", "."); + MainResourceLoader fs(ResourceOptions{}); - const Resource resource { Resource::Unknown, "http://127.0.0.1:3000/cache" }; + const Resource resource{Resource::Unknown, "http://127.0.0.1:3000/cache"}; Response response; std::unique_ptr req1; @@ -47,11 +51,11 @@ TEST(DefaultFileSource, TEST_REQUIRES_SERVER(CacheResponse)) { loop.run(); } -TEST(DefaultFileSource, TEST_REQUIRES_SERVER(CacheRevalidateSame)) { +TEST(MainResourceLoader, TEST_REQUIRES_SERVER(CacheRevalidateSame)) { util::RunLoop loop; - DefaultFileSource fs(":memory:", "."); + MainResourceLoader fs(ResourceOptions{}); - const Resource revalidateSame { Resource::Unknown, "http://127.0.0.1:3000/revalidate-same" }; + const Resource revalidateSame{Resource::Unknown, "http://127.0.0.1:3000/revalidate-same"}; std::unique_ptr req1; std::unique_ptr req2; bool gotResponse = false; @@ -109,12 +113,11 @@ TEST(DefaultFileSource, TEST_REQUIRES_SERVER(CacheRevalidateSame)) { loop.run(); } -TEST(DefaultFileSource, TEST_REQUIRES_SERVER(CacheRevalidateModified)) { +TEST(MainResourceLoader, TEST_REQUIRES_SERVER(CacheRevalidateModified)) { util::RunLoop loop; - DefaultFileSource fs(":memory:", "."); + MainResourceLoader fs(ResourceOptions{}); - const Resource revalidateModified{ Resource::Unknown, - "http://127.0.0.1:3000/revalidate-modified" }; + const Resource revalidateModified{Resource::Unknown, "http://127.0.0.1:3000/revalidate-modified"}; std::unique_ptr req1; std::unique_ptr req2; bool gotResponse = false; @@ -129,7 +132,7 @@ TEST(DefaultFileSource, TEST_REQUIRES_SERVER(CacheRevalidateModified)) { EXPECT_EQ("Response", *res.data); EXPECT_FALSE(bool(res.expires)); EXPECT_TRUE(res.mustRevalidate); - EXPECT_EQ(Timestamp{ Seconds(1420070400) }, *res.modified); + EXPECT_EQ(Timestamp{Seconds(1420070400)}, *res.modified); EXPECT_FALSE(res.etag); // The first response is stored in the cache, but it has 'must-revalidate' set. This means @@ -149,7 +152,7 @@ TEST(DefaultFileSource, TEST_REQUIRES_SERVER(CacheRevalidateModified)) { EXPECT_EQ("Response", *res2.data); EXPECT_TRUE(bool(res2.expires)); EXPECT_TRUE(res2.mustRevalidate); - EXPECT_EQ(Timestamp{ Seconds(1420070400) }, *res2.modified); + EXPECT_EQ(Timestamp{Seconds(1420070400)}, *res2.modified); EXPECT_FALSE(res2.etag); } else { // The test server sends a Cache-Control header with a max-age of 1 second. This @@ -162,7 +165,7 @@ TEST(DefaultFileSource, TEST_REQUIRES_SERVER(CacheRevalidateModified)) { EXPECT_FALSE(res2.data.get()); EXPECT_TRUE(bool(res2.expires)); EXPECT_TRUE(res2.mustRevalidate); - EXPECT_EQ(Timestamp{ Seconds(1420070400) }, *res2.modified); + EXPECT_EQ(Timestamp{Seconds(1420070400)}, *res2.modified); EXPECT_FALSE(res2.etag); loop.stop(); } @@ -172,11 +175,11 @@ TEST(DefaultFileSource, TEST_REQUIRES_SERVER(CacheRevalidateModified)) { loop.run(); } -TEST(DefaultFileSource, TEST_REQUIRES_SERVER(CacheRevalidateEtag)) { +TEST(MainResourceLoader, TEST_REQUIRES_SERVER(CacheRevalidateEtag)) { util::RunLoop loop; - DefaultFileSource fs(":memory:", "."); + MainResourceLoader fs(ResourceOptions{}); - const Resource revalidateEtag { Resource::Unknown, "http://127.0.0.1:3000/revalidate-etag" }; + const Resource revalidateEtag{Resource::Unknown, "http://127.0.0.1:3000/revalidate-etag"}; std::unique_ptr req1; std::unique_ptr req2; @@ -222,16 +225,13 @@ TEST(DefaultFileSource, TEST_REQUIRES_SERVER(CacheRevalidateEtag)) { // will notify as expected, the second one will have bound a DefaultFileRequest* in the lambda that // gets invalidated by the first notify's pending.erase, and when it gets notified, the crash // occurs. - -TEST(DefaultFileSource, TEST_REQUIRES_SERVER(HTTPIssue1369)) { +TEST(MainResourceLoader, TEST_REQUIRES_SERVER(HTTPIssue1369)) { util::RunLoop loop; - DefaultFileSource fs(":memory:", "."); + MainResourceLoader fs(ResourceOptions{}); - const Resource resource { Resource::Unknown, "http://127.0.0.1:3000/test" }; + const Resource resource{Resource::Unknown, "http://127.0.0.1:3000/test"}; - auto req = fs.request(resource, [&](Response) { - ADD_FAILURE() << "Callback should not be called"; - }); + auto req = fs.request(resource, [&](Response) { ADD_FAILURE() << "Callback should not be called"; }); req.reset(); req = fs.request(resource, [&](Response res) { req.reset(); @@ -248,87 +248,77 @@ TEST(DefaultFileSource, TEST_REQUIRES_SERVER(HTTPIssue1369)) { loop.run(); } -TEST(DefaultFileSource, OptionalNonExpired) { +TEST(MainResourceLoader, OptionalNonExpired) { util::RunLoop loop; - DefaultFileSource fs(":memory:", "."); + MainResourceLoader fs(ResourceOptions{}); - const Resource optionalResource { Resource::Unknown, "http://127.0.0.1:3000/test", {}, Resource::LoadingMethod::CacheOnly }; + const Resource optionalResource{ + Resource::Unknown, "http://127.0.0.1:3000/test", {}, Resource::LoadingMethod::CacheOnly}; using namespace std::chrono_literals; Response response; response.data = std::make_shared("Cached value"); response.expires = util::now() + 1h; - fs.put(optionalResource, response); std::unique_ptr req; - req = fs.request(optionalResource, [&](Response res) { - req.reset(); - EXPECT_EQ(nullptr, res.error); - ASSERT_TRUE(res.data.get()); - EXPECT_EQ("Cached value", *res.data); - ASSERT_TRUE(bool(res.expires)); - EXPECT_EQ(*response.expires, *res.expires); - EXPECT_FALSE(res.mustRevalidate); - EXPECT_FALSE(bool(res.modified)); - EXPECT_FALSE(bool(res.etag)); - loop.stop(); + auto dbfs = FileSourceManager::get()->getFileSource(FileSourceType::Database, ResourceOptions{}); + dbfs->forward(optionalResource, response, [&] { + req = fs.request(optionalResource, [&](Response res) { + req.reset(); + EXPECT_EQ(nullptr, res.error); + ASSERT_TRUE(res.data.get()); + EXPECT_EQ("Cached value", *res.data); + ASSERT_TRUE(bool(res.expires)); + EXPECT_EQ(*response.expires, *res.expires); + EXPECT_FALSE(res.mustRevalidate); + EXPECT_FALSE(bool(res.modified)); + EXPECT_FALSE(bool(res.etag)); + loop.stop(); + }); }); loop.run(); } -TEST(DefaultFileSource, OptionalExpired) { +TEST(MainResourceLoader, OptionalExpired) { util::RunLoop loop; - DefaultFileSource fs(":memory:", "."); + MainResourceLoader fs(ResourceOptions{}); - const Resource optionalResource { Resource::Unknown, "http://127.0.0.1:3000/test", {}, Resource::LoadingMethod::CacheOnly }; + const Resource optionalResource{ + Resource::Unknown, "http://127.0.0.1:3000/test", {}, Resource::LoadingMethod::CacheOnly}; using namespace std::chrono_literals; Response response; response.data = std::make_shared("Cached value"); response.expires = util::now() - 1h; - fs.put(optionalResource, response); - + auto dbfs = FileSourceManager::get()->getFileSource(FileSourceType::Database, ResourceOptions{}); std::unique_ptr req; - req = fs.request(optionalResource, [&](Response res) { - req.reset(); - EXPECT_EQ(nullptr, res.error); - ASSERT_TRUE(res.data.get()); - EXPECT_EQ("Cached value", *res.data); - ASSERT_TRUE(bool(res.expires)); - EXPECT_EQ(*response.expires, *res.expires); - EXPECT_FALSE(res.mustRevalidate); - EXPECT_FALSE(bool(res.modified)); - EXPECT_FALSE(bool(res.etag)); - loop.stop(); + dbfs->forward(optionalResource, response, [&] { + req = fs.request(optionalResource, [&](Response res) { + req.reset(); + EXPECT_EQ(nullptr, res.error); + ASSERT_TRUE(res.data.get()); + EXPECT_EQ("Cached value", *res.data); + ASSERT_TRUE(bool(res.expires)); + EXPECT_EQ(*response.expires, *res.expires); + EXPECT_FALSE(res.mustRevalidate); + EXPECT_FALSE(bool(res.modified)); + EXPECT_FALSE(bool(res.etag)); + loop.stop(); + }); }); loop.run(); } -TEST(DefaultFileSource, GetBaseURLAndAccessTokenWhilePaused) { +TEST(MainResourceLoader, OptionalNotFound) { util::RunLoop loop; - DefaultFileSource fs(":memory:", "."); + MainResourceLoader fs(ResourceOptions{}); - fs.pause(); - - auto baseURL = "http://url"; - auto accessToken = "access_token"; - - fs.setAPIBaseURL(baseURL); - fs.setAccessToken(accessToken); - - EXPECT_EQ(fs.getAPIBaseURL(), baseURL); - EXPECT_EQ(fs.getAccessToken(), accessToken); -} - -TEST(DefaultFileSource, OptionalNotFound) { - util::RunLoop loop; - DefaultFileSource fs(":memory:", "."); - - const Resource optionalResource { Resource::Unknown, "http://127.0.0.1:3000/test", {}, Resource::LoadingMethod::CacheOnly }; + const Resource optionalResource{ + Resource::Unknown, "http://127.0.0.1:3000/test", {}, Resource::LoadingMethod::CacheOnly}; using namespace std::chrono_literals; @@ -350,9 +340,9 @@ TEST(DefaultFileSource, OptionalNotFound) { } // Test that a network only request doesn't attempt to load data from the cache. -TEST(DefaultFileSource, TEST_REQUIRES_SERVER(NoCacheRefreshEtagNotModified)) { +TEST(MainResourceLoader, TEST_REQUIRES_SERVER(NoCacheRefreshEtagNotModified)) { util::RunLoop loop; - DefaultFileSource fs(":memory:", "."); + MainResourceLoader fs(ResourceOptions{}); Resource resource { Resource::Unknown, "http://127.0.0.1:3000/revalidate-same" }; resource.loadingMethod = Resource::LoadingMethod::NetworkOnly; @@ -364,30 +354,31 @@ TEST(DefaultFileSource, TEST_REQUIRES_SERVER(NoCacheRefreshEtagNotModified)) { Response response; response.data = std::make_shared("Cached value"); response.expires = util::now() + 1h; - fs.put(resource, response); - std::unique_ptr req; - req = fs.request(resource, [&](Response res) { - req.reset(); - EXPECT_EQ(nullptr, res.error); - EXPECT_TRUE(res.notModified); - EXPECT_FALSE(res.data.get()); - ASSERT_TRUE(bool(res.expires)); - EXPECT_LT(util::now(), *res.expires); - EXPECT_TRUE(res.mustRevalidate); - EXPECT_FALSE(bool(res.modified)); - ASSERT_TRUE(bool(res.etag)); - EXPECT_EQ("snowfall", *res.etag); - loop.stop(); + auto dbfs = FileSourceManager::get()->getFileSource(FileSourceType::Database, ResourceOptions{}); + dbfs->forward(resource, response, [&] { + req = fs.request(resource, [&](Response res) { + req.reset(); + EXPECT_EQ(nullptr, res.error); + EXPECT_TRUE(res.notModified); + EXPECT_FALSE(res.data.get()); + ASSERT_TRUE(bool(res.expires)); + EXPECT_LT(util::now(), *res.expires); + EXPECT_TRUE(res.mustRevalidate); + EXPECT_FALSE(bool(res.modified)); + ASSERT_TRUE(bool(res.etag)); + EXPECT_EQ("snowfall", *res.etag); + loop.stop(); + }); }); loop.run(); } // Test that a network only request doesn't attempt to load data from the cache. -TEST(DefaultFileSource, TEST_REQUIRES_SERVER(NoCacheRefreshEtagModified)) { +TEST(MainResourceLoader, TEST_REQUIRES_SERVER(NoCacheRefreshEtagModified)) { util::RunLoop loop; - DefaultFileSource fs(":memory:", "."); + MainResourceLoader fs(ResourceOptions{}); Resource resource { Resource::Unknown, "http://127.0.0.1:3000/revalidate-same" }; resource.loadingMethod = Resource::LoadingMethod::NetworkOnly; @@ -399,30 +390,31 @@ TEST(DefaultFileSource, TEST_REQUIRES_SERVER(NoCacheRefreshEtagModified)) { Response response; response.data = std::make_shared("Cached value"); response.expires = util::now() + 1h; - fs.put(resource, response); - std::unique_ptr req; - req = fs.request(resource, [&](Response res) { - req.reset(); - EXPECT_EQ(nullptr, res.error); - EXPECT_FALSE(res.notModified); - ASSERT_TRUE(res.data.get()); - EXPECT_EQ("Response", *res.data); - EXPECT_FALSE(bool(res.expires)); - EXPECT_TRUE(res.mustRevalidate); - EXPECT_FALSE(bool(res.modified)); - ASSERT_TRUE(bool(res.etag)); - EXPECT_EQ("snowfall", *res.etag); - loop.stop(); + auto dbfs = FileSourceManager::get()->getFileSource(FileSourceType::Database, ResourceOptions{}); + dbfs->forward(resource, response, [&] { + req = fs.request(resource, [&](Response res) { + req.reset(); + EXPECT_EQ(nullptr, res.error); + EXPECT_FALSE(res.notModified); + ASSERT_TRUE(res.data.get()); + EXPECT_EQ("Response", *res.data); + EXPECT_FALSE(bool(res.expires)); + EXPECT_TRUE(res.mustRevalidate); + EXPECT_FALSE(bool(res.modified)); + ASSERT_TRUE(bool(res.etag)); + EXPECT_EQ("snowfall", *res.etag); + loop.stop(); + }); }); loop.run(); } // Test that a network only request doesn't attempt to load data from the cache. -TEST(DefaultFileSource, TEST_REQUIRES_SERVER(NoCacheFull)) { +TEST(MainResourceLoader, TEST_REQUIRES_SERVER(NoCacheFull)) { util::RunLoop loop; - DefaultFileSource fs(":memory:", "."); + MainResourceLoader fs(ResourceOptions{}); Resource resource { Resource::Unknown, "http://127.0.0.1:3000/revalidate-same" }; resource.loadingMethod = Resource::LoadingMethod::NetworkOnly; @@ -433,21 +425,22 @@ TEST(DefaultFileSource, TEST_REQUIRES_SERVER(NoCacheFull)) { Response response; response.data = std::make_shared("Cached value"); response.expires = util::now() + 1h; - fs.put(resource, response); - std::unique_ptr req; - req = fs.request(resource, [&](Response res) { - req.reset(); - EXPECT_EQ(nullptr, res.error); - EXPECT_FALSE(res.notModified); - ASSERT_TRUE(res.data.get()); - EXPECT_EQ("Response", *res.data); - EXPECT_FALSE(bool(res.expires)); - EXPECT_TRUE(res.mustRevalidate); - EXPECT_FALSE(bool(res.modified)); - ASSERT_TRUE(bool(res.etag)); - EXPECT_EQ("snowfall", *res.etag); - loop.stop(); + auto dbfs = FileSourceManager::get()->getFileSource(FileSourceType::Database, ResourceOptions{}); + dbfs->forward(resource, response, [&] { + req = fs.request(resource, [&](Response res) { + req.reset(); + EXPECT_EQ(nullptr, res.error); + EXPECT_FALSE(res.notModified); + ASSERT_TRUE(res.data.get()); + EXPECT_EQ("Response", *res.data); + EXPECT_FALSE(bool(res.expires)); + EXPECT_TRUE(res.mustRevalidate); + EXPECT_FALSE(bool(res.modified)); + ASSERT_TRUE(bool(res.etag)); + EXPECT_EQ("snowfall", *res.etag); + loop.stop(); + }); }); loop.run(); @@ -455,9 +448,9 @@ TEST(DefaultFileSource, TEST_REQUIRES_SERVER(NoCacheFull)) { // Test that we can make a request with a Modified field that doesn't first try to load // from cache like a regular request -TEST(DefaultFileSource, TEST_REQUIRES_SERVER(NoCacheRefreshModifiedNotModified)) { +TEST(MainResourceLoader, TEST_REQUIRES_SERVER(NoCacheRefreshModifiedNotModified)) { util::RunLoop loop; - DefaultFileSource fs(":memory:", "."); + MainResourceLoader fs(ResourceOptions{}); Resource resource { Resource::Unknown, "http://127.0.0.1:3000/revalidate-modified" }; resource.loadingMethod = Resource::LoadingMethod::NetworkOnly; @@ -469,21 +462,22 @@ TEST(DefaultFileSource, TEST_REQUIRES_SERVER(NoCacheRefreshModifiedNotModified)) Response response; response.data = std::make_shared("Cached value"); response.expires = util::now() + 1h; - fs.put(resource, response); - std::unique_ptr req; - req = fs.request(resource, [&](Response res) { - req.reset(); - EXPECT_EQ(nullptr, res.error); - EXPECT_TRUE(res.notModified); - EXPECT_FALSE(res.data.get()); - ASSERT_TRUE(bool(res.expires)); - EXPECT_LT(util::now(), *res.expires); - EXPECT_TRUE(res.mustRevalidate); - ASSERT_TRUE(bool(res.modified)); - EXPECT_EQ(Timestamp{ Seconds(1420070400) }, *res.modified); - EXPECT_FALSE(bool(res.etag)); - loop.stop(); + auto dbfs = FileSourceManager::get()->getFileSource(FileSourceType::Database, ResourceOptions{}); + dbfs->forward(resource, response, [&] { + req = fs.request(resource, [&](Response res) { + req.reset(); + EXPECT_EQ(nullptr, res.error); + EXPECT_TRUE(res.notModified); + EXPECT_FALSE(res.data.get()); + ASSERT_TRUE(bool(res.expires)); + EXPECT_LT(util::now(), *res.expires); + EXPECT_TRUE(res.mustRevalidate); + ASSERT_TRUE(bool(res.modified)); + EXPECT_EQ(Timestamp{Seconds(1420070400)}, *res.modified); + EXPECT_FALSE(bool(res.etag)); + loop.stop(); + }); }); loop.run(); @@ -491,9 +485,9 @@ TEST(DefaultFileSource, TEST_REQUIRES_SERVER(NoCacheRefreshModifiedNotModified)) // Test that we can make a request with a Modified field that doesn't first try to load // from cache like a regular request -TEST(DefaultFileSource, TEST_REQUIRES_SERVER(NoCacheRefreshModifiedModified)) { +TEST(MainResourceLoader, TEST_REQUIRES_SERVER(NoCacheRefreshModifiedModified)) { util::RunLoop loop; - DefaultFileSource fs(":memory:", "."); + MainResourceLoader fs(ResourceOptions{}); Resource resource { Resource::Unknown, "http://127.0.0.1:3000/revalidate-modified" }; resource.loadingMethod = Resource::LoadingMethod::NetworkOnly; @@ -505,40 +499,49 @@ TEST(DefaultFileSource, TEST_REQUIRES_SERVER(NoCacheRefreshModifiedModified)) { Response response; response.data = std::make_shared("Cached value"); response.expires = util::now() + 1h; - fs.put(resource, response); - std::unique_ptr req; - req = fs.request(resource, [&](Response res) { - req.reset(); - EXPECT_EQ(nullptr, res.error); - EXPECT_FALSE(res.notModified); - ASSERT_TRUE(res.data.get()); - EXPECT_EQ("Response", *res.data); - EXPECT_FALSE(bool(res.expires)); - EXPECT_TRUE(res.mustRevalidate); - EXPECT_EQ(Timestamp{ Seconds(1420070400) }, *res.modified); - EXPECT_FALSE(res.etag); - loop.stop(); + auto dbfs = FileSourceManager::get()->getFileSource(FileSourceType::Database, ResourceOptions{}); + dbfs->forward(resource, response, [&] { + req = fs.request(resource, [&](Response res) { + req.reset(); + EXPECT_EQ(nullptr, res.error); + EXPECT_FALSE(res.notModified); + ASSERT_TRUE(res.data.get()); + EXPECT_EQ("Response", *res.data); + EXPECT_FALSE(bool(res.expires)); + EXPECT_TRUE(res.mustRevalidate); + EXPECT_EQ(Timestamp{Seconds(1420070400)}, *res.modified); + EXPECT_FALSE(res.etag); + loop.stop(); + }); }); loop.run(); } -TEST(DefaultFileSource, TEST_REQUIRES_SERVER(SetResourceTransform)) { +TEST(MainResourceLoader, TEST_REQUIRES_SERVER(SetResourceTransform)) { util::RunLoop loop; - DefaultFileSource fs(":memory:", "."); + MainResourceLoader fs(ResourceOptions{}); + + auto onlinefs = std::static_pointer_cast( + FileSourceManager::get()->getFileSource(FileSourceType::Network, ResourceOptions{})); // Translates the URL "localhost://test to http://127.0.0.1:3000/test - Actor transform(loop, [](Resource::Kind, const std::string& url) -> std::string { - if (url == "localhost://test") { - return "http://127.0.0.1:3000/test"; - } else { - return url; - } - }); + Actor transform( + loop, [](Resource::Kind, const std::string& url, ResourceTransform::FinishedCallback cb) { + if (url == "localhost://test") { + cb("http://127.0.0.1:3000/test"); + } else { + cb(url); + } + }); - fs.setResourceTransform(transform.self()); - const Resource resource1 { Resource::Unknown, "localhost://test" }; + onlinefs->setResourceTransform( + {[actorRef = transform.self()]( + Resource::Kind kind, const std::string& url, ResourceTransform::FinishedCallback cb) { + actorRef.invoke(&ResourceTransform::TransformCallback::operator(), kind, url, std::move(cb)); + }}); + const Resource resource1{Resource::Unknown, "localhost://test"}; std::unique_ptr req; req = fs.request(resource1, [&](Response res) { @@ -555,8 +558,8 @@ TEST(DefaultFileSource, TEST_REQUIRES_SERVER(SetResourceTransform)) { loop.run(); - fs.setResourceTransform({}); - const Resource resource2 { Resource::Unknown, "http://127.0.0.1:3000/test" }; + onlinefs->setResourceTransform({}); + const Resource resource2{Resource::Unknown, "http://127.0.0.1:3000/test"}; req = fs.request(resource2, [&](Response res) { req.reset(); @@ -573,22 +576,19 @@ TEST(DefaultFileSource, TEST_REQUIRES_SERVER(SetResourceTransform)) { loop.run(); } -TEST(DefaultFileSource, SetResourceCachePath) { +TEST(MainResourceLoader, SetResourceCachePath) { util::RunLoop loop; - DefaultFileSource fs(":memory:", "."); - - Actor callback(loop, [&]() -> void { - loop.stop(); - }); - - fs.setResourceCachePath("./new_offline.db", callback.self()); + MainResourceLoader fs(ResourceOptions{}); + auto dbfs = std::static_pointer_cast( + FileSourceManager::get()->getFileSource(FileSourceType::Database, ResourceOptions{})); + dbfs->setDatabasePath("./new_offline.db", [&loop] { loop.stop(); }); loop.run(); } // Test that a stale cache file that has must-revalidate set will trigger a response. -TEST(DefaultFileSource, TEST_REQUIRES_SERVER(RespondToStaleMustRevalidate)) { +TEST(MainResourceLoader, TEST_REQUIRES_SERVER(RespondToStaleMustRevalidate)) { util::RunLoop loop; - DefaultFileSource fs(":memory:", "."); + MainResourceLoader fs(ResourceOptions{}); Resource resource { Resource::Unknown, "http://127.0.0.1:3000/revalidate-same" }; resource.loadingMethod = Resource::LoadingMethod::CacheOnly; @@ -602,37 +602,38 @@ TEST(DefaultFileSource, TEST_REQUIRES_SERVER(RespondToStaleMustRevalidate)) { response.expires = Timestamp(Seconds(1417392000)); response.mustRevalidate = true; response.etag.emplace("snowfall"); - fs.put(resource, response); - std::unique_ptr req; - req = fs.request(resource, [&](Response res) { - req.reset(); - ASSERT_TRUE(res.error.get()); - EXPECT_EQ(Response::Error::Reason::NotFound, res.error->reason); - EXPECT_EQ("Cached resource is unusable", res.error->message); - EXPECT_FALSE(res.notModified); - ASSERT_TRUE(res.data.get()); - EXPECT_EQ("Cached value", *res.data); - ASSERT_TRUE(res.expires); - EXPECT_EQ(Timestamp{ Seconds(1417392000) }, *res.expires); - EXPECT_TRUE(res.mustRevalidate); - ASSERT_TRUE(res.modified); - EXPECT_EQ(Timestamp{ Seconds(1417392000) }, *res.modified); - ASSERT_TRUE(res.etag); - EXPECT_EQ("snowfall", *res.etag); - - resource.priorEtag = res.etag; - resource.priorModified = res.modified; - resource.priorExpires = res.expires; - resource.priorData = res.data; + auto dbfs = FileSourceManager::get()->getFileSource(FileSourceType::Database, ResourceOptions{}); + dbfs->forward(resource, response, [&] { + req = fs.request(resource, [&](Response res) { + req.reset(); + ASSERT_TRUE(res.error.get()); + EXPECT_EQ(Response::Error::Reason::NotFound, res.error->reason); + EXPECT_EQ("Cached resource is unusable", res.error->message); + EXPECT_FALSE(res.notModified); + ASSERT_TRUE(res.data.get()); + EXPECT_EQ("Cached value", *res.data); + ASSERT_TRUE(res.expires); + EXPECT_EQ(Timestamp{Seconds(1417392000)}, *res.expires); + EXPECT_TRUE(res.mustRevalidate); + ASSERT_TRUE(res.modified); + EXPECT_EQ(Timestamp{Seconds(1417392000)}, *res.modified); + ASSERT_TRUE(res.etag); + EXPECT_EQ("snowfall", *res.etag); + + resource.priorEtag = res.etag; + resource.priorModified = res.modified; + resource.priorExpires = res.expires; + resource.priorData = res.data; - loop.stop(); + loop.stop(); + }); }); loop.run(); // Now run this request again, with the data we gathered from the previous stale/unusable - // request. We're replacing the data so that we can check that the DefaultFileSource doesn't + // request. We're replacing the data so that we can check that the MainResourceLoader doesn't // attempt another database access if we already have the value. resource.loadingMethod = Resource::LoadingMethod::NetworkOnly; resource.priorData = std::make_shared("Prior value"); @@ -664,9 +665,9 @@ TEST(DefaultFileSource, TEST_REQUIRES_SERVER(RespondToStaleMustRevalidate)) { } // Test that requests for expired resources have lower priority than requests for new resources -TEST(DefaultFileSource, TEST_REQUIRES_SERVER(CachedResourceLowPriority)) { +TEST(MainResourceLoader, TEST_REQUIRES_SERVER(CachedResourceLowPriority)) { util::RunLoop loop; - DefaultFileSource fs(":memory:", "."); + MainResourceLoader fs(ResourceOptions{}); Response response; std::size_t online_response_counter = 0; @@ -674,16 +675,20 @@ TEST(DefaultFileSource, TEST_REQUIRES_SERVER(CachedResourceLowPriority)) { using namespace std::chrono_literals; response.expires = util::now() - 1h; + auto dbfs = FileSourceManager::get()->getFileSource(FileSourceType::Database, ResourceOptions{}); + auto onlineFs = FileSourceManager::get()->getFileSource(FileSourceType::Network, ResourceOptions{}); + // Put existing values into the cache. Resource resource1{Resource::Unknown, "http://127.0.0.1:3000/load/3", {}, Resource::LoadingMethod::All}; response.data = std::make_shared("Cached Request 3"); - fs.put(resource1, response); + dbfs->forward(resource1, response); Resource resource2{Resource::Unknown, "http://127.0.0.1:3000/load/4", {}, Resource::LoadingMethod::All}; response.data = std::make_shared("Cached Request 4"); - fs.put(resource2, response); + dbfs->forward(resource2, response); - fs.setMaximumConcurrentRequests(1); + onlineFs->setProperty("max-concurrent-requests", 1u); + fs.pause(); NetworkStatus::Set(NetworkStatus::Status::Offline); // Ensure that the online requests for new resources are processed first. @@ -695,14 +700,6 @@ TEST(DefaultFileSource, TEST_REQUIRES_SERVER(CachedResourceLowPriority)) { EXPECT_EQ("Request 1", *res.data); }); - Resource nonCached2{Resource::Unknown, "http://127.0.0.1:3000/load/2", {}, Resource::LoadingMethod::All}; - std::unique_ptr req2 = fs.request(nonCached2, [&](Response res) { - online_response_counter++; - req2.reset(); - EXPECT_EQ(online_response_counter, 2); // make sure this is responded second - EXPECT_EQ("Request 2", *res.data); - }); - bool req3CachedResponseReceived = false; std::unique_ptr req3 = fs.request(resource1, [&](Response res) { // Offline callback is received first @@ -732,6 +729,15 @@ TEST(DefaultFileSource, TEST_REQUIRES_SERVER(CachedResourceLowPriority)) { } }); + Resource nonCached2{Resource::Unknown, "http://127.0.0.1:3000/load/2", {}, Resource::LoadingMethod::All}; + std::unique_ptr req2 = fs.request(nonCached2, [&](Response res) { + online_response_counter++; + req2.reset(); + EXPECT_EQ(online_response_counter, 2); // make sure this is responded second + EXPECT_EQ("Request 2", *res.data); + }); + + fs.resume(); NetworkStatus::Set(NetworkStatus::Status::Online); loop.run(); diff --git a/test/storage/offline_download.test.cpp b/test/storage/offline_download.test.cpp index c1a9bb73f42..a15c96d3915 100644 --- a/test/storage/offline_download.test.cpp +++ b/test/storage/offline_download.test.cpp @@ -6,7 +6,6 @@ #include #include -#include #include #include #include @@ -64,7 +63,7 @@ class OfflineTest { } util::RunLoop loop; - StubOnlineFileSource fileSource; + StubFileSource fileSource; OfflineDatabase db; std::size_t size = 0; @@ -386,7 +385,7 @@ TEST(OfflineDownload, DoesNotFloodTheFileSourceWithRequests) { fileSource.respond(Resource::Kind::Style, test.response("style.json")); test.loop.runOnce(); - EXPECT_EQ(fileSource.getMaximumConcurrentRequests(), fileSource.requests.size()); + EXPECT_EQ(*fileSource.getProperty("max-concurrent-requests").getUint(), fileSource.requests.size()); } TEST(OfflineDownload, GetStatusNoResources) { @@ -822,7 +821,6 @@ TEST(OfflineDownload, AllOfflineRequestsHaveLowPriorityAndOfflineUsage) { test.loop.run(); } - #ifndef __QT__ // Qt doesn't expose the ability to register virtual file system handlers. TEST(OfflineDownload, DiskFull) { FixtureLog log; diff --git a/test/storage/online_file_source.test.cpp b/test/storage/online_file_source.test.cpp index 3e697a99ea0..88dbf519f8f 100644 --- a/test/storage/online_file_source.test.cpp +++ b/test/storage/online_file_source.test.cpp @@ -1,11 +1,12 @@ -#include -#include #include +#include +#include +#include #include +#include #include -#include #include -#include +#include #include @@ -52,9 +53,9 @@ TEST(OnlineFileSource, TEST_REQUIRES_SERVER(TemporaryError)) { OnlineFileSource fs; const auto start = Clock::now(); + int counter = 0; auto req = fs.request({ Resource::Unknown, "http://127.0.0.1:3000/temporary-error" }, [&](Response res) { - static int counter = 0; switch (counter++) { case 0: { const auto duration = std::chrono::duration(Clock::now() - start).count(); @@ -92,10 +93,10 @@ TEST(OnlineFileSource, TEST_REQUIRES_SERVER(ConnectionError)) { OnlineFileSource fs; const auto start = Clock::now(); + int counter = 0; + int wait = 0; std::unique_ptr req = fs.request({ Resource::Unknown, "http://127.0.0.1:3001/" }, [&](Response res) { - static int counter = 0; - static int wait = 0; const auto duration = std::chrono::duration(Clock::now() - start).count(); EXPECT_LT(wait - 0.01, duration) << "Backoff timer didn't wait 1 second"; EXPECT_GT(wait + 0.3, duration) << "Backoff timer fired too late"; @@ -157,10 +158,6 @@ TEST(OnlineFileSource, TEST_REQUIRES_SERVER(RetryDelayOnExpiredTile)) { EXPECT_EQ(nullptr, res.error); EXPECT_GT(util::now(), *res.expires); EXPECT_FALSE(res.mustRevalidate); - }); - - util::Timer timer; - timer.start(Milliseconds(500), Duration::zero(), [&] () { loop.stop(); }); @@ -306,12 +303,13 @@ TEST(OnlineFileSource, TEST_REQUIRES_SERVER(NetworkStatusChange)) { TEST(OnlineFileSource, TEST_REQUIRES_SERVER(NetworkStatusChangePreempt)) { util::RunLoop loop; OnlineFileSource fs; + fs.pause(); const auto start = Clock::now(); + int counter = 0; const Resource resource{ Resource::Unknown, "http://127.0.0.1:3001/test" }; std::unique_ptr req = fs.request(resource, [&](Response res) { - static int counter = 0; const auto duration = std::chrono::duration(Clock::now() - start).count(); if (counter == 0) { EXPECT_GT(0.2, duration) << "Response came in too late"; @@ -341,6 +339,7 @@ TEST(OnlineFileSource, TEST_REQUIRES_SERVER(NetworkStatusChangePreempt)) { mbgl::NetworkStatus::Reachable(); }); + fs.resume(); loop.run(); } @@ -416,14 +415,30 @@ TEST(OnlineFileSource, TEST_REQUIRES_SERVER(RateLimitDefault)) { loop.run(); } +TEST(OnlineFileSource, GetBaseURLAndAccessTokenWhilePaused) { + util::RunLoop loop; + OnlineFileSource fs; + + fs.pause(); + + auto baseURL = "http://url"; + auto accessToken = "access_token"; + + fs.setProperty(API_BASE_URL_KEY, baseURL); + fs.setProperty(ACCESS_TOKEN_KEY, accessToken); + + EXPECT_EQ(*fs.getProperty(API_BASE_URL_KEY).getString(), baseURL); + EXPECT_EQ(*fs.getProperty(ACCESS_TOKEN_KEY).getString(), accessToken); +} + TEST(OnlineFileSource, ChangeAPIBaseURL){ util::RunLoop loop; OnlineFileSource fs; - EXPECT_EQ(mbgl::util::API_BASE_URL, fs.getAPIBaseURL()); + EXPECT_EQ(mbgl::util::API_BASE_URL, *fs.getProperty(API_BASE_URL_KEY).getString()); const std::string customURL = "test.domain"; - fs.setAPIBaseURL(customURL); - EXPECT_EQ(customURL, fs.getAPIBaseURL()); + fs.setProperty(API_BASE_URL_KEY, customURL); + EXPECT_EQ(customURL, *fs.getProperty(API_BASE_URL_KEY).getString()); } @@ -433,34 +448,38 @@ TEST(OnlineFileSource, TEST_REQUIRES_SERVER(LowHighPriorityRequests)) { std::size_t response_counter = 0; const std::size_t NUM_REQUESTS = 3; - fs.setMaximumConcurrentRequests(1); - NetworkStatus::Set(NetworkStatus::Status::Offline); - - // requesting a low priority resource - Resource low_prio{ Resource::Unknown, "http://127.0.0.1:3000/load/1" }; - low_prio.setPriority(Resource::Priority::Low); - std::unique_ptr req_0 = fs.request(low_prio, [&](Response) { + fs.setProperty("max-concurrent-requests", 1u); + // After DefaultFileSource was split, OnlineFileSource lives on a separate + // thread. Pause OnlineFileSource, so that messages are queued for processing. + fs.pause(); + + // First regular request. + Resource regular1{Resource::Unknown, "http://127.0.0.1:3000/load/1"}; + std::unique_ptr req_0 = fs.request(regular1, [&](Response) { response_counter++; req_0.reset(); - EXPECT_EQ(response_counter, NUM_REQUESTS); // make sure this is responded last - loop.stop(); }); - // requesting two "regular" resources - Resource regular1{ Resource::Unknown, "http://127.0.0.1:3000/load/2" }; - std::unique_ptr req_1 = fs.request(regular1, [&](Response) { + // Low priority request that will be queued and should be requested last. + Resource low_prio{Resource::Unknown, "http://127.0.0.1:3000/load/2"}; + low_prio.setPriority(Resource::Priority::Low); + std::unique_ptr req_1 = fs.request(low_prio, [&](Response) { response_counter++; req_1.reset(); + EXPECT_EQ(response_counter, NUM_REQUESTS); // make sure this is responded last + loop.stop(); }); + + // Second regular priority request that should de-preoritize low priority request. Resource regular2{ Resource::Unknown, "http://127.0.0.1:3000/load/3" }; std::unique_ptr req_2 = fs.request(regular2, [&](Response) { response_counter++; req_2.reset(); }); + fs.resume(); NetworkStatus::Set(NetworkStatus::Status::Online); - loop.run(); } @@ -472,10 +491,9 @@ TEST(OnlineFileSource, TEST_REQUIRES_SERVER(LowHighPriorityRequestsMany)) { int correct_low = 0; int correct_regular = 0; - - fs.setMaximumConcurrentRequests(1); - NetworkStatus::Set(NetworkStatus::Status::Offline); + fs.setProperty("max-concurrent-requests", 1u); + fs.pause(); std::vector> collector; @@ -515,8 +533,8 @@ TEST(OnlineFileSource, TEST_REQUIRES_SERVER(LowHighPriorityRequestsMany)) { } } + fs.resume(); NetworkStatus::Set(NetworkStatus::Status::Online); - loop.run(); } @@ -524,11 +542,12 @@ TEST(OnlineFileSource, TEST_REQUIRES_SERVER(MaximumConcurrentRequests)) { util::RunLoop loop; OnlineFileSource fs; - ASSERT_EQ(fs.getMaximumConcurrentRequests(), 20u); + ASSERT_EQ(*fs.getProperty("max-concurrent-requests").getUint(), 20u); - fs.setMaximumConcurrentRequests(10); - ASSERT_EQ(fs.getMaximumConcurrentRequests(), 10u); + fs.setProperty("max-concurrent-requests", 10u); + ASSERT_EQ(*fs.getProperty("max-concurrent-requests").getUint(), 10u); } + TEST(OnlineFileSource, TEST_REQUIRES_SERVER(RequestSameUrlMultipleTimes)) { util::RunLoop loop; OnlineFileSource fs; diff --git a/test/storage/sync_file_source.test.cpp b/test/storage/sync_file_source.test.cpp index 4bd964199d1..3cd6cd9f6ee 100644 --- a/test/storage/sync_file_source.test.cpp +++ b/test/storage/sync_file_source.test.cpp @@ -1,11 +1,12 @@ -#include -#include -#include #include #include #include +#include +#include #include #include +#include +#include #include #include @@ -14,7 +15,7 @@ using namespace mbgl; class SyncFileSource : public FileSource { public: - std::unique_ptr request(const Resource& resource, FileSource::Callback callback) { + std::unique_ptr request(const Resource& resource, FileSource::Callback callback) override { Response response; auto it = assets.find(resource.url); if (it == assets.end()) { @@ -27,6 +28,8 @@ class SyncFileSource : public FileSource { return nullptr; } + bool canRequest(const Resource&) const override { return true; } + void add(std::string const& key, std::string const& data) { assets.emplace(key, std::make_shared(data)); }; diff --git a/test/style/source.test.cpp b/test/style/source.test.cpp index 5eb837d92bf..0286aaaec3c 100644 --- a/test/style/source.test.cpp +++ b/test/style/source.test.cpp @@ -58,7 +58,7 @@ class SourceTest { StubRenderSourceObserver renderSourceObserver; Transform transform; TransformState transformState; - Style style { *fileSource, 1 }; + Style style{fileSource, 1}; AnnotationManager annotationManager { style }; ImageManager imageManager; GlyphManager glyphManager; diff --git a/test/style/style.test.cpp b/test/style/style.test.cpp index b9e19d5a858..c866431ac13 100644 --- a/test/style/style.test.cpp +++ b/test/style/style.test.cpp @@ -18,7 +18,7 @@ using namespace mbgl::style; TEST(Style, Properties) { util::RunLoop loop; - StubFileSource fileSource; + auto fileSource = std::make_shared(); Style::Impl style { fileSource, 1.0 }; style.loadJSON(R"STYLE({"name": "Test"})STYLE"); @@ -60,7 +60,7 @@ TEST(Style, Properties) { TEST(Style, DuplicateSource) { util::RunLoop loop; - StubFileSource fileSource; + auto fileSource = std::make_shared(); Style::Impl style { fileSource, 1.0 }; style.loadJSON(util::read_file("test/fixtures/resources/style-unused-sources.json")); @@ -81,7 +81,7 @@ TEST(Style, RemoveSourceInUse) { auto log = new FixtureLogObserver(); Log::setObserver(std::unique_ptr(log)); - StubFileSource fileSource; + auto fileSource = std::make_shared(); Style::Impl style { fileSource, 1.0 }; style.loadJSON(util::read_file("test/fixtures/resources/style-unused-sources.json")); @@ -106,7 +106,7 @@ TEST(Style, RemoveSourceInUse) { TEST(Style, SourceImplsOrder) { util::RunLoop loop; - StubFileSource fileSource; + auto fileSource = std::make_shared(); Style::Impl style{fileSource, 1.0}; style.addSource(std::make_unique("c", "mapbox://mapbox.mapbox-terrain-v2")); diff --git a/test/style/style_layer.test.cpp b/test/style/style_layer.test.cpp index 1d60197c2de..77e936ff3b0 100644 --- a/test/style/style_layer.test.cpp +++ b/test/style/style_layer.test.cpp @@ -283,7 +283,7 @@ TEST(Layer, DuplicateLayer) { util::RunLoop loop; // Setup style - StubFileSource fileSource; + auto fileSource = std::make_shared(); Style::Impl style { fileSource, 1.0 }; style.loadJSON(util::read_file("test/fixtures/resources/style-unused-sources.json")); @@ -304,7 +304,7 @@ TEST(Layer, IncompatibleLayer) { util::RunLoop loop; // Setup style - StubFileSource fileSource; + auto fileSource = std::make_shared(); Style::Impl style{fileSource, 1.0}; style.loadJSON(util::read_file("test/fixtures/resources/style-unused-sources.json")); diff --git a/test/test-files.json b/test/test-files.json index b997503877b..86915837650 100644 --- a/test/test-files.json +++ b/test/test-files.json @@ -36,10 +36,10 @@ "test/src/mbgl/test/test.cpp", "test/src/mbgl/test/util.cpp", "test/storage/asset_file_source.test.cpp", - "test/storage/default_file_source.test.cpp", "test/storage/headers.test.cpp", "test/storage/http_file_source.test.cpp", "test/storage/local_file_source.test.cpp", + "test/storage/main_resource_loader.test.cpp", "test/storage/offline.test.cpp", "test/storage/offline_database.test.cpp", "test/storage/offline_download.test.cpp", diff --git a/test/tile/custom_geometry_tile.test.cpp b/test/tile/custom_geometry_tile.test.cpp index fb905ac0760..f3d11ab8985 100644 --- a/test/tile/custom_geometry_tile.test.cpp +++ b/test/tile/custom_geometry_tile.test.cpp @@ -25,7 +25,7 @@ class CustomTileTest { std::shared_ptr fileSource = std::make_shared(); TransformState transformState; util::RunLoop loop; - style::Style style { *fileSource, 1 }; + style::Style style{fileSource, 1}; AnnotationManager annotationManager { style }; ImageManager imageManager; GlyphManager glyphManager; diff --git a/test/tile/geojson_tile.test.cpp b/test/tile/geojson_tile.test.cpp index d4bf1e07527..25fd268dc85 100644 --- a/test/tile/geojson_tile.test.cpp +++ b/test/tile/geojson_tile.test.cpp @@ -25,7 +25,7 @@ class GeoJSONTileTest { std::shared_ptr fileSource = std::make_shared(); TransformState transformState; util::RunLoop loop; - style::Style style { *fileSource, 1 }; + style::Style style{fileSource, 1}; AnnotationManager annotationManager { style }; ImageManager imageManager; GlyphManager glyphManager; diff --git a/test/tile/raster_dem_tile.test.cpp b/test/tile/raster_dem_tile.test.cpp index 42e75947203..f5f7610096c 100644 --- a/test/tile/raster_dem_tile.test.cpp +++ b/test/tile/raster_dem_tile.test.cpp @@ -19,7 +19,7 @@ class RasterDEMTileTest { std::shared_ptr fileSource = std::make_shared(); TransformState transformState; util::RunLoop loop; - style::Style style { *fileSource, 1 }; + style::Style style{fileSource, 1}; AnnotationManager annotationManager { style }; ImageManager imageManager; GlyphManager glyphManager; diff --git a/test/tile/raster_tile.test.cpp b/test/tile/raster_tile.test.cpp index f19bd26260e..a5a2875f2e6 100644 --- a/test/tile/raster_tile.test.cpp +++ b/test/tile/raster_tile.test.cpp @@ -19,7 +19,7 @@ class RasterTileTest { std::shared_ptr fileSource = std::make_shared(); TransformState transformState; util::RunLoop loop; - style::Style style { *fileSource, 1 }; + style::Style style{fileSource, 1}; AnnotationManager annotationManager { style }; ImageManager imageManager; GlyphManager glyphManager; diff --git a/test/tile/tile_cache.test.cpp b/test/tile/tile_cache.test.cpp index 7a89ece7564..43b409ae871 100644 --- a/test/tile/tile_cache.test.cpp +++ b/test/tile/tile_cache.test.cpp @@ -28,7 +28,7 @@ class VectorTileTest { std::shared_ptr fileSource = std::make_shared(); TransformState transformState; util::RunLoop loop; - style::Style style{*fileSource, 1}; + style::Style style{fileSource, 1}; AnnotationManager annotationManager{style}; ImageManager imageManager; GlyphManager glyphManager; diff --git a/test/tile/vector_tile.test.cpp b/test/tile/vector_tile.test.cpp index 940c0272db4..d282c874efc 100644 --- a/test/tile/vector_tile.test.cpp +++ b/test/tile/vector_tile.test.cpp @@ -25,7 +25,7 @@ class VectorTileTest { std::shared_ptr fileSource = std::make_shared(); TransformState transformState; util::RunLoop loop; - style::Style style { *fileSource, 1 }; + style::Style style{fileSource, 1}; AnnotationManager annotationManager { style }; ImageManager imageManager; GlyphManager glyphManager;