From c9f213cac589595580a04a89784a44e0ff19c39b Mon Sep 17 00:00:00 2001 From: Vladimir Kryachko Date: Fri, 9 Nov 2018 11:45:20 -0500 Subject: [PATCH 01/25] Fix errorprone annotation processor classpath (#121) The issue results in a missing `META-INF/services/com.google.errorprone.bugpatterns.BugChecker` file in the resulting binary which prevents the errorprone plugin from discovering the custom checks. --- tools/errorprone/errorprone.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/errorprone/errorprone.gradle b/tools/errorprone/errorprone.gradle index e7a329ccf2a..14706959d1b 100644 --- a/tools/errorprone/errorprone.gradle +++ b/tools/errorprone/errorprone.gradle @@ -18,6 +18,7 @@ apply plugin: 'java-library' dependencies { implementation 'com.google.errorprone:error_prone_check_api:2.3.2' implementation 'com.google.auto.service:auto-service:1.0-rc4' + annotationProcessor 'com.google.auto.service:auto-service:1.0-rc4' testImplementation 'junit:junit:4.12' testImplementation 'com.google.errorprone:error_prone_test_helpers:2.3.1' From 0b672e8460bc8bcc0389e982819c12633847af56 Mon Sep 17 00:00:00 2001 From: rsgowman Date: Thu, 22 Nov 2018 09:37:27 -0500 Subject: [PATCH 02/25] Fix handling of sqlite transactions (#136) * Move as much work as possible out of the sqlite txn In particular, don't do anything that could throw in the finally block, as this will mask any exception thrown within the transaction itself. * Use sqlite transaction listeners (rather than doing so manually) Using sqlite's version allows us to not worry about getting the transaction semantics correct in the face of exceptions. --- .../firestore/local/SQLitePersistence.java | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLitePersistence.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLitePersistence.java index 891aa091729..6b5f06eab5b 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLitePersistence.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLitePersistence.java @@ -26,6 +26,7 @@ import android.database.sqlite.SQLiteOpenHelper; import android.database.sqlite.SQLiteProgram; import android.database.sqlite.SQLiteStatement; +import android.database.sqlite.SQLiteTransactionListener; import android.support.annotation.VisibleForTesting; import com.google.common.base.Function; import com.google.firebase.firestore.auth.User; @@ -75,6 +76,21 @@ public static String databaseName(String persistenceKey, DatabaseId databaseId) private final SQLiteQueryCache queryCache; private final SQLiteRemoteDocumentCache remoteDocumentCache; private final SQLiteLruReferenceDelegate referenceDelegate; + private final SQLiteTransactionListener transactionListener = + new SQLiteTransactionListener() { + @Override + public void onBegin() { + referenceDelegate.onTransactionStarted(); + } + + @Override + public void onCommit() { + referenceDelegate.onTransactionCommitted(); + } + + @Override + public void onRollback() {} + }; public SQLitePersistence( Context context, String persistenceKey, DatabaseId databaseId, LocalSerializer serializer) { @@ -143,35 +159,32 @@ RemoteDocumentCache getRemoteDocumentCache() { @Override void runTransaction(String action, Runnable operation) { + Logger.debug(TAG, "Starting transaction: %s", action); + db.beginTransactionWithListener(transactionListener); try { - Logger.debug(TAG, "Starting transaction: %s", action); - referenceDelegate.onTransactionStarted(); - db.beginTransaction(); operation.run(); // Note that an exception in operation.run() will prevent this code from running. db.setTransactionSuccessful(); } finally { db.endTransaction(); - referenceDelegate.onTransactionCommitted(); } } @Override T runTransaction(String action, Supplier operation) { + Logger.debug(TAG, "Starting transaction: %s", action); + T value = null; + db.beginTransactionWithListener(transactionListener); try { - Logger.debug(TAG, "Starting transaction: %s", action); - referenceDelegate.onTransactionStarted(); - db.beginTransaction(); - T value = operation.get(); + value = operation.get(); // Note that an exception in operation.run() will prevent this code from running. db.setTransactionSuccessful(); - return value; } finally { db.endTransaction(); - referenceDelegate.onTransactionCommitted(); } + return value; } /** From 2d9cb81d22a3baff6daff59c096318e8102635a0 Mon Sep 17 00:00:00 2001 From: rsgowman Date: Tue, 27 Nov 2018 17:53:47 -0500 Subject: [PATCH 03/25] CHANGELOG.md updates for 17.1.4 (#143) --- firebase-firestore/CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/firebase-firestore/CHANGELOG.md b/firebase-firestore/CHANGELOG.md index c7c5516695d..745ee973245 100644 --- a/firebase-firestore/CHANGELOG.md +++ b/firebase-firestore/CHANGELOG.md @@ -6,6 +6,13 @@ custom objects (POJOs) by passing a Class instance, e.g. `snapshot.get("field", CustomType.class)`. +# 17.1.4 +- [fixed] Fixed a SQLite transaction handling issue that occasionally masked + exceptions when Firestore closed a transaction that was never started. For + more information, see the issue report in GitHub (https://github.com/firebase/firebase-android-sdk/issues/115). +- [fixed] Fixed a race condition that caused a `SQLiteDatabaseLockedException` + when an app attempted to access the SQLite database from multiple threads. + # 17.1.2 - [changed] Changed how the SDK handles locally-updated documents while syncing those updates with Cloud Firestore servers. This can lead to slight behavior From 6c2e9a5ecd98118c8fa0012a5721f12c68c56d56 Mon Sep 17 00:00:00 2001 From: Gil Date: Tue, 27 Nov 2018 17:54:41 -0800 Subject: [PATCH 04/25] Revert "Better POJO Support for Document Fields" (#140) This reverts commit ddda9268537e096b978803df46d31b4541137d9a. --- firebase-firestore/CHANGELOG.md | 6 -- .../google/firebase/firestore/POJOTest.java | 56 +---------------- .../firestore/ServerTimestampTest.java | 22 ------- .../firebase/firestore/ValidationTest.java | 37 ++++++++--- .../firestore/CollectionReference.java | 19 +++++- .../firebase/firestore/DocumentReference.java | 38 +++++++++-- .../firebase/firestore/DocumentSnapshot.java | 63 ------------------- .../firebase/firestore/Transaction.java | 44 +++++++++++-- .../firebase/firestore/UserDataConverter.java | 59 ++++++++--------- .../google/firebase/firestore/WriteBatch.java | 44 ++++++++++--- .../firestore/util/CustomClassMapper.java | 7 +-- .../firestore/model/FieldValueTest.java | 2 +- 12 files changed, 182 insertions(+), 215 deletions(-) diff --git a/firebase-firestore/CHANGELOG.md b/firebase-firestore/CHANGELOG.md index 745ee973245..e90cc8a9e8c 100644 --- a/firebase-firestore/CHANGELOG.md +++ b/firebase-firestore/CHANGELOG.md @@ -1,10 +1,4 @@ # Unreleased -- [feature] Custom objects (POJOs) can now be passed as a field value in - update(), within `Map<>` objects passed to set(), in array transform - operations, and in query filters. -- [feature] DocumentSnapshot.get() now supports retrieving fields as - custom objects (POJOs) by passing a Class instance, e.g. - `snapshot.get("field", CustomType.class)`. # 17.1.4 - [fixed] Fixed a SQLite transaction handling issue that occasionally masked diff --git a/firebase-firestore/src/androidTest/java/com/google/firebase/firestore/POJOTest.java b/firebase-firestore/src/androidTest/java/com/google/firebase/firestore/POJOTest.java index bb60af5f7c3..5e8c941f89f 100644 --- a/firebase-firestore/src/androidTest/java/com/google/firebase/firestore/POJOTest.java +++ b/firebase-firestore/src/androidTest/java/com/google/firebase/firestore/POJOTest.java @@ -15,18 +15,13 @@ package com.google.firebase.firestore; import static com.google.firebase.firestore.testutil.IntegrationTestUtil.testCollection; -import static com.google.firebase.firestore.testutil.IntegrationTestUtil.testDocument; import static com.google.firebase.firestore.testutil.IntegrationTestUtil.waitFor; import static com.google.firebase.firestore.testutil.TestUtil.expectError; -import static com.google.firebase.firestore.testutil.TestUtil.map; import static junit.framework.Assert.assertEquals; import android.support.test.runner.AndroidJUnit4; -import com.google.android.gms.tasks.Task; -import com.google.android.gms.tasks.Tasks; import com.google.firebase.Timestamp; import com.google.firebase.firestore.testutil.IntegrationTestUtil; -import java.util.ArrayList; import java.util.Date; import org.junit.After; import org.junit.Test; @@ -180,7 +175,7 @@ public void testWriteAndRead() { } @Test - public void testSetMerge() { + public void testUpdate() { CollectionReference collection = testCollection(); POJO data = new POJO(1.0, "a", collection.document()); DocumentReference reference = waitFor(collection.add(data)); @@ -195,55 +190,6 @@ public void testSetMerge() { assertEquals(expected, doc.toObject(POJO.class)); } - // General smoke test that makes sure APIs accept POJOs. - @Test - public void testAPIsAcceptPOJOsForFields() { - DocumentReference ref = testDocument(); - ArrayList> tasks = new ArrayList<>(); - - // as Map<> entries in a set() call. - POJO data = new POJO(1.0, "a", ref); - tasks.add(ref.set(map("a", data, "b", map("c", data)))); - - // as Map<> entries in an update() call. - tasks.add(ref.update(map("a", data))); - - // as field values in an update() call. - tasks.add(ref.update("c", data)); - - // as values in arrayUnion() / arrayRemove(). - tasks.add(ref.update("c", FieldValue.arrayUnion(data))); - tasks.add(ref.update("c", FieldValue.arrayRemove(data))); - - // as Query parameters. - data.setBlob(null); // blobs are broken, see b/117680212 - tasks.add(testCollection().whereEqualTo("field", data).get()); - - waitFor(Tasks.whenAll(tasks)); - } - - @Test - public void testDocumentSnapshotGetWithPOJOs() { - DocumentReference ref = testDocument(); - - // Go offline so that we can verify server timestamp behavior overload. - ref.getFirestore().disableNetwork(); - - POJO pojo = new POJO(1.0, "a", ref); - ref.set(map("field", pojo)); - - DocumentSnapshot snap = waitFor(ref.get()); - - assertEquals(pojo, snap.get("field", POJO.class)); - assertEquals(pojo, snap.get(FieldPath.of("field"), POJO.class)); - assertEquals( - pojo, snap.get("field", POJO.class, DocumentSnapshot.ServerTimestampBehavior.DEFAULT)); - assertEquals( - pojo, - snap.get( - FieldPath.of("field"), POJO.class, DocumentSnapshot.ServerTimestampBehavior.DEFAULT)); - } - @Test public void setFieldMaskMustHaveCorrespondingValue() { CollectionReference collection = testCollection(); diff --git a/firebase-firestore/src/androidTest/java/com/google/firebase/firestore/ServerTimestampTest.java b/firebase-firestore/src/androidTest/java/com/google/firebase/firestore/ServerTimestampTest.java index 0f9e0652969..9b71486e3bb 100644 --- a/firebase-firestore/src/androidTest/java/com/google/firebase/firestore/ServerTimestampTest.java +++ b/firebase-firestore/src/androidTest/java/com/google/firebase/firestore/ServerTimestampTest.java @@ -248,28 +248,6 @@ public void testServerTimestampsUsesPreviousValueFromLocalMutation() { assertThat(remoteSnapshot.get("a")).isInstanceOf(Timestamp.class); } - @Test - public void testServerTimestampBehaviorOverloadsOfDocumentSnapshotGet() { - writeInitialData(); - waitFor(docRef.update(updateData)); - DocumentSnapshot snap = accumulator.awaitLocalEvent(); - - // Default behavior should return null timestamp (via any overload). - assertNull(snap.get("when")); - assertNull(snap.get(FieldPath.of("when"))); - assertNull(snap.get("when", Timestamp.class)); - assertNull(snap.get(FieldPath.of("when"), Timestamp.class)); - - // Estimate should return a Timestamp object (via any overload). - assertThat(snap.get("when", ServerTimestampBehavior.ESTIMATE)).isInstanceOf(Timestamp.class); - assertThat(snap.get(FieldPath.of("when"), ServerTimestampBehavior.ESTIMATE)) - .isInstanceOf(Timestamp.class); - assertThat(snap.get("when", Timestamp.class, ServerTimestampBehavior.ESTIMATE)) - .isInstanceOf(Timestamp.class); - assertThat(snap.get(FieldPath.of("when"), Timestamp.class, ServerTimestampBehavior.ESTIMATE)) - .isInstanceOf(Timestamp.class); - } - @Test public void testServerTimestampsWorkViaTransactionSet() { waitFor( diff --git a/firebase-firestore/src/androidTest/java/com/google/firebase/firestore/ValidationTest.java b/firebase-firestore/src/androidTest/java/com/google/firebase/firestore/ValidationTest.java index dd9ae258d67..7bdc168da09 100644 --- a/firebase-firestore/src/androidTest/java/com/google/firebase/firestore/ValidationTest.java +++ b/firebase-firestore/src/androidTest/java/com/google/firebase/firestore/ValidationTest.java @@ -257,16 +257,22 @@ public void writesMustNotContainReservedFieldNames() { @Test public void setsMustNotContainFieldValueDelete() { - expectSetError( - map("foo", FieldValue.delete()), + // PORTING NOTE: We avoid using expectSetError(), since it hits the POJO overload which + // can't handle FieldValue.delete(). + DocumentReference ref = testDocument(); + expectError( + () -> ref.set(map("foo", FieldValue.delete())), "Invalid data. FieldValue.delete() can only be used with update() and set() with " + "SetOptions.merge() (found in field foo)"); } @Test public void updatesMustNotContainNestedFieldValueDeletes() { - expectUpdateError( - map("foo", map("bar", FieldValue.delete())), + // PORTING NOTE: We avoid using expectSetError(), since it hits the POJO overload which + // can't handle FieldValue.delete(). + DocumentReference ref = testDocument(); + expectError( + () -> ref.update(map("foo", map("bar", FieldValue.delete()))), "Invalid data. FieldValue.delete() can only appear at the top level of your update data " + "(found in field foo.bar)"); } @@ -370,8 +376,8 @@ public void arrayTransformsFailInQueries() { @Test public void arrayTransformsRejectInvalidElements() { DocumentReference doc = testDocument(); - String reason = - "No properties to serialize found on class com.google.firebase.firestore.ValidationTest"; + String reason = "Invalid data. Unsupported type: com.google.firebase.firestore.ValidationTest"; + // TODO: If we get more permissive with POJOs, perhaps we should make this work. expectError(() -> doc.set(map("x", FieldValue.arrayUnion(1, this))), reason); expectError(() -> doc.set(map("x", FieldValue.arrayRemove(1, this))), reason); } @@ -552,8 +558,15 @@ private static void expectWriteError( DocumentReference ref = testDocument(); if (includeSets) { - expectError(() -> ref.set(data), reason); - expectError(() -> ref.getFirestore().batch().set(ref, data), reason); + if (data instanceof Map) { + @SuppressWarnings("unchecked") + Map setMap = (Map) data; + expectError(() -> ref.set(setMap), reason); + expectError(() -> ref.getFirestore().batch().set(ref, setMap), reason); + } else { + expectError(() -> ref.set(data), reason); + expectError(() -> ref.getFirestore().batch().set(ref, data), reason); + } } if (includeUpdates) { @@ -580,7 +593,13 @@ private static void expectWriteError( (Function) transaction -> { if (includeSets) { - expectError(() -> transaction.set(ref, data), reason); + if (data instanceof Map) { + @SuppressWarnings("unchecked") + Map setMap = (Map) data; + expectError(() -> transaction.set(ref, setMap), reason); + } else { + expectError(() -> transaction.set(ref, data), reason); + } } if (includeUpdates) { assertTrue("update() only support Maps.", data instanceof Map); diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/CollectionReference.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/CollectionReference.java index c8364f7a76f..db3cf10a113 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/CollectionReference.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/CollectionReference.java @@ -23,6 +23,7 @@ import com.google.firebase.firestore.model.ResourcePath; import com.google.firebase.firestore.util.Executors; import com.google.firebase.firestore.util.Util; +import java.util.Map; import javax.annotation.Nullable; /** @@ -116,13 +117,12 @@ public DocumentReference document(@NonNull String documentPath) { * Adds a new document to this collection with the specified data, assigning it a document ID * automatically. * - * @param data The data to write to the document (e.g. a Map or a POJO containing the desired - * document contents). + * @param data A Map containing the data for the new document. * @return A Task that will be resolved with the DocumentReference of the newly created document. */ @NonNull @PublicApi - public Task add(@NonNull Object data) { + public Task add(@NonNull Map data) { checkNotNull(data, "Provided data must not be null."); final DocumentReference ref = document(); return ref.set(data) @@ -134,4 +134,17 @@ public Task add(@NonNull Object data) { return ref; }); } + + /** + * Adds a new document to this collection with the specified POJO as contents, assigning it a + * document ID automatically. + * + * @param pojo The POJO that will be used to populate the contents of the document + * @return A Task that will be resolved with the DocumentReference of the newly created document. + */ + @NonNull + @PublicApi + public Task add(@NonNull Object pojo) { + return add(firestore.getDataConverter().convertPOJO(pojo)); + } } diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/DocumentReference.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/DocumentReference.java index cde2b8a3e3c..4fe7323b62f 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/DocumentReference.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/DocumentReference.java @@ -143,13 +143,12 @@ public CollectionReference collection(@NonNull String collectionPath) { * Overwrites the document referred to by this DocumentReference. If the document does not yet * exist, it will be created. If a document already exists, it will be overwritten. * - * @param data The data to write to the document (e.g. a Map or a POJO containing the desired - * document contents). + * @param data A map of the fields and values for the document. * @return A Task that will be resolved when the write finishes. */ @NonNull @PublicApi - public Task set(@NonNull Object data) { + public Task set(@NonNull Map data) { return set(data, SetOptions.OVERWRITE); } @@ -158,14 +157,13 @@ public Task set(@NonNull Object data) { * exist, it will be created. If you pass {@link SetOptions}, the provided data can be merged into * an existing document. * - * @param data The data to write to the document (e.g. a Map or a POJO containing the desired - * document contents). + * @param data A map of the fields and values for the document. * @param options An object to configure the set behavior. * @return A Task that will be resolved when the write finishes. */ @NonNull @PublicApi - public Task set(@NonNull Object data, @NonNull SetOptions options) { + public Task set(@NonNull Map data, @NonNull SetOptions options) { checkNotNull(data, "Provided data must not be null."); checkNotNull(options, "Provided options must not be null."); ParsedSetData parsed = @@ -178,6 +176,34 @@ public Task set(@NonNull Object data, @NonNull SetOptions options) { .continueWith(Executors.DIRECT_EXECUTOR, voidErrorTransformer()); } + /** + * Overwrites the document referred to by this DocumentReference. If the document does not yet + * exist, it will be created. If a document already exists, it will be overwritten. + * + * @param pojo The POJO that will be used to populate the document contents + * @return A Task that will be resolved when the write finishes. + */ + @NonNull + @PublicApi + public Task set(@NonNull Object pojo) { + return set(firestore.getDataConverter().convertPOJO(pojo), SetOptions.OVERWRITE); + } + + /** + * Writes to the document referred to by this DocumentReference. If the document does not yet + * exist, it will be created. If you pass {@link SetOptions}, the provided data can be merged into + * an existing document. + * + * @param pojo The POJO that will be used to populate the document contents + * @param options An object to configure the set behavior. + * @return A Task that will be resolved when the write finishes. + */ + @NonNull + @PublicApi + public Task set(@NonNull Object pojo, @NonNull SetOptions options) { + return set(firestore.getDataConverter().convertPOJO(pojo), options); + } + /** * Updates fields in the document referred to by this DocumentReference. If no document exists * yet, the update will fail. diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/DocumentSnapshot.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/DocumentSnapshot.java index 4a13eefd90a..e3444ab2e5e 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/DocumentSnapshot.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/DocumentSnapshot.java @@ -289,69 +289,6 @@ public Object get( firestore.getFirestoreSettings().areTimestampsInSnapshotsEnabled())); } - /** - * Returns the value at the field, converted to a POJO, or null if the field or document doesn't - * exist. - * - * @param field The path to the field - * @param valueType The Java class to convert the field value to. - * @return The value at the given field or null. - */ - @Nullable - public T get(@NonNull String field, @NonNull Class valueType) { - return get(FieldPath.fromDotSeparatedPath(field), valueType, ServerTimestampBehavior.DEFAULT); - } - - /** - * Returns the value at the field, converted to a POJO, or null if the field or document doesn't - * exist. - * - * @param field The path to the field - * @param valueType The Java class to convert the field value to. - * @param serverTimestampBehavior Configures the behavior for server timestamps that have not yet - * been set to their final value. - * @return The value at the given field or null. - */ - @Nullable - public T get( - @NonNull String field, - @NonNull Class valueType, - @NonNull ServerTimestampBehavior serverTimestampBehavior) { - return get(FieldPath.fromDotSeparatedPath(field), valueType, serverTimestampBehavior); - } - - /** - * Returns the value at the field, converted to a POJO, or null if the field or document doesn't - * exist. - * - * @param fieldPath The path to the field - * @param valueType The Java class to convert the field value to. - * @return The value at the given field or null. - */ - @Nullable - public T get(@NonNull FieldPath fieldPath, @NonNull Class valueType) { - return get(fieldPath, valueType, ServerTimestampBehavior.DEFAULT); - } - - /** - * Returns the value at the field, converted to a POJO, or null if the field or document doesn't - * exist. - * - * @param fieldPath The path to the field - * @param valueType The Java class to convert the field value to. - * @param serverTimestampBehavior Configures the behavior for server timestamps that have not yet - * been set to their final value. - * @return The value at the given field or null. - */ - @Nullable - public T get( - @NonNull FieldPath fieldPath, - @NonNull Class valueType, - @NonNull ServerTimestampBehavior serverTimestampBehavior) { - Object data = get(fieldPath, serverTimestampBehavior); - return data == null ? null : CustomClassMapper.convertToCustomClass(data, valueType); - } - /** * Returns the value of the field as a boolean. If the value is not a boolean this will throw a * runtime exception. diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/Transaction.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/Transaction.java index 8951d3c4b08..11f7601d49c 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/Transaction.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/Transaction.java @@ -60,13 +60,13 @@ public class Transaction { * yet exist, it will be created. If a document already exists, it will be overwritten. * * @param documentRef The DocumentReference to overwrite. - * @param data The data to write to the document (e.g. a Map or a POJO containing the desired - * document contents). + * @param data A map of the fields and values for the document. * @return This Transaction instance. Used for chaining method calls. */ @NonNull @PublicApi - public Transaction set(@NonNull DocumentReference documentRef, @NonNull Object data) { + public Transaction set( + @NonNull DocumentReference documentRef, @NonNull Map data) { return set(documentRef, data, SetOptions.OVERWRITE); } @@ -76,15 +76,16 @@ public Transaction set(@NonNull DocumentReference documentRef, @NonNull Object d * into an existing document. * * @param documentRef The DocumentReference to overwrite. - * @param data The data to write to the document (e.g. a Map or a POJO containing the desired - * document contents). + * @param data A map of the fields and values for the document. * @param options An object to configure the set behavior. * @return This Transaction instance. Used for chaining method calls. */ @NonNull @PublicApi public Transaction set( - @NonNull DocumentReference documentRef, @NonNull Object data, @NonNull SetOptions options) { + @NonNull DocumentReference documentRef, + @NonNull Map data, + @NonNull SetOptions options) { firestore.validateReference(documentRef); checkNotNull(data, "Provided data must not be null."); checkNotNull(options, "Provided options must not be null."); @@ -96,6 +97,37 @@ public Transaction set( return this; } + /** + * Overwrites the document referred to by the provided DocumentReference. If the document does not + * yet exist, it will be created. If a document already exists, it will be overwritten. + * + * @param documentRef The DocumentReference to overwrite. + * @param pojo The POJO that will be used to populate the document contents + * @return This Transaction instance. Used for chaining method calls. + */ + @NonNull + @PublicApi + public Transaction set(@NonNull DocumentReference documentRef, @NonNull Object pojo) { + return set(documentRef, firestore.getDataConverter().convertPOJO(pojo), SetOptions.OVERWRITE); + } + + /** + * Writes to the document referred to by the provided DocumentReference. If the document does not + * yet exist, it will be created. If you pass {@link SetOptions}, the provided data can be merged + * into an existing document. + * + * @param documentRef The DocumentReference to overwrite. + * @param pojo The POJO that will be used to populate the document contents + * @param options An object to configure the set behavior. + * @return This Transaction instance. Used for chaining method calls. + */ + @NonNull + @PublicApi + public Transaction set( + @NonNull DocumentReference documentRef, @NonNull Object pojo, @NonNull SetOptions options) { + return set(documentRef, firestore.getDataConverter().convertPOJO(pojo), options); + } + /** * Updates fields in the document referred to by the provided DocumentReference. If no document * exists yet, the update will fail. diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/UserDataConverter.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/UserDataConverter.java index 898a189fa3a..d02ae4eaac6 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/UserDataConverter.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/UserDataConverter.java @@ -70,16 +70,17 @@ public UserDataConverter(DatabaseId databaseId) { } /** Parse document data from a non-merge set() call. */ - public ParsedSetData parseSetData(Object input) { + public ParsedSetData parseSetData(Map input) { ParseAccumulator accumulator = new ParseAccumulator(UserData.Source.Set); - ObjectValue updateData = convertAndParseDocumentData(input, accumulator.rootContext()); - return accumulator.toSetData(updateData); + FieldValue updateData = parseData(input, accumulator.rootContext()); + + return accumulator.toSetData((ObjectValue) updateData); } /** Parse document data from a set() call with SetOptions.merge() set. */ - public ParsedSetData parseMergeData(Object input, @Nullable FieldMask fieldMask) { + public ParsedSetData parseMergeData(Map input, @Nullable FieldMask fieldMask) { ParseAccumulator accumulator = new ParseAccumulator(UserData.Source.MergeSet); - ObjectValue updateData = convertAndParseDocumentData(input, accumulator.rootContext()); + ObjectValue updateData = (ObjectValue) parseData(input, accumulator.rootContext()); if (fieldMask != null) { // Verify that all elements specified in the field mask are part of the parsed context. @@ -93,6 +94,7 @@ public ParsedSetData parseMergeData(Object input, @Nullable FieldMask fieldMask) } return accumulator.toMergeData(updateData, fieldMask); + } else { return accumulator.toMergeData(updateData); } @@ -116,9 +118,7 @@ public ParsedUpdateData parseUpdateData(Map data) { // Add it to the field mask, but don't add anything to updateData. context.addToFieldMask(fieldPath); } else { - @Nullable - FieldValue parsedValue = - convertAndParseFieldData(fieldValue, context.childContext(fieldPath)); + @Nullable FieldValue parsedValue = parseData(fieldValue, context.childContext(fieldPath)); if (parsedValue != null) { context.addToFieldMask(fieldPath); updateData = updateData.set(fieldPath, parsedValue); @@ -168,8 +168,7 @@ public ParsedUpdateData parseUpdateData(List fieldsAndValues) { // Add it to the field mask, but don't add anything to updateData. context.addToFieldMask(parsedField); } else { - FieldValue parsedValue = - convertAndParseFieldData(fieldValue, context.childContext(parsedField)); + FieldValue parsedValue = parseData(fieldValue, context.childContext(parsedField)); if (parsedValue != null) { context.addToFieldMask(parsedField); updateData = updateData.set(parsedField, parsedValue); @@ -184,7 +183,7 @@ public ParsedUpdateData parseUpdateData(List fieldsAndValues) { public FieldValue parseQueryValue(Object input) { ParseAccumulator accumulator = new ParseAccumulator(UserData.Source.Argument); - @Nullable FieldValue parsed = convertAndParseFieldData(input, accumulator.rootContext()); + @Nullable FieldValue parsed = parseData(input, accumulator.rootContext()); hardAssert(parsed != null, "Parsed data should not be null."); hardAssert( accumulator.getFieldTransforms().isEmpty(), @@ -192,38 +191,32 @@ public FieldValue parseQueryValue(Object input) { return parsed; } - /** Converts a POJO to native types and then parses it into model types. */ - private FieldValue convertAndParseFieldData(Object input, ParseContext context) { - Object converted = CustomClassMapper.convertToPlainJavaTypes(input); - return parseData(converted, context); - } - /** - * Converts a POJO to native types and then parses it into model types. It expects the input to - * conform to document data (i.e. it must parse into an ObjectValue model type) and will throw an - * appropriate error otherwise. + * Converts a POJO into a Map, throwing appropriate errors if it wasn't actually a proper POJO. */ - private ObjectValue convertAndParseDocumentData(Object input, ParseContext context) { - String badDocReason = + public Map convertPOJO(Object pojo) { + checkNotNull(pojo, "Provided data must not be null."); + String reason = "Invalid data. Data must be a Map or a suitable POJO object, but it was "; // Check Array before calling CustomClassMapper since it'll give you a confusing message - // to use List instead, which also won't work in a set(). - if (input.getClass().isArray()) { - throw new IllegalArgumentException(badDocReason + "an array"); + // to use List instead, which also won't work. + if (pojo.getClass().isArray()) { + throw new IllegalArgumentException(reason + "an array"); } - Object converted = CustomClassMapper.convertToPlainJavaTypes(input); - FieldValue value = parseData(converted, context); - - if (!(value instanceof ObjectValue)) { - throw new IllegalArgumentException(badDocReason + "of type: " + Util.typeName(input)); + Object converted = CustomClassMapper.convertToPlainJavaTypes(pojo); + if (!(converted instanceof Map)) { + throw new IllegalArgumentException(reason + "of type: " + Util.typeName(pojo)); } - return (ObjectValue) value; + + @SuppressWarnings("unchecked") // CustomClassMapper promises to map keys to Strings. + Map map = (Map) converted; + return map; } /** - * Recursive helper for parsing user data. + * Internal helper for parsing user data. * * @param input Data to be parsed. * @param context A context object representing the current path being parsed, the source of the @@ -423,7 +416,7 @@ private List parseArrayTransformElements(List elements) { // being unioned or removed are not considered writes since they cannot // contain any FieldValue sentinels, etc. ParseContext context = accumulator.rootContext(); - result.add(convertAndParseFieldData(element, context.childContext(i))); + result.add(parseData(element, context.childContext(i))); } return result; } diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/WriteBatch.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/WriteBatch.java index ec0fdd23c76..6590b9254f0 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/WriteBatch.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/WriteBatch.java @@ -59,13 +59,12 @@ public class WriteBatch { * yet exist, it will be created. If a document already exists, it will be overwritten. * * @param documentRef The DocumentReference to overwrite. - * @param data The data to write to the document (e.g. a Map or a POJO containing the desired - * document contents). + * @param data A map of the fields and values for the document. * @return This WriteBatch instance. Used for chaining method calls. */ @NonNull @PublicApi - public WriteBatch set(@NonNull DocumentReference documentRef, @NonNull Object data) { + public WriteBatch set(@NonNull DocumentReference documentRef, @NonNull Map data) { return set(documentRef, data, SetOptions.OVERWRITE); } @@ -75,18 +74,18 @@ public WriteBatch set(@NonNull DocumentReference documentRef, @NonNull Object da * into an existing document. * * @param documentRef The DocumentReference to overwrite. - * @param data The data to write to the document (e.g. a Map or a POJO containing the desired - * document contents). + * @param data A map of the fields and values for the document. * @param options An object to configure the set behavior. * @return This WriteBatch instance. Used for chaining method calls. */ @NonNull @PublicApi public WriteBatch set( - @NonNull DocumentReference documentRef, @NonNull Object data, @NonNull SetOptions options) { + @NonNull DocumentReference documentRef, + @NonNull Map data, + @NonNull SetOptions options) { firestore.validateReference(documentRef); checkNotNull(data, "Provided data must not be null."); - checkNotNull(options, "Provided options must not be null."); verifyNotCommitted(); ParsedSetData parsed = options.isMerge() @@ -96,6 +95,37 @@ public WriteBatch set( return this; } + /** + * Overwrites the document referred to by the provided DocumentReference. If the document does not + * yet exist, it will be created. If a document already exists, it will be overwritten. + * + * @param documentRef The DocumentReference to overwrite. + * @param pojo The POJO that will be used to populate the document contents. + * @return This WriteBatch instance. Used for chaining method calls. + */ + @NonNull + @PublicApi + public WriteBatch set(@NonNull DocumentReference documentRef, @NonNull Object pojo) { + return set(documentRef, firestore.getDataConverter().convertPOJO(pojo), SetOptions.OVERWRITE); + } + + /** + * Writes to the document referred to by the provided DocumentReference. If the document does not + * yet exist, it will be created. If you pass {@link SetOptions}, the provided data can be merged + * into an existing document. + * + * @param documentRef The DocumentReference to overwrite. + * @param pojo The POJO that will be used to populate the document contents. + * @param options An object to configure the set behavior. + * @return This WriteBatch instance. Used for chaining method calls. + */ + @NonNull + @PublicApi + public WriteBatch set( + @NonNull DocumentReference documentRef, @NonNull Object pojo, @NonNull SetOptions options) { + return set(documentRef, firestore.getDataConverter().convertPOJO(pojo), options); + } + /** * Updates fields in the document referred to by the provided DocumentReference. If no document * exists yet, the update will fail. diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/util/CustomClassMapper.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/util/CustomClassMapper.java index a82b1c83427..2fc95af3d74 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/util/CustomClassMapper.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/util/CustomClassMapper.java @@ -161,8 +161,7 @@ private static Object serialize(T o, ErrorPath path) { || o instanceof Timestamp || o instanceof GeoPoint || o instanceof Blob - || o instanceof DocumentReference - || o instanceof FieldValue) { + || o instanceof DocumentReference) { return o; } else { Class clazz = (Class) o.getClass(); @@ -509,12 +508,12 @@ private static T convertBean(Object o, Class clazz, ErrorPath path) { } } - private static IllegalArgumentException serializeError(ErrorPath path, String reason) { + private static RuntimeException serializeError(ErrorPath path, String reason) { reason = "Could not serialize object. " + reason; if (path.getLength() > 0) { reason = reason + " (found in field '" + path.toString() + "')"; } - return new IllegalArgumentException(reason); + return new RuntimeException(reason); } private static RuntimeException deserializeError(ErrorPath path, String reason) { diff --git a/firebase-firestore/src/test/java/com/google/firebase/firestore/model/FieldValueTest.java b/firebase-firestore/src/test/java/com/google/firebase/firestore/model/FieldValueTest.java index 7855d4bea65..1e1cafe69fc 100644 --- a/firebase-firestore/src/test/java/com/google/firebase/firestore/model/FieldValueTest.java +++ b/firebase-firestore/src/test/java/com/google/firebase/firestore/model/FieldValueTest.java @@ -411,7 +411,7 @@ public void testArraysFail() { wrap(array); fail("wrap should have failed"); } catch (IllegalArgumentException e) { - assertNotEquals(-1, e.getMessage().indexOf("use Lists instead")); + assertNotEquals(-1, e.getMessage().indexOf("use a List instead")); } } From fc71eff0c4f83820d80918d2a980844a1edb3132 Mon Sep 17 00:00:00 2001 From: Konstantin Varlamov Date: Wed, 28 Nov 2018 08:45:48 -0500 Subject: [PATCH 05/25] Performance optimizations to speed up reading large collections (#123) * memoize the encoded form of `Document`s received from the backend and use it when writing to local storage instead of reencoding; * when applying remote event, get all base documents in a single query, don't issue a separate query for each; * when applying remote event, don't reread documents from local storage after processing them. --- .../firestore/local/LocalDocumentsView.java | 36 ++++- .../firestore/local/LocalSerializer.java | 8 +- .../firebase/firestore/local/LocalStore.java | 15 +- .../local/MemoryRemoteDocumentCache.java | 14 ++ .../firestore/local/RemoteDocumentCache.java | 10 ++ .../firestore/local/SQLiteMutationQueue.java | 61 +++----- .../firestore/local/SQLitePersistence.java | 138 ++++++++++++++++++ .../local/SQLiteRemoteDocumentCache.java | 36 +++++ .../firebase/firestore/model/Document.java | 23 +++ .../firestore/remote/AbstractStream.java | 13 +- .../firestore/remote/RemoteSerializer.java | 8 +- .../local/RemoteDocumentCacheTestCase.java | 58 ++++++++ 12 files changed, 364 insertions(+), 56 deletions(-) diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LocalDocumentsView.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LocalDocumentsView.java index 6f82a815ddd..8db4d4b1a6c 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LocalDocumentsView.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LocalDocumentsView.java @@ -69,6 +69,21 @@ private MaybeDocument getDocument(DocumentKey key, List inBatches return document; } + // Returns the view of the given {@code docs} as they would appear after applying all mutations in + // the given {@code batches}. + private Map applyLocalMutationsToDocuments( + Map docs, List batches) { + for (Map.Entry base : docs.entrySet()) { + MaybeDocument localView = base.getValue(); + for (MutationBatch batch : batches) { + localView = batch.applyToLocalView(base.getKey(), localView); + } + base.setValue(localView); + } + + return docs; + } + /** * Gets the local view of the documents identified by {@code keys}. * @@ -76,13 +91,24 @@ private MaybeDocument getDocument(DocumentKey key, List inBatches * for that key in the resulting set. */ ImmutableSortedMap getDocuments(Iterable keys) { + Map docs = remoteDocumentCache.getAll(keys); + return getLocalViewOfDocuments(docs); + } + + /** + * Similar to {@code #getDocuments}, but creates the local view from the given {@code baseDocs} + * without retrieving documents from the local store. + */ + ImmutableSortedMap getLocalViewOfDocuments( + Map baseDocs) { ImmutableSortedMap results = emptyMaybeDocumentMap(); - List batches = mutationQueue.getAllMutationBatchesAffectingDocumentKeys(keys); - for (DocumentKey key : keys) { - // TODO: PERF: Consider fetching all remote documents at once rather than - // one-by-one. - MaybeDocument maybeDoc = getDocument(key, batches); + List batches = + mutationQueue.getAllMutationBatchesAffectingDocumentKeys(baseDocs.keySet()); + Map docs = applyLocalMutationsToDocuments(baseDocs, batches); + for (Map.Entry entry : docs.entrySet()) { + DocumentKey key = entry.getKey(); + MaybeDocument maybeDoc = entry.getValue(); // TODO: Don't conflate missing / deleted. if (maybeDoc == null) { maybeDoc = new NoDocument(key, SnapshotVersion.NONE, /*hasCommittedMutations=*/ false); diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LocalSerializer.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LocalSerializer.java index 55c8bd146ab..3805ecffa6a 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LocalSerializer.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LocalSerializer.java @@ -48,13 +48,19 @@ public LocalSerializer(RemoteSerializer rpcSerializer) { com.google.firebase.firestore.proto.MaybeDocument encodeMaybeDocument(MaybeDocument document) { com.google.firebase.firestore.proto.MaybeDocument.Builder builder = com.google.firebase.firestore.proto.MaybeDocument.newBuilder(); + if (document instanceof NoDocument) { NoDocument noDocument = (NoDocument) document; builder.setNoDocument(encodeNoDocument(noDocument)); builder.setHasCommittedMutations(noDocument.hasCommittedMutations()); } else if (document instanceof Document) { Document existingDocument = (Document) document; - builder.setDocument(encodeDocument(existingDocument)); + // Use the memoized encoded form if it exists. + if (existingDocument.getProto() != null) { + builder.setDocument(existingDocument.getProto()); + } else { + builder.setDocument(encodeDocument(existingDocument)); + } builder.setHasCommittedMutations(existingDocument.hasCommittedMutations()); } else if (document instanceof UnknownDocument) { builder.setUnknownDocument(encodeUnknownDocument((UnknownDocument) document)); diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LocalStore.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LocalStore.java index 6c203edd3ad..46d2d480b62 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LocalStore.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LocalStore.java @@ -35,6 +35,7 @@ import com.google.firebase.firestore.remote.TargetChange; import com.google.firebase.firestore.util.Logger; import com.google.protobuf.ByteString; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -329,14 +330,19 @@ public ImmutableSortedMap applyRemoteEvent(RemoteEve } } - Set changedDocKeys = new HashSet<>(); + Map changedDocs = new HashMap<>(); Map documentUpdates = remoteEvent.getDocumentUpdates(); Set limboDocuments = remoteEvent.getResolvedLimboDocuments(); + // Each loop iteration only affects its "own" doc, so it's safe to get all the remote + // documents in advance in a single call. + Map existingDocs = + remoteDocuments.getAll(documentUpdates.keySet()); + for (Entry entry : documentUpdates.entrySet()) { DocumentKey key = entry.getKey(); MaybeDocument doc = entry.getValue(); - changedDocKeys.add(key); - MaybeDocument existingDoc = remoteDocuments.get(key); + MaybeDocument existingDoc = existingDocs.get(key); + // If a document update isn't authoritative, make sure we don't // apply an old document version to the remote cache. We make an // exception for SnapshotVersion.MIN which can happen for @@ -347,6 +353,7 @@ public ImmutableSortedMap applyRemoteEvent(RemoteEve || (authoritativeUpdates.contains(doc.getKey()) && !existingDoc.hasPendingWrites()) || doc.getVersion().compareTo(existingDoc.getVersion()) >= 0) { remoteDocuments.add(doc); + changedDocs.put(key, doc); } else { Logger.debug( "LocalStore", @@ -376,7 +383,7 @@ public ImmutableSortedMap applyRemoteEvent(RemoteEve queryCache.setLastRemoteSnapshotVersion(remoteVersion); } - return localDocuments.getDocuments(changedDocKeys); + return localDocuments.getLocalViewOfDocuments(changedDocs); }); } diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryRemoteDocumentCache.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryRemoteDocumentCache.java index f4209da214a..86c99a56b5f 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryRemoteDocumentCache.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryRemoteDocumentCache.java @@ -23,6 +23,7 @@ import com.google.firebase.firestore.model.DocumentKey; import com.google.firebase.firestore.model.MaybeDocument; import com.google.firebase.firestore.model.ResourcePath; +import java.util.HashMap; import java.util.Iterator; import java.util.Map; import javax.annotation.Nullable; @@ -53,6 +54,19 @@ public MaybeDocument get(DocumentKey key) { return docs.get(key); } + @Override + public Map getAll(Iterable keys) { + Map result = new HashMap<>(); + + for (DocumentKey key : keys) { + // Make sure each key has a corresponding entry, which is null in case the document is not + // found. + result.put(key, get(key)); + } + + return result; + } + @Override public ImmutableSortedMap getAllDocumentsMatchingQuery(Query query) { ImmutableSortedMap result = emptyDocumentMap(); diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/RemoteDocumentCache.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/RemoteDocumentCache.java index f3e689f221f..1acd0de8df8 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/RemoteDocumentCache.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/RemoteDocumentCache.java @@ -19,6 +19,7 @@ import com.google.firebase.firestore.model.Document; import com.google.firebase.firestore.model.DocumentKey; import com.google.firebase.firestore.model.MaybeDocument; +import java.util.Map; import javax.annotation.Nullable; /** @@ -51,6 +52,15 @@ interface RemoteDocumentCache { @Nullable MaybeDocument get(DocumentKey documentKey); + /** + * Looks up a set of entries in the cache. + * + * @param documentKeys The keys of the entries to look up. + * @return The cached Document or NoDocument entries indexed by key. If an entry is not cached, + * the corresponding key will be mapped to a null value. + */ + Map getAll(Iterable documentKeys); + /** * Executes a query against the cached Document entries * diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLiteMutationQueue.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLiteMutationQueue.java index ccf5bb70490..41d74c1b1d3 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLiteMutationQueue.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLiteMutationQueue.java @@ -32,9 +32,9 @@ import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.MessageLite; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Set; import javax.annotation.Nullable; @@ -277,46 +277,29 @@ public List getAllMutationBatchesAffectingDocumentKey(DocumentKey @Override public List getAllMutationBatchesAffectingDocumentKeys( Iterable documentKeys) { - List result = new ArrayList<>(); - if (!documentKeys.iterator().hasNext()) { - return result; + List args = new ArrayList<>(); + for (DocumentKey key : documentKeys) { + args.add(EncodedPath.encode(key.getPath())); } - // SQLite limits maximum number of host parameters to 999 (see - // https://www.sqlite.org/limits.html). To work around this, split the given keys into several - // smaller sets and issue a separate query for each. - int limit = 900; - Iterator keyIter = documentKeys.iterator(); + SQLitePersistence.LongQuery longQuery = + new SQLitePersistence.LongQuery( + db, + "SELECT DISTINCT dm.batch_id, m.mutations FROM document_mutations dm, mutations m " + + "WHERE dm.uid = ? " + + "AND dm.path IN (", + Arrays.asList(uid), + args, + ") " + + "AND dm.uid = m.uid " + + "AND dm.batch_id = m.batch_id " + + "ORDER BY dm.batch_id"); + + List result = new ArrayList<>(); Set uniqueBatchIds = new HashSet<>(); - int queriesPerformed = 0; - while (keyIter.hasNext()) { - ++queriesPerformed; - StringBuilder placeholdersBuilder = new StringBuilder(); - List args = new ArrayList<>(); - args.add(uid); - - for (int i = 0; keyIter.hasNext() && i < limit; i++) { - DocumentKey key = keyIter.next(); - - if (i > 0) { - placeholdersBuilder.append(", "); - } - placeholdersBuilder.append("?"); - - args.add(EncodedPath.encode(key.getPath())); - } - String placeholders = placeholdersBuilder.toString(); - - db.query( - "SELECT DISTINCT dm.batch_id, m.mutations FROM document_mutations dm, mutations m " - + "WHERE dm.uid = ? " - + "AND dm.path IN (" - + placeholders - + ") " - + "AND dm.uid = m.uid " - + "AND dm.batch_id = m.batch_id " - + "ORDER BY dm.batch_id") - .binding(args.toArray()) + while (longQuery.hasMoreSubqueries()) { + longQuery + .performNextSubquery() .forEach( row -> { int batchId = row.getInt(0); @@ -330,7 +313,7 @@ public List getAllMutationBatchesAffectingDocumentKeys( // If more than one query was issued, batches might be in an unsorted order (batches are ordered // within one query's results, but not across queries). It's likely to be rare, so don't impose // performance penalty on the normal case. - if (queriesPerformed > 1) { + if (longQuery.getSubqueriesPerformed() > 1) { Collections.sort( result, (MutationBatch lhs, MutationBatch rhs) -> diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLitePersistence.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLitePersistence.java index 6b5f06eab5b..7ecf6050c52 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLitePersistence.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLitePersistence.java @@ -36,6 +36,10 @@ import com.google.firebase.firestore.util.Supplier; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; import javax.annotation.Nullable; /** @@ -468,6 +472,140 @@ private Cursor startQuery() { } } + /** + * Encapsulates a query whose parameter list is so long that it might exceed SQLite limit. + * + *

SQLite limits maximum number of host parameters to 999 (see + * https://www.sqlite.org/limits.html). This class wraps most of the messy details of splitting a + * large query into several smaller ones. + * + *

The class is configured to contain a "template" for each subquery: + * + *

    + *
  1. head -- the beginning of the query, will be the same for each subquery + *
  2. tail -- the end of the query, also the same for each subquery + *
+ * + *

Then the host parameters will be inserted in-between head and tail; if there are too many + * arguments for a single query, several subqueries will be issued. Each subquery which will have + * the following form: + * + *

[head][an auto-generated comma-separated list of '?' placeholders][tail] + * + *

To use this class, keep calling {@link #performNextSubquery}, which will issue the next + * subquery, as long as {@link #hasMoreSubqueries} returns true. Note that if the parameter list + * is empty, not even a single query will be issued. + * + *

For example, imagine for demonstration purposes that the limit were 2, and the {@code + * LongQuery} was created like this: + * + * 

+   *     String[] args = {"foo", "bar", "baz", "spam", "eggs"};
+   *     LongQuery longQuery = new LongQuery(
+   *         db,
+   *         "SELECT name WHERE id in (",
+   *         Arrays.asList(args),
+   *         ")"
+   *     );
+   *
+ * + *

Assuming limit of 2, this query will issue three subqueries: + * + * 

+   *     query.performNextSubquery(); // "SELECT name WHERE id in (?, ?)", binding "foo" and "bar"
+   *     query.performNextSubquery(); // "SELECT name WHERE id in (?, ?)", binding "baz" and "spam"
+   *     query.performNextSubquery(); // "SELECT name WHERE id in (?)", binding "eggs"
+   *
+ */ + static class LongQuery { + private final SQLitePersistence db; + // The non-changing beginning of each subquery. + private final String head; + // The non-changing end of each subquery. + private final String tail; + // Arguments that will be prepended in each subquery before the main argument list. + private final List argsHead; + + private int subqueriesPerformed = 0; + private final Iterator argsIter; + + // Limit for the number of host parameters beyond which a query will be split into several + // subqueries. Deliberately set way below 999 as a safety measure because this class doesn't + // attempt to check for placeholders in the query {@link head}; if it only relied on the number + // of placeholders it itself generates, in that situation it would still exceed the SQLite + // limit. + private static final int LIMIT = 900; + + /** + * Creates a new {@code LongQuery} with parameters that describe a template for creating each + * subquery. + * + * @param db The database on which to execute the query. + * @param head The non-changing beginning of the query; each subquery will begin with this. + * @param allArgs The list of host parameters to bind. If the list size exceeds the limit, + * several subqueries will be issued, and the correct number of placeholders will be + * generated for each subquery. + * @param tail The non-changing end of the query; each subquery will end with this. + */ + LongQuery(SQLitePersistence db, String head, List allArgs, String tail) { + this.db = db; + this.head = head; + this.argsHead = Collections.emptyList(); + this.tail = tail; + + argsIter = allArgs.iterator(); + } + + /** + * The longer version of the constructor additionally takes {@code argsHead} parameter that + * contains parameters that will be reissued in each subquery, i.e. subqueries take the form: + * + *

[head][argsHead][an auto-generated comma-separated list of '?' placeholders][tail] + */ + LongQuery( + SQLitePersistence db, + String head, + List argsHead, + List allArgs, + String tail) { + this.db = db; + this.head = head; + this.argsHead = argsHead; + this.tail = tail; + + argsIter = allArgs.iterator(); + } + + /** Whether {@link #performNextSubquery} can be called. */ + boolean hasMoreSubqueries() { + return argsIter.hasNext(); + } + + /** Performs the next subquery and returns a {@link Query} object for method chaining. */ + Query performNextSubquery() { + ++subqueriesPerformed; + + List subqueryArgs = new ArrayList<>(argsHead); + StringBuilder placeholdersBuilder = new StringBuilder(); + for (int i = 0; argsIter.hasNext() && i < LIMIT - argsHead.size(); i++) { + if (i > 0) { + placeholdersBuilder.append(", "); + } + placeholdersBuilder.append("?"); + + subqueryArgs.add(argsIter.next()); + } + String placeholders = placeholdersBuilder.toString(); + + return db.query(head + placeholders + tail).binding(subqueryArgs.toArray()); + } + + /** How many subqueries were performed. */ + int getSubqueriesPerformed() { + return subqueriesPerformed; + } + } + /** * Binds the given arguments to the given SQLite statement or query. * diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLiteRemoteDocumentCache.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLiteRemoteDocumentCache.java index 49171428705..67f1524cb71 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLiteRemoteDocumentCache.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLiteRemoteDocumentCache.java @@ -24,7 +24,9 @@ import com.google.firebase.firestore.model.ResourcePath; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.MessageLite; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import javax.annotation.Nullable; @@ -66,6 +68,40 @@ public MaybeDocument get(DocumentKey documentKey) { .firstValue(row -> decodeMaybeDocument(row.getBlob(0))); } + @Override + public Map getAll(Iterable documentKeys) { + List args = new ArrayList<>(); + for (DocumentKey key : documentKeys) { + args.add(EncodedPath.encode(key.getPath())); + } + + Map results = new HashMap<>(); + for (DocumentKey key : documentKeys) { + // Make sure each key has a corresponding entry, which is null in case the document is not + // found. + results.put(key, null); + } + + SQLitePersistence.LongQuery longQuery = + new SQLitePersistence.LongQuery( + db, + "SELECT contents FROM remote_documents " + "WHERE path IN (", + args, + ") ORDER BY path"); + + while (longQuery.hasMoreSubqueries()) { + longQuery + .performNextSubquery() + .forEach( + row -> { + MaybeDocument decoded = decodeMaybeDocument(row.getBlob(0)); + results.put(decoded.getKey(), decoded); + }); + } + + return results; + } + @Override public ImmutableSortedMap getAllDocumentsMatchingQuery(Query query) { // Use the query path as a prefix for testing if a document matches the query. diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/model/Document.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/model/Document.java index 83eeaa28412..a0be1936ef6 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/model/Document.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/model/Document.java @@ -52,11 +52,34 @@ public static Comparator keyComparator() { private final DocumentState documentState; + /** + * Memoized serialized form of the document for optimization purposes (avoids repeated + * serialization). Might be null. + */ + private final com.google.firestore.v1beta1.Document proto; + + public @Nullable com.google.firestore.v1beta1.Document getProto() { + return proto; + } + public Document( DocumentKey key, SnapshotVersion version, ObjectValue data, DocumentState documentState) { super(key, version); this.data = data; this.documentState = documentState; + this.proto = null; + } + + public Document( + DocumentKey key, + SnapshotVersion version, + ObjectValue data, + DocumentState documentState, + com.google.firestore.v1beta1.Document proto) { + super(key, version); + this.data = data; + this.documentState = documentState; + this.proto = proto; } public ObjectValue getData() { diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/remote/AbstractStream.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/remote/AbstractStream.java index 1cd3d7d1c43..4eede2ee92f 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/remote/AbstractStream.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/remote/AbstractStream.java @@ -109,11 +109,13 @@ public void onHeaders(Metadata headers) { public void onNext(RespT response) { dispatcher.run( () -> { - Logger.debug( - AbstractStream.this.getClass().getSimpleName(), - "(%x) Stream received: %s", - System.identityHashCode(AbstractStream.this), - response); + if (Logger.isDebugEnabled()) { + Logger.debug( + AbstractStream.this.getClass().getSimpleName(), + "(%x) Stream received: %s", + System.identityHashCode(AbstractStream.this), + response); + } AbstractStream.this.onNext(response); }); } @@ -203,6 +205,7 @@ public void run() { this.idleTimerId = idleTimerId; this.listener = listener; this.idleTimeoutRunnable = new IdleTimeoutRunnable(); + backoff = new ExponentialBackoff( workerQueue, diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/remote/RemoteSerializer.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/remote/RemoteSerializer.java index fc4eb36bd30..8a58768f1a6 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/remote/RemoteSerializer.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/remote/RemoteSerializer.java @@ -409,7 +409,7 @@ private Document decodeFoundDocument(BatchGetDocumentsResponse response) { SnapshotVersion version = decodeVersion(response.getFound().getUpdateTime()); hardAssert( !version.equals(SnapshotVersion.NONE), "Got a document response with no snapshot version"); - return new Document(key, version, value, Document.DocumentState.SYNCED); + return new Document(key, version, value, Document.DocumentState.SYNCED, response.getFound()); } private NoDocument decodeMissingDocument(BatchGetDocumentsResponse response) { @@ -1014,7 +1014,11 @@ public WatchChange decodeWatchChange(ListenResponse protoChange) { hardAssert( !version.equals(SnapshotVersion.NONE), "Got a document change without an update time"); ObjectValue data = decodeFields(docChange.getDocument().getFieldsMap()); - Document document = new Document(key, version, data, Document.DocumentState.SYNCED); + // The document may soon be re-serialized back to protos in order to store it in local + // persistence. Memoize the encoded form to avoid encoding it again. + Document document = + new Document( + key, version, data, Document.DocumentState.SYNCED, docChange.getDocument()); watchChange = new WatchChange.DocumentChange(added, removed, document.getKey(), document); break; case DOCUMENT_DELETE: diff --git a/firebase-firestore/src/test/java/com/google/firebase/firestore/local/RemoteDocumentCacheTestCase.java b/firebase-firestore/src/test/java/com/google/firebase/firestore/local/RemoteDocumentCacheTestCase.java index 4340b5f51d2..06c38e7ba7b 100644 --- a/firebase-firestore/src/test/java/com/google/firebase/firestore/local/RemoteDocumentCacheTestCase.java +++ b/firebase-firestore/src/test/java/com/google/firebase/firestore/local/RemoteDocumentCacheTestCase.java @@ -32,6 +32,9 @@ import com.google.firebase.firestore.model.DocumentKey; import com.google.firebase.firestore.model.MaybeDocument; import com.google.firebase.firestore.model.NoDocument; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import javax.annotation.Nullable; @@ -82,6 +85,51 @@ public void testSetAndReadDocument() { } } + @Test + public void testSetAndReadSeveralDocuments() { + String[] paths = {"a/b", "a/b/c/d/e/f"}; + Map written = new HashMap<>(); + for (String path : paths) { + written.put(DocumentKey.fromPathString(path), addTestDocumentAtPath(path)); + } + + Map read = getAll(Arrays.asList(paths)); + assertEquals(written, read); + } + + @Test + public void testReadSeveralDocumentsIncludingMissingDocument() { + String[] paths = {"foo/1", "foo/2"}; + Map written = new HashMap<>(); + for (String path : paths) { + written.put(DocumentKey.fromPathString(path), addTestDocumentAtPath(path)); + } + written.put(DocumentKey.fromPathString("foo/nonexistent"), null); + + List keys = new ArrayList(Arrays.asList(paths)); + keys.add("foo/nonexistent"); + Map read = getAll(keys); + assertEquals(written, read); + } + + // PORTING NOTE: this test only applies to Android, because it's the only platform where the + // implementation of getAll might split the input into several queries. + @Test + public void testSetAndReadLotsOfDocuments() { + // Make sure to force SQLite implementation to split the large query into several smaller ones. + int lotsOfDocuments = 2000; + List paths = new ArrayList<>(); + Map expected = new HashMap<>(); + for (int i = 0; i < lotsOfDocuments; i++) { + String path = "foo/" + String.valueOf(i); + paths.add(path); + expected.put(DocumentKey.fromPathString(path), addTestDocumentAtPath(path)); + } + + Map read = getAll(paths); + assertEquals(expected, read); + } + @Test public void testSetAndReadDeletedDocument() { String path = "a/b"; @@ -147,6 +195,16 @@ private MaybeDocument get(String path) { return remoteDocumentCache.get(key(path)); } + private Map getAll(Iterable paths) { + List keys = new ArrayList<>(); + + for (String path : paths) { + keys.add(key(path)); + } + + return remoteDocumentCache.getAll(keys); + } + private void remove(String path) { persistence.runTransaction("remove entry", () -> remoteDocumentCache.remove(key(path))); } From 03c141e3385d7c3ad5996f59562b34fbc14385a7 Mon Sep 17 00:00:00 2001 From: VinayGuthal Date: Wed, 28 Nov 2018 11:34:54 -0500 Subject: [PATCH 06/25] update versions (#141) --- firebase-database/gradle.properties | 4 ++-- firebase-firestore/gradle.properties | 4 ++-- firebase-functions/gradle.properties | 4 ++-- firebase-inappmessaging-display/gradle.properties | 2 +- firebase-storage/gradle.properties | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/firebase-database/gradle.properties b/firebase-database/gradle.properties index 6f746e26b95..6d52a9fd3bc 100644 --- a/firebase-database/gradle.properties +++ b/firebase-database/gradle.properties @@ -12,5 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -version=16.0.5 -latestReleasedVersion=16.0.4 +version=16.0.6 +latestReleasedVersion=16.0.5 diff --git a/firebase-firestore/gradle.properties b/firebase-firestore/gradle.properties index c442d0be711..0332b866ede 100644 --- a/firebase-firestore/gradle.properties +++ b/firebase-firestore/gradle.properties @@ -1,2 +1,2 @@ -version=17.1.3 -latestReleasedVersion=17.1.2 +version=17.1.4 +latestReleasedVersion=17.1.3 diff --git a/firebase-functions/gradle.properties b/firebase-functions/gradle.properties index 72b6a666ce9..e3aa9a1b902 100644 --- a/firebase-functions/gradle.properties +++ b/firebase-functions/gradle.properties @@ -1,2 +1,2 @@ -version=16.1.3 -latestReleasedVersion=16.1.2 +version=16.1.4 +latestReleasedVersion=16.1.3 diff --git a/firebase-inappmessaging-display/gradle.properties b/firebase-inappmessaging-display/gradle.properties index 29fb94e0127..90ec8c308f9 100644 --- a/firebase-inappmessaging-display/gradle.properties +++ b/firebase-inappmessaging-display/gradle.properties @@ -1 +1 @@ -version=17.0.4 +version=17.0.5 diff --git a/firebase-storage/gradle.properties b/firebase-storage/gradle.properties index 6f746e26b95..6d52a9fd3bc 100644 --- a/firebase-storage/gradle.properties +++ b/firebase-storage/gradle.properties @@ -12,5 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -version=16.0.5 -latestReleasedVersion=16.0.4 +version=16.0.6 +latestReleasedVersion=16.0.5 From 4ae3f1bded6e7608a2acb7e6ce80ddd03a4c3b38 Mon Sep 17 00:00:00 2001 From: rsgowman Date: Wed, 28 Nov 2018 13:23:42 -0500 Subject: [PATCH 07/25] Forbid queries endAt an uncommitted server timestamp. (#138) --- .../firebase/firestore/ValidationTest.java | 53 +++++++++++++++++++ .../com/google/firebase/firestore/Query.java | 13 ++++- 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/firebase-firestore/src/androidTest/java/com/google/firebase/firestore/ValidationTest.java b/firebase-firestore/src/androidTest/java/com/google/firebase/firestore/ValidationTest.java index 7bdc168da09..c92c366d9b9 100644 --- a/firebase-firestore/src/androidTest/java/com/google/firebase/firestore/ValidationTest.java +++ b/firebase-firestore/src/androidTest/java/com/google/firebase/firestore/ValidationTest.java @@ -14,6 +14,8 @@ package com.google.firebase.firestore; +import static com.google.common.truth.Truth.assertThat; +import static com.google.firebase.firestore.testutil.Assert.assertThrows; import static com.google.firebase.firestore.testutil.IntegrationTestUtil.testAlternateFirestore; import static com.google.firebase.firestore.testutil.IntegrationTestUtil.testCollection; import static com.google.firebase.firestore.testutil.IntegrationTestUtil.testCollectionWithDocs; @@ -30,6 +32,7 @@ import android.support.test.InstrumentationRegistry; import android.support.test.runner.AndroidJUnit4; +import com.google.android.gms.tasks.TaskCompletionSource; import com.google.firebase.FirebaseApp; import com.google.firebase.FirebaseOptions; import com.google.firebase.firestore.Transaction.Function; @@ -438,6 +441,56 @@ public void queriesCannotBeCreatedFromDocumentsMissingSortValues() { expectError(() -> query.endAt(snapshot), reason); } + @Test + public void queriesCannotBeSortedByAnUncommittedServerTimestamp() { + CollectionReference collection = testCollection(); + + // Ensure the server timestamp stays uncommitted for the first half of the test + waitFor(collection.firestore.getClient().disableNetwork()); + + TaskCompletionSource offlineCallbackDone = new TaskCompletionSource<>(); + TaskCompletionSource onlineCallbackDone = new TaskCompletionSource<>(); + + collection.addSnapshotListener( + (snapshot, error) -> { + assertNotNull(snapshot); + + // Skip the initial empty snapshot. + if (snapshot.isEmpty()) return; + + assertThat(snapshot.getDocuments()).hasSize(1); + DocumentSnapshot docSnap = snapshot.getDocuments().get(0); + + if (snapshot.getMetadata().hasPendingWrites()) { + // Offline snapshot. Since the server timestamp is uncommitted, we shouldn't be able to + // query by it. + assertThrows( + IllegalArgumentException.class, + () -> + collection + .orderBy("timestamp") + .endAt(docSnap) + .addSnapshotListener((snapshot2, error2) -> {})); + offlineCallbackDone.setResult(null); + } else { + // Online snapshot. Since the server timestamp is committed, we should be able to query + // by it. + collection + .orderBy("timestamp") + .endAt(docSnap) + .addSnapshotListener((snapshot2, error2) -> {}); + onlineCallbackDone.setResult(null); + } + }); + + DocumentReference document = collection.document(); + document.set(map("timestamp", FieldValue.serverTimestamp())); + waitFor(offlineCallbackDone.getTask()); + + waitFor(collection.firestore.getClient().enableNetwork()); + waitFor(onlineCallbackDone.getTask()); + } + @Test public void queriesMustNotHaveMoreComponentsThanOrderBy() { CollectionReference collection = testCollection(); diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/Query.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/Query.java index bb6143edcce..6008bb00f82 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/Query.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/Query.java @@ -39,6 +39,7 @@ import com.google.firebase.firestore.model.ResourcePath; import com.google.firebase.firestore.model.value.FieldValue; import com.google.firebase.firestore.model.value.ReferenceValue; +import com.google.firebase.firestore.model.value.ServerTimestampValue; import com.google.firebase.firestore.util.ExecutorEventListener; import com.google.firebase.firestore.util.Executors; import com.google.firebase.firestore.util.ListenerRegistrationImpl; @@ -582,7 +583,8 @@ public Query endAt(Object... fieldValues) { *

Note that the Bound will always include the key of the document and so only the provided * document will compare equal to the returned position. * - *

Will throw if the document does not contain all fields of the order by of the query. + *

Will throw if the document does not contain all fields of the order by of the query or if + * any of the fields in the order by are an uncommitted server timestamp. */ private Bound boundFromDocumentSnapshot( String methodName, DocumentSnapshot snapshot, boolean before) { @@ -606,7 +608,14 @@ private Bound boundFromDocumentSnapshot( components.add(ReferenceValue.valueOf(firestore.getDatabaseId(), document.getKey())); } else { FieldValue value = document.getField(orderBy.getField()); - if (value != null) { + if (value instanceof ServerTimestampValue) { + throw new IllegalArgumentException( + "Invalid query. You are trying to start or end a query using a document for which " + + "the field '" + + orderBy.getField() + + "' is an uncommitted server timestamp. (Since the value of this field is " + + "unknown, you cannot start/end a query with it.)"); + } else if (value != null) { components.add(value); } else { throw new IllegalArgumentException( From 58a4f73ac93702d9d17d8cd5299b3c8d152732ed Mon Sep 17 00:00:00 2001 From: rsgowman Date: Thu, 29 Nov 2018 09:51:39 -0500 Subject: [PATCH 08/25] Add shutdown logic to the grpc channel (#135) Without this, we get a potential resource leak when Firestore shuts down. Currently, it only shuts down in tests, so this doesn't impact production, however, with teh (soon-to-come) AndroiManagedChannel change, the integration tests will fail, as the Android OS itself tracks resources associated with AndroidManagedChannel and will terminate if they're not properly cleaned up. --- .../firebase/firestore/remote/Datastore.java | 4 ++ .../firestore/remote/RemoteStore.java | 3 +- .../firestore/util/FirestoreChannel.java | 49 +++++++++++++++++-- 3 files changed, 51 insertions(+), 5 deletions(-) diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/remote/Datastore.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/remote/Datastore.java index 9072ff17fe1..c5f20583ca6 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/remote/Datastore.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/remote/Datastore.java @@ -114,6 +114,10 @@ public Datastore( workerQueue, credentialsProvider, channelBuilder.build(), databaseInfo.getDatabaseId()); } + void shutdown() { + channel.shutdown(); + } + AsyncQueue getWorkerQueue() { return workerQueue; } diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/remote/RemoteStore.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/remote/RemoteStore.java index e1fd8a5605c..abfd1146112 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/remote/RemoteStore.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/remote/RemoteStore.java @@ -258,10 +258,9 @@ public void start() { */ public void shutdown() { Logger.debug(LOG_TAG, "Shutting down"); - // For now, all shutdown logic is handled by disableNetworkInternal(). We might expand on this - // in the future. networkEnabled = false; this.disableNetworkInternal(); + datastore.shutdown(); // Set the OnlineState to UNKNOWN (rather than OFFLINE) to avoid potentially triggering // spurious listener events with cached data, etc. onlineStateTracker.updateState(OnlineState.UNKNOWN); diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/util/FirestoreChannel.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/util/FirestoreChannel.java index 2bd3a42e415..84830d8f120 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/util/FirestoreChannel.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/util/FirestoreChannel.java @@ -25,7 +25,6 @@ import com.google.firestore.v1beta1.FirestoreGrpc; import com.google.firestore.v1beta1.FirestoreGrpc.FirestoreStub; import io.grpc.CallOptions; -import io.grpc.Channel; import io.grpc.ClientCall; import io.grpc.ManagedChannel; import io.grpc.Metadata; @@ -33,6 +32,7 @@ import io.grpc.Status; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.TimeUnit; /** * Wrapper class around io.grpc.Channel that adds headers, exception handling and simplifies @@ -58,7 +58,7 @@ public class FirestoreChannel { private final CredentialsProvider credentialsProvider; /** The underlying gRPC channel. */ - private final Channel channel; + private final ManagedChannel channel; /** Call options to be used when invoking RPCs. */ private final CallOptions callOptions; @@ -77,7 +77,7 @@ public FirestoreChannel( FirestoreCallCredentials firestoreHeaders = new FirestoreCallCredentials(credentialsProvider); FirestoreStub firestoreStub = FirestoreGrpc.newStub(grpcChannel).withCallCredentials(firestoreHeaders); - this.channel = firestoreStub.getChannel(); + this.channel = grpcChannel; this.callOptions = firestoreStub.getCallOptions(); this.resourcePrefixValue = @@ -85,6 +85,49 @@ public FirestoreChannel( "projects/%s/databases/%s", databaseId.getProjectId(), databaseId.getDatabaseId()); } + /** + * Shuts down the grpc channel. This is not reversible and renders the FirestoreChannel unusable. + */ + public void shutdown() { + channel.shutdown(); + try { + // TODO(rsgowman): Investigate occasional hangs in channel.shutdown(). + // + // While running the integration tests, channel.shutdown() will occasionally timeout. + // (Typically on ~4-5 different tests, differing from one run to the next.) We should figure + // this out. But in the meantime, just use an exceptionally short timeout here and skip + // straight to shutdownNow() which works every time. (We don't support shutting down + // firestore, so this should only be triggered from the test suite.) + if (!channel.awaitTermination(1, TimeUnit.SECONDS)) { + Logger.debug( + FirestoreChannel.class.getSimpleName(), + "Unable to gracefully shutdown the gRPC ManagedChannel. Will attempt an immediate shutdown."); + channel.shutdownNow(); + + // gRPC docs claim "Although forceful, the shutdown process is still not + // instantaneous; isTerminated() will likely return false immediately after this method + // returns." Therefore, we still need to awaitTermination() again. + if (!channel.awaitTermination(60, TimeUnit.SECONDS)) { + // Something bad has happened. We could assert, but this is just resource cleanup for a + // resource that is likely only released at the end of the execution. So instead, we'll + // just log the error. + Logger.warn( + FirestoreChannel.class.getSimpleName(), + "Unable to forcefully shutdown the gRPC ManagedChannel."); + } + } + } catch (InterruptedException e) { + // (Re-)Cancel if current thread also interrupted + channel.shutdownNow(); + // Similar to above, something bad happened, but it's not worth asserting. Just log it. + Logger.warn( + FirestoreChannel.class.getSimpleName(), + "Interrupted while shutting down the gRPC Managed Channel"); + // Preserve interrupt status + Thread.currentThread().interrupt(); + } + } + /** Creates and starts a new bi-directional streaming RPC. */ public ClientCall runBidiStreamingRpc( MethodDescriptor method, IncomingStreamObserver observer) { From a13019bcd83cb36d015d89ecbf81870d05c03e11 Mon Sep 17 00:00:00 2001 From: rsgowman Date: Thu, 29 Nov 2018 17:04:48 -0500 Subject: [PATCH 09/25] Use gRPC's AndroidChannelBuilder to notice network changes quicker (#103) https://github.com/grpc/grpc-java/blob/master/documentation/android-channel-builder.md --- firebase-firestore/firebase-firestore.gradle | 1 + .../firebase/firestore/remote/StreamTest.java | 26 +++++++++++++++---- .../firestore/core/FirestoreClient.java | 2 +- .../firebase/firestore/remote/Datastore.java | 17 ++++++++++-- .../firestore/remote/MockDatastore.java | 6 +++-- .../firebase/firestore/spec/SpecTestCase.java | 3 ++- 6 files changed, 44 insertions(+), 11 deletions(-) diff --git a/firebase-firestore/firebase-firestore.gradle b/firebase-firestore/firebase-firestore.gradle index eda7161ac84..485f63436f0 100644 --- a/firebase-firestore/firebase-firestore.gradle +++ b/firebase-firestore/firebase-firestore.gradle @@ -97,6 +97,7 @@ dependencies { implementation 'io.grpc:grpc-stub:1.16.1' implementation 'io.grpc:grpc-protobuf-lite:1.16.1' implementation 'io.grpc:grpc-okhttp:1.16.1' + implementation 'io.grpc:grpc-android:1.16.1' implementation "com.google.android.gms:play-services-basement:$playServicesVersion" implementation "com.google.android.gms:play-services-tasks:$playServicesVersion" implementation "com.google.android.gms:play-services-base:$playServicesVersion" diff --git a/firebase-firestore/src/androidTest/java/com/google/firebase/firestore/remote/StreamTest.java b/firebase-firestore/src/androidTest/java/com/google/firebase/firestore/remote/StreamTest.java index 6eef00b8731..f7dd90476d3 100644 --- a/firebase-firestore/src/androidTest/java/com/google/firebase/firestore/remote/StreamTest.java +++ b/firebase-firestore/src/androidTest/java/com/google/firebase/firestore/remote/StreamTest.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import android.support.test.InstrumentationRegistry; import android.support.test.runner.AndroidJUnit4; import com.google.android.gms.tasks.Task; import com.google.firebase.firestore.auth.EmptyCredentialsProvider; @@ -105,7 +106,10 @@ private WriteStream createAndOpenWriteStream( AsyncQueue testQueue, StreamStatusCallback callback) { Datastore datastore = new Datastore( - IntegrationTestUtil.testEnvDatabaseInfo(), testQueue, new EmptyCredentialsProvider()); + IntegrationTestUtil.testEnvDatabaseInfo(), + testQueue, + new EmptyCredentialsProvider(), + InstrumentationRegistry.getContext()); final WriteStream writeStream = datastore.createWriteStream(callback); waitForWriteStreamOpen(testQueue, writeStream, callback); return writeStream; @@ -125,7 +129,10 @@ public void testWatchStreamStopBeforeHandshake() throws Exception { AsyncQueue testQueue = new AsyncQueue(); Datastore datastore = new Datastore( - IntegrationTestUtil.testEnvDatabaseInfo(), testQueue, new EmptyCredentialsProvider()); + IntegrationTestUtil.testEnvDatabaseInfo(), + testQueue, + new EmptyCredentialsProvider(), + InstrumentationRegistry.getContext()); StreamStatusCallback streamCallback = new StreamStatusCallback() {}; final WatchStream watchStream = datastore.createWatchStream(streamCallback); @@ -142,7 +149,10 @@ public void testWriteStreamStopAfterHandshake() throws Exception { AsyncQueue testQueue = new AsyncQueue(); Datastore datastore = new Datastore( - IntegrationTestUtil.testEnvDatabaseInfo(), testQueue, new EmptyCredentialsProvider()); + IntegrationTestUtil.testEnvDatabaseInfo(), + testQueue, + new EmptyCredentialsProvider(), + InstrumentationRegistry.getContext()); final WriteStream[] writeStreamWrapper = new WriteStream[1]; StreamStatusCallback streamCallback = new StreamStatusCallback() { @@ -185,7 +195,10 @@ public void testWriteStreamStopPartial() throws Exception { AsyncQueue testQueue = new AsyncQueue(); Datastore datastore = new Datastore( - IntegrationTestUtil.testEnvDatabaseInfo(), testQueue, new EmptyCredentialsProvider()); + IntegrationTestUtil.testEnvDatabaseInfo(), + testQueue, + new EmptyCredentialsProvider(), + InstrumentationRegistry.getContext()); StreamStatusCallback streamCallback = new StreamStatusCallback() {}; final WriteStream writeStream = datastore.createWriteStream(streamCallback); @@ -261,7 +274,10 @@ public void testStreamRefreshesTokenUponExpiration() throws Exception { MockCredentialsProvider mockCredentialsProvider = new MockCredentialsProvider(); Datastore datastore = new Datastore( - IntegrationTestUtil.testEnvDatabaseInfo(), testQueue, mockCredentialsProvider); + IntegrationTestUtil.testEnvDatabaseInfo(), + testQueue, + mockCredentialsProvider, + InstrumentationRegistry.getContext()); StreamStatusCallback callback = new StreamStatusCallback(); WriteStream writeStream = datastore.createWriteStream(callback); waitForWriteStreamOpen(testQueue, writeStream, callback); diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/core/FirestoreClient.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/core/FirestoreClient.java index 0f68cebde54..9ea2b2f5b7d 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/core/FirestoreClient.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/core/FirestoreClient.java @@ -213,7 +213,7 @@ private void initialize(Context context, User user, boolean usePersistence) { persistence.start(); localStore = new LocalStore(persistence, user); - Datastore datastore = new Datastore(databaseInfo, asyncQueue, credentialsProvider); + Datastore datastore = new Datastore(databaseInfo, asyncQueue, credentialsProvider, context); remoteStore = new RemoteStore(this, localStore, datastore, asyncQueue); syncEngine = new SyncEngine(localStore, remoteStore, user); diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/remote/Datastore.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/remote/Datastore.java index c5f20583ca6..2f809c4ce38 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/remote/Datastore.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/remote/Datastore.java @@ -14,6 +14,7 @@ package com.google.firebase.firestore.remote; +import android.content.Context; import android.support.annotation.VisibleForTesting; import com.google.android.gms.tasks.Task; import com.google.firebase.firestore.FirebaseFirestoreException; @@ -34,6 +35,7 @@ import com.google.firestore.v1beta1.FirestoreGrpc; import io.grpc.ManagedChannelBuilder; import io.grpc.Status; +import io.grpc.android.AndroidChannelBuilder; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -89,7 +91,10 @@ public static void overrideChannelBuilder( } public Datastore( - DatabaseInfo databaseInfo, AsyncQueue workerQueue, CredentialsProvider credentialsProvider) { + DatabaseInfo databaseInfo, + AsyncQueue workerQueue, + CredentialsProvider credentialsProvider, + Context context) { this.databaseInfo = databaseInfo; this.workerQueue = workerQueue; this.serializer = new RemoteSerializer(databaseInfo.getDatabaseId()); @@ -109,9 +114,17 @@ public Datastore( // all calls need to be audited to make sure they are executed on the right thread. channelBuilder.executor(workerQueue.getExecutor()); + // Wrap the ManagedChannelBuilder in an AndroidChannelBuilder. This allows the channel to + // respond more gracefully to network change events (such as switching from cell to wifi). + AndroidChannelBuilder androidChannelBuilder = + AndroidChannelBuilder.fromBuilder(channelBuilder).context(context); + channel = new FirestoreChannel( - workerQueue, credentialsProvider, channelBuilder.build(), databaseInfo.getDatabaseId()); + workerQueue, + credentialsProvider, + androidChannelBuilder.build(), + databaseInfo.getDatabaseId()); } void shutdown() { diff --git a/firebase-firestore/src/test/java/com/google/firebase/firestore/remote/MockDatastore.java b/firebase-firestore/src/test/java/com/google/firebase/firestore/remote/MockDatastore.java index 84c83139501..84911d286cd 100644 --- a/firebase-firestore/src/test/java/com/google/firebase/firestore/remote/MockDatastore.java +++ b/firebase-firestore/src/test/java/com/google/firebase/firestore/remote/MockDatastore.java @@ -16,6 +16,7 @@ import static com.google.firebase.firestore.util.Assert.hardAssert; +import android.content.Context; import com.google.firebase.firestore.auth.EmptyCredentialsProvider; import com.google.firebase.firestore.core.DatabaseInfo; import com.google.firebase.firestore.local.QueryData; @@ -214,12 +215,13 @@ int getWritesSent() { private int writeStreamRequestCount; private int watchStreamRequestCount; - public MockDatastore(AsyncQueue workerQueue) { + public MockDatastore(AsyncQueue workerQueue, Context context) { super( new DatabaseInfo( DatabaseId.forDatabase("project", "database"), "persistenceKey", "host", false), workerQueue, - new EmptyCredentialsProvider()); + new EmptyCredentialsProvider(), + context); this.serializer = new RemoteSerializer(getDatabaseInfo().getDatabaseId()); } diff --git a/firebase-firestore/src/test/java/com/google/firebase/firestore/spec/SpecTestCase.java b/firebase-firestore/src/test/java/com/google/firebase/firestore/spec/SpecTestCase.java index 7751f009a37..32cccd33df4 100644 --- a/firebase-firestore/src/test/java/com/google/firebase/firestore/spec/SpecTestCase.java +++ b/firebase-firestore/src/test/java/com/google/firebase/firestore/spec/SpecTestCase.java @@ -92,6 +92,7 @@ import org.json.JSONException; import org.json.JSONObject; import org.junit.Test; +import org.robolectric.RuntimeEnvironment; import org.robolectric.android.util.concurrent.RoboExecutorService; /** @@ -263,7 +264,7 @@ private void initClient() { queue = new AsyncQueue(); // Set up the sync engine and various stores. - datastore = new MockDatastore(queue); + datastore = new MockDatastore(queue, RuntimeEnvironment.application); remoteStore = new RemoteStore(this, localStore, datastore, queue); syncEngine = new SyncEngine(localStore, remoteStore, currentUser); From 83f4c5955f46c7e970def1718b1036d789238314 Mon Sep 17 00:00:00 2001 From: Vladimir Kryachko Date: Thu, 29 Nov 2018 17:12:46 -0500 Subject: [PATCH 10/25] Adds CI gradle tasks. (#145) * Adds CI gradle tasks. The tasks determine what tests to run based on the contents of the PR as opposed to testing the whole repo. * Addressed review comments. --- .../plugins/ci/AffectedProjectFinder.groovy | 80 ++++++++++ .../ci/ContinuousIntegrationExtension.groovy | 23 +++ .../ci/ContinuousIntegrationPlugin.groovy | 141 ++++++++++++++++++ root-project.gradle | 8 + 4 files changed, 252 insertions(+) create mode 100644 buildSrc/src/main/groovy/com/google/firebase/gradle/plugins/ci/AffectedProjectFinder.groovy create mode 100644 buildSrc/src/main/groovy/com/google/firebase/gradle/plugins/ci/ContinuousIntegrationExtension.groovy create mode 100644 buildSrc/src/main/groovy/com/google/firebase/gradle/plugins/ci/ContinuousIntegrationPlugin.groovy diff --git a/buildSrc/src/main/groovy/com/google/firebase/gradle/plugins/ci/AffectedProjectFinder.groovy b/buildSrc/src/main/groovy/com/google/firebase/gradle/plugins/ci/AffectedProjectFinder.groovy new file mode 100644 index 00000000000..d04607189e9 --- /dev/null +++ b/buildSrc/src/main/groovy/com/google/firebase/gradle/plugins/ci/AffectedProjectFinder.groovy @@ -0,0 +1,80 @@ +// Copyright 2018 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.firebase.gradle.plugins.ci + +import groovy.transform.builder.Builder + +import java.util.regex.Pattern +import org.gradle.api.Project + +/** Determines a set of subprojects that own the 'changedPaths'. */ +class AffectedProjectFinder { + Project project; + Set changedPaths; + + @Builder + AffectedProjectFinder(Project project, + Set changedPaths, + List ignorePaths) { + this.project = project + this.changedPaths = changedPaths.findAll { + for(def path : ignorePaths) { + if(it ==~ path) { + return false + } + } + return true + } + } + + Set find() { + Set paths = changedPaths.collect() + def projects = changedSubProjects(project, paths) + + if(!containsRootProject(projects)) { + return projects + } + return project.subprojects + } + + /** + * Performs a post-order project tree traversal and returns a set of projects that own the + * 'changedPaths'. + */ + private static Set changedSubProjects(Project project, Set changedPaths) { + // project.subprojects include all descendents of a given project, we only want immediate + // children. + Set immediateChildProjects = project.subprojects.findAll { it.parent == project } + + Set projects = immediateChildProjects.collectMany { + changedSubProjects(it, changedPaths) + } + def relativePath = project.rootDir.toURI().relativize(project.projectDir.toURI()).toString() + + Iterator itr = changedPaths.iterator() + while (itr.hasNext()) { + def file = itr.next() + if (file.startsWith(relativePath)) { + itr.remove() + projects.add(project) + } + } + return projects + } + + private static boolean containsRootProject(Set projects) { + return projects.any { it.rootProject == it }; + } +} diff --git a/buildSrc/src/main/groovy/com/google/firebase/gradle/plugins/ci/ContinuousIntegrationExtension.groovy b/buildSrc/src/main/groovy/com/google/firebase/gradle/plugins/ci/ContinuousIntegrationExtension.groovy new file mode 100644 index 00000000000..8539fcf166f --- /dev/null +++ b/buildSrc/src/main/groovy/com/google/firebase/gradle/plugins/ci/ContinuousIntegrationExtension.groovy @@ -0,0 +1,23 @@ +// Copyright 2018 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.firebase.gradle.plugins.ci + +import java.util.regex.Pattern + +/** Contains plugin configuration properties. */ +class ContinuousIntegrationExtension { + /** List of paths that the plugin should ignore when querying the Git commit. */ + List ignorePaths = [] +} \ No newline at end of file diff --git a/buildSrc/src/main/groovy/com/google/firebase/gradle/plugins/ci/ContinuousIntegrationPlugin.groovy b/buildSrc/src/main/groovy/com/google/firebase/gradle/plugins/ci/ContinuousIntegrationPlugin.groovy new file mode 100644 index 00000000000..06c27f96d7d --- /dev/null +++ b/buildSrc/src/main/groovy/com/google/firebase/gradle/plugins/ci/ContinuousIntegrationPlugin.groovy @@ -0,0 +1,141 @@ +// Copyright 2018 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.firebase.gradle.plugins.ci + +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.Task + + +/** + * Provides 'checkChanged' and 'connectedCheckChanged' tasks to the root project. + * + *

The task definition is dynamic and depends on the latest git changes in the project. Namely + * it gets a list of changed files from the latest Git pull/merge and determines which subprojects + * the files belong to. Then, for each affected project, it declares a dependency on the + * 'checkDependents' or 'connectedCheckChanged' task respectively in that project. + * + *

Note: If the commits contain a file that does not belong to any subproject, *all* subprojects + * will be built. + */ +class ContinuousIntegrationPlugin implements Plugin { + + @Override + void apply(Project project) { + + def extension = project.extensions.create( + "firebaseContinuousIntegration", + ContinuousIntegrationExtension) + + project.configure(project.subprojects) { + def checkDependents = it.task('checkDependents') {} + def connectedCheckDependents = it.task('connectedCheckDependents') + + configurations.all { + if (it.name == 'debugUnitTestRuntimeClasspath') { + checkDependents.dependsOn(configurations + .debugUnitTestRuntimeClasspath.getTaskDependencyFromProjectDependency( + false, "checkDependents")) + checkDependents.dependsOn 'check' + } + + if (it.name == 'debugAndroidTestRuntimeClasspath') { + connectedCheckDependents.dependsOn(configurations + .debugAndroidTestRuntimeClasspath.getTaskDependencyFromProjectDependency( + false, "connectedCheckDependents")) + connectedCheckDependents.dependsOn 'connectedCheck' + } + + if (it.name == 'annotationProcessor') { + connectedCheckDependents.dependsOn(configurations + .annotationProcessor.getTaskDependencyFromProjectDependency( + false, "connectedCheckDependents")) + checkDependents.dependsOn(configurations + .annotationProcessor.getTaskDependencyFromProjectDependency( + false, "checkDependents")) + } + } + + afterEvaluate { + // non-android projects need to define the custom configurations due to the way + // getTaskDependencyFromProjectDependency works. + if (!isAndroidProject(it)) { + configurations { + debugUnitTestRuntimeClasspath + debugAndroidTestRuntimeClasspath + annotationProcessor + } + // noop task to avoid having to handle the edge-case of tasks not being + // defined. + tasks.maybeCreate('connectedCheck') + tasks.maybeCreate('check') + } + } + } + + def affectedProjects = AffectedProjectFinder.builder() + .project(project) + .changedPaths(changedPaths(project.rootDir)) + .ignorePaths(extension.ignorePaths) + .build() + .find() + + project.task('checkChanged') { task -> + task.group = 'verification' + task.description = 'Runs the check task in all changed projects.' + affectedProjects.each { + task.dependsOn("$it.path:checkDependents") + } + } + project.task('connectedCheckChanged') { task -> + task.group = 'verification' + task.description = 'Runs the connectedCheck task in all changed projects.' + affectedProjects.each { + task.dependsOn("$it.path:connectedCheckDependents") + } + } + + project.task('ciTasksSanityCheck') { + doLast { + [':firebase-common', ':tools:errorprone'].each { projectName -> + def task = project.project(projectName).tasks.findByName('checkDependents') + def dependents = task.taskDependencies.getDependencies(task).collect { it.path} + + def expectedDependents = [ + 'database', + 'firestore', + 'functions', + 'storage'].collect { ":firebase-$it:checkDependents"} + assert expectedDependents.intersect(dependents) == expectedDependents : + "$projectName:checkDependents does not depend on expected projects" + } + } + } + } + + private static Set changedPaths(File workDir) { + return 'git diff --name-only --submodule=diff HEAD@{0} HEAD@{1}' + .execute([], workDir) + .text + .readLines() + } + + private static final ANDROID_PLUGINS = ["com.android.application", "com.android.library", + "com.android.test"] + + private static boolean isAndroidProject(Project project) { + ANDROID_PLUGINS.find { plugin -> project.plugins.hasPlugin(plugin) } + } +} diff --git a/root-project.gradle b/root-project.gradle index 03d037c7fe0..e5dd5e27fac 100644 --- a/root-project.gradle +++ b/root-project.gradle @@ -47,6 +47,14 @@ ext { } apply plugin: com.google.firebase.gradle.plugins.publish.PublishingPlugin +apply plugin: com.google.firebase.gradle.plugins.ci.ContinuousIntegrationPlugin + +firebaseContinuousIntegration { + ignorePaths = [ + /.*\.gitignore$/, + /.*.md$/, + ] +} configure(subprojects) { repositories { From 9fa8f237d973f7e8fce422e50fc414234cc0742e Mon Sep 17 00:00:00 2001 From: Vladimir Kryachko Date: Mon, 3 Dec 2018 11:42:37 -0500 Subject: [PATCH 11/25] Fix google-services testing tooling. (#148) --- gradle/googleServices.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/googleServices.gradle b/gradle/googleServices.gradle index 52eb58989bb..84b5cdb7a4e 100644 --- a/gradle/googleServices.gradle +++ b/gradle/googleServices.gradle @@ -45,7 +45,7 @@ afterEvaluate { // ./gradlew test at the root project). def isTesting = getGradle().getStartParameter().getTaskNames().any({ String taskName -> def hasProjectTestTask = taskName.contains("$name") && ['AndroidTest', 'connectedCheck'].any({ taskName.contains(it)}) - return hasProjectTestTask || taskName == 'connectedCheck' + return hasProjectTestTask || taskName.contains('connectedCheck') }) if (isTesting) { From 79fc70b6c25818a9565662514a192e611dc8851d Mon Sep 17 00:00:00 2001 From: rsgowman Date: Tue, 4 Dec 2018 10:38:17 -0500 Subject: [PATCH 12/25] Cleanup a number of warnings in the code. (#137) * Eliminate 'badimport' warning It was already reasonably obvious what this is, so the warning is mostly spurious. But it's easier and less boilerplate to just add the extra qualifier than to suppress the warning. https://errorprone.info/bugpattern/BadImport * Suppress EqualsGetClass warnings for public classes These are effectively final and are documented as not being final only to allow mocking within tests. https://errorprone.info/bugpattern/EqualsGetClass * Mark a number of classes final Mostly to avoid a warning, though it's generally recommended to do this anyways. (Note that this makes mocking impossible; if any of these classes need to be mocked, we might be better of just suppressing the warning or using instanceof.) https://errorprone.info/bugpattern/EqualsGetClass * Switch .getClass() to instanceof in some .equals() methods This is for cases where we're dealing with a non-leaf node in the inheritance hierarchy. (Leaf nodes in the heirarchy can just mark the entire class as final.) In order to preserve symmetry of .equals(), also mark the .equals() methods themselves as final. In this case, "non-leaf" nodes also counts effectively final classes which are mocked. (i.e. ViewSnapshot.) https://errorprone.info/bugpattern/EqualsGetClass * Provide toString on FieldMask Since it's used by PatchMutation https://errorprone.info/bugpattern/ObjectToString * Fix a 'missing override' warning https://errorprone.info/bugpattern/MissingOverride * Rework FieldMask to use a Set of FieldPaths rather than a Collection. FieldMask uses .equals() on the container, and that's not well defined for a Collection. The fix is to switch to either a Set or a List. Set seems more appropriate. https://errorprone.info/bugpattern/UndefinedEquals * Add TestUtil.fieldMask(String...) to simplify creating FieldMasks. --- .../firebase/firestore/DocumentReference.java | 2 +- .../com/google/firebase/firestore/Query.java | 2 +- .../google/firebase/firestore/SetOptions.java | 15 ++++++++------- .../firebase/firestore/core/EventManager.java | 1 + .../firebase/firestore/core/UserData.java | 14 +++++++------- .../firebase/firestore/core/ViewSnapshot.java | 4 ++-- .../firestore/local/IndexedQueryEngine.java | 3 +-- .../firebase/firestore/model/BasePath.java | 10 ++++------ .../firebase/firestore/model/Document.java | 2 +- .../firebase/firestore/model/DocumentKey.java | 2 +- .../firebase/firestore/model/DocumentSet.java | 2 +- .../firebase/firestore/model/NoDocument.java | 2 +- .../firestore/model/UnknownDocument.java | 2 +- .../mutation/ArrayTransformOperation.java | 1 + .../firestore/model/mutation/FieldMask.java | 17 +++++++++++------ .../firestore/remote/RemoteSerializer.java | 6 ++++-- .../firestore/remote/WatchChange.java | 6 +++--- .../firestore/local/LocalSerializerTest.java | 5 ++--- .../firestore/model/MutationTest.java | 3 ++- .../firebase/firestore/testutil/TestUtil.java | 19 ++++++++++++++++--- 20 files changed, 69 insertions(+), 49 deletions(-) diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/DocumentReference.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/DocumentReference.java index 4fe7323b62f..459b14378d0 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/DocumentReference.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/DocumentReference.java @@ -555,7 +555,7 @@ public boolean equals(Object o) { if (this == o) { return true; } - if (o == null || getClass() != o.getClass()) { + if (!(o instanceof DocumentReference)) { return false; } diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/Query.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/Query.java index 6008bb00f82..16957d93c37 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/Query.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/Query.java @@ -905,7 +905,7 @@ public boolean equals(Object o) { if (this == o) { return true; } - if (o == null || getClass() != o.getClass()) { + if (!(o instanceof Query)) { return false; } diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/SetOptions.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/SetOptions.java index 4c302738906..adde4525994 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/SetOptions.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/SetOptions.java @@ -20,8 +20,9 @@ import android.support.annotation.Nullable; import com.google.firebase.annotations.PublicApi; import com.google.firebase.firestore.model.mutation.FieldMask; -import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; /** * An options object that configures the behavior of set() calls. By providing one of the SetOptions @@ -78,13 +79,13 @@ public static SetOptions merge() { @NonNull @PublicApi public static SetOptions mergeFields(List fields) { - List fieldPaths = new ArrayList<>(); + Set fieldPaths = new HashSet<>(); for (String field : fields) { fieldPaths.add(FieldPath.fromDotSeparatedPath(field).getInternalPath()); } - return new SetOptions(true, FieldMask.fromCollection(fieldPaths)); + return new SetOptions(true, FieldMask.fromSet(fieldPaths)); } /** @@ -100,13 +101,13 @@ public static SetOptions mergeFields(List fields) { @NonNull @PublicApi public static SetOptions mergeFields(String... fields) { - List fieldPaths = new ArrayList<>(); + Set fieldPaths = new HashSet<>(); for (String field : fields) { fieldPaths.add(FieldPath.fromDotSeparatedPath(field).getInternalPath()); } - return new SetOptions(true, FieldMask.fromCollection(fieldPaths)); + return new SetOptions(true, FieldMask.fromSet(fieldPaths)); } /** @@ -121,13 +122,13 @@ public static SetOptions mergeFields(String... fields) { @NonNull @PublicApi public static SetOptions mergeFieldPaths(List fields) { - List fieldPaths = new ArrayList<>(); + Set fieldPaths = new HashSet<>(); for (FieldPath field : fields) { fieldPaths.add(field.getInternalPath()); } - return new SetOptions(true, FieldMask.fromCollection(fieldPaths)); + return new SetOptions(true, FieldMask.fromSet(fieldPaths)); } @Override diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/core/EventManager.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/core/EventManager.java index e74f837e496..38665628a45 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/core/EventManager.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/core/EventManager.java @@ -137,6 +137,7 @@ public void onError(Query query, Status error) { queries.remove(query); } + @Override public void handleOnlineStateChange(OnlineState onlineState) { this.onlineState = onlineState; for (QueryListenersInfo info : queries.values()) { diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/core/UserData.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/core/UserData.java index 00cc3969799..0cb0f614903 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/core/UserData.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/core/UserData.java @@ -29,9 +29,9 @@ import com.google.firebase.firestore.model.value.ObjectValue; import com.google.firebase.firestore.util.Assert; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; -import java.util.SortedSet; -import java.util.TreeSet; +import java.util.Set; import java.util.regex.Pattern; import javax.annotation.Nullable; @@ -72,8 +72,8 @@ public static class ParseAccumulator { */ private final Source dataSource; - /** Accumulates a list of the field paths found while parsing the data. */ - private final SortedSet fieldMask; + /** Accumulates a set of the field paths found while parsing the data. */ + private final Set fieldMask; /** Accumulates a list of field transforms found while parsing the data. */ private final ArrayList fieldTransforms; @@ -81,7 +81,7 @@ public static class ParseAccumulator { /** @param dataSource Indicates what kind of API method this data came from. */ public ParseAccumulator(Source dataSource) { this.dataSource = dataSource; - this.fieldMask = new TreeSet<>(); + this.fieldMask = new HashSet<>(); this.fieldTransforms = new ArrayList<>(); } @@ -135,7 +135,7 @@ void addToFieldTransforms(FieldPath fieldPath, TransformOperation transformOpera */ public ParsedSetData toMergeData(ObjectValue data) { return new ParsedSetData( - data, FieldMask.fromCollection(fieldMask), unmodifiableList(fieldTransforms)); + data, FieldMask.fromSet(fieldMask), unmodifiableList(fieldTransforms)); } /** @@ -181,7 +181,7 @@ public ParsedSetData toSetData(ObjectValue data) { */ public ParsedUpdateData toUpdateData(ObjectValue data) { return new ParsedUpdateData( - data, FieldMask.fromCollection(fieldMask), unmodifiableList(fieldTransforms)); + data, FieldMask.fromSet(fieldMask), unmodifiableList(fieldTransforms)); } } diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/core/ViewSnapshot.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/core/ViewSnapshot.java index a05cffea965..1e05a3d8e98 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/core/ViewSnapshot.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/core/ViewSnapshot.java @@ -118,11 +118,11 @@ public boolean excludesMetadataChanges() { } @Override - public boolean equals(Object o) { + public final boolean equals(Object o) { if (this == o) { return true; } - if (o == null || getClass() != o.getClass()) { + if (!(o instanceof ViewSnapshot)) { return false; } diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/IndexedQueryEngine.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/IndexedQueryEngine.java index 1dab574afff..ab734d95e31 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/IndexedQueryEngine.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/IndexedQueryEngine.java @@ -22,7 +22,6 @@ import com.google.firebase.firestore.core.Filter; import com.google.firebase.firestore.core.Filter.Operator; import com.google.firebase.firestore.core.IndexRange; -import com.google.firebase.firestore.core.IndexRange.Builder; import com.google.firebase.firestore.core.NaNFilter; import com.google.firebase.firestore.core.NullFilter; import com.google.firebase.firestore.core.Query; @@ -216,7 +215,7 @@ static IndexRange extractBestIndexRange(Query query) { * filter. The determined {@code IndexRange} is likely overselective and requires post-filtering. */ private static IndexRange convertFilterToIndexRange(Filter filter) { - Builder indexRange = IndexRange.builder().setFieldPath(filter.getField()); + IndexRange.Builder indexRange = IndexRange.builder().setFieldPath(filter.getField()); if (filter instanceof RelationFilter) { RelationFilter relationFilter = (RelationFilter) filter; FieldValue filterValue = relationFilter.getValue(); diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/model/BasePath.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/model/BasePath.java index 2edc7875e3b..4da80b39d99 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/model/BasePath.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/model/BasePath.java @@ -161,13 +161,11 @@ public int length() { } @Override - @SuppressWarnings("unchecked") - public boolean equals(Object o) { - if (o == null) { - return false; + public final boolean equals(Object o) { + if (this == o) { + return true; } - // The cast is not unchecked because of the class equality check. - return getClass() == o.getClass() && compareTo((B) o) == 0; + return (o instanceof BasePath) && compareTo((B) o) == 0; } @Override diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/model/Document.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/model/Document.java index a0be1936ef6..911ec3cd87d 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/model/Document.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/model/Document.java @@ -23,7 +23,7 @@ * Represents a document in Firestore with a key, version, data and whether the data has local * mutations applied to it. */ -public class Document extends MaybeDocument { +public final class Document extends MaybeDocument { /** Describes the `hasPendingWrites` state of a document. */ public enum DocumentState { diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/model/DocumentKey.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/model/DocumentKey.java index f4e3b7b5e09..173edfaf96c 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/model/DocumentKey.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/model/DocumentKey.java @@ -23,7 +23,7 @@ import java.util.List; /** DocumentKey represents the location of a document in the Firestore database. */ -public class DocumentKey implements Comparable { +public final class DocumentKey implements Comparable { public static final String KEY_FIELD_NAME = "__name__"; diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/model/DocumentSet.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/model/DocumentSet.java index 66dd581cf1d..47a10987bc1 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/model/DocumentSet.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/model/DocumentSet.java @@ -30,7 +30,7 @@ * An immutable set of documents (unique by key) ordered by the given comparator or ordered by key * by default if no document is present. */ -public class DocumentSet implements Iterable { +public final class DocumentSet implements Iterable { /** Returns an empty DocumentSet sorted by the given comparator, then by keys. */ public static DocumentSet emptySet(final Comparator comparator) { diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/model/NoDocument.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/model/NoDocument.java index 74b6d6cad54..df83be54a54 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/model/NoDocument.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/model/NoDocument.java @@ -15,7 +15,7 @@ package com.google.firebase.firestore.model; /** Represents that no documents exists for the key at the given version. */ -public class NoDocument extends MaybeDocument { +public final class NoDocument extends MaybeDocument { private boolean hasCommittedMutations; diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/model/UnknownDocument.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/model/UnknownDocument.java index dc8cf0d9142..892ffb2d712 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/model/UnknownDocument.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/model/UnknownDocument.java @@ -18,7 +18,7 @@ * A class representing an existing document whose data is unknown (e.g. a document that was updated * without a known base document). */ -public class UnknownDocument extends MaybeDocument { +public final class UnknownDocument extends MaybeDocument { public UnknownDocument(DocumentKey key, SnapshotVersion version) { super(key, version); } diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/model/mutation/ArrayTransformOperation.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/model/mutation/ArrayTransformOperation.java index 89e10ec3364..f8f1c7abd8c 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/model/mutation/ArrayTransformOperation.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/model/mutation/ArrayTransformOperation.java @@ -51,6 +51,7 @@ public FieldValue applyToRemoteDocument(FieldValue previousValue, FieldValue tra } @Override + @SuppressWarnings("EqualsGetClass") // subtype-sensitive equality is intended. public boolean equals(Object o) { if (this == o) { return true; diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/model/mutation/FieldMask.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/model/mutation/FieldMask.java index 0cbef886ccd..7b0f8ee9fb2 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/model/mutation/FieldMask.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/model/mutation/FieldMask.java @@ -15,7 +15,7 @@ package com.google.firebase.firestore.model.mutation; import com.google.firebase.firestore.model.FieldPath; -import java.util.Collection; +import java.util.Set; /** * Provides a set of fields that can be used to partially patch a document. The FieldMask is used in @@ -25,14 +25,14 @@ * companion ObjectValue, the field is deleted. foo.bar - Overwrites only the field bar of the * object foo. If foo is not an object, foo is replaced with an object containing foo. */ -public class FieldMask { - public static FieldMask fromCollection(Collection mask) { +public final class FieldMask { + public static FieldMask fromSet(Set mask) { return new FieldMask(mask); } - private final Collection mask; + private final Set mask; - private FieldMask(Collection mask) { + private FieldMask(Set mask) { this.mask = mask; } @@ -49,6 +49,11 @@ public boolean equals(Object o) { return mask.equals(fieldMask.mask); } + @Override + public String toString() { + return "FieldMask{mask=" + mask.toString() + "}"; + } + /** * Verifies that 'fieldPath' is included by at least one field in this field mask. * @@ -69,7 +74,7 @@ public int hashCode() { return mask.hashCode(); } - public Collection getMask() { + public Set getMask() { return mask; } } diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/remote/RemoteSerializer.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/remote/RemoteSerializer.java index 8a58768f1a6..555639025b1 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/remote/RemoteSerializer.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/remote/RemoteSerializer.java @@ -98,8 +98,10 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; /** Serializer that converts to and from Firestore API protos. */ public final class RemoteSerializer { @@ -532,11 +534,11 @@ private DocumentMask encodeDocumentMask(FieldMask mask) { private FieldMask decodeDocumentMask(DocumentMask mask) { int count = mask.getFieldPathsCount(); - List paths = new ArrayList<>(count); + Set paths = new HashSet<>(count); for (int i = 0; i < count; i++) { paths.add(FieldPath.fromServerFormat(mask.getFieldPaths(i))); } - return FieldMask.fromCollection(paths); + return FieldMask.fromSet(paths); } private DocumentTransform.FieldTransform encodeFieldTransform(FieldTransform fieldTransform) { diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/remote/WatchChange.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/remote/WatchChange.java index e2a882cc797..565c85afa2e 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/remote/WatchChange.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/remote/WatchChange.java @@ -38,7 +38,7 @@ private WatchChange() { * A document change represents a change document and a list of target ids to which this change * applies. If the document has been deleted, the deleted document will be provided. */ - public static class DocumentChange extends WatchChange { + public static final class DocumentChange extends WatchChange { // TODO: figure out if we can actually use arrays here for efficiency /** The new document applies to all of these targets. */ private final List updatedTargetIds; @@ -137,7 +137,7 @@ public int hashCode() { * An ExistenceFilterWatchChange applies to the targets and is required to verify the current * client state against expected state sent from the server. */ - public static class ExistenceFilterWatchChange extends WatchChange { + public static final class ExistenceFilterWatchChange extends WatchChange { private final int targetId; private final ExistenceFilter existenceFilter; @@ -177,7 +177,7 @@ public enum WatchTargetChangeType { } /** The state of a target has changed. This can mean removal, addition, current or reset. */ - public static class WatchTargetChange extends WatchChange { + public static final class WatchTargetChange extends WatchChange { private final WatchTargetChangeType changeType; private final List targetIds; private final ByteString resumeToken; diff --git a/firebase-firestore/src/test/java/com/google/firebase/firestore/local/LocalSerializerTest.java b/firebase-firestore/src/test/java/com/google/firebase/firestore/local/LocalSerializerTest.java index db743396b29..a4429a0b27d 100644 --- a/firebase-firestore/src/test/java/com/google/firebase/firestore/local/LocalSerializerTest.java +++ b/firebase-firestore/src/test/java/com/google/firebase/firestore/local/LocalSerializerTest.java @@ -17,7 +17,7 @@ import static com.google.firebase.firestore.testutil.TestUtil.deleteMutation; import static com.google.firebase.firestore.testutil.TestUtil.deletedDoc; import static com.google.firebase.firestore.testutil.TestUtil.doc; -import static com.google.firebase.firestore.testutil.TestUtil.field; +import static com.google.firebase.firestore.testutil.TestUtil.fieldMask; import static com.google.firebase.firestore.testutil.TestUtil.key; import static com.google.firebase.firestore.testutil.TestUtil.map; import static com.google.firebase.firestore.testutil.TestUtil.setMutation; @@ -33,7 +33,6 @@ import com.google.firebase.firestore.model.NoDocument; import com.google.firebase.firestore.model.SnapshotVersion; import com.google.firebase.firestore.model.UnknownDocument; -import com.google.firebase.firestore.model.mutation.FieldMask; import com.google.firebase.firestore.model.mutation.Mutation; import com.google.firebase.firestore.model.mutation.MutationBatch; import com.google.firebase.firestore.model.mutation.PatchMutation; @@ -71,7 +70,7 @@ public void testEncodesMutationBatch() { new PatchMutation( key("bar/baz"), TestUtil.wrapObject(map("a", "b", "num", 1)), - FieldMask.fromCollection(asList(field("a"))), + fieldMask("a"), com.google.firebase.firestore.model.mutation.Precondition.exists(true)); Mutation del = deleteMutation("baz/quux"); Timestamp writeTime = Timestamp.now(); diff --git a/firebase-firestore/src/test/java/com/google/firebase/firestore/model/MutationTest.java b/firebase-firestore/src/test/java/com/google/firebase/firestore/model/MutationTest.java index d686c4829ef..67c1fba0ac4 100644 --- a/firebase-firestore/src/test/java/com/google/firebase/firestore/model/MutationTest.java +++ b/firebase-firestore/src/test/java/com/google/firebase/firestore/model/MutationTest.java @@ -18,6 +18,7 @@ import static com.google.firebase.firestore.testutil.TestUtil.deletedDoc; import static com.google.firebase.firestore.testutil.TestUtil.doc; import static com.google.firebase.firestore.testutil.TestUtil.field; +import static com.google.firebase.firestore.testutil.TestUtil.fieldMask; import static com.google.firebase.firestore.testutil.TestUtil.key; import static com.google.firebase.firestore.testutil.TestUtil.map; import static com.google.firebase.firestore.testutil.TestUtil.mutationResult; @@ -110,7 +111,7 @@ public void testDeletesValuesFromTheFieldMask() { Document baseDoc = doc("collection/key", 0, data); DocumentKey key = key("collection/key"); - FieldMask mask = FieldMask.fromCollection(Arrays.asList(field("foo.bar"))); + FieldMask mask = fieldMask("foo.bar"); Mutation patch = new PatchMutation(key, ObjectValue.emptyObject(), mask, Precondition.NONE); MaybeDocument patchDoc = patch.applyToLocalView(baseDoc, baseDoc, Timestamp.now()); diff --git a/firebase-firestore/src/testUtil/java/com/google/firebase/firestore/testutil/TestUtil.java b/firebase-firestore/src/testUtil/java/com/google/firebase/firestore/testutil/TestUtil.java index d6d5384fd5f..27caae9bbdc 100644 --- a/firebase-firestore/src/testUtil/java/com/google/firebase/firestore/testutil/TestUtil.java +++ b/firebase-firestore/src/testUtil/java/com/google/firebase/firestore/testutil/TestUtil.java @@ -74,10 +74,13 @@ import java.util.Collections; import java.util.Comparator; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; import javax.annotation.Nullable; /** A set of utilities for tests */ @@ -109,6 +112,14 @@ public static ByteString byteString(int... bytes) { return ByteString.copyFrom(primitive); } + public static FieldMask fieldMask(String... fields) { + FieldPath[] mask = new FieldPath[fields.length]; + for (int i = 0; i < fields.length; i++) { + mask[i] = field(fields[i]); + } + return FieldMask.fromSet(new HashSet<>(Arrays.asList(mask))); + } + public static final Map EMPTY_MAP = new HashMap<>(); public static FieldValue wrap(Object value) { @@ -430,13 +441,15 @@ public static PatchMutation patchMutation( boolean merge = updateMask != null; - // We sort the fieldMaskPaths to make the order deterministic in tests. - Collections.sort(objectMask); + // We sort the fieldMaskPaths to make the order deterministic in tests. (Otherwise, when we + // flatten a Set to a proto repeated field, we'll end up comparing in iterator order and + // possibly consider {foo,bar} != {bar,foo}.) + SortedSet fieldMaskPaths = new TreeSet<>(merge ? updateMask : objectMask); return new PatchMutation( key(path), objectValue, - FieldMask.fromCollection(merge ? updateMask : objectMask), + FieldMask.fromSet(fieldMaskPaths), merge ? Precondition.NONE : Precondition.exists(true)); } From bb69b042892beaee2ca873db2e187422f7524993 Mon Sep 17 00:00:00 2001 From: rsgowman Date: Wed, 5 Dec 2018 15:24:42 -0500 Subject: [PATCH 13/25] Add grpc keepalive timeout (#151) To work around dead connections where the OS doesn't actually close the socket. This allows us to not have to wait for a tcp timeout to occur. --- .../java/com/google/firebase/firestore/remote/Datastore.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/remote/Datastore.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/remote/Datastore.java index 2f809c4ce38..ffab590fd1d 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/remote/Datastore.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/remote/Datastore.java @@ -43,6 +43,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.TimeUnit; /** * Datastore represents a proxy for the remote server, hiding details of the RPC layer. It: @@ -110,6 +111,10 @@ public Datastore( } } + // Ensure gRPC recovers from a dead connection. (Not typically necessary, as the OS will usually + // notify gRPC when a connection dies. But not always. This acts as a failsafe.) + channelBuilder.keepAliveTime(30, TimeUnit.SECONDS); + // This ensures all callbacks are issued on the worker queue. If this call is removed, // all calls need to be audited to make sure they are executed on the right thread. channelBuilder.executor(workerQueue.getExecutor()); From 2d297f49ad1549a5eca63471f0d459a1c1313791 Mon Sep 17 00:00:00 2001 From: zxu Date: Wed, 5 Dec 2018 15:24:55 -0500 Subject: [PATCH 14/25] fix tests (#146) * fix tests * fix format --- .../google/firebase/firestore/ArrayTransformsTest.java | 2 +- .../com/google/firebase/firestore/TransactionTest.java | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/firebase-firestore/src/androidTest/java/com/google/firebase/firestore/ArrayTransformsTest.java b/firebase-firestore/src/androidTest/java/com/google/firebase/firestore/ArrayTransformsTest.java index 172dc1cdf01..46de87b879e 100644 --- a/firebase-firestore/src/androidTest/java/com/google/firebase/firestore/ArrayTransformsTest.java +++ b/firebase-firestore/src/androidTest/java/com/google/firebase/firestore/ArrayTransformsTest.java @@ -114,7 +114,7 @@ public void removeFromArrayViaUpdate() { @Test public void removeFromArrayViaSetMerge() { writeInitialData(map("array", asList(1L, 3L, 1L, 3L))); - waitFor(docRef.update(map("array", FieldValue.arrayRemove(1L, 4L)))); + waitFor(docRef.set(map("array", FieldValue.arrayRemove(1L, 4L)), SetOptions.merge())); expectLocalAndRemoteEvent(map("array", asList(3L, 3L))); } diff --git a/firebase-firestore/src/androidTest/java/com/google/firebase/firestore/TransactionTest.java b/firebase-firestore/src/androidTest/java/com/google/firebase/firestore/TransactionTest.java index 580f864c82a..e7a0615cece 100644 --- a/firebase-firestore/src/androidTest/java/com/google/firebase/firestore/TransactionTest.java +++ b/firebase-firestore/src/androidTest/java/com/google/firebase/firestore/TransactionTest.java @@ -22,6 +22,7 @@ import static junit.framework.Assert.assertFalse; import static junit.framework.Assert.assertNotNull; import static junit.framework.Assert.assertTrue; +import static junit.framework.Assert.fail; import android.support.test.runner.AndroidJUnit4; import com.google.android.gms.tasks.Task; @@ -456,10 +457,9 @@ public void testReadingADocTwiceWithDifferentVersions() { waitFor(doc.set(map("count", 1234.0))); // Get the doc again in the transaction with the new version. DocumentSnapshot snapshot2 = transaction.get(doc); - assertEquals(1234, snapshot2.getDouble("count").intValue()); - // Now try to update the doc from within the transaction. - // This should fail, because we read 15 earlier. - transaction.set(doc, map("count", 16.0)); + // The get itself will fail, because we already read an earlier version of this + // document. + fail("Should have thrown exception"); return null; })); DocumentSnapshot snapshot = waitFor(doc.get()); From 5176fb541e454497e0ad7716a4f1a29030fd87a5 Mon Sep 17 00:00:00 2001 From: Greg Soltis Date: Thu, 6 Dec 2018 10:09:18 -0800 Subject: [PATCH 15/25] Make schema migrations idempotent (#149) * Make schema migrations idempotent * Tests for schema migration regressions --- .../firestore/local/SQLiteSchema.java | 282 ++++++++++++------ .../firestore/local/SQLiteSchemaTest.java | 68 +++++ 2 files changed, 266 insertions(+), 84 deletions(-) diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLiteSchema.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLiteSchema.java index b2270ce8155..0a91de9a09f 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLiteSchema.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLiteSchema.java @@ -17,10 +17,16 @@ import static com.google.firebase.firestore.util.Assert.hardAssert; import android.content.ContentValues; +import android.database.Cursor; import android.database.DatabaseUtils; import android.database.sqlite.SQLiteDatabase; import android.database.sqlite.SQLiteStatement; +import android.support.annotation.VisibleForTesting; +import android.text.TextUtils; +import android.util.Log; import com.google.common.base.Preconditions; +import java.util.ArrayList; +import java.util.List; /** * Migrates schemas from version 0 (empty) to whatever the current version is. @@ -65,14 +71,16 @@ void runMigrations(int fromVersion) { * otherwise for testing. */ void runMigrations(int fromVersion, int toVersion) { - // Each case in this switch statement intentionally falls through to the one below it, making - // it possible to start at the version that's installed and then run through any that haven't - // been applied yet. + /* + * New migrations should be added at the end of the series of `if` statements and should follow + * the pattern. Make sure to increment `VERSION` and to read the comment below about + * requirements for new migrations. + */ if (fromVersion < 1 && toVersion >= 1) { - createMutationQueue(); - createQueryCache(); - createRemoteDocumentCache(); + createV1MutationQueue(); + createV1QueryCache(); + createV1RemoteDocumentCache(); } // Migration 2 to populate the target_globals table no longer needed since migration 3 @@ -82,8 +90,8 @@ void runMigrations(int fromVersion, int toVersion) { // Brand new clients don't need to drop and recreate--only clients that have potentially // corrupt data. if (fromVersion != 0) { - dropQueryCache(); - createQueryCache(); + dropV1QueryCache(); + createV1QueryCache(); } } @@ -104,36 +112,85 @@ void runMigrations(int fromVersion, int toVersion) { ensureSequenceNumbers(); } + /* + * Adding a new migration? READ THIS FIRST! + * + * Be aware that the SDK version may be downgraded then re-upgraded. This means that running + * your new migration must not prevent older versions of the SDK from functioning. Additionally, + * your migration must be able to run multiple times. In practice, this means a few things: + * * Do not delete tables or columns. Older versions may be reading and writing them. + * * Guard schema additions. Check if tables or columns exist before adding them. + * * Data migrations should *probably* always run. Older versions of the SDK will not have + * maintained invariants from later versions, so migrations that update values cannot assume + * that existing values have been properly maintained. Calculate them again, if applicable. + */ + if (fromVersion < INDEXING_SUPPORT_VERSION && toVersion >= INDEXING_SUPPORT_VERSION) { Preconditions.checkState(Persistence.INDEXING_SUPPORT_ENABLED); createLocalDocumentsCollectionIndex(); } } - private void createMutationQueue() { - // A table naming all the mutation queues in the system. - db.execSQL( - "CREATE TABLE mutation_queues (" - + "uid TEXT PRIMARY KEY, " - + "last_acknowledged_batch_id INTEGER, " - + "last_stream_token BLOB)"); + /** + * Used to assert that a set of tables either all exist or not. The supplied function is run if + * none of the tables exist. Use this method to create a set of tables at once. + * + *

If some but not all of the tables exist, an exception will be thrown. + */ + private void ifTablesDontExist(String[] tables, Runnable fn) { + boolean tablesFound = false; + String allTables = "[" + TextUtils.join(", ", tables) + "]"; + for (int i = 0; i < tables.length; i++) { + String table = tables[i]; + boolean tableFound = tableExists(table); + if (i == 0) { + tablesFound = tableFound; + } else if (tableFound != tablesFound) { + String msg = "Expected all of " + allTables + " to either exist or not, but "; + if (tablesFound) { + msg += tables[0] + " exists and " + table + " does not"; + } else { + msg += tables[0] + " does not exist and " + table + " does"; + } + throw new IllegalStateException(msg); + } + } + if (!tablesFound) { + fn.run(); + } else { + Log.d("SQLiteSchema", "Skipping migration because all of " + allTables + " already exist"); + } + } - // All the mutation batches in the system, partitioned by user. - db.execSQL( - "CREATE TABLE mutations (" - + "uid TEXT, " - + "batch_id INTEGER, " - + "mutations BLOB, " - + "PRIMARY KEY (uid, batch_id))"); - - // A manually maintained index of all the mutation batches that affect a given document key. - // the rows in this table are references based on the contents of mutations.mutations. - db.execSQL( - "CREATE TABLE document_mutations (" - + "uid TEXT, " - + "path TEXT, " - + "batch_id INTEGER, " - + "PRIMARY KEY (uid, path, batch_id))"); + private void createV1MutationQueue() { + ifTablesDontExist( + new String[] {"mutation_queues", "mutations", "document_mutations"}, + () -> { + // A table naming all the mutation queues in the system. + db.execSQL( + "CREATE TABLE mutation_queues (" + + "uid TEXT PRIMARY KEY, " + + "last_acknowledged_batch_id INTEGER, " + + "last_stream_token BLOB)"); + + // All the mutation batches in the system, partitioned by user. + db.execSQL( + "CREATE TABLE mutations (" + + "uid TEXT, " + + "batch_id INTEGER, " + + "mutations BLOB, " + + "PRIMARY KEY (uid, batch_id))"); + + // A manually maintained index of all the mutation batches that affect a given document + // key. + // the rows in this table are references based on the contents of mutations.mutations. + db.execSQL( + "CREATE TABLE document_mutations (" + + "uid TEXT, " + + "path TEXT, " + + "batch_id INTEGER, " + + "PRIMARY KEY (uid, path, batch_id))"); + }); } private void removeAcknowledgedMutations() { @@ -168,64 +225,85 @@ private void removeMutationBatch(String uid, int batchId) { new Object[] {uid, batchId}); } - private void createQueryCache() { - // A cache of targets and associated metadata - db.execSQL( - "CREATE TABLE targets (" - + "target_id INTEGER PRIMARY KEY, " - + "canonical_id TEXT, " - + "snapshot_version_seconds INTEGER, " - + "snapshot_version_nanos INTEGER, " - + "resume_token BLOB, " - + "last_listen_sequence_number INTEGER," - + "target_proto BLOB)"); - - db.execSQL("CREATE INDEX query_targets ON targets (canonical_id, target_id)"); - - // Global state tracked across all queries, tracked separately - db.execSQL( - "CREATE TABLE target_globals (" - + "highest_target_id INTEGER, " - + "highest_listen_sequence_number INTEGER, " - + "last_remote_snapshot_version_seconds INTEGER, " - + "last_remote_snapshot_version_nanos INTEGER)"); - - // A Mapping table between targets and document paths - db.execSQL( - "CREATE TABLE target_documents (" - + "target_id INTEGER, " - + "path TEXT, " - + "PRIMARY KEY (target_id, path))"); - - // The document_targets reverse mapping table is just an index on target_documents. - db.execSQL("CREATE INDEX document_targets ON target_documents (path, target_id)"); + private void createV1QueryCache() { + ifTablesDontExist( + new String[] {"targets", "target_globals", "target_documents"}, + () -> { + // A cache of targets and associated metadata + db.execSQL( + "CREATE TABLE targets (" + + "target_id INTEGER PRIMARY KEY, " + + "canonical_id TEXT, " + + "snapshot_version_seconds INTEGER, " + + "snapshot_version_nanos INTEGER, " + + "resume_token BLOB, " + + "last_listen_sequence_number INTEGER," + + "target_proto BLOB)"); + + db.execSQL("CREATE INDEX query_targets ON targets (canonical_id, target_id)"); + + // Global state tracked across all queries, tracked separately + db.execSQL( + "CREATE TABLE target_globals (" + + "highest_target_id INTEGER, " + + "highest_listen_sequence_number INTEGER, " + + "last_remote_snapshot_version_seconds INTEGER, " + + "last_remote_snapshot_version_nanos INTEGER)"); + + // A Mapping table between targets and document paths + db.execSQL( + "CREATE TABLE target_documents (" + + "target_id INTEGER, " + + "path TEXT, " + + "PRIMARY KEY (target_id, path))"); + + // The document_targets reverse mapping table is just an index on target_documents. + db.execSQL("CREATE INDEX document_targets ON target_documents (path, target_id)"); + }); } - private void dropQueryCache() { - db.execSQL("DROP TABLE targets"); - db.execSQL("DROP TABLE target_globals"); - db.execSQL("DROP TABLE target_documents"); + private void dropV1QueryCache() { + // This might be overkill, but if any future migration drops these, it's possible we could try + // dropping tables that don't exist. + if (tableExists("targets")) { + db.execSQL("DROP TABLE targets"); + } + if (tableExists("target_globals")) { + db.execSQL("DROP TABLE target_globals"); + } + if (tableExists("target_documents")) { + db.execSQL("DROP TABLE target_documents"); + } } - private void createRemoteDocumentCache() { - // A cache of documents obtained from the server. - db.execSQL("CREATE TABLE remote_documents (path TEXT PRIMARY KEY, contents BLOB)"); + private void createV1RemoteDocumentCache() { + ifTablesDontExist( + new String[] {"remote_documents"}, + () -> { + // A cache of documents obtained from the server. + db.execSQL("CREATE TABLE remote_documents (path TEXT PRIMARY KEY, contents BLOB)"); + }); } + // TODO(indexing): Put the schema version in this method name. private void createLocalDocumentsCollectionIndex() { - // A per-user, per-collection index for cached documents indexed by a single field's name and - // value. - db.execSQL( - "CREATE TABLE collection_index (" - + "uid TEXT, " - + "collection_path TEXT, " - + "field_path TEXT, " - + "field_value_type INTEGER, " // determines type of field_value fields. - + "field_value_1, " // first component - + "field_value_2, " // second component; required for timestamps, GeoPoints - + "document_id TEXT, " - + "PRIMARY KEY (uid, collection_path, field_path, field_value_type, field_value_1, " - + "field_value_2, document_id))"); + ifTablesDontExist( + new String[] {"collection_index"}, + () -> { + // A per-user, per-collection index for cached documents indexed by a single field's name + // and value. + db.execSQL( + "CREATE TABLE collection_index (" + + "uid TEXT, " + + "collection_path TEXT, " + + "field_path TEXT, " + + "field_value_type INTEGER, " // determines type of field_value fields. + + "field_value_1, " // first component + + "field_value_2, " // second component; required for timestamps, GeoPoints + + "document_id TEXT, " + + "PRIMARY KEY (uid, collection_path, field_path, field_value_type, field_value_1, " + + "field_value_2, document_id))"); + }); } // Note that this runs before we add the target count column, so we don't populate it yet. @@ -241,15 +319,20 @@ private void ensureTargetGlobal() { } private void addTargetCount() { + if (!tableContainsColumn("target_globals", "target_count")) { + db.execSQL("ALTER TABLE target_globals ADD COLUMN target_count INTEGER"); + } + // Even if the column already existed, rerun the data migration to make sure it's correct. long count = DatabaseUtils.queryNumEntries(db, "targets"); - db.execSQL("ALTER TABLE target_globals ADD COLUMN target_count INTEGER"); ContentValues cv = new ContentValues(); cv.put("target_count", count); db.update("target_globals", cv, null, null); } private void addSequenceNumber() { - db.execSQL("ALTER TABLE target_documents ADD COLUMN sequence_number INTEGER"); + if (!tableContainsColumn("target_documents", "sequence_number")) { + db.execSQL("ALTER TABLE target_documents ADD COLUMN sequence_number INTEGER"); + } } /** @@ -281,4 +364,35 @@ private void ensureSequenceNumbers() { hardAssert(tagDocument.executeInsert() != -1, "Failed to insert a sentinel row"); }); } + + private boolean tableContainsColumn(String table, String column) { + List columns = getTableColumns(table); + return columns.indexOf(column) != -1; + } + + @VisibleForTesting + List getTableColumns(String table) { + // NOTE: SQLitePersistence.Query helper binding doesn't work with PRAGMA queries. So, just use + // `rawQuery`. + Cursor c = null; + List columns = new ArrayList<>(); + try { + c = db.rawQuery("PRAGMA table_info(" + table + ")", null); + int nameIndex = c.getColumnIndex("name"); + while (c.moveToNext()) { + columns.add(c.getString(nameIndex)); + } + } finally { + if (c != null) { + c.close(); + } + } + return columns; + } + + private boolean tableExists(String table) { + return !new SQLitePersistence.Query(db, "SELECT 1=1 FROM sqlite_master WHERE tbl_name = ?") + .binding(table) + .isEmpty(); + } } diff --git a/firebase-firestore/src/test/java/com/google/firebase/firestore/local/SQLiteSchemaTest.java b/firebase-firestore/src/test/java/com/google/firebase/firestore/local/SQLiteSchemaTest.java index ec03c6160ed..5f068b5049b 100644 --- a/firebase-firestore/src/test/java/com/google/firebase/firestore/local/SQLiteSchemaTest.java +++ b/firebase-firestore/src/test/java/com/google/firebase/firestore/local/SQLiteSchemaTest.java @@ -16,6 +16,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import android.database.Cursor; @@ -26,6 +27,10 @@ import com.google.firebase.firestore.proto.WriteBatch; import com.google.firestore.v1beta1.Document; import com.google.firestore.v1beta1.Write; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -65,6 +70,69 @@ public void tearDown() { } } + @Test + public void canRerunMigrations() { + schema.runMigrations(); + // Run the whole thing again + schema.runMigrations(); + // Run just a piece. Adds a column, make sure it doesn't throw + schema.runMigrations(4, 6); + } + + private Map> getCurrentSchema() { + Map> tables = new HashMap<>(); + new SQLitePersistence.Query(db, "SELECT tbl_name FROM sqlite_master WHERE type = \"table\"") + .forEach( + c -> { + String table = c.getString(0); + Set columns = new HashSet<>(schema.getTableColumns(table)); + tables.put(table, columns); + }); + return tables; + } + + private void assertNoRemovals( + Map> oldSchema, Map> newSchema, int newVersion) { + for (Map.Entry> entry : oldSchema.entrySet()) { + String table = entry.getKey(); + Set newColumns = newSchema.get(table); + assertNotNull("Table " + table + " was deleted at version " + newVersion, newColumns); + Set oldColumns = entry.getValue(); + // We could use `Set.containsAll()`, but if we iterate we can point out the column that was + // deleted. + for (String column : oldColumns) { + assertTrue( + "Column " + column + " was deleted from table " + table + " at version " + newVersion, + newColumns.contains(column)); + } + } + } + + @Test + public void migrationsDontDeleteTablesOrColumns() { + // In order to support users downgrading the SDK we need to make sure that every prior-released + // version of the SDK can gracefully handle running against an upgraded schema. We can't + // guarantee this in the general case, but this test at least ensures that no schema upgrade + // deletes an existing table or column, which would be very likely to break old versions of the + // SDK relying on that table or column. + Map> tables = new HashMap<>(); + for (int toVersion = 1; toVersion <= SQLiteSchema.VERSION; toVersion++) { + schema.runMigrations(toVersion - 1, toVersion); + Map> newTables = getCurrentSchema(); + assertNoRemovals(tables, newTables, toVersion); + tables = newTables; + } + } + + @Test + public void canRecoverFromDowngrades() { + for (int downgradeVersion = 0; downgradeVersion < SQLiteSchema.VERSION; downgradeVersion++) { + // Upgrade schema to current, then upgrade from `downgradeVersion` to current + schema.runMigrations(); + schema.runMigrations(downgradeVersion, SQLiteSchema.VERSION); + } + } + @Test public void createsMutationsTable() { schema.runMigrations(); From 08840df3edfaf9a3d31461d10824d66901d85a7f Mon Sep 17 00:00:00 2001 From: Vladimir Kryachko Date: Thu, 6 Dec 2018 15:49:41 -0500 Subject: [PATCH 16/25] Fix automatic data collection(b/120600091). (#156) * Fix automatic data collection(b/120600091). Makes setting persistense scoped per FirebaseApp instance instead of being scoped globally. * Addressed review comments, factored out common functionality. --- .../java/com/google/firebase/FirebaseApp.java | 8 ++- .../DataCollectionDefaultDisabledTest.java | 31 +--------- .../DataCollectionDefaultEnabledTest.java | 50 +++++++--------- .../firebase/DataCollectionTestUtil.java | 57 +++++++++++++++++++ 4 files changed, 88 insertions(+), 58 deletions(-) create mode 100644 firebase-common/src/test/java/com/google/firebase/DataCollectionTestUtil.java diff --git a/firebase-common/src/main/java/com/google/firebase/FirebaseApp.java b/firebase-common/src/main/java/com/google/firebase/FirebaseApp.java index 1fcbcb63945..b96067145e4 100644 --- a/firebase-common/src/main/java/com/google/firebase/FirebaseApp.java +++ b/firebase-common/src/main/java/com/google/firebase/FirebaseApp.java @@ -104,7 +104,7 @@ public class FirebaseApp { public static final String DEFAULT_APP_NAME = "[DEFAULT]"; - @VisibleForTesting static final String FIREBASE_APP_PREFS = "com.google.firebase.common.prefs"; + private static final String FIREBASE_APP_PREFS = "com.google.firebase.common.prefs:"; @VisibleForTesting static final String DATA_COLLECTION_DEFAULT_ENABLED = "firebase_data_collection_default_enabled"; @@ -525,7 +525,7 @@ protected FirebaseApp(Context applicationContext, String name, FirebaseOptions o idTokenListenersCountChangedListener = new DefaultIdTokenListenersCountChangedListener(); sharedPreferences = - applicationContext.getSharedPreferences(FIREBASE_APP_PREFS, Context.MODE_PRIVATE); + applicationContext.getSharedPreferences(getSharedPrefsName(name), Context.MODE_PRIVATE); dataCollectionDefaultEnabled = new AtomicBoolean(readAutoDataCollectionEnabled()); List registrars = @@ -540,6 +540,10 @@ protected FirebaseApp(Context applicationContext, String name, FirebaseOptions o publisher = componentRuntime.get(Publisher.class); } + private static String getSharedPrefsName(String appName) { + return FIREBASE_APP_PREFS + appName; + } + private boolean readAutoDataCollectionEnabled() { if (sharedPreferences.contains(DATA_COLLECTION_DEFAULT_ENABLED)) { return sharedPreferences.getBoolean(DATA_COLLECTION_DEFAULT_ENABLED, true); diff --git a/firebase-common/src/test/java/com/google/firebase/DataCollectionDefaultDisabledTest.java b/firebase-common/src/test/java/com/google/firebase/DataCollectionDefaultDisabledTest.java index e518c1d5368..26cf2c23368 100644 --- a/firebase-common/src/test/java/com/google/firebase/DataCollectionDefaultDisabledTest.java +++ b/firebase-common/src/test/java/com/google/firebase/DataCollectionDefaultDisabledTest.java @@ -15,14 +15,14 @@ package com.google.firebase; import static com.google.common.truth.Truth.assertThat; +import static com.google.firebase.DataCollectionTestUtil.getSharedPreferences; +import static com.google.firebase.DataCollectionTestUtil.setSharedPreferencesTo; +import static com.google.firebase.DataCollectionTestUtil.withApp; -import android.content.Context; import android.content.SharedPreferences; -import java.util.function.Consumer; import org.junit.Test; import org.junit.runner.RunWith; import org.robolectric.RobolectricTestRunner; -import org.robolectric.RuntimeEnvironment; import org.robolectric.annotation.Config; @RunWith(RobolectricTestRunner.class) @@ -76,29 +76,4 @@ public void setDataCollectionDefaultEnabledTrue_shouldEmitEvents() { assertThat(changeListener.changes).containsExactly(true, false).inOrder(); }); } - - private static void withApp(Consumer callable) { - FirebaseApp app = - FirebaseApp.initializeApp( - RuntimeEnvironment.application.getApplicationContext(), - new FirebaseOptions.Builder().setApplicationId("appId").build(), - "someApp"); - try { - callable.accept(app); - } finally { - app.delete(); - } - } - - private static SharedPreferences getSharedPreferences() { - return RuntimeEnvironment.application.getSharedPreferences( - FirebaseApp.FIREBASE_APP_PREFS, Context.MODE_PRIVATE); - } - - private static void setSharedPreferencesTo(boolean enabled) { - getSharedPreferences() - .edit() - .putBoolean(FirebaseApp.DATA_COLLECTION_DEFAULT_ENABLED, enabled) - .commit(); - } } diff --git a/firebase-common/src/test/java/com/google/firebase/DataCollectionDefaultEnabledTest.java b/firebase-common/src/test/java/com/google/firebase/DataCollectionDefaultEnabledTest.java index 997f3a4ab2e..1de32014374 100644 --- a/firebase-common/src/test/java/com/google/firebase/DataCollectionDefaultEnabledTest.java +++ b/firebase-common/src/test/java/com/google/firebase/DataCollectionDefaultEnabledTest.java @@ -15,21 +15,18 @@ package com.google.firebase; import static com.google.common.truth.Truth.assertThat; +import static com.google.firebase.DataCollectionTestUtil.getSharedPreferences; +import static com.google.firebase.DataCollectionTestUtil.setSharedPreferencesTo; +import static com.google.firebase.DataCollectionTestUtil.withApp; -import android.content.Context; import android.content.SharedPreferences; -import java.util.function.Consumer; import org.junit.Test; import org.junit.runner.RunWith; import org.robolectric.RobolectricTestRunner; -import org.robolectric.RuntimeEnvironment; @RunWith(RobolectricTestRunner.class) public class DataCollectionDefaultEnabledTest { - private static final String NO_AUTO_DATA_COLLECTION_MANIFEST = - "NoAutoDataCollectionAndroidManifest.xml"; - @Test public void isDataCollectionDefaultEnabled_shouldDefaultToTrue() { withApp(app -> assertThat(app.isDataCollectionDefaultEnabled()).isTrue()); @@ -61,28 +58,25 @@ public void setDataCollectionDefaultEnabledFalse_shouldUpdateSharedPrefs() { }); } - private static void withApp(Consumer callable) { - FirebaseApp app = - FirebaseApp.initializeApp( - RuntimeEnvironment.application.getApplicationContext(), - new FirebaseOptions.Builder().setApplicationId("appId").build(), - "someApp"); - try { - callable.accept(app); - } finally { - app.delete(); - } - } - - private static SharedPreferences getSharedPreferences() { - return RuntimeEnvironment.application.getSharedPreferences( - FirebaseApp.FIREBASE_APP_PREFS, Context.MODE_PRIVATE); - } + @Test + public void setDataCollectionDefaultEnabled_shouldNotAffectOtherFirebaseAppInstances() { + withApp( + "app1", + app1 -> { + withApp( + "app2", + app2 -> { + assertThat(app1.isDataCollectionDefaultEnabled()).isTrue(); + assertThat(app2.isDataCollectionDefaultEnabled()).isTrue(); + }); - private static void setSharedPreferencesTo(boolean enabled) { - getSharedPreferences() - .edit() - .putBoolean(FirebaseApp.DATA_COLLECTION_DEFAULT_ENABLED, enabled) - .commit(); + app1.setDataCollectionDefaultEnabled(false); + withApp( + "app2", + app2 -> { + assertThat(app1.isDataCollectionDefaultEnabled()).isFalse(); + assertThat(app2.isDataCollectionDefaultEnabled()).isTrue(); + }); + }); } } diff --git a/firebase-common/src/test/java/com/google/firebase/DataCollectionTestUtil.java b/firebase-common/src/test/java/com/google/firebase/DataCollectionTestUtil.java new file mode 100644 index 00000000000..a3739d069d2 --- /dev/null +++ b/firebase-common/src/test/java/com/google/firebase/DataCollectionTestUtil.java @@ -0,0 +1,57 @@ +// Copyright 2018 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.firebase; + +import android.content.Context; +import android.content.SharedPreferences; +import java.util.function.Consumer; +import org.robolectric.RuntimeEnvironment; + +public final class DataCollectionTestUtil { + static final String APP_NAME = "someApp"; + + static final String FIREBASE_APP_PREFS = "com.google.firebase.common.prefs:"; + + private DataCollectionTestUtil() {} + + static void withApp(Consumer callable) { + withApp(APP_NAME, callable); + } + + static void withApp(String name, Consumer callable) { + FirebaseApp app = + FirebaseApp.initializeApp( + RuntimeEnvironment.application.getApplicationContext(), + new FirebaseOptions.Builder().setApplicationId("appId").build(), + name); + try { + callable.accept(app); + } finally { + app.delete(); + } + } + + static SharedPreferences getSharedPreferences() { + return RuntimeEnvironment.application.getSharedPreferences( + FIREBASE_APP_PREFS + APP_NAME, Context.MODE_PRIVATE); + } + + static void setSharedPreferencesTo(boolean enabled) { + getSharedPreferences() + .edit() + .putBoolean(FirebaseApp.DATA_COLLECTION_DEFAULT_ENABLED, enabled) + .commit(); + } +} From d00e6161b8f56db2eb6ecf72c3ba769929d5dc6b Mon Sep 17 00:00:00 2001 From: Ashwin Raghav Date: Thu, 6 Dec 2018 16:14:06 -0800 Subject: [PATCH 17/25] Adding version info to manifest for blaze builds (#157) --- firebase-common/src/main/AndroidManifest.xml | 21 +++++++++++-------- .../src/main/AndroidManifest.xml | 4 +++- .../src/androidTest/AndroidManifest.xml | 3 ++- .../src/main/AndroidManifest.xml | 2 ++ .../src/androidTest/AndroidManifest.xml | 1 + .../src/main/AndroidManifest.xml | 3 +++ .../src/test/AndroidManifest.xml | 1 + .../src/androidTest/AndroidManifest.xml | 3 +++ .../src/main/AndroidManifest.xml | 2 ++ .../src/main/AndroidManifest.xml | 2 ++ .../src/androidTest/AndroidManifest.xml | 3 +++ firebase-storage/src/main/AndroidManifest.xml | 2 ++ firebase-storage/src/test/AndroidManifest.xml | 1 + 13 files changed, 37 insertions(+), 11 deletions(-) diff --git a/firebase-common/src/main/AndroidManifest.xml b/firebase-common/src/main/AndroidManifest.xml index c0c662065cb..97bb6edbd84 100644 --- a/firebase-common/src/main/AndroidManifest.xml +++ b/firebase-common/src/main/AndroidManifest.xml @@ -1,12 +1,15 @@ - - + + + + - - + + diff --git a/firebase-database-collection/src/main/AndroidManifest.xml b/firebase-database-collection/src/main/AndroidManifest.xml index 59833a0d91f..c56b75e0188 100644 --- a/firebase-database-collection/src/main/AndroidManifest.xml +++ b/firebase-database-collection/src/main/AndroidManifest.xml @@ -1,4 +1,6 @@ - + + diff --git a/firebase-database/src/androidTest/AndroidManifest.xml b/firebase-database/src/androidTest/AndroidManifest.xml index bd4a6dd5bbd..d43c61d2d36 100644 --- a/firebase-database/src/androidTest/AndroidManifest.xml +++ b/firebase-database/src/androidTest/AndroidManifest.xml @@ -16,7 +16,8 @@ - + diff --git a/firebase-database/src/main/AndroidManifest.xml b/firebase-database/src/main/AndroidManifest.xml index 1b8c9298286..a79682a8e51 100644 --- a/firebase-database/src/main/AndroidManifest.xml +++ b/firebase-database/src/main/AndroidManifest.xml @@ -15,6 +15,8 @@ + + diff --git a/firebase-firestore/src/androidTest/AndroidManifest.xml b/firebase-firestore/src/androidTest/AndroidManifest.xml index 4cea76cbfa5..f3926e89dae 100644 --- a/firebase-firestore/src/androidTest/AndroidManifest.xml +++ b/firebase-firestore/src/androidTest/AndroidManifest.xml @@ -1,5 +1,6 @@ + diff --git a/firebase-firestore/src/main/AndroidManifest.xml b/firebase-firestore/src/main/AndroidManifest.xml index e3ad260008c..e18990f1121 100644 --- a/firebase-firestore/src/main/AndroidManifest.xml +++ b/firebase-firestore/src/main/AndroidManifest.xml @@ -2,6 +2,9 @@ + + + diff --git a/firebase-firestore/src/test/AndroidManifest.xml b/firebase-firestore/src/test/AndroidManifest.xml index 7bbee804d04..74f118943a5 100644 --- a/firebase-firestore/src/test/AndroidManifest.xml +++ b/firebase-firestore/src/test/AndroidManifest.xml @@ -2,6 +2,7 @@ + diff --git a/firebase-functions/src/androidTest/AndroidManifest.xml b/firebase-functions/src/androidTest/AndroidManifest.xml index 1da6c39bdca..4d0b912811d 100644 --- a/firebase-functions/src/androidTest/AndroidManifest.xml +++ b/firebase-functions/src/androidTest/AndroidManifest.xml @@ -1,5 +1,8 @@ + + diff --git a/firebase-functions/src/main/AndroidManifest.xml b/firebase-functions/src/main/AndroidManifest.xml index b27668ca405..605a67ec56b 100644 --- a/firebase-functions/src/main/AndroidManifest.xml +++ b/firebase-functions/src/main/AndroidManifest.xml @@ -2,6 +2,8 @@ + + diff --git a/firebase-inappmessaging-display/src/main/AndroidManifest.xml b/firebase-inappmessaging-display/src/main/AndroidManifest.xml index 7bcd0788986..5a244f51f52 100644 --- a/firebase-inappmessaging-display/src/main/AndroidManifest.xml +++ b/firebase-inappmessaging-display/src/main/AndroidManifest.xml @@ -1,6 +1,8 @@ + + diff --git a/firebase-storage/src/androidTest/AndroidManifest.xml b/firebase-storage/src/androidTest/AndroidManifest.xml index 9c9ff5d2d95..8b386d8e779 100644 --- a/firebase-storage/src/androidTest/AndroidManifest.xml +++ b/firebase-storage/src/androidTest/AndroidManifest.xml @@ -14,6 +14,9 @@ + + diff --git a/firebase-storage/src/main/AndroidManifest.xml b/firebase-storage/src/main/AndroidManifest.xml index 69b9ce7943f..566b5e67928 100644 --- a/firebase-storage/src/main/AndroidManifest.xml +++ b/firebase-storage/src/main/AndroidManifest.xml @@ -16,6 +16,8 @@ + + diff --git a/firebase-storage/src/test/AndroidManifest.xml b/firebase-storage/src/test/AndroidManifest.xml index 4d2ea6007f2..307e8847de0 100644 --- a/firebase-storage/src/test/AndroidManifest.xml +++ b/firebase-storage/src/test/AndroidManifest.xml @@ -16,6 +16,7 @@ + From b40b461c0fb85af0965d12d2ff3ca1e12af26e15 Mon Sep 17 00:00:00 2001 From: Vladimir Kryachko Date: Tue, 11 Dec 2018 18:29:43 -0500 Subject: [PATCH 18/25] Add support for multiple test-app directories to smoke_tests. (#159) * Add support for multiple test-app directories to smoke_tests. * Delete gradle wrapper from test-apps. Instead use the gradlew from the root project. * Increase emulator startup timeout. --- ci/fireci/fireci/commands.py | 26 ++- ci/fireci/fireci/emulator.py | 2 +- ci/fireci/setup.py | 2 +- test-apps/gradle/projectSettings.gradle | 41 ----- test-apps/gradle/wrapper/gradle-wrapper.jar | Bin 54727 -> 0 bytes .../gradle/wrapper/gradle-wrapper.properties | 22 --- test-apps/gradlew | 173 +----------------- test-apps/gradlew.bat | 84 --------- 8 files changed, 21 insertions(+), 329 deletions(-) delete mode 100644 test-apps/gradle/projectSettings.gradle delete mode 100644 test-apps/gradle/wrapper/gradle-wrapper.jar delete mode 100644 test-apps/gradle/wrapper/gradle-wrapper.properties mode change 100755 => 120000 test-apps/gradlew delete mode 100644 test-apps/gradlew.bat diff --git a/ci/fireci/fireci/commands.py b/ci/fireci/fireci/commands.py index d32d4a5e7de..2685bb438a6 100644 --- a/ci/fireci/fireci/commands.py +++ b/ci/fireci/fireci/commands.py @@ -37,16 +37,26 @@ def gradle_command(task, gradle_opts): help= 'App build variant to use while running the smoke Tests. One of release|debug' ) +@click.option( + '--test-apps-dir', + '-d', + multiple=True, + type=click.Path(exists=True, file_okay=False, resolve_path=True), + default=['test-apps'], + help= + 'Directory that contains gradle build with apps to test against. Multiple values are allowed.' +) @ci_command() -def smoke_tests(app_build_variant): +def smoke_tests(app_build_variant, test_apps_dir): """Builds all SDKs in release mode and then tests test-apps against them.""" gradle.run('publishAllToBuildDir') cwd = os.getcwd() - gradle.run( - 'connectedCheck', - '-PtestBuildType=%s' % (app_build_variant), - gradle_opts='-Dmaven.repo.local={}'.format( - os.path.join(cwd, 'build', 'm2repository')), - workdir=os.path.join(cwd, 'test-apps'), - ) + for location in test_apps_dir: + gradle.run( + 'connectedCheck', + '-PtestBuildType=%s' % (app_build_variant), + gradle_opts='-Dmaven.repo.local={}'.format( + os.path.join(cwd, 'build', 'm2repository')), + workdir=location, + ) diff --git a/ci/fireci/fireci/emulator.py b/ci/fireci/fireci/emulator.py index 89edc4d65ca..5ea3aa0cc4b 100644 --- a/ci/fireci/fireci/emulator.py +++ b/ci/fireci/fireci/emulator.py @@ -74,7 +74,7 @@ def __enter__(self): stdout=self._stdout, stderr=self._stderr) try: - self._wait_for_boot(datetime.timedelta(minutes=5)) + self._wait_for_boot(datetime.timedelta(minutes=10)) except: self._kill(self._process) self._close_files() diff --git a/ci/fireci/setup.py b/ci/fireci/setup.py index 16fa88f0dae..09b2e32f727 100755 --- a/ci/fireci/setup.py +++ b/ci/fireci/setup.py @@ -25,7 +25,7 @@ name='fireci', version='0.1', install_requires=[ - 'click==6.7', + 'click==7.0', ], packages=find_packages(exclude=['tests']), entry_points={ diff --git a/test-apps/gradle/projectSettings.gradle b/test-apps/gradle/projectSettings.gradle deleted file mode 100644 index ad2bf56e535..00000000000 --- a/test-apps/gradle/projectSettings.gradle +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -ext { - /** - * Parses the input file and returns a list of subprojects. - * - *

Expected file format: - *

    - *
  • Empty lines are ignored. - *
  • Lines that start with # are ignored(considered comments). - *
  • Other lines are considered project paths with initial ":" removed. - *
- */ - discoverSubprojects = {File subprojectsFile -> - return subprojectsFile.readLines().collect {it.trim()}.findAll { !it.empty && !it.startsWith('#')} - } - - /** Recursively renames build scripts to ${project.name}.gradle. */ - renameBuildScripts = {ProjectDescriptor project -> - project.buildFileName = project.parent ? "${project.name}.gradle" : 'root-project.gradle' - - project.children.each { - renameBuildScripts(it) - } - - } -} diff --git a/test-apps/gradle/wrapper/gradle-wrapper.jar b/test-apps/gradle/wrapper/gradle-wrapper.jar deleted file mode 100644 index 27768f1bbac3ce2d055b20d521f12da78d331e8e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 54727 zcmagFV|ZrKvM!pAZQHhO+qP}9lTNj?q^^Y^VFp)SH8qbSJ)2BQ2girNE| z%tC(^)c?v~^Z#E_K}1nTQbJ9gQ9<%vVRAxVj)8FwL5_iTdUB>&m3fhE=kRWl;g`&m z!W5kh{WsV%fO*%je&j+Lv4xxK~zsEYQls$Q-p&dwID|A)!7uWtJF-=Tm1{V@#x*+kUI$=%KUuf2ka zjiZ{oiL1MXE2EjciJM!jrjFNwCh`~hL>iemrqwqnX?T*MX;U>>8yRcZb{Oy+VKZos zLiFKYPw=LcaaQt8tj=eoo3-@bG_342HQ%?jpgAE?KCLEHC+DmjxAfJ%Og^$dpC8Xw zAcp-)tfJm}BPNq_+6m4gBgBm3+CvmL>4|$2N$^Bz7W(}fz1?U-u;nE`+9`KCLuqg} zwNstNM!J4Uw|78&Y9~9>MLf56to!@qGkJw5Thx%zkzj%Ek9Nn1QA@8NBXbwyWC>9H z#EPwjMNYPigE>*Ofz)HfTF&%PFj$U6mCe-AFw$U%-L?~-+nSXHHKkdgC5KJRTF}`G zE_HNdrE}S0zf4j{r_f-V2imSqW?}3w-4=f@o@-q+cZgaAbZ((hn))@|eWWhcT2pLpTpL!;_5*vM=sRL8 zqU##{U#lJKuyqW^X$ETU5ETeEVzhU|1m1750#f}38_5N9)B_2|v@1hUu=Kt7-@dhA zq_`OMgW01n`%1dB*}C)qxC8q;?zPeF_r;>}%JYmlER_1CUbKa07+=TV45~symC*g8 zW-8(gag#cAOuM0B1xG8eTp5HGVLE}+gYTmK=`XVVV*U!>H`~j4+ROIQ+NkN$LY>h4 zqpwdeE_@AX@PL};e5vTn`Ro(EjHVf$;^oiA%@IBQq>R7_D>m2D4OwwEepkg}R_k*M zM-o;+P27087eb+%*+6vWFCo9UEGw>t&WI17Pe7QVuoAoGHdJ(TEQNlJOqnjZ8adCb zI`}op16D@v7UOEo%8E-~m?c8FL1utPYlg@m$q@q7%mQ4?OK1h%ODjTjFvqd!C z-PI?8qX8{a@6d&Lb_X+hKxCImb*3GFemm?W_du5_&EqRq!+H?5#xiX#w$eLti-?E$;Dhu`{R(o>LzM4CjO>ICf z&DMfES#FW7npnbcuqREgjPQM#gs6h>`av_oEWwOJZ2i2|D|0~pYd#WazE2Bbsa}X@ zu;(9fi~%!VcjK6)?_wMAW-YXJAR{QHxrD5g(ou9mR6LPSA4BRG1QSZT6A?kelP_g- zH(JQjLc!`H4N=oLw=f3{+WmPA*s8QEeEUf6Vg}@!xwnsnR0bl~^2GSa5vb!Yl&4!> zWb|KQUsC$lT=3A|7vM9+d;mq=@L%uWKwXiO9}a~gP4s_4Yohc!fKEgV7WbVo>2ITbE*i`a|V!^p@~^<={#?Gz57 zyPWeM2@p>D*FW#W5Q`1`#5NW62XduP1XNO(bhg&cX`-LYZa|m-**bu|>}S;3)eP8_ zpNTnTfm8 ze+7wDH3KJ95p)5tlwk`S7mbD`SqHnYD*6`;gpp8VdHDz%RR_~I_Ar>5)vE-Pgu7^Y z|9Px+>pi3!DV%E%4N;ii0U3VBd2ZJNUY1YC^-e+{DYq+l@cGtmu(H#Oh%ibUBOd?C z{y5jW3v=0eV0r@qMLgv1JjZC|cZ9l9Q)k1lLgm))UR@#FrJd>w^`+iy$c9F@ic-|q zVHe@S2UAnc5VY_U4253QJxm&Ip!XKP8WNcnx9^cQ;KH6PlW8%pSihSH2(@{2m_o+m zr((MvBja2ctg0d0&U5XTD;5?d?h%JcRJp{_1BQW1xu&BrA3(a4Fh9hon-ly$pyeHq zG&;6q?m%NJ36K1Sq_=fdP(4f{Hop;_G_(i?sPzvB zDM}>*(uOsY0I1j^{$yn3#U(;B*g4cy$-1DTOkh3P!LQ;lJlP%jY8}Nya=h8$XD~%Y zbV&HJ%eCD9nui-0cw!+n`V~p6VCRqh5fRX z8`GbdZ@73r7~myQLBW%db;+BI?c-a>Y)m-FW~M=1^|<21_Sh9RT3iGbO{o-hpN%d6 z7%++#WekoBOP^d0$$|5npPe>u3PLvX_gjH2x(?{&z{jJ2tAOWTznPxv-pAv<*V7r$ z6&glt>7CAClWz6FEi3bToz-soY^{ScrjwVPV51=>n->c(NJngMj6TyHty`bfkF1hc zkJS%A@cL~QV0-aK4>Id!9dh7>0IV;1J9(myDO+gv76L3NLMUm9XyPauvNu$S<)-|F zZS}(kK_WnB)Cl`U?jsdYfAV4nrgzIF@+%1U8$poW&h^c6>kCx3;||fS1_7JvQT~CV zQ8Js+!p)3oW>Df(-}uqC`Tcd%E7GdJ0p}kYj5j8NKMp(KUs9u7?jQ94C)}0rba($~ zqyBx$(1ae^HEDG`Zc@-rXk1cqc7v0wibOR4qpgRDt#>-*8N3P;uKV0CgJE2SP>#8h z=+;i_CGlv+B^+$5a}SicVaSeaNn29K`C&=}`=#Nj&WJP9Xhz4mVa<+yP6hkrq1vo= z1rX4qg8dc4pmEvq%NAkpMK>mf2g?tg_1k2%v}<3`$6~Wlq@ItJ*PhHPoEh1Yi>v57 z4k0JMO)*=S`tKvR5gb-(VTEo>5Y>DZJZzgR+j6{Y`kd|jCVrg!>2hVjz({kZR z`dLlKhoqT!aI8=S+fVp(5*Dn6RrbpyO~0+?fy;bm$0jmTN|t5i6rxqr4=O}dY+ROd zo9Et|x}!u*xi~>-y>!M^+f&jc;IAsGiM_^}+4|pHRn{LThFFpD{bZ|TA*wcGm}XV^ zr*C6~@^5X-*R%FrHIgo-hJTBcyQ|3QEj+cSqp#>&t`ZzB?cXM6S(lRQw$I2?m5=wd z78ki`R?%;o%VUhXH?Z#(uwAn9$m`npJ=cA+lHGk@T7qq_M6Zoy1Lm9E0UUysN)I_x zW__OAqvku^>`J&CB=ie@yNWsaFmem}#L3T(x?a`oZ+$;3O-icj2(5z72Hnj=9Z0w% z<2#q-R=>hig*(t0^v)eGq2DHC%GymE-_j1WwBVGoU=GORGjtaqr0BNigOCqyt;O(S zKG+DoBsZU~okF<7ahjS}bzwXxbAxFfQAk&O@>LsZMsZ`?N?|CDWM(vOm%B3CBPC3o z%2t@%H$fwur}SSnckUm0-k)mOtht`?nwsDz=2#v=RBPGg39i#%odKq{K^;bTD!6A9 zskz$}t)sU^=a#jLZP@I=bPo?f-L}wpMs{Tc!m7-bi!Ldqj3EA~V;4(dltJmTXqH0r z%HAWKGutEc9vOo3P6Q;JdC^YTnby->VZ6&X8f{obffZ??1(cm&L2h7q)*w**+sE6dG*;(H|_Q!WxU{g)CeoT z(KY&bv!Usc|m+Fqfmk;h&RNF|LWuNZ!+DdX*L=s-=_iH=@i` z?Z+Okq^cFO4}_n|G*!)Wl_i%qiMBaH8(WuXtgI7EO=M>=i_+;MDjf3aY~6S9w0K zUuDO7O5Ta6+k40~xh~)D{=L&?Y0?c$s9cw*Ufe18)zzk%#ZY>Tr^|e%8KPb0ht`b( zuP@8#Ox@nQIqz9}AbW0RzE`Cf>39bOWz5N3qzS}ocxI=o$W|(nD~@EhW13Rj5nAp; zu2obEJa=kGC*#3=MkdkWy_%RKcN=?g$7!AZ8vBYKr$ePY(8aIQ&yRPlQ=mudv#q$q z4%WzAx=B{i)UdLFx4os?rZp6poShD7Vc&mSD@RdBJ=_m^&OlkEE1DFU@csgKcBifJ zz4N7+XEJhYzzO=86 z#%eBQZ$Nsf2+X0XPHUNmg#(sNt^NW1Y0|M(${e<0kW6f2q5M!2YE|hSEQ*X-%qo(V zHaFwyGZ0on=I{=fhe<=zo{=Og-_(to3?cvL4m6PymtNsdDINsBh8m>a%!5o3s(en) z=1I z6O+YNertC|OFNqd6P=$gMyvmfa`w~p9*gKDESFqNBy(~Zw3TFDYh}$iudn)9HxPBi zdokK@o~nu?%imcURr5Y~?6oo_JBe}t|pU5qjai|#JDyG=i^V~7+a{dEnO<(y>ahND#_X_fcEBNiZ)uc&%1HVtx8Ts z*H_Btvx^IhkfOB#{szN*n6;y05A>3eARDXslaE>tnLa>+`V&cgho?ED+&vv5KJszf zG4@G;7i;4_bVvZ>!mli3j7~tPgybF5|J6=Lt`u$D%X0l}#iY9nOXH@(%FFJLtzb%p zzHfABnSs;v-9(&nzbZytLiqqDIWzn>JQDk#JULcE5CyPq_m#4QV!}3421haQ+LcfO*>r;rg6K|r#5Sh|y@h1ao%Cl)t*u`4 zMTP!deC?aL7uTxm5^nUv#q2vS-5QbBKP|drbDXS%erB>fYM84Kpk^au99-BQBZR z7CDynflrIAi&ahza+kUryju5LR_}-Z27g)jqOc(!Lx9y)e z{cYc&_r947s9pteaa4}dc|!$$N9+M38sUr7h(%@Ehq`4HJtTpA>B8CLNO__@%(F5d z`SmX5jbux6i#qc}xOhumzbAELh*Mfr2SW99=WNOZRZgoCU4A2|4i|ZVFQt6qEhH#B zK_9G;&h*LO6tB`5dXRSBF0hq0tk{2q__aCKXYkP#9n^)@cq}`&Lo)1KM{W+>5mSed zKp~=}$p7>~nK@va`vN{mYzWN1(tE=u2BZhga5(VtPKk(*TvE&zmn5vSbjo zZLVobTl%;t@6;4SsZ>5+U-XEGUZGG;+~|V(pE&qqrp_f~{_1h@5ZrNETqe{bt9ioZ z#Qn~gWCH!t#Ha^n&fT2?{`}D@s4?9kXj;E;lWV9Zw8_4yM0Qg-6YSsKgvQ*fF{#Pq z{=(nyV>#*`RloBVCs;Lp*R1PBIQOY=EK4CQa*BD0MsYcg=opP?8;xYQDSAJBeJpw5 zPBc_Ft9?;<0?pBhCmOtWU*pN*;CkjJ_}qVic`}V@$TwFi15!mF1*m2wVX+>5p%(+R zQ~JUW*zWkalde{90@2v+oVlkxOZFihE&ZJ){c?hX3L2@R7jk*xjYtHi=}qb+4B(XJ z$gYcNudR~4Kz_WRq8eS((>ALWCO)&R-MXE+YxDn9V#X{_H@j616<|P(8h(7z?q*r+ zmpqR#7+g$cT@e&(%_|ipI&A%9+47%30TLY(yuf&*knx1wNx|%*H^;YB%ftt%5>QM= z^i;*6_KTSRzQm%qz*>cK&EISvF^ovbS4|R%)zKhTH_2K>jP3mBGn5{95&G9^a#4|K zv+!>fIsR8z{^x4)FIr*cYT@Q4Z{y}};rLHL+atCgHbfX*;+k&37DIgENn&=k(*lKD zG;uL-KAdLn*JQ?@r6Q!0V$xXP=J2i~;_+i3|F;_En;oAMG|I-RX#FwnmU&G}w`7R{ z788CrR-g1DW4h_`&$Z`ctN~{A)Hv_-Bl!%+pfif8wN32rMD zJDs$eVWBYQx1&2sCdB0!vU5~uf)=vy*{}t{2VBpcz<+~h0wb7F3?V^44*&83Z2#F` z32!rd4>uc63rQP$3lTH3zb-47IGR}f)8kZ4JvX#toIpXH`L%NnPDE~$QI1)0)|HS4 zVcITo$$oWWwCN@E-5h>N?Hua!N9CYb6f8vTFd>h3q5Jg-lCI6y%vu{Z_Uf z$MU{{^o~;nD_@m2|E{J)q;|BK7rx%`m``+OqZAqAVj-Dy+pD4-S3xK?($>wn5bi90CFAQ+ACd;&m6DQB8_o zjAq^=eUYc1o{#+p+ zn;K<)Pn*4u742P!;H^E3^Qu%2dM{2slouc$AN_3V^M7H_KY3H)#n7qd5_p~Za7zAj|s9{l)RdbV9e||_67`#Tu*c<8!I=zb@ z(MSvQ9;Wrkq6d)!9afh+G`!f$Ip!F<4ADdc*OY-y7BZMsau%y?EN6*hW4mOF%Q~bw z2==Z3^~?q<1GTeS>xGN-?CHZ7a#M4kDL zQxQr~1ZMzCSKFK5+32C%+C1kE#(2L=15AR!er7GKbp?Xd1qkkGipx5Q~FI-6zt< z*PTpeVI)Ngnnyaz5noIIgNZtb4bQdKG{Bs~&tf)?nM$a;7>r36djllw%hQxeCXeW^ z(i6@TEIuxD<2ulwLTt|&gZP%Ei+l!(%p5Yij6U(H#HMkqM8U$@OKB|5@vUiuY^d6X zW}fP3;Kps6051OEO(|JzmVU6SX(8q>*yf*x5QoxDK={PH^F?!VCzES_Qs>()_y|jg6LJlJWp;L zKM*g5DK7>W_*uv}{0WUB0>MHZ#oJZmO!b3MjEc}VhsLD~;E-qNNd?x7Q6~v zR=0$u>Zc2Xr}>x_5$-s#l!oz6I>W?lw;m9Ae{Tf9eMX;TI-Wf_mZ6sVrMnY#F}cDd z%CV*}fDsXUF7Vbw>PuDaGhu631+3|{xp<@Kl|%WxU+vuLlcrklMC!Aq+7n~I3cmQ! z`e3cA!XUEGdEPSu``&lZEKD1IKO(-VGvcnSc153m(i!8ohi`)N2n>U_BemYJ`uY>8B*Epj!oXRLV}XK}>D*^DHQ7?NY*&LJ9VSo`Ogi9J zGa;clWI8vIQqkngv2>xKd91K>?0`Sw;E&TMg&6dcd20|FcTsnUT7Yn{oI5V4@Ow~m zz#k~8TM!A9L7T!|colrC0P2WKZW7PNj_X4MfESbt<-soq*0LzShZ}fyUx!(xIIDwx zRHt^_GAWe0-Vm~bDZ(}XG%E+`XhKpPlMBo*5q_z$BGxYef8O!ToS8aT8pmjbPq)nV z%x*PF5ZuSHRJqJ!`5<4xC*xb2vC?7u1iljB_*iUGl6+yPyjn?F?GOF2_KW&gOkJ?w z3e^qc-te;zez`H$rsUCE0<@7PKGW?7sT1SPYWId|FJ8H`uEdNu4YJjre`8F*D}6Wh z|FQ`xf7yiphHIAkU&OYCn}w^ilY@o4larl?^M7&8YI;hzBIsX|i3UrLsx{QDKwCX< zy;a>yjfJ6!sz`NcVi+a!Fqk^VE^{6G53L?@Tif|j!3QZ0fk9QeUq8CWI;OmO-Hs+F zuZ4sHLA3{}LR2Qlyo+{d@?;`tpp6YB^BMoJt?&MHFY!JQwoa0nTSD+#Ku^4b{5SZVFwU9<~APYbaLO zu~Z)nS#dxI-5lmS-Bnw!(u15by(80LlC@|ynj{TzW)XcspC*}z0~8VRZq>#Z49G`I zgl|C#H&=}n-ajxfo{=pxPV(L*7g}gHET9b*s=cGV7VFa<;Htgjk>KyW@S!|z`lR1( zGSYkEl&@-bZ*d2WQ~hw3NpP=YNHF^XC{TMG$Gn+{b6pZn+5=<()>C!N^jncl0w6BJ zdHdnmSEGK5BlMeZD!v4t5m7ct7{k~$1Ie3GLFoHjAH*b?++s<|=yTF+^I&jT#zuMx z)MLhU+;LFk8bse|_{j+d*a=&cm2}M?*arjBPnfPgLwv)86D$6L zLJ0wPul7IenMvVAK$z^q5<^!)7aI|<&GGEbOr=E;UmGOIa}yO~EIr5xWU_(ol$&fa zR5E(2vB?S3EvJglTXdU#@qfDbCYs#82Yo^aZN6`{Ex#M)easBTe_J8utXu(fY1j|R z9o(sQbj$bKU{IjyhosYahY{63>}$9_+hWxB3j}VQkJ@2$D@vpeRSldU?&7I;qd2MF zSYmJ>zA(@N_iK}m*AMPIJG#Y&1KR)6`LJ83qg~`Do3v^B0>fU&wUx(qefuTgzFED{sJ65!iw{F2}1fQ3= ziFIP{kezQxmlx-!yo+sC4PEtG#K=5VM9YIN0z9~c4XTX?*4e@m;hFM!zVo>A`#566 z>f&3g94lJ{r)QJ5m7Xe3SLau_lOpL;A($wsjHR`;xTXgIiZ#o&vt~ zGR6KdU$FFbLfZCC3AEu$b`tj!9XgOGLSV=QPIYW zjI!hSP#?8pn0@ezuenOzoka8!8~jXTbiJ6+ZuItsWW03uzASFyn*zV2kIgPFR$Yzm zE<$cZlF>R8?Nr2_i?KiripBc+TGgJvG@vRTY2o?(_Di}D30!k&CT`>+7ry2!!iC*X z<@=U0_C#16=PN7bB39w+zPwDOHX}h20Ap);dx}kjXX0-QkRk=cr};GYsjSvyLZa-t zzHONWddi*)RDUH@RTAsGB_#&O+QJaaL+H<<9LLSE+nB@eGF1fALwjVOl8X_sdOYme z0lk!X=S(@25=TZHR7LlPp}fY~yNeThMIjD}pd9+q=j<_inh0$>mIzWVY+Z9p<{D^#0Xk+b_@eNSiR8;KzSZ#7lUsk~NGMcB8C2c=m2l5paHPq`q{S(kdA7Z1a zyfk2Y;w?^t`?@yC5Pz9&pzo}Hc#}mLgDmhKV|PJ3lKOY(Km@Fi2AV~CuET*YfUi}u zfInZnqDX(<#vaS<^fszuR=l)AbqG{}9{rnyx?PbZz3Pyu!eSJK`uwkJU!ORQXy4x83r!PNgOyD33}}L=>xX_93l6njNTuqL8J{l%*3FVn3MG4&Fv*`lBXZ z?=;kn6HTT^#SrPX-N)4EZiIZI!0ByXTWy;;J-Tht{jq1mjh`DSy7yGjHxIaY%*sTx zuy9#9CqE#qi>1misx=KRWm=qx4rk|}vd+LMY3M`ow8)}m$3Ggv&)Ri*ON+}<^P%T5 z_7JPVPfdM=Pv-oH<tecoE}(0O7|YZc*d8`Uv_M*3Rzv7$yZnJE6N_W=AQ3_BgU_TjA_T?a)U1csCmJ&YqMp-lJe`y6>N zt++Bi;ZMOD%%1c&-Q;bKsYg!SmS^#J@8UFY|G3!rtyaTFb!5@e(@l?1t(87ln8rG? z--$1)YC~vWnXiW3GXm`FNSyzu!m$qT=Eldf$sMl#PEfGmzQs^oUd=GIQfj(X=}dw+ zT*oa0*oS%@cLgvB&PKIQ=Ok?>x#c#dC#sQifgMwtAG^l3D9nIg(Zqi;D%807TtUUCL3_;kjyte#cAg?S%e4S2W>9^A(uy8Ss0Tc++ZTjJw1 z&Em2g!3lo@LlDyri(P^I8BPpn$RE7n*q9Q-c^>rfOMM6Pd5671I=ZBjAvpj8oIi$! zl0exNl(>NIiQpX~FRS9UgK|0l#s@#)p4?^?XAz}Gjb1?4Qe4?j&cL$C8u}n)?A@YC zfmbSM`Hl5pQFwv$CQBF=_$Sq zxsV?BHI5bGZTk?B6B&KLdIN-40S426X3j_|ceLla*M3}3gx3(_7MVY1++4mzhH#7# zD>2gTHy*%i$~}mqc#gK83288SKp@y3wz1L_e8fF$Rb}ex+`(h)j}%~Ld^3DUZkgez zOUNy^%>>HHE|-y$V@B}-M|_{h!vXpk01xaD%{l{oQ|~+^>rR*rv9iQen5t?{BHg|% zR`;S|KtUb!X<22RTBA4AAUM6#M?=w5VY-hEV)b`!y1^mPNEoy2K)a>OyA?Q~Q*&(O zRzQI~y_W=IPi?-OJX*&&8dvY0zWM2%yXdFI!D-n@6FsG)pEYdJbuA`g4yy;qrgR?G z8Mj7gv1oiWq)+_$GqqQ$(ZM@#|0j7})=#$S&hZwdoijFI4aCFLVI3tMH5fLreZ;KD zqA`)0l~D2tuIBYOy+LGw&hJ5OyE+@cnZ0L5+;yo2pIMdt@4$r^5Y!x7nHs{@>|W(MzJjATyWGNwZ^4j+EPU0RpAl-oTM@u{lx*i0^yyWPfHt6QwPvYpk9xFMWfBFt!+Gu6TlAmr zeQ#PX71vzN*_-xh&__N`IXv6`>CgV#eA_%e@7wjgkj8jlKzO~Ic6g$cT`^W{R{606 zCDP~+NVZ6DMO$jhL~#+!g*$T!XW63#(ngDn#Qwy71yj^gazS{e;3jGRM0HedGD@pt z?(ln3pCUA(ekqAvvnKy0G@?-|-dh=eS%4Civ&c}s%wF@0K5Bltaq^2Os1n6Z3%?-Q zAlC4goQ&vK6TpgtzkHVt*1!tBYt-`|5HLV1V7*#45Vb+GACuU+QB&hZ=N_flPy0TY zR^HIrdskB#<$aU;HY(K{a3(OQa$0<9qH(oa)lg@Uf>M5g2W0U5 zk!JSlhrw8quBx9A>RJ6}=;W&wt@2E$7J=9SVHsdC?K(L(KACb#z)@C$xXD8^!7|uv zZh$6fkq)aoD}^79VqdJ!Nz-8$IrU(_-&^cHBI;4 z^$B+1aPe|LG)C55LjP;jab{dTf$0~xbXS9!!QdcmDYLbL^jvxu2y*qnx2%jbL%rB z{aP85qBJe#(&O~Prk%IJARcdEypZ)vah%ZZ%;Zk{eW(U)Bx7VlzgOi8)x z`rh4l`@l_Ada7z&yUK>ZF;i6YLGwI*Sg#Fk#Qr0Jg&VLax(nNN$u-XJ5=MsP3|(lEdIOJ7|(x3iY;ea)5#BW*mDV%^=8qOeYO&gIdJVuLLN3cFaN=xZtFB=b zH{l)PZl_j^u+qx@89}gAQW7ofb+k)QwX=aegihossZq*+@PlCpb$rpp>Cbk9UJO<~ zDjlXQ_Ig#W0zdD3&*ei(FwlN#3b%FSR%&M^ywF@Fr>d~do@-kIS$e%wkIVfJ|Ohh=zc zF&Rnic^|>@R%v?@jO}a9;nY3Qrg_!xC=ZWUcYiA5R+|2nsM*$+c$TOs6pm!}Z}dfM zGeBhMGWw3$6KZXav^>YNA=r6Es>p<6HRYcZY)z{>yasbC81A*G-le8~QoV;rtKnkx z;+os8BvEe?0A6W*a#dOudsv3aWs?d% z0oNngyVMjavLjtjiG`!007#?62ClTqqU$@kIY`=x^$2e>iqIy1>o|@Tw@)P)B8_1$r#6>DB_5 zmaOaoE~^9TolgDgooKFuEFB#klSF%9-~d2~_|kQ0Y{Ek=HH5yq9s zDq#1S551c`kSiWPZbweN^A4kWiP#Qg6er1}HcKv{fxb1*BULboD0fwfaNM_<55>qM zETZ8TJDO4V)=aPp_eQjX%||Ud<>wkIzvDlpNjqW>I}W!-j7M^TNe5JIFh#-}zAV!$ICOju8Kx)N z0vLtzDdy*rQN!7r>Xz7rLw8J-(GzQlYYVH$WK#F`i_i^qVlzTNAh>gBWKV@XC$T-` z3|kj#iCquDhiO7NKum07i|<-NuVsX}Q}mIP$jBJDMfUiaWR3c|F_kWBMw0_Sr|6h4 zk`_r5=0&rCR^*tOy$A8K;@|NqwncjZ>Y-75vlpxq%Cl3EgH`}^^~=u zoll6xxY@a>0f%Ddpi;=cY}fyG!K2N-dEyXXmUP5u){4VnyS^T4?pjN@Ot4zjL(Puw z_U#wMH2Z#8Pts{olG5Dy0tZj;N@;fHheu>YKYQU=4Bk|wcD9MbA`3O4bj$hNRHwzb zSLcG0SLV%zywdbuwl(^E_!@&)TdXge4O{MRWk2RKOt@!8E{$BU-AH(@4{gxs=YAz9LIob|Hzto0}9cWoz6Tp2x0&xi#$ zHh$dwO&UCR1Ob2w00-2eG7d4=cN(Y>0R#$q8?||q@iTi+7-w-xR%uMr&StFIthC<# zvK(aPduwuNB}oJUV8+Zl)%cnfsHI%4`;x6XW^UF^e4s3Z@S<&EV8?56Wya;HNs0E> z`$0dgRdiUz9RO9Au3RmYq>K#G=X%*_dUbSJHP`lSfBaN8t-~@F>)BL1RT*9I851A3 z<-+Gb#_QRX>~av#Ni<#zLswtu-c6{jGHR>wflhKLzC4P@b%8&~u)fosoNjk4r#GvC zlU#UU9&0Hv;d%g72Wq?Ym<&&vtA3AB##L}=ZjiTR4hh7J)e>ei} zt*u+>h%MwN`%3}b4wYpV=QwbY!jwfIj#{me)TDOG`?tI!%l=AwL2G@9I~}?_dA5g6 zCKgK(;6Q0&P&K21Tx~k=o6jwV{dI_G+Ba*Zts|Tl6q1zeC?iYJTb{hel*x>^wb|2RkHkU$!+S4OU4ZOKPZjV>9OVsqNnv5jK8TRAE$A&^yRwK zj-MJ3Pl?)KA~fq#*K~W0l4$0=8GRx^9+?w z!QT8*-)w|S^B0)ZeY5gZPI2G(QtQf?DjuK(s^$rMA!C%P22vynZY4SuOE=wX2f8$R z)A}mzJi4WJnZ`!bHG1=$lwaxm!GOnRbR15F$nRC-M*H<*VfF|pQw(;tbSfp({>9^5 zw_M1-SJ9eGF~m(0dvp*P8uaA0Yw+EkP-SWqu zqal$hK8SmM7#Mrs0@OD+%_J%H*bMyZiWAZdsIBj#lkZ!l2c&IpLu(5^T0Ge5PHzR} zn;TXs$+IQ_&;O~u=Jz+XE0wbOy`=6>m9JVG} zJ~Kp1e5m?K3x@@>!D)piw^eMIHjD4RebtR`|IlckplP1;r21wTi8v((KqNqn%2CB< zifaQc&T}*M&0i|LW^LgdjIaX|o~I$`owHolRqeH_CFrqCUCleN130&vH}dK|^kC>) z-r2P~mApHotL4dRX$25lIcRh_*kJaxi^%ZN5-GAAMOxfB!6flLPY-p&QzL9TE%ho( zRwftE3sy5<*^)qYzKkL|rE>n@hyr;xPqncY6QJ8125!MWr`UCWuC~A#G1AqF1@V$kv>@NBvN&2ygy*{QvxolkRRb%Ui zsmKROR%{*g*WjUUod@@cS^4eF^}yQ1>;WlGwOli z+Y$(8I`0(^d|w>{eaf!_BBM;NpCoeem2>J}82*!em=}}ymoXk>QEfJ>G(3LNA2-46 z5PGvjr)Xh9>aSe>vEzM*>xp{tJyZox1ZRl}QjcvX2TEgNc^(_-hir@Es>NySoa1g^ zFow_twnHdx(j?Q_3q51t3XI7YlJ4_q&(0#)&a+RUy{IcBq?)eaWo*=H2UUVIqtp&lW9JTJiP&u zw8+4vo~_IJXZIJb_U^&=GI1nSD%e;P!c{kZALNCm5c%%oF+I3DrA63_@4)(v4(t~JiddILp7jmoy+>cD~ivwoctFfEL zP*#2Rx?_&bCpX26MBgp^4G>@h`Hxc(lnqyj!*t>9sOBcXN(hTwEDpn^X{x!!gPX?1 z*uM$}cYRwHXuf+gYTB}gDTcw{TXSOUU$S?8BeP&sc!Lc{{pEv}x#ELX>6*ipI1#>8 zKes$bHjiJ1OygZge_ak^Hz#k;=od1wZ=o71ba7oClBMq>Uk6hVq|ePPt)@FM5bW$I z;d2Or@wBjbTyZj|;+iHp%Bo!Vy(X3YM-}lasMItEV_QrP-Kk_J4C>)L&I3Xxj=E?| zsAF(IfVQ4w+dRRnJ>)}o^3_012YYgFWE)5TT=l2657*L8_u1KC>Y-R{7w^S`WJ+^JzHuu=JXOHcf zJ+^Jzwr%U1_nvcc-h2LE-KwN2RY@h4{5nr}uU@^@j9Y!eW&RrlxrFJsJ2m$YA_9Zz zQ+{`F1*shE`k2SQa*%|AUxq<=OnLWoUSKBL5S3upsND`EUdf$ctj1W+2<}WUDMj>z za+Wj!+79Vd*#&dxJZUUqcbZTV?^AN-WmS0xbO0L%qI4R5O0}%qTI}x2PsGXxa+rLb zKYys3#s6LbHFE*r;Z_2}f(Ghf&o{3Ff_C17?ImPaYYE29AL74)xG#-HDL8_6uXQ>t z@~fAb>IUp>$h{RVr7A|gHq!P0z4v0 z%ym-k&xgT`bxc8aG@QQ8JLHDtxJ#^AQj{B6HlOY)QN92>Yp?g>2yw}HnKR%z&!o!J zHh!g$kLAqd5xI!0YD~JB*)GzDO&A~Y5SQG(28+=@^q6#)oYAgF8xLiZn4{u z5&5*9C3yVeXSj;Dd*$ZdBVF{))4ZSiWr%r`q0kQfF);z){9>8>7_v z0j1pk4DxiF?YXMc*cWsCy%F;TrjqkXhU0rL6{CQePQ|dt?c<)^jtTc;eqPq{Y37vQ z!Um_nse-}h<3}bh9~QAVU@sm6G5*B{E;eAXO*bbm2f{-DETrR2VCD~%MZ)6BxjBQ0hNIhUE&Yg(gRm~8P(Q=b~wdqYdM7si)_YiR7roGf0Fvq{BME4Ic9H@(QIS)r; z%RcWbmq29@fmvY`Le5<>X=+=Exzppaq}#Q6=>}!cE@wE4#h6Nd$Dli&6tT&@F5;8? zxVcN^_n7Sila;d>GChi{eNm?wEuFB^Jg3wz8cbdJlX+zB zx9CrZ>SJN9B9UZ=FaO7_+(%ux`FAwPwl0C=uSq^YAx1(}!Jd!k&;hv{BGcsbz4Hy8 zAAdqWS4PS)5XeAJrgARBmwnmusufhCE2!DD5`eM&8L@-YID)LY{6+QrK*fs>g~zsMYM+SqIjBEBGjS|TiGPw=;q2{+3&Y~hUR zA)7V)Ccmb?+-Gj^*u*)$M13UF-MGt%#L2)J^BEp-?hE#lXnUXw@yT1>+~J*_pC0gW za9XNlp?hV{PbRT{v1DIPw$-jfl-t6-wMX`B*2m~WkLt@)hd7+v$$(Ds_d594?3ENF zK7RrH>(r^xjjmPYFZCgiA3yN^J;EmS%k;na_(Ab+zh>o-hq{u7D68lPZKYC>G9iUk zgMZPJ1{*;j;6a#>zEvcoS4x`aB1e6N`vhSQ^y9q)z2`?BHNqgO)&0);mP%mHzN7T{ z{CtJkhL?>O+cp7Awx#l0D<+i>_$j0v$|mX@Rvmqz~Evt!KhIoe+PR!9mH1N%>`fQA5Llt(k|4nHQGyjXsyr~nu0F2n&uCPUxg*ZZ$m zQ>}c!rb((Wi^}jC+UF-8X)T@tC=UsFkS?S?mCad5Ee&ARka@As48zq;F||JfIa>AE zD7!lYbGl&^lcF7tL6BY_5Yss)E695VJ|Y?S(2L?GBOC&O-h2w55Twc8__vAW;4EU5 zL=v|Q2Fs)p2<%h0?O{n^T+(vpnactMdY#ZI>Mxvx*|G1zW?sR|49&n0#bt{HHvD?OS)3GC(lC9T5tr-GLsiz%t{7vpmeNX z?>_!LBrWhQe%Ay1_@M&y;|JTn4@o(FM>Bp02V-jkD`R_Nsb7ZrRzlyKGWO;MPLAfk z{z-RCRM3>f`ljSgnrtjMmf1Blu4>l1g<77i?rKW%BLWlD2chD5l1s%A$h5Ajqf zN%Y8F=kj*rDRVIf&lbabE~h%Y(KsxRb)otEXdftJAJ?k@hm)1QAIF~ZYQL8!eYR#E zj#0{{+d2-eNE{P&~wT|4Zo|4Pt5tChPcpb7%yvEZ6Xq+a^2`H6HEDB z_RbBjBA*$TEi$GcUy0GrRoGvMa4#80=bYHhp0uKxL~{37GWYI*0{14sPmyP^s6Rte z<;UlZ=iz=5@P|q{MWctLok&eMQAR z+xp}czZkndEqmwjRou9>qAi&dO8UZoHQ|xQdY5@Mp5FBJId%30Xbbxlxx*DHm{2(+ z*DVqmN6`m^k)XbGdb2^^Nk*ota^r=4R zub4A>hn&sH&D+Y}-NMOS-@^N0)XL^tJHw8L(?Olz^EKF8aSGX~?6-OjKp9=>_O;N6 zz1D_(@`J&EoUM_K_hVQ|*uZNE5s2m#S`^7pbyWh3a38B-5<^V7Z~!S`Oj^>3j>2>n zww4Nf8u>wqv*T)gWa{W)nm+BRrLf>T)U)vhi!qK>@H$L<@mrCkGr?Zl=z8sg{Yo{X zLu(uTU@=RH`P@v+zW37Zg;|g7(_KxPmGK#ck4gQ~guuX}u#4k0bq8#FnC*_0R}~5f z^+dg6F@EbG&cR3;A(1<#vj`mLl72cXpTbc*Es$B|x^g|1m&5ap05(k{or>iFs@6LG zFznY^LF)E;E>G7vTkH-!sWgy2JCyquiRc=g8fh2K__a6Ihd#@-N+r{^20IW)L#~>k z)A_|#dD!PLwk#;pMHUuG)~FKdrE2V$!`}xr`M+UEBq#mH-!dM+hN=4|eomOefvX>D z)kZuJc2-I68UNAdj}#e7C5ZX>3|KK!@)1KE4S*^P@30vrrSj5+i5iB0$#+%i1GAGK zpu!~mo^vl<*9RBTl@Uak>+E+VF~5VmFmZupsqV&$+X?o5K-?DrL`Yhg&eXjGTfm zFdm!`ShSDw&v}g?kKC>DHxuo-K}}Veo?FhWQmbq+KYyun$y1^@L{b@%wyLH>`lDRg z4AI3T(*IZ%nbNy;?E>TSEfG^HI8!$N-p{mb_HIm*K?qlYvjYt=+ zy_jY6Y2aU&NS7%z;J!(@L7397DKC~CrDw8agMQYI0M|7!HqtlJb7;Y}IlnO2fq5p; zSbl19z+M$cv^zRVh3>8C!a+`PB4=Yx&>Uczj%foWIQE&TA9G6&3We9FHm1_vRnc@? z-PB|0Q6g{q`gM=dMP}6TWRM#CK#zcdJ9 zM2z<%l(_D5GVGfbS*uX->S}0e*GIDvmpl{E5fH<%H-e>Ew=`fBVJkL5P*m&OWtk;q zqWPAHi-#P1BOpx6A^rGXi-XhNn(c2r#LVKQb99bacVvV2!wAFlS=lj5SMTC@kf`|w z$kkCPjCdt)wQ1&VjMs1b1P`kSY`neGjtrE^9VM92vaC~*X!=P$JONjDyu5-JiD$Y& zwg|tQ?(V&L!FVm}gmQaX&~cUta}j9*2w z%Joa|qlLj1;8O*`bId|C!Oppr!@4t=uor}l3W8v&8Ym%qqHK;KY)W?Z!IKd?Z>>X* zCxcHX?z3`xaz zp<1-@_+(Ib8|H z&d4wq+6};a?73nhxyD9v+3ZWYwf^2OaAiZ5O}mXN-T~O>va-Gf1=+m1&d9(H%We$; zvv*wPW)%9x-Nx2|NSL>Y`N{!S>S{~Y6wxp74$Zjm6>Rzft@v5#wH{@MEed9MX;k~Y zlKj&Ps`H8d6WpCXB<5NO>?L&wuqLChJ(PqtEtcaI;it#(+CB-~Zyrf&il$1(cPkX2 zn1@Y%wvKq4tBTzX7@b`mCRql>x%v#Ey}GQgK%d6TTqwK;uHt#M9_6axmlOEs+7fDK z_u(PI76S2}?m`|8>{X0YC~nn(KA0FX3=DM16g#uXg81dV`psbp*$qp$EPZS%J2pS% zDg!1R!W&xk9T)NKFOn=I5z@IEBb0zD{Iu4H#!N@9g9?tq3Gt#obh*dT-FS`?j;@FbPTT0$-HIITOie{iW#ms5aW(?% z(GDgt&4PwNO$Aypl6p#HViZ6U@Iswaf(+7-V29liae!YBuNu18rl$eFU?bK40Hrcmdi&e|a4b6!=r%ozk83IZ08a-1HDd z{d&pKQ;{K5Xv^KU262Eq^fK!$K$B;u5vw5|kj7K`DehX1Fy>l>K&6(ro3y_F2hEaa zeXvcToowI@@s*$Ga$682&ELtdaaqI4&HZz7cea;s;9hDUHOe7<)r&e|c3g=3a5*>? z9EwR=-DGe^%2Zg=*van|qK_%V60ownJKWb}bTy~JZIbT6%-K@ADY@YxfhIuRj=CXl zB{%~u%7)C`2srrYCnti$@~Vggob{RpN5xw1vd!R36RLFt($F+xU7C2)1oA3UtKta? z8yh~e>Sl5ZC@7X6%h(KewJUCktskCDDRS!EGU zAH#{m#+(a=SRK*|^@igpyN24<{2r{l)1a_lble9~w2fsnNjz^NbMw4mE6V3cq_ zO2kQskQT_n{+4q{ab<3bH3_X#)h0LN!9MmL1i};0I3s`)%d6jqpWR^tEkrASSinYH|A8`tWtb%rQYTsZ+3_y|EQjpPi}WmXp(&(l3WAI?8hE{FG$H(k^A z`_xQ+S4r2LtV9@J#|`CFtWxF>H%KXM>4P_=;*Bcux*KayBAN2u7&m5)m+lb11TPRK zHje@^%#A{C>FS&pcGOOE@?1#~Oz`cp4GADL`YE4?=+ZGPqB7yYZu}$GI z_$EiV;|)P&Ec8rO{e}ZNAkh%O1`(aw=IOt36XP}Z+B_EZRiOSs=gS`rgWgLgul2Jt zAYE*`NpQa6qK}z1CE!ieH5eDKS5WZdogjf(>ckf3g;7lw~2C8e|nNln8;D)ygtm*1IyCZM!^~`-O;EYX*u1+W;!j6OxAdXuJDeQ>`A$pFwzA4oh^Bbn-uC zdR!7$8$1DYiylREJnnS=wHFwN(GiLL1}a{@`+@(*cIH19-UNTyn3$V7+3WvzD;O1T zEsMktKlHVBv>3qS@0*uLctMbnv&{$rr%bO5jUwhLSZSL?bP&C+&3vP1PDpyEm;G^}_4YP3rTgRXnmj}@Wkio90y`4=(vEj%f{XR3#jSfn05igz z%V_%1n)mu#g|%8cM8De3%$osb2r{x_;-LsSX!AAvL=(EOxX6&hI$xZ*i2A96F#sqy zcT?%EJ408^$~gvoR`-0*3^68ebDnXnCV(XPn~*;7Tg~aIBFk9bFrmD(2HR&575;%d#j|Czya? zM(7N0f={X&X5`;Xa=U-Vqk;iogg4n9vUL+HIdpeL&ZbwP=m0)Lekg?Agdq<+U*yt) z>mqj&d$QjH>1AY}(`7o7PYuVM@pj)UoCDi+B(U)_L@MfMe4>uh#^Z>@S%E-=+b2-y zCFIdZ%Be(v%9})T^`U5?B%|-UQJ46L9-ggKC%|V!#GCHgX(8>BoJQZ+c)bFrIwWYN zWa3Xu*!J4alaOAEL2ZrN;;#CH58P4# zDn5$uP>^~BqyUc}FY&t^x;6*6B_DKT6hB7%t^iI<-dBo(Ux8uRfkaFhCN7RYNxW_r ztbmx$LgIHlw1TSt@i(3UT`QBebfa@1HBbA#%VmL6f{ zC}k+yOy&aa1t;38^#mQV?nCu@GtCg(xzbl}JYT=PGu_(CRSa_P?~a}}+f$#?_a??Q zJ8rYlbU~|ezF>E1;Bn#hCKyhyg}`M;!FMyDA!KhRH3eKP(SJehTrgw}avCvhV_-zs z(FD4Ts)aki5WmpiZcg-hJa2orx#Br&;SGYh@=S5!?JtD%x+WdL-Cf7hW$nEH)@2_p zi1t0BPvITyAnAL?9m(EYpTP4V4Vtd_PSrdg8K3u~E%!&XzY|PUQ2^^{M>^)G)}N%j{GHV#=f48i+g&3iE)X8jgE(LiX{sJ z^T$0nSd>KQRi?CPVKO5v`&3HvPgXVuzP@-swcQenSOYXgsSZ|23L3hQylbxR2EDa&JB2XEc6cK(#YHcbBHS=_x+Iub2 z(G9XYEGh-jd+e1hGs#mCvN_^!*7j}uYeFEkS1|hmyK(7C#-iJx5|m@*T{E`}RBBv_ zMr&-5pmcn&xVwx6##yz^tXWDOqVqs$&sE^EgUVRKF}fc}Yt&Em#(LQ)OQ6D3hzV>J zvgJjw>{xk+AtgoA)fJ;IoDO8g+CBiO1vU*Ixv8^7AViiWwFA6T>LFq=$MThUU~W@J zjh-7eJ?S&#D1g`+2}7zuYD$X5Q=m;&ra2lrV}Oo0SA_bY^e9M6G6~0mEIvej zS=rAIh@7?-gRpi{AlDxVg!{CH>|R1{#ne16rWfR){d3K}#HUD%kwtY=Ce?IU{k7 z(Cs2lQ(h5V6GON6^-G&!-@i$Oq~BuSy0(v?Nu@Q-pQ%&2^dmgYjrNAFeA`$n|5MA( zwK$>a9%G_{4nlnBj~NIThxnin>#v_S(Izm2cflwNlL{wRg=r;vFfz7+kUN}^oe@_x zy;q8BEn}zh*O=`pJqYa@J@WUIt|=2Z11bJ^+aU#HXA}!G4aF4A(O8g>_+Q@rX{E#p zrOXxELqBgI8Phw)@9QOg$+R-%Z-tn_^qm8FGwcQn8!yEKh2HWrv`ZL*NcTs4!ViU#lfD`aTgA<_*4II+`1w^|xt zWzz>SK4lQ(&G1tkO*Hl{I|H}CUkQgbodX;4-_u;y8&v4_q6A`ZG znB(l=Qk$HvGSL!jcpa0C0OYmG%TnbS1I!)+o{H?n3L68X!edkCcScTU_+k~zpsoiZ zM}>QVS0PQqpxXoB35Z?C-M9s251oQAMT+cqOR90Tl4o^77DkdRsj08aiV~sDpp!)c zuTJmGBs#AD;EisiIl&(RF*Bvn5h2>4CTTZTt>(&V_ZQ=`1CfSc*ae&<gclnrV(|U*Y2gXfgzGmUDE8o408lRjaehjc-09O)T$A&N4CPZnEgR*r+!?JCGQ28rs`nJn>9P4efHb|6=`1K5TPj5`TGhSI_KE&_U{@(inu}oE_W!TIO#`XXZSnE3o1Uk zKlw!Sf+}CP3)5VA1!~6i*-7w;E^B35AW;3H;{_xGi5&<*2#M2+>KIb3<>UZuez4t~ zKn>e%=fr#OxC%hZB`&-Q2)~*g!(_6J#4?{ zKQy-g1!Pc>k4{NQ(@-=@(@IEr!*i`>5msEeRf3(T&an<5*xVgdW8%hKOaelna4BrzCfHRf&B;dx5FrbWWCflCBE z)H0arWRt2r<}luboToO%xZL)L(PYey7c3S*f<0T?80udsK5I#{!2NSL>WP|u+h5;O zr+d6-3ydDQ<2WG^qnsk>jNPx1+}wyx$E(Iox3!aXx@O3>?1UqWB*ee+T+f^(ZxqZC zkFsK~I@|)iRY0ovn}3Z5ncn5Bj8~`nV6D3#-rH>*JnpoVCj!DMa(B~yoEp@Ig!gO-iE0z!lKgroH`ph`ps$x=A69^pi}HIuu%I8R zut9t#K>-T}O&Y}9@|+l>ciM<_Qi|>!VoQ6>MRzS(UQ1Fn`vd0_)+t+D42g6$fkZvS z;W5kW<#E&WDwX%^^8)V2RX)KEA`j|KSYU+M-9dDq@_J%*ut&ywLiVNP@OR6dO~e`L zWCd-Ar0Mx$0Iw@?O~4uo)|b+)nz4L179CpE@>v-gLWoNbUBIMWr;7d_d(0A0ZIPf9 zJX8Ls4C}#ypBaxl2-419J-=9~5k+y&F@j?GEp5Q|TTkpjXhlf^g;~DbEX=W|R=Uva zSE`6Kv$b@CN|c52jO6-xX)Ye3B6B?Sp7Ddwv00soW$+{&LYN6$f*^^!{JlM)X?mIt z>5O>HvG#&WeYl1}%H_>CWtzs=uH4M@Q@#BL@$k;Dc{R>-Bk~-f78e2#^V2xplZ9Ix zi$>*SoDCt7dCjsEKpzeq=8{o~Ak~VL$i^aNm{Va=WL92@J##>KH-l0^(dkU1LP?or zR9cBf5){QURXUG#7Q==SnUU`5(1l;8 zwK^S%9B4YM-+Vb6)CCXBD*5s*8J+z`qxLZI;F>w-Zy{R@jJgzro2W>aShSmpNHY8= z_Rj*}yii1+yzu3C`N2+b=|O<3@Z#ZOfn@z05tsMI$SXc+2nb3u29Q<|BNO3S9!;%|JZrez5JG%*}H`^8D23**M( ziCsFloTF>rC?+IsRAR zFdfWhQ{@kuJxF|TPp0NKE6eOX&XM+jm!ug?@i+4>OsIF*txBIBKi(#)Y0`ac;Ke;y z(8WEWa9B_m9d{Uk9H($!nYk;5-u+mj6mtzrLR(q=S5snE2_$lX)krwFxZTY!!YiQq zBg6Ho00OXC`d|!}N*qCxX9P>f=I(32PR-r|cx*emR%~z(?_Sr8;Tprpx0*Y~KcTlF zfUy4Y0_6BycGn_jGrV1c*|A_<5wDPi$O%z&#s;yq*8~Ry($CHiD~7!TL}~;=5ur1* zGRM7Yz07$ak#iw>fLARy2WvM6Cl0qjtY>bujY2otzn1(QlENGUCIRhic8OShng(b0 zsl>^$0!V6yttFspo}r}Jz_~f|8x_XxB-1;S7LaY)-bTCr70Ng!g1Hs_CT~c7=gfbT zFaO7p#BXovWc_W%@+|>sZ2R9(U1IEn1Q0!PknAgCenX>%HPvbFWxX=kQlfvTKV5Tm z;hQ7opV(9(2F6p%7Ru&p08esyaY+$ZRvB=^TZztQGg3O$CbJ(TW;1~$CgU~666C71%h(!VpI{%y(hZHghmY; zn}wj8v@;(Zv*z07E~WT&&OgGVNy=E94q#OtO6bdGU(*WN$PKj_q01Od zH;ysfI@&HKZ;)HEtGPGof9ZqO)q;#?_KlZ>!&utQIWO`2t)?Oa7K6t4zAC2QSLJ(^ z^6y2@|F|lDt6rkyr6v3L;JxM+2j{Cw$)*UIAVsRADa7QF0U;qan@(D-#93=M5bV;K;fe09HXFaKwxS`e3G(v;;8vV~OXcj4+d}FF?Ras2SBO5u$_IVY@yeW_jrU z38H06FIbmVIO(G2K8lxTNvCIqC|qr+JHshp>8#8g3_%uNQ$;ZdQ!qR3_8_|lwd=Cr zD$i6%IN;ckWoURsBWam&htS%pR0|xtm`twT?hlNeHgwN>l4+y@^T@VNb(bRZPwpiSX-g?Iq-zlbV-Rf+%O z_m%x0p`Q6IZu^%%T>|=8jW8l~{|+v`uOZSpDqw;fd7vgfG2d(f!F1koQFO5Z#>(OB z+s7@E>xt%=B%WDOpm^%ZeOSokz3hER{YP~9aIJB&6d6(`cNurv)}^<{KJVw}1M3gk zy*2VieTe}q`Fj0QAWixWKa6&YLiLws+&*lZep{qpBSUN7)RY`U9bpw=n()a}3W7qH zf`sH*e@}FT_2_Nw5+)+Ggi?Pl%Mnre0UVS@zy-=Am@+wqX=Z25t};_fd@@4I>M^`7y(;!ciS}Uxe_iKo(X-7_7b>yJi`2a$=iaYhF6!#LLc$lcx zJqkC5B&5P}Yc@UP{(#Gp@vuDV+E%92nG3)M$UG4O&D0_pn}jpq%%%znyFqeVa<*mg z)!Mt%_KG8^*pW05lYR}Yd8iipe0S5K^0T!2CjM<{AT^5 z5b6wB*i}C#znOKN@!b{WHZo_81W%O=RxRcY7#}(S*mBWxwZ@DV#qE04P;EMqHJ!n@ zG$7m$Ju84{e05wJj&vOsn(85DItgSd;Po`@@Hx|2BH9tvH6R_1qX*IcxkkQhKxHLk zGu`asmC865vX!G!cRNO-nd4uP+09rgw-rS;%czCnBjYR}n^~3UCNg56jWjX}zY1+NpH0iq$hC)NFQX@|X`386tIe4Y62| zES}1OXtFG}qO)hVw?5DIuI%Q@=jCjsVvkS2<*;H^n2&YoGTIB zT(5FK!slnjpG$#SlXUgD*N{Upp<8iqRTR-$g$w8~q%fl^S2MHU>Mq9CDl3nKwqTQm z*Ik&Ng&L>z6H}aY*AC=4sp^q+4X!<(z!A}?gR+FTv7D|E&$N5-evLqgZXQ*hb1XIQ z=b0vppdEam7kK)nZOuf(FGZ9T$tg@tv%Dc+$iho}L^8|5SDMe?GY~@3Tb*G_nQiaU<=X*k1-Q`9EqF$)JI>&0da z(CI(hSH|MSJKaFQ!eWYT!3BMZtcUaoc?Hz>G#0QQZru}S>D^0A5_e=)%VVc;IYBXs zLKRwuAT*q62Vvx#>@!-IY%ZQ8Itv zF2>8GEWIS?N6Qgeb10l4XhO^%LOJ>lG4hP#*x-W{rE#dQjgRny=`xZP(eHA+%t@

i| z%T)<(DrJc1>wW;MNR}y;M~6yB{yhXKDv>S?J88p`<;lWue;~<$7r+fd+b$ElDFfmp zuxZm|(^5f-+7N1B^6OoL|3gT(8asxym{_)bkETNKpcK>hX4TG+6 z%%ATBdi;I=+oa}i2fduW{kKf)e@gWPMe_e;ttb3t)}R69e9#(dDL5sE3@qG()bCtO zZ4M~@U`xa08-l2))oROg$BSpOdG_H7I1C>GE+`auY-Q89ZC#O4JuJN@p?zsNL1vD# z=0tQA_su*Nz)(Fq?cP{OATS9mtVt{`|A`VIu&{gNmWaR?>Y`CMk?0tWLvRu+Ag&#@ zSGbc$RPZGxe##EyX?hH@1sLfGitds98ubqIK%MIOH>5KJipH)3e%)+Bo4KB-@6rIiq34E3w+UMLjO zz3waN8u44BvF+stgcx&j4O;D5vq$G{7%VT_Nm1CcS{S%q>>IUYMG^v%XvPbj9o9%_ z*S`Uv&rEN3GW#Ob2X@@i4kROK1EBphCh0>lyvBwp<0-3Bq8ZWwvTz~1Gvc=e%X~!< zN$E-SG+<~Uv+BNYXtNZB@yj_78|=&zUrt zs*&!}_(xuqYV}GHI_nfgXQG|$;ZyarmyBN+?c+f6n7iAPhi5v}p>N;_YvPP$ReEky zS_v|krw;`>$YtkK%mZ!J-1?BCF_hyGSSN`eY=lEh^pPzl!gpZCh!CR>rFBB&!xz*w zl+-_a`xQ|3nd(&Q+3)q`GyD3AUkx_)55chWOmiKWUFzCJPa8I5yx8fMD$50S^X`%F zIlIORRDGQ>@G{j{W{%g4A18#CC~Hek3s!#%`7t?QGelEN{ynO;l%m;tf!@G4tYPI* z$cg|&v+WO>oS5yD0g#Gfb~J(IQH!o1dS56ZGIF4a``uIG5#_ukE<*Rv-X%UV19Si% zivf7Dj^tzQYf%koBwRF;us>Y8f8#;u!tu=J|5e6={V!96e}4k~$G`F)Wv9bG{*4uh z*0OWoOB`QKSZBweSmdEoQ2u;S3AuTp^zxqIBSJ`yVeRxTmN*NQ%r3$=M9CW$AMrK!d3xXg*jq*>D;s)2g?!blNfvB5)YH$=GJ;+jp#elS(A$ zIMoEE73+I-t}}@!YCnuKZr)vL(LCslbvKd%)0BxI@HsNpix~O^IP_G|dg#`u=Hymp z9B+Xei5-DKNSeR_>Z648Nfp$^V<8Ef7FT>g z8I-OZXD20opU9c3t?p<|cKF{cU+;HJAlFeRDtQVM-?{MgO9{?@(< z&Y&KierF=jZzB<||KIlYpP5L&*yNY}x2MRzO-0skinmYby2<`#^WR;)^Wa(Q#VdME1xl1d&mOMEX$a=!{7CMz&k*CXCmMs|R<-VEvz_8i`5+3NB?DrCJM$ z>UAoLQ5zXHW=+avmFgG*w5P!~wDje&?tQwVY=;{xS|%3h{G(}Yn0*-f%NFwzX-=Zl z$|H!Qsm2Yh6&kH6tWj|}WAHjNm+483e>9!irpcMT7|5}LbJbT$HL5Iu)9;8eE>1&b zFv;=w+Ct~tP=opB$d^lvkMLGn&22p=>Gq>H)auRRt1?H{fgZq^m6f9;O7%2b?RFW$!OG)Q@hbX;;ZONoi18F9TVCILaCBUSSngXiVwu2nXlXX?}O zQdmsO{uG!qE=WZkq1+*~nG_}+JVTm;?g@AwPsTMfPT%7Mp_CvrNZlztie-smo3?!d z*-1X_OJpqr>wvcxq~TSezM#v^M>Uc4?m5wMD>|zQ+w(_f@t_pw(4kbPPHu4L=3o^} zKKs^Qb{maasD7|GO?nkWo)QG5G(wp$X1|4 zo!BJjzG~@Ml?JM7Du9xqQ1>V9=bg6eCZ|q`N&~FR3f1n%9jNj0-qQ95+;dmIbVffF z;e8I|9A_j*KwkUYFkE)=j7`SD~wm9sUELqARwsO z(0j7k%9nXr@C!kjL37Md1Rb~5m>z@U`g_hvB$Buv3`GTII+ZX5wWI2CIP02=R zcw+Tdw`Q&iv3UnkYUdo4F55oTOgehBS;#(m$x}vC$B`MyNSZyvBM&V*PpyHF`KsT} zrYG#4SwH1Zhe&R;boerK1}IJuuzg)=ObKxNpqho7=^nDhc|4^*SmWOD{uP+qPi89Q z_|BV)-xCy(|H~O7sPAAbZsTBV<6!RiZBV56yVGo}|IvKd4QU6>I0@!LD7O?SbU9XFbnHQH&!R ztVoc7e)!ArOl}90$@B9kJl#$}v+aK0=s3Sf4h7e|=pqhS<>vDI()>U9lfP}mRfDaA zg<9+3!qp{4`TS^n;~Xg>jU19)tVlJYFcP zLPzheLJLu%QExXjl4!LQcAvrURslmz7&Q*lX!6$^gO6c0l)sRQ1*O@!UWB58(UbL% z7Ut`uTNyyret`3p)8Kpi2)Xe_Pz|rlh!-0%Vi%JM*%{O+kdV2?zZYxin1KG6h?Fn5 znT(gVTO;R7fUJ|q=Hp7O_jwWDRuv!0Xx*_R-UHb$tg;rI-gk+(KPC@4+#$OCKHW&! z2T`Y1nNTUCnFNUVRB#RDcF*ee4)h5O80HzwEH;f2(aNRdbmc>R8JGRn=;TE+`x^SL z=t903fZYF==#;eiHgEq&Rrimar|78fX#9`*ZbQw|75M3V{^f?z%@smS_OeHSTER>rl|72xv$3C)WQooN;oj~eh*cRvY4f%bWw>b!@= zJlU^Dw^uH&*RAXdZc`KIZ++P6Fy6PL^zU`J^-hPk$;*MSEFS!LEUJRXnzivmGjJ`MB^n0&@Z@357b^WgPz}nyCdSjlS+3xiScjp3 z@qsJJLAlmd=BLiG0uI<42xb>`=dp_jnh|98i)y`Q7d3-}OpKeRDX-oW&W>%Q={_NR zEmi#6r(@NxTteCi>7uB5H%k3==wXH9cFd~Dw&BfQNTBEh(+ca0KiyfJv?L3jlM=l` z8tf{V4=}?PdHU>5tOk7P4J>R%Nqd-~qFr9!0!^apL7y%S>@JIU=IgaT4?97mI_BtL znk2UcyzFj`MRJ>uA~@aMCauts!5`G@ZWmFcX0tIl3)aBu1tEHcUdvOG(C4iJo&Xs7 zHxbob;?1Q~I+fXH)^*l>d&~DVR~XtZV&_wAS^?Wm@A?+1*OeeFG2CK3?EuS`pVBTK z&qTEpsHauBtZ>Ri99?1#$2D~{wrtB>Fm|O#9Umo9lWPGR7RsuSkW^M&u+~$*vudwb5wg^nqzaxfEUL@YS$VY^4CqDkH7KBT+tpCD^*(@ zXY%E+7>Z+oCVzft2qqdEMUHr9ys#7g=O*Bx2?!3=50#3=1r8^R^;w*SdaZ?p%X#Gq zr8$f(fe$;Ly(b)w@}c3{t!;6ZD+&-inIz*Z&D?AB$%3`B}4_GhfUuQQKt+6}= z#Lt1U=FDJ3V9v$V;u58I&lnMYcwH{;qR^p1{b2c^)VT^Wt@pWT4RHmb{_6FWnNI{s zZ3K4S45-b>(7ph$%#bPmMIIWRhfm{13!@41^nxi(z6JVJ!IU-KsVL5&hN|O*1&Ygu zBB&N?YVOQrqT_=6;&|=&C5*svb?#PguF^p5R0tl?<&XX>QLdkME8}2{+0bqck-FOQ z+dD|uiJs52i(XSzt4cekBwdZNntkPwn*EFc1m$A4p8H)J2NQb0bT9v+7C3_Lsd_sc zG?b@Zu{FXLZEvZW8H_PPj}<+vI_fU)A=62pv|x|-Gs;p0LfD%NlWltAaMUPwYQvs_ zZ>LK_V96`SmL5W$?`-=w&TiN9LM$41O#}=cBaM=^wZ1VL>&1M$R*ty7B7q_CLGLnK zmSbEZNIvQFtBaG~C(ZS=@940JGHTR+^_YdiA&2Kw>hMKPp&i*yGujyvcd{RYZ9$om z!@#DeZ1n8edvrx3(GMS01Mf>OQ)PKeir8c!8^mYNXs1cJ6+2;<9DMRlg@Ehaj?Vi@ zi**mv4aF5+C6TQ|NlL{?U#XTN5buK8vQgsf z=h%8R410HG3KMAw3~pnuv-w6%Zj;qaLdpeGDW<^4}BssZrGfZwI(}w#gjGJ93LitkG+ej>PT)eR*WKj(d`z59g z)pp%V8#CsVPoJ<^@6_2YZ!Pe$CLA&GuDr(rWOJ-N4LN~TP$Jr?myz6(jm{91HALEH ze2vP@@34~d9i)=ra5=qV4CogE%Go?!>#8v#2dG!=p&?j0Ao21e$4{S{+d> z)u32&P^z!bWrE8#r|+MM-=@KLIwFGwL#q;L=asaGS9+n;h0f6v08$5>fsytkyQnq? z#B5kxU(lwv;Vm;z(i)Rs?b;7R& zF{b6y*keic#*rRz$c^2m&Q z!4XzUn4ypUVm>C_W^v*OHol3|?}{H{S(~Y0a~G~l^J`UcPtgcfp7s($_(qaav8_A> zmf-axX#{^9#b5{l%r$D4U@acMRSZFukrH{jfN6cJ%Hr%%zWZWM%z9N#*NBW2);oAO zqGM>kNgP)L_6UL^-tVSdM4=8j=nu?)VWinPn z4K#uDb;XQrM06O@aV7#5j{FYZS96d4B(pTO=#&$Tt243<&hS&1_=X=zW16xAYmDua z~4ZT}60~MTRnd=r&Tn66(T#_noy|pa&8b z8hxrF7z=ZBy*ZF1OiZBU_US5Eweo(K*vF(D7WLqAh4g;Z_zGsI7Ni78~TS5xz|9 z2eu|sz@tJTav1oD&ptLduLBA>V^1vW<#1__uvl#dfXGNb=e!v}qsR5O27~M+Nw5p6 z72;#tMz`kQ49A|TN6tXy=HZu*7<;Od`+R%|t#?=)H03UY7Y{6>OX$5sFjTQx@w(!% z)ojO8_kun9Yf3BoNBl`hD4H04f_Nu~>7%BvTtAZpty z>=OWQKoQ^#_^mnezfIp+*Us>7bL3K`BUvQC!mUoL@yMwXCDU^aTo0iU8H%Mp9}1Cy z7&d8|xx=gONFA-N>D%$_C$TfghfR1H;c#MJZ$Mm_Mx6R&lE_B-=;&~weV+5TB*R+47mj0LOs=BC`^<_EX4HrdfFmU1ZwulGRM!K%89-XN)>)7YZlizpRVHP*!!^qQOR;{NI zhj%+VY3>B$yOw;tf86cl;$6v8cGAc)vQqo*!pO6$R+vE)P#y6_b(|rXiPK77u_r5n zgt}ODqB4XfFyQTWxN$2*E%o~Cwla%26U;TVR1Fsl6WJy=Hy&of%8?}8LQRjtXe7Zi zopIp??rU_?E)_1WRqf^aZ5&u9YKu7xFxQr+wQxF@fJK^fx*^5A+TrkQhN=Z8&Ty~}qt2I;cv@o`Y6jHHvZf3^-zH_rOHsU+4Ya>jAd`r)+u{|(|BO2Xq<2-~8aTvkhI_Hux zO}jquyaN|M1S`9$%r2Aws|{w?*q@|vatQUY0-0N6*{tWE#os3Oct1*oz&V6nM)930 zMkwtKWEJQ}iwNr8jrCubg~TEy?-~FmUkWgJw%=J6{$cVjy%e97%mEI6bWhp233*QR z&8%VQU6K?7`%$XK;(;?7>+I@u@rm2czV-%0~$sgIQB%o;Wi6KmW&)z zy3@javfUhiH8=Aq9Z1rJiYS}|!|#E?+Z7U;QJ8u#M+Ou)@UrrG`vos1IIBdJQk~WOeW0-|FJefC@sM%< zAr5%lLG1Fk%5@B%0|s)GK8BVm%bQk-;O(LVmg+!b?1en#I-2kbnJ$hkU0(Dw>kqfd zyyR?!E8ob+?>lMW1f_UzGnM=E7xScGrq00TPDvWgr#6(o?nl|`nbfW`SF5k7$qKENCR1pF?&3JQxTU+#Jr->@wh``K{elj zmDG}aq%$7@tH$d7W#cAqQ^UtmmhOn^LrKT(&m%nUnTpZxLq;@PpX!X75OMP1Af zn-6umF%ZLFzyv1<+N=+KdYHfMk~R}9uyXXKR)!w7T{p^iRswv{wY03pRT?8G!W%`$Tu^r|a>Tp7}?O@t>CCCyX z)4|vtb|Ef4OE2Dr6bsFfm>*~o1iVm_2JSb>sbr#TdS^b zY*D~yvU8yh6sSgnIyKL>ls*r;i(|=eD-egBR&)UcF7F#0bu}*gGnFtXJ_X5ytDo^Z z_vBVfQM7Ji&qLZL2+Rrvtee~^(IaaEzL4A@w6M31nDOX?F=D#pGK38zA3A9d;{)`K z2~~I+LH!LFjIO*oZY6yDzQ!7OJo~^S?}&oj+(6VT>jCji6E68&Z1; z?uPYzZR-go>J;Y=SFVhUE6sm^HG>~C+_lghy^JEGe&b0hta}Ce*P&2s!ssv>FchW$ zj@dE&{!sXb+xFkWPzr!=KA_*H;A>-Rv}KD9Jbf0%pXdfZ%?xwSqY_*Mxv}3_;j%yG*%|#;n%-9h8(;CuGGa;f^P&XQ z0_`ajCli8lbqQc$4NZ$Csq<`9(zGUR-gmtYWWP>^X{h0Oiqe2{PM$T|U9_@K)NMBp zs@;kHqSxe9KS-}}$TOErVaY&jrY%HoFlV7sa#H8y{~UM1F6i`qf9dN+E6pZ(B82mi zx4`OKSS~|y_wB~cat>|?kRx^TwAJb)UTgNwBCcAcb9I_yR)bKsC3ye$?BQgu67wM5 z&kHQBr_Z^D-i4t`J^JSfmT#K7^aBOXp-sB-rWYlN98UQ%E19BVK%sRoz?^;10ujh; ztmZ#eKa1YKhmx`WaPO(rT)jQs=SDd!6&#_9&S{4p^(`ub8b(jM(8Q%gAA<@8X*oCj zWKmY=hBHk^sSj3~p&}&WAYt+}Hq(w`AEx*D4vWhz3zu;?g^%gOkO+rWb~4T$oZxX# z2N&0pA^L%RL+X&Ja!+8TwFh8P^>CAX5ze1>{3IDc^75V36v;$mMEv2V=tZ zwx%qFEqM#jRTzDU!9uF`X{*KU) z!x|MAdW6<`9lkyyE!?b?Idu{Xq;WE_=wP+6#hsRcs;w1cI*QFg1N83{%G_7XaK)c< z*=_on)X(=jzoNBHI?b8-i&5%`pQQN@+FuL4ZwL>W<3?zO;7KP?bJW^X!A1aywn=6g zvz~{2kIgw*#x+Q4p->;xI1jxJJ~_6WQaG$LOB5P1x?|qAp*SC5gXILSc6^CkguE&8 zRWiu~e$C7+An6UZtEV{o)m>1fUFrC7M`cOSwzgcRnQwdWsmGWK>3 zvM_$J75*}Rz`o)YT*o6XgqAJoN5~wEcXO^x77sha4{;?^)PZNojXm^>&!i@_*n6y< z*}J*pHX|3I9gI9Iq2D%8d8A)!vc;9?R#`+dGX~RX`g-99==;yUI@+Sh5tnlUst>o_ zUC+95@LMGk_0-9C@kuyC%{zk=wQxIAF<qw#>Mc6qyY)~G1!?uvF)2tUCj0VXw z-b%?W;{|mp4|C37aLfMf2IRXtA_;GRQrbs|QpXS{NK;Eh1%v^dC4z9I1|v@g+2fS5O39qtu~Dohn=)Uc`N*l>#u`14T^~`IKZ--0Gn@&zcYCM zZN2tcVfBab=#wl3GPHgBk|Hw_8#X=bzB?1T3~^FIq$Q*gyjv50S7WS({UXgB-|a>y zDen#VjTpw5l%5QOreF#`)1KvrP;q>S-Egh(wjN zi>x_+#THvZdajOfk`gDLJzVXu`?5RoJ6<=*WgYwnq)cv0xfCOZZvp;Gm2WePKSTx3 zCqCon7IU^j2*tx|Ec1t_L?H^TI)b(CIQX8a_GgwwZYkwYF8X(>y6-hv6z=XSY=K5s zXrH8oO0C}rMxv_9-WOLg0I?o8`bMaRr(7E7f9!MJJs)^kt>g{HnFI{ES`+2V+Q*x)8v3v%YaFl z1Q@_5E2a4tQVdOYjVLa0zRhXSCo>F-B1X4&FJK<~pxfZU>#YTm3%!pJn>$RZ967Nx z;!+qU_n|iFACcIQitEiOP2Bp9oPNQQ&YYHkn9mcwS!WY(h(WE z;2!O-X0@d^L$(euD=Wax8Q<@im6DbDKkS>eC=I>)F+boLAl7B%hj?=q5KKPs24X#v zFqkkmR|#1?ph{vY@lxUvb&uhJNo#9w)jTOy2iBJfFB)03{ zR*o01Q(8TaN46eM>P~>RY&8U6HlaA_Cj^R9=wmv!dOBi#O^1bTSwhTV?7nWM;r3t) zJs>y_H8zm~!|cCaoLx2yjUW1usH@jw8=kWMJu7zyDlSpONs`10O+{Lxd_#19?Hq>S z7!zjTv+)DynA#Gnoq3x10vJvYbdYM`diF4{TxCQ$eiY~wYl{dNk4H)+hk#p;@hnE? zkZe@Q0V+lD=gGWd-fziqwAx$9^);hf3Wt6=^KNF*;;-cncWTckJ?pmZXku*; z6Vg@4mDAU;GFMzk_xgA_HpzK8yLY^sBP26+)^dI~?zQq16PofR!amwDNY zDKH84l@J*n>WP&b?fV_&fUC#w-kMi4l~fGEc%5)}s)3Qnu$fBls{5~}Nxmb9XL&GJ zK2}pr&`P(y*9VWRuH^BrKE&-@xWV1R;f#zVO!k##dO~2l2MO>HWxMy~y+X;~l`clq z0Wt>iBB3>SlGLQQrIMEp&N8;8t>=`|Hjr4Kt8pVF>}RD>-KHq%ty@%B=u>C{`iZwulwCIpq-Ff{f`|#8yxn(# zY(mv}x$dv!jnRlo%KPGUwBVXpIrw{_39NBAd_apF?`FX9_!LG8l~B4q^Y5UGzE0H_ zzvm29>OBebXGo_BmHpm@{81m(O!Yy3bc0#R^&>Z;o`sPuPsziJQ&j=E)o8|uKtR2e z|0`(akJ0##Npz|jw2R_QjW*RedrZu0;wT_LZbJA0{b(RT?^8x$#aIw}h`=BhaoK2} z0qKN9Ao+rt`+!pxy^-zMSiylx*vc<}~y$}t~l;-6&k4z@BC zIFEED3qPuDVy8NoYH?y5&VKFEPMl@FGEGVDWz2#zT{u$augwBux79jM-#h z%EP_3m&pN&K6DE)T*|RX@5(l@diy(Me+bmAB9tHHI+p_P7h$;?D<|CaF8eKoj5Ezt zRQsCVa|iXoa~ACk+i=+-mrU83X7OND^Jd}v^ByQE$HuotsOJrsbNddJ^qRf)?wVxE z9CAi+_a^z`9PfG2cHIfeBUeN)-=~Njxa591V6lokrbK91=rb2Sk#b)mZ<{l7FO*e* z*mTsyZ@JtE@xB1YeE)5e^y?g0s=90T1?#QL7u6lR)Vfm?&gABqzL6}*hl#8&J*B)> zF#}HFe$w3rB@jWSCR+VrJtgQ<2}-GFI>bxppTN2-9it*-nap~LX{6^ z7qpC~2O~qd`-jlmJ;NQ$8B#0FE*DUWF>9HpXX#d}8l8?7w&R)UZ&j?AoRgHa&U6YW z&1%$|ij|XXO;EJ^nJM+Cno76^^c683TfRajE%oYX%!fIPRCaAAehEEHCtzAqHe^z* zXGF9tHVaLnAt)~5KrWFyv^1o*_&P|d37iIMM2`Gb32(`=hIqKA8>`|qOWHc!FmkPG zs(kTR#a3@*a(JwTD!(X7mTF$iZhF;p1{pz$qPR4)utW_ZRO(|bWEk*GsRT`u+=GNA z$0$@OR_GecM$TIGi5fu&c`Bk2Ba>7N*uj(T46YSie@q2lUF%w7VRs41a(_ueIzG3^ zfhh{9gm0fMuq}EfE#yPIq>}h;`pt1~AUohJfKrE?*)#^cyK%q8=g60OlU8-De`Jd7 zg!EzI63%|8l9W4-+0<5hPrcBVH5rj1?Yq4a+4CUM?q>L^jV9#pdZg)qLL4raJ zjOnM^%{Wmy5u)G0kw_F881()_zRqYa!xvcDi+s1d+Al~5R>mlZtHB2N7rWxsJt3qq z{kSMKK-Q&$C<9Rs%r(kj`dNkMO*63b^QM7~_|y6UoAS3UOID$bQLagtaY|8T^&lD9 zf&Kg`LOA>E{RXz4cIeDROtFD>X>Uq=W@mJdjgpcq1^P}?<7zUeB552J0;d=@>pvWi z_r+{-Ue}7hL8vVtw)D^5#Me|iLxc8#U+msaL>pA{t8S5a4!vq$ze!WUffsR&~bLFarpY7 zjIL%Rybo`AuYulxv$|wrnD>#|ZgB0ALpz%`FRod&PG$t{G30GzM)(OFM)hL8H$25% zp@QKfN-s_<3gW)PK2TTl2=BdzX^ktNa%t;G-#&nS!d?YR8H%9sv)+0g>=y#%$2Z0X zP|c%YZrMjwW7|Y8D42J--5A*hTkQmgY5m-$b87h@;%B|X1!O`ZrJoBK((~C6Y;^!U zV3*eKMX$F23h^^Hp@w06C$;d zc($gluhqy8`T8Xq!G7_^s+axf*hZS(2kYtEmbPRJJl(Ze8mDM8ibZ;UD3#J7_gw)) zqcB`_#EOHF*%Q+Tft+3Ibz@lGUOCex{c*X5xwTETA%*n5)oHV_e4FZHdX?WM z1`8e{cP`4{>|(>YJD^v8oAPMWfqcUppS!;1@hw~Ic5e^o9o=K|UPMo6gaAC`m2I<4 zM#&gwfsz{EhjAJ&zs7IInW%$2+MuN&L>Q6AQq%W^IpasKk%>rL4(%Tw>R~uQ-9Z|k#cR{XSvg*YwS+>2* z@N1mFQRWkkkL5&Jy~Q!QR&Hn!+ZUlVBnj0NBiWq0`d8PFOjJ&lOGzzxjTgXL*ske$>M_22xDw0D zQrJp(UyXX?FJTJG7f}@Y5t+{eOKdcnwE`$$BGDk(ggYg`zflctvBrgZma!HNk2;0$ z(nyADrB)QB*{a{RnR)t))Q{pcW;EbJDJ6(jF+^J1x$VOrGdn9DrXKB8BU zZ8y{AAz5bv51jYmJ)vbdyjN5M?a+W6PcW|z_UI^(QW)Lv=ebKW%h0Paj|^n6bGpKo zyYJh5<#Nr9-g0FTX_gi5Ga3actDmKxIR_w@i2x3cdCTKa_KL0*qgd54S{Trk*P+ok ztZAliu~CL6Ka4;mOzQRc$`_M3!Xts~cKXkT*(lYW%ZYyoY6}@xtM*F@?w8ukbrGc# zvpVU%5SeMidA5yGdGsryiFMiioa@uh#y~ zQ;NP6S{@N+LJ^*0zsGw4lbmv#4<3wM431R78Y4C#OMQ^5SuvITf#L+cVVostaJhwf z1N{tWQ&5~#J@5p$qBtAUAq|nf*C_f8eoNCXLGZj7{8T)Hv3Y8Cf!)yHr>TBi%uB_P zQkA_r%5WvC5G7h3!B2Wm5O-BacXTv(gYUug~F6Y6UaPZvK*BcZ(%r(6S*H`1rZw_B|IEcL6yWXNZ&iqo` z!fJ>>ZUj;KZSLV_yH&81rXvfwqCk;`Q);4-2l8FTrCT#aGZLb2P0 zMoSiL0##i~CGw-!qA8W50+W^ujRTUJc%`zi8WxY#(FG#tUQ>0rCs@%nxiH>4@YF2& zPZG&d6`%Liopk+-3Gga6OXKxr5^D^9u;u(HZhku0B@%9zUVYncs7eS9_h0kO^e zmg55$5B%qIu8M93iA!u}M)(Ogkh}HS_BIhK9I-gr?f3F{JmAUKWDKJ6Jn2!Ttlcf% zPmmuNd^qhH2r5i5UyENI->ZF+<+%2iRx-J=kj`Bsc4|X3GLhX-6L-I2_3LUb#P&3 zCm0f@MpS!rk1xFsuDadU^u6J01`g0!Qv5{t+C-41VPOqex)L9}*{(6HHMR+7qz4vA zTOBtMF=8*IV$JsNh*-__T`U~Q~Ydlt07dFR_g1XNFE620E0zLS!*Blsv+rIJ`L1`c*Rc9;xEw5nhYH<0#zX|9vs$2KCq=nul8|x)`GU-X$d6WaZ-xE znil`vk*9awjfX7yl-Q+?lRqiEv>ZCO$G&`9dBr+=sV{Oth5CG}q-)5v?j+QlPW#c= z<$;I8N2y(7Lj0`^lUSZmCv8Zu+-2fDlIp1tZeO{XK2ysY@O1(yTAbz_rW%7()yz+` z#oLlMi9Ve}OLT7)Vt`Gi8^d|35wO(Xh#wmx)xHB6U(|GIXrYh;bB*EZn9v63>ie(9 z2Iu=Zlj9Z2;mBqh!0)2I5?>})55x+{{ z&nr0YE={JEUkCJYb>00#^3J&Pt>zu>+{AT5_{zH01?{RN->REAYEP5VxA1iJroIqKG%QYIl&#HWL=D z;0r6FR)>e6cZHHn5=k5;mONf@ljl1?N*c3t^hW0{L_M24Ik~iSLm%&U;!QHawWj4% z+7pdh+$3(zw`erKRF!UjAp+-JB9S3LgnbaK#V=+BGRCLA6a&6OYvAaAW6sUESezKQX@x?`vG z3U|!;aMD!^j5f5(*G>e!6fl;$mr=g!#B75}KV)Xgqmz_`V#Hptygx0$t zlf+p^JlTNmdp*^ch;xnU9B09iZ*n%Tb3zm;n?7qF^CP}mfri1Rn_&iC<(n|MW5RzF zN|N*PvE%<1cp3(WUP{( z8j5>|ryJb!EY;?$dN@csnTqz@A#XQ&O8kuXIHAh`FT{9wnLk}xpg!^@oX^kc;w`f3 zF|VN29Fw9lI+yI3G5JMx5dm0;zu8tl_r+5Qu}5$TY8bYNR(<(LJ+!Rla4rqD1%Pu+ z$PHzI$K=|#&v;zB^r7DGs==`l^~e!D-QS6CVJxc!_gut;FX#%anyt}(7ieM3?}@aU z&Cd#F`~t`NnD0d>x`vBYLM*_^+lTN=SJl7#QwINzIRBx+_wW5)m9Jh{Dkx8)i~W~V%1fyCo3g`u0IGT4h0wh}h#P)O#4a*@Wd6a61GB&9OP19Edglj1y> zLVa?WAxZh-*lx~7v82Z;ZT zk(~jzzUd2PY)x3Jq$3%Rh&OQO@UcR-br)%VAF+vY=BZ@TOe*Wi^09oqO4U;f$X%%S zz_vMxAHFrQJK05Q*IkOcl?K;(;3mTV$mr{=OtzhY>ujw3TD3ZE z)Hq`?8thD&dXj%k_z+UgLa`D5kWiJKB5h3$^sR3JaN$XuEvDfJF;NmCUIDV(~H8IcY9>>4;@ z$#RI@7OYy1+Co?z?Onw}-UbGmVO{W)@(ayhK)V)XE<3#mAl z9^LL|pk-3`{XwJft1J9KVh85Q3e7TkTXuf0i{s;u)nOop`clRsy&JV^R5w=^@82TE zN+|)9jKVu}3Zb+7V(=UomCqs#RSF=Ei=7(Gq;}0XZE}j@p_0T)OH?BElmpZ!IikL1 zoB~rhI1DxVJk?Scd{*Z<>Sr~-m`I)lohfdZvdWLb$HiicGbd}Ras=hljgwqx?y6&N zX)!>}uO=zox;7T8sPflb3z7lGgQ&&8zH5LLp2=GL0A7Q_a=#nc5M8Du_h|5SuDU@2 zXmMJwLx4Lwu4sYu;)`as)bTay{v3oCC;#>F_Oq9NalKHDQF*@UEJ()GTz83D&9@r; zMw8PB(woOlQ_!F@R!A+fy?S-EwSX)gp!;NAn^UT0Yj|>Y|JR7eYR{ZjYWr3AsvTmd z@)#;8&3?{??kXMEryihu?eHW9$KTkPYFU(#A0YVR&X8EUMUM?16g$RF?IFQiY}r%x z3b&ZT(W08>Tr@tmfVC^^&{ajE46nudqCEJjjFBq%Ig9XSuf^Y>1c{dWQUG#$0Np;a zC>uVAc91dTuhre)h`BC@>Et8Nyc;RbR{6&ADJZo}E^#GI(z=)aNLrw&eHZ#(q?fk8 zK5vav8KpTWANc{dcw~!}fs7yp#WBh%O+KZo(YdBl5i=?8cuu zAw^`9-uWZ3uy5+YSXwQZ(D7K9B`g-KxiU#O`_#QnEk?5|*|^_4=#;wRM+W5aS~*Yp z{=1Q^xS=F@RXq1_Ka2BX5?*V}+M>{1XOM1u2n+b69GGVE-FY+za`Iut&G}7NS?YC{ zOnPlfc^fG__)`@#GB07VNK_nnNI%VhQ-ZRuyH9ucb3v>nKW+$!Z+XKA2D`kOz3B~( zo1>dRDV(t~8#EwR`Tg}`^$x+d92sOQ1N1h)q94uQ5=-yQRmgNIrQ}3LpQ>1-H-szi zSHp^dSrN7bx)H;OHD#q~i;73+-P=%K^Ac)Rmi1#g`P({ekG3fvH#?_}`ZILyydrmZ zV%oxaC|wmOjNuz`X54aP_;^nrs#Q^mMYrME><^<&Dv`}?oAT*yOKmsls<&c4)g0)bZ%8|Z7|pAbYkAqSyWk4JOlHY6ldu-&5t-F=n) zWxX1SuE-ol1MI`3S_4wQ*dq{-;xJHl*S~r8E;EMep!Z0p37Ia#~14g zCD(V2C|v1RrjIZr+D_UJC|Uji5hg;MO>DwD>iOqhU89S;iBcz_>@!PrP*ZAhVktu3ROj_P-pC?==U^*nc&dQQttradTBgw+ez!LG1 z3m07`$Gxm7fOsPsWq7H%8rjY$b^lHa7-dxeRHRRU9%xyJxf$fxYRynIR%<2E#Dkfk z;@El+Qe8TQGJi5Uordtn$%$)9+%5;QZwFZ&N3#2C=uL_7J?z-kTQ5teyUUo;V!TGi zia^caMGE;k&5NLJkjI@5yH*T5T?Dt(%dQc1?dK`?bT`LG8{{7I6nP+SZOYO@twTZ) z*@QwQ;Pz);f8D^1No_`H25jKs15Uh9|9u1ZZ{PJl4)p!;bq`n2_}2lK+B%ve!dy*c z0dllVn!ymX=C{Ql3z*i9djy^qWJ1`_!f;uPCJFS<7n$%Xq^Q54j zs_lBKDrciL1RM&{ZP>zAMIf(g=qh35yMLjI{{aovZXr~cp7zi>lu@H+yziF*YN7HE z5fx$EjJd>;orv0M0?hB{72jyo9KI97+Eoj_1mL|xpMi& zZc)|h?*?}5lg^pvjlXB?+rCt4n$S_!dS}VZqpP7vYieVyfccg_fi5Lp7+3a#Dtm6kOF-y->2w^m=f55Cn-6*poUC6vk5HF!lGq6vxkOA9Q3C4 zeP4q~dVgKe*ZF2D*g;291DJPFGd4J1ph#tl%aTdi64Wje`sP+<>&HZby_r?&8j`i>8tS{n&lLp4k46L@2UD$pwrJ#k4x|#RT;aIG|Lh=Us-xux^*qp8+^J>G+~OfU znLIG=m*q1+JL&8#ivU9?>t@DL8c*>~G}ehf&jljaC9HpzAnQHC{b6?bIOQqsBm7V3RRPK9-`7VZ1Gpc3BTK|j9t z7B2j2Z3hS5=jRvazCs09EZhO-TA=~wS~-957T8&v8R|Ryj;?QNw3J*YKXSkrD2T8- z-`lRN!*&o%B*#r6`7o+1V)Sbvt~dtEAeh&X&yp&nv=6VI$TyLT80LjHsgL(kI94y5 z@~ltj%7LybHTY4nTrIv;eiow$I>L5>_=u*F(#v+y;VZS>uQ;W`yNSD_pLR*+q;Hl;Vv%OSA`s6a0wS;} zgWi8E*lyvwoDJqACk*2DS@75MlAH;@f<(h9E1eH<_-=HdO(}Isr46a_e%C1One9VW5@%BPDZ!|g4GB#}Vh{rE& zSO^MK;R}*I4s;%B{;TXzBzMRm2F*|F7wY@AU^E^=Q}1^rfs@iihCc3^0VgGjP(c>l zI+GP%z6e%{0pd#h(Wei9(T(HpeflyL+n?4hFBCXaqlYBB_>lw0=8G+BYG=)6M3z_t zk%YSg&>~UM-qF3?^Gw2>iXuiLof2G;RPlwzYY##sGksGi(5;rjbUyYxlG4!Z)!h23 z{gp*LK72T#1#+gE{|K-JN`?r&*C03P7^K0%T_k_)P@j0lf-&xj^fE$-8>e0DyA%6R zP9aKFX4&qNlnU>5`E=;TYET?56LmNya9#X~7NjLH0t_&%;KkAZrzjEUL%PY$9oK_6;4B7I$Jt7<(}-N-5IZKQSB3~4Jsq?D;)ZxmHs2C_mf z+hUD`K@~HAM1XU|GO)Yf_NgHIY`&7TEHm+}D(%H%<`6hCb1AKvsDLe&?>9)r&Sc0Y}gOS(JYJr4&5`1Oxy=0C4>*=zv>2M^g&}8aqRMLsJ`v zKg-)o(NK;KkDXJE$Vk#uu}m<50hY($5JrPfegL@uAi$a!@b@cVWFSE8{saxMynhvd zeA|m6BcdokBOxnF_wq5-PEz_G2dNYR*N>n2v;0tv{lCX#1Y{*dMHCciWkg>h{CMI& z#DK$oe=13Uduu!6zj6NfFaLQ0pzrgi(h9i$@x;I7`TvRPM?3s1dw|_DFZ@zegQ2EkaMuG);0K#VBkx@L2FKXctA7p1M18C7eG#cU*w+v0pAT5R{=){ z6M)vTss2ytl9vpTX9%P4KN2ki5-$+^g&`TxKL5b*$8_u^+_Ws+awY&~5O6X41#S=R zAK?J?HMTRfx0ePa^ft8mPa1`n@Sef+2-<+A+yXk%7Do{YcZfz$pP&&u9G*TUkz*>Ea!13xj~O}zpPCis7< z9S{%}%Rk}$x^}s)^o`1Z4gvzK9RNM@r{W?0OEhU~yOF zUgk6Z$$}~Kzgd3W3@`J({=^gojN-rO^p{hQzhr@ZS>u;k7k{FYs{IoE-$we29E>la zUnaf#2@S0IPtbo&f%g*iW%ih#sPKjW{qujlqyLyo<|W_{fFD-&qx{Gh^Rrk10RPm! zKSI!6KKwF!%+H5Y|NiiQ5_tUgx!_Cqml;8R!jqf)t#1E;|DAQjOQM&m{y&LEEdECH zr~3aFjsKVMFXicf!s}c86a0&*@=Ms4s_Z{uyR82S_Rn61mzXaFfPZ3^IQ|pnA4h2a z+sOD*YWF8A-ClCOlbkmnsV{sb0pj|D?i-{%tD2_+s;C4ZfEoFT;d?l2Cm9ZIVCU*FR~dykvP9 zkNT5^H2$|){v4h9lHg@D;7nG1l$KQBfPCNf(vH#;U{?hOAlcu2S z|E6^R%?tCNI{(M#@@J>X51-4=ati?aZyuPpQlNl!(2v+fMxgfqf6Ke>AAkKnZ|KqV diff --git a/test-apps/gradle/wrapper/gradle-wrapper.properties b/test-apps/gradle/wrapper/gradle-wrapper.properties deleted file mode 100644 index f9fe699b1e2..00000000000 --- a/test-apps/gradle/wrapper/gradle-wrapper.properties +++ /dev/null @@ -1,22 +0,0 @@ -# -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -#Mon May 21 15:18:27 PDT 2018 -distributionBase=GRADLE_USER_HOME -distributionPath=wrapper/dists -zipStoreBase=GRADLE_USER_HOME -zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-4.10.2-all.zip diff --git a/test-apps/gradlew b/test-apps/gradlew deleted file mode 100755 index cccdd3d517f..00000000000 --- a/test-apps/gradlew +++ /dev/null @@ -1,172 +0,0 @@ -#!/usr/bin/env sh - -############################################################################## -## -## Gradle start up script for UN*X -## -############################################################################## - -# Attempt to set APP_HOME -# Resolve links: $0 may be a link -PRG="$0" -# Need this for relative symlinks. -while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi -done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >/dev/null -APP_HOME="`pwd -P`" -cd "$SAVED" >/dev/null - -APP_NAME="Gradle" -APP_BASE_NAME=`basename "$0"` - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS="" - -# Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD="maximum" - -warn () { - echo "$*" -} - -die () { - echo - echo "$*" - echo - exit 1 -} - -# OS specific support (must be 'true' or 'false'). -cygwin=false -msys=false -darwin=false -nonstop=false -case "`uname`" in - CYGWIN* ) - cygwin=true - ;; - Darwin* ) - darwin=true - ;; - MINGW* ) - msys=true - ;; - NONSTOP* ) - nonstop=true - ;; -esac - -CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar - -# Determine the Java command to use to start the JVM. -if [ -n "$JAVA_HOME" ] ; then - if [ -x "$JAVA_HOME/jre/sh/java" ] ; then - # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" - else - JAVACMD="$JAVA_HOME/bin/java" - fi - if [ ! -x "$JAVACMD" ] ; then - die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." - fi -else - JAVACMD="java" - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." -fi - -# Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then - MAX_FD_LIMIT=`ulimit -H -n` - if [ $? -eq 0 ] ; then - if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then - MAX_FD="$MAX_FD_LIMIT" - fi - ulimit -n $MAX_FD - if [ $? -ne 0 ] ; then - warn "Could not set maximum file descriptor limit: $MAX_FD" - fi - else - warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" - fi -fi - -# For Darwin, add options to specify how the application appears in the dock -if $darwin; then - GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" -fi - -# For Cygwin, switch paths to Windows format before running java -if $cygwin ; then - APP_HOME=`cygpath --path --mixed "$APP_HOME"` - CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` - JAVACMD=`cygpath --unix "$JAVACMD"` - - # We build the pattern for arguments to be converted via cygpath - ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` - SEP="" - for dir in $ROOTDIRSRAW ; do - ROOTDIRS="$ROOTDIRS$SEP$dir" - SEP="|" - done - OURCYGPATTERN="(^($ROOTDIRS))" - # Add a user-defined pattern to the cygpath arguments - if [ "$GRADLE_CYGPATTERN" != "" ] ; then - OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" - fi - # Now convert the arguments - kludge to limit ourselves to /bin/sh - i=0 - for arg in "$@" ; do - CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` - CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option - - if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition - eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` - else - eval `echo args$i`="\"$arg\"" - fi - i=$((i+1)) - done - case $i in - (0) set -- ;; - (1) set -- "$args0" ;; - (2) set -- "$args0" "$args1" ;; - (3) set -- "$args0" "$args1" "$args2" ;; - (4) set -- "$args0" "$args1" "$args2" "$args3" ;; - (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; - esac -fi - -# Escape application args -save () { - for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done - echo " " -} -APP_ARGS=$(save "$@") - -# Collect all arguments for the java command, following the shell quoting and substitution rules -eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" - -# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong -if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then - cd "$(dirname "$0")" -fi - -exec "$JAVACMD" "$@" diff --git a/test-apps/gradlew b/test-apps/gradlew new file mode 120000 index 00000000000..502f5a2d3ec --- /dev/null +++ b/test-apps/gradlew @@ -0,0 +1 @@ +../gradlew \ No newline at end of file diff --git a/test-apps/gradlew.bat b/test-apps/gradlew.bat deleted file mode 100644 index e95643d6a2c..00000000000 --- a/test-apps/gradlew.bat +++ /dev/null @@ -1,84 +0,0 @@ -@if "%DEBUG%" == "" @echo off -@rem ########################################################################## -@rem -@rem Gradle startup script for Windows -@rem -@rem ########################################################################## - -@rem Set local scope for the variables with windows NT shell -if "%OS%"=="Windows_NT" setlocal - -set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. -set APP_BASE_NAME=%~n0 -set APP_HOME=%DIRNAME% - -@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS= - -@rem Find java.exe -if defined JAVA_HOME goto findJavaFromJavaHome - -set JAVA_EXE=java.exe -%JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto init - -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:findJavaFromJavaHome -set JAVA_HOME=%JAVA_HOME:"=% -set JAVA_EXE=%JAVA_HOME%/bin/java.exe - -if exist "%JAVA_EXE%" goto init - -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:init -@rem Get command-line arguments, handling Windows variants - -if not "%OS%" == "Windows_NT" goto win9xME_args - -:win9xME_args -@rem Slurp the command line arguments. -set CMD_LINE_ARGS= -set _SKIP=2 - -:win9xME_args_slurp -if "x%~1" == "x" goto execute - -set CMD_LINE_ARGS=%* - -:execute -@rem Setup the command line - -set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar - -@rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% - -:end -@rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd - -:fail -rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of -rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 - -:mainEnd -if "%OS%"=="Windows_NT" endlocal - -:omega From 2857342be9c3235d1fdbd9f7d00d7b8ff1055072 Mon Sep 17 00:00:00 2001 From: Vladimir Kryachko Date: Wed, 12 Dec 2018 14:46:51 -0500 Subject: [PATCH 19/25] Port copyright check to python. (#162) * Port copyright check to python. * Add copyright to test file. --- ci/fireci/fireciplugins/copyright.py | 107 +++++++++++++++++++++++++++ ci/fireci/tests/copyright_test.py | 70 ++++++++++++++++++ ci/fireci/tests/fileutil.py | 2 +- ci/fireci/tests/integ_tests.py | 62 ++++++++++++++++ 4 files changed, 240 insertions(+), 1 deletion(-) create mode 100644 ci/fireci/fireciplugins/copyright.py create mode 100644 ci/fireci/tests/copyright_test.py diff --git a/ci/fireci/fireciplugins/copyright.py b/ci/fireci/fireciplugins/copyright.py new file mode 100644 index 00000000000..473c1f7319e --- /dev/null +++ b/ci/fireci/fireciplugins/copyright.py @@ -0,0 +1,107 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import fnmatch +import click +import contextlib +import os +import re + +from fireci import ci_command + + +@click.option( + '--ignore-path', + '-i', + default=(), + multiple=True, + type=str, + help='Unix path pattern to ignore when searching for matching files. ' + 'Multiple values allowed.', +) +@click.option( + '--include-extension', + '-e', + default=(), + multiple=True, + type=str, + help='File extensions to scan for copyright violation. ' + 'Multiple values allowed.', + required=True, +) +@click.option( + '--expected-regex', + '-r', + default='.*Copyright [0-9]{4} Google LLC', + type=str, + help='Regex expected to be present in the file.', +) +@click.argument( + 'dir_to_scan', + type=click.Path(exists=True, file_okay=False), + default='.', + nargs=1, +) +@ci_command() +def copyright_check(dir_to_scan, ignore_path, include_extension, + expected_regex): + """Checks matching files' content for copyright information.""" + expression = re.compile(expected_regex) + failed_files = [] + with chdir(dir_to_scan): + for x in walk('.', ignore_path, include_extension): + with open(x) as f: + if not match_any(f, lambda line: expression.match(line)): + failed_files.append(x) + + if failed_files: + raise click.ClickException( + "The following files do not have valid copyright information:\n{}" + .format('\n'.join(failed_files))) + + +@contextlib.contextmanager +def chdir(directory): + original_dir = os.getcwd() + os.chdir(directory) + try: + yield + finally: + os.chdir(original_dir) + + +def match_any(iterable, predicate): + """Returns True if at least one item in the iterable matches the predicate.""" + for x in iterable: + if predicate(x): + return True + return False + + +def walk(dir_to_scan, ignore_paths, extensions_to_include): + """Recursively walk the provided directory and yield matching paths.""" + for root, dirs, filenames in os.walk(dir_to_scan): + dirs[:] = ( + x for x in dirs if not matches(os.path.join(root, x), ignore_paths)) + + for f in filenames: + filename = os.path.join(root, f) + if os.path.splitext(f)[1][1:] in extensions_to_include and not matches( + filename, ignore_paths): + yield os.path.normpath(filename) + + +def matches(path, paths): + path = os.path.normpath(path) + return match_any(paths, lambda p: fnmatch.fnmatch(path, p)) diff --git a/ci/fireci/tests/copyright_test.py b/ci/fireci/tests/copyright_test.py new file mode 100644 index 00000000000..d2fe4df7132 --- /dev/null +++ b/ci/fireci/tests/copyright_test.py @@ -0,0 +1,70 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from fireciplugins.copyright import ( + match_any, + matches, + walk, +) +from .fileutil import ( + Artifact, + create_artifacts, + in_tempdir, +) + + +class CopyrightCheckTest(unittest.TestCase): + + def test_match_any(self): + test_data = ( + ((1, 2, 3), lambda x: x == 2, True), + ((1, 2, 3), lambda x: x == 5, False), + ((), lambda x: x == 1, False), + ) + for iterable, predicate, expected_result in test_data: + with self.subTest(): + self.assertEqual(match_any(iterable, predicate), expected_result) + + def test_matches(self): + test_data = ( + ('file.py', '*.py', True), + ('file.xml', '*.py', False), + ('hello/file.py', '*.py', True), + ('hello/file.xml', 'hello/**', True), + ('some/file.xml', 'hello/**', False), + ) + + for path, path_to_match, expected_result in test_data: + pass + with self.subTest("'{}' matches '{}' must be {}".format( + path, path_to_match, expected_result)): + self.assertEqual(matches(path, [path_to_match]), expected_result) + + @in_tempdir + def test_walk_in_empty_dir(self): + paths = walk('.', [], ['py', 'xml']) + self.assertTrue(len(list(paths)) == 0) + + @in_tempdir + def test_walk_should_filter_out_non_matching_files(self): + create_artifacts( + Artifact('hello/world/foo.py'), Artifact('dir1/subdir2/file.py'), + Artifact('hello/world.py'), Artifact('dir1/subdir2/file.py'), + Artifact('dir1/subdir2/file.gradle'), Artifact('dir1/subdir2/file.xml')) + paths = walk('.', ['hello/**'], ['py', 'xml']) + + self.assertEqual( + set(paths), {'dir1/subdir2/file.py', 'dir1/subdir2/file.xml'}) diff --git a/ci/fireci/tests/fileutil.py b/ci/fireci/tests/fileutil.py index 30becae00ac..c9537e85259 100644 --- a/ci/fireci/tests/fileutil.py +++ b/ci/fireci/tests/fileutil.py @@ -38,7 +38,7 @@ def create_artifacts(*artifacts): else: dirname = os.path.dirname(artifact.path) if dirname: - os.makedirs(os.path.dirname(artifact.path)) + os.makedirs(os.path.dirname(artifact.path), exist_ok=True) with open(artifact.path, 'w') as opened_file: opened_file.write(artifact.content) os.chmod(artifact.path, artifact.mode) diff --git a/ci/fireci/tests/integ_tests.py b/ci/fireci/tests/integ_tests.py index 9af3ee33500..114748fceea 100644 --- a/ci/fireci/tests/integ_tests.py +++ b/ci/fireci/tests/integ_tests.py @@ -132,3 +132,65 @@ def test_smoke_test_with_buildType_should_invoke_gradle_with_release_build_type( result = self.runner.invoke(cli, ['smoke_tests', '--app-build-variant', 'debug']) self.assertEqual(result.exit_code, 0) + + @in_tempdir + def test_copyright_check_when_no_violating_files_should_succeed(self): + create_artifacts( + Artifact('dir/file.py', content='# Copyright 2018 Google LLC')) + + result = self.runner.invoke(cli, ['copyright_check', '-e', 'py']) + self.assertEqual(result.exit_code, 0) + + @in_tempdir + def test_copyright_check_when_violating_files_exist_should_fail(self): + create_artifacts( + Artifact('dir/file.py', content='# Copyright 2018 Google LLC'), + Artifact('dir/file2.py', content='# hello'), + Artifact('dir2/file3.xml', content='# hello'), + ) + + result = self.runner.invoke(cli, + ['copyright_check', '-e', 'py', '-e' + 'xml']) + self.assertEqual(result.exit_code, 1) + self.assertFalse('dir/file.py' in result.output) + self.assertTrue('dir/file2.py' in result.output) + self.assertTrue('dir2/file3.xml' in result.output) + + @in_tempdir + def test_copyright_check_when_violating_files_exist_should_fail2(self): + create_artifacts( + Artifact('dir/file.py', content='# Copyright 2018 Google LLC'), + Artifact('dir/file2.py', content='# hello'), + Artifact('dir2/file3.xml', content='# hello'), + Artifact('dir2/subdir/file4.xml', content='# hello'), + ) + + result = self.runner.invoke( + cli, ['copyright_check', '-e', 'py', '-e' + 'xml', '-i', 'dir2/**']) + + self.assertEqual(result.exit_code, 1) + self.assertFalse('dir/file.py' in result.output) + self.assertTrue('dir/file2.py' in result.output) + self.assertFalse('dir2/file3.xml' in result.output) + self.assertFalse('dir2/subdir/file4.xml' in result.output) + + @in_tempdir + def test_copyright_check_when_violating_files_exist_should_fail3(self): + create_artifacts( + Artifact('dir/subdir/file.py', content='# Copyright 2018 Google LLC'), + Artifact('dir/subdir/file2.py', content='# hello'), + Artifact('dir/subdir2/file3.xml', content='# hello'), + Artifact('dir/subdir4/file4.xml', content='# hello'), + ) + + result = self.runner.invoke( + cli, + ['copyright_check', '-e', 'py', '-e' + 'xml', '-i', 'subdir2/**', 'dir']) + + self.assertEqual(result.exit_code, 1) + self.assertFalse('subdir/file.py' in result.output) + self.assertTrue('subdir/file2.py' in result.output) + self.assertTrue('subdir4/file4.xml' in result.output) From a70a42991f638730f9becf6acbf11a3cf5a34330 Mon Sep 17 00:00:00 2001 From: Vladimir Kryachko Date: Thu, 13 Dec 2018 14:31:56 -0500 Subject: [PATCH 20/25] Delete old copyright check. (#167) --- root-project.gradle | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/root-project.gradle b/root-project.gradle index e5dd5e27fac..5d0e4a46ccb 100644 --- a/root-project.gradle +++ b/root-project.gradle @@ -13,7 +13,6 @@ // limitations under the License. import com.google.firebase.gradle.plugins.license.LicenseResolverPlugin -import groovy.io.FileType buildscript { @@ -179,23 +178,3 @@ configure(subprojects) { task clean(type: Delete) { delete rootProject.buildDir } - -task copyrightCheck { - doLast { - def nonConformingFiles = [] - projectDir.traverse(type: FileType.FILES, - nameFilter: ~/.+\.(java|groovy|sh|proto|gradle|py)$/, - excludeFilter: ~/(^\.git|.*\/build\/.*)/) { File currentFile -> - - if (!currentFile.any { it.matches(/.*Copyright [0-9]{4} Google LLC/) }) { - nonConformingFiles.add(currentFile.path - projectDir.path - '/') - } - - } - - if (nonConformingFiles) { - throw new GradleException( - "Add copyright and license header to the following files:\n${nonConformingFiles.join('\n')}.") - } - } -} From b25d99e2e964a223d5c5c5ec0c05df4516a284c3 Mon Sep 17 00:00:00 2001 From: rsgowman Date: Fri, 14 Dec 2018 12:06:14 -0500 Subject: [PATCH 21/25] m40 release notes for firestore (#160) --- firebase-firestore/CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/firebase-firestore/CHANGELOG.md b/firebase-firestore/CHANGELOG.md index e90cc8a9e8c..b173256bc34 100644 --- a/firebase-firestore/CHANGELOG.md +++ b/firebase-firestore/CHANGELOG.md @@ -1,5 +1,15 @@ # Unreleased +# 17.1.5 +- [changed] Firestore now recovers more quickly from bad network states. +- [changed] Improved performance for reading large collections. +- [fixed] Offline persistence now properly records schema downgrades. This is a + forward-looking change that allows you to safely downgrade from future SDK + versions to this version (v17.1.5). You can already safely downgrade versions + now depending on the source version. For example, you can safely downgrade + from v17.1.4 to v17.1.2 because there are no schema changes between those + versions. (#134) + # 17.1.4 - [fixed] Fixed a SQLite transaction handling issue that occasionally masked exceptions when Firestore closed a transaction that was never started. For From 27d084e733c33ed927529e3cdd37af8702f02850 Mon Sep 17 00:00:00 2001 From: Vladimir Kryachko Date: Fri, 14 Dec 2018 12:13:56 -0500 Subject: [PATCH 22/25] Upgrade robolectric to latest version. (#166) * Upgrade robolectric to latest version. * Robolectric now returns correct dimensions of bitmaps. --- firebase-common/firebase-common.gradle | 2 +- firebase-database/firebase-database.gradle | 2 +- firebase-firestore/firebase-firestore.gradle | 2 +- .../firebase-inappmessaging-display.gradle | 8 ++++---- firebase-storage/firebase-storage.gradle | 2 +- .../java/com/google/firebase/storage/DownloadTest.java | 8 ++++---- root-project.gradle | 1 + 7 files changed, 13 insertions(+), 12 deletions(-) diff --git a/firebase-common/firebase-common.gradle b/firebase-common/firebase-common.gradle index 6552ade3e94..be2184a96e3 100644 --- a/firebase-common/firebase-common.gradle +++ b/firebase-common/firebase-common.gradle @@ -59,7 +59,7 @@ dependencies { compileOnly 'com.google.code.findbugs:jsr305:3.0.2' testImplementation 'com.android.support.test:runner:1.0.2' - testImplementation 'org.robolectric:robolectric:4.0-alpha-3' + testImplementation "org.robolectric:robolectric:$robolectricVersion" testImplementation 'junit:junit:4.12' testImplementation "com.google.truth:truth:$googleTruthVersion" testImplementation 'org.mockito:mockito-core:2.21.0' diff --git a/firebase-database/firebase-database.gradle b/firebase-database/firebase-database.gradle index 977885bf8c2..ace96b7b06f 100644 --- a/firebase-database/firebase-database.gradle +++ b/firebase-database/firebase-database.gradle @@ -94,7 +94,7 @@ dependencies { testImplementation 'junit:junit:4.12' testImplementation 'org.mockito:mockito-core:2.21.0' - testImplementation 'org.robolectric:robolectric:3.8' + testImplementation "org.robolectric:robolectric:$robolectricVersion" testImplementation 'com.firebase:firebase-token-generator:2.0.0' testImplementation 'com.fasterxml.jackson.core:jackson-core:2.9.6' testImplementation 'com.fasterxml.jackson.core:jackson-databind:2.9.6' diff --git a/firebase-firestore/firebase-firestore.gradle b/firebase-firestore/firebase-firestore.gradle index 485f63436f0..33ff390f230 100644 --- a/firebase-firestore/firebase-firestore.gradle +++ b/firebase-firestore/firebase-firestore.gradle @@ -111,7 +111,7 @@ dependencies { testImplementation 'junit:junit:4.12' testImplementation 'org.mockito:mockito-core:2.21.0' - testImplementation 'org.robolectric:robolectric:4.0-alpha-3' + testImplementation "org.robolectric:robolectric:$robolectricVersion" testImplementation "com.google.truth:truth:$googleTruthVersion" testImplementation 'com.fasterxml.jackson.core:jackson-databind:2.9.6' testImplementation 'com.google.guava:guava-testlib:12.0-rc2' diff --git a/firebase-inappmessaging-display/firebase-inappmessaging-display.gradle b/firebase-inappmessaging-display/firebase-inappmessaging-display.gradle index 16f9f5d9ca7..a4ffd4b3be4 100644 --- a/firebase-inappmessaging-display/firebase-inappmessaging-display.gradle +++ b/firebase-inappmessaging-display/firebase-inappmessaging-display.gradle @@ -15,11 +15,11 @@ apply plugin: "com.android.library" android { - compileSdkVersion 27 + compileSdkVersion 28 defaultConfig { minSdkVersion 16 - targetSdkVersion 27 + targetSdkVersion 28 versionCode 1 versionName "1.0" multiDexEnabled true @@ -72,7 +72,7 @@ dependencies { annotationProcessor 'com.google.auto.value:auto-value:1.6' annotationProcessor 'com.ryanharter.auto.value:auto-value-parcel:0.2.6' - testImplementation "org.robolectric:robolectric:3.8" + testImplementation "org.robolectric:robolectric:$robolectricVersion" testImplementation "junit:junit:4.12" testImplementation "org.mockito:mockito-core:2.18.3" testImplementation "com.google.truth:truth:$googleTruthVersion" @@ -83,4 +83,4 @@ dependencies { androidTestImplementation "com.android.support:support-annotations:27.1.1" androidTestImplementation "com.android.support.test:runner:1.0.2" androidTestImplementation "com.android.support.test:rules:1.0.2" -} \ No newline at end of file +} diff --git a/firebase-storage/firebase-storage.gradle b/firebase-storage/firebase-storage.gradle index 6429fb1b8b3..1ae9c9a8473 100644 --- a/firebase-storage/firebase-storage.gradle +++ b/firebase-storage/firebase-storage.gradle @@ -88,5 +88,5 @@ dependencies { testImplementation 'junit:junit:4.12' testImplementation 'org.mockito:mockito-core:2.21.0' - testImplementation 'org.robolectric:robolectric:4.0-alpha-3-SNAPSHOT' + testImplementation "org.robolectric:robolectric:$robolectricVersion" } diff --git a/firebase-storage/src/test/java/com/google/firebase/storage/DownloadTest.java b/firebase-storage/src/test/java/com/google/firebase/storage/DownloadTest.java index c25f447bbf0..5343d4a1ae0 100644 --- a/firebase-storage/src/test/java/com/google/firebase/storage/DownloadTest.java +++ b/firebase-storage/src/test/java/com/google/firebase/storage/DownloadTest.java @@ -76,8 +76,8 @@ public void streamDownload() throws Exception { TestDownloadHelper.streamDownload( bitmap -> { assertNotNull(bitmap); - assertEquals(100, bitmap.getWidth()); - assertEquals(100, bitmap.getHeight()); + assertEquals(2560, bitmap.getWidth()); + assertEquals(1710, bitmap.getHeight()); completeHandlerInvoked[0] = true; }, null, @@ -162,8 +162,8 @@ public void streamDownloadWithResume() throws Exception { TestDownloadHelper.streamDownload( bitmap -> { assertNotNull(bitmap); - assertEquals(100, bitmap.getWidth()); - assertEquals(100, bitmap.getHeight()); + assertEquals(2560, bitmap.getWidth()); + assertEquals(1710, bitmap.getHeight()); completeHandlerInvoked[0] = true; }, null, diff --git a/root-project.gradle b/root-project.gradle index 5d0e4a46ccb..3c95165a95c 100644 --- a/root-project.gradle +++ b/root-project.gradle @@ -43,6 +43,7 @@ ext { errorproneVersion = '2.3.2' errorproneJavacVersion = '9+181-r4173-1' googleTruthVersion = '0.40' + robolectricVersion = '4.1' } apply plugin: com.google.firebase.gradle.plugins.publish.PublishingPlugin From 5539d80b660b448bde8ca84884752b94e0baed0a Mon Sep 17 00:00:00 2001 From: David Motsonashvili Date: Mon, 17 Dec 2018 13:40:11 -0800 Subject: [PATCH 23/25] update firebase-firestore gradle.properties (#171) last released version is inaccurate --- firebase-firestore/gradle.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/firebase-firestore/gradle.properties b/firebase-firestore/gradle.properties index 0332b866ede..f3ad92d86ba 100644 --- a/firebase-firestore/gradle.properties +++ b/firebase-firestore/gradle.properties @@ -1,2 +1,2 @@ -version=17.1.4 -latestReleasedVersion=17.1.3 +version=17.1.5 +latestReleasedVersion=17.1.4 From 51fb015fe3d24c95135b7e86438fc6332844b8f2 Mon Sep 17 00:00:00 2001 From: Vladimir Kryachko Date: Wed, 19 Dec 2018 15:38:26 -0500 Subject: [PATCH 24/25] Add the concept of Set-components and dependencies. (#169) * Add the contept of Set-components and dependencies. Set-components are components of type T that contribute values into a larger Set rather than registering one unique T value whose uniqueness is enforced at runtime. * Add license to source file. * Update consumer proguard config. --- firebase-common/proguard.txt | 3 +- .../google/firebase/components/Component.java | 92 ++++++++++++++++++- .../firebase/components/Dependency.java | 21 ++++- .../firebase/components/ComponentTest.java | 64 ++++++++++++- .../firebase/components/DependencyTest.java | 84 +++++++++++++++++ 5 files changed, 251 insertions(+), 13 deletions(-) create mode 100644 firebase-common/src/test/java/com/google/firebase/components/DependencyTest.java diff --git a/firebase-common/proguard.txt b/firebase-common/proguard.txt index 63c0883bd9c..305dc2de368 100644 --- a/firebase-common/proguard.txt +++ b/firebase-common/proguard.txt @@ -1 +1,2 @@ --dontwarn com.google.firebase.components.Component$Instantiation \ No newline at end of file +-dontwarn com.google.firebase.components.Component$Instantiation +-dontwarn com.google.firebase.components.Component$ComponentType \ No newline at end of file diff --git a/firebase-common/src/main/java/com/google/firebase/components/Component.java b/firebase-common/src/main/java/com/google/firebase/components/Component.java index 98581417b86..234455a4188 100644 --- a/firebase-common/src/main/java/com/google/firebase/components/Component.java +++ b/firebase-common/src/main/java/com/google/firebase/components/Component.java @@ -36,17 +36,53 @@ @KeepForSdk public final class Component { + /** Specifies instantiation behavior of a {@link Component}. */ @IntDef({Instantiation.LAZY, Instantiation.ALWAYS_EAGER, Instantiation.EAGER_IN_DEFAULT_APP}) @Retention(RetentionPolicy.SOURCE) private @interface Instantiation { + /** Component is not instantiated until requested by developer or a dependent component. */ int LAZY = 0; + + /** + * Component is unconditionally instantiated upon startup of the {@link ComponentRuntime}. + * + *

Namely when {@link ComponentRuntime#initializeEagerComponents(boolean)} is called. + */ int ALWAYS_EAGER = 1; + + /** + * Component is instantiated upon startup of the {@link ComponentRuntime} if the runtime is + * initialized for the default app. + */ int EAGER_IN_DEFAULT_APP = 2; } + /** Specifies the type of a {@link Component}. */ + @IntDef({ComponentType.VALUE, ComponentType.SET}) + @Retention(RetentionPolicy.SOURCE) + private @interface ComponentType { + /** + * Value components provide scalar values to the {@link ComponentRuntime}. + * + *

Such components can be requested by dependents via {@link ComponentContainer#get(Class)} + * or {@link ComponentContainer#getProvider(Class)}. e.g. {@code FirebaseInstanceId}. + */ + int VALUE = 0; + + /** + * Set components collectively contribute values of type {@code T} to a {@link Set + * Set<T>}. + * + *

Such components can be requested by dependents via {@link ComponentContainer#setOf(Class)} + * or {@link ComponentContainer#setOfProvider(Class)}. + */ + int SET = 1; + } + private final Set> providedInterfaces; private final Set dependencies; private final @Instantiation int instantiation; + private final @ComponentType int type; private final ComponentFactory factory; private final Set> publishedEvents; @@ -54,11 +90,13 @@ private Component( Set> providedInterfaces, Set dependencies, @Instantiation int instantiation, + @ComponentType int type, ComponentFactory factory, Set> publishedEvents) { this.providedInterfaces = Collections.unmodifiableSet(providedInterfaces); this.dependencies = Collections.unmodifiableSet(dependencies); this.instantiation = instantiation; + this.type = type; this.factory = factory; this.publishedEvents = Collections.unmodifiableSet(publishedEvents); } @@ -113,6 +151,11 @@ public boolean isEagerInDefaultApp() { return instantiation == Instantiation.EAGER_IN_DEFAULT_APP; } + /** Returns whether a component is a Value Component or a Set Component. */ + public boolean isValue() { + return type == ComponentType.VALUE; + } + @Override public String toString() { StringBuilder sb = @@ -120,23 +163,26 @@ public String toString() { .append(Arrays.toString(providedInterfaces.toArray())) .append(">{") .append(instantiation) + .append(", type=") + .append(type) .append(", deps=") .append(Arrays.toString(dependencies.toArray())) .append("}"); return sb.toString(); } - @KeepForSdk /** Returns a Component builder. */ + @KeepForSdk public static Component.Builder builder(Class anInterface) { - return new Builder(anInterface); + return new Builder<>(anInterface); } - @KeepForSdk /** Returns a Component builder. */ + @KeepForSdk + @SafeVarargs public static Component.Builder builder( Class anInterface, Class... additionalInterfaces) { - return new Builder(anInterface, additionalInterfaces); + return new Builder<>(anInterface, additionalInterfaces); } /** @@ -151,22 +197,46 @@ public static Component of(Class anInterface, T value) { } /** Wraps a value in a {@link Component} with no dependencies. */ - @SafeVarargs @KeepForSdk + @SafeVarargs public static Component of( T value, Class anInterface, Class... additionalInterfaces) { return builder(anInterface, additionalInterfaces).factory((args) -> value).build(); } + /** + * Provides a builder for a {@link Set}-multibinding {@link Component}. + * + *

Such components can be requested by dependents via {@link ComponentContainer#setOf(Class)} * + * or {@link ComponentContainer#setOfProvider(Class)}. + */ + @KeepForSdk + public static Component.Builder intoSetBuilder(Class anInterface) { + return builder(anInterface).intoSet(); + } + + /** + * Wraps a value in a {@link Set}-multibinding {@link Component} with no dependencies. * + * + *

Such components can be requested by dependents via {@link ComponentContainer#setOf(Class)} * + * or {@link ComponentContainer#setOfProvider(Class)}. + */ + @KeepForSdk + public static Component intoSet(T value, Class anInterface) { + return intoSetBuilder(anInterface).factory(c -> value).build(); + } + /** FirebaseComponent builder. */ @KeepForSdk public static class Builder { private final Set> providedInterfaces = new HashSet<>(); private final Set dependencies = new HashSet<>(); private @Instantiation int instantiation = Instantiation.LAZY; + private @ComponentType int type = ComponentType.VALUE; private ComponentFactory factory; private Set> publishedEvents = new HashSet<>(); + @SafeVarargs private Builder(Class anInterface, Class... additionalInterfaces) { Preconditions.checkNotNull(anInterface, "Null interface"); providedInterfaces.add(anInterface); @@ -176,6 +246,7 @@ private Builder(Class anInterface, Class... additionalInterfaces) Collections.addAll(providedInterfaces, additionalInterfaces); } + /** Add a {@link Dependency} to the {@link Component} being built. */ @KeepForSdk public Builder add(Dependency dependency) { Preconditions.checkNotNull(dependency, "Null dependency"); @@ -184,16 +255,19 @@ public Builder add(Dependency dependency) { return this; } + /** Make the {@link Component} initialize upon startup. */ @KeepForSdk public Builder alwaysEager() { return setInstantiation(Instantiation.ALWAYS_EAGER); } + /** Make the component initialize upon startup in default app. */ @KeepForSdk public Builder eagerInDefaultApp() { return setInstantiation(Instantiation.EAGER_IN_DEFAULT_APP); } + /** Make the {@link Component} eligible to publish events of provided eventType. */ @KeepForSdk public Builder publishes(Class eventType) { publishedEvents.add(eventType); @@ -213,12 +287,19 @@ private void validateInterface(Class anInterface) { "Components are not allowed to depend on interfaces they themselves provide."); } + /** Set the factory that will be used to initialize the {@link Component}. */ @KeepForSdk public Builder factory(ComponentFactory value) { factory = Preconditions.checkNotNull(value, "Null factory"); return this; } + private Builder intoSet() { + type = ComponentType.SET; + return this; + } + + /** Return the built {@link Component} definition. */ @KeepForSdk public Component build() { Preconditions.checkState(factory != null, "Missing required property: factory."); @@ -226,6 +307,7 @@ public Component build() { new HashSet<>(providedInterfaces), new HashSet<>(dependencies), instantiation, + type, factory, publishedEvents); } diff --git a/firebase-common/src/main/java/com/google/firebase/components/Dependency.java b/firebase-common/src/main/java/com/google/firebase/components/Dependency.java index faa0e0e3641..14f03a3723b 100644 --- a/firebase-common/src/main/java/com/google/firebase/components/Dependency.java +++ b/firebase-common/src/main/java/com/google/firebase/components/Dependency.java @@ -24,11 +24,12 @@ @KeepForSdk public final class Dependency { /** Enumerates dependency types. */ - @IntDef({Type.OPTIONAL, Type.REQUIRED}) + @IntDef({Type.OPTIONAL, Type.REQUIRED, Type.SET}) @Retention(RetentionPolicy.SOURCE) private @interface Type { int OPTIONAL = 0; int REQUIRED = 1; + int SET = 2; } @IntDef({Injection.DIRECT, Injection.PROVIDER}) @@ -58,6 +59,11 @@ public static Dependency required(Class anInterface) { return new Dependency(anInterface, Type.REQUIRED, Injection.DIRECT); } + @KeepForSdk + public static Dependency setOf(Class anInterface) { + return new Dependency(anInterface, Type.SET, Injection.DIRECT); + } + @KeepForSdk public static Dependency optionalProvider(Class anInterface) { return new Dependency(anInterface, Type.OPTIONAL, Injection.PROVIDER); @@ -68,6 +74,11 @@ public static Dependency requiredProvider(Class anInterface) { return new Dependency(anInterface, Type.REQUIRED, Injection.PROVIDER); } + @KeepForSdk + public static Dependency setOfProvider(Class anInterface) { + return new Dependency(anInterface, Type.SET, Injection.PROVIDER); + } + public Class getInterface() { return anInterface; } @@ -76,6 +87,10 @@ public boolean isRequired() { return type == Type.REQUIRED; } + public boolean isSet() { + return type == Type.SET; + } + public boolean isDirectInjection() { return injection == Injection.DIRECT; } @@ -105,8 +120,8 @@ public String toString() { StringBuilder sb = new StringBuilder("Dependency{anInterface=") .append(anInterface) - .append(", required=") - .append(type == Type.REQUIRED) + .append(", type=") + .append(type == Type.REQUIRED ? "required" : type == Type.OPTIONAL ? "optional" : "set") .append(", direct=") .append(injection == Injection.DIRECT) .append("}"); diff --git a/firebase-common/src/test/java/com/google/firebase/components/ComponentTest.java b/firebase-common/src/test/java/com/google/firebase/components/ComponentTest.java index e0005c83b92..33cb06b8dc9 100644 --- a/firebase-common/src/test/java/com/google/firebase/components/ComponentTest.java +++ b/firebase-common/src/test/java/com/google/firebase/components/ComponentTest.java @@ -17,6 +17,7 @@ import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.fail; +import java.math.BigDecimal; import java.util.List; import org.junit.Test; import org.junit.runner.RunWith; @@ -26,16 +27,55 @@ public class ComponentTest { interface TestInterface {} - static class TestClass implements TestInterface {} + private static class TestClass implements TestInterface {} private final ComponentFactory nullFactory = container -> null; + @Test + public void of_withMultipleInterfaces_shouldSetCorrectDefaults() { + TestClass testClass = new TestClass(); + Component component = Component.of(testClass, TestClass.class, TestInterface.class); + assertThat(component.getProvidedInterfaces()) + .containsExactly(TestClass.class, TestInterface.class); + assertThat(component.isLazy()).isTrue(); + assertThat(component.isValue()).isTrue(); + assertThat(component.isAlwaysEager()).isFalse(); + assertThat(component.isEagerInDefaultApp()).isFalse(); + assertThat(component.getDependencies()).isEmpty(); + assertThat(component.getFactory().create(null)).isSameAs(testClass); + } + @Test public void builder_shouldSetCorrectDefaults() { Component component = Component.builder(TestClass.class).factory(nullFactory).build(); assertThat(component.getProvidedInterfaces()).containsExactly(TestClass.class); assertThat(component.isLazy()).isTrue(); + assertThat(component.isValue()).isTrue(); + assertThat(component.isAlwaysEager()).isFalse(); + assertThat(component.isEagerInDefaultApp()).isFalse(); + assertThat(component.getDependencies()).isEmpty(); + } + + @Test + public void intoSetBuilder_shouldSetCorrectDefaults() { + Component component = + Component.intoSetBuilder(TestClass.class).factory(nullFactory).build(); + assertThat(component.getProvidedInterfaces()).containsExactly(TestClass.class); + assertThat(component.isLazy()).isTrue(); + assertThat(component.isValue()).isFalse(); + assertThat(component.isAlwaysEager()).isFalse(); + assertThat(component.isEagerInDefaultApp()).isFalse(); + assertThat(component.getDependencies()).isEmpty(); + } + + @Test + public void intoSet_shouldSetCorrectDefaults() { + TestClass testClass = new TestClass(); + Component component = Component.intoSet(testClass, TestClass.class); + assertThat(component.getProvidedInterfaces()).containsExactly(TestClass.class); + assertThat(component.isLazy()).isTrue(); + assertThat(component.isValue()).isFalse(); assertThat(component.isAlwaysEager()).isFalse(); assertThat(component.isEagerInDefaultApp()).isFalse(); assertThat(component.getDependencies()).isEmpty(); @@ -62,7 +102,7 @@ public void eagerInDefaultApp_shouldProperlySetComponentInitialization() { } @Test - public void uptatingInstantiationMultipleTimes_shouldThrow() { + public void updatingInstantiationMultipleTimes_shouldThrow() { Component.Builder builder = Component.builder(TestClass.class).eagerInDefaultApp(); try { @@ -79,18 +119,22 @@ public void add_shouldProperlyAddDependencies() { Component.builder(TestClass.class) .add(Dependency.required(List.class)) .add(Dependency.optional(Integer.class)) + .add(Dependency.setOf(Long.class)) .add(Dependency.requiredProvider(Float.class)) .add(Dependency.optionalProvider(Double.class)) + .add(Dependency.setOfProvider(BigDecimal.class)) .factory(nullFactory) .build(); - assertThat(component.getDependencies()).hasSize(4); + assertThat(component.getDependencies()).hasSize(6); assertThat(component.getDependencies()) .containsExactly( Dependency.required(List.class), Dependency.optional(Integer.class), + Dependency.setOf(Long.class), Dependency.requiredProvider(Float.class), - Dependency.optionalProvider(Double.class)); + Dependency.optionalProvider(Double.class), + Dependency.setOfProvider(BigDecimal.class)); } @Test @@ -113,6 +157,18 @@ public void addOptionalDependency_onSelf_shouldThrow() { } } + @Test + public void publishes_shouldProperlyAddToPublishedEvents() { + Component component = + Component.builder(TestClass.class) + .factory(nullFactory) + .publishes(Integer.class) + .publishes(Float.class) + .build(); + + assertThat(component.getPublishedEvents()).containsExactly(Integer.class, Float.class); + } + @Test public void builder_withMultipleInterfaces_shouldProperlySetInterfaces() { Component component = diff --git a/firebase-common/src/test/java/com/google/firebase/components/DependencyTest.java b/firebase-common/src/test/java/com/google/firebase/components/DependencyTest.java new file mode 100644 index 00000000000..0c3fdaca99c --- /dev/null +++ b/firebase-common/src/test/java/com/google/firebase/components/DependencyTest.java @@ -0,0 +1,84 @@ +// Copyright 2018 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.firebase.components; + +import static com.google.common.truth.Truth.assertThat; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class DependencyTest { + @Test + public void optional_shouldHaveExpectedInvariants() { + Dependency dependency = Dependency.optional(String.class); + + assertThat(dependency.isRequired()).isFalse(); + assertThat(dependency.isSet()).isFalse(); + assertThat(dependency.isDirectInjection()).isTrue(); + assertThat(dependency.getInterface()).isEqualTo(String.class); + } + + @Test + public void required_shouldHaveExpectedInvariants() { + Dependency dependency = Dependency.required(String.class); + + assertThat(dependency.isRequired()).isTrue(); + assertThat(dependency.isSet()).isFalse(); + assertThat(dependency.isDirectInjection()).isTrue(); + assertThat(dependency.getInterface()).isEqualTo(String.class); + } + + @Test + public void setOf_shouldHaveExpectedInvariants() { + Dependency dependency = Dependency.setOf(String.class); + + assertThat(dependency.isRequired()).isFalse(); + assertThat(dependency.isSet()).isTrue(); + assertThat(dependency.isDirectInjection()).isTrue(); + assertThat(dependency.getInterface()).isEqualTo(String.class); + } + + @Test + public void optionalProvider_shouldHaveExpectedInvariants() { + Dependency dependency = Dependency.optionalProvider(String.class); + + assertThat(dependency.isRequired()).isFalse(); + assertThat(dependency.isSet()).isFalse(); + assertThat(dependency.isDirectInjection()).isFalse(); + assertThat(dependency.getInterface()).isEqualTo(String.class); + } + + @Test + public void requiredProvider_shouldHaveExpectedInvariants() { + Dependency dependency = Dependency.requiredProvider(String.class); + + assertThat(dependency.isRequired()).isTrue(); + assertThat(dependency.isSet()).isFalse(); + assertThat(dependency.isDirectInjection()).isFalse(); + assertThat(dependency.getInterface()).isEqualTo(String.class); + } + + @Test + public void setOfProvider_shouldHaveExpectedInvariants() { + Dependency dependency = Dependency.setOfProvider(String.class); + + assertThat(dependency.isRequired()).isFalse(); + assertThat(dependency.isSet()).isTrue(); + assertThat(dependency.isDirectInjection()).isFalse(); + assertThat(dependency.getInterface()).isEqualTo(String.class); + } +} From c627b48e40d3eedbb3a342bba61bf9a4b50088f9 Mon Sep 17 00:00:00 2001 From: Greg Soltis Date: Wed, 19 Dec 2018 15:07:44 -0800 Subject: [PATCH 25/25] Enable LRU GC (#68) * Enable LRU GC * Add public api and comment to CACHE_SIZE_UNLIMITED * Switch default for LRU collection to disabled --- .../firebase/firestore/FirebaseFirestore.java | 7 +- .../firestore/FirebaseFirestoreSettings.java | 53 ++++- .../firestore/core/FirestoreClient.java | 37 ++- .../firebase/firestore/local/LocalStore.java | 4 + .../firebase/firestore/local/LruDelegate.java | 7 +- .../firestore/local/LruGarbageCollector.java | 217 +++++++++++++++++- .../local/MemoryLruReferenceDelegate.java | 41 +++- .../firestore/local/MemoryMutationQueue.java | 8 + .../firestore/local/MemoryPersistence.java | 6 +- .../firestore/local/MemoryQueryCache.java | 8 + .../local/MemoryRemoteDocumentCache.java | 24 ++ .../local/SQLiteLruReferenceDelegate.java | 19 +- .../firestore/local/SQLitePersistence.java | 17 +- .../firebase/firestore/util/AsyncQueue.java | 2 + .../local/LruGarbageCollectorTestCase.java | 123 +++++++++- .../local/MemoryLruGarbageCollectorTest.java | 4 +- .../local/PersistenceTestHelpers.java | 23 +- .../local/SQLiteLruGarbageCollectorTest.java | 4 +- .../firestore/spec/SQLiteSpecTest.java | 4 +- 19 files changed, 559 insertions(+), 49 deletions(-) diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/FirebaseFirestore.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/FirebaseFirestore.java index 2acc26fe4ad..65d61ce490b 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/FirebaseFirestore.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/FirebaseFirestore.java @@ -220,12 +220,7 @@ private void ensureClientConfigured() { new DatabaseInfo(databaseId, persistenceKey, settings.getHost(), settings.isSslEnabled()); client = - new FirestoreClient( - context, - databaseInfo, - settings.isPersistenceEnabled(), - credentialsProvider, - asyncQueue); + new FirestoreClient(context, databaseInfo, settings, credentialsProvider, asyncQueue); } } diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/FirebaseFirestoreSettings.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/FirebaseFirestoreSettings.java index f0598e0b768..750e7ff0159 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/FirebaseFirestoreSettings.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/FirebaseFirestoreSettings.java @@ -24,6 +24,18 @@ /** Settings used to configure a FirebaseFirestore instance. */ @PublicApi public final class FirebaseFirestoreSettings { + /** + * Constant to use with {@link FirebaseFirestoreSettings.Builder#setCacheSizeBytes(long)} to + * disable garbage collection. + */ + @PublicApi public static final long CACHE_SIZE_UNLIMITED = -1; + + private static final long MINIMUM_CACHE_BYTES = 1 * 1024 * 1024; // 1 MB + // TODO(b/121269744): Set this to be the default value after SDK is past version 1.0 + // private static final long DEFAULT_CACHE_SIZE_BYTES = 100 * 1024 * 1024; // 100 MB + // For now, we are rolling this out with collection disabled. Once the SDK has hit version 1.0, + // we will switch the default to the above value, 100 MB. + private static final long DEFAULT_CACHE_SIZE_BYTES = CACHE_SIZE_UNLIMITED; private static final String DEFAULT_HOST = "firestore.googleapis.com"; private static final boolean DEFAULT_TIMESTAMPS_IN_SNAPSHOTS_ENABLED = false; @@ -34,6 +46,7 @@ public static final class Builder { private boolean sslEnabled; private boolean persistenceEnabled; private boolean timestampsInSnapshotsEnabled; + private long cacheSizeBytes; /** Constructs a new FirebaseFirestoreSettings Builder object. */ @PublicApi @@ -42,6 +55,7 @@ public Builder() { sslEnabled = true; persistenceEnabled = true; timestampsInSnapshotsEnabled = DEFAULT_TIMESTAMPS_IN_SNAPSHOTS_ENABLED; + cacheSizeBytes = DEFAULT_CACHE_SIZE_BYTES; } /** @@ -124,6 +138,30 @@ public Builder setTimestampsInSnapshotsEnabled(boolean value) { return this; } + /** + * Sets an approximate cache size threshold for the on-disk data. If the cache grows beyond this + * size, Firestore will start removing data that hasn't been recently used. The size is not a + * guarantee that the cache will stay below that size, only that if the cache exceeds the given + * size, cleanup will be attempted. + * + *

By default, collection is disabled (the value is set to {@link + * FirebaseFirestoreSettings#CACHE_SIZE_UNLIMITED}). In a future release, collection will be + * enabled by default, with a default cache size of 100 MB. The minimum value is 1 MB. + * + * @return A settings object on which the cache size is configured as specified by the given + * {@code value}. + */ + @NonNull + @PublicApi + public Builder setCacheSizeBytes(long value) { + if (value != CACHE_SIZE_UNLIMITED && value < MINIMUM_CACHE_BYTES) { + throw new IllegalArgumentException( + "Cache size must be set to at least " + MINIMUM_CACHE_BYTES + " bytes"); + } + this.cacheSizeBytes = value; + return this; + } + @NonNull @PublicApi public FirebaseFirestoreSettings build() { @@ -139,6 +177,7 @@ public FirebaseFirestoreSettings build() { private final boolean sslEnabled; private final boolean persistenceEnabled; private final boolean timestampsInSnapshotsEnabled; + private final long cacheSizeBytes; /** Constructs a FirebaseFirestoreSettings object based on the values in the Builder. */ private FirebaseFirestoreSettings(Builder builder) { @@ -146,6 +185,7 @@ private FirebaseFirestoreSettings(Builder builder) { sslEnabled = builder.sslEnabled; persistenceEnabled = builder.persistenceEnabled; timestampsInSnapshotsEnabled = builder.timestampsInSnapshotsEnabled; + cacheSizeBytes = builder.cacheSizeBytes; } @Override @@ -161,7 +201,8 @@ public boolean equals(@Nullable Object o) { return host.equals(that.host) && sslEnabled == that.sslEnabled && persistenceEnabled == that.persistenceEnabled - && timestampsInSnapshotsEnabled == that.timestampsInSnapshotsEnabled; + && timestampsInSnapshotsEnabled == that.timestampsInSnapshotsEnabled + && cacheSizeBytes == that.cacheSizeBytes; } @Override @@ -170,6 +211,7 @@ public int hashCode() { result = 31 * result + (sslEnabled ? 1 : 0); result = 31 * result + (persistenceEnabled ? 1 : 0); result = 31 * result + (timestampsInSnapshotsEnabled ? 1 : 0); + result = 31 * result + (int) cacheSizeBytes; return result; } @@ -211,4 +253,13 @@ public boolean isPersistenceEnabled() { public boolean areTimestampsInSnapshotsEnabled() { return timestampsInSnapshotsEnabled; } + + /** + * Returns the threshold for the cache size above which the SDK will attempt to collect the least + * recently used documents. + */ + @PublicApi + public long getCacheSizeBytes() { + return cacheSizeBytes; + } } diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/core/FirestoreClient.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/core/FirestoreClient.java index 9ea2b2f5b7d..123a4268a62 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/core/FirestoreClient.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/core/FirestoreClient.java @@ -27,11 +27,14 @@ import com.google.firebase.firestore.EventListener; import com.google.firebase.firestore.FirebaseFirestoreException; import com.google.firebase.firestore.FirebaseFirestoreException.Code; +import com.google.firebase.firestore.FirebaseFirestoreSettings; import com.google.firebase.firestore.auth.CredentialsProvider; import com.google.firebase.firestore.auth.User; import com.google.firebase.firestore.core.EventManager.ListenOptions; import com.google.firebase.firestore.local.LocalSerializer; import com.google.firebase.firestore.local.LocalStore; +import com.google.firebase.firestore.local.LruDelegate; +import com.google.firebase.firestore.local.LruGarbageCollector; import com.google.firebase.firestore.local.MemoryPersistence; import com.google.firebase.firestore.local.Persistence; import com.google.firebase.firestore.local.SQLitePersistence; @@ -71,10 +74,13 @@ public final class FirestoreClient implements RemoteStore.RemoteStoreCallback { private SyncEngine syncEngine; private EventManager eventManager; + // LRU-related + @Nullable private LruGarbageCollector.Scheduler lruScheduler; + public FirestoreClient( final Context context, DatabaseInfo databaseInfo, - final boolean usePersistence, + FirebaseFirestoreSettings settings, CredentialsProvider credentialsProvider, final AsyncQueue asyncQueue) { this.databaseInfo = databaseInfo; @@ -105,7 +111,11 @@ public FirestoreClient( try { // Block on initial user being available User initialUser = Tasks.await(firstUser.getTask()); - initialize(context, initialUser, usePersistence); + initialize( + context, + initialUser, + settings.isPersistenceEnabled(), + settings.getCacheSizeBytes()); } catch (InterruptedException | ExecutionException e) { throw new RuntimeException(e); } @@ -127,6 +137,9 @@ public Task shutdown() { () -> { remoteStore.shutdown(); persistence.shutdown(); + if (lruScheduler != null) { + lruScheduler.stop(); + } }); } @@ -194,24 +207,38 @@ public Task transaction( () -> syncEngine.transaction(asyncQueue, updateFunction, retries)); } - private void initialize(Context context, User user, boolean usePersistence) { + private void initialize(Context context, User user, boolean usePersistence, long cacheSizeBytes) { // Note: The initialization work must all be synchronous (we can't dispatch more work) since // external write/listen operations could get queued to run before that subsequent work // completes. Logger.debug(LOG_TAG, "Initializing. user=%s", user.getUid()); + LruGarbageCollector gc = null; if (usePersistence) { LocalSerializer serializer = new LocalSerializer(new RemoteSerializer(databaseInfo.getDatabaseId())); - persistence = + LruGarbageCollector.Params params = + LruGarbageCollector.Params.WithCacheSizeBytes(cacheSizeBytes); + SQLitePersistence sqlitePersistence = new SQLitePersistence( - context, databaseInfo.getPersistenceKey(), databaseInfo.getDatabaseId(), serializer); + context, + databaseInfo.getPersistenceKey(), + databaseInfo.getDatabaseId(), + serializer, + params); + LruDelegate lruDelegate = sqlitePersistence.getReferenceDelegate(); + gc = lruDelegate.getGarbageCollector(); + persistence = sqlitePersistence; } else { persistence = MemoryPersistence.createEagerGcMemoryPersistence(); } persistence.start(); localStore = new LocalStore(persistence, user); + if (gc != null) { + lruScheduler = gc.newScheduler(asyncQueue, localStore); + lruScheduler.start(); + } Datastore datastore = new Datastore(databaseInfo, asyncQueue, credentialsProvider, context); remoteStore = new RemoteStore(this, localStore, datastore, asyncQueue); diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LocalStore.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LocalStore.java index 46d2d480b62..dae56aafe29 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LocalStore.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LocalStore.java @@ -570,4 +570,8 @@ private void applyWriteToRemoteDocuments(MutationBatchResult batchResult) { mutationQueue.removeMutationBatch(batch); } + + public LruGarbageCollector.Results collectGarbage(LruGarbageCollector garbageCollector) { + return persistence.runTransaction("Collect garbage", () -> garbageCollector.collect(targetIds)); + } } diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LruDelegate.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LruDelegate.java index 569ef9415ea..2dd128c285e 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LruDelegate.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LruDelegate.java @@ -21,12 +21,12 @@ * Persistence layers intending to use LRU Garbage collection should implement this interface. This * interface defines the operations that the LRU garbage collector needs from the persistence layer. */ -interface LruDelegate { +public interface LruDelegate { /** Enumerates all the targets in the QueryCache. */ void forEachTarget(Consumer consumer); - long getTargetCount(); + long getSequenceNumberCount(); /** Enumerates sequence numbers for documents not associated with a target. */ void forEachOrphanedDocumentSequenceNumber(Consumer consumer); @@ -49,4 +49,7 @@ interface LruDelegate { /** Access to the underlying LRU Garbage collector instance. */ LruGarbageCollector getGarbageCollector(); + + /** Return the size of the cache in bytes. */ + long getByteSize(); } diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LruGarbageCollector.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LruGarbageCollector.java index 91a61731338..d423ca60218 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LruGarbageCollector.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/LruGarbageCollector.java @@ -14,22 +14,156 @@ package com.google.firebase.firestore.local; +import android.support.annotation.Nullable; import android.util.SparseArray; +import com.google.firebase.firestore.FirebaseFirestoreSettings; import com.google.firebase.firestore.core.ListenSequence; +import com.google.firebase.firestore.util.AsyncQueue; +import com.google.firebase.firestore.util.Logger; import java.util.Comparator; +import java.util.Locale; import java.util.PriorityQueue; +import java.util.concurrent.TimeUnit; /** Implements the steps for LRU garbage collection. */ -class LruGarbageCollector { +public class LruGarbageCollector { + /** How long we wait to try running LRU GC after SDK initialization. */ + private static final long INITIAL_GC_DELAY_MS = TimeUnit.MINUTES.toMillis(1); + /** Minimum amount of time between GC checks, after the first one. */ + private static final long REGULAR_GC_DELAY_MS = TimeUnit.MINUTES.toMillis(5); + + public static class Params { + private static final long COLLECTION_DISABLED = FirebaseFirestoreSettings.CACHE_SIZE_UNLIMITED; + private static final long DEFAULT_CACHE_SIZE_BYTES = 100 * 1024 * 1024; // 100mb + /** + * The following two constants are estimates for how we want to tune the garbage collector. If + * we encounter a large cache, we don't want to spend a large chunk of time GCing all of it, we + * would rather make some progress and then try again later. We also don't want to collect + * everything that we possibly could, as our thesis is that recently used items are more likely + * to be used again. + */ + private static final int DEFAULT_COLLECTION_PERCENTILE = 10; + + private static final int DEFAULT_MAX_SEQUENCE_NUMBERS_TO_COLLECT = 1000; + + public static Params Default() { + return new Params( + DEFAULT_CACHE_SIZE_BYTES, + DEFAULT_COLLECTION_PERCENTILE, + DEFAULT_MAX_SEQUENCE_NUMBERS_TO_COLLECT); + } + + public static Params Disabled() { + return new Params(COLLECTION_DISABLED, 0, 0); + } + + public static Params WithCacheSizeBytes(long cacheSizeBytes) { + return new Params(cacheSizeBytes, 10, 1000); + } + + final long minBytesThreshold; + final int percentileToCollect; + final int maximumSequenceNumbersToCollect; + + Params(long minBytesThreshold, int percentileToCollect, int maximumSequenceNumbersToCollect) { + this.minBytesThreshold = minBytesThreshold; + this.percentileToCollect = percentileToCollect; + this.maximumSequenceNumbersToCollect = maximumSequenceNumbersToCollect; + } + } + + public static class Results { + private final boolean hasRun; + private final int sequenceNumbersCollected; + private final int targetsRemoved; + private final int documentsRemoved; + + static Results DidNotRun() { + return new Results(/* hasRun= */ false, 0, 0, 0); + } + + Results( + boolean hasRun, int sequenceNumbersCollected, int targetsRemoved, int documentsRemoved) { + this.hasRun = hasRun; + this.sequenceNumbersCollected = sequenceNumbersCollected; + this.targetsRemoved = targetsRemoved; + this.documentsRemoved = documentsRemoved; + } + + public boolean hasRun() { + return hasRun; + } + + public int getSequenceNumbersCollected() { + return sequenceNumbersCollected; + } + + public int getTargetsRemoved() { + return targetsRemoved; + } + + public int getDocumentsRemoved() { + return documentsRemoved; + } + } + + /** + * This class is responsible for the scheduling of LRU garbage collection. It handles checking + * whether or not GC is enabled, as well as which delay to use before the next run. + */ + public class Scheduler { + private final AsyncQueue asyncQueue; + private final LocalStore localStore; + private boolean hasRun = false; + @Nullable private AsyncQueue.DelayedTask gcTask; + + public Scheduler(AsyncQueue asyncQueue, LocalStore localStore) { + this.asyncQueue = asyncQueue; + this.localStore = localStore; + } + + public void start() { + if (params.minBytesThreshold != Params.COLLECTION_DISABLED) { + scheduleGC(); + } + } + + public void stop() { + if (gcTask != null) { + gcTask.cancel(); + } + } + + private void scheduleGC() { + long delay = hasRun ? REGULAR_GC_DELAY_MS : INITIAL_GC_DELAY_MS; + gcTask = + asyncQueue.enqueueAfterDelay( + AsyncQueue.TimerId.GARBAGE_COLLECTION, + delay, + () -> { + localStore.collectGarbage(LruGarbageCollector.this); + hasRun = true; + scheduleGC(); + }); + } + } + private final LruDelegate delegate; + private final Params params; - LruGarbageCollector(LruDelegate delegate) { + LruGarbageCollector(LruDelegate delegate, Params params) { this.delegate = delegate; + this.params = params; + } + + /** A helper method to create a new scheduler. */ + public Scheduler newScheduler(AsyncQueue asyncQueue, LocalStore localStore) { + return new Scheduler(asyncQueue, localStore); } /** Given a percentile of target to collect, returns the number of targets to collect. */ int calculateQueryCount(int percentile) { - long targetCount = delegate.getTargetCount(); + long targetCount = delegate.getSequenceNumberCount(); return (int) ((percentile / 100.0f) * targetCount); } @@ -66,7 +200,7 @@ long getMaxValue() { } /** Returns the nth sequence number, counting in order from the smallest. */ - long nthSequenceNumber(int count) { + long getNthSequenceNumber(int count) { if (count == 0) { return ListenSequence.INVALID; } @@ -91,4 +225,79 @@ int removeTargets(long upperBound, SparseArray activeTargetIds) { int removeOrphanedDocuments(long upperBound) { return delegate.removeOrphanedDocuments(upperBound); } + + Results collect(SparseArray activeTargetIds) { + if (params.minBytesThreshold == Params.COLLECTION_DISABLED) { + Logger.debug("LruGarbageCollector", "Garbage collection skipped; disabled"); + return Results.DidNotRun(); + } + + long cacheSize = getByteSize(); + if (cacheSize < params.minBytesThreshold) { + Logger.debug( + "LruGarbageCollector", + "Garbage collection skipped; Cache size " + + cacheSize + + " is lower than threshold " + + params.minBytesThreshold); + return Results.DidNotRun(); + } else { + return runGarbageCollection(activeTargetIds); + } + } + + private Results runGarbageCollection(SparseArray liveTargetIds) { + long startTs = System.currentTimeMillis(); + int sequenceNumbers = calculateQueryCount(params.percentileToCollect); + // Cap at the configured max + if (sequenceNumbers > params.maximumSequenceNumbersToCollect) { + Logger.debug( + "LruGarbageCollector", + "Capping sequence numbers to collect down to the maximum of " + + params.maximumSequenceNumbersToCollect + + " from " + + sequenceNumbers); + sequenceNumbers = params.maximumSequenceNumbersToCollect; + } + long countedTargetsTs = System.currentTimeMillis(); + + long upperBound = getNthSequenceNumber(sequenceNumbers); + long foundUpperBoundTs = System.currentTimeMillis(); + + int numTargetsRemoved = removeTargets(upperBound, liveTargetIds); + long removedTargetsTs = System.currentTimeMillis(); + + int numDocumentsRemoved = removeOrphanedDocuments(upperBound); + long removedDocumentsTs = System.currentTimeMillis(); + + if (Logger.isDebugEnabled()) { + String desc = "LRU Garbage Collection:\n"; + desc += "\tCounted targets in " + (countedTargetsTs - startTs) + "ms\n"; + desc += + String.format( + Locale.ROOT, + "\tDetermined least recently used %d sequence numbers in %dms\n", + sequenceNumbers, + (foundUpperBoundTs - countedTargetsTs)); + desc += + String.format( + Locale.ROOT, + "\tRemoved %d targets in %dms\n", + numTargetsRemoved, + (removedTargetsTs - foundUpperBoundTs)); + desc += + String.format( + Locale.ROOT, + "\tRemoved %d documents in %dms\n", + numDocumentsRemoved, + (removedDocumentsTs - removedTargetsTs)); + desc += String.format(Locale.ROOT, "Total Duration: %dms", (removedDocumentsTs - startTs)); + Logger.debug("LruGarbageCollector", desc); + } + return new Results(/* hasRun= */ true, sequenceNumbers, numTargetsRemoved, numDocumentsRemoved); + } + + long getByteSize() { + return delegate.getByteSize(); + } } diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryLruReferenceDelegate.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryLruReferenceDelegate.java index 5376b5f9378..6ff4cb74dcc 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryLruReferenceDelegate.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryLruReferenceDelegate.java @@ -27,19 +27,24 @@ /** Provides LRU garbage collection functionality for MemoryPersistence. */ class MemoryLruReferenceDelegate implements ReferenceDelegate, LruDelegate { private final MemoryPersistence persistence; + private final LocalSerializer serializer; private final Map orphanedSequenceNumbers; private ReferenceSet inMemoryPins; private final LruGarbageCollector garbageCollector; private final ListenSequence listenSequence; private long currentSequenceNumber; - MemoryLruReferenceDelegate(MemoryPersistence persistence) { + MemoryLruReferenceDelegate( + MemoryPersistence persistence, + LruGarbageCollector.Params params, + LocalSerializer serializer) { this.persistence = persistence; + this.serializer = serializer; this.orphanedSequenceNumbers = new HashMap<>(); this.listenSequence = new ListenSequence(persistence.getQueryCache().getHighestListenSequenceNumber()); this.currentSequenceNumber = ListenSequence.INVALID; - this.garbageCollector = new LruGarbageCollector(this); + this.garbageCollector = new LruGarbageCollector(this, params); } @Override @@ -77,14 +82,24 @@ public void forEachTarget(Consumer consumer) { } @Override - public long getTargetCount() { - return persistence.getQueryCache().getTargetCount(); + public long getSequenceNumberCount() { + long targetCount = persistence.getQueryCache().getTargetCount(); + long orphanedCount[] = new long[1]; + forEachOrphanedDocumentSequenceNumber( + sequenceNumber -> { + orphanedCount[0]++; + }); + return targetCount + orphanedCount[0]; } @Override public void forEachOrphanedDocumentSequenceNumber(Consumer consumer) { - for (Long sequenceNumber : orphanedSequenceNumbers.values()) { - consumer.accept(sequenceNumber); + for (Map.Entry entry : orphanedSequenceNumbers.entrySet()) { + // Pass in the exact sequence number as the upper bound so we know it won't be pinned by being + // too recent. + if (!isPinned(entry.getKey(), entry.getValue())) { + consumer.accept(entry.getValue()); + } } } @@ -170,4 +185,18 @@ private boolean isPinned(DocumentKey key, long upperBound) { Long sequenceNumber = orphanedSequenceNumbers.get(key); return sequenceNumber != null && sequenceNumber > upperBound; } + + @Override + public long getByteSize() { + // Note that this method is only used for testing because this delegate is only + // used for testing. The algorithm here (loop through everything, serialize it + // and count bytes) is inefficient and inexact, but won't run in production. + long count = 0; + count += persistence.getQueryCache().getByteSize(serializer); + count += persistence.getRemoteDocumentCache().getByteSize(serializer); + for (MemoryMutationQueue queue : persistence.getMutationQueues()) { + count += queue.getByteSize(serializer); + } + return count; + } } diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryMutationQueue.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryMutationQueue.java index 289bedf2348..05a666691dc 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryMutationQueue.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryMutationQueue.java @@ -371,4 +371,12 @@ private int indexOfExistingBatchId(int batchId, String action) { hardAssert(index >= 0 && index < queue.size(), "Batches must exist to be %s", action); return index; } + + long getByteSize(LocalSerializer serializer) { + long count = 0; + for (MutationBatch batch : queue) { + count += serializer.encodeMutationBatch(batch).getSerializedSize(); + } + return count; + } } diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryPersistence.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryPersistence.java index 5b31f95a247..0f7698c4f03 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryPersistence.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryPersistence.java @@ -44,9 +44,11 @@ public static MemoryPersistence createEagerGcMemoryPersistence() { return persistence; } - public static MemoryPersistence createLruGcMemoryPersistence() { + public static MemoryPersistence createLruGcMemoryPersistence( + LruGarbageCollector.Params params, LocalSerializer serializer) { MemoryPersistence persistence = new MemoryPersistence(); - persistence.setReferenceDelegate(new MemoryLruReferenceDelegate(persistence)); + persistence.setReferenceDelegate( + new MemoryLruReferenceDelegate(persistence, params, serializer)); return persistence; } diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryQueryCache.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryQueryCache.java index e37a451810d..7c0315dc3fe 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryQueryCache.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryQueryCache.java @@ -169,4 +169,12 @@ public ImmutableSortedSet getMatchingKeysForTargetId(int targetId) public boolean containsKey(DocumentKey key) { return references.containsKey(key); } + + long getByteSize(LocalSerializer serializer) { + long count = 0; + for (Map.Entry entry : queries.entrySet()) { + count += serializer.encodeQueryData(entry.getValue()).getSerializedSize(); + } + return count; + } } diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryRemoteDocumentCache.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryRemoteDocumentCache.java index 86c99a56b5f..9bed5c2dd39 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryRemoteDocumentCache.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/MemoryRemoteDocumentCache.java @@ -100,4 +100,28 @@ public ImmutableSortedMap getAllDocumentsMatchingQuery(Qu ImmutableSortedMap getDocuments() { return docs; } + + /** + * Returns an estimate of the number of bytes used to store the given document key in memory. This + * is only an estimate and includes the size of the segments of the path, but not any object + * overhead or path separators. + */ + private static long getKeySize(DocumentKey key) { + ResourcePath path = key.getPath(); + long count = 0; + for (int i = 0; i < path.length(); i++) { + // Strings in java are utf-16, each character is two bytes in memory + count += path.getSegment(i).length() * 2; + } + return count; + } + + long getByteSize(LocalSerializer serializer) { + long count = 0; + for (Map.Entry entry : docs) { + count += getKeySize(entry.getKey()); + count += serializer.encodeMaybeDocument(entry.getValue()).getSerializedSize(); + } + return count; + } } diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLiteLruReferenceDelegate.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLiteLruReferenceDelegate.java index cee90c741cc..d2823ac3990 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLiteLruReferenceDelegate.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLiteLruReferenceDelegate.java @@ -30,10 +30,10 @@ class SQLiteLruReferenceDelegate implements ReferenceDelegate, LruDelegate { private final LruGarbageCollector garbageCollector; private ReferenceSet inMemoryPins; - SQLiteLruReferenceDelegate(SQLitePersistence persistence) { + SQLiteLruReferenceDelegate(SQLitePersistence persistence, LruGarbageCollector.Params params) { this.currentSequenceNumber = ListenSequence.INVALID; this.persistence = persistence; - this.garbageCollector = new LruGarbageCollector(this); + this.garbageCollector = new LruGarbageCollector(this, params); } void start(long highestSequenceNumber) { @@ -70,8 +70,14 @@ public LruGarbageCollector getGarbageCollector() { } @Override - public long getTargetCount() { - return persistence.getQueryCache().getTargetCount(); + public long getSequenceNumberCount() { + long targetCount = persistence.getQueryCache().getTargetCount(); + long orphanedDocumentCount = + persistence + .query( + "SELECT COUNT(*) FROM (SELECT sequence_number FROM target_documents GROUP BY path HAVING COUNT(*) = 1 AND target_id = 0)") + .firstValue(row -> row.getLong(0)); + return targetCount + orphanedDocumentCount; } @Override @@ -179,4 +185,9 @@ private void writeSentinel(DocumentKey key) { path, getCurrentSequenceNumber()); } + + @Override + public long getByteSize() { + return persistence.getByteSize(); + } } diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLitePersistence.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLitePersistence.java index 7ecf6050c52..52799912639 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLitePersistence.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/local/SQLitePersistence.java @@ -34,6 +34,7 @@ import com.google.firebase.firestore.util.Consumer; import com.google.firebase.firestore.util.Logger; import com.google.firebase.firestore.util.Supplier; +import java.io.File; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.util.ArrayList; @@ -76,6 +77,7 @@ public static String databaseName(String persistenceKey, DatabaseId databaseId) private final OpenHelper opener; private final LocalSerializer serializer; private SQLiteDatabase db; + private File databasePath; private boolean started; private final SQLiteQueryCache queryCache; private final SQLiteRemoteDocumentCache remoteDocumentCache; @@ -97,13 +99,18 @@ public void onRollback() {} }; public SQLitePersistence( - Context context, String persistenceKey, DatabaseId databaseId, LocalSerializer serializer) { + Context context, + String persistenceKey, + DatabaseId databaseId, + LocalSerializer serializer, + LruGarbageCollector.Params params) { String databaseName = databaseName(persistenceKey, databaseId); this.opener = new OpenHelper(context, databaseName); + this.databasePath = context.getDatabasePath(databaseName); this.serializer = serializer; this.queryCache = new SQLiteQueryCache(this, this.serializer); this.remoteDocumentCache = new SQLiteRemoteDocumentCache(this, this.serializer); - this.referenceDelegate = new SQLiteLruReferenceDelegate(this); + this.referenceDelegate = new SQLiteLruReferenceDelegate(this, params); } @Override @@ -142,7 +149,7 @@ public boolean isStarted() { } @Override - public ReferenceDelegate getReferenceDelegate() { + public SQLiteLruReferenceDelegate getReferenceDelegate() { return referenceDelegate; } @@ -191,6 +198,10 @@ T runTransaction(String action, Supplier operation) { return value; } + long getByteSize() { + return databasePath.length(); + } + /** * A SQLiteOpenHelper that configures database connections just the way we like them, delegating * to SQLiteSchema to actually do the work of migration. diff --git a/firebase-firestore/src/main/java/com/google/firebase/firestore/util/AsyncQueue.java b/firebase-firestore/src/main/java/com/google/firebase/firestore/util/AsyncQueue.java index bef4ee4b12f..4954c14dc70 100644 --- a/firebase-firestore/src/main/java/com/google/firebase/firestore/util/AsyncQueue.java +++ b/firebase-firestore/src/main/java/com/google/firebase/firestore/util/AsyncQueue.java @@ -68,6 +68,8 @@ public enum TimerId { * set timeout, rather than waiting indefinitely for success or failure. */ ONLINE_STATE_TIMEOUT, + /** A timer used to periodically attempt LRU Garbage collection */ + GARBAGE_COLLECTION } /** diff --git a/firebase-firestore/src/test/java/com/google/firebase/firestore/local/LruGarbageCollectorTestCase.java b/firebase-firestore/src/test/java/com/google/firebase/firestore/local/LruGarbageCollectorTestCase.java index 11cbc6a3493..65bdab22b17 100644 --- a/firebase-firestore/src/test/java/com/google/firebase/firestore/local/LruGarbageCollectorTestCase.java +++ b/firebase-firestore/src/test/java/com/google/firebase/firestore/local/LruGarbageCollectorTestCase.java @@ -55,12 +55,13 @@ public abstract class LruGarbageCollectorTestCase { private MutationQueue mutationQueue; private RemoteDocumentCache documentCache; private LruGarbageCollector garbageCollector; + private LruGarbageCollector.Params lruParams; private int previousTargetId; private int previousDocNum; private long initialSequenceNumber; private ObjectValue testValue; - abstract Persistence createPersistence(); + abstract Persistence createPersistence(LruGarbageCollector.Params params); @Before public void setUp() { @@ -81,7 +82,11 @@ public void tearDown() { } private void newTestResources() { - persistence = createPersistence(); + newTestResources(LruGarbageCollector.Params.Default()); + } + + private void newTestResources(LruGarbageCollector.Params params) { + persistence = createPersistence(params); persistence.getReferenceDelegate().setInMemoryPins(new ReferenceSet()); queryCache = persistence.getQueryCache(); documentCache = persistence.getRemoteDocumentCache(); @@ -89,6 +94,7 @@ private void newTestResources() { mutationQueue = persistence.getMutationQueue(user); initialSequenceNumber = queryCache.getHighestListenSequenceNumber(); garbageCollector = ((LruDelegate) persistence.getReferenceDelegate()).getGarbageCollector(); + lruParams = params; } private QueryData nextQueryData() { @@ -193,7 +199,7 @@ public void testPickSequenceNumberPercentile() { @Test public void testSequenceNumberNoQueries() { - assertEquals(ListenSequence.INVALID, garbageCollector.nthSequenceNumber(0)); + assertEquals(ListenSequence.INVALID, garbageCollector.getNthSequenceNumber(0)); } @Test @@ -203,7 +209,7 @@ public void testSequenceNumberForFiftyQueries() { for (int i = 0; i < 50; i++) { addNextQuery(); } - assertEquals(initialSequenceNumber + 10, garbageCollector.nthSequenceNumber(10)); + assertEquals(initialSequenceNumber + 10, garbageCollector.getNthSequenceNumber(10)); } @Test @@ -220,7 +226,7 @@ public void testSequenceNumberForMultipleQueriesInATransaction() { for (int i = 9; i < 50; i++) { addNextQuery(); } - assertEquals(2 + initialSequenceNumber, garbageCollector.nthSequenceNumber(10)); + assertEquals(2 + initialSequenceNumber, garbageCollector.getNthSequenceNumber(10)); } @Test @@ -240,7 +246,7 @@ public void testAllCollectedQueriesInSingleTransaction() { for (int i = 11; i < 50; i++) { addNextQuery(); } - assertEquals(1 + initialSequenceNumber, garbageCollector.nthSequenceNumber(10)); + assertEquals(1 + initialSequenceNumber, garbageCollector.getNthSequenceNumber(10)); } @Test @@ -251,7 +257,7 @@ public void testSequenceNumbersWithMutationAndSequentialQueries() { for (int i = 0; i < 50; i++) { addNextQuery(); } - assertEquals(10 + initialSequenceNumber, garbageCollector.nthSequenceNumber(10)); + assertEquals(10 + initialSequenceNumber, garbageCollector.getNthSequenceNumber(10)); } @Test @@ -279,7 +285,7 @@ public void testSequenceNumbersWithMutationsInQueries() { addDocumentToTarget(docInQuery, queryData.getTargetId()); }); // This should catch the remaining 8 documents, plus the first two queries we added. - assertEquals(3 + initialSequenceNumber, garbageCollector.nthSequenceNumber(10)); + assertEquals(3 + initialSequenceNumber, garbageCollector.getNthSequenceNumber(10)); } @Test @@ -587,4 +593,105 @@ public void testRemoveTargetsThenGC() { } }); } + + @Test + public void testGetsSize() { + long initialSize = garbageCollector.getByteSize(); + + persistence.runTransaction( + "fill cache", + () -> { + // Simulate a bunch of ack'd mutations + for (int i = 0; i < 50; i++) { + Document doc = cacheADocumentInTransaction(); + markDocumentEligibleForGcInTransaction(doc.getKey()); + } + }); + + long finalSize = garbageCollector.getByteSize(); + assertTrue(finalSize > initialSize); + } + + @Test + public void testDisabled() { + LruGarbageCollector.Params params = LruGarbageCollector.Params.Disabled(); + + // Switch out the test resources for ones with a disabled GC. + persistence.shutdown(); + newTestResources(params); + + persistence.runTransaction( + "Fill cache", + () -> { + // Simulate a bunch of ack'd mutations + for (int i = 0; i < 500; i++) { + Document doc = cacheADocumentInTransaction(); + markDocumentEligibleForGcInTransaction(doc.getKey()); + } + }); + + LruGarbageCollector.Results results = + persistence.runTransaction("GC", () -> garbageCollector.collect(new SparseArray<>())); + + assertFalse(results.hasRun()); + } + + @Test + public void testCacheTooSmall() { + // Default LRU Params are ok for this test. + + persistence.runTransaction( + "Fill cache", + () -> { + // Simulate a bunch of ack'd mutations + for (int i = 0; i < 50; i++) { + Document doc = cacheADocumentInTransaction(); + markDocumentEligibleForGcInTransaction(doc.getKey()); + } + }); + + // Make sure we're under the target size + long cacheSize = garbageCollector.getByteSize(); + assertTrue(cacheSize < lruParams.minBytesThreshold); + + LruGarbageCollector.Results results = + persistence.runTransaction("GC", () -> garbageCollector.collect(new SparseArray<>())); + + assertFalse(results.hasRun()); + } + + @Test + public void testGCRan() { + // Set a low byte threshold so we can guarantee that GC will run. + LruGarbageCollector.Params params = LruGarbageCollector.Params.WithCacheSizeBytes(100); + + // Switch to persistence using our new params. + persistence.shutdown(); + newTestResources(params); + + // Add 100 targets and 10 documents to each + for (int i = 0; i < 100; i++) { + // Use separate transactions so that each target and associated documents get their own + // sequence number. + persistence.runTransaction( + "Add a target and some documents", + () -> { + QueryData queryData = addNextQueryInTransaction(); + for (int j = 0; j < 10; j++) { + Document doc = cacheADocumentInTransaction(); + addDocumentToTarget(doc.getKey(), queryData.getTargetId()); + } + }); + } + + // Mark nothing as live, so everything is eligible. + LruGarbageCollector.Results results = + persistence.runTransaction("GC", () -> garbageCollector.collect(new SparseArray<>())); + + // By default, we collect 10% of the sequence numbers. Since we added 100 targets, + // that should be 10 targets with 10 documents each, for a total of 100 documents. + assertTrue(results.hasRun()); + assertEquals(10, results.getTargetsRemoved()); + assertEquals(100, results.getDocumentsRemoved()); + } } diff --git a/firebase-firestore/src/test/java/com/google/firebase/firestore/local/MemoryLruGarbageCollectorTest.java b/firebase-firestore/src/test/java/com/google/firebase/firestore/local/MemoryLruGarbageCollectorTest.java index 98074286836..52ed3f456b9 100644 --- a/firebase-firestore/src/test/java/com/google/firebase/firestore/local/MemoryLruGarbageCollectorTest.java +++ b/firebase-firestore/src/test/java/com/google/firebase/firestore/local/MemoryLruGarbageCollectorTest.java @@ -22,7 +22,7 @@ @Config(manifest = Config.NONE) public class MemoryLruGarbageCollectorTest extends LruGarbageCollectorTestCase { @Override - Persistence createPersistence() { - return PersistenceTestHelpers.createLRUMemoryPersistence(); + Persistence createPersistence(LruGarbageCollector.Params params) { + return PersistenceTestHelpers.createLRUMemoryPersistence(params); } } diff --git a/firebase-firestore/src/test/java/com/google/firebase/firestore/local/PersistenceTestHelpers.java b/firebase-firestore/src/test/java/com/google/firebase/firestore/local/PersistenceTestHelpers.java index e3dfb2669e5..f816937667b 100644 --- a/firebase-firestore/src/test/java/com/google/firebase/firestore/local/PersistenceTestHelpers.java +++ b/firebase-firestore/src/test/java/com/google/firebase/firestore/local/PersistenceTestHelpers.java @@ -25,10 +25,16 @@ public final class PersistenceTestHelpers { private static int databaseNameCounter = 0; public static SQLitePersistence openSQLitePersistence(String name) { + return openSQLitePersistence(name, LruGarbageCollector.Params.Default()); + } + + public static SQLitePersistence openSQLitePersistence( + String name, LruGarbageCollector.Params params) { DatabaseId databaseId = DatabaseId.forProject("projectId"); LocalSerializer serializer = new LocalSerializer(new RemoteSerializer(databaseId)); Context context = RuntimeEnvironment.application; - SQLitePersistence persistence = new SQLitePersistence(context, name, databaseId, serializer); + SQLitePersistence persistence = + new SQLitePersistence(context, name, databaseId, serializer, params); persistence.start(); return persistence; } @@ -43,10 +49,14 @@ public static String nextSQLiteDatabaseName() { * @return a new SQLitePersistence with an empty database and an up-to-date schema. */ public static SQLitePersistence createSQLitePersistence() { + return createSQLitePersistence(LruGarbageCollector.Params.Default()); + } + + public static SQLitePersistence createSQLitePersistence(LruGarbageCollector.Params params) { // Robolectric's test runner will clear out the application database directory in between test // cases, but sometimes (particularly the spec tests) we create multiple databases per test // case and each should be fresh. A unique name is sufficient to keep these separate. - return openSQLitePersistence(nextSQLiteDatabaseName()); + return openSQLitePersistence(nextSQLiteDatabaseName(), params); } /** Creates and starts a new MemoryPersistence instance for testing. */ @@ -57,7 +67,14 @@ public static MemoryPersistence createEagerGCMemoryPersistence() { } public static MemoryPersistence createLRUMemoryPersistence() { - MemoryPersistence persistence = MemoryPersistence.createLruGcMemoryPersistence(); + return createLRUMemoryPersistence(LruGarbageCollector.Params.Default()); + } + + public static MemoryPersistence createLRUMemoryPersistence(LruGarbageCollector.Params params) { + DatabaseId databaseId = DatabaseId.forProject("projectId"); + LocalSerializer serializer = new LocalSerializer(new RemoteSerializer(databaseId)); + MemoryPersistence persistence = + MemoryPersistence.createLruGcMemoryPersistence(params, serializer); persistence.start(); return persistence; } diff --git a/firebase-firestore/src/test/java/com/google/firebase/firestore/local/SQLiteLruGarbageCollectorTest.java b/firebase-firestore/src/test/java/com/google/firebase/firestore/local/SQLiteLruGarbageCollectorTest.java index 792cce7c096..e3e2838eeff 100644 --- a/firebase-firestore/src/test/java/com/google/firebase/firestore/local/SQLiteLruGarbageCollectorTest.java +++ b/firebase-firestore/src/test/java/com/google/firebase/firestore/local/SQLiteLruGarbageCollectorTest.java @@ -22,7 +22,7 @@ @Config(manifest = Config.NONE) public class SQLiteLruGarbageCollectorTest extends LruGarbageCollectorTestCase { @Override - Persistence createPersistence() { - return PersistenceTestHelpers.createSQLitePersistence(); + Persistence createPersistence(LruGarbageCollector.Params params) { + return PersistenceTestHelpers.createSQLitePersistence(params); } } diff --git a/firebase-firestore/src/test/java/com/google/firebase/firestore/spec/SQLiteSpecTest.java b/firebase-firestore/src/test/java/com/google/firebase/firestore/spec/SQLiteSpecTest.java index 011215b7454..c8940dbc14e 100644 --- a/firebase-firestore/src/test/java/com/google/firebase/firestore/spec/SQLiteSpecTest.java +++ b/firebase-firestore/src/test/java/com/google/firebase/firestore/spec/SQLiteSpecTest.java @@ -14,6 +14,7 @@ package com.google.firebase.firestore.spec; +import com.google.firebase.firestore.local.LruGarbageCollector; import com.google.firebase.firestore.local.Persistence; import com.google.firebase.firestore.local.PersistenceTestHelpers; import java.util.Set; @@ -43,7 +44,8 @@ protected void specTearDown() throws Exception { @Override Persistence getPersistence(boolean garbageCollectionEnabled) { - return PersistenceTestHelpers.openSQLitePersistence(databaseName); + return PersistenceTestHelpers.openSQLitePersistence( + databaseName, LruGarbageCollector.Params.Default()); } @Override