Cache support unbounded requests.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=131696858

library/src/androidTest/java/com/google/android/exoplayer2/upstream/cache/CacheDataSourceTest.java

@@ -0,0 +1,190 @@
+/*
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.upstream.cache;
+
+import android.net.Uri;
+import android.test.InstrumentationTestCase;
+import android.test.MoreAsserts;
+import com.google.android.exoplayer2.C;
+import com.google.android.exoplayer2.testutil.FakeDataSource;
+import com.google.android.exoplayer2.testutil.FakeDataSource.Builder;
+import com.google.android.exoplayer2.testutil.TestUtil;
+import com.google.android.exoplayer2.upstream.DataSpec;
+import com.google.android.exoplayer2.upstream.HttpDataSource.InvalidResponseCodeException;
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+
+/** Unit tests for {@link CacheDataSource}. */
+public class CacheDataSourceTest extends InstrumentationTestCase {
+
+  private static final byte[] TEST_DATA = new byte[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+  private static final int MAX_CACHE_FILE_SIZE = 3;
+  private static final String KEY_1 = "key 1";
+  private static final String KEY_2 = "key 2";
+
+  private File cacheDir;
+  private SimpleCache simpleCache;
+
+  @Override
+  protected void setUp() throws Exception {
+    // Create a temporary folder
+    cacheDir = File.createTempFile("CacheDataSourceTest", null);
+    assertTrue(cacheDir.delete());
+    assertTrue(cacheDir.mkdir());
+
+    simpleCache = new SimpleCache(cacheDir, new NoOpCacheEvictor());
+  }
+
+  @Override
+  protected void tearDown() throws Exception {
+    TestUtil.recursiveDelete(cacheDir);
+  }
+
+  public void testMaxCacheFileSize() throws Exception {
+    CacheDataSource cacheDataSource = createCacheDataSource(false, false, false);
+    assertReadDataContentLength(cacheDataSource, false, false);
+    assertEquals((int) Math.ceil((double) TEST_DATA.length / MAX_CACHE_FILE_SIZE),
+        cacheDir.listFiles().length);
+  }
+
+  public void testCacheAndRead() throws Exception {
+    assertCacheAndRead(false, false);
+  }
+
+  public void testCacheAndReadUnboundedRequest() throws Exception {
+    assertCacheAndRead(true, false);
+  }
+
+  public void testCacheAndReadUnknownLength() throws Exception {
+    assertCacheAndRead(false, true);
+  }
+
+  // Disabled test as we don't support caching of definitely unknown length content
+  public void disabledTestCacheAndReadUnboundedRequestUnknownLength() throws Exception {
+    assertCacheAndRead(true, true);
+  }
+
+  public void testUnsatisfiableRange() throws Exception {
+    // Bounded request but the content length is unknown. This forces all data to be cached but not
+    // the length
+    assertCacheAndRead(false, true);
+
+    // Now do an unbounded request. This will read all of the data from cache and then try to read
+    // more from upstream which will cause to a 416 so CDS will store the length.
+    CacheDataSource cacheDataSource = createCacheDataSource(true, true, true);
+    assertReadDataContentLength(cacheDataSource, true, true);
+
+    // If the user try to access off range then it should throw an IOException
+    try {
+      cacheDataSource = createCacheDataSource(false, false, false);
+      cacheDataSource.open(new DataSpec(Uri.EMPTY, TEST_DATA.length, 5, KEY_1));
+      fail();
+    } catch (TestIOException e) {
+      // success
+    }
+  }
+
+  public void testContentLengthEdgeCases() throws Exception {
+    // Read partial at EOS but don't cross it so length is unknown
+    CacheDataSource cacheDataSource = createCacheDataSource(false, false, true);
+    assertReadData(cacheDataSource, true, TEST_DATA.length - 2, 2);
+    assertEquals(C.LENGTH_UNSET, simpleCache.getContentLength(KEY_1));
+
+    // Now do an unbounded request for whole data. This will cause a bounded request from upstream.
+    // End of data from upstream shouldn't be mixed up with EOS and cause length set wrong.
+    cacheDataSource = createCacheDataSource(true, false, true);
+    assertReadDataContentLength(cacheDataSource, true, true);
+
+    // Now the length set correctly do an unbounded request with offset
+    assertEquals(2, cacheDataSource.open(new DataSpec(Uri.EMPTY, TEST_DATA.length - 2,
+        C.LENGTH_UNSET, KEY_1)));
+
+    // An unbounded request with offset for not cached content
+    assertEquals(C.LENGTH_UNSET, cacheDataSource.open(new DataSpec(Uri.EMPTY, TEST_DATA.length - 2,
+        C.LENGTH_UNSET, KEY_2)));
+  }
+
+  private void assertCacheAndRead(boolean unboundedRequest, boolean simulateUnknownLength)
+      throws IOException {
+    // Read all data from upstream and cache
+    CacheDataSource cacheDataSource = createCacheDataSource(false, false, simulateUnknownLength);
+    assertReadDataContentLength(cacheDataSource, unboundedRequest, simulateUnknownLength);
+
+    // Just read from cache
+    cacheDataSource = createCacheDataSource(false, true, simulateUnknownLength);
+    assertReadDataContentLength(cacheDataSource, unboundedRequest,
+        false /*length is already cached*/);
+  }
+
+  /**
+   * Reads data until EOI and compares it to {@link #TEST_DATA}. Also checks content length returned
+   * from open() call and the cached content length.
+   */
+  private void assertReadDataContentLength(CacheDataSource cacheDataSource,
+      boolean unboundedRequest, boolean unknownLength) throws IOException {
+    int length = unboundedRequest ? C.LENGTH_UNSET : TEST_DATA.length;
+    assertReadData(cacheDataSource, unknownLength, 0, length);
+    assertEquals("When the range specified, CacheDataSource doesn't reach EOS so shouldn't cache "
+        + "content length", !unboundedRequest ? C.LENGTH_UNSET : TEST_DATA.length,
+        simpleCache.getContentLength(KEY_1));
+  }
+
+  private void assertReadData(CacheDataSource cacheDataSource, boolean unknownLength, int position,
+      int length) throws IOException {
+    int actualLength = TEST_DATA.length - position;
+    if (length != C.LENGTH_UNSET) {
+      actualLength = Math.min(actualLength, length);
+    }
+    assertEquals(unknownLength ? length : actualLength,
+        cacheDataSource.open(new DataSpec(Uri.EMPTY, position, length, KEY_1)));
+
+    byte[] buffer = new byte[100];
+    int index = 0;
+    while (true) {
+      int read = cacheDataSource.read(buffer, index, buffer.length - index);
+      if (read == C.RESULT_END_OF_INPUT) {
+        break;
+      }
+      index += read;
+    }
+    assertEquals(actualLength, index);
+    MoreAsserts.assertEquals(Arrays.copyOfRange(TEST_DATA, position, position + actualLength),
+        Arrays.copyOf(buffer, index));
+
+    cacheDataSource.close();
+  }
+
+  private CacheDataSource createCacheDataSource(boolean set416exception, boolean setReadException,
+      boolean simulateUnknownLength) {
+    Builder builder = new Builder();
+    if (setReadException) {
+      builder.appendReadError(new IOException("Shouldn't read from upstream"));
+    }
+    builder.setSimulateUnknownLength(simulateUnknownLength);
+    builder.appendReadData(TEST_DATA);
+    FakeDataSource upstream = builder.build();
+    upstream.setUnsatisfiableRangeException(set416exception
+        ? new InvalidResponseCodeException(416, null, null)
+        : new TestIOException());
+    return new CacheDataSource(simpleCache, upstream,
+        CacheDataSource.FLAG_BLOCK_ON_CACHE | CacheDataSource.FLAG_CACHE_UNBOUNDED_REQUESTS,
+        MAX_CACHE_FILE_SIZE);
+  }
+
+  private static class TestIOException extends IOException {}
+
+}

library/src/androidTest/java/com/google/android/exoplayer2/upstream/cache/SimpleCacheTest.java

@@ -0,0 +1,123 @@
+/*
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.upstream.cache;
+
+import android.test.InstrumentationTestCase;
+
+import com.google.android.exoplayer2.C;
+import com.google.android.exoplayer2.testutil.TestUtil;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.NavigableSet;
+import java.util.Set;
+
+/**
+ * Unit tests for {@link SimpleCache}.
+ */
+public class SimpleCacheTest extends InstrumentationTestCase {
+
+  private static final String KEY_1 = "key1";
+
+  private File cacheDir;
+
+  @Override
+  protected void setUp() throws Exception {
+    // Create a temporary folder
+    cacheDir = File.createTempFile("SimpleCacheTest", null);
+    assertTrue(cacheDir.delete());
+    assertTrue(cacheDir.mkdir());
+  }
+
+  @Override
+  protected void tearDown() throws Exception {
+    TestUtil.recursiveDelete(cacheDir);
+  }
+
+  public void testCommittingOneFile() throws Exception {
+    SimpleCache simpleCache = new SimpleCache(cacheDir, new NoOpCacheEvictor());
+
+    CacheSpan cacheSpan = simpleCache.startReadWrite(KEY_1, 0);
+    assertFalse(cacheSpan.isCached);
+    assertTrue(cacheSpan.isOpenEnded());
+
+    assertNull(simpleCache.startReadWriteNonBlocking(KEY_1, 0));
+
+    assertEquals(0, simpleCache.getKeys().size());
+    NavigableSet<CacheSpan> cachedSpans = simpleCache.getCachedSpans(KEY_1);
+    assertTrue(cachedSpans == null || cachedSpans.size() == 0);
+    assertEquals(0, simpleCache.getCacheSpace());
+    assertEquals(0, cacheDir.listFiles().length);
+
+    addCache(simpleCache, 0, 15);
+
+    Set<String> cachedKeys = simpleCache.getKeys();
+    assertEquals(1, cachedKeys.size());
+    assertTrue(cachedKeys.contains(KEY_1));
+    cachedSpans = simpleCache.getCachedSpans(KEY_1);
+    assertEquals(1, cachedSpans.size());
+    assertTrue(cachedSpans.contains(cacheSpan));
+    assertEquals(15, simpleCache.getCacheSpace());
+
+    cacheSpan = simpleCache.startReadWrite(KEY_1, 0);
+    assertTrue(cacheSpan.isCached);
+    assertFalse(cacheSpan.isOpenEnded());
+    assertEquals(15, cacheSpan.length);
+  }
+
+  public void testSetGetLength() throws Exception {
+    SimpleCache simpleCache = new SimpleCache(cacheDir, new NoOpCacheEvictor());
+
+    assertEquals(C.LENGTH_UNSET, simpleCache.getContentLength(KEY_1));
+    assertTrue(simpleCache.setContentLength(KEY_1, 15));
+    assertEquals(15, simpleCache.getContentLength(KEY_1));
+
+    simpleCache.startReadWrite(KEY_1, 0);
+
+    addCache(simpleCache, 0, 15);
+
+    assertTrue(simpleCache.setContentLength(KEY_1, 150));
+    assertEquals(150, simpleCache.getContentLength(KEY_1));
+
+    addCache(simpleCache, 140, 10);
+
+    // Try to set length shorter then the content
+    assertFalse(simpleCache.setContentLength(KEY_1, 15));
+    assertEquals("Content length should be unchanged.",
+        150, simpleCache.getContentLength(KEY_1));
+
+    /* TODO Enable when the length persistance is fixed
+    // Check if values are kept after cache is reloaded.
+    simpleCache = new SimpleCache(cacheDir, new NoOpCacheEvictor());
+    assertEquals(150, simpleCache.getContentLength(KEY_1));
+    CacheSpan lastSpan = simpleCache.startReadWrite(KEY_1, 145);
+
+    // Removing the last span shouldn't cause the length be change next time cache loaded
+    simpleCache.removeSpan(lastSpan);
+    simpleCache = new SimpleCache(cacheDir, new NoOpCacheEvictor());
+    assertEquals(150, simpleCache.getContentLength(KEY_1));
+     */
+  }
+
+  private void addCache(SimpleCache simpleCache, int position, int length) throws IOException {
+    File file = simpleCache.startFile(KEY_1, position, length);
+    FileOutputStream fos = new FileOutputStream(file);
+    fos.write(new byte[length]);
+    fos.close();
+    simpleCache.commitFile(file);
+  }
+
+}

library/src/main/java/com/google/android/exoplayer2/upstream/cache/Cache.java

@@ -143,11 +143,11 @@ interface Listener {
    *
    * @param key The cache key for the data.
    * @param position The starting position of the data.
-   * @param length The length of the data to be written. Used only to ensure that there is enough
-   *     space in the cache.
+   * @param maxLength The maximum length of the data to be written. Used only to ensure that there
+   *     is enough space in the cache.
    * @return The file into which data should be written.
    */
-  File startFile(String key, long position, long length);
+  File startFile(String key, long position, long maxLength);
 
   /**
    * Commits a file into the cache. Must only be called when holding a corresponding hole
@@ -182,4 +182,22 @@ interface Listener {
   */
   boolean isCached(String key, long position, long length);
 
+  /**
+   * Sets the content length for the given key.
+   *
+   * @param key The cache key for the data.
+   * @param length The length of the data.
+   * @return Whether the length was set successfully. Returns false if the length conflicts with the
+   *     existing contents of the cache.
+   */
+  boolean setContentLength(String key, long length);
+
+  /**
+   * Returns the content length for the given key if one set, or {@link
+   * com.google.android.exoplayer2.C#LENGTH_UNSET} otherwise.
+   *
+   * @param key The cache key for the data.
+   */
+  long getContentLength(String key);
+
 }

library/src/main/java/com/google/android/exoplayer2/upstream/cache/CacheDataSink.java

@@ -64,12 +64,12 @@ public CacheDataSink(Cache cache, long maxCacheFileSize) {
 
   @Override
   public void open(DataSpec dataSpec) throws CacheDataSinkException {
-    // TODO: Support caching for unbounded requests. See TODO in {@link CacheDataSource} for
-    // more details.
-    Assertions.checkState(dataSpec.length != C.LENGTH_UNSET);
+    this.dataSpec = dataSpec;
+    if (dataSpec.length == C.LENGTH_UNSET) {
+      return;
+    }
+    dataSpecBytesWritten = 0;
     try {
-      this.dataSpec = dataSpec;
-      dataSpecBytesWritten = 0;
       openNextOutputStream();
     } catch (FileNotFoundException e) {
       throw new CacheDataSinkException(e);
@@ -78,6 +78,9 @@ public void open(DataSpec dataSpec) throws CacheDataSinkException {
 
   @Override
   public void write(byte[] buffer, int offset, int length) throws CacheDataSinkException {
+    if (dataSpec.length == C.LENGTH_UNSET) {
+      return;
+    }
     try {
       int bytesWritten = 0;
       while (bytesWritten < length) {
@@ -99,6 +102,9 @@ public void write(byte[] buffer, int offset, int length) throws CacheDataSinkExc
 
   @Override
   public void close() throws CacheDataSinkException {
+    if (dataSpec == null || dataSpec.length == C.LENGTH_UNSET) {
+      return;
+    }
     try {
       closeCurrentOutputStream();
     } catch (IOException e) {