Skip to content

Commit

Permalink
s3 external storage listing rework
Browse files Browse the repository at this point in the history 
Signed-off-by: Robin Appelman <robin@icewind.nl>
  • Loading branch information
@icewind1991
icewind1991 committed Oct 15, 2021
1 parent fadeae8 commit 09ffac5
Show file tree
Hide file tree
Showing 3 changed files with 213 additions and 97 deletions.
65 changes: 65 additions & 0 deletions .github/workflows/s3-external.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
name: S3 External storage
on:
push:
branches:
- master
- stable*
paths:
- 'apps/files_external/**'
pull_request:
paths:
- 'apps/files_external/**'

env:
APP_NAME: files_external

jobs:
s3-external-tests:
runs-on: ubuntu-latest

strategy:
# do not stop on another job's failure
fail-fast: false
matrix:
php-versions: ['7.4', '8.0']

name: php${{ matrix.php-versions }}-${{ matrix.ftpd }}

services:
minio:
image: minio/minio:RELEASE.2021-10-06T23-36-31Z
ports:
- "9000:9000"

steps:
- name: Checkout server
uses: actions/checkout@v2
with:
submodules: true

- name: Set up php ${{ matrix.php-versions }}
uses: shivammathur/setup-php@v2
with:
php-version: ${{ matrix.php-versions }}
tools: phpunit
extensions: mbstring, iconv, fileinfo, intl, sqlite, pdo_sqlite, zip, gd

- name: Set up Nextcloud
run: |
mkdir data
./occ maintenance:install --verbose --database=sqlite --database-name=nextcloud --database-host=127.0.0.1 --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password
./occ app:enable --force ${{ env.APP_NAME }}
php -S localhost:8080 &
- name: PHPUnit
run: |
echo "<?php return ['run' => true,'hostname' => 'localhost','key' => 'minioadmin','secret' => 'minioadmin', 'bucket' => 'bucket', 'port' => 9000, 'use_ssl' => false, 'autocreate' => true, 'use_path_style' => true];" > apps/${{ env.APP_NAME }}/tests/config.amazons3.php
phpunit --configuration tests/phpunit-autotest-external.xml apps/files_external/tests/Storage/Amazons3Test.php
s3-external-summary:
runs-on: ubuntu-latest
needs: s3-external-tests

if: always()

steps:
- name: Summary status
run: if ${{ needs.s3-external-tests.result != 'success' }}; then exit 1; fi
243 changes: 147 additions & 96 deletions apps/files_external/lib/Lib/Storage/AmazonS3.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
use Icewind\Streams\IteratorDirectory;
use OC\Cache\CappedMemoryCache;
use OC\Files\Cache\CacheEntry;
use OC\Files\Filesystem;
use OC\Files\ObjectStore\S3ConnectionTrait;
use OC\Files\ObjectStore\S3ObjectTrait;
use OCP\Constants;
Expand All @@ -71,6 +72,9 @@ public function needsPartFile() {
/** @var IMimeTypeDetector */
private $mimeDetector;

/** @var bool|null */
private $versioningEnabled = null;

public function __construct($parameters) {
parent::__construct($parameters);
$this->parseParams($parameters);
Expand Down Expand Up @@ -120,20 +124,28 @@ private function invalidateCache($key) {
unset($this->objectCache[$existingKey]);
}
}
unset($this->directoryCache[$key], $this->filesCache[$key]);
unset($this->filesCache[$key]);
$keys = array_keys($this->directoryCache->getData());
$keyLength = strlen($key);
foreach ($keys as $existingKey) {
if (substr($existingKey, 0, $keyLength) === $key) {
unset($this->directoryCache[$existingKey]);
}
}
unset($this->directoryCache[$key]);
}

/**
* @param $key
* @return Result|boolean
* @return array|false
*/
private function headObject($key) {
if (!isset($this->objectCache[$key])) {
try {
$this->objectCache[$key] = $this->getConnection()->headObject([
'Bucket' => $this->bucket,
'Key' => $key
]);
])->toArray();
} catch (S3Exception $e) {
if ($e->getStatusCode() >= 500) {
throw $e;
Expand All @@ -159,32 +171,44 @@ private function headObject($key) {
* @throws \Exception
*/
private function doesDirectoryExist($path) {
if (!isset($this->directoryCache[$path])) {
if ($path === '.' || $path === '') {
return true;
}

if (isset($this->directoryCache[$path])) {
return $this->directoryCache[$path];
}
try {
// Maybe this isn't an actual key, but a prefix.
// Do a prefix listing of objects to determine.
try {
$result = $this->getConnection()->listObjects([
'Bucket' => $this->bucket,
'Prefix' => rtrim($path, '/'),
'MaxKeys' => 1,
'Delimiter' => '/',
]);
$result = $this->getConnection()->listObjectsV2([
'Bucket' => $this->bucket,
'Prefix' => rtrim($path, '/'),
'MaxKeys' => 1,
]);

if ((isset($result['Contents'][0]['Key']) && $result['Contents'][0]['Key'] === rtrim($path, '/') . '/')
|| isset($result['CommonPrefixes'])) {
$this->directoryCache[$path] = true;
} else {
$this->directoryCache[$path] = false;
}
} catch (S3Exception $e) {
if ($e->getStatusCode() === 403) {
$this->directoryCache[$path] = false;
}
throw $e;
if (isset($result['Contents'])) {
$this->directoryCache[$path] = true;
return true;
}

// empty directories have their own object
$object = $this->headObject($path);

if ($object) {
$this->directoryCache[$path] = true;
return true;
}
} catch (S3Exception $e) {
if ($e->getStatusCode() === 403) {
$this->directoryCache[$path] = false;
}
throw $e;
}

return $this->directoryCache[$path];

$this->directoryCache[$path] = false;
return false;
}

/**
Expand Down Expand Up @@ -284,7 +308,9 @@ public function rmdir($path) {
protected function clearBucket() {
$this->clearCache();
try {
$this->getConnection()->clearBucket($this->bucket);
$this->getConnection()->clearBucket([
"Bucket" => $this->bucket
]);
return true;
// clearBucket() is not working with Ceph, so if it fails we try the slower approach
} catch (\Exception $e) {
Expand Down Expand Up @@ -318,7 +344,9 @@ private function batchDelete($path = null) {
}
// we reached the end when the list is no longer truncated
} while ($objects['IsTruncated']);
$this->deleteObject($path);
if ($path !== '' && $path !== null) {
$this->deleteObject($path);
}
} catch (S3Exception $e) {
\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
return false;
Expand All @@ -327,88 +355,32 @@ private function batchDelete($path = null) {
}

public function opendir($path) {
$path = $this->normalizePath($path);

if ($this->isRoot($path)) {
$path = '';
} else {
$path .= '/';
}

try {
$files = [];
$results = $this->getConnection()->getPaginator('ListObjects', [
'Bucket' => $this->bucket,
'Delimiter' => '/',
'Prefix' => $path,
]);

foreach ($results as $result) {
// sub folders
if (is_array($result['CommonPrefixes'])) {
foreach ($result['CommonPrefixes'] as $prefix) {
$directoryName = trim($prefix['Prefix'], '/');
$files[] = substr($directoryName, strlen($path));
$this->directoryCache[$directoryName] = true;
}
}
if (is_array($result['Contents'])) {
foreach ($result['Contents'] as $object) {
if (isset($object['Key']) && $object['Key'] === $path) {
// it's the directory itself, skip
continue;
}
$file = basename(
isset($object['Key']) ? $object['Key'] : $object['Prefix']
);
$files[] = $file;

// store this information for later usage
$this->filesCache[$path . $file] = [
'ContentLength' => $object['Size'],
'LastModified' => (string)$object['LastModified'],
];
}
}
}

return IteratorDirectory::wrap($files);
} catch (S3Exception $e) {
\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
$content = iterator_to_array($this->getDirectoryContent($path));
return IteratorDirectory::wrap(array_map(function (array $item) {
return $item['name'];
}, $content));
} catch (S3Exception $e) {
return false;
}
}

public function stat($path) {
$path = $this->normalizePath($path);

try {
$stat = [];
if ($this->is_dir($path)) {
$cacheEntry = $this->getCache()->get($path);
if ($cacheEntry instanceof CacheEntry) {
$stat['size'] = $cacheEntry->getSize();
$stat['mtime'] = $cacheEntry->getMTime();
} else {
// Use dummy values
$stat['size'] = -1; // Pending
$stat['mtime'] = time();
}
} else {
$stat['size'] = $this->getContentLength($path);
$stat['mtime'] = strtotime($this->getLastModified($path));
if ($this->is_dir($path)) {
$stat = $this->getDirectoryMetaData($path);
} else {
$object = $this->headObject($path);
if ($object === false) {
return false;
}
$stat['atime'] = time();

return $stat;
} catch (S3Exception $e) {
\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
return false;
$object["Key"] = $path;
$stat = $this->objectToMetaData($object);
}
}
$stat['atime'] = time();

public function hasUpdated($path, $time) {
return $this->getMountOption('filesystem_check_changes', 1) === 1 || parent::hasUpdated($path, $time);
return $stat;
}

/**
Expand Down Expand Up @@ -711,4 +683,83 @@ public function writeBack($tmpFile, $path) {
public static function checkDependencies() {
return true;
}

public function getDirectoryContent($directory): \Traversable {
$path = $this->normalizePath($directory);

if ($this->isRoot($path)) {
$path = '';
} else {
$path .= '/';
}

$results = $this->getConnection()->getPaginator('ListObjectsV2', [
'Bucket' => $this->bucket,
'Delimiter' => '/',
'Prefix' => $path,
]);

foreach ($results as $result) {
// sub folders
if (is_array($result['CommonPrefixes'])) {
foreach ($result['CommonPrefixes'] as $prefix) {
$dir = $this->getDirectoryMetaData($prefix['Prefix']);
if ($dir) {
yield $dir;
}
}
}
if (is_array($result['Contents'])) {
foreach ($result['Contents'] as $object) {
$this->objectCache[$object['Key']] = $object;
if ($object['Key'] !== $path) {
yield $this->objectToMetaData($object);
}
}
}
}
}

private function objectToMetaData(array $object): array {
return [
'name' => basename($object['Key']),
'mimetype' => $this->mimeDetector->detectPath($object['Key']),
'mtime' => strtotime($object['LastModified']),
'storage_mtime' => strtotime($object['LastModified']),
'etag' => $object['ETag'],
'permissions' => Constants::PERMISSION_ALL - Constants::PERMISSION_CREATE,
'size' => (int)($object['Size'] ?? $object['ContentLength']),
];
}

private function getDirectoryMetaData(string $path): ?array {
$path = trim($path, '/');
// when versioning is enabled, delete markers are returned as part of CommonPrefixes
// resulting in "ghost" folders, verify that each folder actually exists
if ($this->versioningEnabled() && !$this->doesDirectoryExist($path)) {
return null;
}
$cacheEntry = $this->getCache()->get($path);
if ($cacheEntry instanceof CacheEntry) {
return $cacheEntry->getData();
} else {
return [
'name' => basename($path),
'mimetype' => 'httpd/unix-directory',
'mtime' => time(),
'storage_mtime' => time(),
'etag' => uniqid(),
'permissions' => Constants::PERMISSION_ALL,
'size' => -1,
];
}
}

public function versioningEnabled(): bool {
if ($this->versioningEnabled === null) {
$result = $this->getConnection()->getBucketVersioning(['Bucket' => $this->getBucket()]);
$this->versioningEnabled = $result->get('Status') === 'Enabled';
}
return $this->versioningEnabled;
}
}
2 changes: 1 addition & 1 deletion lib/private/Files/ObjectStore/S3ObjectTrait.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ public function readObject($urn) {
}
$opts = [
'http' => [
'protocol_version' => 1.1,
'protocol_version' => $request->getProtocolVersion(),
'header' => $headers,
],
];
Expand Down

0 comments on commit 09ffac5

Please sign in to comment.