Skip to content

Commit

Permalink
Merge pull request #40555 from nextcloud/query-optimize-distribute
Browse files Browse the repository at this point in the history
optimize query pattern used by storage filter
  • Loading branch information
icewind1991 authored Feb 16, 2024
2 parents fcfd530 + 3890aa5 commit bb87232
Show file tree
Hide file tree
Showing 20 changed files with 877 additions and 54 deletions.
2 changes: 0 additions & 2 deletions lib/composer/composer/LICENSE
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@

Copyright (c) Nils Adermann, Jordi Boggiano

Permission is hereby granted, free of charge, to any person obtaining a copy
Expand All @@ -18,4 +17,3 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

6 changes: 6 additions & 0 deletions lib/composer/composer/autoload_classmap.php
Original file line number Diff line number Diff line change
Expand Up @@ -1406,9 +1406,15 @@
'OC\\Files\\ObjectStore\\Swift' => $baseDir . '/lib/private/Files/ObjectStore/Swift.php',
'OC\\Files\\ObjectStore\\SwiftFactory' => $baseDir . '/lib/private/Files/ObjectStore/SwiftFactory.php',
'OC\\Files\\ObjectStore\\SwiftV2CachingAuthService' => $baseDir . '/lib/private/Files/ObjectStore/SwiftV2CachingAuthService.php',
'OC\\Files\\Search\\QueryOptimizer\\FlattenNestedBool' => $baseDir . '/lib/private/Files/Search/QueryOptimizer/FlattenNestedBool.php',
'OC\\Files\\Search\\QueryOptimizer\\FlattenSingleArgumentBinaryOperation' => $baseDir . '/lib/private/Files/Search/QueryOptimizer/FlattenSingleArgumentBinaryOperation.php',
'OC\\Files\\Search\\QueryOptimizer\\MergeDistributiveOperations' => $baseDir . '/lib/private/Files/Search/QueryOptimizer/MergeDistributiveOperations.php',
'OC\\Files\\Search\\QueryOptimizer\\OrEqualsToIn' => $baseDir . '/lib/private/Files/Search/QueryOptimizer/OrEqualsToIn.php',
'OC\\Files\\Search\\QueryOptimizer\\PathPrefixOptimizer' => $baseDir . '/lib/private/Files/Search/QueryOptimizer/PathPrefixOptimizer.php',
'OC\\Files\\Search\\QueryOptimizer\\QueryOptimizer' => $baseDir . '/lib/private/Files/Search/QueryOptimizer/QueryOptimizer.php',
'OC\\Files\\Search\\QueryOptimizer\\QueryOptimizerStep' => $baseDir . '/lib/private/Files/Search/QueryOptimizer/QueryOptimizerStep.php',
'OC\\Files\\Search\\QueryOptimizer\\ReplacingOptimizerStep' => $baseDir . '/lib/private/Files/Search/QueryOptimizer/ReplacingOptimizerStep.php',
'OC\\Files\\Search\\QueryOptimizer\\SplitLargeIn' => $baseDir . '/lib/private/Files/Search/QueryOptimizer/SplitLargeIn.php',
'OC\\Files\\Search\\SearchBinaryOperator' => $baseDir . '/lib/private/Files/Search/SearchBinaryOperator.php',
'OC\\Files\\Search\\SearchComparison' => $baseDir . '/lib/private/Files/Search/SearchComparison.php',
'OC\\Files\\Search\\SearchOrder' => $baseDir . '/lib/private/Files/Search/SearchOrder.php',
Expand Down
6 changes: 6 additions & 0 deletions lib/composer/composer/autoload_static.php
Original file line number Diff line number Diff line change
Expand Up @@ -1439,9 +1439,15 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
'OC\\Files\\ObjectStore\\Swift' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/Swift.php',
'OC\\Files\\ObjectStore\\SwiftFactory' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/SwiftFactory.php',
'OC\\Files\\ObjectStore\\SwiftV2CachingAuthService' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/SwiftV2CachingAuthService.php',
'OC\\Files\\Search\\QueryOptimizer\\FlattenNestedBool' => __DIR__ . '/../../..' . '/lib/private/Files/Search/QueryOptimizer/FlattenNestedBool.php',
'OC\\Files\\Search\\QueryOptimizer\\FlattenSingleArgumentBinaryOperation' => __DIR__ . '/../../..' . '/lib/private/Files/Search/QueryOptimizer/FlattenSingleArgumentBinaryOperation.php',
'OC\\Files\\Search\\QueryOptimizer\\MergeDistributiveOperations' => __DIR__ . '/../../..' . '/lib/private/Files/Search/QueryOptimizer/MergeDistributiveOperations.php',
'OC\\Files\\Search\\QueryOptimizer\\OrEqualsToIn' => __DIR__ . '/../../..' . '/lib/private/Files/Search/QueryOptimizer/OrEqualsToIn.php',
'OC\\Files\\Search\\QueryOptimizer\\PathPrefixOptimizer' => __DIR__ . '/../../..' . '/lib/private/Files/Search/QueryOptimizer/PathPrefixOptimizer.php',
'OC\\Files\\Search\\QueryOptimizer\\QueryOptimizer' => __DIR__ . '/../../..' . '/lib/private/Files/Search/QueryOptimizer/QueryOptimizer.php',
'OC\\Files\\Search\\QueryOptimizer\\QueryOptimizerStep' => __DIR__ . '/../../..' . '/lib/private/Files/Search/QueryOptimizer/QueryOptimizerStep.php',
'OC\\Files\\Search\\QueryOptimizer\\ReplacingOptimizerStep' => __DIR__ . '/../../..' . '/lib/private/Files/Search/QueryOptimizer/ReplacingOptimizerStep.php',
'OC\\Files\\Search\\QueryOptimizer\\SplitLargeIn' => __DIR__ . '/../../..' . '/lib/private/Files/Search/QueryOptimizer/SplitLargeIn.php',
'OC\\Files\\Search\\SearchBinaryOperator' => __DIR__ . '/../../..' . '/lib/private/Files/Search/SearchBinaryOperator.php',
'OC\\Files\\Search\\SearchComparison' => __DIR__ . '/../../..' . '/lib/private/Files/Search/SearchComparison.php',
'OC\\Files\\Search\\SearchOrder' => __DIR__ . '/../../..' . '/lib/private/Files/Search/SearchOrder.php',
Expand Down
137 changes: 99 additions & 38 deletions lib/private/Files/Cache/SearchBuilder.php
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,12 @@

/**
* Tools for transforming search queries into database queries
*
* @psalm-import-type ParamSingleValue from ISearchComparison
* @psalm-import-type ParamValue from ISearchComparison
*/
class SearchBuilder {
/** @var array<string, string> */
protected static $searchOperatorMap = [
ISearchComparison::COMPARE_LIKE => 'iLike',
ISearchComparison::COMPARE_LIKE_CASE_SENSITIVE => 'like',
Expand All @@ -48,8 +52,10 @@ class SearchBuilder {
ISearchComparison::COMPARE_LESS_THAN => 'lt',
ISearchComparison::COMPARE_LESS_THAN_EQUAL => 'lte',
ISearchComparison::COMPARE_DEFINED => 'isNotNull',
ISearchComparison::COMPARE_IN => 'in',
];

/** @var array<string, string> */
protected static $searchOperatorNegativeMap = [
ISearchComparison::COMPARE_LIKE => 'notLike',
ISearchComparison::COMPARE_LIKE_CASE_SENSITIVE => 'notLike',
Expand All @@ -59,6 +65,38 @@ class SearchBuilder {
ISearchComparison::COMPARE_LESS_THAN => 'gte',
ISearchComparison::COMPARE_LESS_THAN_EQUAL => 'gt',
ISearchComparison::COMPARE_DEFINED => 'isNull',
ISearchComparison::COMPARE_IN => 'notIn',
];

/** @var array<string, string> */
protected static $fieldTypes = [
'mimetype' => 'string',
'mtime' => 'integer',
'name' => 'string',
'path' => 'string',
'size' => 'integer',
'tagname' => 'string',
'systemtag' => 'string',
'favorite' => 'boolean',
'fileid' => 'integer',
'storage' => 'integer',
'share_with' => 'string',
'share_type' => 'integer',
'owner' => 'string',
];

/** @var array<string, int> */
protected static $paramTypeMap = [
'string' => IQueryBuilder::PARAM_STR,
'integer' => IQueryBuilder::PARAM_INT,
'boolean' => IQueryBuilder::PARAM_BOOL,
];

/** @var array<string, int> */
protected static $paramArrayTypeMap = [
'string' => IQueryBuilder::PARAM_STR_ARRAY,
'integer' => IQueryBuilder::PARAM_INT_ARRAY,
'boolean' => IQueryBuilder::PARAM_INT_ARRAY,
];

public const TAG_FAVORITE = '_$!<Favorite>!$_';
Expand Down Expand Up @@ -142,31 +180,56 @@ private function searchComparisonToDBExpr(
?IMetadataQuery $metadataQuery = null
) {
if ($comparison->getExtra()) {
[$field, $value, $type] = $this->getExtraOperatorField($comparison, $metadataQuery);
[$field, $value, $type, $paramType] = $this->getExtraOperatorField($comparison, $metadataQuery);
} else {
[$field, $value, $type] = $this->getOperatorFieldAndValue($comparison);
[$field, $value, $type, $paramType] = $this->getOperatorFieldAndValue($comparison);
}

if (isset($operatorMap[$type])) {
$queryOperator = $operatorMap[$type];
return $builder->expr()->$queryOperator($field, $this->getParameterForValue($builder, $value));
return $builder->expr()->$queryOperator($field, $this->getParameterForValue($builder, $value, $paramType));
} else {
throw new \InvalidArgumentException('Invalid operator type: ' . $comparison->getType());
}
}

private function getOperatorFieldAndValue(ISearchComparison $operator) {
/**
* @param ISearchComparison $operator
* @return list{string, ParamValue, string, string}
*/
private function getOperatorFieldAndValue(ISearchComparison $operator): array {
$this->validateComparison($operator);

$field = $operator->getField();
$value = $operator->getValue();
$type = $operator->getType();
$pathEqHash = $operator->getQueryHint(ISearchComparison::HINT_PATH_EQ_HASH, true);
return $this->getOperatorFieldAndValueInner($field, $value, $type, $pathEqHash);
}

/**
* @param string $field
* @param ParamValue $value
* @param string $type
* @return list{string, ParamValue, string, string}
*/
private function getOperatorFieldAndValueInner(string $field, mixed $value, string $type, bool $pathEqHash): array {
$paramType = self::$fieldTypes[$field];
if ($type === ISearchComparison::COMPARE_IN) {
$resultField = $field;
$values = [];
foreach ($value as $arrayValue) {
/** @var ParamSingleValue $arrayValue */
[$arrayField, $arrayValue] = $this->getOperatorFieldAndValueInner($field, $arrayValue, ISearchComparison::COMPARE_EQUAL, $pathEqHash);
$resultField = $arrayField;
$values[] = $arrayValue;
}
return [$resultField, $values, ISearchComparison::COMPARE_IN, $paramType];
}
if ($field === 'mimetype') {
$value = (string)$value;
if ($operator->getType() === ISearchComparison::COMPARE_EQUAL) {
if ($type === ISearchComparison::COMPARE_EQUAL) {
$value = (int)$this->mimetypeLoader->getId($value);
} elseif ($operator->getType() === ISearchComparison::COMPARE_LIKE) {
} elseif ($type === ISearchComparison::COMPARE_LIKE) {
// transform "mimetype='foo/%'" to "mimepart='foo'"
if (preg_match('|(.+)/%|', $value, $matches)) {
$field = 'mimepart';
Expand All @@ -183,6 +246,7 @@ private function getOperatorFieldAndValue(ISearchComparison $operator) {
} elseif ($field === 'favorite') {
$field = 'tag.category';
$value = self::TAG_FAVORITE;
$paramType = 'string';
} elseif ($field === 'name') {
$field = 'file.name';
} elseif ($field === 'tagname') {
Expand All @@ -191,53 +255,49 @@ private function getOperatorFieldAndValue(ISearchComparison $operator) {
$field = 'systemtag.name';
} elseif ($field === 'fileid') {
$field = 'file.fileid';
} elseif ($field === 'path' && $type === ISearchComparison::COMPARE_EQUAL && $operator->getQueryHint(ISearchComparison::HINT_PATH_EQ_HASH, true)) {
} elseif ($field === 'path' && $type === ISearchComparison::COMPARE_EQUAL && $pathEqHash) {
$field = 'path_hash';
$value = md5((string)$value);
} elseif ($field === 'owner') {
$field = 'uid_owner';
}
return [$field, $value, $type];
return [$field, $value, $type, $paramType];
}

private function validateComparison(ISearchComparison $operator) {
$types = [
'mimetype' => 'string',
'mtime' => 'integer',
'name' => 'string',
'path' => 'string',
'size' => 'integer',
'tagname' => 'string',
'systemtag' => 'string',
'favorite' => 'boolean',
'fileid' => 'integer',
'storage' => 'integer',
'share_with' => 'string',
'share_type' => 'integer',
'owner' => 'string',
];
$comparisons = [
'mimetype' => ['eq', 'like'],
'mimetype' => ['eq', 'like', 'in'],
'mtime' => ['eq', 'gt', 'lt', 'gte', 'lte'],
'name' => ['eq', 'like', 'clike'],
'path' => ['eq', 'like', 'clike'],
'name' => ['eq', 'like', 'clike', 'in'],
'path' => ['eq', 'like', 'clike', 'in'],
'size' => ['eq', 'gt', 'lt', 'gte', 'lte'],
'tagname' => ['eq', 'like'],
'systemtag' => ['eq', 'like'],
'favorite' => ['eq'],
'fileid' => ['eq'],
'storage' => ['eq'],
'fileid' => ['eq', 'in'],
'storage' => ['eq', 'in'],
'share_with' => ['eq'],
'share_type' => ['eq'],
'owner' => ['eq'],
];

if (!isset($types[$operator->getField()])) {
if (!isset(self::$fieldTypes[$operator->getField()])) {
throw new \InvalidArgumentException('Unsupported comparison field ' . $operator->getField());
}
$type = $types[$operator->getField()];
if (gettype($operator->getValue()) !== $type) {
throw new \InvalidArgumentException('Invalid type for field ' . $operator->getField());
$type = self::$fieldTypes[$operator->getField()];
if ($operator->getType() === ISearchComparison::COMPARE_IN) {
if (!is_array($operator->getValue())) {
throw new \InvalidArgumentException('Invalid type for field ' . $operator->getField());
}
foreach ($operator->getValue() as $arrayValue) {
if (gettype($arrayValue) !== $type) {
throw new \InvalidArgumentException('Invalid type in array for field ' . $operator->getField());
}
}
} else {
if (gettype($operator->getValue()) !== $type) {
throw new \InvalidArgumentException('Invalid type for field ' . $operator->getField());
}
}
if (!in_array($operator->getType(), $comparisons[$operator->getField()])) {
throw new \InvalidArgumentException('Unsupported comparison for field ' . $operator->getField() . ': ' . $operator->getType());
Expand All @@ -246,6 +306,7 @@ private function validateComparison(ISearchComparison $operator) {


private function getExtraOperatorField(ISearchComparison $operator, IMetadataQuery $metadataQuery): array {
$paramType = self::$fieldTypes[$operator->getField()];
$field = $operator->getField();
$value = $operator->getValue();
$type = $operator->getType();
Expand All @@ -259,17 +320,17 @@ private function getExtraOperatorField(ISearchComparison $operator, IMetadataQue
throw new \InvalidArgumentException('Invalid extra type: ' . $operator->getExtra());
}

return [$field, $value, $type];
return [$field, $value, $type, $paramType];
}

private function getParameterForValue(IQueryBuilder $builder, $value) {
private function getParameterForValue(IQueryBuilder $builder, $value, string $paramType) {
if ($value instanceof \DateTime) {
$value = $value->getTimestamp();
}
if (is_numeric($value)) {
$type = IQueryBuilder::PARAM_INT;
if (is_array($value)) {
$type = self::$paramArrayTypeMap[$paramType];
} else {
$type = IQueryBuilder::PARAM_STR;
$type = self::$paramTypeMap[$paramType];
}
return $builder->createNamedParameter($value, $type);
}
Expand Down
29 changes: 29 additions & 0 deletions lib/private/Files/Search/QueryOptimizer/FlattenNestedBool.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
<?php

namespace OC\Files\Search\QueryOptimizer;

use OC\Files\Search\SearchBinaryOperator;
use OCP\Files\Search\ISearchBinaryOperator;
use OCP\Files\Search\ISearchOperator;

class FlattenNestedBool extends QueryOptimizerStep {
public function processOperator(ISearchOperator &$operator) {
if (
$operator instanceof SearchBinaryOperator && (
$operator->getType() === ISearchBinaryOperator::OPERATOR_OR ||
$operator->getType() === ISearchBinaryOperator::OPERATOR_AND
)
) {
$newArguments = [];
foreach ($operator->getArguments() as $oldArgument) {
if ($oldArgument instanceof SearchBinaryOperator && $oldArgument->getType() === $operator->getType()) {
$newArguments = array_merge($newArguments, $oldArgument->getArguments());
} else {
$newArguments[] = $oldArgument;
}
}
$operator->setArguments($newArguments);
}
parent::processOperator($operator);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
<?php

namespace OC\Files\Search\QueryOptimizer;

use OCP\Files\Search\ISearchBinaryOperator;
use OCP\Files\Search\ISearchOperator;

/**
* replace single argument AND and OR operations with their single argument
*/
class FlattenSingleArgumentBinaryOperation extends ReplacingOptimizerStep {
public function processOperator(ISearchOperator &$operator): bool {
parent::processOperator($operator);
if (
$operator instanceof ISearchBinaryOperator &&
count($operator->getArguments()) === 1 &&
(
$operator->getType() === ISearchBinaryOperator::OPERATOR_OR ||
$operator->getType() === ISearchBinaryOperator::OPERATOR_AND
)
) {
$operator = $operator->getArguments()[0];
return true;
}
return false;
}
}
Loading

0 comments on commit bb87232

Please sign in to comment.