Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: search optimizations #791

Merged
merged 25 commits into from
Jun 10, 2021
Merged
Show file tree
Hide file tree
Changes from 23 commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
f120daa
Optimizes wallet and block search
alfonsobries Jun 4, 2021
d852f90
typo
alfonsobries Jun 4, 2021
f75cf53
Optimize transaction search
alfonsobries Jun 4, 2021
9b4d4a2
style: resolve style guide violations
alfonsobries Jun 4, 2021
74f1a99
We can use private methods
alfonsobries Jun 4, 2021
a6b77bb
Search by height fix
alfonsobries Jun 4, 2021
f79d4e3
Validate the address only by the length
alfonsobries Jun 4, 2021
d4efa13
Accept uppercase for usernames
alfonsobries Jun 4, 2021
8399167
style: resolve style guide violations
alfonsobries Jun 4, 2021
f6847b8
Update transaction search test
alfonsobries Jun 4, 2021
e953961
Update SearchPageTest.php
alfonsobries Jun 4, 2021
03b7a45
Merge branch 'refactor/search-optimizations' of github.com:ArkEcosyst…
alfonsobries Jun 4, 2021
8541eec
Update SearchPageTest.php
alfonsobries Jun 4, 2021
ef3c3ea
Update SearchPageTest.php
alfonsobries Jun 4, 2021
c4caabb
Better naming
alfonsobries Jun 4, 2021
d41bf29
Remove/rename unnecessary methods
alfonsobries Jun 4, 2021
358bb68
typo
alfonsobries Jun 4, 2021
d5ceb6f
Remove deprecated scope
alfonsobries Jun 4, 2021
b363126
Merge branch 'deps-and-padding' of github.com:ArkEcosystem/explorer.a…
alfonsobries Jun 7, 2021
e69f7ca
wip
ItsANameToo Jun 8, 2021
d1f81fd
Merge branch 'deps-and-padding' of github.com:ArkEcosystem/explorer.a…
alfonsobries Jun 8, 2021
15eb5e3
Refactor empty scopes
alfonsobries Jun 8, 2021
656253b
Simplify query
alfonsobries Jun 8, 2021
bf81681
Merge branch 'deps-and-padding' into refactor/search-optimizations
ItsANameToo Jun 10, 2021
5adf02e
style: resolve style guide violations
ItsANameToo Jun 10, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions app/Enums/SQLEnum.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
<?php

declare(strict_types=1);

namespace App\Enums;

final class SQLEnum
{
const INT4_MAXVALUE = 2147483647;
}
2 changes: 2 additions & 0 deletions app/Models/Block.php
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
namespace App\Models;

use App\Models\Casts\BigInteger;
use App\Models\Concerns\HasEmptyScope;
use App\Models\Concerns\SearchesCaseInsensitive;
use App\Services\BigNumber;
use Illuminate\Database\Eloquent\Factories\HasFactory;
Expand All @@ -27,6 +28,7 @@ final class Block extends Model
{
use HasFactory;
use SearchesCaseInsensitive;
use HasEmptyScope;

/**
* The "type" of the primary key ID.
Expand Down
18 changes: 18 additions & 0 deletions app/Models/Concerns/HasEmptyScope.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
<?php

declare(strict_types=1);

namespace App\Models\Concerns;

use Illuminate\Database\Eloquent\Builder;

trait HasEmptyScope
{
/**
* Used to force a query with no results.
*/
public function scopeEmpty(Builder $query): Builder
{
return $query->whereRaw('false');
}
}
8 changes: 1 addition & 7 deletions app/Models/Concerns/SearchesCaseInsensitive.php
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,6 @@ trait SearchesCaseInsensitive
public function scopeWhereLower(Builder $query, string $key, string $value): Builder
{
// @phpstan-ignore-next-line
return $query->where(DB::raw("lower($key)"), 'ilike', $value);
}

public function scopeOrWhereLower(Builder $query, string $key, string $value): Builder
{
// @phpstan-ignore-next-line
return $query->orWhere(DB::raw("lower($key)"), 'ilike', $value);
return $query->where(DB::raw("lower($key)"), '=', strtolower($value));
}
}
2 changes: 2 additions & 0 deletions app/Models/Transaction.php
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
namespace App\Models;

use App\Models\Casts\BigInteger;
use App\Models\Concerns\HasEmptyScope;
use App\Models\Concerns\SearchesCaseInsensitive;
use App\Models\Scopes\DelegateRegistrationScope;
use App\Models\Scopes\DelegateResignationScope;
Expand Down Expand Up @@ -42,6 +43,7 @@ final class Transaction extends Model
{
use HasFactory;
use SearchesCaseInsensitive;
use HasEmptyScope;

/**
* A list of transaction scopes used for filtering based on type.
Expand Down
2 changes: 2 additions & 0 deletions app/Models/Wallet.php
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
namespace App\Models;

use App\Models\Casts\BigInteger;
use App\Models\Concerns\HasEmptyScope;
use App\Models\Concerns\SearchesCaseInsensitive;
use App\Services\BigNumber;
use Illuminate\Database\Eloquent\Factories\HasFactory;
Expand All @@ -22,6 +23,7 @@ final class Wallet extends Model
{
use HasFactory;
use SearchesCaseInsensitive;
use HasEmptyScope;

/**
* Indicates if the IDs are auto-incrementing.
Expand Down
23 changes: 16 additions & 7 deletions app/Repositories/WalletRepository.php
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,15 @@

use App\Contracts\WalletRepository as Contract;
use App\Models\Wallet;
use App\Services\Search\Traits\ValidatesTerm;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\DB;

final class WalletRepository implements Contract
{
use ValidatesTerm;

public function allWithUsername(): Builder
{
return Wallet::whereNotNull('attributes->delegate->username')->orderBy('balance');
Expand Down Expand Up @@ -54,13 +57,19 @@ public function findByUsername(string $username): Wallet

public function findByIdentifier(string $identifier): Wallet
{
$username = substr(DB::getPdo()->quote($identifier), 1, -1);
$query = Wallet::query();

if ($this->couldBeAddress($identifier)) {
$query->whereLower('address', $identifier);
} elseif ($this->couldBePublicKey($identifier)) {
$query->whereLower('public_key', $identifier);
} elseif ($this->couldBeUsername($identifier)) {
$username = substr(DB::getPdo()->quote($identifier), 1, -1);
$query->orWhereRaw('lower(attributes::text)::jsonb @> lower(\'{"delegate":{"username":"'.$username.'"}}\')::jsonb');
} else {
$query->empty();
}

/* @phpstan-ignore-next-line */
return Wallet::query()
->whereLower('address', $identifier)
->orWhereLower('public_key', $identifier)
->orWhereRaw('lower(attributes::text)::jsonb @> lower(\'{"delegate":{"username":"'.$username.'"}}\')::jsonb')
->firstOrFail();
return $query->firstOrFail();
}
}
16 changes: 12 additions & 4 deletions app/Services/Search/BlockSearch.php
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,15 @@
use App\Models\Block;
use App\Models\Composers\TimestampRangeComposer;
use App\Models\Composers\ValueRangeComposer;
use App\Services\Search\Traits\ValidatesTerm;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Support\Arr;
use Throwable;

final class BlockSearch implements Search
{
use ValidatesTerm;

public function search(array $parameters): Builder
{
$query = Block::query();
Expand All @@ -24,11 +27,16 @@ public function search(array $parameters): Builder
$term = Arr::get($parameters, 'term');

if (! is_null($term)) {
$query = $query->whereLower('id', $term);

$numericTerm = filter_var($term, FILTER_VALIDATE_FLOAT, FILTER_FLAG_ALLOW_THOUSAND);
if ($this->couldBeBlockID($term)) {
$query = $query->whereLower('id', $term);
} else {
// Forces empty results when it has a term but not possible
// block ID
$query->empty();
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why do we need to force empty here?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

See comment above

}

if (is_numeric($numericTerm)) {
if ($this->couldBeHeightValue($term)) {
$numericTerm = strval(filter_var($term, FILTER_VALIDATE_FLOAT, FILTER_FLAG_ALLOW_THOUSAND));
$query->orWhere('height', $numericTerm);
}

Expand Down
79 changes: 79 additions & 0 deletions app/Services/Search/Traits/ValidatesTerm.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
<?php

declare(strict_types=1);

namespace App\Services\Search\Traits;

use App\Enums\SQLEnum;

trait ValidatesTerm
{
private function couldBeTransactionID(string $term): bool
{
return $this->is64CharsHexadecimalString($term);
}

private function couldBeBlockID(string $term): bool
{
return $this->is64CharsHexadecimalString($term);
}

private function couldBeAddress(string $term): bool
{
return strlen($term) === 34;
}

private function couldBePublicKey(string $term): bool
{
return strlen($term) === 66 && $this->isHexadecimalString($term);
}

/**
* Check if the query can be a username
* Regex source: https://github.com/ArkEcosystem/core/blob/4e149f039b59da97d224db1c593059dbc8e0f385/packages/core-api/src/handlers/shared/schemas/username.ts.
*
* @return bool
*/
private function couldBeUsername(string $term): bool
{
$regex = '/^[a-zA-Z0-9!@$&_.]+$/';

return strlen($term) >= 1
&& strlen($term) <= 20
&& preg_match($regex, $term, $matches) > 0;
}

private function couldBeHeightValue(string $term): bool
{
$numericTerm = strval(filter_var($term, FILTER_VALIDATE_FLOAT, FILTER_FLAG_ALLOW_THOUSAND));

return $this->isOnlyNumbers($numericTerm) && $this->numericTermIsInRange($numericTerm);
}

private function is64CharsHexadecimalString(string $term): bool
{
return $this->isOnlyNumbers($term)
|| (strlen($term) === 64 && $this->isHexadecimalString($term));
}

private function isOnlyNumbers(string $term): bool
{
return ctype_digit($term);
}

private function isHexadecimalString(string $term): bool
{
return ctype_xdigit($term);
}

/**
* Validates that the numnber is smaller that the max size for a type integer
* on pgsql. Searching for a bigger number will result in an SQL exception.
*
* @return bool
*/
private function numericTermIsInRange(string $term): bool
{
return floatval($term) <= SQLEnum::INT4_MAXVALUE;
}
}
38 changes: 26 additions & 12 deletions app/Services/Search/TransactionSearch.php
Original file line number Diff line number Diff line change
Expand Up @@ -10,25 +10,34 @@
use App\Models\Composers\TimestampRangeComposer;
use App\Models\Composers\ValueRangeComposer;
use App\Models\Transaction;
use App\Services\Search\Traits\ValidatesTerm;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Support\Arr;
use Throwable;

final class TransactionSearch implements Search
{
use ValidatesTerm;

public function search(array $parameters): Builder
{
$query = Transaction::query();

$this->applyScopes($query, $parameters);

if (! is_null(Arr::get($parameters, 'term'))) {
$query->whereLower('id', $parameters['term']);
$term = Arr::get($parameters, 'term');

if (! is_null($term)) {
if ($this->couldBeTransactionID($term)) {
$query->whereLower('id', $term);
} else {
$query->empty();
}

// Consider the term to be a wallet
try {
$query->orWhere(function ($query) use ($parameters): void {
$wallet = Wallets::findByIdentifier($parameters['term']);
$query->orWhere(function ($query) use ($parameters, $term): void {
$wallet = Wallets::findByIdentifier($term);

$query->where(function ($query) use ($parameters, $wallet): void {
$query->whereLower('sender_public_key', $wallet->public_key);
Expand All @@ -52,14 +61,19 @@ public function search(array $parameters): Builder
// If this throws then the term was not a valid address, public key or username.
}

// Consider the term to be a block
$query->orWhere(function ($query) use ($parameters): void {
$query->where(fn ($query): Builder => $query->whereLower('block_id', $parameters['term']));

if (is_numeric($parameters['term'])) {
$query->orWhere(fn ($query): Builder => $query->where('block_height', $parameters['term']));
}
});
if ($this->couldBeBlockID($term) || $this->couldBeHeightValue($term)) {
// Consider the term to be a block
$query->orWhere(function ($query) use ($term): void {
if ($this->couldBeBlockID($term)) {
$query->where(fn ($query): Builder => $query->whereLower('block_id', $term));
}

if ($this->couldBeHeightValue($term)) {
$numericTerm = strval(filter_var($term, FILTER_VALIDATE_FLOAT, FILTER_FLAG_ALLOW_THOUSAND));
$query->orWhere(fn ($query): Builder => $query->where('block_height', $numericTerm));
}
});
}
}

return $query;
Expand Down
23 changes: 17 additions & 6 deletions app/Services/Search/WalletSearch.php
Original file line number Diff line number Diff line change
Expand Up @@ -7,24 +7,35 @@
use App\Contracts\Search;
use App\Models\Composers\ValueRangeComposer;
use App\Models\Wallet;
use App\Services\Search\Traits\ValidatesTerm;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Support\Arr;
use Illuminate\Support\Facades\DB;

final class WalletSearch implements Search
{
use ValidatesTerm;

public function search(array $parameters): Builder
{
$query = Wallet::query();

ValueRangeComposer::compose($query, $parameters, 'balance');

if (! is_null(Arr::get($parameters, 'term'))) {
$query->whereLower('address', $parameters['term']);
$query->orWhereLower('public_key', $parameters['term']);

$username = substr(DB::getPdo()->quote($parameters['term']), 1, -1);
$query->orWhereRaw('lower(attributes::text)::jsonb @> lower(\'{"delegate":{"username":"'.$username.'"}}\')::jsonb');
$term = Arr::get($parameters, 'term');

if (! is_null($term)) {
if ($this->couldBeAddress($term)) {
$query->whereLower('address', $term);
} elseif ($this->couldBePublicKey($term)) {
$query->whereLower('public_key', $term);
} elseif ($this->couldBeUsername($term)) {
$username = substr(DB::getPdo()->quote($term), 1, -1);
$query->whereRaw('lower(attributes::text)::jsonb @> lower(\'{"delegate":{"username":"'.$username.'"}}\')::jsonb');
} else {
// Empty results when it has a term but not possible results
$query->empty();
}
}

if (! is_null(Arr::get($parameters, 'username'))) {
Expand Down
4 changes: 2 additions & 2 deletions tests/Feature/Http/Livewire/SearchPageTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -617,13 +617,13 @@
Transaction::factory(10)->create();

$tx = Transaction::factory()->create([
'block_id' => 'blockid',
'block_id' => 'ffff273321907d20bda3278ade259e6364ec2091ecd5993398a2ef2402725a31',
]);

Livewire::test(SearchPage::class)
->set('state.type', 'transaction')
->set('state.transactionType', 'all')
->set('state.term', 'blockid')
->set('state.term', 'ffff273321907d20bda3278ade259e6364ec2091ecd5993398a2ef2402725a31')
->call('performSearch')
->assertSee($tx->id);
});
Expand Down
Loading