From 9ae6061fa3b87ce5b664949141fce2b3b833ec06 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Thu, 25 Mar 2021 04:46:21 -0300 Subject: [PATCH 1/2] fix no_source for reposts --- lbry/wallet/server/db/elasticsearch/search.py | 5 +++++ lbry/wallet/server/db/elasticsearch/sync.py | 2 ++ lbry/wallet/server/db/writer.py | 2 ++ tests/integration/blockchain/test_claim_commands.py | 10 ++++++---- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/lbry/wallet/server/db/elasticsearch/search.py b/lbry/wallet/server/db/elasticsearch/search.py index 3621114893..55e1bd0ddc 100644 --- a/lbry/wallet/server/db/elasticsearch/search.py +++ b/lbry/wallet/server/db/elasticsearch/search.py @@ -429,6 +429,11 @@ def expand_query(**kwargs): query["minimum_should_match"] = 1 query['should'].append({"bool": {"must_not": {"exists": {"field": "signature_digest"}}}}) query['should'].append({"term": {"signature_valid": bool(kwargs["signature_valid"])}}) + if 'has_source' in kwargs: + query.setdefault('should', []) + query["minimum_should_match"] = 1 + query['should'].append({"bool": {"must": [{"match": {"has_source": kwargs['has_source']}}, {"match": {"claim_type": CLAIM_TYPES['stream']}}]}}) + query['should'].append({"bool": {"must_not": [{"match": {"claim_type": CLAIM_TYPES['stream']}}]}}) if kwargs.get('text'): query['must'].append( {"simple_query_string": diff --git a/lbry/wallet/server/db/elasticsearch/sync.py b/lbry/wallet/server/db/elasticsearch/sync.py index 645b7e758f..0255c0c2ab 100644 --- a/lbry/wallet/server/db/elasticsearch/sync.py +++ b/lbry/wallet/server/db/elasticsearch/sync.py @@ -28,12 +28,14 @@ def exec_factory(cursor, statement, bindings): claimtrie.last_take_over_height, (select group_concat(tag, ',,') from tag where tag.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as tags, (select group_concat(language, ' ') from language where language.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as languages, + (select cr.has_source from claim cr where cr.claim_hash = claim.reposted_claim_hash) as reposted_has_source, claim.* FROM claim LEFT JOIN claimtrie USING (claim_hash) WHERE claim.height % {shards_total} = {shard_num} ORDER BY claim.height desc """)): claim = dict(claim._asdict()) + claim['has_source'] = bool(claim.pop('reposted_has_source') or claim['has_source']) claim['censor_type'] = 0 claim['censoring_channel_hash'] = None claim['tags'] = claim['tags'].split(',,') if claim['tags'] else [] diff --git a/lbry/wallet/server/db/writer.py b/lbry/wallet/server/db/writer.py index 80fc4b5561..6f94a3c665 100644 --- a/lbry/wallet/server/db/writer.py +++ b/lbry/wallet/server/db/writer.py @@ -827,6 +827,7 @@ def enqueue_changes(self): claimtrie.last_take_over_height, (select group_concat(tag, ',,') from tag where tag.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as tags, (select group_concat(language, ' ') from language where language.claim_hash in (claim.claim_hash, claim.reposted_claim_hash)) as languages, + (select cr.has_source from claim cr where cr.claim_hash = claim.reposted_claim_hash) as reposted_has_source, claim.* FROM claim LEFT JOIN claimtrie USING (claim_hash) WHERE claim.claim_hash in (SELECT claim_hash FROM changelog) @@ -835,6 +836,7 @@ def enqueue_changes(self): id_set = set(filter(None, (claim['claim_hash'], claim['channel_hash'], claim['reposted_claim_hash']))) claim['censor_type'] = 0 claim['censoring_channel_hash'] = None + claim['has_source'] = bool(claim.pop('reposted_has_source') or claim['has_source']) for reason_id in id_set: if reason_id in self.blocked_streams: claim['censor_type'] = 2 diff --git a/tests/integration/blockchain/test_claim_commands.py b/tests/integration/blockchain/test_claim_commands.py index ab373e7029..a1981614f4 100644 --- a/tests/integration/blockchain/test_claim_commands.py +++ b/tests/integration/blockchain/test_claim_commands.py @@ -180,11 +180,13 @@ async def test_basic_claim_search(self): await self.assertFindsClaims([three], claim_id=self.get_claim_id(three), text='*') async def test_source_filter(self): - no_source = await self.stream_create('no_source', data=None) + no_source = await self.stream_create('no-source', data=None) normal = await self.stream_create('normal', data=b'normal') - await self.assertFindsClaims([no_source], has_no_source=True) - await self.assertFindsClaims([normal], has_source=True) - await self.assertFindsClaims([normal, no_source]) + normal_repost = await self.stream_repost(self.get_claim_id(normal), 'normal-repost') + no_source_repost = await self.stream_repost(self.get_claim_id(no_source), 'no-source-repost') + await self.assertFindsClaims([no_source_repost, no_source], has_no_source=True) + await self.assertFindsClaims([normal_repost, normal], has_source=True) + await self.assertFindsClaims([no_source_repost, normal_repost, normal, no_source]) async def test_pagination(self): await self.create_channel() From fdabf884debc929e7b1fb1e062236559e9783b02 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Fri, 26 Mar 2021 00:27:05 -0300 Subject: [PATCH 2/2] show channels regardless of no_source --- lbry/wallet/server/db/elasticsearch/constants.py | 2 +- lbry/wallet/server/db/elasticsearch/search.py | 6 ++++-- tests/integration/blockchain/test_claim_commands.py | 7 ++++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/lbry/wallet/server/db/elasticsearch/constants.py b/lbry/wallet/server/db/elasticsearch/constants.py index 12483ed10b..cea87c225d 100644 --- a/lbry/wallet/server/db/elasticsearch/constants.py +++ b/lbry/wallet/server/db/elasticsearch/constants.py @@ -42,7 +42,7 @@ 'claims_in_channel', 'channel_join', 'signature_valid', 'effective_amount', 'support_amount', 'trending_group', 'trending_mixed', 'trending_local', 'trending_global', 'channel_id', 'tx_id', 'tx_nout', 'signature', 'signature_digest', 'public_key_bytes', 'public_key_hash', 'public_key_id', '_id', 'tags', - 'reposted_claim_id', 'has_source'} + 'reposted_claim_id'} TEXT_FIELDS = {'author', 'canonical_url', 'channel_id', 'claim_name', 'description', 'claim_id', 'media_type', 'normalized', 'public_key_bytes', 'public_key_hash', 'short_url', 'signature', 'signature_digest', 'stream_type', 'title', 'tx_id', 'fee_currency', 'reposted_claim_id', 'tags'} diff --git a/lbry/wallet/server/db/elasticsearch/search.py b/lbry/wallet/server/db/elasticsearch/search.py index 55e1bd0ddc..a5ef0b8d11 100644 --- a/lbry/wallet/server/db/elasticsearch/search.py +++ b/lbry/wallet/server/db/elasticsearch/search.py @@ -432,8 +432,10 @@ def expand_query(**kwargs): if 'has_source' in kwargs: query.setdefault('should', []) query["minimum_should_match"] = 1 - query['should'].append({"bool": {"must": [{"match": {"has_source": kwargs['has_source']}}, {"match": {"claim_type": CLAIM_TYPES['stream']}}]}}) - query['should'].append({"bool": {"must_not": [{"match": {"claim_type": CLAIM_TYPES['stream']}}]}}) + is_stream_or_repost = {"terms": {"claim_type": [CLAIM_TYPES['stream'], CLAIM_TYPES['repost']]}} + query['should'].append( + {"bool": {"must": [{"match": {"has_source": kwargs['has_source']}}, is_stream_or_repost]}}) + query['should'].append({"bool": {"must_not": [is_stream_or_repost]}}) if kwargs.get('text'): query['must'].append( {"simple_query_string": diff --git a/tests/integration/blockchain/test_claim_commands.py b/tests/integration/blockchain/test_claim_commands.py index a1981614f4..975b20ce75 100644 --- a/tests/integration/blockchain/test_claim_commands.py +++ b/tests/integration/blockchain/test_claim_commands.py @@ -180,13 +180,14 @@ async def test_basic_claim_search(self): await self.assertFindsClaims([three], claim_id=self.get_claim_id(three), text='*') async def test_source_filter(self): + channel = await self.channel_create('@abc') no_source = await self.stream_create('no-source', data=None) normal = await self.stream_create('normal', data=b'normal') normal_repost = await self.stream_repost(self.get_claim_id(normal), 'normal-repost') no_source_repost = await self.stream_repost(self.get_claim_id(no_source), 'no-source-repost') - await self.assertFindsClaims([no_source_repost, no_source], has_no_source=True) - await self.assertFindsClaims([normal_repost, normal], has_source=True) - await self.assertFindsClaims([no_source_repost, normal_repost, normal, no_source]) + await self.assertFindsClaims([no_source_repost, no_source, channel], has_no_source=True) + await self.assertFindsClaims([normal_repost, normal, channel], has_source=True) + await self.assertFindsClaims([no_source_repost, normal_repost, normal, no_source, channel]) async def test_pagination(self): await self.create_channel()