From 41e5214f94dfb9b5261b7090fabad4c5bbe2fd01 Mon Sep 17 00:00:00 2001 From: xdj Date: Fri, 5 May 2023 17:07:17 +0800 Subject: [PATCH 1/5] feat(search): deprecate SearchInput.version and SearchInput.version, remove old version search code --- schema.graphql | 4 +- src/connectors/__test__/tagService.test.ts | 22 +- src/connectors/__test__/userService.test.ts | 26 +- src/connectors/articleService.ts | 284 -------------------- src/connectors/tagService.ts | 263 ------------------ src/connectors/userService.ts | 261 ------------------ src/definitions/schema.d.ts | 6 +- src/queries/system/search.ts | 50 ++-- src/types/system.ts | 3 +- 9 files changed, 53 insertions(+), 866 deletions(-) diff --git a/schema.graphql b/schema.graphql index b4a4f867d..33e8ba81b 100644 --- a/schema.graphql +++ b/schema.graphql @@ -2005,8 +2005,10 @@ input SearchInput { record: Boolean oss: Boolean - """use the api version; omit to use latest version""" + """deprecated, make no effect""" version: SearchAPIVersion + + """deprecated, make no effect""" coefficients: String quicksearch: Boolean } diff --git a/src/connectors/__test__/tagService.test.ts b/src/connectors/__test__/tagService.test.ts index 1620685c3..25ba7c8d3 100644 --- a/src/connectors/__test__/tagService.test.ts +++ b/src/connectors/__test__/tagService.test.ts @@ -28,9 +28,9 @@ test('create', async () => { expect(tag.content).toEqual(content) }) -describe('searchV1', () => { +describe('search', () => { test('empty result', async () => { - const res = await tagService.searchV1({ + const res = await tagService.search({ key: 'not-existed-tag', skip: 0, take: 10, @@ -38,12 +38,12 @@ describe('searchV1', () => { expect(res.totalCount).toBe(0) }) test('prefer exact match', async () => { - const res = await tagService.searchV1({ key: 'tag', skip: 0, take: 10 }) + const res = await tagService.search({ key: 'tag', skip: 0, take: 10 }) expect(res.totalCount).toBe(4) expect(res.nodes[0].content).toBe('tag') }) test('prefer more articles', async () => { - const res = await tagService.searchV1({ + const res = await tagService.search({ key: 't', skip: 0, take: 10, @@ -58,21 +58,21 @@ describe('searchV1', () => { ) }) test('handle prefix #,＃', async () => { - const res1 = await tagService.searchV1({ key: '#tag', skip: 0, take: 10 }) + const res1 = await tagService.search({ key: '#tag', skip: 0, take: 10 }) expect(res1.totalCount).toBe(4) expect(res1.nodes[0].content).toBe('tag') - const res2 = await tagService.searchV1({ key: '＃tag', skip: 0, take: 10 }) + const res2 = await tagService.search({ key: '＃tag', skip: 0, take: 10 }) expect(res2.totalCount).toBe(4) expect(res2.nodes[0].content).toBe('tag') }) test('handle empty string', async () => { - const res1 = await tagService.searchV1({ key: '', skip: 0, take: 10 }) + const res1 = await tagService.search({ key: '', skip: 0, take: 10 }) expect(res1.totalCount).toBe(0) - const res2 = await tagService.searchV1({ key: '#', skip: 0, take: 10 }) + const res2 = await tagService.search({ key: '#', skip: 0, take: 10 }) expect(res2.totalCount).toBe(0) }) test('right totalCount with take and skip', async () => { - const res1 = await tagService.searchV1({ + const res1 = await tagService.search({ key: 'tag', skip: 0, take: 10, @@ -81,7 +81,7 @@ describe('searchV1', () => { console.log(new Date(), 'res:', res1) expect(res1.nodes.length).toBe(4) expect(res1.totalCount).toBe(4) - const res2 = await tagService.searchV1({ + const res2 = await tagService.search({ key: 'tag', skip: 0, take: 1, @@ -89,7 +89,7 @@ describe('searchV1', () => { }) expect(res2.nodes.length).toBe(1) expect(res2.totalCount).toBe(4) - const res3 = await tagService.searchV1({ + const res3 = await tagService.search({ key: 'tag', skip: 1, take: 10, diff --git a/src/connectors/__test__/userService.test.ts b/src/connectors/__test__/userService.test.ts index e3c1599d7..4cb09553c 100644 --- a/src/connectors/__test__/userService.test.ts +++ b/src/connectors/__test__/userService.test.ts @@ -134,9 +134,9 @@ describe('countDonators', () => { }) }) -describe('searchV1', () => { +describe('search', () => { test('empty result', async () => { - const res = await userService.searchV1({ + const res = await userService.search({ key: 'not-exist', take: 1, skip: 0, @@ -144,7 +144,7 @@ describe('searchV1', () => { expect(res.totalCount).toBe(0) }) test('prefer exact match', async () => { - const res = await userService.searchV1({ key: 'test1', take: 3, skip: 0 }) + const res = await userService.search({ key: 'test1', take: 3, skip: 0 }) expect(res.totalCount).toBe(2) expect(res.nodes[0].userName).toBe('test1') }) @@ -156,7 +156,7 @@ describe('searchV1', () => { .where({ id }) .select('num_followers') )[0].numFollowers || 0 - const res = await userService.searchV1({ key: 'test', take: 3, skip: 0 }) + const res = await userService.search({ key: 'test', take: 3, skip: 0 }) expect(await getNumFollowers(res.nodes[0].id)).toBeGreaterThanOrEqual( await getNumFollowers(res.nodes[1].id) ) @@ -165,10 +165,10 @@ describe('searchV1', () => { ) }) test('handle prefix @,＠', async () => { - const res = await userService.searchV1({ key: '@test1', take: 3, skip: 0 }) + const res = await userService.search({ key: '@test1', take: 3, skip: 0 }) expect(res.totalCount).toBe(2) expect(res.nodes[0].userName).toBe('test1') - const res2 = await userService.searchV1({ + const res2 = await userService.search({ key: '＠test1', take: 3, skip: 0, @@ -177,9 +177,9 @@ describe('searchV1', () => { expect(res2.nodes[0].userName).toBe('test1') }) test('handle empty string', async () => { - const res1 = await userService.searchV1({ key: '', take: 3, skip: 0 }) + const res1 = await userService.search({ key: '', take: 3, skip: 0 }) expect(res1.totalCount).toBe(0) - const res2 = await userService.searchV1({ key: '@', take: 3, skip: 0 }) + const res2 = await userService.search({ key: '@', take: 3, skip: 0 }) expect(res2.totalCount).toBe(0) }) test('handle blocked', async () => { @@ -187,10 +187,10 @@ describe('searchV1', () => { .knex('action_user') .insert({ userId: '2', action: USER_ACTION.block, targetId: '1' }) - const res = await userService.searchV1({ key: 'test2', take: 3, skip: 0 }) + const res = await userService.search({ key: 'test2', take: 3, skip: 0 }) expect(res.totalCount).toBe(1) - const res2 = await userService.searchV1({ + const res2 = await userService.search({ key: 'test2', take: 3, skip: 0, @@ -200,13 +200,13 @@ describe('searchV1', () => { expect(res2.totalCount).toBe(0) }) test('right totalCount with take and skip', async () => { - const res1 = await userService.searchV1({ key: 'test', take: 10, skip: 0 }) + const res1 = await userService.search({ key: 'test', take: 10, skip: 0 }) expect(res1.nodes.length).toBe(6) expect(res1.totalCount).toBe(6) - const res2 = await userService.searchV1({ key: 'test', take: 1, skip: 0 }) + const res2 = await userService.search({ key: 'test', take: 1, skip: 0 }) expect(res2.nodes.length).toBe(1) expect(res2.totalCount).toBe(6) - const res3 = await userService.searchV1({ key: 'test', take: 10, skip: 1 }) + const res3 = await userService.search({ key: 'test', take: 10, skip: 1 }) expect(res3.nodes.length).toBe(5) expect(res3.totalCount).toBe(6) }) diff --git a/src/connectors/articleService.ts b/src/connectors/articleService.ts index 5541b0031..505577743 100644 --- a/src/connectors/articleService.ts +++ b/src/connectors/articleService.ts @@ -6,7 +6,6 @@ import { stripHtml, } from '@matters/ipns-site-generator' import slugify from '@matters/slugify' -import bodybuilder from 'bodybuilder' import DataLoader from 'dataloader' import createDebug from 'debug' import { Knex } from 'knex' @@ -913,136 +912,11 @@ export class ArticleService extends BaseService { } } - // the searchV0: TBDeprecated in next release search = async ({ - key, - take, - skip, - oss = false, - filter, - exclude, - viewerId, - }: { - key: string - keyOriginal?: string - author?: string - take: number - skip: number - oss?: boolean - filter?: Record - viewerId?: string | null - exclude?: GQLSearchExclude - }) => { - const searchBody = bodybuilder() - .query('multi_match', { - query: key, - fuzziness: 'AUTO', - fields: [ - 'displayName^15', - 'title^10', - 'title.synonyms^5', - 'content^2', - 'content.synonyms', - ], - type: 'most_fields', - }) - .from(skip) - .size(take) - - // only return active if not in oss - if (!oss) { - searchBody.filter('term', { state: ARTICLE_STATE.active }) - } - - // add filter - if (filter && Object.keys(filter).length > 0) { - searchBody.filter('term', filter) - } - - // gather users that blocked viewer - const excludeBlocked = exclude === GQLSearchExclude.blocked && viewerId - let blockedIds: string[] = [] - if (excludeBlocked) { - blockedIds = ( - await this.knex('action_user') - .select('user_id') - .where({ action: USER_ACTION.block, targetId: viewerId }) - ).map(({ userId }) => userId) - } - - try { - // check if media hash in search key - const re = /^([0-9a-zA-Z]{49,59})$/gi - const match = re.exec(key) - if (match) { - const matched = await this.searchByMediaHash({ - key: match[1], - oss, - filter, - }) - let items = (await this.draftLoader.loadMany( - matched.nodes.map((item) => item.id) - )) as Item[] - - if (excludeBlocked) { - items = items.filter((item) => !blockedIds.includes(item.authorId)) - } - - // TODO: check totalCount - return { nodes: items, totalCount: items.length } - } - - // take the condition that searching for exact article title into consideration - const idsByTitle = [] - if (key.length >= 5 && skip === 0) { - const articles = await this.findByTitle({ title: key, oss, filter }) - for (const article of articles) { - idsByTitle.push(article.id) - } - } - searchBody.notFilter('ids', { values: idsByTitle }) - - const { - hits, // ...rest - } = await this.es.client.search({ - index: this.table, - body: searchBody.build(), - }) - const ids = idsByTitle.concat( - hits.hits.map(({ _id }: { _id: any }) => _id) - ) - - let nodes = (await this.draftLoader.loadMany(ids)) as Item[] - - if (excludeBlocked) { - nodes = nodes.filter((node) => !blockedIds.includes(node.authorId)) - } - - // console.log(new Date(), 'searchBody:', JSON.stringify(searchBody.build()), `elasticsearch got ${hits?.hits?.length} from res:`, JSON.stringify(hits?.total), JSON.stringify(rest)) - - // TODO: check totalCount - // error TS2339: Property 'value' does not exist on type 'number | SearchTotalHits'. - return { nodes, totalCount: (hits?.total as any)?.value ?? nodes.length } - } catch (err) { - console.error( - new Date(), - `es.client.search failed with ERROR:`, - err, - 'searchBody:', - searchBody - ) - logger.error(err) - throw new ServerError('article search failed') - } - } - - searchV1 = async ({ key, keyOriginal, take = 10, skip = 0, - oss = false, - filter, exclude, viewerId, coefficients, @@ -1052,164 +926,6 @@ export class ArticleService extends BaseService { author?: string take: number skip: number - oss?: boolean - filter?: Record - viewerId?: string | null - exclude?: GQLSearchExclude - coefficients?: string - }) => { - let coeffs = [1, 1, 1, 1] - try { - coeffs = JSON.parse(coefficients || '[]') - } catch (err) { - // do nothing - } - - const c0 = +( - coeffs?.[0] || - environment.searchPgArticleCoefficients?.[0] || - 1 - ) - const c1 = +( - coeffs?.[1] || - environment.searchPgArticleCoefficients?.[1] || - 1 - ) - const c2 = +( - coeffs?.[2] || - environment.searchPgArticleCoefficients?.[2] || - 1 - ) - const c3 = +( - coeffs?.[3] || - environment.searchPgArticleCoefficients?.[3] || - 1 - ) - // const c4 = +(coeffs?.[4] || environment.searchPgArticleCoefficients?.[4] || 1) - - // gather users that blocked viewer - const excludeBlocked = exclude === GQLSearchExclude.blocked && viewerId - let blockedIds: string[] = [] - if (excludeBlocked) { - blockedIds = ( - await this.knex('action_user') - .select('user_id') - .where({ action: USER_ACTION.block, targetId: viewerId }) - ).map(({ userId }) => userId) - } - - const baseQuery = this.searchKnex - .from( - this.searchKnex - .select( - '*', - this.searchKnex.raw( - '(_text_cd_rank/(_text_cd_rank + 1)) AS text_cd_rank' - ) - ) - .from( - this.searchKnex - .select( - 'id', - 'num_views', - 'title_orig', // 'title', - 'created_at', - 'last_read_at', // -- title, slug, - this.searchKnex.raw( - `percent_rank() OVER (ORDER BY num_views NULLS FIRST) AS views_rank` - ), - // this.searchKnex.raw('(CASE WHEN title LIKE ? THEN 1 ELSE 0 END) ::float AS title_like_rank', [`%${key}%`]), - this.searchKnex.raw( - 'ts_rank(title_ts, query) AS title_ts_rank' - ), - this.searchKnex.raw( - 'COALESCE(ts_rank(summary_ts, query, 1), 0) ::float AS summary_ts_rank' - ), - this.searchKnex.raw( - 'ts_rank_cd(text_ts, query, 4) AS _text_cd_rank' - ) - ) - .from('search_index.article') - .crossJoin( - this.searchKnex.raw("plainto_tsquery('chinese_zh', ?) query", [ - key, - ]) - ) - .whereIn('state', [ARTICLE_STATE.active]) - .andWhere('author_state', 'NOT IN', [ - // USER_STATE.active, USER_STATE.onboarding, - USER_STATE.archived, - USER_STATE.banned, - ]) - .andWhere('author_id', 'NOT IN', blockedIds) - .andWhereRaw( - `(query @@ title_ts OR query @@ summary_ts OR query @@ text_ts)` - ) - .as('t0') - ) - .as('t1') - ) - .where('title_ts_rank', '>=', SEARCH_TITLE_RANK_THRESHOLD) - .orWhere('text_cd_rank', '>=', SEARCH_DEFAULT_TEXT_RANK_THRESHOLD) - - const records = await this.searchKnex - .select( - '*', - this.searchKnex.raw( - '(? * views_rank + ? * title_ts_rank + ? * summary_ts_rank + ? * text_cd_rank) AS score', - [c0, c1, c2, c3] - ), - this.searchKnex.raw('COUNT(id) OVER() ::int AS total_count') - ) - .from(baseQuery.as('base')) - .orderByRaw('score DESC NULLS LAST') - .orderByRaw('num_views DESC NULLS LAST') - .orderByRaw('id DESC') - .modify((builder: Knex.QueryBuilder) => { - if (take !== undefined && Number.isFinite(take)) { - builder.limit(take) - } - if (skip !== undefined && Number.isFinite(skip)) { - builder.offset(skip) - } - }) - - const nodes = (await this.draftLoader.loadMany( - records.map((item: any) => item.id).filter(Boolean) - )) as Item[] - - const totalCount = records.length === 0 ? 0 : +records[0].totalCount - - debugLog( - // new Date(), - `articleService::searchV1 searchKnex instance got ${nodes.length} nodes from: ${totalCount} total:`, - { key, keyOriginal, baseQuery: baseQuery.toString() }, - // { countRes, articleIds } - { sample: records?.slice(0, 3) } - ) - - return { nodes, totalCount } - } - - // the jieba based schema - searchV2 = async ({ - key, - keyOriginal, - take = 10, - skip = 0, - oss = false, - filter, - exclude, - viewerId, - coefficients, - }: { - key: string - keyOriginal?: string - author?: string - take: number - skip: number - oss?: boolean - filter?: Record viewerId?: string | null exclude?: GQLSearchExclude coefficients?: string diff --git a/src/connectors/tagService.ts b/src/connectors/tagService.ts index 82c6ff9a2..426daae57 100644 --- a/src/connectors/tagService.ts +++ b/src/connectors/tagService.ts @@ -15,7 +15,6 @@ import { VIEW, } from 'common/enums' import { environment } from 'common/environment' -import { ServerError } from 'common/errors' import logger from 'common/logger' import { BaseService } from 'connectors' import { Item, ItemData } from 'definitions' @@ -563,7 +562,6 @@ export class TagService extends BaseService { } } - // the searchV0: TBDeprecated in next release search = async ({ key, keyOriginal, @@ -571,267 +569,6 @@ export class TagService extends BaseService { skip, includeAuthorTags, viewerId, - }: { - key: string - keyOriginal?: string - author?: string - take: number - skip: number - includeAuthorTags?: boolean - viewerId?: string | null - }) => { - const _key = keyOriginal || key - const body = bodybuilder() - .query('match', 'content', _key) - .sort([ - { _score: 'desc' }, - { numArticles: 'desc' }, - { numAuthors: 'desc' }, - { createdAt: 'asc' }, // prefer earlier created one if same number of articles - ]) - .from(skip) - .size(take) - .build() - - try { - const ids = new Set() - let totalCount: number = 0 - if (includeAuthorTags && viewerId) { - const [res, res2] = await Promise.all([ - this.knex - .from(this.knex.ref(VIEW.authors_lasts_view).as('a')) - .joinRaw( - 'CROSS JOIN jsonb_to_recordset(top_tags) AS x(id int, num_articles int, last_use timestamptz)' - ) - .where('a.id', viewerId) - .select('x.id'), - // also get the tags use from last articles in recent days - this.knex - .from('article_tag AS at') - .join('article AS a', 'at.article_id', 'a.id') - .where('a.author_id', viewerId) - .andWhere( - 'at.created_at', - '>=', - this.knex.raw(`CURRENT_DATE - '7 days' ::interval`) - ) - .select(this.knex.raw('DISTINCT at.tag_id ::int')), - ]) - res.forEach(({ id }) => ids.add(+id)) - res2.forEach(({ tagId }) => ids.add(+tagId)) - } - - if (_key) { - try { - const result = await this.es.client.search({ - index: this.table, - body, - }) - - const { hits } = result - - hits.hits.forEach((hit) => ids.add((hit._source as any)?.id)) - if (typeof hits.total === 'number') { - totalCount = hits.total - } else if (hits.total) { - totalCount = hits.total.value // es version upgrade changed internal scheme - } - } catch (err) { - console.error(new Date(), 'es client search ERROR:', err) - } - } - - const queryTags = this.knex - .select( - 'id', - 'content', - 'description', - 'num_articles', - 'num_authors', - 'created_at', - this.knex.raw('(content = ?) AS content_equal_rank', [_key]), - this.knex.raw('(content ILIKE ?) AS content_ilike_rank', [ - `%${_key}%`, - ]), - this.knex.raw('COUNT(id) OVER() ::int AS total_count') - ) - .from(VIEW.tags_lasts_view) - .where((builder: Knex.QueryBuilder) => { - // if either author's freq-use tags have something, or es client told a number - if (ids.size > 0) { - builder.whereIn('id', Array.from(ids)) - } - - if (totalCount === 0) { - // otherwise if es client got nothing, try some slower ilike match, better than nothing - builder.whereILike('content', [`%${_key}%`]) - } - }) - .andWhere((builder: Knex.QueryBuilder) => { - builder.whereNotIn('id', [environment.mattyChoiceTagId]) - }) - .orderByRaw('content_equal_rank DESC') // always show exact match at first - .orderByRaw('content_ilike_rank DESC') // then show inclusive match, by case insensitive - .orderByRaw('num_authors DESC NULLS LAST') - .orderByRaw('num_articles DESC NULLS LAST') - .orderByRaw('id') // fallback earlier ones - .modify((builder: Knex.QueryBuilder) => { - if (skip !== undefined && Number.isFinite(skip)) { - builder.offset(skip) - } - if (take !== undefined && Number.isFinite(take)) { - builder.limit(take) - } - }) - - const nodes = await queryTags - - totalCount = nodes.length === 0 ? 0 : +nodes[0].totalCount - - debugLog( - // new Date(), - `tagService::searchV0 got ${nodes.length} nodes from: ${totalCount} total:`, - { key, keyOriginal, queryTags: queryTags.toString() }, - { sample: nodes?.slice(0, 3) } - ) - - return { nodes, totalCount } - } catch (err) { - logger.error(err) - console.error(new Date(), 'tag searchV0 ERROR:', err) - throw new ServerError('tag search failed') - } - } - - searchV1 = async ({ - key, - keyOriginal, - take, - skip, - includeAuthorTags, - viewerId, - coefficients, - quicksearch, - }: { - key: string - keyOriginal?: string - author?: string - take: number - skip: number - includeAuthorTags?: boolean - viewerId?: string | null - coefficients?: string - quicksearch?: boolean - }) => { - let coeffs = [1, 1, 1, 1] - try { - coeffs = JSON.parse(coefficients || '[]') - } catch (err) { - // do nothing - } - - const a = +(coeffs?.[0] || environment.searchPgTagCoefficients?.[0] || 1) - const b = +(coeffs?.[1] || environment.searchPgTagCoefficients?.[1] || 1) - const c = +(coeffs?.[2] || environment.searchPgTagCoefficients?.[2] || 1) - const d = +(coeffs?.[3] || environment.searchPgTagCoefficients?.[3] || 1) - - // debugLog(new Date(), `searchV1 tag got search key:`, {key, keyOriginal,}) - - const strip0 = key.startsWith('#') || key.startsWith('＃') - const _key = strip0 ? key.slice(1) : key - - if (!_key) { - return { nodes: [], totalCount: 0 } - } - - const mattyChoiceTagIds = environment.mattyChoiceTagId - ? [environment.mattyChoiceTagId] - : [] - - const baseQuery = this.searchKnex - .select( - 'id', - 'content_orig AS content', - 'description', - // 'num_articles', // 'num_followers', - this.searchKnex.raw( - 'percent_rank() OVER (ORDER by num_followers NULLS FIRST) AS followers_rank' - ), - this.searchKnex.raw( - '(CASE WHEN content LIKE ? THEN 1 ELSE 0 END) ::float AS content_like_rank', - [`%${_key}%`] - ), - this.searchKnex.raw('ts_rank(content_ts, query) AS content_rank'), - this.searchKnex.raw( - 'ts_rank(description_ts, query) AS description_rank' - ), - this.searchKnex.raw('COALESCE(num_articles, 0) AS num_articles'), - this.searchKnex.raw('COALESCE(num_authors, 0) AS num_authors') - // this.searchKnex.raw('COALESCE(num_followers, 0) AS num_followers'), - ) - .from('search_index.tag') - .crossJoin( - this.searchKnex.raw(`plainto_tsquery('chinese_zh', ?) query`, key) - ) - .whereNotIn('id', mattyChoiceTagIds) - .andWhere((builder: Knex.QueryBuilder) => { - builder.whereLike('content', `%${_key}%`) - - if (!quicksearch) { - builder - .orWhereRaw('content_ts @@ query') - .orWhereRaw('description_ts @@ query') - } - }) - - const queryTags = this.searchKnex - .select( - '*', - this.searchKnex.raw( - '(? * followers_rank + ? * content_like_rank + ? * content_rank + ? * description_rank) AS score', - [a, b, c, d] - ), - this.searchKnex.raw('COUNT(id) OVER() ::int AS total_count') - ) - .from(baseQuery.as('base')) - .modify((builder: Knex.QueryBuilder) => { - if (quicksearch) { - builder.orderByRaw('content = ? DESC', [_key]) // always show exact match at first - } else { - builder.orderByRaw('score DESC NULLS LAST') - } - }) - .orderByRaw('num_articles DESC NULLS LAST') - .orderByRaw('id') // fallback to earlier first - .modify((builder: Knex.QueryBuilder) => { - if (skip !== undefined && Number.isFinite(skip)) { - builder.offset(skip) - } - if (take !== undefined && Number.isFinite(take)) { - builder.limit(take) - } - }) - - const nodes = (await queryTags) as Item[] - const totalCount = nodes.length === 0 ? 0 : +nodes[0].totalCount - - debugLog( - // new Date(), - `tagService::searchV1 searchKnex instance got ${nodes.length} nodes from: ${totalCount} total:`, - { key, keyOriginal, queryTags: queryTags.toString() }, - { sample: nodes?.slice(0, 3) } - ) - - return { nodes, totalCount } - } - - searchV2 = async ({ - key, - keyOriginal, - take, - skip, - includeAuthorTags, - viewerId, coefficients, quicksearch, }: { diff --git a/src/connectors/userService.ts b/src/connectors/userService.ts index 46d3dd60e..5639b90d5 100644 --- a/src/connectors/userService.ts +++ b/src/connectors/userService.ts @@ -1,5 +1,4 @@ import { compare } from 'bcrypt' -import bodybuilder from 'bodybuilder' import DataLoader from 'dataloader' import createDebug from 'debug' import jwt from 'jsonwebtoken' @@ -35,7 +34,6 @@ import { NameInvalidError, PasswordInvalidError, PasswordNotAvailableError, - ServerError, UserInputError, } from 'common/errors' import logger from 'common/logger' @@ -507,268 +505,11 @@ export class UserService extends BaseService { } } - // the searchV0: TBDeprecated in next release search = async ({ key, keyOriginal, take, skip, - oss = false, - filter, - exclude, - viewerId, - }: { - key: string - keyOriginal?: string - author?: string - take: number - skip: number - oss?: boolean - filter?: Record - viewerId?: string | null - exclude?: GQLSearchExclude - }) => { - const body = bodybuilder() - .from(skip) - .size(take) - .query('match', 'displayName.raw', keyOriginal) - .filter('term', 'state', USER_STATE.active) - .build() as { [key: string]: any } - - body.suggest = { - userName: { - prefix: keyOriginal, - completion: { - field: 'userName', - fuzzy: { - fuzziness: 0, - }, - size: take, - }, - }, - displayName: { - prefix: keyOriginal, - completion: { - field: 'displayName', - fuzzy: { - fuzziness: 0, - }, - size: take, - }, - }, - } - - try { - const result = await this.es.client.search({ - index: this.table, - body, - }) - - const { hits, suggest } = result as typeof result & { - hits: { hits: any[] } - suggest: { userName: any[]; displayName: any[] } - } - - const matchIds = hits.hits.map(({ _id }: { _id: any }) => _id) - - const userNameIds = suggest.userName[0].options.map( - ({ _id }: { _id: any }) => _id - ) - const displayNameIds = suggest.displayName[0].options.map( - ({ _id }: { _id: any }) => _id - ) - - // merge two ID arrays and remove duplicates - let ids = [...new Set([...userNameIds, ...displayNameIds, ...matchIds])] - - // filter out users who blocked viewer - if (exclude === GQLSearchExclude.blocked && viewerId) { - const blockedIds = ( - await this.knex('action_user') - .select('user_id') - .where({ action: USER_ACTION.block, targetId: viewerId }) - ).map(({ userId }) => userId) - - ids = _.difference(ids, blockedIds) - } - const nodes = await this.baseFindByIds(ids) - return { nodes, totalCount: nodes.length } - } catch (err) { - logger.error(err) - console.error(new Date(), 'user searchV0 ERROR:', err) - throw new ServerError('user search failed') - } - } - - searchV1 = async ({ - key, - keyOriginal, - take, - skip, - oss = false, - filter, - exclude, - viewerId, - coefficients, - quicksearch, - }: { - key: string - keyOriginal?: string - author?: string - take: number - skip: number - oss?: boolean - filter?: Record - viewerId?: string | null - exclude?: GQLSearchExclude - coefficients?: string - quicksearch?: boolean - }) => { - let coeffs = [1, 1, 1, 1] - try { - coeffs = JSON.parse(coefficients || '[]') - } catch (err) { - // do nothing - } - - const c0 = +(coeffs?.[0] || environment.searchPgUserCoefficients?.[0] || 1) - const c1 = +(coeffs?.[1] || environment.searchPgUserCoefficients?.[1] || 1) - const c2 = +(coeffs?.[2] || environment.searchPgUserCoefficients?.[2] || 1) - const c3 = +(coeffs?.[3] || environment.searchPgUserCoefficients?.[3] || 1) - const c4 = +(coeffs?.[4] || environment.searchPgUserCoefficients?.[4] || 1) - const c5 = +(coeffs?.[5] || environment.searchPgUserCoefficients?.[5] || 1) - const c6 = +(coeffs?.[6] || environment.searchPgUserCoefficients?.[6] || 1) - - const searchUserName = key.startsWith('@') || key.startsWith('＠') - const strippedName = key.replaceAll(/^[@＠]+/g, '').trim() // (searchUserName ? key.slice(1) : key).trim() - - if (!strippedName) { - return { nodes: [], totalCount: 0 } - } - - // gather users that blocked viewer - const excludeBlocked = exclude === GQLSearchExclude.blocked && viewerId - let blockedIds: string[] = [] - if (excludeBlocked) { - blockedIds = ( - await this.knex('action_user') - .select('user_id') - .where({ action: USER_ACTION.block, targetId: viewerId }) - ).map(({ userId }) => userId) - } - - const baseQuery = this.searchKnex - .select( - '*', - this.searchKnex.raw( - 'percent_rank() OVER (ORDER BY num_followers NULLS FIRST) AS followers_rank' - ), - - this.searchKnex.raw( - '(CASE WHEN user_name = ? THEN 1 ELSE 0 END) ::float AS user_name_equal_rank', - [strippedName] - ), - this.searchKnex.raw( - '(CASE WHEN display_name = ? THEN 1 ELSE 0 END) ::float AS display_name_equal_rank', - [strippedName] - ), - this.searchKnex.raw( - '(CASE WHEN user_name LIKE ? THEN 1 ELSE 0 END) ::float AS user_name_like_rank', - [`%${strippedName}%`] - ), - this.searchKnex.raw( - '(CASE WHEN display_name LIKE ? THEN 1 ELSE 0 END) ::float AS display_name_like_rank', - [`%${strippedName}%`] - ), - this.searchKnex.raw( - 'ts_rank(display_name_ts, query) AS display_name_ts_rank' - ), - this.searchKnex.raw( - 'ts_rank(description_ts, query) AS description_ts_rank' - ) - ) - .from('search_index.user') - .crossJoin( - this.searchKnex.raw(`plainto_tsquery('chinese_zh', ?) query`, key) - ) - .where('state', 'NOT IN ', [ - // USER_STATE.active, USER_STATE.onboarding, - USER_STATE.archived, - USER_STATE.banned, - ]) - .andWhere('id', 'NOT IN', blockedIds) - .andWhere((builder: Knex.QueryBuilder) => { - builder - .whereLike('user_name', `%${strippedName}%`) - .orWhereLike('display_name', `%${strippedName}%`) - - if (!quicksearch) { - builder - .orWhereRaw('display_name_ts @@ query') - .orWhereRaw('description_ts @@ query') - } - }) - - const queryUsers = this.searchKnex - .select( - '*', - this.searchKnex.raw( - '(? * followers_rank + ? * user_name_equal_rank + ? * display_name_equal_rank + ? * user_name_like_rank + ? * display_name_like_rank + ? * display_name_ts_rank + ? * description_ts_rank) AS score', - [c0, c1, c2, c3, c4, c5, c6] - ), - this.searchKnex.raw('COUNT(result.id) OVER() AS total_count') - ) - .from(baseQuery.as('result')) - .modify((builder: Knex.QueryBuilder) => { - if (quicksearch) { - if (searchUserName) { - builder - .orderByRaw('user_name = ? DESC', [strippedName]) - .orderByRaw('display_name = ? DESC', [strippedName]) - } else { - builder - .orderByRaw('display_name = ? DESC', [strippedName]) - .orderByRaw('user_name = ? DESC', [strippedName]) - } - } else { - builder.orderByRaw('score DESC NULLS LAST') - } - }) - .orderByRaw('num_followers DESC NULLS LAST') - .orderByRaw('id') // fallback to earlier first - .modify((builder: Knex.QueryBuilder) => { - if (skip !== undefined && Number.isFinite(skip)) { - builder.offset(skip) - } - if (take !== undefined && Number.isFinite(take)) { - builder.limit(take) - } - }) - - const records = (await queryUsers) as Item[] - const totalCount = records.length === 0 ? 0 : +records[0].totalCount - - debugLog( - // new Date(), - `userService::searchV1 searchKnex instance got ${records.length} nodes from: ${totalCount} total:`, - { key, keyOriginal, queryUsers: queryUsers.toString() }, - { sample: records?.slice(0, 3) } - ) - - const nodes = (await this.dataloader.loadMany( - records.map(({ id }) => id) - )) as Item[] - - return { nodes, totalCount } - } - - searchV2 = async ({ - key, - keyOriginal, - take, - skip, - oss = false, - filter, exclude, viewerId, coefficients, @@ -779,8 +520,6 @@ export class UserService extends BaseService { author?: string take: number skip: number - oss?: boolean - filter?: Record viewerId?: string | null exclude?: GQLSearchExclude coefficients?: string diff --git a/src/definitions/schema.d.ts b/src/definitions/schema.d.ts index 05e3c58e4..74eaa1847 100644 --- a/src/definitions/schema.d.ts +++ b/src/definitions/schema.d.ts @@ -2814,9 +2814,13 @@ export interface GQLSearchInput { oss?: boolean /** - * use the api version; omit to use latest version + * deprecated, make no effect */ version?: GQLSearchAPIVersion + + /** + * deprecated, make no effect + */ coefficients?: string quicksearch?: boolean } diff --git a/src/queries/system/search.ts b/src/queries/system/search.ts index b89488ee5..94b110f67 100644 --- a/src/queries/system/search.ts +++ b/src/queries/system/search.ts @@ -1,4 +1,4 @@ -import _ from 'lodash' +import { compact } from 'lodash' import { SEARCH_ARTICLE_URL_REGEX, @@ -10,19 +10,14 @@ import { fromGlobalId, normalizeQueryInput, } from 'common/utils' -import { - GQLNode, - GQLSearchAPIVersion, - QueryToSearchResolver, -} from 'definitions' +import { GQLNode, QueryToSearchResolver } from 'definitions' // the original ElasticSearch based solution const resolver: QueryToSearchResolver = async ( - root, + _, args, // { input }, - context, // { dataSources: { systemService, articleService, userService, tagService }, viewer, } - info + context // { dataSources: { systemService, articleService, userService, tagService }, viewer, } ) => { const { input } = args const { @@ -61,30 +56,23 @@ const resolver: QueryToSearchResolver = async ( const keyOriginal = input.key input.key = await normalizeQueryInput(keyOriginal) - const connection = await (input.version === GQLSearchAPIVersion.v20230301 - ? serviceMap[input.type].searchV2 - : input.version === GQLSearchAPIVersion.v20221212 - ? serviceMap[input.type].searchV1 - : serviceMap[input.type].search)({ - ...input, - keyOriginal, - take, - skip, - viewerId: viewer.id, - }).then(({ nodes, totalCount }) => { - nodes = _.compact(nodes) - return { - nodes: nodes.map((node: GQLNode) => ({ __type: input.type, ...node })), - totalCount, - } - }) + const connection = await serviceMap[input.type] + .search({ + ...input, + keyOriginal, + take, + skip, + viewerId: viewer.id, + }) + .then(({ nodes, totalCount }) => { + nodes = compact(nodes) + return { + nodes: nodes.map((node: GQLNode) => ({ __type: input.type, ...node })), + totalCount, + } + }) return connectionFromArray(connection.nodes, input, connection.totalCount) - - // let res = resolverV0 - // switch (input.version) {// case GQLSearchAPIVersion.v20230301: case GQLSearchAPIVersion.v20221212: res = resolverV1 break} - - // return res(root, args, context, info) } export default resolver diff --git a/src/types/system.ts b/src/types/system.ts index 172353553..edbcd0d0e 100644 --- a/src/types/system.ts +++ b/src/types/system.ts @@ -234,8 +234,9 @@ export default /* GraphQL */ ` record: Boolean oss: Boolean - "use the api version; omit to use latest version" + "deprecated, make no effect" version: SearchAPIVersion + "deprecated, make no effect" coefficients: String quicksearch: Boolean } From dbca4a3a3eb3e97e17b3741d8a55649168ff3aa1 Mon Sep 17 00:00:00 2001 From: xdj <13580441+gary02@users.noreply.github.com> Date: Tue, 9 May 2023 20:53:58 +0800 Subject: [PATCH 2/5] feat(related-articles): remove es code --- src/connectors/articleService.ts | 55 +---------- src/queries/article/relatedArticles.ts | 130 +++---------------------- 2 files changed, 16 insertions(+), 169 deletions(-) diff --git a/src/connectors/articleService.ts b/src/connectors/articleService.ts index 505577743..ec3b8c489 100644 --- a/src/connectors/articleService.ts +++ b/src/connectors/articleService.ts @@ -28,7 +28,7 @@ import { USER_ACTION, USER_STATE, } from 'common/enums' -import { environment, isTest } from 'common/environment' +import { environment } from 'common/environment' import { ArticleNotFoundError, ServerError } from 'common/errors' import logger from 'common/logger' import { @@ -1062,59 +1062,6 @@ export class ArticleService extends BaseService { return { nodes, totalCount } } - /********************************* - * * - * Recommand * - * * - *********************************/ - related = async ({ - id, - size, - notIn = [], - }: { - id: string - size: number - notIn?: string[] - }) => { - // skip if in test - if (isTest) { - return [] - } - - // get vector score - const scoreResult = await this.es.client.get({ - index: this.table, - id, - }) - - const factors = (scoreResult as any)?._source?.embedding_vector - - // return empty list if we don't have any score - if (!factors) { - return [] - } - - const searchBody = { - index: this.table, - knn: { - field: 'embedding_vector', - query_vector: factors, - k: 10, - num_candidates: size, - }, - filter: { - bool: { - must: { term: { state: ARTICLE_STATE.active } }, - must_not: { ids: { values: notIn.concat([id]) } }, - }, - }, - } - - const body = await this.es.client.knnSearch(searchBody) - // add recommendation - return body.hits.hits.map((hit: any) => ({ ...hit, id: hit._id })) - } - /** * Boost & Score */ diff --git a/src/queries/article/relatedArticles.ts b/src/queries/article/relatedArticles.ts index d164eaa61..e0556e841 100644 --- a/src/queries/article/relatedArticles.ts +++ b/src/queries/article/relatedArticles.ts @@ -1,16 +1,10 @@ import _ from 'lodash' -import { ARTICLE_STATE } from 'common/enums' -import logger from 'common/logger' -import { - connectionFromArray, - fromConnectionArgs, - loadManyFilterError, -} from 'common/utils' +import { connectionFromArray, fromConnectionArgs } from 'common/utils' import { ArticleToRelatedArticlesResolver } from 'definitions' const resolver: ArticleToRelatedArticlesResolver = async ( - { articleId, authorId, title }, + { articleId, authorId }, { input }, { dataSources: { articleService, draftService, tagService } } ) => { @@ -24,88 +18,27 @@ const resolver: ArticleToRelatedArticlesResolver = async ( const addRec = (rec: any[], extra: any[]) => _.uniqBy(rec.concat(extra), 'id').filter((_rec) => _rec.id !== articleId) - // articles in collection for this article as the entrance - // const entranceId = articleId - const collection = ( - await articleService.findCollections({ entranceId: articleId }) - ).map((item: any) => item.articleId) - // const ids: string[] = [] let articles: any[] = [] - let sameIdx = -1 - // get initial recommendation - try { - const relatedArticles = await articleService.related({ - id: articleId, - size: take + buffer, - notIn: collection, - }) - - // articles in collection shall be excluded from recommendation - const relatedArticleIds = relatedArticles.map( - ({ id: aid }: { id: any }) => aid - ) - - // logger.info(`[recommendation] article ${articleId}, title ${title}, ES result ${relatedArticleIds}`) - - // get articles - articles = await articleService.dataloader - .loadMany(relatedArticleIds) - .then(loadManyFilterError) - .then((allArticles) => - allArticles.filter(({ state }) => state === ARTICLE_STATE.active) - ) + // first select from tags + const tagIds = await articleService.findTagIds({ id: articleId }) - // tslint:disable-next-line - if ((sameIdx = articles?.findIndex((item) => item.id === articleId)) >= 0) { - console.log( - new Date(), - `found same article at {${sameIdx}} from articleService.related and remove it`, - sameIdx - ) - articles.splice(sameIdx, 1) + for (const tagId of tagIds) { + if (articles.length >= take + buffer) { + break } - } catch (err) { - logger.error(`error in recommendation via ES: ${JSON.stringify(err)}`) - } - - // fall back to tags - if (articles.length < take + buffer) { - const tagIds = await articleService.findTagIds({ id: articleId }) - - for (const tagId of tagIds) { - if (articles.length >= take + buffer) { - break - } - - const articleIds = await tagService.findArticleIds({ - id: tagId, - take, // : take - ids.length, // this ids.length is always 0?? - skip, - }) - // logger.info(`[recommendation] article ${articleId}, title ${title}, tag result ${articleIds} `) - - // get articles and append - const articlesFromTag = await articleService.dataloader.loadMany( - articleIds - ) + const articleIds = await tagService.findArticleIds({ + id: tagId, + take, // : take - ids.length, // this ids.length is always 0?? + skip, + }) - articles = addRec(articles, articlesFromTag) + // get articles and append + const articlesFromTag = await articleService.dataloader.loadMany(articleIds) - if ( - // tslint:disable-next-line - (sameIdx = articles?.findIndex((item) => item.id === articleId)) >= 0 - ) { - console.log( - new Date(), - `found same article at {${sameIdx}} from articleService.findTagIds and remove it`, - { sameIdx, articleId, tagId } - ) - articles.splice(sameIdx, 1) - } - } + articles = addRec(articles, articlesFromTag) } // fall back to author @@ -113,16 +46,6 @@ const resolver: ArticleToRelatedArticlesResolver = async ( const articlesFromAuthor = await articleService.findByAuthor(authorId) // logger.info(`[recommendation] article ${articleId}, title ${title}, author result ${articlesFromAuthor.map(({ id: aid }: { id: string }) => aid)} `) articles = addRec(articles, articlesFromAuthor) - - // tslint:disable-next-line - if ((sameIdx = articles?.findIndex((item) => item.id === articleId)) >= 0) { - console.log( - new Date(), - `found same article at {${sameIdx}} from articleService.findByAuthor and remove it`, - { sameIdx, articleId } - ) - articles.splice(sameIdx, 1) - } } // random pick for last few elements @@ -132,33 +55,10 @@ const resolver: ArticleToRelatedArticlesResolver = async ( _.sampleSize(articles.slice(take - randomPick), randomPick) ) - // tslint:disable-next-line - if ((sameIdx = articles?.findIndex((item) => item.id === articleId)) >= 0) { - console.log( - new Date(), - `found same article at {${sameIdx}} after randomPick and remove it`, - { sameIdx, articleId } - ) - articles.splice(sameIdx, 1) - } - const nodes = await draftService.dataloader.loadMany( pick.map((item) => item.draftId) ) - if ( - // tslint:disable-next-line - (sameIdx = nodes?.findIndex((item: any) => item.articleId === articleId)) >= - 0 - ) { - console.log( - new Date(), - `found same article at {${sameIdx}} at last step and remove it`, - { sameIdx, articleId } - ) - nodes.splice(sameIdx, 1) - } - return connectionFromArray(nodes, input) } From 9c80375bdca4fabda6fef5e40ec5c6952ebed0cf Mon Sep 17 00:00:00 2001 From: xdj <13580441+gary02@users.noreply.github.com> Date: Tue, 9 May 2023 22:19:15 +0800 Subject: [PATCH 3/5] feat(related-tags): remove es code --- src/connectors/tagService.ts | 53 +------------------------- src/queries/article/tag/recommended.ts | 9 ++--- src/types/__test__/tag.test.ts | 20 ++++++++++ 3 files changed, 25 insertions(+), 57 deletions(-) diff --git a/src/connectors/tagService.ts b/src/connectors/tagService.ts index 426daae57..d3f9a1e1b 100644 --- a/src/connectors/tagService.ts +++ b/src/connectors/tagService.ts @@ -1,17 +1,11 @@ -import bodybuilder from 'bodybuilder' import DataLoader from 'dataloader' import createDebug from 'debug' import { Knex } from 'knex' -// import _ from 'lodash' import { ARTICLE_STATE, DEFAULT_TAKE_PER_PAGE, - // MATERIALIZED_VIEW, - // MAX_TAG_CONTENT_LENGTH, - // MAX_TAG_DESCRIPTION_LENGTH, TAG_ACTION, - TAGS_RECOMMENDED_LIMIT, VIEW, } from 'common/enums' import { environment } from 'common/environment' @@ -21,8 +15,6 @@ import { Item, ItemData } from 'definitions' const debugLog = createDebug('tag-service') -// const SEARCH_DEFAULT_TEXT_RANK_THRESHOLD = 0.0001 - export class TagService extends BaseService { constructor() { super('tag') @@ -1172,18 +1164,8 @@ export class TagService extends BaseService { * top100 at most * */ - findRelatedTags = async ({ - id, - content: tagContent, - }: // skip, take, exclude, - { - id: string - content?: string - // skip?: number - // take?: number - // exclude?: string[] - }) => { - const results = await this.knex + findRelatedTags = async ({ id }: { id: string; content?: string }) => { + return this.knex .from(VIEW.tags_lasts_view) .joinRaw( 'CROSS JOIN jsonb_to_recordset(top_rels) AS x(tag_rel_id int, count_rel int, count_common int, similarity float)' @@ -1191,36 +1173,5 @@ export class TagService extends BaseService { .where(this.knex.raw(`dup_tag_ids @> ARRAY[?] ::int[]`, [id])) .select('x.tag_rel_id AS id') .orderByRaw('x.count_rel * x.similarity DESC NULLS LAST') - - if (results?.length < TAGS_RECOMMENDED_LIMIT && tagContent) { - const body = bodybuilder() - .query('match', 'content', tagContent) - .size(TAGS_RECOMMENDED_LIMIT) // at most 100 - .build() - - const result = await this.es.client.search({ - index: this.table, - body, - }) - - const { hits } = result - if ((hits.hits?.[0]?._source as any)?.content === tagContent) { - hits.hits.shift() // remove the exact match at first, if exists - } - - // hits.hits.forEach((hit) => fromEsTags.add(hit._source)) - - const existingIds = new Set(results.map((item) => item.id)) - for (const hit of hits.hits) { - if (!existingIds.has((hit._source as any).id)) { - results.push({ id: (hit._source as any).id }) - if (results?.length >= TAGS_RECOMMENDED_LIMIT) { - break - } - } - } - } - - return results } } diff --git a/src/queries/article/tag/recommended.ts b/src/queries/article/tag/recommended.ts index 765127680..671de7cae 100644 --- a/src/queries/article/tag/recommended.ts +++ b/src/queries/article/tag/recommended.ts @@ -10,9 +10,9 @@ import { import { Item, TagToRecommendedResolver } from 'definitions' const resolver: TagToRecommendedResolver = async ( - { id, content: inputContent, owner }, + { id }, { input }, - { dataSources: { tagService, userService } } + { dataSources: { tagService } } ) => { const { take, skip } = fromConnectionArgs(input, { allowTakeAll: true, @@ -24,10 +24,7 @@ const resolver: TagToRecommendedResolver = async ( return connectionFromArray([], input) } - const relatedIds = await tagService.findRelatedTags({ - id, - content: inputContent, - }) + const relatedIds = await tagService.findRelatedTags({ id }) const tags = ( (await tagService.dataloader.loadMany( diff --git a/src/types/__test__/tag.test.ts b/src/types/__test__/tag.test.ts index ea3e6c01e..664333ac3 100644 --- a/src/types/__test__/tag.test.ts +++ b/src/types/__test__/tag.test.ts @@ -20,6 +20,15 @@ const QUERY_TAG = /* GraphQL */ ` id content description + recommended(input: {}) { + edges { + node { + ... on Tag { + content + } + } + } + } } } } @@ -627,3 +636,14 @@ describe('manage settings of a tag', () => { expect(leaveData?.editors.length).toBe(1) }) }) + +describe('query tag', () => { + test('tag recommended', async () => { + const server = await testClient() + const { data } = await server.executeOperation({ + query: QUERY_TAG, + variables: { input: { id: toGlobalId({ type: NODE_TYPES.Tag, id: 1 }) } }, + }) + expect(data!.node.recommended.edges).toBeDefined() + }) +}) From b32e988b9284caa470196e0564da0d1b6bf3c447 Mon Sep 17 00:00:00 2001 From: xdj <13580441+gary02@users.noreply.github.com> Date: Tue, 9 May 2023 22:59:29 +0800 Subject: [PATCH 4/5] feat(elasticsearch): remove elasticsearch reindex code --- src/connectors/articleService.ts | 74 ------------------------ src/connectors/atomService.ts | 19 +------ src/connectors/baseService.ts | 14 +---- src/connectors/queue/publication.ts | 11 ---- src/connectors/queue/revision.ts | 8 --- src/connectors/queue/user.ts | 12 ---- src/connectors/tagService.ts | 81 --------------------------- src/connectors/userService.ts | 58 ------------------- src/mutations/article/deleteTags.ts | 8 +-- src/mutations/article/mergeTags.ts | 8 +-- src/mutations/article/putTag.ts | 7 --- src/mutations/article/renameTag.ts | 6 -- src/mutations/user/updateUserInfo.ts | 21 +------ src/mutations/user/updateUserRole.ts | 13 ----- src/mutations/user/updateUserState.ts | 15 ----- src/queries/system/search.ts | 2 - 16 files changed, 7 insertions(+), 350 deletions(-) diff --git a/src/connectors/articleService.ts b/src/connectors/articleService.ts index ec3b8c489..5aae67337 100644 --- a/src/connectors/articleService.ts +++ b/src/connectors/articleService.ts @@ -1,9 +1,6 @@ -// import type { SearchTotalHits } from '@elastic/elasticsearch' - import { ArticlePageContext, makeArticlePage, - stripHtml, } from '@matters/ipns-site-generator' import slugify from '@matters/slugify' import DataLoader from 'dataloader' @@ -30,7 +27,6 @@ import { } from 'common/enums' import { environment } from 'common/environment' import { ArticleNotFoundError, ServerError } from 'common/errors' -import logger from 'common/logger' import { AtomService, BaseService, @@ -663,19 +659,6 @@ export class ArticleService extends BaseService { sticky: false, updatedAt: new Date(), }) - - // update search - try { - await this.es.client.update({ - index: this.table, - id: article.id, - body: { - doc: { state: ARTICLE_STATE.archived }, - }, - }) - } catch (e) { - logger.error(e) - } } } @@ -824,63 +807,6 @@ export class ArticleService extends BaseService { * Search * * * *********************************/ - /** - * Dump all data to ES (Currently only used in test) - */ - initSearch = async () => { - const articles = await this.knex(this.table) - .innerJoin('user', `${this.table}.author_id`, 'user.id') - .select( - `${this.table}.id as id`, - 'title', - 'content', - 'author_id as authorId', - 'user.user_name as userName', - 'user.display_name as displayName' - ) - - return this.es.indexManyItems({ - index: this.table, - items: articles.map( - (article: { content: string; title: string; id: string }) => ({ - ...article, - content: stripHtml(article.content), - }) - ), - }) - } - - addToSearch = async ({ - id, - title, - content, - authorId, - userName, - displayName, - tags, - }: { - [key: string]: any - }) => { - try { - return await this.es.indexItems({ - index: this.table, - items: [ - { - id, - title, - content: stripHtml(content), - state: ARTICLE_STATE.active, - authorId, - userName, - displayName, - tags, - }, - ], - }) - } catch (error) { - logger.error(error) - } - } searchByMediaHash = async ({ key, diff --git a/src/connectors/atomService.ts b/src/connectors/atomService.ts index cac07bd3f..0e273dc27 100644 --- a/src/connectors/atomService.ts +++ b/src/connectors/atomService.ts @@ -3,8 +3,7 @@ import DataLoader from 'dataloader' import { Knex } from 'knex' import { EntityNotFoundError } from 'common/errors' -import logger from 'common/logger' -import { aws, cfsvc, es, knex } from 'connectors' +import { aws, cfsvc, knex } from 'connectors' import { Item, TableName } from 'definitions' interface InitLoaderInput { @@ -77,7 +76,6 @@ interface MaxInput { * This object is a container for data loaders or system wide services. */ export class AtomService extends DataSource { - es: typeof es aws: typeof aws cfsvc: typeof cfsvc knex: Knex @@ -90,7 +88,6 @@ export class AtomService extends DataSource { constructor() { super() - this.es = es this.aws = aws this.cfsvc = cfsvc this.knex = knex @@ -300,18 +297,4 @@ export class AtomService extends DataSource { .first() return parseInt(record ? (record.count as string) : '0', 10) } - - /* Elastic Search */ - - /** - * Delete data stored in elastic search. - */ - deleteSearch = async ({ table, id }: { table: TableName; id: any }) => { - try { - const result = await this.es.client.delete({ index: table, id }) - return result - } catch (error) { - logger.error(error) - } - } } diff --git a/src/connectors/baseService.ts b/src/connectors/baseService.ts index 5df7e6e6f..cb4109fdf 100644 --- a/src/connectors/baseService.ts +++ b/src/connectors/baseService.ts @@ -4,20 +4,10 @@ import { Knex } from 'knex' import _ from 'lodash' import logger from 'common/logger' -import { - aws, - cfsvc, - es, - knex, - meiliClient, - readonlyKnex, - searchKnexDB, -} from 'connectors' +import { aws, cfsvc, knex, readonlyKnex, searchKnexDB } from 'connectors' import { Item, ItemData, TableName } from 'definitions' export class BaseService extends DataSource { - es: typeof es - meili: typeof meiliClient aws: typeof aws cfsvc: typeof cfsvc knex: Knex @@ -28,8 +18,6 @@ export class BaseService extends DataSource { constructor(table: TableName) { super() - this.es = es - this.meili = meiliClient this.knex = knex this.knexRO = readonlyKnex this.searchKnex = searchKnexDB diff --git a/src/connectors/queue/publication.ts b/src/connectors/queue/publication.ts index 582d97833..4385e1dc2 100644 --- a/src/connectors/queue/publication.ts +++ b/src/connectors/queue/publication.ts @@ -215,17 +215,6 @@ export class PublicationQueue extends BaseQueue { article.id ) await job.progress(75) - - // Step 6: add to search; async - this.articleService.addToSearch({ - id: article.id, - title: draft.title, - content: draft.content, - authorId: article.authorId, - userName, - displayName, - tags, - }) } catch (err) { // ignore errors caused by these steps console.error(new Date(), 'optional step failed:', err, job, draft) diff --git a/src/connectors/queue/revision.ts b/src/connectors/queue/revision.ts index 371bf7cff..b07b72e47 100644 --- a/src/connectors/queue/revision.ts +++ b/src/connectors/queue/revision.ts @@ -177,14 +177,6 @@ class RevisionQueue extends BaseQueue { entityTypeId, }) - // Step 6: add to search; async - this.articleService.addToSearch({ - ...article, - content: draft.content, - userName, - displayName, - }) - // Step 7: handle newly added mentions await this.handleMentions({ article: updatedArticle, diff --git a/src/connectors/queue/user.ts b/src/connectors/queue/user.ts index 95143b3f4..79d5b2d2d 100644 --- a/src/connectors/queue/user.ts +++ b/src/connectors/queue/user.ts @@ -132,18 +132,6 @@ class UserQueue extends BaseQueue { data, }) - try { - await this.atomService.es.client.update({ - index: 'user', - id: record.userId, - body: { - doc: data, - }, - }) - } catch (err) { - logger.error(err) - } - await this.userService.baseUpdate( record.id, { archived: true }, diff --git a/src/connectors/tagService.ts b/src/connectors/tagService.ts index d3f9a1e1b..a5a879e07 100644 --- a/src/connectors/tagService.ts +++ b/src/connectors/tagService.ts @@ -9,7 +9,6 @@ import { VIEW, } from 'common/enums' import { environment } from 'common/environment' -import logger from 'common/logger' import { BaseService } from 'connectors' import { Item, ItemData } from 'definitions' @@ -256,15 +255,6 @@ export class TagService extends BaseService { skipCreate, // : content.length > MAX_TAG_CONTENT_LENGTH, // || (description && description.length > MAX_TAG_DESCRIPTION_LENGTH), }) - // add tag into search engine - if (tag) { - this.addToSearch({ - id: tag.id, - content: tag.content, - description: tag.description, - }) - } - return tag } @@ -489,70 +479,6 @@ export class TagService extends BaseService { * Search * * * *********************************/ - initSearch = async () => { - const tags = await this.knex - .from(VIEW.tags_lasts_view) - .select( - 'id', - 'content', - 'description', - 'num_articles', - 'num_authors', - 'created_at', - 'span_days', - 'earliest_use', - 'latest_use' - ) - - return this.es.indexManyItems({ - index: this.table, - items: tags, // .map((tag) => ({...tag,})), - }) - } - - addToSearch = async ({ - id, - content, - description, - }: { - [key: string]: any - }) => { - try { - return await this.es.indexItems({ - index: this.table, - items: [ - { - id, - content, - description, - }, - ], - }) - } catch (error) { - logger.error(error) - } - } - - updateSearch = async ({ - id, - content, - description, - }: { - [key: string]: any - }) => { - try { - const result = await this.es.client.update({ - index: this.table, - id, - body: { - doc: { content, description }, - }, - }) - return result - } catch (error) { - logger.error(error) - } - } search = async ({ key, @@ -1105,13 +1031,6 @@ export class TagService extends BaseService { // create new tag const newTag = await this.create({ content, creator, editors, owner }) - // add tag into search engine - this.addToSearch({ - id: newTag.id, - content: newTag.content, - description: newTag.description, - }) - // move article tags to new tag const articleIds = await this.findArticleIdsByTagIds(tagIds) await this.createArticleTags({ articleIds, creator, tagIds: [newTag.id] }) diff --git a/src/connectors/userService.ts b/src/connectors/userService.ts index 5639b90d5..ab1b9de37 100644 --- a/src/connectors/userService.ts +++ b/src/connectors/userService.ts @@ -137,8 +137,6 @@ export class UserService extends BaseService { ) await this.baseCreate({ userId: user.id }, 'user_notify_setting') - this.addToSearch(user) - return user } @@ -398,19 +396,6 @@ export class UserService extends BaseService { return user }) - // update search - try { - await this.es.client.update({ - index: this.table, - id, - body: { - doc: { state: USER_STATE.archived }, - }, - }) - } catch (e) { - logger.error(e) - } - return archivedUser } @@ -461,49 +446,6 @@ export class UserService extends BaseService { * Search * * * *********************************/ - /** - * Dump all data to ES (Currently only used in test) - */ - initSearch = async () => { - const users = await this.knex(this.table).select( - 'id', - 'description', - 'display_name', - 'user_name' - ) - - return this.es.indexManyItems({ - index: this.table, - items: users.map((user) => ({ - ...user, - })), - }) - } - - addToSearch = async ({ - id, - userName, - displayName, - description, - }: { - [key: string]: string - }) => { - try { - return await this.es.indexItems({ - index: this.table, - items: [ - { - id, - userName, - displayName, - description, - }, - ], - }) - } catch (error) { - logger.error(error) - } - } search = async ({ key, diff --git a/src/mutations/article/deleteTags.ts b/src/mutations/article/deleteTags.ts index 78cc5577f..e68721201 100644 --- a/src/mutations/article/deleteTags.ts +++ b/src/mutations/article/deleteTags.ts @@ -6,9 +6,9 @@ import { CacheService } from 'connectors' import { MutationToDeleteTagsResolver } from 'definitions' const resolver: MutationToDeleteTagsResolver = async ( - root, + _, { input: { ids } }, - { viewer, dataSources: { atomService } } + { dataSources: { atomService } } ) => { const tagIds = ids.map((id) => fromGlobalId(id).id) @@ -28,10 +28,6 @@ const resolver: MutationToDeleteTagsResolver = async ( // delete tags await atomService.deleteMany({ table: 'tag', whereIn: ['id', tagIds] }) - await Promise.all( - tagIds.map((id: string) => atomService.deleteSearch({ table: 'tag', id })) - ) - // manually invalidate cache since it returns nothing const cacheService = new CacheService() await Promise.all( diff --git a/src/mutations/article/mergeTags.ts b/src/mutations/article/mergeTags.ts index 9d6eda855..3ff59e54b 100644 --- a/src/mutations/article/mergeTags.ts +++ b/src/mutations/article/mergeTags.ts @@ -5,9 +5,9 @@ import { fromGlobalId } from 'common/utils' import { MutationToMergeTagsResolver } from 'definitions' const resolver: MutationToMergeTagsResolver = async ( - root, + _, { input: { ids, content } }, - { viewer, dataSources: { atomService, tagService, userService } } + { dataSources: { tagService } } ) => { // assign Matty as tag's editor if (!environment.mattyId) { @@ -22,10 +22,6 @@ const resolver: MutationToMergeTagsResolver = async ( owner: environment.mattyId, }) - await Promise.all( - tagIds.map((id: string) => atomService.deleteSearch({ table: 'tag', id })) - ) - // invalidate extra nodes newTag[CACHE_KEYWORD] = tagIds.map((id) => ({ id, type: NODE_TYPES.Tag })) return newTag diff --git a/src/mutations/article/putTag.ts b/src/mutations/article/putTag.ts index 046841fc7..e44006855 100644 --- a/src/mutations/article/putTag.ts +++ b/src/mutations/article/putTag.ts @@ -129,13 +129,6 @@ const resolver: MutationToPutTagResolver = async ( const updateTag = await tagService.baseUpdate(dbId, updateParams) - // update tag for search engine - await tagService.updateSearch({ - id: updateTag.id, - content: updateTag.content, - description: updateTag.description, - }) - // delete unused tag cover if (tag.cover && tag.cover !== updateTag.cover) { const coverAsset = await tagService.baseFindById(tag.cover, 'asset') diff --git a/src/mutations/article/renameTag.ts b/src/mutations/article/renameTag.ts index 7e3384e13..b07617128 100644 --- a/src/mutations/article/renameTag.ts +++ b/src/mutations/article/renameTag.ts @@ -9,12 +9,6 @@ const resolver: MutationToRenameTagResolver = async ( const { id: dbId } = fromGlobalId(id) const newTag = await tagService.renameTag({ tagId: dbId, content }) - // update tag for search engine - tagService.updateSearch({ - id: newTag.id, - content: newTag.content, - description: newTag.description, - }) return newTag } diff --git a/src/mutations/user/updateUserInfo.ts b/src/mutations/user/updateUserInfo.ts index e71e72e97..8c6321876 100644 --- a/src/mutations/user/updateUserInfo.ts +++ b/src/mutations/user/updateUserInfo.ts @@ -1,4 +1,4 @@ -import { has, isEmpty, isNil, omitBy } from 'lodash' +import { has, isEmpty } from 'lodash' import { v4 } from 'uuid' import { ASSET_TYPE } from 'common/enums' @@ -203,25 +203,6 @@ const resolver: MutationToUpdateUserInfoResolver = async ( }) } - // update user info to es - const { description, displayName, userName } = updateParams - - if (description || displayName || userName) { - const searchable = omitBy({ description, displayName, userName }, isNil) - - try { - await atomService.es.client.update({ - index: 'user', - id: viewer.id, - body: { - doc: searchable, - }, - }) - } catch (err) { - logger.error(err) - } - } - // trigger notifications if (updateParams.paymentPasswordHash) { notificationService.mail.sendPayment({ diff --git a/src/mutations/user/updateUserRole.ts b/src/mutations/user/updateUserRole.ts index ffb0ee2b9..8fcd08253 100644 --- a/src/mutations/user/updateUserRole.ts +++ b/src/mutations/user/updateUserRole.ts @@ -17,19 +17,6 @@ const resolver: MutationToUpdateUserRoleResolver = async ( data, }) - try { - await atomService.es.client.update({ - index: 'user', - id: dbId, - body: { - doc: data, - }, - }) - } catch (err) { - // logger.error(err) - console.error(new Date(), 'ERROR:', err) - } - return user } diff --git a/src/mutations/user/updateUserState.ts b/src/mutations/user/updateUserState.ts index e8098e80f..2e64859f3 100644 --- a/src/mutations/user/updateUserState.ts +++ b/src/mutations/user/updateUserState.ts @@ -1,6 +1,5 @@ import { OFFICIAL_NOTICE_EXTEND_TYPE, USER_STATE } from 'common/enums' import { ActionFailedError, UserInputError } from 'common/errors' -import logger from 'common/logger' import { fromGlobalId, getPunishExpiredDate } from 'common/utils' import { userQueue } from 'connectors/queue' import { MutationToUpdateUserStateResolver } from 'definitions' @@ -122,20 +121,6 @@ const resolver: MutationToUpdateUserStateResolver = async ( }, }) - try { - await atomService.es.client.update({ - index: 'user', - id: dbId, - body: { - doc: { - state, - }, - }, - }) - } catch (err) { - logger.error(err) - } - if (state === USER_STATE.banned) { handleBan(updatedUser.id) } else if (state !== user.state && user.state === USER_STATE.banned) { diff --git a/src/queries/system/search.ts b/src/queries/system/search.ts index 94b110f67..6bb78d814 100644 --- a/src/queries/system/search.ts +++ b/src/queries/system/search.ts @@ -12,8 +12,6 @@ import { } from 'common/utils' import { GQLNode, QueryToSearchResolver } from 'definitions' -// the original ElasticSearch based solution - const resolver: QueryToSearchResolver = async ( _, args, // { input }, From a5b4cb2677344f1ac0b14d77ad8a8b26280120ae Mon Sep 17 00:00:00 2001 From: xdj <13580441+gary02@users.noreply.github.com> Date: Tue, 9 May 2023 23:00:07 +0800 Subject: [PATCH 5/5] feat: remove elasticsearch and meili totally --- .env.example | 4 - README.md | 3 +- docker/docker-compose.yml | 21 -- package-lock.json | 111 ----------- package.json | 1 - src/common/environment.ts | 5 - src/common/utils/initSearchIndices.ts | 269 -------------------------- src/connectors/es/index.ts | 107 ---------- src/connectors/index.ts | 2 - src/connectors/meili/index.ts | 9 - 10 files changed, 1 insertion(+), 531 deletions(-) delete mode 100644 src/common/utils/initSearchIndices.ts delete mode 100644 src/connectors/es/index.ts delete mode 100644 src/connectors/meili/index.ts diff --git a/.env.example b/.env.example index ebfb010a0..f03f0fe4f 100644 --- a/.env.example +++ b/.env.example @@ -40,10 +40,6 @@ MATTERS_JWT_SECRET=QsNmu9 MATTERS_MATTY_ID=6 MATTERS_MATTY_CHOICE_TAG_ID=3906 MATTERS_EMAIL_FROM_ASK=Matters -MATTERS_ELASTICSEARCH_HOST=elasticsearch -MATTERS_ELASTICSEARCH_PORT=9200 -MATTERS_MEILISEARCH_SERVER=http://meili.dev.vpc:7700 -MATTERS_MEILISEARCH_APIKEY=masterKey # the IPFS_HOST & IPFS_PORT will be deprecated MATTERS_IPFS_HOST=ipfs diff --git a/README.md b/README.md index bd19a0159..7f6674627 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ ### Local - Install dependencies: `npm install` -- Start Postgres, Redis, ElasticSearch, stripe-mock, and IPFS daemon +- Start Postgres, Redis, stripe-mock, and IPFS daemon - Setup Environments: `cp .env.example .env` - Run all migrations: `npm run db:migrate` - Populate all seeds data if needed: `npm run db:seed` @@ -51,5 +51,4 @@ AWS resources that we need to put in the same VPC - Pub/Sub - Cache - Queue -- ElasticSearch EC2 instances - IPFS cluster EC2 instances diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 1cb1bbcf5..14023ad96 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -16,7 +16,6 @@ services: - db - redis - s3 - - elasticsearch - ipfs - stripe ports: @@ -49,25 +48,6 @@ services: - '8080:8080' - '4001:4001' - '5001:5001' - elasticsearch: - image: matterslab/elasticsearch:latest - container_name: elasticsearch - environment: - - 'ES_JAVA_OPTS=-Xms750m -Xmx750m' - - 'discovery.type=single-node' - - 'xpack.security.enabled=false' - volumes: - - elasticsearch_data:/usr/share/elasticsearch/data - ports: - - '9200:9200' - - '9300:9300' - kibana: - container_name: kibana - image: kibana:5.4 - ports: - - '5601:5601' - depends_on: - - elasticsearch stripe: container_name: stripe image: stripemock/stripe-mock:latest @@ -77,4 +57,3 @@ services: volumes: postgres_data: - elasticsearch_data: diff --git a/package-lock.json b/package-lock.json index 88e71241b..5c75d5def 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,6 @@ "version": "4.18.0", "license": "Apache-2.0", "dependencies": { - "@elastic/elasticsearch": "^8.2.1", "@ethersproject/abstract-provider": "^5.7.0", "@google-cloud/translate": "^6.2.6", "@graphql-tools/schema": "^7.1.5", @@ -1067,44 +1066,6 @@ "kuler": "^2.0.0" } }, - "node_modules/@elastic/elasticsearch": { - "version": "8.2.1", - "resolved": "https://registry.npmjs.org/@elastic/elasticsearch/-/elasticsearch-8.2.1.tgz", - "integrity": "sha512-Kwerd8DfNZdBGgl7fkn+20kXkw1QePB3goTv5QwW9poo2d4VbPE0EChmh6irpXWAGsVSYiKr8x6bh8dH5YdylA==", - "dependencies": { - "@elastic/transport": "^8.2.0", - "tslib": "^2.4.0" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/@elastic/elasticsearch/node_modules/tslib": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.0.tgz", - "integrity": "sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ==" - }, - "node_modules/@elastic/transport": { - "version": "8.2.0", - "resolved": "https://registry.npmjs.org/@elastic/transport/-/transport-8.2.0.tgz", - "integrity": "sha512-H/HmefMNQfLiBSVTmNExu2lYs5EzwipUnQB53WLr17RCTDaQX0oOLHcWpDsbKQSRhDAMPPzp5YZsZMJxuxPh7A==", - "dependencies": { - "debug": "^4.3.4", - "hpagent": "^1.0.0", - "ms": "^2.1.3", - "secure-json-parse": "^2.4.0", - "tslib": "^2.4.0", - "undici": "^5.1.1" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/@elastic/transport/node_modules/tslib": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.0.tgz", - "integrity": "sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ==" - }, "node_modules/@ethersproject/abi": { "version": "5.7.0", "resolved": "https://registry.npmjs.org/@ethersproject/abi/-/abi-5.7.0.tgz", @@ -9418,14 +9379,6 @@ "integrity": "sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==", "dev": true }, - "node_modules/hpagent": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/hpagent/-/hpagent-1.0.0.tgz", - "integrity": "sha512-SCleE2Uc1bM752ymxg8QXYGW0TWtAV4ZW3TqH1aOnyi6T6YW2xadCcclm5qeVjvMvfQ2RKNtZxO7uVb9CTPt1A==", - "engines": { - "node": ">=14" - } - }, "node_modules/html-encoding-sniffer": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-2.0.1.tgz", @@ -17074,11 +17027,6 @@ "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-2.0.2.tgz", "integrity": "sha512-Ntyt4AIXyaLIuMHF6IOoTakB3K+RWxwtsHNRxllEoA6vPwP9o4866g6YWDLUdnucilZhmkxiHwHr11gAENw+QA==" }, - "node_modules/secure-json-parse": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.4.0.tgz", - "integrity": "sha512-Q5Z/97nbON5t/L/sH6mY2EacfjVGwrCcSi5D3btRO2GZ8pf1K1UN7Z9H5J57hjVU2Qzxr1xO+FmBhOvEkzCMmg==" - }, "node_modules/selectn": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/selectn/-/selectn-1.1.2.tgz", @@ -18266,14 +18214,6 @@ "integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==", "dev": true }, - "node_modules/undici": { - "version": "5.10.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-5.10.0.tgz", - "integrity": "sha512-c8HsD3IbwmjjbLvoZuRI26TZic+TSEe8FPMLLOkN1AfYRhdjnKBU6yL+IwcSCbdZiX4e5t0lfMDLDCqj4Sq70g==", - "engines": { - "node": ">=12.18" - } - }, "node_modules/unified": { "version": "10.1.2", "resolved": "https://registry.npmjs.org/unified/-/unified-10.1.2.tgz", @@ -19865,42 +19805,6 @@ "kuler": "^2.0.0" } }, - "@elastic/elasticsearch": { - "version": "8.2.1", - "resolved": "https://registry.npmjs.org/@elastic/elasticsearch/-/elasticsearch-8.2.1.tgz", - "integrity": "sha512-Kwerd8DfNZdBGgl7fkn+20kXkw1QePB3goTv5QwW9poo2d4VbPE0EChmh6irpXWAGsVSYiKr8x6bh8dH5YdylA==", - "requires": { - "@elastic/transport": "^8.2.0", - "tslib": "^2.4.0" - }, - "dependencies": { - "tslib": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.0.tgz", - "integrity": "sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ==" - } - } - }, - "@elastic/transport": { - "version": "8.2.0", - "resolved": "https://registry.npmjs.org/@elastic/transport/-/transport-8.2.0.tgz", - "integrity": "sha512-H/HmefMNQfLiBSVTmNExu2lYs5EzwipUnQB53WLr17RCTDaQX0oOLHcWpDsbKQSRhDAMPPzp5YZsZMJxuxPh7A==", - "requires": { - "debug": "^4.3.4", - "hpagent": "^1.0.0", - "ms": "^2.1.3", - "secure-json-parse": "^2.4.0", - "tslib": "^2.4.0", - "undici": "^5.1.1" - }, - "dependencies": { - "tslib": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.0.tgz", - "integrity": "sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ==" - } - } - }, "@ethersproject/abi": { "version": "5.7.0", "resolved": "https://registry.npmjs.org/@ethersproject/abi/-/abi-5.7.0.tgz", @@ -26116,11 +26020,6 @@ "integrity": "sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==", "dev": true }, - "hpagent": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/hpagent/-/hpagent-1.0.0.tgz", - "integrity": "sha512-SCleE2Uc1bM752ymxg8QXYGW0TWtAV4ZW3TqH1aOnyi6T6YW2xadCcclm5qeVjvMvfQ2RKNtZxO7uVb9CTPt1A==" - }, "html-encoding-sniffer": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-2.0.1.tgz", @@ -31821,11 +31720,6 @@ } } }, - "secure-json-parse": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.4.0.tgz", - "integrity": "sha512-Q5Z/97nbON5t/L/sH6mY2EacfjVGwrCcSi5D3btRO2GZ8pf1K1UN7Z9H5J57hjVU2Qzxr1xO+FmBhOvEkzCMmg==" - }, "selectn": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/selectn/-/selectn-1.1.2.tgz", @@ -32742,11 +32636,6 @@ "integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==", "dev": true }, - "undici": { - "version": "5.10.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-5.10.0.tgz", - "integrity": "sha512-c8HsD3IbwmjjbLvoZuRI26TZic+TSEe8FPMLLOkN1AfYRhdjnKBU6yL+IwcSCbdZiX4e5t0lfMDLDCqj4Sq70g==" - }, "unified": { "version": "10.1.2", "resolved": "https://registry.npmjs.org/unified/-/unified-10.1.2.tgz", diff --git a/package.json b/package.json index fea3dd116..18f86055d 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,6 @@ "prepare": "husky install" }, "dependencies": { - "@elastic/elasticsearch": "^8.2.1", "@ethersproject/abstract-provider": "^5.7.0", "@google-cloud/translate": "^6.2.6", "@graphql-tools/schema": "^7.1.5", diff --git a/src/common/environment.ts b/src/common/environment.ts index 41ddccf5f..78cf60626 100644 --- a/src/common/environment.ts +++ b/src/common/environment.ts @@ -51,11 +51,6 @@ export const environment = { awsLikecoinUpdateCivicLikerCache: process.env.MATTERS_AWS_LIKECOIN_UPDATE_CIVIC_LIKER_CACHE_QUEUE_URL || '', awsArticlesSnsTopic: process.env.MATTERS_AWS_ARTICLES_SNS_TOPIC || '', - esHost: process.env.MATTERS_ELASTICSEARCH_HOST, - esPort: process.env.MATTERS_ELASTICSEARCH_PORT, - meiliSearch_Server: - process.env.MATTERS_MEILISEARCH_SERVER || 'http://meili.dev.vpc:7700', - meiliSearch_apiKey: process.env.MATTERS_MEILISEARCH_APIKEY || '', awsCloudFrontEndpoint: process.env.MATTERS_AWS_CLOUD_FRONT_ENDPOINT, cloudflareAccountId: process.env.MATTERS_CLOUDFLARE_ACCOUNT_ID, cloudflareAccountHash: process.env.MATTERS_CLOUDFLARE_ACCOUNT_HASH, diff --git a/src/common/utils/initSearchIndices.ts b/src/common/utils/initSearchIndices.ts deleted file mode 100644 index ad8538680..000000000 --- a/src/common/utils/initSearchIndices.ts +++ /dev/null @@ -1,269 +0,0 @@ -import elasticsearch from '@elastic/elasticsearch' -import 'module-alias/register' - -import { environment } from 'common/environment' -import logger from 'common/logger' -import { ArticleService, TagService, UserService } from 'connectors' - -const articleIndexDef = { - index: 'article', - settings: { - analysis: { - analyzer: { - pinyin: { - tokenizer: 'pinyin_tokenizer', - }, - tsconvert: { - type: 'custom', - char_filter: ['tsconvert'], - tokenizer: 'ik_max_word', - }, - synonym: { - type: 'custom', - char_filter: ['tsconvert'], - tokenizer: 'ik_smart', - filter: ['synonym'], - }, - }, - tokenizer: { - tsconvert: { - type: 'stconvert', - delimiter: '#', - keep_both: true, - convert_type: 't2s', - }, - pinyin_tokenizer: { - type: 'pinyin', - keep_first_letter: false, - keep_separate_first_letter: false, - keep_full_pinyin: true, - keep_original: true, - limit_first_letter_length: 16, - lowercase: true, - remove_duplicated_term: true, - }, - }, - filter: { - synonym: { - type: 'synonym', - synonyms_path: 'synonyms.txt', - }, - tsconvert: { - type: 'stconvert', - delimiter: '#', - keep_both: true, - convert_type: 't2s', - }, - }, - char_filter: { - tsconvert: { - type: 'stconvert', - convert_type: 't2s', - }, - }, - }, - }, - mappings: { - properties: { - id: { - type: 'long', - }, - authorId: { - type: 'long', - }, - userName: { - type: 'completion', - }, - displayName: { - type: 'completion', - analyzer: 'pinyin', - fields: { - raw: { - type: 'text', - analyzer: 'tsconvert', - }, - }, - }, - title: { - type: 'text', - index: true, - analyzer: 'tsconvert', - fields: { - synonyms: { - type: 'text', - analyzer: 'synonym', - }, - }, - }, - content: { - type: 'text', - index: true, - analyzer: 'tsconvert', - fields: { - synonyms: { - type: 'text', - analyzer: 'synonym', - }, - }, - }, - embedding_vector: { - type: 'dense_vector', - dims: 20, - index: true, - similarity: 'l2_norm', - }, - factor: { type: 'text' }, - }, - }, -} - -const userIndexDef = { - index: 'user', - settings: { - analysis: { - analyzer: { - pinyin: { - tokenizer: 'pinyin_tokenizer', - }, - tsconvert: { - type: 'custom', - char_filter: ['tsconvert'], - tokenizer: 'ik_max_word', - }, - synonym: { - type: 'custom', - char_filter: ['tsconvert'], - tokenizer: 'ik_smart', - filter: ['synonym'], - }, - }, - tokenizer: { - tsconvert: { - type: 'stconvert', - delimiter: '#', - keep_both: true, - convert_type: 't2s', - }, - pinyin_tokenizer: { - type: 'pinyin', - keep_first_letter: false, - keep_separate_first_letter: false, - keep_full_pinyin: true, - keep_original: true, - limit_first_letter_length: 16, - lowercase: true, - remove_duplicated_term: true, - }, - }, - filter: { - synonym: { - type: 'synonym', - synonyms_path: 'synonyms.txt', - }, - tsconvert: { - type: 'stconvert', - delimiter: '#', - keep_both: true, - convert_type: 't2s', - }, - }, - char_filter: { - tsconvert: { - type: 'stconvert', - convert_type: 't2s', - }, - }, - }, - }, - mappings: { - properties: { - id: { - type: 'long', - }, - userName: { - type: 'completion', - }, - displayName: { - type: 'completion', - analyzer: 'pinyin', - fields: { - raw: { - type: 'text', - analyzer: 'tsconvert', - }, - }, - }, - description: { - type: 'text', - index: true, - analyzer: 'tsconvert', - }, - embedding_vector: { - type: 'dense_vector', - dims: 20, - index: true, - similarity: 'l2_norm', - }, - factor: { - type: 'text', - }, - }, - }, -} - -const indices = ['article', 'user', 'tag'] - -async function main() { - const es = new elasticsearch.Client({ - node: `http://${environment.esHost}:${environment.esPort}`, - }) - - logger.info( - `connecting node: http://${environment.esHost}:${environment.esPort}` - ) - - await Promise.all( - indices.map(async (idx) => { - const exists = await es.indices.exists({ index: idx }) - if (exists) { - logger.info(`deleting es index: ${idx} ...`) - await es.indices.delete({ index: idx }) - } - }) - ) - - logger.info('creating indices: article, user, tag ...') - await Promise.all([ - es.indices.create(articleIndexDef), - es.indices.create(userIndexDef), - es.indices.create({ index: 'tag' }), - ]) - - // logger.info('indexing articles ...') - const articleService = new ArticleService() - // await articleService.initSearch() - - // logger.info('indexing users...') - const userService = new UserService() - // await userService.initSearch() - - // logger.info('indexing tags...') - const tagService = new TagService() - - logger.info('indexing: article, user, tag ...') - await Promise.all([ - articleService.initSearch(), - userService.initSearch(), - tagService.initSearch(), - ]) - - logger.info('done.') - process.exit() -} - -if (require.main === module) { - main().catch((err) => { - console.error(new Date(), 'ERROR:', err) - process.exit(1) - }) -} diff --git a/src/connectors/es/index.ts b/src/connectors/es/index.ts deleted file mode 100644 index 7c0d2b6dd..000000000 --- a/src/connectors/es/index.ts +++ /dev/null @@ -1,107 +0,0 @@ -import elasticsearch from '@elastic/elasticsearch' -import _ from 'lodash' - -import { environment } from 'common/environment' -import logger from 'common/logger' - -interface Item { - [key: string]: any - id: string -} - -class ElasticSearch { - client: elasticsearch.Client - - indices = ['article', 'user', 'tag'] - - constructor() { - this.client = new elasticsearch.Client({ - node: `http://${environment.esHost}:${environment.esPort}`, - }) - - this.init() - } - - init = async () => { - for (const index of this.indices) { - const exists = await this.client.indices.exists({ index }) - if (!exists) { - try { - logger.info(`Creating index ${index}`) - await this.client.indices.create({ index }) - logger.info(`Done`) - } catch (e) { - logger.error((e as Error).message) - } - } - } - } - - // clear = async () => { - // try { - // await this.client.indices.delete({ - // index: '_all' - // }) - - // await this.init() - // logger.info('All search indices are cleared') - // } catch (err) { - // throw err - // } - // } - - /** - * break many items into smaller chunks, then bulk index each chunk - */ - indexManyItems = async ({ - index, - items, - type, - }: { - index: string - type?: string - items: Item[] - }) => { - // break items into chunks - const size = 25 - const chunks: Item[][] = [] - while (items.length) { - chunks.push(items.splice(0, size)) - } - - // index items by chunks - const indexItemsByChunk = async (chks: Item[][]) => { - for (let i = 0; i < chks.length; i++) { - await this.indexItems({ - index, - items: chks[i], - }) - logger.info(`Indexed ${chks[i].length} items into ${index}.`) - } - } - - return indexItemsByChunk(chunks) - } - - indexItems = async ({ index, items }: { index: string; items: Item[] }) => { - const exists = await this.client.indices.exists({ index }) - if (!exists) { - await this.client.indices.create({ index }) - } - - try { - const body = _.flattenDepth( - items.map((item) => [{ index: { _index: index, _id: item.id } }, item]) - ) - - const res = await this.client.bulk({ - body, - }) - return res - } catch (err) { - throw err - } - } -} - -export const es = new ElasticSearch() diff --git a/src/connectors/index.ts b/src/connectors/index.ts index 92c5b0ae1..d86e4e759 100644 --- a/src/connectors/index.ts +++ b/src/connectors/index.ts @@ -2,11 +2,9 @@ export * from './aws' export * from './cache' export * from './cloudflare' export * from './db' -export * from './es' export * from './feed' export * from './ipfs' export * from './mail' -export * from './meili' export * from './likecoin' export * from './gcp' export * from './opensea' diff --git a/src/connectors/meili/index.ts b/src/connectors/meili/index.ts deleted file mode 100644 index 42def428c..000000000 --- a/src/connectors/meili/index.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { MeiliSearch } from 'meilisearch' -// import _ from 'lodash' - -import { environment } from 'common/environment' - -export const meiliClient = new MeiliSearch({ - host: environment.meiliSearch_Server, - apiKey: environment.meiliSearch_apiKey, -})