Squash commit for cherry pick

feat: create elasticsearch module

feat: create search module


Revert "feat: create elasticsearch module"

This reverts commit bcb9dda.

feat(repo): add prettier organize plugin


chore: move isTime function to utils


feat: add course index in env file


feat: update configuration file


feat: add search module to app module


chore(repo): remove unused files


chore(repo): remove unused files


feat: create course search interface


feat: import elasticsearch module in search module


feat: implement search service


feat: implement with course service


feat: add pagination


refactor: change search service to use search request as input


chore: add courseIndex to required configuration


chore(dep): move prettier plugin organize import to root package


refactor: change calling search method from course service to course resolver


chore(dep): update lock file


refactor: move interface to resolver


chore: change return type to Course graphql


chore: remove unused import


feat(repo): add changeset


chore(dep): add opensearch nestjs

refactor: change to use opensearch nestjs

chore: update config

chore: remove unused statement

refactor: update to beta v1.6.0

feat: register opensearch search service

chore(dep): add opensearch nestjs

feat: implement search service

fix: ssl problem

refactor: update query for opensearch lib

chore(dep): update lock file

feat: update template of scraper

feat: update scraper config

feat: create opensearch cli for create index

feat: finish search feature

chore(docs): update index information

chore: implement upsert data from scraper to opensearch

Update .changeset/enhance-search-engine.md

Co-authored-by: Nut Pinyo <bomb.NP@gmail.com>
refactor: move build course query to subdirectory


fix: config error

feat: change cli to execute file

chore: update as review

chore: add go project for opensearch cli

.changeset/enhance-search-engine.md

@@ -0,0 +1,5 @@
+---
+'api': minor
+---
+
+Enhance search engine with Elasticsearch

.gitignore

@@ -51,3 +51,8 @@ data/
 .turbo
 
 out
+
+# opensearch-cli
+.idea
+opensearch.yaml
+apps/opensearch-cli/index/local

apps/api/.env.template

@@ -9,9 +9,12 @@ JWT_SECRET=<insert_jwt_secret>
 
 ADMIN_TOKEN=<insert_admin_token>
 
-ELASTIC_URL=https://localhost:9200
-ELASTIC_USERNAME=cgr-api
-ELASTIC_PASSWORD=<password>
+OPENSEARCH_URL=https://localhost:9200
+OPENSEARCH_USERNAME=cgr-api
+OPENSEARCH_PASSWORD=password
+OPENSEARCH_SKIP_SSL=true
+
+COURSE_INDEX_NAME=course
 
 COMPUTATION_BACKEND_URL=http://localhost:8000
 COMPUTATION_BACKEND_AUTHTOKEN=<Authelia Token, Optional for Connecting to Computation Prod from outside>

apps/api/package.json

@@ -38,6 +38,7 @@
     "@nestjs/passport": "7.1.5",
     "@nestjs/platform-express": "9.4.0",
     "@opensearch-project/opensearch": "2.1.0",
+    "nestjs-opensearch": "^0.3.2",
     "@slack/webhook": "6.0.0",
     "@typegoose/typegoose": "9.1.0",
     "class-transformer": "0.3.1",

apps/api/src/app/app.module.ts

@@ -1,5 +1,5 @@
 import { ApolloDriver, ApolloDriverConfig } from '@nestjs/apollo'
-import { Logger, Module } from '@nestjs/common'
+import { Module } from '@nestjs/common'
 import { ConfigModule, ConfigService } from '@nestjs/config'
 import { GraphQLModule } from '@nestjs/graphql'
 import { MongooseModule } from '@nestjs/mongoose'
@@ -19,6 +19,7 @@ import { ReviewModule } from '../review/review.module'
 import { UserModule } from '../user/user.module'
 import { AppController } from './app.controller'
 import { AppService } from './app.service'
+import { SearchModule } from '@api/search/search.module'
 
 @Module({
   imports: [
@@ -81,6 +82,7 @@ import { AppService } from './app.service'
     ClientLoggingModule,
     OverrideModule,
     ComputationModule,
+    SearchModule,
   ],
   controllers: [AppController],
   providers: [AppService],

apps/api/src/clientlogging/clientlogging.module.ts

@@ -1,32 +1,34 @@
-import { Module, Provider, forwardRef } from '@nestjs/common'
+import { Module, forwardRef } from '@nestjs/common'
 import { ConfigService } from '@nestjs/config'
 
-import { Client } from '@opensearch-project/opensearch'
+import { OpensearchModule } from 'nestjs-opensearch'
 
 import { AuthModule } from '../auth/auth.module'
 import { ClientLoggingController } from './clientlogging.controller'
 import { ClientLoggingService } from './clientlogging.service'
 
-const elasticProvider: Provider<Client> = {
-  provide: Client,
-  useFactory: (configService: ConfigService) =>
-    new Client({
-      node: configService.get('elasticUrl'),
-      auth: {
-        username: configService.get('elasticUsername'),
-        password: configService.get('elasticPassword'),
-      },
-      ssl: {
-        rejectUnauthorized: false,
+@Module({
+  imports: [
+    forwardRef(() => AuthModule),
+    OpensearchModule.forRootAsync({
+      clientName: 'default',
+      inject: [ConfigService],
+      useFactory: async (configService: ConfigService) => {
+        return {
+          nodes: configService.get<string>('opensearchUrl'),
+          auth: {
+            username: configService.get<string>('opensearchUsername'),
+            password: configService.get<string>('opensearchPassword'),
+          },
+          ssl: {
+            rejectUnauthorized: !configService.get<boolean>('opensearchSkipSSL'),
+          },
+        }
       },
     }),
-  inject: [ConfigService],
-}
-
-@Module({
-  providers: [ClientLoggingService, elasticProvider],
+  ],
+  providers: [ClientLoggingService],
   controllers: [ClientLoggingController],
-  imports: [forwardRef(() => AuthModule)],
   exports: [ClientLoggingService],
 })
 export class ClientLoggingModule {}

apps/api/src/clientlogging/clientlogging.service.ts

@@ -1,7 +1,6 @@
 import { Injectable, InternalServerErrorException, Logger } from '@nestjs/common'
-import { ConfigService } from '@nestjs/config'
 
-import { Client } from '@opensearch-project/opensearch'
+import { InjectOpensearchClient, OpensearchClient } from 'nestjs-opensearch'
 import { hostname } from 'os'
 
 export interface GelfLogEntry {
@@ -15,7 +14,9 @@ const CLIENTLOGGING_INDEX = 'cgr-clientlogging'
 
 @Injectable()
 export class ClientLoggingService {
-  constructor(readonly configService: ConfigService, private readonly elasticClient: Client) {}
+  constructor(
+    @InjectOpensearchClient('default') private readonly opensearchClient: OpensearchClient
+  ) {}
 
   async sendLogEntry(entry: GelfLogEntry & Record<string, string>) {
     try {
@@ -25,7 +26,7 @@ export class ClientLoggingService {
         host: hostname(),
       }
 
-      await this.elasticClient.index({
+      await this.opensearchClient.index({
         index: CLIENTLOGGING_INDEX,
         body: record,
         refresh: true,

apps/api/src/config/configuration.ts

@@ -16,9 +16,11 @@ export interface Configuration {
   slackWebhookUrl: string
   reviewDashboardUrl: string
   env: string
-  elasticUrl: string
-  elasticUsername: string
-  elasticPassword: string
+  opensearchUrl: string
+  opensearchUsername: string
+  opensearchPassword: string
+  opensearchSkipSSL: boolean
+  courseIndexName: string
 }
 
 export const configuration = (): Configuration => {
@@ -37,9 +39,11 @@ export const configuration = (): Configuration => {
     slackWebhookUrl: process.env.SLACK_WEBHOOK_URL,
     reviewDashboardUrl: process.env.REVIEW_DASHBOARD_URL,
     env: process.env.ENV || 'development',
-    elasticUrl: process.env.ELASTIC_URL,
-    elasticUsername: process.env.ELASTIC_USERNAME,
-    elasticPassword: process.env.ELASTIC_PASSWORD,
+    opensearchUrl: process.env.OPENSEARCH_URL,
+    opensearchUsername: process.env.OPENSEARCH_USERNAME,
+    opensearchPassword: process.env.OPENSEARCH_PASSWORD,
+    opensearchSkipSSL: process.env.OPENSEARCH_SKIP_SSL === 'true',
+    courseIndexName: process.env.COURSE_INDEX_NAME,
   }
 }
 
@@ -50,9 +54,10 @@ const requiredConfigs = [
   'adminToken',
   'computationBackendUrl',
   'backendPublicUrl',
-  'elasticUrl',
-  'elasticUsername',
-  'elasticPassword',
+  'opensearchUrl',
+  'opensearchUsername',
+  'opensearchPassword',
+  'courseIndexName',
 ]
 
 export function validateConfig(configService: ConfigService<Configuration>): void {

apps/api/src/course/course.module.ts

@@ -1,13 +1,18 @@
 import { Module } from '@nestjs/common'
 import { MongooseModule } from '@nestjs/mongoose'
 
+import { SearchModule } from '@api/search/search.module'
+
 import { CourseSchema, ModelName } from '@cgr/schema'
 
 import { CourseResolver } from './course.resolver'
 import { CourseService } from './course.service'
 
 @Module({
-  imports: [MongooseModule.forFeature([{ name: ModelName.Course, schema: CourseSchema }])],
+  imports: [
+    MongooseModule.forFeature([{ name: ModelName.Course, schema: CourseSchema }]),
+    SearchModule,
+  ],
   providers: [CourseResolver, CourseService],
   exports: [CourseService],
 })

apps/api/src/course/course.resolver.ts

@@ -1,13 +1,48 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import { BadRequestException, Logger } from '@nestjs/common'
+import { ConfigService } from '@nestjs/config'
 import { Args, Query, Resolver } from '@nestjs/graphql'
 
-import { Semester, StudyProgram } from '@cgr/schema'
+import { buildCourseQuery } from '@api/search/queries/course'
+import { SearchService } from '@api/search/search.service'
+import { isTime } from '@api/util/functions'
 
-import { CourseGroupInput, FilterInput, Course as GraphQLCourse } from '../graphql'
+import { Course, CourseDocument, DayOfWeek, GenEdType, Semester, StudyProgram } from '@cgr/schema'
+
+import { CourseGroupInput, FilterInput, Period } from '../graphql'
 import { CourseService } from './course.service'
 
+export interface ICourseSearchDocument {
+  abbrName: string
+  courseNo: string
+  courseNameTh: string
+  courseNameEn: string
+  courseDescTh: string
+  courseDescEn: string
+  genEdType: string
+  studyProgram: string
+  semester: string
+  academicYear: string
+  rawData: Course
+}
+
+export interface ICourseSearchFilter {
+  keyword: string
+  genEdTypes?: GenEdType[]
+  dayOfWeeks?: DayOfWeek[]
+  periodRange: Period
+  studyProgram: string
+  semester: string
+  academicYear: string
+}
+
 @Resolver('Course')
 export class CourseResolver {
-  constructor(private readonly courseService: CourseService) {}
+  constructor(
+    private readonly courseService: CourseService,
+    private readonly configService: ConfigService,
+    private readonly searchService: SearchService
+  ) {}
 
   @Query('courseNos')
   courseNos(): Promise<Record<StudyProgram, string[]>> {
@@ -19,7 +54,7 @@ export class CourseResolver {
     @Args('courseNo') courseNo: string,
     @Args('courseGroup')
     { semester, academicYear, studyProgram }: CourseGroupInput
-  ): Promise<GraphQLCourse> {
+  ): Promise<CourseDocument> {
     return this.courseService.findOne(
       courseNo,
       semester as Semester,
@@ -32,7 +67,41 @@ export class CourseResolver {
   async search(
     @Args('filter') filter: FilterInput,
     @Args('courseGroup') courseGroup: CourseGroupInput
-  ): Promise<GraphQLCourse[]> {
-    return this.courseService.search(filter, courseGroup)
+  ): Promise<Course[]> {
+    if (filter.periodRange) {
+      const { start, end } = filter.periodRange
+      if (!isTime(start) || !isTime(end)) {
+        throw new BadRequestException({
+          reason: 'INVALID_PERIOD_RANGE',
+          message: 'Start time or end time is invalid',
+        })
+      }
+
+      if (start > end) {
+        throw new BadRequestException({
+          reason: 'INVALID_PERIOD_RANGE',
+          message: 'Start time cannot be later than end time',
+        })
+      }
+    }
+
+    const searchResult = await this.searchService.search<ICourseSearchDocument>({
+      index: this.configService.get<string>('courseIndexName'),
+      body: {
+        query: buildCourseQuery({
+          keyword: filter.keyword,
+          genEdTypes: filter.genEdTypes,
+          dayOfWeeks: filter.dayOfWeeks,
+          periodRange: filter.periodRange,
+          studyProgram: courseGroup.studyProgram,
+          semester: courseGroup.semester,
+          academicYear: courseGroup.academicYear,
+        }),
+        from: filter?.offset || 0,
+        size: filter?.limit || 10,
+      },
+    })
+
+    return searchResult.map((item) => item.rawData)
   }
 }