From 24400f284c8823f3da9ce64c0130cb47fbf20da1 Mon Sep 17 00:00:00 2001 From: jiwoo Date: Mon, 15 Jul 2024 16:32:21 +0900 Subject: [PATCH 01/31] =?UTF-8?q?setting:=20Chroma=20Vector=20DB=20?= =?UTF-8?q?=EC=9D=98=EC=A1=B4=EC=84=B1=20=EC=84=A4=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/build.gradle b/build.gradle index 75466795..8054faa2 100644 --- a/build.gradle +++ b/build.gradle @@ -46,6 +46,7 @@ dependencies { // AI implementation "org.springframework.ai:spring-ai-openai-spring-boot-starter:${springAiVersion}" + implementation "org.springframework.ai:spring-ai-chroma-store-spring-boot-starter:${springAiVersion}" // DB implementation 'org.springframework.boot:spring-boot-starter-data-jpa' From 4241964eb2660ded73bbca5ebf7cf93fafd7bb13 Mon Sep 17 00:00:00 2001 From: jiwoo Date: Mon, 15 Jul 2024 19:57:00 +0900 Subject: [PATCH 02/31] =?UTF-8?q?feat:=20=ED=99=98=EA=B2=BD=EC=84=A4?= =?UTF-8?q?=EC=A0=95=20=ED=8C=8C=EC=9D=BC=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/resources/application-local.yml | 6 ++++++ src/main/resources/application-prod.yml | 21 ++++++++++++++++----- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/main/resources/application-local.yml b/src/main/resources/application-local.yml index 57b6aef4..407a7016 100644 --- a/src/main/resources/application-local.yml +++ b/src/main/resources/application-local.yml @@ -44,6 +44,12 @@ spring: embedding: options: model: text-embedding-3-small + vectorstore: + chroma: + collection-name: ${CHROMA_COLLECTION_NAME} + client: + key-token: ${CHROMA_API_KEY} + springdoc: swagger-ui: diff --git a/src/main/resources/application-prod.yml b/src/main/resources/application-prod.yml index a3e27baa..3e44d7bc 100644 --- a/src/main/resources/application-prod.yml +++ b/src/main/resources/application-prod.yml @@ -34,14 +34,25 @@ spring: url: jdbc:${DB_URL} user: ${DB_USER} password: ${DB_PASSWORD} - thymeleaf: - cache: true - prefix: classpath:/templates/ - suffix: .html - enabled: true jackson: serialization: FAIL_ON_EMPTY_BEANS: false + ai: + openai: + api-key: ${AI_API_KEY} + chat: + options: + model: gpt-3.5-turbo + temperature: 0.0 + maxTokens: 300 + embedding: + options: + model: text-embedding-3-small + vectorstore: + chroma: + collection-name: ${CHROMA_COLLECTION_NAME} + client: + key-token: ${CHROMA_API_KEY} decorator: datasource: From b571e8165a1687ac5e6413bea1a500e4e4ac6cd1 Mon Sep 17 00:00:00 2001 From: jiwoo Date: Mon, 15 Jul 2024 21:13:33 +0900 Subject: [PATCH 03/31] =?UTF-8?q?feat(QueryVectorStoreAdapter):=20QueryVec?= =?UTF-8?q?torStoreAdapter=EB=A5=BC=20ChromaVectorStore=EB=A5=BC=20?= =?UTF-8?q?=EC=82=AC=EC=9A=A9=ED=95=98=EC=97=AC=20=EA=B5=AC=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../InMemoryQueryVectorStoreAdapter.java | 2 +- .../persistence/QueryVectorStoreAdapter.java | 19 ++++++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/kustacks/kuring/ai/adapter/out/persistence/InMemoryQueryVectorStoreAdapter.java b/src/main/java/com/kustacks/kuring/ai/adapter/out/persistence/InMemoryQueryVectorStoreAdapter.java index cc45cba8..e61b5337 100644 --- a/src/main/java/com/kustacks/kuring/ai/adapter/out/persistence/InMemoryQueryVectorStoreAdapter.java +++ b/src/main/java/com/kustacks/kuring/ai/adapter/out/persistence/InMemoryQueryVectorStoreAdapter.java @@ -12,7 +12,7 @@ import java.util.stream.Stream; @Slf4j -@Profile("local | dev | test") +@Profile("dev | test") @Component @RequiredArgsConstructor public class InMemoryQueryVectorStoreAdapter implements QueryVectorStorePort { diff --git a/src/main/java/com/kustacks/kuring/ai/adapter/out/persistence/QueryVectorStoreAdapter.java b/src/main/java/com/kustacks/kuring/ai/adapter/out/persistence/QueryVectorStoreAdapter.java index 99651e42..f3ea3931 100644 --- a/src/main/java/com/kustacks/kuring/ai/adapter/out/persistence/QueryVectorStoreAdapter.java +++ b/src/main/java/com/kustacks/kuring/ai/adapter/out/persistence/QueryVectorStoreAdapter.java @@ -2,19 +2,32 @@ import com.kustacks.kuring.ai.application.port.out.QueryVectorStorePort; import lombok.RequiredArgsConstructor; +import org.springframework.ai.document.Document; +import org.springframework.ai.vectorstore.ChromaVectorStore; +import org.springframework.ai.vectorstore.SearchRequest; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Component; -import java.util.Collections; import java.util.List; @Component -@Profile("prod") +@Profile("prod | local") @RequiredArgsConstructor public class QueryVectorStoreAdapter implements QueryVectorStorePort { + private static final double SIMILARITY_THRESHOLD = 0.80; + private static final int TOP_K = 1; + + private final ChromaVectorStore chromaVectorStore; + @Override public List findSimilarityContents(String question) { - return Collections.emptyList(); + return chromaVectorStore.similaritySearch( + SearchRequest.query(question) + .withTopK(TOP_K) + .withSimilarityThreshold(SIMILARITY_THRESHOLD) + ).stream() + .map(Document::getContent) + .toList(); } } From 4ed55779102f0d72609ef32c994ceb0a0855a120 Mon Sep 17 00:00:00 2001 From: jiwoo Date: Tue, 16 Jul 2024 14:09:58 +0900 Subject: [PATCH 04/31] =?UTF-8?q?feat(Notice):=20Notice=20=ED=85=8C?= =?UTF-8?q?=EC=9D=B4=EB=B8=94=EC=97=90=20embedded=20boolean=20=ED=95=84?= =?UTF-8?q?=EB=93=9C=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 해당 공지가 임베딩 되었는지 확인하는 컬럼을 추가한다 --- .../kustacks/kuring/notice/domain/Notice.java | 17 +++++++++--- .../V240716__Add_embedded_to_notice.sql | 2 ++ .../kuring/notice/domain/NoticeTest.java | 27 ++++++++++++++++--- 3 files changed, 39 insertions(+), 7 deletions(-) create mode 100644 src/main/resources/db/migration/V240716__Add_embedded_to_notice.sql diff --git a/src/main/java/com/kustacks/kuring/notice/domain/Notice.java b/src/main/java/com/kustacks/kuring/notice/domain/Notice.java index 72849608..99b7ec1e 100644 --- a/src/main/java/com/kustacks/kuring/notice/domain/Notice.java +++ b/src/main/java/com/kustacks/kuring/notice/domain/Notice.java @@ -5,6 +5,7 @@ import lombok.NoArgsConstructor; import jakarta.persistence.*; + import java.util.Objects; @Entity @@ -26,7 +27,10 @@ public class Notice { private String subject; @Column(name = "important") - private Boolean important = false; + private Boolean important = Boolean.FALSE; + + @Column(name = "embedded") + private Boolean embedded = Boolean.FALSE; @Embedded protected NoticeDateTime noticeDateTime; @@ -40,8 +44,7 @@ public class Notice { public Notice(String articleId, String postedDate, String updatedDate, String subject, CategoryName categoryName, Boolean important, - String fullUrl) - { + String fullUrl) { this.articleId = articleId; this.subject = subject; this.categoryName = categoryName; @@ -74,6 +77,14 @@ public String getUpdatedDate() { return this.noticeDateTime.updatedDateStr(); } + public void embeddedSuccess() { + this.embedded = Boolean.TRUE; + } + + public boolean isEmbedded() { + return this.embedded; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/src/main/resources/db/migration/V240716__Add_embedded_to_notice.sql b/src/main/resources/db/migration/V240716__Add_embedded_to_notice.sql new file mode 100644 index 00000000..48e99399 --- /dev/null +++ b/src/main/resources/db/migration/V240716__Add_embedded_to_notice.sql @@ -0,0 +1,2 @@ +ALTER TABLE notice + ADD column embedded boolean default false; diff --git a/src/test/java/com/kustacks/kuring/notice/domain/NoticeTest.java b/src/test/java/com/kustacks/kuring/notice/domain/NoticeTest.java index 3518a070..451c2d46 100644 --- a/src/test/java/com/kustacks/kuring/notice/domain/NoticeTest.java +++ b/src/test/java/com/kustacks/kuring/notice/domain/NoticeTest.java @@ -2,9 +2,11 @@ import com.kustacks.kuring.common.exception.InternalLogicException; import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; @@ -14,10 +16,12 @@ class NoticeTest { @DisplayName("공지 생성 테스트") @ParameterizedTest @ValueSource(strings = {"https://www.example.com", "http://example.com:8080/path/to/resource", - "https://library.konkuk.ac.kr/library-guide/bulletins/notice/7192", "http://www.konkuk.ac.kr/do/MessageBoard/ArticleRead.do?forum=notice&sort=6&id=5b50736&cat=0000300001", - "http://mae.konkuk.ac.kr/noticeView.do?siteId=MAE&boardSeq=988&menuSeq=6823&categorySeq=0&curBoardDispType=LIST&curPage=12&pageNum=1&seq=179896"}) + "https://library.konkuk.ac.kr/library-guide/bulletins/notice/7192", + "http://www.konkuk.ac.kr/do/MessageBoard/ArticleRead.do?forum=notice&sort=6&id=5b50736&cat=0000300001", + "http://mae.konkuk.ac.kr/noticeView.do?siteId=MAE&boardSeq=988&menuSeq=6823&categorySeq=0&curBoardDispType=LIST&curPage=12&pageNum=1&seq=179896"}) void create_member(String url) { - assertThatCode(() -> new Notice("artice_id", "2024-01-19 17:27:05", "2024-01-19 17:27:05", "subject", CategoryName.BACHELOR, false, url)) + assertThatCode(() -> new Notice("artice_id", "2024-01-19 17:27:05", + "2024-01-19 17:27:05", "subject", CategoryName.BACHELOR, false, url)) .doesNotThrowAnyException(); } @@ -25,8 +29,23 @@ void create_member(String url) { @ParameterizedTest @ValueSource(strings = {"//www.example.com", "https:/www.example.com", "https://"}) void member_invalid_email_id(String url) { - assertThatThrownBy(() -> new Notice("artice_id", "2024-01-19 17:27:05", "2024-01-19 17:27:05", "subject", CategoryName.BACHELOR, false, url)) + assertThatThrownBy(() -> new Notice("artice_id", "2024-01-19 17:27:05", + "2024-01-19 17:27:05", "subject", CategoryName.BACHELOR, false, url)) .isInstanceOf(InternalLogicException.class); } + @DisplayName("공지 임베딩 여부 확인 테스트") + @Test + public void notice_embedded() { + // given + Notice notice = new Notice("artice_id", "2024-01-19 17:27:05", + "2024-01-19 17:27:05", "subject", CategoryName.BACHELOR, + false, "https://www.example.com"); + + // when + notice.embeddedSuccess(); + + // then + assertThat(notice.isEmbedded()).isTrue(); + } } From d9977400befb1ef968585ec286592ca3e68eb3bf Mon Sep 17 00:00:00 2001 From: jiwoo Date: Tue, 16 Jul 2024 20:22:56 +0900 Subject: [PATCH 05/31] =?UTF-8?q?feat(NoticeTextParserTemplate):=20?= =?UTF-8?q?=EA=B3=B5=EC=A7=80=EC=9D=98=20=EB=B3=B8=EB=AC=B8,=20=EC=A0=9C?= =?UTF-8?q?=EB=AA=A9,=20=EC=95=84=EC=9D=B4=EB=94=94=EB=A5=BC=20=ED=8C=8C?= =?UTF-8?q?=EC=8B=B1=ED=95=98=EB=8A=94=20ParserTemplate=20=EA=B5=AC?= =?UTF-8?q?=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../notice/KuisHomepageNoticeTextParser.java | 27 ++ .../notice/NoticeTextParserTemplate.java | 29 ++ .../worker/parser/notice/PageTextDto.java | 13 + .../notice/NoticeTextParserTemplateTest.java | 42 ++ .../resources/notice/bbs-article-2024.html | 424 ++++++++++++++++++ 5 files changed, 535 insertions(+) create mode 100644 src/main/java/com/kustacks/kuring/worker/parser/notice/KuisHomepageNoticeTextParser.java create mode 100644 src/main/java/com/kustacks/kuring/worker/parser/notice/NoticeTextParserTemplate.java create mode 100644 src/main/java/com/kustacks/kuring/worker/parser/notice/PageTextDto.java create mode 100644 src/test/java/com/kustacks/kuring/worker/parser/notice/NoticeTextParserTemplateTest.java create mode 100644 src/test/resources/notice/bbs-article-2024.html diff --git a/src/main/java/com/kustacks/kuring/worker/parser/notice/KuisHomepageNoticeTextParser.java b/src/main/java/com/kustacks/kuring/worker/parser/notice/KuisHomepageNoticeTextParser.java new file mode 100644 index 00000000..5b5ed205 --- /dev/null +++ b/src/main/java/com/kustacks/kuring/worker/parser/notice/KuisHomepageNoticeTextParser.java @@ -0,0 +1,27 @@ +package com.kustacks.kuring.worker.parser.notice; + +import com.kustacks.kuring.notice.domain.CategoryName; +import org.jsoup.nodes.Document; + +public class KuisHomepageNoticeTextParser extends NoticeTextParserTemplate { + + @Override + protected boolean support(CategoryName categoryName) { + return !(categoryName == CategoryName.LIBRARY); + } + + @Override + protected String extractTitle(Document document) { + return document.selectFirst("h2.view-title").text().trim(); + } + + @Override + protected String extractArticleId(Document document) { + return document.selectFirst("dl.view-num dd").text().trim(); + } + + @Override + protected String extractTextBody(Document document) { + return document.selectFirst("div.board_content").text(); + } +} diff --git a/src/main/java/com/kustacks/kuring/worker/parser/notice/NoticeTextParserTemplate.java b/src/main/java/com/kustacks/kuring/worker/parser/notice/NoticeTextParserTemplate.java new file mode 100644 index 00000000..2327f67c --- /dev/null +++ b/src/main/java/com/kustacks/kuring/worker/parser/notice/NoticeTextParserTemplate.java @@ -0,0 +1,29 @@ +package com.kustacks.kuring.worker.parser.notice; + +import com.kustacks.kuring.common.exception.InternalLogicException; +import com.kustacks.kuring.common.exception.code.ErrorCode; +import com.kustacks.kuring.notice.domain.CategoryName; +import org.jsoup.nodes.Document; + +public abstract class NoticeTextParserTemplate { + + public PageTextDto parse(Document document) { + try { + String title = extractTitle(document); + String articleId = extractArticleId(document); + String textBody = extractTextBody(document); + + return new PageTextDto(title, articleId, textBody); + } catch (NullPointerException | IndexOutOfBoundsException e) { + throw new InternalLogicException(ErrorCode.NOTICE_SCRAPER_CANNOT_PARSE, e); + } + } + + protected abstract boolean support(CategoryName categoryName); + + protected abstract String extractTitle(Document document); + + protected abstract String extractArticleId(Document document); + + protected abstract String extractTextBody(Document document); +} diff --git a/src/main/java/com/kustacks/kuring/worker/parser/notice/PageTextDto.java b/src/main/java/com/kustacks/kuring/worker/parser/notice/PageTextDto.java new file mode 100644 index 00000000..ae874c16 --- /dev/null +++ b/src/main/java/com/kustacks/kuring/worker/parser/notice/PageTextDto.java @@ -0,0 +1,13 @@ +package com.kustacks.kuring.worker.parser.notice; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +@Getter +@RequiredArgsConstructor +public class PageTextDto { + + private final String title; + private final String articleId; + private final String text; +} diff --git a/src/test/java/com/kustacks/kuring/worker/parser/notice/NoticeTextParserTemplateTest.java b/src/test/java/com/kustacks/kuring/worker/parser/notice/NoticeTextParserTemplateTest.java new file mode 100644 index 00000000..65cefb23 --- /dev/null +++ b/src/test/java/com/kustacks/kuring/worker/parser/notice/NoticeTextParserTemplateTest.java @@ -0,0 +1,42 @@ +package com.kustacks.kuring.worker.parser.notice; + +import com.kustacks.kuring.support.TestFileLoader; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertAll; + +class NoticeTextParserTemplateTest { + + @DisplayName("공지의 텍스트를 파싱한다") + @Test + public void NoticeTextParser() throws IOException { + // given + Document doc = Jsoup.parse(TestFileLoader.loadHtmlFile("src/test/resources/notice/bbs-article-2024.html")); + + // when + PageTextDto results = new KuisHomepageNoticeTextParser().parse(doc); + + // then + assertAll( + () -> assertThat(results.getArticleId()).isEqualTo("1117110"), + () -> assertThat(results.getTitle()).isEqualTo("2024년도 대학생 청소년교육지원 장학사업 멘토모집 안내"), + () -> assertThat(results.getText()).contains( + "1. 멘토 신청기간: 2024. 3. 18. (월) ~ 2023. 3. 25. (월)", + "2. 모집유형: 나눔지기(멘토) 발굴형", + "3. 신청방법: 기관 협의 완료 후 한국장학재단 사이트에서 신청", + "4. 멘토 선발 및 매칭기간: 2023. 3. 26. (화) ~ 2023. 3. 29. (금)", + "5. 멘토 모집인원: 15명", + "6. 멘토 활동기간: 2024. 4. ~ 2025. 2", + "7. 멘토 선발기준", + "8. 멘토활동 세부사항", + "9. 기타" + ) + ); + } +} diff --git a/src/test/resources/notice/bbs-article-2024.html b/src/test/resources/notice/bbs-article-2024.html new file mode 100644 index 00000000..ef00ae7b --- /dev/null +++ b/src/test/resources/notice/bbs-article-2024.html @@ -0,0 +1,424 @@ + + + + + + + + + + + 건국대학교 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + +
+ + + +
+
+
글번호
+
1117110
+
+

+ + 2024년도 대학생 청소년교육지원 장학사업 멘토모집 안내 +

+ +
+
+ +
+
작성자
+
+ + + 장학복지팀 + + +
+
+
+
조회수
+
2485
+
+
+ +
등록일
+
2024.03.11
+
+
+
수정일
+
2024.03.20
+
+ +
+ +
+
+ +
+ +
+ + +
+
+
+
+
+ + +
+ + +
+

대학생 청소년교육지원 장학사업
멘토 모집 안내

+

 
+
+ 1. 멘토 신청기간: 2024. 3. 18. (월) ~ 2023. 3. 25. (월)

+ 2. 모집유형: 나눔지기(멘토) 발굴형
+ *멘토발굴형: 활동하고자 하는 기관을 멘토가 직접 발굴하는 유형으로, + 신청 전 멘토 활동 관련하여 기관문의 후 협의가 완료되어야 합니다. 기관과 협의된 학생에 한해 장학재단 사이트 신청 후 선발을 진행합니다.
+ (반드시 협의를 완료한 후 장학복지팀 연락)
+ * 활동가능기관: 전국 초·중·고등학교, 지역아동센터, 학교 밖 청소년지원센터, VMS·1365(정보인증포털)에 등록되어 있는 시설 및 + 청소년방과후아카데미 운영시설(한국청소년활동진흥원 인증)로 제한함, 어린이집·유치원·노인복지시설 활동 불가
+
+ 3. 신청방법: 기관 협의 완료 후 + 한국장학재단 사이트에서 신청
+ 가. 홈페이지: 한국장학재단 홈페이지 로그인 > 장학금 > 장학금신청 > 신청서 작성 > 대학생 청소년교육지원장학금
+ 나. 모바일: 한국장학재단 모바일 로그인 > 인재육성 > 대학생지식 멘토링 > 대학생 청소년교육지원장학금 > 신청하기
+
+ 4. 멘토 선발 및 매칭기간: 2023. 3. 26. (화) ~ 2023. 3. 29. (금)
+
+ 5. 멘토 모집인원: 15명
+
+ 6. 멘토 활동기간: 2024. 4. ~ 2025. 2
+  (상황에 따라 조정 가능, 최소 4개월 이상·월별 8시간 이상 활동이 가능한 멘토 신청 요망)
+
+ 7. 멘토 선발기준
+  가. 학부 재학생
+     ※참여불가: 휴학생‧졸업생
+ 나. 한국장학재단의 기본 요건을 충족한 학생(학자금 지원구간 무관)
+  - 직전학기 성적 70점이상 충족 필요.
+  - 1학년 1학기생(신입‧편입‧재입학의 첫 번째 학기) 성적 미적용
+  - 조기취업자 참여 불가(4대보험 가입내역이 확인되는 경우)
+ 다. 지원인원이 모집인원을 초과한 경우
+  1) 교·사대생 등 대학생 튜터링 사업 참여 경험이 있는 학생 
+  2) 교육과 관련된 학과 학생
+  3) 멘토링 경험이 있는 학생
+ - 멘토 신청기간에 증빙자료 장학복지팀(scholarship@konkuk.ac.kr) 이메일로 제출 +

+

  재단을 통한 신청 시에도 자기소개서와 함께 + 관련 증빙자료를 첨부할 것을 권장드립니다.
+
+ 8. 멘토활동 세부사항
+  가. 장학금: 시급은 12,220원이며, 장학금은 익월 셋째주 목요일 오후 6시 이후 지급
 나. 활동 가능시간: 월 + 최대 40시간 (1일최대 8시간, 주 최대 20시간까지 가능, 방학기간 동일하게 운영)
+ 다. 멘토링 활동 외의 시간은 멘토링 활동으로 인정되지 않음.
+ 라. 누적 활동시간이 10시간 이상인 경우에 지급 가능하며, 첫달 누적활동시간이 10시간 미만일 경우,
+ 누적 활동시간이 10시간 이상을 충족한 월의 익월에 장학금 지급
+  마. 멘토링 활동시간 봉사점수·학점시간(교사대 등)으로 인정 불가
+ 바. 멘토링 활동 재택근로로 인정 불가
+
+ 9. 기타
+  가. 반드시 사전에 기관과 협의를 완료한 이후에 장학복지팀으로 연락 요망
+ 나. 타 근로장학과 중복참여 불가, 복수의 기관에서 활동 불가
+  다. 선발된 인원에 대해 4월 1일 이전 개별 연락 예정
+ 라. 출근부 작성, 사전교육 등 활동 관련 매뉴얼은 선발자 관련 안내시 추후 안내 예정.

+


+

10. EBS 화상 멘토링 추진계획 (향후 공지알림 예정)

+ +

  

+

 가. 활동기간 : 2024. 07. ~ 2024. 12. (6개월 예정)

+

 나. 활동내용 : 대학생멘토가 중3학생들의 EBS 영어,수학과목을 온라인 1:1 코칭

+

 다. 선발대상 : 교·사대생, 교직이수중인 학생, 관련 전공 학생 선발 예정

+

 라. 모집방법 : B유형인 “나눔지기(튜터) 발굴형”으로 학교공지를 통해 홍보 예정

+

 마. 지급계획 : 활동 시간 당 장학금(12,220원) 지급예정, EBS에서 교재연구비 등 추가장학금 지급 검토 중.

+


+
+ 11. 문의사항이 있으신 경우, 장학복지팀(02-450-3512)로 연락바랍니다.

+


+ + +
+ +
+ + +
+ + +
+
+ + + + + + + + + + + + +
+ + +
+
+ + +
+
+
+ + +
+ + + + + + + + + From 937e2333e0522533d931f58dcb48f47988a32c4f Mon Sep 17 00:00:00 2001 From: jiwoo Date: Wed, 17 Jul 2024 15:19:00 +0900 Subject: [PATCH 06/31] =?UTF-8?q?test:=20ChromaDB=20test=20container=20?= =?UTF-8?q?=EC=84=A4=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build.gradle | 7 ++++--- .../kuring/config/RAGConfiguration.java | 18 +++++++++++++++++- .../kuring/support/IntegrationTestSupport.java | 12 ++++++++++++ src/test/resources/application-test.yml | 5 +++++ 4 files changed, 38 insertions(+), 4 deletions(-) diff --git a/build.gradle b/build.gradle index 8054faa2..67d38beb 100644 --- a/build.gradle +++ b/build.gradle @@ -103,9 +103,10 @@ dependencies { // Test Container testImplementation 'org.junit.jupiter:junit-jupiter:5.8.1' - testImplementation 'org.testcontainers:testcontainers:1.19.3' - testImplementation 'org.testcontainers:junit-jupiter:1.19.3' - testImplementation 'org.testcontainers:mariadb:1.19.3' + testImplementation 'org.testcontainers:testcontainers:1.19.8' + testImplementation 'org.testcontainers:junit-jupiter:1.19.8' + testImplementation 'org.testcontainers:mariadb:1.19.8' + testImplementation 'org.testcontainers:chromadb:1.19.8' } dependencyManagement { diff --git a/src/main/java/com/kustacks/kuring/config/RAGConfiguration.java b/src/main/java/com/kustacks/kuring/config/RAGConfiguration.java index 084f03b1..1c3eca35 100644 --- a/src/main/java/com/kustacks/kuring/config/RAGConfiguration.java +++ b/src/main/java/com/kustacks/kuring/config/RAGConfiguration.java @@ -1,16 +1,19 @@ package com.kustacks.kuring.config; import lombok.extern.slf4j.Slf4j; +import org.springframework.ai.chroma.ChromaApi; import org.springframework.ai.document.Document; import org.springframework.ai.embedding.EmbeddingModel; import org.springframework.ai.reader.TextReader; import org.springframework.ai.transformer.splitter.TokenTextSplitter; +import org.springframework.ai.vectorstore.ChromaVectorStore; import org.springframework.ai.vectorstore.SimpleVectorStore; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Profile; import org.springframework.core.io.Resource; +import org.springframework.web.client.RestTemplate; import java.io.File; import java.nio.file.Path; @@ -18,7 +21,6 @@ import java.util.List; @Slf4j -@Profile("local") @Configuration public class RAGConfiguration { @@ -28,6 +30,20 @@ public class RAGConfiguration { @Value("vectorstore.json") private String vectorStoreName; + @Profile("test") + @Bean + public ChromaApi chromaApi(RestTemplate restTemplate) { + String chromaUrl = "http://127.0.0.1:8000"; + return new ChromaApi(chromaUrl, restTemplate); + } + + @Profile("test") + @Bean + public ChromaVectorStore chromaVectorStore(EmbeddingModel embeddingModel, ChromaApi chromaApi) { + return new ChromaVectorStore(embeddingModel, chromaApi, false); + } + + @Profile("local") @Bean public SimpleVectorStore simpleVectorStore(EmbeddingModel embeddingModel) { SimpleVectorStore simpleVectorStore = new SimpleVectorStore(embeddingModel); diff --git a/src/test/java/com/kustacks/kuring/support/IntegrationTestSupport.java b/src/test/java/com/kustacks/kuring/support/IntegrationTestSupport.java index 41c1d87b..6a27bcb4 100644 --- a/src/test/java/com/kustacks/kuring/support/IntegrationTestSupport.java +++ b/src/test/java/com/kustacks/kuring/support/IntegrationTestSupport.java @@ -8,7 +8,11 @@ import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.boot.test.web.server.LocalServerPort; +import org.testcontainers.chromadb.ChromaDBContainer; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.DockerImageName; +@Testcontainers @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) public class IntegrationTestSupport { @@ -18,6 +22,14 @@ public class IntegrationTestSupport { public static final String ADMIN_CLIENT_LOGIN_ID = "client@email.com"; public static final String ADMIN_CLIENT_PASSWORD = "client_password"; + protected static final ChromaDBContainer chroma; + + static { + chroma = new ChromaDBContainer(DockerImageName.parse("chromadb/chroma:latest")) + .withExposedPorts(8000); + chroma.start(); + } + @MockBean protected FirebaseSubscribeService firebaseSubscribeService; diff --git a/src/test/resources/application-test.yml b/src/test/resources/application-test.yml index ca921743..8e33c205 100644 --- a/src/test/resources/application-test.yml +++ b/src/test/resources/application-test.yml @@ -40,6 +40,11 @@ spring: embedding: options: model: text-embedding-3-small + vectorstore: + chroma: + collection-name: "test-collection" + client: + port: 8000 testcontainers: reuse: From 64d54af8ceed67d517e819edc7ca64e1c71ed0fa Mon Sep 17 00:00:00 2001 From: jiwoo Date: Wed, 17 Jul 2024 18:44:21 +0900 Subject: [PATCH 07/31] =?UTF-8?q?feat(NoticeApiClient):=20=EB=8B=A8?= =?UTF-8?q?=EC=9D=BC=20=ED=8E=98=EC=9D=B4=EC=A7=80=EB=A5=BC=20scrap?= =?UTF-8?q?=ED=95=98=EB=8A=94=20requestSinglePageWithUrl=20=EA=B5=AC?= =?UTF-8?q?=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../client/notice/KuisHomepageNoticeApiClient.java | 14 ++++++++++++++ .../scrap/client/notice/KuisNoticeApiClient.java | 5 +++++ .../client/notice/LatestPageNoticeApiClient.java | 5 +++++ .../client/notice/LibraryNoticeApiClient.java | 5 +++++ .../scrap/client/notice/NoticeApiClient.java | 2 ++ .../client/notice/RealEstateNoticeApiClient.java | 5 +++++ 6 files changed, 36 insertions(+) diff --git a/src/main/java/com/kustacks/kuring/worker/scrap/client/notice/KuisHomepageNoticeApiClient.java b/src/main/java/com/kustacks/kuring/worker/scrap/client/notice/KuisHomepageNoticeApiClient.java index cd956da2..b6738a95 100644 --- a/src/main/java/com/kustacks/kuring/worker/scrap/client/notice/KuisHomepageNoticeApiClient.java +++ b/src/main/java/com/kustacks/kuring/worker/scrap/client/notice/KuisHomepageNoticeApiClient.java @@ -59,6 +59,20 @@ public List requestAll(KuisHomepageNoticeInfo kuisHomepageNot return Collections.emptyList(); } + @Override + public ScrapingResultDto requestSinglePageWithUrl(KuisHomepageNoticeInfo noticeInfo, String url) { + try { + Document document = jsoupClient.get(url, LATEST_SCRAP_ALL_TIMEOUT); + return new ScrapingResultDto(document, url); + } catch (IOException e) { + log.info("Notice Text Scrap IOException", e); + } catch (NullPointerException | IndexOutOfBoundsException e) { + throw new InternalLogicException(ErrorCode.NOTICE_SCRAPER_CANNOT_PARSE, e); + } + + throw new InternalLogicException(ErrorCode.NOTICE_SCRAPER_CANNOT_PARSE); + } + public int getTotalNoticeSize(String url) throws IOException, IndexOutOfBoundsException, NullPointerException { Document document = jsoupClient.get(url, LATEST_SCRAP_TIMEOUT); diff --git a/src/main/java/com/kustacks/kuring/worker/scrap/client/notice/KuisNoticeApiClient.java b/src/main/java/com/kustacks/kuring/worker/scrap/client/notice/KuisNoticeApiClient.java index 375a6a6a..aa3d77ff 100644 --- a/src/main/java/com/kustacks/kuring/worker/scrap/client/notice/KuisNoticeApiClient.java +++ b/src/main/java/com/kustacks/kuring/worker/scrap/client/notice/KuisNoticeApiClient.java @@ -58,6 +58,11 @@ public List requestAll(KuisNoticeInfo kuisNoticeRequestBo return Collections.emptyList(); } + @Override + public CommonNoticeFormatDto requestSinglePageWithUrl(KuisNoticeInfo noticeInfo, String url) { + throw new InternalLogicException(ErrorCode.NOTICE_SCRAPER_CANNOT_PARSE); + } + private HttpEntity kuisNoticeRequests(KuisNoticeInfo kuisNoticeRequestBody, HttpHeaders noticeRequestHeader) { String encodedNoticeRequestBody = KuisInfo.toUrlEncodedString(kuisNoticeRequestBody); return new HttpEntity<>(encodedNoticeRequestBody, noticeRequestHeader); diff --git a/src/main/java/com/kustacks/kuring/worker/scrap/client/notice/LatestPageNoticeApiClient.java b/src/main/java/com/kustacks/kuring/worker/scrap/client/notice/LatestPageNoticeApiClient.java index b99d60a3..c8802cb1 100644 --- a/src/main/java/com/kustacks/kuring/worker/scrap/client/notice/LatestPageNoticeApiClient.java +++ b/src/main/java/com/kustacks/kuring/worker/scrap/client/notice/LatestPageNoticeApiClient.java @@ -59,6 +59,11 @@ public List requestAll(DeptInfo deptInfo) throws InternalLogi return Collections.emptyList(); } + @Override + public ScrapingResultDto requestSinglePageWithUrl(DeptInfo noticeInfo, String url) { + throw new InternalLogicException(ErrorCode.NOTICE_SCRAPER_CANNOT_PARSE); + } + public int getTotalNoticeSize(String url) throws IOException, IndexOutOfBoundsException, NullPointerException { Document document = jsoupClient.get(url, LATEST_SCRAP_TIMEOUT); diff --git a/src/main/java/com/kustacks/kuring/worker/scrap/client/notice/LibraryNoticeApiClient.java b/src/main/java/com/kustacks/kuring/worker/scrap/client/notice/LibraryNoticeApiClient.java index 7774d4d1..48d050fb 100644 --- a/src/main/java/com/kustacks/kuring/worker/scrap/client/notice/LibraryNoticeApiClient.java +++ b/src/main/java/com/kustacks/kuring/worker/scrap/client/notice/LibraryNoticeApiClient.java @@ -45,6 +45,11 @@ public List requestAll(CategoryName categoryName) throws return Collections.emptyList(); } + @Override + public CommonNoticeFormatDto requestSinglePageWithUrl(CategoryName noticeInfo, String url) { + throw new InternalLogicException(ErrorCode.NOTICE_SCRAPER_CANNOT_PARSE); + } + private List scrapLibraryNoticeDtos() { int offset = 0; int max = 20; diff --git a/src/main/java/com/kustacks/kuring/worker/scrap/client/notice/NoticeApiClient.java b/src/main/java/com/kustacks/kuring/worker/scrap/client/notice/NoticeApiClient.java index 3a704fa0..22b59877 100644 --- a/src/main/java/com/kustacks/kuring/worker/scrap/client/notice/NoticeApiClient.java +++ b/src/main/java/com/kustacks/kuring/worker/scrap/client/notice/NoticeApiClient.java @@ -9,4 +9,6 @@ public interface NoticeApiClient { List request(P p) throws InternalLogicException; List requestAll(P p) throws InternalLogicException; + + T requestSinglePageWithUrl(P noticeInfo, String url); } diff --git a/src/main/java/com/kustacks/kuring/worker/scrap/client/notice/RealEstateNoticeApiClient.java b/src/main/java/com/kustacks/kuring/worker/scrap/client/notice/RealEstateNoticeApiClient.java index 818d902d..8605a7c7 100644 --- a/src/main/java/com/kustacks/kuring/worker/scrap/client/notice/RealEstateNoticeApiClient.java +++ b/src/main/java/com/kustacks/kuring/worker/scrap/client/notice/RealEstateNoticeApiClient.java @@ -79,6 +79,11 @@ public List requestAll(DeptInfo deptInfo) throws InternalLogi return reqResults; } + @Override + public ScrapingResultDto requestSinglePageWithUrl(DeptInfo noticeInfo, String url) { + throw new InternalLogicException(ErrorCode.NOTICE_SCRAPER_CANNOT_PARSE); + } + private int getTotalPageNum(Document document) { Element lastPageNumElement = document.select(".paging > ul > li").last(); Element lastPageBtnElement = lastPageNumElement.getElementsByTag("a").get(1); From d285e3444f06613d315250427fe977c23450b052 Mon Sep 17 00:00:00 2001 From: jiwoo Date: Thu, 18 Jul 2024 15:41:14 +0900 Subject: [PATCH 08/31] =?UTF-8?q?fix(NoticeJdbcRepository):=20=EA=B3=B5?= =?UTF-8?q?=EC=A7=80=EC=97=90=20=EC=B6=94=EA=B0=80=EB=90=9C=20embedded=20?= =?UTF-8?q?=ED=95=84=EB=93=9C=EB=A5=BC=20=EC=9C=84=ED=95=B4=20bulk=20inser?= =?UTF-8?q?t=20method=20=EC=9D=BC=EB=B6=80=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../out/persistence/NoticeJdbcRepository.java | 24 ++++++++-------- .../kustacks/kuring/notice/domain/Notice.java | 1 + .../out/persistence/NoticeRepositoryTest.java | 28 +++++++++++++++++++ 3 files changed, 42 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/kustacks/kuring/notice/adapter/out/persistence/NoticeJdbcRepository.java b/src/main/java/com/kustacks/kuring/notice/adapter/out/persistence/NoticeJdbcRepository.java index 229c5e44..c4de81d3 100644 --- a/src/main/java/com/kustacks/kuring/notice/adapter/out/persistence/NoticeJdbcRepository.java +++ b/src/main/java/com/kustacks/kuring/notice/adapter/out/persistence/NoticeJdbcRepository.java @@ -21,7 +21,7 @@ class NoticeJdbcRepository { @Transactional public void saveAllCategoryNotices(List notices) { - jdbcTemplate.batchUpdate("INSERT INTO notice (article_id, category_name, important, posted_dt, subject, updated_dt, url, dtype) values (?, ?, ?, ?, ?, ?, ?, 'Notice')", + jdbcTemplate.batchUpdate("INSERT INTO notice (article_id, category_name, important, embedded, posted_dt, subject, updated_dt, url, dtype) values (?, ?, ?, ?, ?, ?, ?, ?, 'Notice')", new BatchPreparedStatementSetter() { @Override public void setValues(PreparedStatement ps, int i) throws SQLException { @@ -29,10 +29,11 @@ public void setValues(PreparedStatement ps, int i) throws SQLException { ps.setString(1, notice.getArticleId()); ps.setString(2, notice.getCategoryName().toUpperCase()); ps.setInt(3, notice.isImportant() ? 1 : 0); - ps.setString(4, notice.getPostedDate()); - ps.setString(5, notice.getSubject()); - ps.setString(6, notice.getUpdatedDate()); - ps.setString(7, notice.getUrl()); + ps.setInt(4, notice.isEmbedded() ? 1 : 0); + ps.setString(5, notice.getPostedDate()); + ps.setString(6, notice.getSubject()); + ps.setString(7, notice.getUpdatedDate()); + ps.setString(8, notice.getUrl()); } @Override @@ -44,7 +45,7 @@ public int getBatchSize() { @Transactional public void saveAllDepartmentNotices(List departmentNotices) { - jdbcTemplate.batchUpdate("INSERT INTO notice (article_id, category_name, important, posted_dt, subject, updated_dt, url, department_name, dtype) values (?, ?, ?, ?, ?, ?, ?, ?, 'DepartmentNotice')", + jdbcTemplate.batchUpdate("INSERT INTO notice (article_id, category_name, important, embedded, posted_dt, subject, updated_dt, url, department_name, dtype) values (?, ?, ?, ?, ?, ?, ?, ?, ?, 'DepartmentNotice')", new BatchPreparedStatementSetter() { @Override public void setValues(PreparedStatement ps, int i) throws SQLException { @@ -52,11 +53,12 @@ public void setValues(PreparedStatement ps, int i) throws SQLException { ps.setString(1, departmentNotice.getArticleId()); ps.setString(2, departmentNotice.getCategoryName().toUpperCase()); ps.setInt(3, departmentNotice.isImportant() ? 1 : 0); - ps.setString(4, departmentNotice.getPostedDate()); - ps.setString(5, departmentNotice.getSubject()); - ps.setString(6, departmentNotice.getUpdatedDate()); - ps.setString(7, departmentNotice.getUrl()); - ps.setString(8, DepartmentName.fromName(departmentNotice.getDepartmentName()).name()); + ps.setInt(4, departmentNotice.isEmbedded() ? 1 : 0); + ps.setString(5, departmentNotice.getPostedDate()); + ps.setString(6, departmentNotice.getSubject()); + ps.setString(7, departmentNotice.getUpdatedDate()); + ps.setString(8, departmentNotice.getUrl()); + ps.setString(9, DepartmentName.fromName(departmentNotice.getDepartmentName()).name()); } @Override diff --git a/src/main/java/com/kustacks/kuring/notice/domain/Notice.java b/src/main/java/com/kustacks/kuring/notice/domain/Notice.java index 99b7ec1e..00f1a927 100644 --- a/src/main/java/com/kustacks/kuring/notice/domain/Notice.java +++ b/src/main/java/com/kustacks/kuring/notice/domain/Notice.java @@ -51,6 +51,7 @@ public Notice(String articleId, String postedDate, String updatedDate, this.important = important; this.noticeDateTime = new NoticeDateTime(postedDate, updatedDate); this.url = new Url(fullUrl); + this.embedded = Boolean.FALSE; } public boolean isImportant() { diff --git a/src/test/java/com/kustacks/kuring/notice/adapter/out/persistence/NoticeRepositoryTest.java b/src/test/java/com/kustacks/kuring/notice/adapter/out/persistence/NoticeRepositoryTest.java index f2ab1f48..ff4950f6 100644 --- a/src/test/java/com/kustacks/kuring/notice/adapter/out/persistence/NoticeRepositoryTest.java +++ b/src/test/java/com/kustacks/kuring/notice/adapter/out/persistence/NoticeRepositoryTest.java @@ -14,6 +14,7 @@ import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; +import java.util.ArrayList; import java.util.List; import static org.assertj.core.api.Assertions.assertThat; @@ -25,12 +26,30 @@ class NoticeRepositoryTest extends IntegrationTestSupport { @Autowired private NoticeQueryPort noticeQueryPort; + @Autowired + private NoticeRepository noticeRepository; + @Autowired private NoticeCommandPort noticeCommandPort; @Autowired private UserPersistenceAdapter userPersistenceAdapter; + @DisplayName("jdbc를 사용한 bulk insert 테스트") + @Test + public void jdbcBulkInsert() { + // given + noticeRepository.deleteAll(); + List notices = creatNotices(70); + + // when + noticeCommandPort.saveAllCategoryNotices(notices); + + // then + List findNotices = noticeRepository.findAll(); + assertThat(findNotices).hasSize(70); + } + @DisplayName("사용자가 북마크해둔 공지의 ID로 해당 공지들을 찾아올 수 있다") @Test void lookupAllNoticeByIds() { @@ -90,4 +109,13 @@ void notice_change_important() { List normalArticleIds = noticeQueryPort.findNormalArticleIdsByCategoryName(CategoryName.BACHELOR); assertThat(normalArticleIds).containsExactly("1", "2"); } + + private List creatNotices(int count) { + List notices = new ArrayList<>(); + for(int i = 0; i < count; i++) { + notices.add(new Notice(String.valueOf(i), "2024-01-19 17:27:05", "2023-04-03 17:27:05", + "notice" + i, CategoryName.BACHELOR, false, "https://www.example.com")); + } + return notices; + } } From aa7c80b24cf56b4aab793390d85659f242c4983b Mon Sep 17 00:00:00 2001 From: jiwoo Date: Thu, 18 Jul 2024 16:00:18 +0900 Subject: [PATCH 09/31] =?UTF-8?q?feat(NoticeRepository):=20updateNoticeEmb?= =?UTF-8?q?eddingStatus,=20findNotYetEmbeddingNotice=20=EB=A9=94=EC=84=9C?= =?UTF-8?q?=EB=93=9C=20=EA=B5=AC=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../persistence/NoticePersistenceAdapter.java | 11 +++ .../persistence/NoticeQueryRepository.java | 5 + .../NoticeQueryRepositoryImpl.java | 40 +++++++- .../port/out/NoticeCommandPort.java | 1 + .../application/port/out/NoticeQueryPort.java | 3 + .../scrap/client/notice/NoticeApiClient.java | 2 +- .../out/persistence/NoticeRepositoryTest.java | 98 ++++++++++++++++++- 7 files changed, 157 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/kustacks/kuring/notice/adapter/out/persistence/NoticePersistenceAdapter.java b/src/main/java/com/kustacks/kuring/notice/adapter/out/persistence/NoticePersistenceAdapter.java index 56c42c3d..8d244aa8 100644 --- a/src/main/java/com/kustacks/kuring/notice/adapter/out/persistence/NoticePersistenceAdapter.java +++ b/src/main/java/com/kustacks/kuring/notice/adapter/out/persistence/NoticePersistenceAdapter.java @@ -13,6 +13,7 @@ import lombok.RequiredArgsConstructor; import org.springframework.data.domain.Pageable; +import java.time.LocalDateTime; import java.util.List; @PersistenceAdapter @@ -47,6 +48,11 @@ public void changeNoticeImportantToFalseByArticleId(CategoryName categoryName, L this.noticeRepository.changeNoticeImportantByArticleId(categoryName, articleIds, false); } + @Override + public void updateNoticeEmbeddingStatus(CategoryName categoryName, List articleIds) { + this.noticeRepository.updateNoticeEmbeddingStatus(articleIds, categoryName); + } + @Override public List findNoticesByCategoryWithOffset(CategoryName categoryName, Pageable pageable) { return this.noticeRepository.findNoticesByCategoryWithOffset(categoryName, pageable); @@ -57,6 +63,11 @@ public List findAllByKeywords(List containedNames) { return this.noticeRepository.findAllByKeywords(containedNames); } + @Override + public List findNotYetEmbeddingNotice(CategoryName categoryName, LocalDateTime date) { + return this.noticeRepository.findNotYetEmbeddingNoticeByDate(categoryName, date); + } + @Override public List findNormalArticleIdsByCategory(CategoryName categoryName) { return this.noticeRepository.findNormalArticleIdsByCategory(categoryName); diff --git a/src/main/java/com/kustacks/kuring/notice/adapter/out/persistence/NoticeQueryRepository.java b/src/main/java/com/kustacks/kuring/notice/adapter/out/persistence/NoticeQueryRepository.java index 4a8aecbf..cd409fae 100644 --- a/src/main/java/com/kustacks/kuring/notice/adapter/out/persistence/NoticeQueryRepository.java +++ b/src/main/java/com/kustacks/kuring/notice/adapter/out/persistence/NoticeQueryRepository.java @@ -7,6 +7,7 @@ import com.kustacks.kuring.user.application.port.out.dto.BookmarkDto; import org.springframework.data.domain.Pageable; +import java.time.LocalDateTime; import java.util.List; interface NoticeQueryRepository { @@ -36,4 +37,8 @@ interface NoticeQueryRepository { List findAllByBookmarkIds(List ids); void changeNoticeImportantByArticleId(CategoryName categoryName, List articleIds, boolean important); + + void updateNoticeEmbeddingStatus(List articleIds, CategoryName categoryName); + + List findNotYetEmbeddingNoticeByDate(CategoryName categoryName, LocalDateTime date); } diff --git a/src/main/java/com/kustacks/kuring/notice/adapter/out/persistence/NoticeQueryRepositoryImpl.java b/src/main/java/com/kustacks/kuring/notice/adapter/out/persistence/NoticeQueryRepositoryImpl.java index 43cde2e8..7b9fe1b3 100644 --- a/src/main/java/com/kustacks/kuring/notice/adapter/out/persistence/NoticeQueryRepositoryImpl.java +++ b/src/main/java/com/kustacks/kuring/notice/adapter/out/persistence/NoticeQueryRepositoryImpl.java @@ -18,6 +18,7 @@ import org.springframework.data.domain.Pageable; import org.springframework.transaction.annotation.Transactional; +import java.time.LocalDateTime; import java.util.List; import static com.kustacks.kuring.notice.domain.QDepartmentNotice.departmentNotice; @@ -79,6 +80,31 @@ public List findAllByKeywords(List keywords) { .fetch(); } + @Transactional(readOnly = true) + @Override + public List findNotYetEmbeddingNoticeByDate(CategoryName categoryName, LocalDateTime date) { + StringTemplate postedDate = Expressions.stringTemplate( + DATE_FORMAT_TEMPLATE, + notice.noticeDateTime.postedDate, + ConstantImpl.create(DATE_TIME_TEMPLATE) + ); + + return queryFactory.select( + new QNoticeDto( + notice.articleId, + postedDate, + notice.url.value, + notice.subject, + notice.categoryName.stringValue().toLowerCase(), + notice.important + ) + ).from(notice) + .where(notice.categoryName.eq(categoryName) + .and(notice.embedded.isFalse()) + .and(notice.noticeDateTime.postedDate.after(date))) + .fetch(); + } + @Transactional(readOnly = true) @Override public List findNormalArticleIdsByCategory(CategoryName categoryName) { @@ -247,7 +273,19 @@ public void changeNoticeImportantByArticleId(CategoryName categoryName, List articleIds, CategoryName categoryName) { + queryFactory + .update(notice) + .set(notice.embedded, true) + .where( + notice.categoryName.eq(categoryName) + .and(notice.articleId.in(articleIds)) ).execute(); } diff --git a/src/main/java/com/kustacks/kuring/notice/application/port/out/NoticeCommandPort.java b/src/main/java/com/kustacks/kuring/notice/application/port/out/NoticeCommandPort.java index ddccf265..69fa6207 100644 --- a/src/main/java/com/kustacks/kuring/notice/application/port/out/NoticeCommandPort.java +++ b/src/main/java/com/kustacks/kuring/notice/application/port/out/NoticeCommandPort.java @@ -14,4 +14,5 @@ public interface NoticeCommandPort { void deleteAllByIdsAndCategory(CategoryName categoryName, List articleIds); void deleteAllByIdsAndDepartment(DepartmentName departmentName, List articleIds); void changeNoticeImportantToFalseByArticleId(CategoryName categoryName, List articleIds); + void updateNoticeEmbeddingStatus(CategoryName categoryName, List articleIds); } diff --git a/src/main/java/com/kustacks/kuring/notice/application/port/out/NoticeQueryPort.java b/src/main/java/com/kustacks/kuring/notice/application/port/out/NoticeQueryPort.java index 74ff1740..a7a60535 100644 --- a/src/main/java/com/kustacks/kuring/notice/application/port/out/NoticeQueryPort.java +++ b/src/main/java/com/kustacks/kuring/notice/application/port/out/NoticeQueryPort.java @@ -7,6 +7,7 @@ import com.kustacks.kuring.user.application.port.out.dto.BookmarkDto; import org.springframework.data.domain.Pageable; +import java.time.LocalDateTime; import java.util.List; public interface NoticeQueryPort { @@ -32,4 +33,6 @@ public interface NoticeQueryPort { List findAllByBookmarkIds(List ids); Long count(); + + List findNotYetEmbeddingNotice(CategoryName categoryName, LocalDateTime now); } diff --git a/src/main/java/com/kustacks/kuring/worker/scrap/client/notice/NoticeApiClient.java b/src/main/java/com/kustacks/kuring/worker/scrap/client/notice/NoticeApiClient.java index 22b59877..414a3095 100644 --- a/src/main/java/com/kustacks/kuring/worker/scrap/client/notice/NoticeApiClient.java +++ b/src/main/java/com/kustacks/kuring/worker/scrap/client/notice/NoticeApiClient.java @@ -10,5 +10,5 @@ public interface NoticeApiClient { List requestAll(P p) throws InternalLogicException; - T requestSinglePageWithUrl(P noticeInfo, String url); + T requestSinglePageWithUrl(P p, String url); } diff --git a/src/test/java/com/kustacks/kuring/notice/adapter/out/persistence/NoticeRepositoryTest.java b/src/test/java/com/kustacks/kuring/notice/adapter/out/persistence/NoticeRepositoryTest.java index ff4950f6..27145626 100644 --- a/src/test/java/com/kustacks/kuring/notice/adapter/out/persistence/NoticeRepositoryTest.java +++ b/src/test/java/com/kustacks/kuring/notice/adapter/out/persistence/NoticeRepositoryTest.java @@ -2,6 +2,7 @@ import com.kustacks.kuring.notice.application.port.out.NoticeCommandPort; import com.kustacks.kuring.notice.application.port.out.NoticeQueryPort; +import com.kustacks.kuring.notice.application.port.out.dto.NoticeDto; import com.kustacks.kuring.notice.domain.CategoryName; import com.kustacks.kuring.notice.domain.DepartmentName; import com.kustacks.kuring.notice.domain.DepartmentNotice; @@ -14,11 +15,13 @@ import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; +import java.time.LocalDateTime; import java.util.ArrayList; import java.util.List; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.groups.Tuple.tuple; +import static org.junit.jupiter.api.Assertions.assertAll; @DisplayName("리포지토리 : Notice") class NoticeRepositoryTest extends IntegrationTestSupport { @@ -110,9 +113,102 @@ void notice_change_important() { assertThat(normalArticleIds).containsExactly("1", "2"); } + @DisplayName("Embedding 된 공지의 상태를 성공적으로 영속화 한다") + @Test + void embeddingNotice() { + // given + noticeRepository.deleteAll(); + Notice notice1 = new Notice("1", "2024-03-19 17:27:05", "2023-04-03 17:27:05", + "notice1", CategoryName.BACHELOR, false, "https://www.example.com"); + Notice notice2 = new Notice("2", "2024-01-19 17:27:06", "2023-04-03 17:27:05", + "notice2", CategoryName.BACHELOR, false, "https://www.example.com"); + Notice notice3 = new Notice("3", "2024-01-19 17:27:05", "2023-04-03 17:27:05", + "notice3", CategoryName.BACHELOR, false, "https://www.example.com"); + Notice notice4 = new Notice("4", "2024-01-18 17:27:05", "2023-04-03 17:27:05", + "notice4", CategoryName.BACHELOR, false, "https://www.example.com"); + + notice1.embeddedSuccess(); + notice3.embeddedSuccess(); + + noticeCommandPort.saveAllCategoryNotices(List.of(notice1, notice2, notice3, notice4)); + + // when + List notices = noticeRepository.findAll(); + + // then + assertAll( + () -> assertThat(notices).hasSize(4), + () -> assertThat(notices).extracting("articleId") + .containsExactly("1", "2", "3", "4"), + () -> assertThat(notices).extracting("embedded") + .containsExactly(true, false, true, false) + ); + } + + @DisplayName("지정된 기간 동안 Embedding 되지 않은 공지를 찾아올 수 있다") + @Test + void findNotYetEmbeddingNotice() { + // given + Notice notice1 = new Notice("1", "2024-03-19 17:27:07", "2023-04-03 17:27:05", + "notice1", CategoryName.BACHELOR, false, "https://www.example.com"); + Notice notice2 = new Notice("2", "2024-01-20 00:00:00", "2023-04-03 17:27:05", + "notice2", CategoryName.BACHELOR, false, "https://www.example.com"); + Notice notice3 = new Notice("3", "2024-01-19 17:27:05", "2023-04-03 17:27:05", + "notice3", CategoryName.BACHELOR, false, "https://www.example.com"); + Notice notice4 = new Notice("4", "2024-01-18 17:27:05", "2023-04-03 17:27:05", + "notice4", CategoryName.BACHELOR, false, "https://www.example.com"); + + noticeCommandPort.saveAllCategoryNotices(List.of(notice1, notice2, notice3, notice4)); + + // when + List results = noticeQueryPort.findNotYetEmbeddingNotice( + CategoryName.BACHELOR, + LocalDateTime.of(2024, 3, 19, 17, 27, 5).minusMonths(2) + ); + + // then + assertAll( + () -> assertThat(results).hasSize(2), + () -> assertThat(results).extracting("articleId") + .contains("1", "2") + ); + } + + @DisplayName("Embedding 된 공지의 상태를 변경할 수 있다") + @Test + public void updateNoticeEmbeddingStatus() { + // given + Notice notice1 = new Notice("1", "2024-03-19 17:27:07", "2023-04-03 17:27:05", + "notice1", CategoryName.BACHELOR, false, "https://www.example.com"); + Notice notice2 = new Notice("2", "2024-01-20 00:00:00", "2023-04-03 17:27:05", + "notice2", CategoryName.BACHELOR, false, "https://www.example.com"); + Notice notice3 = new Notice("3", "2024-01-21 17:27:05", "2023-04-03 17:27:05", + "notice3", CategoryName.BACHELOR, false, "https://www.example.com"); + Notice notice4 = new Notice("4", "2024-01-22 17:27:05", "2023-04-03 17:27:05", + "notice4", CategoryName.BACHELOR, false, "https://www.example.com"); + + noticeCommandPort.saveAllCategoryNotices(List.of(notice1, notice2, notice3, notice4)); + + // when + noticeCommandPort.updateNoticeEmbeddingStatus(CategoryName.BACHELOR, List.of("1", "4")); + + // then + List results = noticeQueryPort.findNotYetEmbeddingNotice( + CategoryName.BACHELOR, + LocalDateTime.of(2024, 3, 19, 17, 27, 5).minusMonths(2) + ); + + // then + assertAll( + () -> assertThat(results).hasSize(2), + () -> assertThat(results).extracting("articleId") + .contains("2", "3") + ); + } + private List creatNotices(int count) { List notices = new ArrayList<>(); - for(int i = 0; i < count; i++) { + for (int i = 0; i < count; i++) { notices.add(new Notice(String.valueOf(i), "2024-01-19 17:27:05", "2023-04-03 17:27:05", "notice" + i, CategoryName.BACHELOR, false, "https://www.example.com")); } From 4d65d245e5425fa304cb3cbc197fc83af14ed23e Mon Sep 17 00:00:00 2001 From: jiwoo Date: Thu, 18 Jul 2024 20:36:16 +0900 Subject: [PATCH 10/31] =?UTF-8?q?fix(KuisHomepageNoticeTextParser):=20?= =?UTF-8?q?=EB=B3=B8=EB=AC=B8=EC=9D=84=20=ED=8F=AC=ED=95=A8=ED=95=98?= =?UTF-8?q?=EB=8A=94=20=EC=B6=94=EA=B0=80=20=ED=85=8C=EA=B7=B8=EB=A5=BC=20?= =?UTF-8?q?=ED=8C=8C=EC=8B=B1=ED=95=98=EB=8A=94=20=EB=A1=9C=EC=A7=81=20?= =?UTF-8?q?=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../notice/KuisHomepageNoticeTextParser.java | 15 +- .../notice/NoticeTextParserTemplateTest.java | 50 +- .../resources/notice/bbs-article-2-2024.html | 984 ++++++++++++++++++ 3 files changed, 1032 insertions(+), 17 deletions(-) create mode 100644 src/test/resources/notice/bbs-article-2-2024.html diff --git a/src/main/java/com/kustacks/kuring/worker/parser/notice/KuisHomepageNoticeTextParser.java b/src/main/java/com/kustacks/kuring/worker/parser/notice/KuisHomepageNoticeTextParser.java index 5b5ed205..4d3dd5ed 100644 --- a/src/main/java/com/kustacks/kuring/worker/parser/notice/KuisHomepageNoticeTextParser.java +++ b/src/main/java/com/kustacks/kuring/worker/parser/notice/KuisHomepageNoticeTextParser.java @@ -2,7 +2,10 @@ import com.kustacks.kuring.notice.domain.CategoryName; import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.springframework.stereotype.Component; +@Component public class KuisHomepageNoticeTextParser extends NoticeTextParserTemplate { @Override @@ -22,6 +25,16 @@ protected String extractArticleId(Document document) { @Override protected String extractTextBody(Document document) { - return document.selectFirst("div.board_content").text(); + Element boardContent = document.selectFirst("div.board_content"); + if(boardContent != null) { + return boardContent.text(); + } + + boardContent = document.selectFirst("div.view-con"); + if(boardContent != null) { + return boardContent.text(); + } + + throw new IllegalArgumentException("공지의 본문을 찾을 수 없습니다."); } } diff --git a/src/test/java/com/kustacks/kuring/worker/parser/notice/NoticeTextParserTemplateTest.java b/src/test/java/com/kustacks/kuring/worker/parser/notice/NoticeTextParserTemplateTest.java index 65cefb23..95563dcb 100644 --- a/src/test/java/com/kustacks/kuring/worker/parser/notice/NoticeTextParserTemplateTest.java +++ b/src/test/java/com/kustacks/kuring/worker/parser/notice/NoticeTextParserTemplateTest.java @@ -4,9 +4,13 @@ import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import java.io.IOException; +import java.util.List; +import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertAll; @@ -14,28 +18,42 @@ class NoticeTextParserTemplateTest { @DisplayName("공지의 텍스트를 파싱한다") - @Test - public void NoticeTextParser() throws IOException { + @ParameterizedTest + @MethodSource("noticeTextInputProvider") + void NoticeTextParser( + String path, String articleId, String title, List textBody + ) throws IOException { // given - Document doc = Jsoup.parse(TestFileLoader.loadHtmlFile("src/test/resources/notice/bbs-article-2024.html")); + Document doc = Jsoup.parse(TestFileLoader.loadHtmlFile(path)); // when PageTextDto results = new KuisHomepageNoticeTextParser().parse(doc); // then assertAll( - () -> assertThat(results.getArticleId()).isEqualTo("1117110"), - () -> assertThat(results.getTitle()).isEqualTo("2024년도 대학생 청소년교육지원 장학사업 멘토모집 안내"), - () -> assertThat(results.getText()).contains( - "1. 멘토 신청기간: 2024. 3. 18. (월) ~ 2023. 3. 25. (월)", - "2. 모집유형: 나눔지기(멘토) 발굴형", - "3. 신청방법: 기관 협의 완료 후 한국장학재단 사이트에서 신청", - "4. 멘토 선발 및 매칭기간: 2023. 3. 26. (화) ~ 2023. 3. 29. (금)", - "5. 멘토 모집인원: 15명", - "6. 멘토 활동기간: 2024. 4. ~ 2025. 2", - "7. 멘토 선발기준", - "8. 멘토활동 세부사항", - "9. 기타" + () -> assertThat(results.articleId()).isEqualTo(articleId), + () -> assertThat(results.title()).isEqualTo(title), + () -> assertThat(results.text()).contains(textBody) + ); + } + + private static Stream noticeTextInputProvider() { + return Stream.of( + Arguments.of("src/test/resources/notice/bbs-article-2-2024.html", + "1129848", "2024학년도 2학기 다·부·연계·융합전공 이수 및 포기 신청 안내", + List.of("2024학년도 2학기 다·부·연계·융합전공 이수 신청 안내", "문의사항: 학사팀 02-450-3192") + ), + Arguments.of("src/test/resources/notice/bbs-article-2024.html", + "1117110", "2024년도 대학생 청소년교육지원 장학사업 멘토모집 안내", + List.of("1. 멘토 신청기간: 2024. 3. 18. (월) ~ 2023. 3. 25. (월)", + "2. 모집유형: 나눔지기(멘토) 발굴형", + "3. 신청방법: 기관 협의 완료 후 한국장학재단 사이트에서 신청", + "4. 멘토 선발 및 매칭기간: 2023. 3. 26. (화) ~ 2023. 3. 29. (금)", + "5. 멘토 모집인원: 15명", + "6. 멘토 활동기간: 2024. 4. ~ 2025. 2", + "7. 멘토 선발기준", + "8. 멘토활동 세부사항", + "9. 기타") ) ); } diff --git a/src/test/resources/notice/bbs-article-2-2024.html b/src/test/resources/notice/bbs-article-2-2024.html new file mode 100644 index 00000000..dca6ef46 --- /dev/null +++ b/src/test/resources/notice/bbs-article-2-2024.html @@ -0,0 +1,984 @@ + + + + + + + + + + + 건국대학교 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+ + + + + + + +
+
+
글번호
+
1129848
+
+

+ + 2024학년도 2학기 다·부·연계·융합전공 이수 및 포기 신청 안내 +

+ +
+
+ +
+
작성자
+
+ + + 최가영 + + +
+
+
+
조회수
+
4113
+
+
+ +
등록일
+
2024.06.18
+
+
+
수정일
+
2024.07.17
+
+ +
+ +
+
+ +
+ +
+ + +
+
+
+
+
+ +
+ + + + + + + + +
+

2024학년도 2학기 다·부·연계·융합전공 이수 신청 안내

+
+

 

+

1. 일정

+ + + + + + + + + + + + + + + + + + + +
+

이수 신청

 +

2024. 7. 17.(수) 10:00 ~ 7. 18.(목) 16:00 +

+

학과(전공)별 사정 대상자 안내

 +

2024. 7. 22.(월)

+

학과(전공)별 사정기간

 +

2024. 7. 23.(화) ~ 7. 29.(월)

+

선발자(합격자) 발표

 +

2024. 8. 2.(금) 예정

+

 

+

2. 이수 신청 자격

+

가. 3 ~ 8학기 등록(진급)예정자(편입생 포함)

+

나. 전입/전출 제한학과

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+

구분

 +

예·체능계학과

 +

수의예과

+

수의학과

 +

신산업융합학과

+

K뷰티산업융합학과

 +

국제대학

+

예술디자인대학

 +

사범대학

+

음악/체육교육과

+

전입(일반→해당)

 +

불가능
+ (미술계 학과만 가능)

 +

가능

 +

불가능

 +

불가능

 +

불가능

+

전출(해당→일반)

 +

가능

 +

가능

 +

불가능

 +

불가능

 +

불가능

+

* 예술디자인대학 의상디자인학과는 동일(미술)계열 및 일반 학과 전입/전출 허용 +

+

 

+

3. 신청 방법

+

- 학사정보시스템 → 학적 → 다/부/연계/융합전공관리 해당 메뉴에 따라 각각 신청

+

 

+

4. 선발방법

+ + + + + + + + + + + + + + + + + + + +
+

구분

 +

선발 방법

+

신청인원 ≤ 선발인원

 +

학과별 신청 자격 충족자 전원 선발(적/부 평가)

+

신청인원 ＞ 선발인원

 +

정성평가 + 정량평가(총평점평균) 합산 고득자순 선발

+

0명(미수용) +

+

선발 계획이 없으므로 선발하지 않음

+

가. 학과별 선발인원 및 지정 정성평가 요소: 붙임 문서 참조

+

나. 선발인원보다 신청인원이 적어도 학과 희망 시 별도 선발 기준 지정 및 사정 진행 가능

+

다. 선발인원보다 신청인원이 2배 이상 많을 경우, 학과 재량에 따라 정성평가 대상자를 선발인원의 최대 2배수로 선정하여 평가 가능(정성평가 비대상자 자동 미선발)

+

라. 학과별 사정방법: 사전 회신한 학과별 선발 기준에 따라 자체 진행 (붙임 문서 참조)

+

1) 면접: 학과별 지정 일정에 따라 면접 개별 진행

+

2) 학업계획서/포트폴리오 등: 첨부된 양식에 따라 지정된 기간 내에 학생이 작성하여 제출한 서류 심사

+


+


+


+ + + + + + +
+

2024학년도 2학기 다·부·연계·융합전공 이수 포기 안내

+
+

 

+

1. 일정

+ + + + + + + + + + + +
+

포기 신청

 +

2024. 7. 17.(화) 10:00 ~ 7. 18.(목) 16:00 +

+

포기 처리

 +

제출 즉시 처리

+

 

+

2. 포기 자격

+

○ 현재 다·부·연계·융합전공을 이수 중인 4학기 이상 등록(진급)예정자

+

 

+

3. 포기 방법

+

○ 해당 전공 포기원 작성 후 소속(원전공) 단과대학 행정실에 제출

+

○ 양식: 홈페이지 → 대학생활 → 학교양식 → 검색 후 다운로드 가능

+

 

+

4. 유의사항

+

○ 2개 이상의 다·부·연계·융합전공을 이수 중인 경우, 한 개의 다·부·연계·융합전공만 포기해도 미 포기전공의 기 이수 교과목의 이수구분이 모두 ‘일선’으로 자동 변경되므로, 이수구분 정정 신청 으로 다·부·연계·융합전공으로 변경 필수

+

예시 1) 국어국문학과 다전공과 기계항공공학부 다전공을 동시에 이수 중인 경우, 국어국문학과 다전공을 포기하게 되면 국어국문학과 이수 교과목과 기계항공공학부 이수 교과목이 모두 ‘일선’으로 변경되므로, 기계항공공학부 이수구분 정정 신청 필수

+

예시 2) 국어국문학과 다전공과 기계항공공학부 부전공을 동시에 이수 중인 경우, 국어국문학과 다전공을 포기하게 되면 국어국문학과 이수 교과목만 ‘일선’으로 변경됨

+


+


+


+ + + + + + +
+

2024학년도 2학기 다전공 → 부전공 이수 전환 안내

+

 

+

1. 일정

+ + + + + + + + + + + +
+

포기 신청

 +

2024. 7. 17.(수) 10:00 ~ 7. 18.(목) 16:00 +

+

포기 처리

 +

제출 즉시 처리

+

 

+

2. 전환 자격

+

○ 현재 다전공을 이수 중인 재학생 중 졸업 최종학기 등록예정자

+

 

+

3. 전환 방법

+

○ 하기 3개 서류 단과대학 행정실 제출

+

- 부전공 전환 이수 신청서

+

- 이수구분 정정 신청서 (다전공 주임 교수 날인 취득 필수)

+

- 취득학점확인원 (학사정보시스템 → 졸업 → 졸업자관리 → 졸업시뮬레이션)

+

○ 양식: 홈페이지 → 대학생활 → 학교양식 → 검색 후 다운로드 가능

+

 

+

4. 유의사항

+

○ 부전공 전환 신청시 이수한 다전공의 지정교양 및 전공과목이 모두 일선으로 변경되므로, 전공과목은 별도의 이수구분 정정 신청서를 제출하여 부전공으로 이수구분 정정 필수 +

+

○ 2개 이상의 다전공을 이수 중인 경우, 한 개의 다전공만 전환해도 미포기 다전공의 기이수 교과목의 이수구분이 모두 ‘일선’으로 자동 변경되므로, 이수구분 정정 신청으로 다전공으로 변경 필수

+

예시) 국어국문학과 다전공과 기계항공공학부 다전공을 동시에 이수 중, 국어국문학과 다전공을 부전공으로 전환 시, 국어국문학과 이수 교과목과 기계항공공학부 이수 교과목이 일괄로

+

‘일선’으로 변경되므로, 국어국문학과의 전공과목은 부전공으로, 기계항공공학부의 전공과목은 다전공으로 이수구분 정정 신청 필수

+

   

+

 

+

    

+ + + + + + +
+

2024학년도 2학기 다·부·연계·융합전공 이수 공통 안내

+
+

 

+

1. 사전에 홈페이지 학사 안내 내 전공별 학점이수, 학점인정 등 자세한 사항을 반드시 숙지

+

2. 추가 신청자격, 면접 등과 관련된 자세한 사항 붙임 문서 필히 확인 후 신청

+

3. 문의사항: 학사팀 02-450-3192

+

 

+ + +
+ +
+ + +
+
첨부파일
+
+ +
+
+ + +
+ + +
+
+ + + + + + + + + + + + +
+ + +
+
+ + +
+
+
+ + +
+ + + + + + + + + From 360b0783f4b2047b5d4ab63acc3b0f42b671940e Mon Sep 17 00:00:00 2001 From: jiwoo Date: Thu, 18 Jul 2024 21:15:17 +0900 Subject: [PATCH 11/31] =?UTF-8?q?feat(KuisHomepageNoticeInfo):=20textParse?= =?UTF-8?q?r=20=EC=9D=98=EC=A1=B4=EC=84=B1=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../noticeinfo/BachelorKuisHomepageNoticeInfo.java | 5 ++++- .../noticeinfo/EmploymentKuisHomepageNoticeInfo.java | 3 +++ .../IndustryUnivKuisHomepageNoticeInfo.java | 3 +++ .../scrap/noticeinfo/KuisHomepageNoticeInfo.java | 11 +++++++++++ .../noticeinfo/NationalKuisHomepageNoticeInfo.java | 3 +++ .../noticeinfo/NormalKuisHomepageNoticeInfo.java | 3 +++ .../noticeinfo/ScholarshipKuisHomepageNoticeInfo.java | 3 +++ .../noticeinfo/StudentKuisHomepageNoticeInfo.java | 3 +++ 8 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/kustacks/kuring/worker/scrap/noticeinfo/BachelorKuisHomepageNoticeInfo.java b/src/main/java/com/kustacks/kuring/worker/scrap/noticeinfo/BachelorKuisHomepageNoticeInfo.java index efc83189..e415c851 100644 --- a/src/main/java/com/kustacks/kuring/worker/scrap/noticeinfo/BachelorKuisHomepageNoticeInfo.java +++ b/src/main/java/com/kustacks/kuring/worker/scrap/noticeinfo/BachelorKuisHomepageNoticeInfo.java @@ -1,9 +1,10 @@ package com.kustacks.kuring.worker.scrap.noticeinfo; import com.kustacks.kuring.notice.domain.CategoryName; +import com.kustacks.kuring.worker.parser.notice.KuisHomepageNoticeHtmlParser; +import com.kustacks.kuring.worker.parser.notice.KuisHomepageNoticeTextParser; import com.kustacks.kuring.worker.scrap.client.notice.KuisHomepageNoticeApiClient; import com.kustacks.kuring.worker.scrap.client.notice.property.KuisHomepageNoticeProperties; -import com.kustacks.kuring.worker.parser.notice.KuisHomepageNoticeHtmlParser; import org.springframework.stereotype.Component; @Component @@ -11,11 +12,13 @@ public class BachelorKuisHomepageNoticeInfo extends KuisHomepageNoticeInfo { public BachelorKuisHomepageNoticeInfo( KuisHomepageNoticeApiClient kuisHomepageNoticeApiClient, KuisHomepageNoticeHtmlParser kuisHomepageNoticeHtmlParser, + KuisHomepageNoticeTextParser kuisHomepageNoticeTextParser, KuisHomepageNoticeProperties kuisHomepageNoticeProperties ) { super(); this.noticeApiClient = kuisHomepageNoticeApiClient; this.htmlParser = kuisHomepageNoticeHtmlParser; + this.textParser = kuisHomepageNoticeTextParser; this.kuisHomepageNoticeProperties = kuisHomepageNoticeProperties; this.siteId = 234; this.categoryName = CategoryName.BACHELOR; diff --git a/src/main/java/com/kustacks/kuring/worker/scrap/noticeinfo/EmploymentKuisHomepageNoticeInfo.java b/src/main/java/com/kustacks/kuring/worker/scrap/noticeinfo/EmploymentKuisHomepageNoticeInfo.java index a9cbfc85..3bac30c2 100644 --- a/src/main/java/com/kustacks/kuring/worker/scrap/noticeinfo/EmploymentKuisHomepageNoticeInfo.java +++ b/src/main/java/com/kustacks/kuring/worker/scrap/noticeinfo/EmploymentKuisHomepageNoticeInfo.java @@ -1,5 +1,6 @@ package com.kustacks.kuring.worker.scrap.noticeinfo; +import com.kustacks.kuring.worker.parser.notice.KuisHomepageNoticeTextParser; import com.kustacks.kuring.worker.scrap.client.notice.KuisHomepageNoticeApiClient; import com.kustacks.kuring.worker.scrap.client.notice.property.KuisHomepageNoticeProperties; import com.kustacks.kuring.worker.parser.notice.KuisHomepageNoticeHtmlParser; @@ -12,10 +13,12 @@ public class EmploymentKuisHomepageNoticeInfo extends KuisHomepageNoticeInfo { public EmploymentKuisHomepageNoticeInfo( KuisHomepageNoticeApiClient kuisHomepageNoticeApiClient, KuisHomepageNoticeHtmlParser kuisHomepageNoticeHtmlParser, + KuisHomepageNoticeTextParser kuisHomepageNoticeTextParser, KuisHomepageNoticeProperties kuisHomepageNoticeProperties ) { this.noticeApiClient = kuisHomepageNoticeApiClient; this.htmlParser = kuisHomepageNoticeHtmlParser; + this.textParser = kuisHomepageNoticeTextParser; this.kuisHomepageNoticeProperties = kuisHomepageNoticeProperties; this.category = "job"; this.siteId = 4083; diff --git a/src/main/java/com/kustacks/kuring/worker/scrap/noticeinfo/IndustryUnivKuisHomepageNoticeInfo.java b/src/main/java/com/kustacks/kuring/worker/scrap/noticeinfo/IndustryUnivKuisHomepageNoticeInfo.java index 3ca75d7d..83f31051 100644 --- a/src/main/java/com/kustacks/kuring/worker/scrap/noticeinfo/IndustryUnivKuisHomepageNoticeInfo.java +++ b/src/main/java/com/kustacks/kuring/worker/scrap/noticeinfo/IndustryUnivKuisHomepageNoticeInfo.java @@ -1,6 +1,7 @@ package com.kustacks.kuring.worker.scrap.noticeinfo; import com.kustacks.kuring.notice.domain.CategoryName; +import com.kustacks.kuring.worker.parser.notice.KuisHomepageNoticeTextParser; import com.kustacks.kuring.worker.scrap.client.notice.KuisHomepageNoticeApiClient; import com.kustacks.kuring.worker.scrap.client.notice.property.KuisHomepageNoticeProperties; import com.kustacks.kuring.worker.parser.notice.KuisHomepageNoticeHtmlParser; @@ -11,10 +12,12 @@ public class IndustryUnivKuisHomepageNoticeInfo extends KuisHomepageNoticeInfo { public IndustryUnivKuisHomepageNoticeInfo( KuisHomepageNoticeApiClient kuisHomepageNoticeApiClient, KuisHomepageNoticeHtmlParser kuisHomepageNoticeHtmlParser, + KuisHomepageNoticeTextParser kuisHomepageNoticeTextParser, KuisHomepageNoticeProperties kuisHomepageNoticeProperties ) { this.noticeApiClient = kuisHomepageNoticeApiClient; this.htmlParser = kuisHomepageNoticeHtmlParser; + this.textParser = kuisHomepageNoticeTextParser; this.kuisHomepageNoticeProperties = kuisHomepageNoticeProperties; this.category = "research"; this.siteId = 4214; diff --git a/src/main/java/com/kustacks/kuring/worker/scrap/noticeinfo/KuisHomepageNoticeInfo.java b/src/main/java/com/kustacks/kuring/worker/scrap/noticeinfo/KuisHomepageNoticeInfo.java index 585088e3..4b197f3a 100644 --- a/src/main/java/com/kustacks/kuring/worker/scrap/noticeinfo/KuisHomepageNoticeInfo.java +++ b/src/main/java/com/kustacks/kuring/worker/scrap/noticeinfo/KuisHomepageNoticeInfo.java @@ -2,6 +2,8 @@ import com.kustacks.kuring.notice.domain.CategoryName; import com.kustacks.kuring.worker.dto.ScrapingResultDto; +import com.kustacks.kuring.worker.parser.notice.NoticeTextParserTemplate; +import com.kustacks.kuring.worker.parser.notice.PageTextDto; import com.kustacks.kuring.worker.scrap.client.notice.NoticeApiClient; import com.kustacks.kuring.worker.scrap.client.notice.property.KuisHomepageNoticeProperties; import com.kustacks.kuring.worker.parser.notice.NoticeHtmlParserTemplate; @@ -18,6 +20,7 @@ public class KuisHomepageNoticeInfo { protected NoticeApiClient noticeApiClient; protected KuisHomepageNoticeProperties kuisHomepageNoticeProperties; protected NoticeHtmlParserTemplate htmlParser; + protected NoticeTextParserTemplate textParser; protected CategoryName categoryName; protected String category = "konkuk"; protected Integer siteId; @@ -30,10 +33,18 @@ public List scrapAllPageHtml() { return noticeApiClient.requestAll(this); } + public ScrapingResultDto scrapSinglePageHtml(String url) { + return noticeApiClient.requestSinglePageWithUrl(this, url); + } + public RowsDto parse(Document document) { return htmlParser.parse(document); } + public PageTextDto parseText(Document document) { + return textParser.parse(document); + } + public CategoryName getCategoryName() { return categoryName; } diff --git a/src/main/java/com/kustacks/kuring/worker/scrap/noticeinfo/NationalKuisHomepageNoticeInfo.java b/src/main/java/com/kustacks/kuring/worker/scrap/noticeinfo/NationalKuisHomepageNoticeInfo.java index 3e98c815..ea507cdc 100644 --- a/src/main/java/com/kustacks/kuring/worker/scrap/noticeinfo/NationalKuisHomepageNoticeInfo.java +++ b/src/main/java/com/kustacks/kuring/worker/scrap/noticeinfo/NationalKuisHomepageNoticeInfo.java @@ -1,6 +1,7 @@ package com.kustacks.kuring.worker.scrap.noticeinfo; import com.kustacks.kuring.notice.domain.CategoryName; +import com.kustacks.kuring.worker.parser.notice.KuisHomepageNoticeTextParser; import com.kustacks.kuring.worker.scrap.client.notice.KuisHomepageNoticeApiClient; import com.kustacks.kuring.worker.scrap.client.notice.property.KuisHomepageNoticeProperties; import com.kustacks.kuring.worker.parser.notice.KuisHomepageNoticeHtmlParser; @@ -11,10 +12,12 @@ public class NationalKuisHomepageNoticeInfo extends KuisHomepageNoticeInfo { public NationalKuisHomepageNoticeInfo( KuisHomepageNoticeApiClient kuisHomepageNoticeApiClient, KuisHomepageNoticeHtmlParser kuisHomepageNoticeHtmlParser, + KuisHomepageNoticeTextParser kuisHomepageNoticeTextParser, KuisHomepageNoticeProperties kuisHomepageNoticeProperties ) { this.noticeApiClient = kuisHomepageNoticeApiClient; this.htmlParser = kuisHomepageNoticeHtmlParser; + this.textParser = kuisHomepageNoticeTextParser; this.kuisHomepageNoticeProperties = kuisHomepageNoticeProperties; this.siteId = 237; this.categoryName = CategoryName.NATIONAL; diff --git a/src/main/java/com/kustacks/kuring/worker/scrap/noticeinfo/NormalKuisHomepageNoticeInfo.java b/src/main/java/com/kustacks/kuring/worker/scrap/noticeinfo/NormalKuisHomepageNoticeInfo.java index c5cb475e..3b3bc464 100644 --- a/src/main/java/com/kustacks/kuring/worker/scrap/noticeinfo/NormalKuisHomepageNoticeInfo.java +++ b/src/main/java/com/kustacks/kuring/worker/scrap/noticeinfo/NormalKuisHomepageNoticeInfo.java @@ -1,6 +1,7 @@ package com.kustacks.kuring.worker.scrap.noticeinfo; import com.kustacks.kuring.notice.domain.CategoryName; +import com.kustacks.kuring.worker.parser.notice.KuisHomepageNoticeTextParser; import com.kustacks.kuring.worker.scrap.client.notice.KuisHomepageNoticeApiClient; import com.kustacks.kuring.worker.scrap.client.notice.property.KuisHomepageNoticeProperties; import com.kustacks.kuring.worker.parser.notice.KuisHomepageNoticeHtmlParser; @@ -11,10 +12,12 @@ public class NormalKuisHomepageNoticeInfo extends KuisHomepageNoticeInfo { public NormalKuisHomepageNoticeInfo( KuisHomepageNoticeApiClient kuisHomepageNoticeApiClient, KuisHomepageNoticeHtmlParser kuisHomepageNoticeHtmlParser, + KuisHomepageNoticeTextParser kuisHomepageNoticeTextParser, KuisHomepageNoticeProperties kuisHomepageNoticeProperties ) { this.noticeApiClient = kuisHomepageNoticeApiClient; this.htmlParser = kuisHomepageNoticeHtmlParser; + this.textParser = kuisHomepageNoticeTextParser; this.kuisHomepageNoticeProperties = kuisHomepageNoticeProperties; this.siteId = 240; this.categoryName = CategoryName.NORMAL; diff --git a/src/main/java/com/kustacks/kuring/worker/scrap/noticeinfo/ScholarshipKuisHomepageNoticeInfo.java b/src/main/java/com/kustacks/kuring/worker/scrap/noticeinfo/ScholarshipKuisHomepageNoticeInfo.java index c760aa6a..d4559f7e 100644 --- a/src/main/java/com/kustacks/kuring/worker/scrap/noticeinfo/ScholarshipKuisHomepageNoticeInfo.java +++ b/src/main/java/com/kustacks/kuring/worker/scrap/noticeinfo/ScholarshipKuisHomepageNoticeInfo.java @@ -1,6 +1,7 @@ package com.kustacks.kuring.worker.scrap.noticeinfo; import com.kustacks.kuring.notice.domain.CategoryName; +import com.kustacks.kuring.worker.parser.notice.KuisHomepageNoticeTextParser; import com.kustacks.kuring.worker.scrap.client.notice.KuisHomepageNoticeApiClient; import com.kustacks.kuring.worker.scrap.client.notice.property.KuisHomepageNoticeProperties; import com.kustacks.kuring.worker.parser.notice.KuisHomepageNoticeHtmlParser; @@ -11,10 +12,12 @@ public class ScholarshipKuisHomepageNoticeInfo extends KuisHomepageNoticeInfo { public ScholarshipKuisHomepageNoticeInfo( KuisHomepageNoticeApiClient kuisHomepageNoticeApiClient, KuisHomepageNoticeHtmlParser kuisHomepageNoticeHtmlParser, + KuisHomepageNoticeTextParser kuisHomepageNoticeTextParser, KuisHomepageNoticeProperties kuisHomepageNoticeProperties ) { this.noticeApiClient = kuisHomepageNoticeApiClient; this.htmlParser = kuisHomepageNoticeHtmlParser; + this.textParser = kuisHomepageNoticeTextParser; this.kuisHomepageNoticeProperties = kuisHomepageNoticeProperties; this.siteId = 235; this.categoryName = CategoryName.SCHOLARSHIP; diff --git a/src/main/java/com/kustacks/kuring/worker/scrap/noticeinfo/StudentKuisHomepageNoticeInfo.java b/src/main/java/com/kustacks/kuring/worker/scrap/noticeinfo/StudentKuisHomepageNoticeInfo.java index 82949296..01acbc19 100644 --- a/src/main/java/com/kustacks/kuring/worker/scrap/noticeinfo/StudentKuisHomepageNoticeInfo.java +++ b/src/main/java/com/kustacks/kuring/worker/scrap/noticeinfo/StudentKuisHomepageNoticeInfo.java @@ -1,6 +1,7 @@ package com.kustacks.kuring.worker.scrap.noticeinfo; import com.kustacks.kuring.notice.domain.CategoryName; +import com.kustacks.kuring.worker.parser.notice.KuisHomepageNoticeTextParser; import com.kustacks.kuring.worker.scrap.client.notice.KuisHomepageNoticeApiClient; import com.kustacks.kuring.worker.scrap.client.notice.property.KuisHomepageNoticeProperties; import com.kustacks.kuring.worker.parser.notice.KuisHomepageNoticeHtmlParser; @@ -11,10 +12,12 @@ public class StudentKuisHomepageNoticeInfo extends KuisHomepageNoticeInfo { public StudentKuisHomepageNoticeInfo( KuisHomepageNoticeApiClient kuisHomepageNoticeApiClient, KuisHomepageNoticeHtmlParser kuisHomepageNoticeHtmlParser, + KuisHomepageNoticeTextParser kuisHomepageNoticeTextParser, KuisHomepageNoticeProperties kuisHomepageNoticeProperties ) { this.noticeApiClient = kuisHomepageNoticeApiClient; this.htmlParser = kuisHomepageNoticeHtmlParser; + this.textParser = kuisHomepageNoticeTextParser; this.kuisHomepageNoticeProperties = kuisHomepageNoticeProperties; this.siteId = 238; this.categoryName = CategoryName.STUDENT; From 95dbbae0119bdb205972fb1c6e13d1734f9c5d0f Mon Sep 17 00:00:00 2001 From: jiwoo Date: Thu, 18 Jul 2024 21:19:01 +0900 Subject: [PATCH 12/31] =?UTF-8?q?feat(ChromaVectorStoreAdapter):=20ChromaV?= =?UTF-8?q?ector=20=EA=B5=AC=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../persistence/ChromaVectorStoreAdapter.java | 56 +++++++++++++++++++ ...r.java => InMemoryVectorStoreAdapter.java} | 10 +++- .../persistence/QueryVectorStoreAdapter.java | 33 ----------- .../port/out/CommandVectorStorePort.java | 10 ++++ .../worker/parser/notice/PageTextDto.java | 15 ++--- 5 files changed, 80 insertions(+), 44 deletions(-) create mode 100644 src/main/java/com/kustacks/kuring/ai/adapter/out/persistence/ChromaVectorStoreAdapter.java rename src/main/java/com/kustacks/kuring/ai/adapter/out/persistence/{InMemoryQueryVectorStoreAdapter.java => InMemoryVectorStoreAdapter.java} (76%) delete mode 100644 src/main/java/com/kustacks/kuring/ai/adapter/out/persistence/QueryVectorStoreAdapter.java create mode 100644 src/main/java/com/kustacks/kuring/ai/application/port/out/CommandVectorStorePort.java diff --git a/src/main/java/com/kustacks/kuring/ai/adapter/out/persistence/ChromaVectorStoreAdapter.java b/src/main/java/com/kustacks/kuring/ai/adapter/out/persistence/ChromaVectorStoreAdapter.java new file mode 100644 index 00000000..d9291262 --- /dev/null +++ b/src/main/java/com/kustacks/kuring/ai/adapter/out/persistence/ChromaVectorStoreAdapter.java @@ -0,0 +1,56 @@ +package com.kustacks.kuring.ai.adapter.out.persistence; + +import com.kustacks.kuring.ai.application.port.out.CommandVectorStorePort; +import com.kustacks.kuring.ai.application.port.out.QueryVectorStorePort; +import com.kustacks.kuring.notice.domain.CategoryName; +import com.kustacks.kuring.worker.parser.notice.PageTextDto; +import lombok.RequiredArgsConstructor; +import org.springframework.ai.document.Document; +import org.springframework.ai.reader.TextReader; +import org.springframework.ai.transformer.splitter.TextSplitter; +import org.springframework.ai.vectorstore.ChromaVectorStore; +import org.springframework.ai.vectorstore.SearchRequest; +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Component; + +import java.util.List; + +@Component +@Profile("prod | local") +@RequiredArgsConstructor +public class ChromaVectorStoreAdapter implements QueryVectorStorePort, CommandVectorStorePort { + + private static final double SIMILARITY_THRESHOLD = 0.80; + private static final int TOP_K = 1; + + private final ChromaVectorStore chromaVectorStore; + private final TextSplitter TokenTextSplitter; + + @Override + public List findSimilarityContents(String question) { + return chromaVectorStore.similaritySearch( + SearchRequest.query(question) + .withTopK(TOP_K) + .withSimilarityThreshold(SIMILARITY_THRESHOLD) + ).stream() + .map(Document::getContent) + .toList(); + } + + @Override + public void embedding(List extractTextResults, CategoryName categoryName) { + for (PageTextDto textResult : extractTextResults) { + List documents = createDocuments(categoryName, textResult); + List splitDocuments = TokenTextSplitter.apply(documents); + chromaVectorStore.accept(splitDocuments); + } + } + + private List createDocuments(CategoryName categoryName, PageTextDto textResult) { + TextReader textReader = new TextReader(textResult.text()); + textReader.getCustomMetadata().put("filename", textResult.title()); + textReader.getCustomMetadata().put("articleId", textResult.articleId()); + textReader.getCustomMetadata().put("category", categoryName.getName()); + return textReader.get(); + } +} diff --git a/src/main/java/com/kustacks/kuring/ai/adapter/out/persistence/InMemoryQueryVectorStoreAdapter.java b/src/main/java/com/kustacks/kuring/ai/adapter/out/persistence/InMemoryVectorStoreAdapter.java similarity index 76% rename from src/main/java/com/kustacks/kuring/ai/adapter/out/persistence/InMemoryQueryVectorStoreAdapter.java rename to src/main/java/com/kustacks/kuring/ai/adapter/out/persistence/InMemoryVectorStoreAdapter.java index e61b5337..91ba001b 100644 --- a/src/main/java/com/kustacks/kuring/ai/adapter/out/persistence/InMemoryQueryVectorStoreAdapter.java +++ b/src/main/java/com/kustacks/kuring/ai/adapter/out/persistence/InMemoryVectorStoreAdapter.java @@ -1,6 +1,9 @@ package com.kustacks.kuring.ai.adapter.out.persistence; +import com.kustacks.kuring.ai.application.port.out.CommandVectorStorePort; import com.kustacks.kuring.ai.application.port.out.QueryVectorStorePort; +import com.kustacks.kuring.notice.domain.CategoryName; +import com.kustacks.kuring.worker.parser.notice.PageTextDto; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.ai.document.Document; @@ -15,7 +18,7 @@ @Profile("dev | test") @Component @RequiredArgsConstructor -public class InMemoryQueryVectorStoreAdapter implements QueryVectorStorePort { +public class InMemoryVectorStoreAdapter implements QueryVectorStorePort, CommandVectorStorePort { @Override public List findSimilarityContents(String question) { @@ -28,6 +31,11 @@ public List findSimilarityContents(String question) { .toList(); } + @Override + public void embedding(List extractTextResults, CategoryName categoryName) { + log.info("[InMemoryQueryVectorStoreAdapter] embedding {}", categoryName); + } + private Document createDocument(HashMap metadata) { return new Document( "a5a7414f-f676-409b-9f2e-1042f9846c97", diff --git a/src/main/java/com/kustacks/kuring/ai/adapter/out/persistence/QueryVectorStoreAdapter.java b/src/main/java/com/kustacks/kuring/ai/adapter/out/persistence/QueryVectorStoreAdapter.java deleted file mode 100644 index f3ea3931..00000000 --- a/src/main/java/com/kustacks/kuring/ai/adapter/out/persistence/QueryVectorStoreAdapter.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.kustacks.kuring.ai.adapter.out.persistence; - -import com.kustacks.kuring.ai.application.port.out.QueryVectorStorePort; -import lombok.RequiredArgsConstructor; -import org.springframework.ai.document.Document; -import org.springframework.ai.vectorstore.ChromaVectorStore; -import org.springframework.ai.vectorstore.SearchRequest; -import org.springframework.context.annotation.Profile; -import org.springframework.stereotype.Component; - -import java.util.List; - -@Component -@Profile("prod | local") -@RequiredArgsConstructor -public class QueryVectorStoreAdapter implements QueryVectorStorePort { - - private static final double SIMILARITY_THRESHOLD = 0.80; - private static final int TOP_K = 1; - - private final ChromaVectorStore chromaVectorStore; - - @Override - public List findSimilarityContents(String question) { - return chromaVectorStore.similaritySearch( - SearchRequest.query(question) - .withTopK(TOP_K) - .withSimilarityThreshold(SIMILARITY_THRESHOLD) - ).stream() - .map(Document::getContent) - .toList(); - } -} diff --git a/src/main/java/com/kustacks/kuring/ai/application/port/out/CommandVectorStorePort.java b/src/main/java/com/kustacks/kuring/ai/application/port/out/CommandVectorStorePort.java new file mode 100644 index 00000000..f4792878 --- /dev/null +++ b/src/main/java/com/kustacks/kuring/ai/application/port/out/CommandVectorStorePort.java @@ -0,0 +1,10 @@ +package com.kustacks.kuring.ai.application.port.out; + +import com.kustacks.kuring.notice.domain.CategoryName; +import com.kustacks.kuring.worker.parser.notice.PageTextDto; + +import java.util.List; + +public interface CommandVectorStorePort { + void embedding(List extractTextResults, CategoryName categoryName); +} diff --git a/src/main/java/com/kustacks/kuring/worker/parser/notice/PageTextDto.java b/src/main/java/com/kustacks/kuring/worker/parser/notice/PageTextDto.java index ae874c16..0126bb1e 100644 --- a/src/main/java/com/kustacks/kuring/worker/parser/notice/PageTextDto.java +++ b/src/main/java/com/kustacks/kuring/worker/parser/notice/PageTextDto.java @@ -1,13 +1,8 @@ package com.kustacks.kuring.worker.parser.notice; -import lombok.Getter; -import lombok.RequiredArgsConstructor; - -@Getter -@RequiredArgsConstructor -public class PageTextDto { - - private final String title; - private final String articleId; - private final String text; +public record PageTextDto( + String title, + String articleId, + String text +) { } From 7c7181083c6840548a3442f0c2e4283b2afa0642 Mon Sep 17 00:00:00 2001 From: jiwoo Date: Thu, 18 Jul 2024 21:36:42 +0900 Subject: [PATCH 13/31] =?UTF-8?q?test(KuisHomepageNoticeScraperTemplateTes?= =?UTF-8?q?t):=20=EC=9E=84=EB=B2=A0=EB=94=A9=20=ED=85=8C=EC=8A=A4=ED=8A=B8?= =?UTF-8?q?=20scrapForEmbedding=20=EC=9E=91=EC=84=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...KuisHomepageNoticeScraperTemplateTest.java | 36 +++++++++++++++++-- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/src/test/java/com/kustacks/kuring/worker/scrap/KuisHomepageNoticeScraperTemplateTest.java b/src/test/java/com/kustacks/kuring/worker/scrap/KuisHomepageNoticeScraperTemplateTest.java index a8d2e4f2..3f5beba1 100644 --- a/src/test/java/com/kustacks/kuring/worker/scrap/KuisHomepageNoticeScraperTemplateTest.java +++ b/src/test/java/com/kustacks/kuring/worker/scrap/KuisHomepageNoticeScraperTemplateTest.java @@ -1,9 +1,12 @@ package com.kustacks.kuring.worker.scrap; +import com.kustacks.kuring.notice.application.port.out.dto.NoticeDto; import com.kustacks.kuring.support.IntegrationTestSupport; import com.kustacks.kuring.support.TestFileLoader; import com.kustacks.kuring.worker.dto.ComplexNoticeFormatDto; +import com.kustacks.kuring.worker.parser.notice.PageTextDto; import com.kustacks.kuring.worker.scrap.client.NormalJsoupClient; +import com.kustacks.kuring.worker.scrap.noticeinfo.BachelorKuisHomepageNoticeInfo; import com.kustacks.kuring.worker.scrap.noticeinfo.KuisHomepageNoticeInfo; import com.kustacks.kuring.worker.scrap.noticeinfo.StudentKuisHomepageNoticeInfo; import org.jsoup.Jsoup; @@ -28,12 +31,15 @@ class KuisHomepageNoticeScraperTemplateTest extends IntegrationTestSupport { private NormalJsoupClient normalJsoupClient; @Autowired - private KuisHomepageNoticeScraperTemplate scraperTemplateTest; + private KuisHomepageNoticeScraperTemplate scraperTemplate; @Autowired private StudentKuisHomepageNoticeInfo studentKuisHomepageNoticeInfo; - @DisplayName("Kuis 공지의 최신 페이지를 스크래핑한다.") + @Autowired + private BachelorKuisHomepageNoticeInfo bachelorKuisHomepageNoticeInfo; + + @DisplayName("Kuis 공지의 최신 페이지를 스크래핑한다") @Test void request() throws IOException { // given @@ -43,7 +49,7 @@ void request() throws IOException { when(normalJsoupClient.get(anyString(), anyInt())).thenReturn(doc); // when - List results = scraperTemplateTest.scrap( + List results = scraperTemplate.scrap( studentKuisHomepageNoticeInfo, KuisHomepageNoticeInfo::scrapLatestPageHtml); // then @@ -53,4 +59,28 @@ void request() throws IOException { () -> assertThat(results.get(0).getNormalNoticeList()).hasSize(2) ); } + + @DisplayName("Kuis 공지의 최신 페이지를 embedding 한다") + @Test + void scrapForEmbedding() throws IOException { + // given + Document doc = Jsoup.parse( + TestFileLoader.loadHtmlFile("src/test/resources/notice/bbs-article-2024.html") + ); + + NoticeDto noticeDto = new NoticeDto("1", "2024-01-01 00:00:00", + "http://example.com", "제목", "category", true); + + when(normalJsoupClient.get(anyString(), anyInt())).thenReturn(doc); + + // when + List pageTextDtos = scraperTemplate.scrapForEmbedding(List.of(noticeDto), bachelorKuisHomepageNoticeInfo); + + // then + assertAll( + () -> assertThat(pageTextDtos).hasSize(1), + () -> assertThat(pageTextDtos.get(0).articleId()).isEqualTo("1117110"), + () -> assertThat(pageTextDtos.get(0).title()).isEqualTo("2024년도 대학생 청소년교육지원 장학사업 멘토모집 안내") + ); + } } From 91c6a933f72f0b4e4218f91cd1bb305072f2f213 Mon Sep 17 00:00:00 2001 From: jiwoo Date: Fri, 19 Jul 2024 14:56:48 +0900 Subject: [PATCH 14/31] =?UTF-8?q?feat(RAGConfiguration):=20RAG=20=ED=99=98?= =?UTF-8?q?=EA=B2=BD=EC=84=A4=EC=A0=95=20=EA=B5=AC=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/kustacks/kuring/config/RAGConfiguration.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/com/kustacks/kuring/config/RAGConfiguration.java b/src/main/java/com/kustacks/kuring/config/RAGConfiguration.java index 1c3eca35..ab1b09ff 100644 --- a/src/main/java/com/kustacks/kuring/config/RAGConfiguration.java +++ b/src/main/java/com/kustacks/kuring/config/RAGConfiguration.java @@ -5,6 +5,7 @@ import org.springframework.ai.document.Document; import org.springframework.ai.embedding.EmbeddingModel; import org.springframework.ai.reader.TextReader; +import org.springframework.ai.transformer.splitter.TextSplitter; import org.springframework.ai.transformer.splitter.TokenTextSplitter; import org.springframework.ai.vectorstore.ChromaVectorStore; import org.springframework.ai.vectorstore.SimpleVectorStore; @@ -30,6 +31,11 @@ public class RAGConfiguration { @Value("vectorstore.json") private String vectorStoreName; + @Bean + public TextSplitter tokenTextSplitter() { + return new TokenTextSplitter(); + } + @Profile("test") @Bean public ChromaApi chromaApi(RestTemplate restTemplate) { From f808b155ae2d4dc2971ba67b9e4864baea24e983 Mon Sep 17 00:00:00 2001 From: jiwoo Date: Fri, 19 Jul 2024 15:34:46 +0900 Subject: [PATCH 15/31] =?UTF-8?q?feat(NoticeEmbeddingUpdater):=20=EA=B3=B5?= =?UTF-8?q?=EC=A7=80=20embedding=EC=9D=84=20=EC=9C=84=ED=95=9C=20Updater?= =?UTF-8?q?=20=EA=B5=AC=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../KuisHomepageNoticeScraperTemplate.java | 58 +++++++++++++- .../notice/KuisHomepageNoticeApiClient.java | 4 +- .../update/notice/NoticeEmbeddingUpdater.java | 80 +++++++++++++++++++ 3 files changed, 137 insertions(+), 5 deletions(-) create mode 100644 src/main/java/com/kustacks/kuring/worker/update/notice/NoticeEmbeddingUpdater.java diff --git a/src/main/java/com/kustacks/kuring/worker/scrap/KuisHomepageNoticeScraperTemplate.java b/src/main/java/com/kustacks/kuring/worker/scrap/KuisHomepageNoticeScraperTemplate.java index a9caa151..12caea11 100644 --- a/src/main/java/com/kustacks/kuring/worker/scrap/KuisHomepageNoticeScraperTemplate.java +++ b/src/main/java/com/kustacks/kuring/worker/scrap/KuisHomepageNoticeScraperTemplate.java @@ -2,15 +2,18 @@ import com.kustacks.kuring.common.exception.InternalLogicException; import com.kustacks.kuring.common.exception.code.ErrorCode; +import com.kustacks.kuring.notice.application.port.out.dto.NoticeDto; import com.kustacks.kuring.worker.dto.ComplexNoticeFormatDto; import com.kustacks.kuring.worker.dto.ScrapingResultDto; -import com.kustacks.kuring.worker.scrap.noticeinfo.KuisHomepageNoticeInfo; +import com.kustacks.kuring.worker.parser.notice.PageTextDto; import com.kustacks.kuring.worker.parser.notice.RowsDto; +import com.kustacks.kuring.worker.scrap.noticeinfo.KuisHomepageNoticeInfo; import com.kustacks.kuring.worker.update.notice.dto.response.CommonNoticeFormatDto; import lombok.extern.slf4j.Slf4j; import org.jsoup.nodes.Document; import org.springframework.stereotype.Component; +import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.function.Function; @@ -34,6 +37,19 @@ public List scrap( return noticeDtoList; } + public List scrapForEmbedding( + List scrapResults, + KuisHomepageNoticeInfo noticeInfo + ) throws InternalLogicException { + List requestResults = requestWithDeptInfoForEmbedding(scrapResults, noticeInfo); + + log.debug("[{}] Text extract begin", noticeInfo.getCategoryName()); + List noticeDtoList = htmlTextParsingFromScrapingResult(noticeInfo, requestResults); + log.debug("[{}] Text extract end", noticeInfo.getCategoryName()); + + return noticeDtoList; + } + private void validateScrapedNoticeCountIsNotZero(List noticeDtoList) { for (ComplexNoticeFormatDto complexNoticeFormatDto : noticeDtoList) { if (complexNoticeFormatDto.getNormalNoticeSize() == 0) { @@ -48,9 +64,9 @@ private List requestWithDeptInfo( ) { long startTime = System.currentTimeMillis(); - log.debug("[{}] HTML 요청", kuisNoticeInfo.getCategoryName()); + log.debug("[{}] HTML SCRAP 요청", kuisNoticeInfo.getCategoryName()); List reqResults = decisionMaker.apply(kuisNoticeInfo); - log.debug("[{}] HTML 수신", kuisNoticeInfo.getCategoryName()); + log.debug("[{}] HTML SCRAP 수신", kuisNoticeInfo.getCategoryName()); long endTime = System.currentTimeMillis(); log.debug("[{}] 파싱에 소요된 초 = {}", kuisNoticeInfo.getCategoryName(), (endTime - startTime) / 1000.0); @@ -58,6 +74,42 @@ private List requestWithDeptInfo( return reqResults; } + private List requestWithDeptInfoForEmbedding( + List scrapResults, + KuisHomepageNoticeInfo noticeInfo + ) { + long startTime = System.currentTimeMillis(); + + List scrapResultDtos = new LinkedList<>(); + for (NoticeDto scrapResult : scrapResults) { + log.debug("[{}] HTML SCRAP 요청", noticeInfo.getCategoryName()); + scrapResultDtos.add(noticeInfo.scrapSinglePageHtml(scrapResult.getUrl())); + log.debug("[{}] HTML SCRAP 수신", noticeInfo.getCategoryName()); + } + + long endTime = System.currentTimeMillis(); + log.debug("[{}] 파싱에 소요된 초 = {}", noticeInfo.getCategoryName(), (endTime - startTime) / 1000.0); + + return scrapResultDtos; + } + + private List htmlTextParsingFromScrapingResult( + KuisHomepageNoticeInfo noticeInfo, + List results + ) { + List parsedTexts = new ArrayList<>(); + for (ScrapingResultDto result : results) { + try { + PageTextDto parsedText = noticeInfo.parseText(result.getDocument()); + parsedTexts.add(parsedText); + } catch (InternalLogicException e) { + log.error("Exception extracting url: {}", result.getViewUrl(), e); + } + } + + return parsedTexts; + } + private List htmlParsingFromScrapingResult( KuisHomepageNoticeInfo kuisNoticeInfo, diff --git a/src/main/java/com/kustacks/kuring/worker/scrap/client/notice/KuisHomepageNoticeApiClient.java b/src/main/java/com/kustacks/kuring/worker/scrap/client/notice/KuisHomepageNoticeApiClient.java index b6738a95..fde4349f 100644 --- a/src/main/java/com/kustacks/kuring/worker/scrap/client/notice/KuisHomepageNoticeApiClient.java +++ b/src/main/java/com/kustacks/kuring/worker/scrap/client/notice/KuisHomepageNoticeApiClient.java @@ -62,7 +62,7 @@ public List requestAll(KuisHomepageNoticeInfo kuisHomepageNot @Override public ScrapingResultDto requestSinglePageWithUrl(KuisHomepageNoticeInfo noticeInfo, String url) { try { - Document document = jsoupClient.get(url, LATEST_SCRAP_ALL_TIMEOUT); + Document document = jsoupClient.get(url, LATEST_SCRAP_TIMEOUT); return new ScrapingResultDto(document, url); } catch (IOException e) { log.info("Notice Text Scrap IOException", e); @@ -78,7 +78,7 @@ public int getTotalNoticeSize(String url) throws IOException, IndexOutOfBoundsEx Element totalNoticeSizeElement = document.selectFirst(".util-search strong"); - if(totalNoticeSizeElement == null) { // 총 공지 개수가 없는 경우 650개로 가정 + if (totalNoticeSizeElement == null) { // 총 공지 개수가 없는 경우 650개로 가정 return TOTAL_KUIS_NOTICES_COUNT; } diff --git a/src/main/java/com/kustacks/kuring/worker/update/notice/NoticeEmbeddingUpdater.java b/src/main/java/com/kustacks/kuring/worker/update/notice/NoticeEmbeddingUpdater.java new file mode 100644 index 00000000..e0c71c5f --- /dev/null +++ b/src/main/java/com/kustacks/kuring/worker/update/notice/NoticeEmbeddingUpdater.java @@ -0,0 +1,80 @@ +package com.kustacks.kuring.worker.update.notice; + +import com.kustacks.kuring.ai.application.port.out.CommandVectorStorePort; +import com.kustacks.kuring.notice.application.port.out.NoticeCommandPort; +import com.kustacks.kuring.notice.application.port.out.NoticeQueryPort; +import com.kustacks.kuring.notice.application.port.out.dto.NoticeDto; +import com.kustacks.kuring.notice.domain.CategoryName; +import com.kustacks.kuring.worker.parser.notice.PageTextDto; +import com.kustacks.kuring.worker.scrap.KuisHomepageNoticeScraperTemplate; +import com.kustacks.kuring.worker.scrap.noticeinfo.KuisHomepageNoticeInfo; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import org.springframework.stereotype.Component; + +import java.time.LocalDateTime; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +@Slf4j +@Component +@RequiredArgsConstructor +public class NoticeEmbeddingUpdater { + + private final ThreadPoolTaskExecutor noticeUpdaterThreadTaskExecutor; + private final KuisHomepageNoticeScraperTemplate scrapperTemplate; + private final List kuisNoticeInfoList; + private final CommandVectorStorePort commandVectorStorePort; + private final NoticeCommandPort noticeCommandPort; + private final NoticeQueryPort noticeQueryPort; + + /* + 학사, 장학, 취창업, 국제, 학생, 산학, 일반, 공지 embedding + */ + @Scheduled(cron = "0 0 21 * * *", zone = "Asia/Seoul") // 매일 오후 9시 embedding 작업 수행 + public void update() { + log.info("========== KUIS Hompage Embedding 시작 =========="); + + for (KuisHomepageNoticeInfo kuisNoticeInfo : kuisNoticeInfoList) { + CompletableFuture + .supplyAsync( + () -> lookupNotYetEmbeddingNotice(kuisNoticeInfo), + noticeUpdaterThreadTaskExecutor + ).thenApply( + scrapResults -> scrapNoticeText(scrapResults, kuisNoticeInfo) + ).thenAccept( + scrapResults -> embeddingNotice(scrapResults, kuisNoticeInfo.getCategoryName()) + ); + } + } + + private List lookupNotYetEmbeddingNotice(KuisHomepageNoticeInfo noticeInfo) { + log.debug("lookupNotYetEmbeddingNotice {}", noticeInfo.getCategoryName()); + LocalDateTime startDate = LocalDateTime.now().minusMonths(2); + return noticeQueryPort.findNotYetEmbeddingNotice(noticeInfo.getCategoryName(), startDate); + } + + private List scrapNoticeText( + List scrapResults, + KuisHomepageNoticeInfo noticeInfo + ) { + return scrapperTemplate.scrapForEmbedding(scrapResults, noticeInfo); + } + + private void embeddingNotice(List extractTextResults, CategoryName categoryName) { + if (extractTextResults.isEmpty()) { + log.debug("Embedding {} no more notice to embed", categoryName); + return; + } + log.info("Embedding {}, size = {}", categoryName, extractTextResults.size()); + + commandVectorStorePort.embedding(extractTextResults, categoryName); + List articleIds = extractTextResults.stream() + .map(PageTextDto::articleId) + .toList(); + + noticeCommandPort.updateNoticeEmbeddingStatus(categoryName, articleIds); + } +} From 3c0fca8606faf657533effc433640a21056f659e Mon Sep 17 00:00:00 2001 From: jiwoo Date: Fri, 19 Jul 2024 15:37:31 +0900 Subject: [PATCH 16/31] =?UTF-8?q?feat:=20=EA=B3=B5=EC=A7=80=20updater=20?= =?UTF-8?q?=EC=9E=91=EC=97=85=20=EC=88=98=ED=96=89=20=EC=8B=9C=EA=B0=84=20?= =?UTF-8?q?=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kuring/worker/update/notice/DepartmentNoticeUpdater.java | 4 ++-- .../worker/update/notice/KuisHomepageNoticeUpdater.java | 4 ++-- .../kuring/worker/update/notice/NoticeEmbeddingUpdater.java | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/kustacks/kuring/worker/update/notice/DepartmentNoticeUpdater.java b/src/main/java/com/kustacks/kuring/worker/update/notice/DepartmentNoticeUpdater.java index 1ad57367..51c6cf6c 100644 --- a/src/main/java/com/kustacks/kuring/worker/update/notice/DepartmentNoticeUpdater.java +++ b/src/main/java/com/kustacks/kuring/worker/update/notice/DepartmentNoticeUpdater.java @@ -37,7 +37,7 @@ public class DepartmentNoticeUpdater { private final FirebaseNotificationService notificationService; private final NoticeUpdateSupport noticeUpdateSupport; - @Scheduled(cron = "0 5/10 8-19 * * *", zone = "Asia/Seoul") // 학교 공지는 오전 8:10 ~ 오후 7:55분 사이에 10분마다 업데이트 된다. + @Scheduled(cron = "0 15/20 7-19 * * *", zone = "Asia/Seoul") // 학교 공지는 오전 7:15 ~ 오후 7:55분 사이에 20분마다 업데이트 된다. public void update() { log.info("******** 학과별 최신 공지 업데이트 시작 ********"); @@ -54,7 +54,7 @@ public void update() { } } - @Scheduled(cron = "0 0 2 * * *", zone = "Asia/Seoul") // 전체 업데이트는 매일 오전 2시에 한다. + @Scheduled(cron = "0 0 23 * * 5", zone = "Asia/Seoul") // 전체 업데이트는 매주 금요일 오후 11시에 한다. public void updateAll() { log.info("******** 학과별 전체 공지 업데이트 시작 ********"); diff --git a/src/main/java/com/kustacks/kuring/worker/update/notice/KuisHomepageNoticeUpdater.java b/src/main/java/com/kustacks/kuring/worker/update/notice/KuisHomepageNoticeUpdater.java index a2ae416d..6ee87d16 100644 --- a/src/main/java/com/kustacks/kuring/worker/update/notice/KuisHomepageNoticeUpdater.java +++ b/src/main/java/com/kustacks/kuring/worker/update/notice/KuisHomepageNoticeUpdater.java @@ -40,7 +40,7 @@ public class KuisHomepageNoticeUpdater { /* 학사, 장학, 취창업, 국제, 학생, 산학, 일반, 도서관 공지 갱신 */ - @Scheduled(cron = "0 0/10 6-21 * * *", zone = "Asia/Seoul") // 학교 공지는 오전 6:00 ~ 오후 9:55분 사이에 10분마다 업데이트 된다. + @Scheduled(cron = "0 0/10 7-19 * * *", zone = "Asia/Seoul") // 학교 공지는 오전 7:00 ~ 오후 7:55분 사이에 10분마다 업데이트 된다. public void update() { log.info("========== KUIS Hompage 공지 업데이트 시작 =========="); @@ -59,7 +59,7 @@ public void update() { } } - @Scheduled(cron = "0 0 1 * * *", zone = "Asia/Seoul") // 전체 업데이트는 매일 오전 1시에 한다. + @Scheduled(cron = "0 0 22 * * *", zone = "Asia/Seoul") // 전체 업데이트는 매일 오후 10시에 한다. public void updateAll() { log.info("******** KUIS Hompage 전체 공지 업데이트 시작 ********"); diff --git a/src/main/java/com/kustacks/kuring/worker/update/notice/NoticeEmbeddingUpdater.java b/src/main/java/com/kustacks/kuring/worker/update/notice/NoticeEmbeddingUpdater.java index e0c71c5f..6ce89080 100644 --- a/src/main/java/com/kustacks/kuring/worker/update/notice/NoticeEmbeddingUpdater.java +++ b/src/main/java/com/kustacks/kuring/worker/update/notice/NoticeEmbeddingUpdater.java @@ -33,7 +33,7 @@ public class NoticeEmbeddingUpdater { /* 학사, 장학, 취창업, 국제, 학생, 산학, 일반, 공지 embedding */ - @Scheduled(cron = "0 0 21 * * *", zone = "Asia/Seoul") // 매일 오후 9시 embedding 작업 수행 + @Scheduled(cron = "0 5/20 7-19 * * *", zone = "Asia/Seoul") // 학교 공지는 오전 7:05 ~ 오후 7:55분 사이에 20분마다 업데이트 된다. public void update() { log.info("========== KUIS Hompage Embedding 시작 =========="); From 6b4c10c110e6604f0f9fff255afb8a620c3fa4df Mon Sep 17 00:00:00 2001 From: jiwoo Date: Fri, 19 Jul 2024 16:16:30 +0900 Subject: [PATCH 17/31] =?UTF-8?q?chore:=20=EC=84=A4=EC=A0=95=ED=8C=8C?= =?UTF-8?q?=EC=9D=BC=EC=97=90=20collection-name=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/resources/application-local.yml | 3 ++- src/main/resources/application-prod.yml | 3 ++- src/test/resources/application-test.yml | 5 ----- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/main/resources/application-local.yml b/src/main/resources/application-local.yml index 407a7016..affce7b0 100644 --- a/src/main/resources/application-local.yml +++ b/src/main/resources/application-local.yml @@ -46,9 +46,10 @@ spring: model: text-embedding-3-small vectorstore: chroma: - collection-name: ${CHROMA_COLLECTION_NAME} client: key-token: ${CHROMA_API_KEY} + store: + collection-name: ${CHROMA_COLLECTION_NAME} springdoc: diff --git a/src/main/resources/application-prod.yml b/src/main/resources/application-prod.yml index 3e44d7bc..3e2ada16 100644 --- a/src/main/resources/application-prod.yml +++ b/src/main/resources/application-prod.yml @@ -50,9 +50,10 @@ spring: model: text-embedding-3-small vectorstore: chroma: - collection-name: ${CHROMA_COLLECTION_NAME} client: key-token: ${CHROMA_API_KEY} + store: + collection-name: ${CHROMA_COLLECTION_NAME} decorator: datasource: diff --git a/src/test/resources/application-test.yml b/src/test/resources/application-test.yml index 8e33c205..ca921743 100644 --- a/src/test/resources/application-test.yml +++ b/src/test/resources/application-test.yml @@ -40,11 +40,6 @@ spring: embedding: options: model: text-embedding-3-small - vectorstore: - chroma: - collection-name: "test-collection" - client: - port: 8000 testcontainers: reuse: From 81e06251f55bb9ecc28e39cbb3c07609dff5ebff Mon Sep 17 00:00:00 2001 From: jiwoo Date: Sat, 20 Jul 2024 14:18:47 +0900 Subject: [PATCH 18/31] =?UTF-8?q?fix(ChromaVectorStoreAdapter):=20embeddin?= =?UTF-8?q?g=20=EB=A9=94=EC=84=9C=EB=93=9C=20=EC=88=98=EC=A0=95=EA=B3=BC?= =?UTF-8?q?=20=ED=85=8C=EC=8A=A4=ED=8A=B8=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../persistence/ChromaVectorStoreAdapter.java | 21 ++++-- .../kuring/config/RAGConfiguration.java | 6 -- .../ChromaVectorStoreAdapterTest.java | 64 +++++++++++++++++++ 3 files changed, 80 insertions(+), 11 deletions(-) create mode 100644 src/test/java/com/kustacks/kuring/ai/adapter/out/persistence/ChromaVectorStoreAdapterTest.java diff --git a/src/main/java/com/kustacks/kuring/ai/adapter/out/persistence/ChromaVectorStoreAdapter.java b/src/main/java/com/kustacks/kuring/ai/adapter/out/persistence/ChromaVectorStoreAdapter.java index d9291262..a21b148e 100644 --- a/src/main/java/com/kustacks/kuring/ai/adapter/out/persistence/ChromaVectorStoreAdapter.java +++ b/src/main/java/com/kustacks/kuring/ai/adapter/out/persistence/ChromaVectorStoreAdapter.java @@ -7,10 +7,12 @@ import lombok.RequiredArgsConstructor; import org.springframework.ai.document.Document; import org.springframework.ai.reader.TextReader; -import org.springframework.ai.transformer.splitter.TextSplitter; +import org.springframework.ai.transformer.splitter.TokenTextSplitter; import org.springframework.ai.vectorstore.ChromaVectorStore; import org.springframework.ai.vectorstore.SearchRequest; import org.springframework.context.annotation.Profile; +import org.springframework.core.io.ByteArrayResource; +import org.springframework.core.io.Resource; import org.springframework.stereotype.Component; import java.util.List; @@ -24,7 +26,6 @@ public class ChromaVectorStoreAdapter implements QueryVectorStorePort, CommandVe private static final int TOP_K = 1; private final ChromaVectorStore chromaVectorStore; - private final TextSplitter TokenTextSplitter; @Override public List findSimilarityContents(String question) { @@ -39,16 +40,26 @@ public List findSimilarityContents(String question) { @Override public void embedding(List extractTextResults, CategoryName categoryName) { + TokenTextSplitter textSplitter = new TokenTextSplitter(); + for (PageTextDto textResult : extractTextResults) { + if (textResult.text().isBlank()) continue; + List documents = createDocuments(categoryName, textResult); - List splitDocuments = TokenTextSplitter.apply(documents); + List splitDocuments = textSplitter.apply(documents); chromaVectorStore.accept(splitDocuments); } } private List createDocuments(CategoryName categoryName, PageTextDto textResult) { - TextReader textReader = new TextReader(textResult.text()); - textReader.getCustomMetadata().put("filename", textResult.title()); + Resource resource = new ByteArrayResource(textResult.text().getBytes()) { + @Override + public String getFilename() { + return textResult.title(); + } + }; + + TextReader textReader = new TextReader(resource); textReader.getCustomMetadata().put("articleId", textResult.articleId()); textReader.getCustomMetadata().put("category", categoryName.getName()); return textReader.get(); diff --git a/src/main/java/com/kustacks/kuring/config/RAGConfiguration.java b/src/main/java/com/kustacks/kuring/config/RAGConfiguration.java index ab1b09ff..1c3eca35 100644 --- a/src/main/java/com/kustacks/kuring/config/RAGConfiguration.java +++ b/src/main/java/com/kustacks/kuring/config/RAGConfiguration.java @@ -5,7 +5,6 @@ import org.springframework.ai.document.Document; import org.springframework.ai.embedding.EmbeddingModel; import org.springframework.ai.reader.TextReader; -import org.springframework.ai.transformer.splitter.TextSplitter; import org.springframework.ai.transformer.splitter.TokenTextSplitter; import org.springframework.ai.vectorstore.ChromaVectorStore; import org.springframework.ai.vectorstore.SimpleVectorStore; @@ -31,11 +30,6 @@ public class RAGConfiguration { @Value("vectorstore.json") private String vectorStoreName; - @Bean - public TextSplitter tokenTextSplitter() { - return new TokenTextSplitter(); - } - @Profile("test") @Bean public ChromaApi chromaApi(RestTemplate restTemplate) { diff --git a/src/test/java/com/kustacks/kuring/ai/adapter/out/persistence/ChromaVectorStoreAdapterTest.java b/src/test/java/com/kustacks/kuring/ai/adapter/out/persistence/ChromaVectorStoreAdapterTest.java new file mode 100644 index 00000000..bb477dee --- /dev/null +++ b/src/test/java/com/kustacks/kuring/ai/adapter/out/persistence/ChromaVectorStoreAdapterTest.java @@ -0,0 +1,64 @@ +package com.kustacks.kuring.ai.adapter.out.persistence; + +import com.kustacks.kuring.notice.domain.CategoryName; +import com.kustacks.kuring.worker.parser.notice.PageTextDto; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.ai.vectorstore.ChromaVectorStore; + +import java.util.List; + +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +@ExtendWith(MockitoExtension.class) +class ChromaVectorStoreAdapterTest { + + @Mock + ChromaVectorStore chromaVectorStore; + + @DisplayName("embedding이 성공하는지 확인") + @Test + void embedding() { + // given + ChromaVectorStoreAdapter vectorStoreAdapter = new ChromaVectorStoreAdapter(chromaVectorStore); + + String text = "건국대학교 산학협력단 2024년 하반기 KOICA ODA 영프로페셔널(YP) 채용 공고 건국대학교 「산학협력단」은 연구자가 최고의 성과를 " + + "도출할 수 있는 연구 환경을 조성하고, 연구기획·지원에서 지식재산 관리, 대외 이해관계자와의 협력은 물론, 교내 우수 연구자 및 기술을 " + + "기반으로 건국대 기술지주회사 자회사를 통한 기술사업화 고도화 지원을 위해 2004년 3월 출범하였습니다. 국제개발 협력과 사회공헌 분야에 " + + "관심 있는 인재를 다음과 같이 채용하고자 합니다. 2024년 6월 4일 건국대학교 산학협력단장 1. 채용 분야 및 인원 채용 분야 채용 인원 " + + "근무지 담당업무 KOICA ODA YP 1명 건대·코이카·베트남 국립농업대학교 축산고등교육센터(KUVEC) ODA 및 국제개발 협력 사업수행 실무 " + + "사업수행 과정 모니터링 및 대외협력 업무 등 2. 지원 자격 구분 필수 자격 사항 우대사항 공통 사항 1. 국가공무원법 제33조 각 호에 " + + "해당하는 결격사유가 없는 자 2. 공무원임용시험령 등 관계 법령에 의하여 응시 자격이 정지되지 아니한 자 3. 남자는 병역필 또는 면제자 " + + "4. 해외여행에 결격사유가 없는 자 5. 공무원 채용 신체검사 기준에 결격사유가 없는 자 6. 만 19세 이상 만 34세 이하 대한민국 국적을 " + + "가진 미취업자 (단, 군필자는 해당 법률에 따라 연령 연장) 7. KOICA 개발 협력 사업수행기관 YP로 근무한 경험이 없는 자 1. " + + "사회배려층 우대 - 장애인, 저소득층, 차상위계층, 국가보훈대상자, 지방인재, 북한이탈주민, 여성 가장, 결혼이주자, 다문화가정, " + + "위탁가정 및 아동 보육 시설 재원자(보호 종료 아동)(※ 공고 시작일 기준 등록된 자) 2. 국제 개발협력 관련 활동 경험자 및 전공자 " + + "3. 영어 능통자 우대 4. 한글, 워드, 엑셀, 파워포인트 등 프로그램 활용 가능자 우대 5. KOICA ODA 자격증 보유자 6. 대학교 행정업무" + + " 경험자 우대 ※ 군필자 응시 연령 상한: 「제대군인지원에 관한 법률」 제16조 1항에 의거 제대군인에 대한 채용시험 응시 연령 상한을 다음 " + + "각 호와 같이 연장함 ▷ 2년 이상의 복무기간을 마치고 전역한 제대군인: 만 37세 ▷ 1년 이상 2년 미만의 복무기간을 마치고 전역한 " + + "제대군인: 만 36세 ▷ 1년 미만의 복무기간을 마치고 전역한 제대군인: 만 35세 ※ 장애인 인정 범위: 「장애인복지법」상 장애인 기준에 " + + "해당하는 자 또는「국가유공자 등 예우 및 지원에 관한 법률」상 상이등급 기준에 해당하는 자 ※ 저소득층의 범위: 「국민기초생활보장법」 " + + "제2조제2호·제11호의 규정에 의한 기초생활보장 수급자 및 차상위계층에 속한 자와 「한부모가족지원법」 제5조의 규정에 의하여 보호를 " + + "받는 한부모가족 세대주 ※ 장애인, 저소득층은 각 단계별 전형의 만점 기준 4% 가점 부여 ※ 국가유공자는「국가유공자 등 예우 및 " + + "지원에 관한 법률」에 따른 해당 가점 부여 ※ KOICA 영프로페셔널 사업 개발협력 사업수행기관 YP 기참여자 재지원 불가능, KOICA " + + "해외사무소/재외공관 YP 기 참여자 지원 가능 3. 채용 기간 및 근무 조건 가. 계약기간: 2024. 8. 1. ~ 2025. 2. 28.(7개월) " + + "(연장 없음) 나. 근무 조건: 전일제 (주 5일 09:00 ~ 18:00, 주당 40시간 근무) 다. 보수액: 고용노동부 고시 최저임금 적용 예정" + + "(세전 약 207만 원, 2024년 최저임금 9,860원/시간) ※ 2025년 1, 2월분은 최저임금 인상에 따라 급여 변동 예정 라. 근무 장소: " + + "건국대학교 서울캠퍼스(상허생명과학대학 KUVEC 710-1호) 4. 채용 절차 가. 1차 전형: 서류심사 나. 2차 전형: 면접 심사 " + + "※ 서류전형 합격자는 채용 인원의 5배수 이내로 하며, 면접 일정·장소 등은 서류전형 합격자에 한해 개별 통보 5. 제출 서류 가. " + + "원서 접수 기간에 제출하는 서류 ◦ 첨부된 양식의 입사지원서 1부 ◦ 첨부된 양식의 개인정보 수집 및 이용 동의서 1부 "; + + PageTextDto pageTextDto = new PageTextDto("건국대학교 산학협력단 2024년 하반기 KOICA ODA 영프로페셔널(YP) 채용 공고", "articeId", text); + + // when + vectorStoreAdapter.embedding(List.of(pageTextDto), CategoryName.INDUSTRY_UNIVERSITY); + + // then + verify(chromaVectorStore, times(1)).accept(anyList()); + } +} From 1889237f9712a195ef2902fe5543308ad4432ee5 Mon Sep 17 00:00:00 2001 From: jiwoo Date: Sat, 20 Jul 2024 15:41:54 +0900 Subject: [PATCH 19/31] =?UTF-8?q?feat(ChromaVectorStoreAdapter):=20?= =?UTF-8?q?=EC=9C=A0=EC=82=AC=EB=8F=84=20=EC=9E=84=EA=B3=84=EC=B9=98=20?= =?UTF-8?q?=EC=A0=9C=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 유사도가 낮아도 답변을 꼭 생성하는쪽으로 구현 --- .../ai/adapter/out/model/InMemoryQueryAiModelAdapter.java | 2 +- .../kuring/ai/adapter/out/model/QueryAiModelAdapter.java | 2 +- .../ai/adapter/out/persistence/ChromaVectorStoreAdapter.java | 5 +---- .../kuring/ai/application/service/RAGQueryService.java | 4 ++-- 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/kustacks/kuring/ai/adapter/out/model/InMemoryQueryAiModelAdapter.java b/src/main/java/com/kustacks/kuring/ai/adapter/out/model/InMemoryQueryAiModelAdapter.java index d41e9c5b..a5d5b85d 100644 --- a/src/main/java/com/kustacks/kuring/ai/adapter/out/model/InMemoryQueryAiModelAdapter.java +++ b/src/main/java/com/kustacks/kuring/ai/adapter/out/model/InMemoryQueryAiModelAdapter.java @@ -12,7 +12,7 @@ @Slf4j @Component -@Profile("dev | local | test") +@Profile("dev | test") @RequiredArgsConstructor public class InMemoryQueryAiModelAdapter implements QueryAiModelPort { diff --git a/src/main/java/com/kustacks/kuring/ai/adapter/out/model/QueryAiModelAdapter.java b/src/main/java/com/kustacks/kuring/ai/adapter/out/model/QueryAiModelAdapter.java index 60882ad9..395c3a29 100644 --- a/src/main/java/com/kustacks/kuring/ai/adapter/out/model/QueryAiModelAdapter.java +++ b/src/main/java/com/kustacks/kuring/ai/adapter/out/model/QueryAiModelAdapter.java @@ -11,7 +11,7 @@ @Slf4j @Component -@Profile("prod") +@Profile("prod | local") @RequiredArgsConstructor public class QueryAiModelAdapter implements QueryAiModelPort { diff --git a/src/main/java/com/kustacks/kuring/ai/adapter/out/persistence/ChromaVectorStoreAdapter.java b/src/main/java/com/kustacks/kuring/ai/adapter/out/persistence/ChromaVectorStoreAdapter.java index a21b148e..2278300b 100644 --- a/src/main/java/com/kustacks/kuring/ai/adapter/out/persistence/ChromaVectorStoreAdapter.java +++ b/src/main/java/com/kustacks/kuring/ai/adapter/out/persistence/ChromaVectorStoreAdapter.java @@ -22,7 +22,6 @@ @RequiredArgsConstructor public class ChromaVectorStoreAdapter implements QueryVectorStorePort, CommandVectorStorePort { - private static final double SIMILARITY_THRESHOLD = 0.80; private static final int TOP_K = 1; private final ChromaVectorStore chromaVectorStore; @@ -30,9 +29,7 @@ public class ChromaVectorStoreAdapter implements QueryVectorStorePort, CommandVe @Override public List findSimilarityContents(String question) { return chromaVectorStore.similaritySearch( - SearchRequest.query(question) - .withTopK(TOP_K) - .withSimilarityThreshold(SIMILARITY_THRESHOLD) + SearchRequest.query(question).withTopK(TOP_K) ).stream() .map(Document::getContent) .toList(); diff --git a/src/main/java/com/kustacks/kuring/ai/application/service/RAGQueryService.java b/src/main/java/com/kustacks/kuring/ai/application/service/RAGQueryService.java index 39f0adf2..86a68b0e 100644 --- a/src/main/java/com/kustacks/kuring/ai/application/service/RAGQueryService.java +++ b/src/main/java/com/kustacks/kuring/ai/application/service/RAGQueryService.java @@ -33,8 +33,8 @@ public class RAGQueryService implements RAGQueryUseCase { @Override public Flux askAiModel(String question, String id) { - Prompt completePrompt = buildCompletePrompt(question); ragEventPort.userDecreaseQuestionCountEvent(id); + Prompt completePrompt = buildCompletePrompt(question); return ragChatModel.call(completePrompt); } @@ -45,7 +45,7 @@ private void init() { private Prompt buildCompletePrompt(String question) { List similarDocuments = vectorStorePort.findSimilarityContents(question); - if(similarDocuments.isEmpty()) { + if (similarDocuments.isEmpty()) { throw new InvalidStateException(ErrorCode.AI_SIMILAR_DOCUMENTS_NOT_FOUND); } From 828fd4e4fc453db5bf545c6d26bd92f227b08ba1 Mon Sep 17 00:00:00 2001 From: jiwoo Date: Sat, 20 Jul 2024 15:58:18 +0900 Subject: [PATCH 20/31] =?UTF-8?q?feat:=20=EC=82=AC=EC=9A=A9=ED=95=98?= =?UTF-8?q?=EC=A7=80=20=EC=95=8A=EB=8A=94=20RestTemplateConfig=20=EC=A0=9C?= =?UTF-8?q?=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kuring/config/RestTemplateConfig.java | 23 ------------------- 1 file changed, 23 deletions(-) delete mode 100644 src/main/java/com/kustacks/kuring/config/RestTemplateConfig.java diff --git a/src/main/java/com/kustacks/kuring/config/RestTemplateConfig.java b/src/main/java/com/kustacks/kuring/config/RestTemplateConfig.java deleted file mode 100644 index 31e0acf7..00000000 --- a/src/main/java/com/kustacks/kuring/config/RestTemplateConfig.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.kustacks.kuring.config; - -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.web.client.RestTemplate; - -@Configuration -public class RestTemplateConfig { - - @Bean - public RestTemplate restTemplate() { - return new RestTemplate(); -// HttpComponentsClientHttpRequestFactory factory = new HttpComponentsClientHttpRequestFactory(); -// factory.setConnectTimeout(5000); //5초 -// factory.setReadTimeout(5000); //5초 -// CloseableHttpClient httpClient = HttpClientBuilder.create() -// .setMaxConnTotal(100) -// .setMaxConnPerRoute(5) -// .build(); -// factory.setHttpClient(httpClient); -// return new RestTemplate(factory); - } -} From 63d68b3122ce7e014603574732adb54edf861d99 Mon Sep 17 00:00:00 2001 From: jiwoo Date: Sat, 20 Jul 2024 19:36:55 +0900 Subject: [PATCH 21/31] =?UTF-8?q?chore:=20Public=20=EC=A0=91=EA=B7=BC=20?= =?UTF-8?q?=EC=A0=9C=EC=96=B4=EC=9E=90=20=EC=A0=9C=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../adapter/out/persistence/InMemoryVectorStoreAdapter.java | 6 ++++-- .../worker/parser/notice/KuisHomepageNoticeTextParser.java | 2 +- .../adapter/out/persistence/NoticeRepositoryTest.java | 4 ++-- .../java/com/kustacks/kuring/notice/domain/NoticeTest.java | 2 +- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/kustacks/kuring/ai/adapter/out/persistence/InMemoryVectorStoreAdapter.java b/src/main/java/com/kustacks/kuring/ai/adapter/out/persistence/InMemoryVectorStoreAdapter.java index 91ba001b..9b45f031 100644 --- a/src/main/java/com/kustacks/kuring/ai/adapter/out/persistence/InMemoryVectorStoreAdapter.java +++ b/src/main/java/com/kustacks/kuring/ai/adapter/out/persistence/InMemoryVectorStoreAdapter.java @@ -39,8 +39,10 @@ public void embedding(List extractTextResults, CategoryName categor private Document createDocument(HashMap metadata) { return new Document( "a5a7414f-f676-409b-9f2e-1042f9846c97", - "● 등록금 전액 완납 또는 분할납부 1차분을 정해진 기간에 미납할 경우 분할납부 신청은 자동 취소되며, 미납 등록금은 이후\n" + - "추가 등록기간에 전액 납부해야 함.\n", + """ + ● 등록금 전액 완납 또는 분할납부 1차분을 정해진 기간에 미납할 경우 분할납부 신청은 자동 취소되며, + 미납 등록금은 이후 추가 등록기간에 전액 납부해야 함.\n + """, metadata); } diff --git a/src/main/java/com/kustacks/kuring/worker/parser/notice/KuisHomepageNoticeTextParser.java b/src/main/java/com/kustacks/kuring/worker/parser/notice/KuisHomepageNoticeTextParser.java index 4d3dd5ed..384747df 100644 --- a/src/main/java/com/kustacks/kuring/worker/parser/notice/KuisHomepageNoticeTextParser.java +++ b/src/main/java/com/kustacks/kuring/worker/parser/notice/KuisHomepageNoticeTextParser.java @@ -10,7 +10,7 @@ public class KuisHomepageNoticeTextParser extends NoticeTextParserTemplate { @Override protected boolean support(CategoryName categoryName) { - return !(categoryName == CategoryName.LIBRARY); + return categoryName != CategoryName.LIBRARY; } @Override diff --git a/src/test/java/com/kustacks/kuring/notice/adapter/out/persistence/NoticeRepositoryTest.java b/src/test/java/com/kustacks/kuring/notice/adapter/out/persistence/NoticeRepositoryTest.java index 27145626..3d879c8b 100644 --- a/src/test/java/com/kustacks/kuring/notice/adapter/out/persistence/NoticeRepositoryTest.java +++ b/src/test/java/com/kustacks/kuring/notice/adapter/out/persistence/NoticeRepositoryTest.java @@ -40,7 +40,7 @@ class NoticeRepositoryTest extends IntegrationTestSupport { @DisplayName("jdbc를 사용한 bulk insert 테스트") @Test - public void jdbcBulkInsert() { + void jdbcBulkInsert() { // given noticeRepository.deleteAll(); List notices = creatNotices(70); @@ -176,7 +176,7 @@ void findNotYetEmbeddingNotice() { @DisplayName("Embedding 된 공지의 상태를 변경할 수 있다") @Test - public void updateNoticeEmbeddingStatus() { + void updateNoticeEmbeddingStatus() { // given Notice notice1 = new Notice("1", "2024-03-19 17:27:07", "2023-04-03 17:27:05", "notice1", CategoryName.BACHELOR, false, "https://www.example.com"); diff --git a/src/test/java/com/kustacks/kuring/notice/domain/NoticeTest.java b/src/test/java/com/kustacks/kuring/notice/domain/NoticeTest.java index 451c2d46..f164b924 100644 --- a/src/test/java/com/kustacks/kuring/notice/domain/NoticeTest.java +++ b/src/test/java/com/kustacks/kuring/notice/domain/NoticeTest.java @@ -36,7 +36,7 @@ void member_invalid_email_id(String url) { @DisplayName("공지 임베딩 여부 확인 테스트") @Test - public void notice_embedded() { + void notice_embedded() { // given Notice notice = new Notice("artice_id", "2024-01-19 17:27:05", "2024-01-19 17:27:05", "subject", CategoryName.BACHELOR, From bc886215fb83593a83adca823f21126daa9ee2c3 Mon Sep 17 00:00:00 2001 From: jiwoo Date: Sun, 21 Jul 2024 14:52:51 +0900 Subject: [PATCH 22/31] =?UTF-8?q?feat(ChromaVectorStoreAdapter):=20Top-K?= =?UTF-8?q?=20=EB=A5=BC=202=EB=A1=9C=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ai/adapter/out/persistence/ChromaVectorStoreAdapter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/kustacks/kuring/ai/adapter/out/persistence/ChromaVectorStoreAdapter.java b/src/main/java/com/kustacks/kuring/ai/adapter/out/persistence/ChromaVectorStoreAdapter.java index 2278300b..48f1b7ad 100644 --- a/src/main/java/com/kustacks/kuring/ai/adapter/out/persistence/ChromaVectorStoreAdapter.java +++ b/src/main/java/com/kustacks/kuring/ai/adapter/out/persistence/ChromaVectorStoreAdapter.java @@ -22,7 +22,7 @@ @RequiredArgsConstructor public class ChromaVectorStoreAdapter implements QueryVectorStorePort, CommandVectorStorePort { - private static final int TOP_K = 1; + private static final int TOP_K = 2; private final ChromaVectorStore chromaVectorStore; From 4e52373e99dea1f77b18782147befc3e89966025 Mon Sep 17 00:00:00 2001 From: jiwoo Date: Sun, 21 Jul 2024 15:07:20 +0900 Subject: [PATCH 23/31] =?UTF-8?q?feat(User):=20=ED=95=9C=EB=8B=AC=20?= =?UTF-8?q?=EC=A7=88=EB=AC=B8=20=EA=B0=80=EB=8A=A5=20=ED=9A=9F=EC=88=98?= =?UTF-8?q?=EB=A5=BC=203=EB=B2=88=EC=9C=BC=EB=A1=9C=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/java/com/kustacks/kuring/user/domain/User.java | 2 +- .../java/com/kustacks/kuring/acceptance/AiAcceptanceTest.java | 1 + .../user/adapter/out/persistence/UserRepositoryTest.java | 1 + src/test/java/com/kustacks/kuring/user/domain/UserTest.java | 3 ++- 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/kustacks/kuring/user/domain/User.java b/src/main/java/com/kustacks/kuring/user/domain/User.java index c39aabd7..a2ad2d6c 100644 --- a/src/main/java/com/kustacks/kuring/user/domain/User.java +++ b/src/main/java/com/kustacks/kuring/user/domain/User.java @@ -21,7 +21,7 @@ @SQLRestriction("deleted = false") public class User implements Serializable { - private static final int MONTHLY_QUESTION_COUNT = 2; + private static final int MONTHLY_QUESTION_COUNT = 3; @Id @GeneratedValue(strategy = GenerationType.IDENTITY) diff --git a/src/test/java/com/kustacks/kuring/acceptance/AiAcceptanceTest.java b/src/test/java/com/kustacks/kuring/acceptance/AiAcceptanceTest.java index 3b54677e..1fd4dc26 100644 --- a/src/test/java/com/kustacks/kuring/acceptance/AiAcceptanceTest.java +++ b/src/test/java/com/kustacks/kuring/acceptance/AiAcceptanceTest.java @@ -43,6 +43,7 @@ void ask_to_open_ai_overflow_count() { String question = "교내,외 장학금 및 학자금 대출 관련 전화번호들을 안내를 해줘"; 사용자_질문_요청_REST(question, USER_FCM_TOKEN); 사용자_질문_요청_REST(question, USER_FCM_TOKEN); + 사용자_질문_요청_REST(question, USER_FCM_TOKEN); // when var 모델_응답 = 사용자_질문_요청_REST(question, USER_FCM_TOKEN); diff --git a/src/test/java/com/kustacks/kuring/user/adapter/out/persistence/UserRepositoryTest.java b/src/test/java/com/kustacks/kuring/user/adapter/out/persistence/UserRepositoryTest.java index cdb1cfbc..53009ca2 100644 --- a/src/test/java/com/kustacks/kuring/user/adapter/out/persistence/UserRepositoryTest.java +++ b/src/test/java/com/kustacks/kuring/user/adapter/out/persistence/UserRepositoryTest.java @@ -71,6 +71,7 @@ void deleteALl() { void decreaseQuestionCount() { // given User savedUser = userPersistenceAdapter.findByToken(USER_FCM_TOKEN).get(); + savedUser.decreaseQuestionCount(); // 3 -> 2 savedUser.decreaseQuestionCount(); // 2 -> 1 savedUser.decreaseQuestionCount(); // 1 -> 0 em.flush(); diff --git a/src/test/java/com/kustacks/kuring/user/domain/UserTest.java b/src/test/java/com/kustacks/kuring/user/domain/UserTest.java index 81785e65..e54555c5 100644 --- a/src/test/java/com/kustacks/kuring/user/domain/UserTest.java +++ b/src/test/java/com/kustacks/kuring/user/domain/UserTest.java @@ -135,7 +135,7 @@ void decrease_question_count() { int leftCount = user.decreaseQuestionCount(); // then - assertThat(leftCount).isEqualTo(1); + assertThat(leftCount).isEqualTo(2); } @DisplayName("질문 카운트가 0보다 큰 경우에만 질문이 가능하다") @@ -143,6 +143,7 @@ void decrease_question_count() { void is_enough_question_count() { // given User newUser = new User("token_one"); + newUser.decreaseQuestionCount(); // after 2 newUser.decreaseQuestionCount(); // after 1 newUser.decreaseQuestionCount(); // after 0 From 2aee9bfee76912b6104db105c9ef5424dab92b90 Mon Sep 17 00:00:00 2001 From: jiwoo Date: Sun, 21 Jul 2024 17:43:05 +0900 Subject: [PATCH 24/31] =?UTF-8?q?feat(UserUpdater#questionCountReset):=20?= =?UTF-8?q?=EB=A7=A4=EB=8B=AC=20=EB=A7=88=EC=A7=80=EB=A7=89=EB=82=A0=20?= =?UTF-8?q?=EC=82=AC=EC=9A=A9=EC=9E=90=20=EC=A7=88=EB=AC=B8=20=EC=B9=B4?= =?UTF-8?q?=EC=9A=B4=ED=8A=B8=20=EC=B4=88=EA=B8=B0=ED=99=94=20=EC=9E=91?= =?UTF-8?q?=EC=97=85=20=EA=B5=AC=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../persistence/UserPersistenceAdapter.java | 5 +++ .../out/persistence/UserQueryRepository.java | 2 + .../persistence/UserQueryRepositoryImpl.java | 10 +++++ .../application/port/out/UserCommandPort.java | 1 + .../com/kustacks/kuring/user/domain/User.java | 2 +- .../worker/update/user/UserUpdater.java | 7 +++ .../worker/update/user/UserUpdaterTest.java | 45 +++++++++++++++++++ 7 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 src/test/java/com/kustacks/kuring/worker/update/user/UserUpdaterTest.java diff --git a/src/main/java/com/kustacks/kuring/user/adapter/out/persistence/UserPersistenceAdapter.java b/src/main/java/com/kustacks/kuring/user/adapter/out/persistence/UserPersistenceAdapter.java index 86fac6ef..b216ea9c 100644 --- a/src/main/java/com/kustacks/kuring/user/adapter/out/persistence/UserPersistenceAdapter.java +++ b/src/main/java/com/kustacks/kuring/user/adapter/out/persistence/UserPersistenceAdapter.java @@ -57,4 +57,9 @@ public User save(User user) { public void deleteAll(List allInvalidUsers) { userRepository.deleteAll(allInvalidUsers); } + + @Override + public void resetAllUserQuestionCount() { + userRepository.resetAllUserQuestionCount(); + } } diff --git a/src/main/java/com/kustacks/kuring/user/adapter/out/persistence/UserQueryRepository.java b/src/main/java/com/kustacks/kuring/user/adapter/out/persistence/UserQueryRepository.java index 50441654..4c0324e9 100644 --- a/src/main/java/com/kustacks/kuring/user/adapter/out/persistence/UserQueryRepository.java +++ b/src/main/java/com/kustacks/kuring/user/adapter/out/persistence/UserQueryRepository.java @@ -11,4 +11,6 @@ interface UserQueryRepository { List findAllFeedbackByPageRequest(Pageable pageable); List findByPageRequest(Pageable pageable); + + void resetAllUserQuestionCount(); } diff --git a/src/main/java/com/kustacks/kuring/user/adapter/out/persistence/UserQueryRepositoryImpl.java b/src/main/java/com/kustacks/kuring/user/adapter/out/persistence/UserQueryRepositoryImpl.java index 4927fe89..18643644 100644 --- a/src/main/java/com/kustacks/kuring/user/adapter/out/persistence/UserQueryRepositoryImpl.java +++ b/src/main/java/com/kustacks/kuring/user/adapter/out/persistence/UserQueryRepositoryImpl.java @@ -6,11 +6,13 @@ import com.querydsl.jpa.impl.JPAQueryFactory; import lombok.RequiredArgsConstructor; import org.springframework.data.domain.Pageable; +import org.springframework.transaction.annotation.Transactional; import java.util.List; import static com.kustacks.kuring.user.domain.QFeedback.feedback; import static com.kustacks.kuring.user.domain.QUser.user; +import static com.kustacks.kuring.user.domain.User.MONTHLY_QUESTION_COUNT; @RequiredArgsConstructor class UserQueryRepositoryImpl implements UserQueryRepository { @@ -35,4 +37,12 @@ public List findByPageRequest(Pageable pageable) { .limit(pageable.getPageSize()) .fetch(); } + + @Transactional + @Override + public void resetAllUserQuestionCount() { + queryFactory.update(user) + .set(user.questionCount, MONTHLY_QUESTION_COUNT) + .execute(); + } } diff --git a/src/main/java/com/kustacks/kuring/user/application/port/out/UserCommandPort.java b/src/main/java/com/kustacks/kuring/user/application/port/out/UserCommandPort.java index 44a82b0f..43fed83e 100644 --- a/src/main/java/com/kustacks/kuring/user/application/port/out/UserCommandPort.java +++ b/src/main/java/com/kustacks/kuring/user/application/port/out/UserCommandPort.java @@ -7,4 +7,5 @@ public interface UserCommandPort { User save(User user); void deleteAll(List allInvalidUsers); + void resetAllUserQuestionCount(); } diff --git a/src/main/java/com/kustacks/kuring/user/domain/User.java b/src/main/java/com/kustacks/kuring/user/domain/User.java index a2ad2d6c..11d396c8 100644 --- a/src/main/java/com/kustacks/kuring/user/domain/User.java +++ b/src/main/java/com/kustacks/kuring/user/domain/User.java @@ -21,7 +21,7 @@ @SQLRestriction("deleted = false") public class User implements Serializable { - private static final int MONTHLY_QUESTION_COUNT = 3; + public static final int MONTHLY_QUESTION_COUNT = 3; @Id @GeneratedValue(strategy = GenerationType.IDENTITY) diff --git a/src/main/java/com/kustacks/kuring/worker/update/user/UserUpdater.java b/src/main/java/com/kustacks/kuring/worker/update/user/UserUpdater.java index 442cd586..c60405a7 100644 --- a/src/main/java/com/kustacks/kuring/worker/update/user/UserUpdater.java +++ b/src/main/java/com/kustacks/kuring/worker/update/user/UserUpdater.java @@ -25,6 +25,13 @@ public class UserUpdater { private final UserCommandPort userCommandPort; private final UserQueryPort userQueryPort; + @Scheduled(cron = "0 59 23 L * ?") // 매달 마지막날 23:59에 실행 + public void questionCountReset() { + log.info("========== RAG 질문 토큰 초기화 시작 =========="); + userCommandPort.resetAllUserQuestionCount(); + log.info("========== RAG 질문 토큰 초기화 종료 =========="); + } + @Transactional @Scheduled(fixedRate = 30, timeUnit = TimeUnit.DAYS) public void update() { diff --git a/src/test/java/com/kustacks/kuring/worker/update/user/UserUpdaterTest.java b/src/test/java/com/kustacks/kuring/worker/update/user/UserUpdaterTest.java new file mode 100644 index 00000000..87bfa127 --- /dev/null +++ b/src/test/java/com/kustacks/kuring/worker/update/user/UserUpdaterTest.java @@ -0,0 +1,45 @@ +package com.kustacks.kuring.worker.update.user; + +import com.kustacks.kuring.support.IntegrationTestSupport; +import com.kustacks.kuring.user.adapter.out.persistence.UserPersistenceAdapter; +import com.kustacks.kuring.user.domain.User; +import jakarta.persistence.EntityManager; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.transaction.annotation.Transactional; + +import static org.assertj.core.api.Assertions.assertThat; + +class UserUpdaterTest extends IntegrationTestSupport { + + @Autowired + EntityManager em; + + @Autowired + UserPersistenceAdapter userPersistenceAdapter; + + @Autowired + UserUpdater userUpdater; + + @Transactional // em.flush(), clear()를 위해 설정 + @DisplayName("사용자 질문 카운트가 감소해도 매월 초에 초기값으로 다시 설정된다") + @Test + void questionCountReset() { + // given + User savedUser = userPersistenceAdapter.findByToken(USER_FCM_TOKEN).get(); + savedUser.decreaseQuestionCount(); // 3 -> 2 + savedUser.decreaseQuestionCount(); // 2 -> 1 + savedUser.decreaseQuestionCount(); // 1 -> 0 + em.flush(); + em.clear(); + + // when + userUpdater.questionCountReset(); + + // then + User findUser = userPersistenceAdapter.findByToken(USER_FCM_TOKEN).get(); + int leftCount = findUser.decreaseQuestionCount(); + assertThat(leftCount).isEqualTo(User.MONTHLY_QUESTION_COUNT - 1); + } +} From 10488b2ecf0ee7d0c8db014ff9dff1ae64df2beb Mon Sep 17 00:00:00 2001 From: jiwoo Date: Sun, 21 Jul 2024 18:04:10 +0900 Subject: [PATCH 25/31] =?UTF-8?q?feat(UserRegisterNonChainingFilter):=20?= =?UTF-8?q?=EC=82=AC=EC=9A=A9=EC=9E=90=20=EC=A4=91=EB=B3=B5=20=EB=93=B1?= =?UTF-8?q?=EB=A1=9D=20=EC=98=88=EC=99=B8=20=EB=A1=9C=EA=B7=B8=EB=A5=BC=20?= =?UTF-8?q?=EB=82=A8=EA=B8=B0=EB=8F=84=EB=A1=9D=20=EC=B2=98=EB=A6=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../UserRegisterNonChainingFilter.java | 18 +++++++++++++----- .../KuisHomepageNoticeScraperTemplate.java | 2 +- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/kustacks/kuring/auth/interceptor/UserRegisterNonChainingFilter.java b/src/main/java/com/kustacks/kuring/auth/interceptor/UserRegisterNonChainingFilter.java index 94b38742..f8434dfd 100644 --- a/src/main/java/com/kustacks/kuring/auth/interceptor/UserRegisterNonChainingFilter.java +++ b/src/main/java/com/kustacks/kuring/auth/interceptor/UserRegisterNonChainingFilter.java @@ -5,21 +5,24 @@ import com.kustacks.kuring.auth.exception.RegisterException; import com.kustacks.kuring.auth.handler.AuthenticationFailureHandler; import com.kustacks.kuring.auth.handler.AuthenticationSuccessHandler; +import com.kustacks.kuring.common.properties.ServerProperties; import com.kustacks.kuring.message.application.port.in.FirebaseWithUserUseCase; import com.kustacks.kuring.message.application.port.in.dto.UserSubscribeCommand; -import com.kustacks.kuring.common.properties.ServerProperties; import com.kustacks.kuring.user.application.port.out.UserCommandPort; import com.kustacks.kuring.user.domain.User; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.dao.DuplicateKeyException; import org.springframework.web.servlet.HandlerInterceptor; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.servlet.http.HttpServletResponse; import java.io.IOException; import java.util.stream.Collectors; import static com.kustacks.kuring.message.application.service.FirebaseSubscribeService.ALL_DEVICE_SUBSCRIBED_TOPIC; +@Slf4j @RequiredArgsConstructor public class UserRegisterNonChainingFilter implements HandlerInterceptor { @@ -35,7 +38,7 @@ public class UserRegisterNonChainingFilter implements HandlerInterceptor { @Override public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws Exception { try { - if(request.getMethod().equals(REGISTER_HTTP_METHOD)) { + if (request.getMethod().equals(REGISTER_HTTP_METHOD)) { String userFcmToken = convert(request); register(userFcmToken); afterAuthentication(request, response); @@ -50,7 +53,12 @@ public boolean preHandle(HttpServletRequest request, HttpServletResponse respons } private void register(String userFcmToken) { - userCommandPort.save(new User(userFcmToken)); + try { + userCommandPort.save(new User(userFcmToken)); + } catch (DuplicateKeyException e) { // 이미 등록된 사용자에 대한 처리는 필요없다 + log.warn("User already exists: {}", userFcmToken, e); + } + UserSubscribeCommand command = new UserSubscribeCommand( userFcmToken, diff --git a/src/main/java/com/kustacks/kuring/worker/scrap/KuisHomepageNoticeScraperTemplate.java b/src/main/java/com/kustacks/kuring/worker/scrap/KuisHomepageNoticeScraperTemplate.java index 12caea11..c80c5df4 100644 --- a/src/main/java/com/kustacks/kuring/worker/scrap/KuisHomepageNoticeScraperTemplate.java +++ b/src/main/java/com/kustacks/kuring/worker/scrap/KuisHomepageNoticeScraperTemplate.java @@ -103,7 +103,7 @@ private List htmlTextParsingFromScrapingResult( PageTextDto parsedText = noticeInfo.parseText(result.getDocument()); parsedTexts.add(parsedText); } catch (InternalLogicException e) { - log.error("Exception extracting url: {}", result.getViewUrl(), e); + log.warn("Exception extracting url: {}", result.getViewUrl(), e); } } From d8944be81eb1f4cf60c33b3d750ad9461fa572aa Mon Sep 17 00:00:00 2001 From: jiwoo Date: Sun, 21 Jul 2024 20:31:03 +0900 Subject: [PATCH 26/31] =?UTF-8?q?feat(UserUpdater):=20=EC=82=AC=EC=9A=A9?= =?UTF-8?q?=EC=9E=90=20=EC=A0=9C=EA=B1=B0=EC=9E=91=EC=97=85=20=EC=A4=91?= =?UTF-8?q?=EC=A7=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kustacks/kuring/worker/update/user/UserUpdater.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/kustacks/kuring/worker/update/user/UserUpdater.java b/src/main/java/com/kustacks/kuring/worker/update/user/UserUpdater.java index c60405a7..70f2d3aa 100644 --- a/src/main/java/com/kustacks/kuring/worker/update/user/UserUpdater.java +++ b/src/main/java/com/kustacks/kuring/worker/update/user/UserUpdater.java @@ -10,11 +10,9 @@ import org.springframework.data.domain.PageRequest; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; -import org.springframework.transaction.annotation.Transactional; import java.util.LinkedList; import java.util.List; -import java.util.concurrent.TimeUnit; @Slf4j @Component @@ -32,8 +30,9 @@ public void questionCountReset() { log.info("========== RAG 질문 토큰 초기화 종료 =========="); } - @Transactional - @Scheduled(fixedRate = 30, timeUnit = TimeUnit.DAYS) + // 사용자 제거 로직에 일부 오류가 있는것 같다, 정상 사용자가 제거 되었다. + //@Transactional + //@Scheduled(fixedRate = 30, timeUnit = TimeUnit.DAYS) public void update() { log.info("========== 토큰 유효성 필터링 시작 =========="); From 835198278a44397927ecc31a9031d8d8ee2684e4 Mon Sep 17 00:00:00 2001 From: jiwoo Date: Sun, 21 Jul 2024 20:44:32 +0900 Subject: [PATCH 27/31] =?UTF-8?q?setting:=20ai=20max=20token=201000?= =?UTF-8?q?=EC=9C=BC=EB=A1=9C=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/resources/application-local.yml | 2 +- src/main/resources/application-prod.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/application-local.yml b/src/main/resources/application-local.yml index affce7b0..2271eebe 100644 --- a/src/main/resources/application-local.yml +++ b/src/main/resources/application-local.yml @@ -40,7 +40,7 @@ spring: options: model: gpt-3.5-turbo temperature: 0.0 - maxTokens: 300 + maxTokens: 1000 embedding: options: model: text-embedding-3-small diff --git a/src/main/resources/application-prod.yml b/src/main/resources/application-prod.yml index 3e2ada16..a9502420 100644 --- a/src/main/resources/application-prod.yml +++ b/src/main/resources/application-prod.yml @@ -44,7 +44,7 @@ spring: options: model: gpt-3.5-turbo temperature: 0.0 - maxTokens: 300 + maxTokens: 1000 embedding: options: model: text-embedding-3-small From d00db858a8f11bcf9e55df644d3598b70f2ec1b8 Mon Sep 17 00:00:00 2001 From: jiwoo Date: Sun, 21 Jul 2024 21:57:03 +0900 Subject: [PATCH 28/31] =?UTF-8?q?feat(RAGQueryApiV2):=20RAGQueryApi=20?= =?UTF-8?q?=EB=AC=B8=EC=84=9C=ED=99=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kustacks/kuring/ai/adapter/in/web/RAGQueryApiV2.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/kustacks/kuring/ai/adapter/in/web/RAGQueryApiV2.java b/src/main/java/com/kustacks/kuring/ai/adapter/in/web/RAGQueryApiV2.java index 47013ed4..88b059d2 100644 --- a/src/main/java/com/kustacks/kuring/ai/adapter/in/web/RAGQueryApiV2.java +++ b/src/main/java/com/kustacks/kuring/ai/adapter/in/web/RAGQueryApiV2.java @@ -3,7 +3,9 @@ import com.kustacks.kuring.ai.application.port.in.RAGQueryUseCase; import com.kustacks.kuring.common.annotation.RestWebAdapter; import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.security.SecurityRequirement; +import io.swagger.v3.oas.annotations.tags.Tag; import lombok.RequiredArgsConstructor; import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.GetMapping; @@ -11,6 +13,7 @@ import org.springframework.web.bind.annotation.RequestParam; import reactor.core.publisher.Flux; +@Tag(name = "AI-Query", description = "AI Assistant") @RequiredArgsConstructor @RestWebAdapter(path = "/api/v2/ai/messages") public class RAGQueryApiV2 { @@ -20,10 +23,10 @@ public class RAGQueryApiV2 { private final RAGQueryUseCase ragQueryUseCase; @Operation(summary = "사용자 AI에 질문요청", description = "사용자가 궁금한 학교 정보를 AI에게 질문합니다.") - @SecurityRequirement(name = "User-Token") + @SecurityRequirement(name = USER_TOKEN_HEADER_KEY) @GetMapping(produces = MediaType.TEXT_EVENT_STREAM_VALUE) public Flux askAIQuery( - @RequestParam("question") String question, + @Parameter(description = "사용자 질문") @RequestParam("question") String question, @RequestHeader(USER_TOKEN_HEADER_KEY) String id ) { return ragQueryUseCase.askAiModel(question, id); From 736e835d5d262ed4200024dcfe861e61f08c3f8c Mon Sep 17 00:00:00 2001 From: jiwoo Date: Mon, 22 Jul 2024 19:39:18 +0900 Subject: [PATCH 29/31] =?UTF-8?q?refactor:=20SecurityRequirement=EC=97=90?= =?UTF-8?q?=EC=84=9C=20=EC=83=81=EC=88=98=EB=A5=BC=20=EC=82=AC=EC=9A=A9?= =?UTF-8?q?=ED=95=98=EB=8F=84=EB=A1=9D=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../user/adapter/in/web/UserCommandApiV2.java | 15 ++++++--------- .../user/adapter/in/web/UserQueryApiV2.java | 15 +++++++-------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/src/main/java/com/kustacks/kuring/user/adapter/in/web/UserCommandApiV2.java b/src/main/java/com/kustacks/kuring/user/adapter/in/web/UserCommandApiV2.java index d71a586d..fb7ea8d1 100644 --- a/src/main/java/com/kustacks/kuring/user/adapter/in/web/UserCommandApiV2.java +++ b/src/main/java/com/kustacks/kuring/user/adapter/in/web/UserCommandApiV2.java @@ -1,10 +1,5 @@ package com.kustacks.kuring.user.adapter.in.web; -import static com.kustacks.kuring.common.dto.ResponseCodeAndMessages.BOOKMAKR_SAVE_SUCCESS; -import static com.kustacks.kuring.common.dto.ResponseCodeAndMessages.CATEGORY_SUBSCRIBE_SUCCESS; -import static com.kustacks.kuring.common.dto.ResponseCodeAndMessages.DEPARTMENTS_SUBSCRIBE_SUCCESS; -import static com.kustacks.kuring.common.dto.ResponseCodeAndMessages.FEEDBACK_SAVE_SUCCESS; - import com.kustacks.kuring.common.annotation.RestWebAdapter; import com.kustacks.kuring.common.dto.BaseResponse; import com.kustacks.kuring.user.adapter.in.web.dto.UserBookmarkRequest; @@ -29,6 +24,8 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestHeader; +import static com.kustacks.kuring.common.dto.ResponseCodeAndMessages.*; + @Tag(name = "User-Command", description = "사용자가 주체가 되는 정보 수정") @Slf4j @Validated @@ -41,7 +38,7 @@ class UserCommandApiV2 { private final UserCommandUseCase userCommandUseCase; @Operation(summary = "사용자 카테고리 수정", description = "사용자가 구독한 카테고리 목록을 추가, 삭제 합니다") - @SecurityRequirement(name = "User-Token") + @SecurityRequirement(name = USER_TOKEN_HEADER_KEY) @PostMapping(value = "/subscriptions/categories", consumes = MediaType.APPLICATION_JSON_VALUE) public ResponseEntity> editUserSubscribeCategories( @Valid @RequestBody UserCategoriesSubscribeRequest request, @@ -52,7 +49,7 @@ public ResponseEntity> editUserSubscribeCategories( } @Operation(summary = "사용자 학과 수정", description = "사용자가 구독한 학과 목록을 추가, 삭제 합니다") - @SecurityRequirement(name = "User-Token") + @SecurityRequirement(name = USER_TOKEN_HEADER_KEY) @PostMapping(value = "/subscriptions/departments", consumes = MediaType.APPLICATION_JSON_VALUE) public ResponseEntity> editUserSubscribeDepartments( @Valid @RequestBody UserDepartmentsSubscribeRequest request, @@ -63,7 +60,7 @@ public ResponseEntity> editUserSubscribeDepartments( } @Operation(summary = "사용자 피드백 작성", description = "사용자가 피드백을 작성하여 저장합니다") - @SecurityRequirement(name = "User-Token") + @SecurityRequirement(name = USER_TOKEN_HEADER_KEY) @PostMapping("/feedbacks") public ResponseEntity> saveFeedback( @Valid @RequestBody UserFeedbackRequest request, @@ -74,7 +71,7 @@ public ResponseEntity> saveFeedback( } @Operation(summary = "사용자 북마크 작성", description = "사용자가 원하는 공지를 북마크 하여 저장합니다") - @SecurityRequirement(name = "User-Token") + @SecurityRequirement(name = USER_TOKEN_HEADER_KEY) @PostMapping("/bookmarks") public ResponseEntity> saveBookmark( @Valid @RequestBody UserBookmarkRequest request, diff --git a/src/main/java/com/kustacks/kuring/user/adapter/in/web/UserQueryApiV2.java b/src/main/java/com/kustacks/kuring/user/adapter/in/web/UserQueryApiV2.java index e7993daf..0c1f7453 100644 --- a/src/main/java/com/kustacks/kuring/user/adapter/in/web/UserQueryApiV2.java +++ b/src/main/java/com/kustacks/kuring/user/adapter/in/web/UserQueryApiV2.java @@ -1,9 +1,5 @@ package com.kustacks.kuring.user.adapter.in.web; -import static com.kustacks.kuring.common.dto.ResponseCodeAndMessages.BOOKMARK_LOOKUP_SUCCESS; -import static com.kustacks.kuring.common.dto.ResponseCodeAndMessages.CATEGORY_USER_SUBSCRIBES_LOOKUP_SUCCESS; -import static com.kustacks.kuring.common.dto.ResponseCodeAndMessages.DEPARTMENTS_USER_SUBSCRIBES_LOOKUP_SUCCESS; - import com.kustacks.kuring.common.annotation.RestWebAdapter; import com.kustacks.kuring.common.dto.BaseResponse; import com.kustacks.kuring.user.adapter.in.web.dto.UserBookmarkResponse; @@ -13,7 +9,6 @@ import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.security.SecurityRequirement; import io.swagger.v3.oas.annotations.tags.Tag; -import java.util.List; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.http.ResponseEntity; @@ -21,6 +16,10 @@ import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestHeader; +import java.util.List; + +import static com.kustacks.kuring.common.dto.ResponseCodeAndMessages.*; + @Tag(name = "User-Query", description = "사용자가 주체가 되는 정보 조회") @Slf4j @Validated @@ -33,7 +32,7 @@ class UserQueryApiV2 { private final UserQueryUseCase userQueryUseCase; @Operation(summary = "사용자 카테고리 조회", description = "사용자가 구독한 카테고리 목록을 조회합니다") - @SecurityRequirement(name = "User-Token") + @SecurityRequirement(name = USER_TOKEN_HEADER_KEY) @GetMapping("/subscriptions/categories") public ResponseEntity>> lookupUserSubscribeCategories( @RequestHeader(USER_TOKEN_HEADER_KEY) String userToken @@ -47,7 +46,7 @@ public ResponseEntity>> lookupUserSu } @Operation(summary = "사용자 학과 조회", description = "사용자가 구독한 학과의 목록을 조회합니다") - @SecurityRequirement(name = "User-Token") + @SecurityRequirement(name = USER_TOKEN_HEADER_KEY) @GetMapping("/subscriptions/departments") public ResponseEntity>> lookupUserSubscribeDepartments( @RequestHeader(USER_TOKEN_HEADER_KEY) String userToken @@ -61,7 +60,7 @@ public ResponseEntity>> lookupUser } @Operation(summary = "사용자 북마크 조회", description = "사용자가 북마크한 공지의 목록을 조회합니다") - @SecurityRequirement(name = "User-Token") + @SecurityRequirement(name = USER_TOKEN_HEADER_KEY) @GetMapping("/bookmarks") public ResponseEntity>> lookupUserBookmarks( @RequestHeader(USER_TOKEN_HEADER_KEY) String userToken From 645cb7793c6b34b1c18aa8106bbc393f00c3d5a3 Mon Sep 17 00:00:00 2001 From: jiwoo Date: Mon, 22 Jul 2024 19:56:10 +0900 Subject: [PATCH 30/31] =?UTF-8?q?feat(User):=20=EC=82=AC=EC=9A=A9=EC=9E=90?= =?UTF-8?q?=20=EC=A7=88=EB=AC=B8=20=ED=9A=9F=EC=88=98=202=EB=A1=9C=20?= =?UTF-8?q?=EC=A0=9C=ED=95=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/java/com/kustacks/kuring/user/domain/User.java | 2 +- .../user/adapter/out/persistence/UserRepositoryTest.java | 1 - src/test/java/com/kustacks/kuring/user/domain/UserTest.java | 3 +-- .../kustacks/kuring/worker/update/user/UserUpdaterTest.java | 1 - 4 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/kustacks/kuring/user/domain/User.java b/src/main/java/com/kustacks/kuring/user/domain/User.java index 11d396c8..e704e2c0 100644 --- a/src/main/java/com/kustacks/kuring/user/domain/User.java +++ b/src/main/java/com/kustacks/kuring/user/domain/User.java @@ -21,7 +21,7 @@ @SQLRestriction("deleted = false") public class User implements Serializable { - public static final int MONTHLY_QUESTION_COUNT = 3; + public static final int MONTHLY_QUESTION_COUNT = 2; @Id @GeneratedValue(strategy = GenerationType.IDENTITY) diff --git a/src/test/java/com/kustacks/kuring/user/adapter/out/persistence/UserRepositoryTest.java b/src/test/java/com/kustacks/kuring/user/adapter/out/persistence/UserRepositoryTest.java index 53009ca2..cdb1cfbc 100644 --- a/src/test/java/com/kustacks/kuring/user/adapter/out/persistence/UserRepositoryTest.java +++ b/src/test/java/com/kustacks/kuring/user/adapter/out/persistence/UserRepositoryTest.java @@ -71,7 +71,6 @@ void deleteALl() { void decreaseQuestionCount() { // given User savedUser = userPersistenceAdapter.findByToken(USER_FCM_TOKEN).get(); - savedUser.decreaseQuestionCount(); // 3 -> 2 savedUser.decreaseQuestionCount(); // 2 -> 1 savedUser.decreaseQuestionCount(); // 1 -> 0 em.flush(); diff --git a/src/test/java/com/kustacks/kuring/user/domain/UserTest.java b/src/test/java/com/kustacks/kuring/user/domain/UserTest.java index e54555c5..81785e65 100644 --- a/src/test/java/com/kustacks/kuring/user/domain/UserTest.java +++ b/src/test/java/com/kustacks/kuring/user/domain/UserTest.java @@ -135,7 +135,7 @@ void decrease_question_count() { int leftCount = user.decreaseQuestionCount(); // then - assertThat(leftCount).isEqualTo(2); + assertThat(leftCount).isEqualTo(1); } @DisplayName("질문 카운트가 0보다 큰 경우에만 질문이 가능하다") @@ -143,7 +143,6 @@ void decrease_question_count() { void is_enough_question_count() { // given User newUser = new User("token_one"); - newUser.decreaseQuestionCount(); // after 2 newUser.decreaseQuestionCount(); // after 1 newUser.decreaseQuestionCount(); // after 0 diff --git a/src/test/java/com/kustacks/kuring/worker/update/user/UserUpdaterTest.java b/src/test/java/com/kustacks/kuring/worker/update/user/UserUpdaterTest.java index 87bfa127..7d40f698 100644 --- a/src/test/java/com/kustacks/kuring/worker/update/user/UserUpdaterTest.java +++ b/src/test/java/com/kustacks/kuring/worker/update/user/UserUpdaterTest.java @@ -28,7 +28,6 @@ class UserUpdaterTest extends IntegrationTestSupport { void questionCountReset() { // given User savedUser = userPersistenceAdapter.findByToken(USER_FCM_TOKEN).get(); - savedUser.decreaseQuestionCount(); // 3 -> 2 savedUser.decreaseQuestionCount(); // 2 -> 1 savedUser.decreaseQuestionCount(); // 1 -> 0 em.flush(); From a508dd5db40a9cff09083ee3ba2906dd1293df90 Mon Sep 17 00:00:00 2001 From: jiwoo Date: Mon, 22 Jul 2024 20:38:47 +0900 Subject: [PATCH 31/31] =?UTF-8?q?feat(RAGConfiguration):=20=ED=85=8C?= =?UTF-8?q?=EC=8A=A4=ED=8A=B8=EC=9A=A9=20ChromaVectorStore=EB=A5=BC=20prod?= =?UTF-8?q?=20=ED=99=98=EA=B2=BD=EC=97=90=EC=84=9C=EB=8F=84=20bean?= =?UTF-8?q?=EC=9C=BC=EB=A1=9C=20=EB=93=B1=EB=A1=9D=ED=95=98=EB=8F=84?= =?UTF-8?q?=EB=A1=9D=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/kustacks/kuring/config/RAGConfiguration.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/kustacks/kuring/config/RAGConfiguration.java b/src/main/java/com/kustacks/kuring/config/RAGConfiguration.java index 1c3eca35..7fa958f4 100644 --- a/src/main/java/com/kustacks/kuring/config/RAGConfiguration.java +++ b/src/main/java/com/kustacks/kuring/config/RAGConfiguration.java @@ -30,14 +30,14 @@ public class RAGConfiguration { @Value("vectorstore.json") private String vectorStoreName; - @Profile("test") + @Profile("dev | test") @Bean public ChromaApi chromaApi(RestTemplate restTemplate) { String chromaUrl = "http://127.0.0.1:8000"; return new ChromaApi(chromaUrl, restTemplate); } - @Profile("test") + @Profile("dev | test") @Bean public ChromaVectorStore chromaVectorStore(EmbeddingModel embeddingModel, ChromaApi chromaApi) { return new ChromaVectorStore(embeddingModel, chromaApi, false);