From 76ede1b952d2081b306051f1683b2e3fda9e3383 Mon Sep 17 00:00:00 2001 From: dvince Date: Fri, 18 Aug 2023 11:48:01 -0400 Subject: [PATCH 1/3] task(hmi): Implement new code asset in back end Adding a new code type asset in the back end, and hooking the front up as-is. This right now is very similar to the old code artifact. --- .../client/hmi-client/src/services/code.ts | 143 ++++++++++++++++++ .../models/dataservice/code/Code.java | 69 +++++++++ 2 files changed, 212 insertions(+) create mode 100644 packages/client/hmi-client/src/services/code.ts create mode 100644 packages/services/hmi-server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/code/Code.java diff --git a/packages/client/hmi-client/src/services/code.ts b/packages/client/hmi-client/src/services/code.ts new file mode 100644 index 0000000000..21dff3b92e --- /dev/null +++ b/packages/client/hmi-client/src/services/code.ts @@ -0,0 +1,143 @@ +import API from '@/api/api'; +import { AssetType, Code, ProgrammingLanguage } from '@/types/Types'; +import { Ref } from 'vue'; +import { addAsset } from '@/services/project'; +import { logger } from '@/utils/logger'; + +async function getCodeFileAsText(codeAssetId: string, fileName: string): Promise { + const response = await API.get( + `/code-asset/${codeAssetId}/download-code-as-text?filename=${fileName}`, + {} + ); + + if (!response || response.status >= 400) { + logger.error('Error getting code file as text'); + return null; + } + + return response.data; +} + +async function uploadCodeToProject( + projectId: string, + file: File, + progress: Ref +): Promise { + // Create a new code asset with the same name as the file and post the metadata to TDS + const codeAsset: Code = { + name: file.name, + description: file.name, + filename: file.name, + language: getProgrammingLanguage(file.name) + }; + + const newCodeAsset: Code | null = await createNewCodeAsset(codeAsset); + if (!newCodeAsset || !newCodeAsset.id) return null; + + const successfulUpload = await addFileToCodeAsset(newCodeAsset.id, file, progress); + if (!successfulUpload) return null; + + const resp = addAsset(projectId, AssetType.Code, newCodeAsset.id); + if (!resp) return null; + + return newCodeAsset; +} + +/** + * This is a helper function to take an arbitrary file from a github repo and create a new code asset from it + * @param repoOwnerAndName + * @param path + * @param userName + * @param projectId + */ +async function uploadCodeToProjectFromGithub( + repoOwnerAndName: string, + path: string, + userName: string, + projectId: string, + url: string +): Promise { + // Find the file name by removing the path portion + const fileName: string | undefined = path.split('/').pop(); + + if (!fileName) return null; + + // Create a new code asset with the same name as the file and post the metadata to TDS + const codeAsset: Code = { + name: file.name, + description: file.name, + filename: file.name, + language: getProgrammingLanguage(file.name), + repoUrl: url + }; + + const newCode: Code | null = await createNewCodeAsset(codeAsset); + if (!newCode || !newCode.id) return null; + + const urlResponse = await API.put( + `/code-asset/${newCode.id}/uploadCodeFromGithub?filename=${fileName}&path=${path}&repoOwnerAndName=${repoOwnerAndName}`, + { + timeout: 30000 + } + ); + + if (!urlResponse || urlResponse.status >= 400) { + logger.error(`Failed to upload code from github: ${urlResponse}`); + return null; + } + + const resp = addAsset(projectId, AssetType.Code, newCodeAsset.id); + + if (!resp) return null; + + return newCode; +} + +async function createNewCodeAsset(codeAsset: Code): Promise { + const response = await API.post('/code-asset', codeAsset); + if (!response || response.status >= 400) return null; + return response.data; +} + +async function addFileToCodeAsset( + codeAssetId: string, + file: File, + progress: Ref +): Promise { + const formData = new FormData(); + formData.append('file', file); + + const response = await API.put(`/code-asset/${codeAssetId}/uploadFile`, formData, { + params: { + filename: file.name + }, + headers: { + 'Content-Type': 'multipart/form-data' + }, + onUploadProgress(progressEvent) { + progress.value = Math.min( + 100, + Math.round((progressEvent.loaded * 100) / (progressEvent?.total ?? 100)) + ); + } + }); + + return response && response.status < 400; +} + +function getProgrammingLanguage(fileName: string): ProgrammingLanguage { + // given the extension of a file, return the programming language + const fileExtensiopn: string = fileName.split('.').pop() || ''; + switch (fileExtensiopn) { + case 'py': + return ProgrammingLanguage.Python; + case 'jl': + return ProgrammingLanguage.Julia; + case 'r': + return ProgrammingLanguage.R; + default: + return ProgrammingLanguage.Python; // TODO do we need an "unknown" language? + } +} + +export { uploadCodeToProject, getCodeFileAsText, uploadCodeToProjectFromGithub }; diff --git a/packages/services/hmi-server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/code/Code.java b/packages/services/hmi-server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/code/Code.java new file mode 100644 index 0000000000..d14941f652 --- /dev/null +++ b/packages/services/hmi-server/src/main/java/software/uncharted/terarium/hmiserver/models/dataservice/code/Code.java @@ -0,0 +1,69 @@ +package software.uncharted.terarium.hmiserver.models.dataservice.code; + +import com.fasterxml.jackson.annotation.JsonAlias; +import com.fasterxml.jackson.annotation.JsonValue; +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import lombok.experimental.Accessors; +import software.uncharted.terarium.hmiserver.annotations.TSModel; +import software.uncharted.terarium.hmiserver.annotations.TSOptional; + +import java.time.Instant; + +@Data +@Accessors(chain = true) +@TSModel +public class Code { + + /* The id of the code. */ + @TSOptional + private String id; + + /* Timestamp of creation */ + @TSOptional + private Instant timestamp; + + /* The name of the code. */ + private String name; + + /* The description of the code. */ + private String description; + + /* The name of the file in this code*/ + private String filename; + + /* The optional URL for where this code came from */ + @TSOptional + @JsonAlias("repo_url") + private String repoUrl; + + /* The programming language of this code */ + private ProgrammingLanguage language; + + /* The optional metadata for this code */ + @TSOptional + private JsonNode metadata; + + + public enum ProgrammingLanguage { + PYTHON("python"), + R("r"), + Julia("julia"); + + public final String language; + + ProgrammingLanguage(final String language) { + this.language = language; + } + + @Override + @JsonValue + public String toString() { + return language; + } + + public static ProgrammingLanguage fromString(final String language) { + return ProgrammingLanguage.valueOf(language.toUpperCase()); + } + } +} From 79c1bfe46817972752fafd1b9b687768fafbf372 Mon Sep 17 00:00:00 2001 From: dvince Date: Fri, 18 Aug 2023 11:49:14 -0400 Subject: [PATCH 2/3] task(hmi): Implement new code asset in back end Adding a new code type asset in the back end, and hooking the front up as-is. This right now is very similar to the old code artifact. --- .../extracting/tera-drag-n-drop-importer.vue | 4 - .../widgets/tera-import-github-file.vue | 12 +- .../src/components/widgets/tera-tab-group.vue | 4 +- .../components/tera-project-overview.vue | 102 ++++++---- .../project/components/tera-project-page.vue | 41 ++-- .../components/tera-resource-sidebar.vue | 6 +- .../src/page/project/tera-project.vue | 14 +- .../client/hmi-client/src/router/routes.ts | 4 +- .../client/hmi-client/src/services/project.ts | 2 +- .../client/hmi-client/src/types/Project.ts | 9 +- packages/client/hmi-client/src/types/Types.ts | 19 ++ .../client/hmi-client/src/types/common.ts | 4 - .../hmiserver/models/code/GithubRepo.java | 2 +- .../models/dataservice/Artifact.java | 2 +- .../hmiserver/models/dataservice/Assets.java | 6 +- .../proxies/dataservice/CodeProxy.java | 56 ++++++ .../resources/dataservice/CodeResource.java | 187 ++++++++++++++++++ 17 files changed, 383 insertions(+), 91 deletions(-) create mode 100644 packages/services/hmi-server/src/main/java/software/uncharted/terarium/hmiserver/proxies/dataservice/CodeProxy.java create mode 100644 packages/services/hmi-server/src/main/java/software/uncharted/terarium/hmiserver/resources/dataservice/CodeResource.java diff --git a/packages/client/hmi-client/src/components/extracting/tera-drag-n-drop-importer.vue b/packages/client/hmi-client/src/components/extracting/tera-drag-n-drop-importer.vue index f371378697..3c4dff3e8b 100644 --- a/packages/client/hmi-client/src/components/extracting/tera-drag-n-drop-importer.vue +++ b/packages/client/hmi-client/src/components/extracting/tera-drag-n-drop-importer.vue @@ -85,8 +85,6 @@ const props = defineProps({ AcceptedTypes.TXT, AcceptedTypes.MD, AcceptedTypes.PY, - AcceptedTypes.M, - AcceptedTypes.JS, AcceptedTypes.R, AcceptedTypes.JL ].every((v) => value.includes(v)) @@ -101,8 +99,6 @@ const props = defineProps({ AcceptedExtensions.TXT, AcceptedExtensions.MD, AcceptedExtensions.PY, - AcceptedExtensions.M, - AcceptedExtensions.JS, AcceptedExtensions.R, AcceptedExtensions.JL ].every((v) => value.includes(v)) diff --git a/packages/client/hmi-client/src/components/widgets/tera-import-github-file.vue b/packages/client/hmi-client/src/components/widgets/tera-import-github-file.vue index 933437fac3..548a12988e 100644 --- a/packages/client/hmi-client/src/components/widgets/tera-import-github-file.vue +++ b/packages/client/hmi-client/src/components/widgets/tera-import-github-file.vue @@ -192,6 +192,7 @@ import { createNewDatasetFromGithubFile } from '@/services/dataset'; import { createNewArtifactFromGithubFile } from '@/services/artifact'; import { extractPDF } from '@/services/models/extractions'; import useAuthStore from '@/stores/auth'; +import { uploadCodeToProjectFromGithub } from '@/services/code'; const props = defineProps<{ urlString: string; @@ -362,8 +363,15 @@ async function importDocumentFiles(githubFiles: GithubFile[]) { * @param githubFiles The code files to open */ async function openCodeFiles(githubFiles: GithubFile[]) { - // For now just throw to the document path as they're all artifacts - await importDocumentFiles(githubFiles); + githubFiles.forEach(async (githubFile) => { + await uploadCodeToProjectFromGithub( + repoOwnerAndName.value, + githubFile.path, + props.project?.username ?? '', + props.project?.id ?? '', + githubFile.htmlUrl + ); + }); } diff --git a/packages/client/hmi-client/src/components/widgets/tera-tab-group.vue b/packages/client/hmi-client/src/components/widgets/tera-tab-group.vue index e57c9a48c7..446d2138e8 100644 --- a/packages/client/hmi-client/src/components/widgets/tera-tab-group.vue +++ b/packages/client/hmi-client/src/components/widgets/tera-tab-group.vue @@ -60,7 +60,6 @@ const loadingTabIndex = ref(); const getTabName = (tab: Tab) => { if (tab.assetName) return tab.assetName; if (tab.pageType === ProjectPages.OVERVIEW) return 'Overview'; - // DVINCE TODO if (tab.pageType === AssetType.CODE) return 'New File'; const assets = resourceStore.activeProjectAssets; if (assets) { @@ -71,6 +70,9 @@ const getTabName = (tab: Tab) => { // https://github.com/DARPA-ASKEM/data-service/issues/299 return (tab.pageType === AssetType.Publications ? asset?.title : asset?.name) ?? 'n/a'; } + + if (tab.pageType === AssetType.Code) return 'New File'; + return 'n/a'; }; diff --git a/packages/client/hmi-client/src/page/project/components/tera-project-overview.vue b/packages/client/hmi-client/src/page/project/components/tera-project-overview.vue index dcd8bb9855..f3c4bd48e6 100644 --- a/packages/client/hmi-client/src/page/project/components/tera-project-overview.vue +++ b/packages/client/hmi-client/src/page/project/components/tera-project-overview.vue @@ -184,8 +184,6 @@ AcceptedTypes.TXT, AcceptedTypes.MD, AcceptedTypes.PY, - AcceptedTypes.JS, - AcceptedTypes.M, AcceptedTypes.R, AcceptedTypes.JL ]" @@ -195,8 +193,6 @@ AcceptedExtensions.TXT, AcceptedExtensions.MD, AcceptedExtensions.PY, - AcceptedExtensions.M, - AcceptedExtensions.JS, AcceptedExtensions.R, AcceptedExtensions.JL ]" @@ -283,6 +279,7 @@ import TeraMultiSelectModal from '@/components/widgets/tera-multi-select-modal.v import { useTabStore } from '@/stores/tabs'; import { extractPDF } from '@/services/models/extractions'; import useAuthStore from '@/stores/auth'; +import { uploadCodeToProject } from '@/services/code'; const props = defineProps<{ project: IProject; @@ -351,40 +348,77 @@ const assets = computed(() => { async function processFiles(files: File[], csvDescription: string) { return files.map(async (file) => { - if (file.type === AcceptedTypes.CSV) { - const addedCSV: CsvAsset | null = await createNewDatasetFromCSV( - progress, - file, - auth.name ?? '', - props.project.id, - csvDescription - ); - - if (addedCSV !== null) { - const text: string = addedCSV?.csv?.join('\r\n') ?? ''; - const images = []; - - return { file, error: false, response: { text, images } }; - } - return { file, error: true, response: { text: '', images: [] } }; + switch (file.type) { + case AcceptedTypes.CSV: + return processDataset(file, csvDescription); + case AcceptedTypes.PDF: + case AcceptedTypes.TXT: + case AcceptedTypes.MD: + return processArtifact(file); + case AcceptedTypes.PY: + case AcceptedTypes.R: + case AcceptedTypes.JL: + return processCode(file); + default: + return { file, error: true, response: { text: '', images: [] } }; } - - // This is pdf, txt, md files - const artifact: Artifact | null = await uploadArtifactToProject( - progress, - file, - props.project.username ?? '', - props.project.id, - '' - ); - if (artifact && file.name.toLowerCase().endsWith('.pdf')) { - extractPDF(artifact); - return { file, error: false, response: { text: '', images: [] } }; - } - return { file, error: true, response: { text: '', images: [] } }; }); } +/** + * Process a python, R or Julia file into a code asset + * @param file + */ +async function processCode(file: File) { + // This is pdf, txt, md files + await uploadCodeToProject(props.project.id, file, progress); + + return { file, error: true, response: { text: '', images: [] } }; +} + +/** + * Process a pdf, txt, md file into an artifact + * @param file + */ +async function processArtifact(file: File) { + // This is pdf, txt, md files + const artifact: Artifact | null = await uploadArtifactToProject( + progress, + file, + props.project.username ?? '', + props.project.id, + '' + ); + if (artifact && file.name.toLowerCase().endsWith('.pdf')) { + await extractPDF(artifact); + return { file, error: false, response: { text: '', images: [] } }; + } + return { file, error: true, response: { text: '', images: [] } }; +} + +/** + * Process a csv file into a dataset + * @param file + * @param description + */ +async function processDataset(file: File, description: string) { + const addedCSV: CsvAsset | null = await createNewDatasetFromCSV( + progress, + file, + auth.name ?? '', + props.project.id, + description + ); + + if (addedCSV !== null) { + const text: string = addedCSV?.csv?.join('\r\n') ?? ''; + const images = []; + + return { file, error: false, response: { text, images } }; + } + return { file, error: true, response: { text: '', images: [] } }; +} + const onRowSelect = (selectedRows) => { // show multi select modal when there are selectedRows otherwise hide showMultiSelect.value = selectedRows.length !== 0; diff --git a/packages/client/hmi-client/src/page/project/components/tera-project-page.vue b/packages/client/hmi-client/src/page/project/components/tera-project-page.vue index 3b2f76eeb4..dc66c04cd0 100644 --- a/packages/client/hmi-client/src/page/project/components/tera-project-page.vue +++ b/packages/client/hmi-client/src/page/project/components/tera-project-page.vue @@ -5,15 +5,15 @@ :project="project" @asset-loaded="emit('asset-loaded')" /> - + />