From b094b50bff635504fa46427a6a548cd9d2e13993 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Sza=C5=82owski?= Date: Wed, 24 Apr 2024 14:10:42 +0200 Subject: [PATCH] fix missing ga data --- .../src/consts/externalDataModalConfig.ts | 1 + .../queries/useGetProposalsInfiniteQuery.ts | 9 ++++++-- .../src/hooks/queries/useGetProposalsQuery.ts | 5 ++-- govtool/frontend/src/i18n/locales/en.ts | 2 +- .../frontend/src/models/metadataValidation.ts | 2 ++ govtool/frontend/src/services/API.ts | 3 +++ .../src/services/requests/getDRepVotes.ts | 9 ++++++-- .../src/services/requests/getProposal.ts | 6 ++--- ...getMetadataDataMissingStatusTranslation.ts | 1 + ...tadataDataMissingStatusTranslation.test.ts | 2 +- .../utils/tests/validateMetadataHash.test.ts | 14 +++++++---- .../src/utils/validateMetadataHash.ts | 17 ++++++++++---- .../metadata-validation/src/app.service.ts | 7 ++++-- .../src/schemas/cipStandardSchema.ts | 10 +++++--- .../src/types/validateMetadata.ts | 1 + .../metadata-validation/src/utils/index.ts | 1 + .../src/utils/parseMetadata.ts | 23 +++++++++++++++++++ 17 files changed, 87 insertions(+), 26 deletions(-) create mode 100644 govtool/metadata-validation/src/utils/parseMetadata.ts diff --git a/govtool/frontend/src/consts/externalDataModalConfig.ts b/govtool/frontend/src/consts/externalDataModalConfig.ts index 132b20103..36dd10836 100644 --- a/govtool/frontend/src/consts/externalDataModalConfig.ts +++ b/govtool/frontend/src/consts/externalDataModalConfig.ts @@ -36,6 +36,7 @@ export const storageInformationErrorModals: Record< >["state"] > = { [MetadataValidationStatus.URL_NOT_FOUND]: urlCannotBeFound, + [MetadataValidationStatus.INCORRECT_FORMAT]: externalDataDoesntMatchModal, [MetadataValidationStatus.INVALID_JSONLD]: externalDataDoesntMatchModal, [MetadataValidationStatus.INVALID_HASH]: externalDataDoesntMatchModal, }; diff --git a/govtool/frontend/src/hooks/queries/useGetProposalsInfiniteQuery.ts b/govtool/frontend/src/hooks/queries/useGetProposalsInfiniteQuery.ts index 7bde94cee..3bae61e10 100644 --- a/govtool/frontend/src/hooks/queries/useGetProposalsInfiniteQuery.ts +++ b/govtool/frontend/src/hooks/queries/useGetProposalsInfiniteQuery.ts @@ -24,11 +24,16 @@ export const useGetProposalsInfiniteQuery = ({ }); const mappedElements = await Promise.all( data.elements.map(async (proposal: ActionType) => { - const isDataMissing = await checkIsMissingGAMetadata({ + const { metadata, status } = await checkIsMissingGAMetadata({ hash: proposal?.metadataHash ?? "", url: proposal?.url ?? "", }); - return { ...proposal, isDataMissing }; + // workaround for the missing data in db-sync + return { + ...proposal, + ...metadata, + isDataMissing: status || false, + }; }), ); diff --git a/govtool/frontend/src/hooks/queries/useGetProposalsQuery.ts b/govtool/frontend/src/hooks/queries/useGetProposalsQuery.ts index 2bcb4f3c5..e6b0d6a38 100644 --- a/govtool/frontend/src/hooks/queries/useGetProposalsQuery.ts +++ b/govtool/frontend/src/hooks/queries/useGetProposalsQuery.ts @@ -33,11 +33,12 @@ export const useGetProposalsQuery = ({ allProposals .flatMap((proposal) => proposal.elements) .map(async (proposal) => { - const isDataMissing = await checkIsMissingGAMetadata({ + const { metadata, status } = await checkIsMissingGAMetadata({ hash: proposal?.metadataHash ?? "", url: proposal?.url ?? "", }); - return { ...proposal, isDataMissing }; + // workaround for the missing data in db-sync + return { ...proposal, ...metadata, isDataMissing: status || false }; }), ); diff --git a/govtool/frontend/src/i18n/locales/en.ts b/govtool/frontend/src/i18n/locales/en.ts index 979d0f6ca..e59acaa2a 100644 --- a/govtool/frontend/src/i18n/locales/en.ts +++ b/govtool/frontend/src/i18n/locales/en.ts @@ -751,7 +751,7 @@ export const en = { dataMissingErrors: { dataMissing: "Data Missing", notVerifiable: "Not Verifiable", - incorrectFormat: "Incorrect Format", + incorrectFormat: "Data Formatted Incorrectly", }, about: "About", abstain: "Abstain", diff --git a/govtool/frontend/src/models/metadataValidation.ts b/govtool/frontend/src/models/metadataValidation.ts index 00910e9f5..bb4a4edcc 100644 --- a/govtool/frontend/src/models/metadataValidation.ts +++ b/govtool/frontend/src/models/metadataValidation.ts @@ -9,6 +9,8 @@ export enum MetadataValidationStatus { export type ValidateMetadataResult = { status?: MetadataValidationStatus; valid: boolean; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + metadata?: any; }; export enum MetadataStandard { diff --git a/govtool/frontend/src/services/API.ts b/govtool/frontend/src/services/API.ts index 2e084a29b..54c09aa7f 100644 --- a/govtool/frontend/src/services/API.ts +++ b/govtool/frontend/src/services/API.ts @@ -6,11 +6,14 @@ import { PATHS } from "@consts"; const TIMEOUT_IN_SECONDS = 30 * 1000; // 1000 ms is 1 s then its 10 s const BASE_URL = import.meta.env.VITE_BASE_URL; +// Validation should be performed directly on the server +// than no metadata service is needed and `/api` might be removed export const API = axios.create({ baseURL: `${BASE_URL}/api`, timeout: TIMEOUT_IN_SECONDS, }); +// TODO: Remove this service and use the API service export const METADATA_VALIDATION_API = axios.create({ baseURL: `${BASE_URL}/metadata-validation`, timeout: TIMEOUT_IN_SECONDS, diff --git a/govtool/frontend/src/services/requests/getDRepVotes.ts b/govtool/frontend/src/services/requests/getDRepVotes.ts index c01f421c1..2b7efd682 100644 --- a/govtool/frontend/src/services/requests/getDRepVotes.ts +++ b/govtool/frontend/src/services/requests/getDRepVotes.ts @@ -21,14 +21,19 @@ export const getDRepVotes = async ({ const mappedData = (await Promise.all( data.map(async (proposal) => { - const isDataMissing = await checkIsMissingGAMetadata({ + const { metadata, status } = await checkIsMissingGAMetadata({ hash: proposal?.proposal?.metadataHash, url: proposal?.proposal?.url, }); return { vote: proposal.vote, - proposal: { ...proposal.proposal, isDataMissing }, + proposal: { + ...proposal.proposal, + // workaround for the missing data in db-sync + ...metadata, + isDataMissing: status || false, + }, }; }), )) as VotedProposalToDisplay[]; diff --git a/govtool/frontend/src/services/requests/getProposal.ts b/govtool/frontend/src/services/requests/getProposal.ts index faa1d89ab..16b89fd81 100644 --- a/govtool/frontend/src/services/requests/getProposal.ts +++ b/govtool/frontend/src/services/requests/getProposal.ts @@ -8,10 +8,10 @@ export const getProposal = async (proposalId: string, drepId?: string) => { `/proposal/get/${encodedHash}?drepId=${drepId}`, ); - const isDataMissing = await checkIsMissingGAMetadata({ + const { metadata, status } = await checkIsMissingGAMetadata({ hash: data?.proposal.metadataHash, url: data?.proposal.url, }); - - return { ...data, isDataMissing }; + // workaround for the missing data in db-sync + return { ...data, ...metadata, isDataMissing: status || false }; }; diff --git a/govtool/frontend/src/utils/getMetadataDataMissingStatusTranslation.ts b/govtool/frontend/src/utils/getMetadataDataMissingStatusTranslation.ts index 0a91531a6..c7d388e57 100644 --- a/govtool/frontend/src/utils/getMetadataDataMissingStatusTranslation.ts +++ b/govtool/frontend/src/utils/getMetadataDataMissingStatusTranslation.ts @@ -13,6 +13,7 @@ export const getMetadataDataMissingStatusTranslation = ( const errorKey = { [MetadataValidationStatus.URL_NOT_FOUND]: "dataMissing", [MetadataValidationStatus.INVALID_JSONLD]: "incorrectFormat", + [MetadataValidationStatus.INCORRECT_FORMAT]: "incorrectFormat", [MetadataValidationStatus.INVALID_HASH]: "notVerifiable", }[status] as "dataMissing" | "incorrectFormat" | "notVerifiable"; return i18n.t(`dataMissingErrors.${errorKey || "dataMissing"}`); diff --git a/govtool/frontend/src/utils/tests/getMetadataDataMissingStatusTranslation.test.ts b/govtool/frontend/src/utils/tests/getMetadataDataMissingStatusTranslation.test.ts index 230dedf48..2cfec8d28 100644 --- a/govtool/frontend/src/utils/tests/getMetadataDataMissingStatusTranslation.test.ts +++ b/govtool/frontend/src/utils/tests/getMetadataDataMissingStatusTranslation.test.ts @@ -13,7 +13,7 @@ describe("getMetadataDataMissingStatusTranslation", () => { const translation = getMetadataDataMissingStatusTranslation( MetadataValidationStatus.INVALID_JSONLD, ); - expect(translation).toBe("Incorrect Format"); + expect(translation).toBe("Data Formatted Incorrectly"); }); it("should return the correct translation for INVALID_HASH status", () => { diff --git a/govtool/frontend/src/utils/tests/validateMetadataHash.test.ts b/govtool/frontend/src/utils/tests/validateMetadataHash.test.ts index d04f8d2fc..ea9b1e78e 100644 --- a/govtool/frontend/src/utils/tests/validateMetadataHash.test.ts +++ b/govtool/frontend/src/utils/tests/validateMetadataHash.test.ts @@ -13,14 +13,18 @@ const mockPostValidate = postValidate as jest.MockedFunction< >; describe("checkIsMissingGAMetadata", () => { - it("returns false when there are no issues with the metadata", async () => { + it("returns metadata when there are no issues with the validation", async () => { mockPostValidate.mockResolvedValueOnce({ valid: true, + metadata: { some: "metadata" }, }); const result = await checkIsMissingGAMetadata({ url, hash }); - expect(result).toBe(false); + expect(result).toStrictEqual({ + valid: true, + metadata: { some: "metadata" }, + }); expect(mockPostValidate).toHaveBeenCalledWith({ url, hash, @@ -36,7 +40,7 @@ describe("checkIsMissingGAMetadata", () => { const result = await checkIsMissingGAMetadata({ url, hash }); - expect(result).toBe(MetadataValidationStatus.INVALID_HASH); + expect(result.status).toBe(MetadataValidationStatus.INVALID_HASH); expect(mockPostValidate).toHaveBeenCalledWith({ url, hash, @@ -52,7 +56,7 @@ describe("checkIsMissingGAMetadata", () => { const result = await checkIsMissingGAMetadata({ url, hash }); - expect(result).toBe(MetadataValidationStatus.INVALID_JSONLD); + expect(result.status).toBe(MetadataValidationStatus.INVALID_JSONLD); expect(mockPostValidate).toHaveBeenCalledWith({ url, hash, @@ -65,7 +69,7 @@ describe("checkIsMissingGAMetadata", () => { const result = await checkIsMissingGAMetadata({ url, hash }); - expect(result).toBe(MetadataValidationStatus.URL_NOT_FOUND); + expect(result.status).toBe(MetadataValidationStatus.URL_NOT_FOUND); expect(mockPostValidate).toHaveBeenCalledWith({ url, hash, diff --git a/govtool/frontend/src/utils/validateMetadataHash.ts b/govtool/frontend/src/utils/validateMetadataHash.ts index d9eb0e1e8..6868cc3d6 100644 --- a/govtool/frontend/src/utils/validateMetadataHash.ts +++ b/govtool/frontend/src/utils/validateMetadataHash.ts @@ -2,24 +2,31 @@ import { postValidate } from "@services"; import { MetadataStandard, MetadataValidationStatus } from "@/models"; +type CheckIsMissingGAMetadataResponse = { + status?: MetadataValidationStatus; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + metadata?: any; + valid: boolean; +}; + export const checkIsMissingGAMetadata = async ({ url, hash, }: { url: string; hash: string; -}): Promise => { +}): Promise => { try { - const { status } = await postValidate({ + const { status, metadata, valid } = await postValidate({ url, hash, standard: MetadataStandard.CIP108, }); if (status) { - return status; + return { status, valid }; } - return false; + return { metadata, valid }; } catch (error) { - return MetadataValidationStatus.URL_NOT_FOUND; + return { status: MetadataValidationStatus.URL_NOT_FOUND, valid: false }; } }; diff --git a/govtool/metadata-validation/src/app.service.ts b/govtool/metadata-validation/src/app.service.ts index c29cf16fa..fb31dd4fd 100644 --- a/govtool/metadata-validation/src/app.service.ts +++ b/govtool/metadata-validation/src/app.service.ts @@ -5,7 +5,7 @@ import * as blake from 'blakejs'; import { ValidateMetadataDTO } from '@dto'; import { MetadataValidationStatus } from '@enums'; -import { canonizeJSON, validateMetadataStandard } from '@utils'; +import { canonizeJSON, validateMetadataStandard, parseMetadata } from '@utils'; import { ValidateMetadataResult } from '@types'; @Injectable() @@ -18,6 +18,7 @@ export class AppService { standard, }: ValidateMetadataDTO): Promise { let status: MetadataValidationStatus; + let metadata: any; try { const { data } = await firstValueFrom( this.httpService.get(url).pipe( @@ -31,6 +32,8 @@ export class AppService { await validateMetadataStandard(data, standard); } + metadata = parseMetadata(data.body, standard); + let canonizedMetadata; try { canonizedMetadata = await canonizeJSON(data); @@ -48,6 +51,6 @@ export class AppService { } } - return { status, valid: !Boolean(status) }; + return { status, valid: !Boolean(status), metadata }; } } diff --git a/govtool/metadata-validation/src/schemas/cipStandardSchema.ts b/govtool/metadata-validation/src/schemas/cipStandardSchema.ts index 913b9e952..0551ad172 100644 --- a/govtool/metadata-validation/src/schemas/cipStandardSchema.ts +++ b/govtool/metadata-validation/src/schemas/cipStandardSchema.ts @@ -32,9 +32,13 @@ export const cipStandardSchema: StandardSpecification = { references: Joi.array().items( Joi.object({ '@type': Joi.string(), - label: Joi.object({ '@value': Joi.string().required() }), - uri: Joi.object({ '@value': Joi.string().uri().required() }), - referenceHash: Joi.object({ + 'CIP108:reference-label': Joi.object({ + '@value': Joi.string().required(), + }), + 'CIP108:reference-uri': Joi.object({ + '@value': Joi.string().uri().required(), + }), + 'CIP108:reference-hash': Joi.object({ hashDigest: Joi.string().required(), hashAlgorithm: Joi.string().required(), }), diff --git a/govtool/metadata-validation/src/types/validateMetadata.ts b/govtool/metadata-validation/src/types/validateMetadata.ts index dcdf28363..fa20f24fe 100644 --- a/govtool/metadata-validation/src/types/validateMetadata.ts +++ b/govtool/metadata-validation/src/types/validateMetadata.ts @@ -7,4 +7,5 @@ export enum MetadataStandard { export type ValidateMetadataResult = { status?: MetadataValidationStatus; valid: boolean; + metadata: any; }; diff --git a/govtool/metadata-validation/src/utils/index.ts b/govtool/metadata-validation/src/utils/index.ts index 648150db2..a074ea4fe 100644 --- a/govtool/metadata-validation/src/utils/index.ts +++ b/govtool/metadata-validation/src/utils/index.ts @@ -1,2 +1,3 @@ export * from './canonizeJSON'; export * from './validateMetadataStandard'; +export * from './parseMetadata'; diff --git a/govtool/metadata-validation/src/utils/parseMetadata.ts b/govtool/metadata-validation/src/utils/parseMetadata.ts new file mode 100644 index 000000000..64baff226 --- /dev/null +++ b/govtool/metadata-validation/src/utils/parseMetadata.ts @@ -0,0 +1,23 @@ +import { MetadataStandard } from '@/types'; + +const CIP_108_VALUE_KEYS = ['abstract', 'motivation', 'rationale', 'title']; +export const parseMetadata = (metadata: any, standard: MetadataStandard) => { + const parsedMetadata = {}; + switch (standard) { + case MetadataStandard.CIP108: + for (const [key, value] of Object.entries(metadata)) { + if (CIP_108_VALUE_KEYS.includes(key)) { + parsedMetadata[key] = value['@value']; + } + + if (key === 'references') { + parsedMetadata[key] = (Array.isArray(value) ? value : [])?.map( + (reference) => reference['CIP108:reference-uri']['@value'], + ); + } + } + return parsedMetadata; + default: + return metadata; + } +};