From 3a8d4d904688176352e351d18666b019df60daf4 Mon Sep 17 00:00:00 2001 From: jankun4 Date: Tue, 30 Apr 2024 01:59:01 +0200 Subject: [PATCH 1/4] [#876] add metadata/validate endpoint Signed-off-by: jankun4 --- CHANGELOG.md | 1 + govtool/backend/app/Main.hs | 11 ++- govtool/backend/example-config.json | 4 +- govtool/backend/src/VVA/API.hs | 57 ++++++++----- govtool/backend/src/VVA/API/Types.hs | 101 +++++++++++++++++++++++- govtool/backend/src/VVA/Config.hs | 40 ++++++++-- govtool/backend/src/VVA/Metadata.hs | 48 +++++++++++ govtool/backend/src/VVA/Types.hs | 17 +++- govtool/backend/vva-be.cabal | 1 + govtool/metadata-validation/src/main.ts | 2 +- 10 files changed, 249 insertions(+), 33 deletions(-) create mode 100644 govtool/backend/src/VVA/Metadata.hs diff --git a/CHANGELOG.md b/CHANGELOG.md index fee94d409..e19d14f47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ changes. ### Added +- added `metadata/validate` endpoint [Issue 876](https://github.com/IntersectMBO/govtool/issues/876) - added pagination to `drep/list` [Issue 756](https://github.com/IntersectMBO/govtool/issues/756) - added search query param to the `drep/getVotes` [Issue 640](https://github.com/IntersectMBO/govtool/issues/640) - added filtering and sorting capabilities to the `drep/list` [Issue 722](https://github.com/IntersectMBO/govtool/issues/722) diff --git a/govtool/backend/app/Main.hs b/govtool/backend/app/Main.hs index f4df1c03c..5ca505210 100644 --- a/govtool/backend/app/Main.hs +++ b/govtool/backend/app/Main.hs @@ -70,8 +70,10 @@ import VVA.API.Types import VVA.CommandLine import VVA.Config import VVA.Types (AppEnv (..), - AppError (CriticalError, NotFoundError, ValidationError), + AppError (CriticalError, NotFoundError, ValidationError, InternalError), CacheEnv (..)) +import Network.HTTP.Client hiding (Proxy, Request) +import Network.HTTP.Client.TLS proxyAPI :: Proxy (VVAApi :<|> SwaggerAPI) proxyAPI = Proxy @@ -113,6 +115,7 @@ startApp vvaConfig = do dRepVotingPowerCache <- newCache dRepListCache <- newCache networkMetricsCache <- newCache + metadataValidationCache <- newCache return $ CacheEnv { proposalListCache , getProposalCache @@ -124,10 +127,12 @@ startApp vvaConfig = do , dRepVotingPowerCache , dRepListCache , networkMetricsCache + , metadataValidationCache } connectionPool <- createPool (connectPostgreSQL (encodeUtf8 (dbSyncConnectionString $ getter vvaConfig))) close 1 1 60 + vvaTlsManager <- newManager tlsManagerSettings - let appEnv = AppEnv {vvaConfig=vvaConfig, vvaCache=cacheEnv, vvaConnectionPool=connectionPool} + let appEnv = AppEnv {vvaConfig=vvaConfig, vvaCache=cacheEnv, vvaConnectionPool=connectionPool, vvaTlsManager} server' <- mkVVAServer appEnv runSettings settings server' @@ -252,7 +257,7 @@ liftServer appEnv = handleErrors (Left (ValidationError msg)) = throwError $ err400 { errBody = BS.fromStrict $ encodeUtf8 msg } handleErrors (Left (NotFoundError msg)) = throwError $ err404 { errBody = BS.fromStrict $ encodeUtf8 msg } handleErrors (Left (CriticalError msg)) = throwError $ err500 { errBody = BS.fromStrict $ encodeUtf8 msg } - + handleErrors (Left (InternalError msg)) = throwError $ err500 { errBody = BS.fromStrict $ encodeUtf8 msg } -- * Swagger type SwaggerAPI = SwaggerSchemaUI "swagger-ui" "swagger.json" diff --git a/govtool/backend/example-config.json b/govtool/backend/example-config.json index 7d7d8ba41..756e75282 100644 --- a/govtool/backend/example-config.json +++ b/govtool/backend/example-config.json @@ -9,5 +9,7 @@ "port" : 9999, "host" : "localhost", "cachedurationseconds": 20, - "sentrydsn": "https://username:password@senty.host/id" + "sentrydsn": "https://username:password@senty.host/id", + "metadatavalidationhost": "localhost", + "metadatavalidationport": 3001 } diff --git a/govtool/backend/src/VVA/API.hs b/govtool/backend/src/VVA/API.hs index d982c61dd..d860fd391 100644 --- a/govtool/backend/src/VVA/API.hs +++ b/govtool/backend/src/VVA/API.hs @@ -11,7 +11,7 @@ module VVA.API where import Control.Exception (throw) import Control.Monad.Except (throwError) import Control.Monad.Reader - +import Data.Aeson (Result(Error, Success), fromJSON) import Data.Bool (Bool) import Data.List (sortOn) import qualified Data.Map as Map @@ -39,8 +39,9 @@ import qualified VVA.Proposal as Proposal import qualified VVA.Transaction as Transaction import qualified VVA.Types as Types import VVA.Types (App, AppEnv (..), - AppError (CriticalError, ValidationError), + AppError (CriticalError, ValidationError, InternalError), CacheEnv (..)) +import qualified VVA.Metadata as Metadata type VVAApi = "drep" :> "list" @@ -73,6 +74,7 @@ type VVAApi = :<|> "transaction" :> "status" :> Capture "transactionId" HexText :> Get '[JSON] GetTransactionStatusResponse :<|> "throw500" :> Get '[JSON] () :<|> "network" :> "metrics" :> Get '[JSON] GetNetworkMetricsResponse + :<|> "metadata" :> "validate" :> ReqBody '[JSON] MetadataValidationParams :> Post '[JSON] MetadataValidationResponse server :: App m => ServerT VVAApi m server = drepList @@ -87,6 +89,7 @@ server = drepList :<|> getTransactionStatus :<|> throw500 :<|> getNetworkMetrics + :<|> validateMetadata mapDRepType :: Types.DRepType -> DRepType @@ -174,8 +177,8 @@ getVotingPower (unHexText -> dRepId) = do cacheRequest dRepVotingPowerCache dRepId $ DRep.getVotingPower dRepId -proposalToResponse :: Types.Proposal -> ProposalResponse -proposalToResponse Types.Proposal {..} = +proposalToResponse :: Types.Proposal -> MetadataValidationResponse -> ProposalResponse +proposalToResponse Types.Proposal {..} metadataValidationResponse = ProposalResponse { proposalResponseId = pack $ show proposalId, proposalResponseTxHash = HexText proposalTxHash, @@ -196,7 +199,8 @@ proposalToResponse Types.Proposal {..} = proposalResponseReferences = GovernanceActionReferences <$> proposalReferences, proposalResponseYesVotes = proposalYesVotes, proposalResponseNoVotes = proposalNoVotes, - proposalResponseAbstainVotes = proposalAbstainVotes + proposalResponseAbstainVotes = proposalAbstainVotes, + proposalResponseMetadataStatus = Just metadataValidationResponse } voteToResponse :: Types.Vote -> VoteParams @@ -214,16 +218,17 @@ voteToResponse Types.Vote {..} = mapSortAndFilterProposals - :: [GovernanceActionType] + :: App m + => [GovernanceActionType] -> Maybe GovernanceActionSortMode -> [Types.Proposal] - -> [ProposalResponse] -mapSortAndFilterProposals selectedTypes sortMode proposals = - let mappedProposals = - map - proposalToResponse + -> m [ProposalResponse] +mapSortAndFilterProposals selectedTypes sortMode proposals = do + mappedProposals <- + mapM + (\proposal@Types.Proposal {proposalUrl, proposalDocHash} -> proposalToResponse proposal <$> validateMetadata (MetadataValidationParams proposalUrl $ HexText proposalDocHash)) proposals - filteredProposals = + let filteredProposals = if null selectedTypes then mappedProposals else @@ -232,19 +237,19 @@ mapSortAndFilterProposals selectedTypes sortMode proposals = proposalResponseType `elem` selectedTypes ) mappedProposals - sortedProposals = case sortMode of + let sortedProposals = case sortMode of Nothing -> filteredProposals Just NewestCreated -> sortOn (Down . proposalResponseCreatedDate) filteredProposals Just SoonestToExpire -> sortOn proposalResponseExpiryDate filteredProposals Just MostYesVotes -> sortOn (Down . proposalResponseYesVotes) filteredProposals - in sortedProposals + return sortedProposals getVotes :: App m => HexText -> [GovernanceActionType] -> Maybe GovernanceActionSortMode -> Maybe Text -> m [VoteResponse] getVotes (unHexText -> dRepId) selectedTypes sortMode mSearch = do CacheEnv {dRepGetVotesCache} <- asks vvaCache (votes, proposals) <- cacheRequest dRepGetVotesCache dRepId $ DRep.getVotes dRepId [] let voteMap = Map.fromList $ map (\vote@Types.Vote {..} -> (voteProposalId, vote)) votes - let processedProposals = filter (isProposalSearchedFor mSearch) $ mapSortAndFilterProposals selectedTypes sortMode proposals + processedProposals <- filter (isProposalSearchedFor mSearch) <$> mapSortAndFilterProposals selectedTypes sortMode proposals return $ [ VoteResponse { voteResponseVote = voteToResponse (voteMap Map.! read (unpack proposalResponseId)) @@ -321,12 +326,14 @@ listProposals selectedTypes sortMode mPage mPageSize mDrepRaw mSearchQuery = do CacheEnv {proposalListCache} <- asks vvaCache - mappedAndSortedProposals <- - filter + mappedAndSortedProposals <- do + proposals <- cacheRequest proposalListCache () Proposal.listProposals + mappedSortedAndFilteredProposals <- mapSortAndFilterProposals selectedTypes sortMode proposals + return $ filter ( \p@ProposalResponse {proposalResponseId} -> proposalResponseId `notElem` proposalsToRemove && isProposalSearchedFor mSearchQuery p - ) . mapSortAndFilterProposals selectedTypes sortMode <$> cacheRequest proposalListCache () Proposal.listProposals + ) mappedSortedAndFilteredProposals let total = length mappedAndSortedProposals :: Int @@ -343,7 +350,9 @@ getProposal :: App m => GovActionId -> Maybe HexText -> m GetProposalResponse getProposal g@(GovActionId govActionTxHash govActionIndex) mDrepId' = do let mDrepId = unHexText <$> mDrepId' CacheEnv {getProposalCache} <- asks vvaCache - proposalResponse <- proposalToResponse <$> cacheRequest getProposalCache (unHexText govActionTxHash, govActionIndex) (Proposal.getProposal (unHexText govActionTxHash) govActionIndex) + proposal@Types.Proposal {proposalUrl, proposalDocHash} <- cacheRequest getProposalCache (unHexText govActionTxHash, govActionIndex) (Proposal.getProposal (unHexText govActionTxHash) govActionIndex) + metadataStatus <- validateMetadata $ MetadataValidationParams proposalUrl $ HexText proposalDocHash + let proposalResponse = proposalToResponse proposal metadataStatus voteResponse <- case mDrepId of Nothing -> return Nothing Just drepId -> do @@ -390,3 +399,13 @@ getNetworkMetrics = do , getNetworkMetricsResponseAlwaysAbstainVotingPower = networkMetricsAlwaysAbstainVotingPower , getNetworkMetricsResponseAlwaysNoConfidenceVotingPower = networkMetricsAlwaysNoConfidenceVotingPower } + +validateMetadata :: App m => MetadataValidationParams -> m MetadataValidationResponse +validateMetadata MetadataValidationParams {..} = do + CacheEnv {metadataValidationCache} <- asks vvaCache + result <- cacheRequest metadataValidationCache (metadataValidationParamsUrl, unHexText metadataValidationParamsHash) + $ Metadata.validateMetadata metadataValidationParamsUrl (unHexText metadataValidationParamsHash) + + case fromJSON result of + Error e -> throwError $ InternalError $ pack $ show e + Success a -> return a diff --git a/govtool/backend/src/VVA/API/Types.hs b/govtool/backend/src/VVA/API/Types.hs index e911afa35..6eebd5d7c 100644 --- a/govtool/backend/src/VVA/API/Types.hs +++ b/govtool/backend/src/VVA/API/Types.hs @@ -90,6 +90,102 @@ instance ToSchema HexText where & schema . format ?~ "hex" & schema . example ?~ toJSON (HexText "a1b2c3") +newtype AnyValue + = AnyValue { unAnyValue :: Maybe Value } + deriving newtype (Show) + +instance FromJSON AnyValue where + parseJSON = pure . AnyValue . Just + +instance ToJSON AnyValue where + toJSON (AnyValue Nothing) = Null + toJSON (AnyValue (Just params)) = toJSON params + +exampleAnyValue :: Text +exampleAnyValue = + "{ \"any\": \"value\"}" + +instance ToSchema AnyValue where + declareNamedSchema _ = pure $ NamedSchema (Just "AnyValue") $ mempty + & type_ ?~ OpenApiObject + & description ?~ "Any value" + & example + ?~ toJSON exampleAnyValue + +data MetadataValidationStatus + = IncorrectFormat + | IncorrectJSONLD + | IncorrectHash + | UrlNotFound + deriving (Show, Eq) + +instance ToJSON MetadataValidationStatus where + toJSON IncorrectFormat = "INCORRECT_FORMTAT" + toJSON IncorrectJSONLD = "INVALID_JSONLD" + toJSON IncorrectHash = "INVALID_HASH" + toJSON UrlNotFound = "URL_NOT_FOUND" + +instance FromJSON MetadataValidationStatus where + parseJSON (String s) = case s of + "INCORRECT_FORMTAT" -> pure IncorrectFormat + "INVALID_JSONLD" -> pure IncorrectJSONLD + "INVALID_HASH" -> pure IncorrectHash + "URL_NOT_FOUND" -> pure UrlNotFound + _ -> fail "Invalid MetadataValidationStatus" + parseJSON _ = fail "Invalid MetadataValidationStatus" + +instance ToSchema MetadataValidationStatus where + declareNamedSchema _ = pure $ NamedSchema (Just "MetadataValidationStatus") $ mempty + & type_ ?~ OpenApiString + & description ?~ "Metadata Validation Status" + & enum_ ?~ map toJSON [IncorrectFormat, IncorrectJSONLD, IncorrectHash, UrlNotFound] + +data MetadataValidationResponse + = MetadataValidationResponse + { metadataValidationResponseStatus :: Maybe MetadataValidationStatus + , metadataValidationResponseValid :: Bool + } + deriving (Generic, Show) + +deriveJSON (jsonOptions "metadataValidationResponse") ''MetadataValidationResponse + +instance ToSchema MetadataValidationResponse where + declareNamedSchema _ = do + NamedSchema name_ schema_ <- + genericDeclareNamedSchema + ( fromAesonOptions $ jsonOptions "metadataValidationResponse" ) + (Proxy :: Proxy MetadataValidationResponse) + return $ + NamedSchema name_ $ + schema_ + & description ?~ "Metadata Validation Response" + & example + ?~ toJSON ("{\"status\": \"INCORRECT_FORMTAT\", \"valid\":false}" :: Text) + +data MetadataValidationParams + = MetadataValidationParams + { metadataValidationParamsUrl :: Text + , metadataValidationParamsHash :: HexText + } + deriving (Generic, Show) + +deriveJSON (jsonOptions "metadataValidationParams") ''MetadataValidationParams + +instance ToSchema MetadataValidationParams where + declareNamedSchema proxy = do + NamedSchema name_ schema_ <- + genericDeclareNamedSchema + ( fromAesonOptions $ jsonOptions "metadataValidationParams" ) + proxy + return $ + NamedSchema name_ $ + schema_ + & description ?~ "Metadata Validation Params" + & example + ?~ toJSON ("{\"url\": \"https://metadata.xyz\", \"hash\": \"9af10e89979e51b8cdc827c963124a1ef4920d1253eef34a1d5cfe76438e3f11\"}" :: Text) + + + data GovActionId = GovActionId { govActionIdTxHash :: HexText @@ -349,6 +445,7 @@ data ProposalResponse , proposalResponseYesVotes :: Integer , proposalResponseNoVotes :: Integer , proposalResponseAbstainVotes :: Integer + , proposalResponseMetadataStatus :: Maybe MetadataValidationResponse } deriving (Generic, Show) @@ -374,7 +471,8 @@ exampleProposalResponse = "{ \"id\": \"proposalId123\"," <> "\"references\": [{\"uri\": \"google.com\", \"@type\": \"Other\", \"label\": \"example label\"}]," <> "\"yesVotes\": 0," <> "\"noVotes\": 0," - <> "\"abstainVotes\": 0}" + <> "\"abstainVotes\": 0" + <> "\"metadataStatus\": {\"status\": null, \"valid\": true}}" instance ToSchema ProposalResponse where declareNamedSchema proxy = do @@ -841,3 +939,4 @@ instance ToSchema GetNetworkMetricsResponse where & description ?~ "GetNetworkMetricsResponse" & example ?~ toJSON exampleGetNetworkMetricsResponse + diff --git a/govtool/backend/src/VVA/Config.hs b/govtool/backend/src/VVA/Config.hs index 7055c33ed..c4781681f 100644 --- a/govtool/backend/src/VVA/Config.hs +++ b/govtool/backend/src/VVA/Config.hs @@ -24,6 +24,8 @@ module VVA.Config , getServerHost , getServerPort , vvaConfigToText + , getMetadataValidationHost + , getMetadataValidationPort ) where import Conferer @@ -69,15 +71,19 @@ instance DefaultConfig DBConfig where data VVAConfigInternal = VVAConfigInternal { -- | db-sync database access. - vVAConfigInternalDbsyncconfig :: DBConfig + vVAConfigInternalDbsyncconfig :: DBConfig -- | Server port. - , vVAConfigInternalPort :: Int + , vVAConfigInternalPort :: Int -- | Server host. - , vVAConfigInternalHost :: Text + , vVAConfigInternalHost :: Text -- | Request cache duration - , vVaConfigInternalCacheDurationSeconds :: Int + , vVaConfigInternalCacheDurationSeconds :: Int -- | Sentry DSN - , vVAConfigInternalSentrydsn :: String + , vVAConfigInternalSentrydsn :: String + -- | Metadata validation service host + , vVAConfigInternalMetadataValidationHost :: Text + -- | Metadata validation service port + , vVAConfigInternalMetadataValidationPort :: Int } deriving (FromConfig, Generic, Show) @@ -88,7 +94,9 @@ instance DefaultConfig VVAConfigInternal where vVAConfigInternalPort = 3000, vVAConfigInternalHost = "localhost", vVaConfigInternalCacheDurationSeconds = 20, - vVAConfigInternalSentrydsn = "https://username:password@senty.host/id" + vVAConfigInternalSentrydsn = "https://username:password@senty.host/id", + vVAConfigInternalMetadataValidationHost = "localhost", + vVAConfigInternalMetadataValidationPort = 3001 } -- | DEX configuration. @@ -104,6 +112,10 @@ data VVAConfig , cacheDurationSeconds :: Int -- | Sentry DSN , sentryDSN :: String + -- | Metadata validation service host + , metadataValidationHost :: Text + -- | Metadata validation service port + , metadataValidationPort :: Int } deriving (Generic, Show, ToJSON) @@ -143,7 +155,9 @@ convertConfig VVAConfigInternal {..} = serverPort = vVAConfigInternalPort, serverHost = vVAConfigInternalHost, cacheDurationSeconds = vVaConfigInternalCacheDurationSeconds, - sentryDSN = vVAConfigInternalSentrydsn + sentryDSN = vVAConfigInternalSentrydsn, + metadataValidationHost = vVAConfigInternalMetadataValidationHost, + metadataValidationPort = vVAConfigInternalMetadataValidationPort } -- | Load configuration from a file specified on the command line. Load from @@ -181,3 +195,15 @@ getServerHost :: (Has VVAConfig r, MonadReader r m) => m Text getServerHost = asks (serverHost . getter) + +-- | Access MetadataValidationService host +getMetadataValidationHost :: + (Has VVAConfig r, MonadReader r m) => + m Text +getMetadataValidationHost = asks (metadataValidationHost . getter) + +-- | Access MetadataValidationService port +getMetadataValidationPort :: + (Has VVAConfig r, MonadReader r m) => + m Int +getMetadataValidationPort = asks (metadataValidationPort . getter) diff --git a/govtool/backend/src/VVA/Metadata.hs b/govtool/backend/src/VVA/Metadata.hs new file mode 100644 index 000000000..a2a6a6490 --- /dev/null +++ b/govtool/backend/src/VVA/Metadata.hs @@ -0,0 +1,48 @@ +{-# LANGUAGE FlexibleContexts #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE TemplateHaskell #-} +{-# LANGUAGE TypeApplications #-} + +module VVA.Metadata where + +import Control.Monad.Except (MonadError, throwError) +import Control.Monad.Reader + +import Data.Aeson (Value, decode) +import Data.Maybe (fromJust) +import Data.ByteString (ByteString) +import Data.FileEmbed (embedFile) +import Data.Has (Has, getter) +import Data.String (fromString) +import Data.Text (Text, unpack) +import qualified Data.Text.Encoding as Text +import Data.Time.Clock + +import qualified Database.PostgreSQL.Simple as SQL + +import VVA.Config +import VVA.Pool (ConnectionPool, withPool) +import VVA.Types +import Network.HTTP.Client +import Network.HTTP.Client.TLS +import Data.Aeson (encode, object, (.=)) + +validateMetadata + :: (Has VVAConfig r, Has Manager r, MonadReader r m, MonadIO m, MonadError AppError m) + => Text + -> Text + -> m Value +validateMetadata url hash = do + metadataHost <- getMetadataValidationHost + metadataPort <- getMetadataValidationPort + manager <- asks getter + let requestBody = encode $ object ["url" .= unpack url, "hash" .= unpack hash] + initialRequest <- liftIO $ parseRequest (unpack metadataHost <> ":" <> show metadataPort <> "/validate") + let request = initialRequest + { method = "POST" + , requestBody = RequestBodyLBS requestBody + , requestHeaders = [("Content-Type", "application/json")] + } + response <- liftIO $ httpLbs request manager + return $ fromJust $ decode $ responseBody response diff --git a/govtool/backend/src/VVA/Types.hs b/govtool/backend/src/VVA/Types.hs index 7b128045c..fba1e2321 100644 --- a/govtool/backend/src/VVA/Types.hs +++ b/govtool/backend/src/VVA/Types.hs @@ -23,6 +23,7 @@ import Database.PostgreSQL.Simple (Connection) import VVA.Cache import VVA.Config +import Network.HTTP.Client (Manager) type App m = (MonadReader AppEnv m, MonadIO m, MonadFail m, MonadError AppError m) @@ -31,6 +32,7 @@ data AppEnv { vvaConfig :: VVAConfig , vvaCache :: CacheEnv , vvaConnectionPool :: Pool Connection + , vvaTlsManager :: Manager } instance Has VVAConfig AppEnv where @@ -45,10 +47,15 @@ instance Has (Pool Connection) AppEnv where getter AppEnv {vvaConnectionPool} = vvaConnectionPool modifier f a@AppEnv {vvaConnectionPool} = a {vvaConnectionPool = f vvaConnectionPool} +instance Has Manager AppEnv where + getter AppEnv {vvaTlsManager} = vvaTlsManager + modifier f a@AppEnv {vvaTlsManager} = a {vvaTlsManager = f vvaTlsManager} + data AppError = ValidationError Text | NotFoundError Text | CriticalError Text + | InternalError Text deriving (Show) instance Exception AppError @@ -138,6 +145,7 @@ data CacheEnv , dRepVotingPowerCache :: Cache.Cache Text Integer , dRepListCache :: Cache.Cache () [DRepRegistration] , networkMetricsCache :: Cache.Cache () NetworkMetrics + , metadataValidationCache :: Cache.Cache (Text, Text) Value } data NetworkMetrics @@ -159,4 +167,11 @@ data Delegation { delegationDRepHash :: Maybe Text , delegationDRepView :: Text , delegationTxHash :: Text - } \ No newline at end of file + } + + +data MetadataValidationStatus + = IncorrectFormat + | IncorrectJSONLD + | IncorrectHash + | UrlNotFound \ No newline at end of file diff --git a/govtool/backend/vva-be.cabal b/govtool/backend/vva-be.cabal index ccd247dc4..cc7ed2b63 100644 --- a/govtool/backend/vva-be.cabal +++ b/govtool/backend/vva-be.cabal @@ -114,3 +114,4 @@ library , VVA.Pool , VVA.Types , VVA.Network + , VVA.Metadata diff --git a/govtool/metadata-validation/src/main.ts b/govtool/metadata-validation/src/main.ts index c27be2ac4..7ab38cb38 100644 --- a/govtool/metadata-validation/src/main.ts +++ b/govtool/metadata-validation/src/main.ts @@ -5,7 +5,7 @@ import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger'; import { AppModule } from './app.module'; async function bootstrap() { - const app = await NestFactory.create(AppModule); + const app = await NestFactory.create(AppModule, { cors: true }); const config = new DocumentBuilder() .setTitle('Submission Tool') From b8e3aa3afe850b5671f9377c3f30e1fd8de1c477 Mon Sep 17 00:00:00 2001 From: jankun4 Date: Wed, 8 May 2024 12:37:49 +0200 Subject: [PATCH 2/4] [#876] add metadata/validate endpoint --- CHANGELOG.md | 1 + govtool/backend/app/Main.hs | 11 ++- govtool/backend/example-config.json | 4 +- govtool/backend/src/VVA/API.hs | 57 ++++++++----- govtool/backend/src/VVA/API/Types.hs | 101 +++++++++++++++++++++++- govtool/backend/src/VVA/Config.hs | 40 ++++++++-- govtool/backend/src/VVA/Metadata.hs | 45 +++++++++++ govtool/backend/src/VVA/Types.hs | 17 +++- govtool/backend/vva-be.cabal | 4 +- govtool/metadata-validation/src/main.ts | 2 +- 10 files changed, 248 insertions(+), 34 deletions(-) create mode 100644 govtool/backend/src/VVA/Metadata.hs diff --git a/CHANGELOG.md b/CHANGELOG.md index fee94d409..e19d14f47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ changes. ### Added +- added `metadata/validate` endpoint [Issue 876](https://github.com/IntersectMBO/govtool/issues/876) - added pagination to `drep/list` [Issue 756](https://github.com/IntersectMBO/govtool/issues/756) - added search query param to the `drep/getVotes` [Issue 640](https://github.com/IntersectMBO/govtool/issues/640) - added filtering and sorting capabilities to the `drep/list` [Issue 722](https://github.com/IntersectMBO/govtool/issues/722) diff --git a/govtool/backend/app/Main.hs b/govtool/backend/app/Main.hs index f4df1c03c..5ca505210 100644 --- a/govtool/backend/app/Main.hs +++ b/govtool/backend/app/Main.hs @@ -70,8 +70,10 @@ import VVA.API.Types import VVA.CommandLine import VVA.Config import VVA.Types (AppEnv (..), - AppError (CriticalError, NotFoundError, ValidationError), + AppError (CriticalError, NotFoundError, ValidationError, InternalError), CacheEnv (..)) +import Network.HTTP.Client hiding (Proxy, Request) +import Network.HTTP.Client.TLS proxyAPI :: Proxy (VVAApi :<|> SwaggerAPI) proxyAPI = Proxy @@ -113,6 +115,7 @@ startApp vvaConfig = do dRepVotingPowerCache <- newCache dRepListCache <- newCache networkMetricsCache <- newCache + metadataValidationCache <- newCache return $ CacheEnv { proposalListCache , getProposalCache @@ -124,10 +127,12 @@ startApp vvaConfig = do , dRepVotingPowerCache , dRepListCache , networkMetricsCache + , metadataValidationCache } connectionPool <- createPool (connectPostgreSQL (encodeUtf8 (dbSyncConnectionString $ getter vvaConfig))) close 1 1 60 + vvaTlsManager <- newManager tlsManagerSettings - let appEnv = AppEnv {vvaConfig=vvaConfig, vvaCache=cacheEnv, vvaConnectionPool=connectionPool} + let appEnv = AppEnv {vvaConfig=vvaConfig, vvaCache=cacheEnv, vvaConnectionPool=connectionPool, vvaTlsManager} server' <- mkVVAServer appEnv runSettings settings server' @@ -252,7 +257,7 @@ liftServer appEnv = handleErrors (Left (ValidationError msg)) = throwError $ err400 { errBody = BS.fromStrict $ encodeUtf8 msg } handleErrors (Left (NotFoundError msg)) = throwError $ err404 { errBody = BS.fromStrict $ encodeUtf8 msg } handleErrors (Left (CriticalError msg)) = throwError $ err500 { errBody = BS.fromStrict $ encodeUtf8 msg } - + handleErrors (Left (InternalError msg)) = throwError $ err500 { errBody = BS.fromStrict $ encodeUtf8 msg } -- * Swagger type SwaggerAPI = SwaggerSchemaUI "swagger-ui" "swagger.json" diff --git a/govtool/backend/example-config.json b/govtool/backend/example-config.json index 7d7d8ba41..756e75282 100644 --- a/govtool/backend/example-config.json +++ b/govtool/backend/example-config.json @@ -9,5 +9,7 @@ "port" : 9999, "host" : "localhost", "cachedurationseconds": 20, - "sentrydsn": "https://username:password@senty.host/id" + "sentrydsn": "https://username:password@senty.host/id", + "metadatavalidationhost": "localhost", + "metadatavalidationport": 3001 } diff --git a/govtool/backend/src/VVA/API.hs b/govtool/backend/src/VVA/API.hs index d982c61dd..d860fd391 100644 --- a/govtool/backend/src/VVA/API.hs +++ b/govtool/backend/src/VVA/API.hs @@ -11,7 +11,7 @@ module VVA.API where import Control.Exception (throw) import Control.Monad.Except (throwError) import Control.Monad.Reader - +import Data.Aeson (Result(Error, Success), fromJSON) import Data.Bool (Bool) import Data.List (sortOn) import qualified Data.Map as Map @@ -39,8 +39,9 @@ import qualified VVA.Proposal as Proposal import qualified VVA.Transaction as Transaction import qualified VVA.Types as Types import VVA.Types (App, AppEnv (..), - AppError (CriticalError, ValidationError), + AppError (CriticalError, ValidationError, InternalError), CacheEnv (..)) +import qualified VVA.Metadata as Metadata type VVAApi = "drep" :> "list" @@ -73,6 +74,7 @@ type VVAApi = :<|> "transaction" :> "status" :> Capture "transactionId" HexText :> Get '[JSON] GetTransactionStatusResponse :<|> "throw500" :> Get '[JSON] () :<|> "network" :> "metrics" :> Get '[JSON] GetNetworkMetricsResponse + :<|> "metadata" :> "validate" :> ReqBody '[JSON] MetadataValidationParams :> Post '[JSON] MetadataValidationResponse server :: App m => ServerT VVAApi m server = drepList @@ -87,6 +89,7 @@ server = drepList :<|> getTransactionStatus :<|> throw500 :<|> getNetworkMetrics + :<|> validateMetadata mapDRepType :: Types.DRepType -> DRepType @@ -174,8 +177,8 @@ getVotingPower (unHexText -> dRepId) = do cacheRequest dRepVotingPowerCache dRepId $ DRep.getVotingPower dRepId -proposalToResponse :: Types.Proposal -> ProposalResponse -proposalToResponse Types.Proposal {..} = +proposalToResponse :: Types.Proposal -> MetadataValidationResponse -> ProposalResponse +proposalToResponse Types.Proposal {..} metadataValidationResponse = ProposalResponse { proposalResponseId = pack $ show proposalId, proposalResponseTxHash = HexText proposalTxHash, @@ -196,7 +199,8 @@ proposalToResponse Types.Proposal {..} = proposalResponseReferences = GovernanceActionReferences <$> proposalReferences, proposalResponseYesVotes = proposalYesVotes, proposalResponseNoVotes = proposalNoVotes, - proposalResponseAbstainVotes = proposalAbstainVotes + proposalResponseAbstainVotes = proposalAbstainVotes, + proposalResponseMetadataStatus = Just metadataValidationResponse } voteToResponse :: Types.Vote -> VoteParams @@ -214,16 +218,17 @@ voteToResponse Types.Vote {..} = mapSortAndFilterProposals - :: [GovernanceActionType] + :: App m + => [GovernanceActionType] -> Maybe GovernanceActionSortMode -> [Types.Proposal] - -> [ProposalResponse] -mapSortAndFilterProposals selectedTypes sortMode proposals = - let mappedProposals = - map - proposalToResponse + -> m [ProposalResponse] +mapSortAndFilterProposals selectedTypes sortMode proposals = do + mappedProposals <- + mapM + (\proposal@Types.Proposal {proposalUrl, proposalDocHash} -> proposalToResponse proposal <$> validateMetadata (MetadataValidationParams proposalUrl $ HexText proposalDocHash)) proposals - filteredProposals = + let filteredProposals = if null selectedTypes then mappedProposals else @@ -232,19 +237,19 @@ mapSortAndFilterProposals selectedTypes sortMode proposals = proposalResponseType `elem` selectedTypes ) mappedProposals - sortedProposals = case sortMode of + let sortedProposals = case sortMode of Nothing -> filteredProposals Just NewestCreated -> sortOn (Down . proposalResponseCreatedDate) filteredProposals Just SoonestToExpire -> sortOn proposalResponseExpiryDate filteredProposals Just MostYesVotes -> sortOn (Down . proposalResponseYesVotes) filteredProposals - in sortedProposals + return sortedProposals getVotes :: App m => HexText -> [GovernanceActionType] -> Maybe GovernanceActionSortMode -> Maybe Text -> m [VoteResponse] getVotes (unHexText -> dRepId) selectedTypes sortMode mSearch = do CacheEnv {dRepGetVotesCache} <- asks vvaCache (votes, proposals) <- cacheRequest dRepGetVotesCache dRepId $ DRep.getVotes dRepId [] let voteMap = Map.fromList $ map (\vote@Types.Vote {..} -> (voteProposalId, vote)) votes - let processedProposals = filter (isProposalSearchedFor mSearch) $ mapSortAndFilterProposals selectedTypes sortMode proposals + processedProposals <- filter (isProposalSearchedFor mSearch) <$> mapSortAndFilterProposals selectedTypes sortMode proposals return $ [ VoteResponse { voteResponseVote = voteToResponse (voteMap Map.! read (unpack proposalResponseId)) @@ -321,12 +326,14 @@ listProposals selectedTypes sortMode mPage mPageSize mDrepRaw mSearchQuery = do CacheEnv {proposalListCache} <- asks vvaCache - mappedAndSortedProposals <- - filter + mappedAndSortedProposals <- do + proposals <- cacheRequest proposalListCache () Proposal.listProposals + mappedSortedAndFilteredProposals <- mapSortAndFilterProposals selectedTypes sortMode proposals + return $ filter ( \p@ProposalResponse {proposalResponseId} -> proposalResponseId `notElem` proposalsToRemove && isProposalSearchedFor mSearchQuery p - ) . mapSortAndFilterProposals selectedTypes sortMode <$> cacheRequest proposalListCache () Proposal.listProposals + ) mappedSortedAndFilteredProposals let total = length mappedAndSortedProposals :: Int @@ -343,7 +350,9 @@ getProposal :: App m => GovActionId -> Maybe HexText -> m GetProposalResponse getProposal g@(GovActionId govActionTxHash govActionIndex) mDrepId' = do let mDrepId = unHexText <$> mDrepId' CacheEnv {getProposalCache} <- asks vvaCache - proposalResponse <- proposalToResponse <$> cacheRequest getProposalCache (unHexText govActionTxHash, govActionIndex) (Proposal.getProposal (unHexText govActionTxHash) govActionIndex) + proposal@Types.Proposal {proposalUrl, proposalDocHash} <- cacheRequest getProposalCache (unHexText govActionTxHash, govActionIndex) (Proposal.getProposal (unHexText govActionTxHash) govActionIndex) + metadataStatus <- validateMetadata $ MetadataValidationParams proposalUrl $ HexText proposalDocHash + let proposalResponse = proposalToResponse proposal metadataStatus voteResponse <- case mDrepId of Nothing -> return Nothing Just drepId -> do @@ -390,3 +399,13 @@ getNetworkMetrics = do , getNetworkMetricsResponseAlwaysAbstainVotingPower = networkMetricsAlwaysAbstainVotingPower , getNetworkMetricsResponseAlwaysNoConfidenceVotingPower = networkMetricsAlwaysNoConfidenceVotingPower } + +validateMetadata :: App m => MetadataValidationParams -> m MetadataValidationResponse +validateMetadata MetadataValidationParams {..} = do + CacheEnv {metadataValidationCache} <- asks vvaCache + result <- cacheRequest metadataValidationCache (metadataValidationParamsUrl, unHexText metadataValidationParamsHash) + $ Metadata.validateMetadata metadataValidationParamsUrl (unHexText metadataValidationParamsHash) + + case fromJSON result of + Error e -> throwError $ InternalError $ pack $ show e + Success a -> return a diff --git a/govtool/backend/src/VVA/API/Types.hs b/govtool/backend/src/VVA/API/Types.hs index e911afa35..6eebd5d7c 100644 --- a/govtool/backend/src/VVA/API/Types.hs +++ b/govtool/backend/src/VVA/API/Types.hs @@ -90,6 +90,102 @@ instance ToSchema HexText where & schema . format ?~ "hex" & schema . example ?~ toJSON (HexText "a1b2c3") +newtype AnyValue + = AnyValue { unAnyValue :: Maybe Value } + deriving newtype (Show) + +instance FromJSON AnyValue where + parseJSON = pure . AnyValue . Just + +instance ToJSON AnyValue where + toJSON (AnyValue Nothing) = Null + toJSON (AnyValue (Just params)) = toJSON params + +exampleAnyValue :: Text +exampleAnyValue = + "{ \"any\": \"value\"}" + +instance ToSchema AnyValue where + declareNamedSchema _ = pure $ NamedSchema (Just "AnyValue") $ mempty + & type_ ?~ OpenApiObject + & description ?~ "Any value" + & example + ?~ toJSON exampleAnyValue + +data MetadataValidationStatus + = IncorrectFormat + | IncorrectJSONLD + | IncorrectHash + | UrlNotFound + deriving (Show, Eq) + +instance ToJSON MetadataValidationStatus where + toJSON IncorrectFormat = "INCORRECT_FORMTAT" + toJSON IncorrectJSONLD = "INVALID_JSONLD" + toJSON IncorrectHash = "INVALID_HASH" + toJSON UrlNotFound = "URL_NOT_FOUND" + +instance FromJSON MetadataValidationStatus where + parseJSON (String s) = case s of + "INCORRECT_FORMTAT" -> pure IncorrectFormat + "INVALID_JSONLD" -> pure IncorrectJSONLD + "INVALID_HASH" -> pure IncorrectHash + "URL_NOT_FOUND" -> pure UrlNotFound + _ -> fail "Invalid MetadataValidationStatus" + parseJSON _ = fail "Invalid MetadataValidationStatus" + +instance ToSchema MetadataValidationStatus where + declareNamedSchema _ = pure $ NamedSchema (Just "MetadataValidationStatus") $ mempty + & type_ ?~ OpenApiString + & description ?~ "Metadata Validation Status" + & enum_ ?~ map toJSON [IncorrectFormat, IncorrectJSONLD, IncorrectHash, UrlNotFound] + +data MetadataValidationResponse + = MetadataValidationResponse + { metadataValidationResponseStatus :: Maybe MetadataValidationStatus + , metadataValidationResponseValid :: Bool + } + deriving (Generic, Show) + +deriveJSON (jsonOptions "metadataValidationResponse") ''MetadataValidationResponse + +instance ToSchema MetadataValidationResponse where + declareNamedSchema _ = do + NamedSchema name_ schema_ <- + genericDeclareNamedSchema + ( fromAesonOptions $ jsonOptions "metadataValidationResponse" ) + (Proxy :: Proxy MetadataValidationResponse) + return $ + NamedSchema name_ $ + schema_ + & description ?~ "Metadata Validation Response" + & example + ?~ toJSON ("{\"status\": \"INCORRECT_FORMTAT\", \"valid\":false}" :: Text) + +data MetadataValidationParams + = MetadataValidationParams + { metadataValidationParamsUrl :: Text + , metadataValidationParamsHash :: HexText + } + deriving (Generic, Show) + +deriveJSON (jsonOptions "metadataValidationParams") ''MetadataValidationParams + +instance ToSchema MetadataValidationParams where + declareNamedSchema proxy = do + NamedSchema name_ schema_ <- + genericDeclareNamedSchema + ( fromAesonOptions $ jsonOptions "metadataValidationParams" ) + proxy + return $ + NamedSchema name_ $ + schema_ + & description ?~ "Metadata Validation Params" + & example + ?~ toJSON ("{\"url\": \"https://metadata.xyz\", \"hash\": \"9af10e89979e51b8cdc827c963124a1ef4920d1253eef34a1d5cfe76438e3f11\"}" :: Text) + + + data GovActionId = GovActionId { govActionIdTxHash :: HexText @@ -349,6 +445,7 @@ data ProposalResponse , proposalResponseYesVotes :: Integer , proposalResponseNoVotes :: Integer , proposalResponseAbstainVotes :: Integer + , proposalResponseMetadataStatus :: Maybe MetadataValidationResponse } deriving (Generic, Show) @@ -374,7 +471,8 @@ exampleProposalResponse = "{ \"id\": \"proposalId123\"," <> "\"references\": [{\"uri\": \"google.com\", \"@type\": \"Other\", \"label\": \"example label\"}]," <> "\"yesVotes\": 0," <> "\"noVotes\": 0," - <> "\"abstainVotes\": 0}" + <> "\"abstainVotes\": 0" + <> "\"metadataStatus\": {\"status\": null, \"valid\": true}}" instance ToSchema ProposalResponse where declareNamedSchema proxy = do @@ -841,3 +939,4 @@ instance ToSchema GetNetworkMetricsResponse where & description ?~ "GetNetworkMetricsResponse" & example ?~ toJSON exampleGetNetworkMetricsResponse + diff --git a/govtool/backend/src/VVA/Config.hs b/govtool/backend/src/VVA/Config.hs index 7055c33ed..c4781681f 100644 --- a/govtool/backend/src/VVA/Config.hs +++ b/govtool/backend/src/VVA/Config.hs @@ -24,6 +24,8 @@ module VVA.Config , getServerHost , getServerPort , vvaConfigToText + , getMetadataValidationHost + , getMetadataValidationPort ) where import Conferer @@ -69,15 +71,19 @@ instance DefaultConfig DBConfig where data VVAConfigInternal = VVAConfigInternal { -- | db-sync database access. - vVAConfigInternalDbsyncconfig :: DBConfig + vVAConfigInternalDbsyncconfig :: DBConfig -- | Server port. - , vVAConfigInternalPort :: Int + , vVAConfigInternalPort :: Int -- | Server host. - , vVAConfigInternalHost :: Text + , vVAConfigInternalHost :: Text -- | Request cache duration - , vVaConfigInternalCacheDurationSeconds :: Int + , vVaConfigInternalCacheDurationSeconds :: Int -- | Sentry DSN - , vVAConfigInternalSentrydsn :: String + , vVAConfigInternalSentrydsn :: String + -- | Metadata validation service host + , vVAConfigInternalMetadataValidationHost :: Text + -- | Metadata validation service port + , vVAConfigInternalMetadataValidationPort :: Int } deriving (FromConfig, Generic, Show) @@ -88,7 +94,9 @@ instance DefaultConfig VVAConfigInternal where vVAConfigInternalPort = 3000, vVAConfigInternalHost = "localhost", vVaConfigInternalCacheDurationSeconds = 20, - vVAConfigInternalSentrydsn = "https://username:password@senty.host/id" + vVAConfigInternalSentrydsn = "https://username:password@senty.host/id", + vVAConfigInternalMetadataValidationHost = "localhost", + vVAConfigInternalMetadataValidationPort = 3001 } -- | DEX configuration. @@ -104,6 +112,10 @@ data VVAConfig , cacheDurationSeconds :: Int -- | Sentry DSN , sentryDSN :: String + -- | Metadata validation service host + , metadataValidationHost :: Text + -- | Metadata validation service port + , metadataValidationPort :: Int } deriving (Generic, Show, ToJSON) @@ -143,7 +155,9 @@ convertConfig VVAConfigInternal {..} = serverPort = vVAConfigInternalPort, serverHost = vVAConfigInternalHost, cacheDurationSeconds = vVaConfigInternalCacheDurationSeconds, - sentryDSN = vVAConfigInternalSentrydsn + sentryDSN = vVAConfigInternalSentrydsn, + metadataValidationHost = vVAConfigInternalMetadataValidationHost, + metadataValidationPort = vVAConfigInternalMetadataValidationPort } -- | Load configuration from a file specified on the command line. Load from @@ -181,3 +195,15 @@ getServerHost :: (Has VVAConfig r, MonadReader r m) => m Text getServerHost = asks (serverHost . getter) + +-- | Access MetadataValidationService host +getMetadataValidationHost :: + (Has VVAConfig r, MonadReader r m) => + m Text +getMetadataValidationHost = asks (metadataValidationHost . getter) + +-- | Access MetadataValidationService port +getMetadataValidationPort :: + (Has VVAConfig r, MonadReader r m) => + m Int +getMetadataValidationPort = asks (metadataValidationPort . getter) diff --git a/govtool/backend/src/VVA/Metadata.hs b/govtool/backend/src/VVA/Metadata.hs new file mode 100644 index 000000000..d6033633f --- /dev/null +++ b/govtool/backend/src/VVA/Metadata.hs @@ -0,0 +1,45 @@ +{-# LANGUAGE FlexibleContexts #-} +{-# LANGUAGE OverloadedStrings #-} + +module VVA.Metadata where + +import Control.Monad.Except (MonadError, throwError) +import Control.Monad.Reader + +import Data.Aeson (Value, decode, encode, object, (.=)) +import Data.Maybe (fromJust) +import Data.ByteString (ByteString) +import Data.FileEmbed (embedFile) +import Data.Has (Has, getter) +import Data.String (fromString) +import Data.Text (Text, unpack) +import qualified Data.Text.Encoding as Text +import Data.Time.Clock + +import qualified Database.PostgreSQL.Simple as SQL + +import VVA.Config +import VVA.Pool (ConnectionPool, withPool) +import VVA.Types +import Network.HTTP.Client +import Network.HTTP.Client.TLS +import Data.Aeson (encode, object, (.=)) + +validateMetadata + :: (Has VVAConfig r, Has Manager r, MonadReader r m, MonadIO m, MonadError AppError m) + => Text + -> Text + -> m Value +validateMetadata url hash = do + metadataHost <- getMetadataValidationHost + metadataPort <- getMetadataValidationPort + manager <- asks getter + let requestBody = encode $ object ["url" .= unpack url, "hash" .= unpack hash] + initialRequest <- liftIO $ parseRequest (unpack metadataHost <> ":" <> show metadataPort <> "/validate") + let request = initialRequest + { method = "POST" + , requestBody = RequestBodyLBS requestBody + , requestHeaders = [("Content-Type", "application/json")] + } + response <- liftIO $ httpLbs request manager + return $ fromJust $ decode $ responseBody response diff --git a/govtool/backend/src/VVA/Types.hs b/govtool/backend/src/VVA/Types.hs index 7b128045c..fba1e2321 100644 --- a/govtool/backend/src/VVA/Types.hs +++ b/govtool/backend/src/VVA/Types.hs @@ -23,6 +23,7 @@ import Database.PostgreSQL.Simple (Connection) import VVA.Cache import VVA.Config +import Network.HTTP.Client (Manager) type App m = (MonadReader AppEnv m, MonadIO m, MonadFail m, MonadError AppError m) @@ -31,6 +32,7 @@ data AppEnv { vvaConfig :: VVAConfig , vvaCache :: CacheEnv , vvaConnectionPool :: Pool Connection + , vvaTlsManager :: Manager } instance Has VVAConfig AppEnv where @@ -45,10 +47,15 @@ instance Has (Pool Connection) AppEnv where getter AppEnv {vvaConnectionPool} = vvaConnectionPool modifier f a@AppEnv {vvaConnectionPool} = a {vvaConnectionPool = f vvaConnectionPool} +instance Has Manager AppEnv where + getter AppEnv {vvaTlsManager} = vvaTlsManager + modifier f a@AppEnv {vvaTlsManager} = a {vvaTlsManager = f vvaTlsManager} + data AppError = ValidationError Text | NotFoundError Text | CriticalError Text + | InternalError Text deriving (Show) instance Exception AppError @@ -138,6 +145,7 @@ data CacheEnv , dRepVotingPowerCache :: Cache.Cache Text Integer , dRepListCache :: Cache.Cache () [DRepRegistration] , networkMetricsCache :: Cache.Cache () NetworkMetrics + , metadataValidationCache :: Cache.Cache (Text, Text) Value } data NetworkMetrics @@ -159,4 +167,11 @@ data Delegation { delegationDRepHash :: Maybe Text , delegationDRepView :: Text , delegationTxHash :: Text - } \ No newline at end of file + } + + +data MetadataValidationStatus + = IncorrectFormat + | IncorrectJSONLD + | IncorrectHash + | UrlNotFound \ No newline at end of file diff --git a/govtool/backend/vva-be.cabal b/govtool/backend/vva-be.cabal index ccd247dc4..be278ec8f 100644 --- a/govtool/backend/vva-be.cabal +++ b/govtool/backend/vva-be.cabal @@ -98,7 +98,8 @@ library , data-has , resource-pool , swagger2 - + , http-client + , http-client-tls exposed-modules: VVA.Config , VVA.CommandLine @@ -114,3 +115,4 @@ library , VVA.Pool , VVA.Types , VVA.Network + , VVA.Metadata diff --git a/govtool/metadata-validation/src/main.ts b/govtool/metadata-validation/src/main.ts index c27be2ac4..7ab38cb38 100644 --- a/govtool/metadata-validation/src/main.ts +++ b/govtool/metadata-validation/src/main.ts @@ -5,7 +5,7 @@ import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger'; import { AppModule } from './app.module'; async function bootstrap() { - const app = await NestFactory.create(AppModule); + const app = await NestFactory.create(AppModule, { cors: true }); const config = new DocumentBuilder() .setTitle('Submission Tool') From b1ce62ff7188a8296734dc61d1571753861be32f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Sworze=C5=84?= Date: Wed, 8 May 2024 16:17:13 +0200 Subject: [PATCH 3/4] change way to valid metadata --- govtool/frontend/src/models/metadataValidation.ts | 5 ----- govtool/frontend/src/services/API.ts | 12 ++---------- .../requests/metadataValidation/postValidate.ts | 6 +++--- .../src/utils/tests/validateMetadataHash.test.ts | 6 +----- govtool/frontend/src/utils/validateMetadataHash.ts | 3 +-- 5 files changed, 7 insertions(+), 25 deletions(-) diff --git a/govtool/frontend/src/models/metadataValidation.ts b/govtool/frontend/src/models/metadataValidation.ts index bb4a4edcc..5b471ed27 100644 --- a/govtool/frontend/src/models/metadataValidation.ts +++ b/govtool/frontend/src/models/metadataValidation.ts @@ -13,12 +13,7 @@ export type ValidateMetadataResult = { metadata?: any; }; -export enum MetadataStandard { - CIP108 = "CIP108", -} - export type MetadataValidationDTO = { url: string; hash: string; - standard?: MetadataStandard; }; diff --git a/govtool/frontend/src/services/API.ts b/govtool/frontend/src/services/API.ts index 54c09aa7f..3e3d8e1d7 100644 --- a/govtool/frontend/src/services/API.ts +++ b/govtool/frontend/src/services/API.ts @@ -3,19 +3,11 @@ import { NavigateFunction } from "react-router-dom"; import { PATHS } from "@consts"; -const TIMEOUT_IN_SECONDS = 30 * 1000; // 1000 ms is 1 s then its 10 s +const TIMEOUT_IN_SECONDS = 30 * 1000; // 1000 ms is 1 s then its 30 s const BASE_URL = import.meta.env.VITE_BASE_URL; -// Validation should be performed directly on the server -// than no metadata service is needed and `/api` might be removed export const API = axios.create({ - baseURL: `${BASE_URL}/api`, - timeout: TIMEOUT_IN_SECONDS, -}); - -// TODO: Remove this service and use the API service -export const METADATA_VALIDATION_API = axios.create({ - baseURL: `${BASE_URL}/metadata-validation`, + baseURL: `${BASE_URL}`, timeout: TIMEOUT_IN_SECONDS, }); diff --git a/govtool/frontend/src/services/requests/metadataValidation/postValidate.ts b/govtool/frontend/src/services/requests/metadataValidation/postValidate.ts index 4dd95281b..5ff52b8c0 100644 --- a/govtool/frontend/src/services/requests/metadataValidation/postValidate.ts +++ b/govtool/frontend/src/services/requests/metadataValidation/postValidate.ts @@ -1,9 +1,9 @@ import type { MetadataValidationDTO, ValidateMetadataResult } from "@models"; -import { METADATA_VALIDATION_API } from "../../API"; +import { API } from "@services"; export const postValidate = async (body: MetadataValidationDTO) => { - const response = await METADATA_VALIDATION_API.post( - `/validate`, + const response = await API.post( + `/metadata/validate`, body, ); diff --git a/govtool/frontend/src/utils/tests/validateMetadataHash.test.ts b/govtool/frontend/src/utils/tests/validateMetadataHash.test.ts index ea9b1e78e..c1b29a37f 100644 --- a/govtool/frontend/src/utils/tests/validateMetadataHash.test.ts +++ b/govtool/frontend/src/utils/tests/validateMetadataHash.test.ts @@ -1,7 +1,7 @@ import { vi } from "vitest"; import { postValidate } from "@services"; import { checkIsMissingGAMetadata } from ".."; -import { MetadataStandard, MetadataValidationStatus } from "@/models"; +import { MetadataValidationStatus } from "@/models"; const url = "https://example.com"; const hash = "abcdefg"; @@ -28,7 +28,6 @@ describe("checkIsMissingGAMetadata", () => { expect(mockPostValidate).toHaveBeenCalledWith({ url, hash, - standard: MetadataStandard.CIP108, }); }); @@ -44,7 +43,6 @@ describe("checkIsMissingGAMetadata", () => { expect(mockPostValidate).toHaveBeenCalledWith({ url, hash, - standard: MetadataStandard.CIP108, }); }); @@ -60,7 +58,6 @@ describe("checkIsMissingGAMetadata", () => { expect(mockPostValidate).toHaveBeenCalledWith({ url, hash, - standard: MetadataStandard.CIP108, }); }); @@ -73,7 +70,6 @@ describe("checkIsMissingGAMetadata", () => { expect(mockPostValidate).toHaveBeenCalledWith({ url, hash, - standard: MetadataStandard.CIP108, }); }); }); diff --git a/govtool/frontend/src/utils/validateMetadataHash.ts b/govtool/frontend/src/utils/validateMetadataHash.ts index 6868cc3d6..76ac3ecfa 100644 --- a/govtool/frontend/src/utils/validateMetadataHash.ts +++ b/govtool/frontend/src/utils/validateMetadataHash.ts @@ -1,6 +1,6 @@ import { postValidate } from "@services"; -import { MetadataStandard, MetadataValidationStatus } from "@/models"; +import { MetadataValidationStatus } from "@/models"; type CheckIsMissingGAMetadataResponse = { status?: MetadataValidationStatus; @@ -20,7 +20,6 @@ export const checkIsMissingGAMetadata = async ({ const { status, metadata, valid } = await postValidate({ url, hash, - standard: MetadataStandard.CIP108, }); if (status) { return { status, valid }; From 3221dceaff7f918bafd30814c83bbcd34f5e0c3d Mon Sep 17 00:00:00 2001 From: Bartlomiej Sworzen Date: Thu, 9 May 2024 12:58:28 +0200 Subject: [PATCH 4/4] Update API.ts Signed-off-by: Bartlomiej Sworzen --- govtool/frontend/src/services/API.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/govtool/frontend/src/services/API.ts b/govtool/frontend/src/services/API.ts index 3e3d8e1d7..a3ad6398a 100644 --- a/govtool/frontend/src/services/API.ts +++ b/govtool/frontend/src/services/API.ts @@ -7,7 +7,7 @@ const TIMEOUT_IN_SECONDS = 30 * 1000; // 1000 ms is 1 s then its 30 s const BASE_URL = import.meta.env.VITE_BASE_URL; export const API = axios.create({ - baseURL: `${BASE_URL}`, + baseURL: BASE_URL, timeout: TIMEOUT_IN_SECONDS, });