[#876] add metadata/validate endpoint

CHANGELOG.md

@@ -22,6 +22,7 @@ changes.
 
 ### Added
 
+- added `metadata/validate` endpoint [Issue 876](https://github.com/IntersectMBO/govtool/issues/876)
 - added pagination to `drep/list` [Issue 756](https://github.com/IntersectMBO/govtool/issues/756)
 - added search query param to the `drep/getVotes` [Issue 640](https://github.com/IntersectMBO/govtool/issues/640)
 - added filtering and sorting capabilities to the `drep/list` [Issue 722](https://github.com/IntersectMBO/govtool/issues/722)

govtool/backend/app/Main.hs

@@ -70,8 +70,10 @@ import           VVA.API.Types
 import           VVA.CommandLine
 import           VVA.Config
 import           VVA.Types                              (AppEnv (..),
-                                                         AppError (CriticalError, NotFoundError, ValidationError),
+                                                         AppError (CriticalError, NotFoundError, ValidationError, InternalError),
                                                          CacheEnv (..))
+import Network.HTTP.Client hiding (Proxy, Request)
+import Network.HTTP.Client.TLS
 
 proxyAPI :: Proxy (VVAApi :<|> SwaggerAPI)
 proxyAPI = Proxy
@@ -113,6 +115,7 @@ startApp vvaConfig = do
     dRepVotingPowerCache <- newCache
     dRepListCache <- newCache
     networkMetricsCache <- newCache
+    metadataValidationCache <- newCache
     return $ CacheEnv
       { proposalListCache
       , getProposalCache
@@ -124,10 +127,12 @@ startApp vvaConfig = do
       , dRepVotingPowerCache
       , dRepListCache
       , networkMetricsCache
+      , metadataValidationCache
       }
   connectionPool <- createPool (connectPostgreSQL (encodeUtf8 (dbSyncConnectionString $ getter vvaConfig))) close 1 1 60
+  vvaTlsManager <- newManager tlsManagerSettings
 
-  let appEnv = AppEnv {vvaConfig=vvaConfig, vvaCache=cacheEnv, vvaConnectionPool=connectionPool}
+  let appEnv = AppEnv {vvaConfig=vvaConfig, vvaCache=cacheEnv, vvaConnectionPool=connectionPool, vvaTlsManager}
   server' <- mkVVAServer appEnv
   runSettings settings server'
 
@@ -252,7 +257,7 @@ liftServer appEnv =
     handleErrors (Left (ValidationError msg)) = throwError $ err400 { errBody = BS.fromStrict $ encodeUtf8 msg }
     handleErrors (Left (NotFoundError msg)) = throwError $ err404 { errBody = BS.fromStrict $ encodeUtf8 msg }
     handleErrors (Left (CriticalError msg)) = throwError $ err500 { errBody = BS.fromStrict $ encodeUtf8 msg }
-
+    handleErrors (Left (InternalError msg)) = throwError $ err500 { errBody = BS.fromStrict $ encodeUtf8 msg }
 -- * Swagger
 
 type SwaggerAPI = SwaggerSchemaUI "swagger-ui" "swagger.json"

govtool/backend/example-config.json

@@ -9,5 +9,7 @@
     "port" : 9999,
     "host" : "localhost",
     "cachedurationseconds": 20,
-    "sentrydsn": "https://username:password@senty.host/id"
+    "sentrydsn": "https://username:password@senty.host/id",
+    "metadatavalidationhost": "localhost",
+    "metadatavalidationport": 3001
 }

govtool/backend/src/VVA/API.hs

@@ -11,7 +11,7 @@ module VVA.API where
 import           Control.Exception    (throw)
 import           Control.Monad.Except (throwError)
 import           Control.Monad.Reader
-
+import           Data.Aeson           (Result(Error, Success), fromJSON)
 import           Data.Bool            (Bool)
 import           Data.List            (sortOn)
 import qualified Data.Map             as Map
@@ -39,8 +39,9 @@ import qualified VVA.Proposal         as Proposal
 import qualified VVA.Transaction      as Transaction
 import qualified VVA.Types            as Types
 import           VVA.Types            (App, AppEnv (..),
-                                       AppError (CriticalError, ValidationError),
+                                       AppError (CriticalError, ValidationError, InternalError),
                                        CacheEnv (..))
+import qualified VVA.Metadata         as Metadata
 
 type VVAApi =
          "drep" :> "list"
@@ -73,6 +74,7 @@ type VVAApi =
     :<|> "transaction" :> "status" :> Capture "transactionId" HexText :> Get '[JSON] GetTransactionStatusResponse
     :<|> "throw500" :> Get '[JSON] ()
     :<|> "network" :> "metrics" :> Get '[JSON] GetNetworkMetricsResponse
+    :<|> "metadata" :> "validate" :> ReqBody '[JSON] MetadataValidationParams :> Post '[JSON] MetadataValidationResponse
 
 server :: App m => ServerT VVAApi m
 server = drepList
@@ -87,6 +89,7 @@ server = drepList
     :<|> getTransactionStatus
     :<|> throw500
     :<|> getNetworkMetrics
+    :<|> validateMetadata
 
 
 mapDRepType :: Types.DRepType -> DRepType
@@ -174,8 +177,8 @@ getVotingPower (unHexText -> dRepId) = do
   cacheRequest dRepVotingPowerCache dRepId $ DRep.getVotingPower dRepId
 
 
-proposalToResponse :: Types.Proposal -> ProposalResponse
-proposalToResponse Types.Proposal {..} =
+proposalToResponse :: Types.Proposal -> MetadataValidationResponse -> ProposalResponse
+proposalToResponse Types.Proposal {..} metadataValidationResponse =
   ProposalResponse
   { proposalResponseId = pack $ show proposalId,
     proposalResponseTxHash = HexText proposalTxHash,
@@ -196,7 +199,8 @@ proposalToResponse Types.Proposal {..} =
     proposalResponseReferences = GovernanceActionReferences <$> proposalReferences,
     proposalResponseYesVotes = proposalYesVotes,
     proposalResponseNoVotes = proposalNoVotes,
-    proposalResponseAbstainVotes = proposalAbstainVotes
+    proposalResponseAbstainVotes = proposalAbstainVotes,
+    proposalResponseMetadataStatus = Just metadataValidationResponse
   }
 
 voteToResponse :: Types.Vote -> VoteParams
@@ -214,16 +218,17 @@ voteToResponse Types.Vote {..} =
 
 
 mapSortAndFilterProposals
-  :: [GovernanceActionType]
+  :: App m
+  => [GovernanceActionType]
   -> Maybe GovernanceActionSortMode
   -> [Types.Proposal]
-  -> [ProposalResponse]
-mapSortAndFilterProposals selectedTypes sortMode proposals =
-  let mappedProposals =
-        map
-          proposalToResponse
+  -> m [ProposalResponse]
+mapSortAndFilterProposals selectedTypes sortMode proposals = do
+  mappedProposals <-
+        mapM
+          (\proposal@Types.Proposal {proposalUrl, proposalDocHash} -> proposalToResponse proposal <$> validateMetadata (MetadataValidationParams proposalUrl $ HexText proposalDocHash))
           proposals
-      filteredProposals =
+  let filteredProposals =
         if null selectedTypes
           then mappedProposals
           else
@@ -232,19 +237,19 @@ mapSortAndFilterProposals selectedTypes sortMode proposals =
                   proposalResponseType `elem` selectedTypes
               )
               mappedProposals
-      sortedProposals = case sortMode of
+  let sortedProposals = case sortMode of
         Nothing              -> filteredProposals
         Just NewestCreated   -> sortOn (Down . proposalResponseCreatedDate) filteredProposals
         Just SoonestToExpire -> sortOn proposalResponseExpiryDate filteredProposals
         Just MostYesVotes    -> sortOn (Down . proposalResponseYesVotes) filteredProposals
-  in sortedProposals
+  return sortedProposals
 
 getVotes :: App m => HexText -> [GovernanceActionType] -> Maybe GovernanceActionSortMode -> Maybe Text -> m [VoteResponse]
 getVotes (unHexText -> dRepId) selectedTypes sortMode mSearch = do
   CacheEnv {dRepGetVotesCache} <- asks vvaCache
   (votes, proposals) <- cacheRequest dRepGetVotesCache dRepId $ DRep.getVotes dRepId []
   let voteMap = Map.fromList $ map (\vote@Types.Vote {..} -> (voteProposalId, vote)) votes
-  let processedProposals = filter (isProposalSearchedFor mSearch) $ mapSortAndFilterProposals selectedTypes sortMode proposals
+  processedProposals <- filter (isProposalSearchedFor mSearch) <$> mapSortAndFilterProposals selectedTypes sortMode proposals
   return $
     [ VoteResponse
       { voteResponseVote = voteToResponse (voteMap Map.! read (unpack proposalResponseId))
@@ -321,12 +326,14 @@ listProposals selectedTypes sortMode mPage mPageSize mDrepRaw mSearchQuery = do
 
 
   CacheEnv {proposalListCache} <- asks vvaCache
-  mappedAndSortedProposals <-
-    filter
+  mappedAndSortedProposals <- do
+    proposals <- cacheRequest proposalListCache () Proposal.listProposals
+    mappedSortedAndFilteredProposals <- mapSortAndFilterProposals selectedTypes sortMode proposals
+    return $ filter
       ( \p@ProposalResponse {proposalResponseId} ->
           proposalResponseId `notElem` proposalsToRemove
           && isProposalSearchedFor mSearchQuery p
-      ) . mapSortAndFilterProposals selectedTypes sortMode <$> cacheRequest proposalListCache () Proposal.listProposals
+      ) mappedSortedAndFilteredProposals
 
   let total = length mappedAndSortedProposals :: Int
 
@@ -343,7 +350,9 @@ getProposal :: App m => GovActionId -> Maybe HexText -> m GetProposalResponse
 getProposal g@(GovActionId govActionTxHash govActionIndex) mDrepId' = do
   let mDrepId = unHexText <$> mDrepId'
   CacheEnv {getProposalCache} <- asks vvaCache
-  proposalResponse <- proposalToResponse <$> cacheRequest getProposalCache (unHexText govActionTxHash, govActionIndex) (Proposal.getProposal (unHexText govActionTxHash) govActionIndex)
+  proposal@Types.Proposal {proposalUrl, proposalDocHash} <- cacheRequest getProposalCache (unHexText govActionTxHash, govActionIndex) (Proposal.getProposal (unHexText govActionTxHash) govActionIndex)
+  metadataStatus <- validateMetadata $ MetadataValidationParams proposalUrl $ HexText proposalDocHash
+  let proposalResponse = proposalToResponse proposal metadataStatus
   voteResponse <- case mDrepId of
     Nothing -> return Nothing
     Just drepId -> do
@@ -390,3 +399,13 @@ getNetworkMetrics = do
     , getNetworkMetricsResponseAlwaysAbstainVotingPower = networkMetricsAlwaysAbstainVotingPower
     , getNetworkMetricsResponseAlwaysNoConfidenceVotingPower = networkMetricsAlwaysNoConfidenceVotingPower
     }
+
+validateMetadata :: App m => MetadataValidationParams -> m MetadataValidationResponse
+validateMetadata MetadataValidationParams {..} = do
+  CacheEnv {metadataValidationCache} <- asks vvaCache
+  result <- cacheRequest metadataValidationCache (metadataValidationParamsUrl, unHexText metadataValidationParamsHash)
+      $ Metadata.validateMetadata metadataValidationParamsUrl (unHexText metadataValidationParamsHash)
+
+  case fromJSON result of
+    Error e -> throwError $ InternalError $ pack $ show e
+    Success a -> return a

govtool/backend/src/VVA/API/Types.hs

@@ -90,6 +90,102 @@ instance ToSchema HexText where
       & schema . format ?~ "hex"
       & schema . example ?~ toJSON (HexText "a1b2c3")
 
+newtype AnyValue
+  = AnyValue { unAnyValue :: Maybe Value }
+  deriving newtype (Show)
+
+instance FromJSON AnyValue where
+  parseJSON = pure . AnyValue . Just
+
+instance ToJSON AnyValue where
+  toJSON (AnyValue Nothing)       = Null
+  toJSON (AnyValue (Just params)) = toJSON params
+
+exampleAnyValue :: Text
+exampleAnyValue =
+  "{ \"any\": \"value\"}"
+
+instance ToSchema AnyValue where
+    declareNamedSchema _ = pure $ NamedSchema (Just "AnyValue") $ mempty
+        & type_ ?~ OpenApiObject
+        & description ?~ "Any value"
+        & example
+          ?~ toJSON exampleAnyValue
+
+data MetadataValidationStatus
+  = IncorrectFormat
+  | IncorrectJSONLD
+  | IncorrectHash
+  | UrlNotFound
+  deriving (Show, Eq)
+
+instance ToJSON MetadataValidationStatus where
+  toJSON IncorrectFormat = "INCORRECT_FORMTAT"
+  toJSON IncorrectJSONLD = "INVALID_JSONLD"
+  toJSON IncorrectHash = "INVALID_HASH"
+  toJSON UrlNotFound = "URL_NOT_FOUND"
+
+instance FromJSON MetadataValidationStatus where
+  parseJSON (String s) = case s of
+    "INCORRECT_FORMTAT" -> pure IncorrectFormat
+    "INVALID_JSONLD" -> pure IncorrectJSONLD
+    "INVALID_HASH" -> pure IncorrectHash
+    "URL_NOT_FOUND" -> pure UrlNotFound
+    _ -> fail "Invalid MetadataValidationStatus"
+  parseJSON _ = fail "Invalid MetadataValidationStatus"
+
+instance ToSchema MetadataValidationStatus where
+    declareNamedSchema _ = pure $ NamedSchema (Just "MetadataValidationStatus") $ mempty
+        & type_ ?~ OpenApiString
+        & description ?~ "Metadata Validation Status"
+        & enum_ ?~ map toJSON [IncorrectFormat, IncorrectJSONLD, IncorrectHash, UrlNotFound]
+
+data MetadataValidationResponse
+  = MetadataValidationResponse
+      { metadataValidationResponseStatus :: Maybe MetadataValidationStatus
+      , metadataValidationResponseValid :: Bool
+      }
+  deriving (Generic, Show)
+
+deriveJSON (jsonOptions "metadataValidationResponse") ''MetadataValidationResponse
+
+instance ToSchema MetadataValidationResponse where
+    declareNamedSchema _ = do
+      NamedSchema name_ schema_ <-
+        genericDeclareNamedSchema
+        ( fromAesonOptions $ jsonOptions "metadataValidationResponse" )
+        (Proxy :: Proxy MetadataValidationResponse)
+      return $
+        NamedSchema name_ $
+          schema_
+            & description ?~ "Metadata Validation Response"
+            & example
+              ?~ toJSON ("{\"status\": \"INCORRECT_FORMTAT\", \"valid\":false}" :: Text)
+
+data MetadataValidationParams
+  = MetadataValidationParams
+      { metadataValidationParamsUrl :: Text
+      , metadataValidationParamsHash :: HexText
+      }
+  deriving (Generic, Show)
+
+deriveJSON (jsonOptions "metadataValidationParams") ''MetadataValidationParams
+
+instance ToSchema MetadataValidationParams where
+    declareNamedSchema proxy = do
+      NamedSchema name_ schema_ <-
+        genericDeclareNamedSchema
+        ( fromAesonOptions $ jsonOptions "metadataValidationParams" )
+        proxy
+      return $
+        NamedSchema name_ $
+          schema_
+            & description ?~ "Metadata Validation Params"
+            & example
+              ?~ toJSON ("{\"url\": \"https://metadata.xyz\", \"hash\": \"9af10e89979e51b8cdc827c963124a1ef4920d1253eef34a1d5cfe76438e3f11\"}" :: Text)
+
+
+
 data GovActionId
   = GovActionId
       { govActionIdTxHash :: HexText
@@ -349,6 +445,7 @@ data ProposalResponse
       , proposalResponseYesVotes       :: Integer
       , proposalResponseNoVotes        :: Integer
       , proposalResponseAbstainVotes   :: Integer
+      , proposalResponseMetadataStatus :: Maybe MetadataValidationResponse
       }
   deriving (Generic, Show)
 
@@ -374,7 +471,8 @@ exampleProposalResponse = "{ \"id\": \"proposalId123\","
                   <> "\"references\": [{\"uri\": \"google.com\", \"@type\": \"Other\", \"label\": \"example label\"}],"
                   <> "\"yesVotes\": 0,"
                   <> "\"noVotes\": 0,"
-                  <> "\"abstainVotes\": 0}"
+                  <> "\"abstainVotes\": 0"
+                  <> "\"metadataStatus\": {\"status\": null, \"valid\": true}}"
 
 instance ToSchema ProposalResponse where
   declareNamedSchema proxy = do
@@ -841,3 +939,4 @@ instance ToSchema GetNetworkMetricsResponse where
         & description ?~ "GetNetworkMetricsResponse"
         & example
           ?~ toJSON exampleGetNetworkMetricsResponse
+