adding new mage target to dereference schemas

go.mod

@@ -25,7 +25,9 @@ require (
 )
 
 require (
+	github.com/buger/jsonparser v1.1.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/flowstack-com/jsonschema v0.1.2 // indirect
 	github.com/go-playground/locales v0.14.0 // indirect
 	github.com/go-playground/universal-translator v0.18.0 // indirect
 	github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect
@@ -41,6 +43,7 @@ require (
 	github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
+	golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect
 	golang.org/x/sys v0.0.0-20221006211917-84dc82d7e875 // indirect
 	golang.org/x/term v0.0.0-20220919170432-7a66f970e087 // indirect
 	golang.org/x/text v0.3.7 // indirect

go.sum

@@ -1,5 +1,7 @@
 github.com/bits-and-blooms/bitset v1.3.3 h1:R1XWiopGiXf66xygsiLpzLo67xEYvMkHw3w+rCOSAwg=
 github.com/bits-and-blooms/bitset v1.3.3/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
+github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
+github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -8,6 +10,8 @@ github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn
 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d/go.mod h1:tmAIfUFEirG/Y8jhZ9M+h36obRZAk/1fcSpXwAVlfqE=
 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4=
 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc=
+github.com/flowstack-com/jsonschema v0.1.2 h1:qO2Ed0OHxY88yqqfNwUGfWxF64rVQCdbJ68lmlHSIXs=
+github.com/flowstack-com/jsonschema v0.1.2/go.mod h1:Ce82M1ocyTfow5soqyAcHShvlSUqM8odW0UZVIIK4pQ=
 github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU=
@@ -99,6 +103,8 @@ golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 h1:Y/gsMcFOcR+6S6f3YeMKl5
 golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20221005025214-4161e89ecf1b h1:huxqepDufQpLLIRXiVkTvnxrzJlpwmIWAObmcCcUFr0=
 golang.org/x/crypto v0.0.0-20221005025214-4161e89ecf1b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

magefile.go

@@ -5,6 +5,7 @@ package main
 import (
 	"fmt"
 	"io"
+	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
@@ -14,15 +15,17 @@ import (
 	"strings"
 	"syscall"
 
+	"github.com/flowstack-com/jsonschema"
 	"github.com/magefile/mage/mg"
 	"github.com/magefile/mage/sh"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/crypto/ssh/terminal"
 )
 
 const (
-	Go       = "go"
-	gomobile = "gomobile"
+	Go              = "go"
+	gomobile        = "gomobile"
+	schemaDirectory = "./schema/known_schemas/"
 )
 
 // Build builds the library.
@@ -281,3 +284,62 @@ func Vuln() error {
 func installGoVulnIfNotPresent() error {
 	return installIfNotPresent("govulncheck", "golang.org/x/vuln/cmd/govulncheck@latest")
 }
+
+// DerefSchemas takes our known schemas and dereferences the schema's $ref http links to be a part of the json schema object.
+// This makes our code faster when doing validation checks and allows us to not ping outside sources for schemas refs which may go down or change.
+// TODO: (Neal) Currently we do not use these dereferenced schemas in code because there is more work to be done here.
+// Currently these dereferenced schemas are missing some information and fail validation with our known json objects
+// I believe some more work in the investigation library needs to be done and we need to handle circular dependencies
+func DerefSchemas() error {
+	files, err := ioutil.ReadDir(schemaDirectory)
+	if err != nil {
+		logrus.WithError(err).Fatal("problem reading directory at: " + schemaDirectory)
+		return err
+	}
+
+	os.Chmod(schemaDirectory, 0777)
+
+	for _, file := range files {
+
+		// dont deref already deref'd json schemas
+		if strings.Contains(file.Name(), "-deref") {
+			continue
+		}
+
+		logrus.Println("dereferenceing file at: " + file.Name())
+
+		fileBytes, err := os.ReadFile(schemaDirectory + file.Name())
+		if err != nil {
+			logrus.WithError(err).Fatal("problem reading file at: " + schemaDirectory + file.Name())
+			continue
+		}
+
+		sch, err := jsonschema.New(fileBytes)
+		if err != nil {
+			logrus.WithError(err).Fatal("problem creating schema")
+			continue
+		}
+
+		// dereference schema
+		err = sch.DeRef()
+		if err != nil {
+			logrus.WithError(err).Fatal("problem dereferenceing schema")
+			continue
+		}
+
+		schemaBytes, err := sch.MarshalJSON()
+		if err != nil {
+			logrus.WithError(err).Fatal("problem marshalling schema json")
+			continue
+		}
+
+		err = os.WriteFile("./schema/known_schemas/"+strings.ReplaceAll(file.Name(), ".json", "")+"-deref.json", schemaBytes, 0644)
+		if err != nil {
+			logrus.WithError(err).Fatal("problem writing deref json to file")
+			continue
+		}
+
+	}
+	logrus.Println("\n\nFinished dereferenceing schemas")
+	return nil
+}

schema/known_schemas/cm-credential-application-deref.json

@@ -0,0 +1 @@
+{"$schema":"http://json-schema.org/draft-07/schema#","title":"Credential Application","type":"object","definitions":{"descriptor":{"type":"object","properties":{"format":{"type":"string","enum":["jwt","jwt_vc","jwt_vp","ldp","ldp_vc","ldp_vp"]},"id":{"type":"string"},"path":{"type":"string"},"path_nested":{"type":"object","properties":{"format":{"type":"string","enum":["jwt","jwt_vc","jwt_vp","ldp","ldp_vc","ldp_vp"]},"id":{"type":"string"},"path":{"type":"string"},"path_nested":{"$ref":"#/definitions/descriptor","type":"object"}},"required":["id","path","format"],"additionalProperties":false}},"required":["id","path","format"],"additionalProperties":false}},"properties":{"format":{"type":"object","patternProperties":{"^jwt$|^jwt_vc$|^jwt_vp$":{"type":"object","properties":{"alg":{"type":"array","items":{"type":"string"},"minItems":1}},"required":["alg"],"additionalProperties":false},"^ldp_vc$|^ldp_vp$|^ldp$":{"type":"object","properties":{"proof_type":{"type":"array","items":{"type":"string"},"minItems":1}},"required":["proof_type"],"additionalProperties":false}},"additionalProperties":false},"id":{"type":"string"},"manifest_id":{"type":"string"},"presentation_submission":{"type":"object","properties":{"definition_id":{"type":"string"},"descriptor_map":{"type":"array","items":{"type":"object","properties":{"format":{"type":"string","enum":["jwt","jwt_vc","jwt_vp","ldp","ldp_vc","ldp_vp"]},"id":{"type":"string"},"path":{"type":"string"},"path_nested":{"$ref":"#/definitions/descriptor","type":"object"}},"required":["id","path","format"],"additionalProperties":false}},"id":{"type":"string"}},"required":["id","definition_id","descriptor_map"],"additionalProperties":false},"spec_version":{"type":"string"}},"required":["id","spec_version","manifest_id","format"],"additionalProperties":false}

schema/known_schemas/cm-credential-manifest-deref.json

@@ -0,0 +1 @@
+{"$schema":"http://json-schema.org/draft-07/schema","title":"Credential Manifest","type":"object","properties":{"format":{"$schema":"http://json-schema.org/draft-07/schema#","$id":"https://identity.foundation/claim-format-registry/schemas/presentation-definition-claim-format-designations.json","title":"Presentation Definition Claim Format Designations","type":"object","patternProperties":{"^jwt$|^jwt_vc$|^jwt_vp$":{"type":"object","properties":{"alg":{"type":"array","items":{"type":"string"},"minItems":1}},"additionalProperties":false},"^ldp_vc$|^ldp_vp$|^ldp$":{"type":"object","properties":{"proof_type":{"type":"array","items":{"type":"string"},"minItems":1}},"additionalProperties":false}},"additionalProperties":false},"id":{"type":"string"},"issuer":{"type":"object","properties":{"id":{"type":"string"},"name":{"type":"string"},"styles":{"$schema":"http://json-schema.org/draft-07/schema#","$id":"https://identity.foundation/wallet-rendering/schemas/entity-styles.json","title":"Entity Styles","type":"object","definitions":{"color":{"type":"object","properties":{"color":{"type":"string","pattern":"^#([A-Fa-f0-9]{6}|[A-Fa-f0-9]{3})$"}},"required":["color"]},"image":{"type":"object","properties":{"alt":{"type":"string"},"uri":{"type":"string","format":"uri"}},"required":["uri","alt"]}},"properties":{"background":{"$ref":"#/definitions/color"},"hero":{"$ref":"#/definitions/image"},"text":{"$ref":"#/definitions/color"},"thumbnail":{"$ref":"#/definitions/image"}}}},"required":["id"],"additionalProperties":false},"output_descriptors":{"type":"array","items":{"type":"object","properties":{"description":{"type":"string"},"display":{"type":"object","properties":{"description":{"$schema":"http://json-schema.org/draft-07/schema#","$id":"https://identity.foundation/wallet-rendering/schemas/display-mapping-object.json","title":"Display Mapping Object","oneOf":[{"type":"object","properties":{"fallback":{"type":"string"},"path":{"type":"array","items":{"type":"string"}},"schema":{"oneOf":[{"type":"object","properties":{"type":{"type":"string","enum":["boolean","number","integer"]}},"required":["type"],"additionalProperties":false},{"anyOf":[{"type":"object","properties":{"format":{"type":"string","enum":["date-time","time","date","email","idn-email","hostname","idn-hostname","ipv4","ipv6","uri","uri-reference","iri","iri-reference"]},"type":{"type":"string","enum":["string"]}},"required":["type"],"additionalProperties":false},{"type":"object","properties":{"contentEncoding":{"type":"string","enum":["7bit","8bit","binary","quoted-printable","base16","base32","base64"]},"contentMediaType":{"type":"string"},"type":{"type":"string","enum":["string"]}},"required":["type"],"additionalProperties":false}]}]}},"required":["path","schema"],"additionalProperties":false},{"type":"object","properties":{"text":{"type":"string"}},"required":["text"],"additionalProperties":false}]},"properties":{"type":"array","items":{"$schema":"http://json-schema.org/draft-07/schema#","$id":"https://identity.foundation/wallet-rendering/schemas/labeled-display-mapping-object.json","title":"Labeled Display Mapping Object","oneOf":[{"type":"object","properties":{"fallback":{"type":"string"},"label":{"type":"string"},"path":{"type":"array","items":{"type":"string"}},"schema":{"oneOf":[{"type":"object","properties":{"type":{"type":"string","enum":["boolean","number","integer"]}},"required":["type"],"additionalProperties":false},{"anyOf":[{"type":"object","properties":{"format":{"type":"string","enum":["date-time","time","date","email","idn-email","hostname","idn-hostname","ipv4","ipv6","uri","uri-reference","iri","iri-reference"]},"type":{"type":"string","enum":["string"]}},"required":["type"],"additionalProperties":false},{"type":"object","properties":{"contentEncoding":{"type":"string","enum":["7bit","8bit","binary","quoted-printable","base16","base32","base64"]},"contentMediaType":{"type":"string"},"type":{"type":"string","enum":["string"]}},"required":["type"],"additionalProperties":false}]}]}},"required":["path","schema","label"],"additionalProperties":false},{"type":"object","properties":{"label":{"type":"string"},"text":{"type":"string"}},"required":["text","label"],"additionalProperties":false}]}},"subtitle":{"$schema":"http://json-schema.org/draft-07/schema#","$id":"https://identity.foundation/wallet-rendering/schemas/display-mapping-object.json","title":"Display Mapping Object","oneOf":[{"type":"object","properties":{"fallback":{"type":"string"},"path":{"type":"array","items":{"type":"string"}},"schema":{"oneOf":[{"type":"object","properties":{"type":{"type":"string","enum":["boolean","number","integer"]}},"required":["type"],"additionalProperties":false},{"anyOf":[{"type":"object","properties":{"format":{"type":"string","enum":["date-time","time","date","email","idn-email","hostname","idn-hostname","ipv4","ipv6","uri","uri-reference","iri","iri-reference"]},"type":{"type":"string","enum":["string"]}},"required":["type"],"additionalProperties":false},{"type":"object","properties":{"contentEncoding":{"type":"string","enum":["7bit","8bit","binary","quoted-printable","base16","base32","base64"]},"contentMediaType":{"type":"string"},"type":{"type":"string","enum":["string"]}},"required":["type"],"additionalProperties":false}]}]}},"required":["path","schema"],"additionalProperties":false},{"type":"object","properties":{"text":{"type":"string"}},"required":["text"],"additionalProperties":false}]},"title":{"$schema":"http://json-schema.org/draft-07/schema#","$id":"https://identity.foundation/wallet-rendering/schemas/display-mapping-object.json","title":"Display Mapping Object","oneOf":[{"type":"object","properties":{"fallback":{"type":"string"},"path":{"type":"array","items":{"type":"string"}},"schema":{"oneOf":[{"type":"object","properties":{"type":{"type":"string","enum":["boolean","number","integer"]}},"required":["type"],"additionalProperties":false},{"anyOf":[{"type":"object","properties":{"format":{"type":"string","enum":["date-time","time","date","email","idn-email","hostname","idn-hostname","ipv4","ipv6","uri","uri-reference","iri","iri-reference"]},"type":{"type":"string","enum":["string"]}},"required":["type"],"additionalProperties":false},{"type":"object","properties":{"contentEncoding":{"type":"string","enum":["7bit","8bit","binary","quoted-printable","base16","base32","base64"]},"contentMediaType":{"type":"string"},"type":{"type":"string","enum":["string"]}},"required":["type"],"additionalProperties":false}]}]}},"required":["path","schema"],"additionalProperties":false},{"type":"object","properties":{"text":{"type":"string"}},"required":["text"],"additionalProperties":false}]}},"additionalProperties":false},"id":{"type":"string"},"name":{"type":"string"},"schema":{"type":"string","format":"uri"},"styles":{"$schema":"http://json-schema.org/draft-07/schema#","$id":"https://identity.foundation/wallet-rendering/schemas/entity-styles.json","title":"Entity Styles","type":"object","definitions":{"color":{"type":"object","properties":{"color":{"type":"string","pattern":"^#([A-Fa-f0-9]{6}|[A-Fa-f0-9]{3})$"}},"required":["color"]},"image":{"type":"object","properties":{"alt":{"type":"string"},"uri":{"type":"string","format":"uri"}},"required":["uri","alt"]}},"properties":{"background":{"$ref":"#/definitions/color"},"hero":{"$ref":"#/definitions/image"},"text":{"$ref":"#/definitions/color"},"thumbnail":{"$ref":"#/definitions/image"}}}},"required":["id","schema"],"additionalProperties":false}},"presentation_definition":{"$schema":"http://json-schema.org/draft-07/schema#","$id":"https://identity.foundation/presentation-exchange/schemas/presentation-definition.json","title":"Presentation Definition","type":"object","definitions":{"field":{"type":"object","oneOf":[{"properties":{"filter":{"$ref":"http://json-schema.org/draft-07/schema#"},"id":{"type":"string"},"intent_to_retain":{"type":"boolean"},"path":{"type":"array","items":{"type":"string"}},"purpose":{"type":"string"}},"required":["path"],"additionalProperties":false},{"properties":{"filter":{"$ref":"http://json-schema.org/draft-07/schema#"},"id":{"type":"string"},"intent_to_retain":{"type":"boolean"},"path":{"type":"array","items":{"type":"string"}},"predicate":{"type":"string","enum":["required","preferred"]},"purpose":{"type":"string"}},"required":["path","filter","predicate"],"additionalProperties":false}]},"input_descriptor":{"type":"object","properties":{"constraints":{"type":"object","properties":{"fields":{"type":"array","items":{"$ref":"#/definitions/field"}},"is_holder":{"type":"array","items":{"type":"object","properties":{"directive":{"type":"string","enum":["required","preferred"]},"field_id":{"type":"array","items":{"type":"string"}}},"required":["field_id","directive"],"additionalProperties":false}},"limit_disclosure":{"type":"string","enum":["required","preferred"]},"same_subject":{"type":"array","items":{"type":"object","properties":{"directive":{"type":"string","enum":["required","preferred"]},"field_id":{"type":"array","items":{"type":"string"}}},"required":["field_id","directive"],"additionalProperties":false}},"statuses":{"type":"object","properties":{"active":{"$ref":"#/definitions/status_directive"},"revoked":{"$ref":"#/definitions/status_directive"},"suspended":{"$ref":"#/definitions/status_directive"}},"additionalProperties":false},"subject_is_issuer":{"type":"string","enum":["required","preferred"]}},"additionalProperties":false},"format":{"$ref":"http://identity.foundation/claim-format-registry/schemas/presentation-definition-claim-format-designations.json"},"group":{"type":"array","items":{"type":"string"}},"id":{"type":"string"},"name":{"type":"string"},"purpose":{"type":"string"}},"required":["id","constraints"],"additionalProperties":false},"status_directive":{"type":"object","properties":{"directive":{"type":"string","enum":["required","allowed","disallowed"]},"type":{"type":"array","items":{"type":"string"},"minItems":1}},"additionalProperties":false},"submission_requirement":{"type":"object","oneOf":[{"properties":{"count":{"type":"integer","minimum":1},"from":{"type":"string"},"max":{"type":"integer","minimum":0},"min":{"type":"integer","minimum":0},"name":{"type":"string"},"purpose":{"type":"string"},"rule":{"type":"string","enum":["all","pick"]}},"required":["rule","from"],"additionalProperties":false},{"properties":{"count":{"type":"integer","minimum":1},"from_nested":{"type":"array","items":{"$ref":"#/definitions/submission_requirement"},"minItems":1},"max":{"type":"integer","minimum":0},"min":{"type":"integer","minimum":0},"name":{"type":"string"},"purpose":{"type":"string"},"rule":{"type":"string","enum":["all","pick"]}},"required":["rule","from_nested"],"additionalProperties":false}]}},"properties":{"format":{"$ref":"http://identity.foundation/claim-format-registry/schemas/presentation-definition-claim-format-designations.json"},"frame":{"type":"object","additionalProperties":true},"id":{"type":"string"},"input_descriptors":{"type":"array","items":{"$ref":"#/definitions/input_descriptor"}},"name":{"type":"string"},"purpose":{"type":"string"},"submission_requirements":{"type":"array","items":{"$ref":"#/definitions/submission_requirement"}}},"required":["id","input_descriptors"],"additionalProperties":false},"spec_version":{"type":"string"}},"required":["id","spec_version","issuer","output_descriptors"],"additionalProperties":false}