Re-enabled JSON Parsing async channel

The channel is used by vacuum and the validator, it is required for schema validation. but it also slows things down considerably when done synchronously. I have moved this code back to async, it cuts parsing time in half for vaccum, and restores super speed. Signed-off-by: quobix <dave@quobix.com>
pb33f · Nov 9, 2023 · 8b9ef11 · 8b9ef11
1 parent 6a6d6d6
commit 8b9ef11
Show file tree

Hide file tree

Showing 2 changed files with 38 additions and 24 deletions.
diff --git a/datamodel/spec_info.go b/datamodel/spec_info.go
@@ -101,17 +101,19 @@ func ExtractSpecInfoWithDocumentCheck(spec []byte, bypass bool) (*SpecInfo, erro
 			spec.APISchema = OpenAPI2SchemaData
 		}
 
-		if utils.IsYAML(string(bytes)) {
-			_ = parsedNode.Decode(&jsonSpec)
-			b, _ := json.Marshal(&jsonSpec)
-			spec.SpecJSONBytes = &b
-			spec.SpecJSON = &jsonSpec
-		} else {
-			_ = json.Unmarshal(bytes, &jsonSpec)
-			spec.SpecJSONBytes = &bytes
-			spec.SpecJSON = &jsonSpec
-		}
-		close(spec.JsonParsingChannel) // this needs removing at some point
+		go func() {
+			if utils.IsYAML(string(bytes)) {
+				_ = parsedNode.Decode(&jsonSpec)
+				b, _ := json.Marshal(&jsonSpec)
+				spec.SpecJSONBytes = &b
+				spec.SpecJSON = &jsonSpec
+			} else {
+				_ = json.Unmarshal(bytes, &jsonSpec)
+				spec.SpecJSONBytes = &bytes
+				spec.SpecJSON = &jsonSpec
+			}
+			close(spec.JsonParsingChannel)
+		}()
 	}
 
 	if !bypass {
@@ -177,23 +179,25 @@ func ExtractSpecInfoWithDocumentCheck(spec []byte, bypass bool) (*SpecInfo, erro
 
 		if specInfo.SpecType == "" {
 			// parse JSON
-			parseJSON(spec, specInfo, &parsedSpec)
+			go parseJSON(spec, specInfo, &parsedSpec)
 			specInfo.Error = errors.New("spec type not supported by libopenapi, sorry")
 			return specInfo, specInfo.Error
 		}
 	} else {
-		var jsonSpec map[string]interface{}
-		if utils.IsYAML(string(spec)) {
-			_ = parsedSpec.Decode(&jsonSpec)
-			b, _ := json.Marshal(&jsonSpec)
-			specInfo.SpecJSONBytes = &b
-			specInfo.SpecJSON = &jsonSpec
-		} else {
-			_ = json.Unmarshal(spec, &jsonSpec)
-			specInfo.SpecJSONBytes = &spec
-			specInfo.SpecJSON = &jsonSpec
-		}
-		close(specInfo.JsonParsingChannel) // this needs removing at some point
+		go func() {
+			var jsonSpec map[string]interface{}
+			if utils.IsYAML(string(spec)) {
+				_ = parsedSpec.Decode(&jsonSpec)
+				b, _ := json.Marshal(&jsonSpec)
+				specInfo.SpecJSONBytes = &b
+				specInfo.SpecJSON = &jsonSpec
+			} else {
+				_ = json.Unmarshal(spec, &jsonSpec)
+				specInfo.SpecJSONBytes = &spec
+				specInfo.SpecJSON = &jsonSpec
+			}
+			close(specInfo.JsonParsingChannel) // this needs removing at some point
+		}()
 	}
 
 	// detect the original whitespace indentation

diff --git a/datamodel/spec_info_test.go b/datamodel/spec_info_test.go
@@ -116,6 +116,7 @@ info:
 
 func TestExtractSpecInfo_ValidJSON(t *testing.T) {
 	r, e := ExtractSpecInfo([]byte(goodJSON))
+	<-r.JsonParsingChannel
 	assert.Greater(t, len(*r.SpecJSONBytes), 0)
 	assert.Error(t, e)
 }
@@ -132,6 +133,7 @@ func TestExtractSpecInfo_Nothing(t *testing.T) {
 
 func TestExtractSpecInfo_ValidYAML(t *testing.T) {
 	r, e := ExtractSpecInfo([]byte(goodYAML))
+	<-r.JsonParsingChannel
 	assert.Greater(t, len(*r.SpecJSONBytes), 0)
 	assert.Error(t, e)
 }
@@ -149,6 +151,7 @@ func TestExtractSpecInfo_InvalidOpenAPIVersion(t *testing.T) {
 func TestExtractSpecInfo_OpenAPI3(t *testing.T) {
 
 	r, e := ExtractSpecInfo([]byte(OpenApi3Spec))
+	<-r.JsonParsingChannel
 	assert.Nil(t, e)
 	assert.Equal(t, utils.OpenApi3, r.SpecType)
 	assert.Equal(t, "3.0.1", r.Version)
@@ -159,6 +162,7 @@ func TestExtractSpecInfo_OpenAPI3(t *testing.T) {
 func TestExtractSpecInfo_OpenAPIWat(t *testing.T) {
 
 	r, e := ExtractSpecInfo([]byte(OpenApiWat))
+	<-r.JsonParsingChannel
 	assert.Nil(t, e)
 	assert.Equal(t, OpenApi3, r.SpecType)
 	assert.Equal(t, "3.2", r.Version)
@@ -167,6 +171,7 @@ func TestExtractSpecInfo_OpenAPIWat(t *testing.T) {
 func TestExtractSpecInfo_OpenAPI31(t *testing.T) {
 
 	r, e := ExtractSpecInfo([]byte(OpenApi31))
+	<-r.JsonParsingChannel
 	assert.Nil(t, e)
 	assert.Equal(t, OpenApi3, r.SpecType)
 	assert.Equal(t, "3.1", r.Version)
@@ -183,6 +188,7 @@ why:
   yes: no`
 
 	r, e := ExtractSpecInfoWithDocumentCheck([]byte(random), true)
+	<-r.JsonParsingChannel
 	assert.Nil(t, e)
 	assert.NotNil(t, r.RootNode)
 	assert.Equal(t, "something", r.RootNode.Content[0].Content[0].Value)
@@ -194,6 +200,7 @@ func TestExtractSpecInfo_AnyDocument_JSON(t *testing.T) {
 	random := `{ "something" : "yeah"}`
 
 	r, e := ExtractSpecInfoWithDocumentCheck([]byte(random), true)
+	<-r.JsonParsingChannel
 	assert.Nil(t, e)
 	assert.NotNil(t, r.RootNode)
 	assert.Equal(t, "something", r.RootNode.Content[0].Content[0].Value)
@@ -212,6 +219,7 @@ why:
 	r, e := ExtractSpecInfoWithConfig([]byte(random), &DocumentConfiguration{
 		BypassDocumentCheck: true,
 	})
+	<-r.JsonParsingChannel
 	assert.Nil(t, e)
 	assert.NotNil(t, r.RootNode)
 	assert.Equal(t, "something", r.RootNode.Content[0].Content[0].Value)
@@ -228,6 +236,7 @@ func TestExtractSpecInfo_OpenAPIFalse(t *testing.T) {
 func TestExtractSpecInfo_OpenAPI2(t *testing.T) {
 
 	r, e := ExtractSpecInfo([]byte(OpenApi2Spec))
+	<-r.JsonParsingChannel
 	assert.Nil(t, e)
 	assert.Equal(t, OpenApi2, r.SpecType)
 	assert.Equal(t, "2.0.1", r.Version)
@@ -246,6 +255,7 @@ func TestExtractSpecInfo_OpenAPI2_OddVersion(t *testing.T) {
 func TestExtractSpecInfo_AsyncAPI(t *testing.T) {
 
 	r, e := ExtractSpecInfo([]byte(AsyncAPISpec))
+	<-r.JsonParsingChannel
 	assert.Nil(t, e)
 	assert.Equal(t, AsyncApi, r.SpecType)
 	assert.Equal(t, "2.0.0", r.Version)