[Java.Interop.Tools.JavaSource] Fix remaining parsing errors (#1036)

All 431 instances of `JavadocImport-Error` that would occur when generating API docs for `Mono.Android.dll` have been fixed. Problematic scenarios that have been fixed include: * `@hide` and `@inheritDoc` tags with trailing content * `<tt>` or `<i>` elements with no closing tag * `<pre>` elements with an attribute and/or no closing elements * `<a name="">` or `<a id="">` elements inside of an unclosed `<p>` element * `<a href="">` elements with mixed casing or spacing * `{@link foo` tags without a closing bracket * Any extra occurrences of `@`, `{`, or `}`
dotnet · Sep 28, 2022 · a0728ed · a0728ed
1 parent 9e858bb
commit a0728ed
Show file tree

Hide file tree

Showing 7 changed files with 677 additions and 12 deletions.
diff --git a/...avaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.BlockTagsBnfTerms.cs b/...avaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.BlockTagsBnfTerms.cs
@@ -102,12 +102,16 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar)
 				};
 
 				// Ignore @hide tags
-				HideDeclaration.Rule = "@hide";
+				HideDeclaration.Rule = "@hide"
+					| "@hide" + BlockValues
+					;
 				HideDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
 					FinishParse (context, parseNode);
 				};
 
-				InheritDocDeclaration.Rule = "@inheritDoc";
+				InheritDocDeclaration.Rule = "@inheritDoc"
+					| "@inheritDoc" + BlockValues
+					;
 				InheritDocDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
 					if (!grammar.ShouldImport (ImportJavadoc.InheritDocTag)) {
 						return;

diff --git a/...ols.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.HtmlBnfTerms.cs b/...ols.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.HtmlBnfTerms.cs
@@ -23,6 +23,7 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar)
 				AllHtmlTerms.Rule = TopLevelInlineDeclaration
 					| PBlockDeclaration
 					| PreBlockDeclaration
+					| IgnorableElementDeclaration
 					;
 
 				var inlineDeclaration = new NonTerminal ("<html inline decl>", ConcatChildNodes) {
@@ -59,11 +60,11 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar)
 					parseNode.AstNode       = "";
 				};
 
-				var fontstyle_tt    = CreateHtmlToCrefElement (grammar, "tt",   "c", InlineDeclarations);
-				var fontstyle_i     = CreateHtmlToCrefElement (grammar, "i",    "i", InlineDeclarations);
+				var fontstyle_tt    = CreateHtmlToCrefElement (grammar, "tt",   "c", InlineDeclarations, optionalEnd: true);
+				var fontstyle_i     = CreateHtmlToCrefElement (grammar, "i",    "i", InlineDeclarations, optionalEnd: true);
 
 				var preText = new PreBlockDeclarationBodyTerminal ();
-				PreBlockDeclaration.Rule = CreateStartElement ("pre", grammar) + preText + CreateEndElement ("pre", grammar);
+				PreBlockDeclaration.Rule = CreateStartElementIgnoreAttribute ("pre") + preText + CreateEndElement ("pre", grammar, optional: true);
 				PreBlockDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
 					if (!grammar.ShouldImport (ImportJavadoc.Remarks)) {
 						parseNode.AstNode   = "";
@@ -125,6 +126,16 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar)
 						parseNode.AstNode = astNodeElement;
 					}
 				};
+
+				// Start to trim out unusable HTML elements/tags, but not any inner values
+				IgnorableElementDeclaration.Rule =
+					CreateStartElementIgnoreAttribute ("a", "name") + InlineDeclarations + CreateEndElement ("a", grammar, optional: true)
+					| CreateStartElementIgnoreAttribute ("a", "id") + InlineDeclarations + CreateEndElement ("a", grammar, optional: true)
+					;
+				IgnorableElementDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
+					var aElementValue = new XText (parseNode.ChildNodes [1].AstNode.ToString ());
+					parseNode.AstNode = aElementValue;
+				};
 			}
 
 			static IEnumerable<XElement> GetParagraphs (ParseTreeNodeList children)
@@ -193,8 +204,9 @@ static IEnumerable<XElement> GetParagraphs (ParseTreeNodeList children)
 			public  readonly    NonTerminal PBlockDeclaration           = new NonTerminal (nameof (PBlockDeclaration), ConcatChildNodes);
 			public  readonly    NonTerminal PreBlockDeclaration         = new NonTerminal (nameof (PreBlockDeclaration), ConcatChildNodes);
 			public  readonly    NonTerminal InlineHyperLinkDeclaration  = new NonTerminal (nameof (InlineHyperLinkDeclaration), ConcatChildNodes);
+			public  readonly    NonTerminal IgnorableElementDeclaration = new NonTerminal (nameof (IgnorableElementDeclaration), ConcatChildNodes);
 
-			public	readonly	Terminal	InlineHyperLinkOpenTerm		= new RegexBasedTerminal ("<a href=", @"<a\s*href=") {
+			public  readonly    Terminal    InlineHyperLinkOpenTerm     = new RegexBasedTerminal ("<a href=", @"(?i)<a\s*href\s*=") {
 				AstConfig = new AstNodeConfig {
 					NodeCreator = (context, parseNode) => parseNode.AstNode = "",
 				},
@@ -231,6 +243,20 @@ static NonTerminal CreateStartElement (string startElement, Grammar grammar)
 				return start;
 			}
 
+			static RegexBasedTerminal CreateStartElementIgnoreAttribute (string startElement, string attribute)
+			{
+				return new RegexBasedTerminal ($"<{startElement} {attribute}", $@"(?i)<{startElement}\s*{attribute}[^>]*>") {
+					AstConfig = new AstNodeConfig {
+						NodeCreator = (context, parseNode) => parseNode.AstNode = "",
+					},
+				};
+			}
+
+			static RegexBasedTerminal CreateStartElementIgnoreAttribute (string startElement)
+			{
+				return CreateStartElementIgnoreAttribute (startElement, "");
+			}
+
 			static NonTerminal CreateEndElement (string endElement, Grammar grammar, bool optional = false)
 			{
 				var end	    = new NonTerminal (endElement, nodeCreator: (context, parseNode) => parseNode.AstNode = "") {

diff --git a/...vaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.InlineTagsBnfTerms.cs b/...vaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.InlineTagsBnfTerms.cs
@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Data;
 using System.IO;
 using System.Linq;
 using System.Xml.Linq;
@@ -28,6 +29,7 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar)
 					| LiteralDeclaration
 					| SeeDeclaration
 					| ValueDeclaration
+					| IgnorableDeclaration
 					;
 
 				CodeDeclaration.Rule = grammar.ToTerm ("{@code") + InlineValue + "}";
@@ -54,12 +56,20 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar)
 					parseNode.AstNode = new XText ("To be added");
 				};
 
-				LinkDeclaration.Rule = grammar.ToTerm ("{@link") + InlineValue + "}";
+				LinkDeclaration.Rule = grammar.ToTerm ("{@link") + InlineValue + new NonTerminal ("Optional }", nodeCreator: (context, parseNode) => parseNode.AstNode = "") {
+					Rule = grammar.ToTerm ("}") | grammar.Empty,
+				};
 				LinkDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
 					// TODO: *everything*; {@link target label}, but target can contain spaces!
 					// Also need to convert to appropriate CREF value, use code text for now.
+					// Also some {@link tags are missing a closing bracket, use plain text rather than throwing for now.
 					var target = parseNode.ChildNodes [1].AstNode;
-					parseNode.AstNode = new XElement ("c", target);
+					var optionalBracketMatch = parseNode.ChildNodes [2];
+					if (optionalBracketMatch.ChildNodes.Count > 0 && optionalBracketMatch.ChildNodes [0].Term.Name == "}") {
+						parseNode.AstNode = new XElement ("c", target);
+					} else {
+						parseNode.AstNode = new XText (target.ToString ());
+					}
 				};
 
 				LinkplainDeclaration.Rule = grammar.ToTerm ("{@linkplain") + InlineValue + "}";
@@ -97,6 +107,15 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar)
 						parseNode.AstNode = new XText ("To be added");
 					}
 				};
+
+				// Inline content may contain reserved characters with no tags or special parsing rules, do not throw when encountering them
+				IgnorableDeclaration.Rule = grammar.ToTerm ("@ ")
+					| grammar.ToTerm ("{")
+					| grammar.ToTerm ("}")
+					;
+				IgnorableDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
+					parseNode.AstNode = new XText (parseNode.ChildNodes [0].Term.Name.Trim ());
+				};
 			}
 
 			public  readonly    NonTerminal AllInlineTerms              = new NonTerminal (nameof (AllInlineTerms), ConcatChildNodes);
@@ -129,6 +148,9 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar)
 
 			// https://docs.oracle.com/javase/7/docs/technotes/tools/windows/javadoc.html#value
 			public  readonly    NonTerminal ValueDeclaration            = new NonTerminal (nameof (ValueDeclaration));
+
+			public  readonly    NonTerminal IgnorableDeclaration        = new NonTerminal (nameof (IgnorableDeclaration));
+
 		}
 	}
 }
diff --git a/...ava.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.BlockTagsBnfTermsTests.cs b/...ava.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.BlockTagsBnfTermsTests.cs
@@ -65,6 +65,11 @@ public void InheritDocDeclaration ()
 			Assert.IsFalse (r.HasErrors (), "@inheritDoc: " + DumpMessages (r, p));
 			// TODO: Enable after adding support for @inheritDoc
 			Assert.IsNull (r.Root.AstNode, "@inheritDoc should be ignored, but node was not null.");
+
+			r = p.Parse ("@inheritDoc With extra");
+			Assert.IsFalse (r.HasErrors (), "@inheritDoc with trailing content " + DumpMessages (r, p));
+			// TODO: Enable after adding support for @inheritDoc
+			Assert.IsNull (r.Root.AstNode, "@inheritDoc with trailing content should be ignored, but node was not null.");
 		}
 
 		[Test]
@@ -75,6 +80,9 @@ public void HideDeclaration ()
 			var r = p.Parse ("@hide");
 			Assert.IsFalse (r.HasErrors (), "@hide: " + DumpMessages (r, p));
 			Assert.IsNull (r.Root.AstNode, "@hide should be ignored, but node was not null.");
+			r = p.Parse ("@hide Method is broken");
+			Assert.IsFalse (r.HasErrors (), "@hide with trailing content: " + DumpMessages (r, p));
+			Assert.IsNull (r.Root.AstNode, "@hide with trailing content should be ignored, but node was not null.");
 		}
 
 		[Test]

diff --git a/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.HtmlBnfTermsTests.cs b/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.HtmlBnfTermsTests.cs
@@ -46,6 +46,10 @@ public void PreBlockDeclaration ()
 			Assert.AreEqual ("<code lang=\"text/java\">this @contains &lt;arbitrary/&gt; text.</code>",
 					r.Root.AstNode.ToString ());
 
+			r = p.Parse ("<pre class=\"prettyprint\">ColorSpace cs = ColorSpace.get(ColorSpace.Named.DCI_P3);");
+			Assert.IsFalse (r.HasErrors (), DumpMessages (r, p));
+			Assert.AreEqual ($"<code lang=\"text/java\">ColorSpace cs = ColorSpace.get(ColorSpace.Named.DCI_P3);</code>",
+					r.Root.AstNode.ToString ());
 		}
 
 		[Test]