feat: support basic regex analysers in XML

The `<analyse>` element contains a regex to match against the input, and
a score if the pattern matches.

The scores of all matching patterns for a lexer are summed.

Replaces #815, #813 and #826.

lexer.go

@@ -57,6 +57,18 @@ type Config struct {
 	//
 	// If this is 0 it will be treated as a default of 1.
 	Priority float32 `xml:"priority,omitempty"`
+
+	// Analyse is a list of regexes to match against the input.
+	//
+	// The sum of all the score of matching patterns will be
+	// used as the final score.
+	Analyse []AnalyseConfig `xml:"analyse,omitempty"`
+}
+
+// AnalyseConfig defines a single regex analyser pattern.
+type AnalyseConfig struct {
+	Regex string  `xml:"regex,attr"`
+	Score float32 `xml:"score,attr"`
 }
 
 // Token output to formatter.

lexers/embedded/c++.xml

@@ -15,9 +15,12 @@
     <filename>*.H</filename>
     <filename>*.cp</filename>
     <filename>*.CPP</filename>
+    <filename>*.tpp</filename>
     <mime_type>text/x-c++hdr</mime_type>
     <mime_type>text/x-c++src</mime_type>
     <ensure_nl>true</ensure_nl>
+    <analyse regex="#include &lt;[a-z_]+>" score="0.2" />
+    <analyse regex="using namespace " score="0.4" />
   </config>
   <rules>
     <state name="classname">

lexers/embedded/c.xml

@@ -11,6 +11,8 @@
     <mime_type>image/x-xbitmap</mime_type>
     <mime_type>image/x-xpixmap</mime_type>
     <ensure_nl>true</ensure_nl>
+    <analyse regex="(?m)^\s*#include &lt;" score="0.1"/>
+    <analyse regex="(?m)^\s*#ifn?def " score="0.1" />
   </config>
   <rules>
     <state name="statement">

lexers/testdata/analysis/c.ifdef.expected

@@ -0,0 +1 @@
+0.1

lexers/testdata/analysis/c.ifndef.expected

@@ -0,0 +1 @@
+0.1

lexers/testdata/analysis/c.include.expected

@@ -0,0 +1 @@
+0.1

lexers/testdata/analysis/cpp.include.expected

@@ -0,0 +1 @@
+0.2

lexers/testdata/analysis/cpp.namespace.expected

@@ -0,0 +1 @@
+0.4

serialise.go

@@ -11,6 +11,8 @@ import (
 	"reflect"
 	"regexp"
 	"strings"
+
+	"github.com/dlclark/regexp2"
 )
 
 // Serialisation of Chroma rules to XML. The format is:
@@ -107,7 +109,7 @@ func fastUnmarshalConfig(from fs.FS, path string) (*Config, error) {
 			var config Config
 			err = dec.DecodeElement(&config, &se)
 			if err != nil {
-				panic(err)
+				return nil, fmt.Errorf("%s: %w", path, err)
 			}
 			return &config, nil
 		}
@@ -135,8 +137,29 @@ func NewXMLLexer(from fs.FS, path string) (*RegexLexer, error) {
 			return nil, fmt.Errorf("%s: %q is not a valid glob: %w", config.Name, glob, err)
 		}
 	}
+	type regexAnalyse struct {
+		re    *regexp2.Regexp
+		score float32
+	}
+	regexAnalysers := make([]regexAnalyse, 0, len(config.Analyse))
+	for _, ra := range config.Analyse {
+		re, err := regexp2.Compile(ra.Regex, regexp2.None)
+		if err != nil {
+			return nil, fmt.Errorf("%s: %q is not a valid analyser regex: %w", config.Name, ra.Regex, err)
+		}
+		regexAnalysers = append(regexAnalysers, regexAnalyse{re, ra.Score})
+	}
 	return &RegexLexer{
 		config: config,
+		analyser: func(text string) float32 {
+			var score float32
+			for _, ra := range regexAnalysers {
+				if ok, _ := ra.re.MatchString(text); ok {
+					score += ra.score
+				}
+			}
+			return score
+		},
 		fetchRulesFunc: func() (Rules, error) {
 			var lexer struct {
 				Config