main: consider foreign languages when solving the extra specifying _e…

…xtra regex flag

The original code assumed the parser specified in a regex pattern
was the owner of the extra specified with the _extra regex flag in
the pattern.

For an example, in the option

    --regex-Foo=/.../{_extra=anextra}...

, ctags assumed "anextra" was a specific extra of "Foo" parser.
The original code didn't consider an option like

    --regex-Foo=/.../{_language=Bar}{_extra=anextra}...

In this case, ctags should consider "anextra" as a specific
extra of "Bar" parser.

NOTE: the language specified with _language flag must be
declared with _foreignLanguage= in --langdef-Foo.

Signed-off-by: Masatake YAMATO <yamato@redhat.com>

Tmain/list-mline-regex-flags.d/stdout-expected.txt

@@ -9,7 +9,7 @@ p       pcre2                                         use pcre2 regex engine
 -       warning="MESSAGE"                             print the given MESSAGE at WARNING level
 -       _advanceTo=N[start|end]                       a group in pattern from where the next scan starts [0end]
 -       _anonymous=PREFIX                             make an anonymous tag with PREFIX
--       _extra=EXTRA                                  record the tag only when the extra is enabled
+-       _extra=EXTRA                                  record the tag only when the (foreign) extra is enabled
 -       _field=FIELD:VALUE                            record the matched string(VALUE) to the (foreign) language specific FIELD of the tag
 -       _guest=PARSERSPEC,N0[start|end],N1[start|end] run guest parser on the area
 -       _language=LANG                                make a foreign tag for LANG

Tmain/list-mtable-regex-flags.d/stdout-expected.txt

@@ -15,7 +15,7 @@ p       pcre2                                         use pcre2 regex engine
 -       warning="MESSAGE"                             print the given MESSAGE at WARNING level
 -       _advanceTo=N[start|end]                       a group in pattern from where the next scan starts [0end]
 -       _anonymous=PREFIX                             make an anonymous tag with PREFIX
--       _extra=EXTRA                                  record the tag only when the extra is enabled
+-       _extra=EXTRA                                  record the tag only when the (foreign) extra is enabled
 -       _field=FIELD:VALUE                            record the matched string(VALUE) to the (foreign) language specific FIELD of the tag
 -       _guest=PARSERSPEC,N0[start|end],N1[start|end] run guest parser on the area
 -       _language=LANG                                make a foreign tag for LANG

Tmain/list-regex-flags.d/stdout-expected.txt

@@ -10,7 +10,7 @@ x       exclusive                                     skip testing the other pat
 -       scope=ACTION                                  use scope stack: ACTION = ref|push|pop|clear|set|replace|intervaltab
 -       warning="MESSAGE"                             print the given MESSAGE at WARNING level
 -       _anonymous=PREFIX                             make an anonymous tag with PREFIX
--       _extra=EXTRA                                  record the tag only when the extra is enabled
+-       _extra=EXTRA                                  record the tag only when the (foreign) extra is enabled
 -       _field=FIELD:VALUE                            record the matched string(VALUE) to the (foreign) language specific FIELD of the tag
 -       _guest=PARSERSPEC,N0[start|end],N1[start|end] run guest parser on the area
 -       _language=LANG                                make a foreign tag for LANG

Tmain/parser-own-extras-for-foreign-lang.d/input-0.x1

@@ -0,0 +1,3 @@
+D:def00
+d:def01
+v:var0

Tmain/parser-own-extras-for-foreign-lang.d/input-1.x1

@@ -0,0 +1,3 @@
+D:def10
+d:def11
+v:var1

Tmain/parser-own-extras-for-foreign-lang.d/run.sh

@@ -0,0 +1,17 @@
+# Copyright: 2024 Masatake YAMATO
+# License: GPL-2
+
+. ../utils.sh
+
+CTAGS=$1
+
+V=
+# V=valgrind
+
+printf "# %s\n" --extras-X0=+'{iname}'
+${V} ${CTAGS} --quiet --options=NONE --options=./x0.ctags --options=./x1.ctags \
+	 --extras-X0=+'{iname}' --fields=+'{extras}{language}' -o - input-0.x1
+
+printf "# %s\n" --extras-X0=-'{iname}'
+${V} ${CTAGS} --quiet --options=NONE --options=./x0.ctags --options=./x1.ctags \
+	 --extras-X0=-'{iname}' --fields=+'{extras}{language}' -o - input-1.x1

Tmain/parser-own-extras-for-foreign-lang.d/stdout-expected.txt

@@ -0,0 +1,7 @@
+# --extras-X0=+{iname}
+__def01__	input-0.x1	/^d:def01$/;"	d	language:X0	extras:iname
+def00	input-0.x1	/^D:def00$/;"	d	language:X0
+var0	input-0.x1	/^v:var0$/;"	v	language:X1
+# --extras-X0=-{iname}
+def10	input-1.x1	/^D:def10$/;"	d	language:X0
+var1	input-1.x1	/^v:var1$/;"	v	language:X1

Tmain/parser-own-extras-for-foreign-lang.d/x0.ctags

@@ -0,0 +1,3 @@
+--langdef=X0
+--kinddef-X0=d,def,definitions
+--_extradef-X0=iname,internal name like __x__

Tmain/parser-own-extras-for-foreign-lang.d/x1.ctags

@@ -0,0 +1,6 @@
+--langdef=X1{_foreignLanguage=X0}
+--map-X1=+.x1
+--kinddef-X1=v,var,variables
+--regex-X1=/D:([a-z0-9]+)$/\1/d/{_language=X0}
+--regex-X1=/d:([a-z0-9]+)$/__\1__/d/{_language=X0}{_extra=iname}
+--regex-X1=/v:([a-z0-9]+)$/\1/v/

docs/optlib.rst

@@ -371,6 +371,10 @@ The pattern matching is done only when the ``main`` is enabled.
 	$ ctags --options=python-main.ctags -o - --extras-Python='+{main}' input.py
 	__main__	input.py	/^if __name__ == '__main__':$/;"	f
 
+By default, ctags assumes the extra is a part of the language specified
+with `<LANG>` in ``--regex-<LANG>``. Together with ``{_language=<LANG>}``
+flag, you can switch the language of the extra. See ":ref:`foreigntag`".
+The combination of these flags is new in version 6.2.0.
 
 .. TODO: this "fields" section should probably be moved up this document, as a
 	subsection in the "Regex option argument flags" section
@@ -1971,6 +1975,9 @@ the output for input.docc:
 ``{_language=<LANG>}`` flag affects ``{_field=FIELDNAME:GROUP}`` flag; ctags looks up
 the field defintion in `<LANG>`.
 
+``{_language=<LANG>}`` flag affects ``{_extra=XNAME}`` flag; ctags looks up
+the extra defintion in `<LANG>`.
+
 .. END: NOT REVIEWED YET
 
 .. _optlib2c:

main/lregex.c

@@ -1084,9 +1084,12 @@ static void common_flag_extra_long (const char* const s, const char* const v, vo
 		return;
 	}
 
-	cdata->ptrn->xtagType = getXtagTypeForNameAndLanguage (v, cdata->owner);
+	langType lang = (cdata->ptrn->foreign_lang == LANG_IGNORE)
+		? cdata->owner
+		: cdata->ptrn->foreign_lang;
+	cdata->ptrn->xtagType = getXtagTypeForNameAndLanguage (v, lang);
 	if (cdata->ptrn->xtagType == XTAG_UNKNOWN)
-		error (WARNING, "no such extra \"%s\" in %s", v, getLanguageName(cdata->owner));
+		error (WARNING, "no such extra \"%s\" in %s", v, getLanguageName(lang));
 }
 
 
@@ -1271,7 +1274,7 @@ static flagDefinition commonSpecFlagDef[] = {
 	  "\"MESSAGE\"", "print the given MESSAGE at WARNING level"},
 #define EXPERIMENTAL "_"
 	{ '\0',  EXPERIMENTAL "extra", NULL, common_flag_extra_long ,
-	  "EXTRA", "record the tag only when the extra is enabled"},
+	  "EXTRA", "record the tag only when the (foreign) extra is enabled"},
 	{ '\0',  EXPERIMENTAL "field", NULL, common_flag_field_long ,
 	  "FIELD:VALUE", "record the matched string(VALUE) to the (foreign) language specific FIELD of the tag"},
 	{ '\0',  EXPERIMENTAL "role", NULL, common_flag_role_long,