optlib: allow users to set fields and check extras defined in a foreign language

Tmain/flags-fielddef-datatype.d/exit-expected.txt

@@ -0,0 +1 @@
+0

Tmain/flags-fielddef-datatype.d/run.sh

@@ -0,0 +1,11 @@
+# Copyright: 2023 Masatake YAMATO
+# License: GPL-2
+
+CTAGS=$1
+
+${CTAGS} --quiet --options=NONE --langdef=FIELDTEST \
+		 --_fielddef-FIELDTEST=strfield,"a field having string value"'{datatype=str}' \
+		 --_fielddef-FIELDTEST=boolfield,"a field having boolean value"'{datatype=bool}' \
+		 --_fielddef-FIELDTEST=intfield,"a field having integer value"'{datatype=int}' \
+		 --_fielddef-FIELDTEST=deffield,"a field that type is not specified" \
+		 --list-fields=FIELDTEST

Tmain/flags-fielddef-datatype.d/stdout-expected.txt

@@ -0,0 +1,5 @@
+#LETTER NAME      ENABLED LANGUAGE  JSTYPE FIXED OP DESCRIPTION
+-       boolfield no      FIELDTEST --b    no    rw a field having boolean value
+-       deffield  no      FIELDTEST s--    no    -- a field that type is not specified
+-       intfield  no      FIELDTEST -i-    no    rw a field having integer value
+-       strfield  no      FIELDTEST s--    no    rw a field having string value

Tmain/list-fielddef-flags.d/exit-expected.txt

@@ -0,0 +1 @@
+0

Tmain/list-fielddef-flags.d/run.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+# Copyright: 2023 Masatake YAMATO
+# License: GPL-2
+
+CTAGS=$1
+
+${CTAGS} --quiet --options=NONE --_list-fielddef-flags

Tmain/list-fielddef-flags.d/stdout-expected.txt

@@ -0,0 +1,2 @@
+#LETTER NAME          DESCRIPTION
+-       datatype=TYPE acceaptable datatype of the field (str|bool|int)

Tmain/parser-own-fields-for-foreign-lang.d/input.unknownx

@@ -3,3 +3,6 @@ protected func bar(n);
 private func baz(n,...);
 X:tagme@iamowner
 Y:iamowner2=tagme2
+Z:tagme-z@iamowner-z
+eset:a
+enoset:b

Tmain/parser-own-fields-for-foreign-lang.d/knownz.ctags

@@ -1,4 +1,12 @@
 --langdef=knownz
 --kinddef-knownz=m,mark,makers
 
---_fielddef-knownz=owner,the owner of the markers
+--_fielddef-knownz=owner,the owner of the markers{datatype=str}
+
+--_fielddef-knownz=len,the length of owner string{datatype=int}
+--fields-knownz=+{len}
+--_fielddef-knownz=lenplus,the length of owner string + 1{datatype=int}
+--fields-knownz=+{lenplus}
+
+--_fielddef-knownz=exported,whether the marker is exported or not{datatype=bool}
+--fields-knownz=+{exported}

Tmain/parser-own-fields-for-foreign-lang.d/stdout-expected.txt

@@ -1,5 +1,9 @@
+7_exported	input.unknownx	/^enoset:b$/;"	m	language:knownz
+a	input.unknownx	/^eset:a$/;"	m	language:knownz	exported:t
+b	input.unknownx	/^enoset:b$/;"	m	language:knownz	exported:
 bar	input.unknownx	/^protected func bar(n);$/;"	f	language:unknownx	protection:protected 	signature:(n)
 baz	input.unknownx	/^private func baz(n,...);$/;"	f	language:unknownx	protection:private 	signature:(n,...)
 foo	input.unknownx	/^public func foo(n, m);$/;"	f	language:unknownx	protection:public 	signature:(n, m)
 tagme	input.unknownx	/^X:tagme@iamowner$/;"	m	language:knownz	owner:iamowner
+tagme-z	input.unknownx	/^Z:tagme-z@iamowner-z$/;"	m	language:knownz	owner:iamowner-z	len:10	lenplus:11
 tagme2	input.unknownx	/^Y:iamowner2=tagme2$/;"	m	language:knownz	owner:iamowner2

Tmain/parser-own-fields-for-foreign-lang.d/unknownx.ctags

@@ -5,6 +5,35 @@
 --_fielddef-unknownx=protection,protections
 --_fielddef-unknownx=signature,signatures
 
+--_prelude-unknownx={{
+   /exported false def
+}}
+
 --regex-unknownx=/^((public|protected|private) +)?func ([^\(]+)\((.*)\)/\3/f/{_field=protection:\1}{_field=signature:(\4)}
 --regex-unknownx=/^X:([a-z]+)@([a-z]+)/\1/m/{_language=knownz}{_field=owner:\2}
 --regex-unknownx=/^Y:([a-z0-9]+)=([a-z0-9]+)/\2/m/{_field=owner:\1}{_language=knownz}
+--regex-unknownx=/^Z:([-a-z]+)@([-a-z]+)/\1/m/{_language=knownz}{{
+    . \2 knownz.owner:
+    . :knownz.owner {
+        . exch length knownz.len:
+    } if
+    . :knownz.len {
+        1 add
+        . exch knownz.lenplus:
+    } if
+}}
+
+--regex-unknownx=/^eset:([-a-z]+)/\1/m/{_language=knownz}{{
+    /exported . def
+    . true knownz.exported:
+}}
+
+--regex-unknownx=/^enoset:([-a-z]+)/\1/m/{_language=knownz}{{
+    . false knownz.exported:
+    exported :knownz.exported and {
+       mark exported 0 string cvs (_exported) _buildstring
+       /knownz
+       /mark
+       1@ _foreigntag _commit pop
+    } if
+}}

main/field.c

@@ -75,17 +75,17 @@
 static bool doesContainAnyCharInFieldScope (const tagEntryInfo *const tag, const char *value, const char *chars);
 static bool doesContainAnyCharInSignature (const tagEntryInfo *const tag, const char *value, const char *chars);
 
-static bool     isTyperefFieldAvailable   (const tagEntryInfo *const tag);
-static bool     isFileFieldAvailable      (const tagEntryInfo *const tag);
-static bool     isInheritsFieldAvailable  (const tagEntryInfo *const tag);
-static bool     isAccessFieldAvailable    (const tagEntryInfo *const tag);
-static bool     isImplementationFieldAvailable (const tagEntryInfo *const tag);
-static bool     isSignatureFieldAvailable (const tagEntryInfo *const tag);
-static bool     isExtrasFieldAvailable    (const tagEntryInfo *const tag);
-static bool     isXpathFieldAvailable     (const tagEntryInfo *const tag);
-static bool     isEndFieldAvailable       (const tagEntryInfo *const tag);
-static bool     isEpochAvailable          (const tagEntryInfo *const tag);
-static bool     isNthAvailable            (const tagEntryInfo *const tag);
+static bool     isTyperefFieldAvailable   (const tagEntryInfo *const tag, const fieldDefinition *fdef);
+static bool     isFileFieldAvailable      (const tagEntryInfo *const tag, const fieldDefinition *fdef);
+static bool     isInheritsFieldAvailable  (const tagEntryInfo *const tag, const fieldDefinition *fdef);
+static bool     isAccessFieldAvailable    (const tagEntryInfo *const tag, const fieldDefinition *fdef);
+static bool     isImplementationFieldAvailable (const tagEntryInfo *const tag, const fieldDefinition *fdef);
+static bool     isSignatureFieldAvailable (const tagEntryInfo *const tag, const fieldDefinition *fdef);
+static bool     isExtrasFieldAvailable    (const tagEntryInfo *const tag, const fieldDefinition *fdef);
+static bool     isXpathFieldAvailable     (const tagEntryInfo *const tag, const fieldDefinition *fdef);
+static bool     isEndFieldAvailable       (const tagEntryInfo *const tag, const fieldDefinition *fdef);
+static bool     isEpochAvailable          (const tagEntryInfo *const tag, const fieldDefinition *fdef);
+static bool     isNthAvailable            (const tagEntryInfo *const tag, const fieldDefinition *fdef);
 
 static EsObject* getFieldValueForName (const tagEntryInfo *, const fieldDefinition *);
 static EsObject* setFieldValueForName (tagEntryInfo *, const fieldDefinition *, const EsObject *);
@@ -614,7 +614,8 @@
 extern bool doesFieldHaveValue (fieldType type, const tagEntryInfo *tag)
 {
 	if (getFieldObject(type)->def->isValueAvailable)
-		return getFieldObject(type)->def->isValueAvailable(tag);
+		return getFieldObject(type)->def->isValueAvailable(tag,
+														   getFieldObject(type)->def);
 	else
 		return true;
 }
@@ -1164,43 +1165,43 @@
 #undef buf_len
 }
 
-static bool     isTyperefFieldAvailable  (const tagEntryInfo *const tag)
+static bool isTyperefFieldAvailable (const tagEntryInfo *const tag, const fieldDefinition *fdef CTAGS_ATTR_UNUSED)
 {
 	return (tag->extensionFields.typeRef [0] != NULL
 		&& tag->extensionFields.typeRef [1] != NULL)? true: false;
 }
 
-static bool     isFileFieldAvailable  (const tagEntryInfo *const tag)
+static bool isFileFieldAvailable  (const tagEntryInfo *const tag, const fieldDefinition *fdef CTAGS_ATTR_UNUSED)
 {
 	return tag->isFileScope? true: false;
 }
 
-static bool     isInheritsFieldAvailable (const tagEntryInfo *const tag)
+static bool isInheritsFieldAvailable (const tagEntryInfo *const tag, const fieldDefinition *fdef CTAGS_ATTR_UNUSED)
 {
 	return (tag->extensionFields.inheritance != NULL)? true: false;
 }
 
-static bool     isAccessFieldAvailable   (const tagEntryInfo *const tag)
+static bool isAccessFieldAvailable (const tagEntryInfo *const tag, const fieldDefinition *fdef CTAGS_ATTR_UNUSED)
 {
 	return (tag->extensionFields.access != NULL)? true: false;
 }
 
-static bool     isImplementationFieldAvailable (const tagEntryInfo *const tag)
+static bool isImplementationFieldAvailable (const tagEntryInfo *const tag, const fieldDefinition *fdef CTAGS_ATTR_UNUSED)
 {
 	return (tag->extensionFields.implementation != NULL)? true: false;
 }
 
-static bool     isSignatureFieldAvailable (const tagEntryInfo *const tag)
+static bool isSignatureFieldAvailable (const tagEntryInfo *const tag, const fieldDefinition *fdef CTAGS_ATTR_UNUSED)
 {
 	return (tag->extensionFields.signature != NULL)? true: false;
 }
 
-static bool     isExtrasFieldAvailable     (const tagEntryInfo *const tag)
+static bool isExtrasFieldAvailable (const tagEntryInfo *const tag, const fieldDefinition *fdef CTAGS_ATTR_UNUSED)
 {
 	return isTagExtra (tag);
 }
 
-static bool     isXpathFieldAvailable      (const tagEntryInfo *const tag)
+static bool isXpathFieldAvailable (const tagEntryInfo *const tag, const fieldDefinition *fdef CTAGS_ATTR_UNUSED)
 {
 #ifdef HAVE_LIBXML
 	return (tag->extensionFields.xpath != NULL)? true: false;
@@ -1209,19 +1210,19 @@
 #endif
 }
 
-static bool     isEndFieldAvailable       (const tagEntryInfo *const tag)
+static bool isEndFieldAvailable (const tagEntryInfo *const tag, const fieldDefinition *fdef CTAGS_ATTR_UNUSED)
 {
 	return (tag->extensionFields._endLine != 0)? true: false;
 }
 
-static bool isEpochAvailable (const tagEntryInfo *const tag)
+static bool isEpochAvailable (const tagEntryInfo *const tag, const fieldDefinition *fdef CTAGS_ATTR_UNUSED)
 {
 	return (tag->kindIndex == KIND_FILE_INDEX)
 		? true
 		: false;
 }
 
-static bool isNthAvailable (const tagEntryInfo *const tag)
+static bool isNthAvailable (const tagEntryInfo *const tag, const fieldDefinition *fdef CTAGS_ATTR_UNUSED)
 {
 	Assert (tag->langType >= NO_NTH_FIELD);
 	return (tag->extensionFields.nth != NO_NTH_FIELD)? true: false;
@@ -1315,6 +1316,11 @@
 	return renderEscapedString (value, tag, buffer);
 }
 
+static bool isValueAvailableGeneric (const tagEntryInfo *const e, const fieldDefinition *fdef)
+{
+	return getParserFieldValueForType(e, fdef->ftype)? true: false;
+}
+
 extern int defineField (fieldDefinition *def, langType language)
 {
 	fieldObject *fobj;
@@ -1347,6 +1353,9 @@
 	if (! def->dataType)
 		def->dataType = FIELDTYPE_STRING;
 
+	if (def->isValueAvailable == NULL)
+		def->isValueAvailable = isValueAvailableGeneric;
+
 	fobj->def = def;
 
 	fobj->buffer = NULL;
@@ -1362,6 +1371,8 @@
 	fobj->sibling  = FIELD_UNKNOWN;
 
 	updateSiblingField (def->ftype, def->name);
+	installOptscriptFieldAccessor (def->ftype);
+
 	return def->ftype;
 }
 
@@ -1914,3 +1925,62 @@
 
 	return es_false;
 }
+
+extern EsObject* getFieldValueGeneric (const tagEntryInfo *tag, const fieldDefinition *fdef)
+{
+	const char *value = getParserFieldValueForType(tag, fdef->ftype);
+
+	if (value == NULL)
+		return es_nil;
+
+	unsigned int dt = fdef->dataType;
+	if (dt & FIELDTYPE_STRING)
+		return (dt & FIELDTYPE_BOOL && value[0] == '\0')
+			? es_false
+			: opt_string_new_from_cstr (value);
+	else if (dt & FIELDTYPE_INTEGER)
+	{
+		long tmp;
+		if (strToLong (value, 10, &tmp))
+			return es_integer_new ((int)tmp); /* TODO: if tmp is not in the range of int, return es_nil */
+		else
+		{
+			AssertNotReached ();
+			return es_nil;
+		}
+	}
+	else if (dt & FIELDTYPE_BOOL)
+		return value [0]? es_true: es_false;
+	else
+	{
+		AssertNotReached ();
+		return es_nil;
+	}
+}
+
+extern EsObject* setFieldValueGeneric (tagEntryInfo *tag, const fieldDefinition *fdef, const EsObject *obj)
+{
+	unsigned int dt = fdef->dataType;
+	const char * val;
+	char buf[1 /* [+-] */ + 20 + 1 /* for \0 */];
+
+	if (dt & FIELDTYPE_STRING)
+		val = opt_string_get_cstr (obj);
+	else if (dt & FIELDTYPE_INTEGER)
+	{
+		int tmp = es_integer_get (obj);
+		/* 2^64 => "18446744073709551616" */
+		snprintf(buf, 22, "%d", tmp);
+		val = buf;
+	}
+	else if (dt & FIELDTYPE_BOOL)
+		val = es_boolean_get (obj)? "t": "";
+	else
+	{
+		val = "";
+		AssertNotReached ();
+	}
+
+	attachParserField (tag, fdef->ftype, val);
+	return es_false;
+}

main/field.h

@@ -77,6 +77,17 @@ typedef enum eFieldDataType {
 
 	/* used in --list-fields */
 	FIELDTYPE_END_MARKER = 1 << 3,
+
+	/* If you want to allow a parser code written in optscript
+	 * to access the field, append FIELDTYPE_SCRIPTABLE to
+	 * dataType field of your fieldDefinition.
+	 *
+	 * From a optlib parser, pass {datatype=TYPE} flag to
+	 * --_fielddef-<LANV>=... option. Just specifying a
+	 * type is enough; FIELDTYPE_SCRIPTABLE is automatically
+	 * append the filed definition. If you don't pass the
+	 * flag explicitly, FIELDTYPE_SCRIPTABLE is not set. */
+	FIELDTYPE_SCRIPTABLE = FIELDTYPE_END_MARKER,
 } fieldDataType;
 
 typedef const char* (*fieldRenderer)(const tagEntryInfo *const,
@@ -96,7 +107,7 @@ struct sFieldDefinition {
 	fieldRenderer renderNoEscaping;
 	bool (* doesContainAnyChar) (const tagEntryInfo *const, const char*, const char *);
 
-	bool (* isValueAvailable) (const tagEntryInfo *const);
+	bool (* isValueAvailable) (const tagEntryInfo *const, const fieldDefinition *);
 
 	const char * getterValueType;
 	struct _EsObject * (* getValueObject) (const tagEntryInfo *, const fieldDefinition *);
@@ -119,4 +130,7 @@ struct sFieldDefinition {
 
 extern bool isFieldEnabled (fieldType type);
 
+extern struct _EsObject* getFieldValueGeneric (const tagEntryInfo *tag, const fieldDefinition *fdef);
+extern struct _EsObject* setFieldValueGeneric (tagEntryInfo *tag, const fieldDefinition *fdef, const struct _EsObject *obj);
+
 #endif	/* CTAGS_MAIN_FIELD_H */

main/lregex.c

@@ -4564,6 +4564,19 @@ static struct optscriptOperatorRegistration lropOperators [] = {
 	},
 };
 
+extern bool installOptscriptFieldAccessor (fieldType ftype)
+{
+	/* This function is called via defineField().
+	   defineField() is called via --fielddef option of built-in parsers
+	   defining their specific fields.
+	   built-in parsers are initialized lazily in the current implementation.
+	   optvm is initialized earlier than parsers. */
+	Assert (!es_null(lregex_dict));
+
+	optscriptInstallFieldAccessor (lregex_dict, ftype);
+	return true;
+}
+
 extern void initRegexOptscript (void)
 {
 	if (!regexAvailable)

main/lregex_p.h

@@ -19,6 +19,7 @@
 *   INCLUDE FILES
 */
 #include "general.h"
+#include "field.h"
 #include "flags_p.h"
 #include "kind_p.h"
 #include "lregex.h"
@@ -114,6 +115,8 @@ extern void extendRegexTable (struct lregexControlBlock *lcb, const char *src, c
 extern void initRegexOptscript (void);
 extern void listRegexOpscriptOperators (FILE *fp);
 
+extern bool installOptscriptFieldAccessor (fieldType ftype);
+
 extern void addOptscriptToHook (struct lregexControlBlock *lcb, enum scriptHook hook, const char *code);
 extern void propagateParamToOptscript (struct lregexControlBlock *lcb, const char *param, const char *value);