Python: resolve real kinds for tags with unknown kinds when a hint fi…

…le is given

As the first pass, make a tags file with --fields=+{language}.
In the second pass, specified the tags file created in the first pass
with --_hint=<tags file>.

Signed-off-by: Masatake YAMATO <yamato@redhat.com>

Tmain/list-roles.d/stdout-expected.txt

@@ -56,9 +56,15 @@ Perl           M/module          unused             on      specified in `no' bu
 Perl           M/module          used               on      specified in `use' built-in function
 Protobuf       D/protodef        imported           on      imported
 Protobuf       m/message         extension          on      extending the message
+Python         c/class           imported           on      imported module/imported from the other module
+Python         c/class           indirectlyImported on      classes/variables/functions/modules imported in alternative name
+Python         f/function        imported           on      imported module/imported from the other module
+Python         f/function        indirectlyImported on      classes/variables/functions/modules imported in alternative name
 Python         i/module          imported           on      imported module/imported from the other module
 Python         i/module          indirectlyImported on      classes/variables/functions/modules imported in alternative name
 Python         i/module          namespace          on      namespace from where classes/variables/functions are imported
+Python         v/variable        imported           on      imported module/imported from the other module
+Python         v/variable        indirectlyImported on      classes/variables/functions/modules imported in alternative name
 Python         x/unknown         imported           on      imported module/imported from the other module
 Python         x/unknown         indirectlyImported on      classes/variables/functions/modules imported in alternative name
 R              l/library         library            on      library attached by library function
@@ -141,9 +147,15 @@ Perl           M/module          unused             on      specified in `no' bu
 Perl           M/module          used               on      specified in `use' built-in function
 Protobuf       D/protodef        imported           on      imported
 Protobuf       m/message         extension          on      extending the message
+Python         c/class           imported           on      imported module/imported from the other module
+Python         c/class           indirectlyImported on      classes/variables/functions/modules imported in alternative name
+Python         f/function        imported           on      imported module/imported from the other module
+Python         f/function        indirectlyImported on      classes/variables/functions/modules imported in alternative name
 Python         i/module          imported           on      imported module/imported from the other module
 Python         i/module          indirectlyImported on      classes/variables/functions/modules imported in alternative name
 Python         i/module          namespace          on      namespace from where classes/variables/functions are imported
+Python         v/variable        imported           on      imported module/imported from the other module
+Python         v/variable        indirectlyImported on      classes/variables/functions/modules imported in alternative name
 Python         x/unknown         imported           on      imported module/imported from the other module
 Python         x/unknown         indirectlyImported on      classes/variables/functions/modules imported in alternative name
 R              l/library         library            on      library attached by library function

Units/parser-python.r/resolve-unknown-kind.d/args.ctags

@@ -0,0 +1,5 @@
+# This must be fixed
+--_hint=Units/parser-python.r/resolve-unknown-kind.d/hint.tags
+
+--extras=+r
+--fields=+rEK

Units/parser-python.r/resolve-unknown-kind.d/expected.tags

@@ -0,0 +1,6 @@
+C0	input.py	/^from mymod0 import C0$/;"	class	module:mymod0	roles:imported	extras:reference
+f0	input.py	/^from mymod0 import f0$/;"	function	module:mymod0	roles:imported	extras:reference
+mymod0	input.py	/^from mymod0 import C0$/;"	module	roles:namespace	extras:reference
+mymod0	input.py	/^from mymod0 import f0$/;"	module	roles:namespace	extras:reference
+mymod0	input.py	/^from mymod0 import v0$/;"	module	roles:namespace	extras:reference
+v0	input.py	/^from mymod0 import v0$/;"	variable	module:mymod0	roles:imported	extras:reference

Units/parser-python.r/resolve-unknown-kind.d/hint.ctags

@@ -0,0 +1,3 @@
+# u-ctags -o hint.tags --options=hint.ctags mymod0.py
+--sort=no
+--fields=+{language}

Units/parser-python.r/resolve-unknown-kind.d/hint.tags

@@ -0,0 +1,15 @@
+!_TAG_FILE_FORMAT	2	/extended format; --format=1 will not append ;" to lines/
+!_TAG_FILE_SORTED	0	/0=unsorted, 1=sorted, 2=foldcase/
+!_TAG_PROGRAM_AUTHOR	Universal Ctags Team	//
+!_TAG_PROGRAM_NAME	Universal Ctags	/Derived from Exuberant Ctags/
+!_TAG_PROGRAM_URL	https://ctags.io/	/official site/
+!_TAG_PROGRAM_VERSION	5.9.0	/8ca48ff0/
+!_TAG_OUTPUT_MODE	u-ctags	/u-ctags or e-ctags/
+!_TAG_OUTPUT_FILESEP	slash	/slash or backslash/
+!_TAG_PATTERN_LENGTH_LIMIT	96	/0 for no limit/
+!_TAG_PROC_CWD	/home/jet/var/ctags/Units/parser-python.r/resolve-unknown-kind.d/	//
+!_TAG_OUTPUT_EXCMD	mixed	/number, pattern, mixed, or combineV2/
+C0	mymod0.py	/^class C0:$/;"	c	language:Python
+f0	mymod0.py	/^    def f0(self):$/;"	m	language:Python	class:C0
+f0	mymod0.py	/^def f0():$/;"	f	language:Python
+v0	mymod0.py	/^v0 = 1$/;"	v	language:Python

Units/parser-python.r/resolve-unknown-kind.d/input.py

@@ -0,0 +1,3 @@
+from mymod0 import C0
+from mymod0 import f0
+from mymod0 import v0

Units/parser-python.r/resolve-unknown-kind.d/mymod0.py

@@ -0,0 +1,8 @@
+class C0:
+    def f0(self):
+        pass
+
+def f0():
+    pass
+
+v0 = 1

parsers/python.c

@@ -23,6 +23,9 @@
 #include "debug.h"
 #include "xtag.h"
 #include "objpool.h"
+#include "hint.h"
+#include "routines.h"
+#include "field.h"
 
 #define isIdentifierChar(c) \
 	(isalnum (c) || (c) == '_' || (c) >= 0x80)
@@ -133,12 +136,18 @@ static roleDefinition PythonModuleRoles [] = {
 };
 
 defineRolesWithCommoneElements (Unknown);
+defineRolesWithCommoneElements (Class);
+defineRolesWithCommoneElements (Variable);
+defineRolesWithCommoneElements (Function);
 
 static kindDefinition PythonKinds[COUNT_KIND] = {
-	{true, 'c', "class",    "classes"},
-	{true, 'f', "function", "functions"},
+	{true, 'c', "class",    "classes",
+	 .referenceOnly = false, ATTACH_ROLES(PythonClassRoles)},
+	{true, 'f', "function", "functions",
+	 .referenceOnly = false, ATTACH_ROLES(PythonFunctionRoles)},
 	{true, 'm', "member",   "class members"},
-	{true, 'v', "variable", "variables"},
+	{true, 'v', "variable", "variables",
+	 .referenceOnly = false, ATTACH_ROLES(PythonVariableRoles)},
 	{true, 'I', "namespace", "name referring a module defined in other file"},
 	{true, 'i', "module",    "modules",
 	 .referenceOnly = true,  ATTACH_ROLES(PythonModuleRoles)},
@@ -1057,6 +1066,91 @@ static bool parseClassOrDef (tokenInfo *const token,
 	return true;
 }
 
+struct foreachHintData
+{
+	const char *module_name;
+	size_t module_namelen;
+	int kind;
+};
+
+static bool langobjIsInTheModule (const char *name, hintEntry *hint, void *data)
+{
+	struct foreachHintData *hdata = data;
+
+	bool isPython = false;
+	const char *fname = getFieldName (FIELD_LANGUAGE);
+	const char *pname = getLanguageName (Lang_python);
+	for (unsigned int i = 0; i < hint->fields.count; i++)
+	{
+		if (strcmp (hint->fields.list [i].key, fname) == 0
+			&& hint->fields.list [i].value
+			&& strcmp (hint->fields.list [i].value, pname) == 0)
+		{
+			isPython = true;
+			break;
+		}
+	}
+	if (!isPython)
+		return true;			/* continue the finding */
+
+	const char *f = strrstr (hint->file, hdata->module_name);
+	if (f == NULL)
+		return true;			/* continue the finding */
+
+	size_t len = hdata->module_namelen
+		? hdata->module_namelen
+		: (hdata->module_namelen = strlen(hdata->module_name)
+		   , hdata->module_namelen);
+	const char * suffix = f + len;
+	if (strcmp(suffix, ".py") != 0)
+		return true;			/* continue the finding */
+
+	if (!(hint->kind && *hint->kind))
+		return true;
+
+	kindDefinition *kdef = NULL;
+	if (hint->kind[1] == '\0')
+		kdef = getLanguageKindForLetter (Lang_python, *hint->kind);
+	else
+		kdef = getLanguageKindForName (Lang_python, hint->kind);
+
+	if (kdef == NULL)
+	{
+		/* Not found; please continue the finding. */
+		return true;
+	}
+
+	switch (kdef->id)
+	{
+	case K_CLASS:
+	case K_FUNCTION:
+	case K_VARIABLE:
+		hdata->kind = kdef->id;
+		/* Found; please stop the finding. */
+		return false;
+	default:
+		/* Not found; please continue the finding. */
+		return true;
+	}
+}
+
+static int resolveKindWithHints (const char *name, int moduleIndex)
+{
+	tagEntryInfo *e = getEntryInCorkQueue (moduleIndex);
+	if (!e)
+		return K_UNKNOWN;
+
+	struct foreachHintData data = {
+		.kind = K_UNKNOWN,
+		.module_name = e->name,
+		.module_namelen = 0,
+	};
+
+	foreachHints (name, TAG_FULLMATCH, langobjIsInTheModule, &data);
+
+	return data.kind;
+}
+
 static bool parseImport (tokenInfo *const token)
 {
 	tokenInfo *fromModule = NULL;
@@ -1120,7 +1214,8 @@ static bool parseImport (tokenInfo *const token)
 							int index;
 
 							/* Y */
-							index = makeSimplePythonRefTag (name, NULL, K_UNKNOWN,
+							int kind  = resolveKindWithHints (vStringValue (name->string), moduleIndex);
+							index = makeSimplePythonRefTag (name, NULL, kind,
 															PYTHON_COMMON_INDIRECTLY_IMPORTED,
 															XTAG_UNKNOWN);
 							/* fill the scope field for Y */
@@ -1129,11 +1224,11 @@ static bool parseImport (tokenInfo *const token)
 								e->extensionFields.scopeIndex = moduleIndex;
 
 							/* Z */
-							index = makeSimplePythonTag (token, K_UNKNOWN);
+							index = makeSimplePythonTag (token, kind);
 							/* fill the nameref filed for Y */
 							if (PythonFields[F_NAMEREF].enabled)
 							{
-								vString *nameref = vStringNewInit (PythonKinds [K_UNKNOWN].name);
+								vString *nameref = vStringNewInit (PythonKinds [kind].name);
 								vStringPut (nameref, ':');
 								vStringCat (nameref, name->string);
 								attachParserFieldToCorkEntry (index, PythonFields[F_NAMEREF].ftype,
@@ -1178,7 +1273,8 @@ static bool parseImport (tokenInfo *const token)
 						   x = (kind:module,  role:namespace),
 						   Y = (kind:unknown, role:imported, scope:module:x) */
 						/* Y */
-						int index = makeSimplePythonRefTag (name, NULL, K_UNKNOWN,
+						int kind  = resolveKindWithHints (vStringValue (name->string), moduleIndex);
+						int index = makeSimplePythonRefTag (name, NULL, kind,
 															PYTHON_COMMON_IMPORTED,
 															XTAG_UNKNOWN);
 						/* fill the scope field for Y */