Skip to content

Commit

Permalink
Python: resolve real kinds for tags with unknown kinds when a hint fi…
Browse files Browse the repository at this point in the history
…le is given

As the first pass, make a tags file with --fields=+{language}.
In the second pass, specified the tags file created in the first pass
with --_hint=<tags file>.

Signed-off-by: Masatake YAMATO <yamato@redhat.com>
  • Loading branch information
masatake committed Nov 29, 2020
1 parent 032079e commit 563d0a2
Show file tree
Hide file tree
Showing 8 changed files with 155 additions and 7 deletions.
12 changes: 12 additions & 0 deletions Tmain/list-roles.d/stdout-expected.txt
Original file line number Diff line number Diff line change
Expand Up @@ -56,9 +56,15 @@ Perl M/module unused on specified in `no' bu
Perl M/module used on specified in `use' built-in function
Protobuf D/protodef imported on imported
Protobuf m/message extension on extending the message
Python c/class imported on imported module/imported from the other module
Python c/class indirectlyImported on classes/variables/functions/modules imported in alternative name
Python f/function imported on imported module/imported from the other module
Python f/function indirectlyImported on classes/variables/functions/modules imported in alternative name
Python i/module imported on imported module/imported from the other module
Python i/module indirectlyImported on classes/variables/functions/modules imported in alternative name
Python i/module namespace on namespace from where classes/variables/functions are imported
Python v/variable imported on imported module/imported from the other module
Python v/variable indirectlyImported on classes/variables/functions/modules imported in alternative name
Python x/unknown imported on imported module/imported from the other module
Python x/unknown indirectlyImported on classes/variables/functions/modules imported in alternative name
R l/library library on library attached by library function
Expand Down Expand Up @@ -141,9 +147,15 @@ Perl M/module unused on specified in `no' bu
Perl M/module used on specified in `use' built-in function
Protobuf D/protodef imported on imported
Protobuf m/message extension on extending the message
Python c/class imported on imported module/imported from the other module
Python c/class indirectlyImported on classes/variables/functions/modules imported in alternative name
Python f/function imported on imported module/imported from the other module
Python f/function indirectlyImported on classes/variables/functions/modules imported in alternative name
Python i/module imported on imported module/imported from the other module
Python i/module indirectlyImported on classes/variables/functions/modules imported in alternative name
Python i/module namespace on namespace from where classes/variables/functions are imported
Python v/variable imported on imported module/imported from the other module
Python v/variable indirectlyImported on classes/variables/functions/modules imported in alternative name
Python x/unknown imported on imported module/imported from the other module
Python x/unknown indirectlyImported on classes/variables/functions/modules imported in alternative name
R l/library library on library attached by library function
Expand Down
5 changes: 5 additions & 0 deletions Units/parser-python.r/resolve-unknown-kind.d/args.ctags
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# This must be fixed
--_hint=Units/parser-python.r/resolve-unknown-kind.d/hint.tags

--extras=+r
--fields=+rEK
6 changes: 6 additions & 0 deletions Units/parser-python.r/resolve-unknown-kind.d/expected.tags
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
C0 input.py /^from mymod0 import C0$/;" class module:mymod0 roles:imported extras:reference
f0 input.py /^from mymod0 import f0$/;" function module:mymod0 roles:imported extras:reference
mymod0 input.py /^from mymod0 import C0$/;" module roles:namespace extras:reference
mymod0 input.py /^from mymod0 import f0$/;" module roles:namespace extras:reference
mymod0 input.py /^from mymod0 import v0$/;" module roles:namespace extras:reference
v0 input.py /^from mymod0 import v0$/;" variable module:mymod0 roles:imported extras:reference
3 changes: 3 additions & 0 deletions Units/parser-python.r/resolve-unknown-kind.d/hint.ctags
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# u-ctags -o hint.tags --options=hint.ctags mymod0.py
--sort=no
--fields=+{language}
15 changes: 15 additions & 0 deletions Units/parser-python.r/resolve-unknown-kind.d/hint.tags
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
!_TAG_FILE_FORMAT 2 /extended format; --format=1 will not append ;" to lines/
!_TAG_FILE_SORTED 0 /0=unsorted, 1=sorted, 2=foldcase/
!_TAG_PROGRAM_AUTHOR Universal Ctags Team //
!_TAG_PROGRAM_NAME Universal Ctags /Derived from Exuberant Ctags/
!_TAG_PROGRAM_URL https://ctags.io/ /official site/
!_TAG_PROGRAM_VERSION 5.9.0 /8ca48ff0/
!_TAG_OUTPUT_MODE u-ctags /u-ctags or e-ctags/
!_TAG_OUTPUT_FILESEP slash /slash or backslash/
!_TAG_PATTERN_LENGTH_LIMIT 96 /0 for no limit/
!_TAG_PROC_CWD /home/jet/var/ctags/Units/parser-python.r/resolve-unknown-kind.d/ //
!_TAG_OUTPUT_EXCMD mixed /number, pattern, mixed, or combineV2/
C0 mymod0.py /^class C0:$/;" c language:Python
f0 mymod0.py /^ def f0(self):$/;" m language:Python class:C0
f0 mymod0.py /^def f0():$/;" f language:Python
v0 mymod0.py /^v0 = 1$/;" v language:Python
3 changes: 3 additions & 0 deletions Units/parser-python.r/resolve-unknown-kind.d/input.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
from mymod0 import C0
from mymod0 import f0
from mymod0 import v0
8 changes: 8 additions & 0 deletions Units/parser-python.r/resolve-unknown-kind.d/mymod0.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
class C0:
def f0(self):
pass

def f0():
pass

v0 = 1
110 changes: 103 additions & 7 deletions parsers/python.c
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@
#include "debug.h"
#include "xtag.h"
#include "objpool.h"
#include "hint.h"
#include "routines.h"
#include "field.h"

#define isIdentifierChar(c) \
(isalnum (c) || (c) == '_' || (c) >= 0x80)
Expand Down Expand Up @@ -133,12 +136,18 @@ static roleDefinition PythonModuleRoles [] = {
};

defineRolesWithCommoneElements (Unknown);
defineRolesWithCommoneElements (Class);
defineRolesWithCommoneElements (Variable);
defineRolesWithCommoneElements (Function);

static kindDefinition PythonKinds[COUNT_KIND] = {
{true, 'c', "class", "classes"},
{true, 'f', "function", "functions"},
{true, 'c', "class", "classes",
.referenceOnly = false, ATTACH_ROLES(PythonClassRoles)},
{true, 'f', "function", "functions",
.referenceOnly = false, ATTACH_ROLES(PythonFunctionRoles)},
{true, 'm', "member", "class members"},
{true, 'v', "variable", "variables"},
{true, 'v', "variable", "variables",
.referenceOnly = false, ATTACH_ROLES(PythonVariableRoles)},
{true, 'I', "namespace", "name referring a module defined in other file"},
{true, 'i', "module", "modules",
.referenceOnly = true, ATTACH_ROLES(PythonModuleRoles)},
Expand Down Expand Up @@ -1057,6 +1066,91 @@ static bool parseClassOrDef (tokenInfo *const token,
return true;
}

struct foreachHintData
{
const char *module_name;
size_t module_namelen;
int kind;
};

static bool langobjIsInTheModule (const char *name, hintEntry *hint, void *data)
{
struct foreachHintData *hdata = data;

bool isPython = false;
const char *fname = getFieldName (FIELD_LANGUAGE);
const char *pname = getLanguageName (Lang_python);
for (unsigned int i = 0; i < hint->fields.count; i++)
{
if (strcmp (hint->fields.list [i].key, fname) == 0
&& hint->fields.list [i].value
&& strcmp (hint->fields.list [i].value, pname) == 0)
{
isPython = true;
break;
}
}
if (!isPython)
return true; /* continue the finding */

const char *f = strrstr (hint->file, hdata->module_name);
if (f == NULL)
return true; /* continue the finding */

size_t len = hdata->module_namelen
? hdata->module_namelen
: (hdata->module_namelen = strlen(hdata->module_name)
, hdata->module_namelen);
const char * suffix = f + len;
if (strcmp(suffix, ".py") != 0)
return true; /* continue the finding */

if (!(hint->kind && *hint->kind))
return true;

kindDefinition *kdef = NULL;
if (hint->kind[1] == '\0')
kdef = getLanguageKindForLetter (Lang_python, *hint->kind);
else
kdef = getLanguageKindForName (Lang_python, hint->kind);

if (kdef == NULL)
{
/* Not found; please continue the finding. */
return true;
}

switch (kdef->id)
{
case K_CLASS:
case K_FUNCTION:
case K_VARIABLE:
hdata->kind = kdef->id;
/* Found; please stop the finding. */
return false;
default:
/* Not found; please continue the finding. */
return true;
}
}

static int resolveKindWithHints (const char *name, int moduleIndex)
{
tagEntryInfo *e = getEntryInCorkQueue (moduleIndex);
if (!e)
return K_UNKNOWN;

struct foreachHintData data = {
.kind = K_UNKNOWN,
.module_name = e->name,
.module_namelen = 0,
};

foreachHints (name, TAG_FULLMATCH, langobjIsInTheModule, &data);

return data.kind;
}

static bool parseImport (tokenInfo *const token)
{
tokenInfo *fromModule = NULL;
Expand Down Expand Up @@ -1120,7 +1214,8 @@ static bool parseImport (tokenInfo *const token)
int index;

/* Y */
index = makeSimplePythonRefTag (name, NULL, K_UNKNOWN,
int kind = resolveKindWithHints (vStringValue (name->string), moduleIndex);
index = makeSimplePythonRefTag (name, NULL, kind,
PYTHON_COMMON_INDIRECTLY_IMPORTED,
XTAG_UNKNOWN);
/* fill the scope field for Y */
Expand All @@ -1129,11 +1224,11 @@ static bool parseImport (tokenInfo *const token)
e->extensionFields.scopeIndex = moduleIndex;

/* Z */
index = makeSimplePythonTag (token, K_UNKNOWN);
index = makeSimplePythonTag (token, kind);
/* fill the nameref filed for Y */
if (PythonFields[F_NAMEREF].enabled)
{
vString *nameref = vStringNewInit (PythonKinds [K_UNKNOWN].name);
vString *nameref = vStringNewInit (PythonKinds [kind].name);
vStringPut (nameref, ':');
vStringCat (nameref, name->string);
attachParserFieldToCorkEntry (index, PythonFields[F_NAMEREF].ftype,
Expand Down Expand Up @@ -1178,7 +1273,8 @@ static bool parseImport (tokenInfo *const token)
x = (kind:module, role:namespace),
Y = (kind:unknown, role:imported, scope:module:x) */
/* Y */
int index = makeSimplePythonRefTag (name, NULL, K_UNKNOWN,
int kind = resolveKindWithHints (vStringValue (name->string), moduleIndex);
int index = makeSimplePythonRefTag (name, NULL, kind,
PYTHON_COMMON_IMPORTED,
XTAG_UNKNOWN);
/* fill the scope field for Y */
Expand Down

0 comments on commit 563d0a2

Please sign in to comment.