Merge pull request #3000 from masatake/ipython-cell

IPythonCell: a new subparsre stacked on Python parser

Makefile.am

@@ -254,6 +254,7 @@ man_MANS = \
 	man/ctags-optlib.7 \
 	man/ctags-faq.7 \
 	\
+	man/ctags-lang-iPythonCell.7 \
 	man/ctags-lang-julia.7 \
 	man/ctags-lang-python.7 \
 	man/ctags-lang-verilog.7 \

Tmain/extras-long.d/stdout-expected.txt

@@ -1,60 +1,65 @@
 # resetting
--       anonymous         no      NONE     no    Include tags for non-named objects like lambda
-F       fileScope         no      NONE     no    Include tags of file scope
-f       inputFile         no      NONE     no    Include an entry for the base file name of every input file
-g       guest             no      NONE     no    Include tags generated by guest parsers
-p       pseudo            no      NONE     no    Include pseudo tags
-q       qualified         no      NONE     no    Include an extra class-qualified tag entry for each tag
-r       reference         no      NONE     no    Include reference tags
-s       subparser         yes     NONE     no    Include tags generated by subparsers
--       configPrefixed    yes     Kconfig  no    prepend CONFIG_ to config names
--       funcmap           yes     QemuHX   no    Include mapping SQMP to C function name
--       whitespaceSwapped yes     Robot    no    Include tags swapping whitespace and underscore chars
+-       anonymous         no      NONE        no    Include tags for non-named objects like lambda
+F       fileScope         no      NONE        no    Include tags of file scope
+f       inputFile         no      NONE        no    Include an entry for the base file name of every input file
+g       guest             no      NONE        no    Include tags generated by guest parsers
+p       pseudo            no      NONE        no    Include pseudo tags
+q       qualified         no      NONE        no    Include an extra class-qualified tag entry for each tag
+r       reference         no      NONE        no    Include reference tags
+s       subparser         yes     NONE        no    Include tags generated by subparsers
+-       doubleSharps      no      IPythonCell no     Include cells starting from ##
+-       configPrefixed    yes     Kconfig     no    prepend CONFIG_ to config names
+-       funcmap           yes     QemuHX      no    Include mapping SQMP to C function name
+-       whitespaceSwapped yes     Robot       no    Include tags swapping whitespace and underscore chars
 # enabling 1
--       anonymous         yes     NONE     no    Include tags for non-named objects like lambda
-F       fileScope         yes     NONE     no    Include tags of file scope
-f       inputFile         no      NONE     no    Include an entry for the base file name of every input file
-g       guest             no      NONE     no    Include tags generated by guest parsers
-p       pseudo            yes     NONE     no    Include pseudo tags
-q       qualified         no      NONE     no    Include an extra class-qualified tag entry for each tag
-r       reference         no      NONE     no    Include reference tags
-s       subparser         yes     NONE     no    Include tags generated by subparsers
--       configPrefixed    yes     Kconfig  no    prepend CONFIG_ to config names
--       funcmap           yes     QemuHX   no    Include mapping SQMP to C function name
--       whitespaceSwapped yes     Robot    no    Include tags swapping whitespace and underscore chars
+-       anonymous         yes     NONE        no    Include tags for non-named objects like lambda
+F       fileScope         yes     NONE        no    Include tags of file scope
+f       inputFile         no      NONE        no    Include an entry for the base file name of every input file
+g       guest             no      NONE        no    Include tags generated by guest parsers
+p       pseudo            yes     NONE        no    Include pseudo tags
+q       qualified         no      NONE        no    Include an extra class-qualified tag entry for each tag
+r       reference         no      NONE        no    Include reference tags
+s       subparser         yes     NONE        no    Include tags generated by subparsers
+-       doubleSharps      no      IPythonCell no     Include cells starting from ##
+-       configPrefixed    yes     Kconfig     no    prepend CONFIG_ to config names
+-       funcmap           yes     QemuHX      no    Include mapping SQMP to C function name
+-       whitespaceSwapped yes     Robot       no    Include tags swapping whitespace and underscore chars
 # disabling 1
--       anonymous         yes     NONE     no    Include tags for non-named objects like lambda
-F       fileScope         no      NONE     no    Include tags of file scope
-f       inputFile         no      NONE     no    Include an entry for the base file name of every input file
-g       guest             no      NONE     no    Include tags generated by guest parsers
-p       pseudo            yes     NONE     no    Include pseudo tags
-q       qualified         no      NONE     no    Include an extra class-qualified tag entry for each tag
-r       reference         no      NONE     no    Include reference tags
-s       subparser         yes     NONE     no    Include tags generated by subparsers
--       configPrefixed    yes     Kconfig  no    prepend CONFIG_ to config names
--       funcmap           yes     QemuHX   no    Include mapping SQMP to C function name
--       whitespaceSwapped yes     Robot    no    Include tags swapping whitespace and underscore chars
+-       anonymous         yes     NONE        no    Include tags for non-named objects like lambda
+F       fileScope         no      NONE        no    Include tags of file scope
+f       inputFile         no      NONE        no    Include an entry for the base file name of every input file
+g       guest             no      NONE        no    Include tags generated by guest parsers
+p       pseudo            yes     NONE        no    Include pseudo tags
+q       qualified         no      NONE        no    Include an extra class-qualified tag entry for each tag
+r       reference         no      NONE        no    Include reference tags
+s       subparser         yes     NONE        no    Include tags generated by subparsers
+-       doubleSharps      no      IPythonCell no     Include cells starting from ##
+-       configPrefixed    yes     Kconfig     no    prepend CONFIG_ to config names
+-       funcmap           yes     QemuHX      no    Include mapping SQMP to C function name
+-       whitespaceSwapped yes     Robot       no    Include tags swapping whitespace and underscore chars
 # combination
--       anonymous         yes     NONE     no    Include tags for non-named objects like lambda
-F       fileScope         no      NONE     no    Include tags of file scope
-f       inputFile         yes     NONE     no    Include an entry for the base file name of every input file
-g       guest             no      NONE     no    Include tags generated by guest parsers
-p       pseudo            yes     NONE     no    Include pseudo tags
-q       qualified         no      NONE     no    Include an extra class-qualified tag entry for each tag
-r       reference         yes     NONE     no    Include reference tags
-s       subparser         yes     NONE     no    Include tags generated by subparsers
--       configPrefixed    yes     Kconfig  no    prepend CONFIG_ to config names
--       funcmap           yes     QemuHX   no    Include mapping SQMP to C function name
--       whitespaceSwapped yes     Robot    no    Include tags swapping whitespace and underscore chars
+-       anonymous         yes     NONE        no    Include tags for non-named objects like lambda
+F       fileScope         no      NONE        no    Include tags of file scope
+f       inputFile         yes     NONE        no    Include an entry for the base file name of every input file
+g       guest             no      NONE        no    Include tags generated by guest parsers
+p       pseudo            yes     NONE        no    Include pseudo tags
+q       qualified         no      NONE        no    Include an extra class-qualified tag entry for each tag
+r       reference         yes     NONE        no    Include reference tags
+s       subparser         yes     NONE        no    Include tags generated by subparsers
+-       doubleSharps      no      IPythonCell no     Include cells starting from ##
+-       configPrefixed    yes     Kconfig     no    prepend CONFIG_ to config names
+-       funcmap           yes     QemuHX      no    Include mapping SQMP to C function name
+-       whitespaceSwapped yes     Robot       no    Include tags swapping whitespace and underscore chars
 # combination with letters
--       anonymous         yes     NONE     no    Include tags for non-named objects like lambda
-F       fileScope         no      NONE     no    Include tags of file scope
-f       inputFile         no      NONE     no    Include an entry for the base file name of every input file
-g       guest             no      NONE     no    Include tags generated by guest parsers
-p       pseudo            yes     NONE     no    Include pseudo tags
-q       qualified         yes     NONE     no    Include an extra class-qualified tag entry for each tag
-r       reference         yes     NONE     no    Include reference tags
-s       subparser         yes     NONE     no    Include tags generated by subparsers
--       configPrefixed    yes     Kconfig  no    prepend CONFIG_ to config names
--       funcmap           yes     QemuHX   no    Include mapping SQMP to C function name
--       whitespaceSwapped yes     Robot    no    Include tags swapping whitespace and underscore chars
+-       anonymous         yes     NONE        no    Include tags for non-named objects like lambda
+F       fileScope         no      NONE        no    Include tags of file scope
+f       inputFile         no      NONE        no    Include an entry for the base file name of every input file
+g       guest             no      NONE        no    Include tags generated by guest parsers
+p       pseudo            yes     NONE        no    Include pseudo tags
+q       qualified         yes     NONE        no    Include an extra class-qualified tag entry for each tag
+r       reference         yes     NONE        no    Include reference tags
+s       subparser         yes     NONE        no    Include tags generated by subparsers
+-       doubleSharps      no      IPythonCell no     Include cells starting from ##
+-       configPrefixed    yes     Kconfig     no    prepend CONFIG_ to config names
+-       funcmap           yes     QemuHX      no    Include mapping SQMP to C function name
+-       whitespaceSwapped yes     Robot       no    Include tags swapping whitespace and underscore chars

Tmain/json-ptag-in-list-extras.d/stdout-expected.txt

@@ -1,6 +1,6 @@
 # json regular file
-p       pseudo            yes     NONE     no    Include pseudo tags
+p       pseudo            yes     NONE        no    Include pseudo tags
 # json -
-p       pseudo            no      NONE     no    Include pseudo tags
+p       pseudo            no      NONE        no    Include pseudo tags
 # json NOTHING
-p       pseudo            no      NONE     no    Include pseudo tags
+p       pseudo            no      NONE        no    Include pseudo tags

Tmain/list-extras.d/stdout-expected.txt

@@ -1,15 +1,16 @@
-#LETTER NAME              ENABLED LANGUAGE FIXED DESCRIPTION
--       anonymous         yes     NONE     no    Include tags for non-named objects like lambda
-F       fileScope         yes     NONE     no    Include tags of file scope
-f       inputFile         yes     NONE     no    Include an entry for the base file name of every input file
-g       guest             yes     NONE     no    Include tags generated by guest parsers
-p       pseudo            yes     NONE     no    Include pseudo tags
-q       qualified         yes     NONE     no    Include an extra class-qualified tag entry for each tag
-r       reference         yes     NONE     no    Include reference tags
-s       subparser         yes     NONE     no    Include tags generated by subparsers
--       configPrefixed    yes     Kconfig  no    prepend CONFIG_ to config names
--       funcmap           yes     QemuHX   no    Include mapping SQMP to C function name
--       whitespaceSwapped yes     Robot    no    Include tags swapping whitespace and underscore chars
+#LETTER NAME              ENABLED LANGUAGE    FIXED DESCRIPTION
+-       anonymous         yes     NONE        no    Include tags for non-named objects like lambda
+F       fileScope         yes     NONE        no    Include tags of file scope
+f       inputFile         yes     NONE        no    Include an entry for the base file name of every input file
+g       guest             yes     NONE        no    Include tags generated by guest parsers
+p       pseudo            yes     NONE        no    Include pseudo tags
+q       qualified         yes     NONE        no    Include an extra class-qualified tag entry for each tag
+r       reference         yes     NONE        no    Include reference tags
+s       subparser         yes     NONE        no    Include tags generated by subparsers
+-       doubleSharps      no      IPythonCell no     Include cells starting from ##
+-       configPrefixed    yes     Kconfig     no    prepend CONFIG_ to config names
+-       funcmap           yes     QemuHX      no    Include mapping SQMP to C function name
+-       whitespaceSwapped yes     Robot       no    Include tags swapping whitespace and underscore chars
 #LETTER	NAME	ENABLED	LANGUAGE	FIXED	DESCRIPTION
 -	anonymous	yes	NONE	no	Include tags for non-named objects like lambda
 F	fileScope	yes	NONE	no	Include tags of file scope
@@ -19,6 +20,7 @@ p	pseudo	yes	NONE	no	Include pseudo tags
 q	qualified	yes	NONE	no	Include an extra class-qualified tag entry for each tag
 r	reference	yes	NONE	no	Include reference tags
 s	subparser	yes	NONE	no	Include tags generated by subparsers
+-	doubleSharps	no	IPythonCell	no	 Include cells starting from ##
 -	configPrefixed	yes	Kconfig	no	prepend CONFIG_ to config names
 -	funcmap	yes	QemuHX	no	Include mapping SQMP to C function name
 -	whitespaceSwapped	yes	Robot	no	Include tags swapping whitespace and underscore chars

Tmain/list-subparsers-all.d/stdout-expected.txt

@@ -5,6 +5,7 @@ Automake             Make       base <= sub {dedicated}
 DBusIntrospect       XML        base <> sub {bidirectional}
 FunctionParameters   Perl       base <> sub {bidirectional}
 Glade                XML        base <> sub {bidirectional}
+IPythonCell          Python     base => sub {shared}
 ITcl                 Tcl        base <> sub {bidirectional}
 Maven2               XML        base <> sub {bidirectional}
 Moose                Perl       base <> sub {bidirectional}

Tmain/xref-ptag-in-list-extras.d/stdout-expected.txt

@@ -1,12 +1,12 @@
 # tags regular file
-p       pseudo            yes     NONE     no    Include pseudo tags
+p       pseudo            yes     NONE        no    Include pseudo tags
 # tags -
-p       pseudo            no      NONE     no    Include pseudo tags
+p       pseudo            no      NONE        no    Include pseudo tags
 # tags NOTHING
-p       pseudo            yes     NONE     no    Include pseudo tags
+p       pseudo            yes     NONE        no    Include pseudo tags
 # xref regular file
-p       pseudo            no      NONE     no    Include pseudo tags
+p       pseudo            no      NONE        no    Include pseudo tags
 # xref -
-p       pseudo            no      NONE     no    Include pseudo tags
+p       pseudo            no      NONE        no    Include pseudo tags
 # xref NOTHING
-p       pseudo            no      NONE     no    Include pseudo tags
+p       pseudo            no      NONE        no    Include pseudo tags

Units/parser-iPythonCell.r/customizing.d/args.ctags

@@ -0,0 +1,4 @@
+--sort=no
+--fields=+{language}{extras}
+--extras=+{subparser}
+--regex-IPythonCell=/[ \t]*# CTAGS:[ ]?(.*)$/\1/c/

Units/parser-iPythonCell.r/customizing.d/expected.tags

@@ -0,0 +1,4 @@
+x	input.py	/^x=1$/;"	v	language:Python
+DEFINE F	input.py	/^# CTAGS: DEFINE F$/;"	c	language:IPythonCell	extras:subparser
+F	input.py	/^def F():$/;"	f	language:Python
+DO NOTING	input.py	/^    # CTAGS: DO NOTING$/;"	c	language:IPythonCell	extras:subparser

Units/parser-iPythonCell.r/customizing.d/input.py

@@ -0,0 +1,5 @@
+x=1
+# CTAGS: DEFINE F
+def F():
+    # CTAGS: DO NOTING
+    pass

Units/parser-iPythonCell.r/default-formats.d/args.ctags

@@ -0,0 +1,3 @@
+--sort=no
+--fields=+{language}
+--extras=+{subparser}

Units/parser-iPythonCell.r/default-formats.d/expected.tags

@@ -0,0 +1,27 @@
+np	input.py	/^import numpy as np$/;"	I	language:Python	nameref:module:numpy
+plt	input.py	/^from matplotlib import pyplot as plt$/;"	x	language:Python	nameref:unknown:pyplot
+generate sound	input.py	/^# %% generate sound$/;"	c	language:IPythonCell
+f	input.py	/^f = 12000$/;"	v	language:Python
+fs	input.py	/^fs = 44100$/;"	v	language:Python
+t	input.py	/^t = np.arange(0, 1, 1\/fs)$/;"	v	language:Python
+sound	input.py	/^sound = np.sin(2*np.pi*f * t)$/;"	v	language:Python
+plot sound	input.py	/^# %% plot sound$/;"	c	language:IPythonCell
+play sound	input.py	/^# %% play sound$/;"	c	language:IPythonCell
+no space	input.py	/^#%% no space$/;"	c	language:IPythonCell
+a	input.py	/^a=1$/;"	v	language:Python
+% extra percent chars with space	input.py	/^# %%% extra percent chars with space$/;"	c	language:IPythonCell
+b	input.py	/^b=1$/;"	v	language:Python
+% extra percent chars without space	input.py	/^#%%% extra percent chars without space$/;"	c	language:IPythonCell
+c	input.py	/^c=1$/;"	v	language:Python
+f	input.py	/^def f():$/;"	f	language:Python
+no space with prefix	input.py	/^    #%% no space with prefix$/;"	c	language:IPythonCell
+g	input.py	/^def g():$/;"	f	language:Python
+% extra percent chars with space and prefix	input.py	/^    # %%% extra percent chars with space and prefix$/;"	c	language:IPythonCell
+h	input.py	/^def h():$/;"	f	language:Python
+% extra percent chars without space with prefix	input.py	/^    #%%% extra percent chars without space with prefix$/;"	c	language:IPythonCell
+triangle without space	input.py	/^# <codecell>triangle without space$/;"	c	language:IPythonCell
+A	input.py	/^A=1$/;"	v	language:Python
+triangle with space	input.py	/^# <codecell> triangle with space$/;"	c	language:IPythonCell
+B	input.py	/^B=1$/;"	v	language:Python
+F	input.py	/^def F():$/;"	f	language:Python
+triangle with space and prefix	input.py	/^    # <codecell> triangle with space and prefix$/;"	c	language:IPythonCell

Units/parser-iPythonCell.r/default-formats.d/input.py

@@ -0,0 +1,53 @@
+# Derived from https://github.com/universal-ctags/ctags/issues/2978
+# submitted by @gerazov.
+import numpy as np
+from matplotlib import pyplot as plt
+from scipy.io import wavfile
+import os
+
+# %% generate sound
+f = 12000
+fs = 44100
+t = np.arange(0, 1, 1/fs)
+sound = np.sin(2*np.pi*f * t)
+
+# %% plot sound
+plt.plot(t, sound)
+
+# %% play sound
+wavfile.write('sound.wav', fs, np.int16(sound * 2**15))
+os.system('play sound.wav')
+
+#%% no space
+a=1
+
+# %%% extra percent chars with space
+b=1
+
+#%%% extra percent chars without space
+c=1
+
+def f():
+    #%% no space with prefix
+    pass
+
+def g():
+    # %%% extra percent chars with space and prefix
+    pass
+
+def h():
+    #%%% extra percent chars without space with prefix
+    pass
+
+# <codecell>triangle without space
+A=1
+
+# <codecell> triangle with space
+B=1
+
+def F():
+    # <codecell> triangle with space and prefix
+    pass
+
+## IGNORE ME
+

Units/parser-iPythonCell.r/double-sharps.d/args.ctags

@@ -0,0 +1,4 @@
+--sort=no
+--fields=+{language}{extras}
+--extras=+{subparser}
+--extras-IPythonCell=+{doubleSharps}