Merge pull request #3365 from masatake/macro-expansion--ldscript

LdScript: expand C preprocessor macros

Units/parser-ldscript.r/lds-macro-expansion.d/args.ctags

@@ -0,0 +1,5 @@
+--sort=no
+--param-CPreProcessor._expand=1
+--fields=+{signature}{roles}
+--fields-CPreProcessor=+{macrodef}
+--extras=+r

Units/parser-ldscript.r/lds-macro-expansion.d/expected.tags

@@ -0,0 +1,10 @@
+MEM_DISCARD	input.lds.S	/^#define MEM_DISCARD(/;"	d	file:	signature:(sec)	roles:def	macrodef:*(.mem##sec)
+INIT_TEXT	input.lds.S	/^#define INIT_TEXT(/;"	d	file:	signature:(X,A)	roles:def	macrodef:*A *X MEM_DISCARD(init.text*)
+INIT_TEXT_SECTION	input.lds.S	/^#define INIT_TEXT_SECTION(/;"	d	file:	signature:(inittext_align,Y,B)	roles:def	macrodef:. = ALIGN(inittext_align); .init.text : AT(ADDR(.init.text) - LOAD_OFFSET) { _sinittext = .; INIT_TEXT(Y,B) _einittext = .; }
+.init.text	input.lds.S	/^	INIT_TEXT_SECTION(PAGE_SIZE,(.text.startup),(.init.text .init.text.*))$/;"	S	roles:def
+_sinittext	input.lds.S	/^	INIT_TEXT_SECTION(PAGE_SIZE,(.text.startup),(.init.text .init.text.*))$/;"	s	section:.init.text	roles:def
+.init.text	input.lds.S	/^	INIT_TEXT_SECTION(PAGE_SIZE,(.text.startup),(.init.text .init.text.*))$/;"	i	section:.init.text	roles:mapped
+.init.text.	input.lds.S	/^	INIT_TEXT_SECTION(PAGE_SIZE,(.text.startup),(.init.text .init.text.*))$/;"	i	section:.init.text	roles:mapped
+.text.startup	input.lds.S	/^	INIT_TEXT_SECTION(PAGE_SIZE,(.text.startup),(.init.text .init.text.*))$/;"	i	section:.init.text	roles:mapped
+.meminit.text	input.lds.S	/^	INIT_TEXT_SECTION(PAGE_SIZE,(.text.startup),(.init.text .init.text.*))$/;"	i	section:.init.text	roles:mapped
+_einittext	input.lds.S	/^	INIT_TEXT_SECTION(PAGE_SIZE,(.text.startup),(.init.text .init.text.*))$/;"	s	section:.init.text	roles:def

Units/parser-ldscript.r/lds-macro-expansion.d/input.lds.S

@@ -0,0 +1,27 @@
+/* Based on linux/include/asm-generic/vmlinux.lds.h and
+ * linux/arch/x86/kernel/vmlinux.lds.S */
+
+#define MEM_DISCARD(sec) *(.mem##sec)
+
+#define INIT_TEXT(X,A)				\
+	*A					\
+	*X					\
+	MEM_DISCARD(init.text*)
+
+#define INIT_TEXT_SECTION(inittext_align,Y,B)				\
+	. = ALIGN(inittext_align);					\
+	.init.text : AT(ADDR(.init.text) - LOAD_OFFSET) {		\
+		_sinittext = .;						\
+		INIT_TEXT(Y,B)						\
+		_einittext = .;						\
+	}
+
+
+SECTIONS
+{
+	INIT_TEXT_SECTION(PAGE_SIZE,(.text.startup),(.init.text .init.text.*))
+#ifdef CONFIG_X86_64
+	:init
+#endif
+}
+

parsers/cpreprocessor.c

@@ -1925,7 +1925,7 @@ static bool buildMacroInfoFromTagEntry (int corkIndex,
 {
 	cppMacroInfo **info = data;
 
-	if (entry->langType == Cpp.clientLang
+	if ((entry->langType == Cpp.clientLang || entry->langType == Cpp.lang)
 		&& entry->kindIndex == Cpp.defineMacroKindIndex
 		&& isRoleAssigned (entry, ROLE_DEFINITION_INDEX))
 	{
@@ -2031,6 +2031,54 @@ extern vString * cppBuildMacroReplacement(
 	return ret;
 }
 
+// We stop applying macro replacements if the unget buffer gets too big
+// as it is a sign of recursive macro expansion
+#define CPP_PARSER_MAXIMUM_UNGET_BUFFER_SIZE_FOR_MACRO_REPLACEMENTS 65536
+
+extern void cppBuildMacroReplacementWithPtrArrayAndUngetResult(
+		cppMacroInfo * macro,
+		const ptrArray * args)
+{
+	vString * replacement = NULL;
+
+	// Detect other cases of nasty macro expansion that cause
+	// the unget buffer to grow fast (but the token chain to grow slowly)
+	//    -D'p=a' -D'a=p+p'
+	if ((cppUngetBufferSize() < CPP_PARSER_MAXIMUM_UNGET_BUFFER_SIZE_FOR_MACRO_REPLACEMENTS)
+		&& macro->replacements)
+	{
+		int argc = 0;
+		const char ** argv = NULL;
+
+		if (args)
+		{
+			argc = ptrArrayCount (args);
+			argv = (const char **)eMalloc (sizeof(char *) * argc);
+			for (int i = 0; i < argc; i++)
+			{
+				TRACE_PRINT("Arg[%d] for %s<%p>: %s",
+							i, macro->name, macro, ptrArrayItem (args, i));
+				argv[i] = ptrArrayItem (args, i);
+			}
+		}
+
+		replacement = cppBuildMacroReplacement(macro, argv, argc);
+
+		if (argv)
+			eFree ((void *)argv);
+	}
+
+	if (replacement)
+	{
+		cppUngetStringBuiltByMacro(vStringValue(replacement), vStringLength(replacement),
+								   macro);
+		TRACE_PRINT("Replacement for %s<%p>: %s", macro->name, macro, vStringValue (replacement));
+		vStringDelete (replacement);
+	}
+	else
+		TRACE_PRINT("Replacement for %s<%p>: ", macro->name, macro);
+
+}
 
 static void saveIgnoreToken(const char * ignoreToken)
 {
@@ -2093,8 +2141,8 @@ static void saveIgnoreToken(const char * ignoreToken)
 	}
 	info->useCount = 0;
 	info->next = NULL;
-
-	hashTablePutItem(cmdlineMacroTable,eStrndup(tokenBegin,tokenEnd - tokenBegin),info);
+	info->name = eStrndup(tokenBegin,tokenEnd - tokenBegin);
+	hashTablePutItem(cmdlineMacroTable,info->name,info);
 
 	verbose ("    ignore token: %s\n", ignoreToken);
 }
@@ -2389,7 +2437,8 @@ static cppMacroInfo * saveMacro(hashTable *table, const char * macro)
 			ADD_CONSTANT_REPLACEMENT(begin,c - begin);
 	}
 
-	hashTablePutItem(table,eStrndup(identifierBegin,identifierEnd - identifierBegin),info);
+	info->name = eStrndup(identifierBegin,identifierEnd - identifierBegin);
+	hashTablePutItem(table,info->name,info);
 	CXX_DEBUG_LEAVE();
 
 	return info;
@@ -2408,6 +2457,7 @@ static void freeMacroInfo(cppMacroInfo * info)
 		pPart = pPart->next;
 		eFree(pPartToDelete);
 	}
+	eFree(info->name);
 	eFree(info);
 }
 
@@ -2417,7 +2467,7 @@ static hashTable *makeMacroTable (void)
 		1024,
 		hashCstrhash,
 		hashCstreq,
-		eFree,
+		NULL,					/* Keys refers values' name fields. */
 		(void (*)(void *))freeMacroInfo
 		);
 }

parsers/cpreprocessor.h

@@ -13,6 +13,7 @@
 *   INCLUDE FILES
 */
 #include "general.h"  /* must always come first */
+#include "ptrarray.h"
 #include "types.h"
 #include "vstring.h"
 
@@ -115,6 +116,7 @@ typedef struct sCppMacroReplacementPartInfo {
 } cppMacroReplacementPartInfo;
 
 typedef struct sCppMacroInfo {
+	char *name;			/* the name of macro. Useful for debugging. */
 	bool hasParameterList; /* true if the macro has a trailing () */
 	cppMacroReplacementPartInfo * replacements;
 	int useCount;
@@ -136,4 +138,10 @@ extern vString * cppBuildMacroReplacement(
 		int parameterCount
 	);
 
+/* Do the same as cppBuildMacroReplacement with ptrArray<const char*>,
+ * and unget the result of expansion to input cpp stream. */
+extern void cppBuildMacroReplacementWithPtrArrayAndUngetResult(
+		cppMacroInfo * macro,
+		const ptrArray * args);
+
 #endif  /* CTAGS_MAIN_GET_H */

parsers/cxx/cxx_parser_tokenizer.c

@@ -1341,16 +1341,16 @@ bool cxxParserParseNextToken(void)
 			{
 				/* If the macro is overly used, report it here. */
 				CXX_DEBUG_PRINT("Overly uesd macro %s <%p> useCount: %d (> %d)",
-								vStringValue(t->pszWord),
+								pMacro->name,
 								pMacro, pMacro->useCount,
 								CXX_PARSER_MAXIMUM_MACRO_USE_COUNT);
 			}
 #endif
 
 			if(pMacro && (pMacro->useCount < CXX_PARSER_MAXIMUM_MACRO_USE_COUNT))
 			{
-				CXX_DEBUG_PRINT("Macro %s <%p> useCount: %d", vStringValue(t->pszWord),
-								pMacro, pMacro? pMacro->useCount: -1);
+				CXX_DEBUG_PRINT("Macro %s <%p> useCount: %d", pMacro->name,
+								pMacro, pMacro->useCount);
 
 				cxxTokenChainDestroyLast(g_cxx.pTokenChain);
 

parsers/fypp.c

@@ -450,9 +450,10 @@ static void fyppSetGuestParser (const langType language CTAGS_ATTR_UNUSED,
 }
 
 static parameterHandlerTable FyppParameterHandlerTable [] = {
-	{ .name = "guest",
-	  .desc = "parser run after Fypp parser parses the original input (\"NONE\" or a parser name [Fortran])" ,
-	  .handleParameter = fyppSetGuestParser,
+	{
+		.name = "guest",
+		.desc = "parser run after Fypp parser parses the original input (\"NONE\" or a parser name [Fortran])" ,
+		.handleParameter = fyppSetGuestParser,
 	},
 };