Merge pull request #3684 from masatake/terraform-parser

* Terraform: new parser
* added _foreignLanguage flag to --langdef option
* added _language flag to --regex-<LANG>, --mline-regex-<LANG>, and --_mtable-regex-<LANG>

Tmain/list-map-extensions.d/stdout-expected.txt

@@ -1,9 +1,9 @@
 ## all|grep LdScript
-#LANGUAGE         EXTENSION
-LdScript          lds
-LdScript          scr
-LdScript          ld
-LdScript          ldi
+#LANGUAGE           EXTENSION
+LdScript            lds
+LdScript            scr
+LdScript            ld
+LdScript            ldi
 ## LdScript
 #EXTENSION
 lds

Tmain/list-mline-regex-flags.d/stdout-expected.txt

@@ -11,4 +11,5 @@ p       pcre2                                         use pcre2 regex engine
 -       _extra=EXTRA                                  record the tag only when the extra is enabled
 -       _field=FIELD:VALUE                            record the matched string(VALUE) to parser own FIELD of the tag
 -       _guest=PARSERSPEC,N0[start|end],N1[start|end] run guest parser on the area
+-       _language=LANG                                make a foreign tag for LANG
 -       _role=ROLE                                    set the given ROLE to the roles field

Tmain/list-mtable-regex-flags.d/stdout-expected.txt

@@ -18,4 +18,5 @@ p       pcre2                                         use pcre2 regex engine
 -       _extra=EXTRA                                  record the tag only when the extra is enabled
 -       _field=FIELD:VALUE                            record the matched string(VALUE) to parser own FIELD of the tag
 -       _guest=PARSERSPEC,N0[start|end],N1[start|end] run guest parser on the area
+-       _language=LANG                                make a foreign tag for LANG
 -       _role=ROLE                                    set the given ROLE to the roles field

Tmain/list-regex-flags.d/stdout-expected.txt

@@ -12,4 +12,5 @@ x       exclusive                                     skip testing the other pat
 -       _extra=EXTRA                                  record the tag only when the extra is enabled
 -       _field=FIELD:VALUE                            record the matched string(VALUE) to parser own FIELD of the tag
 -       _guest=PARSERSPEC,N0[start|end],N1[start|end] run guest parser on the area
+-       _language=LANG                                make a foreign tag for LANG
 -       _role=ROLE                                    set the given ROLE to the roles field

Tmain/list-roles.d/stdout-expected.txt

@@ -102,6 +102,7 @@ SystemdUnit    u/unit            RequiredBy         on      referred in Required
 SystemdUnit    u/unit            Requires           on      referred in Requires key
 SystemdUnit    u/unit            WantedBy           on      referred in WantedBy key
 SystemdUnit    u/unit            Wants              on      referred in Wants key
+Terraform      v/variable        assigned           on      assigned in Variable Definitions (.tfvars) files
 Tex            e/environment     used               off     environment usage introduced by \begin{MyEnv}
 Tex            i/xinput          bibliography       on      bibliography (.bib) file
 Tex            i/xinput          included           on      external input file specified with \include
@@ -221,6 +222,7 @@ SystemdUnit    u/unit            RequiredBy         on      referred in Required
 SystemdUnit    u/unit            Requires           on      referred in Requires key
 SystemdUnit    u/unit            WantedBy           on      referred in WantedBy key
 SystemdUnit    u/unit            Wants              on      referred in Wants key
+Terraform      v/variable        assigned           on      assigned in Variable Definitions (.tfvars) files
 Tex            e/environment     used               off     environment usage introduced by \begin{MyEnv}
 Tex            i/xinput          bibliography       on      bibliography (.bib) file
 Tex            i/xinput          included           on      external input file specified with \include

Units/parser-terraform.r/data.d/expected.tags

@@ -0,0 +1 @@
+example_database_password	input.tf	/^data "aws_ssm_parameter" "example_database_password" {$/;"	d

Units/parser-terraform.r/data.d/input.tf

@@ -0,0 +1,3 @@
+data "aws_ssm_parameter" "example_database_password" {
+  name = "example-database-password"
+}

Units/parser-terraform.r/module.d/expected.tags

@@ -0,0 +1 @@
+database	input.tf	/^module "database" {$/;"	m

Units/parser-terraform.r/module.d/input.tf

@@ -0,0 +1,3 @@
+module "database" {
+  source = "../../modules/database"
+}

Units/parser-terraform.r/output.d/expected.tags

@@ -0,0 +1 @@
+password	input.tf	/^output "password" {$/;"	o

Units/parser-terraform.r/output.d/input.tf

@@ -0,0 +1,3 @@
+output "password" {
+  value = data.aws_ssm_parameter.example_database_password.value
+}

Units/parser-terraform.r/provider.d/expected.tags

@@ -0,0 +1 @@
+aws	input.tf	/^provider "aws" {$/;"	p

Units/parser-terraform.r/provider.d/input.tf

@@ -0,0 +1,3 @@
+provider "aws" {
+  region = "us-east-1"
+}

Units/parser-terraform.r/resource.d/expected.tags

@@ -0,0 +1 @@
+example_events_bus	input.tf	/^resource "aws_cloudwatch_event_bus" "example_events_bus" {$/;"	r

Units/parser-terraform.r/resource.d/input.tf

@@ -0,0 +1,3 @@
+resource "aws_cloudwatch_event_bus" "example_events_bus" {
+  name = var.events_bus_name
+}

Units/parser-terraform.r/simple-terraform.d/args.ctags

@@ -0,0 +1,3 @@
+--sort=no
+--extras=+r
+--fields=+rl

Units/parser-terraform.r/simple-terraform.d/expected.tags

@@ -0,0 +1,7 @@
+aws	input.tf	/^provider "aws" {$/;"	p	language:Terraform	roles:def
+events_bus_name	input.tf	/^variable "events_bus_name" {$/;"	v	language:Terraform	roles:def
+example_events_bus	input.tf	/^resource "aws_cloudwatch_event_bus" "example_events_bus" {$/;"	r	language:Terraform	roles:def
+database	input.tf	/^module "database" {$/;"	m	language:Terraform	roles:def
+example_database_password	input.tf	/^data "aws_ssm_parameter" "example_database_password" {$/;"	d	language:Terraform	roles:def
+password	input.tf	/^output "password" {$/;"	o	language:Terraform	roles:def
+events_bus_name	input-0.tfvars	/^events_bus_name = "hyper-connector"$/;"	v	language:Terraform	roles:assigned

Units/parser-terraform.r/simple-terraform.d/input-0.tfvars

@@ -0,0 +1 @@
+events_bus_name = "hyper-connector"

Units/parser-terraform.r/simple-terraform.d/input.tf

@@ -0,0 +1,27 @@
+provider "aws" {
+  region = "us-east-1"
+}
+
+variable "events_bus_name" {
+  type = string
+  default = "hello-world"
+}
+
+# variable "dont_extract_me0" {
+// variable "dont_extract_me1" {
+
+resource "aws_cloudwatch_event_bus" "example_events_bus" {
+  name = var.events_bus_name
+}
+
+module "database" {
+  source = "../../modules/database"
+}
+
+data "aws_ssm_parameter" "example_database_password" {
+  name = "example-database-password"
+}
+
+output "password" {
+  value = data.aws_ssm_parameter.example_database_password.value
+}

Units/parser-terraform.r/variable.d/expected.tags

@@ -0,0 +1 @@
+events_bus_name	input.tf	/^variable "events_bus_name" {$/;"	v

Units/parser-terraform.r/variable.d/input.tf

@@ -0,0 +1,4 @@
+variable "events_bus_name" {
+  type = string
+  default = "hello-world"
+}

docs/man-pages.rst

@@ -31,6 +31,7 @@ Man pages
 	ctags-lang-sql(7) <man/ctags-lang-sql.7.rst>
 	ctags-lang-systemtap(7) <man/ctags-lang-systemtap.7.rst>
 	ctags-lang-tcl(7) <man/ctags-lang-tcl.7.rst>
+	ctags-lang-terraform(7) <man/ctags-lang-terraform.7.rst>
 	ctags-lang-verilog(7) <man/ctags-lang-verilog.7.rst>
 
 	readtags(1) <man/readtags.1.rst>

docs/man/ctags-lang-terraform.7.rst

@@ -0,0 +1,47 @@
+.. _ctags-lang-terraform(7):
+
+==============================================================
+ctags-lang-terraform
+==============================================================
+
+Random notes about tagging Terraform files with Universal Ctags
+
+:Version: 6.0.0
+:Manual group: Universal Ctags
+:Manual section: 7
+
+SYNOPSIS
+--------
+|	**ctags** ... --languages=+Terraform ...
+|	**ctags** ... --language-force=Terraform ...
+|	**ctags** ... --map-Terraform=+.tf ...
+|
+|	**ctags** ... --extras=+{reference} --languages=+TerraformVariables ...
+|	**ctags** ... --extras=+{reference} --language-force=TerraformVariables ...
+|	**ctags** ... --extras=+{reference} --map-Terraform=+.tfvars ...
+
+DESCRIPTION
+-----------
+This man page gathers random notes about tagging Terraform files.
+
+TIPS
+-----------
+
+Extracting variables assigned in Variable definitions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Specify ``--extras=+{reference}`` and ``--languages=+TerraformVariables``
+to extract variables assigned in variables definitions (`*.tfvars`).
+The TerraformVariables parser extracts variables in `*.tfvars` files
+with ``variable`` kind with ``assigned`` role of ``Terraform`` language.
+
+KNOWN BUGS
+----------
+* The parser doesn't ignore strings inside a block comment (``/* ... */``).
+
+* The parser doesn't extract variables defined with ``locals`` keyword.
+
+SEE ALSO
+--------
+:ref:`ctags(1) <ctags(1)>`,
+`Configuration Syntax <https://developer.hashicorp.com/terraform/language/syntax/configuration>`_ (https://developer.hashicorp.com/terraform/language/syntax/configuration),
+`Variable Definitions (.tfvars) Files <https://developer.hashicorp.com/terraform/language/values/variables#variable-definitions-tfvars-files>`_ (https://developer.hashicorp.com/terraform/language/values/variables#variable-definitions-tfvars-files)

docs/news.rst

@@ -474,6 +474,8 @@ The following parsers have been added:
 * SystemVerilog
 * SVG *libxml*
 * TclOO (see :ref:`The new Tcl parser <tcl>`)
+* Terraform (HCL) *optlib*
+* TerraformVariables *optlib*
 * Thrift *peg/packcc*
 * TTCN
 * Txt2tags

main/lregex.c

@@ -167,6 +167,7 @@ typedef struct {
 	char *pattern_string;
 
 	char *anonymous_tag_prefix;
+	langType foreign_lang;
 
 	struct {
 		errorSelection selection;
@@ -397,10 +398,14 @@ extern void freeLregexControlBlock (struct lregexControlBlock* lcb)
 
 static void initRegexTag (tagEntryInfo *e,
 		const char * name, int kindIndex, int roleIndex, int scopeIndex, int placeholder,
-		unsigned long line, MIOPos *pos, int xtag_type)
+		unsigned long line, MIOPos *pos, int xtag_type, langType foreign_lang)
 {
 	Assert (name != NULL  &&  ((name[0] != '\0') || placeholder));
-	initRefTagEntry (e, name, kindIndex, roleIndex);
+
+	if (foreign_lang == LANG_IGNORE)
+		initRefTagEntry (e, name, kindIndex, roleIndex);
+	else
+		initForeignRefTagEntry (e, name, foreign_lang, kindIndex, roleIndex);
 	e->extensionFields.scopeIndex = scopeIndex;
 	e->placeholder = !!placeholder;
 	if (line)
@@ -672,6 +677,8 @@ static regexPattern * newPattern (regexCompiledCode* const pattern,
 	ptrn->optscript = NULL;
 	ptrn->optscript_src = NULL;
 
+	ptrn->foreign_lang = LANG_IGNORE;
+
 	return ptrn;
 }
 
@@ -1112,7 +1119,7 @@ static void common_flag_role_long (const char* const s, const char* const v, voi
 		return;
 	}
 
-	role = getLanguageRoleForName(cdata->owner,
+	role = getLanguageRoleForName((ptrn->foreign_lang == LANG_IGNORE? cdata->owner: ptrn->foreign_lang),
 								  ptrn->u.tag.kindIndex, v);
 	if (!role)
 	{
@@ -1154,6 +1161,39 @@ static void common_flag_anonymous_long (const char* const s, const char* const v
 	ptrn->anonymous_tag_prefix = eStrdup (v);
 }
 
+static void precommon_flag_foreign_lang (const char* const s, const char* const v, void* data)
+{
+	struct commonFlagData * cdata = data;
+	regexPattern *ptrn = cdata->ptrn;
+
+	Assert (ptrn);
+
+	if (ptrn->foreign_lang != LANG_IGNORE)
+		error (FATAL, "foreign language for this pattern (%s) is already given: %s",
+			   ptrn->pattern_string? ptrn->pattern_string: "",
+			   getLanguageName(ptrn->foreign_lang));
+
+	if (!v)
+		error (FATAL, "no LANG for foreign flag is given (pattern == %s)",
+			   ptrn->pattern_string? ptrn->pattern_string: "");
+
+	langType lang = getNamedLanguage (v, 0);
+	if (lang == LANG_IGNORE)
+		error (FATAL, "language named '%s' specified is not found or not initialized yet",
+			   v);
+
+	if (!doesLanguageHaveForeignDependency (cdata->owner, lang))
+		error (FATAL, "%s is not declared as a foreign language in %s parser",
+			   v, getLanguageName (cdata->owner));
+
+	ptrn->foreign_lang = lang;
+}
+
+static flagDefinition preCommonSpecFlagDef[] = {
+	{ '\0',  EXPERIMENTAL "language", NULL, precommon_flag_foreign_lang,
+	  "LANG", "make a foreign tag for LANG"},
+};
+
 static flagDefinition commonSpecFlagDef[] = {
 	{ '\0',  "fatal", NULL, common_flag_msg_long ,
 	  "\"MESSAGE\"", "print the given MESSAGE and exit"},
@@ -1262,15 +1302,17 @@ static void setKind(regexPattern * ptrn, const langType owner,
 	Assert (ptrn);
 	Assert (ptrn->u.tag.name_pattern);
 	Assert (kindName);
-	kindDefinition *kdef = getLanguageKindForLetter (owner, kindLetter);
+
+	langType lang = (ptrn->foreign_lang == LANG_IGNORE? owner: ptrn->foreign_lang);
+	kindDefinition *kdef = getLanguageKindForLetter (lang, kindLetter);
 
 	if (kdef)
 	{
 		if (strcmp (kdef->name, kindName) && (strcmp(kindName, KIND_REGEX_DEFAULT_NAME)))
 			/* When using a same kind letter for multiple regex patterns, the name of kind
 			   should be the same. */
 			error  (WARNING, "Don't reuse the kind letter `%c' in a language %s (old: \"%s\", new: \"%s\")",
-					kdef->letter, getLanguageName (owner),
+					kdef->letter, getLanguageName (lang),
 					kdef->name, kindName);
 		ptrn->u.tag.kindIndex = kdef->id;
 	}
@@ -1282,11 +1324,25 @@ static void setKind(regexPattern * ptrn, const langType owner,
 	else
 	{
 		kdef = kindNew (kindLetter, kindName, description);
-		defineLanguageKind (owner, kdef, kindFree);
+		defineLanguageKind (lang, kdef, kindFree);
 		ptrn->u.tag.kindIndex = kdef->id;
 	}
 }
 
+static void prePatternEvalFlags(struct lregexControlBlock *lcb,
+							  regexPattern * ptrn,
+							  enum regexParserType regptype,
+							  const char* flags)
+{
+	struct commonFlagData commonFlagData = {
+		.owner = lcb->owner,
+		.lcb = lcb,
+		.ptrn = ptrn
+	};
+
+	flagsEval (flags, preCommonSpecFlagDef, ARRAY_SIZE(preCommonSpecFlagDef), &commonFlagData);
+}
+
 static void patternEvalFlags (struct lregexControlBlock *lcb,
 							  regexPattern * ptrn,
 							  enum regexParserType regptype,
@@ -1347,6 +1403,7 @@ static regexPattern *addCompiledTagPattern (struct lregexControlBlock *lcb,
 	ptrn->u.tag.name_pattern = eStrdup (name);
 	ptrn->disabled = disabled;
 
+	prePatternEvalFlags (lcb, ptrn, regptype, flags);
 	setKind(ptrn, lcb->owner, kindLetter, kindName, description, kind_explicitly_defined);
 	patternEvalFlags (lcb, ptrn, regptype, flags);
 
@@ -1680,7 +1737,7 @@ static void matchTagPattern (struct lregexControlBlock *lcb,
 		roleBits = patbuf->u.tag.roleBits;
 
 		initRegexTag (&e, vStringValue (name), kind, ROLE_DEFINITION_INDEX, scope, placeholder,
-					  ln, ln == 0? NULL: &pos, patbuf->xtagType);
+					  ln, ln == 0? NULL: &pos, patbuf->xtagType, patbuf->foreign_lang);
 
 		if (field_trashbox == NULL)
 		{
@@ -2472,6 +2529,7 @@ extern void printRegexFlags (bool withListHeader, bool machinable, const char *f
 		flagsColprintAddDefinitions (table, prePtrnFlagDef, ARRAY_SIZE (prePtrnFlagDef));
 		flagsColprintAddDefinitions (table, guestPtrnFlagDef, ARRAY_SIZE (guestPtrnFlagDef));
 		flagsColprintAddDefinitions (table, scopePtrnFlagDef, ARRAY_SIZE (scopePtrnFlagDef));
+		flagsColprintAddDefinitions (table, preCommonSpecFlagDef, ARRAY_SIZE (preCommonSpecFlagDef));
 		flagsColprintAddDefinitions (table, commonSpecFlagDef, ARRAY_SIZE (commonSpecFlagDef));
 	}
 
@@ -2497,6 +2555,7 @@ extern void printMultilineRegexFlags (bool withListHeader, bool machinable, cons
 		flagsColprintAddDefinitions (table, backendCommonRegexFlagDefs, ARRAY_SIZE(backendCommonRegexFlagDefs));
 		flagsColprintAddDefinitions (table, multilinePtrnFlagDef, ARRAY_SIZE (multilinePtrnFlagDef));
 		flagsColprintAddDefinitions (table, guestPtrnFlagDef, ARRAY_SIZE (guestPtrnFlagDef));
+		flagsColprintAddDefinitions (table, preCommonSpecFlagDef, ARRAY_SIZE (preCommonSpecFlagDef));
 		flagsColprintAddDefinitions (table, commonSpecFlagDef, ARRAY_SIZE (commonSpecFlagDef));
 	}
 
@@ -2524,6 +2583,7 @@ extern void printMultitableRegexFlags (bool withListHeader, bool machinable, con
 		flagsColprintAddDefinitions (table, multitablePtrnFlagDef, ARRAY_SIZE (multitablePtrnFlagDef));
 		flagsColprintAddDefinitions (table, guestPtrnFlagDef, ARRAY_SIZE (guestPtrnFlagDef));
 		flagsColprintAddDefinitions (table, scopePtrnFlagDef, ARRAY_SIZE (scopePtrnFlagDef));
+		flagsColprintAddDefinitions (table, preCommonSpecFlagDef, ARRAY_SIZE (preCommonSpecFlagDef));
 		flagsColprintAddDefinitions (table, commonSpecFlagDef, ARRAY_SIZE (commonSpecFlagDef));
 	}
 
@@ -3247,7 +3307,7 @@ static EsObject* lrop_make_tag (OptVM *vm, EsObject *name)
 	tagEntryInfo *e = xMalloc (1, tagEntryInfo);
 	initRegexTag (e, eStrdup (n),
 				  kind_index, ROLE_DEFINITION_INDEX, CORK_NIL, 0,
-				  loc? loc->line: 0, loc? &loc->pos: NULL, XTAG_UNKNOWN);
+				  loc? loc->line: 0, loc? &loc->pos: NULL, XTAG_UNKNOWN, LANG_IGNORE);
 	EsObject *obj = es_pointer_new (OPT_TYPE_TAG, e);
 	if (es_error_p (obj))
 		return obj;
@@ -3324,7 +3384,7 @@ static EsObject* lrop_make_reftag (OptVM *vm, EsObject *name)
 				  loc? loc->line: 0, loc? &loc->pos: NULL,
 				  role_index == ROLE_DEFINITION_INDEX
 				  ? XTAG_UNKNOWN
-				  : XTAG_REFERENCE_TAGS);
+				  : XTAG_REFERENCE_TAGS, LANG_IGNORE);
 	EsObject *obj = es_pointer_new (OPT_TYPE_TAG, e);
 	if (es_error_p (obj))
 		return obj;