From 82fb3b03803110a8957770773f2affc56fb20f10 Mon Sep 17 00:00:00 2001 From: xescugc Date: Mon, 3 Aug 2020 09:05:26 +0200 Subject: [PATCH 1/6] provider: Add the 'Groups' function to the Provider It allows the Provider to return the groups of a Resource ID --- provider/aws/aws.go | 20 ++++++++++++++++++++ provider/nop_provider.go | 5 +++++ provider/provider.go | 3 +++ 3 files changed, 28 insertions(+) diff --git a/provider/aws/aws.go b/provider/aws/aws.go index 8a66772..d362ee3 100644 --- a/provider/aws/aws.go +++ b/provider/aws/aws.go @@ -23,6 +23,8 @@ var ( "ingress", "source_security_group_id", "security_group_id", + "vpc_id", + "vpc_security_group_ids", } ) @@ -123,3 +125,21 @@ func (a Provider) ResourceInOut(id, rs string, cfgs map[string]map[string]interf func (a Provider) UsedAttributes() []string { return usedAttributes } + +// Groups returns all the groups for the resource id with the config cfg +func (a Provider) Groups(id string, cfg map[string]map[string]interface{}) []string { + groups := make([]string, 0) + if v, ok := cfg[id]["vpc_id"]; ok { + groups = append(groups, v.(string)) + } + + // If it has vpc_security_group_ids it means it has IDs + // to the config, so we'll recall Groups + if v, ok := cfg[id]["vpc_security_group_ids"]; ok { + for _, s := range v.([]interface{}) { + groups = append(groups, a.Groups(s.(string), cfg)...) + } + } + + return groups +} diff --git a/provider/nop_provider.go b/provider/nop_provider.go index 2839fdc..1e224bd 100644 --- a/provider/nop_provider.go +++ b/provider/nop_provider.go @@ -47,3 +47,8 @@ func (n NopProvider) UsedAttributes() []string { return nil } func (n NopProvider) PreProcess(cfg map[string]map[string]interface{}) [][]string { return nil } + +// Groups returns all the groups for the resource id with the config cfg +func (n NopProvider) Groups(rid string, cfg map[string]map[string]interface{}) []string { + return nil +} diff --git a/provider/provider.go b/provider/provider.go index d8c0252..ae0f8de 100644 --- a/provider/provider.go +++ b/provider/provider.go @@ -45,4 +45,7 @@ type Provider interface { // [_][0] is the source of the edge // [_][1] is the target of the edge PreProcess(cfg map[string]map[string]interface{}) [][]string + + // Groups returns all the groups for the resource id with the config cfg + Groups(id string, cfg map[string]map[string]interface{}) []string } From b7025dc0893a03ebb55d4ee2d4756753433cb138 Mon Sep 17 00:00:00 2001 From: xescugc Date: Mon, 3 Aug 2020 09:07:07 +0200 Subject: [PATCH 2/6] graph: Add the 'Node.AddGroupIDs' That allows to add the GroupIDs in which the Node belongs to --- graph/graph.go | 4 ++++ graph/node.go | 24 ++++++++++++++++++++++++ graph/node_test.go | 22 ++++++++++++++++++++++ 3 files changed, 50 insertions(+) create mode 100644 graph/node_test.go diff --git a/graph/graph.go b/graph/graph.go index c3e3ad3..148c499 100644 --- a/graph/graph.go +++ b/graph/graph.go @@ -153,6 +153,10 @@ func (g *Graph) Replace(srcID, repID string) error { return err } + // We add the Groups the srcNode has to the + // end Node so we'll keep them + repNode.AddGroupIDs(srcNode.GroupIDs...) + // mutualEdge is the edge that connects this 2 Nodes var mutualEdge *Edge for _, e := range srcEdges { diff --git a/graph/node.go b/graph/node.go index 11171b1..4fdd73b 100644 --- a/graph/node.go +++ b/graph/node.go @@ -23,4 +23,28 @@ type Node struct { // Weight is the addition of the Directions // of the Node Weight int + + // GroupIDs is the IDs of all the groups + // it may belong to + GroupIDs []string + + // mGroupIDs is used to know which GroupIDs + // are already on the GroupIDs slice so we + // do not repeat them + mGroupIDs map[string]struct{} +} + +// AddGroupIDs adds the ids to the internal list, if +// one is repeated it'll be ignored +func (n *Node) AddGroupIDs(gids ...string) { + if n.mGroupIDs == nil { + n.mGroupIDs = make(map[string]struct{}) + } + + for _, c := range gids { + if _, ok := n.mGroupIDs[c]; !ok { + n.mGroupIDs[c] = struct{}{} + n.GroupIDs = append(n.GroupIDs, c) + } + } } diff --git a/graph/node_test.go b/graph/node_test.go new file mode 100644 index 0000000..5e4fc4c --- /dev/null +++ b/graph/node_test.go @@ -0,0 +1,22 @@ +package graph_test + +import ( + "testing" + + "github.com/cycloidio/inframap/graph" + "github.com/stretchr/testify/assert" +) + +func TestAddGroupIDs(t *testing.T) { + t.Run("Success", func(t *testing.T) { + n := graph.Node{} + n.AddGroupIDs("a") + assert.Equal(t, []string{"a"}, n.GroupIDs) + n.AddGroupIDs("b") + assert.Equal(t, []string{"a", "b"}, n.GroupIDs) + n.AddGroupIDs("b") + assert.Equal(t, []string{"a", "b"}, n.GroupIDs) + n.AddGroupIDs("a", "b", "c") + assert.Equal(t, []string{"a", "b", "c"}, n.GroupIDs) + }) +} From 48eb0f91050e615e8d1dcb31940296d68bf99674 Mon Sep 17 00:00:00 2001 From: xescugc Date: Mon, 3 Aug 2020 09:08:37 +0200 Subject: [PATCH 3/6] generate: Add the logic to get and set GroupIDs to Nodes This way we have the info on the Node as we want --- generate/helper_test.go | 6 +- generate/state.go | 9 + generate/state_test.go | 59 ++++++ generate/testdata/aws_state_vpc.json | 300 +++++++++++++++++++++++++++ 4 files changed, 369 insertions(+), 5 deletions(-) create mode 100644 generate/testdata/aws_state_vpc.json diff --git a/generate/helper_test.go b/generate/helper_test.go index 938467b..9ef8544 100644 --- a/generate/helper_test.go +++ b/generate/helper_test.go @@ -50,11 +50,7 @@ func assertEqualGraph(t *testing.T, expected, actual *graph.Graph, actualCfg map for _, en := range expected.Nodes { if an, ok := nodeCans[en.Canonical]; ok { - en.ID = an.ID - en.TFID = an.TFID - en.Resource = an.Resource - en.Weight = an.Weight - assert.Equal(t, en, an) + assert.Equal(t, en.GroupIDs, an.GroupIDs) } else { assert.Failf(t, "Fail", "The Node with Canonical %q is missing", en.Canonical) } diff --git a/generate/state.go b/generate/state.go index f7ee8c5..4c44862 100644 --- a/generate/state.go +++ b/generate/state.go @@ -120,6 +120,15 @@ func FromState(tfstate json.RawMessage, opt Options) (*graph.Graph, map[string]i } } + // Now that we have the full config we can set the Groups + for _, n := range g.Nodes { + pv, _, err := getProviderAndResource(n.Canonical, opt) + if err != nil { + return nil, nil, err + } + n.AddGroupIDs(pv.Groups(n.ID, cfg)...) + } + for sourceID, edges := range nodeIDEdges { edgeIDs := make([]string, 0) for _, e := range edges { diff --git a/generate/state_test.go b/generate/state_test.go index 5f9a7d2..184f984 100644 --- a/generate/state_test.go +++ b/generate/state_test.go @@ -147,6 +147,65 @@ func TestFromState_AWS(t *testing.T) { require.NotNil(t, cfg) assert.Len(t, g.Nodes, 2) }) + t.Run("SuccessVPC", func(t *testing.T) { + src, err := ioutil.ReadFile("./testdata/aws_state_vpc.json") + require.NoError(t, err) + + g, cfg, err := generate.FromState(src, generate.Options{Clean: true, Connections: true}) + require.NoError(t, err) + require.NotNil(t, g) + require.NotNil(t, cfg) + + eg := &graph.Graph{ + Nodes: []*graph.Node{ + &graph.Node{ + Canonical: "aws_elb.front", + GroupIDs: []string{"vpc-0d96ad69"}, + }, + &graph.Node{ + Canonical: "aws_instance.front", + GroupIDs: []string{"vpc-0d96ad69"}, + }, + &graph.Node{ + Canonical: "aws_db_instance.magento", + GroupIDs: []string{"vpc-0d96ad69"}, + }, + &graph.Node{ + Canonical: "aws_elasticache_cluster.redis", + GroupIDs: []string{"vpc-0d96ad69"}, + }, + }, + Edges: []*graph.Edge{ + &graph.Edge{ + Source: "aws_elb.front", + Target: "aws_instance.front", + Canonicals: []string{ + "aws_security_group.elb-front", + "aws_security_group.front", + "aws_security_group_rule.elb_to_front_http", + }, + }, + &graph.Edge{ + Source: "aws_instance.front", + Target: "aws_db_instance.magento", + Canonicals: []string{ + "aws_security_group.front", + "aws_security_group.rds", + }, + }, + &graph.Edge{ + Source: "aws_instance.front", + Target: "aws_elasticache_cluster.redis", + Canonicals: []string{ + "aws_security_group.front", + "aws_security_group.redis", + }, + }, + }, + } + + assertEqualGraph(t, eg, g, cfg) + }) } func TestFromState_OpenStack(t *testing.T) { diff --git a/generate/testdata/aws_state_vpc.json b/generate/testdata/aws_state_vpc.json new file mode 100644 index 0000000..2903637 --- /dev/null +++ b/generate/testdata/aws_state_vpc.json @@ -0,0 +1,300 @@ +{ + "version": 4, + "terraform_version": "0.12.28", + "serial": 1, + "lineage": "7ad13269-c000-d247-1b1c-3075a222f1df", + "resources": [ + { + "module": "module.magento", + "mode": "managed", + "type": "aws_db_instance", + "name": "magento", + "provider": "provider.aws", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "sample-magento-rds-prod", + "vpc_security_group_ids": [ + "sg-057cbd30abf1dd923" + ] + }, + "depends_on": [ + "aws_db_subnet_group.rds-subnet[0]", + "aws_security_group.rds" + ] + } + ] + }, + { + "module": "module.magento", + "mode": "managed", + "type": "aws_elasticache_cluster", + "name": "redis", + "provider": "provider.aws", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "cy323i4p3rf0szblrtmu" + }, + "depends_on": [ + "aws_elasticache_subnet_group.cache-subnet[0]", + "aws_security_group.redis" + ] + } + ] + }, + { + "module": "module.magento", + "mode": "managed", + "type": "aws_elb", + "name": "front", + "provider": "provider.aws", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "sample-magento-front-prod", + "source_security_group_id": "sg-02831399bd1302ed1" + }, + "depends_on": [ + "aws_instance.front", + "aws_security_group.elb-front" + ] + } + ] + }, + { + "module": "module.magento", + "mode": "managed", + "type": "aws_instance", + "name": "front", + "each": "list", + "provider": "provider.aws", + "instances": [ + { + "index_key": 0, + "schema_version": 1, + "attributes": { + "id": "i-08ffccfdf54168280", + "vpc_security_group_ids": [ + "sg-0d6e0e07fbc6ec626", + "sg-ddeee6bb" + ] + }, + "depends_on": [ + "aws_iam_instance_profile.front_profile", + "aws_security_group.front", + "data.aws_ami.debian" + ] + } + ] + }, + { + "module": "module.magento", + "mode": "managed", + "type": "aws_security_group", + "name": "elb-front", + "provider": "provider.aws", + "instances": [ + { + "schema_version": 1, + "attributes": { + "egress": [ + { + "cidr_blocks": [ + "0.0.0.0/0" + ], + "description": "", + "from_port": 0, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "-1", + "security_groups": [], + "self": false, + "to_port": 0 + } + ], + "id": "sg-02831399bd1302ed1", + "ingress": [ + { + "cidr_blocks": [ + "0.0.0.0/0" + ], + "description": "", + "from_port": 443, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [], + "self": false, + "to_port": 443 + }, + { + "cidr_blocks": [ + "0.0.0.0/0" + ], + "description": "", + "from_port": 80, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [], + "self": false, + "to_port": 80 + } + ], + "vpc_id": "vpc-0d96ad69" + } + } + ] + }, + { + "module": "module.magento", + "mode": "managed", + "type": "aws_security_group", + "name": "front", + "provider": "provider.aws", + "instances": [ + { + "schema_version": 1, + "attributes": { + "egress": [ + { + "cidr_blocks": [ + "0.0.0.0/0" + ], + "description": "", + "from_port": 0, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "-1", + "security_groups": [], + "self": false, + "to_port": 0 + } + ], + "id": "sg-0d6e0e07fbc6ec626", + "ingress": [], + "vpc_id": "vpc-0d96ad69" + } + } + ] + }, + { + "module": "module.magento", + "mode": "managed", + "type": "aws_security_group", + "name": "rds", + "provider": "provider.aws", + "instances": [ + { + "schema_version": 1, + "attributes": { + "egress": [], + "id": "sg-057cbd30abf1dd923", + "ingress": [ + { + "cidr_blocks": [], + "description": "", + "from_port": 3306, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [ + "sg-0d6e0e07fbc6ec626" + ], + "self": false, + "to_port": 3306 + } + ], + "vpc_id": "vpc-0d96ad69" + }, + "depends_on": [ + "aws_security_group.front" + ] + } + ] + }, + { + "module": "module.magento", + "mode": "managed", + "type": "aws_security_group", + "name": "redis", + "provider": "provider.aws", + "instances": [ + { + "schema_version": 1, + "attributes": { + "egress": [], + "id": "sg-078ba1d75bddac084", + "ingress": [ + { + "cidr_blocks": [], + "description": "", + "from_port": 6379, + "ipv6_cidr_blocks": [], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [ + "sg-0d6e0e07fbc6ec626" + ], + "self": false, + "to_port": 6379 + } + ], + "vpc_id": "vpc-0d96ad69" + }, + "depends_on": [ + "aws_security_group.front" + ] + } + ] + }, + { + "module": "module.magento", + "mode": "managed", + "type": "aws_security_group_rule", + "name": "bastion_to_front_ssh", + "each": "list", + "provider": "provider.aws", + "instances": [ + { + "index_key": 0, + "schema_version": 2, + "attributes": { + "id": "sgrule-252974847", + "security_group_id": "sg-0d6e0e07fbc6ec626", + "source_security_group_id": "sg-ddeee6bb" + }, + "depends_on": [ + "aws_security_group.front" + ] + } + ] + }, + { + "module": "module.magento", + "mode": "managed", + "type": "aws_security_group_rule", + "name": "elb_to_front_http", + "provider": "provider.aws", + "instances": [ + { + "schema_version": 2, + "attributes": { + "id": "sgrule-2124073583", + "security_group_id": "sg-0d6e0e07fbc6ec626", + "source_security_group_id": "sg-02831399bd1302ed1" + }, + "depends_on": [ + "aws_security_group.elb-front", + "aws_security_group.front" + ] + } + ] + } + ] +} + From 263bd574651a7ef927ab179097c39b802342b5a1 Mon Sep 17 00:00:00 2001 From: xescugc Date: Mon, 3 Aug 2020 09:11:12 +0200 Subject: [PATCH 4/6] fixup! generate: Add the logic to get and set GroupIDs to Nodes --- generate/hcl.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/generate/hcl.go b/generate/hcl.go index bc7f6c8..80628f7 100644 --- a/generate/hcl.go +++ b/generate/hcl.go @@ -117,6 +117,15 @@ func FromHCL(fs afero.Fs, path string, opt Options) (*graph.Graph, error) { } } + // Now that we have the full config we can set the Groups + for _, n := range g.Nodes { + pv, _, err := getProviderAndResource(n.Canonical, opt) + if err != nil { + return nil, err + } + n.AddGroupIDs(pv.Groups(n.ID, resourcesRawConfig)...) + } + for nid, resources := range nodeIDEdges { for _, rkattr := range resources { keys := strings.Split(rkattr, ".") From 55009b1a243ae45bbaf6aba218145469cbf1d777 Mon Sep 17 00:00:00 2001 From: xescugc Date: Mon, 3 Aug 2020 09:11:50 +0200 Subject: [PATCH 5/6] printer: Add the logic to print GropuIDs with DOT We use the 'cluster' logic to generate a subgraph, it has it's limitations but it works for now --- printer/dot/printer.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/printer/dot/printer.go b/printer/dot/printer.go index 993d6c8..f723f7a 100644 --- a/printer/dot/printer.go +++ b/printer/dot/printer.go @@ -87,6 +87,14 @@ func (d Dot) Print(g *graph.Graph, opt printer.Options, w io.Writer) error { } } + for _, g := range n.GroupIDs { + clusterName := fmt.Sprintf("%q", fmt.Sprintf("cluster_%s", g)) + graph.AddSubGraph(parentName, clusterName, map[string]string{ + "label": clusterName, + }) + graph.AddNode(clusterName, fmt.Sprintf("%q", n.Canonical), attr) + } + graph.AddNode(parentName, fmt.Sprintf("%q", n.Canonical), attr) } From eaedd66c8a8d62471e9a0996077501a5c5f1d163 Mon Sep 17 00:00:00 2001 From: xescugc Date: Mon, 3 Aug 2020 09:13:16 +0200 Subject: [PATCH 6/6] CHANGELOG: Add Groups logic for VPCs on AWS --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2431414..24bbe56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ ## [Unreleased] +### Added + +- Add Groups logic for VPCs on AWS + ([Issue #6](https://github.com/cycloidio/inframap/issues/6)) + ### Fixed - Google graph generation from HCL