-
Notifications
You must be signed in to change notification settings - Fork 52
/
agent_nodes.tf
363 lines (311 loc) · 18 KB
/
agent_nodes.tf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
locals {
// Generate a map of all agents annotations in order to manage them through this module. This
// generation is made in two steps:
// - generate a list of objects representing all annotations, following this
// 'template' {key = node_name|annotation_name, value = annotation_value}
// - generate a map based on the generated list (using the field key as map key)
agent_annotations_list = flatten([
for nk, nv in var.agents : [
// Because we need node name and annotation name when we remove the annotation resource, we need
// to share them through the annotation key (each.value are not avaible on destruction).
for ak, av in try(nv.annotations, {}) : av == null ? { key : "" } : { key : "${nk}${var.separator}${ak}", value : av }
]
])
agent_annotations = local.managed_annotation_enabled ? { for o in local.agent_annotations_list : o.key => o.value if o.key != "" } : {}
// Generate a map of all agents labels in order to manage them through this module. This
// generation is made in two steps, following the same process than annotation's map.
agent_labels_list = flatten([
for nk, nv in var.agents : [
// Because we need node name and label name when we remove the label resource, we need
// to share them through the label key (each.value are not avaible on destruction).
for lk, lv in try(nv.labels, {}) : lv == null ? { key : "" } : { key : "${nk}${var.separator}${lk}", value : lv }
]
])
agent_labels = local.managed_label_enabled ? { for o in local.agent_labels_list : o.key => o.value if o.key != "" } : {}
// Generate a map of all agents taints in order to manage them through this module. This
// generation is made in two steps, following the same process than annotation's map.
agent_taints_list = flatten([
for nk, nv in var.agents : [
// Because we need node name and taint name when we remove the taint resource, we need
// to share them through the taint key (each.value are not avaible on destruction).
for tk, tv in try(nv.taints, {}) : tv == null ? { key : "" } : { key : "${nk}${var.separator}${tk}", value : tv }
]
])
agent_taints = local.managed_taint_enabled ? { for o in local.agent_taints_list : o.key => o.value if o.key != "" } : {}
// Generate a map of all calculated agent fields, used during k3s installation.
agents_metadata = {
for key, agent in var.agents :
key => {
name = try(agent.name, key)
ip = agent.ip
flags = join(" ", compact(concat(
[
"--node-ip ${agent.ip}",
"--node-name '${try(agent.name, key)}'",
"--server https://${local.root_advertise_ip_k3s}:6443",
"--token ${nonsensitive(random_password.k3s_cluster_secret.result)}", # NOTE: nonsensitive is used to show logs during provisioning
],
var.global_flags,
try(agent.flags, []),
[for key, value in try(agent.taints, {}) : "--node-taint '${key}=${value}'" if value != null]
)))
immutable_fields_hash = sha1(join("", concat(
[var.cluster_domain],
var.global_flags,
try(agent.flags, []),
)))
}
}
kubectl_cmd = var.use_sudo ? "sudo kubectl" : "kubectl"
}
// Install k3s agent
resource "null_resource" "agents_install" {
for_each = var.agents
depends_on = [null_resource.servers_install]
triggers = {
on_immutable_changes = local.agents_metadata[each.key].immutable_fields_hash
on_new_version = local.k3s_version
}
connection {
type = try(each.value.connection.type, "ssh")
host = try(each.value.connection.host, each.value.ip)
user = try(each.value.connection.user, null)
password = try(each.value.connection.password, null)
port = try(each.value.connection.port, null)
timeout = try(each.value.connection.timeout, null)
script_path = try(each.value.connection.script_path, null)
private_key = try(each.value.connection.private_key, null)
certificate = try(each.value.connection.certificate, null)
agent = try(each.value.connection.agent, null)
agent_identity = try(each.value.connection.agent_identity, null)
host_key = try(each.value.connection.host_key, null)
https = try(each.value.connection.https, null)
insecure = try(each.value.connection.insecure, null)
use_ntlm = try(each.value.connection.use_ntlm, null)
cacert = try(each.value.connection.cacert, null)
bastion_host = try(each.value.connection.bastion_host, null)
bastion_host_key = try(each.value.connection.bastion_host_key, null)
bastion_port = try(each.value.connection.bastion_port, null)
bastion_user = try(each.value.connection.bastion_user, null)
bastion_password = try(each.value.connection.bastion_password, null)
bastion_private_key = try(each.value.connection.bastion_private_key, null)
bastion_certificate = try(each.value.connection.bastion_certificate, null)
}
// Upload k3s install script
provisioner "file" {
content = data.http.k3s_installer.response_body
destination = "/tmp/k3s-installer"
}
// Install k3s
provisioner "remote-exec" {
inline = [
"${local.install_env_vars} INSTALL_K3S_VERSION=${local.k3s_version} sh /tmp/k3s-installer agent ${local.agents_metadata[each.key].flags}",
"until systemctl is-active --quiet k3s-agent.service; do sleep 1; done"
]
}
}
// Drain k3s node on destruction in order to safely move all workflows to another node.
resource "null_resource" "agents_drain" {
for_each = var.agents
depends_on = [null_resource.agents_install]
triggers = {
// Because some fields must be used on destruction, we need to store them into the current
// object. The only way to do that is to use triggers to store theses fields.
agent_name = local.agents_metadata[split(var.separator, each.key)[0]].name
connection_json = base64encode(jsonencode(local.root_server_connection))
drain_timeout = var.drain_timeout
kubectl_cmd = local.kubectl_cmd
}
// Because we use triggers as memory area, we need to ignore all changes on it.
lifecycle { ignore_changes = [triggers] }
connection {
type = jsondecode(base64decode(self.triggers.connection_json)).type
host = jsondecode(base64decode(self.triggers.connection_json)).host
user = jsondecode(base64decode(self.triggers.connection_json)).user
password = jsondecode(base64decode(self.triggers.connection_json)).password
port = jsondecode(base64decode(self.triggers.connection_json)).port
timeout = jsondecode(base64decode(self.triggers.connection_json)).timeout
script_path = jsondecode(base64decode(self.triggers.connection_json)).script_path
private_key = jsondecode(base64decode(self.triggers.connection_json)).private_key
certificate = jsondecode(base64decode(self.triggers.connection_json)).certificate
agent = jsondecode(base64decode(self.triggers.connection_json)).agent
agent_identity = jsondecode(base64decode(self.triggers.connection_json)).agent_identity
host_key = jsondecode(base64decode(self.triggers.connection_json)).host_key
https = jsondecode(base64decode(self.triggers.connection_json)).https
insecure = jsondecode(base64decode(self.triggers.connection_json)).insecure
use_ntlm = jsondecode(base64decode(self.triggers.connection_json)).use_ntlm
cacert = jsondecode(base64decode(self.triggers.connection_json)).cacert
bastion_host = jsondecode(base64decode(self.triggers.connection_json)).bastion_host
bastion_host_key = jsondecode(base64decode(self.triggers.connection_json)).bastion_host_key
bastion_port = jsondecode(base64decode(self.triggers.connection_json)).bastion_port
bastion_user = jsondecode(base64decode(self.triggers.connection_json)).bastion_user
bastion_password = jsondecode(base64decode(self.triggers.connection_json)).bastion_password
bastion_private_key = jsondecode(base64decode(self.triggers.connection_json)).bastion_private_key
bastion_certificate = jsondecode(base64decode(self.triggers.connection_json)).bastion_certificate
}
provisioner "remote-exec" {
when = destroy
inline = [
"${self.triggers.kubectl_cmd} drain ${self.triggers.agent_name} --delete-local-data --force --ignore-daemonsets --timeout=${self.triggers.drain_timeout}"
]
}
}
// Add/remove manually annotation on k3s agent
resource "null_resource" "agents_annotation" {
for_each = local.agent_annotations
depends_on = [null_resource.agents_install]
triggers = {
agent_name = local.agents_metadata[split(var.separator, each.key)[0]].name
annotation_name = split(var.separator, each.key)[1]
on_value_changes = each.value
// Because some fields must be used on destruction, we need to store them into the current
// object. The only way to do that is to use triggers to store theses fields.
connection_json = base64encode(jsonencode(local.root_server_connection))
kubectl_cmd = local.kubectl_cmd
}
// Because we dont care about connection modification, we ignore its changes.
lifecycle { ignore_changes = [triggers["connection_json"], triggers["kubectl_cmd"]] }
connection {
type = jsondecode(base64decode(self.triggers.connection_json)).type
host = jsondecode(base64decode(self.triggers.connection_json)).host
user = jsondecode(base64decode(self.triggers.connection_json)).user
password = jsondecode(base64decode(self.triggers.connection_json)).password
port = jsondecode(base64decode(self.triggers.connection_json)).port
timeout = jsondecode(base64decode(self.triggers.connection_json)).timeout
script_path = jsondecode(base64decode(self.triggers.connection_json)).script_path
private_key = jsondecode(base64decode(self.triggers.connection_json)).private_key
certificate = jsondecode(base64decode(self.triggers.connection_json)).certificate
agent = jsondecode(base64decode(self.triggers.connection_json)).agent
agent_identity = jsondecode(base64decode(self.triggers.connection_json)).agent_identity
host_key = jsondecode(base64decode(self.triggers.connection_json)).host_key
https = jsondecode(base64decode(self.triggers.connection_json)).https
insecure = jsondecode(base64decode(self.triggers.connection_json)).insecure
use_ntlm = jsondecode(base64decode(self.triggers.connection_json)).use_ntlm
cacert = jsondecode(base64decode(self.triggers.connection_json)).cacert
bastion_host = jsondecode(base64decode(self.triggers.connection_json)).bastion_host
bastion_host_key = jsondecode(base64decode(self.triggers.connection_json)).bastion_host_key
bastion_port = jsondecode(base64decode(self.triggers.connection_json)).bastion_port
bastion_user = jsondecode(base64decode(self.triggers.connection_json)).bastion_user
bastion_password = jsondecode(base64decode(self.triggers.connection_json)).bastion_password
bastion_private_key = jsondecode(base64decode(self.triggers.connection_json)).bastion_private_key
bastion_certificate = jsondecode(base64decode(self.triggers.connection_json)).bastion_certificate
}
provisioner "remote-exec" {
inline = [
"until kubectl get node ${self.triggers.agent_name}; do sleep 1; done",
"${self.triggers.kubectl_cmd} annotate --overwrite node ${self.triggers.agent_name} ${self.triggers.annotation_name}=${self.triggers.on_value_changes}"
]
}
provisioner "remote-exec" {
when = destroy
inline = [
"${self.triggers.kubectl_cmd} annotate node ${self.triggers.agent_name} ${self.triggers.annotation_name}-"
]
}
}
// Add/remove manually label on k3s agent
resource "null_resource" "agents_label" {
for_each = local.agent_labels
depends_on = [null_resource.agents_install]
triggers = {
agent_name = local.agents_metadata[split(var.separator, each.key)[0]].name
label_name = split(var.separator, each.key)[1]
on_value_changes = each.value
// Because some fields must be used on destruction, we need to store them into the current
// object. The only way to do that is to use triggers to store theses fields.
connection_json = base64encode(jsonencode(local.root_server_connection))
kubectl_cmd = local.kubectl_cmd
}
// Because we dont care about connection modification, we ignore its changes.
lifecycle { ignore_changes = [triggers["connection_json"], triggers["kubectl_cmd"]] }
connection {
type = jsondecode(base64decode(self.triggers.connection_json)).type
host = jsondecode(base64decode(self.triggers.connection_json)).host
user = jsondecode(base64decode(self.triggers.connection_json)).user
password = jsondecode(base64decode(self.triggers.connection_json)).password
port = jsondecode(base64decode(self.triggers.connection_json)).port
timeout = jsondecode(base64decode(self.triggers.connection_json)).timeout
script_path = jsondecode(base64decode(self.triggers.connection_json)).script_path
private_key = jsondecode(base64decode(self.triggers.connection_json)).private_key
certificate = jsondecode(base64decode(self.triggers.connection_json)).certificate
agent = jsondecode(base64decode(self.triggers.connection_json)).agent
agent_identity = jsondecode(base64decode(self.triggers.connection_json)).agent_identity
host_key = jsondecode(base64decode(self.triggers.connection_json)).host_key
https = jsondecode(base64decode(self.triggers.connection_json)).https
insecure = jsondecode(base64decode(self.triggers.connection_json)).insecure
use_ntlm = jsondecode(base64decode(self.triggers.connection_json)).use_ntlm
cacert = jsondecode(base64decode(self.triggers.connection_json)).cacert
bastion_host = jsondecode(base64decode(self.triggers.connection_json)).bastion_host
bastion_host_key = jsondecode(base64decode(self.triggers.connection_json)).bastion_host_key
bastion_port = jsondecode(base64decode(self.triggers.connection_json)).bastion_port
bastion_user = jsondecode(base64decode(self.triggers.connection_json)).bastion_user
bastion_password = jsondecode(base64decode(self.triggers.connection_json)).bastion_password
bastion_private_key = jsondecode(base64decode(self.triggers.connection_json)).bastion_private_key
bastion_certificate = jsondecode(base64decode(self.triggers.connection_json)).bastion_certificate
}
provisioner "remote-exec" {
inline = [
"until ${self.triggers.kubectl_cmd} get node ${self.triggers.agent_name}; do sleep 1; done",
"${self.triggers.kubectl_cmd} label --overwrite node ${self.triggers.agent_name} ${self.triggers.label_name}=${self.triggers.on_value_changes}"
]
}
provisioner "remote-exec" {
when = destroy
inline = [
"${self.triggers.kubectl_cmd} label node ${self.triggers.agent_name} ${self.triggers.label_name}-"
]
}
}
// Add manually taint on k3s agent
resource "null_resource" "agents_taint" {
for_each = local.agent_taints
depends_on = [null_resource.agents_install]
triggers = {
agent_name = local.agents_metadata[split(var.separator, each.key)[0]].name
taint_name = split(var.separator, each.key)[1]
on_value_changes = each.value
// Because some fields must be used on destruction, we need to store them into the current
// object. The only way to do that is to use triggers to store theses fields.
connection_json = base64encode(jsonencode(local.root_server_connection))
kubectl_cmd = local.kubectl_cmd
}
// Because we dont care about connection modification, we ignore its changes.
lifecycle { ignore_changes = [triggers["connection_json"], triggers["kubectl_cmd"]] }
connection {
type = jsondecode(base64decode(self.triggers.connection_json)).type
host = jsondecode(base64decode(self.triggers.connection_json)).host
user = jsondecode(base64decode(self.triggers.connection_json)).user
password = jsondecode(base64decode(self.triggers.connection_json)).password
port = jsondecode(base64decode(self.triggers.connection_json)).port
timeout = jsondecode(base64decode(self.triggers.connection_json)).timeout
script_path = jsondecode(base64decode(self.triggers.connection_json)).script_path
private_key = jsondecode(base64decode(self.triggers.connection_json)).private_key
certificate = jsondecode(base64decode(self.triggers.connection_json)).certificate
agent = jsondecode(base64decode(self.triggers.connection_json)).agent
agent_identity = jsondecode(base64decode(self.triggers.connection_json)).agent_identity
host_key = jsondecode(base64decode(self.triggers.connection_json)).host_key
https = jsondecode(base64decode(self.triggers.connection_json)).https
insecure = jsondecode(base64decode(self.triggers.connection_json)).insecure
use_ntlm = jsondecode(base64decode(self.triggers.connection_json)).use_ntlm
cacert = jsondecode(base64decode(self.triggers.connection_json)).cacert
bastion_host = jsondecode(base64decode(self.triggers.connection_json)).bastion_host
bastion_host_key = jsondecode(base64decode(self.triggers.connection_json)).bastion_host_key
bastion_port = jsondecode(base64decode(self.triggers.connection_json)).bastion_port
bastion_user = jsondecode(base64decode(self.triggers.connection_json)).bastion_user
bastion_password = jsondecode(base64decode(self.triggers.connection_json)).bastion_password
bastion_private_key = jsondecode(base64decode(self.triggers.connection_json)).bastion_private_key
bastion_certificate = jsondecode(base64decode(self.triggers.connection_json)).bastion_certificate
}
provisioner "remote-exec" {
inline = [
"until ${self.triggers.kubectl_cmd} get node ${self.triggers.agent_name}; do sleep 1; done",
"${self.triggers.kubectl_cmd} taint node ${self.triggers.agent_name} ${self.triggers.taint_name}=${self.triggers.on_value_changes} --overwrite"
]
}
provisioner "remote-exec" {
when = destroy
inline = [
"${self.triggers.kubectl_cmd} taint node ${self.triggers.agent_name} ${self.triggers.taint_name}-"
]
}
}