-
Notifications
You must be signed in to change notification settings - Fork 8
/
cloudformation.yml
2155 lines (2141 loc) · 77.4 KB
/
cloudformation.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
AWSTemplateFormatVersion: "2010-09-09"
Metadata:
AWS::CloudFormation::Interface:
ParameterGroups:
- Label: { default: "Neo4J License" }
Parameters: [AcceptLicense]
- Label: { default: "Global configurations" }
Parameters:
- VpcId
- KeyName
- ECSAMI
- NodeSecurityGroups
- Neo4jAccessFromIP
- SNSTopicArn
- Label: { default: "Core Nodes Configuration" }
Parameters:
- ClusterInstanceType
- SubnetID
- DesiredCapacity
- EBSSize
- EBSIOPS
- EBSType
- Label: { default: "Read replicas configuration" }
Parameters:
- ReplicasInstanceType
- ReplicasCount
- ReplicasSubnetID
- Label: { default: "Docker image configurations" }
Parameters: [DockerImage, DockerECRARN]
- Label: { default: "Neo4j users configuration" }
Parameters:
[AdminUser, AdminPassword, ReadOnlyUser, ReadOnlyUserPassword]
- Label: { default: "Cloud Map discovery settings" }
Parameters:
- CloudMapNamespaceID
- CloudMapNamespaceName
- Neo4jCoreSubdomain
- Neo4jReplicasSubdomain
- Label: { default: "Neo4J operations" }
Parameters:
- AllowUpgrade
- BackupBatchQueueARN
- BackupBatchInstanceType
- BackupBatchMaxVCPUs
- BackupPath
- UpgradeMode
- BackupBatchDiskSize
- BackupBatchDiskType
- BackupBatchDiskIOPS
- BackupHourlyStoreForDays
- BackupDailyStoreForDays
- BackupBatchJobTimeout
- IsDrainSupported
- SlowQueryLog
- CaptureDebugLogs
ParameterLabels:
AcceptLicense: { default: "Do you agree?" }
IsDrainSupported:
default: "Does your AWS account has system for automatic ECS instance draining deployed?"
Parameters:
KeyName:
Type: AWS::EC2::KeyPair::KeyName
Description: Name of an existing EC2 KeyPair to enable SSH access to the ECS instances.
ECSAMI:
Description: AMI ID
Type: AWS::SSM::Parameter::Value<AWS::EC2::Image::Id>
Default: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id
VpcId:
Type: AWS::EC2::VPC::Id
Description: Select a VPC that allows instances access to the Internet.
SubnetID:
Type: List<AWS::EC2::Subnet::Id>
Description: "!!! Number of selected subnets must match the DesiredCapacity !!!"
# RexRay plugin can't work with many volumes with the same name in the same AZ at the same time
# see issue: https://github.com/rexray/rexray/issues/1314
ReplicasSubnetID:
Type: List<AWS::EC2::Subnet::Id>
Description: |
1+ number of subnets to place the Replicas in.
Even if no replicas used, some subnet must be provided here
otherwise template validation will fail ¯\_(ツ)_/¯
ReplicasCount:
Type: Number
Default: 0
Description: "If set to 0, all other replica related settings can be ignored"
NodeSecurityGroups:
Type: List<AWS::EC2::SecurityGroup::Id>
Description: Select at least one additional security group to apply to Neo4j nodes (e.g. your SSH group)
DesiredCapacity:
Type: Number
Default: "3"
AllowedValues: ["3", "5", "7"]
Description: Number of instances to launch in your ECS cluster.
EBSSize:
Description: Size in GiBs of data volumes to be created for Neo4j nodes.
Type: Number
Default: "10"
EBSType:
Description: Type of data volumes to be created for Neo4j nodes (gp3 is recommended)
Type: String
Default: gp3
AllowedValues: [standard, gp3, gp2, io1, io2, sc1, st1]
EBSIOPS:
Description: Custom IOPS size for a disk, applicable only for gp3, io1 and io2 disks (omitted if 0)
Type: Number
Default: 0
DockerImage:
Description: Docker image of Neo4j enterprise, e.g. 111111111111.dkr.ecr.us-east-1.amazonaws.com/neo:c531de3a6655b8c885330ca91b867431760392bf
Type: String
DockerECRARN:
Description: ARN of Docker ECR repository for images, e.g. arn:aws:ecr:us-east-1:111111111111:repository/neo
Type: String
AdminUser:
Description: Username for Neo4j database admin access.
Type: String
AllowedValues: [neo4j]
AdminPassword:
NoEcho: true
Description: Password for Neo4j database admin access.
Type: String
ReadOnlyUser:
Description: Username for Neo4j database read-only access.
Type: String
ReadOnlyUserPassword:
NoEcho: true
Description: Password for Neo4j database read-only access.
Type: String
CloudMapNamespaceID:
Type: String
Description: |
If you want to use existing CloudMap Namespace, please provide the ID here.
Leave empty to create new namespace.
CloudMapNamespaceName:
Type: String
Description: |
If "CloudMapNamespaceID" were not provided, then new CloudMap will be created.
The name of the namespace is actually the Route53 private domain that will be used to create dns entry for Neo4J
Default: neo4j.testing
Neo4jCoreSubdomain:
Type: String
Default: core
Description: |
The sudbdomain that well be used for the neo4j core cluster. Will look like
<subdomain>.<namespace>. For example: core.neo4j.testing
Neo4jReplicasSubdomain:
Type: String
Default: replica
Description: |
The sudbdomain that well be used for the neo4j replicas. Will look like
<subdomain>.<namespace>. For example: replica.neo4j.testing
BackupPath:
Description: |
(FOR RESTORE ONLY) Start cluster from a database backup snapshot (zip). Full S3 path `<bucket_name>/path/to/backup.zip`
IMPORTANT: Rolling backup restore is not possible. You need to stop the cluster and then do the import.
See https://neo4j.com/docs/operations-manual/current/backup/restoring/#backup-restoring-causal-cluster
Type: String
ClusterInstanceType:
Description: EC2 instance type for Neo4J cluster
Type: String
Default: t2.medium
AllowedValues: [
spot.medium,
t2.medium,
t3.medium,
t3a.medium,
m3.medium, # 4gb
spot.large,
spot.r.large,
t2.large,
t3.large,
t3a.large,
m3.large,
m4.large,
m5.large,
m5a.large, # 8gb
spot.xlarge,
t2.xlarge,
t3.xlarge,
t3a.xlarge,
m3.xlarge,
m4.xlarge,
m5.xlarge,
m5a.xlarge, # 16gb
spot.2xlarge,
t2.2xlarge,
t3.2xlarge,
t3a.2xlarge,
m3.2xlarge,
m4.2xlarge,
m5.2xlarge,
m5a.2xlarge, # 32gb
] # NOTE: Feel free to extend the list of instances but define Memory maps for them.
# m4.4xlarge, m5.4xlarge, m5a.4xlarge, # 64gb
# m4.10xlarge, m5.10xlarge, m5a.10xlarge, # 160gb
# m4.16xlarge, m5.16xlarge, m5a.16xlarge, # 256gb
ConstraintDescription: Please choose a valid instance type.
ReplicasInstanceType:
Description: EC2 instance type for Neo4J Replicas node
Type: String
Default: t2.medium
AllowedValues: [
spot.medium,
t2.medium,
t3.medium,
t3a.medium,
m3.medium, # 4gb
spot.large,
spot.r.large,
t2.large,
t3.large,
t3a.large,
m3.large,
m4.large,
m5.large,
m5a.large, # 8gb
spot.xlarge,
t2.xlarge,
t3.xlarge,
t3a.xlarge,
m3.xlarge,
m4.xlarge,
m5.xlarge,
m5a.xlarge, # 16gb
spot.2xlarge,
t2.2xlarge,
t3.2xlarge,
t3a.2xlarge,
m3.2xlarge,
m4.2xlarge,
m5.2xlarge,
m5a.2xlarge, # 32gb
] # NOTE: Feel free to extend the list of instances but define Memory maps for them.
# m4.4xlarge, m5.4xlarge, m5a.4xlarge, # 64gb
# m4.10xlarge, m5.10xlarge, m5a.10xlarge, # 160gb
# m4.16xlarge, m5.16xlarge, m5a.16xlarge, # 256gb
ConstraintDescription: Please choose a valid instance type.
SNSTopicArn:
Description: SNS topic to send CloudWatch Alarms notifications (if you leave it empty new topic will be created)
Type: String
UpgradeMode:
Type: String
Description: |
!!!!! Warning !!!!!!
This flag will break your cluster into independent nodes for a upgrade.
Use for upgrade ONLY. Should be used with the BackupPath
Default: "false"
AllowedValues: ["false", "true"]
AllowUpgrade:
Type: String
AllowedValues: ["true", "false"]
Default: "true"
Description: |
Neo4j parameter which allows automatic schema upgrades on version change.
This parameter is "true" if "UpgradeMode"
AcceptLicense:
Description: |
In order to use Neo4j Enterprise Edition you must accept the license agreement.
(c) Neo4j Sweden AB. 2019. All Rights Reserved.
Use of this Software without a proper commercial license with Neo4j,
Inc. or its affiliates is prohibited.
Email inquiries can be directed to: licensing@neo4j.com
More information is also available at: https://neo4j.com/licensing/
Type: String
Default: no
AllowedValues: [yes, no]
IsDrainSupported:
Type: String
Default: false
AllowedValues: [false, true]
Description: |
For graceful updates automatic drain of ECS Instances needed.
External tools needed for this, for example: https://github.com/getsocial-rnd/ecs-drain-lambda
If drain is not supported changes to AutoScaling Group will not be applied automatically.
SlowQueryLog:
Type: String
AllowedValues:
[disabled, 0ms, 100ms, 300ms, 500ms, 700ms, 1000ms, 1500ms, 2000ms]
Default: disabled
Description: Log all queries slower than ...
CaptureDebugLogs:
Type: String
AllowedValues: [true, false]
Default: true
Description: |
Capture and send to CloudWatch Logs the Neo4j debug logs.
This will generate quite a lot of data to CLoudWatch logs,
but it is quite impossible to debug without it.
BackupHourlyStoreForDays:
Type: Number
Default: 1
Description: Number of days to store hourly backups
BackupDailyStoreForDays:
Type: Number
Default: 14
Description: Number of days to store daily backups (backup made at midnight considered as daily)
BackupBatchQueueARN:
Description: |
(OPTIONAL)
Use existing Batch Jobs Queue to perform Neo4j cluster backups.
However, need to make sure, that it has all needed permissions
and can access backup ports of Neo4j Cluster nodes.
If not provided, separate AutoScaled Spot environment for Batch will be created
Type: String
BackupBatchInstanceType:
Description: |
EC2 instance type to use for Batch Environment for backups.
(This also used to define Backup Job container memory restrictions)
Type: String
Default: m3.medium
AllowedValues: [
m3.medium, # 4gb
m3.large,
m4.large,
m5.large,
m5a.large, # 8gb
m3.xlarge,
m4.xlarge,
m5.xlarge,
m5a.xlarge, # 16gb
m3.2xlarge,
m4.2xlarge,
m5.2xlarge,
m5a.2xlarge, # 32gb
r3.large, # r3.large on spot is actually cheaper then m4.large
] # NOTE: Feel free to extend the list of instances but define Memory maps for them.
ConstraintDescription: Please choose a valid instance type.
BackupBatchMaxVCPUs:
Type: Number
Default: 1
Description: |
Defines how many servers can be launched in parallel for batch jobs.
If backup takes longer than 1 hour you may want more then one server to meet the schedule.
For example, if you choose the m3.medium server for running backup jobs, which has 1 vCPU,
and set MaxVCPUs to 1, this mean that only one job can be run in parallel.
However, if you set MaxVCPUs to 3, up to 3 servers may be started if needed.
And if you choose the r3.large you have to set this value to 2 at least, otherwise, batch
server will never start.
BackupBatchDiskSize:
Description: |
Size of the disk to perform backups on. Take in mind that
Consistency Check may consume a lot of disk IOPS and the bigger
the disk the more performance it has (IOPS)
Type: Number
Default: 30
BackupBatchDiskType:
Description: Type of data volumes to be created for Neo4j backup nodes (gp3 is recommended)
Type: String
Default: gp3
AllowedValues: [standard, gp3, gp2, io1, io2, sc1, st1]
BackupBatchDiskIOPS:
Description: Custom IOPS size for a disk, applicable only for gp3, io1 and io2 disks (omitted if 0)
Type: Number
Default: 0
BackupBatchJobTimeout:
Description: |
Timeout to complete backup job
Type: Number
Default: 3600
Neo4jAccessFromIP:
Description: Open HTTP and Bolt interfaces for IP CIDR
Type: String
Mappings:
InstanceTypes:
spot.medium: # for testing
Memory: 4gb
vCPUs: 2
Instances:
- InstanceType: t3a.medium
- InstanceType: t3.medium
- InstanceType: t2.medium
spot.large: # for production
Memory: 8gb
vCPUs: 2
Instances:
- InstanceType: t3a.large
- InstanceType: t3.large
- InstanceType: m5a.large
- InstanceType: t2.large
- InstanceType: m5.large
- InstanceType: m4.large
- InstanceType: m3.large
spot.xlarge: # for production
Memory: 16gb
vCPUs: 4
Instances:
- InstanceType: t3a.xlarge
- InstanceType: t3.xlarge
- InstanceType: m5a.xlarge
- InstanceType: t2.xlarge
- InstanceType: m5.xlarge
- InstanceType: m4.xlarge
- InstanceType: m3.xlarge
# r5 is also possible to use here, but they have less cpu
# - InstanceType: r5.xlarge
# - InstanceType: r5a.xlarge
spot.r.large: # for production
Memory: 16gb
vCPUs: 2
Instances:
# there are also r5ad, but not completely sure
# it will work OK with the RexRay storage
- InstanceType: r5a.large
- InstanceType: r5.large
- InstanceType: r4.large
- InstanceType: r3.large
spot.2xlarge: # for production
Memory: 32gb
vCPUs: 8
Instances:
- InstanceType: t3a.2xlarge
- InstanceType: t3.2xlarge
- InstanceType: m5a.2xlarge
- InstanceType: t2.2xlarge
- InstanceType: m5.2xlarge
- InstanceType: m4.2xlarge
- InstanceType: m3.2xlarge
t3a.medium: { Instances: [], Memory: 4gb, vCPUs: 2 }
t3.medium: { Instances: [], Memory: 4gb, vCPUs: 2 }
t2.medium: { Instances: [], Memory: 4gb, vCPUs: 2 }
m3.medium: { Instances: [], Memory: 4gb, vCPUs: 1 }
t3a.large: { Instances: [], Memory: 8gb, vCPUs: 2 }
t3.large: { Instances: [], Memory: 8gb, vCPUs: 2 }
t2.large: { Instances: [], Memory: 8gb, vCPUs: 2 }
m5a.large: { Instances: [], Memory: 8gb, vCPUs: 2 }
m5.large: { Instances: [], Memory: 8gb, vCPUs: 2 }
m4.large: { Instances: [], Memory: 8gb, vCPUs: 2 }
m3.large: { Instances: [], Memory: 8gb, vCPUs: 2 }
t3a.xlarge: { Instances: [], Memory: 16gb, vCPUs: 4 }
t3.xlarge: { Instances: [], Memory: 16gb, vCPUs: 4 }
t2.xlarge: { Instances: [], Memory: 16gb, vCPUs: 4 }
m5a.xlarge: { Instances: [], Memory: 16gb, vCPUs: 4 }
m5.xlarge: { Instances: [], Memory: 16gb, vCPUs: 4 }
m4.xlarge: { Instances: [], Memory: 16gb, vCPUs: 4 }
m3.xlarge: { Instances: [], Memory: 16gb, vCPUs: 4 }
t3a.2xlarge: { Instances: [], Memory: 32gb, vCPUs: 8 }
t3.2xlarge: { Instances: [], Memory: 32gb, vCPUs: 8 }
t2.2xlarge: { Instances: [], Memory: 32gb, vCPUs: 8 }
m5a.2xlarge: { Instances: [], Memory: 32gb, vCPUs: 8 }
m5.2xlarge: { Instances: [], Memory: 32gb, vCPUs: 8 }
m4.2xlarge: { Instances: [], Memory: 32gb, vCPUs: 8 }
m3.2xlarge: { Instances: [], Memory: 32gb, vCPUs: 8 }
## r3.large instance as spot is actually cheaper then m4.large
## perfect candidate for backups for example
r3.large: { Instances: [], Memory: 16gb, vCPUs: 2 }
MemoryMap:
4gb:
ContainerMemory: 3500
JavaHeap: 1800m
JavaPage: 800m
8gb:
ContainerMemory: 7000
JavaHeap: 3300m
JavaPage: 1800m
16gb:
ContainerMemory: 15000
JavaHeap: 5000m
JavaPage: 6400m
32gb:
ContainerMemory: 31000
JavaHeap: 11400m
JavaPage: 11800m
64gb:
ContainerMemory: 63000
JavaHeap: 23g
JavaPage: 27g
160gb:
ContainerMemory: 155000
JavaHeap: 31g
JavaPage: 106g
256gb:
ContainerMemory: 250000
JavaHeap: 31g
JavaPage: 198g
Conditions:
IsUpgrade: !Equals [!Ref UpgradeMode, true]
ReplicasEnabled:
!And [!Not [!Equals [!Ref ReplicasCount, 0]], !Not [Condition: IsUpgrade]]
CreateSNSTopic: !Equals [!Ref SNSTopicArn, ""]
AcceptedLicense: !Equals [!Ref AcceptLicense, "true"]
CreateCloudMapNamespace: !Equals [!Ref CloudMapNamespaceID, ""]
SpotCore:
!Or [
!Equals [!Ref ClusterInstanceType, "spot.medium"],
!Equals [!Ref ClusterInstanceType, "spot.large"],
!Equals [!Ref ClusterInstanceType, "spot.r.large"],
!Equals [!Ref ClusterInstanceType, "spot.xlarge"],
!Equals [!Ref ClusterInstanceType, "spot.2xlarge"],
]
SpotReplica:
!Or [
!Equals [!Ref ReplicasInstanceType, "spot.medium"],
!Equals [!Ref ReplicasInstanceType, "spot.large"],
!Equals [!Ref ClusterInstanceType, "spot.r.large"],
!Equals [!Ref ReplicasInstanceType, "spot.xlarge"],
!Equals [!Ref ReplicasInstanceType, "spot.2xlarge"],
]
EnableASGHook: !Equals [!Ref IsDrainSupported, "true"]
EnableReplicaASGHook:
!And [!Condition EnableASGHook, !Condition ReplicasEnabled]
SlowQueryLogEnabled: !Not [!Equals [!Ref SlowQueryLog, "disabled"]]
SlowQueryLogReplicaEnabled:
!And [!Condition SlowQueryLogEnabled, !Condition ReplicasEnabled]
CreateBatchEnv: !Equals [!Ref BackupBatchQueueARN, ""]
IsPublicAccessible: !Not [!Equals [!Ref Neo4jAccessFromIP, ""]]
CaptureDebugLogsEnabled: !Equals [!Ref CaptureDebugLogs, true]
CaptureDebugLogsReplicaEnabled:
!And [!Condition CaptureDebugLogsEnabled, !Condition ReplicasEnabled]
IsIOPSSupported:
!And [
!Or [
!Equals [!Ref EBSType, gp3],
!Equals [!Ref EBSType, io1],
!Equals [!Ref EBSType, io2],
],
!Not [!Equals [!Ref EBSIOPS, 0]],
]
IsIOPSBatchSupported:
!And [
!Or [
!Equals [!Ref BackupBatchDiskType, gp3],
!Equals [!Ref BackupBatchDiskType, io1],
!Equals [!Ref BackupBatchDiskType, io2],
],
!Not [!Equals [!Ref BackupBatchDiskIOPS, 0]],
]
Resources:
Neo4jCluster:
Type: AWS::ECS::Cluster
Neo4jReplicas:
Condition: ReplicasEnabled
Type: AWS::ECS::Cluster
Neo4jClientSecurityGroup:
Type: AWS::EC2::SecurityGroup
Properties:
GroupDescription: Neo4j Client Security Group, which has access to the Neo4j Cluster
VpcId: !Ref "VpcId"
Neo4jSecurityGroup:
Type: AWS::EC2::SecurityGroup
Properties:
GroupDescription: Neo4j Cluster Security Group
VpcId: !Ref "VpcId"
#### Start of inbound rules for cluster nodes communications with each other
Neo4jSecurityGroupHTTPinboundFromCluster:
Type: AWS::EC2::SecurityGroupIngress
Properties:
GroupId: !Ref "Neo4jSecurityGroup"
IpProtocol: tcp
FromPort: 7473
ToPort: 7474
SourceSecurityGroupId: !Ref "Neo4jSecurityGroup"
Description: Neo4j HTTP(S) communication between nodes
Neo4jSecurityGroupBoltInboundFromCluster:
Type: AWS::EC2::SecurityGroupIngress
Properties:
GroupId: !Ref "Neo4jSecurityGroup"
IpProtocol: tcp
FromPort: 7687
ToPort: 7687
SourceSecurityGroupId: !Ref "Neo4jSecurityGroup"
Description: Neo4j Bolt communication between nodes
Neo4jSecurityGroupDiscoveryInboundFromCluster:
Type: AWS::EC2::SecurityGroupIngress
Properties:
GroupId: !Ref "Neo4jSecurityGroup"
IpProtocol: tcp
FromPort: 5000
ToPort: 5000
SourceSecurityGroupId: !Ref "Neo4jSecurityGroup"
Description: Neo4j Discovery communication between nodes
Neo4jSecurityGroupTransactionInboundFromCluster:
Type: AWS::EC2::SecurityGroupIngress
Properties:
GroupId: !Ref "Neo4jSecurityGroup"
IpProtocol: tcp
FromPort: 6000
ToPort: 6000
SourceSecurityGroupId: !Ref "Neo4jSecurityGroup"
Description: Neo4j Transactions communication between nodes
Neo4jSecurityGroupRaftInboundFromCluster:
Type: AWS::EC2::SecurityGroupIngress
Properties:
GroupId: !Ref "Neo4jSecurityGroup"
IpProtocol: tcp
FromPort: 7000
ToPort: 7000
SourceSecurityGroupId: !Ref "Neo4jSecurityGroup"
Description: Neo4j Raft communication between nodes
Neo4jSecurityGroupHTTPInboundFromClient:
Type: AWS::EC2::SecurityGroupIngress
Properties:
GroupId: !Ref "Neo4jSecurityGroup"
IpProtocol: tcp
FromPort: 7473
ToPort: 7474
SourceSecurityGroupId: !Ref "Neo4jClientSecurityGroup"
Description: Neo4j HTTP(S) external access
Neo4jSecurityGroupBoltInboundFromClient:
Type: AWS::EC2::SecurityGroupIngress
Properties:
GroupId: !Ref "Neo4jSecurityGroup"
IpProtocol: tcp
FromPort: 7687
ToPort: 7687
SourceSecurityGroupId: !Ref "Neo4jClientSecurityGroup"
Description: Neo4j Bolt communication from client to cluster
Neo4jSecurityGroupHTTPinbound:
Condition: IsPublicAccessible
Type: AWS::EC2::SecurityGroupIngress
Properties:
GroupId: !Ref "Neo4jSecurityGroup"
IpProtocol: tcp
FromPort: 7473
ToPort: 7474
CidrIp: !Ref Neo4jAccessFromIP
Description: Neo4j HTTP(S) external access
Neo4jSecurityGroupBoltInbound:
Condition: IsPublicAccessible
Type: AWS::EC2::SecurityGroupIngress
Properties:
GroupId: !Ref "Neo4jSecurityGroup"
IpProtocol: tcp
FromPort: 7687
ToPort: 7687
CidrIp: !Ref Neo4jAccessFromIP
Description: Neo4j Bolt external access
Neo4jSecurityGroupBackupsInbound:
Type: AWS::EC2::SecurityGroupIngress
Properties:
GroupId: !Ref "Neo4jSecurityGroup"
IpProtocol: tcp
FromPort: 6362
ToPort: 6372
SourceSecurityGroupId: !Ref "Neo4jSecurityGroup"
Description: Neo4j Backup external access
CloudwatchLogsGroup:
Type: AWS::Logs::LogGroup
Properties:
LogGroupName: !Sub ${AWS::StackName}
RetentionInDays: 14
CloudwatchSlowQueryLogsGroup:
Condition: SlowQueryLogEnabled
Type: AWS::Logs::LogGroup
Properties:
LogGroupName: !Sub ${AWS::StackName}/slowquery
RetentionInDays: 14
Neo4jClusterTask:
Type: AWS::ECS::TaskDefinition
Properties:
Family: !Sub "${AWS::StackName}-cluster"
TaskRoleArn: !GetAtt ECSTaskRole.Arn
NetworkMode: host
Volumes:
- Name: !Sub ${AWS::StackName}-data
DockerVolumeConfiguration:
Scope: shared
Autoprovision: true
Driver: rexray/ebs
DriverOpts:
volumetype: !Ref EBSType
size: !Ref EBSSize
iops: !If [IsIOPSSupported, !Ref EBSIOPS, !Ref AWS::NoValue]
ContainerDefinitions:
- Name: neo4j
Essential: true
Image: !Ref "DockerImage"
MemoryReservation:
Fn::FindInMap:
- MemoryMap
- Fn::FindInMap:
- InstanceTypes
- !Ref "ClusterInstanceType"
- Memory
- ContainerMemory
MountPoints:
- ContainerPath: /neo4j
SourceVolume: !Sub ${AWS::StackName}-data
ReadOnly: false
PortMappings:
# discovery port
- ContainerPort: 5000
HostPort: 5000
# transaction port
- ContainerPort: 6000
HostPort: 6000
# raft port
- ContainerPort: 7000
HostPort: 7000
# http interface
- ContainerPort: 7474
HostPort: 7474
# https interface
- ContainerPort: 7473
HostPort: 7473
# bolt interface
- ContainerPort: 7687
HostPort: 7687
Environment:
- Name: UPGRADE_MODE
Value: !Ref UpgradeMode
- Name: CLOUDMAP_SERVICE_ID
Value: !Ref DiscoveryServiceCoreSRV
- Name: NEO4J_dbms_import_csv_legacy__quote__escaping
Value: "false"
- Name: NEO4J_causal__clustering_discovery__type
# Value: SRV # discovery based on the SRV record created by CloudMap
# Value: DNS
Value: LIST
- Name: NEO4J_causal__clustering_initial__discovery__members
# Value: !Sub ${DiscoveryServiceCoreSRV.Name}.${CloudMapNamespaceName}:0
Value: !Sub ${DiscoveryServiceCoreA.Name}.${CloudMapNamespaceName}:5000
- Name: NEO4J_causal__clustering_minimum__core__cluster__size__at__formation
Value: !Ref DesiredCapacity
- Name: NEO4J_causal__clustering_minimum__core__cluster__size__at__runtime
Value: "2"
- Name: NEO4J_DATA_ROOT
Value: /neo4j
- Name: AWS_REGION
Value: !Ref AWS::Region
- Name: NEO4J_dbms_allow__upgrade
Value: !If [IsUpgrade, "true", !Ref AllowUpgrade]
- Name: NEO4J_dbms_mode
Value: !If [IsUpgrade, "SINGLE", "CORE"]
- Name: NEO4J_dbms_memory_heap_max__size
Value:
Fn::FindInMap:
- MemoryMap
- Fn::FindInMap:
- InstanceTypes
- !Ref "ClusterInstanceType"
- Memory
- JavaHeap
- Name: NEO4J_dbms_memory_heap_initial__size
Value:
Fn::FindInMap:
- MemoryMap
- Fn::FindInMap:
- InstanceTypes
- !Ref "ClusterInstanceType"
- Memory
- JavaHeap
- Name: NEO4J_dbms_memory_pagecache_size
Value:
Fn::FindInMap:
- MemoryMap
- Fn::FindInMap:
- InstanceTypes
- !Ref "ClusterInstanceType"
- Memory
- JavaPage
- Name: NEO4J_ADMIN_PASSWORD
Value: !Ref AdminPassword
- Name: NEO4J_GUEST_AUTH
Value:
!Join ["", [!Ref ReadOnlyUser, "/", !Ref ReadOnlyUserPassword]]
- Name: SNAPSHOT_PATH
Value: !Ref BackupPath
- Name: AWS_BACKUP_BUCKET
Value: !Ref BackupBucket
- Name: STACK_NAME
Value: !Ref AWS::StackName
- Name: STACK_ID
Value: !Ref AWS::StackId
- Name: NEO4J_ACCEPT_LICENSE_AGREEMENT
Value: !If [AcceptedLicense, "yes", "no"]
- !If
- SlowQueryLogEnabled
- Name: NEO4J_QUERY_LOG
Value: enabled
- Name: NEO4J_QUERY_LOG
Value: disabled
- !If
- SlowQueryLogEnabled
- Name: NEO4J_dbms_logs_query_threshold
Value: !Ref SlowQueryLog
- !Ref AWS::NoValue
- Name: NEO4J_apoc_ttl_schedule
Value: 60
Ulimits:
- Name: nofile
HardLimit: 400000
SoftLimit: 400000
LogConfiguration:
LogDriver: awslogs
Options:
awslogs-group: !Ref "CloudwatchLogsGroup"
awslogs-region: !Ref "AWS::Region"
awslogs-stream-prefix: !Ref Neo4jCoreSubdomain
HealthCheck: !If
- IsUpgrade
- !Ref AWS::NoValue
# https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_definition_parameters.html#container_definition_healthcheck
- Command: [CMD-SHELL, "/healthcheck.sh"]
# make longer intervals if restoring from backup
# to be sure, that container will not be killed by healthcheck
# during the restore
Interval: 30
Retries: 10
StartPeriod: 300 # 300 is a max value
Timeout: 15
- !If
- SlowQueryLogEnabled
- Name: tail-slowquery-log
Image: alpine:latest
MemoryReservation: 25
Essential: true
DependsOn:
- Condition: START
ContainerName: neo4j
MountPoints:
- ContainerPath: /neo4j
SourceVolume: !Sub ${AWS::StackName}-data
ReadOnly: true
Command: [tail, -Fn, "+1", /neo4j/logs/slow_query.log]
StopTimeout: 3 # tail doesn't gracefully exits on SIGTERM, so give it 3 secs to read all the remaining logs and send KILL
LogConfiguration:
LogDriver: awslogs
Options:
awslogs-group: !Ref "CloudwatchSlowQueryLogsGroup"
awslogs-region: !Ref "AWS::Region"
awslogs-stream-prefix: !Ref Neo4jCoreSubdomain
awslogs-multiline-pattern: '^\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}' # date like 2019-08-29 13:44:21.146
- !Ref AWS::NoValue
- !If
- CaptureDebugLogsEnabled
- Name: tail-debug-log
Image: alpine:latest
MemoryReservation: 25
Essential: true
DependsOn:
- Condition: START
ContainerName: neo4j
MountPoints:
- ContainerPath: /neo4j
SourceVolume: !Sub ${AWS::StackName}-data
ReadOnly: true
Command: [tail, -Fn, "+1", /neo4j/logs/debug.log]
StopTimeout: 3 # tail doesn't gracefully exits on SIGTERM, so give it 3 secs to read all the remaining logs and send KILL
LogConfiguration:
LogDriver: awslogs
Options:
awslogs-group: !Ref "CloudwatchLogsGroup"
awslogs-region: !Ref "AWS::Region"
awslogs-stream-prefix: !Ref Neo4jCoreSubdomain
awslogs-multiline-pattern: '^\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}' # date like 2019-08-29 13:44:21.146
- !Ref AWS::NoValue
Neo4jReplicasTask:
Condition: ReplicasEnabled
Type: AWS::ECS::TaskDefinition
Properties:
Family: !Sub "${AWS::StackName}-replica"
TaskRoleArn: !GetAtt ECSTaskRole.Arn
NetworkMode: host
Volumes:
- Name: exports
Host:
SourcePath: "/tmp/exports"
- Name: data
Host:
SourcePath: "/opt/neo4j"
ContainerDefinitions:
- Name: neo4j
Essential: true
Image: !Ref "DockerImage"
Memory:
Fn::FindInMap:
- MemoryMap
- Fn::FindInMap:
- InstanceTypes
- !Ref "ReplicasInstanceType"
- Memory
- "ContainerMemory"
MountPoints:
- ContainerPath: /tmp/exports
SourceVolume: exports
ReadOnly: false
- ContainerPath: /neo4j
SourceVolume: data
ReadOnly: false
PortMappings:
# discovery port
- ContainerPort: 5000
HostPort: 5000
# transaction port
- ContainerPort: 6000
HostPort: 6000
# raft port
- ContainerPort: 7000
HostPort: 7000
# http interface
- ContainerPort: 7474
HostPort: 7474
# https interface
- ContainerPort: 7473
HostPort: 7473
# bolt interface
- ContainerPort: 7687
HostPort: 7687
Environment:
- Name: UPGRADE_MODE
Value: !Ref UpgradeMode
- Name: CLOUDMAP_SERVICE_ID
Value: !Ref DiscoveryServiceCoreSRV
- Name: NEO4J_causal__clustering_discovery__type
# Value: SRV # discovery based on the SRV record created by CloudMap
Value: LIST
- Name: NEO4J_causal__clustering_initial__discovery__members
Value: !Sub ${DiscoveryServiceCoreSRV.Name}.${CloudMapNamespaceName}:0
- Name: AWS_REGION
Value: !Ref AWS::Region
- Name: NEO4J_dbms_allow__upgrade
Value: !If [IsUpgrade, "true", !Ref AllowUpgrade]
- Name: NEO4J_dbms_mode
Value: READ_REPLICA
- Name: NEO4J_dbms_memory_heap_max__size
Value:
Fn::FindInMap:
- MemoryMap
- Fn::FindInMap:
- InstanceTypes
- !Ref "ReplicasInstanceType"
- Memory
- "JavaHeap"
- Name: NEO4J_dbms_memory_heap_initial__size
Value:
Fn::FindInMap:
- MemoryMap
- Fn::FindInMap:
- InstanceTypes
- !Ref "ReplicasInstanceType"
- Memory
- "JavaHeap"
- Name: NEO4J_dbms_memory_pagecache_size
Value:
Fn::FindInMap:
- MemoryMap
- Fn::FindInMap:
- InstanceTypes
- !Ref "ReplicasInstanceType"
- Memory
- "JavaPage"
- Name: NEO4J_ADMIN_PASSWORD
Value: !Ref AdminPassword
- Name: NEO4J_GUEST_AUTH
Value:
!Join ["", [!Ref ReadOnlyUser, "/", !Ref ReadOnlyUserPassword]]
# Remove read replica snapshot restoration because of the problem
# 2019-09-18 09:07:20.751+0000 ERROR Failed to start Neo4j: Starting Neo4j failed: Component 'org.neo4j.server.database.LifecycleManagingDatabase@7da31a40'
# was successfully initialized, but failed to start. Please see the attached cause exception "Timer already cancelled.".
# Starting Neo4j failed: Component 'org.neo4j.server.database.LifecycleManagingDatabase@7da31a40' was successfully initialized,
# but failed to start. Please see the attached cause exception "Timer already cancelled.".
# - Name: SNAPSHOT_PATH
# Value: !Ref BackupPath
- Name: AWS_BACKUP_BUCKET
Value: !Ref BackupBucket
- Name: STACK_NAME
Value: !Ref AWS::StackName
- Name: STACK_ID
Value: !Ref AWS::StackId
- Name: NEO4J_ACCEPT_LICENSE_AGREEMENT
Value: !If [AcceptedLicense, "yes", "no"]
- Name: NEO4J_DATA_ROOT
Value: /neo4j
- !If
- SlowQueryLogEnabled
- Name: NEO4J_QUERY_LOG
Value: enabled
- Name: NEO4J_QUERY_LOG
Value: disabled
- !If
- SlowQueryLogEnabled
- Name: NEO4J_dbms_logs_query_threshold
Value: !Ref SlowQueryLog
- !Ref AWS::NoValue
Ulimits:
- Name: nofile
HardLimit: 400000
SoftLimit: 400000
LogConfiguration:
LogDriver: awslogs
Options:
awslogs-group: !Ref "CloudwatchLogsGroup"
awslogs-region: !Ref "AWS::Region"
awslogs-stream-prefix: !Ref Neo4jReplicasSubdomain
HealthCheck: !If
- IsUpgrade
- !Ref AWS::NoValue
- Command: [CMD-SHELL, "/healthcheck.sh"]
Interval: 60
Retries: 10
StartPeriod: 300
Timeout: 15
- !If
- SlowQueryLogReplicaEnabled
- Name: tail-slowquery-log
Image: alpine:latest
MemoryReservation: 25
Essential: true
DependsOn:
- Condition: START