diff --git a/.gitignore b/.gitignore index a3ea0d1f..24c92262 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ -goat pkg/ +bin/ diff --git a/Dockerfile.build b/Dockerfile.build new file mode 100644 index 00000000..a4cd4e05 --- /dev/null +++ b/Dockerfile.build @@ -0,0 +1,37 @@ +FROM golang:1.11-stretch as build + +ENV REPO sevagh/goat +ENV GO111MODULE=on + +RUN apt-get update -y &&\ + apt-get install -y zip + +COPY . /go/src/github.com/$REPO +WORKDIR /go/src/github.com/$REPO + +RUN make build zip tarball + +FROM fedora as rpmbuild + +RUN dnf update -y &&\ + dnf install -y make rpm-build alien + +COPY ./rpm /goat-rpm-pkg/ + +COPY --from=build /go/src/github.com/sevagh/goat/bin/goat /goat-rpm-pkg/goat +COPY --from=build /go/src/github.com/sevagh/goat/bin/version-file /goat-rpm-pkg/version-file + +WORKDIR goat-rpm-pkg + +RUN make rpm + +WORKDIR /root/rpmbuild/RPMS/x86_64/ + +RUN alien ./*.rpm + +FROM alpine + +COPY --from=build /go/src/github.com/sevagh/goat/pkg /_goat-pkg +COPY --from=rpmbuild /root/rpmbuild/RPMS/x86_64/* /_goat-pkg/ + +CMD cp _goat-pkg/* goat-pkg/ diff --git a/GNUmakefile b/GNUmakefile index 4b073d65..e7cd2852 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -1,63 +1,47 @@ NAME:=goat -VERSION:=1.0.0 +VERSION:=$(shell git describe --tags) OS:=linux ARCH:=amd64 -GOAT_FILES?=$$(find . -name '*.go' | grep -v vendor) -BINPATH=usr/sbin +GOAT_FILES:=$$(find . -name '*.go' | grep -v vendor) +BINPATH:=$(PWD)/bin +PKGDIR?=$(PWD)/pkg +DEBIANDIR:=debian/$(NAME)-$(VERSION) all: build +docker-build: + mkdir -p $(PKGDIR) + docker build --no-cache -t "goat-builder" -f Dockerfile.build . + docker run -v $(PKGDIR):/goat-pkg goat-builder + build: CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build -a -tags netgo -ldflags '-w -extldflags "-static" -X main.VERSION=$(VERSION)' -o $(BINPATH)/$(NAME) strip $(BINPATH)/$(NAME) + echo $(VERSION) > $(BINPATH)/version-file -test: - @go vet ./... - @go test -v ./... - -fmt: - @gofmt -s -w $(GOAT_FILES) - -lint: - -gometalinter.v2 --enable-all $(GOAT_FILES) --exclude=_test.go +pkgclean: + rm -rf $(PKGDIR) -package_all: pkgclean build deb rpm zip +tarball: + @tar -czf $(PKGDIR)/$(NAME)_$(VERSION)_$(OS)_$(ARCH).tar.gz -C $(BINPATH) $(NAME) zip: - @zip pkg/$(NAME)_$(VERSION)_$(OS)_$(ARCH).zip -j $(BINPATH)/$(NAME) - -deb: - @mkdir -p pkg - fpm -s dir -t deb -n $(NAME) -v $(VERSION) -C . \ - -p pkg/$(NAME)_VERSION_ARCH.deb \ - -d "mdadm" \ - --deb-systemd ./goat@.service \ - $(BINPATH) - -rpm: - @mkdir -p pkg - fpm -s dir -t rpm -n $(NAME) -v $(VERSION) -C . \ - -p pkg/$(NAME)_VERSION_ARCH.rpm \ - -d "mdadm" \ - --rpm-systemd ./goat@.service \ - $(BINPATH) + @zip $(PKGDIR)/$(NAME)_$(VERSION)_$(OS)_$(ARCH).zip -j $(BINPATH)/$(NAME) -pkgclean: - @rm -rf pkg +test: + @go test -v ./... -lintsetup: - @go get -u gopkg.in/alecthomas/gometalinter.v2 - @gometalinter.v2 --install 2>&1 >/dev/null - @go install ./... +fmt: + @gofmt -s -w $(GOAT_FILES) dev-env: ## Build a local development environment using Docker @docker run -it --rm \ -v $(shell pwd):/go/src/github.com/sevagh/$(NAME) \ -w /go/src/github.com/sevagh/$(NAME) \ - golang:1.10 \ + golang:1.11 \ /bin/bash -c 'make install; bash' install: ## Build and install locally the binary (dev purpose) go install . -.PHONY: dev-env clean install test +.PHONY: docker-build dev-env clean install test diff --git a/README.md b/README.md index bf1d11d7..643a2bcc 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,18 @@ -# goat :goat: +# goat ### Attach EBS volumes and ENIs to running EC2 instances -`goat` is a Go application which runs from inside the EC2 instance. +`goat` is a Go program which runs from inside the EC2 instance. -By setting your tags correctly, `goat` can discover and attach EBS volumes and ENIs. Furthermore, for EBS volumes, it can perform additional actions such as RAID (with mdadm), mkfs, and mount EBS volumes to the EC2 instance where it's running. - -The `goat` package consists of the subcommands [goat ebs](./docs/EBS.md) and [goat eni](./docs/ENI.md). - -### Permission model - -It's necessary for the instance to have an IAM Role with _at least_ access to the EBS and ENI resources that it will be attaching - see [here](./docs/hcl-example/iam_role.tf). Your roles can be even more permissive (i.e. full EC2 access) but that comes with its own risks. - -Unfortunately, resource-level permissions are [currently not supported](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/ec2-api-permissions.html#ec2-api-unsupported-resource-permissions) for attaching network interfaces. This means that to use `goat@eni`, your instances must have full permissions for __all__ ENIs. +By setting your tags correctly, `goat` can discover and attach EBS volumes and ENIs. For EBS volumes, it can perform additional actions such as RAID (with mdadm), mkfs, and mount EBS volumes to the EC2 instance where it's running. ### Install and run -Goat is a Go binary that should be able to run on any Linux instance. - -In the releases tab you can find a zip of the binary, and a `.deb` and `.rpm` package with systemd support generated by [fpm](https://github.com/jordansissel/fpm). - -To use goat, run it during the launch process of your EC2 instance - you can `systemctl enable goat@TARGET` and `systemctl start goat@TARGET` (where TARGET is one of ebs or eni). +Goat is a Go binary that should be able to run on any Linux instance. In the releases tab you can find a zip of the binary, and a `.deb` and `.rpm` package with systemd support. Goat needs `mdadm` to perform RAID (which is a dependency in the deb and rpm). -It's a Linux-specific tool which needs `mdadm` to work for the RAID features. +To use goat, run it during the launch process of your EC2 instance - you can `systemctl enable goat@TARGET` and `systemctl start goat@TARGET` (where TARGET is one of ebs or eni) in the EC2 user data script. [Full Terraform example here](./terraform-example). -#### Usage +### Usage In the most basic case, you should run `goat ebs` or `goat eni`. @@ -44,23 +32,75 @@ OPTIONS Display version and exit ``` -### Environment variables - -You can use the environment variables: +You can set `-tagPrefix` and `-logLevel` with environment variables (which take precedence): * `GOAT_LOG_LEVEL` * `GOAT_TAG_PREFIX` -Set these the same as `-logLevel` and `-tagPrefix` - the environment variables take precedence. +### Tags + +These are the tags you need (recall that the `GOAT-IN` prefix is configurable): + +| Tag Name | Description | Resource type | Required | Effect | +| :------------------- | :---------------------- | :---------------- | :------------------ | :---------- | +| GOAT-IN:Prefix | Logical app name | EC2, EBS, ENI | :heavy_check_mark: | attach | +| GOAT-IN:NodeId | Node id | EC2, EBS, ENI | :heavy_check_mark: | attach | +| GOAT-IN:VolumeName | Distinct volume name | EBS | | | +| GOAT-IN:VolumeSize | # of disks in vol group | EBS | | mdadm | +| GOAT-IN:RaidLevel | level of RAID (0 or 1) | EBS | | mdadm | +| GOAT-IN:MountPath | Linux path to mount vol | EBS | | mount | +| GOAT-IN:FsType | Linux filesystem type | EBS | | mkfs | + +If non-required tags are omitted, that step is skipped. The barest case will simply attach the EBS volumes and perform no further actions. + +The filesystem and fstab entries are created with the label `GOAT-{VolumeName}` for convenience. Running `goat` multiple times will result in it detecting the existing label it intended to create and not proceeding. + +Aside from the `mount` syscall, `goat` shells out to `mdadm`, `blkid`, and `mkfs`. If the mount and RAID steps are performed, the configs will be persisted to `/etc/fstab` and `/etc/mdadm.conf`. + +Check the [Terraform example](./terraform-example) for example tag values. + +### Permissions + +It's necessary for the instance to have an IAM Role with _at least_ access to the EBS and ENI resources that it will be attaching - see [here](./terraform-example/iam_role.tf). Your roles can be even more permissive (i.e. full EC2 access) but that comes with its own risks. + +Unfortunately, resource-level permissions are [currently not supported](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/ec2-api-permissions.html#ec2-api-unsupported-resource-permissions) for attaching network interfaces. This means that to use `goat@eni`, your instances must have full permissions for __all__ ENIs. + +### Example EBS usecase - attaching old disks to a new instance + +The specific use-case that `goat` was developed to solve is the following. Say you have 3 instances with their own respective disks, and you receive a termination notice for instance 1. I want the `goat` workflow to be: + +1. Terminate instance 1 +2. Create a new instance with the same GOAT tags (to indicate to `goat` that it's the logical equivalent as the machine it is replacing) + 1. **No need to modify or manipulate the EBS volumes or their tags** +4. On boot, everything works magically + +After `goat` ran on the first fresh run, the EBS volumes got the correct filesystems, labels, and in the case of RAID, `mdadm` metadata on them. + +The event flow on a re-created instance is: + +1. Get EC2 metadata on the running instance, create an EC2 client, and search EBS volumes +2. Attach the volumes it needs based on their tags +3. Discover that `/dev/disk/by-label` already contains the correct disks + 1. From `mdadm` magic, after the EBS attachment the RAID array is already detected correctly +4. Proceed to perform the `fstab` and `mount` phases - skip `mdadm`, `mkfs` + +**CAVEAT**: the mdadm metadata will have the hostname of the previous EC2 instance: + +``` +[centos@ip-172-31-29-69 ~]$ sudo mdadm --detail --scan --verbose +ARRAY /dev/md127 level=raid0 num-devices=3 metadata=1.2 name="ip-172-31-25-105:'GOAT-data'" UUID=2d08b310:fd13bd21:bc2417a4:56a1ec57 + devices=/dev/xvdb,/dev/xvdc,/dev/xvdd +[centos@ip-172-31-29-69 ~]$ +``` -### Additional dependencies for ENI +To avoid this, define a good/persistent hostname for the EC2 instance, that you will then re-apply to any instance taking ownership of the previous instance's disks. -Goat by itself is sufficient for the EBS feature, but needs help for setting up an ENI. Refer to [this](./docs/ENI.md#setting-up-the-eni---ec2-net-utils) document. +### ENI notes -### Hack +As mentioned, the only action `goat` will do for ENIs is attaching them. You can try to use [ec2-net-utils](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-eni.html#ec2-net-utils), a tool available on Amazon Linux AMIs, or [this port to CentOS/systemd](https://github.com/etuttle/ec2-utils), to configure an ENI after `goat` attaches it. -If you have docker locally, you can use the following command in order to quickly get a development env ready: `make dev-env`. +ENI attachments take a parameter called [DeviceIndex](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-ec2-network-interface-attachment.html). Goat isn't smart, and always starts from DeviceIndex `1`. This means that your EC2 instance should have no attached ENIs to use `goat`. If it does, they should be the ones that `goat` was going to attach anyway, not external ENIs that have no `goat` tags. -### Examples +### Build and develop -[Link to the example Terraform HCL scripts](./docs/hcl-example). +The deb, rpm, and zip are generated from a multi-stage [Dockerfile.build](./Dockerfile.build). Invoke it with `make docker-build`. If you have docker locally, you can use the following command in order to quickly get a development env ready: `make dev-env`. diff --git a/docs/EBS.md b/docs/EBS.md deleted file mode 100644 index 182ac592..00000000 --- a/docs/EBS.md +++ /dev/null @@ -1,75 +0,0 @@ -### Goat for EBS - -#### Behavior - -`goat ebs` should behave correctly with no parameters. It is configured entirely with tags (explained [below](#tags)). It logs to `stderr` by default. - -It takes some options: - -* `-logLevel=` - logrus log levels (i.e. debug, info, warn, error, fatal, panic) -* `-debug` - an interactive debug mode which prompts to continue after every phase so you can explore the state between phases - -#### Tags - -These are the tags you need: - -| Tag Name | Description | EC2 | EBS | Required | Tag Value (examples) | -| -------------------- | ----------------------- | ------- | ----- | -------- | ---------------------------------------------------------------- | -| GOAT-IN:Prefix | Logical stack name | *Yes* | *Yes* | *YES* | `my_app_v1.3.4` | -| GOAT-IN:NodeId | EC2 id within stack | *Yes* | *Yes* | *YES* | `0`, `1`, `2` for 3-node kafka | -| GOAT-IN:VolumeName | Distinct volume name | | *Yes* | no | `data`, `log` - RAID disks must use the same VolumeName | -| GOAT-IN:VolumeSize | # of disks in vol group | | *Yes* | no | 2 for 2-disk RAID, 1 for single disk/no RAID | -| GOAT-IN:RaidLevel | level of RAID (0 or 1) | | *Yes* | no | 0 or 1 for RAID, ignored if VolumeSize == 1 | -| GOAT-IN:MountPath | Linux path to mount vol | | *Yes* | no | `/var/kafka_data` | -| GOAT-IN:FsType | Linux filesystem type | | *Yes* | no | `ext4`, `vfat` | - -#### Missing tags - -If a non-required tag is missing, that step will be skipped. E.g. without RaidLevel or VolumeSize, `mdadm` won't be run. Without a filesystem, `mkfs` won't be run. Without a mount path, `mount` won't be run. Without a volume name, nothing will be run. - -*The barest case will simply attach the EBS volumes and perform no further actions* - -#### Fresh run - -The event flow is roughly the following: - -* Get EC2 metadata on the running instance -* Use metadata to establish an EC2 client and scan EBS volumes -* Attach the volumes it needs based on their tags -* Use typical Linux disk tools to mount the drives correctly: - * `mdadm` for RAID volumes (if needed) - * `blkid` to check for an existing filesystem - * `mkfs` to make the filesystem - * `/etc/fstab` entries to preserve behavior on reboot - -The filesystem and fstab entries are created with the label `GOAT-{volumeName}` for convenience. Running `goat` multiple times will result in it detecting the existing label it intended to create and not proceeding. - -#### Attaching old disks to a new instance - -The specific use-case that `goat` was developed to solve is the following. Say you have 3 instances with their own respective disks, and you receive a termination notice for instance 1. I want the `goat` workflow to be: - -* Terminate instance 1 -* Create a new one with the same GOAT tags (to indicate that it's the same as the machine it is replacing) -* Everything works magically - -After `goat` ran on the previous instance with fresh disks, the disks have the correct filesystems, labels, and in the case of RAID, `mdadm` metadata on them. - -The event flow on a re-created instance (with disks that were previously attached to another instance) is: - -* Get EC2 metadata -* Use metadata to establish an EC2 client and scan EBS volumes -* Attach the volumes it needs based on their tags -* Discover that `/dev/disk/by-label` already contains the correct disks - * From `mdadm` magic, after the EBS attachment the RAID array is already detected correctly -* Proceed to perform the `fstab` and `mount` phases - skip `mdadm`, `mkfs` - -**CAVEAT**: the mdadm metadata will have the hostname of the previous EC2 instance: - -``` -[centos@ip-172-31-29-69 ~]$ sudo mdadm --detail --scan --verbose -ARRAY /dev/md127 level=raid0 num-devices=3 metadata=1.2 name="ip-172-31-25-105:'GOAT-data'" UUID=2d08b310:fd13bd21:bc2417a4:56a1ec57 - devices=/dev/xvdb,/dev/xvdc,/dev/xvdd -[centos@ip-172-31-29-69 ~]$ -``` - -To avoid this, define a good/persistent hostname for EC2 instance, that you will then re-apply to any instance taking over this instance's disks. diff --git a/docs/ENI.md b/docs/ENI.md deleted file mode 100644 index 0b89e032..00000000 --- a/docs/ENI.md +++ /dev/null @@ -1,40 +0,0 @@ -### Goat for ENI - -#### Behavior - -`goat eni` should behave correctly with no parameters. It is configured entirely with tags (explained [below](#tags)). It logs to `stderr` by default. - -It takes some options: - -* `-logLevel=` - logrus log levels (i.e. debug, info, warn, error, fatal, panic) -* `-debug` - an interactive debug mode which prompts to continue after every phase so you can explore the state between phases - -#### Fresh run - -The event flow is roughly the following: - -* Get EC2 metadata on the running instance -* Use metadata to establish an EC2 client and scan ENIs -* Attach the ENIs it needs based on their tags - -#### Setting up the ENI - ec2-net-utils - -There's a project to port [ec2-net-utils](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-eni.html#ec2-net-utils), a tool available on Amazon Linux AMIs, to CentOS/systemd, [here](https://github.com/etuttle/ec2-utils). - - -#### DeviceIndex - -ENI attachments take a parameter called [DeviceIndex](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-ec2-network-interface-attachment.html). Goat isn't smart, and always starts from DeviceIndex `1`. - -This means that your EC2 instance should have no attached ENIs. - -If it does, they should be the ones that `goat` was going to attach anyway, not external ENIs that have no `goat` tags. - -#### Tags - -These are the tags you need: - -| Tag Name | Description | EC2 | ENI | Tag Value (examples) | -| -------------------- | ----------------------- | ------- | ----- | ---------------------------------------------------------------- | -| GOAT-IN:Prefix | Logical stack name | *Yes* | *Yes* | `my_app_v1.3.4` | -| GOAT-IN:NodeId | EC2 id within stack | *Yes* | *Yes* | `0`, `1`, `2` for 3-node kafka | diff --git a/rpm/GNUmakefile b/rpm/GNUmakefile new file mode 100644 index 00000000..9697b0c1 --- /dev/null +++ b/rpm/GNUmakefile @@ -0,0 +1,6 @@ +VERSION:=$(shell cat version-file | tr - _) + +all: rpm + +rpm: + @rpmbuild -ba specfile.spec --define "_sourcedir $(PWD)" --define "_version $(VERSION)" diff --git a/goat@.service b/rpm/goat@.service similarity index 100% rename from goat@.service rename to rpm/goat@.service diff --git a/rpm/specfile.spec b/rpm/specfile.spec new file mode 100644 index 00000000..d3ffd5dd --- /dev/null +++ b/rpm/specfile.spec @@ -0,0 +1,63 @@ +%define pkgname goat + +Name: %{pkgname} +Version: %{_version} +Release: 1%{?dist} +Summary: Attach and mount EBS and ENI + +License: BSD 3-clause +URL: https://github.com/sevagh/goat +Source0: %{pkgname} +Source1: %{pkgname}@.service + +Requires: systemd mdadm + + +%description +Automatically attach AWS resources to a running EC2 instance. + + +#%prep +#%setup +#%build + + +%install +%{__mkdir} -p %{buildroot}/%{_sbindir} +%{__mkdir} -p %{buildroot}/%{_unitdir} +%{__install} -m0775 %{SOURCE0} %{buildroot}/%{_sbindir}/%{pkgname} +%{__install} -m0777 %{SOURCE1} %{buildroot}/%{_unitdir}/%{pkgname}@.service + + +%files +%{_sbindir}/%{pkgname} +%{_unitdir}/%{pkgname}@.service + + +%post +if [ $1 -eq 1 ]; then + /bin/systemctl daemon-reload >/dev/null 2>&1 || : +fi + + +%preun +if [ $1 -eq 0 ] ; then + # Package removal, not upgrade + /bin/systemctl disable %{pkgname}@*.service >/dev/null 2>&1 || : + /bin/systemctl stop %{pkgname}@*.service >/dev/null 2>&1 || : +fi + + +%postun +/bin/systemctl daemon-reload >/dev/null 2>&1 || : + + +%changelog +* Tue Apr 17 2018 Sevag Hanssian +- Recombine commands +* Tue Mar 06 2018 Sevag Hanssian +- Split subcommands into two binaries +* Thu Aug 10 2017 Sevag Hanssian +- Goat subcommands +* Tue Jul 11 2017 Sevag Hanssian +- First RPM package for goat diff --git a/docs/hcl-example/.gitignore b/terraform-example/.gitignore similarity index 100% rename from docs/hcl-example/.gitignore rename to terraform-example/.gitignore diff --git a/docs/hcl-example/LICENSE b/terraform-example/LICENSE similarity index 100% rename from docs/hcl-example/LICENSE rename to terraform-example/LICENSE diff --git a/docs/hcl-example/README.md b/terraform-example/README.md similarity index 100% rename from docs/hcl-example/README.md rename to terraform-example/README.md diff --git a/docs/hcl-example/bootstrap.tpl b/terraform-example/bootstrap.tpl similarity index 100% rename from docs/hcl-example/bootstrap.tpl rename to terraform-example/bootstrap.tpl diff --git a/docs/hcl-example/iam_role.tf b/terraform-example/iam_role.tf similarity index 100% rename from docs/hcl-example/iam_role.tf rename to terraform-example/iam_role.tf diff --git a/docs/hcl-example/instance.tf b/terraform-example/instance.tf similarity index 100% rename from docs/hcl-example/instance.tf rename to terraform-example/instance.tf diff --git a/docs/hcl-example/network.tf b/terraform-example/network.tf similarity index 100% rename from docs/hcl-example/network.tf rename to terraform-example/network.tf diff --git a/docs/hcl-example/provider.tf b/terraform-example/provider.tf similarity index 100% rename from docs/hcl-example/provider.tf rename to terraform-example/provider.tf diff --git a/docs/hcl-example/sg.tf b/terraform-example/sg.tf similarity index 100% rename from docs/hcl-example/sg.tf rename to terraform-example/sg.tf diff --git a/docs/hcl-example/subnet.tf b/terraform-example/subnet.tf similarity index 100% rename from docs/hcl-example/subnet.tf rename to terraform-example/subnet.tf diff --git a/docs/hcl-example/variables.tf b/terraform-example/variables.tf similarity index 100% rename from docs/hcl-example/variables.tf rename to terraform-example/variables.tf diff --git a/docs/hcl-example/volumes.tf b/terraform-example/volumes.tf similarity index 100% rename from docs/hcl-example/volumes.tf rename to terraform-example/volumes.tf