Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add grafanacloud-specific install script #291

Merged
merged 10 commits into from
Dec 16, 2020
52 changes: 52 additions & 0 deletions cmd/agentctl/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,16 @@
package main

import (
"context"
"fmt"
"os"
"path/filepath"
"sort"
"time"

// Adds version information
_ "github.com/grafana/agent/pkg/build"
"github.com/grafana/agent/pkg/client/grafanacloud"
"github.com/olekukonko/tablewriter"
"github.com/prometheus/common/version"

Expand All @@ -35,6 +38,7 @@ func main() {
walStatsCmd(),
targetStatsCmd(),
samplesCmd(),
cmdCloudConfig(),
)

_ = cmd.Execute()
Expand Down Expand Up @@ -262,6 +266,54 @@ deletion but then comes back at some point).`,
}
}

func cmdCloudConfig() *cobra.Command {
var (
stackID string
apiKey string
)

cmd := &cobra.Command{
Use: "cloud-config",
Short: "Retrieves the cloud config for the Grafana Cloud Agent",
Long: `cloud-config connects to Grafana Cloud and retrieves the generated
config that may be used with this agent.`,
Args: cobra.ExactArgs(0),

// Hidden, this is only expected to be used by scripts.
Hidden: true,

RunE: func(_ *cobra.Command, args []string) error {
if stackID == "" {
return fmt.Errorf("--stack must be provided")
}
if apiKey == "" {
return fmt.Errorf("--api-key must be provided")
}

cli := grafanacloud.NewClient(nil, apiKey)

ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()

cfg, err := cli.AgentConfig(ctx, stackID)
if err != nil {
fmt.Fprintf(os.Stderr, "could not retrieve agent cloud config: %s\n", err)
os.Exit(1)
}

fmt.Println(cfg)
return nil
},
}

cmd.Flags().StringVarP(&stackID, "stack", "u", "", "stack ID to get a config for")
cmd.Flags().StringVarP(&apiKey, "api-key", "p", "", "API key authenticate against Grafana Cloud's API with")
rfratto marked this conversation as resolved.
Show resolved Hide resolved
must(cmd.MarkFlagRequired("stack"))
must(cmd.MarkFlagRequired("api-key"))

return cmd
}

func must(err error) {
if err != nil {
panic(err)
Expand Down
80 changes: 80 additions & 0 deletions pkg/client/grafanacloud/client.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
// Package grafanacloud provides an interface to the Grafana Cloud API.
package grafanacloud
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is out of scope but we really should start consolidating our GCom clients 🙂


import (
"context"
"fmt"
"net/http"
"strings"

"gopkg.in/yaml.v2"
)

const integrationsApiUrl = "https://integrations-api.grafana.net"

// Client is a grafanacloud API client.
type Client struct {
c *http.Client
apiKey string
}

// NewClient creates a new Grafana Cloud client. All requests made will be
// performed using the provided http.Client c. If c is nil, the default
// http client will be used instead.
//
// apiKey will be used to authenticate against the API.
func NewClient(c *http.Client, apiKey string) *Client {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What do you think about adding NewClientFromEnv that assumes a configured GCLOUD_API_KEY?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think the client code is too low level for environment variable logic. The caller can be responsible for retrieving environment variables, if necessary.

(That being said, I don't really like environment variables for configuration, and I think having the flag here is enough given this is a hidden command we don't want to document)

if c == nil {
c = http.DefaultClient
}
return &Client{c: c, apiKey: apiKey}
}

// AgentConfig generates a Grafana Cloud Agent config from the given stack.
// The config is returned as a string in YAML form.
func (c *Client) AgentConfig(ctx context.Context, stackID string) (string, error) {
req, err := http.NewRequestWithContext(
ctx, "GET",
fmt.Sprintf("%s/stacks/%s/agent_config", integrationsApiUrl, stackID),
nil,
)
if err != nil {
return "", fmt.Errorf("failed to generate request: %w", err)
}
req.Header.Add("Authorization", "Bearer "+c.apiKey)

resp, err := c.c.Do(req)
if err != nil {
return "", fmt.Errorf("failed to make request: %w", err)
}
defer resp.Body.Close()

if resp.StatusCode != 200 {
return "", fmt.Errorf("unexpected status code %d", resp.StatusCode)
}

// Even though the API returns json, we'll parse it as YAML here so we can
// re-encode it with the same order it was decoded in.
payload := struct {
Status string `yaml:"status"`
Data yaml.MapSlice `yaml:"data"`
Error string `yaml:"error"`
}{}

dec := yaml.NewDecoder(resp.Body)
if err := dec.Decode(&payload); err != nil {
return "", fmt.Errorf("failed to read response: %w", err)
}

if payload.Status != "success" {
return "", fmt.Errorf("request was not successful: %s", payload.Error)
}

// Convert the data to YAML
var sb strings.Builder
if err := yaml.NewEncoder(&sb).Encode(payload.Data); err != nil {
return "", fmt.Errorf("failed to generate YAML config: %w", err)
}

return sb.String(), nil
}
98 changes: 98 additions & 0 deletions pkg/client/grafanacloud/client_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
package grafanacloud

import (
"context"
"crypto/tls"
"fmt"
"net"
"net/http"
"net/http/httptest"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

const (
testSecret = "secret-key"
testStackID = "12345"
)

func TestClient_AgentConfig(t *testing.T) {
httpClient := testClient(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, "/stacks/"+testStackID+"/agent_config", r.URL.Path)
assert.Equal(t, "Bearer "+testSecret, r.Header.Get("Authorization"))

_, err := w.Write([]byte(`{
"status": "success",
"data": {
"server": {
"http_listen_port": 12345
},
"integrations": {
"agent": {
"enabled": true
}
}
}
}`))
assert.NoError(t, err)
}))

cli := NewClient(httpClient, testSecret)
cfg, err := cli.AgentConfig(context.Background(), testStackID)
require.NoError(t, err)
fmt.Println(cfg)

expect := `
server:
http_listen_port: 12345
integrations:
agent:
enabled: true
`

require.YAMLEq(t, expect, cfg)
}

func TestClient_AgentConfig_Error(t *testing.T) {
httpClient := testClient(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNotFound)
}))

cli := NewClient(httpClient, testSecret)
_, err := cli.AgentConfig(context.Background(), testStackID)
require.Error(t, err, "unexpected status code 404")
}

func TestClient_AgentConfig_ErrorMessage(t *testing.T) {
httpClient := testClient(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, err := w.Write([]byte(`{
"status": "error",
"error": "Something went wrong"
}`))
assert.NoError(t, err)
}))

cli := NewClient(httpClient, testSecret)
_, err := cli.AgentConfig(context.Background(), testStackID)
require.Error(t, err, "request was not successful: Something went wrong")
}

func testClient(t *testing.T, handler http.HandlerFunc) *http.Client {
h := httptest.NewTLSServer(handler)
t.Cleanup(func() {
h.Close()
})

return &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
return net.Dial(network, h.Listener.Addr().String())
},
},
}
}
111 changes: 111 additions & 0 deletions production/grafanacloud-install.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
#!/usr/bin/env bash
rfratto marked this conversation as resolved.
Show resolved Hide resolved
# grafanacloud-install.sh installs the Grafana Cloud Agent on supported x86_64
# Linux systems for Grafana Cloud users. Those who aren't users of Grafana Cloud
# or need to install the Agent on a different architecture or platform should
# try another installation method.
#
# grafanacloud-install.sh has a hard dependency on being run on a supported
# Linux system. Currently only systems that can install deb or rpm packages
# are supported. The target system will try to be detected, but if it cannot,
# PACKAGE_SYSTEM can be passed as an environment variable with either rpm or
# deb.
set -euo pipefail

log() {
echo "$@" >&2
}

fatal() {
log "$@"
exit 1
}

#
# REQUIRED environment variables.
#
GCLOUD_STACK_ID=${GCLOUD_STACK_ID:=} # Stack ID where integrations are installed
GCLOUD_API_KEY=${GCLOUD_API_KEY:=} # API key to communicate to the integrations API

[ -z "$GCLOUD_STACK_ID" ] && fatal "Required environment variable $$GCLOUD_STACK_ID not set."
rfratto marked this conversation as resolved.
Show resolved Hide resolved
[ -z "$GCLOUD_API_KEY" ] && fatal "Required environment variable $$GCLOUD_API_KEY not set."
rfratto marked this conversation as resolved.
Show resolved Hide resolved

#
# OPTIONAL environment variables.
#

# Package system to install the Agent with. If not empty, MUST be either rpm or
# deb. If empty, the script will try to detect the host OS and the appropriate
# package system to use.
PACKAGE_SYSTEM=${PACKAGE_SYSTEM:=}

#
# Global constants.
#
RELEASE_VERSION="0.9.0"

RELEASE_URL="https://github.com/grafana/agent/releases/download/v${RELEASE_VERSION}"
DEB_URL="${RELEASE_URL}/grafana-agent-${RELEASE_VERSION}-1.x86_64.deb"
RPM_URL="${RELEASE_URL}/grafana-agent-${RELEASE_VERSION}-1.x86_64.rpm"

main() {
if [ -z "$PACKAGE_SYSTEM" ]; then
PACKAGE_SYSTEM=$(detect_package_system)
fi
log "--- Using package system $PACKAGE_SYSTEM. Downloading and installing package"

case "$PACKAGE_SYSTEM" in
deb)
install_deb
;;
rpm)
install_rpm
;;
*)
fatal "Unsupported PACKAGE_SYSTEM value $PACKAGE_SYSTEM. Must be either rpm or deb".
;;
esac

log '--- Retrieving config and placing in /etc/grafana-agent.yaml'
retrieve_config | sudo tee /etc/grafana-agent.yaml

log '--- Enabling and starting grafana-agent.service'
sudo systemctl enable grafana-agent.service
sudo systemctl start grafana-agent.service
}

# detect_package_system tries to detect the host distribution to determine if
# deb or rpm should be used for installing the Agent. Prints out either "deb"
# or "rpm". Calls fatal if the host OS is not supported.
detect_package_system() {
command -v dpkg >/dev/null 2>&1 && { echo "deb"; return; }
command -v rpm >/dev/null 2>&1 && { echo "rpm"; return; }

case "$(uname)" in
Darwin)
fatal 'macOS not supported'
;;
*)
fatal "Unknown unsupported OS: $(uname)"
;;
esac
}

# install_deb downloads and installs the deb package of the Grafana Cloud Agent.
install_deb() {
curl -sL "${DEB_URL}" -o /tmp/grfana-agent.deb
rfratto marked this conversation as resolved.
Show resolved Hide resolved
sudo dpkg -i /tmp/grafana-agent.deb
rm /tmp/grafana-agent.deb
}

# install_rpm downloads and installs the deb package of the Grafana Cloud Agent.
install_rpm() {
sudo rpm --reinstall "${RPM_URL}"
}

# retrieve_config downloads the config file for the Agent and prints out its
# contents to stdout.
retrieve_config() {
grafana-agentctl cloud-config -u "${GCLOUD_STACK_ID}" -p "${GCLOUD_API_KEY}" || fatal 'Failed to retrieve config'
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I probably miss something but why not use curl ${API_ENDPOINT}/stacks/${GCLOUD_STACK_ID}/agent.yaml instead? The GCom client seems to be overhead unless we'll add more logic I am not aware of.

Copy link
Member Author

@rfratto rfratto Dec 16, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The implementation of cloud-config will probably change over time when integrations are decoupled from the API implementation. I'm comfortable maintaining a tiny package on the client side that's responsible for stitching together the final config file (which is where I think the logic should live anyway - the integrations API should, ideally, just give me config for integrations and remote_write info, and the Agent should be responsible for knowing how it should be configured outside of that scope).

It may turn out that we even remove the package later on, but I suspect it's also where the C&C client-side code will be written.

}

main