Skip to content

Commit

Permalink
Fix prometheus-operator test to wait for the CRD to be ready before use
Browse files Browse the repository at this point in the history
  • Loading branch information
@metral
metral committed Jun 18, 2020
1 parent 7c4a21b commit 91cfb18
Show file tree
Hide file tree
Showing 6 changed files with 189 additions and 127 deletions.
4 changes: 4 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
## HEAD (Unreleased)

### Improvements

- Fix prometheus-operator test to wait for the CRD to be ready before use (https://github.com/pulumi/pulumi-kubernetes/pull/1172)

## 2.3.1 (June 17, 2020)

### Improvements
Expand Down
57 changes: 28 additions & 29 deletions tests/examples/examples_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -188,35 +188,34 @@ func TestAccHelmLocal(t *testing.T) {
integration.ProgramTest(t, &test)
}

// TODO: uncomment once https://github.com/pulumi/pulumi-kubernetes/issues/1137 is fixed.
//func TestAccPrometheusOperator(t *testing.T) {
// skipIfShort(t)
// test := getBaseOptions(t).
// With(integration.ProgramTestOptions{
// Dir: path.Join(getCwd(t), "prometheus-operator"),
// SkipRefresh: true,
// ExtraRuntimeValidation: func(
// t *testing.T, stackInfo integration.RuntimeValidationStackInfo,
// ) {
// assert.NotNil(t, stackInfo.Deployment)
// assert.Equal(t, 10, len(stackInfo.Deployment.Resources))
// },
// EditDirs: []integration.EditDir{
// {
// Dir: path.Join(getCwd(t), "prometheus-operator", "steps"),
// Additive: true,
// ExtraRuntimeValidation: func(
// t *testing.T, stackInfo integration.RuntimeValidationStackInfo,
// ) {
// assert.NotNil(t, stackInfo.Deployment)
// assert.Equal(t, 10, len(stackInfo.Deployment.Resources))
// },
// },
// },
// })
//
// integration.ProgramTest(t, &test)
//}
func TestAccPrometheusOperator(t *testing.T) {
skipIfShort(t)
test := getBaseOptions(t).
With(integration.ProgramTestOptions{
Dir: path.Join(getCwd(t), "prometheus-operator"),
SkipRefresh: true,
ExtraRuntimeValidation: func(
t *testing.T, stackInfo integration.RuntimeValidationStackInfo,
) {
assert.NotNil(t, stackInfo.Deployment)
assert.Equal(t, 10, len(stackInfo.Deployment.Resources))
},
EditDirs: []integration.EditDir{
{
Dir: path.Join(getCwd(t), "prometheus-operator", "steps"),
Additive: true,
ExtraRuntimeValidation: func(
t *testing.T, stackInfo integration.RuntimeValidationStackInfo,
) {
assert.NotNil(t, stackInfo.Deployment)
assert.Equal(t, 10, len(stackInfo.Deployment.Resources))
},
},
},
})

integration.ProgramTest(t, &test)
}

func TestAccMariadb(t *testing.T) {
skipIfShort(t)
Expand Down
113 changes: 64 additions & 49 deletions tests/examples/prometheus-operator/index.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import * as k8s from "@pulumi/kubernetes";
import * as pulumi from "@pulumi/pulumi";
import * as k8sClient from "@kubernetes/client-node";
import * as utils from "./utils";

// PrometheusOperatorArgs are the options to configure on the CoreOS
// PrometheusOperator.
Expand All @@ -9,30 +11,29 @@ interface PrometheusOperatorArgs {
}

// PrometheusOperator implements the CoreOS Prometheus Operator.
export class PrometheusOperator extends pulumi.ComponentResource {
class PrometheusOperator extends pulumi.ComponentResource {
public readonly configFile: k8s.yaml.ConfigFile;
public readonly serviceMonitorReady: boolean;
constructor(
name: string,
args: PrometheusOperatorArgs,
opts?: pulumi.ComponentResourceOptions,
) {
super('pulumi:monitoring/v1:PrometheusOperator', name, {}, opts);

this.configFile = new k8s.yaml.ConfigFile(
name,
{
file: `https://github.com/coreos/prometheus-operator/raw/release-${args.version || '0.38'}/bundle.yaml`,
transformations: [
obj => {
if (obj.metadata.namespace) {
obj.metadata.namespace = args.namespace;
}
if (obj.kind === 'ClusterRoleBinding') {
obj.subjects[0].namespace = args.namespace;
}
},
],
}, { parent: this });
this.configFile = new k8s.yaml.ConfigFile(name, {
file: `https://github.com/coreos/prometheus-operator/raw/release-${args.version || '0.38'}/bundle.yaml`,
transformations: [
obj => {
if (obj.metadata.namespace) {
obj.metadata.namespace = args.namespace;
}
if (obj.kind === 'ClusterRoleBinding') {
obj.subjects[0].namespace = args.namespace;
}
},
],
}, {parent: this});
}
}

Expand All @@ -41,38 +42,52 @@ const prometheusOperator = new PrometheusOperator("prometheus", {
namespace: "default",
});

// Use the k8s JS client (https://github.com/kubernetes-client/javascript)
// to retrieve resource created by the operator as a workaround for:
// https://github.com/pulumi/pulumi-kubernetes/issues/1056
const kc = new k8sClient.KubeConfig();
kc.loadFromDefault();
const kApi = kc.makeApiClient(k8sClient.ApiextensionsV1Api)

// Create the Prometheus Operator ServiceMonitor.
const myMonitoring = new k8s.apiextensions.CustomResource('my-monitoring', {
apiVersion: 'monitoring.coreos.com/v1',
kind: 'ServiceMonitor',
spec: {
selector: {
matchLabels: { app: 'my-app' },
},
endpoints: [
{
port: 'http',
interval: '65s',
// start with the following
relabelings: [
{
regex: '(.*)',
targetLabel: 'stackdriver',
replacement: 'true',
action: 'replace'
}
],
// try to add the following in replacement of above in steps/step1.ts
// metricRelabelings: [
// {
// sourceLabels: ['__name__'],
// regex: 'typhoon_(.*)',
// targetLabel: 'stackdriver',
// replacement: 'true',
// action: 'replace'
// }
// ]
const myMonitoring = pulumi.output(utils.checkCrdStatus(kApi)).apply(ready => {
if(!ready){
throw new Error("CRD is not ready");
}
return new k8s.apiextensions.CustomResource('my-monitoring', {
apiVersion: 'monitoring.coreos.com/v1',
kind: 'ServiceMonitor',
spec: {
selector: {
matchLabels: { app: 'my-app' },
},
],
},
}, {dependsOn: prometheusOperator});
endpoints: [
{
port: 'http',
interval: '65s',
// start with the following
relabelings: [
{
regex: '(.*)',
targetLabel: 'stackdriver',
replacement: 'true',
action: 'replace'
}
],
// try to add the following in replacement of above in steps/step1.ts
// metricRelabelings: [
// {
// sourceLabels: ['__name__'],
// regex: 'typhoon_(.*)',
// targetLabel: 'stackdriver',
// replacement: 'true',
// action: 'replace'
// }
// ]
},
],
},
});
})

export const myMonitoringName = myMonitoring.id;
2 changes: 2 additions & 0 deletions tests/examples/prometheus-operator/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
"name": "prometheus-operator",
"version": "0.1.0",
"dependencies": {
"@kubernetes/client-node": "latest",
"@pulumi/kubernetes": "^2.3.1",
"@pulumi/pulumi": "latest"
},
"peerDependencies": {
Expand Down
113 changes: 64 additions & 49 deletions tests/examples/prometheus-operator/step1/index.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import * as k8s from "@pulumi/kubernetes";
import * as pulumi from "@pulumi/pulumi";
import * as k8sClient from "@kubernetes/client-node";
import * as utils from "./utils";

// PrometheusOperatorArgs are the options to configure on the CoreOS
// PrometheusOperator.
Expand All @@ -9,30 +11,29 @@ interface PrometheusOperatorArgs {
}

// PrometheusOperator implements the CoreOS Prometheus Operator.
export class PrometheusOperator extends pulumi.ComponentResource {
class PrometheusOperator extends pulumi.ComponentResource {
public readonly configFile: k8s.yaml.ConfigFile;
public readonly serviceMonitorReady: boolean;
constructor(
name: string,
args: PrometheusOperatorArgs,
opts?: pulumi.ComponentResourceOptions,
) {
super('pulumi:monitoring/v1:PrometheusOperator', name, {}, opts);

this.configFile = new k8s.yaml.ConfigFile(
name,
{
file: `https://github.com/coreos/prometheus-operator/raw/release-${args.version || '0.38'}/bundle.yaml`,
transformations: [
obj => {
if (obj.metadata.namespace) {
obj.metadata.namespace = args.namespace;
}
if (obj.kind === 'ClusterRoleBinding') {
obj.subjects[0].namespace = args.namespace;
}
},
],
}, { parent: this });
this.configFile = new k8s.yaml.ConfigFile(name, {
file: `https://github.com/coreos/prometheus-operator/raw/release-${args.version || '0.38'}/bundle.yaml`,
transformations: [
obj => {
if (obj.metadata.namespace) {
obj.metadata.namespace = args.namespace;
}
if (obj.kind === 'ClusterRoleBinding') {
obj.subjects[0].namespace = args.namespace;
}
},
],
}, {parent: this});
}
}

Expand All @@ -41,38 +42,52 @@ const prometheusOperator = new PrometheusOperator("prometheus", {
namespace: "default",
});

// Use the k8s JS client (https://github.com/kubernetes-client/javascript)
// to retrieve resource created by the operator as a workaround for:
// https://github.com/pulumi/pulumi-kubernetes/issues/1056
const kc = new k8sClient.KubeConfig();
kc.loadFromDefault();
const kApi = kc.makeApiClient(k8sClient.ApiextensionsV1Api)

// Create the Prometheus Operator ServiceMonitor.
const myMonitoring = new k8s.apiextensions.CustomResource('my-monitoring', {
apiVersion: 'monitoring.coreos.com/v1',
kind: 'ServiceMonitor',
spec: {
selector: {
matchLabels: { app: 'my-app' },
},
endpoints: [
{
port: 'http',
interval: '65s',
// removing the following in index.ts in favor of below
// relabelings: [
// {
// regex: '(.*)',
// targetLabel: 'stackdriver',
// replacement: 'true',
// action: 'replace'
// }
// ],
// add the following in replacement of above in index.ts
metricRelabelings: [
{
sourceLabels: ['__name__'],
regex: 'typhoon_(.*)',
targetLabel: 'stackdriver',
replacement: 'true',
action: 'replace'
}
]
const myMonitoring = pulumi.output(utils.checkCrdStatus(kApi)).apply(ready => {
if(!ready){
throw new Error("CRD is not ready");
}
return new k8s.apiextensions.CustomResource('my-monitoring', {
apiVersion: 'monitoring.coreos.com/v1',
kind: 'ServiceMonitor',
spec: {
selector: {
matchLabels: { app: 'my-app' },
},
],
},
}, {dependsOn: prometheusOperator});
endpoints: [
{
port: 'http',
interval: '65s',
// removing the following in index.ts in favor of below
// relabelings: [
// {
// regex: '(.*)',
// targetLabel: 'stackdriver',
// replacement: 'true',
// action: 'replace'
// }
// ],
// add the following in replacement of above in index.ts
metricRelabelings: [
{
sourceLabels: ['__name__'],
regex: 'typhoon_(.*)',
targetLabel: 'stackdriver',
replacement: 'true',
action: 'replace'
}
]
},
],
},
});
})

export const myMonitoringName = myMonitoring.id;
27 changes: 27 additions & 0 deletions tests/examples/prometheus-operator/utils.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
import * as k8sClient from "@kubernetes/client-node";

// Wait for the CRD to be established.
// Use the k8s JS client (https://github.com/kubernetes-client/javascript)
// to retrieve resource created by the operator as a workaround for:
// https://github.com/pulumi/pulumi-kubernetes/issues/1056
export async function checkCrdStatus(client: k8sClient.ApiextensionsV1Api): Promise<boolean> {
for (let i = 0; i < 20; i++){
try {
const crd = await client.readCustomResourceDefinition("servicemonitors.monitoring.coreos.com");
let conditions = crd?.body?.status?.conditions;
if (conditions) {
for (let c of conditions) {
if (c.type == "Established" && c.status == "True") {
console.log("ServiceMonitor CRD is ready");
return true;
}
}
}
break;
} catch(e) {
console.log(`Waiting for ServiceMonitor CRD to be ready (${i+1})`)
await new Promise(resolve => setTimeout(resolve, 3 * 1000));
}
}
return false;
}

0 comments on commit 91cfb18

Please sign in to comment.