Skip to content

Commit

Permalink
Revert "Move YAML decode logic into provider (#925)"
Browse files Browse the repository at this point in the history
This reverts commit c75f754.
  • Loading branch information
lblackstone committed Jan 8, 2020
1 parent d4d2fd7 commit b47f9d1
Show file tree
Hide file tree
Showing 20 changed files with 361 additions and 150 deletions.
81 changes: 75 additions & 6 deletions pkg/gen/nodejs-templates/helm/v2/helm.ts
Original file line number Diff line number Diff line change
Expand Up @@ -219,24 +219,93 @@ export class Chart extends yaml.CollectionComponentResource {
}

parseTemplate(
text: string,
yamlStream: string,
transformations: ((o: any, opts: pulumi.CustomResourceOptions) => void)[] | undefined,
resourcePrefix: string | undefined,
dependsOn: pulumi.Resource[],
): pulumi.Output<{ [key: string]: pulumi.CustomResource }> {
const promise = pulumi.runtime.invoke(
"kubernetes:yaml:decode", {text}, {async: true});
return pulumi.output(promise).apply(p => yaml.parse(
// NOTE: We must manually split the YAML stream because of js-yaml#456. Perusing the code
// and the spec, it looks like a YAML stream is delimited by `^---`, though it is difficult
// to know for sure.
//
// NOTE: We use `{json: true, schema: jsyaml.CORE_SCHEMA}` here so that we conform to Helm's
// YAML parsing semantics. Specifically, `json: true` to ensure that a duplicate key
// overrides its predecessory, rather than throwing an exception, and `schema:
// jsyaml.CORE_SCHEMA` to avoid using additional YAML parsing rules not supported by the
// YAML parser used by Kubernetes.
const objs = yamlStream.split(/^---/m)
.map(yaml => jsyaml.safeLoad(yaml, {json: true, schema: jsyaml.CORE_SCHEMA}))
.filter(a => a != null && "kind" in a)
.sort(helmSort);
return yaml.parse(
{
resourcePrefix: resourcePrefix,
objs: p.result,
yaml: objs.map(o => jsyaml.safeDump(o)),
transformations: transformations || [],
},
{ parent: this, dependsOn: dependsOn }
));
);
}
}

// helmSort is a JavaScript implementation of the Helm Kind sorter[1]. It provides a
// best-effort topology of Kubernetes kinds, which in most cases should ensure that resources
// that must be created first, are.
//
// [1]: https://github.com/helm/helm/blob/094b97ab5d7e2f6eda6d0ab0f2ede9cf578c003c/pkg/tiller/kind_sorter.go
/** @ignore */ export function helmSort(a: { kind: string }, b: { kind: string }): number {
const installOrder = [
"Namespace",
"ResourceQuota",
"LimitRange",
"PodSecurityPolicy",
"Secret",
"ConfigMap",
"StorageClass",
"PersistentVolume",
"PersistentVolumeClaim",
"ServiceAccount",
"CustomResourceDefinition",
"ClusterRole",
"ClusterRoleBinding",
"Role",
"RoleBinding",
"Service",
"DaemonSet",
"Pod",
"ReplicationController",
"ReplicaSet",
"Deployment",
"StatefulSet",
"Job",
"CronJob",
"Ingress",
"APIService"
];

const ordering: { [key: string]: number } = {};
installOrder.forEach((_, i) => {
ordering[installOrder[i]] = i;
});

const aKind = a["kind"];
const bKind = b["kind"];

if (!(aKind in ordering) && !(bKind in ordering)) {
return aKind.localeCompare(bKind);
}

if (!(aKind in ordering)) {
return 1;
}

if (!(bKind in ordering)) {
return -1;
}

return ordering[aKind] - ordering[bKind];
}

/**
* Additional options to customize the fetching of the Helm chart.
*/
Expand Down
35 changes: 24 additions & 11 deletions pkg/gen/nodejs-templates/yaml.ts.mustache
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ import * as outputs from "../types/output";

export interface ConfigOpts {
/** JavaScript objects representing Kubernetes resources. */
objs: pulumi.Input<pulumi.Input<any>[]>;
objs: any[];
/**
* A set of transformations to apply to Kubernetes resource definitions before registering
Expand Down Expand Up @@ -120,10 +120,14 @@ import * as outputs from "../types/output";
resourcePrefix?: string;
}

function yamlLoadAll(text: string): Promise<any[]> {
const promise = pulumi.runtime.invoke(
"kubernetes:yaml:decode", {text}, {async: true});
return promise.then(p => p.result);
function yamlLoadAll(text: string): any[] {
// NOTE[pulumi-kubernetes#501]: Use `loadAll` with `JSON_SCHEMA` here instead of
// `safeLoadAll` because the latter is incompatible with `JSON_SCHEMA`. It is
// important to use `JSON_SCHEMA` here because the fields of the Kubernetes core
// API types are all tagged with `json:`, and they don't deal very well with things
// like dates.
const jsyaml = require("js-yaml");
return jsyaml.loadAll(text, undefined, {schema: jsyaml.JSON_SCHEMA});
}
/** @ignore */ export function parse(
Expand Down Expand Up @@ -336,13 +340,22 @@ import * as outputs from "../types/output";
config: ConfigOpts,
opts?: pulumi.CustomResourceOptions,
): pulumi.Output<{[key: string]: pulumi.CustomResource}> {
return pulumi.output(config.objs).apply(configObjs => {
const objs = configObjs
.map(obj => parseYamlObject(obj, config.transformations, config.resourcePrefix, opts))
.reduce((array, objs) => (array.concat(...objs)), []);
const objs: pulumi.Output<{name: string, resource: pulumi.CustomResource}>[] = [];

return pulumi.output(objs).apply(objs => objs
.reduce((map: {[key: string]: pulumi.CustomResource}, val) => (map[val.name] = val.resource, map), {}))
for (const obj of config.objs) {
const fileObjects: pulumi.Output<{name: string, resource: pulumi.CustomResource}>[] =
parseYamlObject(obj, config.transformations, config.resourcePrefix, opts);
for (const fileObject of fileObjects) {
objs.push(fileObject);
}
}
return pulumi.all(objs).apply(xs => {
let resources: {[key: string]: pulumi.CustomResource} = {};
for (const x of xs) {
resources[x.name] = x.resource
}

return resources;
});
}

Expand Down
7 changes: 3 additions & 4 deletions pkg/gen/python-templates/helm/v2/helm.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from typing import Any, Callable, List, Optional, TextIO, Tuple, Union

import pulumi.runtime
import yaml
from pulumi_kubernetes.yaml import _parse_yaml_document


Expand Down Expand Up @@ -347,12 +348,10 @@ def _parse_chart(all_config: Tuple[str, Union[ChartOpts, LocalChartOpts], pulumi
cmd.extend(home_arg)

chart_resources = pulumi.Output.all(cmd, data).apply(_run_helm_cmd)
objects = chart_resources.apply(
lambda text: pulumi.runtime.invoke('kubernetes:yaml:decode', {'text': text}).value['result'])

# Parse the manifest and create the specified resources.
resources = objects.apply(
lambda objects: _parse_yaml_document(objects, opts, config.transformations))
resources = chart_resources.apply(
lambda yaml_str: _parse_yaml_document(yaml.safe_load_all(yaml_str), opts, config.transformations))

pulumi.Output.all(file, chart_dir, resources).apply(_cleanup_temp_dir)
return resources
Expand Down
9 changes: 5 additions & 4 deletions pkg/gen/python-templates/yaml.py.mustache
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ from typing import Callable, Dict, List, Optional

import pulumi.runtime
import requests
import yaml
import pulumi_kubernetes
from pulumi_kubernetes.apiextensions import CustomResource

from . import tables
Expand All @@ -23,6 +25,7 @@ class ConfigFile(pulumi.ComponentResource):
Kubernetes resources contained in this ConfigFile.
"""


def __init__(self, name, file_id, opts=None, transformations=None, resource_prefix=None):
"""
:param str name: A name for a resource.
Expand Down Expand Up @@ -60,8 +63,7 @@ class ConfigFile(pulumi.ComponentResource):
# Note: Unlike NodeJS, Python requires that we "pull" on our futures in order to get them scheduled for
# execution. In order to do this, we leverage the engine's RegisterResourceOutputs to wait for the
# resolution of all resources that this YAML document created.
__ret__ = pulumi.runtime.invoke('kubernetes:yaml:decode', {'text': text}).value['result']
self.resources = _parse_yaml_document(__ret__, opts, transformations, resource_prefix)
self.resources = _parse_yaml_document(yaml.safe_load_all(text), opts, transformations, resource_prefix)
self.register_outputs({"resources": self.resources})

def translate_output_property(self, prop: str) -> str:
Expand All @@ -82,13 +84,12 @@ class ConfigFile(pulumi.ComponentResource):

# `id` will either be `${name}` or `${namespace}/${name}`.
id = pulumi.Output.from_input(name)
if namespace is not None:
if namespace != None:
id = pulumi.Output.concat(namespace, '/', name)

resource_id = id.apply(lambda x: f'{group_version_kind}:{x}')
return resource_id.apply(lambda x: self.resources[x])


def _read_url(url: str) -> str:
response = requests.get(url)
response.raise_for_status()
Expand Down
49 changes: 0 additions & 49 deletions pkg/provider/invoke_decode_yaml.go

This file was deleted.

38 changes: 2 additions & 36 deletions pkg/provider/provider.go
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,6 @@ const (
streamInvokeList = "kubernetes:kubernetes:list"
streamInvokeWatch = "kubernetes:kubernetes:watch"
streamInvokePodLogs = "kubernetes:kubernetes:podLogs"
invokeDecodeYaml = "kubernetes:yaml:decode"
lastAppliedConfigKey = "kubectl.kubernetes.io/last-applied-configuration"
initialApiVersionKey = "__initialApiVersion"
)
Expand Down Expand Up @@ -377,42 +376,9 @@ func (k *kubeProvider) Configure(_ context.Context, req *pulumirpc.ConfigureRequ
func (k *kubeProvider) Invoke(ctx context.Context,
req *pulumirpc.InvokeRequest) (*pulumirpc.InvokeResponse, error) {

// Always fail.
tok := req.GetTok()
label := fmt.Sprintf("%s.Invoke(%s)", k.label(), tok)
args, err := plugin.UnmarshalProperties(
req.GetArgs(), plugin.MarshalOptions{Label: label, KeepUnknowns: true})
if err != nil {
return nil, pkgerrors.Wrapf(err, "failed to unmarshal %v args during an Invoke call", tok)
}

switch tok {
case invokeDecodeYaml:
var text string
if args["text"].HasValue() {
text = args["text"].StringValue()
} else {
return nil, fmt.Errorf("missing required field 'text' on input: %#v", args)
}

result, err := decodeYaml(text)
if err != nil {
return nil, err
}

objProps, err := plugin.MarshalProperties(
resource.NewPropertyMapFromMap(map[string]interface{}{"result": result}),
plugin.MarshalOptions{
Label: label, KeepUnknowns: true, SkipNulls: true,
})
if err != nil {
return nil, err
}

return &pulumirpc.InvokeResponse{Return: objProps}, nil

default:
return nil, fmt.Errorf("unknown Invoke type %q", tok)
}
return nil, fmt.Errorf("Unknown Invoke type '%s'", tok)
}

// StreamInvoke dynamically executes a built-in function in the provider. The result is streamed
Expand Down
Loading

0 comments on commit b47f9d1

Please sign in to comment.