-
Notifications
You must be signed in to change notification settings - Fork 1.9k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Expose Paths not returned from the API #9379
Comments
Hi @jorgemarey! I tested this with the current I used following job derived from the jobspec you provided above: test.nomadjob "test" {
datacenters = ["dc1"]
type = "service"
group "api" {
count = 1
network {
mode = "bridge"
port "api_expose_healthcheck" {
to = -1
}
}
service {
name = "test"
port = "8001"
check {
name = "http-check"
port = "api_expose_healthcheck"
type = "http"
path = "/health.json"
interval = "10s"
timeout = "2s"
}
connect {
sidecar_service {
proxy {
expose {
path {
path = "/health.json"
protocol = "http"
local_path_port = 8001
listener_port = "api_expose_healthcheck"
}
}
}
}
}
}
task "api" {
driver = "docker"
config {
image = "busybox:1"
command = "httpd"
args = ["-v", "-f", "-p", "8001", "-h", "/www"]
volumes = ["/srv/www:/www:ro"]
}
resources {
cpu = 200
memory = 100
}
}
}
} I then queried the job endpoint via curl (and also {
"Affinities": null,
"AllAtOnce": false,
"Constraints": null,
"ConsulToken": "",
"CreateIndex": 10,
"Datacenters": [
"dc1"
],
"Dispatched": false,
"ID": "test",
"JobModifyIndex": 10,
"Meta": null,
"ModifyIndex": 18,
"Multiregion": null,
"Name": "test",
"Namespace": "default",
"NomadTokenID": "",
"ParameterizedJob": null,
"ParentID": "",
"Payload": null,
"Periodic": null,
"Priority": 50,
"Region": "global",
"Spreads": null,
"Stable": true,
"Status": "running",
"StatusDescription": "",
"Stop": false,
"SubmitTime": 1606770419319729400,
"TaskGroups": [
{
"Affinities": null,
"Constraints": null,
"Count": 1,
"EphemeralDisk": {
"Migrate": false,
"SizeMB": 300,
"Sticky": false
},
"Meta": null,
"Migrate": {
"HealthCheck": "checks",
"HealthyDeadline": 300000000000,
"MaxParallel": 1,
"MinHealthyTime": 10000000000
},
"Name": "api",
"Networks": [
{
"CIDR": "",
"DNS": null,
"Device": "",
"DynamicPorts": [
{
"HostNetwork": "default",
"Label": "api_expose_healthcheck",
"To": -1,
"Value": 0
},
{
"HostNetwork": "default",
"Label": "connect-proxy-test",
"To": -1,
"Value": 0
}
],
"IP": "",
"MBits": 0,
"Mode": "bridge",
"ReservedPorts": null
}
],
"ReschedulePolicy": {
"Attempts": 0,
"Delay": 30000000000,
"DelayFunction": "exponential",
"Interval": 0,
"MaxDelay": 3600000000000,
"Unlimited": true
},
"RestartPolicy": {
"Attempts": 2,
"Delay": 15000000000,
"Interval": 1800000000000,
"Mode": "fail"
},
"Scaling": null,
"Services": [
{
"AddressMode": "auto",
"CanaryMeta": null,
"CanaryTags": null,
"Checks": [
{
"AddressMode": "",
"Args": null,
"CheckRestart": null,
"Command": "",
"Expose": false,
"FailuresBeforeCritical": 0,
"GRPCService": "",
"GRPCUseTLS": false,
"Header": null,
"InitialStatus": "",
"Interval": 10000000000,
"Method": "",
"Name": "http-check",
"Path": "/health.json",
"PortLabel": "api_expose_healthcheck",
"Protocol": "",
"SuccessBeforePassing": 0,
"TLSSkipVerify": false,
"TaskName": "",
"Timeout": 2000000000,
"Type": "http"
}
],
"Connect": {
"Gateway": null,
"Native": false,
"SidecarService": {
"Port": "",
"Proxy": {
"Config": null,
"ExposeConfig": {
"Path": [
{
"ListenerPort": "api_expose_healthcheck",
"LocalPathPort": 8001,
"Path": "/health.json",
"Protocol": "http"
}
]
},
"LocalServiceAddress": "",
"LocalServicePort": 0,
"Upstreams": null
},
"Tags": null
},
"SidecarTask": null
},
"EnableTagOverride": false,
"Meta": null,
"Name": "test",
"PortLabel": "8001",
"Tags": null,
"TaskName": ""
}
],
"ShutdownDelay": null,
"Spreads": null,
"StopAfterClientDisconnect": null,
"Tasks": [
{
"Affinities": null,
"Artifacts": null,
"CSIPluginConfig": null,
"Config": {
"command": "httpd",
"image": "busybox:1",
"volumes": [
"/srv/www:/www:ro"
],
"args": [
"-v",
"-f",
"-p",
"8001",
"-h",
"/www"
]
},
"Constraints": null,
"DispatchPayload": null,
"Driver": "docker",
"Env": null,
"KillSignal": "",
"KillTimeout": 5000000000,
"Kind": "",
"Leader": false,
"Lifecycle": null,
"LogConfig": {
"MaxFileSizeMB": 10,
"MaxFiles": 10
},
"Meta": null,
"Name": "api",
"Resources": {
"CPU": 200,
"Devices": null,
"DiskMB": 0,
"IOPS": 0,
"MemoryMB": 100,
"Networks": null
},
"RestartPolicy": {
"Attempts": 2,
"Delay": 15000000000,
"Interval": 1800000000000,
"Mode": "fail"
},
"ScalingPolicies": null,
"Services": null,
"ShutdownDelay": 0,
"Templates": null,
"User": "",
"Vault": null,
"VolumeMounts": null
},
{
"Affinities": null,
"Artifacts": null,
"CSIPluginConfig": null,
"Config": {
"args": [
"-c",
"${NOMAD_SECRETS_DIR}/envoy_bootstrap.json",
"-l",
"${meta.connect.log_level}",
"--disable-hot-restart"
],
"image": "envoyproxy/envoy:v${NOMAD_envoy_version}"
},
"Constraints": [
{
"LTarget": "${attr.consul.version}",
"Operand": "semver",
"RTarget": ">= 1.6.0-beta1"
}
],
"DispatchPayload": null,
"Driver": "docker",
"Env": null,
"KillSignal": "",
"KillTimeout": 5000000000,
"Kind": "connect-proxy:test",
"Leader": false,
"Lifecycle": {
"Hook": "prestart",
"Sidecar": true
},
"LogConfig": {
"MaxFileSizeMB": 2,
"MaxFiles": 2
},
"Meta": null,
"Name": "connect-proxy-test",
"Resources": {
"CPU": 250,
"Devices": null,
"DiskMB": 0,
"IOPS": 0,
"MemoryMB": 128,
"Networks": null
},
"RestartPolicy": {
"Attempts": 2,
"Delay": 15000000000,
"Interval": 1800000000000,
"Mode": "fail"
},
"ScalingPolicies": null,
"Services": null,
"ShutdownDelay": 5000000000,
"Templates": null,
"User": "",
"Vault": null,
"VolumeMounts": null
}
],
"Update": {
"AutoPromote": false,
"AutoRevert": false,
"Canary": 0,
"HealthCheck": "checks",
"HealthyDeadline": 300000000000,
"MaxParallel": 1,
"MinHealthyTime": 10000000000,
"ProgressDeadline": 600000000000,
"Stagger": 30000000000
},
"Volumes": null
}
],
"Type": "service",
"Update": {
"AutoPromote": false,
"AutoRevert": false,
"Canary": 0,
"HealthCheck": "",
"HealthyDeadline": 0,
"MaxParallel": 1,
"MinHealthyTime": 0,
"ProgressDeadline": 0,
"Stagger": 30000000000
},
"VaultNamespace": "",
"VaultToken": "",
"Version": 0
} |
Quick follow-up, I was not able to reproduce this with Nomad 0.12.7:
{
"Stop": false,
"Region": "global",
"Namespace": "default",
"ID": "test",
"ParentID": "",
"Name": "test",
"Type": "service",
"Priority": 50,
"AllAtOnce": false,
"Datacenters": [
"dc1"
],
"Constraints": null,
"Affinities": null,
"Spreads": null,
"TaskGroups": [
{
"Name": "api",
"Count": 1,
"Update": {
"Stagger": 30000000000,
"MaxParallel": 1,
"HealthCheck": "checks",
"MinHealthyTime": 10000000000,
"HealthyDeadline": 300000000000,
"ProgressDeadline": 600000000000,
"AutoRevert": false,
"AutoPromote": false,
"Canary": 0
},
"Migrate": {
"MaxParallel": 1,
"HealthCheck": "checks",
"MinHealthyTime": 10000000000,
"HealthyDeadline": 300000000000
},
"Constraints": null,
"Scaling": null,
"RestartPolicy": {
"Attempts": 2,
"Interval": 1800000000000,
"Delay": 15000000000,
"Mode": "fail"
},
"Tasks": [
{
"Name": "api",
"Driver": "docker",
"User": "",
"Config": {
"image": "busybox:1",
"volumes": [
"/srv/www:/www:ro"
],
"args": [
"-v",
"-f",
"-p",
"8001",
"-h",
"/www"
],
"command": "httpd"
},
"Env": null,
"Services": null,
"Vault": null,
"Templates": null,
"Constraints": null,
"Affinities": null,
"Resources": {
"CPU": 200,
"MemoryMB": 100,
"DiskMB": 0,
"IOPS": 0,
"Networks": null,
"Devices": null
},
"RestartPolicy": {
"Attempts": 2,
"Interval": 1800000000000,
"Delay": 15000000000,
"Mode": "fail"
},
"DispatchPayload": null,
"Lifecycle": null,
"Meta": null,
"KillTimeout": 5000000000,
"LogConfig": {
"MaxFiles": 10,
"MaxFileSizeMB": 10
},
"Artifacts": null,
"Leader": false,
"ShutdownDelay": 0,
"VolumeMounts": null,
"KillSignal": "",
"Kind": "",
"CSIPluginConfig": null
},
{
"Name": "connect-proxy-test",
"Driver": "docker",
"User": "",
"Config": {
"image": "${meta.connect.sidecar_image}",
"args": [
"-c",
"${NOMAD_SECRETS_DIR}/envoy_bootstrap.json",
"-l",
"${meta.connect.log_level}",
"--disable-hot-restart"
]
},
"Env": null,
"Services": null,
"Vault": null,
"Templates": null,
"Constraints": [
{
"LTarget": "${attr.consul.version}",
"RTarget": ">= 1.6.0-beta1",
"Operand": "semver"
}
],
"Affinities": null,
"Resources": {
"CPU": 250,
"MemoryMB": 128,
"DiskMB": 0,
"IOPS": 0,
"Networks": null,
"Devices": null
},
"RestartPolicy": {
"Attempts": 2,
"Interval": 1800000000000,
"Delay": 15000000000,
"Mode": "fail"
},
"DispatchPayload": null,
"Lifecycle": {
"Hook": "prestart",
"Sidecar": true
},
"Meta": null,
"KillTimeout": 5000000000,
"LogConfig": {
"MaxFiles": 2,
"MaxFileSizeMB": 2
},
"Artifacts": null,
"Leader": false,
"ShutdownDelay": 5000000000,
"VolumeMounts": null,
"KillSignal": "",
"Kind": "connect-proxy:test",
"CSIPluginConfig": null
}
],
"EphemeralDisk": {
"Sticky": false,
"SizeMB": 300,
"Migrate": false
},
"Meta": null,
"ReschedulePolicy": {
"Attempts": 0,
"Interval": 0,
"Delay": 30000000000,
"DelayFunction": "exponential",
"MaxDelay": 3600000000000,
"Unlimited": true
},
"Affinities": null,
"Spreads": null,
"Networks": [
{
"Mode": "bridge",
"Device": "",
"CIDR": "",
"IP": "",
"MBits": 10,
"DNS": null,
"ReservedPorts": null,
"DynamicPorts": [
{
"Label": "api_expose_healthcheck",
"Value": 0,
"To": -1,
"HostNetwork": "default"
},
{
"Label": "connect-proxy-test",
"Value": 0,
"To": -1,
"HostNetwork": "default"
}
]
}
],
"Services": [
{
"Name": "test",
"TaskName": "",
"PortLabel": "8001",
"AddressMode": "auto",
"EnableTagOverride": false,
"Tags": null,
"CanaryTags": null,
"Checks": [
{
"Name": "http-check",
"Type": "http",
"Command": "",
"Args": null,
"Path": "/health.json",
"Protocol": "",
"PortLabel": "api_expose_healthcheck",
"Expose": false,
"AddressMode": "",
"Interval": 10000000000,
"Timeout": 2000000000,
"InitialStatus": "",
"TLSSkipVerify": false,
"Method": "",
"Header": null,
"CheckRestart": null,
"GRPCService": "",
"GRPCUseTLS": false,
"TaskName": "",
"SuccessBeforePassing": 0,
"FailuresBeforeCritical": 0
}
],
"Connect": {
"Native": false,
"SidecarService": {
"Tags": null,
"Port": "",
"Proxy": {
"LocalServiceAddress": "",
"LocalServicePort": 0,
"Upstreams": null,
"Expose": {
"Paths": [
{
"Path": "/health.json",
"Protocol": "http",
"LocalPathPort": 8001,
"ListenerPort": "api_expose_healthcheck"
}
]
},
"Config": null
}
},
"SidecarTask": null,
"Gateway": null
},
"Meta": null,
"CanaryMeta": null
}
],
"Volumes": null,
"ShutdownDelay": null,
"StopAfterClientDisconnect": null
}
],
"Update": {
"Stagger": 30000000000,
"MaxParallel": 1,
"HealthCheck": "",
"MinHealthyTime": 0,
"HealthyDeadline": 0,
"ProgressDeadline": 0,
"AutoRevert": false,
"AutoPromote": false,
"Canary": 0
},
"Multiregion": null,
"Periodic": null,
"ParameterizedJob": null,
"Dispatched": false,
"Payload": null,
"Meta": null,
"ConsulToken": "",
"VaultToken": "",
"VaultNamespace": "",
"NomadTokenID": "",
"Status": "running",
"StatusDescription": "",
"Stable": true,
"Version": 0,
"SubmitTime": 1606771059815018200,
"CreateIndex": 10,
"ModifyIndex": 22,
"JobModifyIndex": 10
}
Is this example job very close to what you're running? Maybe there's something specific to the job you're running that's hitting some edge case? |
Hi @tgross. I think that you actually did reproduce it with the 0.12.7. If you see the SidecarService.Proxy part of the json. The values are:
But with the 1.0.0 those are:
Note the difference between ExposeConfig and Expose, and Path vs Paths. In fact, if I do an inspect instead of the curl, the expose config will de empty (I guess that when doing the unmarshall as the values are not correclty presented they are lost):
Could there be a problem with the codec library version used on the 0.12.X? |
🤦 Thank you for pointing that out... I was so focused on the values I wasn't looking at the keys. That's embarrassing!
I don't think so... the codec library version hasn't been changed in a while and the last time we touched the formatting code was in 887f23a (released as 0.11.2). There's an unfortunate architectural issue in Nomad where the Both v1.0.0-beta2 and 0.12.7 versions of the // ConsulProxy represents a Consul Connect sidecar proxy jobspec stanza.
type ConsulProxy struct {
// Expose configures the consul proxy.expose stanza to "open up" endpoints
// used by task-group level service checks using HTTP or gRPC protocols.
//
// Use json tag to match with field name in api/
Expose *ConsulExposeConfig `json:"ExposeConfig"`
} We made changes in the // ConsulProxy represents a Consul Connect sidecar proxy jobspec stanza.
type ConsulProxy struct {
LocalServiceAddress string `mapstructure:"local_service_address" hcl:"local_service_address,optional"`
LocalServicePort int `mapstructure:"local_service_port" hcl:"local_service_port,optional"`
ExposeConfig *ConsulExposeConfig `mapstructure:"expose" hcl:"expose,block"`
Upstreams []*ConsulUpstream `hcl:"upstreams,block"`
Config map[string]interface{} `hcl:"config,block"`
} And 0.12.7 has: // ConsulProxy represents a Consul Connect sidecar proxy jobspec stanza.
type ConsulProxy struct {
LocalServiceAddress string `mapstructure:"local_service_address"`
LocalServicePort int `mapstructure:"local_service_port"`
ExposeConfig *ConsulExposeConfig `mapstructure:"expose"`
Upstreams []*ConsulUpstream
Config map[string]interface{}
} If you make a curl request you're hitting the At first I suspected that the 0.12.7 bug was in the implicit conversion we're doing between In any case we're not going to backport a fix for 0.12.x for this bug, but it does worry me that we might have fixed a bug accidentally rather than intentionally so I do want to get to the bottom of this. We're going to be shipping the 1.0-rc1 soon so I'm focused there but I'll circle back to this issue later. |
If this is fixed on 1.0 then that's great! We don't really need this to be fixed on 0.12.X, so it's not a problem for us that this is not backported. We encountered this when modifying a job from the UI, as the expose configuration was lost. If you need me to provide any more information, I'll be happy to do that. |
Verified this is still as expected in 1.0+. Unfortunately this portion of the response body isn't documented in the
As far as the origin of the change I discovered that the behavior in the released 0.12.7 version differs from that you'll get if you build from the As mentioned, we're not going to backport this bugfix but at least now we know where it came from. Thanks for your patience! |
Hi again @tgross, I think we're seeing a issue related to this using nomad v1.1.5. |
Just looking into this. I think it has to do with the Expose field in the struct.
And that could miss the
That's set to map between the struct defined in the api package and the one used in the structs package. Could adding the following work whitout breaking anything else?
|
I'm going to lock this issue because it has been closed for 120 days ⏳. This helps our maintainers find and focus on the active issues. |
Nomad version
Nomad v0.12.7
Issue
When running jobs with
Expose Paths
in the sidecar_service stanza those are not returned when querying the job API.The job runs correctly when using the HCL and the nomad client, but if we query the job (via curl or
nomad inspect
) the expose configuration is not presented. This is a problem also if we try to modify the job directly via the UI.Reproduction steps
It seems that the problem is when encoding the struct back to the client. Maybe some problem with the msgpack codec? Maybe the tags are not used correctly?
Job file (if appropriate)
The text was updated successfully, but these errors were encountered: