Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

multiple data-race detected #12098

Open
shoenig opened this issue Feb 20, 2022 · 3 comments
Open

multiple data-race detected #12098

shoenig opened this issue Feb 20, 2022 · 3 comments
Assignees
@schmichael
Labels
stage/accepted Confirmed, and intend to work on. No timeline committment though. theme/testing Test related issues type/bug

Comments

@shoenig
Copy link
Member

shoenig commented Feb 20, 2022 •
edited

While messing around with running CI in GHA

tests-nomad

WARNING: DATA RACE
Write at 0x00c00049a760 by goroutine 30:
  github.com/hashicorp/nomad/nomad/state.(*StateStore).upsertAllocsImpl()
      /home/runner/work/nomad/nomad/nomad/state/state_store.go:3378 +0x294
  github.com/hashicorp/nomad/nomad/state.(*StateStore).UpsertAllocs()
      /home/runner/work/nomad/nomad/nomad/state/state_store.go:3336 +0x129
  github.com/hashicorp/nomad/nomad/drainer.TestDrainingJobWatcher_DrainJobs()
      /home/runner/work/nomad/nomad/nomad/drainer/watch_jobs_test.go:192 +0xd6f
  testing.tRunner()
      /opt/hostedtoolcache/go/1.17.7/x64/src/testing/testing.go:1259 +0x22f
  testing.(*T).Run·dwrap·21()
      /opt/hostedtoolcache/go/1.17.7/x64/src/testing/testing.go:1306 +0x47

Previous read at 0x00c00049a760 by goroutine 62:
  github.com/hashicorp/nomad/nomad/drainer.(*drainingJobWatcher).getJobAllocsImpl()
      /home/runner/work/nomad/nomad/nomad/drainer/watch_jobs.go:466 +0x4f1
  github.com/hashicorp/nomad/nomad/drainer.(*drainingJobWatcher).getJobAllocsImpl-fm()
      /home/runner/work/nomad/nomad/nomad/drainer/watch_jobs.go:441 +0x59
  github.com/hashicorp/nomad/nomad/state.(*StateStore).BlockingQuery()
      /home/runner/work/nomad/nomad/nomad/state/state_store.go:308 +0xee
  github.com/hashicorp/nomad/nomad/drainer.(*drainingJobWatcher).getJobAllocs()
      /home/runner/work/nomad/nomad/nomad/drainer/watch_jobs.go:429 +0xfb
  github.com/hashicorp/nomad/nomad/drainer.(*drainingJobWatcher).watch()
      /home/runner/work/nomad/nomad/nomad/drainer/watch_jobs.go:151 +0x21e
  github.com/hashicorp/nomad/nomad/drainer.NewDrainingJobWatcher·dwrap·18()
      /home/runner/work/nomad/nomad/nomad/drainer/watch_jobs.go:88 +0x39

tests-client

WARNING: DATA RACE
Write at 0x00c00060c1c0 by goroutine 111:
  github.com/hashicorp/nomad/client.(*heartbeatStop).setLastOk()
      /home/runner/work/nomad/nomad/client/heartbeatstop.go:129 +0xfa
  github.com/hashicorp/nomad/client.(*Client).registerNode()
      /home/runner/work/nomad/nomad/client/client.go:1851 +0x5fe
  github.com/hashicorp/nomad/client.(*Client).retryRegisterNode()
      /home/runner/work/nomad/nomad/client/client.go:1803 +0x69
  github.com/hashicorp/nomad/client.(*Client).registerAndHeartbeat()
      /home/runner/work/nomad/nomad/client/client.go:1586 +0x5b
  github.com/hashicorp/nomad/client.(*Client).registerAndHeartbeat-fm()
      /home/runner/work/nomad/nomad/client/client.go:1584 +0x39
  github.com/hashicorp/nomad/client.(*group).Go.func1()
      /home/runner/work/nomad/nomad/client/client.go:3104 +0x9d

Previous write at 0x00c00060c1c0 by goroutine 161:
  github.com/hashicorp/nomad/client.(*heartbeatStop).watch()
      /home/runner/work/nomad/nomad/client/heartbeatstop.go:73 +0x7c
  github.com/hashicorp/nomad/client.NewClient·dwrap·16()
      /home/runner/work/nomad/nomad/client/client.go:484 +0x39

WARNING: DATA RACE
Read at 0x00c001496108 by goroutine 19:
  runtime.racereadrange()
      <autogenerated>:1 +0x1b
  github.com/hashicorp/nomad/nomad/structs.(*Task).Copy()
      /home/runner/work/nomad/nomad/nomad/structs/structs.go:6890 +0x778
  github.com/hashicorp/nomad/nomad/structs.(*TaskGroup).Copy()
      /home/runner/work/nomad/nomad/nomad/structs/structs.go:6124 +0xa8f
  github.com/hashicorp/nomad/nomad/structs.(*Job).Copy()
      /home/runner/work/nomad/nomad/nomad/structs/structs.go:4257 +0x94f
  github.com/hashicorp/nomad/nomad/state.(*StateStore).setJobStatus()
      /home/runner/work/nomad/nomad/nomad/state/state_store.go:4625 +0x127
  github.com/hashicorp/nomad/nomad/state.(*StateStore).setJobStatuses()
      /home/runner/work/nomad/nomad/nomad/state/state_store.go:4589 +0x2a7
  github.com/hashicorp/nomad/nomad/state.(*StateStore).upsertAllocsImpl()
      /home/runner/work/nomad/nomad/nomad/state/state_store.go:3456 +0x130d
  github.com/hashicorp/nomad/nomad/state.(*StateStore).UpsertPlanResults()
      /home/runner/work/nomad/nomad/nomad/state/state_store.go:401 +0xb0d
  github.com/hashicorp/nomad/nomad.(*nomadFSM).applyPlanResults()
      /home/runner/work/nomad/nomad/nomad/fsm.go:1005 +0x2f3
  github.com/hashicorp/nomad/nomad.(*nomadFSM).Apply()
      /home/runner/work/nomad/nomad/nomad/fsm.go:238 +0x9dc
  github.com/hashicorp/raft.(*Raft).runFSM.func1()
      /home/runner/go/pkg/mod/github.com/hashicorp/raft@v1.3.5/fsm.go:98 +0x3bc
  github.com/hashicorp/raft.(*Raft).runFSM.func2()
      /home/runner/go/pkg/mod/github.com/hashicorp/raft@v1.3.5/fsm.go:121 +0x8e8
  github.com/hashicorp/raft.(*Raft).runFSM()
      /home/runner/go/pkg/mod/github.com/hashicorp/raft@v1.3.5/fsm.go:231 +0x471
  github.com/hashicorp/raft.(*Raft).runFSM-fm()
      /home/runner/go/pkg/mod/github.com/hashicorp/raft@v1.3.5/fsm.go:77 +0x39
  github.com/hashicorp/raft.(*raftState).goFunc.func1()
      /home/runner/go/pkg/mod/github.com/hashicorp/raft@v1.3.5/state.go:146 +0x82

Previous write at 0x00c001496108 by goroutine 83:
  github.com/hashicorp/nomad/nomad/structs.(*Resources).Canonicalize()
      /home/runner/work/nomad/nomad/nomad/structs/structs.go:2420 +0xa8
  github.com/hashicorp/nomad/nomad/structs.(*Task).Canonicalize()
      /home/runner/work/nomad/nomad/nomad/structs/structs.go:6943 +0x378
  github.com/hashicorp/nomad/nomad/structs.(*TaskGroup).Canonicalize()
      /home/runner/work/nomad/nomad/nomad/structs/structs.go:6193 +0xb4a
  github.com/hashicorp/nomad/nomad/structs.(*Job).Canonicalize()
      /home/runner/work/nomad/nomad/nomad/structs/structs.go:4225 +0x1c5
  github.com/hashicorp/nomad/nomad/structs.(*Allocation).Canonicalize()
      /home/runner/work/nomad/nomad/nomad/structs/structs.go:9500 +0x1e4
  github.com/hashicorp/nomad/nomad/state.(*StateStore).UpsertPlanResults()
      /home/runner/work/nomad/nomad/nomad/state/state_store.go:398 +0xad3
  github.com/hashicorp/nomad/nomad.(*planner).applyPlan()
      /home/runner/work/nomad/nomad/nomad/plan_apply.go:313 +0x17db
  github.com/hashicorp/nomad/nomad.(*planner).planApply()
      /home/runner/work/nomad/nomad/nomad/plan_apply.go:167 +0x77b
  github.com/hashicorp/nomad/nomad.(*Server).establishLeadership·dwrap·219()
      /home/runner/work/nomad/nomad/nomad/leader.go:250 +0x39

WARNING: DATA RACE
Read at 0x00c001a8bd10 by goroutine 449:
  runtime.mapaccess2_faststr()
      /opt/hostedtoolcache/go/1.17.7/x64/src/runtime/map_faststr.go:107 +0x0
  github.com/hashicorp/nomad/client/pluginmanager/csimanager.(*csiManager).MounterForPlugin()
      /home/runner/work/nomad/nomad/client/pluginmanager/csimanager/manager.go:78 +0x7b
  github.com/hashicorp/nomad/client.(*CSI).NodeDetachVolume()
      /home/runner/work/nomad/nomad/client/csi_endpoint.go:464 +0x588
  runtime.call32()
      /opt/hostedtoolcache/go/1.17.7/x64/src/runtime/asm_amd64.s:626 +0x48
  reflect.Value.Call()
      /opt/hostedtoolcache/go/1.17.7/x64/src/reflect/value.go:339 +0xd7
  net/rpc.(*service).call()
      /opt/hostedtoolcache/go/1.17.7/x64/src/net/rpc/server.go:377 +0x28c
  net/rpc.(*Server).ServeRequest()
      /opt/hostedtoolcache/go/1.17.7/x64/src/net/rpc/server.go:498 +0x1a4
  github.com/hashicorp/nomad/client.(*Client).ClientRPC()
      /home/runner/work/nomad/nomad/client/rpc.go:36 +0x1a4
  github.com/hashicorp/nomad/client.TestCSINode_DetachVolume.func2()
      /home/runner/work/nomad/nomad/client/csi_endpoint_test.go:967 +0x327
  testing.tRunner()
      /opt/hostedtoolcache/go/1.17.7/x64/src/testing/testing.go:1259 +0x22f
  testing.(*T).Run·dwrap·21()
      /opt/hostedtoolcache/go/1.17.7/x64/src/testing/testing.go:1306 +0x47

Previous write at 0x00c001a8bd10 by goroutine 377:
  runtime.mapassign_faststr()
      /opt/hostedtoolcache/go/1.17.7/x64/src/runtime/map_faststr.go:202 +0x0
  github.com/hashicorp/nomad/client/pluginmanager/csimanager.(*csiManager).instancesForType()
      /home/runner/work/nomad/nomad/client/pluginmanager/csimanager/manager.go:197 +0x216
  github.com/hashicorp/nomad/client/pluginmanager/csimanager.(*csiManager).resyncPluginsFromRegistry()
      /home/runner/work/nomad/nomad/client/pluginmanager/csimanager/manager.go:135 +0x254
  github.com/hashicorp/nomad/client/pluginmanager/csimanager.(*csiManager).runLoop()
      /home/runner/work/nomad/nomad/client/pluginmanager/csimanager/manager.go:104 +0x2cb
  github.com/hashicorp/nomad/client/pluginmanager/csimanager.(*csiManager).Run·dwrap·3()
      /home/runner/work/nomad/nomad/client/pluginmanager/csimanager/manager.go:93 +0x39

WARNING: DATA RACE
Read at 0x00c0000900df by goroutine 26:
  github.com/hashicorp/nomad/client/dynamicplugins.TestPluginEventBroadcaster_SendsMessagesToAllClients.func2()
      /home/runner/work/nomad/nomad/client/dynamicplugins/registry_test.go:34 +0x39
  github.com/stretchr/testify/assert.Eventually.func1()
      /home/runner/go/pkg/mod/github.com/stretchr/testify@v1.7.0/assert/assertions.go:1655 +0x39

Previous write at 0x00c0000900df by goroutine 16:
  github.com/hashicorp/nomad/client/dynamicplugins.TestPluginEventBroadcaster_SendsMessagesToAllClients.func1()
      /home/runner/work/nomad/nomad/client/dynamicplugins/registry_test.go:24 +0x44
  github.com/hashicorp/nomad/client/dynamicplugins.TestPluginEventBroadcaster_SendsMessagesToAllClients·dwrap·10()
      /home/runner/work/nomad/nomad/client/dynamicplugins/registry_test.go:28 +0x58
@shoenig shoenig changed the title data-race detected in state store multiple data-race detected Feb 21, 2022
@lgfa29 lgfa29 added stage/accepted Confirmed, and intend to work on. No timeline committment though. theme/testing Test related issues labels Feb 22, 2022
tgross added a commit that referenced this issue Apr 12, 2022
@tgross
CSI: fix data race in plugin manager
109666a 
The plugin manager for CSI hands out instances of a plugin for callers
that need to mount a volume. The `MounterForPlugin` method accesses
the internal instances map without a lock, and can be called
concurrently from outside the plugin manager's main run-loop.

The original commit for the instances map included a warning that it
needed to be accessed only from the main loop but that comment was
unfortunately ignored shortly thereafter, so this bug has existed in
the code for a couple years without being detected until we ran tests
with `-race` in #12098. Lesson learned here: comments make for lousy
enforcement of invariants!
@tgross tgross mentioned this issue Apr 12, 2022
Merged
tgross added a commit that referenced this issue Apr 12, 2022
@tgross
CSI: fix data race in plugin manager
762acf8 
The plugin manager for CSI hands out instances of a plugin for callers
that need to mount a volume. The `MounterForPlugin` method accesses
the internal instances map without a lock, and can be called
concurrently from outside the plugin manager's main run-loop.

The original commit for the instances map included a warning that it
needed to be accessed only from the main loop but that comment was
unfortunately ignored shortly thereafter, so this bug has existed in
the code for a couple years without being detected until we ran tests
with `-race` in #12098. Lesson learned here: comments make for lousy
enforcement of invariants!
tgross added a commit that referenced this issue Apr 12, 2022
@tgross
fix data race in dynamic plugin registry tests
1bb0958 
These tests have a data race where the test assertion is reading a
value that's being set in the `listenFunc` goroutines that are
subscribing to registry update events. Move the assertion into the
subscribing goroutine to remove the race. This bug was discovered
in #12098 but does not impact production Nomad code.
@tgross tgross mentioned this issue Apr 12, 2022
Merged
@tgross
Copy link
Member

tgross commented Apr 12, 2022

Two fixes for the CSI-related items: #12553 #12554

tgross added a commit that referenced this issue Apr 12, 2022
@tgross
CSI: fix data race in plugin manager (#12553)
9d5b3bc 
The plugin manager for CSI hands out instances of a plugin for callers
that need to mount a volume. The `MounterForPlugin` method accesses
the internal instances map without a lock, and can be called
concurrently from outside the plugin manager's main run-loop.

The original commit for the instances map included a warning that it
needed to be accessed only from the main loop but that comment was
unfortunately ignored shortly thereafter, so this bug has existed in
the code for a couple years without being detected until we ran tests
with `-race` in #12098. Lesson learned here: comments make for lousy
enforcement of invariants!
tgross added a commit that referenced this issue Apr 14, 2022
@tgross
fix data race in dynamic plugin registry tests
f30e8a8 
These tests have a data race where the test assertion is reading a
value that's being set in the `listenFunc` goroutines that are
subscribing to registry update events. Move the assertion into the
subscribing goroutine to remove the race. This bug was discovered
in #12098 but does not impact production Nomad code.
tgross added a commit that referenced this issue Apr 14, 2022
@tgross
fix data race in dynamic plugin registry tests (#12554)
d2aab5d 
These tests have a data race where the test assertion is reading a
value that's being set in the `listenFunc` goroutines that are
subscribing to registry update events. Move the assertion into the
subscribing goroutine to remove the race. This bug was discovered
in #12098 but does not impact production Nomad code.
This was referenced Apr 20, 2022
Closed
Closed
lgfa29 pushed a commit that referenced this issue Apr 20, 2022
@tgross @lgfa29
CSI: fix data race in plugin manager (#12553)
543101e 
The plugin manager for CSI hands out instances of a plugin for callers
that need to mount a volume. The `MounterForPlugin` method accesses
the internal instances map without a lock, and can be called
concurrently from outside the plugin manager's main run-loop.

The original commit for the instances map included a warning that it
needed to be accessed only from the main loop but that comment was
unfortunately ignored shortly thereafter, so this bug has existed in
the code for a couple years without being detected until we ran tests
with `-race` in #12098. Lesson learned here: comments make for lousy
enforcement of invariants!
lgfa29 pushed a commit that referenced this issue Apr 20, 2022
@tgross @lgfa29
CSI: fix data race in plugin manager (#12553)
0cfe4e3 
The plugin manager for CSI hands out instances of a plugin for callers
that need to mount a volume. The `MounterForPlugin` method accesses
the internal instances map without a lock, and can be called
concurrently from outside the plugin manager's main run-loop.

The original commit for the instances map included a warning that it
needed to be accessed only from the main loop but that comment was
unfortunately ignored shortly thereafter, so this bug has existed in
the code for a couple years without being detected until we ran tests
with `-race` in #12098. Lesson learned here: comments make for lousy
enforcement of invariants!
@tgross
Copy link
Member

tgross commented Jul 20, 2022

Another possible in https://app.circleci.com/pipelines/github/hashicorp/nomad/28486/workflows/88d021ae-09b1-47b3-a777-e53777ea08ad/jobs/314220.

@shoenig
Copy link
Member Author

shoenig commented Aug 2, 2022

New race in CaptureFiles

    operator_debug_test.go:437: [TEST] Client for test TestDebug_CapturedFiles ready, id: df6ec249-b6bd-bbe3-e0d3-9e6857b0bd85, region: global
    operator_debug_test.go:439: serverName: TestDebug_CapturedFiles.global, clientID, df6ec249-b6bd-bbe3-e0d3-9e6857b0bd85
2022-08-02T13:07:14.945Z [TRACE] client/client.go:1919: client: next heartbeat: period=13.097315477s
2022-08-02T13:07:14.945Z [DEBUG] agent/http.go:579: http: request complete: method=GET path=/v1/nodes duration="171.768µs"
2022-08-02T13:07:14.946Z [DEBUG] agent/http.go:579: http: request complete: method=GET path=/v1/nodes?prefix=df6ec249-b6bd-bbe3-e0d3-9e6857b0bd85 duration="156.114µs"
2022-08-02T13:07:14.946Z [DEBUG] agent/http.go:579: http: request complete: method=GET path=/v1/agent/members duration="129.636µs"
2022-08-02T13:07:14.947Z [DEBUG] agent/http.go:579: http: request complete: method=GET path=/v1/status/leader duration="146.175µs"
fatal error: concurrent map read and map write

goroutine 39028 [running]:
runtime.throw({0x2abda8e?, 0x296a0a0?})
	/usr/local/go/src/runtime/panic.go:992 +0x71 fp=0xc005f789f0 sp=0xc005f789c0 pc=0x43c2f1
runtime.mapaccess2(0xc0094c4bb0?, 0xc001e9ec30?, 0x4588680?)
	/usr/local/go/src/runtime/map.go:476 +0x205 fp=0xc005f78a30 sp=0xc005f789f0 pc=0x412945
reflect.mapaccess(0x254a060?, 0xc0094c4bb0?, 0xc0094c4bb0?)
	/usr/local/go/src/runtime/map.go:1331 +0x19 fp=0xc005f78a58 sp=0xc005f78a30 pc=0x468f19
reflect.Value.MapIndex({0x24c5160?, 0xc001e9ec80?, 0xc005a26640?}, {0x254a060, 0xc0094c4bb0, 0x94})
	/usr/local/go/src/reflect/value.go:1616 +0x151 fp=0xc005f78ac8 sp=0xc005f78a58 pc=0x4e33b1
github.com/mitchellh/reflectwalk.walkMap({0x24c5160?, 0xc001e9ec80?, 0xc005a26640?}, {0x296a0a0, 0xc005a26640})
	/home/circleci/.go_workspace/pkg/mod/github.com/mitchellh/reflectwalk@v1.0.2/reflectwalk.go:252 +0x315 fp=0xc005f78ba8 sp=0xc005f78ac8 pc=0x702275
github.com/mitchellh/reflectwalk.walk({0x24c5160?, 0xc001e9ec80?, 0xc005a26640?}, {0x296a0a0, 0xc005a26640})
	/home/circleci/.go_workspace/pkg/mod/github.com/mitchellh/reflectwalk@v1.0.2/reflectwalk.go:200 +0x5aa fp=0xc005f78c90 sp=0xc005f78ba8 pc=0x701b4a
github.com/mitchellh/reflectwalk.walkStruct({0x2925420?, 0xc001e9ec60?, 0xc005a26640?}, {0x296a0a0, 0xc005a26640})
	/home/circleci/.go_workspace/pkg/mod/github.com/mitchellh/reflectwalk@v1.0.2/reflectwalk.go:404 +0x3d5 fp=0xc005f78e10 sp=0xc005f78c90 pc=0x702c35
github.com/mitchellh/reflectwalk.walk({0x290e760?, 0xc007c1e040?, 0xc005a26640?}, {0x296a0a0, 0xc005a26640})
	/home/circleci/.go_workspace/pkg/mod/github.com/mitchellh/reflectwalk@v1.0.2/reflectwalk.go:206 +0x6af fp=0xc005f78ef8 sp=0xc005f78e10 pc=0x701c4f
github.com/mitchellh/reflectwalk.walkStruct({0x29f1740?, 0xc007c1e000?, 0xc005a26640?}, {0x296a0a0, 0xc005a26640})
	/home/circleci/.go_workspace/pkg/mod/github.com/mitchellh/reflectwalk@v1.0.2/reflectwalk.go:404 +0x3d5 fp=0xc005f79078 sp=0xc005f78ef8 pc=0x702c35
github.com/mitchellh/reflectwalk.walk({0x230c420?, 0xc002964530?, 0xc005a26640?}, {0x296a0a0, 0xc005a26640})
	/home/circleci/.go_workspace/pkg/mod/github.com/mitchellh/reflectwalk@v1.0.2/reflectwalk.go:206 +0x6af fp=0xc005f79160 sp=0xc005f79078 pc=0x701c4f
github.com/mitchellh/reflectwalk.walkStruct({0x29dc200?, 0xc002964400?, 0xc005a26640?}, {0x296a0a0, 0xc005a26640})
	/home/circleci/.go_workspace/pkg/mod/github.com/mitchellh/reflectwalk@v1.0.2/reflectwalk.go:404 +0x3d5 fp=0xc005f792e0 sp=0xc005f79160 pc=0x702c35
github.com/mitchellh/reflectwalk.walk({0x25862a0?, 0xc002964400?, 0xc005a26640?}, {0x296a0a0, 0xc005a26640})
	/home/circleci/.go_workspace/pkg/mod/github.com/mitchellh/reflectwalk@v1.0.2/reflectwalk.go:206 +0x6af fp=0xc005f793c8 sp=0xc005f792e0 pc=0x701c4f
github.com/mitchellh/reflectwalk.Walk({0x25862a0?, 0xc002964400?}, {0x296a0a0, 0xc005a26640})
	/home/circleci/.go_workspace/pkg/mod/github.com/mitchellh/reflectwalk@v1.0.2/reflectwalk.go:99 +0x146 fp=0xc005f79438 sp=0xc005f793c8 pc=0x701526
github.com/mitchellh/copystructure.Config.Copy({0x0?, 0x0?, 0x0?}, {0x25862a0?, 0xc002964400})
	/home/circleci/.go_workspace/pkg/mod/github.com/mitchellh/copystructure@v1.2.0/copystructure.go:116 +0x188 fp=0xc005f794b0 sp=0xc005f79438 pc=0x703188
github.com/mitchellh/copystructure.Copy(...)
	/home/circleci/.go_workspace/pkg/mod/github.com/mitchellh/copystructure@v1.2.0/copystructure.go:36
github.com/hashicorp/nomad/command/agent.(*HTTPServer).AgentSelfRequest(0xc003bb88c0, {0x0?, 0xc005f79778?}, 0x4950d7?)
	/home/circleci/go/src/github.com/hashicorp/nomad/command/agent/agent_endpoint.go:84 +0x33e fp=0xc005f79758 sp=0xc005f794b0 pc=0x1fd499e
github.com/hashicorp/nomad/command/agent.(*HTTPServer).AgentSelfRequest-fm({0x3025740?, 0xc000fa9360?}, 0xc00096f704?)
	<autogenerated>:1 +0x3c fp=0xc005f79788 sp=0xc005f79758 pc=0x2021cfc
github.com/hashicorp/nomad/command/agent.(*HTTPServer).wrap.func1({0x3025740, 0xc000fa9360}, 0xc0044c3a00)
	/home/circleci/go/src/github.com/hashicorp/nomad/command/agent/http.go:581 +0x154 fp=0xc005f798a0 sp=0xc005f79788 pc=0x1fff954
net/http.HandlerFunc.ServeHTTP(0xc007aed880?, {0x3025740?, 0xc000fa9360?}, 0xc005f79908?)
	/usr/local/go/src/net/http/server.go:2084 +0x2f fp=0xc005f798c8 sp=0xc005f798a0 pc=0x803d2f
net/http.(*ServeMux).ServeHTTP(0x0?, {0x3025740, 0xc000fa9360}, 0xc0044c3a00)
	/usr/local/go/src/net/http/server.go:2462 +0x149 fp=0xc005f79918 sp=0xc005f798c8 pc=0x805b89
github.com/gorilla/handlers.CompressHandlerLevel.func1({0x3023d60?, 0xc000d0d260}, 0xc0044c3a00)
	/home/circleci/.go_workspace/pkg/mod/github.com/gorilla/handlers@v1.5.1/compress.go:141 +0x663 fp=0xc005f79a70 sp=0xc005f79918 pc=0x1f4b7a3
net/http.HandlerFunc.ServeHTTP(0x0?, {0x3023d60?, 0xc000d0d260?}, 0x4554960?)
	/usr/local/go/src/net/http/server.go:2084 +0x2f fp=0xc005f79a98 sp=0xc005f79a70 pc=0x803d2f
net/http.serverHandler.ServeHTTP({0xc006836d50?}, {0x3023d60, 0xc000d0d260}, 0xc0044c3a00)
	/usr/local/go/src/net/http/server.go:2916 +0x43b fp=0xc005f79b58 sp=0xc005f79a98 pc=0x8077db
net/http.(*conn).serve(0xc005a26000, {0x3025bd8, 0xc007a4e8a0})
	/usr/local/go/src/net/http/server.go:1966 +0x5d7 fp=0xc005f79fb8 sp=0xc005f79b58 pc=0x8027d7
net/http.(*Server).Serve.func3()
	/usr/local/go/src/net/http/server.go:3071 +0x2e fp=0xc005f79fe0 sp=0xc005f79fb8 pc=0x80812e
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1571 +0x1 fp=0xc005f79fe8 sp=0xc005f79fe0 pc=0x46fe21
created by net/http.(*Server).Serve
	/usr/local/go/src/net/http/server.go:3071 +0x4db

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees

@schmichael schmichael

Labels
stage/accepted Confirmed, and intend to work on. No timeline committment though. theme/testing Test related issues type/bug
Projects
None yet
Milestone
No milestone
Development

No branches or pull requests
4 participants
@schmichael @shoenig @lgfa29 @tgross