Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

sql/colexec: TestMergeJoiner failed under stress #42774

Closed
cockroach-teamcity opened this issue Nov 26, 2019 · 6 comments · Fixed by #43720
Closed

sql/colexec: TestMergeJoiner failed under stress #42774

cockroach-teamcity opened this issue Nov 26, 2019 · 6 comments · Fixed by #43720
Assignees
@yuzefovich
Labels
C-test-failure Broken test (automatically or manually discovered). O-robot Originated from a bot.
Milestone
20.1

Comments

@cockroach-teamcity
Copy link
Member

SHA: https://github.com/cockroachdb/cockroach/commits/78f8482fd864376a26f32dd983afeb679d1c5977

Parameters:

TAGS=
GOFLAGS=-parallel=4

To repro, try:

# Don't forget to check out a clean suitable branch and experiment with the
# stress invocation until the desired results present themselves. For example,
# using stress instead of stressrace and passing the '-p' stressflag which
# controls concurrency.
./scripts/gceworker.sh start && ./scripts/gceworker.sh mosh
cd ~/go/src/github.com/cockroachdb/cockroach && \
stdbuf -oL -eL \
make stressrace TESTS=TestMergeJoiner PKG=github.com/cockroachdb/cockroach/pkg/sql/colexec TESTTIMEOUT=5m STRESSFLAGS='-maxtime 20m -timeout 10m' 2>&1 | tee /tmp/stress.log

Failed test: https://teamcity.cockroachdb.com/viewLog.html?buildId=1611682&tab=buildLog


panic: runtime error: slice bounds out of range [recovered]
	panic: runtime error: slice bounds out of range

goroutine 3527 [running]:
testing.tRunner.func1(0xc000598200)
	/usr/local/go/src/testing/testing.go:830 +0x392
panic(0x1eef680, 0x3b989a0)
	/usr/local/go/src/runtime/panic.go:522 +0x1b5
github.com/cockroachdb/cockroach/pkg/col/coldata.(*Bytes).Get(...)
	/go/src/github.com/cockroachdb/cockroach/pkg/col/coldata/bytes.go:96
github.com/cockroachdb/cockroach/pkg/col/coldata.(*memColumn).Append(0xc000d2db40, 0x1, 0x271c080, 0xc000d2d980, 0xc000436800, 0x400, 0x400, 0x0, 0x1, 0x2)
	/go/src/github.com/cockroachdb/cockroach/pkg/col/coldata/vec.eg.go:56 +0x2230
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Allocator).Append.func1()
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/allocator.go:118 +0xa6
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Allocator).performOperation(0xc00040da40, 0xc000977600, 0x1, 0x1, 0xc000977610)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/allocator.go:96 +0xfd
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Allocator).Append(0xc00040da40, 0x271c080, 0xc000d2db40, 0x1, 0x271c080, 0xc000d2d980, 0xc000436800, 0x400, 0x400, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/allocator.go:118 +0xfe
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*mergeJoinBase).appendToBufferedGroup(0xc000e3c000, 0xc000e3c028, 0x2706540, 0xc000d2d8c0, 0xc000436800, 0x400, 0x400, 0x1, 0x1)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/mergejoiner.go:421 +0x231
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*mergeJoinFullOuterOp).probeBodyLSeltrueRSeltrue(0xc000e3c000, 0x26d5b00, 0xc000052108)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/mergejoiner_fullouter.eg.go:7985 +0x1a9c5
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*mergeJoinFullOuterOp).probe(0xc000e3c000, 0x26d5b00, 0xc000052108)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/mergejoiner_fullouter.eg.go:38833 +0x1aa
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*mergeJoinFullOuterOp).Next(0xc000e3c000, 0x26d5b00, 0xc000052108, 0x207c420, 0xc000d18101)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/mergejoiner_fullouter.eg.go:39025 +0x2f4
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*opTestOutput).next(0xc000d2d700, 0x26d5b00, 0xc000052108, 0xc000834000, 0xc000d181b0, 0x2706540)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/utils_test.go:765 +0xbb
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*opTestOutput).VerifyAnyOrder(0xc000d2d700, 0x1000, 0x1000)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/utils_test.go:802 +0x78
github.com/cockroachdb/cockroach/pkg/sql/colexec.TestMergeJoiner.func1(0xc000d2d700, 0xc000d2d700, 0x2)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/mergejoiner_test.go:1634 +0xcd
github.com/cockroachdb/cockroach/pkg/sql/colexec.runTestsWithoutAllNullsInjection.func1(0xc000598200, 0xc00040da20, 0x2, 0x2)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/utils_test.go:211 +0x174
github.com/cockroachdb/cockroach/pkg/sql/colexec.runTestsWithFn.func1(0xc000598200)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/utils_test.go:347 +0x298
testing.tRunner(0xc000598200, 0xc0004c6d20)
	/usr/local/go/src/testing/testing.go:865 +0xc0
created by testing.(*T).Run
	/usr/local/go/src/testing/testing.go:916 +0x35a

goroutine 1 [chan receive]:
testing.(*T).Run(0xc0001f2a00, 0x2156ebc, 0xf, 0x220ae70, 0x4c4701)
	/usr/local/go/src/testing/testing.go:917 +0x381
testing.runTests.func1(0xc0005aa300)
	/usr/local/go/src/testing/testing.go:1157 +0x78
testing.tRunner(0xc0005aa300, 0xc0006a5dc8)
	/usr/local/go/src/testing/testing.go:865 +0xc0
testing.runTests(0xc0002ed020, 0x3bba220, 0x5f, 0x5f, 0x0)
	/usr/local/go/src/testing/testing.go:1155 +0x2a9
testing.(*M).Run(0xc0004cd400, 0x0)
	/usr/local/go/src/testing/testing.go:1072 +0x162
github.com/cockroachdb/cockroach/pkg/sql/colexec.TestMain.func1(0xc0004cd400, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/main_test.go:46 +0x1b4
github.com/cockroachdb/cockroach/pkg/sql/colexec.TestMain(0xc0004cd400)
@cockroach-teamcity cockroach-teamcity added C-test-failure Broken test (automatically or manually discovered). O-robot Originated from a bot. labels Nov 26, 2019
@cockroach-teamcity cockroach-teamcity added this to the 20.1 milestone Nov 26, 2019
@rohany rohany assigned yuzefovich and unassigned georgeutsin Nov 26, 2019
@yuzefovich
Copy link
Member

The problem here is that Vec.Append always gets the value even when it is NULL. It is (in a hacky way) fixed by the second commit in #42522.

@yuzefovich yuzefovich mentioned this issue Nov 27, 2019
Merged
@cockroach-teamcity
Copy link
Member Author

SHA: https://github.com/cockroachdb/cockroach/commits/f97dc13163020a032b098ef3eb88e4d9f54a04ba

Parameters:

TAGS=
GOFLAGS=-parallel=4

To repro, try:

# Don't forget to check out a clean suitable branch and experiment with the
# stress invocation until the desired results present themselves. For example,
# using stress instead of stressrace and passing the '-p' stressflag which
# controls concurrency.
./scripts/gceworker.sh start && ./scripts/gceworker.sh mosh
cd ~/go/src/github.com/cockroachdb/cockroach && \
stdbuf -oL -eL \
make stressrace TESTS=TestMergeJoiner PKG=github.com/cockroachdb/cockroach/pkg/sql/colexec TESTTIMEOUT=5m STRESSFLAGS='-maxtime 20m -timeout 10m' 2>&1 | tee /tmp/stress.log

Failed test: https://teamcity.cockroachdb.com/viewLog.html?buildId=1615057&tab=buildLog


panic: runtime error: slice bounds out of range [recovered]
	panic: runtime error: slice bounds out of range

goroutine 3500 [running]:
testing.tRunner.func1(0xc0001f2500)
	/usr/local/go/src/testing/testing.go:830 +0x392
panic(0x1eef7c0, 0x3b989a0)
	/usr/local/go/src/runtime/panic.go:522 +0x1b5
github.com/cockroachdb/cockroach/pkg/col/coldata.(*Bytes).Get(...)
	/go/src/github.com/cockroachdb/cockroach/pkg/col/coldata/bytes.go:96
github.com/cockroachdb/cockroach/pkg/col/coldata.(*memColumn).Append(0xc000b5d4c0, 0x1, 0x271c200, 0xc000b5d300, 0xc000523000, 0x400, 0x400, 0x0, 0x1, 0x2)
	/go/src/github.com/cockroachdb/cockroach/pkg/col/coldata/vec.eg.go:56 +0x2230
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Allocator).Append.func1()
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/allocator.go:118 +0xa6
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Allocator).performOperation(0xc0002f68e0, 0xc000739600, 0x1, 0x1, 0xc000739610)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/allocator.go:96 +0xfd
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Allocator).Append(0xc0002f68e0, 0x271c200, 0xc000b5d4c0, 0x1, 0x271c200, 0xc000b5d300, 0xc000523000, 0x400, 0x400, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/allocator.go:118 +0xfe
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*mergeJoinBase).appendToBufferedGroup(0xc0004702c0, 0xc0004702e8, 0x27066c0, 0xc000b5d240, 0xc000523000, 0x400, 0x400, 0x1, 0x1)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/mergejoiner.go:421 +0x231
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*mergeJoinFullOuterOp).probeBodyLSeltrueRSeltrue(0xc0004702c0, 0x26d5c80, 0xc000052108)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/mergejoiner_fullouter.eg.go:7985 +0x1a9c5
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*mergeJoinFullOuterOp).probe(0xc0004702c0, 0x26d5c80, 0xc000052108)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/mergejoiner_fullouter.eg.go:38833 +0x1aa
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*mergeJoinFullOuterOp).Next(0xc0004702c0, 0x26d5c80, 0xc000052108, 0x207c560, 0xc000227001)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/mergejoiner_fullouter.eg.go:39025 +0x2f4
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*opTestOutput).next(0xc000b5d080, 0x26d5c80, 0xc000052108, 0xc000bce000, 0xc000227080, 0x27066c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/utils_test.go:765 +0xbb
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*opTestOutput).VerifyAnyOrder(0xc000b5d080, 0x1000, 0x1000)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/utils_test.go:802 +0x78
github.com/cockroachdb/cockroach/pkg/sql/colexec.TestMergeJoiner.func1(0xc000b5d080, 0xc000b5d080, 0x2)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/mergejoiner_test.go:1634 +0xcd
github.com/cockroachdb/cockroach/pkg/sql/colexec.runTestsWithoutAllNullsInjection.func1(0xc0001f2500, 0xc0002f68c0, 0x2, 0x2)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/utils_test.go:211 +0x174
github.com/cockroachdb/cockroach/pkg/sql/colexec.runTestsWithFn.func1(0xc0001f2500)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/utils_test.go:347 +0x298
testing.tRunner(0xc0001f2500, 0xc0004c50e0)
	/usr/local/go/src/testing/testing.go:865 +0xc0
created by testing.(*T).Run
	/usr/local/go/src/testing/testing.go:916 +0x35a

goroutine 1 [chan receive]:
testing.(*T).Run(0xc0004f6600, 0x2156ffc, 0xf, 0x220afe0, 0x4c4701)
	/usr/local/go/src/testing/testing.go:917 +0x381
testing.runTests.func1(0xc0001f2300)
	/usr/local/go/src/testing/testing.go:1157 +0x78
testing.tRunner(0xc0001f2300, 0xc0006bbdc8)
	/usr/local/go/src/testing/testing.go:865 +0xc0
testing.runTests(0xc0002753e0, 0x3bba220, 0x5f, 0x5f, 0x0)
	/usr/local/go/src/testing/testing.go:1155 +0x2a9
testing.(*M).Run(0xc0001e1380, 0x0)
	/usr/local/go/src/testing/testing.go:1072 +0x162
github.com/cockroachdb/cockroach/pkg/sql/colexec.TestMain.func1(0xc0001e1380, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/main_test.go:46 +0x1b4
github.com/cockroachdb/cockroach/pkg/sql/colexec.TestMain(0xc0001e1380)

@cockroach-teamcity
Copy link
Member Author

SHA: https://github.com/cockroachdb/cockroach/commits/28f216e1bd53da872a759a98779144c7f70f33a3

Parameters:

TAGS=
GOFLAGS=-parallel=4

To repro, try:

# Don't forget to check out a clean suitable branch and experiment with the
# stress invocation until the desired results present themselves. For example,
# using stress instead of stressrace and passing the '-p' stressflag which
# controls concurrency.
./scripts/gceworker.sh start && ./scripts/gceworker.sh mosh
cd ~/go/src/github.com/cockroachdb/cockroach && \
stdbuf -oL -eL \
make stressrace TESTS=TestMergeJoiner PKG=github.com/cockroachdb/cockroach/pkg/sql/colexec TESTTIMEOUT=5m STRESSFLAGS='-maxtime 20m -timeout 10m' 2>&1 | tee /tmp/stress.log

Failed test: https://teamcity.cockroachdb.com/viewLog.html?buildId=1630224&tab=buildLog


panic: runtime error: slice bounds out of range [recovered]
	panic: runtime error: slice bounds out of range

goroutine 3510 [running]:
testing.tRunner.func1(0xc000680500)
	/usr/local/go/src/testing/testing.go:830 +0x392
panic(0x1f06000, 0x3bb3960)
	/usr/local/go/src/runtime/panic.go:522 +0x1b5
github.com/cockroachdb/cockroach/pkg/col/coldata.(*Bytes).Get(...)
	/go/src/github.com/cockroachdb/cockroach/pkg/col/coldata/bytes.go:96
github.com/cockroachdb/cockroach/pkg/col/coldata.(*memColumn).Append(0xc000e47440, 0x1, 0x27323c0, 0xc000e47280, 0xc0005e8000, 0x400, 0x400, 0x0, 0x1, 0x2)
	/go/src/github.com/cockroachdb/cockroach/pkg/col/coldata/vec.eg.go:56 +0x2230
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Allocator).Append.func1()
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/allocator.go:112 +0xa6
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Allocator).performOperation(0xc000359960, 0xc0006a7600, 0x1, 0x1, 0xc0006a7610)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/allocator.go:90 +0xfd
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Allocator).Append(0xc000359960, 0x27323c0, 0xc000e47440, 0x1, 0x27323c0, 0xc000e47280, 0xc0005e8000, 0x400, 0x400, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/allocator.go:112 +0xfe
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*mergeJoinBase).appendToBufferedGroup(0xc0004418c0, 0xc0004418e8, 0x271cd00, 0xc000e471c0, 0xc0005e8000, 0x400, 0x400, 0x1, 0x1)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/mergejoiner.go:421 +0x231
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*mergeJoinFullOuterOp).probeBodyLSeltrueRSeltrue(0xc0004418c0, 0x26ebfc0, 0xc000052108)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/mergejoiner_fullouter.eg.go:7985 +0x1a9c5
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*mergeJoinFullOuterOp).probe(0xc0004418c0, 0x26ebfc0, 0xc000052108)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/mergejoiner_fullouter.eg.go:39031 +0x1aa
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*mergeJoinFullOuterOp).Next(0xc0004418c0, 0x26ebfc0, 0xc000052108, 0x50, 0xc000a83101)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/mergejoiner_fullouter.eg.go:39223 +0x2f4
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*opTestOutput).next(0xc000e47000, 0x26ebfc0, 0xc000052108, 0xc000a36000, 0xc000a831a0, 0x271cd00)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/utils_test.go:768 +0xbb
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*opTestOutput).VerifyAnyOrder(0xc000e47000, 0x1000, 0x1000)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/utils_test.go:805 +0x78
github.com/cockroachdb/cockroach/pkg/sql/colexec.TestMergeJoiner.func1(0xc000e47000, 0xc000e47000, 0x2)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/mergejoiner_test.go:1634 +0xcd
github.com/cockroachdb/cockroach/pkg/sql/colexec.runTestsWithoutAllNullsInjection.func1(0xc000680500, 0xc000359940, 0x2, 0x2)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/utils_test.go:214 +0x174
github.com/cockroachdb/cockroach/pkg/sql/colexec.runTestsWithFn.func1(0xc000680500)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/utils_test.go:350 +0x298
testing.tRunner(0xc000680500, 0xc0000aaae0)
	/usr/local/go/src/testing/testing.go:865 +0xc0
created by testing.(*T).Run
	/usr/local/go/src/testing/testing.go:916 +0x35a

goroutine 1 [chan receive]:
testing.(*T).Run(0xc000680100, 0x216ded1, 0xf, 0x2221c30, 0x4c4701)
	/usr/local/go/src/testing/testing.go:917 +0x381
testing.runTests.func1(0xc0001e4600)
	/usr/local/go/src/testing/testing.go:1157 +0x78
testing.tRunner(0xc0001e4600, 0xc0006dfdc8)
	/usr/local/go/src/testing/testing.go:865 +0xc0
testing.runTests(0xc000277780, 0x3bd53c0, 0x5f, 0x5f, 0x0)
	/usr/local/go/src/testing/testing.go:1155 +0x2a9
testing.(*M).Run(0xc000471300, 0x0)
	/usr/local/go/src/testing/testing.go:1072 +0x162
github.com/cockroachdb/cockroach/pkg/sql/colexec.TestMain.func1(0xc000471300, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/main_test.go:46 +0x1b4
github.com/cockroachdb/cockroach/pkg/sql/colexec.TestMain(0xc000471300)

@cockroach-teamcity
Copy link
Member Author

SHA: https://github.com/cockroachdb/cockroach/commits/28f216e1bd53da872a759a98779144c7f70f33a3

Parameters:

TAGS=
GOFLAGS=-race -parallel=2

To repro, try:

# Don't forget to check out a clean suitable branch and experiment with the
# stress invocation until the desired results present themselves. For example,
# using stress instead of stressrace and passing the '-p' stressflag which
# controls concurrency.
./scripts/gceworker.sh start && ./scripts/gceworker.sh mosh
cd ~/go/src/github.com/cockroachdb/cockroach && \
stdbuf -oL -eL \
make stressrace TESTS=TestMergeJoiner PKG=github.com/cockroachdb/cockroach/pkg/sql/colexec TESTTIMEOUT=5m STRESSFLAGS='-maxtime 20m -timeout 10m' 2>&1 | tee /tmp/stress.log

Failed test: https://teamcity.cockroachdb.com/viewLog.html?buildId=1630225&tab=buildLog


panic: runtime error: slice bounds out of range [recovered]
	panic: runtime error: slice bounds out of range

goroutine 3524 [running]:
testing.tRunner.func1(0xc0004fc100)
	/usr/local/go/src/testing/testing.go:830 +0x69d
panic(0x2ca5ba0, 0x52cda40)
	/usr/local/go/src/runtime/panic.go:522 +0x1b5
github.com/cockroachdb/cockroach/pkg/col/coldata.(*Bytes).Get(...)
	/go/src/github.com/cockroachdb/cockroach/pkg/col/coldata/bytes.go:96
github.com/cockroachdb/cockroach/pkg/col/coldata.(*memColumn).Append(0xc00073bdc0, 0x1, 0x3b0cf40, 0xc00073bc00, 0xc000782000, 0x400, 0x400, 0x0, 0x1, 0x2)
	/go/src/github.com/cockroachdb/cockroach/pkg/col/coldata/vec.eg.go:56 +0x2e4d
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Allocator).Append.func1()
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/allocator.go:112 +0xc7
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Allocator).performOperation(0xc0002f1220, 0xc000563398, 0x1, 0x1, 0xc0005573a8)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/allocator.go:90 +0x161
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Allocator).Append(0xc0002f1220, 0x3b0cf40, 0xc00073bdc0, 0x1, 0x3b0cf40, 0xc00073bc00, 0xc000782000, 0x400, 0x400, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/allocator.go:112 +0x10f
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*mergeJoinBase).appendToBufferedGroup(0xc0004f62c0, 0xc0004f62e8, 0x3af3cc0, 0xc00073bb40, 0xc000782000, 0x400, 0x400, 0x1, 0x1)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/mergejoiner.go:421 +0x31d
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*mergeJoinFullOuterOp).probeBodyLSeltrueRSeltrue(0xc0004f62c0, 0x3abfaa0, 0xc0000e0010)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/mergejoiner_fullouter.eg.go:7985 +0x27213
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*mergeJoinFullOuterOp).probe(0xc0004f62c0, 0x3abfaa0, 0xc0000e0010)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/mergejoiner_fullouter.eg.go:39031 +0x352
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*mergeJoinFullOuterOp).Next(0xc0004f62c0, 0x3abfaa0, 0xc0000e0010, 0x0, 0xc000aa1d38)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/mergejoiner_fullouter.eg.go:39223 +0x776
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*opTestOutput).next(0xc00073b980, 0x3abfaa0, 0xc0000e0010, 0x0, 0x0, 0x801)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/utils_test.go:768 +0x19e
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*opTestOutput).VerifyAnyOrder(0xc00073b980, 0xc000aa1cf0, 0x2838563)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/utils_test.go:805 +0xba
github.com/cockroachdb/cockroach/pkg/sql/colexec.TestMergeJoiner.func1(0xc00073b980, 0x40, 0x2)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/mergejoiner_test.go:1634 +0x13b
github.com/cockroachdb/cockroach/pkg/sql/colexec.runTestsWithoutAllNullsInjection.func1(0xc0004fc100, 0xc0002f1200, 0x2, 0x2)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/utils_test.go:214 +0x218
github.com/cockroachdb/cockroach/pkg/sql/colexec.runTestsWithFn.func1(0xc0004fc100)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/utils_test.go:350 +0x414
testing.tRunner(0xc0004fc100, 0xc0005a3680)
	/usr/local/go/src/testing/testing.go:865 +0x164
created by testing.(*T).Run
	/usr/local/go/src/testing/testing.go:916 +0x65b

goroutine 1 [chan receive]:
testing.(*T).Run(0xc000219400, 0x2f0ec09, 0xf, 0x2fc2b70, 0x1)
	/usr/local/go/src/testing/testing.go:917 +0x693
testing.runTests.func1(0xc000219400)
	/usr/local/go/src/testing/testing.go:1157 +0xa9
testing.tRunner(0xc000219400, 0xc000719d10)
	/usr/local/go/src/testing/testing.go:865 +0x164
testing.runTests(0xc000290220, 0x52ef4a0, 0x5f, 0x5f, 0x0)
	/usr/local/go/src/testing/testing.go:1155 +0x524
testing.(*M).Run(0xc0001fe680, 0x0)
	/usr/local/go/src/testing/testing.go:1072 +0x2ec
github.com/cockroachdb/cockroach/pkg/sql/colexec.TestMain.func1(0xc0001fe680, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/main_test.go:46 +0x35a
github.com/cockroachdb/cockroach/pkg/sql/colexec.TestMain(0xc0001fe680)

@cockroach-teamcity
Copy link
Member Author

SHA: https://github.com/cockroachdb/cockroach/commits/9d697080f227324220c09d1b63da7bbf969133ac

Parameters:

TAGS=
GOFLAGS=-race -parallel=2

To repro, try:

# Don't forget to check out a clean suitable branch and experiment with the
# stress invocation until the desired results present themselves. For example,
# using stress instead of stressrace and passing the '-p' stressflag which
# controls concurrency.
./scripts/gceworker.sh start && ./scripts/gceworker.sh mosh
cd ~/go/src/github.com/cockroachdb/cockroach && \
stdbuf -oL -eL \
make stressrace TESTS=TestMergeJoiner PKG=github.com/cockroachdb/cockroach/pkg/sql/colexec TESTTIMEOUT=5m STRESSFLAGS='-maxtime 20m -timeout 10m' 2>&1 | tee /tmp/stress.log

Failed test: https://teamcity.cockroachdb.com/viewLog.html?buildId=1633809&tab=buildLog


panic: runtime error: slice bounds out of range [recovered]
	panic: runtime error: slice bounds out of range

goroutine 3499 [running]:
testing.tRunner.func1(0xc00021a400)
	/usr/local/go/src/testing/testing.go:830 +0x69d
panic(0x2caa160, 0x52e5a50)
	/usr/local/go/src/runtime/panic.go:522 +0x1b5
github.com/cockroachdb/cockroach/pkg/col/coldata.(*Bytes).Get(...)
	/go/src/github.com/cockroachdb/cockroach/pkg/col/coldata/bytes.go:96
github.com/cockroachdb/cockroach/pkg/col/coldata.(*memColumn).Append(0xc00071a9c0, 0x1, 0x3b231a0, 0xc00071a800, 0xc000376000, 0x400, 0x400, 0x0, 0x1, 0x2)
	/go/src/github.com/cockroachdb/cockroach/pkg/col/coldata/vec.eg.go:56 +0x2e4d
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Allocator).Append.func1()
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/allocator.go:112 +0xc7
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Allocator).performOperation(0xc00038a220, 0xc00059b398, 0x1, 0x1, 0xc0002773a8)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/allocator.go:90 +0x161
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*Allocator).Append(0xc00038a220, 0x3b231a0, 0xc00071a9c0, 0x1, 0x3b231a0, 0xc00071a800, 0xc000376000, 0x400, 0x400, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/allocator.go:112 +0x10f
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*mergeJoinBase).appendToBufferedGroup(0xc0002a7600, 0xc0002a7628, 0x3b0a0a0, 0xc00071a740, 0xc000376000, 0x400, 0x400, 0x1, 0x1)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/mergejoiner.go:421 +0x31d
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*mergeJoinFullOuterOp).probeBodyLSeltrueRSeltrue(0xc0002a7600, 0x3ad5d60, 0xc0000e0010)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/mergejoiner_fullouter.eg.go:7985 +0x27213
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*mergeJoinFullOuterOp).probe(0xc0002a7600, 0x3ad5d60, 0xc0000e0010)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/mergejoiner_fullouter.eg.go:39031 +0x352
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*mergeJoinFullOuterOp).Next(0xc0002a7600, 0x3ad5d60, 0xc0000e0010, 0x0, 0xc000279d38)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/mergejoiner_fullouter.eg.go:39223 +0x776
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*opTestOutput).next(0xc00071a580, 0x3ad5d60, 0xc0000e0010, 0x0, 0x0, 0x801)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/utils_test.go:768 +0x19e
github.com/cockroachdb/cockroach/pkg/sql/colexec.(*opTestOutput).VerifyAnyOrder(0xc00071a580, 0xc000279cf0, 0x283c9d3)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/utils_test.go:805 +0xba
github.com/cockroachdb/cockroach/pkg/sql/colexec.TestMergeJoiner.func1(0xc00071a580, 0x40, 0x2)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/mergejoiner_test.go:1634 +0x13b
github.com/cockroachdb/cockroach/pkg/sql/colexec.runTestsWithoutAllNullsInjection.func1(0xc00021a400, 0xc00038a200, 0x2, 0x2)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/utils_test.go:214 +0x218
github.com/cockroachdb/cockroach/pkg/sql/colexec.runTestsWithFn.func1(0xc00021a400)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/utils_test.go:350 +0x414
testing.tRunner(0xc00021a400, 0xc000688360)
	/usr/local/go/src/testing/testing.go:865 +0x164
created by testing.(*T).Run
	/usr/local/go/src/testing/testing.go:916 +0x65b

goroutine 1 [chan receive]:
testing.(*T).Run(0xc00021b500, 0x2f13495, 0xf, 0x2fc7518, 0x1)
	/usr/local/go/src/testing/testing.go:917 +0x693
testing.runTests.func1(0xc00021b500)
	/usr/local/go/src/testing/testing.go:1157 +0xa9
testing.tRunner(0xc00021b500, 0xc000735d10)
	/usr/local/go/src/testing/testing.go:865 +0x164
testing.runTests(0xc0000d8220, 0x5307de0, 0x5f, 0x5f, 0x0)
	/usr/local/go/src/testing/testing.go:1155 +0x524
testing.(*M).Run(0xc000464500, 0x0)
	/usr/local/go/src/testing/testing.go:1072 +0x2ec
github.com/cockroachdb/cockroach/pkg/sql/colexec.TestMain.func1(0xc000464500, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/colexec/main_test.go:46 +0x35a
github.com/cockroachdb/cockroach/pkg/sql/colexec.TestMain(0xc000464500)

@andreimatei
Copy link
Contributor

Happened to me too without stress.

@yuzefovich yuzefovich mentioned this issue Jan 3, 2020
Merged
@craig craig bot closed this as completed in f899d1c Jan 15, 2020
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees

@yuzefovich yuzefovich

Labels
C-test-failure Broken test (automatically or manually discovered). O-robot Originated from a bot.
Projects
None yet
Milestone
20.1
Development

Successfully merging a pull request may close this issue.

coldata: fix behavior of Vec.Append in some cases when NULLs are present yuzefovich/cockroach
4 participants
@andreimatei @yuzefovich @georgeutsin @cockroach-teamcity