sentry: conn_executor.go:640: panic while executing 1 statements: UPSERT INTO _(_, _) VALUES ($1, $2): caused by <redacted>

[tbg]

https://sentry.io/cockroach-labs/cockroachdb/issues/763899111/

conn_executor.go:640: panic while executing 1 statements: UPSERT INTO (, _) VALUES ($1, $2): caused by

github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn.func1

stacktrace: {u'frames': [{u'function': u'func4', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go', u'module': u'github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl', u'filename': u'github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go', u'lineno': 309, u'in_app': True}, {u'function': u'ServeConn', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go', u'module': u'github.com/cockroachdb/cockroach/pkg/sql.(*Server)', u'filename': u'github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go', u'lineno': 390, u'in_app': True}, {u'function': u'run', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go', u'module': u'github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor)', u'filename': u'github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go', u'lineno': 1239, u'in_app': True}, {u'function': u'txnStateTransitionsApplyWrapper', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go', u'module': u'github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor)', u'filename': u'github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go', u'lineno': 1884, u'in_app': True}, {u'function': u'ApplyWithPayload', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/util/fsm/fsm.go', u'module': u'github.com/cockroachdb/cockroach/pkg/util/fsm.(*Machine)', u'filename': u'github.com/cockroachdb/cockroach/pkg/util/fsm/fsm.go', u'lineno': 136, u'in_app': True}, {u'function': u'apply', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/util/fsm/fsm.go', u'module': u'github.com/cockroachdb/cockroach/pkg/util/fsm.Transitions', u'filename': u'github.com/cockroachdb/cockroach/pkg/util/fsm/fsm.go', u'lineno': 107, u'in_app': True}, {u'function': u'func1', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_fsm.go', u'module': u'github.com/cockroachdb/cockroach/pkg/sql.glob.', u'filename': u'github.com/cockroachdb/cockroach/pkg/sql/conn_fsm.go', u'lineno': 256, u'in_app': True}, {u'function': u'noTxnToOpen', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_fsm.go', u'module': u'github.com/cockroachdb/cockroach/pkg/sql.(*txnState)', u'filename': u'github.com/cockroachdb/cockroach/pkg/sql/conn_fsm.go', u'lineno': 544, u'in_app': True}, {u'function': u'resetForNewSQLTxn', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/sql/txn_state.go', u'module': u'github.com/cockroachdb/cockroach/pkg/sql.(*txnState)', u'filename': u'github.com/cockroachdb/cockroach/pkg/sql/txn_state.go', u'lineno': 208, u'in_app': True}, {u'function': u'NewTxn', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go', u'module': u'github.com/cockroachdb/cockroach/pkg/internal/client', u'filename': u'github.com/cockroachdb/cockroach/pkg/internal/client/txn.go', u'lineno': 103, u'in_app': True}, {u'function': u'MakeTransaction', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/data.go', u'module': u'github.com/cockroachdb/cockroach/pkg/roachpb', u'filename': u'github.com/cockroachdb/cockroach/pkg/roachpb/data.go', u'lineno': 736, u'in_app': True}, {u'function': u'FastMakeV4', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/util/uuid/uuid_wrapper.go', u'module': u'github.com/cockroachdb/cockroach/pkg/util/uuid', u'filename': u'github.com/cockroachdb/cockroach/pkg/util/uuid/uuid_wrapper.go', u'lineno': 102, u'in_app': True}, {u'function': u'NewV4', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/util/uuid/generator.go', u'module': u'github.com/cockroachdb/cockroach/pkg/util/uuid.(*Gen)', u'filename': u'github.com/cockroachdb/cockroach/pkg/util/uuid/generator.go', u'lineno': 198, u'in_app': True}, {u'function': u'ReadFull', u'abs_path': u'/usr/local/go/src/io/io.go', u'module': u'io', u'filename': u'io/io.go', u'lineno': 327, u'in_app': False}, {u'function': u'ReadAtLeast', u'abs_path': u'/usr/local/go/src/io/io.go', u'module': u'io', u'filename': u'io/io.go', u'lineno': 309, u'in_app': False}, {u'function': u'Read', u'abs_path': u'/usr/local/go/src/math/rand/rand.go', u'module': u'math/rand.(*Rand)', u'filename': u'math/rand/rand.go', u'lineno': 261, u'in_app': False}, {u'function': u'read', u'abs_path': u'/usr/local/go/src/math/rand/rand.go', u'module': u'math/rand', u'filename': u'math/rand/rand.go', u'lineno': 269, u'in_app': False}, {u'function': u'Int63-fm', u'abs_path': u'/usr/local/go/src/math/rand/rand.go', u'module': u'math/rand.(*Rand)', u'filename': u'math/rand/rand.go', u'lineno': 261, u'in_app': False}, {u'function': u'Int63', u'abs_path': u'/usr/local/go/src/math/rand/rand.go', u'module': u'math/rand.(*Rand)', u'filename': u'math/rand/rand.go', u'lineno': 82, u'in_app': False}, {u'function': u'Int63', u'abs_path': u'/usr/local/go/src/math/rand/rng.go', u'module': u'math/rand.(*rngSource)', u'filename': u'math/rand/rng.go', u'lineno': 231, u'in_app': False}, {u'function': u'Int63', u'abs_path': u'/usr/local/go/src/math/rand/rng.go', u'module': u'math/rand.(*rngSource)', u'filename': u'math/rand/rng.go', u'lineno': 246, u'in_app': False}, {u'function': u'panicindex', u'abs_path': u'/usr/local/go/src/runtime/panic.go', u'module': u'runtime', u'filename': u'runtime/panic.go', u'lineno': 28, u'in_app': False}, {u'function': u'gopanic', u'abs_path': u'/usr/local/go/src/runtime/panic.go', u'module': u'runtime', u'filename': u'runtime/panic.go', u'lineno': 502, u'in_app': False}, {u'function': u'call32', u'abs_path': u'/usr/local/go/src/runtime/asm_amd64.s', u'module': u'runtime', u'filename': u'runtime/asm_amd64.s', u'lineno': 573, u'in_app': False}, {u'function': u'func1', u'abs_path': u'/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go', u'module': u'github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn', u'filename': u'github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go', u'lineno': 388, u'in_app': True}]}
type: *log.safeError
value: conn_executor.go:640: panic while executing 1 statements: UPSERT INTO (, _) VALUES ($1, $2): caused by

[jordanlewis]

for triage

[nvanbenschoten]

This failure was on cyan:4. @jordanlewis anything you'd like me to pull?

[nvanbenschoten]

*log.safeError: conn_executor.go:640: panic while executing 1 statements: UPSERT INTO _(_, _) VALUES ($1, $2): caused by <redacted>
  File "github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go", line 388, in func1
  File "runtime/asm_amd64.s", line 573, in call32
  File "runtime/panic.go", line 502, in gopanic
  File "runtime/panic.go", line 28, in panicindex
  File "math/rand/rng.go", line 246, in Int63
  File "math/rand/rng.go", line 231, in Int63
  File "math/rand/rand.go", line 82, in Int63
  File "math/rand/rand.go", line 261, in Int63-fm
  File "math/rand/rand.go", line 269, in read
  File "math/rand/rand.go", line 261, in Read
  File "io/io.go", line 309, in ReadAtLeast
  File "io/io.go", line 327, in ReadFull
  File "github.com/cockroachdb/cockroach/pkg/util/uuid/generator.go", line 198, in NewV4
  File "github.com/cockroachdb/cockroach/pkg/util/uuid/uuid_wrapper.go", line 102, in FastMakeV4
  File "github.com/cockroachdb/cockroach/pkg/roachpb/data.go", line 736, in MakeTransaction
  File "github.com/cockroachdb/cockroach/pkg/internal/client/txn.go", line 103, in NewTxn
  File "github.com/cockroachdb/cockroach/pkg/sql/txn_state.go", line 208, in resetForNewSQLTxn
  File "github.com/cockroachdb/cockroach/pkg/sql/conn_fsm.go", line 544, in noTxnToOpen
  File "github.com/cockroachdb/cockroach/pkg/sql/conn_fsm.go", line 256, in func1
  File "github.com/cockroachdb/cockroach/pkg/util/fsm/fsm.go", line 107, in apply
  File "github.com/cockroachdb/cockroach/pkg/util/fsm/fsm.go", line 136, in ApplyWithPayload
  File "github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go", line 1884, in txnStateTransitionsApplyWrapper
  File "github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go", line 1239, in run
  File "github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go", line 390, in ServeConn
  File "github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go", line 309, in func4

@ajwerner, looks like this was likely caused by #32238. Mind investigating. It's possible that it was already resolved by #32356.

[ajwerner]

Indeed this was caused by #32238 and should be resolved by #32356. This is exactly the stack traces from golang/go#17629 and golang/go#3611 which occurred due to races on rand.Sources.