Arvados

It looks like this can happen when there is an "unachievable" block (referenced by a collection, but not returned by any keepstore index).

fatal error: concurrent map read and map write

goroutine 33154 [running]:
runtime.throw({0x9ca24b, 0x0})
        /usr/local/go/src/runtime/panic.go:1198 +0x71 fp=0xc125a09768 sp=0xc125a09738 pc=0x437151
runtime.mapaccess1_faststr(0x927060, 0xc125a09870, {0xc106a34bd0, 0x29})
        /usr/local/go/src/runtime/map_faststr.go:21 +0x3a5 fp=0xc125a097d0 sp=0xc125a09768 pc=0x414245
main.(*BlockStateMap).get(...)
        /arvados/services/keep-balance/block_state.go:91
main.(*BlockStateMap).GetConfirmedReplication(0xc42ba0c330, {0xc86d7ee000, 0x9f8bc, 0x1010101}, {0xc0f75aeb00, 0x1, 0xc320947560})
        /arvados/services/keep-balance/block_state.go:154 +0x217 fp=0xc125a09b08 sp=0xc125a097d0 pc=0x8be617
main.(*Balancer).updateCollections.func2()
        /arvados/services/keep-balance/collection.go:187 +0x41a fp=0xc125a09fa8 sp=0xc125a09b08 pc=0x8c099a
main.goSendErr.func1()
        /arvados/services/keep-balance/collection.go:256 +0x26 fp=0xc125a09fe0 sp=0xc125a09fa8 pc=0x8c1866
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:1581 +0x1 fp=0xc125a09fe8 sp=0xc125a09fe0 pc=0x46a1c1
created by main.goSendErr
        /arvados/services/keep-balance/collection.go:255 +0x77

goroutine 33104 [runnable]:
main.(*BlockStateMap).GetConfirmedReplication(0xc42ba0c330, {0xc867b66000, 0x947a5, 0xc012cf1500}, {0xc0f0864880, 0x1, 0x1})
        /arvados/services/keep-balance/block_state.go:161 +0x49b
main.(*Balancer).updateCollections.func2()
        /arvados/services/keep-balance/collection.go:187 +0x41a
main.goSendErr.func1()
        /arvados/services/keep-balance/collection.go:256 +0x26
created by main.goSendErr
        /arvados/services/keep-balance/collection.go:255 +0x77

goroutine 33157 [runnable]:
main.(*BlockStateMap).GetConfirmedReplication(0xc42ba0c330, {0xc8614be000, 0x9e192, 0x1}, {0xc0e7fd1840, 0x1, 0x4146f9})
        /arvados/services/keep-balance/block_state.go:172 +0x2ec
main.(*Balancer).updateCollections.func2()
        /arvados/services/keep-balance/collection.go:187 +0x41a
main.goSendErr.func1()
        /arvados/services/keep-balance/collection.go:256 +0x26
created by main.goSendErr
        /arvados/services/keep-balance/collection.go:255 +0x77