failed to start timer due to controller isn't ready

[wenfengwang]

Bug

[centos@ip-172-31-59-20 deploy]$ kubectl logs vanus-timer-7f76d7bcf4-z8j4f -n vanus -p
time="2022-10-28T15:15:33.221803197Z" level=debug msg="logger level has been set" log_level=DEBUG
time="2022-10-28T15:16:01.041176943Z" level=info msg="new leaderelection manager" key=/vanus/internal/resource/resourcelock/timer lease_duration=15 name=timer
time="2022-10-28T15:16:01.042643436Z" level=info msg="new timingwheel manager" ctrl_endpoints="[vanus-controller-0.vanus-controller.vanus.svc:2048 vanus-controller-1.vanus-controller.vanus.svc:2048 vanus-controller-2.vanus-controller.vanus.svc:2048]" etcd_endpoints="[vanus-controller-0.vanus-controller:2379 vanus-controller-1.vanus-controller:2379 vanus-controller-2.vanus-controller:2379]" key_prefix=/vanus layers=4 tick=1s wheel_size=32
time="2022-10-28T15:16:01.043000244Z" level=info msg="init timingwheel"
time="2022-10-28T15:16:01.043212457Z" level=info msg="start leaderelection"
time="2022-10-28T15:16:01.045269767Z" level=info msg="acquired lock" identity=timer lock=/vanus/internal/resource/resourcelock/timer
time="2022-10-28T15:16:01.045288296Z" level=info msg="leaderelection finish, become leader"
time="2022-10-28T15:16:01.047628941Z" level=info msg="start timingwheel" leader=true
time="2022-10-28T15:16:01.724331721Z" level=error msg="create eventbus failed" error="rpc error: code = Unknown desc = {\"code\":\"UNKNOWN\",\"message\":\"rpc error: code = Internal desc = runtime error: invalid memory address or nil pointer dereference\"}" eventbus=__Timer_DS
time="2022-10-28T15:16:01.724388349Z" level=error msg="start timer wheel failed" error="rpc error: code = Unknown desc = {\"code\":\"UNKNOWN\",\"message\":\"rpc error: code = Internal desc = runtime error: invalid memory address or nil pointer dereference\"}"
[centos@ip-172-31-59-20 deploy]$ kubectl logs vanus-controller-0 -n vanus | grep "panic"
time="2022-10-28T15:16:01.724032215Z" level=error msg="goroutine panicked" error="runtime error: invalid memory address or nil pointer dereference" stack="goroutine 8385 [running]:\nruntime/debug.Stack()\n\truntime/debug/stack.go:24 +0x65\nmain.main.func1({0x153ab80, 0xc001a92780}, {0x118de60, 0x1e7e860})\n\t./main.go:118 +0xca\ngithub.com/grpc-ecosystem/go-grpc-middleware/recovery.recoverFrom({0x153ab80?, 0xc001a92780?}, {0x118de60?, 0x1e7e860?}, 0xc0014d4f70?)\n\tgithub.com/grpc-ecosystem/[email protected]/recovery/interceptors.go:61 +0x36\ngithub.com/grpc-ecosystem/go-grpc-middleware/recovery.UnaryServerInterceptor.func1.1()\n\tgithub.com/grpc-ecosystem/[email protected]/recovery/interceptors.go:29 +0x7b\npanic({0x118de60, 0x1e7e860})\n\truntime/panic.go:838 +0x207\ngithub.com/linkall-labs/vanus/internal/controller/eventbus/eventlog.(*eventlogManager).AcquireEventLog(0x1e91020, {0x153ab80, 0xc001a92a20}, 0x3000010)\n\tgithub.com/linkall-labs/vanus/internal/controller/eventbus/eventlog/eventlog.go:182 +0x2e4\ngithub.com/linkall-labs/vanus/internal/controller/eventbus.(*controller).CreateEventBus(0xc0004c6c80, {0x153ab80, 0xc001a92a20}, 0xc001af5300)\n\tgithub.com/linkall-labs/vanus/internal/controller/eventbus/controller.go:149 +0x79d\ngithub.com/linkall-labs/vanus/proto/pkg/controller._EventBusController_CreateEventBus_Handler.func1({0x153ab80, 0xc001a92a20}, {0x1251ea0?, 0xc001af5300})\n\tgithub.com/linkall-labs/vanus/[email protected]/pkg/controller/controller.pb.go:2918 +0x78\ngo.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x153ab80, 0xc001a92780}, {0x1251ea0, 0xc001af5300}, 0xc00058e820, 0xc00176cc30)\n\tgo.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/[email protected]/interceptor.go:325 +0x664\ngoogle.golang.org/grpc.chainUnaryInterceptors.func1.1({0x153ab80?, 0xc001a92780?}, {0x1251ea0?, 0xc001af5300?})\n\tgoogle.golang.org/[email protected]/server.go:1135 +0x5b\ngithub.com/linkall-labs/vanus/internal/primitive/interceptor/memberinterceptor.UnaryServerInterceptor.func1({0x153ab80, 0xc001a92780}, {0x1251ea0, 0xc001af5300}, 0xc001ab48a0?, 0xc001af5340)\n\tgithub.com/linkall-labs/vanus/internal/primitive/interceptor/memberinterceptor/member.go:46 +0x102\ngoogle.golang.org/grpc.chainUnaryInterceptors.func1.1({0x153ab80?, 0xc001a92780?}, {0x1251ea0?, 0xc001af5300?})\n\tgoogle.golang.org/[email protected]/server.go:1138 +0x83\ngithub.com/grpc-ecosystem/go-grpc-middleware/recovery.UnaryServerInterceptor.func1({0x153ab80?, 0xc001a92780?}, {0x1251ea0?, 0xc001af5300?}, 0x203000?, 0x40?)\n\tgithub.com/grpc-ecosystem/[email protected]/recovery/interceptors.go:33 +0xc4\ngoogle.golang.org/grpc.chainUnaryInterceptors.func1.1({0x153ab80?, 0xc001a92780?}, {0x1251ea0?, 0xc001af5300?})\n\tgoogle.golang.org/[email protected]/server.go:1138 +0x83\ngithub.com/linkall-labs/vanus/internal/primitive/interceptor/errinterceptor.UnaryServerInterceptor.func1({0x153ab80?, 0xc001a92780?}, {0x1251ea0?, 0xc001af5300?}, 0x12767a0?, 0x7f899b917501?)\n\tgithub.com/linkall-labs/vanus/internal/primitive/interceptor/errinterceptor/error.go:41 +0x30\ngoogle.golang.org/grpc.chainUnaryInterceptors.func1.1({0x153ab80?, 0xc001a92780?}, {0x1251ea0?, 0xc001af5300?})\n\tgoogle.golang.org/[email protected]/server.go:1138 +0x83\ngoogle.golang.org/grpc.chainUnaryInterceptors.func1({0x153ab80, 0xc001a92780}, {0x1251ea0, 0xc001af5300}, 0xc00058e820, 0xc00176cc30)\n\tgoogle.golang.org/[email protected]/server.go:1140 +0x12b\ngithub.com/linkall-labs/vanus/proto/pkg/controller._EventBusController_CreateEventBus_Handler({0x12f5700?, 0xc0004c6c80}, {0x153ab80, 0xc001a92780}, 0xc0000b7110, 0xc00074ba00)\n\tgithub.com/linkall-labs/vanus/[email protected]/pkg/controller/controller.pb.go:2920 +0x138\ngoogle.golang.org/grpc.(*Server).processUnaryRPC(0xc0000014a0, {0x1543fb0, 0xc001ab6340}, 0xc001aeec60, 0xc001bd4900, 0x1e8f6e0, 0x0)\n\tgoogle.golang.org/[email protected]/server.go:1301 +0xb0b\ngoogle.golang.org/grpc.(*Server).handleStream(0xc0000014a0, {0x1543fb0, 0xc001ab6340}, 0xc001aeec60, 0x0)\n\tgoogle.golang.org/[email protected]/server.go:1642 +0xa1b\ngoogle.golang.org/grpc.(*Server).serveStreams.func1.2()\n\tgoogle.golang.org/[email protected]/server.go:938 +0x98\ncreated by google.golang.org/grpc.(*Server).serveStreams.func1\n\tgoogle.golang.org/[email protected]/server.go:936 +0x28a\n"