[2021/12/02 23:20:50.633 +00:00] [INFO] [printer.go:54] ["Welcome to dm-master"] ["Release Version"=v5.2.0-master] ["Git Commit Hash"=ee4a7f889449a557081a2529da5c9265c9d015fc] ["Git Branch"=master] ["UTC Build Time"="2021-12-02 23:16:44"] ["Go Version"="go version go1.16.10 linux/amd64"] [2021/12/02 23:20:50.633 +00:00] [INFO] [main.go:63] ["dm-master config"="{\"log-level\":\"info\",\"log-file\":\"/log/dm-master-1.log\",\"log-format\":\"text\",\"log-rotate\":\"\",\"rpc-timeout\":\"30s\",\"rpc-rate-limit\":10,\"rpc-rate-burst\":40,\"master-addr\":\"0.0.0.0:8261\",\"advertise-addr\":\"dm-master-1.dm-master.default:8261\",\"config-file\":\"\",\"name\":\"dm-master-1\",\"data-dir\":\"/data\",\"peer-urls\":\"http://:8291\",\"advertise-peer-urls\":\"http://dm-master-1.dm-master.default:8291\",\"initial-cluster\":\"dm-master-0=http://dm-master-0.dm-master.default:8291,dm-master-1=http://dm-master-1.dm-master.default:8291,dm-master-2=http://dm-master-2.dm-master.default:8291\",\"initial-cluster-state\":\"new\",\"join\":\"\",\"max-txn-ops\":2048,\"max-request-bytes\":1572864,\"auto-compaction-mode\":\"periodic\",\"auto-compaction-retention\":\"1h\",\"quota-backend-bytes\":2147483648,\"v1-sources-path\":\"\",\"ssl-ca\":\"\",\"ssl-cert\":\"\",\"ssl-key\":\"\",\"cert-allowed-cn\":null,\"ExperimentalFeatures\":{\"OpenAPI\":false}}"] [2021/12/02 23:20:50.633 +00:00] [INFO] [config.go:437] ["Set log level of etcd to `warn`, if you want to log more message about etcd, change log-level to `debug` in master configuration file"] [2021/12/02 23:20:50.633 +00:00] [INFO] [server.go:148] ["config after join prepared"] [config="{\"log-level\":\"info\",\"log-file\":\"/log/dm-master-1.log\",\"log-format\":\"text\",\"log-rotate\":\"\",\"rpc-timeout\":\"30s\",\"rpc-rate-limit\":10,\"rpc-rate-burst\":40,\"master-addr\":\"0.0.0.0:8261\",\"advertise-addr\":\"dm-master-1.dm-master.default:8261\",\"config-file\":\"\",\"name\":\"dm-master-1\",\"data-dir\":\"/data\",\"peer-urls\":\"http://:8291\",\"advertise-peer-urls\":\"http://dm-master-1.dm-master.default:8291\",\"initial-cluster\":\"dm-master-0=http://dm-master-0.dm-master.default:8291,dm-master-1=http://dm-master-1.dm-master.default:8291,dm-master-2=http://dm-master-2.dm-master.default:8291\",\"initial-cluster-state\":\"new\",\"join\":\"\",\"max-txn-ops\":2048,\"max-request-bytes\":1572864,\"auto-compaction-mode\":\"periodic\",\"auto-compaction-retention\":\"1h\",\"quota-backend-bytes\":2147483648,\"v1-sources-path\":\"\",\"ssl-ca\":\"\",\"ssl-cert\":\"\",\"ssl-key\":\"\",\"cert-allowed-cn\":null,\"ExperimentalFeatures\":{\"OpenAPI\":false}}"] [2021/12/02 23:20:50.673 +00:00] [WARN] [netutil.go:121] ["failed to resolve URL Host"] [component="embed etcd"] [url=http://dm-master-1.dm-master.default:8291] [host=dm-master-1.dm-master.default:8291] [retry-interval=1s] [error="lookup dm-master-1.dm-master.default on 10.43.0.10:53: no such host"] [2021/12/02 23:20:50.705 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-master-1.dm-master.default:8261 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-master-1.dm-master.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:20:51.678 +00:00] [WARN] [netutil.go:121] ["failed to resolve URL Host"] [component="embed etcd"] [url=http://dm-master-1.dm-master.default:8291] [host=dm-master-1.dm-master.default:8291] [retry-interval=1s] [error="lookup dm-master-1.dm-master.default on 10.43.0.10:53: no such host"] [2021/12/02 23:20:51.707 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-master-1.dm-master.default:8261 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-master-1.dm-master.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:20:52.680 +00:00] [WARN] [netutil.go:121] ["failed to resolve URL Host"] [component="embed etcd"] [url=http://dm-master-1.dm-master.default:8291] [host=dm-master-1.dm-master.default:8291] [retry-interval=1s] [error="lookup dm-master-1.dm-master.default on 10.43.0.10:53: no such host"] [2021/12/02 23:20:53.335 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-master-1.dm-master.default:8261 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-master-1.dm-master.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:20:53.683 +00:00] [WARN] [netutil.go:121] ["failed to resolve URL Host"] [component="embed etcd"] [url=http://dm-master-1.dm-master.default:8291] [host=dm-master-1.dm-master.default:8291] [retry-interval=1s] [error="lookup dm-master-1.dm-master.default on 10.43.0.10:53: no such host"] [2021/12/02 23:20:54.686 +00:00] [WARN] [netutil.go:121] ["failed to resolve URL Host"] [component="embed etcd"] [url=http://dm-master-1.dm-master.default:8291] [host=dm-master-1.dm-master.default:8291] [retry-interval=1s] [error="lookup dm-master-1.dm-master.default on 10.43.0.10:53: no such host"] [2021/12/02 23:20:55.528 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-master-1.dm-master.default:8261 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-master-1.dm-master.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:20:55.694 +00:00] [WARN] [store.go:1366] ["simple token is not cryptographically signed"] [component="embed etcd"] [2021/12/02 23:20:55.714 +00:00] [WARN] [stream.go:277] ["established TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream MsgApp v2"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id=c6ec7df2da09b327] [2021/12/02 23:20:55.714 +00:00] [WARN] [stream.go:277] ["established TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream Message"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id=c6ec7df2da09b327] [2021/12/02 23:20:56.796 +00:00] [INFO] [server.go:253] ["listening gRPC API and status request"] [address=0.0.0.0:8261] [2021/12/02 23:20:56.900 +00:00] [INFO] [election.go:301] ["get response from election observe"] [component=election] [key=/dm-master/leader/29c17d7d736a8d04] [value="{\"id\":\"dm-master-1\",\"addr\":\"dm-master-1.dm-master.default:8261\"}"] [2021/12/02 23:20:56.900 +00:00] [INFO] [election.go:332] ["become leader"] [component=election] ["current member"="{\"id\":\"dm-master-1\",\"addr\":\"dm-master-1.dm-master.default:8261\"}"] [2021/12/02 23:20:56.900 +00:00] [INFO] [election.go:57] ["current member become the leader"] ["current member"=dm-master-1] [2021/12/02 23:20:56.900 +00:00] [INFO] [bootstrap.go:73] ["bootstrap before scheduler start"] [2021/12/02 23:20:56.903 +00:00] [INFO] [upgrade.go:110] ["fetch previous version"] [preVer="{\"internal-no\":0,\"release-ver\":\"\"}"] [2021/12/02 23:20:56.908 +00:00] [INFO] [upgrade.go:253] ["no old KVs, skipping"] ["etcd path"=/dm-master/upstream/config/] [2021/12/02 23:20:56.913 +00:00] [INFO] [upgrade.go:253] ["no old KVs, skipping"] ["etcd path"=/dm-master/stage/relay/] [2021/12/02 23:20:56.913 +00:00] [INFO] [scheduler.go:200] ["the scheduler is starting"] [component=scheduler] [2021/12/02 23:20:56.944 +00:00] [INFO] [scheduler.go:278] ["the scheduler has started"] [component=scheduler] [2021/12/02 23:20:56.945 +00:00] [INFO] [pessimist.go:74] ["the shard DDL pessimist is starting"] [component="shard DDL pessimist"] [2021/12/02 23:20:56.948 +00:00] [INFO] [pessimist.go:139] ["get history shard DDL info"] [component="shard DDL pessimist"] [info={}] [revision=6] [2021/12/02 23:20:56.950 +00:00] [INFO] [pessimist.go:148] ["get history shard DDL lock operation"] [component="shard DDL pessimist"] [operation={}] [revision=6] [2021/12/02 23:20:56.950 +00:00] [INFO] [pessimist.go:95] ["the shard DDL pessimist has started"] [component="shard DDL pessimist"] [2021/12/02 23:20:56.950 +00:00] [INFO] [optimist.go:68] ["the shard DDL optimist is starting"] [component="shard DDL optimist"] [2021/12/02 23:20:56.952 +00:00] [INFO] [optimist.go:235] ["get history initial source tables"] [component="shard DDL optimist"] [revision=6] [2021/12/02 23:20:56.953 +00:00] [INFO] [optimist.go:243] ["get history shard DDL info"] [component="shard DDL optimist"] [revision=6] [2021/12/02 23:20:56.954 +00:00] [INFO] [optimist.go:251] ["get history shard DDL lock operation"] [component="shard DDL optimist"] [revision=6] [2021/12/02 23:20:56.955 +00:00] [INFO] [optimist.go:257] ["get all init schemas"] [component="shard DDL optimist"] [revision=6] [2021/12/02 23:20:56.956 +00:00] [INFO] [optimist.go:383] ["build lock joined and tts"] [component="shard DDL optimist"] [2021/12/02 23:20:56.956 +00:00] [INFO] [optimist.go:386] ["rebuild locks and tables"] [component="shard DDL optimist"] [2021/12/02 23:20:56.956 +00:00] [INFO] [optimist.go:92] ["the shard DDL optimist has started"] [component="shard DDL optimist"] [2021/12/02 23:20:56.956 +00:00] [INFO] [bootstrap.go:52] ["start bootstrapping"] [2021/12/02 23:20:56.958 +00:00] [INFO] [upgrade.go:68] ["fetch previous version"] [preVer="{\"internal-no\":0,\"release-ver\":\"min-ver\"}"] [2021/12/02 23:20:56.959 +00:00] [INFO] [upgrade.go:101] ["upgrade cluster version"] [version="{\"internal-no\":4,\"release-ver\":\"v5.2.0-master\"}"] [] [2021/12/02 23:20:56.959 +00:00] [INFO] [bootstrap.go:68] ["successful bootstrapped"] [2021/12/02 23:20:58.247 +00:00] [WARN] [stream.go:277] ["established TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream MsgApp v2"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id=94ddedb20f20c0c7] [2021/12/02 23:20:58.247 +00:00] [WARN] [stream.go:277] ["established TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream Message"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id=94ddedb20f20c0c7] [2021/12/02 23:21:04.748 +00:00] [INFO] [server.go:2389] [payload="name:\"dm-worker-0\" address:\"dm-worker-0.dm-worker.default:8262\" "] [request=RegisterWorker] [2021/12/02 23:21:04.750 +00:00] [INFO] [server.go:318] ["register worker successfully"] [name=dm-worker-0] [address=dm-worker-0.dm-worker.default:8262] [2021/12/02 23:21:04.764 +00:00] [INFO] [keepalive.go:209] ["receive dm-worker keep alive event"] [operation=PUT] [kv=/dm-worker/a/646d2d776f726b65722d30] [2021/12/02 23:21:04.764 +00:00] [INFO] [scheduler.go:1744] ["receive worker status change event"] [component=scheduler] [delete=false] [event="{\"worker-name\":\"dm-worker-0\",\"join-time\":\"2021-12-02T23:21:04.754945494Z\"}"] [2021/12/02 23:21:04.764 +00:00] [INFO] [scheduler.go:1989] ["no unbound sources need to bound"] [component=scheduler] [worker="{\"name\":\"dm-worker-0\",\"addr\":\"dm-worker-0.dm-worker.default:8262\"}"] [2021/12/02 23:21:04.835 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-worker-0.dm-worker.default:8262 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-worker-0.dm-worker.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:21:04.940 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-worker-0.dm-worker.default:8262 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-worker-0.dm-worker.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:21:05.114 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-worker-0.dm-worker.default:8262 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-worker-0.dm-worker.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:21:05.366 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-worker-0.dm-worker.default:8262 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-worker-0.dm-worker.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:21:05.771 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-worker-0.dm-worker.default:8262 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-worker-0.dm-worker.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:21:06.386 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-worker-0.dm-worker.default:8262 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-worker-0.dm-worker.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:21:06.756 +00:00] [INFO] [server.go:2389] [payload="name:\"dm-worker-1\" address:\"dm-worker-1.dm-worker.default:8262\" "] [request=RegisterWorker] [2021/12/02 23:21:06.758 +00:00] [INFO] [server.go:318] ["register worker successfully"] [name=dm-worker-1] [address=dm-worker-1.dm-worker.default:8262] [2021/12/02 23:21:06.769 +00:00] [INFO] [keepalive.go:209] ["receive dm-worker keep alive event"] [operation=PUT] [kv=/dm-worker/a/646d2d776f726b65722d31] [2021/12/02 23:21:06.769 +00:00] [INFO] [scheduler.go:1744] ["receive worker status change event"] [component=scheduler] [delete=false] [event="{\"worker-name\":\"dm-worker-1\",\"join-time\":\"2021-12-02T23:21:06.763621545Z\"}"] [2021/12/02 23:21:06.769 +00:00] [INFO] [scheduler.go:1989] ["no unbound sources need to bound"] [component=scheduler] [worker="{\"name\":\"dm-worker-1\",\"addr\":\"dm-worker-1.dm-worker.default:8262\"}"] [2021/12/02 23:21:06.910 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-worker-1.dm-worker.default:8262 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-worker-1.dm-worker.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:21:07.012 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-worker-1.dm-worker.default:8262 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-worker-1.dm-worker.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:21:07.173 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-worker-1.dm-worker.default:8262 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-worker-1.dm-worker.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:21:07.446 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-worker-0.dm-worker.default:8262 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-worker-0.dm-worker.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:21:07.451 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-worker-1.dm-worker.default:8262 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-worker-1.dm-worker.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:21:07.933 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-worker-1.dm-worker.default:8262 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-worker-1.dm-worker.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:21:08.480 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-worker-1.dm-worker.default:8262 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-worker-1.dm-worker.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:21:08.869 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-worker-0.dm-worker.default:8262 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-worker-0.dm-worker.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:21:09.632 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-worker-1.dm-worker.default:8262 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-worker-1.dm-worker.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:21:11.181 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-worker-1.dm-worker.default:8262 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-worker-1.dm-worker.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:21:13.497 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-worker-1.dm-worker.default:8262 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-worker-1.dm-worker.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:21:13.823 +00:00] [INFO] [server.go:2389] [payload="name:\"dm-worker-2\" address:\"dm-worker-2.dm-worker.default:8262\" "] [request=RegisterWorker] [2021/12/02 23:21:13.825 +00:00] [INFO] [server.go:318] ["register worker successfully"] [name=dm-worker-2] [address=dm-worker-2.dm-worker.default:8262] [2021/12/02 23:21:13.853 +00:00] [INFO] [keepalive.go:209] ["receive dm-worker keep alive event"] [operation=PUT] [kv=/dm-worker/a/646d2d776f726b65722d32] [2021/12/02 23:21:13.853 +00:00] [INFO] [scheduler.go:1744] ["receive worker status change event"] [component=scheduler] [delete=false] [event="{\"worker-name\":\"dm-worker-2\",\"join-time\":\"2021-12-02T23:21:13.836557798Z\"}"] [2021/12/02 23:21:13.853 +00:00] [INFO] [scheduler.go:1989] ["no unbound sources need to bound"] [component=scheduler] [worker="{\"name\":\"dm-worker-2\",\"addr\":\"dm-worker-2.dm-worker.default:8262\"}"] [2021/12/02 23:21:14.023 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-worker-2.dm-worker.default:8262 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-worker-2.dm-worker.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:21:14.125 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-worker-2.dm-worker.default:8262 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-worker-2.dm-worker.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:21:14.316 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-worker-2.dm-worker.default:8262 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-worker-2.dm-worker.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:21:14.594 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-worker-2.dm-worker.default:8262 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-worker-2.dm-worker.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:21:15.054 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-worker-2.dm-worker.default:8262 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-worker-2.dm-worker.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:21:15.216 +00:00] [INFO] [server.go:2389] [payload="name:\"dm-worker-3\" address:\"dm-worker-3.dm-worker.default:8262\" "] [request=RegisterWorker] [2021/12/02 23:21:15.220 +00:00] [INFO] [server.go:318] ["register worker successfully"] [name=dm-worker-3] [address=dm-worker-3.dm-worker.default:8262] [2021/12/02 23:21:15.230 +00:00] [INFO] [keepalive.go:209] ["receive dm-worker keep alive event"] [operation=PUT] [kv=/dm-worker/a/646d2d776f726b65722d33] [2021/12/02 23:21:15.230 +00:00] [INFO] [scheduler.go:1744] ["receive worker status change event"] [component=scheduler] [delete=false] [event="{\"worker-name\":\"dm-worker-3\",\"join-time\":\"2021-12-02T23:21:15.224790107Z\"}"] [2021/12/02 23:21:15.230 +00:00] [INFO] [scheduler.go:1989] ["no unbound sources need to bound"] [component=scheduler] [worker="{\"name\":\"dm-worker-3\",\"addr\":\"dm-worker-3.dm-worker.default:8262\"}"] [2021/12/02 23:21:15.323 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-worker-3.dm-worker.default:8262 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-worker-3.dm-worker.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:21:15.427 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-worker-3.dm-worker.default:8262 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-worker-3.dm-worker.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:21:15.617 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-worker-3.dm-worker.default:8262 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-worker-3.dm-worker.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:21:15.647 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-worker-2.dm-worker.default:8262 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-worker-2.dm-worker.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:21:15.904 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-worker-3.dm-worker.default:8262 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-worker-3.dm-worker.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:21:16.262 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-worker-3.dm-worker.default:8262 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-worker-3.dm-worker.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:21:16.399 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-worker-1.dm-worker.default:8262 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-worker-1.dm-worker.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:21:16.785 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-worker-2.dm-worker.default:8262 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-worker-2.dm-worker.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:21:17.022 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-worker-3.dm-worker.default:8262 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-worker-3.dm-worker.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:21:18.280 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-worker-3.dm-worker.default:8262 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-worker-3.dm-worker.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:21:18.406 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-worker-2.dm-worker.default:8262 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-worker-2.dm-worker.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:21:20.174 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-worker-3.dm-worker.default:8262 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-worker-3.dm-worker.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:21:22.740 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-worker-3.dm-worker.default:8262 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-worker-3.dm-worker.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:21:25.669 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-worker-3.dm-worker.default:8262 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-worker-3.dm-worker.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:21:36.243 +00:00] [INFO] [server.go:2389] [payload=] [request=ListMember] [2021/12/02 23:21:46.253 +00:00] [INFO] [server.go:2389] [payload=] [request=ListMember] [2021/12/02 23:21:56.262 +00:00] [INFO] [server.go:2389] [payload=] [request=ListMember] [2021/12/02 23:22:06.270 +00:00] [INFO] [server.go:2389] [payload=] [request=ListMember] [2021/12/02 23:22:16.282 +00:00] [INFO] [server.go:2389] [payload=] [request=ListMember] [2021/12/02 23:22:16.293 +00:00] [INFO] [server.go:2389] [payload="{HidePasswordObject=\"op:StartSource config:\\\"enable-gtid: true\\\\nauto-fix-gtid: false\\\\nrelay-dir: \\\\\\\"\\\\\\\"\\\\nmeta-dir: \\\\\\\"\\\\\\\"\\\\nflavor: \\\\\\\"\\\\\\\"\\\\ncharset: \\\\\\\"\\\\\\\"\\\\nenable-relay: true\\\\nrelay-binlog-name: \\\\\\\"\\\\\\\"\\\\nrelay-binlog-gtid: \\\\\\\"\\\\\\\"\\\\nsource-id: replica-01\\\\nfrom:\\\\n host: mysql57-0.sources\\\\n port: 3306\\\\n user: root\\\\n password: \\\\\\\"******\\\\\\\"\\\\n max-allowed-packet: null\\\\n session:\\\\n explicit_defaults_for_timestamp: \\\\\\\"on\\\\\\\"\\\\n sql_mode: \\\\\\\"\\\\\\\"\\\\n security: null\\\\npurge:\\\\n interval: 3600\\\\n expires: 0\\\\n remain-space: 15\\\\nchecker:\\\\n check-enable: true\\\\n backoff-rollback: 5m0s\\\\n backoff-max: 5s\\\\n check-interval: 5s\\\\n backoff-min: 1s\\\\n backoff-jitter: true\\\\n backoff-factor: 2\\\\nserver-id: 0\\\\ntracer: {}\\\\ncase-sensitive: false\\\\nfilters: []\\\\n\\\" config:\\\"enable-gtid: false\\\\nauto-fix-gtid: false\\\\nrelay-dir: \\\\\\\"\\\\\\\"\\\\nmeta-dir: \\\\\\\"\\\\\\\"\\\\nflavor: \\\\\\\"\\\\\\\"\\\\ncharset: \\\\\\\"\\\\\\\"\\\\nenable-relay: false\\\\nrelay-binlog-name: \\\\\\\"\\\\\\\"\\\\nrelay-binlog-gtid: \\\\\\\"\\\\\\\"\\\\nsource-id: replica-02\\\\nfrom:\\\\n host: mysql8-0.sources\\\\n port: 3306\\\\n user: root\\\\n password: \\\\\\\"******\\\\\\\"\\\\n max-allowed-packet: null\\\\n session:\\\\n explicit_defaults_for_timestamp: \\\\\\\"on\\\\\\\"\\\\n sql_mode: \\\\\\\"\\\\\\\"\\\\n security: null\\\\npurge:\\\\n interval: 3600\\\\n expires: 0\\\\n remain-space: 15\\\\nchecker:\\\\n check-enable: true\\\\n backoff-rollback: 5m0s\\\\n backoff-max: 5s\\\\n check-interval: 5s\\\\n backoff-min: 1s\\\\n backoff-jitter: true\\\\n backoff-factor: 2\\\\nserver-id: 0\\\\ntracer: {}\\\\ncase-sensitive: false\\\\nfilters: []\\\\n\\\" config:\\\"enable-gtid: true\\\\nauto-fix-gtid: false\\\\nrelay-dir: \\\\\\\"\\\\\\\"\\\\nmeta-dir: \\\\\\\"\\\\\\\"\\\\nflavor: \\\\\\\"\\\\\\\"\\\\ncharset: \\\\\\\"\\\\\\\"\\\\nenable-relay: true\\\\nrelay-binlog-name: \\\\\\\"\\\\\\\"\\\\nrelay-binlog-gtid: \\\\\\\"\\\\\\\"\\\\nsource-id: replica-03\\\\nfrom:\\\\n host: mariadb-0.sources\\\\n port: 3306\\\\n user: root\\\\n password: \\\\\\\"******\\\\\\\"\\\\n max-allowed-packet: null\\\\n session:\\\\n sql_mode: \\\\\\\"\\\\\\\"\\\\n security: null\\\\npurge:\\\\n interval: 3600\\\\n expires: 0\\\\n remain-space: 15\\\\nchecker:\\\\n check-enable: true\\\\n backoff-rollback: 5m0s\\\\n backoff-max: 5s\\\\n check-interval: 5s\\\\n backoff-min: 1s\\\\n backoff-jitter: true\\\\n backoff-factor: 2\\\\nserver-id: 0\\\\ntracer: {}\\\\ncase-sensitive: false\\\\nfilters: []\\\\n\\\" \"}"] [request=OperateSource] [2021/12/02 23:22:16.318 +00:00] [INFO] [scheduler.go:2086] ["found free worker when source bound"] [component=scheduler] [worker=dm-worker-1] [source=replica-01] [2021/12/02 23:22:16.320 +00:00] [INFO] [scheduler.go:2130] ["bound the source to worker"] [component=scheduler] [bound="{\"source\":\"replica-01\",\"worker\":\"dm-worker-1\"}"] [2021/12/02 23:22:16.322 +00:00] [INFO] [scheduler.go:2086] ["found free worker when source bound"] [component=scheduler] [worker=dm-worker-0] [source=replica-02] [2021/12/02 23:22:16.325 +00:00] [INFO] [scheduler.go:2130] ["bound the source to worker"] [component=scheduler] [bound="{\"source\":\"replica-02\",\"worker\":\"dm-worker-0\"}"] [2021/12/02 23:22:16.327 +00:00] [INFO] [scheduler.go:2086] ["found free worker when source bound"] [component=scheduler] [worker=dm-worker-2] [source=replica-03] [2021/12/02 23:22:16.332 +00:00] [INFO] [scheduler.go:2130] ["bound the source to worker"] [component=scheduler] [bound="{\"source\":\"replica-03\",\"worker\":\"dm-worker-2\"}"] [2021/12/02 23:22:16.521 +00:00] [INFO] [server.go:2389] [payload="op:Stop name:\"task_single\" "] [request=OperateTask] [2021/12/02 23:22:16.534 +00:00] [INFO] [server.go:2389] [payload="op:Stop name:\"task_pessimistic\" "] [request=OperateTask] [2021/12/02 23:22:16.536 +00:00] [INFO] [server.go:2389] [payload="op:Stop name:\"task_optimistic\" "] [request=OperateTask] [2021/12/02 23:22:17.308 +00:00] [INFO] [server.go:2389] [payload="{HidePasswordObject=\"task:\\\"name: task_single\\\\ntask-mode: all\\\\nis-sharding: false\\\\nshard-mode: \\\\\\\"\\\\\\\"\\\\nignore-checking-items: []\\\\nmeta-schema: \\\\\\\"\\\\\\\"\\\\nenable-heartbeat: false\\\\nheartbeat-update-interval: 0\\\\nheartbeat-report-interval: 0\\\\ntimezone: \\\\\\\"\\\\\\\"\\\\ncase-sensitive: false\\\\ntarget-database:\\\\n host: tidb-0.tidb\\\\n port: 4000\\\\n user: root\\\\n password: \\\\\\\"******\\\\\\\"\\\\n max-allowed-packet: null\\\\n session: {}\\\\n security: null\\\\nmysql-instances:\\\\n- source-id: replica-01\\\\n meta: null\\\\n filter-rules: []\\\\n column-mapping-rules: []\\\\n route-rules: []\\\\n expression-filters: []\\\\n black-white-list: instance\\\\n block-allow-list: instance\\\\n mydumper-config-name: \\\\\\\"\\\\\\\"\\\\n mydumper:\\\\n mydumper-path: ./bin/mydumper\\\\n threads: 4\\\\n chunk-filesize: \\\\\\\"64\\\\\\\"\\\\n statement-size: 0\\\\n rows: 0\\\\n where: \\\\\\\"\\\\\\\"\\\\n skip-tz-utc: true\\\\n extra-args: \\\\\\\"\\\\\\\"\\\\n mydumper-thread: 4\\\\n loader-config-name: \\\\\\\"\\\\\\\"\\\\n loader:\\\\n pool-size: 16\\\\n dir: ./dumped_data\\\\n loader-thread: 16\\\\n syncer-config-name: global\\\\n syncer: null\\\\n syncer-thread: 0\\\\nonline-ddl: false\\\\nshadow-table-rules: []\\\\ntrash-table-rules: []\\\\nonline-ddl-scheme: \\\\\\\"\\\\\\\"\\\\nroutes: {}\\\\nfilters: {}\\\\ncolumn-mappings: {}\\\\nexpression-filter: {}\\\\nblack-white-list:\\\\n instance:\\\\n do-tables: []\\\\n do-dbs:\\\\n - db_single\\\\n ignore-tables: []\\\\n ignore-dbs: []\\\\nblock-allow-list:\\\\n instance:\\\\n do-tables: []\\\\n do-dbs:\\\\n - db_single\\\\n ignore-tables: []\\\\n ignore-dbs: []\\\\nmydumpers: {}\\\\nloaders: {}\\\\nsyncers:\\\\n global:\\\\n meta-file: \\\\\\\"\\\\\\\"\\\\n worker-count: 16\\\\n batch: 100\\\\n queue-size: 1024\\\\n checkpoint-flush-interval: 30\\\\n compact: true\\\\n multiple-rows: true\\\\n max-retry: 0\\\\n auto-fix-gtid: false\\\\n enable-gtid: false\\\\n disable-detect: false\\\\n safe-mode: false\\\\n enable-ansi-quotes: false\\\\nclean-dump-file: false\\\\nansi-quotes: false\\\\nremove-meta: false\\\\ntidb: null\\\\n\\\" \"}"] [request=StartTask] [2021/12/02 23:22:17.309 +00:00] [WARN] [task.go:709] ["heartbeat is deprecated, needn't set it anymore."] [2021/12/02 23:22:17.409 +00:00] [INFO] [checker.go:236] ["\n************ task task_single checking items ************\nmysql_version\nmysql_server_id\nmysql_binlog_enable\nmysql_binlog_format\nmysql_binlog_row_image\nsource db dump privilege checker\nsource db replication privilege checker\ntable structure compatibility check\n************ task task_single checking items ************"] [unit="task check"] [2021/12/02 23:22:17.483 +00:00] [INFO] [server.go:434] ["task name"=task_single] [task="{\"name\":\"task_single\",\"task-mode\":\"all\",\"is-sharding\":false,\"shard-mode\":\"\",\"ignore-checking-items\":[],\"meta-schema\":\"\",\"enable-heartbeat\":false,\"heartbeat-update-interval\":0,\"heartbeat-report-interval\":0,\"timezone\":\"\",\"case-sensitive\":false,\"target-database\":{\"host\":\"tidb-0.tidb\",\"port\":4000,\"user\":\"root\",\"max-allowed-packet\":null,\"session\":{\"tidb_txn_mode\":\"optimistic\"},\"security\":null},\"mysql-instances\":[{\"SourceID\":\"replica-01\",\"Meta\":null,\"FilterRules\":[],\"ColumnMappingRules\":[],\"RouteRules\":[],\"ExpressionFilters\":[],\"BWListName\":\"instance\",\"BAListName\":\"instance\",\"MydumperConfigName\":\"\",\"Mydumper\":{\"mydumper-path\":\"./bin/mydumper\",\"threads\":4,\"chunk-filesize\":\"64\",\"statement-size\":0,\"rows\":0,\"where\":\"\",\"skip-tz-utc\":true,\"extra-args\":\"\"},\"MydumperThread\":4,\"LoaderConfigName\":\"\",\"Loader\":{\"pool-size\":16,\"dir\":\"./dumped_data\"},\"LoaderThread\":16,\"SyncerConfigName\":\"global\",\"Syncer\":{\"meta-file\":\"\",\"worker-count\":16,\"batch\":100,\"queue-size\":1024,\"checkpoint-flush-interval\":30,\"compact\":true,\"multiple-rows\":true,\"max-retry\":0,\"auto-fix-gtid\":false,\"enable-gtid\":false,\"disable-detect\":false,\"safe-mode\":false,\"enable-ansi-quotes\":false},\"SyncerThread\":0}],\"online-ddl\":false,\"shadow-table-rules\":[],\"trash-table-rules\":[],\"online-ddl-scheme\":\"\",\"routes\":{},\"filters\":{},\"column-mappings\":{},\"expression-filter\":{},\"black-white-list\":{\"instance\":{\"do-tables\":[],\"do-dbs\":[\"db_single\"],\"ignore-tables\":[],\"ignore-dbs\":[]}},\"block-allow-list\":{\"instance\":{\"do-tables\":[],\"do-dbs\":[\"db_single\"],\"ignore-tables\":[],\"ignore-dbs\":[]}},\"mydumpers\":{},\"loaders\":{},\"syncers\":{\"global\":{\"meta-file\":\"\",\"worker-count\":16,\"batch\":100,\"queue-size\":1024,\"checkpoint-flush-interval\":30,\"compact\":true,\"multiple-rows\":true,\"max-retry\":0,\"auto-fix-gtid\":false,\"enable-gtid\":false,\"disable-detect\":false,\"safe-mode\":false,\"enable-ansi-quotes\":false}},\"clean-dump-file\":false,\"ansi-quotes\":false,\"RemoveMeta\":false,\"tidb\":null}"] [request=StartTask] [2021/12/02 23:22:17.492 +00:00] [INFO] [server.go:1750] ["fail to get expect operation result"] [retryNum=0] [task=task_single] [source=replica-01] [expect=Running] [resp="result:true msg:\"no sub task started\" sourceStatus: > "] [2021/12/02 23:22:18.397 +00:00] [INFO] [scheduler.go:2392] ["receive load task"] [component=scheduler] [delete=false] [task=task_single] [source=replica-01] [worker=dm-worker-1] [2021/12/02 23:22:18.559 +00:00] [INFO] [server.go:2389] [payload="{HidePasswordObject=\"task:\\\"name: task_optimistic\\\\ntask-mode: all\\\\nis-sharding: false\\\\nshard-mode: optimistic\\\\nignore-checking-items:\\\\n- schema_of_shard_tables\\\\n- auto_increment_ID\\\\nmeta-schema: \\\\\\\"\\\\\\\"\\\\nenable-heartbeat: false\\\\nheartbeat-update-interval: 0\\\\nheartbeat-report-interval: 0\\\\ntimezone: \\\\\\\"\\\\\\\"\\\\ncase-sensitive: false\\\\ntarget-database:\\\\n host: tidb-0.tidb\\\\n port: 4000\\\\n user: root\\\\n password: \\\\\\\"******\\\\\\\"\\\\n max-allowed-packet: null\\\\n session: {}\\\\n security: null\\\\nmysql-instances:\\\\n- source-id: replica-01\\\\n meta: null\\\\n filter-rules: []\\\\n column-mapping-rules: []\\\\n route-rules: []\\\\n expression-filters: []\\\\n black-white-list: instance\\\\n block-allow-list: instance\\\\n mydumper-config-name: \\\\\\\"\\\\\\\"\\\\n mydumper:\\\\n mydumper-path: ./bin/mydumper\\\\n threads: 4\\\\n chunk-filesize: \\\\\\\"64\\\\\\\"\\\\n statement-size: 0\\\\n rows: 0\\\\n where: \\\\\\\"\\\\\\\"\\\\n skip-tz-utc: true\\\\n extra-args: \\\\\\\"\\\\\\\"\\\\n mydumper-thread: 4\\\\n loader-config-name: \\\\\\\"\\\\\\\"\\\\n loader:\\\\n pool-size: 16\\\\n dir: ./dumped_data\\\\n loader-thread: 16\\\\n syncer-config-name: global\\\\n syncer: null\\\\n syncer-thread: 0\\\\n- source-id: replica-02\\\\n meta: null\\\\n filter-rules: []\\\\n column-mapping-rules: []\\\\n route-rules: []\\\\n expression-filters: []\\\\n black-white-list: instance\\\\n block-allow-list: instance\\\\n mydumper-config-name: \\\\\\\"\\\\\\\"\\\\n mydumper:\\\\n mydumper-path: ./bin/mydumper\\\\n threads: 4\\\\n chunk-filesize: \\\\\\\"64\\\\\\\"\\\\n statement-size: 0\\\\n rows: 0\\\\n where: \\\\\\\"\\\\\\\"\\\\n skip-tz-utc: true\\\\n extra-args: \\\\\\\"\\\\\\\"\\\\n mydumper-thread: 4\\\\n loader-config-name: \\\\\\\"\\\\\\\"\\\\n loader:\\\\n pool-size: 16\\\\n dir: ./dumped_data\\\\n loader-thread: 16\\\\n syncer-config-name: global\\\\n syncer: null\\\\n syncer-thread: 0\\\\n- source-id: replica-03\\\\n meta: null\\\\n filter-rules: []\\\\n column-mapping-rules: []\\\\n route-rules: []\\\\n expression-filters: []\\\\n black-white-list: instance\\\\n block-allow-list: instance\\\\n mydumper-config-name: \\\\\\\"\\\\\\\"\\\\n mydumper:\\\\n mydumper-path: ./bin/mydumper\\\\n threads: 4\\\\n chunk-filesize: \\\\\\\"64\\\\\\\"\\\\n statement-size: 0\\\\n rows: 0\\\\n where: \\\\\\\"\\\\\\\"\\\\n skip-tz-utc: true\\\\n extra-args: \\\\\\\"\\\\\\\"\\\\n mydumper-thread: 4\\\\n loader-config-name: \\\\\\\"\\\\\\\"\\\\n loader:\\\\n pool-size: 16\\\\n dir: ./dumped_data\\\\n loader-thread: 16\\\\n syncer-config-name: global\\\\n syncer: null\\\\n syncer-thread: 0\\\\nonline-ddl: false\\\\nshadow-table-rules: []\\\\ntrash-table-rules: []\\\\nonline-ddl-scheme: \\\\\\\"\\\\\\\"\\\\nroutes: {}\\\\nfilters: {}\\\\ncolumn-mappings: {}\\\\nexpression-filter: {}\\\\nblack-white-list:\\\\n instance:\\\\n do-tables: []\\\\n do-dbs:\\\\n - db_optimistic\\\\n ignore-tables: []\\\\n ignore-dbs: []\\\\nblock-allow-list:\\\\n instance:\\\\n do-tables: []\\\\n do-dbs:\\\\n - db_optimistic\\\\n ignore-tables: []\\\\n ignore-dbs: []\\\\nmydumpers: {}\\\\nloaders: {}\\\\nsyncers:\\\\n global:\\\\n meta-file: \\\\\\\"\\\\\\\"\\\\n worker-count: 16\\\\n batch: 100\\\\n queue-size: 1024\\\\n checkpoint-flush-interval: 30\\\\n compact: true\\\\n multiple-rows: true\\\\n max-retry: 0\\\\n auto-fix-gtid: false\\\\n enable-gtid: false\\\\n disable-detect: false\\\\n safe-mode: false\\\\n enable-ansi-quotes: false\\\\nclean-dump-file: false\\\\nansi-quotes: false\\\\nremove-meta: false\\\\ntidb: null\\\\n\\\" \"}"] [request=StartTask] [2021/12/02 23:22:18.559 +00:00] [WARN] [task.go:709] ["heartbeat is deprecated, needn't set it anymore."] [2021/12/02 23:22:18.637 +00:00] [INFO] [checker.go:236] ["\n************ task task_optimistic checking items ************\nmysql_version\nmysql_server_id\nmysql_binlog_enable\nmysql_binlog_format\nmysql_binlog_row_image\nsource db dump privilege checker\nsource db replication privilege checker\ntable structure compatibility check\nmysql_version\nmysql_server_id\nmysql_binlog_enable\nmysql_binlog_format\nmysql_binlog_row_image\nsource db dump privilege checker\nsource db replication privilege checker\ntable structure compatibility check\nmysql_version\nmysql_server_id\nmysql_binlog_enable\nmysql_binlog_format\nmysql_binlog_row_image\nsource db dump privilege checker\nsource db replication privilege checker\ntable structure compatibility check\n************ task task_optimistic checking items ************"] [unit="task check"] [2021/12/02 23:22:18.660 +00:00] [INFO] [server.go:2389] [payload="{HidePasswordObject=\"task:\\\"name: task_pessimistic\\\\ntask-mode: all\\\\nis-sharding: false\\\\nshard-mode: pessimistic\\\\nignore-checking-items:\\\\n- schema_of_shard_tables\\\\n- auto_increment_ID\\\\nmeta-schema: \\\\\\\"\\\\\\\"\\\\nenable-heartbeat: false\\\\nheartbeat-update-interval: 0\\\\nheartbeat-report-interval: 0\\\\ntimezone: \\\\\\\"\\\\\\\"\\\\ncase-sensitive: false\\\\ntarget-database:\\\\n host: tidb-0.tidb\\\\n port: 4000\\\\n user: root\\\\n password: \\\\\\\"******\\\\\\\"\\\\n max-allowed-packet: null\\\\n session: {}\\\\n security: null\\\\nmysql-instances:\\\\n- source-id: replica-01\\\\n meta: null\\\\n filter-rules: []\\\\n column-mapping-rules: []\\\\n route-rules: []\\\\n expression-filters: []\\\\n black-white-list: instance\\\\n block-allow-list: instance\\\\n mydumper-config-name: \\\\\\\"\\\\\\\"\\\\n mydumper:\\\\n mydumper-path: ./bin/mydumper\\\\n threads: 4\\\\n chunk-filesize: \\\\\\\"64\\\\\\\"\\\\n statement-size: 0\\\\n rows: 0\\\\n where: \\\\\\\"\\\\\\\"\\\\n skip-tz-utc: true\\\\n extra-args: \\\\\\\"\\\\\\\"\\\\n mydumper-thread: 4\\\\n loader-config-name: \\\\\\\"\\\\\\\"\\\\n loader:\\\\n pool-size: 16\\\\n dir: ./dumped_data\\\\n loader-thread: 16\\\\n syncer-config-name: global\\\\n syncer: null\\\\n syncer-thread: 0\\\\n- source-id: replica-02\\\\n meta: null\\\\n filter-rules: []\\\\n column-mapping-rules: []\\\\n route-rules: []\\\\n expression-filters: []\\\\n black-white-list: instance\\\\n block-allow-list: instance\\\\n mydumper-config-name: \\\\\\\"\\\\\\\"\\\\n mydumper:\\\\n mydumper-path: ./bin/mydumper\\\\n threads: 4\\\\n chunk-filesize: \\\\\\\"64\\\\\\\"\\\\n statement-size: 0\\\\n rows: 0\\\\n where: \\\\\\\"\\\\\\\"\\\\n skip-tz-utc: true\\\\n extra-args: \\\\\\\"\\\\\\\"\\\\n mydumper-thread: 4\\\\n loader-config-name: \\\\\\\"\\\\\\\"\\\\n loader:\\\\n pool-size: 16\\\\n dir: ./dumped_data\\\\n loader-thread: 16\\\\n syncer-config-name: global\\\\n syncer: null\\\\n syncer-thread: 0\\\\n- source-id: replica-03\\\\n meta: null\\\\n filter-rules: []\\\\n column-mapping-rules: []\\\\n route-rules: []\\\\n expression-filters: []\\\\n black-white-list: instance\\\\n block-allow-list: instance\\\\n mydumper-config-name: \\\\\\\"\\\\\\\"\\\\n mydumper:\\\\n mydumper-path: ./bin/mydumper\\\\n threads: 4\\\\n chunk-filesize: \\\\\\\"64\\\\\\\"\\\\n statement-size: 0\\\\n rows: 0\\\\n where: \\\\\\\"\\\\\\\"\\\\n skip-tz-utc: true\\\\n extra-args: \\\\\\\"\\\\\\\"\\\\n mydumper-thread: 4\\\\n loader-config-name: \\\\\\\"\\\\\\\"\\\\n loader:\\\\n pool-size: 16\\\\n dir: ./dumped_data\\\\n loader-thread: 16\\\\n syncer-config-name: global\\\\n syncer: null\\\\n syncer-thread: 0\\\\nonline-ddl: false\\\\nshadow-table-rules: []\\\\ntrash-table-rules: []\\\\nonline-ddl-scheme: \\\\\\\"\\\\\\\"\\\\nroutes: {}\\\\nfilters: {}\\\\ncolumn-mappings: {}\\\\nexpression-filter: {}\\\\nblack-white-list:\\\\n instance:\\\\n do-tables: []\\\\n do-dbs:\\\\n - db_pessimistic\\\\n ignore-tables: []\\\\n ignore-dbs: []\\\\nblock-allow-list:\\\\n instance:\\\\n do-tables: []\\\\n do-dbs:\\\\n - db_pessimistic\\\\n ignore-tables: []\\\\n ignore-dbs: []\\\\nmydumpers: {}\\\\nloaders: {}\\\\nsyncers:\\\\n global:\\\\n meta-file: \\\\\\\"\\\\\\\"\\\\n worker-count: 16\\\\n batch: 100\\\\n queue-size: 1024\\\\n checkpoint-flush-interval: 30\\\\n compact: true\\\\n multiple-rows: true\\\\n max-retry: 0\\\\n auto-fix-gtid: false\\\\n enable-gtid: false\\\\n disable-detect: false\\\\n safe-mode: false\\\\n enable-ansi-quotes: false\\\\nclean-dump-file: false\\\\nansi-quotes: false\\\\nremove-meta: false\\\\ntidb: null\\\\n\\\" \"}"] [request=StartTask] [2021/12/02 23:22:18.660 +00:00] [WARN] [task.go:709] ["heartbeat is deprecated, needn't set it anymore."] [2021/12/02 23:22:18.736 +00:00] [INFO] [server.go:434] ["task name"=task_optimistic] [task="{\"name\":\"task_optimistic\",\"task-mode\":\"all\",\"is-sharding\":false,\"shard-mode\":\"optimistic\",\"ignore-checking-items\":[\"schema_of_shard_tables\",\"auto_increment_ID\"],\"meta-schema\":\"\",\"enable-heartbeat\":false,\"heartbeat-update-interval\":0,\"heartbeat-report-interval\":0,\"timezone\":\"\",\"case-sensitive\":false,\"target-database\":{\"host\":\"tidb-0.tidb\",\"port\":4000,\"user\":\"root\",\"max-allowed-packet\":null,\"session\":{\"tidb_txn_mode\":\"optimistic\"},\"security\":null},\"mysql-instances\":[{\"SourceID\":\"replica-01\",\"Meta\":null,\"FilterRules\":[],\"ColumnMappingRules\":[],\"RouteRules\":[],\"ExpressionFilters\":[],\"BWListName\":\"instance\",\"BAListName\":\"instance\",\"MydumperConfigName\":\"\",\"Mydumper\":{\"mydumper-path\":\"./bin/mydumper\",\"threads\":4,\"chunk-filesize\":\"64\",\"statement-size\":0,\"rows\":0,\"where\":\"\",\"skip-tz-utc\":true,\"extra-args\":\"\"},\"MydumperThread\":4,\"LoaderConfigName\":\"\",\"Loader\":{\"pool-size\":16,\"dir\":\"./dumped_data\"},\"LoaderThread\":16,\"SyncerConfigName\":\"global\",\"Syncer\":{\"meta-file\":\"\",\"worker-count\":16,\"batch\":100,\"queue-size\":1024,\"checkpoint-flush-interval\":30,\"compact\":true,\"multiple-rows\":true,\"max-retry\":0,\"auto-fix-gtid\":false,\"enable-gtid\":false,\"disable-detect\":false,\"safe-mode\":false,\"enable-ansi-quotes\":false},\"SyncerThread\":0},{\"SourceID\":\"replica-02\",\"Meta\":null,\"FilterRules\":[],\"ColumnMappingRules\":[],\"RouteRules\":[],\"ExpressionFilters\":[],\"BWListName\":\"instance\",\"BAListName\":\"instance\",\"MydumperConfigName\":\"\",\"Mydumper\":{\"mydumper-path\":\"./bin/mydumper\",\"threads\":4,\"chunk-filesize\":\"64\",\"statement-size\":0,\"rows\":0,\"where\":\"\",\"skip-tz-utc\":true,\"extra-args\":\"\"},\"MydumperThread\":4,\"LoaderConfigName\":\"\",\"Loader\":{\"pool-size\":16,\"dir\":\"./dumped_data\"},\"LoaderThread\":16,\"SyncerConfigName\":\"global\",\"Syncer\":{\"meta-file\":\"\",\"worker-count\":16,\"batch\":100,\"queue-size\":1024,\"checkpoint-flush-interval\":30,\"compact\":true,\"multiple-rows\":true,\"max-retry\":0,\"auto-fix-gtid\":false,\"enable-gtid\":false,\"disable-detect\":false,\"safe-mode\":false,\"enable-ansi-quotes\":false},\"SyncerThread\":0},{\"SourceID\":\"replica-03\",\"Meta\":null,\"FilterRules\":[],\"ColumnMappingRules\":[],\"RouteRules\":[],\"ExpressionFilters\":[],\"BWListName\":\"instance\",\"BAListName\":\"instance\",\"MydumperConfigName\":\"\",\"Mydumper\":{\"mydumper-path\":\"./bin/mydumper\",\"threads\":4,\"chunk-filesize\":\"64\",\"statement-size\":0,\"rows\":0,\"where\":\"\",\"skip-tz-utc\":true,\"extra-args\":\"\"},\"MydumperThread\":4,\"LoaderConfigName\":\"\",\"Loader\":{\"pool-size\":16,\"dir\":\"./dumped_data\"},\"LoaderThread\":16,\"SyncerConfigName\":\"global\",\"Syncer\":{\"meta-file\":\"\",\"worker-count\":16,\"batch\":100,\"queue-size\":1024,\"checkpoint-flush-interval\":30,\"compact\":true,\"multiple-rows\":true,\"max-retry\":0,\"auto-fix-gtid\":false,\"enable-gtid\":false,\"disable-detect\":false,\"safe-mode\":false,\"enable-ansi-quotes\":false},\"SyncerThread\":0}],\"online-ddl\":false,\"shadow-table-rules\":[],\"trash-table-rules\":[],\"online-ddl-scheme\":\"\",\"routes\":{},\"filters\":{},\"column-mappings\":{},\"expression-filter\":{},\"black-white-list\":{\"instance\":{\"do-tables\":[],\"do-dbs\":[\"db_optimistic\"],\"ignore-tables\":[],\"ignore-dbs\":[]}},\"block-allow-list\":{\"instance\":{\"do-tables\":[],\"do-dbs\":[\"db_optimistic\"],\"ignore-tables\":[],\"ignore-dbs\":[]}},\"mydumpers\":{},\"loaders\":{},\"syncers\":{\"global\":{\"meta-file\":\"\",\"worker-count\":16,\"batch\":100,\"queue-size\":1024,\"checkpoint-flush-interval\":30,\"compact\":true,\"multiple-rows\":true,\"max-retry\":0,\"auto-fix-gtid\":false,\"enable-gtid\":false,\"disable-detect\":false,\"safe-mode\":false,\"enable-ansi-quotes\":false}},\"clean-dump-file\":false,\"ansi-quotes\":false,\"RemoveMeta\":false,\"tidb\":null}"] [request=StartTask] [2021/12/02 23:22:18.757 +00:00] [INFO] [server.go:1750] ["fail to get expect operation result"] [retryNum=0] [task=task_optimistic] [source=replica-01] [expect=Running] [resp="result:true sourceStatus: > subTaskStatus: "] [2021/12/02 23:22:18.760 +00:00] [INFO] [server.go:1750] ["fail to get expect operation result"] [retryNum=0] [task=task_optimistic] [source=replica-02] [expect=Running] [resp="result:true msg:\"no sub task started\" sourceStatus: "] [2021/12/02 23:22:18.767 +00:00] [INFO] [server.go:1750] ["fail to get expect operation result"] [retryNum=0] [task=task_optimistic] [source=replica-03] [expect=Running] [resp="result:true msg:\"no sub task started\" sourceStatus: > "] [2021/12/02 23:22:18.772 +00:00] [INFO] [checker.go:236] ["\n************ task task_pessimistic checking items ************\nmysql_version\nmysql_server_id\nmysql_binlog_enable\nmysql_binlog_format\nmysql_binlog_row_image\nsource db dump privilege checker\nsource db replication privilege checker\ntable structure compatibility check\nmysql_version\nmysql_server_id\nmysql_binlog_enable\nmysql_binlog_format\nmysql_binlog_row_image\nsource db dump privilege checker\nsource db replication privilege checker\ntable structure compatibility check\nmysql_version\nmysql_server_id\nmysql_binlog_enable\nmysql_binlog_format\nmysql_binlog_row_image\nsource db dump privilege checker\nsource db replication privilege checker\ntable structure compatibility check\n************ task task_pessimistic checking items ************"] [unit="task check"] [2021/12/02 23:22:18.850 +00:00] [INFO] [server.go:434] ["task name"=task_pessimistic] [task="{\"name\":\"task_pessimistic\",\"task-mode\":\"all\",\"is-sharding\":false,\"shard-mode\":\"pessimistic\",\"ignore-checking-items\":[\"schema_of_shard_tables\",\"auto_increment_ID\"],\"meta-schema\":\"\",\"enable-heartbeat\":false,\"heartbeat-update-interval\":0,\"heartbeat-report-interval\":0,\"timezone\":\"\",\"case-sensitive\":false,\"target-database\":{\"host\":\"tidb-0.tidb\",\"port\":4000,\"user\":\"root\",\"max-allowed-packet\":null,\"session\":{\"tidb_txn_mode\":\"optimistic\"},\"security\":null},\"mysql-instances\":[{\"SourceID\":\"replica-01\",\"Meta\":null,\"FilterRules\":[],\"ColumnMappingRules\":[],\"RouteRules\":[],\"ExpressionFilters\":[],\"BWListName\":\"instance\",\"BAListName\":\"instance\",\"MydumperConfigName\":\"\",\"Mydumper\":{\"mydumper-path\":\"./bin/mydumper\",\"threads\":4,\"chunk-filesize\":\"64\",\"statement-size\":0,\"rows\":0,\"where\":\"\",\"skip-tz-utc\":true,\"extra-args\":\"\"},\"MydumperThread\":4,\"LoaderConfigName\":\"\",\"Loader\":{\"pool-size\":16,\"dir\":\"./dumped_data\"},\"LoaderThread\":16,\"SyncerConfigName\":\"global\",\"Syncer\":{\"meta-file\":\"\",\"worker-count\":16,\"batch\":100,\"queue-size\":1024,\"checkpoint-flush-interval\":30,\"compact\":true,\"multiple-rows\":true,\"max-retry\":0,\"auto-fix-gtid\":false,\"enable-gtid\":false,\"disable-detect\":false,\"safe-mode\":false,\"enable-ansi-quotes\":false},\"SyncerThread\":0},{\"SourceID\":\"replica-02\",\"Meta\":null,\"FilterRules\":[],\"ColumnMappingRules\":[],\"RouteRules\":[],\"ExpressionFilters\":[],\"BWListName\":\"instance\",\"BAListName\":\"instance\",\"MydumperConfigName\":\"\",\"Mydumper\":{\"mydumper-path\":\"./bin/mydumper\",\"threads\":4,\"chunk-filesize\":\"64\",\"statement-size\":0,\"rows\":0,\"where\":\"\",\"skip-tz-utc\":true,\"extra-args\":\"\"},\"MydumperThread\":4,\"LoaderConfigName\":\"\",\"Loader\":{\"pool-size\":16,\"dir\":\"./dumped_data\"},\"LoaderThread\":16,\"SyncerConfigName\":\"global\",\"Syncer\":{\"meta-file\":\"\",\"worker-count\":16,\"batch\":100,\"queue-size\":1024,\"checkpoint-flush-interval\":30,\"compact\":true,\"multiple-rows\":true,\"max-retry\":0,\"auto-fix-gtid\":false,\"enable-gtid\":false,\"disable-detect\":false,\"safe-mode\":false,\"enable-ansi-quotes\":false},\"SyncerThread\":0},{\"SourceID\":\"replica-03\",\"Meta\":null,\"FilterRules\":[],\"ColumnMappingRules\":[],\"RouteRules\":[],\"ExpressionFilters\":[],\"BWListName\":\"instance\",\"BAListName\":\"instance\",\"MydumperConfigName\":\"\",\"Mydumper\":{\"mydumper-path\":\"./bin/mydumper\",\"threads\":4,\"chunk-filesize\":\"64\",\"statement-size\":0,\"rows\":0,\"where\":\"\",\"skip-tz-utc\":true,\"extra-args\":\"\"},\"MydumperThread\":4,\"LoaderConfigName\":\"\",\"Loader\":{\"pool-size\":16,\"dir\":\"./dumped_data\"},\"LoaderThread\":16,\"SyncerConfigName\":\"global\",\"Syncer\":{\"meta-file\":\"\",\"worker-count\":16,\"batch\":100,\"queue-size\":1024,\"checkpoint-flush-interval\":30,\"compact\":true,\"multiple-rows\":true,\"max-retry\":0,\"auto-fix-gtid\":false,\"enable-gtid\":false,\"disable-detect\":false,\"safe-mode\":false,\"enable-ansi-quotes\":false},\"SyncerThread\":0}],\"online-ddl\":false,\"shadow-table-rules\":[],\"trash-table-rules\":[],\"online-ddl-scheme\":\"\",\"routes\":{},\"filters\":{},\"column-mappings\":{},\"expression-filter\":{},\"black-white-list\":{\"instance\":{\"do-tables\":[],\"do-dbs\":[\"db_pessimistic\"],\"ignore-tables\":[],\"ignore-dbs\":[]}},\"block-allow-list\":{\"instance\":{\"do-tables\":[],\"do-dbs\":[\"db_pessimistic\"],\"ignore-tables\":[],\"ignore-dbs\":[]}},\"mydumpers\":{},\"loaders\":{},\"syncers\":{\"global\":{\"meta-file\":\"\",\"worker-count\":16,\"batch\":100,\"queue-size\":1024,\"checkpoint-flush-interval\":30,\"compact\":true,\"multiple-rows\":true,\"max-retry\":0,\"auto-fix-gtid\":false,\"enable-gtid\":false,\"disable-detect\":false,\"safe-mode\":false,\"enable-ansi-quotes\":false}},\"clean-dump-file\":false,\"ansi-quotes\":false,\"RemoveMeta\":false,\"tidb\":null}"] [request=StartTask] [2021/12/02 23:22:19.191 +00:00] [INFO] [scheduler.go:2392] ["receive load task"] [component=scheduler] [delete=true] [task=task_single] [source=replica-01] [worker=] [2021/12/02 23:22:19.582 +00:00] [INFO] [optimist.go:508] ["receive source tables"] [component="shard DDL optimist"] ["source tables"="{\"task\":\"task_optimistic\",\"source\":\"replica-02\",\"tables\":{\"db_optimistic\":{\"eeodta\":{\"db_optimistic\":{\"eeodta\":{}}},\"infmawmls\":{\"db_optimistic\":{\"infmawmls\":{}}},\"mmqinfn\":{\"db_optimistic\":{\"mmqinfn\":{}}},\"qejfvbyl\":{\"db_optimistic\":{\"qejfvbyl\":{}}},\"rykwvonj\":{\"db_optimistic\":{\"rykwvonj\":{}}},\"tkwux\":{\"db_optimistic\":{\"tkwux\":{}}},\"ubxretux\":{\"db_optimistic\":{\"ubxretux\":{}}},\"ukglzbz\":{\"db_optimistic\":{\"ukglzbz\":{}}},\"uqgqzm\":{\"db_optimistic\":{\"uqgqzm\":{}}},\"wihxkukz\":{\"db_optimistic\":{\"wihxkukz\":{}}}}}}"] ["is deleted"=false] [updated=true] [2021/12/02 23:22:19.582 +00:00] [INFO] [optimist.go:508] ["receive source tables"] [component="shard DDL optimist"] ["source tables"="{\"task\":\"task_optimistic\",\"source\":\"replica-01\",\"tables\":{\"db_optimistic\":{\"eeodta\":{\"db_optimistic\":{\"eeodta\":{}}},\"infmawmls\":{\"db_optimistic\":{\"infmawmls\":{}}},\"mmqinfn\":{\"db_optimistic\":{\"mmqinfn\":{}}},\"qejfvbyl\":{\"db_optimistic\":{\"qejfvbyl\":{}}},\"rykwvonj\":{\"db_optimistic\":{\"rykwvonj\":{}}},\"tkwux\":{\"db_optimistic\":{\"tkwux\":{}}},\"ubxretux\":{\"db_optimistic\":{\"ubxretux\":{}}},\"ukglzbz\":{\"db_optimistic\":{\"ukglzbz\":{}}},\"uqgqzm\":{\"db_optimistic\":{\"uqgqzm\":{}}},\"wihxkukz\":{\"db_optimistic\":{\"wihxkukz\":{}}}}}}"] ["is deleted"=false] [updated=true] [2021/12/02 23:22:19.642 +00:00] [INFO] [server.go:1750] ["fail to get expect operation result"] [retryNum=0] [task=task_pessimistic] [source=replica-02] [expect=Running] [resp="result:true sourceStatus: subTaskStatus: "] [2021/12/02 23:22:19.645 +00:00] [INFO] [server.go:1750] ["fail to get expect operation result"] [retryNum=0] [task=task_pessimistic] [source=replica-01] [expect=Running] [resp="result:true sourceStatus: > subTaskStatus: "] [2021/12/02 23:22:19.823 +00:00] [INFO] [scheduler.go:2392] ["receive load task"] [component=scheduler] [delete=false] [task=task_optimistic] [source=replica-01] [worker=dm-worker-1] [2021/12/02 23:22:20.013 +00:00] [INFO] [scheduler.go:2392] ["receive load task"] [component=scheduler] [delete=false] [task=task_optimistic] [source=replica-02] [worker=dm-worker-0] [2021/12/02 23:22:20.027 +00:00] [INFO] [optimist.go:508] ["receive source tables"] [component="shard DDL optimist"] ["source tables"="{\"task\":\"task_optimistic\",\"source\":\"replica-03\",\"tables\":{\"db_optimistic\":{\"eeodta\":{\"db_optimistic\":{\"eeodta\":{}}},\"infmawmls\":{\"db_optimistic\":{\"infmawmls\":{}}},\"mmqinfn\":{\"db_optimistic\":{\"mmqinfn\":{}}},\"qejfvbyl\":{\"db_optimistic\":{\"qejfvbyl\":{}}},\"rykwvonj\":{\"db_optimistic\":{\"rykwvonj\":{}}},\"tkwux\":{\"db_optimistic\":{\"tkwux\":{}}},\"ubxretux\":{\"db_optimistic\":{\"ubxretux\":{}}},\"ukglzbz\":{\"db_optimistic\":{\"ukglzbz\":{}}},\"uqgqzm\":{\"db_optimistic\":{\"uqgqzm\":{}}},\"wihxkukz\":{\"db_optimistic\":{\"wihxkukz\":{}}}}}}"] ["is deleted"=false] [updated=true] [2021/12/02 23:22:20.108 +00:00] [INFO] [server.go:1750] ["fail to get expect operation result"] [retryNum=0] [task=task_pessimistic] [source=replica-03] [expect=Running] [resp="result:true sourceStatus: > subTaskStatus: "] [2021/12/02 23:22:20.210 +00:00] [INFO] [scheduler.go:2392] ["receive load task"] [component=scheduler] [delete=false] [task=task_optimistic] [source=replica-03] [worker=dm-worker-2] [2021/12/02 23:22:20.770 +00:00] [INFO] [scheduler.go:2392] ["receive load task"] [component=scheduler] [delete=true] [task=task_optimistic] [source=replica-01] [worker=] [2021/12/02 23:22:20.866 +00:00] [INFO] [scheduler.go:2392] ["receive load task"] [component=scheduler] [delete=true] [task=task_optimistic] [source=replica-03] [worker=] [2021/12/02 23:22:20.934 +00:00] [INFO] [scheduler.go:2392] ["receive load task"] [component=scheduler] [delete=false] [task=task_pessimistic] [source=replica-01] [worker=dm-worker-1] [2021/12/02 23:22:21.215 +00:00] [INFO] [scheduler.go:2392] ["receive load task"] [component=scheduler] [delete=false] [task=task_pessimistic] [source=replica-02] [worker=dm-worker-0] [2021/12/02 23:22:21.433 +00:00] [INFO] [scheduler.go:2392] ["receive load task"] [component=scheduler] [delete=false] [task=task_pessimistic] [source=replica-03] [worker=dm-worker-2] [2021/12/02 23:22:21.664 +00:00] [INFO] [scheduler.go:2392] ["receive load task"] [component=scheduler] [delete=true] [task=task_pessimistic] [source=replica-01] [worker=] [2021/12/02 23:22:22.288 +00:00] [INFO] [scheduler.go:2392] ["receive load task"] [component=scheduler] [delete=true] [task=task_optimistic] [source=replica-02] [worker=] [2021/12/02 23:22:23.640 +00:00] [INFO] [scheduler.go:2392] ["receive load task"] [component=scheduler] [delete=true] [task=task_pessimistic] [source=replica-02] [worker=] [2021/12/02 23:22:23.740 +00:00] [INFO] [scheduler.go:2392] ["receive load task"] [component=scheduler] [delete=true] [task=task_pessimistic] [source=replica-03] [worker=] [2021/12/02 23:22:43.260 +00:00] [INFO] [main.go:84] ["got signal to exit"] [signal=terminated] [2021/12/02 23:22:43.261 +00:00] [INFO] [server.go:262] ["closing server"] [2021/12/02 23:22:43.261 +00:00] [INFO] [election.go:190] ["election is closing"] [component=election] ["current member"="{\"id\":\"dm-master-1\",\"addr\":\"dm-master-1.dm-master.default:8261\"}"] [2021/12/02 23:22:43.261 +00:00] [INFO] [election.go:335] ["retire from leader"] [component=election] ["current member"="{\"id\":\"dm-master-1\",\"addr\":\"dm-master-1.dm-master.default:8261\"}"] [2021/12/02 23:22:43.261 +00:00] [WARN] [election.go:359] ["ignore notify the leader's information because context canceled"] [component=election] ["current member"="{\"id\":\"dm-master-1\",\"addr\":\"dm-master-1.dm-master.default:8261\"}"] [leader=null] [2021/12/02 23:22:43.261 +00:00] [INFO] [election.go:240] ["break campaign loop, context is done"] [component=election] ["current member"="{\"id\":\"dm-master-1\",\"addr\":\"dm-master-1.dm-master.default:8261\"}"] [error="context canceled"] [2021/12/02 23:22:43.261 +00:00] [INFO] [scheduler.go:2264] ["observeLoadTask will quit now"] [2021/12/02 23:22:43.261 +00:00] [INFO] [keepalive.go:194] ["watch keepalive worker quit due to context canceled"] [2021/12/02 23:22:43.261 +00:00] [INFO] [scheduler.go:1809] ["observeWorkerEvent will quit now"] [2021/12/02 23:22:43.261 +00:00] [INFO] [pessimist.go:123] ["pessimist will quit now"] [2021/12/02 23:22:43.264 +00:00] [INFO] [election.go:198] ["election is closed"] [component=election] ["current member"="{\"id\":\"dm-master-1\",\"addr\":\"dm-master-1.dm-master.default:8261\"}"] [2021/12/02 23:22:43.266 +00:00] [WARN] [grpclog.go:66] ["transport: http2Server.HandleStreams failed to read frame: read tcp 10.88.0.18:8261->10.88.0.25:53452: use of closed network connection"] [component="embed etcd"] [2021/12/02 23:22:43.266 +00:00] [WARN] [grpclog.go:66] ["transport: http2Server.HandleStreams failed to read frame: read tcp 10.88.0.18:8261->10.88.0.20:50102: use of closed network connection"] [component="embed etcd"] [2021/12/02 23:22:43.266 +00:00] [WARN] [grpclog.go:66] ["transport: http2Server.HandleStreams failed to read frame: read tcp 10.88.0.18:8261->10.88.0.16:38398: use of closed network connection"] [component="embed etcd"] [2021/12/02 23:22:43.266 +00:00] [WARN] [grpclog.go:66] ["transport: http2Server.HandleStreams failed to read frame: read tcp 10.88.0.18:8261->10.88.0.18:36690: use of closed network connection"] [component="embed etcd"] [2021/12/02 23:22:43.266 +00:00] [WARN] [grpclog.go:66] ["transport: http2Server.HandleStreams failed to read frame: read tcp 10.88.0.18:8261->10.88.0.27:47180: use of closed network connection"] [component="embed etcd"] [2021/12/02 23:22:43.266 +00:00] [WARN] [grpclog.go:66] ["transport: http2Server.HandleStreams failed to read frame: read tcp 10.88.0.18:8261->10.88.0.31:52254: use of closed network connection"] [component="embed etcd"] [2021/12/02 23:22:43.266 +00:00] [WARN] [grpclog.go:66] ["transport: http2Server.HandleStreams failed to read frame: read tcp 10.88.0.18:8261->10.88.0.32:53888: use of closed network connection"] [component="embed etcd"] [2021/12/02 23:22:43.267 +00:00] [WARN] [grpclog.go:66] ["transport: http2Server.HandleStreams failed to read frame: read tcp 10.88.0.18:8261->10.88.0.1:65342: use of closed network connection"] [component="embed etcd"] [2021/12/02 23:22:43.267 +00:00] [WARN] [grpclog.go:66] ["transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:8261->127.0.0.1:49760: use of closed network connection"] [component="embed etcd"] [2021/12/02 23:22:43.280 +00:00] [WARN] [stream.go:291] ["closed TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream MsgApp v2"] [remote-peer-id=94ddedb20f20c0c7] [2021/12/02 23:22:43.280 +00:00] [WARN] [stream.go:301] ["stopped TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream MsgApp v2"] [remote-peer-id=94ddedb20f20c0c7] [2021/12/02 23:22:43.281 +00:00] [WARN] [stream.go:291] ["closed TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream Message"] [remote-peer-id=94ddedb20f20c0c7] [2021/12/02 23:22:43.281 +00:00] [WARN] [stream.go:301] ["stopped TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream Message"] [remote-peer-id=94ddedb20f20c0c7] [2021/12/02 23:22:43.281 +00:00] [WARN] [stream.go:436] ["lost TCP streaming connection with remote peer"] [component="embed etcd"] [stream-reader-type="stream MsgApp v2"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id=94ddedb20f20c0c7] [error="context canceled"] [2021/12/02 23:22:43.281 +00:00] [WARN] [peer_status.go:68] ["peer became inactive (message send to peer failed)"] [component="embed etcd"] [peer-id=94ddedb20f20c0c7] [error="failed to read 94ddedb20f20c0c7 on stream MsgApp v2 (context canceled)"] [2021/12/02 23:22:43.281 +00:00] [WARN] [stream.go:436] ["lost TCP streaming connection with remote peer"] [component="embed etcd"] [stream-reader-type="stream Message"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id=94ddedb20f20c0c7] [error="context canceled"] [2021/12/02 23:22:43.281 +00:00] [WARN] [stream.go:291] ["closed TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream MsgApp v2"] [remote-peer-id=c6ec7df2da09b327] [2021/12/02 23:22:43.281 +00:00] [WARN] [stream.go:301] ["stopped TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream MsgApp v2"] [remote-peer-id=c6ec7df2da09b327] [2021/12/02 23:22:43.288 +00:00] [WARN] [stream.go:291] ["closed TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream Message"] [remote-peer-id=c6ec7df2da09b327] [2021/12/02 23:22:43.288 +00:00] [WARN] [stream.go:301] ["stopped TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream Message"] [remote-peer-id=c6ec7df2da09b327] [2021/12/02 23:22:43.288 +00:00] [WARN] [stream.go:436] ["lost TCP streaming connection with remote peer"] [component="embed etcd"] [stream-reader-type="stream MsgApp v2"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id=c6ec7df2da09b327] [error="context canceled"] [2021/12/02 23:22:43.288 +00:00] [WARN] [peer_status.go:68] ["peer became inactive (message send to peer failed)"] [component="embed etcd"] [peer-id=c6ec7df2da09b327] [error="failed to read c6ec7df2da09b327 on stream MsgApp v2 (context canceled)"] [2021/12/02 23:22:43.288 +00:00] [WARN] [stream.go:436] ["lost TCP streaming connection with remote peer"] [component="embed etcd"] [stream-reader-type="stream Message"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id=c6ec7df2da09b327] [error="http: read on closed response body"] [2021/12/02 23:22:43.289 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-master-1.dm-master.default:8261 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp 10.88.0.18:8261: connect: connection refused\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:22:43.294 +00:00] [WARN] [http.go:448] ["failed to find remote peer in cluster"] [component="embed etcd"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id-stream-handler=7f72ccbf7d68a9c1] [remote-peer-id-from=94ddedb20f20c0c7] [cluster-id=b93483a4fb43c8bb] [2021/12/02 23:22:43.294 +00:00] [WARN] [http.go:448] ["failed to find remote peer in cluster"] [component="embed etcd"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id-stream-handler=7f72ccbf7d68a9c1] [remote-peer-id-from=94ddedb20f20c0c7] [cluster-id=b93483a4fb43c8bb] [2021/12/02 23:22:43.297 +00:00] [INFO] [server.go:264] ["server closed"] [2021/12/02 23:22:43.298 +00:00] [INFO] [main.go:91] ["dm-master exit"] [2021/12/02 23:23:14.333 +00:00] [INFO] [printer.go:54] ["Welcome to dm-master"] ["Release Version"=v5.2.0-master] ["Git Commit Hash"=ee4a7f889449a557081a2529da5c9265c9d015fc] ["Git Branch"=master] ["UTC Build Time"="2021-12-02 23:16:44"] ["Go Version"="go version go1.16.10 linux/amd64"] [2021/12/02 23:23:14.336 +00:00] [INFO] [main.go:63] ["dm-master config"="{\"log-level\":\"info\",\"log-file\":\"/log/dm-master-1.log\",\"log-format\":\"text\",\"log-rotate\":\"\",\"rpc-timeout\":\"30s\",\"rpc-rate-limit\":10,\"rpc-rate-burst\":40,\"master-addr\":\"0.0.0.0:8261\",\"advertise-addr\":\"dm-master-1.dm-master.default:8261\",\"config-file\":\"\",\"name\":\"dm-master-1\",\"data-dir\":\"/data\",\"peer-urls\":\"http://:8291\",\"advertise-peer-urls\":\"http://dm-master-1.dm-master.default:8291\",\"initial-cluster\":\"dm-master-0=http://dm-master-0.dm-master.default:8291,dm-master-1=http://dm-master-1.dm-master.default:8291,dm-master-2=http://dm-master-2.dm-master.default:8291\",\"initial-cluster-state\":\"new\",\"join\":\"\",\"max-txn-ops\":2048,\"max-request-bytes\":1572864,\"auto-compaction-mode\":\"periodic\",\"auto-compaction-retention\":\"1h\",\"quota-backend-bytes\":2147483648,\"v1-sources-path\":\"\",\"ssl-ca\":\"\",\"ssl-cert\":\"\",\"ssl-key\":\"\",\"cert-allowed-cn\":null,\"ExperimentalFeatures\":{\"OpenAPI\":false}}"] [2021/12/02 23:23:14.336 +00:00] [INFO] [config.go:437] ["Set log level of etcd to `warn`, if you want to log more message about etcd, change log-level to `debug` in master configuration file"] [2021/12/02 23:23:14.336 +00:00] [INFO] [server.go:148] ["config after join prepared"] [config="{\"log-level\":\"info\",\"log-file\":\"/log/dm-master-1.log\",\"log-format\":\"text\",\"log-rotate\":\"\",\"rpc-timeout\":\"30s\",\"rpc-rate-limit\":10,\"rpc-rate-burst\":40,\"master-addr\":\"0.0.0.0:8261\",\"advertise-addr\":\"dm-master-1.dm-master.default:8261\",\"config-file\":\"\",\"name\":\"dm-master-1\",\"data-dir\":\"/data\",\"peer-urls\":\"http://:8291\",\"advertise-peer-urls\":\"http://dm-master-1.dm-master.default:8291\",\"initial-cluster\":\"dm-master-0=http://dm-master-0.dm-master.default:8291,dm-master-1=http://dm-master-1.dm-master.default:8291,dm-master-2=http://dm-master-2.dm-master.default:8291\",\"initial-cluster-state\":\"new\",\"join\":\"\",\"max-txn-ops\":2048,\"max-request-bytes\":1572864,\"auto-compaction-mode\":\"periodic\",\"auto-compaction-retention\":\"1h\",\"quota-backend-bytes\":2147483648,\"v1-sources-path\":\"\",\"ssl-ca\":\"\",\"ssl-cert\":\"\",\"ssl-key\":\"\",\"cert-allowed-cn\":null,\"ExperimentalFeatures\":{\"OpenAPI\":false}}"] [2021/12/02 23:23:14.343 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-master-1.dm-master.default:8261 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-master-1.dm-master.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:23:14.355 +00:00] [WARN] [store.go:1366] ["simple token is not cryptographically signed"] [component="embed etcd"] [2021/12/02 23:23:14.782 +00:00] [INFO] [server.go:253] ["listening gRPC API and status request"] [address=0.0.0.0:8261] [2021/12/02 23:23:14.794 +00:00] [INFO] [election.go:301] ["get response from election observe"] [component=election] [key=/dm-master/leader/33277d7d7360be04] [value="{\"id\":\"dm-master-0\",\"addr\":\"dm-master-0.dm-master.default:8261\"}"] [2021/12/02 23:23:14.794 +00:00] [INFO] [election.go:325] ["current member is not the leader"] [component=election] ["current member"="{\"id\":\"dm-master-1\",\"addr\":\"dm-master-1.dm-master.default:8261\"}"] [leader="{\"id\":\"dm-master-0\",\"addr\":\"dm-master-0.dm-master.default:8261\"}"] [2021/12/02 23:23:14.796 +00:00] [INFO] [election.go:97] ["get new leader"] [leader=dm-master-0] ["current member"=dm-master-1] [2021/12/02 23:23:14.817 +00:00] [INFO] [election.go:279] ["fail to campaign"] [component=election] ["current member"="{\"id\":\"dm-master-1\",\"addr\":\"dm-master-1.dm-master.default:8261\"}"] [error="context canceled"] [2021/12/02 23:23:14.826 +00:00] [INFO] [election.go:301] ["get response from election observe"] [component=election] [key=/dm-master/leader/33277d7d7360be04] [value="{\"id\":\"dm-master-0\",\"addr\":\"dm-master-0.dm-master.default:8261\"}"] [2021/12/02 23:23:15.347 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-master-1.dm-master.default:8261 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-master-1.dm-master.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:23:17.173 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-master-1.dm-master.default:8261 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-master-1.dm-master.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:23:17.392 +00:00] [WARN] [stream.go:277] ["established TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream Message"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id=94ddedb20f20c0c7] [2021/12/02 23:23:17.392 +00:00] [WARN] [stream.go:277] ["established TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream MsgApp v2"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id=94ddedb20f20c0c7] [2021/12/02 23:23:17.394 +00:00] [WARN] [stream.go:277] ["established TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream Message"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id=c6ec7df2da09b327] [2021/12/02 23:23:17.400 +00:00] [WARN] [stream.go:277] ["established TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream MsgApp v2"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id=c6ec7df2da09b327] [2021/12/02 23:26:43.040 +00:00] [INFO] [main.go:84] ["got signal to exit"] [signal=terminated] [2021/12/02 23:26:43.041 +00:00] [INFO] [server.go:262] ["closing server"] [2021/12/02 23:26:43.041 +00:00] [INFO] [election.go:190] ["election is closing"] [component=election] ["current member"="{\"id\":\"dm-master-1\",\"addr\":\"dm-master-1.dm-master.default:8261\"}"] [2021/12/02 23:26:43.041 +00:00] [INFO] [election.go:325] ["current member is not the leader"] [component=election] ["current member"="{\"id\":\"dm-master-1\",\"addr\":\"dm-master-1.dm-master.default:8261\"}"] [leader="{\"id\":\"dm-master-0\",\"addr\":\"dm-master-0.dm-master.default:8261\"}"] [2021/12/02 23:26:43.064 +00:00] [WARN] [election.go:272] ["failed to clean election key"] [component=election] [error="context canceled"] [2021/12/02 23:26:43.064 +00:00] [INFO] [election.go:279] ["fail to campaign"] [component=election] ["current member"="{\"id\":\"dm-master-1\",\"addr\":\"dm-master-1.dm-master.default:8261\"}"] [error="context canceled"] [2021/12/02 23:26:43.064 +00:00] [INFO] [election.go:240] ["break campaign loop, context is done"] [component=election] ["current member"="{\"id\":\"dm-master-1\",\"addr\":\"dm-master-1.dm-master.default:8261\"}"] [error="context canceled"] [2021/12/02 23:26:43.068 +00:00] [INFO] [election.go:198] ["election is closed"] [component=election] ["current member"="{\"id\":\"dm-master-1\",\"addr\":\"dm-master-1.dm-master.default:8261\"}"] [2021/12/02 23:26:43.068 +00:00] [WARN] [grpclog.go:66] ["transport: http2Server.HandleStreams failed to read frame: read tcp 10.88.0.36:8261->10.88.0.37:39396: use of closed network connection"] [component="embed etcd"] [2021/12/02 23:26:43.068 +00:00] [WARN] [grpclog.go:66] ["transport: http2Server.HandleStreams failed to read frame: read tcp 10.88.0.36:8261->10.88.0.31:42636: use of closed network connection"] [component="embed etcd"] [2021/12/02 23:26:43.068 +00:00] [WARN] [grpclog.go:66] ["transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:8261->127.0.0.1:60664: use of closed network connection"] [component="embed etcd"] [2021/12/02 23:26:43.068 +00:00] [WARN] [grpclog.go:66] ["transport: http2Server.HandleStreams failed to read frame: read tcp 10.88.0.36:8261->10.88.0.36:46586: use of closed network connection"] [component="embed etcd"] [2021/12/02 23:26:43.068 +00:00] [WARN] [grpclog.go:66] ["transport: http2Server.HandleStreams failed to read frame: read tcp 10.88.0.36:8261->10.88.0.32:41970: use of closed network connection"] [component="embed etcd"] [2021/12/02 23:26:43.068 +00:00] [WARN] [grpclog.go:66] ["transport: http2Server.HandleStreams failed to read frame: read tcp 10.88.0.36:8261->10.88.0.25:51032: use of closed network connection"] [component="embed etcd"] [2021/12/02 23:26:43.080 +00:00] [WARN] [stream.go:291] ["closed TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream MsgApp v2"] [remote-peer-id=94ddedb20f20c0c7] [2021/12/02 23:26:43.080 +00:00] [WARN] [stream.go:301] ["stopped TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream MsgApp v2"] [remote-peer-id=94ddedb20f20c0c7] [2021/12/02 23:26:43.083 +00:00] [WARN] [stream.go:291] ["closed TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream Message"] [remote-peer-id=94ddedb20f20c0c7] [2021/12/02 23:26:43.083 +00:00] [WARN] [stream.go:301] ["stopped TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream Message"] [remote-peer-id=94ddedb20f20c0c7] [2021/12/02 23:26:43.084 +00:00] [WARN] [stream.go:436] ["lost TCP streaming connection with remote peer"] [component="embed etcd"] [stream-reader-type="stream MsgApp v2"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id=94ddedb20f20c0c7] [error="context canceled"] [2021/12/02 23:26:43.084 +00:00] [WARN] [peer_status.go:68] ["peer became inactive (message send to peer failed)"] [component="embed etcd"] [peer-id=94ddedb20f20c0c7] [error="failed to read 94ddedb20f20c0c7 on stream MsgApp v2 (context canceled)"] [2021/12/02 23:26:43.099 +00:00] [WARN] [stream.go:436] ["lost TCP streaming connection with remote peer"] [component="embed etcd"] [stream-reader-type="stream Message"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id=94ddedb20f20c0c7] [error="context canceled"] [2021/12/02 23:26:43.099 +00:00] [WARN] [stream.go:291] ["closed TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream MsgApp v2"] [remote-peer-id=c6ec7df2da09b327] [2021/12/02 23:26:43.099 +00:00] [WARN] [stream.go:301] ["stopped TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream MsgApp v2"] [remote-peer-id=c6ec7df2da09b327] [2021/12/02 23:26:43.100 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-master-1.dm-master.default:8261 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp 10.88.0.36:8261: connect: connection refused\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:26:43.101 +00:00] [WARN] [stream.go:291] ["closed TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream Message"] [remote-peer-id=c6ec7df2da09b327] [2021/12/02 23:26:43.101 +00:00] [WARN] [stream.go:301] ["stopped TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream Message"] [remote-peer-id=c6ec7df2da09b327] [2021/12/02 23:26:43.101 +00:00] [WARN] [stream.go:436] ["lost TCP streaming connection with remote peer"] [component="embed etcd"] [stream-reader-type="stream MsgApp v2"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id=c6ec7df2da09b327] [error="context canceled"] [2021/12/02 23:26:43.101 +00:00] [WARN] [peer_status.go:68] ["peer became inactive (message send to peer failed)"] [component="embed etcd"] [peer-id=c6ec7df2da09b327] [error="failed to read c6ec7df2da09b327 on stream MsgApp v2 (context canceled)"] [2021/12/02 23:26:43.101 +00:00] [WARN] [stream.go:436] ["lost TCP streaming connection with remote peer"] [component="embed etcd"] [stream-reader-type="stream Message"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id=c6ec7df2da09b327] [error="context canceled"] [2021/12/02 23:26:43.101 +00:00] [WARN] [http.go:448] ["failed to find remote peer in cluster"] [component="embed etcd"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id-stream-handler=7f72ccbf7d68a9c1] [remote-peer-id-from=94ddedb20f20c0c7] [cluster-id=b93483a4fb43c8bb] [2021/12/02 23:26:43.101 +00:00] [WARN] [http.go:448] ["failed to find remote peer in cluster"] [component="embed etcd"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id-stream-handler=7f72ccbf7d68a9c1] [remote-peer-id-from=94ddedb20f20c0c7] [cluster-id=b93483a4fb43c8bb] [2021/12/02 23:26:43.115 +00:00] [WARN] [http.go:448] ["failed to find remote peer in cluster"] [component="embed etcd"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id-stream-handler=7f72ccbf7d68a9c1] [remote-peer-id-from=c6ec7df2da09b327] [cluster-id=b93483a4fb43c8bb] [2021/12/02 23:26:43.115 +00:00] [WARN] [http.go:448] ["failed to find remote peer in cluster"] [component="embed etcd"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id-stream-handler=7f72ccbf7d68a9c1] [remote-peer-id-from=c6ec7df2da09b327] [cluster-id=b93483a4fb43c8bb] [2021/12/02 23:26:43.123 +00:00] [INFO] [server.go:264] ["server closed"] [2021/12/02 23:26:43.123 +00:00] [INFO] [main.go:91] ["dm-master exit"] [2021/12/02 23:27:14.450 +00:00] [INFO] [printer.go:54] ["Welcome to dm-master"] ["Release Version"=v5.2.0-master] ["Git Commit Hash"=ee4a7f889449a557081a2529da5c9265c9d015fc] ["Git Branch"=master] ["UTC Build Time"="2021-12-02 23:16:44"] ["Go Version"="go version go1.16.10 linux/amd64"] [2021/12/02 23:27:14.450 +00:00] [INFO] [main.go:63] ["dm-master config"="{\"log-level\":\"info\",\"log-file\":\"/log/dm-master-1.log\",\"log-format\":\"text\",\"log-rotate\":\"\",\"rpc-timeout\":\"30s\",\"rpc-rate-limit\":10,\"rpc-rate-burst\":40,\"master-addr\":\"0.0.0.0:8261\",\"advertise-addr\":\"dm-master-1.dm-master.default:8261\",\"config-file\":\"\",\"name\":\"dm-master-1\",\"data-dir\":\"/data\",\"peer-urls\":\"http://:8291\",\"advertise-peer-urls\":\"http://dm-master-1.dm-master.default:8291\",\"initial-cluster\":\"dm-master-0=http://dm-master-0.dm-master.default:8291,dm-master-1=http://dm-master-1.dm-master.default:8291,dm-master-2=http://dm-master-2.dm-master.default:8291\",\"initial-cluster-state\":\"new\",\"join\":\"\",\"max-txn-ops\":2048,\"max-request-bytes\":1572864,\"auto-compaction-mode\":\"periodic\",\"auto-compaction-retention\":\"1h\",\"quota-backend-bytes\":2147483648,\"v1-sources-path\":\"\",\"ssl-ca\":\"\",\"ssl-cert\":\"\",\"ssl-key\":\"\",\"cert-allowed-cn\":null,\"ExperimentalFeatures\":{\"OpenAPI\":false}}"] [2021/12/02 23:27:14.450 +00:00] [INFO] [config.go:437] ["Set log level of etcd to `warn`, if you want to log more message about etcd, change log-level to `debug` in master configuration file"] [2021/12/02 23:27:14.450 +00:00] [INFO] [server.go:148] ["config after join prepared"] [config="{\"log-level\":\"info\",\"log-file\":\"/log/dm-master-1.log\",\"log-format\":\"text\",\"log-rotate\":\"\",\"rpc-timeout\":\"30s\",\"rpc-rate-limit\":10,\"rpc-rate-burst\":40,\"master-addr\":\"0.0.0.0:8261\",\"advertise-addr\":\"dm-master-1.dm-master.default:8261\",\"config-file\":\"\",\"name\":\"dm-master-1\",\"data-dir\":\"/data\",\"peer-urls\":\"http://:8291\",\"advertise-peer-urls\":\"http://dm-master-1.dm-master.default:8291\",\"initial-cluster\":\"dm-master-0=http://dm-master-0.dm-master.default:8291,dm-master-1=http://dm-master-1.dm-master.default:8291,dm-master-2=http://dm-master-2.dm-master.default:8291\",\"initial-cluster-state\":\"new\",\"join\":\"\",\"max-txn-ops\":2048,\"max-request-bytes\":1572864,\"auto-compaction-mode\":\"periodic\",\"auto-compaction-retention\":\"1h\",\"quota-backend-bytes\":2147483648,\"v1-sources-path\":\"\",\"ssl-ca\":\"\",\"ssl-cert\":\"\",\"ssl-key\":\"\",\"cert-allowed-cn\":null,\"ExperimentalFeatures\":{\"OpenAPI\":false}}"] [2021/12/02 23:27:14.457 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-master-1.dm-master.default:8261 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-master-1.dm-master.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:27:14.735 +00:00] [WARN] [store.go:1366] ["simple token is not cryptographically signed"] [component="embed etcd"] [2021/12/02 23:27:14.804 +00:00] [WARN] [util.go:121] ["failed to apply request"] [component="embed etcd"] [took=7.435088ms] [request="header: lease_revoke:"] [response=size:28] [error="lease not found"] [2021/12/02 23:27:14.984 +00:00] [INFO] [server.go:253] ["listening gRPC API and status request"] [address=0.0.0.0:8261] [2021/12/02 23:27:15.031 +00:00] [INFO] [election.go:301] ["get response from election observe"] [component=election] [key=/dm-master/leader/33277d7d7360be04] [value="{\"id\":\"dm-master-0\",\"addr\":\"dm-master-0.dm-master.default:8261\"}"] [2021/12/02 23:27:15.031 +00:00] [INFO] [election.go:325] ["current member is not the leader"] [component=election] ["current member"="{\"id\":\"dm-master-1\",\"addr\":\"dm-master-1.dm-master.default:8261\"}"] [leader="{\"id\":\"dm-master-0\",\"addr\":\"dm-master-0.dm-master.default:8261\"}"] [2021/12/02 23:27:15.031 +00:00] [INFO] [election.go:97] ["get new leader"] [leader=dm-master-0] ["current member"=dm-master-1] [2021/12/02 23:27:15.058 +00:00] [INFO] [election.go:279] ["fail to campaign"] [component=election] ["current member"="{\"id\":\"dm-master-1\",\"addr\":\"dm-master-1.dm-master.default:8261\"}"] [error="context canceled"] [2021/12/02 23:27:15.062 +00:00] [INFO] [election.go:301] ["get response from election observe"] [component=election] [key=/dm-master/leader/33277d7d7360be04] [value="{\"id\":\"dm-master-0\",\"addr\":\"dm-master-0.dm-master.default:8261\"}"] [2021/12/02 23:27:15.459 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-master-1.dm-master.default:8261 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-master-1.dm-master.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:27:16.947 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-master-1.dm-master.default:8261 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-master-1.dm-master.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:27:17.324 +00:00] [WARN] [stream.go:277] ["established TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream Message"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id=c6ec7df2da09b327] [2021/12/02 23:27:17.324 +00:00] [WARN] [stream.go:277] ["established TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream MsgApp v2"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id=c6ec7df2da09b327] [2021/12/02 23:27:17.375 +00:00] [WARN] [stream.go:277] ["established TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream MsgApp v2"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id=94ddedb20f20c0c7] [2021/12/02 23:27:17.386 +00:00] [WARN] [stream.go:277] ["established TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream Message"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id=94ddedb20f20c0c7] [2021/12/02 23:28:43.068 +00:00] [WARN] [stream.go:436] ["lost TCP streaming connection with remote peer"] [component="embed etcd"] [stream-reader-type="stream MsgApp v2"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id=94ddedb20f20c0c7] [error=EOF] [2021/12/02 23:28:43.078 +00:00] [WARN] [peer_status.go:68] ["peer became inactive (message send to peer failed)"] [component="embed etcd"] [peer-id=94ddedb20f20c0c7] [error="failed to dial 94ddedb20f20c0c7 on stream MsgApp v2 (peer 94ddedb20f20c0c7 failed to find local node 7f72ccbf7d68a9c1)"] [2021/12/02 23:28:43.078 +00:00] [WARN] [stream.go:436] ["lost TCP streaming connection with remote peer"] [component="embed etcd"] [stream-reader-type="stream Message"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id=94ddedb20f20c0c7] [error=EOF] [2021/12/02 23:28:48.115 +00:00] [WARN] [stream.go:193] ["lost TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream Message"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id=94ddedb20f20c0c7] [2021/12/02 23:28:48.115 +00:00] [WARN] [stream.go:193] ["lost TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream MsgApp v2"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id=94ddedb20f20c0c7] [2021/12/02 23:28:49.755 +00:00] [WARN] [probing_status.go:70] ["prober detected unhealthy status"] [component="embed etcd"] [round-tripper-name=ROUND_TRIPPER_RAFT_MESSAGE] [remote-peer-id=94ddedb20f20c0c7] [rtt=3.683726ms] [error="dial tcp 10.88.0.20:8291: connect: connection refused"] [2021/12/02 23:28:49.790 +00:00] [WARN] [probing_status.go:70] ["prober detected unhealthy status"] [component="embed etcd"] [round-tripper-name=ROUND_TRIPPER_SNAPSHOT] [remote-peer-id=94ddedb20f20c0c7] [rtt=1.679074ms] [error="dial tcp 10.88.0.20:8291: connect: connection refused"] [2021/12/02 23:28:54.763 +00:00] [WARN] [probing_status.go:70] ["prober detected unhealthy status"] [component="embed etcd"] [round-tripper-name=ROUND_TRIPPER_RAFT_MESSAGE] [remote-peer-id=94ddedb20f20c0c7] [rtt=3.683726ms] [error="dial tcp: lookup dm-master-2.dm-master.default on 10.43.0.10:53: no such host"] [2021/12/02 23:28:54.790 +00:00] [WARN] [probing_status.go:70] ["prober detected unhealthy status"] [component="embed etcd"] [round-tripper-name=ROUND_TRIPPER_SNAPSHOT] [remote-peer-id=94ddedb20f20c0c7] [rtt=1.679074ms] [error="dial tcp: lookup dm-master-2.dm-master.default on 10.43.0.10:53: no such host"] [2021/12/02 23:28:59.767 +00:00] [WARN] [probing_status.go:70] ["prober detected unhealthy status"] [component="embed etcd"] [round-tripper-name=ROUND_TRIPPER_RAFT_MESSAGE] [remote-peer-id=94ddedb20f20c0c7] [rtt=3.683726ms] [error="dial tcp: lookup dm-master-2.dm-master.default on 10.43.0.10:53: no such host"] [2021/12/02 23:28:59.791 +00:00] [WARN] [probing_status.go:70] ["prober detected unhealthy status"] [component="embed etcd"] [round-tripper-name=ROUND_TRIPPER_SNAPSHOT] [remote-peer-id=94ddedb20f20c0c7] [rtt=1.679074ms] [error="dial tcp: lookup dm-master-2.dm-master.default on 10.43.0.10:53: no such host"] [2021/12/02 23:29:04.768 +00:00] [WARN] [probing_status.go:70] ["prober detected unhealthy status"] [component="embed etcd"] [round-tripper-name=ROUND_TRIPPER_RAFT_MESSAGE] [remote-peer-id=94ddedb20f20c0c7] [rtt=3.683726ms] [error="dial tcp: lookup dm-master-2.dm-master.default on 10.43.0.10:53: no such host"] [2021/12/02 23:29:04.791 +00:00] [WARN] [probing_status.go:70] ["prober detected unhealthy status"] [component="embed etcd"] [round-tripper-name=ROUND_TRIPPER_SNAPSHOT] [remote-peer-id=94ddedb20f20c0c7] [rtt=1.679074ms] [error="dial tcp: lookup dm-master-2.dm-master.default on 10.43.0.10:53: no such host"] [2021/12/02 23:29:09.768 +00:00] [WARN] [probing_status.go:70] ["prober detected unhealthy status"] [component="embed etcd"] [round-tripper-name=ROUND_TRIPPER_RAFT_MESSAGE] [remote-peer-id=94ddedb20f20c0c7] [rtt=3.683726ms] [error="dial tcp: lookup dm-master-2.dm-master.default on 10.43.0.10:53: no such host"] [2021/12/02 23:29:09.792 +00:00] [WARN] [probing_status.go:70] ["prober detected unhealthy status"] [component="embed etcd"] [round-tripper-name=ROUND_TRIPPER_SNAPSHOT] [remote-peer-id=94ddedb20f20c0c7] [rtt=1.679074ms] [error="dial tcp: lookup dm-master-2.dm-master.default on 10.43.0.10:53: no such host"] [2021/12/02 23:29:14.775 +00:00] [WARN] [probing_status.go:70] ["prober detected unhealthy status"] [component="embed etcd"] [round-tripper-name=ROUND_TRIPPER_RAFT_MESSAGE] [remote-peer-id=94ddedb20f20c0c7] [rtt=3.683726ms] [error="dial tcp: lookup dm-master-2.dm-master.default on 10.43.0.10:53: no such host"] [2021/12/02 23:29:14.792 +00:00] [WARN] [probing_status.go:70] ["prober detected unhealthy status"] [component="embed etcd"] [round-tripper-name=ROUND_TRIPPER_SNAPSHOT] [remote-peer-id=94ddedb20f20c0c7] [rtt=1.679074ms] [error="dial tcp: lookup dm-master-2.dm-master.default on 10.43.0.10:53: no such host"] [2021/12/02 23:29:15.399 +00:00] [WARN] [stream.go:277] ["established TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream MsgApp v2"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id=94ddedb20f20c0c7] [2021/12/02 23:29:15.399 +00:00] [WARN] [stream.go:277] ["established TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream Message"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id=94ddedb20f20c0c7] [2021/12/02 23:29:15.471 +00:00] [WARN] [peer_status.go:68] ["peer became inactive (message send to peer failed)"] [component="embed etcd"] [peer-id=94ddedb20f20c0c7] [error="failed to dial 94ddedb20f20c0c7 on stream MsgApp v2 (dial tcp: lookup dm-master-2.dm-master.default on 10.43.0.10:53: no such host)"] [2021/12/02 23:29:19.777 +00:00] [WARN] [probing_status.go:70] ["prober detected unhealthy status"] [component="embed etcd"] [round-tripper-name=ROUND_TRIPPER_RAFT_MESSAGE] [remote-peer-id=94ddedb20f20c0c7] [rtt=3.683726ms] [error="dial tcp: lookup dm-master-2.dm-master.default on 10.43.0.10:53: no such host"] [2021/12/02 23:29:19.794 +00:00] [WARN] [probing_status.go:70] ["prober detected unhealthy status"] [component="embed etcd"] [round-tripper-name=ROUND_TRIPPER_SNAPSHOT] [remote-peer-id=94ddedb20f20c0c7] [rtt=1.679074ms] [error="dial tcp: lookup dm-master-2.dm-master.default on 10.43.0.10:53: no such host"] [2021/12/02 23:36:43.051 +00:00] [INFO] [main.go:84] ["got signal to exit"] [signal=terminated] [2021/12/02 23:36:43.059 +00:00] [INFO] [server.go:262] ["closing server"] [2021/12/02 23:36:43.059 +00:00] [INFO] [election.go:190] ["election is closing"] [component=election] ["current member"="{\"id\":\"dm-master-1\",\"addr\":\"dm-master-1.dm-master.default:8261\"}"] [2021/12/02 23:36:43.059 +00:00] [INFO] [election.go:325] ["current member is not the leader"] [component=election] ["current member"="{\"id\":\"dm-master-1\",\"addr\":\"dm-master-1.dm-master.default:8261\"}"] [leader="{\"id\":\"dm-master-0\",\"addr\":\"dm-master-0.dm-master.default:8261\"}"] [2021/12/02 23:36:43.059 +00:00] [WARN] [election.go:359] ["ignore notify the leader's information because context canceled"] [component=election] ["current member"="{\"id\":\"dm-master-1\",\"addr\":\"dm-master-1.dm-master.default:8261\"}"] [leader="{\"id\":\"dm-master-0\",\"addr\":\"dm-master-0.dm-master.default:8261\"}"] [2021/12/02 23:36:43.085 +00:00] [WARN] [election.go:272] ["failed to clean election key"] [component=election] [error="context canceled"] [2021/12/02 23:36:43.085 +00:00] [INFO] [election.go:279] ["fail to campaign"] [component=election] ["current member"="{\"id\":\"dm-master-1\",\"addr\":\"dm-master-1.dm-master.default:8261\"}"] [error="context canceled"] [2021/12/02 23:36:43.090 +00:00] [INFO] [election.go:240] ["break campaign loop, context is done"] [component=election] ["current member"="{\"id\":\"dm-master-1\",\"addr\":\"dm-master-1.dm-master.default:8261\"}"] [error="context canceled"] [2021/12/02 23:36:43.095 +00:00] [INFO] [election.go:198] ["election is closed"] [component=election] ["current member"="{\"id\":\"dm-master-1\",\"addr\":\"dm-master-1.dm-master.default:8261\"}"] [2021/12/02 23:36:43.096 +00:00] [WARN] [grpclog.go:66] ["transport: http2Server.HandleStreams failed to read frame: read tcp 10.88.0.38:8261->10.88.0.41:38104: use of closed network connection"] [component="embed etcd"] [2021/12/02 23:36:43.096 +00:00] [WARN] [grpclog.go:66] ["transport: http2Server.HandleStreams failed to read frame: read tcp 10.88.0.38:8261->10.88.0.42:55150: use of closed network connection"] [component="embed etcd"] [2021/12/02 23:36:43.096 +00:00] [WARN] [grpclog.go:66] ["transport: http2Server.HandleStreams failed to read frame: read tcp 127.0.0.1:8261->127.0.0.1:41266: use of closed network connection"] [component="embed etcd"] [2021/12/02 23:36:43.096 +00:00] [WARN] [grpclog.go:66] ["transport: http2Server.HandleStreams failed to read frame: read tcp 10.88.0.38:8261->10.88.0.38:42400: use of closed network connection"] [component="embed etcd"] [2021/12/02 23:36:43.096 +00:00] [WARN] [grpclog.go:66] ["transport: http2Server.HandleStreams failed to read frame: read tcp 10.88.0.38:8261->10.88.0.40:56322: use of closed network connection"] [component="embed etcd"] [2021/12/02 23:36:43.096 +00:00] [WARN] [grpclog.go:66] ["transport: http2Server.HandleStreams failed to read frame: read tcp 10.88.0.38:8261->10.88.0.32:40140: use of closed network connection"] [component="embed etcd"] [2021/12/02 23:36:43.096 +00:00] [WARN] [stream.go:291] ["closed TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream MsgApp v2"] [remote-peer-id=94ddedb20f20c0c7] [2021/12/02 23:36:43.096 +00:00] [WARN] [stream.go:301] ["stopped TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream MsgApp v2"] [remote-peer-id=94ddedb20f20c0c7] [2021/12/02 23:36:43.097 +00:00] [WARN] [stream.go:291] ["closed TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream Message"] [remote-peer-id=94ddedb20f20c0c7] [2021/12/02 23:36:43.097 +00:00] [WARN] [stream.go:301] ["stopped TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream Message"] [remote-peer-id=94ddedb20f20c0c7] [2021/12/02 23:36:43.097 +00:00] [WARN] [stream.go:436] ["lost TCP streaming connection with remote peer"] [component="embed etcd"] [stream-reader-type="stream MsgApp v2"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id=94ddedb20f20c0c7] [error="context canceled"] [2021/12/02 23:36:43.097 +00:00] [WARN] [peer_status.go:68] ["peer became inactive (message send to peer failed)"] [component="embed etcd"] [peer-id=94ddedb20f20c0c7] [error="failed to read 94ddedb20f20c0c7 on stream MsgApp v2 (context canceled)"] [2021/12/02 23:36:43.097 +00:00] [WARN] [stream.go:436] ["lost TCP streaming connection with remote peer"] [component="embed etcd"] [stream-reader-type="stream Message"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id=94ddedb20f20c0c7] [error="context canceled"] [2021/12/02 23:36:43.097 +00:00] [WARN] [stream.go:291] ["closed TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream MsgApp v2"] [remote-peer-id=c6ec7df2da09b327] [2021/12/02 23:36:43.097 +00:00] [WARN] [stream.go:301] ["stopped TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream MsgApp v2"] [remote-peer-id=c6ec7df2da09b327] [2021/12/02 23:36:43.097 +00:00] [WARN] [stream.go:291] ["closed TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream Message"] [remote-peer-id=c6ec7df2da09b327] [2021/12/02 23:36:43.097 +00:00] [WARN] [stream.go:301] ["stopped TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream Message"] [remote-peer-id=c6ec7df2da09b327] [2021/12/02 23:36:43.097 +00:00] [WARN] [stream.go:436] ["lost TCP streaming connection with remote peer"] [component="embed etcd"] [stream-reader-type="stream MsgApp v2"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id=c6ec7df2da09b327] [error="context canceled"] [2021/12/02 23:36:43.097 +00:00] [WARN] [peer_status.go:68] ["peer became inactive (message send to peer failed)"] [component="embed etcd"] [peer-id=c6ec7df2da09b327] [error="failed to read c6ec7df2da09b327 on stream MsgApp v2 (context canceled)"] [2021/12/02 23:36:43.098 +00:00] [WARN] [stream.go:436] ["lost TCP streaming connection with remote peer"] [component="embed etcd"] [stream-reader-type="stream Message"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id=c6ec7df2da09b327] [error="context canceled"] [2021/12/02 23:36:43.108 +00:00] [WARN] [http.go:448] ["failed to find remote peer in cluster"] [component="embed etcd"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id-stream-handler=7f72ccbf7d68a9c1] [remote-peer-id-from=94ddedb20f20c0c7] [cluster-id=b93483a4fb43c8bb] [2021/12/02 23:36:43.109 +00:00] [WARN] [http.go:448] ["failed to find remote peer in cluster"] [component="embed etcd"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id-stream-handler=7f72ccbf7d68a9c1] [remote-peer-id-from=c6ec7df2da09b327] [cluster-id=b93483a4fb43c8bb] [2021/12/02 23:36:43.111 +00:00] [WARN] [http.go:448] ["failed to find remote peer in cluster"] [component="embed etcd"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id-stream-handler=7f72ccbf7d68a9c1] [remote-peer-id-from=c6ec7df2da09b327] [cluster-id=b93483a4fb43c8bb] [2021/12/02 23:36:43.112 +00:00] [WARN] [http.go:448] ["failed to find remote peer in cluster"] [component="embed etcd"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id-stream-handler=7f72ccbf7d68a9c1] [remote-peer-id-from=94ddedb20f20c0c7] [cluster-id=b93483a4fb43c8bb] [2021/12/02 23:36:43.117 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-master-1.dm-master.default:8261 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp 10.88.0.38:8261: connect: connection refused\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:36:43.121 +00:00] [INFO] [server.go:264] ["server closed"] [2021/12/02 23:36:43.121 +00:00] [INFO] [main.go:91] ["dm-master exit"] [2021/12/02 23:37:14.822 +00:00] [INFO] [printer.go:54] ["Welcome to dm-master"] ["Release Version"=v5.2.0-master] ["Git Commit Hash"=ee4a7f889449a557081a2529da5c9265c9d015fc] ["Git Branch"=master] ["UTC Build Time"="2021-12-02 23:16:44"] ["Go Version"="go version go1.16.10 linux/amd64"] [2021/12/02 23:37:14.823 +00:00] [INFO] [main.go:63] ["dm-master config"="{\"log-level\":\"info\",\"log-file\":\"/log/dm-master-1.log\",\"log-format\":\"text\",\"log-rotate\":\"\",\"rpc-timeout\":\"30s\",\"rpc-rate-limit\":10,\"rpc-rate-burst\":40,\"master-addr\":\"0.0.0.0:8261\",\"advertise-addr\":\"dm-master-1.dm-master.default:8261\",\"config-file\":\"\",\"name\":\"dm-master-1\",\"data-dir\":\"/data\",\"peer-urls\":\"http://:8291\",\"advertise-peer-urls\":\"http://dm-master-1.dm-master.default:8291\",\"initial-cluster\":\"dm-master-0=http://dm-master-0.dm-master.default:8291,dm-master-1=http://dm-master-1.dm-master.default:8291,dm-master-2=http://dm-master-2.dm-master.default:8291\",\"initial-cluster-state\":\"new\",\"join\":\"\",\"max-txn-ops\":2048,\"max-request-bytes\":1572864,\"auto-compaction-mode\":\"periodic\",\"auto-compaction-retention\":\"1h\",\"quota-backend-bytes\":2147483648,\"v1-sources-path\":\"\",\"ssl-ca\":\"\",\"ssl-cert\":\"\",\"ssl-key\":\"\",\"cert-allowed-cn\":null,\"ExperimentalFeatures\":{\"OpenAPI\":false}}"] [2021/12/02 23:37:14.823 +00:00] [INFO] [config.go:437] ["Set log level of etcd to `warn`, if you want to log more message about etcd, change log-level to `debug` in master configuration file"] [2021/12/02 23:37:14.823 +00:00] [INFO] [server.go:148] ["config after join prepared"] [config="{\"log-level\":\"info\",\"log-file\":\"/log/dm-master-1.log\",\"log-format\":\"text\",\"log-rotate\":\"\",\"rpc-timeout\":\"30s\",\"rpc-rate-limit\":10,\"rpc-rate-burst\":40,\"master-addr\":\"0.0.0.0:8261\",\"advertise-addr\":\"dm-master-1.dm-master.default:8261\",\"config-file\":\"\",\"name\":\"dm-master-1\",\"data-dir\":\"/data\",\"peer-urls\":\"http://:8291\",\"advertise-peer-urls\":\"http://dm-master-1.dm-master.default:8291\",\"initial-cluster\":\"dm-master-0=http://dm-master-0.dm-master.default:8291,dm-master-1=http://dm-master-1.dm-master.default:8291,dm-master-2=http://dm-master-2.dm-master.default:8291\",\"initial-cluster-state\":\"new\",\"join\":\"\",\"max-txn-ops\":2048,\"max-request-bytes\":1572864,\"auto-compaction-mode\":\"periodic\",\"auto-compaction-retention\":\"1h\",\"quota-backend-bytes\":2147483648,\"v1-sources-path\":\"\",\"ssl-ca\":\"\",\"ssl-cert\":\"\",\"ssl-key\":\"\",\"cert-allowed-cn\":null,\"ExperimentalFeatures\":{\"OpenAPI\":false}}"] [2021/12/02 23:37:14.828 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-master-1.dm-master.default:8261 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-master-1.dm-master.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:37:15.849 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-master-1.dm-master.default:8261 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-master-1.dm-master.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:37:16.135 +00:00] [WARN] [store.go:1366] ["simple token is not cryptographically signed"] [component="embed etcd"] [2021/12/02 23:37:16.347 +00:00] [WARN] [util.go:121] ["failed to apply request"] [component="embed etcd"] [took=438.604µs] [request="header: lease_revoke:"] [response=size:28] [error="lease not found"] [2021/12/02 23:37:16.419 +00:00] [INFO] [server.go:253] ["listening gRPC API and status request"] [address=0.0.0.0:8261] [2021/12/02 23:37:16.426 +00:00] [INFO] [election.go:301] ["get response from election observe"] [component=election] [key=/dm-master/leader/33277d7d7360be04] [value="{\"id\":\"dm-master-0\",\"addr\":\"dm-master-0.dm-master.default:8261\"}"] [2021/12/02 23:37:16.426 +00:00] [INFO] [election.go:325] ["current member is not the leader"] [component=election] ["current member"="{\"id\":\"dm-master-1\",\"addr\":\"dm-master-1.dm-master.default:8261\"}"] [leader="{\"id\":\"dm-master-0\",\"addr\":\"dm-master-0.dm-master.default:8261\"}"] [2021/12/02 23:37:16.427 +00:00] [INFO] [election.go:97] ["get new leader"] [leader=dm-master-0] ["current member"=dm-master-1] [2021/12/02 23:37:16.433 +00:00] [INFO] [election.go:279] ["fail to campaign"] [component=election] ["current member"="{\"id\":\"dm-master-1\",\"addr\":\"dm-master-1.dm-master.default:8261\"}"] [error="context canceled"] [2021/12/02 23:37:16.444 +00:00] [INFO] [election.go:301] ["get response from election observe"] [component=election] [key=/dm-master/leader/33277d7d7360be04] [value="{\"id\":\"dm-master-0\",\"addr\":\"dm-master-0.dm-master.default:8261\"}"] [2021/12/02 23:37:17.535 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-master-1.dm-master.default:8261 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp: lookup dm-master-1.dm-master.default on 10.43.0.10:53: no such host\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:37:18.213 +00:00] [WARN] [stream.go:277] ["established TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream Message"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id=94ddedb20f20c0c7] [2021/12/02 23:37:18.214 +00:00] [WARN] [stream.go:277] ["established TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream MsgApp v2"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id=94ddedb20f20c0c7] [2021/12/02 23:37:18.308 +00:00] [WARN] [stream.go:277] ["established TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream Message"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id=c6ec7df2da09b327] [2021/12/02 23:37:18.309 +00:00] [WARN] [stream.go:277] ["established TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream MsgApp v2"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id=c6ec7df2da09b327] [2021/12/02 23:40:43.118 +00:00] [WARN] [grpclog.go:42] ["grpc: addrConn.createTransport failed to connect to {dm-master-0.dm-master.default:8261 0 }. Err: connection error: desc = \"transport: Error while dialing dial tcp 10.88.0.16:8261: connect: connection refused\". Reconnecting..."] [component="embed etcd"] [2021/12/02 23:40:43.120 +00:00] [INFO] [election.go:301] ["get response from election observe"] [component=election] [key=/dm-master/leader/40c77d7d7b0a4304] [value="{\"id\":\"dm-master-2\",\"addr\":\"dm-master-2.dm-master.default:8261\"}"] [2021/12/02 23:40:43.124 +00:00] [INFO] [election.go:325] ["current member is not the leader"] [component=election] ["current member"="{\"id\":\"dm-master-1\",\"addr\":\"dm-master-1.dm-master.default:8261\"}"] [leader="{\"id\":\"dm-master-2\",\"addr\":\"dm-master-2.dm-master.default:8261\"}"] [2021/12/02 23:40:43.124 +00:00] [INFO] [election.go:97] ["get new leader"] [leader=dm-master-2] ["current member"=dm-master-1] [2021/12/02 23:40:43.237 +00:00] [WARN] [stream.go:436] ["lost TCP streaming connection with remote peer"] [component="embed etcd"] [stream-reader-type="stream MsgApp v2"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id=c6ec7df2da09b327] [error=EOF] [2021/12/02 23:40:43.238 +00:00] [WARN] [stream.go:436] ["lost TCP streaming connection with remote peer"] [component="embed etcd"] [stream-reader-type="stream Message"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id=c6ec7df2da09b327] [error=EOF] [2021/12/02 23:40:43.244 +00:00] [WARN] [peer_status.go:68] ["peer became inactive (message send to peer failed)"] [component="embed etcd"] [peer-id=c6ec7df2da09b327] [error="failed to dial c6ec7df2da09b327 on stream Message (peer c6ec7df2da09b327 failed to find local node 7f72ccbf7d68a9c1)"] [2021/12/02 23:40:49.571 +00:00] [WARN] [stream.go:193] ["lost TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream MsgApp v2"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id=c6ec7df2da09b327] [2021/12/02 23:40:49.572 +00:00] [WARN] [stream.go:193] ["lost TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream Message"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id=c6ec7df2da09b327] [2021/12/02 23:40:51.277 +00:00] [WARN] [probing_status.go:70] ["prober detected unhealthy status"] [component="embed etcd"] [round-tripper-name=ROUND_TRIPPER_SNAPSHOT] [remote-peer-id=c6ec7df2da09b327] [rtt=3.899215ms] [error="dial tcp: lookup dm-master-0.dm-master.default on 10.43.0.10:53: no such host"] [2021/12/02 23:40:51.280 +00:00] [WARN] [probing_status.go:70] ["prober detected unhealthy status"] [component="embed etcd"] [round-tripper-name=ROUND_TRIPPER_RAFT_MESSAGE] [remote-peer-id=c6ec7df2da09b327] [rtt=5.829526ms] [error="dial tcp: lookup dm-master-0.dm-master.default on 10.43.0.10:53: no such host"] [2021/12/02 23:40:51.471 +00:00] [WARN] [util.go:163] ["apply request took too long"] [component="embed etcd"] [took=8.151158634s] [expected-duration=100ms] [prefix=] [request="header: txn: success: request_put: request_put: request_put: request_put: request_put:> failure:<>>"] [response=size:61] [] [2021/12/02 23:40:51.474 +00:00] [WARN] [util.go:163] ["apply request took too long"] [component="embed etcd"] [took=8.146190596s] [expected-duration=100ms] [prefix="read-only range "] [request="key:\"/dm-master/v2/upstream/config/7265706c6963612d3033\" revision:8263 "] [response="range_response_count:1 size:693"] [] [2021/12/02 23:40:51.493 +00:00] [WARN] [util.go:163] ["apply request took too long"] [component="embed etcd"] [took=1.357641206s] [expected-duration=100ms] [prefix="read-only range "] [request="key:\"/dm-master/leader\" range_end:\"/dm-master/leades\" "] [response="range_response_count:2 size:251"] [] [2021/12/02 23:40:51.501 +00:00] [INFO] [election.go:274] ["successful manually clean election key"] [component=election] ["campaign error"="context canceled"] [2021/12/02 23:40:51.501 +00:00] [INFO] [election.go:279] ["fail to campaign"] [component=election] ["current member"="{\"id\":\"dm-master-1\",\"addr\":\"dm-master-1.dm-master.default:8261\"}"] [error="context canceled"] [2021/12/02 23:40:51.506 +00:00] [INFO] [election.go:301] ["get response from election observe"] [component=election] [key=/dm-master/leader/40c77d7d7b0a4304] [value="{\"id\":\"dm-master-2\",\"addr\":\"dm-master-2.dm-master.default:8261\"}"] [2021/12/02 23:40:53.845 +00:00] [WARN] [util.go:163] ["apply request took too long"] [component="embed etcd"] [took=2.180904846s] [expected-duration=100ms] [prefix=] [request="header: txn: success:> failure:<>>"] [response=size:16] [] [2021/12/02 23:40:56.278 +00:00] [WARN] [probing_status.go:70] ["prober detected unhealthy status"] [component="embed etcd"] [round-tripper-name=ROUND_TRIPPER_SNAPSHOT] [remote-peer-id=c6ec7df2da09b327] [rtt=3.899215ms] [error="dial tcp: lookup dm-master-0.dm-master.default on 10.43.0.10:53: no such host"] [2021/12/02 23:40:56.280 +00:00] [WARN] [probing_status.go:70] ["prober detected unhealthy status"] [component="embed etcd"] [round-tripper-name=ROUND_TRIPPER_RAFT_MESSAGE] [remote-peer-id=c6ec7df2da09b327] [rtt=5.829526ms] [error="dial tcp: lookup dm-master-0.dm-master.default on 10.43.0.10:53: no such host"] [2021/12/02 23:41:01.279 +00:00] [WARN] [probing_status.go:70] ["prober detected unhealthy status"] [component="embed etcd"] [round-tripper-name=ROUND_TRIPPER_SNAPSHOT] [remote-peer-id=c6ec7df2da09b327] [rtt=3.899215ms] [error="dial tcp: lookup dm-master-0.dm-master.default on 10.43.0.10:53: no such host"] [2021/12/02 23:41:01.281 +00:00] [WARN] [probing_status.go:70] ["prober detected unhealthy status"] [component="embed etcd"] [round-tripper-name=ROUND_TRIPPER_RAFT_MESSAGE] [remote-peer-id=c6ec7df2da09b327] [rtt=5.829526ms] [error="dial tcp: lookup dm-master-0.dm-master.default on 10.43.0.10:53: no such host"] [2021/12/02 23:41:06.280 +00:00] [WARN] [probing_status.go:70] ["prober detected unhealthy status"] [component="embed etcd"] [round-tripper-name=ROUND_TRIPPER_SNAPSHOT] [remote-peer-id=c6ec7df2da09b327] [rtt=3.899215ms] [error="dial tcp: lookup dm-master-0.dm-master.default on 10.43.0.10:53: no such host"] [2021/12/02 23:41:06.282 +00:00] [WARN] [probing_status.go:70] ["prober detected unhealthy status"] [component="embed etcd"] [round-tripper-name=ROUND_TRIPPER_RAFT_MESSAGE] [remote-peer-id=c6ec7df2da09b327] [rtt=5.829526ms] [error="dial tcp: lookup dm-master-0.dm-master.default on 10.43.0.10:53: no such host"] [2021/12/02 23:41:11.280 +00:00] [WARN] [probing_status.go:70] ["prober detected unhealthy status"] [component="embed etcd"] [round-tripper-name=ROUND_TRIPPER_SNAPSHOT] [remote-peer-id=c6ec7df2da09b327] [rtt=3.899215ms] [error="dial tcp: lookup dm-master-0.dm-master.default on 10.43.0.10:53: no such host"] [2021/12/02 23:41:11.283 +00:00] [WARN] [probing_status.go:70] ["prober detected unhealthy status"] [component="embed etcd"] [round-tripper-name=ROUND_TRIPPER_RAFT_MESSAGE] [remote-peer-id=c6ec7df2da09b327] [rtt=5.829526ms] [error="dial tcp: lookup dm-master-0.dm-master.default on 10.43.0.10:53: no such host"] [2021/12/02 23:41:16.283 +00:00] [WARN] [probing_status.go:70] ["prober detected unhealthy status"] [component="embed etcd"] [round-tripper-name=ROUND_TRIPPER_SNAPSHOT] [remote-peer-id=c6ec7df2da09b327] [rtt=3.899215ms] [error="dial tcp: lookup dm-master-0.dm-master.default on 10.43.0.10:53: no such host"] [2021/12/02 23:41:16.283 +00:00] [WARN] [probing_status.go:70] ["prober detected unhealthy status"] [component="embed etcd"] [round-tripper-name=ROUND_TRIPPER_RAFT_MESSAGE] [remote-peer-id=c6ec7df2da09b327] [rtt=5.829526ms] [error="dial tcp: lookup dm-master-0.dm-master.default on 10.43.0.10:53: no such host"] [2021/12/02 23:41:16.516 +00:00] [WARN] [stream.go:277] ["established TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream MsgApp v2"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id=c6ec7df2da09b327] [2021/12/02 23:41:16.516 +00:00] [WARN] [stream.go:277] ["established TCP streaming connection with remote peer"] [component="embed etcd"] [stream-writer-type="stream Message"] [local-member-id=7f72ccbf7d68a9c1] [remote-peer-id=c6ec7df2da09b327] [2021/12/02 23:41:16.565 +00:00] [WARN] [peer_status.go:68] ["peer became inactive (message send to peer failed)"] [component="embed etcd"] [peer-id=c6ec7df2da09b327] [error="failed to dial c6ec7df2da09b327 on stream MsgApp v2 (dial tcp: lookup dm-master-0.dm-master.default on 10.43.0.10:53: no such host)"]