SDS/RDS sends two DiscoveryRequest with same config resource name #5744
Description
- envoy use config file
config_dump.txt
in the config, there are only two unique sdsconfig (defaultandROOTCA); so server side should receive two stream requests(one fordefaultand one forROOTCA) when envoy start (envoy may send more requests later depends on server side resp)
"tls_context": {
"common_tls_context": {
"alpn_protocols": [
"istio",
"h2"
],
"tls_certificate_sds_secret_configs": [
{
"name": "default",
"sds_config": {
"api_config_source": {
"api_type": "GRPC",
"grpc_services": [
{
"google_grpc": {
"target_uri": "unix:/var/run/sds/uds_path",
"channel_credentials": {
"local_credentials": {}
},
"call_credentials": [
{
"from_plugin": {
"name": "envoy.grpc_credentials.file_based_metadata",
"config": {
"secret_data": {
"filename": "/var/run/secrets/kubernetes.io/serviceaccount/token"
},
"header_key": "istio_sds_credentail_header-bin"
}
}
}
],
"stat_prefix": "sdsstat",
"credentials_factory_name": "envoy.grpc_credentials.file_based_metadata"
}
}
]
}
}
}
],
"combined_validation_context": {
"default_validation_context": {
"verify_subject_alt_name": [
"spiffe://cluster.local/ns/istio-system/sa/istio-pilot-service-account"
]
},
"validation_context_sds_secret_config": {
"name": "ROOTCA",
"sds_config": {
"api_config_source": {
"api_type": "GRPC",
"grpc_services": [
{
"google_grpc": {
"target_uri": "unix:/var/run/sds/uds_path",
"channel_credentials": {
"local_credentials": {}
},
"call_credentials": [
{
"from_plugin": {
"name": "envoy.grpc_credentials.file_based_metadata",
"config": {
"secret_data": {
"filename": "/var/run/secrets/kubernetes.io/serviceaccount/token"
},
"header_key": "istio_sds_credentail_header-bin"
}
}
}
],
"stat_prefix": "sdsstat",
"credentials_factory_name": "envoy.grpc_credentials.file_based_metadata"
}
}
]
}
}
}
}
},
- however, from both envoy log
envoy1.log
and server side log, envoy send two requests(for same sdsconfig resource name), almost at the same time when new pod starts.
The question is why envoy sends two same requests for same sdsconfig resource ?
envoy logs
[2019-01-28 23:00:25.890][23][trace][upstream] [external/envoy/source/common/config/grpc_mux_impl.cc:63] draining discovery requests 1
[2019-01-28 23:00:25.890][23][trace][upstream] [external/envoy/source/common/config/grpc_mux_impl.cc:114] Sending DiscoveryRequest for type.googleapis.com/envoy.api.v2.auth.Secret: node {
id: "router~10.8.2.83~istio-ingressgateway-6bd597dc46-sfc4t.istio-system~istio-system.svc.cluster.local"
cluster: "istio-ingressgateway"
metadata {
fields {
key: "CONFIG_NAMESPACE"
value {
string_value: "istio-system"
}
}
fields {
key: "ISTIO_META_INSTANCE_IPS"
value {
string_value: "10.8.2.83,10.8.2.83,fe80::3017:e8ff:feab:f9a8"
}
}
fields {
key: "ISTIO_PROXY_SHA"
value {
string_value: "istio-proxy:7738fa3d506d3a1a7be002c1f06377e6516bc245"
}
}
fields {
key: "ISTIO_PROXY_VERSION"
value {
string_value: "1.1.0"
}
}
fields {
key: "ISTIO_VERSION"
value {
string_value: "1.0-dev"
}
}
fields {
key: "POD_NAME"
value {
string_value: "istio-ingressgateway-6bd597dc46-sfc4t"
}
}
fields {
key: "ROUTER_MODE"
value {
string_value: "sni-dnat"
}
}
fields {
key: "istio"
value {
string_value: "sidecar"
}
}
}
locality {
}
build_version: "7738fa3d506d3a1a7be002c1f06377e6516bc245/1.10.0-dev/Clean/RELEASE/BoringSSL"
}
resource_names: "default"
type_url: "type.googleapis.com/envoy.api.v2.auth.Secret"
[2019-01-28 23:00:25.890][23][trace][grpc] [external/envoy/source/common/grpc/google_async_client_impl.cc:197] Queued message to write (609 bytes)
[2019-01-28 23:00:25.890][23][debug][config] [bazel-out/k8-opt/bin/external/envoy/source/common/config/_virtual_includes/grpc_mux_subscription_lib/common/config/grpc_mux_subscription_impl.h:60] gRPC config for type.googleapis.com/envoy.api.v2.ClusterLoadAssignment accepted with 1 resources: [cluster_name: "outbound|80||istio-egressgateway.istio-system.svc.cluster.local"
endpoints {
locality {
region: "us-central1"
zone: "us-central1-a"
}
lb_endpoints {
endpoint {
address {
socket_address {
address: "10.8.2.85"
port_value: 80
}
}
}
metadata {
filter_metadata {
key: "istio"
value {
fields {
key: "uid"
value {
string_value: "kubernetes://istio-egressgateway-7dc5557587-cvljv.istio-system"
}
}
}
}
}
}
load_balancing_weight {
value: 1
}
}
]
[2019-01-28 23:00:25.890][23][debug][upstream] [external/envoy/source/common/upstream/eds.cc:153] EDS hosts or locality weights changed for cluster: outbound|9093||istio-citadel.istio-system.svc.cluster.local current hosts 0 priority 0
[2019-01-28 23:00:25.890][23][trace][upstream] [external/envoy/source/common/upstream/upstream_impl.cc:899] Local locality:
[2019-01-28 23:00:25.890][23][debug][upstream] [external/envoy/source/common/upstream/upstream_impl.cc:685] initializing secondary cluster outbound|9093||istio-citadel.istio-system.svc.cluster.local completed
[2019-01-28 23:00:25.890][23][debug][upstream] [external/envoy/source/common/upstream/cluster_manager_impl.cc:928] membership update for TLS cluster outbound|9093||istio-citadel.istio-system.svc.cluster.local added 1 removed 0
[2019-01-28 23:00:25.891][23][debug][upstream] [external/envoy/source/common/upstream/cluster_manager_impl.cc:91] cm init: init complete: cluster=outbound|9093||istio-citadel.istio-system.svc.cluster.local primary=0 secondary=4
[2019-01-28 23:00:25.891][23][debug][config] [bazel-out/k8-opt/bin/external/envoy/source/common/config/_virtual_includes/grpc_mux_subscription_lib/common/config/grpc_mux_subscription_impl.h:60] gRPC config for type.googleapis.com/envoy.api.v2.ClusterLoadAssignment accepted with 1 resources: [cluster_name: "outbound|9093||istio-citadel.istio-system.svc.cluster.local"
endpoints {
locality {
region: "us-central1"
zone: "us-central1-a"
}
lb_endpoints {
endpoint {
address {
socket_address {
address: "10.8.3.45"
port_value: 9093
}
}
}
metadata {
filter_metadata {
key: "istio"
value {
fields {
key: "uid"
value {
string_value: "kubernetes://istio-citadel-7bbd9df944-r5pmt.istio-system"
}
}
}
}
}
}
load_balancing_weight {
value: 1
}
}
]
[2019-01-28 23:00:25.891][23][debug][upstream] [external/envoy/source/common/upstream/eds.cc:153] EDS hosts or locality weights changed for cluster: outbound|8060||istio-citadel.istio-system.svc.cluster.local current hosts 0 priority 0
[2019-01-28 23:00:25.891][23][trace][upstream] [external/envoy/source/common/upstream/upstream_impl.cc:899] Local locality:
[2019-01-28 23:00:25.891][23][debug][upstream] [external/envoy/source/common/upstream/upstream_impl.cc:685] initializing secondary cluster outbound|8060||istio-citadel.istio-system.svc.cluster.local completed
[2019-01-28 23:00:25.891][23][debug][upstream] [external/envoy/source/common/config/grpc_mux_impl.cc:137] gRPC mux subscribe for type.googleapis.com/envoy.api.v2.auth.Secret
[2019-01-28 23:00:25.891][23][trace][upstream] [external/envoy/source/common/config/grpc_mux_impl.cc:63] draining discovery requests 1
[2019-01-28 23:00:25.891][23][debug][upstream] [external/envoy/source/common/config/grpc_mux_impl.cc:89] No stream available to sendDiscoveryRequest for type.googleapis.com/envoy.api.v2.auth.Secret
[2019-01-28 23:00:25.892][23][debug][upstream] [external/envoy/source/common/config/grpc_mux_impl.cc:48] Establishing new gRPC bidi stream for rpc StreamSecrets(stream .envoy.api.v2.DiscoveryRequest) returns (stream .envoy.api.v2.DiscoveryResponse);
[2019-01-28 23:00:25.892][23][trace][upstream] [external/envoy/source/common/config/grpc_mux_impl.cc:63] draining discovery requests 1
[2019-01-28 23:00:25.892][23][trace][upstream] [external/envoy/source/common/config/grpc_mux_impl.cc:114] Sending DiscoveryRequest for type.googleapis.com/envoy.api.v2.auth.Secret: node {
id: "router~10.8.2.83~istio-ingressgateway-6bd597dc46-sfc4t.istio-system~istio-system.svc.cluster.local"
cluster: "istio-ingressgateway"
metadata {
fields {
key: "CONFIG_NAMESPACE"
value {
string_value: "istio-system"
}
}
fields {
key: "ISTIO_META_INSTANCE_IPS"
value {
string_value: "10.8.2.83,10.8.2.83,fe80::3017:e8ff:feab:f9a8"
}
}
fields {
key: "ISTIO_PROXY_SHA"
value {
string_value: "istio-proxy:7738fa3d506d3a1a7be002c1f06377e6516bc245"
}
}
fields {
key: "ISTIO_PROXY_VERSION"
value {
string_value: "1.1.0"
}
}
fields {
key: "ISTIO_VERSION"
value {
string_value: "1.0-dev"
}
}
fields {
key: "POD_NAME"
value {
string_value: "istio-ingressgateway-6bd597dc46-sfc4t"
}
}
fields {
key: "ROUTER_MODE"
value {
string_value: "sni-dnat"
}
}
fields {
key: "istio"
value {
string_value: "sidecar"
}
}
}
locality {
}
build_version: "7738fa3d506d3a1a7be002c1f06377e6516bc245/1.10.0-dev/Clean/RELEASE/BoringSSL"
}
resource_names: "default"
type_url: "type.googleapis.com/envoy.api.v2.auth.Secret"
[2019-01-28 23:00:25.892][23][trace][grpc] [external/envoy/source/common/grpc/google_async_client_impl.cc:197] Queued message to write (609 bytes)
server side logs -
2019-01-28T22:59:54.629905Z info *******received StreamSecrets
2019-01-28T22:59:54.629993Z info *******token "eyJhbGciOiJSUzI1NiIsImtpZCI6IiJ9.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJiYXIiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlY3JldC5uYW1lIjoiZGVmYXVsdC10b2tlbi1wNmdsOCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50Lm5hbWUiOiJkZWZhdWx0Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQudWlkIjoiNWE2NTk0NDItMjM1MC0xMWU5LTg3MjQtNDIwMTBhODAwMmJhIiwic3ViIjoic3lzdGVtOnNlcnZpY2VhY2NvdW50OmJhcjpkZWZhdWx0In0.pqZjvd3a33jkgpGehLLhM-U13OhZmlkGrujDWt_fbvpwhk4isCfjKntLjsUTY1ZRCTZBdlY1K77Uf4xR5V8HQdWkv6Eo0u82HBAFUaCdTciqbx6IaxHd9siWAlJlB1HAIOs7AALMU2lSjB1CVC-FTKyXqgxE1cqfPAxdrf5bQj5AT-ESYE2t7CjdUtrV8gJLwPEveGoBpafRqN-FoOserX7NDTwyhQfTF_TKsnPPBiLNAk5gXrkdYTu8IpW-AHakKwmWs_k4HEp3Gy23RFn0VVzR-413nvQ06krww3SGnTfbMI5mxhMw5_JW06LLyMt1XEfAY3lLBBJj1qPx6h_WgA"
2019-01-28T22:59:54.631478Z info *******received StreamSecrets
2019-01-28T22:59:54.631522Z info *******token "eyJhbGciOiJSUzI1NiIsImtpZCI6IiJ9.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJiYXIiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlY3JldC5uYW1lIjoiZGVmYXVsdC10b2tlbi1wNmdsOCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50Lm5hbWUiOiJkZWZhdWx0Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQudWlkIjoiNWE2NTk0NDItMjM1MC0xMWU5LTg3MjQtNDIwMTBhODAwMmJhIiwic3ViIjoic3lzdGVtOnNlcnZpY2VhY2NvdW50OmJhcjpkZWZhdWx0In0.pqZjvd3a33jkgpGehLLhM-U13OhZmlkGrujDWt_fbvpwhk4isCfjKntLjsUTY1ZRCTZBdlY1K77Uf4xR5V8HQdWkv6Eo0u82HBAFUaCdTciqbx6IaxHd9siWAlJlB1HAIOs7AALMU2lSjB1CVC-FTKyXqgxE1cqfPAxdrf5bQj5AT-ESYE2t7CjdUtrV8gJLwPEveGoBpafRqN-FoOserX7NDTwyhQfTF_TKsnPPBiLNAk5gXrkdYTu8IpW-AHakKwmWs_k4HEp3Gy23RFn0VVzR-413nvQ06krww3SGnTfbMI5mxhMw5_JW06LLyMt1XEfAY3lLBBJj1qPx6h_WgA"
2019-01-28T22:59:54.634379Z info *******received StreamSecrets
2019-01-28T22:59:54.634687Z info *******token "eyJhbGciOiJSUzI1NiIsImtpZCI6IiJ9.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJiYXIiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlY3JldC5uYW1lIjoiZGVmYXVsdC10b2tlbi1wNmdsOCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50Lm5hbWUiOiJkZWZhdWx0Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQudWlkIjoiNWE2NTk0NDItMjM1MC0xMWU5LTg3MjQtNDIwMTBhODAwMmJhIiwic3ViIjoic3lzdGVtOnNlcnZpY2VhY2NvdW50OmJhcjpkZWZhdWx0In0.pqZjvd3a33jkgpGehLLhM-U13OhZmlkGrujDWt_fbvpwhk4isCfjKntLjsUTY1ZRCTZBdlY1K77Uf4xR5V8HQdWkv6Eo0u82HBAFUaCdTciqbx6IaxHd9siWAlJlB1HAIOs7AALMU2lSjB1CVC-FTKyXqgxE1cqfPAxdrf5bQj5AT-ESYE2t7CjdUtrV8gJLwPEveGoBpafRqN-FoOserX7NDTwyhQfTF_TKsnPPBiLNAk5gXrkdYTu8IpW-AHakKwmWs_k4HEp3Gy23RFn0VVzR-413nvQ06krww3SGnTfbMI5mxhMw5_JW06LLyMt1XEfAY3lLBBJj1qPx6h_WgA"
2019-01-28T22:59:54.635054Z info ***request proxyID "sidecar~10.8.1.36~httpbin-5676f44967-dpvbv.bar~bar.svc.cluster.local", versionInfo "", resourcename "ROOTCA"
2019-01-28T22:59:54.635597Z info ***request proxyID "sidecar~10.8.1.36~httpbin-5676f44967-dpvbv.bar~bar.svc.cluster.local", versionInfo "", resourcename "ROOTCA"
2019-01-28T22:59:54.635798Z info ***request proxyID "sidecar~10.8.1.36~httpbin-5676f44967-dpvbv.bar~bar.svc.cluster.local", versionInfo "", resourcename "default"
2019-01-28T22:59:54.643131Z info *******received StreamSecrets
2019-01-28T22:59:54.643205Z info *******token "eyJhbGciOiJSUzI1NiIsImtpZCI6IiJ9.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJiYXIiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlY3JldC5uYW1lIjoiZGVmYXVsdC10b2tlbi1wNmdsOCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50Lm5hbWUiOiJkZWZhdWx0Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQudWlkIjoiNWE2NTk0NDItMjM1MC0xMWU5LTg3MjQtNDIwMTBhODAwMmJhIiwic3ViIjoic3lzdGVtOnNlcnZpY2VhY2NvdW50OmJhcjpkZWZhdWx0In0.pqZjvd3a33jkgpGehLLhM-U13OhZmlkGrujDWt_fbvpwhk4isCfjKntLjsUTY1ZRCTZBdlY1K77Uf4xR5V8HQdWkv6Eo0u82HBAFUaCdTciqbx6IaxHd9siWAlJlB1HAIOs7AALMU2lSjB1CVC-FTKyXqgxE1cqfPAxdrf5bQj5AT-ESYE2t7CjdUtrV8gJLwPEveGoBpafRqN-FoOserX7NDTwyhQfTF_TKsnPPBiLNAk5gXrkdYTu8IpW-AHakKwmWs_k4HEp3Gy23RFn0VVzR-413nvQ06krww3SGnTfbMI5mxhMw5_JW06LLyMt1XEfAY3lLBBJj1qPx6h_WgA"
2019-01-28T22:59:54.643268Z info ***request proxyID "sidecar~10.8.1.36~httpbin-5676f44967-dpvbv.bar~bar.svc.cluster.local", versionInfo "", resourcename "default"