Envoy continues to send requests to an unhealthy endpoint

[Vsevolod Molchanov]

Hi everyone!

Can anyone please explain why envoy continues to send requests to an unhealthy endpoint when using weighted clusters? In the configuration matcher-services-2 routes to a non-existing address, so it gets UH and should be ejected, however it seems like that doesn’t happen and I keep getting either a valid response or no healthy upstream from the listener.

admin:
  access_log:
    typed_config:
      "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog
      path: /tmp/admin_access.log
  address:
    socket_address:
      address: 0.0.0.0
      port_value: {{ .Values.envoy.adminPort }}

cluster_manager:
  outlier_detection:
    event_log_path: /dev/stdout

static_resources:
  listeners:
    - name: grpc_matcher_listener
      address:
        socket_address:
          address: 0.0.0.0
          port_value: {{ .Values.matcher.envoyPort }}
      filter_chains:
        - filters:
            - name: envoy.http_connection_manager
              typed_config:
                "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
                stat_prefix: egress_http
                codec_type: HTTP2
                stream_idle_timeout: {{ .Values.envoy.stream_idle_timeout }}
                access_log:
                  - name: envoy.access_loggers.file
                    typed_config:
                      "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog
                      path: /dev/stdout
                      log_format: {
                        json_format: {{- tpl (.Files.Get "resources/envoy_access_log_config.json") . | nindent 24 }}
                      }
                http_filters:
                  - name: envoy.filters.http.router
                route_config:
                  name: routes
                  virtual_hosts:
                    - name: matcher_grpc_service
                      domains:
                        - ["*"]
                      routes:
                        - match:
                            prefix: "/"
                            grpc: {}
                          route:
                            weighted_clusters:
                              clusters:
                                - name: matcher-services
                                  weight: 50
                                - name: matcher-services-2
                                  weight: 50
                            auto_host_rewrite: true
                            timeout: {{ .Values.envoy.timeout }}

  clusters:
    - name: matcher-services
      connectTimeout: {{ .Values.envoy.connectTimeout }}
      type: STRICT_DNS
      lb_policy: LEAST_REQUEST
      common_lb_config:
        healthy_panic_threshold:
          value: 0
      dns_lookup_family: V4_ONLY
      typed_extension_protocol_options:
        envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
          "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
          explicit_http_config:
            http2_protocol_options:
              max_concurrent_streams: 1
      load_assignment:
        cluster_name: matcher-services
        endpoints:
          - lb_endpoints:
              - endpoint:
                  address:
                    socket_address:
                      address: yc-crater-matcher.tcsbank.ru
                      port_value: 5001
      outlier_detection:
        consecutive_5xx: 1
        interval: 5s
        base_ejection_time: 30s
        max_ejection_percent: 100
      health_checks:
        - interval: 3s
          timeout: 2s
          unhealthy_threshold: 1
          healthy_threshold: 2
          event_log_path: /dev/stdout
          always_log_health_check_failures: true
          grpc_health_check:
            authority: yc-crater-matcher.tcsbank.ru
    - name: matcher-services-2
      connectTimeout: {{ .Values.envoy.connectTimeout }}
      type: STRICT_DNS
      lb_policy: LEAST_REQUEST
      common_lb_config:
        healthy_panic_threshold:
          value: 100
      dns_lookup_family: V4_ONLY
      typed_extension_protocol_options:
        envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
          "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
          explicit_http_config:
            http2_protocol_options:
              max_concurrent_streams: 1
      load_assignment:
        cluster_name: matcher-services-2
        endpoints:
          - lb_endpoints:
              - endpoint:
                  address:
                    socket_address:
                      address: matcher-svc
                      port_value: 5001
      outlier_detection:
        consecutive_5xx: 1
        interval: 5s
        base_ejection_time: 30s
        max_ejection_percent: 100
      health_checks:
        - interval: 3s
          timeout: 2s
          unhealthy_threshold: 1
          healthy_threshold: 2
          event_log_path: /dev/stdout
          always_log_health_check_failures: true
          grpc_health_check:
            authority: matcher-svc