Jaeger UI doesn't work with Elastic search version 7.10.2

[Manisai Bashetty]

Hello team,

We are currently using Jaeger v1.19.2 and recently we upgraded our Elastic search from 7.8.1 to 7.10.2. Till Elastic search v7.8.1, the UI was working fine. But while using ES v7.10.2, we are getting the error in the UI as "Error 400 (Bad Request): all shards failed [type=search_phase_execution_exception] ". We have faced a similar issue with v7.8.1 and we have resolved it by applying index templates manually.  So, should we have to create these templates manually every time or is there a fix for this issue in the later releases? Also does Jaeger support ES v7.10? 

Following are the templates that are applied manually while using ES v7.8.1

For Jaeger-service:

{
  "priority": 10,
  "index_patterns": [
    "*jaeger-service-*"
  ],
  "template": {
    "settings": {
      "index": {
        "mapping": {
          "nested_fields": {
            "limit": "50"
          }
        },
        "requests": {
          "cache": {
            "enable": "true"
          }
        },
        "number_of_shards": "5",
        "number_of_replicas": "1"
      }
    },
    "mappings": {
      "dynamic_templates": [
        {
          "span_tags_map": {
            "path_match": "tag.*",
            "mapping": {
              "ignore_above": 256,
              "type": "keyword"
            }
          }
        },
        {
          "process_tags_map": {
            "path_match": "process.tag.*",
            "mapping": {
              "ignore_above": 256,
              "type": "keyword"
            }
          }
        }
      ],
      "properties": {
        "operationName": {
          "ignore_above": 256,
          "type": "keyword"
        },
        "serviceName": {
          "ignore_above": 256,
          "type": "keyword"
        }
      }
    },
    "aliases": {}
  }
}

For Jaeger-span:

{
  "priority": 11,
  "index_patterns": [
    "*jaeger-span-*"
  ],
  "template": {
    "settings": {
      "index": {
        "mapping": {
          "nested_fields": {
            "limit": "50"
          }
        },
        "requests": {
          "cache": {
            "enable": "true"
          }
        },
        "number_of_shards": "5",
        "number_of_replicas": "1"
      }
    },
    "mappings": {
      "dynamic_templates": [
        {
          "span_tags_map": {
            "path_match": "tag.*",
            "mapping": {
              "ignore_above": 256,
              "type": "keyword"
            }
          }
        },
        {
          "process_tags_map": {
            "path_match": "process.tag.*",
            "mapping": {
              "ignore_above": 256,
              "type": "keyword"
            }
          }
        }
      ],
      "properties": {
        "traceID": {
          "ignore_above": 256,
          "type": "keyword"
        },
        "process": {
          "properties": {
            "tag": {
              "type": "object"
            },
            "serviceName": {
              "ignore_above": 256,
              "type": "keyword"
            },
            "tags": {
              "dynamic": false,
              "type": "nested",
              "properties": {
                "tagType": {
                  "ignore_above": 256,
                  "type": "keyword"
                },
                "value": {
                  "ignore_above": 256,
                  "type": "keyword"
                },
                "key": {
                  "ignore_above": 256,
                  "type": "keyword"
                }
              }
            }
          }
        },
        "startTimeMillis": {
          "format": "epoch_millis",
          "type": "date"
        },
        "references": {
          "dynamic": false,
          "type": "nested",
          "properties": {
            "traceID": {
              "ignore_above": 256,
              "type": "keyword"
            },
            "spanID": {
              "ignore_above": 256,
              "type": "keyword"
            },
            "refType": {
              "ignore_above": 256,
              "type": "keyword"
            }
          }
        },
        "flags": {
          "type": "integer"
        },
        "operationName": {
          "ignore_above": 256,
          "type": "keyword"
        },
        "parentSpanID": {
          "ignore_above": 256,
          "type": "keyword"
        },
        "tags": {
          "dynamic": false,
          "type": "nested",
          "properties": {
            "tagType": {
              "ignore_above": 256,
              "type": "keyword"
            },
            "value": {
              "ignore_above": 256,
              "type": "keyword"
            },
            "key": {
              "ignore_above": 256,
              "type": "keyword"
            }
          }
        },
        "spanID": {
          "ignore_above": 256,
          "type": "keyword"
        },
        "duration": {
          "type": "long"
        },
        "startTime": {
          "type": "long"
        },
        "tag": {
          "type": "object"
        },
        "logs": {
          "dynamic": false,
          "type": "nested",
          "properties": {
            "fields": {
              "dynamic": false,
              "type": "nested",
              "properties": {
                "tagType": {
                  "ignore_above": 256,
                  "type": "keyword"
                },
                "value": {
                  "ignore_above": 256,
                  "type": "keyword"
                },
                "key": {
                  "ignore_above": 256,
                  "type": "keyword"
                }
              }
            },
            "timestamp": {
              "type": "long"
            }
          }
        }
      }
    },
    "aliases": {}
  }
}

[Manisai Bashetty]

I'm getting the following error in Jaeger query logs when using Jaeger v1.28.0 & ES v 7.10.2, and even I couldn't access UI.

{"level":"error","ts":1640801425.8547792,"caller":"app/http_handler.go:487","msg":"HTTP handler, Internal Server Error","error":"search services failed: elastic: Error 400 (Bad Request): all shards failed [type=search_phase_execution_exception]","stacktrace":"github.com/jaegertracing/jaeger/cmd/query/app.(*APIHandler).handleError\n\tgithub.com/jaegertracing/jaeger/cmd/query/app/http_handler.go:487\ngithub.com/jaegertracing/jaeger/cmd/query/app.(*APIHandler).getServices\n\tgithub.com/jaegertracing/jaeger/cmd/query/app/http_handler.go:158\nnet/http.HandlerFunc.ServeHTTP\n\tnet/http/server.go:2046\ngithub.com/opentracing-contrib/go-stdlib/nethttp.MiddlewareFunc.func5\n\tgithub.com/opentracing-contrib/go-s...@v1.0.0/nethttp/server.go:154\nnet/http.HandlerFunc.ServeHTTP\n\tnet/http/server.go:2046\nnet/http.HandlerFunc.ServeHTTP\n\tnet/http/server.go:2046\ngithub.com/gorilla/mux.(*Router).ServeHTTP\n\tgithub.com/gorilla/m...@v1.8.0/mux.go:210\ngithub.com/jaegertracing/jaeger/cmd/query/app.additionalHeadersHandler.func1\n\tgithub.com/jaegertracing/jaeger/cmd/query/app/additional_headers_handler.go:28\nnet/http.HandlerFunc.ServeHTTP\n\tnet/http/server.go:2046\ngithub.com/gorilla/handlers.CompressHandlerLevel.func1\n\tgithub.com/gorilla/hand...@v1.5.1/compress.go:141\nnet/http.HandlerFunc.ServeHTTP\n\tnet/http/server.go:2046\ngithub.com/gorilla/handlers.recoveryHandler.ServeHTTP\n\tgithub.com/gorilla/hand...@v1.5.1/recovery.go:78\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:2878\nnet/http.(*conn).serve\n\tnet/http/server.go:1929"}