Does Prometheus automatically scrape metrics by default from all Namespaces?

[vikram yerneni]

Fellas,

Quick question here: 

Does Prometheus automatically scrape metrics by default from all Namespaces or do we have to specifically add it to the Prometheus configuration for the metrics to get scraped?

I have default scrape_configs from the stable helm setups and they are running fine, however when I deploy a new application on the same EKS cluster, the metrics are not getting scraped by default.

Am I missing something here or do I need to setup an additional configuration to scrape all metrics of deployed application by default?

Thanks

Vikram Y

[Stuart Clark]

There is usually some sort of annotation needed to indicate that something should be scraped. Check you scrape config to see what you currently have configured.

--
Sent from my Android device with K-9 Mail. Please excuse my brevity.

[vikram yerneni]

Thanks Staurt for the reply here. 

And nope, I didn't added the scrape_config for the metrics for this new application that we deployed recently. Before I add the scrape configuration, I want to make sure that Prometheus doesn't scrape all metrics by default (unless its been told). 

Now you confirmed the same.

Thanks again...

[Stuart Clark]

On 23/06/2020 16:40, vikram yerneni wrote:
> Thanks Staurt for the reply here.
>
> And nope, I didn't added the scrape_config for the metrics for this
> new application that we deployed recently. Before I add the scrape
> configuration, I want to make sure that Prometheus doesn't scrape all
> metrics by default (unless its been told).
> Now you confirmed the same.

What is your existing scrape configuration?

[vikram yerneni]

Here it is:

scrape_configs:

  - job_name: prometheus

    static_configs:

      - targets:

        - localhost:9090

  - job_name: 'kubernetes-apiservers'

    kubernetes_sd_configs:

      - role: endpoints

    scheme: https

    tls_config:

      ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt

      insecure_skip_verify: true

    bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

    relabel_configs:

      - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]

        action: keep

        regex: default;kubernetes;https

  - job_name: 'kubernetes-nodes'

    scheme: https

    tls_config:

      ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt

      insecure_skip_verify: true

    bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

    kubernetes_sd_configs:

      - role: node

    relabel_configs:

      - action: labelmap

        regex: __meta_kubernetes_node_label_(.+)

      - target_label: __address__

        replacement: kubernetes.default.svc:443

      - source_labels: [__meta_kubernetes_node_name]

        regex: (.+)

        target_label: __metrics_path__

        replacement: /api/v1/nodes/$1/proxy/metrics

  - job_name: 'kubernetes-nodes-cadvisor'

    scheme: https

    tls_config:

      ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt

      insecure_skip_verify: true

    bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

    kubernetes_sd_configs:

      - role: node

    relabel_configs:

      - action: labelmap

        regex: __meta_kubernetes_node_label_(.+)

      - target_label: __address__

        replacement: kubernetes.default.svc:443

      - source_labels: [__meta_kubernetes_node_name]

        regex: (.+)

        target_label: __metrics_path__

        replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor

    metric_relabel_configs:

      - action: replace

        source_labels: [id]

        regex: '^/machine\.slice/machine-rkt\\x2d([^\\]+)\\.+/([^/]+)\.service$'

        target_label: rkt_container_name

        replacement: '${2}-${1}'

      - action: replace

        source_labels: [id]

        regex: '^/system\.slice/(.+)\.service$'

        target_label: systemd_service_name

        replacement: '${1}'

  - job_name: 'kubernetes-service-endpoints'

    kubernetes_sd_configs:

      - role: endpoints

    relabel_configs:

      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]

        action: keep

        regex: true

      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]

        action: replace

        target_label: __scheme__

        regex: (https?)

      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]

        action: replace

        target_label: __metrics_path__

        regex: (.+)

      - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]

        action: replace

        target_label: __address__

        regex: ([^:]+)(?::\d+)?;(\d+)

        replacement: $1:$2

      - action: labelmap

        regex: __meta_kubernetes_service_label_(.+)

      - source_labels: [__meta_kubernetes_namespace]

        action: replace

        target_label: kubernetes_namespace

      - source_labels: [__meta_kubernetes_service_name]

        action: replace

        target_label: kubernetes_name

      - source_labels: [__meta_kubernetes_pod_node_name]

        action: replace

        target_label: kubernetes_node

  - job_name: 'kubernetes-service-endpoints-slow'

    scrape_interval: 5m

    scrape_timeout: 30s

    kubernetes_sd_configs:

      - role: endpoints

    relabel_configs:

      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape_slow]

        action: keep

        regex: true

      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]

        action: replace

        target_label: __scheme__

        regex: (https?)

      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]

        action: replace

        target_label: __metrics_path__

        regex: (.+)

      - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]

        action: replace

        target_label: __address__

        regex: ([^:]+)(?::\d+)?;(\d+)

        replacement: $1:$2

      - action: labelmap

        regex: __meta_kubernetes_service_label_(.+)

      - source_labels: [__meta_kubernetes_namespace]

        action: replace

        target_label: kubernetes_namespace

      - source_labels: [__meta_kubernetes_service_name]

        action: replace

        target_label: kubernetes_name

      - source_labels: [__meta_kubernetes_pod_node_name]

        action: replace

        target_label: kubernetes_node

  - job_name: 'prometheus-pushgateway'

    honor_labels: true

    kubernetes_sd_configs:

      - role: service

    relabel_configs:

      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe]

        action: keep

        regex: pushgateway

  - job_name: 'kubernetes-services'

    metrics_path: /probe

    params:

      module: [http_2xx]

    kubernetes_sd_configs:

      - role: service

    relabel_configs:

      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe]

        action: keep

        regex: true

      - source_labels: [__address__]

        target_label: __param_target

      - target_label: __address__

        replacement: blackbox

      - source_labels: [__param_target]

        target_label: instance

      - action: labelmap

        regex: __meta_kubernetes_service_label_(.+)

      - source_labels: [__meta_kubernetes_namespace]

        target_label: kubernetes_namespace

      - source_labels: [__meta_kubernetes_service_name]

        target_label: kubernetes_name

  - job_name: 'kubernetes-pods'

    kubernetes_sd_configs:

      - role: pod

    relabel_configs:

      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]

        action: keep

        regex: true

      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]

        action: replace

        target_label: __metrics_path__

        regex: (.+)

      - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]

        action: replace

        regex: ([^:]+)(?::\d+)?;(\d+)

        replacement: $1:$2

        target_label: __address__

      - action: labelmap

        regex: __meta_kubernetes_pod_label_(.+)

      - source_labels: [__meta_kubernetes_namespace]

        action: replace

        target_label: kubernetes_namespace

      - source_labels: [__meta_kubernetes_pod_name]

        action: replace

        target_label: kubernetes_pod_name

  - job_name: 'kubernetes-pods-slow'

    scrape_interval: 5m

    scrape_timeout: 30s

    kubernetes_sd_configs:

      - role: pod

    relabel_configs:

      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape_slow]

        action: keep

        regex: true

      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]

        action: replace

        target_label: __metrics_path__

        regex: (.+)

      - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]

        action: replace

        regex: ([^:]+)(?::\d+)?;(\d+)

        replacement: $1:$2

        target_label: __address__

      - action: labelmap

        regex: __meta_kubernetes_pod_label_(.+)

      - source_labels: [__meta_kubernetes_namespace]

        action: replace

        target_label: kubernetes_namespace

      - source_labels: [__meta_kubernetes_pod_name]

        action: replace

        target_label: kubernetes_pod_name

Thanks