Accessing target servers in prometheus showing Down state

[Gita cliff]

Hello all 

i have set up a docker-compose file for monitoring metrics from node exporter and cAdvisor as seen below .
My question is why is it that under targets in prometheus all the servers are down as seen below in the screenshot?

global:

scrape_interval: 5s

scrape_configs:

- job_name: "prometheus"

static_configs:

- targets: ["localhost:9090"]

- job_name: "grafana"

static_configs:

- targets: ["localhost:4000"]

- job_name: "node"

static_configs:

- targets: ["localhost:9100"]

- job_name: "cAdvisor"

static_configs:

- targets: ["localhost:8088"]

version: '3.9'

services:

grafana:

image: grafana/grafana-oss:latest

container_name: grafana

ports:

- "3000:3000"

volumes:

- grafana-data:/var/lib/grafana

environment:

GF_SECURITY_ADMIN_USER: ${GF_SECURITY_ADMIN_USER:-admin}

GF_SECURITY_ADMIN_PASSWORD: ${GF_SECURITY_ADMIN_PASSWORD:-password}

GF_AUTH_GENERIC_OAUTH_ROLE_ATTRIBUTE_PATH: "contains(roles[*], 'admin') && 'Admin' || contains(roles[*], 'editor') && 'Editor' || 'Viewer'"

GF_SERVER_DOMAIN: ${GF_SERVER_DOMAIN}

GF_SERVER_ROOT_URL: ${KC_GRAFANA_ROOT_URL}

configs:

- target: /etc/grafana/grafana.ini

source: grafana.ini

- target: /etc/grafana/provisioning/datasources/datasource.yml

source: datasource.yml

- target: /etc/grafana/provisioning/dashboards/dashboard.yml

source: dashboard.yml

- target: /etc/grafana/provisioning/dashboards/nodes/node-exporter-full_rev27.json

source: node-exporter-full_rev27.json

- target: /etc/grafana/provisioning/dashboards/containers/logging-universal-dashboard_rev1.json

source: logging-universal-dashboard_rev1.json

networks:

- prometheus-network

prometheus:

image: prom/prometheus:latest

container_name: prometheus

restart: unless-stopped

ports:

- "9090:9090"

volumes:

- prometheus-data:/prometheus

- /var/run/docker.sock:/var/run/docker.sock:ro

- ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro

configs:

- target: /etc/prometheus/prometheus.yml

source: prometheus.yml

command:

- '--config.file=/etc/prometheus/prometheus.yml'

- '--storage.tsdb.path=/prometheus'

- '--web.console.libraries=/etc/prometheus/console_libraries'

- '--web.console.templates=/etc/prometheus/consoles'

networks:

- prometheus-network

cadvisor:

image: gcr.io/cadvisor/cadvisor:v0.47.0

command: -docker_only

container_name: cadvisor

restart: unless-stopped

privileged: true

ports:

- "8088:8080"

volumes:

- /:/rootfs:ro

- /var/run:/var/run:ro

- /sys:/sys:ro

- /var/lib/docker/:/var/lib/docker:ro

- /dev/disk/:/dev/disk:ro

deploy:

mode: global

networks:

- prometheus-network

node-exporter:

image: quay.io/prometheus/node-exporter:latest

container_name: node-exporter

hostname: "{{.Node.ID}}"

command:

- '--path.rootfs=/host'

ports:

- "9100:9100"

volumes:

- '/:/host:ro'

deploy:

mode: global

networks:

- prometheus-network

configs:

grafana.ini:

file: ./grafana/grafana.ini

name: grafana.ini-$${grafana_ini_DIGEST:?err}

labels:

name: grafana

datasource.yml:

file: ./grafana/datasource.yml

name: datasource.yml-$${datasource_yml_DIGEST:?err}

labels:

name: grafana

dashboard.yml:

file: ./grafana/dashboards/dashboard.yml

name: dashboard.yml-$${dashboard_yml_DIGEST:?err}

labels:

name: grafana

node-exporter-full_rev27.json:

file: ./grafana/dashboards/nodes/node-exporter-full_rev27.json

name: node-exporter-full_rev27.json-$${node_exporter_full_rev27_json_DIGEST:?err}

labels:

name: grafana

logging-universal-dashboard_rev1.json:

file: ./grafana/dashboards/containers/logging-universal-dashboard_rev1.json

name: logging-universal-dashboard_rev1.json-$${logging_universal_dashboard_rev1_json_DIGEST:?err}

labels:

name: grafana

prometheus.yml:

file: ./prometheus/prometheus.yml

name: prometheus.yml-$${prometheus_yml_DIGEST:?err}

labels:

name: prometheus

volumes:

prometheus-data:

grafana-data:

networks:

prometheus-network:

name: prometheus-network

driver: bridge

[Bryan Boreham]

The reason they are showing Down is given on the right, in the text ending "... connection refused".

Since the IP address is 127.0.0.1 (specified as "localhost") I can guess this relates to the processes running in different containers, which each have their own network namespace.

Either run them all in the same network namespace or use Service Discovery to find the appropriate address of each.

By the way, this question is suitable for 'prometheus-users' mailing list; it has nothing to do with the development of Prometheus.

Bryan

[Abdur Rofi]

Hello Gita, if you're use the bridge network on docker-compose file, you can't access the cadvisor, node-exporter with localhost. You can set like this cadvisor:8080, node-exporter:9100

https://docs.docker.com/network/drivers/

Maybe you can read this about docker network ⬆️

--
You received this message because you are subscribed to the Google Groups "Prometheus Developers" group.
To unsubscribe from this group and stop receiving emails from it, send an email to prometheus-devel...@googlegroups.com.
To view this discussion on the web visit https://groups.google.com/d/msgid/prometheus-developers/fae7c1aa-95a6-4fa5-9ef4-4a9c42f8a21dn%40googlegroups.com.

[Gita cliff]

thanks Brynn and Rofi 
i have updated the docker-compose file and the prometheus.yml file as seen below, now i have node and prometheus visible 

global:

scrape_interval: 3s

scrape_configs:

- job_name: "prometheus"

static_configs:

- targets: ["prometheus:9090"]

- job_name: "grafana"

static_configs:

- targets: ["grafana:4000"]

- job_name: "node"

static_configs:

- targets: ["node_exporter:9100"]

- job_name: "cAdvisor"

static_configs:

- targets: ["cadvisor:8088"]

version: '3.9'

services:

grafana:

image: grafana/grafana-oss:latest

container_name: grafana

ports:

- "3000:3000"

volumes:

- grafana-data:/var/lib/grafana

environment:

GF_SECURITY_ADMIN_USER: ${GF_SECURITY_ADMIN_USER:-admin}

GF_SECURITY_ADMIN_PASSWORD: ${GF_SECURITY_ADMIN_PASSWORD:-password}

networks:

- prometheus-network

prometheus:

image: prom/prometheus:latest

container_name: prometheus

restart: unless-stopped

ports:

- "9090:9090"

volumes:

- prometheus-data:/prometheus

- /var/run/docker.sock:/var/run/docker.sock:ro

- ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro

configs:

- target: /etc/prometheus/prometheus.yml

source: prometheus.yml

command:

- '--config.file=/etc/prometheus/prometheus.yml'

- '--storage.tsdb.path=/prometheus'

- '--web.console.libraries=/etc/prometheus/console_libraries'

- '--web.console.templates=/etc/prometheus/consoles'

- '--web.enable-lifecycle'

depends_on:

- cadvisor

networks:

- prometheus-network

cadvisor:

image: gcr.io/cadvisor/cadvisor:v0.47.0

container_name: cadvisor

network_mode: "host"

restart: unless-stopped

privileged: true

ports:

- "8088:8080"

volumes:

- /:/rootfs:ro

- /var/run:/var/run:ro

- /sys:/sys:ro

- /var/lib/docker/:/var/lib/docker:ro

- /dev/disk/:/dev/disk:ro

- /etc/machine-id:/etc/machine-id:ro

- /var/lib/dbus/machine-id:/var/lib/dbus/machine-id:ro

devices:

- /dev/kmsg:/dev/kmsg

networks:

- prometheus-network

depends_on:

- redis

redis:

image: redis:latest

container_name: redis_monitor

ports:

- 6378:6379

networks:

- prometheus-network

node_exporter:

image: quay.io/prometheus/node-exporter:latest

container_name: node_exporter

hostname: "{{.Node.ID}}"

network_mode: "host"

driver: local

driver_opts:

type: none

device: ${GRAFANA_DATA:-./grafana/data}

o: bind

networks:

prometheus-network:

name: prometheus-network

driver: bridge