Prometheus unable to scrap External ETCD metrics

108 views

Skip to first unread message

Nabarun Sen

unread,

Feb 28, 2021, 12:07:35 PM2/28/21

to Prometheus Users

[root@ip-172-33-31-234 ssl]# cat /etc/kubernetes/manifests/kube-apiserver.yaml|grep etcd

- --etcd-cafile=/etc/ssl/etcd/ssl/ca.pem

- --etcd-certfile=/etc/ssl/etcd/ssl/node-ip-172-33-31-234.ap-southeast-1.compute.internal.pem

- --etcd-keyfile=/etc/ssl/etcd/ssl/node-ip-172-33-31-234.ap-southeast-1.compute.internal-key.pem

- --etcd-servers=https://172.33.31.234:2379,https://172.33.47.146:2379,https://172.33.74.37:2379

- --storage-backend=etcd3

- mountPath: /etc/ssl/etcd/ssl

name: etcd-certs-0

path: /etc/ssl/etcd/ssl

name: etcd-certs-0

[root@ip-172-33-31-234 ssl]# ls -rlt /etc/etcd/ssl/^C

[root@ip-172-33-31-234 ssl]# ls -rlt /etc/ssl/etcd/ssl/ca.pem /etc/ssl/etcd/ssl/node-ip-172-33-31-234.ap-southeast-1.compute.internal.pem /etc/ssl/etcd/ssl/node-ip-172-33-31-234.ap-southeast-1.compute.internal-key.pem

-rwx------. 1 kube root 1090 Feb 28 05:48 /etc/ssl/etcd/ssl/ca.pem

-rwx------. 1 kube root 1675 Feb 28 05:49 /etc/ssl/etcd/ssl/node-ip-172-33-31-234.ap-southeast-1.compute.internal-key.pem

-rwx------. 1 kube root 1639 Feb 28 05:49 /etc/ssl/etcd/ssl/node-ip-172-33-31-234.ap-southeast-1.compute.internal.pem

[root@ip-172-33-31-234 ssl]# mkdir /tmp/test

[root@ip-172-33-31-234 ssl]# cp -p /etc/ssl/etcd/ssl/ca.pem /etc/ssl/etcd/ssl/node-ip-172-33-31-234.ap-southeast-1.compute.internal.pem /etc/ssl/etcd/ssl/node-ip-172-33-31-234.ap-southeast-1.compute.internal-key.pem /tmp/test

[root@ip-172-33-31-234 ssl]# cd /tmp/test

[root@ip-172-33-31-234 test]# ls -rlt

total 12

-rwx------. 1 kube root 1090 Feb 28 05:48 ca.pem

-rwx------. 1 kube root 1675 Feb 28 05:49 node-ip-172-33-31-234.ap-southeast-1.compute.internal-key.pem

-rwx------. 1 kube root 1639 Feb 28 05:49 node-ip-172-33-31-234.ap-southeast-1.compute.internal.pem

[root@ip-172-33-31-234 test]# cp -p /etc/ssl/etcd/ssl/ca-key.pem .

[root@ip-172-33-31-234 test]# ls -rlt

total 16

-rwx------. 1 kube root 1675 Feb 28 05:48 ca-key.pem

-rwx------. 1 kube root 1090 Feb 28 05:48 ca.pem

-rwx------. 1 kube root 1675 Feb 28 05:49 node-ip-172-33-31-234.ap-southeast-1.compute.internal-key.pem

-rwx------. 1 kube root 1639 Feb 28 05:49 node-ip-172-33-31-234.ap-southeast-1.compute.internal.pem

[root@ip-172-33-31-234 test]# mv mv node-ip-172-33-31-234.ap-southeast-1.compute.internal.pem^C

[root@ip-172-33-31-234 test]# mv node-ip-172-33-31-234.ap-southeast-1.compute.internal.pem etcd.pem

[root@ip-172-33-31-234 test]# mv node-ip-172-33-31-234.ap-southeast-1.compute.internal-key.pem etcd-key.pem

[root@ip-172-33-31-234 test]# ls -rlt

total 16

-rwx------. 1 kube root 1675 Feb 28 05:48 ca-key.pem

-rwx------. 1 kube root 1090 Feb 28 05:48 ca.pem

-rwx------. 1 kube root 1675 Feb 28 05:49 etcd-key.pem

-rwx------. 1 kube root 1639 Feb 28 05:49 etcd.pem

[root@ip-172-33-31-234 test]# vi openssl.cnf

[root@ip-172-33-31-234 test]# openssl genrsa -out etcd.key 2048

Generating RSA private key, 2048 bit long modulus

...................................+++

.....................+++

e is 65537 (0x10001)

[root@ip-172-33-31-234 test]# ls -lrt

total 24

-rwx------. 1 kube root 1675 Feb 28 05:48 ca-key.pem

-rwx------. 1 kube root 1090 Feb 28 05:48 ca.pem

-rwx------. 1 kube root 1675 Feb 28 05:49 etcd-key.pem

-rwx------. 1 kube root 1639 Feb 28 05:49 etcd.pem

-rw-r--r--. 1 root root 243 Feb 28 16:45 openssl.cnf

-rw-r--r--. 1 root root 1675 Feb 28 16:46 etcd.key

[root@ip-172-33-31-234 test]# openssl req -new -key etcd.key -out etcd.csr -subj "/CN=etcd" -config openssl.cnf

[root@ip-172-33-31-234 test]# ls -lrt

total 28

-rwx------. 1 kube root 1675 Feb 28 05:48 ca-key.pem

-rwx------. 1 kube root 1090 Feb 28 05:48 ca.pem

-rwx------. 1 kube root 1675 Feb 28 05:49 etcd-key.pem

-rwx------. 1 kube root 1639 Feb 28 05:49 etcd.pem

-rw-r--r--. 1 root root 243 Feb 28 16:45 openssl.cnf

-rw-r--r--. 1 root root 1675 Feb 28 16:46 etcd.key

-rw-r--r--. 1 root root 980 Feb 28 16:46 etcd.csr

[root@ip-172-33-31-234 test]# openssl x509 -req -in etcd.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out etcd.crt -days 1825 -extensions v3_req -extfile openssl.cnf

Signature ok

subject=/CN=etcd

Getting CA Private Key

[root@ip-172-33-31-234 test]# ls -lrt

total 36

-rwx------. 1 kube root 1675 Feb 28 05:48 ca-key.pem

-rwx------. 1 kube root 1090 Feb 28 05:48 ca.pem

-rwx------. 1 kube root 1675 Feb 28 05:49 etcd-key.pem

-rwx------. 1 kube root 1639 Feb 28 05:49 etcd.pem

-rw-r--r--. 1 root root 243 Feb 28 16:45 openssl.cnf

-rw-r--r--. 1 root root 1675 Feb 28 16:46 etcd.key

-rw-r--r--. 1 root root 980 Feb 28 16:46 etcd.csr

-rw-r--r--. 1 root root 17 Feb 28 16:47 ca.srl

-rw-r--r--. 1 root root 1054 Feb 28 16:47 etcd.crt

[root@ip-172-33-31-234 test]# cat <<-EOF > etcd-cert-secret.yaml

> apiVersion: v1

> data:

> etcd-client-ca.crt: "$(cat ca.pem | base64 --wrap=0)"

> etcd-client.crt: "$(cat etcd.crt | base64 --wrap=0)"

> etcd-client.key: "$(cat etcd.key | base64 --wrap=0)"

> kind: Secret

> metadata:

> name: etcd-client-cert

> namespace: monitoring

> type: Opaque

> EOF

[root@ip-172-33-31-234 test]# ls -lrt

total 44

-rwx------. 1 kube root 1675 Feb 28 05:48 ca-key.pem

-rwx------. 1 kube root 1090 Feb 28 05:48 ca.pem

-rwx------. 1 kube root 1675 Feb 28 05:49 etcd-key.pem

-rwx------. 1 kube root 1639 Feb 28 05:49 etcd.pem

-rw-r--r--. 1 root root 243 Feb 28 16:45 openssl.cnf

-rw-r--r--. 1 root root 1675 Feb 28 16:46 etcd.key

-rw-r--r--. 1 root root 980 Feb 28 16:46 etcd.csr

-rw-r--r--. 1 root root 17 Feb 28 16:47 ca.srl

-rw-r--r--. 1 root root 1054 Feb 28 16:47 etcd.crt

-rw-r--r--. 1 root root 5275 Feb 28 16:47 etcd-cert-secret.yaml

[root@ip-172-33-31-234 test]# vi etcd-cert-secret.yaml

[root@ip-172-33-31-234 test]# kubectl get secret -n monitoring

NAME TYPE DATA AGE

alertmanager-prometheus-kube-prometheus-alertmanager Opaque 1 57m

default-token-h4mm8 kubernetes.io/service-account-token 3 70m

prometheus-grafana Opaque 3 57m

prometheus-grafana-test-token-l9274 kubernetes.io/service-account-token 3 57m

prometheus-grafana-token-n6bfb kubernetes.io/service-account-token 3 57m

prometheus-kube-prometheus-admission Opaque 3 69m

prometheus-kube-prometheus-alertmanager-token-dlfqc kubernetes.io/service-account-token 3 57m

prometheus-kube-prometheus-operator-token-jpx2c kubernetes.io/service-account-token 3 57m

prometheus-kube-prometheus-prometheus-token-xp7bs kubernetes.io/service-account-token 3 57m

prometheus-kube-state-metrics-token-7mnvg kubernetes.io/service-account-token 3 57m

prometheus-prometheus-kube-prometheus-prometheus Opaque 1 57m

prometheus-prometheus-kube-prometheus-prometheus-tls-assets Opaque 0 57m

prometheus-prometheus-node-exporter-token-f7h2c kubernetes.io/service-account-token 3 57m

sh.helm.release.v1.prometheus.v1 helm.sh/release.v1 1 57m

[root@ip-172-33-31-234 test]# vi etcd-cert-secret.yaml

[root@ip-172-33-31-234 test]# kubectl apply -f etcd-cert-secret.yaml

secret/etcd-client-cert created

[root@ip-172-33-31-234 test]# kubectl describe secret etcd-client-cert -n monitoring

Name: etcd-client-cert

Namespace: monitoring

Labels: <none>

Annotations: <none>

Type: Opaque

Data

====

etcd-client.key: 1675 bytes

etcd-client-ca.crt: 1090 bytes

etcd-client.crt: 1054 bytes

Reply all

Reply to author

Forward

0 new messages