relable config

[Phan]

Hi, 

     I have my cassadra metrics exposed on port 9103 on cassandra node (https://github.com/datastax/metric-collector-for-apache-cassandra).. I see my metrics showing .. 

i have to send my metrics to cortex using miniprom .. i was able to see some metrics but relable metrics is not showing .. 

i'm not seeing "mcac" in the grafana,, 

global:

  scrape_interval: 15s

  scrape_timeout: 10s

  evaluation_interval: 15s

scrape_configs:

- job_name: default

  file_sd_configs:

  - files:

    - /var/run/mpproxy/sd.json

    refresh_interval: 5m

- job_name: mcac

  scrape_interval: 15s

  scrape_timeout: 15s

  metrics_path: /metrics

  honor_labels: true

  file_sd_configs:

    - files:

      - 'tg_mcac.json'

  relabel_configs:

   #drop metrics we can calculate from prometheus directly

   - source_labels: [__name__]

     regex: .*rate_(mean|1m|5m|15m)

     action: drop

   #save the original name for all metrics

   - source_labels: [__name__]

     regex: (collectd_mcac_.+)

     target_label: prom_name

     replacement: ${1}

     action: replace

   - source_labels: ["prom_name"]

     regex: .+_bucket_(\d+)

     target_label: le

     replacement: ${1}

     action: replace

   - source_labels: ["prom_name"]

     regex: .+_bucket_inf

     target_label: le

     replacement: +Inf

     action: replace

   - source_labels: ["prom_name"]

     regex: .*_histogram_p(\d+)

     target_label: quantile

     replacement: .${1}

     action: replace

   - source_labels: ["prom_name"]

     regex: .*_histogram_min

     target_label: quantile

     replacement: "0"

     action: replace

   - source_labels: ["prom_name"]

     regex: .*_histogram_max

     target_label: quantile

     replacement: "1"

     action: replace

   #Table Metrics *ALL* we can drop

   - source_labels: ["mcac"]

     regex: org\.apache\.cassandra\.metrics\.table\.(\w+)

     action: keep

   #Table Metrics

   - source_labels: ["mcac"]

     regex: org\.apache\.cassandra\.metrics\.table\.(\w+)\.(\w+)\.(\w+)

     target_label: table

     replacement: ${3}

     action: replace

   - source_labels: ["mcac"]

     regex: org\.apache\.cassandra\.metrics\.table\.(\w+)\.(\w+)\.(\w+)

     target_label: keyspace

     replacement: ${2}

     action: replace

   - source_labels: ["mcac"]

     regex: org\.apache\.cassandra\.metrics\.table\.(\w+)\.(\w+)\.(\w+)

     target_label: __name__

     replacement: mcac_table_${1}

     action: replace

   #Keyspace Metrics

   - source_labels: ["mcac"]

     regex: org\.apache\.cassandra\.metrics\.keyspace\.(\w+)\.(\w+)

     target_label: keyspace

     replacement: ${2}

     action: replace

   - source_labels: ["mcac"]

     regex: org\.apache\.cassandra\.metrics\.keyspace\.(\w+)\.(\w+)

     target_label: __name__

     replacement: mcac_keyspace_${1}

     action: replace

   #ThreadPool Metrics (one type is repair.task so we just ignore the second part)

   - source_labels: ["mcac"]

     regex: org\.apache\.cassandra\.metrics\.thread_pools\.(\w+)\.(\w+)\.(\w+).*

     target_label: pool_type

     replacement: ${2}

     action: replace

   - source_labels: ["mcac"]

     regex: org\.apache\.cassandra\.metrics\.thread_pools\.(\w+)\.(\w+)\.(\w+).*

     target_label: pool_name

     replacement: ${3}

     action: replace

   - source_labels: ["mcac"]

     regex: org\.apache\.cassandra\.metrics\.thread_pools\.(\w+)\.(\w+)\.(\w+).*

     target_label: __name__

     replacement: mcac_thread_pools_${1}

     action: replace

   #ClientRequest Metrics

   - source_labels: ["mcac"]

     regex: org\.apache\.cassandra\.metrics\.client_request\.(\w+)\.(\w+)$

     target_label: request_type

     replacement: ${2}

     action: replace

   - source_labels: ["mcac"]

     regex: org\.apache\.cassandra\.metrics\.client_request\.(\w+)\.(\w+)$

     target_label: __name__

     replacement: mcac_client_request_${1}

     action: replace

   - source_labels: ["mcac"]

     regex: org\.apache\.cassandra\.metrics\.client_request\.(\w+)\.(\w+)\.(\w+)$

     target_label: cl

     replacement: ${3}

     action: replace

   - source_labels: ["mcac"]

     regex: org\.apache\.cassandra\.metrics\.client_request\.(\w+)\.(\w+)\.(\w+)$

     target_label: request_type

     replacement: ${2}

     action: replace

   - source_labels: ["mcac"]

     regex: org\.apache\.cassandra\.metrics\.client_request\.(\w+)\.(\w+)\.(\w+)$

     target_label: __name__

     replacement: mcac_client_request_${1}_cl

     action: replace

   #Cache Metrics

   - source_labels: ["mcac"]

     regex: org\.apache\.cassandra\.metrics\.cache\.(\w+)\.(\w+)

     target_label: cache_name

     replacement: ${2}

     action: replace

   - source_labels: ["mcac"]

     regex: org\.apache\.cassandra\.metrics\.cache\.(\w+)\.(\w+)

     target_label: __name__

     replacement: mcac_cache_${1}

     action: replace

   #CQL Metrics

   - source_labels: ["mcac"]

     regex: org\.apache\.cassandra\.metrics\.cql\.(\w+)

     target_label: __name__

     replacement: mcac_cql_${1}

     action: replace

   #Dropped Message Metrics

   - source_labels: ["mcac"]

     regex: org\.apache\.cassandra\.metrics\.dropped_message\.(\w+)\.(\w+)

     target_label: message_type

     replacement: ${2}

     action: replace

   - source_labels: ["mcac"]

     regex: org\.apache\.cassandra\.metrics\.dropped_message\.(\w+)\.(\w+)

     target_label: __name__

     replacement: mcac_dropped_message_${1}

     action: replace

   #Streaming Metrics

   - source_labels: ["mcac"]

     regex: org\.apache\.cassandra\.metrics\.streaming\.(\w+)\.(.+)$

     target_label: peer_ip

     replacement: ${2}

     action: replace

   - source_labels: ["mcac"]

     regex: org\.apache\.cassandra\.metrics\.streaming\.(\w+)\.(.+)$

     target_label: __name__

     replacement: mcac_streaming_${1}

     action: replace

   - source_labels: ["mcac"]

     regex: org\.apache\.cassandra\.metrics\.streaming\.(\w+)$

     target_label: __name__

     replacement: mcac_streaming_${1}

     action: replace

   #CommitLog Metrics

   - source_labels: ["mcac"]

     regex: org\.apache\.cassandra\.metrics\.commit_log\.(\w+)

     target_label: __name__

     replacement: mcac_commit_log_${1}

     action: replace

   #Compaction Metrics

   - source_labels: ["mcac"]

     regex: org\.apache\.cassandra\.metrics\.compaction\.(\w+)

     target_label: __name__

     replacement: mcac_compaction_${1}

     action: replace

   #Storage Metrics

   - source_labels: ["mcac"]

     regex: org\.apache\.cassandra\.metrics\.storage\.(\w+)

     target_label: __name__

     replacement: mcac_storage_${1}

     action: replace

   #Batch Metrics

   - source_labels: ["mcac"]

     regex: org\.apache\.cassandra\.metrics\.batch\.(\w+)

     target_label: __name__

     replacement: mcac_batch_${1}

     action: replace

   #Client Metrics

   - source_labels: ["mcac"]

     regex: org\.apache\.cassandra\.metrics\.client\.(\w+)

     target_label: __name__

     replacement: mcac_client_${1}

     action: replace

   #BufferPool Metrics

   - source_labels: ["mcac"]

     regex: org\.apache\.cassandra\.metrics\.buffer_pool\.(\w+)

     target_label: __name__

     replacement: mcac_buffer_pool_${1}

     action: replace

   #Index Metrics

   - source_labels: ["mcac"]

     regex: org\.apache\.cassandra\.metrics\.index\.(\w+)

     target_label: __name__

     replacement: mcac_sstable_index_${1}

     action: replace

   #HintService Metrics

   - source_labels: ["mcac"]

     regex: org\.apache\.cassandra\.metrics\.hinted_hand_off_manager\.([^\-]+)-(\w+)

     target_label: peer_ip

     replacement: ${2}

     action: replace

   - source_labels: ["mcac"]

     regex: org\.apache\.cassandra\.metrics\.hinted_hand_off_manager\.([^\-]+)-(\w+)

     target_label: __name__

     replacement: mcac_hints_${1}

     action: replace

   #HintService Metrics

   - source_labels: ["mcac"]

     regex: org\.apache\.cassandra\.metrics\.hints_service\.hints_delays\-(\w+)

     target_label: peer_ip

     replacement: ${1}

     action: replace

   - source_labels: ["mcac"]

     regex: org\.apache\.cassandra\.metrics\.hints_service\.hints_delays\-(\w+)

     target_label: __name__

     replacement: mcac_hints_hints_delays

     action: replace

   - source_labels: ["mcac"]

     regex: org\.apache\.cassandra\.metrics\.hints_service\.([^\-]+)

     target_label: __name__

     replacement: mcac_hints_${1}

     action: replace

   # Misc

   - source_labels: ["mcac"]

     regex: org\.apache\.cassandra\.metrics\.memtable_pool\.(\w+)

     target_label: __name__

     replacement: mcac_memtable_pool_${1}

     action: replace

   - source_labels: ["mcac"]

     regex: com\.datastax\.bdp\.type\.performance_objects\.name\.cql_slow_log\.metrics\.queries_latency

     target_label: __name__

     replacement: mcac_cql_slow_log_query_latency

     action: replace

   - source_labels: ["mcac"]

     regex: org\.apache\.cassandra\.metrics\.read_coordination\.(.*)

     target_label: read_type

     replacement: $1

     action: replace

   - source_labels: ["mcac"]

     regex: org\.apache\.cassandra\.metrics\.read_coordination\.(.*)

     target_label: __name__

     replacement: mcac_read_coordination_requests

     action: replace

   #GC Metrics

   - source_labels: ["mcac"]

     regex: jvm\.gc\.(\w+)\.(\w+)

     target_label: collector_type

     replacement: ${1}

     action: replace

   - source_labels: ["mcac"]

     regex: jvm\.gc\.(\w+)\.(\w+)

     target_label: __name__

     replacement: mcac_jvm_gc_${2}

     action: replace

   #JVM Metrics

   - source_labels: ["mcac"]

     regex: jvm\.memory\.(\w+)\.(\w+)

     target_label: memory_type

     replacement: ${1}

     action: replace

   - source_labels: ["mcac"]

     regex: jvm\.memory\.(\w+)\.(\w+)

     target_label: __name__

     replacement: mcac_jvm_memory_${2}

     action: replace

   - source_labels: ["mcac"]

     regex: jvm\.memory\.pools\.(\w+)\.(\w+)

     target_label: pool_name

     replacement: ${2}

     action: replace

   - source_labels: ["mcac"]

     regex: jvm\.memory\.pools\.(\w+)\.(\w+)

     target_label: __name__

     replacement: mcac_jvm_memory_pool_${2}

     action: replace

   - source_labels: ["mcac"]

     regex: jvm\.fd\.usage

     target_label: __name__

     replacement: mcac_jvm_fd_usage

     action: replace

   - source_labels: ["mcac"]

     regex: jvm\.buffers\.(\w+)\.(\w+)

     target_label: buffer_type

     replacement: ${1}

     action: replace

   - source_labels: ["mcac"]

     regex: jvm\.buffers\.(\w+)\.(\w+)

     target_label: __name__

     replacement: mcac_jvm_buffer_${2}

     action: replace

   #Append the prom types back to formatted names

   - source_labels: [__name__, "prom_name"]

     regex: (mcac_.*);.*(_micros_bucket|_bucket|_micros_count_total|_count_total|_total|_micros_sum|_sum|_stddev).*

     separator: ;

     target_label: __name__

     replacement: ${1}${2}

     action: replace

   - regex: prom_name

     action: labeldrop

remote_write:

- url: https://aws1217.com/api/prom/push

  bearer_token_file: /etc/monitoringpack/private/bearer-token

  queue_config:

    capacity: 5000

    max_shards: 20

    min_shards: 5

    max_samples_per_send: 1000

logs:- 

level=info ts=2020-05-28T20:15:33.158Z caller=main.go:331 msg="Starting Prometheus" version="(version=2.16.0, branch=HEAD, revision=b90be6f32a33c03163d700e1452b54454ddce0ec)"

level=info ts=2020-05-28T20:15:33.158Z caller=main.go:332 build_context="(go=go1.13.8, user=root@7ea0ae865f12, date=20200213-23:50:02)"

level=info ts=2020-05-28T20:15:33.159Z caller=main.go:333 host_details="(Linux 4.14.146-93.123.amzn1.x86_64 #1 SMP Tue Sep 24 00:45:23 UTC 2019 x86_64 (none))"

level=info ts=2020-05-28T20:15:33.159Z caller=main.go:334 fd_limits="(soft=1024, hard=4096)"

level=info ts=2020-05-28T20:15:33.159Z caller=main.go:335 vm_limits="(soft=unlimited, hard=unlimited)"

level=info ts=2020-05-28T20:15:33.160Z caller=main.go:661 msg="Starting TSDB ..."

level=info ts=2020-05-28T20:15:33.160Z caller=web.go:508 component=web msg="Start listening for connections" address=127.0.0.1:10101

level=info ts=2020-05-28T20:15:33.163Z caller=head.go:577 component=tsdb msg="replaying WAL, this may take awhile"

level=info ts=2020-05-28T20:15:33.164Z caller=head.go:625 component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0

level=info ts=2020-05-28T20:15:33.165Z caller=main.go:676 fs_type=EXT4_SUPER_MAGIC

level=info ts=2020-05-28T20:15:33.165Z caller=main.go:677 msg="TSDB started"

level=debug ts=2020-05-28T20:15:33.165Z caller=main.go:678 msg="TSDB options" MinBlockDuration=2h MaxBlockDuration=31d MaxBytes=100MiB NoLockfile=false RetentionDuration=0s WALSegmentSize=0B AllowOverlappingBlocks=false WALCompression=false

level=info ts=2020-05-28T20:15:33.165Z caller=main.go:747 msg="Loading configuration file" filename=/etc/monitoringpack/miniprom.yaml

ts=2020-05-28T20:15:33.170Z caller=dedupe.go:112 component=remote level=info remote_name=ae29e2 url=https://aws1217/api/prom/push msg="starting WAL watcher" queue=ae29e2

level=debug ts=2020-05-28T20:15:33.170Z caller=manager.go:224 component="discovery manager scrape" msg="Starting provider" provider=*file.SDConfig/0 subs=[default]

level=debug ts=2020-05-28T20:15:33.170Z caller=manager.go:224 component="discovery manager scrape" msg="Starting provider" provider=*file.SDConfig/1 subs=[mcac]

level=info ts=2020-05-28T20:15:33.170Z caller=main.go:775 msg="Completed loading of configuration file" filename=/etc/monitoringpack/miniprom.yaml

level=info ts=2020-05-28T20:15:33.170Z caller=main.go:630 msg="Server is ready to receive web requests."