Rate function is misbehaving
Eduard Feliu
What did you do?
I queried the following metric:
sum(rate(nginx_ingress_controller_requests{account="xxxx",exported_namespace="xxxxx",ingress=~"xxxxx.*"}[2m]))
What did you expect to see?
I expect to see a rate of 24/25 request per seconds given the raw values of the counter metric.
What did you see instead? Under which circumstances?
I see a bump of the request per second up to 10k per second that does not match with the raw metric datapoints.
Environment
System information:
Linux 4.14.171-105.231.amzn1.x86_64 x86_64
Prometheus version: 2.15.2
Raw datapoints request:
90618044-b9ee2400-e20f-11ea-80e7-c2c4246c9fb0.png
{
"status": "success",
"data": {
"resultType": "matrix",
"result": [
{
"metric": {
"__name__": "nginx_ingress_controller_requests",
"account": "xxxxxx",
"cluster": "prod01",
"controller_class": "nginx",
"controller_pod": "schip-nginx-ingress-controller-fbbf9854-vmvg9",
"exported_namespace": "xxxxx",
"ingress": "xxxxx",
"instance": "172.17.201.41:10254",
"job": "schip-nginx-ingress-controller-metrics",
"namespace": "subito-pro",
"prometheus": "platform-services/ingress-metrics-prometheus",
"service": "areariservata",
"status": "200"
},
"values": [
[
1597054680,
"1058296"
],
[
1597054681,
"1058296"
],
[
1597054682,
"1058296"
],
[
1597054683,
"1058296"
],
[
1597054684,
"1058296"
],
[
1597054685,
"1058325"
],
[
1597054686,
"1058325"
],
[
1597054687,
"1058325"
],
[
1597054688,
"1058325"
],
[
1597054689,
"1058325"
],
[
1597054690,
"1058325"
],
[
1597054691,
"1058325"
],
[
1597054692,
"1058325"
],
[
1597054693,
"1058325"
],
[
1597054694,
"1058325"
],
[
1597054695,
"1058325"
],
[
1597054696,
"1058325"
],
[
1597054697,
"1058325"
],
[
1597054698,
"1058325"
],
[
1597054699,
"1058325"
],
[
1597054700,
"1058325"
],
[
1597054701,
"1058325"
],
[
1597054702,
"1058325"
],
[
1597054703,
"1058325"
],
[
1597054704,
"1058325"
],
[
1597054705,
"1058325"
],
[
1597054706,
"1058325"
],
[
1597054707,
"1058325"
],
[
1597054708,
"1058325"
],
[
1597054709,
"1058325"
],
[
1597054710,
"1058325"
],
[
1597054711,
"1058325"
],
[
1597054712,
"1058325"
],
[
1597054713,
"1058325"
],
[
1597054714,
"1058325"
],
[
1597054715,
"1058382"
],
[
1597054716,
"1058382"
],
[
1597054717,
"1058382"
],
[
1597054718,
"1058382"
],
[
1597054719,
"1058382"
],
[
1597054720,
"1058382"
],
[
1597054721,
"1058382"
],
[
1597054722,
"1058382"
],
[
1597054723,
"1058382"
],
[
1597054724,
"1058382"
],
[
1597054725,
"1058382"
],
[
1597054726,
"1058382"
],
[
1597054727,
"1058382"
],
[
1597054728,
"1058382"
],
[
1597054729,
"1058382"
],
[
1597054730,
"1058382"
],
[
1597054731,
"1058382"
],
[
1597054732,
"1058382"
],
[
1597054733,
"1058382"
],
[
1597054734,
"1058382"
],
[
1597054735,
"1058382"
],
[
1597054736,
"1058382"
],
[
1597054737,
"1058382"
],
[
1597054738,
"1058382"
],
[
1597054739,
"1058382"
],
[
1597054740,
"1058382"
],
[
1597054741,
"1058382"
],
[
1597054742,
"1058382"
],
[
1597054743,
"1058382"
],
[
1597054744,
"1058382"
],
[
1597054745,
"1058401"
],
[
1597054746,
"1058401"
],
[
1597054747,
"1058401"
],
[
1597054748,
"1058401"
],
[
1597054749,
"1058401"
],
[
1597054750,
"1058401"
],
[
1597054751,
"1058401"
],
[
1597054752,
"1058401"
],
[
1597054753,
"1058401"
],
[
1597054754,
"1058401"
],
[
1597054755,
"1058401"
],
[
1597054756,
"1058401"
],
[
1597054757,
"1058401"
],
[
1597054758,
"1058401"
],
[
1597054759,
"1058401"
],
[
1597054760,
"1058401"
],
[
1597054761,
"1058401"
],
[
1597054762,
"1058401"
],
[
1597054763,
"1058401"
],
[
1597054764,
"1058401"
],
[
1597054765,
"1058401"
],
[
1597054766,
"1058401"
],
[
1597054767,
"1058401"
],
[
1597054768,
"1058401"
],
[
1597054769,
"1058401"
],
[
1597054770,
"1058401"
],
[
1597054771,
"1058401"
],
[
1597054772,
"1058401"
],
[
1597054773,
"1058401"
],
[
1597054774,
"1058401"
],
[
1597054775,
"1058401"
],
[
1597054776,
"1058401"
],
[
1597054777,
"1058401"
],
[
1597054778,
"1058401"
],
[
1597054779,
"1058401"
],
[
1597054780,
"1058401"
],
[
1597054781,
"1058401"
],
[
1597054782,
"1058401"
],
[
1597054783,
"1058401"
],
[
1597054784,
"1058401"
],
[
1597054785,
"1058401"
],
[
1597054786,
"1058401"
],
[
1597054787,
"1058401"
],
[
1597054788,
"1058401"
],
[
1597054789,
"1058401"
],
[
1597054790,
"1058401"
],
[
1597054791,
"1058401"
],
[
1597054792,
"1058401"
],
[
1597054793,
"1058401"
],
[
1597054794,
"1058401"
],
[
1597054795,
"1058401"
],
[
1597054796,
"1058401"
],
[
1597054797,
"1058401"
],
[
1597054798,
"1058401"
],
[
1597054799,
"1058401"
],
[
1597054800,
"1058401"
],
[
1597054801,
"1058401"
],
[
1597054802,
"1058401"
],
[
1597054803,
"1058401"
],
[
1597054804,
"1058401"
],
[
1597054805,
"1058462"
],
[
1597054806,
"1058462"
],
[
1597054807,
"1058462"
],
[
1597054808,
"1058462"
],
[
1597054809,
"1058462"
],
[
1597054810,
"1058462"
],
[
1597054811,
"1058462"
],
[
1597054812,
"1058462"
],
[
1597054813,
"1058462"
],
[
1597054814,
"1058462"
],
[
1597054815,
"1058462"
],
[
1597054816,
"1058462"
],
[
1597054817,
"1058462"
],
[
1597054818,
"1058462"
],
[
1597054819,
"1058462"
],
[
1597054820,
"1058462"
],
[
1597054821,
"1058462"
],
[
1597054822,
"1058462"
],
[
1597054823,
"1058462"
],
[
1597054824,
"1058462"
],
[
1597054825,
"1058462"
],
[
1597054826,
"1058462"
],
[
1597054827,
"1058462"
],
[
1597054828,
"1058462"
],
[
1597054829,
"1058462"
],
[
1597054830,
"1058462"
],
[
1597054831,
"1058462"
],
[
1597054832,
"1058462"
],
[
1597054833,
"1058462"
],
[
1597054834,
"1058462"
],
[
1597054835,
"1058486"
],
[
1597054836,
"1058486"
],
[
1597054837,
"1058486"
],
[
1597054838,
"1058486"
],
[
1597054839,
"1058486"
],
[
1597054840,
"1058486"
],
[
1597054841,
"1058486"
],
[
1597054842,
"1058486"
],
[
1597054843,
"1058486"
],
[
1597054844,
"1058486"
],
[
1597054845,
"1058486"
],
[
1597054846,
"1058486"
],
[
1597054847,
"1058486"
],
[
1597054848,
"1058486"
],
[
1597054849,
"1058486"
],
[
1597054850,
"1058486"
],
[
1597054851,
"1058486"
],
[
1597054852,
"1058486"
],
[
1597054853,
"1058486"
],
[
1597054854,
"1058486"
],
[
1597054855,
"1058486"
],
[
1597054856,
"1058486"
],
[
1597054857,
"1058486"
],
[
1597054858,
"1058486"
],
[
1597054859,
"1058486"
],
[
1597054860,
"1058486"
],
[
1597054861,
"1058486"
],
[
1597054862,
"1058486"
],
[
1597054863,
"1058486"
],
[
1597054864,
"1058486"
],
[
1597054865,
"1058521"
],
[
1597054866,
"1058521"
],
[
1597054867,
"1058521"
],
[
1597054868,
"1058521"
],
[
1597054869,
"1058521"
],
[
1597054870,
"1058521"
],
[
1597054871,
"1058521"
],
[
1597054872,
"1058521"
],
[
1597054873,
"1058521"
],
[
1597054874,
"1058521"
],
[
1597054875,
"1058521"
],
[
1597054876,
"1058521"
],
[
1597054877,
"1058521"
],
[
1597054878,
"1058521"
],
[
1597054879,
"1058521"
],
[
1597054880,
"1058521"
],
[
1597054881,
"1058521"
],
[
1597054882,
"1058521"
],
[
1597054883,
"1058521"
],
[
1597054884,
"1058521"
],
[
1597054885,
"1058521"
],
[
1597054886,
"1058521"
],
[
1597054887,
"1058521"
],
[
1597054888,
"1058521"
],
[
1597054889,
"1058521"
],
[
1597054890,
"1058521"
],
[
1597054891,
"1058521"
],
[
1597054892,
"1058521"
],
[
1597054893,
"1058521"
],
[
1597054894,
"1058521"
],
[
1597054895,
"1058551"
],
[
1597054896,
"1058551"
],
[
1597054897,
"1058551"
],
[
1597054898,
"1058551"
],
[
1597054899,
"1058551"
],
[
1597054900,
"1058551"
],
[
1597054901,
"1058551"
],
[
1597054902,
"1058551"
],
[
1597054903,
"1058551"
],
[
1597054904,
"1058551"
],
[
1597054905,
"1058551"
],
[
1597054906,
"1058551"
],
[
1597054907,
"1058551"
],
[
1597054908,
"1058551"
],
[
1597054909,
"1058551"
],
[
1597054910,
"1058551"
],
[
1597054911,
"1058551"
],
[
1597054912,
"1058551"
],
[
1597054913,
"1058551"
],
[
1597054914,
"1058551"
],
[
1597054915,
"1058551"
],
[
1597054916,
"1058551"
],
[
1597054917,
"1058551"
],
[
1597054918,
"1058551"
],
[
1597054919,
"1058551"
],
[
1597054920,
"1058551"
]
]
}
]
}
}
Misbehaving (rate) PromQL query:
90617978-a3e06380-e20f-11ea-82c5-bbdbc1a68b1b.png
{
"status": "success",
"data": {
"resultType": "matrix",
"result": [
{
"metric": {},
"values": [
[
1597054680,
"25.13337491476297"
],
[
1597054681,
"25.177421203122556"
],
[
1597054682,
"24.994101734978706"
],
[
1597054683,
"25.142458348874193"
],
[
1597054684,
"24.958432376638036"
],
[
1597054685,
"24.7015704838093"
],
[
1597054686,
"24.757469407235032"
],
[
1597054687,
"25.804760828255237"
],
[
1597054688,
"25.44198030274553"
],
[
1597054689,
"25.550485386637174"
],
[
1597054690,
"26.62268495093776"
],
[
1597054691,
"25.672258131258218"
],
[
1597054692,
"25.716408825540984"
],
[
1597054693,
"25.29706062863792"
],
[
1597054694,
"25.364570490856156"
],
[
1597054695,
"25.236277359189412"
],
[
1597054696,
"25.410578612583674"
],
[
1597054697,
"25.49037287107598"
],
[
1597054698,
"25.072735675573274"
],
[
1597054699,
"25.29807128212767"
],
[
1597054700,
"25.31097544214572"
],
[
1597054701,
"25.35675409652581"
],
[
1597054702,
"25.355340860878446"
],
[
1597054703,
"25.397333797821943"
],
[
1597054704,
"25.247715321507485"
],
[
1597054705,
"25.279032834054348"
],
[
1597054706,
"24.918360791377314"
],
[
1597054707,
"24.960486509350375"
],
[
1597054708,
"24.68512136640581"
],
[
1597054709,
"24.836994460020552"
],
[
1597054710,
"24.880442622877965"
],
[
1597054711,
"25.02891009495187"
],
[
1597054712,
"25.435189123714633"
],
[
1597054713,
"25.510512490109363"
],
[
1597054714,
"25.758571936517487"
],
[
1597054715,
"27.08897304525644"
],
[
1597054716,
"27.035077814280587"
],
[
1597054717,
"27.07346107641112"
],
[
1597054718,
"27.35926182805818"
],
[
1597054719,
"27.21035895783728"
],
[
1597054720,
"26.94242433974758"
],
[
1597054721,
"26.988345399398916"
],
[
1597054722,
"26.704200464987906"
],
[
1597054723,
"26.67769972718591"
],
[
1597054724,
"26.591509657399104"
],
[
1597054725,
"26.30101612939375"
],
[
1597054726,
"26.255207769460835"
],
[
1597054727,
"26.6546837886257"
],
[
1597054728,
"27.15280821314472"
],
[
1597054729,
"27.087258143130015"
],
[
1597054730,
"27.035093032069735"
],
[
1597054731,
"26.67893463520782"
],
[
1597054732,
"26.297761726173974"
],
[
1597054733,
"26.24033994862165"
],
[
1597054734,
"26.054393005096045"
],
[
1597054735,
"26.20924322701089"
],
[
1597054736,
"26.00497698904199"
],
[
1597054737,
"26.11718464544267"
],
[
1597054738,
"25.97476578936332"
],
[
1597054739,
"25.922362673156535"
],
[
1597054740,
"25.697902126452806"
],
[
1597054741,
"25.678461183051024"
],
[
1597054742,
"25.37192640680701"
],
[
1597054743,
"25.684298546087444"
],
[
1597054744,
"25.633810635302396"
],
[
1597054745,
"9972.378145925248"
],
[
1597054746,
"10070.730880141627"
],
[
1597054747,
"10168.817534438527"
],
[
1597054748,
"10265.785179105953"
],
[
1597054749,
"10363.762240436727"
],
[
1597054750,
"10462.974089769286"
],
[
1597054751,
"11785.98641694534"
],
[
1597054752,
"11786.044878858751"
],
[
1597054753,
"11785.145559088425"
],
[
1597054754,
"11785.221295716115"
],
[
1597054755,
"11784.95228519218"
],
[
1597054756,
"11785.136539004478"
],
[
1597054757,
"11785.349673591227"
],
[
1597054758,
"11785.350807509443"
],
[
1597054759,
"11785.987408902483"
],
[
1597054760,
"11785.598956789163"
],
[
1597054761,
"11785.645064180613"
],
[
1597054762,
"11785.68558623082"
],
[
1597054763,
"11785.730576009844"
],
[
1597054764,
"11785.127351844258"
],
[
1597054765,
"11785.136328932136"
],
[
1597054766,
"11784.828978317904"
],
[
1597054767,
"11784.851233376936"
],
[
1597054768,
"11784.182650849909"
],
[
1597054769,
"11784.332292872343"
],
[
1597054770,
"10412.393963283745"
],
[
1597054771,
"10314.48573677096"
],
[
1597054772,
"10216.149088727398"
],
[
1597054773,
"10118.16301343832"
],
[
1597054774,
"10020.192499047573"
],
[
1597054775,
"11783.8143142388"
],
[
1597054776,
"11783.724427337505"
],
[
1597054777,
"11783.760816268885"
],
[
1597054778,
"11784.086932613129"
],
[
1597054779,
"11783.97420953304"
],
[
1597054780,
"11783.99110685853"
],
[
1597054781,
"11784.11365901503"
],
[
1597054782,
"11783.957683567689"
],
[
1597054783,
"13404.021528418634"
],
[
1597054784,
"13257.705088854684"
],
[
1597054785,
"13111.715111807864"
],
[
1597054786,
"12965.972977157091"
],
[
1597054787,
"12821.032661559278"
],
[
1597054788,
"12675.887675178332"
],
[
1597054789,
"12530.05191012842"
],
[
1597054790,
"12384.309151376483"
],
[
1597054791,
"12237.404233670963"
],
[
1597054792,
"12091.23263055466"
],
[
1597054793,
"11945.50249783114"
],
[
1597054794,
"11800.621859310277"
],
[
1597054795,
"11655.025838913687"
],
[
1597054796,
"11509.090808820163"
],
[
1597054797,
"11363.570569165075"
],
[
1597054798,
"11217.67761305693"
],
[
1597054799,
"11071.940621268199"
],
[
1597054800,
"10926.522744449027"
],
[
1597054801,
"10780.807762115166"
],
[
1597054802,
"10634.766871543703"
],
[
1597054803,
"10489.380218823051"
],
[
1597054804,
"10343.825843138653"
],
[
1597054805,
"11854.658185892622"
],
[
1597054806,
"11854.616597400958"
],
[
1597054807,
"11855.016851680537"
],
[
1597054808,
"11855.029073163885"
],
[
1597054809,
"11855.078954120732"
],
[
1597054810,
"11856.14594677747"
],
[
1597054811,
"11856.099349973172"
],
[
1597054812,
"11856.156901205386"
],
[
1597054813,
"11855.951423504292"
],
[
1597054814,
"11856.024846699946"
],
[
1597054815,
"11855.703316848107"
],
[
1597054816,
"11855.932993750597"
],
[
1597054817,
"11856.269064313887"
],
[
1597054818,
"11855.493160060953"
],
[
1597054819,
"11855.41034402063"
],
[
1597054820,
"11854.997562949593"
],
[
1597054821,
"11855.038209831886"
],
[
1597054822,
"11855.073332868778"
],
[
1597054823,
"11855.112862696806"
],
[
1597054824,
"11855.024181322311"
],
[
1597054825,
"11855.056535351012"
],
[
1597054826,
"11855.219122858065"
],
[
1597054827,
"11855.243017224197"
],
[
1597054828,
"11855.188558556642"
],
[
1597054829,
"11854.957021658383"
],
[
1597054830,
"11855.076722860631"
],
[
1597054831,
"11855.246989836383"
],
[
1597054832,
"11854.652329379052"
],
[
1597054833,
"11854.71757731908"
],
[
1597054834,
"11854.758240490535"
],
[
1597054835,
"11725.677298091205"
],
[
1597054836,
"11725.57168570679"
],
[
1597054837,
"11725.590743851712"
],
[
1597054838,
"11725.74247035833"
],
[
1597054839,
"11725.404393582088"
],
[
1597054840,
"11725.330437109373"
],
[
1597054841,
"11725.499917367346"
],
[
1597054842,
"11725.97921737398"
],
[
1597054843,
"11725.920288772693"
],
[
1597054844,
"11726.2376652233"
],
[
1597054845,
"11726.059270674215"
],
[
1597054846,
"11725.99769356021"
],
[
1597054847,
"11726.236055734656"
],
[
1597054848,
"11727.07682694825"
],
[
1597054849,
"11727.246795979594"
],
[
1597054850,
"11727.193943555985"
],
[
1597054851,
"11726.527967234684"
],
[
1597054852,
"11726.441669813084"
],
[
1597054853,
"11726.416658409522"
],
[
1597054854,
"11727.239906449442"
],
[
1597054855,
"11727.37904518294"
],
[
1597054856,
"11727.426152855925"
],
[
1597054857,
"11727.574405254567"
],
[
1597054858,
"11727.313889391644"
],
[
1597054859,
"11727.273590606621"
],
[
1597054860,
"11727.393293753788"
],
[
1597054861,
"11727.389057608205"
],
[
1597054862,
"11727.122277377317"
],
[
1597054863,
"11727.419755494198"
],
[
1597054864,
"11727.380133126371"
],
[
1597054865,
"24.554652145183244"
],
[
1597054866,
"24.11353705413868"
],
[
1597054867,
"23.803615709276457"
],
[
1597054868,
"23.823972075106774"
],
[
1597054869,
"24.30867012969323"
],
[
1597054870,
"24.84794564815718"
],
[
1597054871,
"23.602747295162384"
],
[
1597054872,
"23.66484877216017"
],
[
1597054873,
"23.383523137568947"
],
[
1597054874,
"23.756626586023494"
],
[
1597054875,
"23.148501771740378"
],
[
1597054876,
"23.408328190400706"
],
[
1597054877,
"23.202317528175215"
],
[
1597054878,
"22.966506553803086"
],
[
1597054879,
"22.847014629598036"
],
[
1597054880,
"22.180060824002297"
],
[
1597054881,
"22.184477481228654"
],
[
1597054882,
"22.17630665428842"
],
[
1597054883,
"22.178791079794227"
],
[
1597054884,
"22.013214874967485"
],
[
1597054885,
"22.006293185140745"
],
[
1597054886,
"22.53928802680475"
],
[
1597054887,
"22.523219856610858"
],
[
1597054888,
"22.488973509814112"
],
[
1597054889,
"21.963105408885035"
],
[
1597054890,
"21.95447749815804"
],
[
1597054891,
"22.426112899287087"
],
[
1597054892,
"22.09349400960449"
],
[
1597054893,
"22.086105001555282"
],
[
1597054894,
"22.073701357333103"
],
[
1597054895,
"22.21325791504528"
],
[
1597054896,
"22.933295741082716"
],
[
1597054897,
"22.929894183881608"
],
[
1597054898,
"23.51612499149476"
],
[
1597054899,
"23.80345544980171"
],
[
1597054900,
"23.948210908184077"
],
[
1597054901,
"24.114268887489853"
],
[
1597054902,
"23.986231867663307"
],
[
1597054903,
"23.950356564346524"
],
[
1597054904,
"23.843726494031483"
],
[
1597054905,
"23.752460521380183"
],
[
1597054906,
"23.710061627901705"
],
[
1597054907,
"23.585015028353506"
],
[
1597054908,
"23.744835543231684"
],
[
1597054909,
"24.15259086814034"
],
[
1597054910,
"24.10811640682759"
],
[
1597054911,
"24.06364194551484"
],
[
1597054912,
"23.541026378504665"
],
[
1597054913,
"23.514025888442863"
],
[
1597054914,
"23.586807183488006"
],
[
1597054915,
"23.782333628454072"
],
[
1597054916,
"24.03122172452061"
],
[
1597054917,
"24.3143837856913"
],
[
1597054918,
"24.10852702234158"
],
[
1597054919,
"24.087524010515867"
],
[
1597054920,
"24.232981699222034"
]
]
}
]
}
}
Thanks in advance!
Ben Kochie
Also, if you swap rate() for deriv(), what do you see?
--
You received this message because you are subscribed to the Google Groups "Prometheus Users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to prometheus-use...@googlegroups.com.
To view this discussion on the web visit https://groups.google.com/d/msgid/prometheus-users/45f41f24-065c-4b7c-b6c9-dcaba0559a6dn%40googlegroups.com.
