Application Metrics - IPC2023
0 likes25 views
The document discusses the importance of application metrics and monitoring using Prometheus, focusing on service level indicators (SLIs), service level objectives (SLOs), and service level agreements (SLAs). It covers how metrics inform the health of services, architectural optimizations, and user experience expectations, alongside practical examples of implementing metrics in code. Ultimately, it emphasizes that actionable metrics are vital for effective monitoring and responsive system management.
![<?php
use PrometheusCounter;
use PrometheusHistogram;
use PrometheusStorageAPC;
require_once 'vendor/autoload.php';
$adapter = new APC();
$histogram = new Histogram(
$adapter,
'my_app',
'response_time_ms',
'This measures ....',
['status', 'url'],
[0, 10, 50, 100]
);
$histogram->observe(15, ['200', '/url']);
$counter = new Counter($adapter, 'my_app', 'count_total',
'How many...', ['status', 'url']);
$counter->inc(['200', '/url']);
$counter->incBy(5, ['200', '/url']);](https://image.slidesharecdn.com/applicationmetrics-ipc2023-230525202940-a364eaf4/85/Application-Metrics-IPC2023-70-320.jpg)
![<?php
use PrometheusCounter;
use PrometheusHistogram;
use PrometheusStorageAPC;
$adapter = new APC();
$histogram = new Histogram(
$adapter,
'my_app',
'response_time_ms',
'This measures ....',
['status', 'url'],
[0, 10, 50, 100]
);
$histogram->observe(15, ['200', '/url']);
$counter = new Counter($adapter, 'my_app', 'count_total',
'How many...', ['status', 'url']);
$counter->inc(['200', '/url']);
$counter->incBy(5, ['200', '/url']);](https://image.slidesharecdn.com/applicationmetrics-ipc2023-230525202940-a364eaf4/85/Application-Metrics-IPC2023-71-320.jpg)
![<?php
use PrometheusCounter;
use PrometheusHistogram;
use PrometheusStorageAPC;
$adapter = new APC();
$histogram = new Histogram(
$adapter,
'my_app',
'response_time_ms',
'This measures ....',
['status', 'url'],
[0, 10, 50, 100]
);
$histogram->observe(15, ['200', '/url']);
$counter = new Counter($adapter, 'my_app', 'count_total',
'How many...', ['status', 'url']);
$counter->inc(['200', '/url']);
$counter->incBy(5, ['200', '/url']);
APC / APCu
Redis](https://image.slidesharecdn.com/applicationmetrics-ipc2023-230525202940-a364eaf4/85/Application-Metrics-IPC2023-72-320.jpg)
![<?php
use PrometheusCounter;
use PrometheusHistogram;
use PrometheusStorageAPC;
$adapter = new APC();
$histogram = new Histogram(
$adapter,
'my_app',
'response_time_ms',
'This measures ....',
['status', 'url'],
[0, 10, 50, 100]
);
$histogram->observe(15, ['200', '/url']);
$counter = new Counter($adapter, 'my_app', 'count_total',
'How many...', ['status', 'url']);
$counter->inc(['200', '/url']);
$counter->incBy(5, ['200', '/url']);
namespace
metric name
help
label names
buckets](https://image.slidesharecdn.com/applicationmetrics-ipc2023-230525202940-a364eaf4/85/Application-Metrics-IPC2023-73-320.jpg)
![<?php
use PrometheusCounter;
use PrometheusHistogram;
use PrometheusStorageAPC;
$adapter = new APC();
$histogram = new Histogram(
$adapter,
'my_app',
'response_time_ms',
'This measures ....',
['status', 'url'],
[0, 10, 50, 100]
);
$histogram->observe(15, ['200', '/url']);
$counter = new Counter($adapter, 'my_app', 'count_total',
'How many...', ['status', 'url']);
$counter->inc(['200', '/url']);
$counter->incBy(5, ['200', '/url']);
measurement
label values](https://image.slidesharecdn.com/applicationmetrics-ipc2023-230525202940-a364eaf4/85/Application-Metrics-IPC2023-74-320.jpg)
![<?php
use PrometheusCounter;
use PrometheusHistogram;
use PrometheusStorageAPC;
$adapter = new APC();
$histogram = new Histogram(
$adapter,
'my_app',
'response_time_ms',
'This measures ....',
['status', 'url'],
[0, 10, 50, 100]
);
$histogram->observe(15, ['200', '/url']);
$counter = new Counter($adapter, 'my_app', 'count_total',
'How many...', ['status', 'url']);
$counter->inc(['200', '/url']);
$counter->incBy(5, ['200', '/url']);
namespace
metric name
help
labels](https://image.slidesharecdn.com/applicationmetrics-ipc2023-230525202940-a364eaf4/85/Application-Metrics-IPC2023-75-320.jpg)
![<?php
use PrometheusCounter;
use PrometheusHistogram;
use PrometheusStorageAPC;
$adapter = new APC();
$histogram = new Histogram(
$adapter,
'my_app',
'response_time_ms',
'This measures ....',
['status', 'url'],
[0, 10, 50, 100]
);
$histogram->observe(15, ['200', '/url']);
$counter = new Counter($adapter, 'my_app', 'count_total',
'How many...', ['status', 'url']);
$counter->inc(['200', '/url']);
$counter->incBy(5, ['200', '/url']);](https://image.slidesharecdn.com/applicationmetrics-ipc2023-230525202940-a364eaf4/85/Application-Metrics-IPC2023-76-320.jpg)
![<?php
use PrometheusCounter;
use PrometheusHistogram;
use PrometheusStorageAPC;
require_once 'vendor/autoload.php';
$adapter = new APC();
$histogram = new Histogram(
$adapter,
'my_app',
'response_time_ms',
'This measures ....',
['status', 'url'],
[0, 10, 50, 100]
);
$histogram->observe(15, ['200', '/url']);
$counter = new Counter($adapter, 'my_app', 'count_total',
'How many...', ['status', 'url']);
$counter->inc(['200', '/url']);
$counter->incBy(5, ['200', '/url']);](https://image.slidesharecdn.com/applicationmetrics-ipc2023-230525202940-a364eaf4/85/Application-Metrics-IPC2023-77-320.jpg)
Application Metrics - IPC2023
- 1. Application Metrics with Prometheus examples Rafael Dohms @rdohms Principal Architect @
- 2. How do you approach metrics?
- 5. I hope not.
- 7. Let’s talk about metrics. But let’s do it with a concrete example.
- 8. Kafka / DDD / Autonomous Microservices / Monitoring 2019
- 9. Kafka / DDD / Autonomous Microservices / Monitoring 2019
- 10. Kafka / DDD / Autonomous Microservices / Monitoring 2019
- 11. Metrics are insights into the current state of your application.
- 12. Metrics tell you if your service is healthy.
- 13. Metrics tell you what is wrong.
- 14. Metrics tell you what is right.
- 15. Metrics tell you what will soon be wrong.
- 16. Metrics tell you where to start looking.
- 19. SLIs Service Level Indicators “A quantitative measure of some aspect of your application” The response time of a request was 150ms Source: Site Reliability Engineering - O’Reilly
- 20. SLOs Service Level Objectives “A target value or a range of values for something measured by an SLI” Request response times should be below 200ms Source: Site Reliability Engineering - O’Reilly
- 21. Help you drive architectural decisions, like optimisation SLOs Response time SLO: 150 ms 95th Percentile of Processing time (PHP time): 5ms As a result we decided to invest more time in exploring the problem domain and not optimising our stack.
- 22. SLAs Service Level Agreements “An explicit or implicit contract with your customer,that includes consequences of missing their SLOs” The 99th percentile of requests response times should meet our SLO,or we will refund users Source: Site Reliability Engineering - O’Reilly
- 23. Measuring
- 24. –Etsy Engineering “If it moves, we track it.” https://codeascraft.com/2011/02/15/measure-anything-measure-everything/
- 25. Metrics Statistics What is happening right now? How often does this happen? Telemetry
- 26. Telemetry “the process of recording and transmitting the readings of an instrument”
- 27. Statistics / Analytics “the practice of collecting and analysing numerical data in large quantities”
- 28. Statistics / Analytics “the practice of collecting and analysing numerical data in large quantities”
- 29. I really miss Ayrton Senna Statistics / Analytics “the practice of collecting and analysing numerical data in large quantities”
- 30. Statistics Incoming feedback items with origin information Telemetry response time of public endpoints
- 31. “If it moves, we track it.”
- 32. Request Latency System Throughput Error Rate Availability Resource Usage “If it moves, we track it.”
- 33. Request Latency System Throughput Error Rate Availability Resource Usage “If it moves, we track it.” Incoming Data Peak frequency CPU Memory Disk Space Bandwith node PHP NginX Database
- 34. Request Latency System Throughput Error Rate Availability Resource Usage “If it moves, we track it.” Incoming Data Peak frequency CPU Memory Disk Space Bandwith node PHP NginX Database Measure Monitoring Measure measurements
- 39. SLIs may change according to who is looking at the data.
- 40. Understanding the nature of your system
- 43. Big Data Systems? throughput,end-to-end latency
- 44. User-Facing and Big Data Systems
- 45. ๏SLIs - Response time in the“receive”endpoint - Turn around time,from“receive” to“show”. - Individual processing time per step - Data counting: how many,what nature User-Facing and Big Data Systems
- 46. ๏SLIs - Response time in the“receive”endpoint - Turn around time,from“receive” to“show”. - Individual processing time per step - Data counting: how many,what nature User-Facing and Big Data Systems More relevant to development team
- 47. ๏SLIs - Response time in the“receive”endpoint - Turn around time,from“receive” to“show”. - Individual processing time per step - Data counting: how many,what nature ๏Other Metrics - node,nginx,php-fpm,java metrics - server metrics: cpu,memory,disk space - Size of cluster - Kafka health User-Facing and Big Data Systems More relevant to development team
- 48. ๏SLIs - Response time in the“receive”endpoint - Turn around time,from“receive” to“show”. - Individual processing time per step - Data counting: how many,what nature ๏Other Metrics - node,nginx,php-fpm,java metrics - server metrics: cpu,memory,disk space - Size of cluster - Kafka health User-Facing and Big Data Systems More relevant to development team More relevant to Infrastructure team
- 49. Picking Targets
- 50. Target value SLI value >= target Target Range lower bound <= SLI value <= upper bound
- 51. Don’t pick a target based on current performance What is the business need? What are users trying to achieve? How much impact does it have on the user experience?
- 52. How long can it take between the user clicking submit and a confirmation that our servers received the data?
- 53. How long can it take between the user clicking submit and a confirmation that our servers received the data? “Immediate" “We sell as real time” “500ms,too much HTML“ “I don’t know”
- 54. How long can it take between the user clicking submit and a confirmation that our servers received the data? “Immediate" “We sell as real time” “500ms,too much HTML“ “I don’t know” What is human perception of immediate? 100ms Collection API should respond within 150ms
- 55. Some, but not too many. can you settle an argument or choose a priority based on it?
- 56. Don’t over achieve. The Chubby example.
- 57. Adapt. Evolve. re-define SLO’s as your product evolves.
- 59. Attach consequences to your Objectives.
- 60. The night is dark and full of loopholes. take a friend from legal with you.
- 61. Safety Margins. like setting the alarm 5 minutes before the meeting.
- 63. prometheus.io
- 67. Gauge Histogram Counter Summary Cumulative metric the represents a single number that only increases Samples and count of observations over time A counter,that can go up or down Same as a histogram but with stream of quantiles over a sliding window.
- 69. reads from /metrics reads from local storage writes to local storage your code /metrics
- 70. <?php use PrometheusCounter; use PrometheusHistogram; use PrometheusStorageAPC; require_once 'vendor/autoload.php'; $adapter = new APC(); $histogram = new Histogram( $adapter, 'my_app', 'response_time_ms', 'This measures ....', ['status', 'url'], [0, 10, 50, 100] ); $histogram->observe(15, ['200', '/url']); $counter = new Counter($adapter, 'my_app', 'count_total', 'How many...', ['status', 'url']); $counter->inc(['200', '/url']); $counter->incBy(5, ['200', '/url']);
- 71. <?php use PrometheusCounter; use PrometheusHistogram; use PrometheusStorageAPC; $adapter = new APC(); $histogram = new Histogram( $adapter, 'my_app', 'response_time_ms', 'This measures ....', ['status', 'url'], [0, 10, 50, 100] ); $histogram->observe(15, ['200', '/url']); $counter = new Counter($adapter, 'my_app', 'count_total', 'How many...', ['status', 'url']); $counter->inc(['200', '/url']); $counter->incBy(5, ['200', '/url']);
- 72. <?php use PrometheusCounter; use PrometheusHistogram; use PrometheusStorageAPC; $adapter = new APC(); $histogram = new Histogram( $adapter, 'my_app', 'response_time_ms', 'This measures ....', ['status', 'url'], [0, 10, 50, 100] ); $histogram->observe(15, ['200', '/url']); $counter = new Counter($adapter, 'my_app', 'count_total', 'How many...', ['status', 'url']); $counter->inc(['200', '/url']); $counter->incBy(5, ['200', '/url']); APC / APCu Redis
- 73. <?php use PrometheusCounter; use PrometheusHistogram; use PrometheusStorageAPC; $adapter = new APC(); $histogram = new Histogram( $adapter, 'my_app', 'response_time_ms', 'This measures ....', ['status', 'url'], [0, 10, 50, 100] ); $histogram->observe(15, ['200', '/url']); $counter = new Counter($adapter, 'my_app', 'count_total', 'How many...', ['status', 'url']); $counter->inc(['200', '/url']); $counter->incBy(5, ['200', '/url']); namespace metric name help label names buckets
- 74. <?php use PrometheusCounter; use PrometheusHistogram; use PrometheusStorageAPC; $adapter = new APC(); $histogram = new Histogram( $adapter, 'my_app', 'response_time_ms', 'This measures ....', ['status', 'url'], [0, 10, 50, 100] ); $histogram->observe(15, ['200', '/url']); $counter = new Counter($adapter, 'my_app', 'count_total', 'How many...', ['status', 'url']); $counter->inc(['200', '/url']); $counter->incBy(5, ['200', '/url']); measurement label values
- 75. <?php use PrometheusCounter; use PrometheusHistogram; use PrometheusStorageAPC; $adapter = new APC(); $histogram = new Histogram( $adapter, 'my_app', 'response_time_ms', 'This measures ....', ['status', 'url'], [0, 10, 50, 100] ); $histogram->observe(15, ['200', '/url']); $counter = new Counter($adapter, 'my_app', 'count_total', 'How many...', ['status', 'url']); $counter->inc(['200', '/url']); $counter->incBy(5, ['200', '/url']); namespace metric name help labels
- 76. <?php use PrometheusCounter; use PrometheusHistogram; use PrometheusStorageAPC; $adapter = new APC(); $histogram = new Histogram( $adapter, 'my_app', 'response_time_ms', 'This measures ....', ['status', 'url'], [0, 10, 50, 100] ); $histogram->observe(15, ['200', '/url']); $counter = new Counter($adapter, 'my_app', 'count_total', 'How many...', ['status', 'url']); $counter->inc(['200', '/url']); $counter->incBy(5, ['200', '/url']);
- 77. <?php use PrometheusCounter; use PrometheusHistogram; use PrometheusStorageAPC; require_once 'vendor/autoload.php'; $adapter = new APC(); $histogram = new Histogram( $adapter, 'my_app', 'response_time_ms', 'This measures ....', ['status', 'url'], [0, 10, 50, 100] ); $histogram->observe(15, ['200', '/url']); $counter = new Counter($adapter, 'my_app', 'count_total', 'How many...', ['status', 'url']); $counter->inc(['200', '/url']); $counter->incBy(5, ['200', '/url']);
- 78. <?php use PrometheusRenderTextFormat; use PrometheusStorageAPC; require_once 'vendor/autoload.php'; $adapter = new APC(); $renderer = new RenderTextFormat(); $result = $renderer->render($adapter->collect()); echo $result;
- 79. <?php use PrometheusRenderTextFormat; use PrometheusStorageAPC; require_once 'vendor/autoload.php'; $adapter = new APC(); $renderer = new RenderTextFormat(); $result = $renderer->render($adapter->collect()); echo $result;
- 80. <?php use PrometheusRenderTextFormat; use PrometheusStorageAPC; require_once 'vendor/autoload.php'; $adapter = new APC(); # HELP my_app_count_total How many... # TYPE my_app_count_total counter my_app_count_total{status="200",url="/url"} 6 # HELP my_app_response_time_ms This measures .... # TYPE my_app_response_time_ms histogram my_app_response_time_ms_bucket{status="200",url="/url",le="0"} 0 my_app_response_time_ms_bucket{status="200",url="/url",le="10"} 0 my_app_response_time_ms_bucket{status="200",url="/url",le="50"} 1 my_app_response_time_ms_bucket{status="200",url="/url",le="100"} 1 my_app_response_time_ms_bucket{status="200",url="/url",le="+Inf"} 1 my_app_response_time_ms_count{status="200",url="/url"} 1 my_app_response_time_ms_sum{status="200",url="/url"} 16 $renderer = new RenderTextFormat(); $result = $renderer->render($adapter->collect()); echo $result;
- 81. –Also Rafael (today) “I’ll just try this live demo again.” http://localhost:9090/graph http://localhost:8180/metrics –Rafael (yesterday) “Demos always fail.” http://localhost:8180/index https://github.com/rdohms/talk-app-metrics
- 82. You can’t act on what you can’t see.
- 85. Metrics without actionability are just numbers on a screen.
- 86. Act as soon as an SLO is threatened .