feat(server): add OpenTelemetry observability stack
This commit is contained in:
RainbowBird
@@ -5,3 +5,5 @@ AUTH_GOOGLE_CLIENT_SECRET="change-me"
|
||||
|
||||
AUTH_GITHUB_CLIENT_ID="change-me"
|
||||
AUTH_GITHUB_CLIENT_SECRET="change-me"
|
||||
|
||||
# OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4318"
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
name: proj-airi-otel
|
||||
|
||||
services:
|
||||
# ============================================================
|
||||
# OpenTelemetry Collector
|
||||
# Receives traces, metrics, and logs from the application
|
||||
# and exports them to the appropriate backends.
|
||||
# ============================================================
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.120.0
|
||||
command: ['--config=/etc/otelcol/otel-collector.yaml']
|
||||
volumes:
|
||||
- ./otel/collector/otel-collector.yaml:/etc/otelcol/otel-collector.yaml:ro
|
||||
ports:
|
||||
- '4317:4317' # OTLP gRPC
|
||||
- '4318:4318' # OTLP HTTP
|
||||
depends_on:
|
||||
loki:
|
||||
condition: service_started
|
||||
tempo:
|
||||
condition: service_started
|
||||
healthcheck:
|
||||
test: ['CMD', 'wget', '--spider', '-q', 'http://localhost:13133/']
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
restart: unless-stopped
|
||||
|
||||
# ============================================================
|
||||
# Prometheus - Metrics storage and querying
|
||||
# ============================================================
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.2.1
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yaml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time=7d
|
||||
- --web.enable-remote-write-receiver
|
||||
- --enable-feature=exemplar-storage
|
||||
- --enable-feature=native-histograms
|
||||
volumes:
|
||||
- ./otel/prometheus/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro
|
||||
- prometheus_data:/prometheus
|
||||
healthcheck:
|
||||
test: ['CMD', 'wget', '--spider', '-q', 'http://localhost:9090/-/healthy']
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
restart: unless-stopped
|
||||
|
||||
# ============================================================
|
||||
# Loki - Log aggregation
|
||||
# ============================================================
|
||||
loki:
|
||||
image: grafana/loki:3.4.3
|
||||
command: -config.file=/etc/loki/loki.yaml
|
||||
volumes:
|
||||
- ./otel/loki/loki.yaml:/etc/loki/loki.yaml:ro
|
||||
- loki_data:/loki
|
||||
healthcheck:
|
||||
test: ['CMD', 'wget', '--spider', '-q', 'http://localhost:3100/ready']
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
restart: unless-stopped
|
||||
|
||||
# ============================================================
|
||||
# Tempo - Distributed tracing backend
|
||||
# ============================================================
|
||||
tempo:
|
||||
image: grafana/tempo:2.7.2
|
||||
command: ['-config.file=/etc/tempo/tempo.yaml']
|
||||
volumes:
|
||||
- ./otel/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
|
||||
- tempo_data:/var/tempo
|
||||
healthcheck:
|
||||
test: ['CMD', 'wget', '--spider', '-q', 'http://localhost:3200/ready']
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
restart: unless-stopped
|
||||
|
||||
# ============================================================
|
||||
# Grafana - Visualization and dashboards
|
||||
# ============================================================
|
||||
grafana:
|
||||
image: grafana/grafana:11.5.2
|
||||
environment:
|
||||
- GF_SECURITY_ADMIN_USER=admin
|
||||
- GF_SECURITY_ADMIN_PASSWORD=admin
|
||||
- GF_AUTH_ANONYMOUS_ENABLED=true
|
||||
- GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer
|
||||
- GF_FEATURE_TOGGLES_ENABLE=traceqlEditor tempoSearch tempoServiceGraph
|
||||
volumes:
|
||||
- ./otel/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./otel/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
- grafana_data:/var/lib/grafana
|
||||
ports:
|
||||
- '3001:3000'
|
||||
depends_on:
|
||||
prometheus:
|
||||
condition: service_healthy
|
||||
loki:
|
||||
condition: service_healthy
|
||||
tempo:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ['CMD', 'wget', '--spider', '-q', 'http://localhost:3000/api/health']
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
prometheus_data:
|
||||
driver: local
|
||||
loki_data:
|
||||
driver: local
|
||||
tempo_data:
|
||||
driver: local
|
||||
grafana_data:
|
||||
driver: local
|
||||
@@ -0,0 +1,88 @@
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
|
||||
processors:
|
||||
batch:
|
||||
timeout: 5s
|
||||
send_batch_size: 1024
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 512
|
||||
spike_limit_mib: 128
|
||||
resource:
|
||||
attributes:
|
||||
- key: service.namespace
|
||||
value: proj-airi
|
||||
action: upsert
|
||||
# Tail-based sampling: keeps all errors and slow requests,
|
||||
# samples a percentage of normal traffic to reduce storage costs.
|
||||
tail_sampling:
|
||||
decision_wait: 10s
|
||||
num_traces: 100000
|
||||
policies:
|
||||
# Always keep traces that contain errors
|
||||
- name: errors-policy
|
||||
type: status_code
|
||||
status_code:
|
||||
status_codes: [ERROR]
|
||||
# Always keep slow requests (> 500ms)
|
||||
- name: slow-requests-policy
|
||||
type: latency
|
||||
latency:
|
||||
threshold_ms: 500
|
||||
# Sample 10% of remaining normal traffic
|
||||
- name: probabilistic-policy
|
||||
type: probabilistic
|
||||
probabilistic:
|
||||
sampling_percentage: 10
|
||||
|
||||
exporters:
|
||||
# Prometheus exporter for metrics
|
||||
prometheus:
|
||||
endpoint: 0.0.0.0:8889
|
||||
namespace: airi
|
||||
resource_to_telemetry_conversion:
|
||||
enabled: true
|
||||
|
||||
# Loki exporter for logs
|
||||
loki:
|
||||
endpoint: http://loki:3100/loki/api/v1/push
|
||||
|
||||
# Tempo exporter for traces
|
||||
otlp/tempo:
|
||||
endpoint: tempo:4317
|
||||
tls:
|
||||
insecure: true
|
||||
|
||||
debug:
|
||||
verbosity: basic
|
||||
|
||||
extensions:
|
||||
health_check:
|
||||
endpoint: 0.0.0.0:13133
|
||||
zpages:
|
||||
endpoint: 0.0.0.0:55679
|
||||
|
||||
service:
|
||||
extensions: [health_check, zpages]
|
||||
pipelines:
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, resource, tail_sampling, batch]
|
||||
exporters: [otlp/tempo, debug]
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, resource, batch]
|
||||
exporters: [prometheus, debug]
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, resource, batch]
|
||||
exporters: [loki, debug]
|
||||
telemetry:
|
||||
logs:
|
||||
level: info
|
||||
@@ -0,0 +1,493 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": []
|
||||
},
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 1,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
|
||||
"id": 100,
|
||||
"title": "HTTP Overview",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 100 },
|
||||
{ "color": "red", "value": 500 }
|
||||
]
|
||||
},
|
||||
"unit": "reqps"
|
||||
}
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 1 },
|
||||
"id": 1,
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto"
|
||||
},
|
||||
"title": "Request Rate",
|
||||
"type": "stat",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(http_server_request_duration_milliseconds_count{service_name=~\"$service\"}[$__rate_interval]))",
|
||||
"legendFormat": "req/s"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 0.01 },
|
||||
{ "color": "red", "value": 0.05 }
|
||||
]
|
||||
},
|
||||
"unit": "percentunit"
|
||||
}
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 1 },
|
||||
"id": 2,
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"] },
|
||||
"colorMode": "value",
|
||||
"graphMode": "area"
|
||||
},
|
||||
"title": "Error Rate (5xx)",
|
||||
"type": "stat",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(http_server_request_duration_milliseconds_count{service_name=~\"$service\", http_status_code=~\"5..\"}[$__rate_interval])) / sum(rate(http_server_request_duration_milliseconds_count{service_name=~\"$service\"}[$__rate_interval]))",
|
||||
"legendFormat": "error rate"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 200 },
|
||||
{ "color": "red", "value": 1000 }
|
||||
]
|
||||
},
|
||||
"unit": "ms"
|
||||
}
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 1 },
|
||||
"id": 3,
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"] },
|
||||
"colorMode": "value",
|
||||
"graphMode": "area"
|
||||
},
|
||||
"title": "P95 Latency",
|
||||
"type": "stat",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum(rate(http_server_request_duration_milliseconds_bucket{service_name=~\"$service\"}[$__rate_interval])) by (le))",
|
||||
"legendFormat": "p95"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 20,
|
||||
"stacking": { "mode": "none" }
|
||||
},
|
||||
"unit": "reqps"
|
||||
}
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 },
|
||||
"id": 4,
|
||||
"title": "Request Rate by Route",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (http_route) (rate(http_server_request_duration_milliseconds_count{service_name=~\"$service\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{http_route}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 20
|
||||
},
|
||||
"unit": "ms"
|
||||
}
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 9 },
|
||||
"id": 5,
|
||||
"title": "Latency by Route (P95)",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_milliseconds_bucket{service_name=~\"$service\"}[$__rate_interval])))",
|
||||
"legendFormat": "{{http_route}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "bars",
|
||||
"fillOpacity": 80,
|
||||
"stacking": { "mode": "normal" }
|
||||
},
|
||||
"unit": "reqps"
|
||||
}
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 17 },
|
||||
"id": 6,
|
||||
"title": "Response Status Codes",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (http_status_code) (rate(http_server_request_duration_milliseconds_count{service_name=~\"$service\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{http_status_code}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 25 },
|
||||
"id": 101,
|
||||
"title": "Node.js Runtime",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 20
|
||||
},
|
||||
"unit": "bytes"
|
||||
}
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 26 },
|
||||
"id": 7,
|
||||
"title": "Heap Memory Usage",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "process_runtime_nodejs_memory_heap_total{service_name=~\"$service\"}",
|
||||
"legendFormat": "heap total"
|
||||
},
|
||||
{
|
||||
"expr": "process_runtime_nodejs_memory_heap_used{service_name=~\"$service\"}",
|
||||
"legendFormat": "heap used"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 20
|
||||
},
|
||||
"unit": "bytes"
|
||||
}
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 26 },
|
||||
"id": 8,
|
||||
"title": "RSS & External Memory",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "process_runtime_nodejs_memory_rss{service_name=~\"$service\"}",
|
||||
"legendFormat": "RSS"
|
||||
},
|
||||
{
|
||||
"expr": "process_runtime_nodejs_memory_array_buffers{service_name=~\"$service\"}",
|
||||
"legendFormat": "array buffers"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 20
|
||||
},
|
||||
"unit": "short"
|
||||
}
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 34 },
|
||||
"id": 9,
|
||||
"title": "Event Loop Lag",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "process_runtime_nodejs_event_loop_lag_p99{service_name=~\"$service\"}",
|
||||
"legendFormat": "p99 lag"
|
||||
},
|
||||
{
|
||||
"expr": "process_runtime_nodejs_event_loop_lag_p50{service_name=~\"$service\"}",
|
||||
"legendFormat": "p50 lag"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 20
|
||||
},
|
||||
"unit": "short"
|
||||
}
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 34 },
|
||||
"id": 10,
|
||||
"title": "Active Handles & Requests",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "process_runtime_nodejs_active_handles_total{service_name=~\"$service\"}",
|
||||
"legendFormat": "active handles"
|
||||
},
|
||||
{
|
||||
"expr": "process_runtime_nodejs_active_requests_total{service_name=~\"$service\"}",
|
||||
"legendFormat": "active requests"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 42 },
|
||||
"id": 102,
|
||||
"title": "PostgreSQL",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 20
|
||||
},
|
||||
"unit": "ms"
|
||||
}
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 43 },
|
||||
"id": 11,
|
||||
"title": "DB Query Duration (P95)",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum by (le, db_operation) (rate(db_client_operation_duration_milliseconds_bucket{service_name=~\"$service\"}[$__rate_interval])))",
|
||||
"legendFormat": "{{db_operation}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 20
|
||||
},
|
||||
"unit": "ops"
|
||||
}
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 43 },
|
||||
"id": 12,
|
||||
"title": "DB Query Rate",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (db_operation) (rate(db_client_operation_duration_milliseconds_count{service_name=~\"$service\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{db_operation}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 20
|
||||
},
|
||||
"unit": "short"
|
||||
}
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 51 },
|
||||
"id": 13,
|
||||
"title": "DB Connection Pool",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "db_client_connections_usage{service_name=~\"$service\", state=\"used\"}",
|
||||
"legendFormat": "used"
|
||||
},
|
||||
{
|
||||
"expr": "db_client_connections_usage{service_name=~\"$service\", state=\"idle\"}",
|
||||
"legendFormat": "idle"
|
||||
},
|
||||
{
|
||||
"expr": "db_client_connections_pending_requests{service_name=~\"$service\"}",
|
||||
"legendFormat": "pending"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 59 },
|
||||
"id": 103,
|
||||
"title": "Redis",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 20
|
||||
},
|
||||
"unit": "ms"
|
||||
}
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 60 },
|
||||
"id": 14,
|
||||
"title": "Redis Command Duration (P95)",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum by (le, db_operation) (rate(db_client_operation_duration_milliseconds_bucket{service_name=~\"$service\", db_system=\"redis\"}[$__rate_interval])))",
|
||||
"legendFormat": "{{db_operation}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 20
|
||||
},
|
||||
"unit": "ops"
|
||||
}
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 60 },
|
||||
"id": 15,
|
||||
"title": "Redis Command Rate",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (db_operation) (rate(db_client_operation_duration_milliseconds_count{service_name=~\"$service\", db_system=\"redis\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{db_operation}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 68 },
|
||||
"id": 104,
|
||||
"title": "Application Logs",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "loki", "uid": "grafanacloud-projairi-logs" },
|
||||
"gridPos": { "h": 10, "w": 24, "x": 0, "y": 69 },
|
||||
"id": 16,
|
||||
"title": "Application Logs",
|
||||
"type": "logs",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "{service_name=~\"$service\"} |= ``",
|
||||
"legendFormat": ""
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"showTime": true,
|
||||
"showLabels": true,
|
||||
"showCommonLabels": false,
|
||||
"wrapLogMessage": true,
|
||||
"prettifyLogMessage": false,
|
||||
"enableLogDetails": true,
|
||||
"sortOrder": "Descending"
|
||||
}
|
||||
}
|
||||
],
|
||||
"schemaVersion": 39,
|
||||
"tags": ["airi", "observability", "grafana-cloud"],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"current": { "selected": false, "text": "server", "value": "server" },
|
||||
"datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" },
|
||||
"definition": "label_values(http_server_request_duration_milliseconds_count, service_name)",
|
||||
"hide": 0,
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
"name": "service",
|
||||
"query": "label_values(http_server_request_duration_milliseconds_count, service_name)",
|
||||
"refresh": 2,
|
||||
"type": "query"
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": { "from": "now-1h", "to": "now" },
|
||||
"title": "AIRI Server Overview (Cloud)",
|
||||
"uid": "airi-server-overview-cloud"
|
||||
}
|
||||
@@ -0,0 +1,503 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": []
|
||||
},
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 1,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
|
||||
"id": 100,
|
||||
"title": "HTTP Overview",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 100 },
|
||||
{ "color": "red", "value": 500 }
|
||||
]
|
||||
},
|
||||
"unit": "reqps"
|
||||
}
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 1 },
|
||||
"id": 1,
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto"
|
||||
},
|
||||
"title": "Request Rate",
|
||||
"type": "stat",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(airi_http_server_request_duration_milliseconds_count{service_name=~\"$service\"}[$__rate_interval]))",
|
||||
"legendFormat": "req/s"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 0.01 },
|
||||
{ "color": "red", "value": 0.05 }
|
||||
]
|
||||
},
|
||||
"unit": "percentunit"
|
||||
}
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 1 },
|
||||
"id": 2,
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"] },
|
||||
"colorMode": "value",
|
||||
"graphMode": "area"
|
||||
},
|
||||
"title": "Error Rate (5xx)",
|
||||
"type": "stat",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(airi_http_server_request_duration_milliseconds_count{service_name=~\"$service\", http_status_code=~\"5..\"}[$__rate_interval])) / sum(rate(airi_http_server_request_duration_milliseconds_count{service_name=~\"$service\"}[$__rate_interval]))",
|
||||
"legendFormat": "error rate"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 200 },
|
||||
{ "color": "red", "value": 1000 }
|
||||
]
|
||||
},
|
||||
"unit": "ms"
|
||||
}
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 1 },
|
||||
"id": 3,
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"] },
|
||||
"colorMode": "value",
|
||||
"graphMode": "area"
|
||||
},
|
||||
"title": "P95 Latency",
|
||||
"type": "stat",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum(rate(airi_http_server_request_duration_milliseconds_bucket{service_name=~\"$service\"}[$__rate_interval])) by (le))",
|
||||
"legendFormat": "p95"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 20,
|
||||
"stacking": { "mode": "none" }
|
||||
},
|
||||
"unit": "reqps"
|
||||
}
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 },
|
||||
"id": 4,
|
||||
"title": "Request Rate by Route",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (http_route) (rate(airi_http_server_request_duration_milliseconds_count{service_name=~\"$service\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{http_route}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 20
|
||||
},
|
||||
"unit": "ms"
|
||||
}
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 9 },
|
||||
"id": 5,
|
||||
"title": "Latency by Route (P95)",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum by (le, http_route) (rate(airi_http_server_request_duration_milliseconds_bucket{service_name=~\"$service\"}[$__rate_interval])))",
|
||||
"legendFormat": "{{http_route}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "bars",
|
||||
"fillOpacity": 80,
|
||||
"stacking": { "mode": "normal" }
|
||||
},
|
||||
"unit": "reqps"
|
||||
}
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 17 },
|
||||
"id": 6,
|
||||
"title": "Response Status Codes",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (http_status_code) (rate(airi_http_server_request_duration_milliseconds_count{service_name=~\"$service\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{http_status_code}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 25 },
|
||||
"id": 101,
|
||||
"title": "Node.js Runtime",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 20
|
||||
},
|
||||
"unit": "bytes"
|
||||
}
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 26 },
|
||||
"id": 7,
|
||||
"title": "Heap Memory Usage",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "airi_process_runtime_nodejs_memory_heap_total{service_name=~\"$service\"}",
|
||||
"legendFormat": "heap total"
|
||||
},
|
||||
{
|
||||
"expr": "airi_process_runtime_nodejs_memory_heap_used{service_name=~\"$service\"}",
|
||||
"legendFormat": "heap used"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 20
|
||||
},
|
||||
"unit": "bytes"
|
||||
}
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 26 },
|
||||
"id": 8,
|
||||
"title": "RSS & External Memory",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "airi_process_runtime_nodejs_memory_rss{service_name=~\"$service\"}",
|
||||
"legendFormat": "RSS"
|
||||
},
|
||||
{
|
||||
"expr": "airi_process_runtime_nodejs_memory_array_buffers{service_name=~\"$service\"}",
|
||||
"legendFormat": "array buffers"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 20
|
||||
},
|
||||
"unit": "short"
|
||||
}
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 34 },
|
||||
"id": 9,
|
||||
"title": "Event Loop Lag",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "airi_process_runtime_nodejs_event_loop_lag_p99{service_name=~\"$service\"}",
|
||||
"legendFormat": "p99 lag"
|
||||
},
|
||||
{
|
||||
"expr": "airi_process_runtime_nodejs_event_loop_lag_p50{service_name=~\"$service\"}",
|
||||
"legendFormat": "p50 lag"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 20
|
||||
},
|
||||
"unit": "short"
|
||||
}
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 34 },
|
||||
"id": 10,
|
||||
"title": "Active Handles & Requests",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "airi_process_runtime_nodejs_active_handles_total{service_name=~\"$service\"}",
|
||||
"legendFormat": "active handles"
|
||||
},
|
||||
{
|
||||
"expr": "airi_process_runtime_nodejs_active_requests_total{service_name=~\"$service\"}",
|
||||
"legendFormat": "active requests"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 42 },
|
||||
"id": 102,
|
||||
"title": "PostgreSQL",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 20
|
||||
},
|
||||
"unit": "ms"
|
||||
}
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 43 },
|
||||
"id": 11,
|
||||
"title": "DB Query Duration (P95)",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum by (le, db_operation) (rate(airi_db_client_operation_duration_milliseconds_bucket{service_name=~\"$service\"}[$__rate_interval])))",
|
||||
"legendFormat": "{{db_operation}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 20
|
||||
},
|
||||
"unit": "ops"
|
||||
}
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 43 },
|
||||
"id": 12,
|
||||
"title": "DB Query Rate",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (db_operation) (rate(airi_db_client_operation_duration_milliseconds_count{service_name=~\"$service\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{db_operation}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 20
|
||||
},
|
||||
"unit": "short"
|
||||
}
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 51 },
|
||||
"id": 13,
|
||||
"title": "DB Connection Pool",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "airi_db_client_connections_usage{service_name=~\"$service\", state=\"used\"}",
|
||||
"legendFormat": "used"
|
||||
},
|
||||
{
|
||||
"expr": "airi_db_client_connections_usage{service_name=~\"$service\", state=\"idle\"}",
|
||||
"legendFormat": "idle"
|
||||
},
|
||||
{
|
||||
"expr": "airi_db_client_connections_pending_requests{service_name=~\"$service\"}",
|
||||
"legendFormat": "pending"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 59 },
|
||||
"id": 103,
|
||||
"title": "Redis",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 20
|
||||
},
|
||||
"unit": "ms"
|
||||
}
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 60 },
|
||||
"id": 14,
|
||||
"title": "Redis Command Duration (P95)",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum by (le, db_operation) (rate(airi_db_client_operation_duration_milliseconds_bucket{service_name=~\"$service\", db_system=\"redis\"}[$__rate_interval])))",
|
||||
"legendFormat": "{{db_operation}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 20
|
||||
},
|
||||
"unit": "ops"
|
||||
}
|
||||
},
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 60 },
|
||||
"id": 15,
|
||||
"title": "Redis Command Rate",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (db_operation) (rate(airi_db_client_operation_duration_milliseconds_count{service_name=~\"$service\", db_system=\"redis\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{db_operation}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 68 },
|
||||
"id": 104,
|
||||
"title": "Application Logs",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"gridPos": { "h": 10, "w": 24, "x": 0, "y": 69 },
|
||||
"id": 16,
|
||||
"title": "Application Logs",
|
||||
"type": "logs",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "{service_name=~\"$service\"} |= ``",
|
||||
"legendFormat": ""
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"showTime": true,
|
||||
"showLabels": true,
|
||||
"showCommonLabels": false,
|
||||
"wrapLogMessage": true,
|
||||
"prettifyLogMessage": false,
|
||||
"enableLogDetails": true,
|
||||
"sortOrder": "Descending"
|
||||
}
|
||||
}
|
||||
],
|
||||
"schemaVersion": 39,
|
||||
"tags": ["airi", "observability"],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"current": { "selected": false, "text": "Prometheus", "value": "Prometheus" },
|
||||
"hide": 0,
|
||||
"includeAll": false,
|
||||
"name": "datasource",
|
||||
"options": [],
|
||||
"query": "prometheus",
|
||||
"refresh": 1,
|
||||
"type": "datasource"
|
||||
},
|
||||
{
|
||||
"current": { "selected": false, "text": "airi-server", "value": "airi-server" },
|
||||
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||||
"definition": "label_values(airi_http_server_request_duration_milliseconds_count, service_name)",
|
||||
"hide": 0,
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
"name": "service",
|
||||
"query": "label_values(airi_http_server_request_duration_milliseconds_count, service_name)",
|
||||
"refresh": 2,
|
||||
"type": "query"
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": { "from": "now-1h", "to": "now" },
|
||||
"title": "AIRI Server Overview",
|
||||
"uid": "airi-server-overview"
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: AIRI Dashboards
|
||||
orgId: 1
|
||||
folder: AIRI
|
||||
type: file
|
||||
disableDeletion: false
|
||||
editable: true
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
foldersFromFilesStructure: false
|
||||
@@ -0,0 +1,45 @@
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: true
|
||||
jsonData:
|
||||
httpMethod: POST
|
||||
exemplarTraceIdDestinations:
|
||||
- name: traceID
|
||||
datasourceUid: tempo
|
||||
|
||||
- name: Loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
editable: true
|
||||
jsonData:
|
||||
derivedFields:
|
||||
- datasourceUid: tempo
|
||||
matcherRegex: '"traceId":"(\w+)"'
|
||||
name: TraceID
|
||||
url: '$${__value.raw}'
|
||||
|
||||
- name: Tempo
|
||||
type: tempo
|
||||
access: proxy
|
||||
url: http://tempo:3200
|
||||
uid: tempo
|
||||
editable: true
|
||||
jsonData:
|
||||
tracesToLogsV2:
|
||||
datasourceUid: loki
|
||||
filterByTraceID: true
|
||||
tracesToMetrics:
|
||||
datasourceUid: prometheus
|
||||
serviceMap:
|
||||
datasourceUid: prometheus
|
||||
nodeGraph:
|
||||
enabled: true
|
||||
lokiSearch:
|
||||
datasourceUid: loki
|
||||
@@ -0,0 +1,36 @@
|
||||
auth_enabled: false
|
||||
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
|
||||
common:
|
||||
path_prefix: /loki
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
|
||||
schema_config:
|
||||
configs:
|
||||
- from: '2024-01-01'
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
|
||||
limits_config:
|
||||
allow_structured_metadata: true
|
||||
volume_enabled: true
|
||||
|
||||
query_range:
|
||||
results_cache:
|
||||
cache:
|
||||
embedded_cache:
|
||||
enabled: true
|
||||
max_size_mb: 100
|
||||
@@ -0,0 +1,16 @@
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
|
||||
scrape_configs:
|
||||
# Scrape OTEL Collector's Prometheus exporter
|
||||
- job_name: otel-collector
|
||||
static_configs:
|
||||
- targets: ['otel-collector:8889']
|
||||
labels:
|
||||
service: otel-collector
|
||||
|
||||
# Scrape OTEL Collector's own metrics
|
||||
- job_name: otel-collector-internal
|
||||
static_configs:
|
||||
- targets: ['otel-collector:8888']
|
||||
@@ -0,0 +1,47 @@
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
local:
|
||||
path: /var/tempo/traces
|
||||
wal:
|
||||
path: /var/tempo/wal
|
||||
|
||||
metrics_generator:
|
||||
registry:
|
||||
external_labels:
|
||||
source: tempo
|
||||
cluster: docker-compose
|
||||
storage:
|
||||
path: /var/tempo/generator/wal
|
||||
remote_write:
|
||||
- url: http://prometheus:9090/api/v1/write
|
||||
send_exemplars: true
|
||||
traces_storage:
|
||||
path: /var/tempo/generator/traces
|
||||
processor:
|
||||
service_graphs:
|
||||
dimensions:
|
||||
- http.method
|
||||
- http.target
|
||||
span_metrics:
|
||||
dimensions:
|
||||
- http.method
|
||||
- http.target
|
||||
- http.status_code
|
||||
|
||||
overrides:
|
||||
defaults:
|
||||
metrics_generator:
|
||||
processors: [service-graphs, span-metrics]
|
||||
@@ -12,10 +12,25 @@
|
||||
"db:push": "pnpm run apply:env -- drizzle-kit push"
|
||||
},
|
||||
"dependencies": {
|
||||
"@better-auth/drizzle-adapter": "^1.5.3",
|
||||
"@dotenvx/dotenvx": "^1.52.0",
|
||||
"@electric-sql/pglite": "catalog:",
|
||||
"@guiiai/logg": "catalog:",
|
||||
"@hono/node-server": "^1.19.9",
|
||||
"@hono/node-server": "^1.19.11",
|
||||
"@opentelemetry/api": "^1.9.0",
|
||||
"@opentelemetry/exporter-logs-otlp-proto": "^0.213.0",
|
||||
"@opentelemetry/exporter-metrics-otlp-proto": "^0.213.0",
|
||||
"@opentelemetry/exporter-trace-otlp-proto": "^0.213.0",
|
||||
"@opentelemetry/instrumentation-http": "^0.213.0",
|
||||
"@opentelemetry/instrumentation-ioredis": "^0.61.0",
|
||||
"@opentelemetry/instrumentation-pg": "^0.65.0",
|
||||
"@opentelemetry/instrumentation-runtime-node": "^0.12.2",
|
||||
"@opentelemetry/resources": "^2.6.0",
|
||||
"@opentelemetry/sdk-logs": "^0.213.0",
|
||||
"@opentelemetry/sdk-metrics": "^2.6.0",
|
||||
"@opentelemetry/sdk-node": "^0.213.0",
|
||||
"@opentelemetry/sdk-trace-node": "^2.6.0",
|
||||
"@opentelemetry/semantic-conventions": "^1.40.0",
|
||||
"@proj-airi/drizzle-orm-browser-migrator": "^0.1.4",
|
||||
"@proj-airi/server-schema": "workspace:*",
|
||||
"better-auth": "^1.4.19",
|
||||
@@ -28,7 +43,7 @@
|
||||
"valibot": "catalog:"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@better-auth/cli": "^1.4.19",
|
||||
"@better-auth/cli": "^1.4.21",
|
||||
"drizzle-kit": "^0.31.9"
|
||||
}
|
||||
}
|
||||
|
||||
+29
-4
@@ -8,12 +8,14 @@ import { Hono } from 'hono'
|
||||
import { bodyLimit } from 'hono/body-limit'
|
||||
import { cors } from 'hono/cors'
|
||||
import { logger as honoLogger } from 'hono/logger'
|
||||
import { createLoggLogger, injeca } from 'injeca'
|
||||
import { createLoggLogger, injeca, lifecycle } from 'injeca'
|
||||
|
||||
import { createAuth } from './libs/auth'
|
||||
import { createDrizzle, migrateDatabase } from './libs/db'
|
||||
import { parsedEnv } from './libs/env'
|
||||
import { initOtel } from './libs/otel'
|
||||
import { sessionMiddleware } from './middlewares/auth'
|
||||
import { otelMiddleware } from './middlewares/otel'
|
||||
import { createCharacterRoutes } from './routes/characters'
|
||||
import { createChatRoutes } from './routes/chats'
|
||||
import { createProviderRoutes } from './routes/providers'
|
||||
@@ -28,17 +30,20 @@ type CharacterService = ReturnType<typeof createCharacterService>
|
||||
type ChatService = ReturnType<typeof createChatService>
|
||||
type ProviderService = ReturnType<typeof createProviderService>
|
||||
|
||||
type OtelMetrics = ReturnType<typeof initOtel>
|
||||
|
||||
interface AppDeps {
|
||||
auth: AuthService
|
||||
characterService: CharacterService
|
||||
chatService: ChatService
|
||||
providerService: ProviderService
|
||||
otel: OtelMetrics | null
|
||||
}
|
||||
|
||||
function buildApp({ auth, characterService, chatService, providerService }: AppDeps) {
|
||||
function buildApp({ auth, characterService, chatService, providerService, otel }: AppDeps) {
|
||||
const logger = useLogger('app').useGlobalConfig()
|
||||
|
||||
return new Hono<HonoEnv>()
|
||||
const app = new Hono<HonoEnv>()
|
||||
.use(
|
||||
'/api/*',
|
||||
cors({
|
||||
@@ -47,6 +52,12 @@ function buildApp({ auth, characterService, chatService, providerService }: AppD
|
||||
}),
|
||||
)
|
||||
.use(honoLogger())
|
||||
|
||||
if (otel) {
|
||||
app.use('*', otelMiddleware(otel))
|
||||
}
|
||||
|
||||
return app
|
||||
.use('*', sessionMiddleware(auth))
|
||||
.use('*', bodyLimit({ maxSize: 1024 * 1024 }))
|
||||
.onError((err, c) => {
|
||||
@@ -102,6 +113,19 @@ async function createApp() {
|
||||
injeca.setLogger(createLoggLogger(useLogger('injeca').useGlobalConfig()))
|
||||
const logger = useLogger('app').useGlobalConfig()
|
||||
|
||||
const otel = injeca.provide('otel', {
|
||||
dependsOn: { env: parsedEnv, lifecycle },
|
||||
build: ({ dependsOn }) => {
|
||||
const o = initOtel(dependsOn.env)
|
||||
if (!o)
|
||||
return null
|
||||
|
||||
o.start()
|
||||
dependsOn.lifecycle.appHooks.onStop(() => o.shutdown())
|
||||
return o
|
||||
},
|
||||
})
|
||||
|
||||
const db = injeca.provide('services:db', {
|
||||
dependsOn: { env: parsedEnv },
|
||||
build: async ({ dependsOn }) => {
|
||||
@@ -136,12 +160,13 @@ async function createApp() {
|
||||
})
|
||||
|
||||
await injeca.start()
|
||||
const resolved = await injeca.resolve({ auth, characterService, chatService, providerService })
|
||||
const resolved = await injeca.resolve({ auth, characterService, chatService, providerService, otel })
|
||||
const app = buildApp({
|
||||
auth: resolved.auth,
|
||||
characterService: resolved.characterService,
|
||||
chatService: resolved.chatService,
|
||||
providerService: resolved.providerService,
|
||||
otel: resolved.otel,
|
||||
})
|
||||
|
||||
logger.withFields({ port: 3000 }).log('Server started')
|
||||
|
||||
@@ -15,6 +15,14 @@ const EnvSchema = object({
|
||||
AUTH_GOOGLE_CLIENT_SECRET: pipe(string(), nonEmpty('AUTH_GOOGLE_CLIENT_SECRET is required')),
|
||||
AUTH_GITHUB_CLIENT_ID: pipe(string(), nonEmpty('AUTH_GITHUB_CLIENT_ID is required')),
|
||||
AUTH_GITHUB_CLIENT_SECRET: pipe(string(), nonEmpty('AUTH_GITHUB_CLIENT_SECRET is required')),
|
||||
|
||||
// OpenTelemetry
|
||||
OTEL_SERVICE_NAMESPACE: optional(string(), 'airi'),
|
||||
OTEL_SERVICE_NAME: optional(string(), 'server'),
|
||||
OTEL_TRACES_SAMPLING_RATIO: optional(string(), '1.0'),
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: optional(string()),
|
||||
OTEL_EXPORTER_OTLP_HEADERS: optional(string()),
|
||||
OTEL_DEBUG: optional(string()),
|
||||
})
|
||||
|
||||
export type Env = InferOutput<typeof EnvSchema>
|
||||
|
||||
@@ -0,0 +1,167 @@
|
||||
import type { IncomingMessage } from 'node:http'
|
||||
|
||||
import type { Env } from './env'
|
||||
|
||||
import { env as processEnv } from 'node:process'
|
||||
|
||||
import { useLogger } from '@guiiai/logg'
|
||||
import { diag, DiagConsoleLogger, DiagLogLevel, metrics } from '@opentelemetry/api'
|
||||
import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-proto'
|
||||
import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-proto'
|
||||
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-proto'
|
||||
import { HttpInstrumentation } from '@opentelemetry/instrumentation-http'
|
||||
import { IORedisInstrumentation } from '@opentelemetry/instrumentation-ioredis'
|
||||
import { PgInstrumentation } from '@opentelemetry/instrumentation-pg'
|
||||
import { RuntimeNodeInstrumentation } from '@opentelemetry/instrumentation-runtime-node'
|
||||
import { resourceFromAttributes } from '@opentelemetry/resources'
|
||||
import { BatchLogRecordProcessor } from '@opentelemetry/sdk-logs'
|
||||
import { PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'
|
||||
import { NodeSDK } from '@opentelemetry/sdk-node'
|
||||
import { BatchSpanProcessor, ParentBasedSampler, TraceIdRatioBasedSampler } from '@opentelemetry/sdk-trace-node'
|
||||
import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions'
|
||||
|
||||
const logger = useLogger('otel')
|
||||
|
||||
export function initOtel(env: Env) {
|
||||
const otlpEndpoint = env.OTEL_EXPORTER_OTLP_ENDPOINT
|
||||
const serviceName = env.OTEL_SERVICE_NAME
|
||||
|
||||
if (!otlpEndpoint) {
|
||||
logger.log('OpenTelemetry disabled (set OTEL_EXPORTER_OTLP_ENDPOINT to enable)')
|
||||
return
|
||||
}
|
||||
|
||||
if (env.OTEL_DEBUG === 'true') {
|
||||
diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.DEBUG)
|
||||
}
|
||||
|
||||
// Parse OTEL_EXPORTER_OTLP_HEADERS (format: "key=value,key2=value2")
|
||||
const headers: Record<string, string> = {}
|
||||
const rawHeaders = env.OTEL_EXPORTER_OTLP_HEADERS
|
||||
if (rawHeaders) {
|
||||
for (const pair of rawHeaders.split(',')) {
|
||||
const idx = pair.indexOf('=')
|
||||
if (idx > 0) {
|
||||
headers[pair.slice(0, idx).trim()] = pair.slice(idx + 1).trim()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const resource = resourceFromAttributes({
|
||||
[ATTR_SERVICE_NAME]: serviceName,
|
||||
[ATTR_SERVICE_VERSION]: processEnv.npm_package_version || '0.0.0',
|
||||
'service.namespace': env.OTEL_SERVICE_NAMESPACE,
|
||||
'deployment.environment': processEnv.NODE_ENV || 'development',
|
||||
})
|
||||
|
||||
const traceExporter = new OTLPTraceExporter({
|
||||
url: `${otlpEndpoint}/v1/traces`,
|
||||
headers,
|
||||
})
|
||||
|
||||
const metricExporter = new OTLPMetricExporter({
|
||||
url: `${otlpEndpoint}/v1/metrics`,
|
||||
headers,
|
||||
})
|
||||
|
||||
const logExporter = new OTLPLogExporter({
|
||||
url: `${otlpEndpoint}/v1/logs`,
|
||||
headers,
|
||||
})
|
||||
|
||||
// Head-based sampling ratio: 1.0 = 100% (default), 0.1 = 10%, etc.
|
||||
// Metrics are always 100% accurate regardless of this setting.
|
||||
const samplingRatio = Number.parseFloat(env.OTEL_TRACES_SAMPLING_RATIO)
|
||||
const sampler = new ParentBasedSampler({
|
||||
root: new TraceIdRatioBasedSampler(samplingRatio),
|
||||
})
|
||||
|
||||
const sdk = new NodeSDK({
|
||||
resource,
|
||||
sampler,
|
||||
spanProcessors: [new BatchSpanProcessor(traceExporter)],
|
||||
metricReader: new PeriodicExportingMetricReader({
|
||||
exporter: metricExporter,
|
||||
exportIntervalMillis: 15000,
|
||||
}),
|
||||
logRecordProcessors: [new BatchLogRecordProcessor(logExporter)],
|
||||
instrumentations: [
|
||||
new HttpInstrumentation({
|
||||
ignoreIncomingRequestHook: (req: IncomingMessage) => {
|
||||
// Ignore health check requests to reduce noise
|
||||
return req.url === '/health'
|
||||
},
|
||||
}),
|
||||
new PgInstrumentation({
|
||||
enhancedDatabaseReporting: true,
|
||||
}),
|
||||
new IORedisInstrumentation(),
|
||||
new RuntimeNodeInstrumentation(),
|
||||
],
|
||||
})
|
||||
|
||||
const start = () => {
|
||||
sdk.start()
|
||||
logger.log(`OpenTelemetry initialized, exporting to ${otlpEndpoint}, sampling ratio: ${samplingRatio}`)
|
||||
}
|
||||
|
||||
const meter = metrics.getMeter(serviceName)
|
||||
|
||||
// Custom application metrics
|
||||
const httpRequestDuration = meter.createHistogram('http.server.request.duration', {
|
||||
description: 'HTTP server request duration in milliseconds',
|
||||
unit: 'ms',
|
||||
})
|
||||
|
||||
const httpActiveRequests = meter.createUpDownCounter('http.server.active_requests', {
|
||||
description: 'Number of active HTTP requests',
|
||||
})
|
||||
|
||||
const dbQueryDuration = meter.createHistogram('db.client.operation.duration', {
|
||||
description: 'Database operation duration in milliseconds',
|
||||
unit: 'ms',
|
||||
})
|
||||
|
||||
const redisCommandDuration = meter.createHistogram('redis.client.command.duration', {
|
||||
description: 'Redis command duration in milliseconds',
|
||||
unit: 'ms',
|
||||
})
|
||||
|
||||
const authAttempts = meter.createCounter('auth.attempts', {
|
||||
description: 'Number of authentication attempts',
|
||||
})
|
||||
|
||||
const authFailures = meter.createCounter('auth.failures', {
|
||||
description: 'Number of failed authentication attempts',
|
||||
})
|
||||
|
||||
const stripeEvents = meter.createCounter('stripe.events', {
|
||||
description: 'Number of Stripe webhook events processed',
|
||||
})
|
||||
|
||||
// Graceful shutdown
|
||||
const shutdown = async () => {
|
||||
try {
|
||||
await sdk.shutdown()
|
||||
logger.log('OpenTelemetry shut down successfully')
|
||||
}
|
||||
catch (err) {
|
||||
logger.withError(err).error('Error shutting down OpenTelemetry')
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
sdk,
|
||||
meter,
|
||||
httpRequestDuration,
|
||||
httpActiveRequests,
|
||||
dbQueryDuration,
|
||||
redisCommandDuration,
|
||||
authAttempts,
|
||||
authFailures,
|
||||
stripeEvents,
|
||||
|
||||
start,
|
||||
shutdown,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
import type { MiddlewareHandler } from 'hono'
|
||||
|
||||
import type { HonoEnv } from '../types/hono'
|
||||
|
||||
import { context, SpanStatusCode, trace } from '@opentelemetry/api'
|
||||
|
||||
const tracer = trace.getTracer('airi-server-hono')
|
||||
|
||||
/**
|
||||
* Hono middleware that creates spans for each request and records
|
||||
* custom HTTP metrics (duration, active requests, status codes).
|
||||
*/
|
||||
export function otelMiddleware(otelMetrics: {
|
||||
httpRequestDuration: { record: (value: number, attributes?: Record<string, string | number>) => void }
|
||||
httpActiveRequests: { add: (value: number, attributes?: Record<string, string>) => void }
|
||||
}): MiddlewareHandler<HonoEnv> {
|
||||
return async (c, next) => {
|
||||
const startTime = performance.now()
|
||||
const method = c.req.method
|
||||
const path = c.req.routePath || c.req.path
|
||||
const attributes = {
|
||||
'http.method': method,
|
||||
'http.route': path,
|
||||
'http.url': c.req.url,
|
||||
}
|
||||
|
||||
otelMetrics.httpActiveRequests.add(1, { 'http.method': method, 'http.route': path })
|
||||
|
||||
const span = tracer.startSpan(`${method} ${path}`, { attributes })
|
||||
|
||||
try {
|
||||
await context.with(trace.setSpan(context.active(), span), () => next())
|
||||
|
||||
const status = c.res.status
|
||||
span.setAttribute('http.status_code', status)
|
||||
|
||||
if (status >= 500) {
|
||||
span.setStatus({ code: SpanStatusCode.ERROR, message: `HTTP ${status}` })
|
||||
}
|
||||
|
||||
otelMetrics.httpRequestDuration.record(performance.now() - startTime, {
|
||||
'http.method': method,
|
||||
'http.route': path,
|
||||
'http.status_code': status,
|
||||
})
|
||||
}
|
||||
catch (err) {
|
||||
span.setStatus({ code: SpanStatusCode.ERROR, message: err instanceof Error ? err.message : 'Unknown error' })
|
||||
span.recordException(err instanceof Error ? err : new Error(String(err)))
|
||||
throw err
|
||||
}
|
||||
finally {
|
||||
otelMetrics.httpActiveRequests.add(-1, { 'http.method': method, 'http.route': path })
|
||||
span.end()
|
||||
}
|
||||
}
|
||||
}
|
||||
Generated
+934
-125
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user