chore(server): remove otel config
This commit is contained in:
RainbowBird
@@ -1,122 +0,0 @@
|
||||
name: proj-airi-otel
|
||||
|
||||
services:
|
||||
# ============================================================
|
||||
# OpenTelemetry Collector
|
||||
# Receives traces, metrics, and logs from the application
|
||||
# and exports them to the appropriate backends.
|
||||
# ============================================================
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.120.0
|
||||
command: ['--config=/etc/otelcol/otel-collector.yaml']
|
||||
volumes:
|
||||
- ./otel/collector/otel-collector.yaml:/etc/otelcol/otel-collector.yaml:ro
|
||||
ports:
|
||||
- '4317:4317' # OTLP gRPC
|
||||
- '4318:4318' # OTLP HTTP
|
||||
depends_on:
|
||||
loki:
|
||||
condition: service_started
|
||||
tempo:
|
||||
condition: service_started
|
||||
healthcheck:
|
||||
test: ['CMD', 'wget', '--spider', '-q', 'http://localhost:13133/']
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
restart: unless-stopped
|
||||
|
||||
# ============================================================
|
||||
# Prometheus - Metrics storage and querying
|
||||
# ============================================================
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.2.1
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yaml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time=7d
|
||||
- --web.enable-remote-write-receiver
|
||||
- --enable-feature=exemplar-storage
|
||||
- --enable-feature=native-histograms
|
||||
volumes:
|
||||
- ./otel/prometheus/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro
|
||||
- prometheus_data:/prometheus
|
||||
healthcheck:
|
||||
test: ['CMD', 'wget', '--spider', '-q', 'http://localhost:9090/-/healthy']
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
restart: unless-stopped
|
||||
|
||||
# ============================================================
|
||||
# Loki - Log aggregation
|
||||
# ============================================================
|
||||
loki:
|
||||
image: grafana/loki:3.4.3
|
||||
command: -config.file=/etc/loki/loki.yaml
|
||||
volumes:
|
||||
- ./otel/loki/loki.yaml:/etc/loki/loki.yaml:ro
|
||||
- loki_data:/loki
|
||||
healthcheck:
|
||||
test: ['CMD', 'wget', '--spider', '-q', 'http://localhost:3100/ready']
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
restart: unless-stopped
|
||||
|
||||
# ============================================================
|
||||
# Tempo - Distributed tracing backend
|
||||
# ============================================================
|
||||
tempo:
|
||||
image: grafana/tempo:2.7.2
|
||||
command: ['-config.file=/etc/tempo/tempo.yaml']
|
||||
volumes:
|
||||
- ./otel/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
|
||||
- tempo_data:/var/tempo
|
||||
healthcheck:
|
||||
test: ['CMD', 'wget', '--spider', '-q', 'http://localhost:3200/ready']
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
restart: unless-stopped
|
||||
|
||||
# ============================================================
|
||||
# Grafana - Visualization and dashboards
|
||||
# ============================================================
|
||||
grafana:
|
||||
image: grafana/grafana:11.5.2
|
||||
environment:
|
||||
- GF_SECURITY_ADMIN_USER=admin
|
||||
- GF_SECURITY_ADMIN_PASSWORD=admin
|
||||
- GF_AUTH_ANONYMOUS_ENABLED=true
|
||||
- GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer
|
||||
- GF_FEATURE_TOGGLES_ENABLE=traceqlEditor tempoSearch tempoServiceGraph
|
||||
volumes:
|
||||
- ./otel/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./otel/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
- grafana_data:/var/lib/grafana
|
||||
ports:
|
||||
- '3001:3000'
|
||||
depends_on:
|
||||
prometheus:
|
||||
condition: service_healthy
|
||||
loki:
|
||||
condition: service_healthy
|
||||
tempo:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ['CMD', 'wget', '--spider', '-q', 'http://localhost:3000/livez']
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
prometheus_data:
|
||||
driver: local
|
||||
loki_data:
|
||||
driver: local
|
||||
tempo_data:
|
||||
driver: local
|
||||
grafana_data:
|
||||
driver: local
|
||||
@@ -1,88 +0,0 @@
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
|
||||
processors:
|
||||
batch:
|
||||
timeout: 5s
|
||||
send_batch_size: 1024
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 512
|
||||
spike_limit_mib: 128
|
||||
resource:
|
||||
attributes:
|
||||
- key: service.namespace
|
||||
value: proj-airi
|
||||
action: upsert
|
||||
# Tail-based sampling: keeps all errors and slow requests,
|
||||
# samples a percentage of normal traffic to reduce storage costs.
|
||||
tail_sampling:
|
||||
decision_wait: 10s
|
||||
num_traces: 100000
|
||||
policies:
|
||||
# Always keep traces that contain errors
|
||||
- name: errors-policy
|
||||
type: status_code
|
||||
status_code:
|
||||
status_codes: [ERROR]
|
||||
# Always keep slow requests (> 500ms)
|
||||
- name: slow-requests-policy
|
||||
type: latency
|
||||
latency:
|
||||
threshold_ms: 500
|
||||
# Sample 10% of remaining normal traffic
|
||||
- name: probabilistic-policy
|
||||
type: probabilistic
|
||||
probabilistic:
|
||||
sampling_percentage: 10
|
||||
|
||||
exporters:
|
||||
# Prometheus exporter for metrics
|
||||
prometheus:
|
||||
endpoint: 0.0.0.0:8889
|
||||
namespace: airi
|
||||
resource_to_telemetry_conversion:
|
||||
enabled: true
|
||||
|
||||
# Loki exporter for logs
|
||||
loki:
|
||||
endpoint: http://loki:3100/loki/api/v1/push
|
||||
|
||||
# Tempo exporter for traces
|
||||
otlp/tempo:
|
||||
endpoint: tempo:4317
|
||||
tls:
|
||||
insecure: true
|
||||
|
||||
debug:
|
||||
verbosity: basic
|
||||
|
||||
extensions:
|
||||
health_check:
|
||||
endpoint: 0.0.0.0:13133
|
||||
zpages:
|
||||
endpoint: 0.0.0.0:55679
|
||||
|
||||
service:
|
||||
extensions: [health_check, zpages]
|
||||
pipelines:
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, resource, tail_sampling, batch]
|
||||
exporters: [otlp/tempo, debug]
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, resource, batch]
|
||||
exporters: [prometheus, debug]
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, resource, batch]
|
||||
exporters: [loki, debug]
|
||||
telemetry:
|
||||
logs:
|
||||
level: info
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,156 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { checkDashboardLayoutReferences, dashboard } from './build'
|
||||
|
||||
/**
|
||||
* Narrows unknown dashboard nodes into indexable records for assertions.
|
||||
*/
|
||||
function asRecord(value: unknown, label: string): Record<string, unknown> {
|
||||
if (!value || typeof value !== 'object')
|
||||
throw new TypeError(`${label} is not an object`)
|
||||
|
||||
return value as Record<string, unknown>
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a generated panel title from the dashboard object.
|
||||
*/
|
||||
function panelTitle(panelName: string): string {
|
||||
const panel = asRecord(dashboard.elements[panelName], panelName)
|
||||
const spec = asRecord(panel.spec, `${panelName}.spec`)
|
||||
if (typeof spec.title !== 'string')
|
||||
throw new TypeError(`${panelName}.spec.title is not a string`)
|
||||
|
||||
return spec.title
|
||||
}
|
||||
|
||||
/**
|
||||
* Collects PromQL expression strings from nested Grafana panel objects.
|
||||
*/
|
||||
function collectQueryExpressions(value: unknown, expressions: string[] = []): string[] {
|
||||
if (!value || typeof value !== 'object')
|
||||
return expressions
|
||||
|
||||
const record = value as Record<string, unknown>
|
||||
if (typeof record.expr === 'string')
|
||||
expressions.push(record.expr)
|
||||
|
||||
for (const nestedValue of Object.values(record))
|
||||
collectQueryExpressions(nestedValue, expressions)
|
||||
|
||||
return expressions
|
||||
}
|
||||
|
||||
describe('grafana dashboard builder', () => {
|
||||
/**
|
||||
* @example
|
||||
* const result = checkDashboardLayoutReferences(dashboard)
|
||||
* expect(result.orphanRefs).toEqual([])
|
||||
*/
|
||||
it('keeps every generated panel connected to the row layout', () => {
|
||||
const result = checkDashboardLayoutReferences(dashboard)
|
||||
|
||||
expect(result.orphanRefs).toEqual([])
|
||||
expect(result.unusedElems).toEqual([])
|
||||
})
|
||||
|
||||
it('keeps the product analytics row focused on Prometheus-safe engagement signals', () => {
|
||||
expect(panelTitle('panel-95')).toBe('Product Events (range)')
|
||||
expect(panelTitle('panel-96')).toBe('Product Failure %')
|
||||
expect(panelTitle('panel-97')).toBe('Top Product Actions (range)')
|
||||
expect(panelTitle('panel-98')).toBe('Product Event Rate')
|
||||
expect(panelTitle('panel-99')).toBe('DAU Trend')
|
||||
})
|
||||
|
||||
/**
|
||||
* @example
|
||||
* const rendered = JSON.stringify(dashboard.elements['panel-101'])
|
||||
* expect(rendered).not.toContain('voice_id')
|
||||
*/
|
||||
it('keeps high-cardinality voice fields out of Prometheus queries', () => {
|
||||
const productPanelExpressions = collectQueryExpressions([
|
||||
dashboard.elements['panel-95'],
|
||||
dashboard.elements['panel-96'],
|
||||
dashboard.elements['panel-97'],
|
||||
dashboard.elements['panel-98'],
|
||||
dashboard.elements['panel-99'],
|
||||
]).join('\n')
|
||||
|
||||
expect(productPanelExpressions).not.toContain('voice_id')
|
||||
expect(productPanelExpressions).not.toContain('voice_pack_id')
|
||||
expect(productPanelExpressions).not.toContain('user_id')
|
||||
expect(productPanelExpressions).not.toContain('session_id')
|
||||
expect(productPanelExpressions).not.toContain('request_id')
|
||||
})
|
||||
|
||||
it('preserves histogram buckets until HTTP and LLM latency quantiles are calculated', () => {
|
||||
// ROOT CAUSE:
|
||||
//
|
||||
// The latency panels previously summed cumulative `_bucket` rates after
|
||||
// dropping `le`. Grafana then labelled that request-rate-derived value as
|
||||
// seconds, making millisecond HTTP routes appear to take 20+ seconds.
|
||||
//
|
||||
// Quantiles must retain `le` while replicas are merged, then call
|
||||
// histogram_quantile over the merged histogram.
|
||||
const httpLatency = collectQueryExpressions(dashboard.elements['panel-20']).join('\n')
|
||||
const llmLatency = collectQueryExpressions(dashboard.elements['panel-21']).join('\n')
|
||||
|
||||
expect(httpLatency).toContain('histogram_quantile(0.95')
|
||||
expect(httpLatency).toContain('sum by (le, http_route)')
|
||||
expect(llmLatency.match(/histogram_quantile\(0\.95/g)).toHaveLength(2)
|
||||
expect(llmLatency).toContain('gen_ai_client_first_token_duration_seconds_bucket')
|
||||
expect(llmLatency).toContain('sum by (le)')
|
||||
})
|
||||
|
||||
it('does not clamp rate denominators to one request per second', () => {
|
||||
// ROOT CAUSE:
|
||||
//
|
||||
// `clamp_min(rate, 1)` changes the denominator whenever traffic is below
|
||||
// 1 req/s, so low-volume provider and fallback failures are underreported.
|
||||
for (const panelName of ['panel-4', 'panel-62', 'panel-68']) {
|
||||
const expressions = collectQueryExpressions(dashboard.elements[panelName]).join('\n')
|
||||
|
||||
expect(expressions).not.toContain('clamp_min')
|
||||
}
|
||||
})
|
||||
|
||||
it('uses visualizations and reductions that match each query shape', () => {
|
||||
const statusPanel = JSON.stringify(dashboard.elements['panel-40'])
|
||||
const modelMixPanel = JSON.stringify(dashboard.elements['panel-11'])
|
||||
|
||||
expect(statusPanel).toContain('"group":"timeseries"')
|
||||
expect(statusPanel).not.toContain('"group":"heatmap"')
|
||||
expect(modelMixPanel).toContain('increase(')
|
||||
expect(modelMixPanel).toContain('"instant":true')
|
||||
expect(modelMixPanel).toContain('"calcs":["lastNotNull"]')
|
||||
})
|
||||
|
||||
it('filters structured Loki severity without parsing plain-text bodies as JSON', () => {
|
||||
const errorLogsPanel = JSON.stringify(dashboard.elements['panel-91'])
|
||||
|
||||
expect(panelTitle('panel-91')).toBe('Warn / Error Logs')
|
||||
expect(errorLogsPanel).toContain('detected_level=~\\"warn|error\\"')
|
||||
expect(errorLogsPanel).not.toContain('| json')
|
||||
})
|
||||
|
||||
it('keeps one DAU visualization and refreshes the operations dashboard', () => {
|
||||
expect(dashboard.elements['panel-80']).toBeUndefined()
|
||||
expect(panelTitle('panel-99')).toBe('DAU Trend')
|
||||
expect(dashboard.timeSettings.autoRefresh).toBe('30s')
|
||||
})
|
||||
|
||||
it('queries cluster-wide distinct online websocket users as an instant value', () => {
|
||||
// ROOT CAUSE:
|
||||
//
|
||||
// Counting WebSocket contexts measures tabs/connections, not people. The
|
||||
// online-user gauge counts unique Redis broadcast channels cluster-wide,
|
||||
// and every replica reports that same shared value.
|
||||
const wsOnlinePanel = JSON.stringify(dashboard.elements['panel-93'])
|
||||
|
||||
expect(panelTitle('panel-93')).toBe('Online Users')
|
||||
expect(wsOnlinePanel).toContain('max(ws_users_online')
|
||||
expect(wsOnlinePanel).not.toContain('ws_connections_active')
|
||||
expect(wsOnlinePanel).toContain('"instant":true')
|
||||
expect(wsOnlinePanel).toContain('"range":false')
|
||||
})
|
||||
})
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,12 +0,0 @@
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: AIRI Dashboards
|
||||
orgId: 1
|
||||
folder: AIRI
|
||||
type: file
|
||||
disableDeletion: false
|
||||
editable: true
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
foldersFromFilesStructure: false
|
||||
@@ -1,45 +0,0 @@
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: true
|
||||
jsonData:
|
||||
httpMethod: POST
|
||||
exemplarTraceIdDestinations:
|
||||
- name: traceID
|
||||
datasourceUid: tempo
|
||||
|
||||
- name: Loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
editable: true
|
||||
jsonData:
|
||||
derivedFields:
|
||||
- datasourceUid: tempo
|
||||
matcherRegex: '"traceId":"(\w+)"'
|
||||
name: TraceID
|
||||
url: '$${__value.raw}'
|
||||
|
||||
- name: Tempo
|
||||
type: tempo
|
||||
access: proxy
|
||||
url: http://tempo:3200
|
||||
uid: tempo
|
||||
editable: true
|
||||
jsonData:
|
||||
tracesToLogsV2:
|
||||
datasourceUid: loki
|
||||
filterByTraceID: true
|
||||
tracesToMetrics:
|
||||
datasourceUid: prometheus
|
||||
serviceMap:
|
||||
datasourceUid: prometheus
|
||||
nodeGraph:
|
||||
enabled: true
|
||||
lokiSearch:
|
||||
datasourceUid: loki
|
||||
@@ -1,36 +0,0 @@
|
||||
auth_enabled: false
|
||||
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
|
||||
common:
|
||||
path_prefix: /loki
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
|
||||
schema_config:
|
||||
configs:
|
||||
- from: '2024-01-01'
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
|
||||
limits_config:
|
||||
allow_structured_metadata: true
|
||||
volume_enabled: true
|
||||
|
||||
query_range:
|
||||
results_cache:
|
||||
cache:
|
||||
embedded_cache:
|
||||
enabled: true
|
||||
max_size_mb: 100
|
||||
@@ -1,16 +0,0 @@
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
|
||||
scrape_configs:
|
||||
# Scrape OTEL Collector's Prometheus exporter
|
||||
- job_name: otel-collector
|
||||
static_configs:
|
||||
- targets: ['otel-collector:8889']
|
||||
labels:
|
||||
service: otel-collector
|
||||
|
||||
# Scrape OTEL Collector's own metrics
|
||||
- job_name: otel-collector-internal
|
||||
static_configs:
|
||||
- targets: ['otel-collector:8888']
|
||||
@@ -1,47 +0,0 @@
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
local:
|
||||
path: /var/tempo/traces
|
||||
wal:
|
||||
path: /var/tempo/wal
|
||||
|
||||
metrics_generator:
|
||||
registry:
|
||||
external_labels:
|
||||
source: tempo
|
||||
cluster: docker-compose
|
||||
storage:
|
||||
path: /var/tempo/generator/wal
|
||||
remote_write:
|
||||
- url: http://prometheus:9090/api/v1/write
|
||||
send_exemplars: true
|
||||
traces_storage:
|
||||
path: /var/tempo/generator/traces
|
||||
processor:
|
||||
service_graphs:
|
||||
dimensions:
|
||||
- http.method
|
||||
- http.target
|
||||
span_metrics:
|
||||
dimensions:
|
||||
- http.method
|
||||
- http.target
|
||||
- http.status_code
|
||||
|
||||
overrides:
|
||||
defaults:
|
||||
metrics_generator:
|
||||
processors: [service-graphs, span-metrics]
|
||||
Reference in New Issue
Block a user