1// What a valid OpenTelemetry trace looks like in Jaeger
2// After implementing auto-instrumentation, every request produces this:
3 
4{
5  "traceID": "abc123def456789",
6  "spans": [
7    {
8      "operationName": "GET /api/users/:id",
9      "serviceName": "my-service",
10      "duration": 145,    // microseconds
11      "tags": {
12        "http.method": "GET",
13        "http.url": "/api/users/42",
14        "http.status_code": 200,
15        "service.version": "1.2.3"
16      }
17    },
18    {
19      "operationName": "pg.query",
20      "serviceName": "my-service",
21      "parentSpanID": "root-span-id",
22      "duration": 130,    // DB query took 130ms — visible as child span
23      "tags": {
24        "db.type": "postgresql",
25        "db.statement": "SELECT * FROM users WHERE id = $1",
26        "db.rows_affected": 1
27      }
28    }
29  ]
30}
31 
32// Circuit breaker state transitions — opossum emits these events:
33circuitBreaker.on('open', () => {
34  // Fires when failure rate exceeds threshold (50%)
35  // All subsequent calls return fallback immediately (no DB hit)
36  metrics.increment('circuit_breaker.opened', { service: 'db' });
37});
38 
39circuitBreaker.on('halfOpen', () => {
40  // After resetTimeout (30s), allows ONE test request through
41  // If it succeeds -> close. If it fails -> open again.
42  metrics.increment('circuit_breaker.half_open', { service: 'db' });
43});
44 
45circuitBreaker.on('close', () => {
46  // Test request succeeded — circuit closed, normal operation resumed
47  metrics.increment('circuit_breaker.closed', { service: 'db' });
48});

1// production-ready-service.ts
2// Full implementation: OpenTelemetry + /health + /ready + SIGTERM + circuit breaker
3// Run: npx ts-node --require ./tracing.ts production-ready-service.ts
4 
5// ── tracing.ts (import FIRST) ─────────────────────────────────────
6import { NodeSDK } from '@opentelemetry/sdk-node';
7import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node';
8import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
9import { Resource } from '@opentelemetry/resources';
10export const sdk = new NodeSDK({
11  resource: new Resource({ 'service.name': 'my-service' }),
12  traceExporter: new OTLPTraceExporter({ url: process.env.OTEL_ENDPOINT }),
13  instrumentations: [getNodeAutoInstrumentations()],
14});
15sdk.start();
16// Every HTTP, Express, pg, redis call now creates spans automatically.
17 
18// ── server.ts ─────────────────────────────────────────────────────
19import express from 'express';
20import CircuitBreaker from 'opossum';
21import { Pool } from 'pg';
22 
23const app = express();
24const db = new Pool({ connectionString: process.env.DATABASE_URL });
25 
26// ── Pillar 4: Circuit breaker on DB calls ─────────────────────────
27async function queryDb(sql: string, params: unknown[]) {
28  const result = await db.query(sql, params as string[]);
29  return result.rows;
30}
31 
32const dbBreaker = new CircuitBreaker(queryDb, {
33  errorThresholdPercentage: 50,  // open when 50%+ of calls fail
34  resetTimeout: 30000,           // try again after 30 seconds
35  timeout: 3000,                 // treat calls taking > 3s as failures
36  volumeThreshold: 10,           // need at least 10 calls before tripping
37});
38 
39dbBreaker.fallback(() => []);    // return empty array when circuit is open
40 
41// ── Pillar 2: Health + Readiness ──────────────────────────────────
42app.get('/health', (_req, res) => {
43  res.json({ status: 'ok', uptime: process.uptime() });
44});
45 
46app.get('/ready', async (_req, res) => {
47  try {
48    await db.query('SELECT 1');  // verify DB is reachable
49    res.json({ status: 'ready' });
50  } catch (err) {
51    res.status(503).json({
52      status: 'not ready',
53      error: (err as Error).message,
54    });
55  }
56});
57 
58// ── Application routes ────────────────────────────────────────────
59app.get('/api/users/:id', async (req, res) => {
60  const users = await dbBreaker.fire(
61    'SELECT id, name, email FROM users WHERE id = $1',
62    [req.params.id],
63  );
64  res.json(users[0] ?? { error: 'not found' });
65});
66 
67// ── Pillar 3: Graceful shutdown ───────────────────────────────────
68const server = app.listen(8080, () => {
69  console.log('Listening on :8080');
70});
71 
72let isShuttingDown = false;
73 
74process.on('SIGTERM', () => {
75  console.log('SIGTERM received — starting graceful shutdown');
76  isShuttingDown = true;
77 
78  // Stop accepting new connections
79  server.close(() => {
80    console.log('HTTP server closed — all in-flight requests drained');
81    // Shutdown OpenTelemetry exporter (flush remaining spans)
82    sdk.shutdown().finally(() => {
83      process.exit(0);
84    });
85  });
86 
87  // Force exit after 25 seconds (K8s terminationGracePeriodSeconds is 30)
88  setTimeout(() => {
89    console.error('Graceful shutdown timeout — forcing exit');
90    process.exit(1);
91  }, 25_000);
92});
93 
94// Middleware: reject new requests during shutdown
95app.use((_req, res, next) => {
96  if (isShuttingDown) {
97    res.setHeader('Connection', 'close');
98    return res.status(503).json({ error: 'Service shutting down' });
99  }
100  next();
101});

1# Kubernetes deployment with liveness + readiness probes + graceful termination
2apiVersion: apps/v1
3kind: Deployment
4metadata:
5  name: my-service
6spec:
7  replicas: 3
8  template:
9    spec:
10      terminationGracePeriodSeconds: 30  # K8s waits 30s before SIGKILL
11      containers:
12        - name: my-service
13          image: my-service:1.2.3
14          ports:
15            - containerPort: 8080
16 
17          # Liveness probe: restart pod if process is deadlocked
18          livenessProbe:
19            httpGet:
20              path: /health
21              port: 8080
22            initialDelaySeconds: 5    # wait 5s before first check
23            periodSeconds: 10
24            failureThreshold: 3       # restart after 3 consecutive failures
25 
26          # Readiness probe: stop routing traffic if dependencies down
27          readinessProbe:
28            httpGet:
29              path: /ready
30              port: 8080
31            initialDelaySeconds: 10   # wait for DB connection to establish
32            periodSeconds: 5
33            failureThreshold: 2       # stop traffic after 2 consecutive failures
34 
35          env:
36            - name: DATABASE_URL
37              valueFrom:
38                secretKeyRef:
39                  name: db-credentials
40                  key: url
41            - name: OTEL_ENDPOINT
42              value: "http://jaeger-collector:4318/v1/traces"