Monitoring and Logging

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

# Prometheus 클라이언트를 사용한 RED 메트릭 구현 예시
from prometheus_client import Counter, Histogram, start_http_server
import time

# 요청 수 카운터
REQUEST_COUNT = Counter('app_request_count', 'Application Request Count', ['method', 'endpoint', 'status'])

# 요청 지속 시간 히스토그램
REQUEST_LATENCY = Histogram('app_request_latency_seconds', 'Application Request Latency', ['method', 'endpoint'])

def process_request(request):
    # 요청 시작 시간
    start_time = time.time()
    
    try:
        # 요청 처리
        result = handle_request(request)
        status = 'success'
    except Exception as e:
        status = 'error'
        raise e
    finally:
        # 요청 처리 시간 기록
        latency = time.time() - start_time
        REQUEST_LATENCY.labels(request.method, request.endpoint).observe(latency)
        REQUEST_COUNT.labels(request.method, request.endpoint, status).inc()
        
    return result

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

# prometheus.yml 기본 설정 예시
global:
  scrape_interval: 15s

scrape_configs:
  - job_name: 'spring-boot-app'
    metrics_path: '/actuator/prometheus'
    static_configs:
      - targets: ['localhost:8080']
  
  - job_name: 'node-exporter'
    static_configs:
      - targets: ['localhost:9100']

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

# logstash.conf 예시
input {
  file {
    path => "/var/log/application.log"
    type => "application"
  }
}

filter {
  if [type] == "application" {
    grok {
      match => { "message" => "%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} %{GREEDYDATA:message}" }
    }
    date {
      match => [ "timestamp", "ISO8601" ]
    }
  }
}

output {
  elasticsearch {
    hosts => ["localhost:9200"]
    index => "application-logs-%{+YYYY.MM.dd}"
  }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.ThreadContext;

public class UserService {
    private static final Logger logger = LogManager.getLogger(UserService.class);
    
    public User getUserById(String userId) {
        // 컨텍스트 정보 추가
        ThreadContext.put("userId", userId);
        ThreadContext.put("operation", "getUserById");
        
        try {
            logger.info("Fetching user data");
            User user = userRepository.findById(userId);
            
            if (user == null) {
                logger.warn("User not found");
                return null;
            }
            
            logger.info("User data retrieved successfully");
            return user;
        } catch (Exception e) {
            logger.error("Error fetching user data", e);
            throw e;
        } finally {
            // 컨텍스트 정보 제거
            ThreadContext.clearAll();
        }
    }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

<!-- log4j2.xml -->
<Configuration>
  <Appenders>
    <RollingFile name="RollingFile" fileName="logs/app.log" filePattern="logs/app-%d{MM-dd-yyyy}.log.gz">
      <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
      <Policies>
        <TimeBasedTriggeringPolicy />
      </Policies>
    </RollingFile>
    
    <!-- 비동기 래퍼 -->
    <Async name="AsyncAppender" bufferSize="1000" blocking="false">
      <AppenderRef ref="RollingFile"/>
    </Async>
  </Appenders>
  
  <Loggers>
    <Root level="info">
      <AppenderRef ref="AsyncAppender"/>
    </Root>
  </Loggers>
</Configuration>

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

# filebeat.yml
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/application/*.log
  tags: ["application"]
  fields:
    service: user-service
    environment: production

output.elasticsearch:
  hosts: ["elasticsearch:9200"]
  index: "application-%{+yyyy.MM.dd}"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45

import io.opentelemetry.api.OpenTelemetry;
import io.opentelemetry.api.trace.Span;
import io.opentelemetry.api.trace.Tracer;
import io.opentelemetry.context.Scope;

public class OrderService {
    private final Tracer tracer;
    
    public OrderService(OpenTelemetry openTelemetry) {
        this.tracer = openTelemetry.getTracer("order-service");
    }
    
    public Order processOrder(String orderId) {
        Span span = tracer.spanBuilder("processOrder").startSpan();
        try (Scope scope = span.makeCurrent()) {
            span.setAttribute("orderId", orderId);
            
            // 주문 검증
            span.addEvent("Validating order");
            validateOrder(orderId);
            
            // 재고 확인
            span.addEvent("Checking inventory");
            checkInventory(orderId);
            
            // 결제 처리
            span.addEvent("Processing payment");
            processPayment(orderId);
            
            // 주문 완료
            Order order = finalizeOrder(orderId);
            span.addEvent("Order completed");
            
            return order;
        } catch (Exception e) {
            span.recordException(e);
            span.setStatus(StatusCode.ERROR, e.getMessage());
            throw e;
        } finally {
            span.end();
        }
    }
    
    // 기타 메서드…
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21

# Prometheus AlertManager 설정
groups:
- name: slo-alerts
  rules:
  - alert: HighLatency
    expr: histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m])) > 0.5
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "High latency detected"
      description: "95th percentile latency is above 500ms (current value: {{ $value }}s)"
  
  - alert: ErrorBudgetBurn
    expr: sum(rate(http_requests_total{status=~"5.."}[1h])) / sum(rate(http_requests_total[1h])) > 0.001
    for: 15m
    labels:
      severity: critical
    annotations:
      summary: "Error budget burn rate too high"
      description: "Error rate is above 0.1%, burning error budget too quickly"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42

# 자가 복구 설정이 포함된 Deployment
apiVersion: apps/v1
kind: Deployment
metadata:
  name: web-application
spec:
  replicas: 3
  selector:
    matchLabels:
      app: web
  template:
    metadata:
      labels:
        app: web
    spec:
      containers:
      - name: web-app
        image: my-web-app:latest
        ports:
        - containerPort: 8080
        # 헬스 체크 설정
        livenessProbe:
          httpGet:
            path: /health
            port: 8080
          initialDelaySeconds: 30
          periodSeconds: 10
        # 준비 상태 체크
        readinessProbe:
          httpGet:
            path: /ready
            port: 8080
          initialDelaySeconds: 5
          periodSeconds: 5
        # 리소스 제한
        resources:
          limits:
            cpu: "1"
            memory: "512Mi"
          requests:
            cpu: "0.5"
            memory: "256Mi"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

{
  "AWSTemplateFormatVersion": "2010-09-09",
  "Resources": {
    "WebServerGroup": {
      "Type": "AWS::AutoScaling::AutoScalingGroup",
      "Properties": {
        "AvailabilityZones": ["us-east-1a", "us-east-1b"],
        "LaunchConfigurationName": { "Ref": "LaunchConfig" },
        "MinSize": "2",
        "MaxSize": "10",
        "DesiredCapacity": "2",
        "HealthCheckType": "ELB",
        "HealthCheckGracePeriod": 300,
        "TargetGroupARNs": [{ "Ref": "ALBTargetGroup" }]
      }
    },
    "CPUScalingPolicy": {
      "Type": "AWS::AutoScaling::ScalingPolicy",
      "Properties": {
        "AutoScalingGroupName": { "Ref": "WebServerGroup" },
        "PolicyType": "TargetTrackingScaling",
        "TargetTrackingConfiguration": {
          "PredefinedMetricSpecification": {
            "PredefinedMetricType": "ASGAverageCPUUtilization"
          },
          "TargetValue": 70.0
        }
      }
    }
  }
}