# app_metrics.py - アプリケーションメトリクス
from prometheus_client import Counter, Histogram, Gauge, generate_latest
import time

# メトリクスの定義
request_count = Counter('http_requests_total', 
                       'Total HTTP requests', 
                       ['method', 'endpoint', 'status'])

request_duration = Histogram('http_request_duration_seconds',
                           'HTTP request latency',
                           ['method', 'endpoint'])

active_users = Gauge('active_users', 'Number of active users')

# メトリクスの記録
@app.route('/api/users')
@request_duration.labels(method='GET', endpoint='/api/users').time()
def get_users():
    request_count.labels(method='GET', endpoint='/api/users', status=200).inc()
    # 実際の処理
    return jsonify(users)

# メトリクスエンドポイント
@app.route('/metrics')
def metrics():
    return generate_latest()

# structured_logging.py
import json
import logging
from datetime import datetime

class StructuredLogger:
    def __init__(self, name):
        self.logger = logging.getLogger(name)
        handler = logging.StreamHandler()
        handler.setFormatter(self.JsonFormatter())
        self.logger.addHandler(handler)
        self.logger.setLevel(logging.INFO)
    
    class JsonFormatter(logging.Formatter):
        def format(self, record):
            log_obj = {
                'timestamp': datetime.utcnow().isoformat(),
                'level': record.levelname,
                'logger': record.name,
                'message': record.getMessage(),
                'module': record.module,
                'function': record.funcName,
                'line': record.lineno
            }
            
            # 追加のコンテキスト情報
            if hasattr(record, 'user_id'):
                log_obj['user_id'] = record.user_id
            if hasattr(record, 'request_id'):
                log_obj['request_id'] = record.request_id
            if hasattr(record, 'duration'):
                log_obj['duration'] = record.duration
            
            return json.dumps(log_obj)

# 使用例
logger = StructuredLogger(__name__).logger

def process_order(order_id, user_id):
    start_time = time.time()
    logger.info("Processing order", extra={
        'user_id': user_id,
        'order_id': order_id,
        'request_id': str(uuid.uuid4())
    })
    
    try:
        # 注文処理
        result = perform_order_processing(order_id)
        
        duration = time.time() - start_time
        logger.info("Order processed successfully", extra={
            'user_id': user_id,
            'order_id': order_id,
            'duration': duration,
            'status': 'success'
        })
        
    except Exception as e:
        logger.error("Order processing failed", extra={
            'user_id': user_id,
            'order_id': order_id,
            'error': str(e),
            'status': 'failed'
        })
        raise

# distributed_tracing.py - OpenTelemetryによるトレーシング
from opentelemetry import trace
from opentelemetry.exporter.jaeger import JaegerExporter
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.instrumentation.flask import FlaskInstrumentor
from opentelemetry.instrumentation.requests import RequestsInstrumentor

# トレーサーの設定
trace.set_tracer_provider(TracerProvider())
tracer = trace.get_tracer(__name__)

# Jaegerエクスポーターの設定
jaeger_exporter = JaegerExporter(
    agent_host_name="localhost",
    agent_port=6831,
)

# スパンプロセッサーの追加
span_processor = BatchSpanProcessor(jaeger_exporter)
trace.get_tracer_provider().add_span_processor(span_processor)

# 自動計装
FlaskInstrumentor().instrument_app(app)
RequestsInstrumentor().instrument()

# カスタムスパン
@app.route('/api/process')
def process_request():
    with tracer.start_as_current_span("process_request") as span:
        span.set_attribute("user.id", get_user_id())
        
        # データベース操作
        with tracer.start_as_current_span("database_query"):
            data = fetch_from_database()
        
        # 外部API呼び出し
        with tracer.start_as_current_span("external_api_call"):
            result = call_external_api(data)
        
        # 結果の処理
        with tracer.start_as_current_span("process_result"):
            processed = process_data(result)
        
        return jsonify(processed)

# anomaly_detection.py - 機械学習による異常検知
import numpy as np
from sklearn.ensemble import IsolationForest
from prometheus_api_client import PrometheusConnect
import pandas as pd

class AnomalyDetector:
    def __init__(self, prometheus_url):
        self.prom = PrometheusConnect(url=prometheus_url)
        self.model = IsolationForest(contamination=0.1)
        
    def fetch_metrics(self, query, start_time, end_time):
        """Prometheusからメトリクスを取得"""
        result = self.prom.custom_query_range(
            query=query,
            start_time=start_time,
            end_time=end_time,
            step='1m'
        )
        return result
    
    def prepare_features(self, metrics_data):
        """特徴量の準備"""
        features = []
        
        for metric in metrics_data:
            values = [float(v[1]) for v in metric['values']]
            
            # 統計的特徴量
            features.append({
                'mean': np.mean(values),
                'std': np.std(values),
                'max': np.max(values),
                'min': np.min(values),
                'percentile_95': np.percentile(values, 95),
                'rate_of_change': np.mean(np.diff(values))
            })
        
        return pd.DataFrame(features)
    
    def train(self, historical_data):
        """正常時のデータで学習"""
        features = self.prepare_features(historical_data)
        self.model.fit(features)
    
    def detect_anomalies(self, current_data):
        """異常を検知"""
        features = self.prepare_features(current_data)
        predictions = self.model.predict(features)
        
        anomalies = []
        for i, pred in enumerate(predictions):
            if pred == -1:  # 異常
                anomalies.append({
                    'metric': current_data[i]['metric'],
                    'severity': self.calculate_severity(features.iloc[i]),
                    'timestamp': datetime.now()
                })
        
        return anomalies
    
    def calculate_severity(self, feature_row):
        """異常の深刻度を計算"""
        # 平均からの乖離度に基づいて深刻度を決定
        deviation = abs(feature_row['mean'] - feature_row['percentile_95'])
        
        if deviation > feature_row['std'] * 3:
            return 'critical'
        elif deviation > feature_row['std'] * 2:
            return 'warning'
        else:
            return 'info'

# 使用例
detector = AnomalyDetector('http://prometheus:9090')

# 正常時のデータで学習
historical_metrics = detector.fetch_metrics(
    'rate(http_requests_total[5m])',
    datetime.now() - timedelta(days=7),
    datetime.now() - timedelta(days=1)
)
detector.train(historical_metrics)

# リアルタイム監視
while True:
    current_metrics = detector.fetch_metrics(
        'rate(http_requests_total[5m])',
        datetime.now() - timedelta(minutes=10),
        datetime.now()
    )
    
    anomalies = detector.detect_anomalies(current_metrics)
    
    for anomaly in anomalies:
        if anomaly['severity'] == 'critical':
            send_alert(anomaly)
    
    time.sleep(60)

# incident_response.py - 自動インシデント対応
import boto3
from datetime import datetime
import requests

class IncidentResponder:
    def __init__(self):
        self.ec2 = boto3.client('ec2')
        self.sns = boto3.client('sns')
        self.ssm = boto3.client('ssm')
        
    def respond_to_high_cpu(self, instance_id, cpu_usage):
        """高CPU使用率への対応"""
        print(f"High CPU detected on {instance_id}: {cpu_usage}%")
        
        # 1. スナップショットの作成（証拠保全）
        self.create_snapshot(instance_id)
        
        # 2. プロセス情報の収集
        process_info = self.collect_process_info(instance_id)
        
        # 3. 自動スケーリング
        if cpu_usage > 90:
            self.scale_horizontally(instance_id)
        
        # 4. 通知
        self.notify_team(
            f"High CPU Alert: {instance_id}",
            f"CPU: {cpu_usage}%\nTop processes:\n{process_info}"
        )
    
    def respond_to_disk_full(self, instance_id, disk_usage):
        """ディスク容量不足への対応"""
        print(f"Disk full on {instance_id}: {disk_usage}%")
        
        # 1. 一時ファイルのクリーンアップ
        self.cleanup_temp_files(instance_id)
        
        # 2. 古いログのアーカイブ
        self.archive_old_logs(instance_id)
        
        # 3. EBSボリュームの拡張
        if disk_usage > 95:
            self.expand_ebs_volume(instance_id)
    
    def collect_process_info(self, instance_id):
        """SSMを使用してプロセス情報を収集"""
        response = self.ssm.send_command(
            InstanceIds=[instance_id],
            DocumentName="AWS-RunShellScript",
            Parameters={
                'commands': [
                    'ps aux --sort=-%cpu | head -10',
                    'free -m',
                    'df -h'
                ]
            }
        )
        
        # コマンド実行結果を取得
        command_id = response['Command']['CommandId']
        output = self.ssm.get_command_invocation(
            CommandId=command_id,
            InstanceId=instance_id
        )
        
        return output['StandardOutputContent']
    
    def scale_horizontally(self, instance_id):
        """水平スケーリングの実行"""
        # Auto Scalingグループの特定
        asg = self.get_auto_scaling_group(instance_id)
        
        if asg:
            # 希望容量を増やす
            current_capacity = asg['DesiredCapacity']
            new_capacity = min(current_capacity + 1, asg['MaxSize'])
            
            self.autoscaling.set_desired_capacity(
                AutoScalingGroupName=asg['AutoScalingGroupName'],
                DesiredCapacity=new_capacity
            )
            
            print(f"Scaled ASG from {current_capacity} to {new_capacity}")
    
    def create_playbook(self, incident_type):
        """インシデント対応プレイブックの生成"""
        playbooks = {
            'high_cpu': [
                "1. CPU使用率の詳細を確認（top, htop）",
                "2. 高負荷プロセスの特定",
                "3. アプリケーションログの確認",
                "4. 必要に応じてプロセスの再起動",
                "5. スケーリングの検討"
            ],
            'memory_leak': [
                "1. メモリ使用状況の確認（free -m）",
                "2. メモリリークの疑いがあるプロセスの特定",
                "3. ヒープダンプの取得",
                "4. アプリケーションの再起動",
                "5. 根本原因の調査"
            ],
            'disk_full': [
                "1. ディスク使用状況の確認（df -h）",
                "2. 大きなファイルの特定（du -sh）",
                "3. 不要なログファイルの削除",
                "4. ログローテーションの確認",
                "5. ディスク容量の拡張"
            ]
        }
        
        return playbooks.get(incident_type, ["インシデントタイプが不明です"])

# log_analysis_pipeline.py
import re
import json
from datetime import datetime
from collections import defaultdict
from elasticsearch import Elasticsearch
import matplotlib.pyplot as plt

class LogAnalyzer:
    def __init__(self, es_host='localhost:9200'):
        self.es = Elasticsearch([es_host])
        self.patterns = {
            'error': re.compile(r'ERROR|CRITICAL|FATAL', re.I),
            'warning': re.compile(r'WARN|WARNING', re.I),
            'ip_address': re.compile(r'\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b'),
            'response_time': re.compile(r'response_time=(\d+\.?\d*)'),
            'status_code': re.compile(r'status[_-]?code["\s:=]+(\d{3})')
        }
    
    def parse_log_line(self, line):
        """ログ行をパース"""
        parsed = {
            'timestamp': self.extract_timestamp(line),
            'level': self.extract_log_level(line),
            'ip_addresses': self.patterns['ip_address'].findall(line),
            'message': line
        }
        
        # レスポンスタイムの抽出
        rt_match = self.patterns['response_time'].search(line)
        if rt_match:
            parsed['response_time'] = float(rt_match.group(1))
        
        # ステータスコードの抽出
        sc_match = self.patterns['status_code'].search(line)
        if sc_match:
            parsed['status_code'] = int(sc_match.group(1))
        
        return parsed
    
    def extract_timestamp(self, line):
        """タイムスタンプの抽出"""
        # ISO形式
        iso_pattern = r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}'
        match = re.search(iso_pattern, line)
        if match:
            return datetime.fromisoformat(match.group())
        
        # 一般的なログ形式
        log_pattern = r'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}'
        match = re.search(log_pattern, line)
        if match:
            return datetime.strptime(match.group(), '%Y-%m-%d %H:%M:%S')
        
        return datetime.now()
    
    def extract_log_level(self, line):
        """ログレベルの抽出"""
        if self.patterns['error'].search(line):
            return 'ERROR'
        elif self.patterns['warning'].search(line):
            return 'WARNING'
        else:
            return 'INFO'
    
    def analyze_patterns(self, logs):
        """ログパターンの分析"""
        analysis = {
            'error_rate': 0,
            'avg_response_time': 0,
            'status_codes': defaultdict(int),
            'top_ips': defaultdict(int),
            'error_patterns': defaultdict(int)
        }
        
        total_logs = len(logs)
        error_count = 0
        response_times = []
        
        for log in logs:
            # エラー率
            if log['level'] == 'ERROR':
                error_count += 1
                # エラーパターンの抽出
                error_msg = log['message'].lower()
                if 'timeout' in error_msg:
                    analysis['error_patterns']['timeout'] += 1
                elif 'connection refused' in error_msg:
                    analysis['error_patterns']['connection_refused'] += 1
                elif 'out of memory' in error_msg:
                    analysis['error_patterns']['out_of_memory'] += 1
            
            # レスポンスタイム
            if 'response_time' in log:
                response_times.append(log['response_time'])
            
            # ステータスコード
            if 'status_code' in log:
                analysis['status_codes'][log['status_code']] += 1
            
            # IPアドレス
            for ip in log['ip_addresses']:
                analysis['top_ips'][ip] += 1
        
        # 集計
        analysis['error_rate'] = (error_count / total_logs) * 100 if total_logs > 0 else 0
        analysis['avg_response_time'] = sum(response_times) / len(response_times) if response_times else 0
        
        # Top 10 IPアドレス
        analysis['top_ips'] = dict(sorted(analysis['top_ips'].items(), 
                                        key=lambda x: x[1], 
                                        reverse=True)[:10])
        
        return analysis
    
    def detect_anomalies(self, logs, window_size=100):
        """異常なパターンの検出"""
        anomalies = []
        
        for i in range(len(logs) - window_size):
            window = logs[i:i+window_size]
            
            # エラー率の急上昇
            error_count = sum(1 for log in window if log['level'] == 'ERROR')
            error_rate = error_count / window_size
            
            if error_rate > 0.1:  # 10%以上のエラー率
                anomalies.append({
                    'type': 'high_error_rate',
                    'timestamp': window[-1]['timestamp'],
                    'error_rate': error_rate,
                    'severity': 'critical' if error_rate > 0.2 else 'warning'
                })
            
            # レスポンスタイムの異常
            response_times = [log['response_time'] for log in window if 'response_time' in log]
            if response_times:
                avg_rt = sum(response_times) / len(response_times)
                if avg_rt > 1000:  # 1秒以上
                    anomalies.append({
                        'type': 'slow_response',
                        'timestamp': window[-1]['timestamp'],
                        'avg_response_time': avg_rt,
                        'severity': 'warning'
                    })
        
        return anomalies
    
    def visualize_analysis(self, analysis):
        """分析結果の可視化"""
        fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(12, 10))
        
        # エラーパターン
        if analysis['error_patterns']:
            ax1.bar(analysis['error_patterns'].keys(), 
                   analysis['error_patterns'].values())
            ax1.set_title('Error Patterns')
            ax1.set_xlabel('Error Type')
            ax1.set_ylabel('Count')
        
        # ステータスコード分布
        if analysis['status_codes']:
            ax2.pie(analysis['status_codes'].values(), 
                   labels=analysis['status_codes'].keys(), 
                   autopct='%1.1f%%')
            ax2.set_title('Status Code Distribution')
        
        # Top IPアドレス
        if analysis['top_ips']:
            ips = list(analysis['top_ips'].keys())[:5]
            counts = [analysis['top_ips'][ip] for ip in ips]
            ax3.barh(ips, counts)
            ax3.set_title('Top 5 IP Addresses')
            ax3.set_xlabel('Request Count')
        
        # メトリクスサマリー
        metrics_text = f"""
        Error Rate: {analysis['error_rate']:.2f}%
        Avg Response Time: {analysis['avg_response_time']:.2f}ms
        Total Status Codes: {sum(analysis['status_codes'].values())}
        """
        ax4.text(0.1, 0.5, metrics_text, fontsize=12, verticalalignment='center')
        ax4.set_title('Summary Metrics')
        ax4.axis('off')
        
        plt.tight_layout()
        plt.savefig('log_analysis_report.png')
        plt.show()

# 使用例
analyzer = LogAnalyzer()

# ログファイルの読み込みと分析
with open('application.log', 'r') as f:
    logs = [analyzer.parse_log_line(line) for line in f]

# パターン分析
analysis = analyzer.analyze_patterns(logs)
print("Analysis Results:", json.dumps(analysis, indent=2))

# 異常検出
anomalies = analyzer.detect_anomalies(logs)
for anomaly in anomalies:
    print(f"Anomaly detected: {anomaly}")

# 可視化
analyzer.visualize_analysis(analysis)

# distributed_tracing_demo.py
from flask import Flask, request, jsonify
import requests
import time
import random
from opentelemetry import trace
from opentelemetry.exporter.jaeger import JaegerExporter
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.instrumentation.flask import FlaskInstrumentor
from opentelemetry.instrumentation.requests import RequestsInstrumentor
import logging

# アプリケーションの設定
app = Flask(__name__)
logging.basicConfig(level=logging.INFO)

# トレーシングの設定
trace.set_tracer_provider(TracerProvider())
tracer = trace.get_tracer(__name__)

jaeger_exporter = JaegerExporter(
    agent_host_name="localhost",
    agent_port=6831,
)

span_processor = BatchSpanProcessor(jaeger_exporter)
trace.get_tracer_provider().add_span_processor(span_processor)

# 自動計装
FlaskInstrumentor().instrument_app(app)
RequestsInstrumentor().instrument()

# サービスのシミュレーション
class MicroserviceSimulator:
    def __init__(self, service_name):
        self.service_name = service_name
        self.tracer = trace.get_tracer(service_name)
    
    def process_request(self, request_id):
        """リクエストの処理をシミュレート"""
        with self.tracer.start_as_current_span(f"{self.service_name}_process") as span:
            span.set_attribute("request.id", request_id)
            span.set_attribute("service.name", self.service_name)
            
            # 処理時間のシミュレート
            processing_time = random.uniform(0.1, 0.5)
            time.sleep(processing_time)
            
            # エラーのシミュレート（5%の確率）
            if random.random() < 0.05:
                span.set_status(trace.Status(trace.StatusCode.ERROR))
                span.record_exception(Exception(f"Random error in {self.service_name}"))
                raise Exception(f"Error in {self.service_name}")
            
            return {
                "service": self.service_name,
                "processing_time": processing_time,
                "status": "success"
            }

# APIゲートウェイ
@app.route('/api/order', methods=['POST'])
def create_order():
    with tracer.start_as_current_span("create_order") as span:
        order_data = request.json
        order_id = order_data.get('order_id', 'unknown')
        
        span.set_attribute("order.id", order_id)
        span.set_attribute("order.amount", order_data.get('amount', 0))
        
        try:
            # 在庫確認サービス
            with tracer.start_as_current_span("check_inventory"):
                inventory_response = requests.post(
                    'http://localhost:5001/inventory/check',
                    json={'order_id': order_id, 'items': order_data.get('items', [])}
                )
                inventory_result = inventory_response.json()
            
            # 決済サービス
            with tracer.start_as_current_span("process_payment"):
                payment_response = requests.post(
                    'http://localhost:5002/payment/process',
                    json={'order_id': order_id, 'amount': order_data.get('amount', 0)}
                )
                payment_result = payment_response.json()
            
            # 配送サービス
            with tracer.start_as_current_span("arrange_shipping"):
                shipping_response = requests.post(
                    'http://localhost:5003/shipping/arrange',
                    json={'order_id': order_id, 'address': order_data.get('address', '')}
                )
                shipping_result = shipping_response.json()
            
            return jsonify({
                'order_id': order_id,
                'status': 'completed',
                'inventory': inventory_result,
                'payment': payment_result,
                'shipping': shipping_result
            })
            
        except Exception as e:
            span.record_exception(e)
            span.set_status(trace.Status(trace.StatusCode.ERROR))
            return jsonify({'error': str(e)}), 500

# 在庫確認サービス
inventory_service = MicroserviceSimulator("inventory_service")

@app.route('/inventory/check', methods=['POST'])
def check_inventory():
    data = request.json
    return jsonify(inventory_service.process_request(data['order_id']))

# 決済サービス
payment_service = MicroserviceSimulator("payment_service")

@app.route('/payment/process', methods=['POST'])
def process_payment():
    data = request.json
    return jsonify(payment_service.process_request(data['order_id']))

# 配送サービス
shipping_service = MicroserviceSimulator("shipping_service")

@app.route('/shipping/arrange', methods=['POST'])
def arrange_shipping():
    data = request.json
    return jsonify(shipping_service.process_request(data['order_id']))

# トレース分析エンドポイント
@app.route('/api/trace/analyze/<trace_id>')
def analyze_trace(trace_id):
    """トレースの分析結果を返す"""
    # 実際の実装では、Jaegerからトレースデータを取得
    analysis = {
        'trace_id': trace_id,
        'total_duration': 1234,  # ms
        'service_count': 4,
        'span_count': 12,
        'error_count': 0,
        'critical_path': [
            {'service': 'api_gateway', 'duration': 10},
            {'service': 'inventory_service', 'duration': 423},
            {'service': 'payment_service', 'duration': 567},
            {'service': 'shipping_service', 'duration': 234}
        ],
        'bottleneck': 'payment_service'
    }
    return jsonify(analysis)

if __name__ == '__main__':
    app.run(debug=True, port=5000)

# predictive_alerting.py
import numpy as np
from statsmodels.tsa.arima.model import ARIMA
from sklearn.preprocessing import StandardScaler
import pandas as pd
from datetime import datetime, timedelta

class PredictiveAlerting:
    def __init__(self, lookback_hours=24, forecast_hours=6):
        self.lookback_hours = lookback_hours
        self.forecast_hours = forecast_hours
        self.scaler = StandardScaler()
        self.models = {}
        
    def train_models(self, historical_data):
        """各メトリクスに対してARIMAモデルを訓練"""
        for metric_name, time_series in historical_data.items():
            # データの準備
            df = pd.DataFrame(time_series)
            df['timestamp'] = pd.to_datetime(df['timestamp'])
            df.set_index('timestamp', inplace=True)
            
            # ARIMAモデルの訓練
            model = ARIMA(df['value'], order=(2, 1, 2))
            fitted_model = model.fit()
            
            self.models[metric_name] = fitted_model
            
            logging.info(f"Trained ARIMA model for {metric_name}")
    
    def predict_future(self, metric_name, current_value):
        """将来の値を予測"""
        if metric_name not in self.models:
            return None
        
        model = self.models[metric_name]
        
        # 予測
        forecast = model.forecast(steps=self.forecast_hours)
        
        # 信頼区間の計算
        forecast_df = pd.DataFrame({
            'forecast': forecast,
            'timestamp': pd.date_range(
                start=datetime.now(),
                periods=self.forecast_hours,
                freq='H'
            )
        })
        
        return forecast_df
    
    def detect_future_anomalies(self, predictions, thresholds):
        """予測値から将来の異常を検出"""
        future_alerts = []
        
        for metric_name, forecast_df in predictions.items():
            threshold = thresholds.get(metric_name, {})
            
            for idx, row in forecast_df.iterrows():
                predicted_value = row['forecast']
                timestamp = row['timestamp']
                
                # 閾値チェック
                if 'max' in threshold and predicted_value > threshold['max']:
                    future_alerts.append({
                        'metric': metric_name,
                        'predicted_value': predicted_value,
                        'threshold': threshold['max'],
                        'predicted_time': timestamp,
                        'severity': 'warning',
                        'message': f"{metric_name} is predicted to exceed {threshold['max']} at {timestamp}"
                    })
                
                if 'critical' in threshold and predicted_value > threshold['critical']:
                    future_alerts[-1]['severity'] = 'critical'
        
        return future_alerts
    
    def generate_recommendations(self, future_alerts):
        """予測されるアラートに基づいて推奨アクションを生成"""
        recommendations = []
        
        for alert in future_alerts:
            if alert['metric'] == 'cpu_usage':
                if alert['severity'] == 'critical':
                    recommendations.append({
                        'action': 'scale_out',
                        'urgency': 'high',
                        'deadline': alert['predicted_time'] - timedelta(hours=1),
                        'details': 'Add 2 more instances to handle predicted CPU load'
                    })
                else:
                    recommendations.append({
                        'action': 'optimize',
                        'urgency': 'medium',
                        'deadline': alert['predicted_time'] - timedelta(hours=2),
                        'details': 'Review and optimize CPU-intensive processes'
                    })
            
            elif alert['metric'] == 'memory_usage':
                recommendations.append({
                    'action': 'clear_cache',
                    'urgency': 'medium',
                    'deadline': alert['predicted_time'] - timedelta(hours=1),
                    'details': 'Clear application caches and restart services'
                })
            
            elif alert['metric'] == 'disk_usage':
                recommendations.append({
                    'action': 'cleanup',
                    'urgency': 'low',
                    'deadline': alert['predicted_time'] - timedelta(hours=4),
                    'details': 'Archive old logs and clean temporary files'
                })
        
        return recommendations

# 使用例
predictor = PredictiveAlerting()

# 履歴データの準備（実際にはPrometheusなどから取得）
historical_data = {
    'cpu_usage': [
        {'timestamp': datetime.now() - timedelta(hours=i), 
         'value': 50 + 10 * np.sin(i/6) + np.random.normal(0, 5)}
        for i in range(168, 0, -1)  # 過去1週間
    ],
    'memory_usage': [
        {'timestamp': datetime.now() - timedelta(hours=i),
         'value': 60 + 5 * np.sin(i/12) + np.random.normal(0, 3)}
        for i in range(168, 0, -1)
    ]
}

# モデルの訓練
predictor.train_models(historical_data)

# 予測の実行
predictions = {}
for metric in ['cpu_usage', 'memory_usage']:
    predictions[metric] = predictor.predict_future(metric, None)

# 閾値の設定
thresholds = {
    'cpu_usage': {'max': 80, 'critical': 90},
    'memory_usage': {'max': 85, 'critical': 95}
}

# 将来の異常を検出
future_alerts = predictor.detect_future_anomalies(predictions, thresholds)

# 推奨アクションの生成
recommendations = predictor.generate_recommendations(future_alerts)

# 結果の表示
for alert in future_alerts:
    print(f"Predicted Alert: {alert}")

for rec in recommendations:
    print(f"Recommendation: {rec}")