nginx实时请求监测的项目实践

Nginx 实时请求监测项目实践

项目概述

实现一个实时监控Nginx请求的系统，能够实时查看请求流量、响应状态、响应时间等关键指标。

一、技术架构设计

1.1 架构图

┌─────────────┐    ┌─────────────┐    ┌─────────────┐
│   Nginx     │───▶│  访问日志   │───▶│ 日志收集器  │
│  服务器     │    │ (JSON格式)  │    │ (Filebeat)  │
└─────────────┘    └─────────────┘    └─────────────┘
                                          │
                                          ▼
┌─────────────┐    ┌─────────────┐    ┌─────────────┐
│   前端      │◀───│   API网关   │◀───│ 消息队列    │
│   Dashboard │    │  (Node.js)  │    │ (Redis/Kafka)│
└─────────────┘    └─────────────┘    └─────────────┘
                                          ▲
                                          │
                                    ┌─────────────┐
                                    │  流处理      │
                                    │  (Flink)    │
                                    └─────────────┘

1.2 技术栈选择

日志收集: Filebeat/Fluentd
消息队列: Redis Streams/Kafka
流处理: Apache Flink/Node.js实时处理
后端API: Node.js + Socket.io
前端: Vue.js/React + ECharts
存储: Redis(实时数据) + Elasticsearch(历史数据)

二、Nginx配置优化

2.1 自定义JSON日志格式

http {
    log_format json_log escape=json 
    '{'
        '"timestamp":"$time_iso8601",'
        '"remote_addr":"$remote_addr",'
        '"remote_user":"$remote_user",'
        '"request":"$request",'
        '"status":"$status",'
        '"body_bytes_sent":"$body_bytes_sent",'
        '"request_time":"$request_time",'
        '"upstream_response_time":"$upstream_response_time",'
        '"http_referer":"$http_referer",'
        '"http_user_agent":"$http_user_agent",'
        '"http_x_forwarded_for":"$http_x_forwarded_for",'
        '"request_id":"$request_id",'
        '"server_name":"$server_name"'
    '}';

    access_log /var/log/nginx/access.json.log json_log buffer=32k flush=5s;
}

2.2 实时状态模块

# 启用状态模块
location /nginx_status {
    stub_status on;
    access_log off;
    allow 127.0.0.1;
    deny all;
}

# Prometheus格式的metrics
location /metrics {
    stub_status on;
    access_log off;

    content_by_lua_block {
        local metric = require "nginx.metric"
        ngx.say(metric.get_prometheus_format())
    }
}

三、日志收集与处理

3.1 Filebeat配置

filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.json.log
  json:
    keys_under_root: true
    add_error_key: true

processors:
  - decode_json_fields:
      fields: ["message"]
      target: ""

output.redis:
  hosts: ["localhost:6379"]
  key: "nginx_access_logs"
  db: 0
  timeout: 5

3.2 实时流处理（Node.js实现）

const redis = require('redis');
const { createClient } = require('redis');

class NginxLogProcessor {
  constructor() {
    this.client = createClient();
    this.stats = {
      totalRequests: 0,
      statusCodes: {},
      responseTimes: [],
      endpoints: {},
      realTimeData: []
    };

    this.init();
  }

  async init() {
    await this.client.connect();
    this.startProcessing();
  }

  async startProcessing() {
    while (true) {
      try {
        const result = await this.client.xRead(
          { key: 'nginx_access_logs', id: '$' },
          { COUNT: 100, BLOCK: 5000 }
        );

        if (result) {
          for (const entry of result[0].messages) {
            await this.processLog(entry.message);
          }
        }
      } catch (error) {
        console.error('Error processing logs:', error);
      }
    }
  }

  async processLog(logData) {
    // 解析日志
    const log = JSON.parse(logData);

    // 更新实时统计
    this.updateStats(log);

    // 检测异常
    this.detectAnomalies(log);

    // 发送到WebSocket
    this.broadcastUpdate(log);
  }

  updateStats(log) {
    this.stats.totalRequests++;

    // 状态码统计
    const status = log.status;
    this.stats.statusCodes[status] = (this.stats.statusCodes[status] || 0) + 1;

    // 响应时间
    const responseTime = parseFloat(log.request_time) * 1000; // 转为毫秒
    this.stats.responseTimes.push(responseTime);

    // 保持最近1000个响应时间
    if (this.stats.responseTimes.length > 1000) {
      this.stats.responseTimes.shift();
    }

    // 端点统计
    const endpoint = this.extractEndpoint(log.request);
    this.stats.endpoints[endpoint] = (this.stats.endpoints[endpoint] || 0) + 1;

    // 实时数据点（最近60秒）
    const now = Date.now();
    this.stats.realTimeData.push({
      timestamp: now,
      status: status,
      responseTime: responseTime,
      endpoint: endpoint
    });

    // 清理过期数据
    this.stats.realTimeData = this.stats.realTimeData.filter(
      d => now - d.timestamp < 60000
    );
  }

  extractEndpoint(request) {
    if (!request) return 'unknown';
    const parts = request.split(' ');
    return parts[1] || 'unknown';
  }

  detectAnomalies(log) {
    const responseTime = parseFloat(log.request_time) * 1000;

    // 响应时间异常
    if (responseTime > 5000) { // 5秒阈值
      this.triggerAlert('SLOW_RESPONSE', {
        url: log.request,
        responseTime: responseTime,
        timestamp: log.timestamp
      });
    }

    // 错误状态码
    if (log.status >= 500) {
      this.triggerAlert('SERVER_ERROR', {
        status: log.status,
        url: log.request,
        timestamp: log.timestamp
      });
    }

    // 异常请求频率检测
    this.detectRateAnomalies(log);
  }

  getSummary() {
    const responseTimes = this.stats.responseTimes;
    const avgResponseTime = responseTimes.length > 0 
      ? responseTimes.reduce((a, b) => a + b, 0) / responseTimes.length 
      : 0;

    return {
      totalRequests: this.stats.totalRequests,
      statusCodes: { ...this.stats.statusCodes },
      avgResponseTime: avgResponseTime.toFixed(2),
      p95ResponseTime: this.calculatePercentile(95),
      p99ResponseTime: this.calculatePercentile(99),
      topEndpoints: this.getTopEndpoints(10),
      requestsPerSecond: this.calculateRPS(),
      activeConnections: 0 // 可以从nginx_status获取
    };
  }

  calculatePercentile(p) {
    const times = [...this.stats.responseTimes].sort((a, b) => a - b);
    if (times.length === 0) return 0;

    const index = Math.ceil(p * times.length / 100) - 1;
    return times[Math.max(0, index)].toFixed(2);
  }

  getTopEndpoints(limit) {
    return Object.entries(this.stats.endpoints)
      .sort((a, b) => b[1] - a[1])
      .slice(0, limit)
      .map(([endpoint, count]) => ({ endpoint, count }));
  }

  calculateRPS() {
    const now = Date.now();
    const lastMinute = this.stats.realTimeData.filter(
      d => now - d.timestamp < 60000
    );
    return (lastMinute.length / 60).toFixed(2);
  }
}

四、WebSocket实时API

const WebSocket = require('ws');
const http = require('http');

class RealtimeMonitor {
  constructor(port = 8080) {
    this.server = http.createServer();
    this.wss = new WebSocket.Server({ server: this.server });
    this.clients = new Set();
    this.logProcessor = new NginxLogProcessor();

    this.init();
    this.startBroadcasting();
  }

  init() {
    this.wss.on('connection', (ws) => {
      this.clients.add(ws);
      console.log('New client connected');

      // 发送当前状态
      ws.send(JSON.stringify({
        type: 'INIT',
        data: this.logProcessor.getSummary()
      }));

      ws.on('close', () => {
        this.clients.delete(ws);
      });
    });

    this.server.listen(port, () => {
      console.log(`WebSocket server running on port ${port}`);
    });
  }

  startBroadcasting() {
    // 每秒广播一次汇总数据
    setInterval(() => {
      const summary = this.logProcessor.getSummary();
      this.broadcast({
        type: 'SUMMARY_UPDATE',
        data: summary,
        timestamp: Date.now()
      });
    }, 1000);

    // 实时日志流（可选，控制频率）
    setInterval(() => {
      const recentLogs = this.logProcessor.stats.realTimeData.slice(-10);
      if (recentLogs.length > 0) {
        this.broadcast({
          type: 'RECENT_LOGS',
          data: recentLogs,
          timestamp: Date.now()
        });
      }
    }, 2000);
  }

  broadcast(message) {
    const data = JSON.stringify(message);
    this.clients.forEach(client => {
      if (client.readyState === WebSocket.OPEN) {
        client.send(data);
      }
    });
  }
}

五、前端Dashboard

5.1 Vue组件示例

<template>
  <div class="nginx-monitor">
    <!-- 头部概览 -->
    <div class="overview-cards">
      <div class="card">
        <h3>总请求数</h3>
        <div class="value">{{ overview.totalRequests }}</div>
      </div>
      <div class="card">
        <h3>平均响应时间</h3>
        <div class="value">{{ overview.avgResponseTime }}ms</div>
      </div>
      <div class="card">
        <h3>QPS</h3>
        <div class="value">{{ overview.requestsPerSecond }}</div>
      </div>
      <div class="card">
        <h3>错误率</h3>
        <div class="value">{{ errorRate }}%</div>
      </div>
    </div>

    <!-- 实时图表 -->
    <div class="charts-row">
      <div class="chart-container">
        <h3>请求频率 (每分钟)</h3>
        <div ref="requestChart" style="width: 100%; height: 300px;"></div>
      </div>
      <div class="chart-container">
        <h3>响应时间分布</h3>
        <div ref="responseTimeChart" style="width: 100%; height: 300px;"></div>
      </div>
    </div>

    <!-- 状态码分布 -->
    <div class="chart-container">
      <h3>HTTP状态码分布</h3>
      <div ref="statusChart" style="width: 100%; height: 250px;"></div>
    </div>

    <!-- 实时日志流 -->
    <div class="log-stream">
      <h3>实时请求日志</h3>
      <div class="log-list">
        <div v-for="log in recentLogs" :key="log.id" class="log-item">
          <span :class="`status status-${log.status}`">{{ log.status }}</span>
          <span class="method">{{ log.method }}</span>
          <span class="url">{{ log.url }}</span>
          <span class="time">{{ log.responseTime }}ms</span>
          <span class="timestamp">{{ formatTime(log.timestamp) }}</span>
        </div>
      </div>
    </div>

    <!-- 端点排行 -->
    <div class="endpoints-ranking">
      <h3>热门端点排行</h3>
      <table>
        <thead>
          <tr>
            <th>排名</th>
            <th>端点</th>
            <th>请求数</th>
            <th>平均响应时间</th>
          </tr>
        </thead>
        <tbody>
          <tr v-for="(endpoint, index) in topEndpoints" :key="endpoint.path">
            <td>{{ index + 1 }}</td>
            <td>{{ endpoint.path }}</td>
            <td>{{ endpoint.count }}</td>
            <td>{{ endpoint.avgTime }}ms</td>
          </tr>
        </tbody>
      </table>
    </div>
  </div>
</template>

<script>
import * as echarts from 'echarts';

export default {
  data() {
    return {
      ws: null,
      overview: {
        totalRequests: 0,
        avgResponseTime: 0,
        requestsPerSecond: 0,
        statusCodes: {}
      },
      requestData: [],
      responseTimeData: [],
      recentLogs: [],
      topEndpoints: []
    };
  },

  computed: {
    errorRate() {
      const errorCodes = ['500', '502', '503', '504'];
      let errorCount = 0;
      let totalCount = 0;

      Object.entries(this.overview.statusCodes).forEach(([code, count]) => {
        totalCount += count;
        if (errorCodes.includes(code.toString())) {
          errorCount += count;
        }
      });

      return totalCount > 0 ? ((errorCount / totalCount) * 100).toFixed(2) : '0.00';
    }
  },

  mounted() {
    this.initWebSocket();
    this.initCharts();
  },

  methods: {
    initWebSocket() {
      this.ws = new WebSocket('ws://localhost:8080');

      this.ws.onmessage = (event) => {
        const data = JSON.parse(event.data);

        switch (data.type) {
          case 'INIT':
          case 'SUMMARY_UPDATE':
            this.overview = data.data;
            this.updateRequestChart();
            this.updateResponseTimeChart();
            this.updateStatusChart();
            this.topEndpoints = data.data.topEndpoints || [];
            break;

          case 'RECENT_LOGS':
            this.recentLogs = data.data.map(log => ({
              id: Date.now() + Math.random(),
              status: log.status,
              method: log.request?.split(' ')[0] || 'GET',
              url: log.request?.split(' ')[1] || '',
              responseTime: log.responseTime?.toFixed(2),
              timestamp: log.timestamp
            }));
            break;

          case 'ALERT':
            this.showAlert(data.data);
            break;
        }
      };

      this.ws.onclose = () => {
        console.log('WebSocket连接关闭，5秒后重连...');
        setTimeout(() => this.initWebSocket(), 5000);
      };
    },

    initCharts() {
      this.requestChart = echarts.init(this.$refs.requestChart);
      this.responseTimeChart = echarts.init(this.$refs.responseTimeChart);
      this.statusChart = echarts.init(this.$refs.statusChart);

      this.setupRequestChart();
      this.setupResponseTimeChart();
      this.setupStatusChart();
    },

    setupRequestChart() {
      const option = {
        tooltip: {
          trigger: 'axis'
        },
        xAxis: {
          type: 'time',
          boundaryGap: false
        },
        yAxis: {
          type: 'value',
          name: '请求数/分钟'
        },
        series: [{
          name: '请求量',
          type: 'line',
          smooth: true,
          data: this.requestData
        }]
      };
      this.requestChart.setOption(option);
    },

    updateRequestChart() {
      const now = Date.now();
      this.requestData.push([now, this.overview.requestsPerSecond * 60]);

      // 保持最近60个数据点
      if (this.requestData.length > 60) {
        this.requestData.shift();
      }

      this.requestChart.setOption({
        series: [{
          data: this.requestData
        }]
      });
    },

    formatTime(timestamp) {
      return new Date(timestamp).toLocaleTimeString();
    }
  }
};
</script>

<style scoped>
.nginx-monitor {
  padding: 20px;
  font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
}

.overview-cards {
  display: grid;
  grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
  gap: 20px;
  margin-bottom: 30px;
}

.card {
  background: white;
  padding: 20px;
  border-radius: 8px;
  box-shadow: 0 2px 8px rgba(0,0,0,0.1);
}

.card h3 {
  margin: 0 0 10px 0;
  font-size: 14px;
  color: #666;
}

.card .value {
  font-size: 28px;
  font-weight: bold;
  color: #1890ff;
}

.charts-row {
  display: grid;
  grid-template-columns: 1fr 1fr;
  gap: 20px;
  margin-bottom: 30px;
}

.chart-container {
  background: white;
  padding: 20px;
  border-radius: 8px;
  box-shadow: 0 2px 8px rgba(0,0,0,0.1);
}

.log-stream {
  background: white;
  padding: 20px;
  border-radius: 8px;
  box-shadow: 0 2px 8px rgba(0,0,0,0.1);
  margin-bottom: 30px;
}

.log-list {
  max-height: 300px;
  overflow-y: auto;
}

.log-item {
  display: grid;
  grid-template-columns: 60px 80px 1fr 100px 150px;
  gap: 10px;
  padding: 8px 0;
  border-bottom: 1px solid #eee;
  align-items: center;
}

.status {
  padding: 2px 8px;
  border-radius: 4px;
  font-size: 12px;
  font-weight: bold;
}

.status-200 { background: #f6ffed; color: #52c41a; }
.status-404 { background: #fff7e6; color: #fa8c16; }
.status-500 { background: #fff1f0; color: #f5222d; }

.endpoints-ranking table {
  width: 100%;
  border-collapse: collapse;
}

.endpoints-ranking th,
.endpoints-ranking td {
  padding: 12px;
  text-align: left;
  border-bottom: 1px solid #eee;
}

.endpoints-ranking th {
  background: #fafafa;
  font-weight: 600;
}
</style>

六、告警系统

class AlertSystem {
  constructor() {
    this.alerts = [];
    this.thresholds = {
      responseTime: 5000, // 5秒
      errorRate: 5, // 5%
      qps: 1000 // 每秒1000请求
    };
  }

  checkThresholds(stats) {
    // 检查响应时间
    if (stats.p95ResponseTime > this.thresholds.responseTime) {
      this.triggerAlert({
        type: 'HIGH_RESPONSE_TIME',
        message: `P95响应时间超过阈值: ${stats.p95ResponseTime}ms`,
        severity: 'WARNING',
        data: stats
      });
    }

    // 检查错误率
    const totalRequests = stats.totalRequests;
    const errorCount = Object.entries(stats.statusCodes)
      .filter(([code]) => code >= '500')
      .reduce((sum, [, count]) => sum + count, 0);

    const errorRate = (errorCount / totalRequests) * 100;
    if (errorRate > this.thresholds.errorRate) {
      this.triggerAlert({
        type: 'HIGH_ERROR_RATE',
        message: `错误率超过阈值: ${errorRate.toFixed(2)}%`,
        severity: 'ERROR',
        data: { errorRate, totalRequests, errorCount }
      });
    }

    // 检查QPS
    if (stats.requestsPerSecond > this.thresholds.qps) {
      this.triggerAlert({
        type: 'HIGH_QPS',
        message: `QPS超过阈值: ${stats.requestsPerSecond}`,
        severity: 'WARNING',
        data: stats
      });
    }
  }

  triggerAlert(alert) {
    alert.timestamp = new Date().toISOString();
    alert.id = `alert_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;

    this.alerts.unshift(alert);

    // 保留最近1000条告警
    if (this.alerts.length > 1000) {
      this.alerts.pop();
    }

    // 发送通知
    this.sendNotification(alert);

    // 广播到前端
    this.broadcastAlert(alert);
  }

  sendNotification(alert) {
    // 可以集成多种通知方式
    const notifiers = [
      this.sendSlackNotification,
      this.sendEmailNotification,
      this.sendWebhookNotification
    ];

    notifiers.forEach(notifier => {
      try {
        notifier.call(this, alert);
      } catch (error) {
        console.error('发送通知失败:', error);
      }
    });
  }

  sendSlackNotification(alert) {
    // Slack通知实现
    const webhookUrl = process.env.SLACK_WEBHOOK_URL;
    if (!webhookUrl) return;

    const message = {
      text: `🚨 Nginx监控告警`,
      attachments: [{
        color: alert.severity === 'ERROR' ? '#ff0000' : '#ffa500',
        fields: [
          { title: '告警类型', value: alert.type, short: true },
          { title: '严重程度', value: alert.severity, short: true },
          { title: '时间', value: alert.timestamp, short: true },
          { title: '详情', value: alert.message, short: false }
        ]
      }]
    };

    // 发送到Slack
    fetch(webhookUrl, {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify(message)
    });
  }
}

七、部署与监控

7.1 Docker部署

# Dockerfile
FROM node:18-alpine

WORKDIR /app

COPY package*.json ./
RUN npm ci --only=production

COPY . .

EXPOSE 3000
EXPOSE 8080

CMD ["node", "server.js"]

# docker-compose.yml
version: '3.8'
services:
  nginx:
    image: nginx:alpine
    volumes:
      - ./nginx.conf:/etc/nginx/nginx.conf
      - ./logs:/var/log/nginx
    ports:
      - "80:80"
      - "443:443"
    networks:
      - monitor-net

  filebeat:
    image: docker.elastic.co/beats/filebeat:8.10.0
    volumes:
      - ./filebeat.yml:/usr/share/filebeat/filebeat.yml
      - ./logs:/var/log/nginx
      - /var/lib/docker/containers:/var/lib/docker/containers:ro
    depends_on:
      - redis
    networks:
      - monitor-net

  redis:
    image: redis:alpine
    ports:
      - "6379:6379"
    volumes:
      - redis-data:/data
    networks:
      - monitor-net

  monitor-api:
    build: .
    ports:
      - "3000:3000"
      - "8080:8080"
    environment:
      - REDIS_HOST=redis
      - REDIS_PORT=6379
    depends_on:
      - redis
    networks:
      - monitor-net

  frontend:
    image: nginx:alpine
    volumes:
      - ./dist:/usr/share/nginx/html
    ports:
      - "8081:80"
    networks:
      - monitor-net

networks:
  monitor-net:
    driver: bridge

volumes:
  redis-data:

7.2 性能优化建议

日志轮转

# logrotate配置
/var/log/nginx/access.json.log {
 daily
 rotate 7
 compress
 delaycompress
 missingok
 notifempty
 create 644 www-data www-data
 sharedscripts
 postrotate
     [ -f /var/run/nginx.pid ] && kill -USR1 `cat /var/run/nginx.pid`
 endscript
}

Redis优化配置

# redis.conf
maxmemory 1gb
maxmemory-policy allkeys-lru
save 900 1
save 300 10
save 60 10000

监控指标收集


# 使用Prometheus收集指标
scrape_configs:

job_name: 'nginx-monitor' static_configs:
- targets: ['monitor-api:3000']

八、安全考虑

访问控制

WebSocket API添加认证
限制访问IP
使用HTTPS

数据保护

敏感信息脱敏
日志数据加密存储
定期清理历史数据

防护措施

防止DDoS攻击
请求频率限制
输入验证

九、扩展功能

机器学习异常检测

基于历史数据训练异常检测模型
自动调整阈值

分布式追踪集成

集成Jaeger或Zipkin
全链路追踪

自动化运维

自动扩容缩容
故障自愈
配置热更新

总结

这个Nginx实时请求监测系统提供了完整的解决方案，具有以下特点：

实时性: WebSocket实现毫秒级监控 可扩展性: 微服务架构，易于水平扩展 可视化: 丰富的图表和实时展示 告警能力: 多维度阈值告警 易于部署: Docker容器化部署

系统可以根据实际需求进行调整，比如替换消息队列、增加存储后端、优化前端展示等。对于生产环境，建议进行充分的压力测试和容量规划。