From 8faa52ce908df64af7e0e69dbdb86553fa71e797 Mon Sep 17 00:00:00 2001 From: augushong Date: Sat, 8 Aug 2026 10:33:20 +0800 Subject: [PATCH] =?UTF-8?q?feat(nginx-log):=20=E5=A4=9A=E8=8A=82=E7=82=B9?= =?UTF-8?q?=E9=80=82=E9=85=8D=EF=BC=88node=5Fid=20=E5=85=A8=E9=93=BE?= =?UTF-8?q?=E8=B7=AF=20+=20=E5=85=A8=E5=B1=80/=E6=8C=89=E8=8A=82=E7=82=B9?= =?UTF-8?q?=E8=81=9A=E5=90=88=20+=20=E8=8A=82=E7=82=B9=E7=AD=9B=E9=80=89?= =?UTF-8?q?=20+=20v2.4.0=20=E5=8D=87=E7=BA=A7=E8=84=9A=E6=9C=AC=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 6 张表加 node_id 字段 + position/stat 改复合唯一键（migration） - 6 个 Scheme 同步 node_id 注解 + 唯一键 - Reader 带 node_id 参数 + 懒加载接管（getLastPosition 回退查 node_id=''） - Aggregator 全局行（node_id=''）+ 按节点循环 insertAggregatesForScope - Stat 所有查询方法加 where node_id 条件 - Dashboard 控制器/视图加节点筛选下拉框 + AJAX 带 node_id - AccessLog 列表加节点列 + 采集进度卡片显示节点信息 - import 定时任务 run_type 改 all（每节点采集自己的日志） - v2.4.0 升级脚本追加 ALTER TABLE（幂等 check-then-alter） - 菜单调整：去掉 Nginx 顶级菜单 + 读取位置管理，改为系统管理下挂两个子菜单 - ulthon-timer 文档补充 run_type=all 多节点部署注意事项 --- .agents/skills/ulthon-timer/SKILL.md | 2 + app/admin/scheme/NginxAccessLog.php | 4 + app/admin/scheme/NginxLogPosition.php | 5 +- app/admin/scheme/NginxStatHour.php | 5 +- app/admin/scheme/NginxStatReferer.php | 5 +- app/admin/scheme/NginxStatUa.php | 5 +- app/admin/scheme/NginxStatUrl.php | 5 +- app/admin/view/nginx/access_log/index.html | 42 +++ app/admin/view/nginx/access_log/index.js | 1 + app/admin/view/nginx/access_log/read.html | 2 +- .../20260801125222_nginx_log_add_node_id.php | 70 ++++ .../admin/controller/nginx/AccessLogBase.php | 18 + .../controller/system/NginxLogStatBase.php | 33 +- .../service/adminInitData/SystemMenu.php | 26 +- .../service/adminUpdateCodeData/v2.4.0.php | 129 +++++-- .../view/system/nginx_log_stat/index.html | 8 + .../admin/view/system/nginx_log_stat/index.js | 19 +- extend/base/common/command/timer/config.php | 2 +- .../service/NginxLogAggregatorServiceBase.php | 339 +++++++++++------- .../service/NginxLogReaderServiceBase.php | 63 +++- .../controller/timer/NginxLogImportBase.php | 1 + 21 files changed, 578 insertions(+), 206 deletions(-) create mode 100644 database/migrations/20260801125222_nginx_log_add_node_id.php diff --git a/.agents/skills/ulthon-timer/SKILL.md b/.agents/skills/ulthon-timer/SKILL.md index 8c4d242..9949450 100644 --- a/.agents/skills/ulthon-timer/SKILL.md +++ b/.agents/skills/ulthon-timer/SKILL.md @@ -201,6 +201,8 @@ nohup php /var/www/html/think timer --local --quiet & 多节点部署：每个容器各自启动 timer 进程，`runtime/node_id.lock` 在各自容器内独立生成（天然隔离），连同一个数据库即自动组成多节点集群。 +> **多节点部署注意（run_type=all 的任务）**：`run_type=all` 的定时任务（如 `nginx_log_import`）要求每节点各自采集本机数据，每节点必须用 `php think timer --local` 启动，确保 site 请求发往本机 `localhost` 而非共享 `site_domain`/LB，否则请求会落在随机节点上，导致采集错位（节点 A 的日志被节点 B 处理）。框架内置 Docker 部署栈已默认带 `--local`，手动/裸机部署时务必显式加上。 + ### 本地调试（指定请求 Host） site 任务会按站点域名发起请求，默认从 `sysconfig('site','site_domain')` 读取。 diff --git a/app/admin/scheme/NginxAccessLog.php b/app/admin/scheme/NginxAccessLog.php index 40fbbda..b6fb0e6 100644 --- a/app/admin/scheme/NginxAccessLog.php +++ b/app/admin/scheme/NginxAccessLog.php @@ -9,6 +9,7 @@ use app\common\scheme\attribute\Component; use app\common\scheme\attribute\Index; #[Table(name: 'ul_nginx_access_log', comment: 'nginx 访问日志明细（解析入库）')] +#[Index(columns: ['node_id', 'time_local'], name: 'idx_node_time', type: 'NORMAL')] #[Index(columns: ['time_local'], name: 'idx_time_local', type: 'NORMAL')] #[Index(columns: ['remote_addr'], name: 'idx_remote_addr', type: 'NORMAL')] #[Index(columns: ['status'], name: 'idx_status', type: 'NORMAL')] @@ -19,6 +20,9 @@ class NginxAccessLog extends BaseScheme #[Field(type: 'bigint', nullable: false, unsigned: true, autoIncrement: true, primary: true, comment: '主键ID')] public $id; + #[Field(type: 'varchar', length: 64, nullable: false, default: '', comment: '节点ID（区分不同采集节点）')] + public $node_id; + #[Field(type: 'varchar', length: 500, nullable: false, comment: '来源日志文件路径')] #[Component(type: 'text', options: [])] public $file_path; diff --git a/app/admin/scheme/NginxLogPosition.php b/app/admin/scheme/NginxLogPosition.php index a027479..ca8560b 100644 --- a/app/admin/scheme/NginxLogPosition.php +++ b/app/admin/scheme/NginxLogPosition.php @@ -9,12 +9,15 @@ use app\common\scheme\attribute\Component; use app\common\scheme\attribute\Index; #[Table(name: 'ul_nginx_log_position', comment: 'nginx 日志文件读取位置（增量读取状态）')] -#[Index(columns: ['file_path'], name: 'uniq_file_path', type: 'UNIQUE')] +#[Index(columns: ['node_id', 'file_path'], name: 'uniq_node_file', type: 'UNIQUE')] class NginxLogPosition extends BaseScheme { #[Field(type: 'int', length: 11, nullable: false, unsigned: true, autoIncrement: true, primary: true)] public $id; + #[Field(type: 'varchar', length: 64, nullable: false, default: '', comment: '节点ID（区分不同采集节点）')] + public $node_id; + #[Field(type: 'varchar', length: 500, nullable: false, comment: '日志文件绝对路径')] #[Component(type: 'text', options: [])] public $file_path; diff --git a/app/admin/scheme/NginxStatHour.php b/app/admin/scheme/NginxStatHour.php index 887173f..526275f 100644 --- a/app/admin/scheme/NginxStatHour.php +++ b/app/admin/scheme/NginxStatHour.php @@ -8,13 +8,16 @@ use app\common\scheme\attribute\Field; use app\common\scheme\attribute\Index; #[Table(name: 'nginx_stat_hour', comment: 'nginx 按小时聚合统计')] -#[Index(columns: ['stat_date', 'stat_hour'], name: 'uniq_date_hour', type: 'UNIQUE')] +#[Index(columns: ['node_id', 'stat_date', 'stat_hour'], name: 'uniq_node_date_hour', type: 'UNIQUE')] #[Index(columns: ['stat_date'], name: 'idx_stat_date', type: 'NORMAL')] class NginxStatHour extends BaseScheme { #[Field(type: 'int', length: 11, nullable: false, unsigned: true, autoIncrement: true, primary: true)] public $id; + #[Field(type: 'varchar', length: 64, nullable: false, default: '', comment: '节点ID（区分不同采集节点）')] + public $node_id; + #[Field(type: 'int', length: 11, nullable: false, unsigned: true, comment: '统计日期(YYYYMMDD)')] public $stat_date; diff --git a/app/admin/scheme/NginxStatReferer.php b/app/admin/scheme/NginxStatReferer.php index 3c64b5d..c3a17ce 100644 --- a/app/admin/scheme/NginxStatReferer.php +++ b/app/admin/scheme/NginxStatReferer.php @@ -8,13 +8,16 @@ use app\common\scheme\attribute\Field; use app\common\scheme\attribute\Index; #[Table(name: 'nginx_stat_referer', comment: 'nginx 按 Referer 来源域名聚合统计')] -#[Index(columns: ['stat_date', 'referer_domain'], name: 'uniq_date_referer', type: 'UNIQUE')] +#[Index(columns: ['node_id', 'stat_date', 'referer_domain'], name: 'uniq_node_date_referer', type: 'UNIQUE')] #[Index(columns: ['stat_date'], name: 'idx_stat_date', type: 'NORMAL')] class NginxStatReferer extends BaseScheme { #[Field(type: 'int', length: 11, nullable: false, unsigned: true, autoIncrement: true, primary: true)] public $id; + #[Field(type: 'varchar', length: 64, nullable: false, default: '', comment: '节点ID（区分不同采集节点）')] + public $node_id; + #[Field(type: 'int', length: 11, nullable: false, unsigned: true, comment: '统计日期(YYYYMMDD)')] public $stat_date; diff --git a/app/admin/scheme/NginxStatUa.php b/app/admin/scheme/NginxStatUa.php index 5171001..73649b1 100644 --- a/app/admin/scheme/NginxStatUa.php +++ b/app/admin/scheme/NginxStatUa.php @@ -8,13 +8,16 @@ use app\common\scheme\attribute\Field; use app\common\scheme\attribute\Index; #[Table(name: 'nginx_stat_ua', comment: 'nginx 按 User-Agent 聚合统计')] -#[Index(columns: ['stat_date', 'ua_type', 'ua_name'], name: 'uniq_date_type_name', type: 'UNIQUE')] +#[Index(columns: ['node_id', 'stat_date', 'ua_type', 'ua_name'], name: 'uniq_node_date_type_name', type: 'UNIQUE')] #[Index(columns: ['stat_date'], name: 'idx_stat_date', type: 'NORMAL')] class NginxStatUa extends BaseScheme { #[Field(type: 'int', length: 11, nullable: false, unsigned: true, autoIncrement: true, primary: true)] public $id; + #[Field(type: 'varchar', length: 64, nullable: false, default: '', comment: '节点ID（区分不同采集节点）')] + public $node_id; + #[Field(type: 'int', length: 11, nullable: false, unsigned: true, comment: '统计日期(YYYYMMDD)')] public $stat_date; diff --git a/app/admin/scheme/NginxStatUrl.php b/app/admin/scheme/NginxStatUrl.php index 84cffd3..1202491 100644 --- a/app/admin/scheme/NginxStatUrl.php +++ b/app/admin/scheme/NginxStatUrl.php @@ -8,13 +8,16 @@ use app\common\scheme\attribute\Field; use app\common\scheme\attribute\Index; #[Table(name: 'nginx_stat_url', comment: 'nginx 按 URL 聚合统计')] -#[Index(columns: ['stat_date', 'uri'], name: 'uniq_date_uri', type: 'UNIQUE')] +#[Index(columns: ['node_id', 'stat_date', 'uri'], name: 'uniq_node_date_uri', type: 'UNIQUE')] #[Index(columns: ['stat_date'], name: 'idx_stat_date', type: 'NORMAL')] class NginxStatUrl extends BaseScheme { #[Field(type: 'int', length: 11, nullable: false, unsigned: true, autoIncrement: true, primary: true)] public $id; + #[Field(type: 'varchar', length: 64, nullable: false, default: '', comment: '节点ID（区分不同采集节点）')] + public $node_id; + #[Field(type: 'int', length: 11, nullable: false, unsigned: true, comment: '统计日期(YYYYMMDD)')] public $stat_date; diff --git a/app/admin/view/nginx/access_log/index.html b/app/admin/view/nginx/access_log/index.html index ef77867..2d1ba11 100644 --- a/app/admin/view/nginx/access_log/index.html +++ b/app/admin/view/nginx/access_log/index.html @@ -1,5 +1,47 @@
+ {if condition="$positions->isEmpty()"} +
+
+ + 暂无采集位置记录，请确认 nginx_log.enable 已开启且 timer 正在运行。 +
+
+ {else/} + {volist name="positions" id="pos"} +
+
+
+
+ 监控文件
+ {$pos.file_path} +
+
+ 节点
+ {$pos.node_id} +
+
+ 已读取
+ {$pos.offset_human} +
+
+ 最后采集
+ {$pos.last_read_time_text} +
+
+ 解析失败
+ {if condition="$pos.parse_fail_count > 0"} + {$pos.parse_fail_count} 次 + {else/} + 0 次 + {/if} +
+
+
+
+ {/volist} + {/if} + = 500) cls = 'layui-badge layui-bg-red'; return '' + d.status + ''; }}, + {field: 'node_id', title: '节点', width: 120, searchOp: '='}, {field: 'remote_addr', title: '客户端IP', width: 140, searchOp: '%*%'}, {field: 'request_time', title: '耗时(秒)', width: 100, search: 'number_limit', templet: function(d) { var t = parseFloat(d.request_time); diff --git a/app/admin/view/nginx/access_log/read.html b/app/admin/view/nginx/access_log/read.html index ca0f137..dae6157 100644 --- a/app/admin/view/nginx/access_log/read.html +++ b/app/admin/view/nginx/access_log/read.html @@ -4,7 +4,7 @@
-

#{$row.id} {$title}

+

#{$row.id} {$title|default=''}

ID: {$row.id}
diff --git a/database/migrations/20260801125222_nginx_log_add_node_id.php b/database/migrations/20260801125222_nginx_log_add_node_id.php new file mode 100644 index 0000000..3de0079 --- /dev/null +++ b/database/migrations/20260801125222_nginx_log_add_node_id.php @@ -0,0 +1,70 @@ +table('nginx_log_position') + ->addColumn('node_id', 'string', ['limit' => 64, 'default' => '', 'comment' => '采集节点 ID（空=全局汇总）']) + ->removeIndexByName('uniq_file_path') + ->addIndex(['node_id', 'file_path'], ['unique' => true, 'name' => 'uniq_node_file']) + ->update(); + + // ==================== 2. nginx_access_log（raw 表）==================== + // 仅加字段 + 普通索引（按节点 + 时间查询），无唯一键变更 + $this->table('nginx_access_log') + ->addColumn('node_id', 'string', ['limit' => 64, 'default' => '', 'comment' => '采集节点 ID（空=全局汇总）']) + ->addIndex(['node_id', 'time_local'], ['name' => 'idx_node_time']) + ->update(); + + // ==================== 3. nginx_stat_hour ==================== + // 旧唯一键 uniq_date_hour(stat_date, stat_hour) → uniq_node_date_hour(node_id, stat_date, stat_hour) + $this->table('nginx_stat_hour') + ->addColumn('node_id', 'string', ['limit' => 64, 'default' => '', 'comment' => '采集节点 ID（空=全局汇总）']) + ->removeIndexByName('uniq_date_hour') + ->addIndex(['node_id', 'stat_date', 'stat_hour'], ['unique' => true, 'name' => 'uniq_node_date_hour']) + ->update(); + + // ==================== 4. nginx_stat_url ==================== + // 旧唯一键 uniq_date_uri(stat_date, uri) → uniq_node_date_uri(node_id, stat_date, uri) + $this->table('nginx_stat_url') + ->addColumn('node_id', 'string', ['limit' => 64, 'default' => '', 'comment' => '采集节点 ID（空=全局汇总）']) + ->removeIndexByName('uniq_date_uri') + ->addIndex(['node_id', 'stat_date', 'uri'], ['unique' => true, 'name' => 'uniq_node_date_uri']) + ->update(); + + // ==================== 5. nginx_stat_referer ==================== + // 旧唯一键 uniq_date_referer(stat_date, referer_domain) → uniq_node_date_referer(node_id, stat_date, referer_domain) + $this->table('nginx_stat_referer') + ->addColumn('node_id', 'string', ['limit' => 64, 'default' => '', 'comment' => '采集节点 ID（空=全局汇总）']) + ->removeIndexByName('uniq_date_referer') + ->addIndex(['node_id', 'stat_date', 'referer_domain'], ['unique' => true, 'name' => 'uniq_node_date_referer']) + ->update(); + + // ==================== 6. nginx_stat_ua ==================== + // 旧唯一键 uniq_date_type_name(stat_date, ua_type, ua_name) → uniq_node_date_type_name(node_id, stat_date, ua_type, ua_name) + $this->table('nginx_stat_ua') + ->addColumn('node_id', 'string', ['limit' => 64, 'default' => '', 'comment' => '采集节点 ID（空=全局汇总）']) + ->removeIndexByName('uniq_date_type_name') + ->addIndex(['node_id', 'stat_date', 'ua_type', 'ua_name'], ['unique' => true, 'name' => 'uniq_node_date_type_name']) + ->update(); + } +} diff --git a/extend/base/admin/controller/nginx/AccessLogBase.php b/extend/base/admin/controller/nginx/AccessLogBase.php index 2a69f03..52f7d29 100644 --- a/extend/base/admin/controller/nginx/AccessLogBase.php +++ b/extend/base/admin/controller/nginx/AccessLogBase.php @@ -56,6 +56,24 @@ class AccessLogBase extends AdminController return json($data); } + // 采集位置信息（合并自原"读取位置管理"独立页面） + $positions = \app\admin\model\NginxLogPosition::order('id', 'desc')->select()->map(function ($item) { + $bytes = (int) $item['offset']; + if ($bytes >= 1048576) { + $item['offset_human'] = number_format($bytes / 1048576, 2) . ' MB'; + } elseif ($bytes >= 1024) { + $item['offset_human'] = number_format($bytes / 1024, 2) . ' KB'; + } else { + $item['offset_human'] = $bytes . ' B'; + } + $item['last_read_time_text'] = $item['last_read_time'] > 0 + ? date('Y-m-d H:i:s', (int) $item['last_read_time']) + : '-'; + + return $item; + }); + $this->assign('positions', $positions); + return $this->fetch(); } diff --git a/extend/base/admin/controller/system/NginxLogStatBase.php b/extend/base/admin/controller/system/NginxLogStatBase.php index 5e391e8..fbfc3d2 100644 --- a/extend/base/admin/controller/system/NginxLogStatBase.php +++ b/extend/base/admin/controller/system/NginxLogStatBase.php @@ -45,11 +45,13 @@ class NginxLogStatBase extends AdminController $period = 'day'; } + $nodeId = (string) $request->param('node_id', ''); + // 解析时间范围（YYYYMMDD int 表示） [$startDate, $endDate] = $this->resolveDateRange($period); if ($request->isAjax()) { - $data = $this->buildStatData($period, $startDate, $endDate); + $data = $this->buildStatData($period, $startDate, $endDate, $nodeId); return json([ 'code' => 0, @@ -63,6 +65,11 @@ class NginxLogStatBase extends AdminController $this->assign('start', $startDate); $this->assign('end', $endDate); + // 在线节点列表（供视图渲染节点筛选下拉） + $nodes = \app\admin\model\SystemHost::where('status', 1)->order('node_id', 'asc')->select(); + $this->assign('nodes', $nodes); + $this->assign('node_id', $nodeId); + return $this->fetch(); } @@ -140,20 +147,20 @@ class NginxLogStatBase extends AdminController /** * 组装完整统计数据 */ - protected function buildStatData(string $period, int $startDate, int $endDate): array + protected function buildStatData(string $period, int $startDate, int $endDate, string $nodeId = ''): array { // 时间范围不合法 → 空数据（不崩） if ($startDate <= 0 || $endDate <= 0 || $startDate > $endDate) { return $this->emptyData($period); } - $summary = $this->querySummary($startDate, $endDate); - $trend = $this->queryTrend($period, $startDate, $endDate); + $summary = $this->querySummary($startDate, $endDate, $nodeId); + $trend = $this->queryTrend($period, $startDate, $endDate, $nodeId); $topN = $this->getTopN(); - $topUrls = $this->queryTopByDateRange(NginxStatUrl::class, 'uri', $startDate, $endDate, $topN); - $topReferers = $this->queryTopByDateRange(NginxStatReferer::class, 'referer_domain', $startDate, $endDate, $topN); - $topUas = $this->queryTopByDateRange(NginxStatUa::class, 'ua_name', $startDate, $endDate, $topN); + $topUrls = $this->queryTopByDateRange(NginxStatUrl::class, 'uri', $startDate, $endDate, $topN, $nodeId); + $topReferers = $this->queryTopByDateRange(NginxStatReferer::class, 'referer_domain', $startDate, $endDate, $topN, $nodeId); + $topUas = $this->queryTopByDateRange(NginxStatUa::class, 'ua_name', $startDate, $endDate, $topN, $nodeId); // 跨周期 UV 估算标注：仅 day 为精确值，week/month/year/custom 均为按日累加的估算值 $uvIsEstimate = $period !== 'day'; @@ -208,7 +215,7 @@ class NginxLogStatBase extends AdminController * 整体指标：SUM stat_hour * avg_request_time 按 PV 加权平均（避免简单 AVG 被低 PV 时段拉偏） */ - protected function querySummary(int $startDate, int $endDate): array + protected function querySummary(int $startDate, int $endDate, string $nodeId = ''): array { $sumFields = [ 'IFNULL(SUM(pv),0) AS pv', @@ -223,6 +230,7 @@ class NginxLogStatBase extends AdminController ]; $row = NginxStatHour::where('stat_date', 'between', [$startDate, $endDate]) + ->where('node_id', $nodeId) ->field(implode(',', $sumFields)) ->find(); @@ -252,10 +260,11 @@ class NginxLogStatBase extends AdminController * - year → 12 个月（按 stat_date DIV 100 分组，补全 1~12 月） * - custom→ 按 stat_date 分组，补全 start~end 日期序列 */ - protected function queryTrend(string $period, int $startDate, int $endDate): array + protected function queryTrend(string $period, int $startDate, int $endDate, string $nodeId = ''): array { if ($period === 'day') { $rows = NginxStatHour::where('stat_date', 'between', [$startDate, $endDate]) + ->where('node_id', $nodeId) ->field('stat_hour, IFNULL(SUM(pv),0) AS pv, IFNULL(SUM(uv),0) AS uv') ->group('stat_hour') ->select() @@ -280,6 +289,7 @@ class NginxLogStatBase extends AdminController if ($period === 'year') { $rows = NginxStatHour::where('stat_date', 'between', [$startDate, $endDate]) + ->where('node_id', $nodeId) ->field('(stat_date DIV 100) AS ym, IFNULL(SUM(pv),0) AS pv, IFNULL(SUM(uv),0) AS uv') ->group('ym') ->order('ym asc') @@ -307,6 +317,7 @@ class NginxLogStatBase extends AdminController // week / month / custom → 按 stat_date 分组，补全日期序列 $rows = NginxStatHour::where('stat_date', 'between', [$startDate, $endDate]) + ->where('node_id', $nodeId) ->field('stat_date, IFNULL(SUM(pv),0) AS pv, IFNULL(SUM(uv),0) AS uv') ->group('stat_date') ->order('stat_date asc') @@ -340,10 +351,12 @@ class NginxLogStatBase extends AdminController * @param string $modelClass 模型类（依赖倒置：调 app/admin/model 入口） * @param string $labelField 分组与展示的字段名 * @param int $topN 取前 N 条；0 不限制 + * @param string $nodeId 节点 ID（空字符串=全局聚合行） */ - protected function queryTopByDateRange(string $modelClass, string $labelField, int $startDate, int $endDate, int $topN): array + protected function queryTopByDateRange(string $modelClass, string $labelField, int $startDate, int $endDate, int $topN, string $nodeId = ''): array { $query = $modelClass::where('stat_date', 'between', [$startDate, $endDate]) + ->where('node_id', $nodeId) ->field($labelField . ' AS label, IFNULL(SUM(pv),0) AS pv, IFNULL(SUM(uv),0) AS uv') ->group($labelField) ->order('pv desc'); diff --git a/extend/base/admin/service/adminInitData/SystemMenu.php b/extend/base/admin/service/adminInitData/SystemMenu.php index 3b94d43..1fc467e 100644 --- a/extend/base/admin/service/adminInitData/SystemMenu.php +++ b/extend/base/admin/service/adminInitData/SystemMenu.php @@ -209,20 +209,9 @@ $ul_system_menu = array( "sort" => 0, "status" => 1, ), - array( - "id" => 300, - "pid" => 0, - "title" => "Nginx 日志分析", - "icon" => "fa fa-chart-bar", - "href" => "", - "params" => "", - "target" => "_self", - "sort" => 0, - "status" => 1, - ), array( "id" => 301, - "pid" => 300, + "pid" => 228, "title" => "访问统计仪表盘", "icon" => "fa fa-line-chart", "href" => "system.nginx_log_stat/index", @@ -233,7 +222,7 @@ $ul_system_menu = array( ), array( "id" => 302, - "pid" => 300, + "pid" => 228, "title" => "原始访问日志", "icon" => "fa fa-file-text", "href" => "nginx.access_log/index", @@ -242,17 +231,6 @@ $ul_system_menu = array( "sort" => 0, "status" => 1, ), - array( - "id" => 303, - "pid" => 300, - "title" => "读取位置管理", - "icon" => "fa fa-map-marker", - "href" => "nginx.log_position/index", - "params" => "", - "target" => "_self", - "sort" => 0, - "status" => 1, - ), ); diff --git a/extend/base/admin/service/adminUpdateCodeData/v2.4.0.php b/extend/base/admin/service/adminUpdateCodeData/v2.4.0.php index 5ab6e4b..a7f7e2f 100644 --- a/extend/base/admin/service/adminUpdateCodeData/v2.4.0.php +++ b/extend/base/admin/service/adminUpdateCodeData/v2.4.0.php @@ -13,11 +13,12 @@ * stat_retention_days 统计聚合保留天数（0=永久） * topn_default Top N 默认值 * exclude_static 是否排除静态资源访问 - * - system_menu 新增 4 条菜单（顶级分组 300 + 3 个子菜单 301/302/303） - * 300 Nginx 日志分析（顶级分组，href 为空） + * - system_menu 新增 2 条菜单（挂到系统管理 pid=228 下） * 301 访问统计仪表盘 system.nginx_log_stat/index * 302 原始访问日志 nginx.access_log/index - * 303 读取位置管理 nginx.log_position/index + * - ALTER TABLE：6 张 nginx_log 表新增 node_id 字段 + 唯一键前置 node_id（多节点采集支持） + * 与 migration 20260801125222_nginx_log_add_node_id.php 保持一致， + * 存量库通过 admin:update 幂等执行（check-then-alter） * * 设计说明： * - 幂等 check-then-insert（先查询存在性，不存在才插入） @@ -82,23 +83,12 @@ class UpdateFunction /** * 待新增菜单. * 字段顺序与 adminInitData/SystemMenu.php 一致：id/pid/title/icon/href/params/target/sort/status. - * id 显式分配 300-303，与 adminInitData 保持一致（顶级 300 + 3 个子菜单）。 + * 挂到系统管理（pid=228）下，不创建独立顶级分组。 */ public $newMenus = [ - [ - 'id' => 300, - 'pid' => 0, - 'title' => 'Nginx 日志分析', - 'icon' => 'fa fa-chart-bar', - 'href' => '', - 'params' => '', - 'target' => '_self', - 'sort' => 0, - 'status' => 1, - ], [ 'id' => 301, - 'pid' => 300, + 'pid' => 228, 'title' => '访问统计仪表盘', 'icon' => 'fa fa-line-chart', 'href' => 'system.nginx_log_stat/index', @@ -109,7 +99,7 @@ class UpdateFunction ], [ 'id' => 302, - 'pid' => 300, + 'pid' => 228, 'title' => '原始访问日志', 'icon' => 'fa fa-file-text', 'href' => 'nginx.access_log/index', @@ -118,24 +108,13 @@ class UpdateFunction 'sort' => 0, 'status' => 1, ], - [ - 'id' => 303, - 'pid' => 300, - 'title' => '读取位置管理', - 'icon' => 'fa fa-map-marker', - 'href' => 'nginx.log_position/index', - 'params' => '', - 'target' => '_self', - 'sort' => 0, - 'status' => 1, - ], ]; public function update() { $this->output->writeln('更新代码'); - $this->output->info('v2.4.0：nginx_log 分析模块存量升级（补齐 4 条 sysconfig + 4 条菜单）'); + $this->output->info('v2.4.0：nginx_log 分析模块存量升级（补齐 4 条 sysconfig + 2 条菜单）'); $configTable = config('database.connections.mysql.prefix', 'ul_') . 'system_config'; $menuTable = config('database.connections.mysql.prefix', 'ul_') . 'system_menu'; @@ -190,6 +169,98 @@ class UpdateFunction $totalSkipped = $configSkipped + $menuSkipped; $this->output->info("本次新增 {$configAdded} 条配置、{$menuAdded} 条菜单，跳过 {$totalSkipped} 条已存在"); + + // ==================== node_id 字段 + 唯一键变更（多节点采集支持）==================== + // 与 migration 20260801125222_nginx_log_add_node_id.php 保持一致 + // 6 张表全部新增 node_id varchar(64) default '' + // position / 4 张 stat 表的唯一键前置 node_id（多节点隔离，避免聚合 INSERT 触发 duplicate key） + // raw 表（nginx_access_log）加 node_id + time_local 普通索引，无唯一键变更 + $this->output->info('v2.4.0：nginx_log 6 张表新增 node_id 字段与唯一键调整'); + + $prefix = config('database.connections.mysql.prefix', 'ul_'); + + // 表短名 => 索引变更配置 + // drop_idx: 要删除的旧索引名（raw 表无） + // add_idx: 新索引 [name, cols, unique] + $nodeIdTables = [ + 'nginx_log_position' => [ + 'drop_idx' => 'uniq_file_path', + 'add_idx' => ['name' => 'uniq_node_file', 'cols' => ['node_id', 'file_path'], 'unique' => true], + ], + 'nginx_access_log' => [ + 'add_idx' => ['name' => 'idx_node_time', 'cols' => ['node_id', 'time_local'], 'unique' => false], + ], + 'nginx_stat_hour' => [ + 'drop_idx' => 'uniq_date_hour', + 'add_idx' => ['name' => 'uniq_node_date_hour', 'cols' => ['node_id', 'stat_date', 'stat_hour'], 'unique' => true], + ], + 'nginx_stat_url' => [ + 'drop_idx' => 'uniq_date_uri', + 'add_idx' => ['name' => 'uniq_node_date_uri', 'cols' => ['node_id', 'stat_date', 'uri'], 'unique' => true], + ], + 'nginx_stat_referer' => [ + 'drop_idx' => 'uniq_date_referer', + 'add_idx' => ['name' => 'uniq_node_date_referer', 'cols' => ['node_id', 'stat_date', 'referer_domain'], 'unique' => true], + ], + 'nginx_stat_ua' => [ + 'drop_idx' => 'uniq_date_type_name', + 'add_idx' => ['name' => 'uniq_node_date_type_name', 'cols' => ['node_id', 'stat_date', 'ua_type', 'ua_name'], 'unique' => true], + ], + ]; + + $colAdded = 0; + $colSkipped = 0; + foreach ($nodeIdTables as $tableShort => $idxCfg) { + $fullTable = $prefix . $tableShort; + + // 防御性检查：表是否存在（未运行 migration 时跳过，不报错） + $tableExists = Db::query( + "SELECT COUNT(*) AS cnt FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ?", + [$fullTable] + ); + if (empty($tableExists) || (int) $tableExists[0]['cnt'] === 0) { + $this->output->writeln(" - {$fullTable} 表不存在，跳过（请先执行 migrate:run）"); + continue; + } + + // 幂等 1：检查 node_id 字段是否存在 + $colExists = Db::query( + "SELECT COUNT(*) AS cnt FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND COLUMN_NAME = 'node_id'", + [$fullTable] + ); + if (!empty($colExists) && (int) $colExists[0]['cnt'] > 0) { + $this->output->writeln(" - {$fullTable} node_id 字段已存在，跳过"); + ++$colSkipped; + } else { + Db::execute("ALTER TABLE `{$fullTable}` ADD COLUMN `node_id` VARCHAR(64) NOT NULL DEFAULT '' COMMENT '采集节点 ID（空=全局汇总）'"); + $this->output->writeln(" - {$fullTable} 新增 node_id 字段完成"); + ++$colAdded; + } + + // 幂等 2：唯一键/索引变更（检查新索引是否存在） + $newIdx = $idxCfg['add_idx']['name'] ?? null; + if ($newIdx !== null) { + $idxExists = Db::query( + "SELECT COUNT(*) AS cnt FROM information_schema.STATISTICS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND INDEX_NAME = ?", + [$fullTable, $newIdx] + ); + if (empty($idxExists) || (int) $idxExists[0]['cnt'] === 0) { + // 删除旧索引（position / stat 表有，raw 表无） + if (!empty($idxCfg['drop_idx'])) { + Db::execute("ALTER TABLE `{$fullTable}` DROP INDEX `{$idxCfg['drop_idx']}`"); + } + // 新增新索引 + $idxType = !empty($idxCfg['add_idx']['unique']) ? 'UNIQUE' : 'INDEX'; + $colsStr = implode(', ', array_map(function ($c) { + return "`{$c}`"; + }, $idxCfg['add_idx']['cols'])); + Db::execute("ALTER TABLE `{$fullTable}` ADD {$idxType} `{$newIdx}` ({$colsStr})"); + $this->output->writeln(" - {$fullTable} 索引调整：{$newIdx}"); + } + } + } + + $this->output->info("node_id 迁移完成（新增字段 {$colAdded} 表，跳过 {$colSkipped} 表已存在）"); $this->output->writeln('更新代码完成'); } } diff --git a/extend/base/admin/view/system/nginx_log_stat/index.html b/extend/base/admin/view/system/nginx_log_stat/index.html index 8e2b463..ad9946a 100644 --- a/extend/base/admin/view/system/nginx_log_stat/index.html +++ b/extend/base/admin/view/system/nginx_log_stat/index.html @@ -19,6 +19,14 @@ +
+ +
diff --git a/extend/base/admin/view/system/nginx_log_stat/index.js b/extend/base/admin/view/system/nginx_log_stat/index.js index d1d5baf..5db29e1 100644 --- a/extend/base/admin/view/system/nginx_log_stat/index.js +++ b/extend/base/admin/view/system/nginx_log_stat/index.js @@ -6,11 +6,15 @@ * 后端 URL 不硬编码：用 ua.url('system.nginx_log_stat/index') 由 CONFIG.ADMIN 拼接。 * AJAX 触发方式：URL 带 _ajax=1（不依赖 Accept header，命令行测试友好）。 */ -layui.use(['table', 'laydate', 'util'], function () { +layui.use(['table', 'laydate', 'util', 'form'], function () { var table = layui.table; var laydate = layui.laydate; var util = layui.util; + var form = layui.form; + + // 渲染节点下拉框 + form.render(); // ── 配置（URL 不硬编码：由 ua.url 拼接 /{ADMIN}/system.nginx_log_stat/index）── var STAT_INDEX_URL = ua.url('system.nginx_log_stat/index'); @@ -129,6 +133,15 @@ layui.use(['table', 'laydate', 'util'], function () { loadData({ period: 'custom', start: startVal, end: endVal }); }); + // ================================================================ + // 节点下拉框切换 + // ================================================================ + + form.on('select(nodeFilter)', function (data) { + // 切换节点时用当前 period 重新加载 + loadData({ period: currentPeriod }); + }); + // ================================================================ // AJAX 加载数据 // ================================================================ @@ -141,6 +154,10 @@ layui.use(['table', 'laydate', 'util'], function () { function loadData(extra) { extra = extra || {}; + // 当前选中的节点 ID（默认空字符串=全部节点） + var nodeId = $('#node-filter').val() || ''; + extra.node_id = nodeId; + var params = $.extend({ _ajax: 1, period: currentPeriod }, extra); ua.request.get({ diff --git a/extend/base/common/command/timer/config.php b/extend/base/common/command/timer/config.php index 2793bf4..03c6c3b 100644 --- a/extend/base/common/command/timer/config.php +++ b/extend/base/common/command/timer/config.php @@ -27,7 +27,7 @@ return [ 'target' => '/tools/timer.NginxLogImport/do', 'frequency' => 300, 'concurrency' => 1, - 'run_type' => 'auto', + 'run_type' => 'all', ], [ 'name' => 'nginx_log_aggregate', diff --git a/extend/base/common/service/NginxLogAggregatorServiceBase.php b/extend/base/common/service/NginxLogAggregatorServiceBase.php index 82d2ad3..5a265aa 100644 --- a/extend/base/common/service/NginxLogAggregatorServiceBase.php +++ b/extend/base/common/service/NginxLogAggregatorServiceBase.php @@ -24,9 +24,9 @@ class NginxLogAggregatorServiceBase /** * 聚合指定日期的指定小时. * - * 流程： - * 1. stat_hour：DELETE WHERE stat_date=:d AND stat_hour=:h → INSERT 1 行（该小时聚合） - * 2. stat_url / stat_referer / stat_ua：DELETE WHERE stat_date=:d（整天重跑）→ INSERT top N 行 + * 流程（每张表生成 全局行 node_id='' + 各节点行）： + * 1. stat_hour：DELETE WHERE stat_date=:d AND stat_hour=:h → INSERT 全局 1 行 + 每节点 1 行 + * 2. stat_url / stat_referer / stat_ua：DELETE WHERE stat_date=:d（整天重跑）→ INSERT 全局 topN + 每节点 topN * * @param int $statDate 日期 YYYYMMDD（如 20260728） * @param int $statHour 小时 0-23 @@ -65,142 +65,50 @@ class NginxLogAggregatorServiceBase Db::startTrans(); try { - // 1. stat_hour：DELETE + INSERT 1 行 + // DELETE（逻辑不变）：stat_hour 按小时，stat_url/referer/ua 按整天 NginxStatHour::where('stat_date', $statDate) ->where('stat_hour', $statHour) ->delete(); - - $hourSql = "INSERT INTO `{$hourTable}` - (stat_date, stat_hour, pv, uv, total_bytes, status_2xx, status_3xx, status_4xx, status_5xx, status_other, avg_request_time) - SELECT :stat_date, :stat_hour, - COUNT(*) AS pv, - COUNT(DISTINCT remote_addr) AS uv, - COALESCE(SUM(body_bytes_sent), 0) AS total_bytes, - COALESCE(SUM(CASE WHEN status BETWEEN 200 AND 299 THEN 1 ELSE 0 END), 0) AS status_2xx, - COALESCE(SUM(CASE WHEN status BETWEEN 300 AND 399 THEN 1 ELSE 0 END), 0) AS status_3xx, - COALESCE(SUM(CASE WHEN status BETWEEN 400 AND 499 THEN 1 ELSE 0 END), 0) AS status_4xx, - COALESCE(SUM(CASE WHEN status BETWEEN 500 AND 599 THEN 1 ELSE 0 END), 0) AS status_5xx, - COALESCE(SUM(CASE WHEN status < 200 OR status > 599 THEN 1 ELSE 0 END), 0) AS status_other, - COALESCE(AVG(request_time), 0) AS avg_request_time - FROM `{$rawTable}` - WHERE time_local >= :hour_start AND time_local < :hour_end - HAVING COUNT(*) > 0"; - - $statHourRows = (int) Db::execute($hourSql, [ - 'stat_date' => $statDate, - 'stat_hour' => $statHour, - 'hour_start' => $hourStart, - 'hour_end' => $hourEnd, - ]); - - // 2. stat_url：DELETE（整天）+ INSERT topN NginxStatUrl::where('stat_date', $statDate)->delete(); - - $urlLimit = $topn > 0 ? 'LIMIT ' . $topn : ''; - $urlSql = "INSERT INTO `{$urlTable}` - (stat_date, uri, pv, uv, total_bytes, avg_request_time) - SELECT :stat_date, uri, - COUNT(*) AS pv, - COUNT(DISTINCT remote_addr) AS uv, - COALESCE(SUM(body_bytes_sent), 0) AS total_bytes, - COALESCE(AVG(request_time), 0) AS avg_request_time - FROM `{$rawTable}` - WHERE time_local >= :day_start AND time_local < :day_end - GROUP BY uri - ORDER BY pv DESC - {$urlLimit}"; - - $statUrlRows = (int) Db::execute($urlSql, [ - 'stat_date' => $statDate, - 'day_start' => $dayStart, - 'day_end' => $dayEnd, - ]); - - // 3. stat_referer：DELETE + INSERT topN（从 http_referer 提取 domain） NginxStatReferer::where('stat_date', $statDate)->delete(); - - $refererLimit = $topn > 0 ? 'LIMIT ' . $topn : ''; - $refererSql = "INSERT INTO `{$refererTable}` - (stat_date, referer_domain, pv, uv) - SELECT :stat_date, - CASE - WHEN http_referer IS NULL OR http_referer = '' OR http_referer = '-' THEN '-' - ELSE SUBSTRING_INDEX( - SUBSTRING_INDEX( - REPLACE(REPLACE(http_referer, 'https://', ''), 'http://', ''), - '/', 1 - ), - '?', 1 - ) - END AS referer_domain, - COUNT(*) AS pv, - COUNT(DISTINCT remote_addr) AS uv - FROM `{$rawTable}` - WHERE time_local >= :day_start AND time_local < :day_end - GROUP BY referer_domain - ORDER BY pv DESC - {$refererLimit}"; - - $statRefererRows = (int) Db::execute($refererSql, [ - 'stat_date' => $statDate, - 'day_start' => $dayStart, - 'day_end' => $dayEnd, - ]); - - // 4. stat_ua：DELETE + INSERT topN（从 http_user_agent 分类） NginxStatUa::where('stat_date', $statDate)->delete(); - $uaLimit = $topn > 0 ? 'LIMIT ' . $topn : ''; - $uaSql = "INSERT INTO `{$uaTable}` - (stat_date, ua_type, ua_name, pv, uv) - SELECT :stat_date, - CASE - WHEN http_user_agent LIKE '%bot%' - OR http_user_agent LIKE '%spider%' - OR http_user_agent LIKE '%crawl%' - OR http_user_agent LIKE '%slurp%' - OR http_user_agent LIKE '%bingpreview%' - OR http_user_agent LIKE '%facebookexternalhit%' - OR http_user_agent LIKE '%twitterbot%' THEN 'spider' - WHEN http_user_agent IS NULL OR http_user_agent = '' OR http_user_agent = '-' THEN 'unknown' - WHEN http_user_agent LIKE '%Mozilla%' - OR http_user_agent LIKE '%Chrome%' - OR http_user_agent LIKE '%Safari%' - OR http_user_agent LIKE '%Firefox%' - OR http_user_agent LIKE '%Edg%' - OR http_user_agent LIKE '%Opera%' - OR http_user_agent LIKE '%MSIE%' - OR http_user_agent LIKE '%Trident%' THEN 'browser' - ELSE 'unknown' - END AS ua_type, - CASE - WHEN http_user_agent LIKE '%Googlebot%' THEN 'Googlebot' - WHEN http_user_agent LIKE '%Baiduspider%' THEN 'Baiduspider' - WHEN http_user_agent LIKE '%bingbot%' THEN 'Bingbot' - WHEN http_user_agent LIKE '%DuckDuckBot%' THEN 'DuckDuckBot' - WHEN http_user_agent LIKE '%YandexBot%' THEN 'YandexBot' - WHEN http_user_agent LIKE '%Edg/%' THEN 'Microsoft Edge' - WHEN http_user_agent LIKE '%OPR/%' OR http_user_agent LIKE '%Opera%' THEN 'Opera' - WHEN http_user_agent LIKE '%Firefox/%' THEN 'Firefox' - WHEN http_user_agent LIKE '%Chrome/%' THEN 'Chrome' - WHEN http_user_agent LIKE '%Safari/%' THEN 'Safari' - WHEN http_user_agent LIKE '%MSIE%' OR http_user_agent LIKE '%Trident%' THEN 'Internet Explorer' - WHEN http_user_agent IS NULL OR http_user_agent = '' OR http_user_agent = '-' THEN 'Unknown' - ELSE 'Other' - END AS ua_name, - COUNT(*) AS pv, - COUNT(DISTINCT remote_addr) AS uv - FROM `{$rawTable}` - WHERE time_local >= :day_start AND time_local < :day_end - GROUP BY ua_type, ua_name - ORDER BY pv DESC - {$uaLimit}"; + $statHourRows = 0; + $statUrlRows = 0; + $statRefererRows = 0; + $statUaRows = 0; - $statUaRows = (int) Db::execute($uaSql, [ - 'stat_date' => $statDate, - 'day_start' => $dayStart, - 'day_end' => $dayEnd, - ]); + // 批次 A：全局行（node_id=''，跨所有节点聚合，UV 精确） + $global = $this->insertAggregatesForScope( + '', null, + $statDate, $statHour, $hourStart, $hourEnd, $dayStart, $dayEnd, $topn, + $rawTable, $hourTable, $urlTable, $refererTable, $uaTable + ); + $statHourRows += $global['stat_hour']; + $statUrlRows += $global['stat_url']; + $statRefererRows += $global['stat_referer']; + $statUaRows += $global['stat_ua']; + + // 批次 B：按节点行（循环每个 node_id） + // 用 day 窗口取节点列表：保证 day 窗口表（url/referer/ua）的 per-node 行不会因 + // 某节点"本小时无活动"而被 DELETE 后丢失；stat_hour 的 per-node 行由 + // HAVING COUNT(*)>0 自然过滤掉本小时无活动的节点。 + $nodeRows = Db::query( + "SELECT DISTINCT node_id FROM `{$rawTable}` WHERE time_local >= :day_start AND time_local < :day_end AND node_id != ''", + ['day_start' => $dayStart, 'day_end' => $dayEnd] + ); + foreach ($nodeRows as $nodeRow) { + $nid = (string) $nodeRow['node_id']; + $perNode = $this->insertAggregatesForScope( + $nid, $nid, + $statDate, $statHour, $hourStart, $hourEnd, $dayStart, $dayEnd, $topn, + $rawTable, $hourTable, $urlTable, $refererTable, $uaTable + ); + $statHourRows += $perNode['stat_hour']; + $statUrlRows += $perNode['stat_url']; + $statRefererRows += $perNode['stat_referer']; + $statUaRows += $perNode['stat_ua']; + } Db::commit(); @@ -216,6 +124,175 @@ class NginxLogAggregatorServiceBase } } + /** + * 为单个作用域（全局或某节点）执行 4 段 INSERT. + * + * - 全局作用域：$nodeId='' 且 $nodeFilter=null（不追加 node WHERE，跨所有节点聚合） + * - 节点作用域：$nodeId=节点ID 且 $nodeFilter=节点ID（追加 AND node_id=:node_filter） + * + * stat_hour：1 行（无 GROUP BY，HAVING COUNT(*)>0） + * stat_url / stat_referer / stat_ua：topN（各自 GROUP BY + LIMIT） + * + * @param string $nodeId INSERT 的 node_id 列值（全局为 ''） + * @param string|null $nodeFilter null=全局（不追加 WHERE）；非 null=追加节点过滤 + * @return array {stat_hour:int, stat_url:int, stat_referer:int, stat_ua:int} + */ + protected function insertAggregatesForScope( + string $nodeId, + ?string $nodeFilter, + int $statDate, + int $statHour, + int $hourStart, + int $hourEnd, + int $dayStart, + int $dayEnd, + int $topn, + string $rawTable, + string $hourTable, + string $urlTable, + string $refererTable, + string $uaTable + ): array { + // 节点过滤片段：全局行无；节点行追加 AND node_id = :node_filter + $nodeWhere = $nodeFilter === null ? '' : 'AND node_id = :node_filter'; + + // hour 窗口绑定（stat_hour 专用） + $hourParams = [ + 'node_id' => $nodeId, + 'stat_date' => $statDate, + 'stat_hour' => $statHour, + 'hour_start' => $hourStart, + 'hour_end' => $hourEnd, + ]; + if ($nodeFilter !== null) { + $hourParams['node_filter'] = $nodeFilter; + } + + // day 窗口绑定（stat_url/referer/ua 共用） + $dayParams = [ + 'node_id' => $nodeId, + 'stat_date' => $statDate, + 'day_start' => $dayStart, + 'day_end' => $dayEnd, + ]; + if ($nodeFilter !== null) { + $dayParams['node_filter'] = $nodeFilter; + } + + // 1. stat_hour：1 行（无 GROUP BY） + $hourSql = "INSERT INTO `{$hourTable}` + (node_id, stat_date, stat_hour, pv, uv, total_bytes, status_2xx, status_3xx, status_4xx, status_5xx, status_other, avg_request_time) + SELECT :node_id, :stat_date, :stat_hour, + COUNT(*) AS pv, + COUNT(DISTINCT remote_addr) AS uv, + COALESCE(SUM(body_bytes_sent), 0) AS total_bytes, + COALESCE(SUM(CASE WHEN status BETWEEN 200 AND 299 THEN 1 ELSE 0 END), 0) AS status_2xx, + COALESCE(SUM(CASE WHEN status BETWEEN 300 AND 399 THEN 1 ELSE 0 END), 0) AS status_3xx, + COALESCE(SUM(CASE WHEN status BETWEEN 400 AND 499 THEN 1 ELSE 0 END), 0) AS status_4xx, + COALESCE(SUM(CASE WHEN status BETWEEN 500 AND 599 THEN 1 ELSE 0 END), 0) AS status_5xx, + COALESCE(SUM(CASE WHEN status < 200 OR status > 599 THEN 1 ELSE 0 END), 0) AS status_other, + COALESCE(AVG(request_time), 0) AS avg_request_time + FROM `{$rawTable}` + WHERE time_local >= :hour_start AND time_local < :hour_end {$nodeWhere} + HAVING COUNT(*) > 0"; + $statHourRows = (int) Db::execute($hourSql, $hourParams); + + // 2. stat_url：topN（GROUP BY uri） + $urlLimit = $topn > 0 ? 'LIMIT ' . $topn : ''; + $urlSql = "INSERT INTO `{$urlTable}` + (node_id, stat_date, uri, pv, uv, total_bytes, avg_request_time) + SELECT :node_id, :stat_date, uri, + COUNT(*) AS pv, + COUNT(DISTINCT remote_addr) AS uv, + COALESCE(SUM(body_bytes_sent), 0) AS total_bytes, + COALESCE(AVG(request_time), 0) AS avg_request_time + FROM `{$rawTable}` + WHERE time_local >= :day_start AND time_local < :day_end {$nodeWhere} + GROUP BY uri + ORDER BY pv DESC + {$urlLimit}"; + $statUrlRows = (int) Db::execute($urlSql, $dayParams); + + // 3. stat_referer：topN（从 http_referer 提取 domain） + $refererLimit = $topn > 0 ? 'LIMIT ' . $topn : ''; + $refererSql = "INSERT INTO `{$refererTable}` + (node_id, stat_date, referer_domain, pv, uv) + SELECT :node_id, :stat_date, + CASE + WHEN http_referer IS NULL OR http_referer = '' OR http_referer = '-' THEN '-' + ELSE SUBSTRING_INDEX( + SUBSTRING_INDEX( + REPLACE(REPLACE(http_referer, 'https://', ''), 'http://', ''), + '/', 1 + ), + '?', 1 + ) + END AS referer_domain, + COUNT(*) AS pv, + COUNT(DISTINCT remote_addr) AS uv + FROM `{$rawTable}` + WHERE time_local >= :day_start AND time_local < :day_end {$nodeWhere} + GROUP BY referer_domain + ORDER BY pv DESC + {$refererLimit}"; + $statRefererRows = (int) Db::execute($refererSql, $dayParams); + + // 4. stat_ua：topN（从 http_user_agent 分类） + $uaLimit = $topn > 0 ? 'LIMIT ' . $topn : ''; + $uaSql = "INSERT INTO `{$uaTable}` + (node_id, stat_date, ua_type, ua_name, pv, uv) + SELECT :node_id, :stat_date, + CASE + WHEN http_user_agent LIKE '%bot%' + OR http_user_agent LIKE '%spider%' + OR http_user_agent LIKE '%crawl%' + OR http_user_agent LIKE '%slurp%' + OR http_user_agent LIKE '%bingpreview%' + OR http_user_agent LIKE '%facebookexternalhit%' + OR http_user_agent LIKE '%twitterbot%' THEN 'spider' + WHEN http_user_agent IS NULL OR http_user_agent = '' OR http_user_agent = '-' THEN 'unknown' + WHEN http_user_agent LIKE '%Mozilla%' + OR http_user_agent LIKE '%Chrome%' + OR http_user_agent LIKE '%Safari%' + OR http_user_agent LIKE '%Firefox%' + OR http_user_agent LIKE '%Edg%' + OR http_user_agent LIKE '%Opera%' + OR http_user_agent LIKE '%MSIE%' + OR http_user_agent LIKE '%Trident%' THEN 'browser' + ELSE 'unknown' + END AS ua_type, + CASE + WHEN http_user_agent LIKE '%Googlebot%' THEN 'Googlebot' + WHEN http_user_agent LIKE '%Baiduspider%' THEN 'Baiduspider' + WHEN http_user_agent LIKE '%bingbot%' THEN 'Bingbot' + WHEN http_user_agent LIKE '%DuckDuckBot%' THEN 'DuckDuckBot' + WHEN http_user_agent LIKE '%YandexBot%' THEN 'YandexBot' + WHEN http_user_agent LIKE '%Edg/%' THEN 'Microsoft Edge' + WHEN http_user_agent LIKE '%OPR/%' OR http_user_agent LIKE '%Opera%' THEN 'Opera' + WHEN http_user_agent LIKE '%Firefox/%' THEN 'Firefox' + WHEN http_user_agent LIKE '%Chrome/%' THEN 'Chrome' + WHEN http_user_agent LIKE '%Safari/%' THEN 'Safari' + WHEN http_user_agent LIKE '%MSIE%' OR http_user_agent LIKE '%Trident%' THEN 'Internet Explorer' + WHEN http_user_agent IS NULL OR http_user_agent = '' OR http_user_agent = '-' THEN 'Unknown' + ELSE 'Other' + END AS ua_name, + COUNT(*) AS pv, + COUNT(DISTINCT remote_addr) AS uv + FROM `{$rawTable}` + WHERE time_local >= :day_start AND time_local < :day_end {$nodeWhere} + GROUP BY ua_type, ua_name + ORDER BY pv DESC + {$uaLimit}"; + $statUaRows = (int) Db::execute($uaSql, $dayParams); + + return [ + 'stat_hour' => $statHourRows, + 'stat_url' => $statUrlRows, + 'stat_referer' => $statRefererRows, + 'stat_ua' => $statUaRows, + ]; + } + /** * 将 int 日期 YYYYMMDD 格式化为 "YYYY-MM-DD". */ diff --git a/extend/base/common/service/NginxLogReaderServiceBase.php b/extend/base/common/service/NginxLogReaderServiceBase.php index aa76970..76a3343 100644 --- a/extend/base/common/service/NginxLogReaderServiceBase.php +++ b/extend/base/common/service/NginxLogReaderServiceBase.php @@ -3,6 +3,7 @@ namespace base\common\service; use app\admin\model\NginxLogPosition; +use app\common\service\HostService; /** * Nginx 日志增量读取 service（Base 层）. @@ -10,12 +11,17 @@ use app\admin\model\NginxLogPosition; * 职责： * - 按文件 offset 增量读取日志行（Generator，不读整个文件到内存） * - 检测日志轮转（inode 变化 OR filesize < offset，覆盖 rename+rebuild 与 copytruncate 两种模式） - * - 将读取进度持久化到 NginxLogPosition 表（upsert） + * - 将读取进度持久化到 NginxLogPosition 表（upsert，按 node_id 隔离） + * + * 多节点适配：position 表按 (node_id, file_path) 唯一定位读取进度。 + * 旧数据（node_id 为空）通过懒加载接管（见 getLastPosition）：首个查询到它的本节点原子 UPDATE node_id， + * 避免 migration 全表回填（多节点共享 DB 时 migration 只跑一次，只有懒加载能确保每节点各自接管）。 * * 不负责：解析（T5 Parser）、聚合（T7 Aggregator）。 * * 依赖倒置：内部 model 调用走 `app\admin\model\NginxLogPosition`（App 入口类）， * 使用者可在 app/ 重写该 model 拦截行为；本类不直接 `new NginxLogPosition`（静态调用满足多态）。 + * 节点身份走 `app\common\service\HostService`（App 入口类）。 */ class NginxLogReaderServiceBase { @@ -159,15 +165,27 @@ class NginxLogReaderServiceBase /** * 查询文件的上次读取位置. * + * 多节点适配：where 条件带 node_id 隔离各节点进度。 + * 懒加载接管：多节点共享 DB 时 migration 只跑一次，旧记录 node_id 为空， + * 首个查询到它的本节点原子 UPDATE node_id 接管，保证每节点各自有独立 position 记录。 + * * @param string $filePath 日志文件绝对路径 * * @return array|null 命中时返回 [inode, offset, last_line_hash, parse_fail_count, parse_fail_samples]，无记录返回 null */ public function getLastPosition(string $filePath): ?array { - $row = NginxLogPosition::where('file_path', $filePath)->find(); + $nodeId = HostService::getNodeId(); + + $row = NginxLogPosition::where('file_path', $filePath) + ->where('node_id', $nodeId) + ->find(); if ($row === null) { - return null; + // 懒加载接管：旧记录 node_id 为空，原子抢占接管到当前节点 + $row = $this->lazyAdoptPosition($filePath, $nodeId); + if ($row === null) { + return null; + } } return [ @@ -179,6 +197,39 @@ class NginxLogReaderServiceBase ]; } + /** + * 懒加载接管：把旧的无 node_id 记录原子接管到当前节点. + * + * 多节点共享 DB 时 migration 只跑一次，无法为每节点预置 position 记录。 + * 旧记录（node_id=''）由首个查询到它的本节点通过原子条件 UPDATE 接管： + * UPDATE ... SET node_id=current WHERE node_id='' AND file_path=X + * 并发安全：WHERE node_id='' 保证只有一个节点能 affected=1，其他节点查询时已被接管。 + * + * @return \think\Model|null 接管成功返回接管后的记录，无旧记录或被其他节点抢占返回 null + */ + protected function lazyAdoptPosition(string $filePath, string $nodeId) + { + $legacy = NginxLogPosition::where('file_path', $filePath) + ->where('node_id', '') + ->find(); + if ($legacy === null) { + return null; + } + + // 原子条件 UPDATE：只有 node_id='' 时才能被接管（防并发多节点同时接管同一记录） + $affected = NginxLogPosition::where('id', $legacy->getData('id')) + ->where('node_id', '') + ->update(['node_id' => $nodeId]); + if ($affected === 0) { + // 被其他节点抢先接管，本节点放弃（下次 savePosition 会创建新记录） + return null; + } + + return NginxLogPosition::where('file_path', $filePath) + ->where('node_id', $nodeId) + ->find(); + } + /** * 保存读取位置（upsert）. * @@ -196,8 +247,11 @@ class NginxLogReaderServiceBase public function savePosition(string $filePath, int $inode, int $offset, ?string $lastLineHash, int $failCount = 0, ?string $failSamples = null): void { $now = time(); + $nodeId = HostService::getNodeId(); - $existing = NginxLogPosition::where('file_path', $filePath)->find(); + $existing = NginxLogPosition::where('file_path', $filePath) + ->where('node_id', $nodeId) + ->find(); if ($existing !== null) { $existing->save([ 'inode' => $inode, @@ -212,6 +266,7 @@ class NginxLogReaderServiceBase } NginxLogPosition::create([ + 'node_id' => $nodeId, 'file_path' => $filePath, 'inode' => $inode, 'offset' => $offset, diff --git a/extend/base/tools/controller/timer/NginxLogImportBase.php b/extend/base/tools/controller/timer/NginxLogImportBase.php index 1ad764e..d6a4c47 100644 --- a/extend/base/tools/controller/timer/NginxLogImportBase.php +++ b/extend/base/tools/controller/timer/NginxLogImportBase.php @@ -191,6 +191,7 @@ class NginxLogImportBase extends TimerController { $now = time(); return [ + 'node_id' => \app\common\service\HostService::getNodeId(), 'file_path' => $filePath, 'remote_addr' => $parsed['remote_addr'], 'remote_user' => $parsed['remote_user'] ?? '',