feat(nginx-log): 多节点适配(node_id 全链路 + 全局/按节点聚合 + 节点筛选 + v2.4.0 升级脚本)

- 6 张表加 node_id 字段 + position/stat 改复合唯一键(migration)
- 6 个 Scheme 同步 node_id 注解 + 唯一键
- Reader 带 node_id 参数 + 懒加载接管(getLastPosition 回退查 node_id='')
- Aggregator 全局行(node_id='')+ 按节点循环 insertAggregatesForScope
- Stat 所有查询方法加 where node_id 条件
- Dashboard 控制器/视图加节点筛选下拉框 + AJAX 带 node_id
- AccessLog 列表加节点列 + 采集进度卡片显示节点信息
- import 定时任务 run_type 改 all(每节点采集自己的日志)
- v2.4.0 升级脚本追加 ALTER TABLE(幂等 check-then-alter)
- 菜单调整:去掉 Nginx 顶级菜单 + 读取位置管理,改为系统管理下挂两个子菜单
- ulthon-timer 文档补充 run_type=all 多节点部署注意事项
This commit is contained in:
augushong
2026-08-08 10:33:20 +08:00
parent f1a2e8fd5d
commit 8faa52ce90
21 changed files with 578 additions and 206 deletions

View File

@@ -201,6 +201,8 @@ nohup php /var/www/html/think timer --local --quiet &
多节点部署:每个容器各自启动 timer 进程,`runtime/node_id.lock` 在各自容器内独立生成(天然隔离),连同一个数据库即自动组成多节点集群。
> **多节点部署注意run_type=all 的任务)**`run_type=all` 的定时任务(如 `nginx_log_import`)要求每节点各自采集本机数据,每节点必须用 `php think timer --local` 启动,确保 site 请求发往本机 `localhost` 而非共享 `site_domain`/LB否则请求会落在随机节点上导致采集错位节点 A 的日志被节点 B 处理)。框架内置 Docker 部署栈已默认带 `--local`,手动/裸机部署时务必显式加上。
### 本地调试(指定请求 Host
site 任务会按站点域名发起请求,默认从 `sysconfig('site','site_domain')` 读取。

View File

@@ -9,6 +9,7 @@ use app\common\scheme\attribute\Component;
use app\common\scheme\attribute\Index;
#[Table(name: 'ul_nginx_access_log', comment: 'nginx 访问日志明细(解析入库)')]
#[Index(columns: ['node_id', 'time_local'], name: 'idx_node_time', type: 'NORMAL')]
#[Index(columns: ['time_local'], name: 'idx_time_local', type: 'NORMAL')]
#[Index(columns: ['remote_addr'], name: 'idx_remote_addr', type: 'NORMAL')]
#[Index(columns: ['status'], name: 'idx_status', type: 'NORMAL')]
@@ -19,6 +20,9 @@ class NginxAccessLog extends BaseScheme
#[Field(type: 'bigint', nullable: false, unsigned: true, autoIncrement: true, primary: true, comment: '主键ID')]
public $id;
#[Field(type: 'varchar', length: 64, nullable: false, default: '', comment: '节点ID区分不同采集节点')]
public $node_id;
#[Field(type: 'varchar', length: 500, nullable: false, comment: '来源日志文件路径')]
#[Component(type: 'text', options: [])]
public $file_path;

View File

@@ -9,12 +9,15 @@ use app\common\scheme\attribute\Component;
use app\common\scheme\attribute\Index;
#[Table(name: 'ul_nginx_log_position', comment: 'nginx 日志文件读取位置(增量读取状态)')]
#[Index(columns: ['file_path'], name: 'uniq_file_path', type: 'UNIQUE')]
#[Index(columns: ['node_id', 'file_path'], name: 'uniq_node_file', type: 'UNIQUE')]
class NginxLogPosition extends BaseScheme
{
#[Field(type: 'int', length: 11, nullable: false, unsigned: true, autoIncrement: true, primary: true)]
public $id;
#[Field(type: 'varchar', length: 64, nullable: false, default: '', comment: '节点ID区分不同采集节点')]
public $node_id;
#[Field(type: 'varchar', length: 500, nullable: false, comment: '日志文件绝对路径')]
#[Component(type: 'text', options: [])]
public $file_path;

View File

@@ -8,13 +8,16 @@ use app\common\scheme\attribute\Field;
use app\common\scheme\attribute\Index;
#[Table(name: 'nginx_stat_hour', comment: 'nginx 按小时聚合统计')]
#[Index(columns: ['stat_date', 'stat_hour'], name: 'uniq_date_hour', type: 'UNIQUE')]
#[Index(columns: ['node_id', 'stat_date', 'stat_hour'], name: 'uniq_node_date_hour', type: 'UNIQUE')]
#[Index(columns: ['stat_date'], name: 'idx_stat_date', type: 'NORMAL')]
class NginxStatHour extends BaseScheme
{
#[Field(type: 'int', length: 11, nullable: false, unsigned: true, autoIncrement: true, primary: true)]
public $id;
#[Field(type: 'varchar', length: 64, nullable: false, default: '', comment: '节点ID区分不同采集节点')]
public $node_id;
#[Field(type: 'int', length: 11, nullable: false, unsigned: true, comment: '统计日期(YYYYMMDD)')]
public $stat_date;

View File

@@ -8,13 +8,16 @@ use app\common\scheme\attribute\Field;
use app\common\scheme\attribute\Index;
#[Table(name: 'nginx_stat_referer', comment: 'nginx 按 Referer 来源域名聚合统计')]
#[Index(columns: ['stat_date', 'referer_domain'], name: 'uniq_date_referer', type: 'UNIQUE')]
#[Index(columns: ['node_id', 'stat_date', 'referer_domain'], name: 'uniq_node_date_referer', type: 'UNIQUE')]
#[Index(columns: ['stat_date'], name: 'idx_stat_date', type: 'NORMAL')]
class NginxStatReferer extends BaseScheme
{
#[Field(type: 'int', length: 11, nullable: false, unsigned: true, autoIncrement: true, primary: true)]
public $id;
#[Field(type: 'varchar', length: 64, nullable: false, default: '', comment: '节点ID区分不同采集节点')]
public $node_id;
#[Field(type: 'int', length: 11, nullable: false, unsigned: true, comment: '统计日期(YYYYMMDD)')]
public $stat_date;

View File

@@ -8,13 +8,16 @@ use app\common\scheme\attribute\Field;
use app\common\scheme\attribute\Index;
#[Table(name: 'nginx_stat_ua', comment: 'nginx 按 User-Agent 聚合统计')]
#[Index(columns: ['stat_date', 'ua_type', 'ua_name'], name: 'uniq_date_type_name', type: 'UNIQUE')]
#[Index(columns: ['node_id', 'stat_date', 'ua_type', 'ua_name'], name: 'uniq_node_date_type_name', type: 'UNIQUE')]
#[Index(columns: ['stat_date'], name: 'idx_stat_date', type: 'NORMAL')]
class NginxStatUa extends BaseScheme
{
#[Field(type: 'int', length: 11, nullable: false, unsigned: true, autoIncrement: true, primary: true)]
public $id;
#[Field(type: 'varchar', length: 64, nullable: false, default: '', comment: '节点ID区分不同采集节点')]
public $node_id;
#[Field(type: 'int', length: 11, nullable: false, unsigned: true, comment: '统计日期(YYYYMMDD)')]
public $stat_date;

View File

@@ -8,13 +8,16 @@ use app\common\scheme\attribute\Field;
use app\common\scheme\attribute\Index;
#[Table(name: 'nginx_stat_url', comment: 'nginx 按 URL 聚合统计')]
#[Index(columns: ['stat_date', 'uri'], name: 'uniq_date_uri', type: 'UNIQUE')]
#[Index(columns: ['node_id', 'stat_date', 'uri'], name: 'uniq_node_date_uri', type: 'UNIQUE')]
#[Index(columns: ['stat_date'], name: 'idx_stat_date', type: 'NORMAL')]
class NginxStatUrl extends BaseScheme
{
#[Field(type: 'int', length: 11, nullable: false, unsigned: true, autoIncrement: true, primary: true)]
public $id;
#[Field(type: 'varchar', length: 64, nullable: false, default: '', comment: '节点ID区分不同采集节点')]
public $node_id;
#[Field(type: 'int', length: 11, nullable: false, unsigned: true, comment: '统计日期(YYYYMMDD)')]
public $stat_date;

View File

@@ -1,5 +1,47 @@
<div class="layuimini-container">
<div class="layuimini-main">
{if condition="$positions->isEmpty()"}
<div class="layui-card" style="margin-bottom: 10px;">
<div class="layui-card-body" style="padding: 12px 15px;">
<i class="layui-icon layui-icon-tips" style="color: #FFB800; font-size: 16px;"></i>
<span style="color: #666;">暂无采集位置记录,请确认 nginx_log.enable 已开启且 timer 正在运行。</span>
</div>
</div>
{else/}
{volist name="positions" id="pos"}
<div class="layui-card" style="margin-bottom: 10px;">
<div class="layui-card-body" style="padding: 12px 15px;">
<div style="display: flex; flex-wrap: wrap; gap: 24px; align-items: center; font-size: 13px;">
<div>
<span style="color: #999;">监控文件</span><br>
<code style="font-size: 12px;">{$pos.file_path}</code>
</div>
<div>
<span style="color: #999;">节点</span><br>
<strong>{$pos.node_id}</strong>
</div>
<div>
<span style="color: #999;">已读取</span><br>
<strong>{$pos.offset_human}</strong>
</div>
<div>
<span style="color: #999;">最后采集</span><br>
<span>{$pos.last_read_time_text}</span>
</div>
<div>
<span style="color: #999;">解析失败</span><br>
{if condition="$pos.parse_fail_count > 0"}
<span style="color: #FF5722; font-weight: bold;">{$pos.parse_fail_count} </span>
{else/}
<span style="color: #5FB878;">0 </span>
{/if}
</div>
</div>
</div>
</div>
{/volist}
{/if}
<table id="currentTable" class="layui-table layui-hide"
data-auth-index="{:auth('nginx.access_log/index')}"
data-auth-read="{:auth('nginx.access_log/read')}"

View File

@@ -17,6 +17,7 @@ $(function(){
else if (s >= 500) cls = 'layui-badge layui-bg-red';
return '<span class="' + cls + '">' + d.status + '</span>';
}},
{field: 'node_id', title: '节点', width: 120, searchOp: '='},
{field: 'remote_addr', title: '客户端IP', width: 140, searchOp: '%*%'},
{field: 'request_time', title: '耗时(秒)', width: 100, search: 'number_limit', templet: function(d) {
var t = parseFloat(d.request_time);

View File

@@ -4,7 +4,7 @@
<div class="layui-card-header detail-header">
<div class="layui-row">
<div class="layui-col-md9">
<h2 class="detail-title">#{$row.id} {$title}</h2>
<h2 class="detail-title">#{$row.id} {$title|default=''}</h2>
<div class="detail-id">ID: {$row.id}</div>
</div>
<div class="layui-col-md3 text-right detail-actions">

View File

@@ -0,0 +1,70 @@
<?php
use think\migration\Migrator;
/**
* nginx 日志表新增 node_id 字段(多节点采集支持)
*
* 变更内容:
* - 6 张表全部新增 node_id varchar(64) default ''(采集节点 ID空=全局汇总)
* - position / 4 张 stat 表的唯一键前置 node_id多节点隔离避免聚合 INSERT 触发 duplicate key
* - raw 表nginx_access_log加 node_id + time_local 普通索引,无唯一键变更
*
* 注意:
* - 不回填历史 raw/stat 数据(旧数据 node_id 为空,视为全局汇总)
* - 不在 migration 中 UPDATE position 的 node_id多节点共享 DB 下第一个节点会抢走所有 position
* position 迁移改为 getLastPosition() 懒加载接管,见配套 Task
* - phinx 表名不带 ul_ 前缀phinx 自动加前缀)
*/
class NginxLogAddNodeId extends Migrator
{
public function change()
{
// ==================== 1. nginx_log_position ====================
// 旧唯一键 uniq_file_path(file_path) → 新 uniq_node_file(node_id, file_path)
$this->table('nginx_log_position')
->addColumn('node_id', 'string', ['limit' => 64, 'default' => '', 'comment' => '采集节点 ID空=全局汇总)'])
->removeIndexByName('uniq_file_path')
->addIndex(['node_id', 'file_path'], ['unique' => true, 'name' => 'uniq_node_file'])
->update();
// ==================== 2. nginx_access_lograw 表)====================
// 仅加字段 + 普通索引(按节点 + 时间查询),无唯一键变更
$this->table('nginx_access_log')
->addColumn('node_id', 'string', ['limit' => 64, 'default' => '', 'comment' => '采集节点 ID空=全局汇总)'])
->addIndex(['node_id', 'time_local'], ['name' => 'idx_node_time'])
->update();
// ==================== 3. nginx_stat_hour ====================
// 旧唯一键 uniq_date_hour(stat_date, stat_hour) → uniq_node_date_hour(node_id, stat_date, stat_hour)
$this->table('nginx_stat_hour')
->addColumn('node_id', 'string', ['limit' => 64, 'default' => '', 'comment' => '采集节点 ID空=全局汇总)'])
->removeIndexByName('uniq_date_hour')
->addIndex(['node_id', 'stat_date', 'stat_hour'], ['unique' => true, 'name' => 'uniq_node_date_hour'])
->update();
// ==================== 4. nginx_stat_url ====================
// 旧唯一键 uniq_date_uri(stat_date, uri) → uniq_node_date_uri(node_id, stat_date, uri)
$this->table('nginx_stat_url')
->addColumn('node_id', 'string', ['limit' => 64, 'default' => '', 'comment' => '采集节点 ID空=全局汇总)'])
->removeIndexByName('uniq_date_uri')
->addIndex(['node_id', 'stat_date', 'uri'], ['unique' => true, 'name' => 'uniq_node_date_uri'])
->update();
// ==================== 5. nginx_stat_referer ====================
// 旧唯一键 uniq_date_referer(stat_date, referer_domain) → uniq_node_date_referer(node_id, stat_date, referer_domain)
$this->table('nginx_stat_referer')
->addColumn('node_id', 'string', ['limit' => 64, 'default' => '', 'comment' => '采集节点 ID空=全局汇总)'])
->removeIndexByName('uniq_date_referer')
->addIndex(['node_id', 'stat_date', 'referer_domain'], ['unique' => true, 'name' => 'uniq_node_date_referer'])
->update();
// ==================== 6. nginx_stat_ua ====================
// 旧唯一键 uniq_date_type_name(stat_date, ua_type, ua_name) → uniq_node_date_type_name(node_id, stat_date, ua_type, ua_name)
$this->table('nginx_stat_ua')
->addColumn('node_id', 'string', ['limit' => 64, 'default' => '', 'comment' => '采集节点 ID空=全局汇总)'])
->removeIndexByName('uniq_date_type_name')
->addIndex(['node_id', 'stat_date', 'ua_type', 'ua_name'], ['unique' => true, 'name' => 'uniq_node_date_type_name'])
->update();
}
}

View File

@@ -56,6 +56,24 @@ class AccessLogBase extends AdminController
return json($data);
}
// 采集位置信息(合并自原"读取位置管理"独立页面)
$positions = \app\admin\model\NginxLogPosition::order('id', 'desc')->select()->map(function ($item) {
$bytes = (int) $item['offset'];
if ($bytes >= 1048576) {
$item['offset_human'] = number_format($bytes / 1048576, 2) . ' MB';
} elseif ($bytes >= 1024) {
$item['offset_human'] = number_format($bytes / 1024, 2) . ' KB';
} else {
$item['offset_human'] = $bytes . ' B';
}
$item['last_read_time_text'] = $item['last_read_time'] > 0
? date('Y-m-d H:i:s', (int) $item['last_read_time'])
: '-';
return $item;
});
$this->assign('positions', $positions);
return $this->fetch();
}

View File

@@ -45,11 +45,13 @@ class NginxLogStatBase extends AdminController
$period = 'day';
}
$nodeId = (string) $request->param('node_id', '');
// 解析时间范围YYYYMMDD int 表示)
[$startDate, $endDate] = $this->resolveDateRange($period);
if ($request->isAjax()) {
$data = $this->buildStatData($period, $startDate, $endDate);
$data = $this->buildStatData($period, $startDate, $endDate, $nodeId);
return json([
'code' => 0,
@@ -63,6 +65,11 @@ class NginxLogStatBase extends AdminController
$this->assign('start', $startDate);
$this->assign('end', $endDate);
// 在线节点列表(供视图渲染节点筛选下拉)
$nodes = \app\admin\model\SystemHost::where('status', 1)->order('node_id', 'asc')->select();
$this->assign('nodes', $nodes);
$this->assign('node_id', $nodeId);
return $this->fetch();
}
@@ -140,20 +147,20 @@ class NginxLogStatBase extends AdminController
/**
* 组装完整统计数据
*/
protected function buildStatData(string $period, int $startDate, int $endDate): array
protected function buildStatData(string $period, int $startDate, int $endDate, string $nodeId = ''): array
{
// 时间范围不合法 → 空数据(不崩)
if ($startDate <= 0 || $endDate <= 0 || $startDate > $endDate) {
return $this->emptyData($period);
}
$summary = $this->querySummary($startDate, $endDate);
$trend = $this->queryTrend($period, $startDate, $endDate);
$summary = $this->querySummary($startDate, $endDate, $nodeId);
$trend = $this->queryTrend($period, $startDate, $endDate, $nodeId);
$topN = $this->getTopN();
$topUrls = $this->queryTopByDateRange(NginxStatUrl::class, 'uri', $startDate, $endDate, $topN);
$topReferers = $this->queryTopByDateRange(NginxStatReferer::class, 'referer_domain', $startDate, $endDate, $topN);
$topUas = $this->queryTopByDateRange(NginxStatUa::class, 'ua_name', $startDate, $endDate, $topN);
$topUrls = $this->queryTopByDateRange(NginxStatUrl::class, 'uri', $startDate, $endDate, $topN, $nodeId);
$topReferers = $this->queryTopByDateRange(NginxStatReferer::class, 'referer_domain', $startDate, $endDate, $topN, $nodeId);
$topUas = $this->queryTopByDateRange(NginxStatUa::class, 'ua_name', $startDate, $endDate, $topN, $nodeId);
// 跨周期 UV 估算标注:仅 day 为精确值week/month/year/custom 均为按日累加的估算值
$uvIsEstimate = $period !== 'day';
@@ -208,7 +215,7 @@ class NginxLogStatBase extends AdminController
* 整体指标SUM stat_hour
* avg_request_time 按 PV 加权平均(避免简单 AVG 被低 PV 时段拉偏)
*/
protected function querySummary(int $startDate, int $endDate): array
protected function querySummary(int $startDate, int $endDate, string $nodeId = ''): array
{
$sumFields = [
'IFNULL(SUM(pv),0) AS pv',
@@ -223,6 +230,7 @@ class NginxLogStatBase extends AdminController
];
$row = NginxStatHour::where('stat_date', 'between', [$startDate, $endDate])
->where('node_id', $nodeId)
->field(implode(',', $sumFields))
->find();
@@ -252,10 +260,11 @@ class NginxLogStatBase extends AdminController
* - year → 12 个月(按 stat_date DIV 100 分组,补全 1~12 月)
* - custom→ 按 stat_date 分组,补全 start~end 日期序列
*/
protected function queryTrend(string $period, int $startDate, int $endDate): array
protected function queryTrend(string $period, int $startDate, int $endDate, string $nodeId = ''): array
{
if ($period === 'day') {
$rows = NginxStatHour::where('stat_date', 'between', [$startDate, $endDate])
->where('node_id', $nodeId)
->field('stat_hour, IFNULL(SUM(pv),0) AS pv, IFNULL(SUM(uv),0) AS uv')
->group('stat_hour')
->select()
@@ -280,6 +289,7 @@ class NginxLogStatBase extends AdminController
if ($period === 'year') {
$rows = NginxStatHour::where('stat_date', 'between', [$startDate, $endDate])
->where('node_id', $nodeId)
->field('(stat_date DIV 100) AS ym, IFNULL(SUM(pv),0) AS pv, IFNULL(SUM(uv),0) AS uv')
->group('ym')
->order('ym asc')
@@ -307,6 +317,7 @@ class NginxLogStatBase extends AdminController
// week / month / custom → 按 stat_date 分组,补全日期序列
$rows = NginxStatHour::where('stat_date', 'between', [$startDate, $endDate])
->where('node_id', $nodeId)
->field('stat_date, IFNULL(SUM(pv),0) AS pv, IFNULL(SUM(uv),0) AS uv')
->group('stat_date')
->order('stat_date asc')
@@ -340,10 +351,12 @@ class NginxLogStatBase extends AdminController
* @param string $modelClass 模型类(依赖倒置:调 app/admin/model 入口)
* @param string $labelField 分组与展示的字段名
* @param int $topN 取前 N 条0 不限制
* @param string $nodeId 节点 ID空字符串=全局聚合行)
*/
protected function queryTopByDateRange(string $modelClass, string $labelField, int $startDate, int $endDate, int $topN): array
protected function queryTopByDateRange(string $modelClass, string $labelField, int $startDate, int $endDate, int $topN, string $nodeId = ''): array
{
$query = $modelClass::where('stat_date', 'between', [$startDate, $endDate])
->where('node_id', $nodeId)
->field($labelField . ' AS label, IFNULL(SUM(pv),0) AS pv, IFNULL(SUM(uv),0) AS uv')
->group($labelField)
->order('pv desc');

View File

@@ -209,20 +209,9 @@ $ul_system_menu = array(
"sort" => 0,
"status" => 1,
),
array(
"id" => 300,
"pid" => 0,
"title" => "Nginx 日志分析",
"icon" => "fa fa-chart-bar",
"href" => "",
"params" => "",
"target" => "_self",
"sort" => 0,
"status" => 1,
),
array(
"id" => 301,
"pid" => 300,
"pid" => 228,
"title" => "访问统计仪表盘",
"icon" => "fa fa-line-chart",
"href" => "system.nginx_log_stat/index",
@@ -233,7 +222,7 @@ $ul_system_menu = array(
),
array(
"id" => 302,
"pid" => 300,
"pid" => 228,
"title" => "原始访问日志",
"icon" => "fa fa-file-text",
"href" => "nginx.access_log/index",
@@ -242,17 +231,6 @@ $ul_system_menu = array(
"sort" => 0,
"status" => 1,
),
array(
"id" => 303,
"pid" => 300,
"title" => "读取位置管理",
"icon" => "fa fa-map-marker",
"href" => "nginx.log_position/index",
"params" => "",
"target" => "_self",
"sort" => 0,
"status" => 1,
),
);

View File

@@ -13,11 +13,12 @@
* stat_retention_days 统计聚合保留天数0=永久)
* topn_default Top N 默认值
* exclude_static 是否排除静态资源访问
* - system_menu 新增 4 条菜单(顶级分组 300 + 3 个子菜单 301/302/303
* 300 Nginx 日志分析顶级分组href 为空)
* - system_menu 新增 2 条菜单(挂到系统管理 pid=228 下
* 301 访问统计仪表盘 system.nginx_log_stat/index
* 302 原始访问日志 nginx.access_log/index
* 303 读取位置管理 nginx.log_position/index
* - ALTER TABLE6 张 nginx_log 表新增 node_id 字段 + 唯一键前置 node_id多节点采集支持
* 与 migration 20260801125222_nginx_log_add_node_id.php 保持一致,
* 存量库通过 admin:update 幂等执行check-then-alter
*
* 设计说明:
* - 幂等 check-then-insert先查询存在性不存在才插入
@@ -82,23 +83,12 @@ class UpdateFunction
/**
* 待新增菜单.
* 字段顺序与 adminInitData/SystemMenu.php 一致id/pid/title/icon/href/params/target/sort/status.
* id 显式分配 300-303与 adminInitData 保持一致(顶级 300 + 3 个子菜单)
* 挂到系统管理pid=228不创建独立顶级分组
*/
public $newMenus = [
[
'id' => 300,
'pid' => 0,
'title' => 'Nginx 日志分析',
'icon' => 'fa fa-chart-bar',
'href' => '',
'params' => '',
'target' => '_self',
'sort' => 0,
'status' => 1,
],
[
'id' => 301,
'pid' => 300,
'pid' => 228,
'title' => '访问统计仪表盘',
'icon' => 'fa fa-line-chart',
'href' => 'system.nginx_log_stat/index',
@@ -109,7 +99,7 @@ class UpdateFunction
],
[
'id' => 302,
'pid' => 300,
'pid' => 228,
'title' => '原始访问日志',
'icon' => 'fa fa-file-text',
'href' => 'nginx.access_log/index',
@@ -118,24 +108,13 @@ class UpdateFunction
'sort' => 0,
'status' => 1,
],
[
'id' => 303,
'pid' => 300,
'title' => '读取位置管理',
'icon' => 'fa fa-map-marker',
'href' => 'nginx.log_position/index',
'params' => '',
'target' => '_self',
'sort' => 0,
'status' => 1,
],
];
public function update()
{
$this->output->writeln('更新代码');
$this->output->info('v2.4.0nginx_log 分析模块存量升级(补齐 4 条 sysconfig + 4 条菜单)');
$this->output->info('v2.4.0nginx_log 分析模块存量升级(补齐 4 条 sysconfig + 2 条菜单)');
$configTable = config('database.connections.mysql.prefix', 'ul_') . 'system_config';
$menuTable = config('database.connections.mysql.prefix', 'ul_') . 'system_menu';
@@ -190,6 +169,98 @@ class UpdateFunction
$totalSkipped = $configSkipped + $menuSkipped;
$this->output->info("本次新增 {$configAdded} 条配置、{$menuAdded} 条菜单,跳过 {$totalSkipped} 条已存在");
// ==================== node_id 字段 + 唯一键变更(多节点采集支持)====================
// 与 migration 20260801125222_nginx_log_add_node_id.php 保持一致
// 6 张表全部新增 node_id varchar(64) default ''
// position / 4 张 stat 表的唯一键前置 node_id多节点隔离避免聚合 INSERT 触发 duplicate key
// raw 表nginx_access_log加 node_id + time_local 普通索引,无唯一键变更
$this->output->info('v2.4.0nginx_log 6 张表新增 node_id 字段与唯一键调整');
$prefix = config('database.connections.mysql.prefix', 'ul_');
// 表短名 => 索引变更配置
// drop_idx: 要删除的旧索引名raw 表无)
// add_idx: 新索引 [name, cols, unique]
$nodeIdTables = [
'nginx_log_position' => [
'drop_idx' => 'uniq_file_path',
'add_idx' => ['name' => 'uniq_node_file', 'cols' => ['node_id', 'file_path'], 'unique' => true],
],
'nginx_access_log' => [
'add_idx' => ['name' => 'idx_node_time', 'cols' => ['node_id', 'time_local'], 'unique' => false],
],
'nginx_stat_hour' => [
'drop_idx' => 'uniq_date_hour',
'add_idx' => ['name' => 'uniq_node_date_hour', 'cols' => ['node_id', 'stat_date', 'stat_hour'], 'unique' => true],
],
'nginx_stat_url' => [
'drop_idx' => 'uniq_date_uri',
'add_idx' => ['name' => 'uniq_node_date_uri', 'cols' => ['node_id', 'stat_date', 'uri'], 'unique' => true],
],
'nginx_stat_referer' => [
'drop_idx' => 'uniq_date_referer',
'add_idx' => ['name' => 'uniq_node_date_referer', 'cols' => ['node_id', 'stat_date', 'referer_domain'], 'unique' => true],
],
'nginx_stat_ua' => [
'drop_idx' => 'uniq_date_type_name',
'add_idx' => ['name' => 'uniq_node_date_type_name', 'cols' => ['node_id', 'stat_date', 'ua_type', 'ua_name'], 'unique' => true],
],
];
$colAdded = 0;
$colSkipped = 0;
foreach ($nodeIdTables as $tableShort => $idxCfg) {
$fullTable = $prefix . $tableShort;
// 防御性检查:表是否存在(未运行 migration 时跳过,不报错)
$tableExists = Db::query(
"SELECT COUNT(*) AS cnt FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ?",
[$fullTable]
);
if (empty($tableExists) || (int) $tableExists[0]['cnt'] === 0) {
$this->output->writeln(" - {$fullTable} 表不存在,跳过(请先执行 migrate:run");
continue;
}
// 幂等 1检查 node_id 字段是否存在
$colExists = Db::query(
"SELECT COUNT(*) AS cnt FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND COLUMN_NAME = 'node_id'",
[$fullTable]
);
if (!empty($colExists) && (int) $colExists[0]['cnt'] > 0) {
$this->output->writeln(" - {$fullTable} node_id 字段已存在,跳过");
++$colSkipped;
} else {
Db::execute("ALTER TABLE `{$fullTable}` ADD COLUMN `node_id` VARCHAR(64) NOT NULL DEFAULT '' COMMENT '采集节点 ID空=全局汇总)'");
$this->output->writeln(" - {$fullTable} 新增 node_id 字段完成");
++$colAdded;
}
// 幂等 2唯一键/索引变更(检查新索引是否存在)
$newIdx = $idxCfg['add_idx']['name'] ?? null;
if ($newIdx !== null) {
$idxExists = Db::query(
"SELECT COUNT(*) AS cnt FROM information_schema.STATISTICS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND INDEX_NAME = ?",
[$fullTable, $newIdx]
);
if (empty($idxExists) || (int) $idxExists[0]['cnt'] === 0) {
// 删除旧索引position / stat 表有raw 表无)
if (!empty($idxCfg['drop_idx'])) {
Db::execute("ALTER TABLE `{$fullTable}` DROP INDEX `{$idxCfg['drop_idx']}`");
}
// 新增新索引
$idxType = !empty($idxCfg['add_idx']['unique']) ? 'UNIQUE' : 'INDEX';
$colsStr = implode(', ', array_map(function ($c) {
return "`{$c}`";
}, $idxCfg['add_idx']['cols']));
Db::execute("ALTER TABLE `{$fullTable}` ADD {$idxType} `{$newIdx}` ({$colsStr})");
$this->output->writeln(" - {$fullTable} 索引调整:{$newIdx}");
}
}
}
$this->output->info("node_id 迁移完成(新增字段 {$colAdded} 表,跳过 {$colSkipped} 表已存在)");
$this->output->writeln('更新代码完成');
}
}

View File

@@ -19,6 +19,14 @@
<input type="text" class="layui-input" id="dateEnd" placeholder="结束日期" autocomplete="off" style="width:130px;" readonly>
<button type="button" class="layui-btn layui-btn-sm layui-btn-normal" id="customSearchBtn">查询</button>
</span>
<div class="layui-input-inline" style="margin-left: 15px; width: 180px; vertical-align: middle;">
<select id="node-filter" lay-filter="nodeFilter">
<option value="">全部节点</option>
{volist name="nodes" id="node"}
<option value="{$node.node_id}" {if condition="$node_id eq $node.node_id"}selected{/if}>{$node.node_id}</option>
{/volist}
</select>
</div>
</div>
</div>

View File

@@ -6,11 +6,15 @@
* 后端 URL 不硬编码:用 ua.url('system.nginx_log_stat/index') 由 CONFIG.ADMIN 拼接。
* AJAX 触发方式URL 带 _ajax=1不依赖 Accept header命令行测试友好
*/
layui.use(['table', 'laydate', 'util'], function () {
layui.use(['table', 'laydate', 'util', 'form'], function () {
var table = layui.table;
var laydate = layui.laydate;
var util = layui.util;
var form = layui.form;
// 渲染节点下拉框
form.render();
// ── 配置URL 不硬编码:由 ua.url 拼接 /{ADMIN}/system.nginx_log_stat/index──
var STAT_INDEX_URL = ua.url('system.nginx_log_stat/index');
@@ -129,6 +133,15 @@ layui.use(['table', 'laydate', 'util'], function () {
loadData({ period: 'custom', start: startVal, end: endVal });
});
// ================================================================
// 节点下拉框切换
// ================================================================
form.on('select(nodeFilter)', function (data) {
// 切换节点时用当前 period 重新加载
loadData({ period: currentPeriod });
});
// ================================================================
// AJAX 加载数据
// ================================================================
@@ -141,6 +154,10 @@ layui.use(['table', 'laydate', 'util'], function () {
function loadData(extra) {
extra = extra || {};
// 当前选中的节点 ID默认空字符串=全部节点)
var nodeId = $('#node-filter').val() || '';
extra.node_id = nodeId;
var params = $.extend({ _ajax: 1, period: currentPeriod }, extra);
ua.request.get({

View File

@@ -27,7 +27,7 @@ return [
'target' => '/tools/timer.NginxLogImport/do',
'frequency' => 300,
'concurrency' => 1,
'run_type' => 'auto',
'run_type' => 'all',
],
[
'name' => 'nginx_log_aggregate',

View File

@@ -24,9 +24,9 @@ class NginxLogAggregatorServiceBase
/**
* 聚合指定日期的指定小时.
*
* 流程:
* 1. stat_hourDELETE WHERE stat_date=:d AND stat_hour=:h → INSERT 1 行(该小时聚合)
* 2. stat_url / stat_referer / stat_uaDELETE WHERE stat_date=:d整天重跑→ INSERT top N 行
* 流程(每张表生成 全局行 node_id='' + 各节点行)
* 1. stat_hourDELETE WHERE stat_date=:d AND stat_hour=:h → INSERT 全局 1 行 + 每节点 1 行
* 2. stat_url / stat_referer / stat_uaDELETE WHERE stat_date=:d整天重跑→ INSERT 全局 topN + 每节点 topN
*
* @param int $statDate 日期 YYYYMMDD如 20260728
* @param int $statHour 小时 0-23
@@ -65,142 +65,50 @@ class NginxLogAggregatorServiceBase
Db::startTrans();
try {
// 1. stat_hourDELETE + INSERT 1 行
// DELETE逻辑不变stat_hour 按小时stat_url/referer/ua 按整天
NginxStatHour::where('stat_date', $statDate)
->where('stat_hour', $statHour)
->delete();
$hourSql = "INSERT INTO `{$hourTable}`
(stat_date, stat_hour, pv, uv, total_bytes, status_2xx, status_3xx, status_4xx, status_5xx, status_other, avg_request_time)
SELECT :stat_date, :stat_hour,
COUNT(*) AS pv,
COUNT(DISTINCT remote_addr) AS uv,
COALESCE(SUM(body_bytes_sent), 0) AS total_bytes,
COALESCE(SUM(CASE WHEN status BETWEEN 200 AND 299 THEN 1 ELSE 0 END), 0) AS status_2xx,
COALESCE(SUM(CASE WHEN status BETWEEN 300 AND 399 THEN 1 ELSE 0 END), 0) AS status_3xx,
COALESCE(SUM(CASE WHEN status BETWEEN 400 AND 499 THEN 1 ELSE 0 END), 0) AS status_4xx,
COALESCE(SUM(CASE WHEN status BETWEEN 500 AND 599 THEN 1 ELSE 0 END), 0) AS status_5xx,
COALESCE(SUM(CASE WHEN status < 200 OR status > 599 THEN 1 ELSE 0 END), 0) AS status_other,
COALESCE(AVG(request_time), 0) AS avg_request_time
FROM `{$rawTable}`
WHERE time_local >= :hour_start AND time_local < :hour_end
HAVING COUNT(*) > 0";
$statHourRows = (int) Db::execute($hourSql, [
'stat_date' => $statDate,
'stat_hour' => $statHour,
'hour_start' => $hourStart,
'hour_end' => $hourEnd,
]);
// 2. stat_urlDELETE整天+ INSERT topN
NginxStatUrl::where('stat_date', $statDate)->delete();
$urlLimit = $topn > 0 ? 'LIMIT ' . $topn : '';
$urlSql = "INSERT INTO `{$urlTable}`
(stat_date, uri, pv, uv, total_bytes, avg_request_time)
SELECT :stat_date, uri,
COUNT(*) AS pv,
COUNT(DISTINCT remote_addr) AS uv,
COALESCE(SUM(body_bytes_sent), 0) AS total_bytes,
COALESCE(AVG(request_time), 0) AS avg_request_time
FROM `{$rawTable}`
WHERE time_local >= :day_start AND time_local < :day_end
GROUP BY uri
ORDER BY pv DESC
{$urlLimit}";
$statUrlRows = (int) Db::execute($urlSql, [
'stat_date' => $statDate,
'day_start' => $dayStart,
'day_end' => $dayEnd,
]);
// 3. stat_refererDELETE + INSERT topN从 http_referer 提取 domain
NginxStatReferer::where('stat_date', $statDate)->delete();
$refererLimit = $topn > 0 ? 'LIMIT ' . $topn : '';
$refererSql = "INSERT INTO `{$refererTable}`
(stat_date, referer_domain, pv, uv)
SELECT :stat_date,
CASE
WHEN http_referer IS NULL OR http_referer = '' OR http_referer = '-' THEN '-'
ELSE SUBSTRING_INDEX(
SUBSTRING_INDEX(
REPLACE(REPLACE(http_referer, 'https://', ''), 'http://', ''),
'/', 1
),
'?', 1
)
END AS referer_domain,
COUNT(*) AS pv,
COUNT(DISTINCT remote_addr) AS uv
FROM `{$rawTable}`
WHERE time_local >= :day_start AND time_local < :day_end
GROUP BY referer_domain
ORDER BY pv DESC
{$refererLimit}";
$statRefererRows = (int) Db::execute($refererSql, [
'stat_date' => $statDate,
'day_start' => $dayStart,
'day_end' => $dayEnd,
]);
// 4. stat_uaDELETE + INSERT topN从 http_user_agent 分类)
NginxStatUa::where('stat_date', $statDate)->delete();
$uaLimit = $topn > 0 ? 'LIMIT ' . $topn : '';
$uaSql = "INSERT INTO `{$uaTable}`
(stat_date, ua_type, ua_name, pv, uv)
SELECT :stat_date,
CASE
WHEN http_user_agent LIKE '%bot%'
OR http_user_agent LIKE '%spider%'
OR http_user_agent LIKE '%crawl%'
OR http_user_agent LIKE '%slurp%'
OR http_user_agent LIKE '%bingpreview%'
OR http_user_agent LIKE '%facebookexternalhit%'
OR http_user_agent LIKE '%twitterbot%' THEN 'spider'
WHEN http_user_agent IS NULL OR http_user_agent = '' OR http_user_agent = '-' THEN 'unknown'
WHEN http_user_agent LIKE '%Mozilla%'
OR http_user_agent LIKE '%Chrome%'
OR http_user_agent LIKE '%Safari%'
OR http_user_agent LIKE '%Firefox%'
OR http_user_agent LIKE '%Edg%'
OR http_user_agent LIKE '%Opera%'
OR http_user_agent LIKE '%MSIE%'
OR http_user_agent LIKE '%Trident%' THEN 'browser'
ELSE 'unknown'
END AS ua_type,
CASE
WHEN http_user_agent LIKE '%Googlebot%' THEN 'Googlebot'
WHEN http_user_agent LIKE '%Baiduspider%' THEN 'Baiduspider'
WHEN http_user_agent LIKE '%bingbot%' THEN 'Bingbot'
WHEN http_user_agent LIKE '%DuckDuckBot%' THEN 'DuckDuckBot'
WHEN http_user_agent LIKE '%YandexBot%' THEN 'YandexBot'
WHEN http_user_agent LIKE '%Edg/%' THEN 'Microsoft Edge'
WHEN http_user_agent LIKE '%OPR/%' OR http_user_agent LIKE '%Opera%' THEN 'Opera'
WHEN http_user_agent LIKE '%Firefox/%' THEN 'Firefox'
WHEN http_user_agent LIKE '%Chrome/%' THEN 'Chrome'
WHEN http_user_agent LIKE '%Safari/%' THEN 'Safari'
WHEN http_user_agent LIKE '%MSIE%' OR http_user_agent LIKE '%Trident%' THEN 'Internet Explorer'
WHEN http_user_agent IS NULL OR http_user_agent = '' OR http_user_agent = '-' THEN 'Unknown'
ELSE 'Other'
END AS ua_name,
COUNT(*) AS pv,
COUNT(DISTINCT remote_addr) AS uv
FROM `{$rawTable}`
WHERE time_local >= :day_start AND time_local < :day_end
GROUP BY ua_type, ua_name
ORDER BY pv DESC
{$uaLimit}";
$statHourRows = 0;
$statUrlRows = 0;
$statRefererRows = 0;
$statUaRows = 0;
$statUaRows = (int) Db::execute($uaSql, [
'stat_date' => $statDate,
'day_start' => $dayStart,
'day_end' => $dayEnd,
]);
// 批次 A全局行node_id=''跨所有节点聚合UV 精确)
$global = $this->insertAggregatesForScope(
'', null,
$statDate, $statHour, $hourStart, $hourEnd, $dayStart, $dayEnd, $topn,
$rawTable, $hourTable, $urlTable, $refererTable, $uaTable
);
$statHourRows += $global['stat_hour'];
$statUrlRows += $global['stat_url'];
$statRefererRows += $global['stat_referer'];
$statUaRows += $global['stat_ua'];
// 批次 B按节点行循环每个 node_id
// 用 day 窗口取节点列表:保证 day 窗口表url/referer/ua的 per-node 行不会因
// 某节点"本小时无活动"而被 DELETE 后丢失stat_hour 的 per-node 行由
// HAVING COUNT(*)>0 自然过滤掉本小时无活动的节点。
$nodeRows = Db::query(
"SELECT DISTINCT node_id FROM `{$rawTable}` WHERE time_local >= :day_start AND time_local < :day_end AND node_id != ''",
['day_start' => $dayStart, 'day_end' => $dayEnd]
);
foreach ($nodeRows as $nodeRow) {
$nid = (string) $nodeRow['node_id'];
$perNode = $this->insertAggregatesForScope(
$nid, $nid,
$statDate, $statHour, $hourStart, $hourEnd, $dayStart, $dayEnd, $topn,
$rawTable, $hourTable, $urlTable, $refererTable, $uaTable
);
$statHourRows += $perNode['stat_hour'];
$statUrlRows += $perNode['stat_url'];
$statRefererRows += $perNode['stat_referer'];
$statUaRows += $perNode['stat_ua'];
}
Db::commit();
@@ -216,6 +124,175 @@ class NginxLogAggregatorServiceBase
}
}
/**
* 为单个作用域(全局或某节点)执行 4 段 INSERT.
*
* - 全局作用域:$nodeId='' 且 $nodeFilter=null不追加 node WHERE跨所有节点聚合
* - 节点作用域:$nodeId=节点ID 且 $nodeFilter=节点ID追加 AND node_id=:node_filter
*
* stat_hour1 行(无 GROUP BYHAVING COUNT(*)>0
* stat_url / stat_referer / stat_uatopN各自 GROUP BY + LIMIT
*
* @param string $nodeId INSERT 的 node_id 列值(全局为 ''
* @param string|null $nodeFilter null=全局(不追加 WHERE非 null=追加节点过滤
* @return array {stat_hour:int, stat_url:int, stat_referer:int, stat_ua:int}
*/
protected function insertAggregatesForScope(
string $nodeId,
?string $nodeFilter,
int $statDate,
int $statHour,
int $hourStart,
int $hourEnd,
int $dayStart,
int $dayEnd,
int $topn,
string $rawTable,
string $hourTable,
string $urlTable,
string $refererTable,
string $uaTable
): array {
// 节点过滤片段:全局行无;节点行追加 AND node_id = :node_filter
$nodeWhere = $nodeFilter === null ? '' : 'AND node_id = :node_filter';
// hour 窗口绑定stat_hour 专用)
$hourParams = [
'node_id' => $nodeId,
'stat_date' => $statDate,
'stat_hour' => $statHour,
'hour_start' => $hourStart,
'hour_end' => $hourEnd,
];
if ($nodeFilter !== null) {
$hourParams['node_filter'] = $nodeFilter;
}
// day 窗口绑定stat_url/referer/ua 共用)
$dayParams = [
'node_id' => $nodeId,
'stat_date' => $statDate,
'day_start' => $dayStart,
'day_end' => $dayEnd,
];
if ($nodeFilter !== null) {
$dayParams['node_filter'] = $nodeFilter;
}
// 1. stat_hour1 行(无 GROUP BY
$hourSql = "INSERT INTO `{$hourTable}`
(node_id, stat_date, stat_hour, pv, uv, total_bytes, status_2xx, status_3xx, status_4xx, status_5xx, status_other, avg_request_time)
SELECT :node_id, :stat_date, :stat_hour,
COUNT(*) AS pv,
COUNT(DISTINCT remote_addr) AS uv,
COALESCE(SUM(body_bytes_sent), 0) AS total_bytes,
COALESCE(SUM(CASE WHEN status BETWEEN 200 AND 299 THEN 1 ELSE 0 END), 0) AS status_2xx,
COALESCE(SUM(CASE WHEN status BETWEEN 300 AND 399 THEN 1 ELSE 0 END), 0) AS status_3xx,
COALESCE(SUM(CASE WHEN status BETWEEN 400 AND 499 THEN 1 ELSE 0 END), 0) AS status_4xx,
COALESCE(SUM(CASE WHEN status BETWEEN 500 AND 599 THEN 1 ELSE 0 END), 0) AS status_5xx,
COALESCE(SUM(CASE WHEN status < 200 OR status > 599 THEN 1 ELSE 0 END), 0) AS status_other,
COALESCE(AVG(request_time), 0) AS avg_request_time
FROM `{$rawTable}`
WHERE time_local >= :hour_start AND time_local < :hour_end {$nodeWhere}
HAVING COUNT(*) > 0";
$statHourRows = (int) Db::execute($hourSql, $hourParams);
// 2. stat_urltopNGROUP BY uri
$urlLimit = $topn > 0 ? 'LIMIT ' . $topn : '';
$urlSql = "INSERT INTO `{$urlTable}`
(node_id, stat_date, uri, pv, uv, total_bytes, avg_request_time)
SELECT :node_id, :stat_date, uri,
COUNT(*) AS pv,
COUNT(DISTINCT remote_addr) AS uv,
COALESCE(SUM(body_bytes_sent), 0) AS total_bytes,
COALESCE(AVG(request_time), 0) AS avg_request_time
FROM `{$rawTable}`
WHERE time_local >= :day_start AND time_local < :day_end {$nodeWhere}
GROUP BY uri
ORDER BY pv DESC
{$urlLimit}";
$statUrlRows = (int) Db::execute($urlSql, $dayParams);
// 3. stat_referertopN从 http_referer 提取 domain
$refererLimit = $topn > 0 ? 'LIMIT ' . $topn : '';
$refererSql = "INSERT INTO `{$refererTable}`
(node_id, stat_date, referer_domain, pv, uv)
SELECT :node_id, :stat_date,
CASE
WHEN http_referer IS NULL OR http_referer = '' OR http_referer = '-' THEN '-'
ELSE SUBSTRING_INDEX(
SUBSTRING_INDEX(
REPLACE(REPLACE(http_referer, 'https://', ''), 'http://', ''),
'/', 1
),
'?', 1
)
END AS referer_domain,
COUNT(*) AS pv,
COUNT(DISTINCT remote_addr) AS uv
FROM `{$rawTable}`
WHERE time_local >= :day_start AND time_local < :day_end {$nodeWhere}
GROUP BY referer_domain
ORDER BY pv DESC
{$refererLimit}";
$statRefererRows = (int) Db::execute($refererSql, $dayParams);
// 4. stat_uatopN从 http_user_agent 分类)
$uaLimit = $topn > 0 ? 'LIMIT ' . $topn : '';
$uaSql = "INSERT INTO `{$uaTable}`
(node_id, stat_date, ua_type, ua_name, pv, uv)
SELECT :node_id, :stat_date,
CASE
WHEN http_user_agent LIKE '%bot%'
OR http_user_agent LIKE '%spider%'
OR http_user_agent LIKE '%crawl%'
OR http_user_agent LIKE '%slurp%'
OR http_user_agent LIKE '%bingpreview%'
OR http_user_agent LIKE '%facebookexternalhit%'
OR http_user_agent LIKE '%twitterbot%' THEN 'spider'
WHEN http_user_agent IS NULL OR http_user_agent = '' OR http_user_agent = '-' THEN 'unknown'
WHEN http_user_agent LIKE '%Mozilla%'
OR http_user_agent LIKE '%Chrome%'
OR http_user_agent LIKE '%Safari%'
OR http_user_agent LIKE '%Firefox%'
OR http_user_agent LIKE '%Edg%'
OR http_user_agent LIKE '%Opera%'
OR http_user_agent LIKE '%MSIE%'
OR http_user_agent LIKE '%Trident%' THEN 'browser'
ELSE 'unknown'
END AS ua_type,
CASE
WHEN http_user_agent LIKE '%Googlebot%' THEN 'Googlebot'
WHEN http_user_agent LIKE '%Baiduspider%' THEN 'Baiduspider'
WHEN http_user_agent LIKE '%bingbot%' THEN 'Bingbot'
WHEN http_user_agent LIKE '%DuckDuckBot%' THEN 'DuckDuckBot'
WHEN http_user_agent LIKE '%YandexBot%' THEN 'YandexBot'
WHEN http_user_agent LIKE '%Edg/%' THEN 'Microsoft Edge'
WHEN http_user_agent LIKE '%OPR/%' OR http_user_agent LIKE '%Opera%' THEN 'Opera'
WHEN http_user_agent LIKE '%Firefox/%' THEN 'Firefox'
WHEN http_user_agent LIKE '%Chrome/%' THEN 'Chrome'
WHEN http_user_agent LIKE '%Safari/%' THEN 'Safari'
WHEN http_user_agent LIKE '%MSIE%' OR http_user_agent LIKE '%Trident%' THEN 'Internet Explorer'
WHEN http_user_agent IS NULL OR http_user_agent = '' OR http_user_agent = '-' THEN 'Unknown'
ELSE 'Other'
END AS ua_name,
COUNT(*) AS pv,
COUNT(DISTINCT remote_addr) AS uv
FROM `{$rawTable}`
WHERE time_local >= :day_start AND time_local < :day_end {$nodeWhere}
GROUP BY ua_type, ua_name
ORDER BY pv DESC
{$uaLimit}";
$statUaRows = (int) Db::execute($uaSql, $dayParams);
return [
'stat_hour' => $statHourRows,
'stat_url' => $statUrlRows,
'stat_referer' => $statRefererRows,
'stat_ua' => $statUaRows,
];
}
/**
* 将 int 日期 YYYYMMDD 格式化为 "YYYY-MM-DD".
*/

View File

@@ -3,6 +3,7 @@
namespace base\common\service;
use app\admin\model\NginxLogPosition;
use app\common\service\HostService;
/**
* Nginx 日志增量读取 serviceBase 层).
@@ -10,12 +11,17 @@ use app\admin\model\NginxLogPosition;
* 职责:
* - 按文件 offset 增量读取日志行Generator不读整个文件到内存
* - 检测日志轮转inode 变化 OR filesize < offset覆盖 rename+rebuild 与 copytruncate 两种模式)
* - 将读取进度持久化到 NginxLogPosition 表upsert
* - 将读取进度持久化到 NginxLogPosition 表upsert,按 node_id 隔离
*
* 多节点适配position 表按 (node_id, file_path) 唯一定位读取进度。
* 旧数据node_id 为空)通过懒加载接管(见 getLastPosition首个查询到它的本节点原子 UPDATE node_id
* 避免 migration 全表回填(多节点共享 DB 时 migration 只跑一次,只有懒加载能确保每节点各自接管)。
*
* 不负责解析T5 Parser、聚合T7 Aggregator
*
* 依赖倒置:内部 model 调用走 `app\admin\model\NginxLogPosition`App 入口类),
* 使用者可在 app/ 重写该 model 拦截行为;本类不直接 `new NginxLogPosition`(静态调用满足多态)。
* 节点身份走 `app\common\service\HostService`App 入口类)。
*/
class NginxLogReaderServiceBase
{
@@ -159,15 +165,27 @@ class NginxLogReaderServiceBase
/**
* 查询文件的上次读取位置.
*
* 多节点适配where 条件带 node_id 隔离各节点进度。
* 懒加载接管:多节点共享 DB 时 migration 只跑一次,旧记录 node_id 为空,
* 首个查询到它的本节点原子 UPDATE node_id 接管,保证每节点各自有独立 position 记录。
*
* @param string $filePath 日志文件绝对路径
*
* @return array|null 命中时返回 [inode, offset, last_line_hash, parse_fail_count, parse_fail_samples],无记录返回 null
*/
public function getLastPosition(string $filePath): ?array
{
$row = NginxLogPosition::where('file_path', $filePath)->find();
$nodeId = HostService::getNodeId();
$row = NginxLogPosition::where('file_path', $filePath)
->where('node_id', $nodeId)
->find();
if ($row === null) {
return null;
// 懒加载接管:旧记录 node_id 为空,原子抢占接管到当前节点
$row = $this->lazyAdoptPosition($filePath, $nodeId);
if ($row === null) {
return null;
}
}
return [
@@ -179,6 +197,39 @@ class NginxLogReaderServiceBase
];
}
/**
* 懒加载接管:把旧的无 node_id 记录原子接管到当前节点.
*
* 多节点共享 DB 时 migration 只跑一次,无法为每节点预置 position 记录。
* 旧记录node_id='')由首个查询到它的本节点通过原子条件 UPDATE 接管:
* UPDATE ... SET node_id=current WHERE node_id='' AND file_path=X
* 并发安全WHERE node_id='' 保证只有一个节点能 affected=1其他节点查询时已被接管。
*
* @return \think\Model|null 接管成功返回接管后的记录,无旧记录或被其他节点抢占返回 null
*/
protected function lazyAdoptPosition(string $filePath, string $nodeId)
{
$legacy = NginxLogPosition::where('file_path', $filePath)
->where('node_id', '')
->find();
if ($legacy === null) {
return null;
}
// 原子条件 UPDATE只有 node_id='' 时才能被接管(防并发多节点同时接管同一记录)
$affected = NginxLogPosition::where('id', $legacy->getData('id'))
->where('node_id', '')
->update(['node_id' => $nodeId]);
if ($affected === 0) {
// 被其他节点抢先接管,本节点放弃(下次 savePosition 会创建新记录)
return null;
}
return NginxLogPosition::where('file_path', $filePath)
->where('node_id', $nodeId)
->find();
}
/**
* 保存读取位置upsert.
*
@@ -196,8 +247,11 @@ class NginxLogReaderServiceBase
public function savePosition(string $filePath, int $inode, int $offset, ?string $lastLineHash, int $failCount = 0, ?string $failSamples = null): void
{
$now = time();
$nodeId = HostService::getNodeId();
$existing = NginxLogPosition::where('file_path', $filePath)->find();
$existing = NginxLogPosition::where('file_path', $filePath)
->where('node_id', $nodeId)
->find();
if ($existing !== null) {
$existing->save([
'inode' => $inode,
@@ -212,6 +266,7 @@ class NginxLogReaderServiceBase
}
NginxLogPosition::create([
'node_id' => $nodeId,
'file_path' => $filePath,
'inode' => $inode,
'offset' => $offset,

View File

@@ -191,6 +191,7 @@ class NginxLogImportBase extends TimerController
{
$now = time();
return [
'node_id' => \app\common\service\HostService::getNodeId(),
'file_path' => $filePath,
'remote_addr' => $parsed['remote_addr'],
'remote_user' => $parsed['remote_user'] ?? '',