mirror of
https://gitee.com/ulthon/ulthon_admin.git
synced 2026-08-30 04:35:33 +08:00
feat(nginx-log): 多节点适配(node_id 全链路 + 全局/按节点聚合 + 节点筛选 + v2.4.0 升级脚本)
- 6 张表加 node_id 字段 + position/stat 改复合唯一键(migration) - 6 个 Scheme 同步 node_id 注解 + 唯一键 - Reader 带 node_id 参数 + 懒加载接管(getLastPosition 回退查 node_id='') - Aggregator 全局行(node_id='')+ 按节点循环 insertAggregatesForScope - Stat 所有查询方法加 where node_id 条件 - Dashboard 控制器/视图加节点筛选下拉框 + AJAX 带 node_id - AccessLog 列表加节点列 + 采集进度卡片显示节点信息 - import 定时任务 run_type 改 all(每节点采集自己的日志) - v2.4.0 升级脚本追加 ALTER TABLE(幂等 check-then-alter) - 菜单调整:去掉 Nginx 顶级菜单 + 读取位置管理,改为系统管理下挂两个子菜单 - ulthon-timer 文档补充 run_type=all 多节点部署注意事项
This commit is contained in:
@@ -201,6 +201,8 @@ nohup php /var/www/html/think timer --local --quiet &
|
||||
|
||||
多节点部署:每个容器各自启动 timer 进程,`runtime/node_id.lock` 在各自容器内独立生成(天然隔离),连同一个数据库即自动组成多节点集群。
|
||||
|
||||
> **多节点部署注意(run_type=all 的任务)**:`run_type=all` 的定时任务(如 `nginx_log_import`)要求每节点各自采集本机数据,每节点必须用 `php think timer --local` 启动,确保 site 请求发往本机 `localhost` 而非共享 `site_domain`/LB,否则请求会落在随机节点上,导致采集错位(节点 A 的日志被节点 B 处理)。框架内置 Docker 部署栈已默认带 `--local`,手动/裸机部署时务必显式加上。
|
||||
|
||||
### 本地调试(指定请求 Host)
|
||||
|
||||
site 任务会按站点域名发起请求,默认从 `sysconfig('site','site_domain')` 读取。
|
||||
|
||||
@@ -9,6 +9,7 @@ use app\common\scheme\attribute\Component;
|
||||
use app\common\scheme\attribute\Index;
|
||||
|
||||
#[Table(name: 'ul_nginx_access_log', comment: 'nginx 访问日志明细(解析入库)')]
|
||||
#[Index(columns: ['node_id', 'time_local'], name: 'idx_node_time', type: 'NORMAL')]
|
||||
#[Index(columns: ['time_local'], name: 'idx_time_local', type: 'NORMAL')]
|
||||
#[Index(columns: ['remote_addr'], name: 'idx_remote_addr', type: 'NORMAL')]
|
||||
#[Index(columns: ['status'], name: 'idx_status', type: 'NORMAL')]
|
||||
@@ -19,6 +20,9 @@ class NginxAccessLog extends BaseScheme
|
||||
#[Field(type: 'bigint', nullable: false, unsigned: true, autoIncrement: true, primary: true, comment: '主键ID')]
|
||||
public $id;
|
||||
|
||||
#[Field(type: 'varchar', length: 64, nullable: false, default: '', comment: '节点ID(区分不同采集节点)')]
|
||||
public $node_id;
|
||||
|
||||
#[Field(type: 'varchar', length: 500, nullable: false, comment: '来源日志文件路径')]
|
||||
#[Component(type: 'text', options: [])]
|
||||
public $file_path;
|
||||
|
||||
@@ -9,12 +9,15 @@ use app\common\scheme\attribute\Component;
|
||||
use app\common\scheme\attribute\Index;
|
||||
|
||||
#[Table(name: 'ul_nginx_log_position', comment: 'nginx 日志文件读取位置(增量读取状态)')]
|
||||
#[Index(columns: ['file_path'], name: 'uniq_file_path', type: 'UNIQUE')]
|
||||
#[Index(columns: ['node_id', 'file_path'], name: 'uniq_node_file', type: 'UNIQUE')]
|
||||
class NginxLogPosition extends BaseScheme
|
||||
{
|
||||
#[Field(type: 'int', length: 11, nullable: false, unsigned: true, autoIncrement: true, primary: true)]
|
||||
public $id;
|
||||
|
||||
#[Field(type: 'varchar', length: 64, nullable: false, default: '', comment: '节点ID(区分不同采集节点)')]
|
||||
public $node_id;
|
||||
|
||||
#[Field(type: 'varchar', length: 500, nullable: false, comment: '日志文件绝对路径')]
|
||||
#[Component(type: 'text', options: [])]
|
||||
public $file_path;
|
||||
|
||||
@@ -8,13 +8,16 @@ use app\common\scheme\attribute\Field;
|
||||
use app\common\scheme\attribute\Index;
|
||||
|
||||
#[Table(name: 'nginx_stat_hour', comment: 'nginx 按小时聚合统计')]
|
||||
#[Index(columns: ['stat_date', 'stat_hour'], name: 'uniq_date_hour', type: 'UNIQUE')]
|
||||
#[Index(columns: ['node_id', 'stat_date', 'stat_hour'], name: 'uniq_node_date_hour', type: 'UNIQUE')]
|
||||
#[Index(columns: ['stat_date'], name: 'idx_stat_date', type: 'NORMAL')]
|
||||
class NginxStatHour extends BaseScheme
|
||||
{
|
||||
#[Field(type: 'int', length: 11, nullable: false, unsigned: true, autoIncrement: true, primary: true)]
|
||||
public $id;
|
||||
|
||||
#[Field(type: 'varchar', length: 64, nullable: false, default: '', comment: '节点ID(区分不同采集节点)')]
|
||||
public $node_id;
|
||||
|
||||
#[Field(type: 'int', length: 11, nullable: false, unsigned: true, comment: '统计日期(YYYYMMDD)')]
|
||||
public $stat_date;
|
||||
|
||||
|
||||
@@ -8,13 +8,16 @@ use app\common\scheme\attribute\Field;
|
||||
use app\common\scheme\attribute\Index;
|
||||
|
||||
#[Table(name: 'nginx_stat_referer', comment: 'nginx 按 Referer 来源域名聚合统计')]
|
||||
#[Index(columns: ['stat_date', 'referer_domain'], name: 'uniq_date_referer', type: 'UNIQUE')]
|
||||
#[Index(columns: ['node_id', 'stat_date', 'referer_domain'], name: 'uniq_node_date_referer', type: 'UNIQUE')]
|
||||
#[Index(columns: ['stat_date'], name: 'idx_stat_date', type: 'NORMAL')]
|
||||
class NginxStatReferer extends BaseScheme
|
||||
{
|
||||
#[Field(type: 'int', length: 11, nullable: false, unsigned: true, autoIncrement: true, primary: true)]
|
||||
public $id;
|
||||
|
||||
#[Field(type: 'varchar', length: 64, nullable: false, default: '', comment: '节点ID(区分不同采集节点)')]
|
||||
public $node_id;
|
||||
|
||||
#[Field(type: 'int', length: 11, nullable: false, unsigned: true, comment: '统计日期(YYYYMMDD)')]
|
||||
public $stat_date;
|
||||
|
||||
|
||||
@@ -8,13 +8,16 @@ use app\common\scheme\attribute\Field;
|
||||
use app\common\scheme\attribute\Index;
|
||||
|
||||
#[Table(name: 'nginx_stat_ua', comment: 'nginx 按 User-Agent 聚合统计')]
|
||||
#[Index(columns: ['stat_date', 'ua_type', 'ua_name'], name: 'uniq_date_type_name', type: 'UNIQUE')]
|
||||
#[Index(columns: ['node_id', 'stat_date', 'ua_type', 'ua_name'], name: 'uniq_node_date_type_name', type: 'UNIQUE')]
|
||||
#[Index(columns: ['stat_date'], name: 'idx_stat_date', type: 'NORMAL')]
|
||||
class NginxStatUa extends BaseScheme
|
||||
{
|
||||
#[Field(type: 'int', length: 11, nullable: false, unsigned: true, autoIncrement: true, primary: true)]
|
||||
public $id;
|
||||
|
||||
#[Field(type: 'varchar', length: 64, nullable: false, default: '', comment: '节点ID(区分不同采集节点)')]
|
||||
public $node_id;
|
||||
|
||||
#[Field(type: 'int', length: 11, nullable: false, unsigned: true, comment: '统计日期(YYYYMMDD)')]
|
||||
public $stat_date;
|
||||
|
||||
|
||||
@@ -8,13 +8,16 @@ use app\common\scheme\attribute\Field;
|
||||
use app\common\scheme\attribute\Index;
|
||||
|
||||
#[Table(name: 'nginx_stat_url', comment: 'nginx 按 URL 聚合统计')]
|
||||
#[Index(columns: ['stat_date', 'uri'], name: 'uniq_date_uri', type: 'UNIQUE')]
|
||||
#[Index(columns: ['node_id', 'stat_date', 'uri'], name: 'uniq_node_date_uri', type: 'UNIQUE')]
|
||||
#[Index(columns: ['stat_date'], name: 'idx_stat_date', type: 'NORMAL')]
|
||||
class NginxStatUrl extends BaseScheme
|
||||
{
|
||||
#[Field(type: 'int', length: 11, nullable: false, unsigned: true, autoIncrement: true, primary: true)]
|
||||
public $id;
|
||||
|
||||
#[Field(type: 'varchar', length: 64, nullable: false, default: '', comment: '节点ID(区分不同采集节点)')]
|
||||
public $node_id;
|
||||
|
||||
#[Field(type: 'int', length: 11, nullable: false, unsigned: true, comment: '统计日期(YYYYMMDD)')]
|
||||
public $stat_date;
|
||||
|
||||
|
||||
@@ -1,5 +1,47 @@
|
||||
<div class="layuimini-container">
|
||||
<div class="layuimini-main">
|
||||
{if condition="$positions->isEmpty()"}
|
||||
<div class="layui-card" style="margin-bottom: 10px;">
|
||||
<div class="layui-card-body" style="padding: 12px 15px;">
|
||||
<i class="layui-icon layui-icon-tips" style="color: #FFB800; font-size: 16px;"></i>
|
||||
<span style="color: #666;">暂无采集位置记录,请确认 nginx_log.enable 已开启且 timer 正在运行。</span>
|
||||
</div>
|
||||
</div>
|
||||
{else/}
|
||||
{volist name="positions" id="pos"}
|
||||
<div class="layui-card" style="margin-bottom: 10px;">
|
||||
<div class="layui-card-body" style="padding: 12px 15px;">
|
||||
<div style="display: flex; flex-wrap: wrap; gap: 24px; align-items: center; font-size: 13px;">
|
||||
<div>
|
||||
<span style="color: #999;">监控文件</span><br>
|
||||
<code style="font-size: 12px;">{$pos.file_path}</code>
|
||||
</div>
|
||||
<div>
|
||||
<span style="color: #999;">节点</span><br>
|
||||
<strong>{$pos.node_id}</strong>
|
||||
</div>
|
||||
<div>
|
||||
<span style="color: #999;">已读取</span><br>
|
||||
<strong>{$pos.offset_human}</strong>
|
||||
</div>
|
||||
<div>
|
||||
<span style="color: #999;">最后采集</span><br>
|
||||
<span>{$pos.last_read_time_text}</span>
|
||||
</div>
|
||||
<div>
|
||||
<span style="color: #999;">解析失败</span><br>
|
||||
{if condition="$pos.parse_fail_count > 0"}
|
||||
<span style="color: #FF5722; font-weight: bold;">{$pos.parse_fail_count} 次</span>
|
||||
{else/}
|
||||
<span style="color: #5FB878;">0 次</span>
|
||||
{/if}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{/volist}
|
||||
{/if}
|
||||
|
||||
<table id="currentTable" class="layui-table layui-hide"
|
||||
data-auth-index="{:auth('nginx.access_log/index')}"
|
||||
data-auth-read="{:auth('nginx.access_log/read')}"
|
||||
|
||||
@@ -17,6 +17,7 @@ $(function(){
|
||||
else if (s >= 500) cls = 'layui-badge layui-bg-red';
|
||||
return '<span class="' + cls + '">' + d.status + '</span>';
|
||||
}},
|
||||
{field: 'node_id', title: '节点', width: 120, searchOp: '='},
|
||||
{field: 'remote_addr', title: '客户端IP', width: 140, searchOp: '%*%'},
|
||||
{field: 'request_time', title: '耗时(秒)', width: 100, search: 'number_limit', templet: function(d) {
|
||||
var t = parseFloat(d.request_time);
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<div class="layui-card-header detail-header">
|
||||
<div class="layui-row">
|
||||
<div class="layui-col-md9">
|
||||
<h2 class="detail-title">#{$row.id} {$title}</h2>
|
||||
<h2 class="detail-title">#{$row.id} {$title|default=''}</h2>
|
||||
<div class="detail-id">ID: {$row.id}</div>
|
||||
</div>
|
||||
<div class="layui-col-md3 text-right detail-actions">
|
||||
|
||||
70
database/migrations/20260801125222_nginx_log_add_node_id.php
Normal file
70
database/migrations/20260801125222_nginx_log_add_node_id.php
Normal file
@@ -0,0 +1,70 @@
|
||||
<?php
|
||||
|
||||
use think\migration\Migrator;
|
||||
|
||||
/**
|
||||
* nginx 日志表新增 node_id 字段(多节点采集支持)
|
||||
*
|
||||
* 变更内容:
|
||||
* - 6 张表全部新增 node_id varchar(64) default ''(采集节点 ID,空=全局汇总)
|
||||
* - position / 4 张 stat 表的唯一键前置 node_id(多节点隔离,避免聚合 INSERT 触发 duplicate key)
|
||||
* - raw 表(nginx_access_log)加 node_id + time_local 普通索引,无唯一键变更
|
||||
*
|
||||
* 注意:
|
||||
* - 不回填历史 raw/stat 数据(旧数据 node_id 为空,视为全局汇总)
|
||||
* - 不在 migration 中 UPDATE position 的 node_id(多节点共享 DB 下第一个节点会抢走所有 position,
|
||||
* position 迁移改为 getLastPosition() 懒加载接管,见配套 Task)
|
||||
* - phinx 表名不带 ul_ 前缀(phinx 自动加前缀)
|
||||
*/
|
||||
class NginxLogAddNodeId extends Migrator
|
||||
{
|
||||
public function change()
|
||||
{
|
||||
// ==================== 1. nginx_log_position ====================
|
||||
// 旧唯一键 uniq_file_path(file_path) → 新 uniq_node_file(node_id, file_path)
|
||||
$this->table('nginx_log_position')
|
||||
->addColumn('node_id', 'string', ['limit' => 64, 'default' => '', 'comment' => '采集节点 ID(空=全局汇总)'])
|
||||
->removeIndexByName('uniq_file_path')
|
||||
->addIndex(['node_id', 'file_path'], ['unique' => true, 'name' => 'uniq_node_file'])
|
||||
->update();
|
||||
|
||||
// ==================== 2. nginx_access_log(raw 表)====================
|
||||
// 仅加字段 + 普通索引(按节点 + 时间查询),无唯一键变更
|
||||
$this->table('nginx_access_log')
|
||||
->addColumn('node_id', 'string', ['limit' => 64, 'default' => '', 'comment' => '采集节点 ID(空=全局汇总)'])
|
||||
->addIndex(['node_id', 'time_local'], ['name' => 'idx_node_time'])
|
||||
->update();
|
||||
|
||||
// ==================== 3. nginx_stat_hour ====================
|
||||
// 旧唯一键 uniq_date_hour(stat_date, stat_hour) → uniq_node_date_hour(node_id, stat_date, stat_hour)
|
||||
$this->table('nginx_stat_hour')
|
||||
->addColumn('node_id', 'string', ['limit' => 64, 'default' => '', 'comment' => '采集节点 ID(空=全局汇总)'])
|
||||
->removeIndexByName('uniq_date_hour')
|
||||
->addIndex(['node_id', 'stat_date', 'stat_hour'], ['unique' => true, 'name' => 'uniq_node_date_hour'])
|
||||
->update();
|
||||
|
||||
// ==================== 4. nginx_stat_url ====================
|
||||
// 旧唯一键 uniq_date_uri(stat_date, uri) → uniq_node_date_uri(node_id, stat_date, uri)
|
||||
$this->table('nginx_stat_url')
|
||||
->addColumn('node_id', 'string', ['limit' => 64, 'default' => '', 'comment' => '采集节点 ID(空=全局汇总)'])
|
||||
->removeIndexByName('uniq_date_uri')
|
||||
->addIndex(['node_id', 'stat_date', 'uri'], ['unique' => true, 'name' => 'uniq_node_date_uri'])
|
||||
->update();
|
||||
|
||||
// ==================== 5. nginx_stat_referer ====================
|
||||
// 旧唯一键 uniq_date_referer(stat_date, referer_domain) → uniq_node_date_referer(node_id, stat_date, referer_domain)
|
||||
$this->table('nginx_stat_referer')
|
||||
->addColumn('node_id', 'string', ['limit' => 64, 'default' => '', 'comment' => '采集节点 ID(空=全局汇总)'])
|
||||
->removeIndexByName('uniq_date_referer')
|
||||
->addIndex(['node_id', 'stat_date', 'referer_domain'], ['unique' => true, 'name' => 'uniq_node_date_referer'])
|
||||
->update();
|
||||
|
||||
// ==================== 6. nginx_stat_ua ====================
|
||||
// 旧唯一键 uniq_date_type_name(stat_date, ua_type, ua_name) → uniq_node_date_type_name(node_id, stat_date, ua_type, ua_name)
|
||||
$this->table('nginx_stat_ua')
|
||||
->addColumn('node_id', 'string', ['limit' => 64, 'default' => '', 'comment' => '采集节点 ID(空=全局汇总)'])
|
||||
->removeIndexByName('uniq_date_type_name')
|
||||
->addIndex(['node_id', 'stat_date', 'ua_type', 'ua_name'], ['unique' => true, 'name' => 'uniq_node_date_type_name'])
|
||||
->update();
|
||||
}
|
||||
}
|
||||
@@ -56,6 +56,24 @@ class AccessLogBase extends AdminController
|
||||
return json($data);
|
||||
}
|
||||
|
||||
// 采集位置信息(合并自原"读取位置管理"独立页面)
|
||||
$positions = \app\admin\model\NginxLogPosition::order('id', 'desc')->select()->map(function ($item) {
|
||||
$bytes = (int) $item['offset'];
|
||||
if ($bytes >= 1048576) {
|
||||
$item['offset_human'] = number_format($bytes / 1048576, 2) . ' MB';
|
||||
} elseif ($bytes >= 1024) {
|
||||
$item['offset_human'] = number_format($bytes / 1024, 2) . ' KB';
|
||||
} else {
|
||||
$item['offset_human'] = $bytes . ' B';
|
||||
}
|
||||
$item['last_read_time_text'] = $item['last_read_time'] > 0
|
||||
? date('Y-m-d H:i:s', (int) $item['last_read_time'])
|
||||
: '-';
|
||||
|
||||
return $item;
|
||||
});
|
||||
$this->assign('positions', $positions);
|
||||
|
||||
return $this->fetch();
|
||||
}
|
||||
|
||||
|
||||
@@ -45,11 +45,13 @@ class NginxLogStatBase extends AdminController
|
||||
$period = 'day';
|
||||
}
|
||||
|
||||
$nodeId = (string) $request->param('node_id', '');
|
||||
|
||||
// 解析时间范围(YYYYMMDD int 表示)
|
||||
[$startDate, $endDate] = $this->resolveDateRange($period);
|
||||
|
||||
if ($request->isAjax()) {
|
||||
$data = $this->buildStatData($period, $startDate, $endDate);
|
||||
$data = $this->buildStatData($period, $startDate, $endDate, $nodeId);
|
||||
|
||||
return json([
|
||||
'code' => 0,
|
||||
@@ -63,6 +65,11 @@ class NginxLogStatBase extends AdminController
|
||||
$this->assign('start', $startDate);
|
||||
$this->assign('end', $endDate);
|
||||
|
||||
// 在线节点列表(供视图渲染节点筛选下拉)
|
||||
$nodes = \app\admin\model\SystemHost::where('status', 1)->order('node_id', 'asc')->select();
|
||||
$this->assign('nodes', $nodes);
|
||||
$this->assign('node_id', $nodeId);
|
||||
|
||||
return $this->fetch();
|
||||
}
|
||||
|
||||
@@ -140,20 +147,20 @@ class NginxLogStatBase extends AdminController
|
||||
/**
|
||||
* 组装完整统计数据
|
||||
*/
|
||||
protected function buildStatData(string $period, int $startDate, int $endDate): array
|
||||
protected function buildStatData(string $period, int $startDate, int $endDate, string $nodeId = ''): array
|
||||
{
|
||||
// 时间范围不合法 → 空数据(不崩)
|
||||
if ($startDate <= 0 || $endDate <= 0 || $startDate > $endDate) {
|
||||
return $this->emptyData($period);
|
||||
}
|
||||
|
||||
$summary = $this->querySummary($startDate, $endDate);
|
||||
$trend = $this->queryTrend($period, $startDate, $endDate);
|
||||
$summary = $this->querySummary($startDate, $endDate, $nodeId);
|
||||
$trend = $this->queryTrend($period, $startDate, $endDate, $nodeId);
|
||||
|
||||
$topN = $this->getTopN();
|
||||
$topUrls = $this->queryTopByDateRange(NginxStatUrl::class, 'uri', $startDate, $endDate, $topN);
|
||||
$topReferers = $this->queryTopByDateRange(NginxStatReferer::class, 'referer_domain', $startDate, $endDate, $topN);
|
||||
$topUas = $this->queryTopByDateRange(NginxStatUa::class, 'ua_name', $startDate, $endDate, $topN);
|
||||
$topUrls = $this->queryTopByDateRange(NginxStatUrl::class, 'uri', $startDate, $endDate, $topN, $nodeId);
|
||||
$topReferers = $this->queryTopByDateRange(NginxStatReferer::class, 'referer_domain', $startDate, $endDate, $topN, $nodeId);
|
||||
$topUas = $this->queryTopByDateRange(NginxStatUa::class, 'ua_name', $startDate, $endDate, $topN, $nodeId);
|
||||
|
||||
// 跨周期 UV 估算标注:仅 day 为精确值,week/month/year/custom 均为按日累加的估算值
|
||||
$uvIsEstimate = $period !== 'day';
|
||||
@@ -208,7 +215,7 @@ class NginxLogStatBase extends AdminController
|
||||
* 整体指标:SUM stat_hour
|
||||
* avg_request_time 按 PV 加权平均(避免简单 AVG 被低 PV 时段拉偏)
|
||||
*/
|
||||
protected function querySummary(int $startDate, int $endDate): array
|
||||
protected function querySummary(int $startDate, int $endDate, string $nodeId = ''): array
|
||||
{
|
||||
$sumFields = [
|
||||
'IFNULL(SUM(pv),0) AS pv',
|
||||
@@ -223,6 +230,7 @@ class NginxLogStatBase extends AdminController
|
||||
];
|
||||
|
||||
$row = NginxStatHour::where('stat_date', 'between', [$startDate, $endDate])
|
||||
->where('node_id', $nodeId)
|
||||
->field(implode(',', $sumFields))
|
||||
->find();
|
||||
|
||||
@@ -252,10 +260,11 @@ class NginxLogStatBase extends AdminController
|
||||
* - year → 12 个月(按 stat_date DIV 100 分组,补全 1~12 月)
|
||||
* - custom→ 按 stat_date 分组,补全 start~end 日期序列
|
||||
*/
|
||||
protected function queryTrend(string $period, int $startDate, int $endDate): array
|
||||
protected function queryTrend(string $period, int $startDate, int $endDate, string $nodeId = ''): array
|
||||
{
|
||||
if ($period === 'day') {
|
||||
$rows = NginxStatHour::where('stat_date', 'between', [$startDate, $endDate])
|
||||
->where('node_id', $nodeId)
|
||||
->field('stat_hour, IFNULL(SUM(pv),0) AS pv, IFNULL(SUM(uv),0) AS uv')
|
||||
->group('stat_hour')
|
||||
->select()
|
||||
@@ -280,6 +289,7 @@ class NginxLogStatBase extends AdminController
|
||||
|
||||
if ($period === 'year') {
|
||||
$rows = NginxStatHour::where('stat_date', 'between', [$startDate, $endDate])
|
||||
->where('node_id', $nodeId)
|
||||
->field('(stat_date DIV 100) AS ym, IFNULL(SUM(pv),0) AS pv, IFNULL(SUM(uv),0) AS uv')
|
||||
->group('ym')
|
||||
->order('ym asc')
|
||||
@@ -307,6 +317,7 @@ class NginxLogStatBase extends AdminController
|
||||
|
||||
// week / month / custom → 按 stat_date 分组,补全日期序列
|
||||
$rows = NginxStatHour::where('stat_date', 'between', [$startDate, $endDate])
|
||||
->where('node_id', $nodeId)
|
||||
->field('stat_date, IFNULL(SUM(pv),0) AS pv, IFNULL(SUM(uv),0) AS uv')
|
||||
->group('stat_date')
|
||||
->order('stat_date asc')
|
||||
@@ -340,10 +351,12 @@ class NginxLogStatBase extends AdminController
|
||||
* @param string $modelClass 模型类(依赖倒置:调 app/admin/model 入口)
|
||||
* @param string $labelField 分组与展示的字段名
|
||||
* @param int $topN 取前 N 条;0 不限制
|
||||
* @param string $nodeId 节点 ID(空字符串=全局聚合行)
|
||||
*/
|
||||
protected function queryTopByDateRange(string $modelClass, string $labelField, int $startDate, int $endDate, int $topN): array
|
||||
protected function queryTopByDateRange(string $modelClass, string $labelField, int $startDate, int $endDate, int $topN, string $nodeId = ''): array
|
||||
{
|
||||
$query = $modelClass::where('stat_date', 'between', [$startDate, $endDate])
|
||||
->where('node_id', $nodeId)
|
||||
->field($labelField . ' AS label, IFNULL(SUM(pv),0) AS pv, IFNULL(SUM(uv),0) AS uv')
|
||||
->group($labelField)
|
||||
->order('pv desc');
|
||||
|
||||
@@ -209,20 +209,9 @@ $ul_system_menu = array(
|
||||
"sort" => 0,
|
||||
"status" => 1,
|
||||
),
|
||||
array(
|
||||
"id" => 300,
|
||||
"pid" => 0,
|
||||
"title" => "Nginx 日志分析",
|
||||
"icon" => "fa fa-chart-bar",
|
||||
"href" => "",
|
||||
"params" => "",
|
||||
"target" => "_self",
|
||||
"sort" => 0,
|
||||
"status" => 1,
|
||||
),
|
||||
array(
|
||||
"id" => 301,
|
||||
"pid" => 300,
|
||||
"pid" => 228,
|
||||
"title" => "访问统计仪表盘",
|
||||
"icon" => "fa fa-line-chart",
|
||||
"href" => "system.nginx_log_stat/index",
|
||||
@@ -233,7 +222,7 @@ $ul_system_menu = array(
|
||||
),
|
||||
array(
|
||||
"id" => 302,
|
||||
"pid" => 300,
|
||||
"pid" => 228,
|
||||
"title" => "原始访问日志",
|
||||
"icon" => "fa fa-file-text",
|
||||
"href" => "nginx.access_log/index",
|
||||
@@ -242,17 +231,6 @@ $ul_system_menu = array(
|
||||
"sort" => 0,
|
||||
"status" => 1,
|
||||
),
|
||||
array(
|
||||
"id" => 303,
|
||||
"pid" => 300,
|
||||
"title" => "读取位置管理",
|
||||
"icon" => "fa fa-map-marker",
|
||||
"href" => "nginx.log_position/index",
|
||||
"params" => "",
|
||||
"target" => "_self",
|
||||
"sort" => 0,
|
||||
"status" => 1,
|
||||
),
|
||||
);
|
||||
|
||||
|
||||
|
||||
@@ -13,11 +13,12 @@
|
||||
* stat_retention_days 统计聚合保留天数(0=永久)
|
||||
* topn_default Top N 默认值
|
||||
* exclude_static 是否排除静态资源访问
|
||||
* - system_menu 新增 4 条菜单(顶级分组 300 + 3 个子菜单 301/302/303)
|
||||
* 300 Nginx 日志分析(顶级分组,href 为空)
|
||||
* - system_menu 新增 2 条菜单(挂到系统管理 pid=228 下)
|
||||
* 301 访问统计仪表盘 system.nginx_log_stat/index
|
||||
* 302 原始访问日志 nginx.access_log/index
|
||||
* 303 读取位置管理 nginx.log_position/index
|
||||
* - ALTER TABLE:6 张 nginx_log 表新增 node_id 字段 + 唯一键前置 node_id(多节点采集支持)
|
||||
* 与 migration 20260801125222_nginx_log_add_node_id.php 保持一致,
|
||||
* 存量库通过 admin:update 幂等执行(check-then-alter)
|
||||
*
|
||||
* 设计说明:
|
||||
* - 幂等 check-then-insert(先查询存在性,不存在才插入)
|
||||
@@ -82,23 +83,12 @@ class UpdateFunction
|
||||
/**
|
||||
* 待新增菜单.
|
||||
* 字段顺序与 adminInitData/SystemMenu.php 一致:id/pid/title/icon/href/params/target/sort/status.
|
||||
* id 显式分配 300-303,与 adminInitData 保持一致(顶级 300 + 3 个子菜单)。
|
||||
* 挂到系统管理(pid=228)下,不创建独立顶级分组。
|
||||
*/
|
||||
public $newMenus = [
|
||||
[
|
||||
'id' => 300,
|
||||
'pid' => 0,
|
||||
'title' => 'Nginx 日志分析',
|
||||
'icon' => 'fa fa-chart-bar',
|
||||
'href' => '',
|
||||
'params' => '',
|
||||
'target' => '_self',
|
||||
'sort' => 0,
|
||||
'status' => 1,
|
||||
],
|
||||
[
|
||||
'id' => 301,
|
||||
'pid' => 300,
|
||||
'pid' => 228,
|
||||
'title' => '访问统计仪表盘',
|
||||
'icon' => 'fa fa-line-chart',
|
||||
'href' => 'system.nginx_log_stat/index',
|
||||
@@ -109,7 +99,7 @@ class UpdateFunction
|
||||
],
|
||||
[
|
||||
'id' => 302,
|
||||
'pid' => 300,
|
||||
'pid' => 228,
|
||||
'title' => '原始访问日志',
|
||||
'icon' => 'fa fa-file-text',
|
||||
'href' => 'nginx.access_log/index',
|
||||
@@ -118,24 +108,13 @@ class UpdateFunction
|
||||
'sort' => 0,
|
||||
'status' => 1,
|
||||
],
|
||||
[
|
||||
'id' => 303,
|
||||
'pid' => 300,
|
||||
'title' => '读取位置管理',
|
||||
'icon' => 'fa fa-map-marker',
|
||||
'href' => 'nginx.log_position/index',
|
||||
'params' => '',
|
||||
'target' => '_self',
|
||||
'sort' => 0,
|
||||
'status' => 1,
|
||||
],
|
||||
];
|
||||
|
||||
public function update()
|
||||
{
|
||||
$this->output->writeln('更新代码');
|
||||
|
||||
$this->output->info('v2.4.0:nginx_log 分析模块存量升级(补齐 4 条 sysconfig + 4 条菜单)');
|
||||
$this->output->info('v2.4.0:nginx_log 分析模块存量升级(补齐 4 条 sysconfig + 2 条菜单)');
|
||||
|
||||
$configTable = config('database.connections.mysql.prefix', 'ul_') . 'system_config';
|
||||
$menuTable = config('database.connections.mysql.prefix', 'ul_') . 'system_menu';
|
||||
@@ -190,6 +169,98 @@ class UpdateFunction
|
||||
|
||||
$totalSkipped = $configSkipped + $menuSkipped;
|
||||
$this->output->info("本次新增 {$configAdded} 条配置、{$menuAdded} 条菜单,跳过 {$totalSkipped} 条已存在");
|
||||
|
||||
// ==================== node_id 字段 + 唯一键变更(多节点采集支持)====================
|
||||
// 与 migration 20260801125222_nginx_log_add_node_id.php 保持一致
|
||||
// 6 张表全部新增 node_id varchar(64) default ''
|
||||
// position / 4 张 stat 表的唯一键前置 node_id(多节点隔离,避免聚合 INSERT 触发 duplicate key)
|
||||
// raw 表(nginx_access_log)加 node_id + time_local 普通索引,无唯一键变更
|
||||
$this->output->info('v2.4.0:nginx_log 6 张表新增 node_id 字段与唯一键调整');
|
||||
|
||||
$prefix = config('database.connections.mysql.prefix', 'ul_');
|
||||
|
||||
// 表短名 => 索引变更配置
|
||||
// drop_idx: 要删除的旧索引名(raw 表无)
|
||||
// add_idx: 新索引 [name, cols, unique]
|
||||
$nodeIdTables = [
|
||||
'nginx_log_position' => [
|
||||
'drop_idx' => 'uniq_file_path',
|
||||
'add_idx' => ['name' => 'uniq_node_file', 'cols' => ['node_id', 'file_path'], 'unique' => true],
|
||||
],
|
||||
'nginx_access_log' => [
|
||||
'add_idx' => ['name' => 'idx_node_time', 'cols' => ['node_id', 'time_local'], 'unique' => false],
|
||||
],
|
||||
'nginx_stat_hour' => [
|
||||
'drop_idx' => 'uniq_date_hour',
|
||||
'add_idx' => ['name' => 'uniq_node_date_hour', 'cols' => ['node_id', 'stat_date', 'stat_hour'], 'unique' => true],
|
||||
],
|
||||
'nginx_stat_url' => [
|
||||
'drop_idx' => 'uniq_date_uri',
|
||||
'add_idx' => ['name' => 'uniq_node_date_uri', 'cols' => ['node_id', 'stat_date', 'uri'], 'unique' => true],
|
||||
],
|
||||
'nginx_stat_referer' => [
|
||||
'drop_idx' => 'uniq_date_referer',
|
||||
'add_idx' => ['name' => 'uniq_node_date_referer', 'cols' => ['node_id', 'stat_date', 'referer_domain'], 'unique' => true],
|
||||
],
|
||||
'nginx_stat_ua' => [
|
||||
'drop_idx' => 'uniq_date_type_name',
|
||||
'add_idx' => ['name' => 'uniq_node_date_type_name', 'cols' => ['node_id', 'stat_date', 'ua_type', 'ua_name'], 'unique' => true],
|
||||
],
|
||||
];
|
||||
|
||||
$colAdded = 0;
|
||||
$colSkipped = 0;
|
||||
foreach ($nodeIdTables as $tableShort => $idxCfg) {
|
||||
$fullTable = $prefix . $tableShort;
|
||||
|
||||
// 防御性检查:表是否存在(未运行 migration 时跳过,不报错)
|
||||
$tableExists = Db::query(
|
||||
"SELECT COUNT(*) AS cnt FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ?",
|
||||
[$fullTable]
|
||||
);
|
||||
if (empty($tableExists) || (int) $tableExists[0]['cnt'] === 0) {
|
||||
$this->output->writeln(" - {$fullTable} 表不存在,跳过(请先执行 migrate:run)");
|
||||
continue;
|
||||
}
|
||||
|
||||
// 幂等 1:检查 node_id 字段是否存在
|
||||
$colExists = Db::query(
|
||||
"SELECT COUNT(*) AS cnt FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND COLUMN_NAME = 'node_id'",
|
||||
[$fullTable]
|
||||
);
|
||||
if (!empty($colExists) && (int) $colExists[0]['cnt'] > 0) {
|
||||
$this->output->writeln(" - {$fullTable} node_id 字段已存在,跳过");
|
||||
++$colSkipped;
|
||||
} else {
|
||||
Db::execute("ALTER TABLE `{$fullTable}` ADD COLUMN `node_id` VARCHAR(64) NOT NULL DEFAULT '' COMMENT '采集节点 ID(空=全局汇总)'");
|
||||
$this->output->writeln(" - {$fullTable} 新增 node_id 字段完成");
|
||||
++$colAdded;
|
||||
}
|
||||
|
||||
// 幂等 2:唯一键/索引变更(检查新索引是否存在)
|
||||
$newIdx = $idxCfg['add_idx']['name'] ?? null;
|
||||
if ($newIdx !== null) {
|
||||
$idxExists = Db::query(
|
||||
"SELECT COUNT(*) AS cnt FROM information_schema.STATISTICS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND INDEX_NAME = ?",
|
||||
[$fullTable, $newIdx]
|
||||
);
|
||||
if (empty($idxExists) || (int) $idxExists[0]['cnt'] === 0) {
|
||||
// 删除旧索引(position / stat 表有,raw 表无)
|
||||
if (!empty($idxCfg['drop_idx'])) {
|
||||
Db::execute("ALTER TABLE `{$fullTable}` DROP INDEX `{$idxCfg['drop_idx']}`");
|
||||
}
|
||||
// 新增新索引
|
||||
$idxType = !empty($idxCfg['add_idx']['unique']) ? 'UNIQUE' : 'INDEX';
|
||||
$colsStr = implode(', ', array_map(function ($c) {
|
||||
return "`{$c}`";
|
||||
}, $idxCfg['add_idx']['cols']));
|
||||
Db::execute("ALTER TABLE `{$fullTable}` ADD {$idxType} `{$newIdx}` ({$colsStr})");
|
||||
$this->output->writeln(" - {$fullTable} 索引调整:{$newIdx}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$this->output->info("node_id 迁移完成(新增字段 {$colAdded} 表,跳过 {$colSkipped} 表已存在)");
|
||||
$this->output->writeln('更新代码完成');
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,6 +19,14 @@
|
||||
<input type="text" class="layui-input" id="dateEnd" placeholder="结束日期" autocomplete="off" style="width:130px;" readonly>
|
||||
<button type="button" class="layui-btn layui-btn-sm layui-btn-normal" id="customSearchBtn">查询</button>
|
||||
</span>
|
||||
<div class="layui-input-inline" style="margin-left: 15px; width: 180px; vertical-align: middle;">
|
||||
<select id="node-filter" lay-filter="nodeFilter">
|
||||
<option value="">全部节点</option>
|
||||
{volist name="nodes" id="node"}
|
||||
<option value="{$node.node_id}" {if condition="$node_id eq $node.node_id"}selected{/if}>{$node.node_id}</option>
|
||||
{/volist}
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -6,11 +6,15 @@
|
||||
* 后端 URL 不硬编码:用 ua.url('system.nginx_log_stat/index') 由 CONFIG.ADMIN 拼接。
|
||||
* AJAX 触发方式:URL 带 _ajax=1(不依赖 Accept header,命令行测试友好)。
|
||||
*/
|
||||
layui.use(['table', 'laydate', 'util'], function () {
|
||||
layui.use(['table', 'laydate', 'util', 'form'], function () {
|
||||
|
||||
var table = layui.table;
|
||||
var laydate = layui.laydate;
|
||||
var util = layui.util;
|
||||
var form = layui.form;
|
||||
|
||||
// 渲染节点下拉框
|
||||
form.render();
|
||||
|
||||
// ── 配置(URL 不硬编码:由 ua.url 拼接 /{ADMIN}/system.nginx_log_stat/index)──
|
||||
var STAT_INDEX_URL = ua.url('system.nginx_log_stat/index');
|
||||
@@ -129,6 +133,15 @@ layui.use(['table', 'laydate', 'util'], function () {
|
||||
loadData({ period: 'custom', start: startVal, end: endVal });
|
||||
});
|
||||
|
||||
// ================================================================
|
||||
// 节点下拉框切换
|
||||
// ================================================================
|
||||
|
||||
form.on('select(nodeFilter)', function (data) {
|
||||
// 切换节点时用当前 period 重新加载
|
||||
loadData({ period: currentPeriod });
|
||||
});
|
||||
|
||||
// ================================================================
|
||||
// AJAX 加载数据
|
||||
// ================================================================
|
||||
@@ -141,6 +154,10 @@ layui.use(['table', 'laydate', 'util'], function () {
|
||||
function loadData(extra) {
|
||||
extra = extra || {};
|
||||
|
||||
// 当前选中的节点 ID(默认空字符串=全部节点)
|
||||
var nodeId = $('#node-filter').val() || '';
|
||||
extra.node_id = nodeId;
|
||||
|
||||
var params = $.extend({ _ajax: 1, period: currentPeriod }, extra);
|
||||
|
||||
ua.request.get({
|
||||
|
||||
@@ -27,7 +27,7 @@ return [
|
||||
'target' => '/tools/timer.NginxLogImport/do',
|
||||
'frequency' => 300,
|
||||
'concurrency' => 1,
|
||||
'run_type' => 'auto',
|
||||
'run_type' => 'all',
|
||||
],
|
||||
[
|
||||
'name' => 'nginx_log_aggregate',
|
||||
|
||||
@@ -24,9 +24,9 @@ class NginxLogAggregatorServiceBase
|
||||
/**
|
||||
* 聚合指定日期的指定小时.
|
||||
*
|
||||
* 流程:
|
||||
* 1. stat_hour:DELETE WHERE stat_date=:d AND stat_hour=:h → INSERT 1 行(该小时聚合)
|
||||
* 2. stat_url / stat_referer / stat_ua:DELETE WHERE stat_date=:d(整天重跑)→ INSERT top N 行
|
||||
* 流程(每张表生成 全局行 node_id='' + 各节点行):
|
||||
* 1. stat_hour:DELETE WHERE stat_date=:d AND stat_hour=:h → INSERT 全局 1 行 + 每节点 1 行
|
||||
* 2. stat_url / stat_referer / stat_ua:DELETE WHERE stat_date=:d(整天重跑)→ INSERT 全局 topN + 每节点 topN
|
||||
*
|
||||
* @param int $statDate 日期 YYYYMMDD(如 20260728)
|
||||
* @param int $statHour 小时 0-23
|
||||
@@ -65,142 +65,50 @@ class NginxLogAggregatorServiceBase
|
||||
|
||||
Db::startTrans();
|
||||
try {
|
||||
// 1. stat_hour:DELETE + INSERT 1 行
|
||||
// DELETE(逻辑不变):stat_hour 按小时,stat_url/referer/ua 按整天
|
||||
NginxStatHour::where('stat_date', $statDate)
|
||||
->where('stat_hour', $statHour)
|
||||
->delete();
|
||||
|
||||
$hourSql = "INSERT INTO `{$hourTable}`
|
||||
(stat_date, stat_hour, pv, uv, total_bytes, status_2xx, status_3xx, status_4xx, status_5xx, status_other, avg_request_time)
|
||||
SELECT :stat_date, :stat_hour,
|
||||
COUNT(*) AS pv,
|
||||
COUNT(DISTINCT remote_addr) AS uv,
|
||||
COALESCE(SUM(body_bytes_sent), 0) AS total_bytes,
|
||||
COALESCE(SUM(CASE WHEN status BETWEEN 200 AND 299 THEN 1 ELSE 0 END), 0) AS status_2xx,
|
||||
COALESCE(SUM(CASE WHEN status BETWEEN 300 AND 399 THEN 1 ELSE 0 END), 0) AS status_3xx,
|
||||
COALESCE(SUM(CASE WHEN status BETWEEN 400 AND 499 THEN 1 ELSE 0 END), 0) AS status_4xx,
|
||||
COALESCE(SUM(CASE WHEN status BETWEEN 500 AND 599 THEN 1 ELSE 0 END), 0) AS status_5xx,
|
||||
COALESCE(SUM(CASE WHEN status < 200 OR status > 599 THEN 1 ELSE 0 END), 0) AS status_other,
|
||||
COALESCE(AVG(request_time), 0) AS avg_request_time
|
||||
FROM `{$rawTable}`
|
||||
WHERE time_local >= :hour_start AND time_local < :hour_end
|
||||
HAVING COUNT(*) > 0";
|
||||
|
||||
$statHourRows = (int) Db::execute($hourSql, [
|
||||
'stat_date' => $statDate,
|
||||
'stat_hour' => $statHour,
|
||||
'hour_start' => $hourStart,
|
||||
'hour_end' => $hourEnd,
|
||||
]);
|
||||
|
||||
// 2. stat_url:DELETE(整天)+ INSERT topN
|
||||
NginxStatUrl::where('stat_date', $statDate)->delete();
|
||||
|
||||
$urlLimit = $topn > 0 ? 'LIMIT ' . $topn : '';
|
||||
$urlSql = "INSERT INTO `{$urlTable}`
|
||||
(stat_date, uri, pv, uv, total_bytes, avg_request_time)
|
||||
SELECT :stat_date, uri,
|
||||
COUNT(*) AS pv,
|
||||
COUNT(DISTINCT remote_addr) AS uv,
|
||||
COALESCE(SUM(body_bytes_sent), 0) AS total_bytes,
|
||||
COALESCE(AVG(request_time), 0) AS avg_request_time
|
||||
FROM `{$rawTable}`
|
||||
WHERE time_local >= :day_start AND time_local < :day_end
|
||||
GROUP BY uri
|
||||
ORDER BY pv DESC
|
||||
{$urlLimit}";
|
||||
|
||||
$statUrlRows = (int) Db::execute($urlSql, [
|
||||
'stat_date' => $statDate,
|
||||
'day_start' => $dayStart,
|
||||
'day_end' => $dayEnd,
|
||||
]);
|
||||
|
||||
// 3. stat_referer:DELETE + INSERT topN(从 http_referer 提取 domain)
|
||||
NginxStatReferer::where('stat_date', $statDate)->delete();
|
||||
|
||||
$refererLimit = $topn > 0 ? 'LIMIT ' . $topn : '';
|
||||
$refererSql = "INSERT INTO `{$refererTable}`
|
||||
(stat_date, referer_domain, pv, uv)
|
||||
SELECT :stat_date,
|
||||
CASE
|
||||
WHEN http_referer IS NULL OR http_referer = '' OR http_referer = '-' THEN '-'
|
||||
ELSE SUBSTRING_INDEX(
|
||||
SUBSTRING_INDEX(
|
||||
REPLACE(REPLACE(http_referer, 'https://', ''), 'http://', ''),
|
||||
'/', 1
|
||||
),
|
||||
'?', 1
|
||||
)
|
||||
END AS referer_domain,
|
||||
COUNT(*) AS pv,
|
||||
COUNT(DISTINCT remote_addr) AS uv
|
||||
FROM `{$rawTable}`
|
||||
WHERE time_local >= :day_start AND time_local < :day_end
|
||||
GROUP BY referer_domain
|
||||
ORDER BY pv DESC
|
||||
{$refererLimit}";
|
||||
|
||||
$statRefererRows = (int) Db::execute($refererSql, [
|
||||
'stat_date' => $statDate,
|
||||
'day_start' => $dayStart,
|
||||
'day_end' => $dayEnd,
|
||||
]);
|
||||
|
||||
// 4. stat_ua:DELETE + INSERT topN(从 http_user_agent 分类)
|
||||
NginxStatUa::where('stat_date', $statDate)->delete();
|
||||
|
||||
$uaLimit = $topn > 0 ? 'LIMIT ' . $topn : '';
|
||||
$uaSql = "INSERT INTO `{$uaTable}`
|
||||
(stat_date, ua_type, ua_name, pv, uv)
|
||||
SELECT :stat_date,
|
||||
CASE
|
||||
WHEN http_user_agent LIKE '%bot%'
|
||||
OR http_user_agent LIKE '%spider%'
|
||||
OR http_user_agent LIKE '%crawl%'
|
||||
OR http_user_agent LIKE '%slurp%'
|
||||
OR http_user_agent LIKE '%bingpreview%'
|
||||
OR http_user_agent LIKE '%facebookexternalhit%'
|
||||
OR http_user_agent LIKE '%twitterbot%' THEN 'spider'
|
||||
WHEN http_user_agent IS NULL OR http_user_agent = '' OR http_user_agent = '-' THEN 'unknown'
|
||||
WHEN http_user_agent LIKE '%Mozilla%'
|
||||
OR http_user_agent LIKE '%Chrome%'
|
||||
OR http_user_agent LIKE '%Safari%'
|
||||
OR http_user_agent LIKE '%Firefox%'
|
||||
OR http_user_agent LIKE '%Edg%'
|
||||
OR http_user_agent LIKE '%Opera%'
|
||||
OR http_user_agent LIKE '%MSIE%'
|
||||
OR http_user_agent LIKE '%Trident%' THEN 'browser'
|
||||
ELSE 'unknown'
|
||||
END AS ua_type,
|
||||
CASE
|
||||
WHEN http_user_agent LIKE '%Googlebot%' THEN 'Googlebot'
|
||||
WHEN http_user_agent LIKE '%Baiduspider%' THEN 'Baiduspider'
|
||||
WHEN http_user_agent LIKE '%bingbot%' THEN 'Bingbot'
|
||||
WHEN http_user_agent LIKE '%DuckDuckBot%' THEN 'DuckDuckBot'
|
||||
WHEN http_user_agent LIKE '%YandexBot%' THEN 'YandexBot'
|
||||
WHEN http_user_agent LIKE '%Edg/%' THEN 'Microsoft Edge'
|
||||
WHEN http_user_agent LIKE '%OPR/%' OR http_user_agent LIKE '%Opera%' THEN 'Opera'
|
||||
WHEN http_user_agent LIKE '%Firefox/%' THEN 'Firefox'
|
||||
WHEN http_user_agent LIKE '%Chrome/%' THEN 'Chrome'
|
||||
WHEN http_user_agent LIKE '%Safari/%' THEN 'Safari'
|
||||
WHEN http_user_agent LIKE '%MSIE%' OR http_user_agent LIKE '%Trident%' THEN 'Internet Explorer'
|
||||
WHEN http_user_agent IS NULL OR http_user_agent = '' OR http_user_agent = '-' THEN 'Unknown'
|
||||
ELSE 'Other'
|
||||
END AS ua_name,
|
||||
COUNT(*) AS pv,
|
||||
COUNT(DISTINCT remote_addr) AS uv
|
||||
FROM `{$rawTable}`
|
||||
WHERE time_local >= :day_start AND time_local < :day_end
|
||||
GROUP BY ua_type, ua_name
|
||||
ORDER BY pv DESC
|
||||
{$uaLimit}";
|
||||
$statHourRows = 0;
|
||||
$statUrlRows = 0;
|
||||
$statRefererRows = 0;
|
||||
$statUaRows = 0;
|
||||
|
||||
$statUaRows = (int) Db::execute($uaSql, [
|
||||
'stat_date' => $statDate,
|
||||
'day_start' => $dayStart,
|
||||
'day_end' => $dayEnd,
|
||||
]);
|
||||
// 批次 A:全局行(node_id='',跨所有节点聚合,UV 精确)
|
||||
$global = $this->insertAggregatesForScope(
|
||||
'', null,
|
||||
$statDate, $statHour, $hourStart, $hourEnd, $dayStart, $dayEnd, $topn,
|
||||
$rawTable, $hourTable, $urlTable, $refererTable, $uaTable
|
||||
);
|
||||
$statHourRows += $global['stat_hour'];
|
||||
$statUrlRows += $global['stat_url'];
|
||||
$statRefererRows += $global['stat_referer'];
|
||||
$statUaRows += $global['stat_ua'];
|
||||
|
||||
// 批次 B:按节点行(循环每个 node_id)
|
||||
// 用 day 窗口取节点列表:保证 day 窗口表(url/referer/ua)的 per-node 行不会因
|
||||
// 某节点"本小时无活动"而被 DELETE 后丢失;stat_hour 的 per-node 行由
|
||||
// HAVING COUNT(*)>0 自然过滤掉本小时无活动的节点。
|
||||
$nodeRows = Db::query(
|
||||
"SELECT DISTINCT node_id FROM `{$rawTable}` WHERE time_local >= :day_start AND time_local < :day_end AND node_id != ''",
|
||||
['day_start' => $dayStart, 'day_end' => $dayEnd]
|
||||
);
|
||||
foreach ($nodeRows as $nodeRow) {
|
||||
$nid = (string) $nodeRow['node_id'];
|
||||
$perNode = $this->insertAggregatesForScope(
|
||||
$nid, $nid,
|
||||
$statDate, $statHour, $hourStart, $hourEnd, $dayStart, $dayEnd, $topn,
|
||||
$rawTable, $hourTable, $urlTable, $refererTable, $uaTable
|
||||
);
|
||||
$statHourRows += $perNode['stat_hour'];
|
||||
$statUrlRows += $perNode['stat_url'];
|
||||
$statRefererRows += $perNode['stat_referer'];
|
||||
$statUaRows += $perNode['stat_ua'];
|
||||
}
|
||||
|
||||
Db::commit();
|
||||
|
||||
@@ -216,6 +124,175 @@ class NginxLogAggregatorServiceBase
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 为单个作用域(全局或某节点)执行 4 段 INSERT.
|
||||
*
|
||||
* - 全局作用域:$nodeId='' 且 $nodeFilter=null(不追加 node WHERE,跨所有节点聚合)
|
||||
* - 节点作用域:$nodeId=节点ID 且 $nodeFilter=节点ID(追加 AND node_id=:node_filter)
|
||||
*
|
||||
* stat_hour:1 行(无 GROUP BY,HAVING COUNT(*)>0)
|
||||
* stat_url / stat_referer / stat_ua:topN(各自 GROUP BY + LIMIT)
|
||||
*
|
||||
* @param string $nodeId INSERT 的 node_id 列值(全局为 '')
|
||||
* @param string|null $nodeFilter null=全局(不追加 WHERE);非 null=追加节点过滤
|
||||
* @return array {stat_hour:int, stat_url:int, stat_referer:int, stat_ua:int}
|
||||
*/
|
||||
protected function insertAggregatesForScope(
|
||||
string $nodeId,
|
||||
?string $nodeFilter,
|
||||
int $statDate,
|
||||
int $statHour,
|
||||
int $hourStart,
|
||||
int $hourEnd,
|
||||
int $dayStart,
|
||||
int $dayEnd,
|
||||
int $topn,
|
||||
string $rawTable,
|
||||
string $hourTable,
|
||||
string $urlTable,
|
||||
string $refererTable,
|
||||
string $uaTable
|
||||
): array {
|
||||
// 节点过滤片段:全局行无;节点行追加 AND node_id = :node_filter
|
||||
$nodeWhere = $nodeFilter === null ? '' : 'AND node_id = :node_filter';
|
||||
|
||||
// hour 窗口绑定(stat_hour 专用)
|
||||
$hourParams = [
|
||||
'node_id' => $nodeId,
|
||||
'stat_date' => $statDate,
|
||||
'stat_hour' => $statHour,
|
||||
'hour_start' => $hourStart,
|
||||
'hour_end' => $hourEnd,
|
||||
];
|
||||
if ($nodeFilter !== null) {
|
||||
$hourParams['node_filter'] = $nodeFilter;
|
||||
}
|
||||
|
||||
// day 窗口绑定(stat_url/referer/ua 共用)
|
||||
$dayParams = [
|
||||
'node_id' => $nodeId,
|
||||
'stat_date' => $statDate,
|
||||
'day_start' => $dayStart,
|
||||
'day_end' => $dayEnd,
|
||||
];
|
||||
if ($nodeFilter !== null) {
|
||||
$dayParams['node_filter'] = $nodeFilter;
|
||||
}
|
||||
|
||||
// 1. stat_hour:1 行(无 GROUP BY)
|
||||
$hourSql = "INSERT INTO `{$hourTable}`
|
||||
(node_id, stat_date, stat_hour, pv, uv, total_bytes, status_2xx, status_3xx, status_4xx, status_5xx, status_other, avg_request_time)
|
||||
SELECT :node_id, :stat_date, :stat_hour,
|
||||
COUNT(*) AS pv,
|
||||
COUNT(DISTINCT remote_addr) AS uv,
|
||||
COALESCE(SUM(body_bytes_sent), 0) AS total_bytes,
|
||||
COALESCE(SUM(CASE WHEN status BETWEEN 200 AND 299 THEN 1 ELSE 0 END), 0) AS status_2xx,
|
||||
COALESCE(SUM(CASE WHEN status BETWEEN 300 AND 399 THEN 1 ELSE 0 END), 0) AS status_3xx,
|
||||
COALESCE(SUM(CASE WHEN status BETWEEN 400 AND 499 THEN 1 ELSE 0 END), 0) AS status_4xx,
|
||||
COALESCE(SUM(CASE WHEN status BETWEEN 500 AND 599 THEN 1 ELSE 0 END), 0) AS status_5xx,
|
||||
COALESCE(SUM(CASE WHEN status < 200 OR status > 599 THEN 1 ELSE 0 END), 0) AS status_other,
|
||||
COALESCE(AVG(request_time), 0) AS avg_request_time
|
||||
FROM `{$rawTable}`
|
||||
WHERE time_local >= :hour_start AND time_local < :hour_end {$nodeWhere}
|
||||
HAVING COUNT(*) > 0";
|
||||
$statHourRows = (int) Db::execute($hourSql, $hourParams);
|
||||
|
||||
// 2. stat_url:topN(GROUP BY uri)
|
||||
$urlLimit = $topn > 0 ? 'LIMIT ' . $topn : '';
|
||||
$urlSql = "INSERT INTO `{$urlTable}`
|
||||
(node_id, stat_date, uri, pv, uv, total_bytes, avg_request_time)
|
||||
SELECT :node_id, :stat_date, uri,
|
||||
COUNT(*) AS pv,
|
||||
COUNT(DISTINCT remote_addr) AS uv,
|
||||
COALESCE(SUM(body_bytes_sent), 0) AS total_bytes,
|
||||
COALESCE(AVG(request_time), 0) AS avg_request_time
|
||||
FROM `{$rawTable}`
|
||||
WHERE time_local >= :day_start AND time_local < :day_end {$nodeWhere}
|
||||
GROUP BY uri
|
||||
ORDER BY pv DESC
|
||||
{$urlLimit}";
|
||||
$statUrlRows = (int) Db::execute($urlSql, $dayParams);
|
||||
|
||||
// 3. stat_referer:topN(从 http_referer 提取 domain)
|
||||
$refererLimit = $topn > 0 ? 'LIMIT ' . $topn : '';
|
||||
$refererSql = "INSERT INTO `{$refererTable}`
|
||||
(node_id, stat_date, referer_domain, pv, uv)
|
||||
SELECT :node_id, :stat_date,
|
||||
CASE
|
||||
WHEN http_referer IS NULL OR http_referer = '' OR http_referer = '-' THEN '-'
|
||||
ELSE SUBSTRING_INDEX(
|
||||
SUBSTRING_INDEX(
|
||||
REPLACE(REPLACE(http_referer, 'https://', ''), 'http://', ''),
|
||||
'/', 1
|
||||
),
|
||||
'?', 1
|
||||
)
|
||||
END AS referer_domain,
|
||||
COUNT(*) AS pv,
|
||||
COUNT(DISTINCT remote_addr) AS uv
|
||||
FROM `{$rawTable}`
|
||||
WHERE time_local >= :day_start AND time_local < :day_end {$nodeWhere}
|
||||
GROUP BY referer_domain
|
||||
ORDER BY pv DESC
|
||||
{$refererLimit}";
|
||||
$statRefererRows = (int) Db::execute($refererSql, $dayParams);
|
||||
|
||||
// 4. stat_ua:topN(从 http_user_agent 分类)
|
||||
$uaLimit = $topn > 0 ? 'LIMIT ' . $topn : '';
|
||||
$uaSql = "INSERT INTO `{$uaTable}`
|
||||
(node_id, stat_date, ua_type, ua_name, pv, uv)
|
||||
SELECT :node_id, :stat_date,
|
||||
CASE
|
||||
WHEN http_user_agent LIKE '%bot%'
|
||||
OR http_user_agent LIKE '%spider%'
|
||||
OR http_user_agent LIKE '%crawl%'
|
||||
OR http_user_agent LIKE '%slurp%'
|
||||
OR http_user_agent LIKE '%bingpreview%'
|
||||
OR http_user_agent LIKE '%facebookexternalhit%'
|
||||
OR http_user_agent LIKE '%twitterbot%' THEN 'spider'
|
||||
WHEN http_user_agent IS NULL OR http_user_agent = '' OR http_user_agent = '-' THEN 'unknown'
|
||||
WHEN http_user_agent LIKE '%Mozilla%'
|
||||
OR http_user_agent LIKE '%Chrome%'
|
||||
OR http_user_agent LIKE '%Safari%'
|
||||
OR http_user_agent LIKE '%Firefox%'
|
||||
OR http_user_agent LIKE '%Edg%'
|
||||
OR http_user_agent LIKE '%Opera%'
|
||||
OR http_user_agent LIKE '%MSIE%'
|
||||
OR http_user_agent LIKE '%Trident%' THEN 'browser'
|
||||
ELSE 'unknown'
|
||||
END AS ua_type,
|
||||
CASE
|
||||
WHEN http_user_agent LIKE '%Googlebot%' THEN 'Googlebot'
|
||||
WHEN http_user_agent LIKE '%Baiduspider%' THEN 'Baiduspider'
|
||||
WHEN http_user_agent LIKE '%bingbot%' THEN 'Bingbot'
|
||||
WHEN http_user_agent LIKE '%DuckDuckBot%' THEN 'DuckDuckBot'
|
||||
WHEN http_user_agent LIKE '%YandexBot%' THEN 'YandexBot'
|
||||
WHEN http_user_agent LIKE '%Edg/%' THEN 'Microsoft Edge'
|
||||
WHEN http_user_agent LIKE '%OPR/%' OR http_user_agent LIKE '%Opera%' THEN 'Opera'
|
||||
WHEN http_user_agent LIKE '%Firefox/%' THEN 'Firefox'
|
||||
WHEN http_user_agent LIKE '%Chrome/%' THEN 'Chrome'
|
||||
WHEN http_user_agent LIKE '%Safari/%' THEN 'Safari'
|
||||
WHEN http_user_agent LIKE '%MSIE%' OR http_user_agent LIKE '%Trident%' THEN 'Internet Explorer'
|
||||
WHEN http_user_agent IS NULL OR http_user_agent = '' OR http_user_agent = '-' THEN 'Unknown'
|
||||
ELSE 'Other'
|
||||
END AS ua_name,
|
||||
COUNT(*) AS pv,
|
||||
COUNT(DISTINCT remote_addr) AS uv
|
||||
FROM `{$rawTable}`
|
||||
WHERE time_local >= :day_start AND time_local < :day_end {$nodeWhere}
|
||||
GROUP BY ua_type, ua_name
|
||||
ORDER BY pv DESC
|
||||
{$uaLimit}";
|
||||
$statUaRows = (int) Db::execute($uaSql, $dayParams);
|
||||
|
||||
return [
|
||||
'stat_hour' => $statHourRows,
|
||||
'stat_url' => $statUrlRows,
|
||||
'stat_referer' => $statRefererRows,
|
||||
'stat_ua' => $statUaRows,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* 将 int 日期 YYYYMMDD 格式化为 "YYYY-MM-DD".
|
||||
*/
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
namespace base\common\service;
|
||||
|
||||
use app\admin\model\NginxLogPosition;
|
||||
use app\common\service\HostService;
|
||||
|
||||
/**
|
||||
* Nginx 日志增量读取 service(Base 层).
|
||||
@@ -10,12 +11,17 @@ use app\admin\model\NginxLogPosition;
|
||||
* 职责:
|
||||
* - 按文件 offset 增量读取日志行(Generator,不读整个文件到内存)
|
||||
* - 检测日志轮转(inode 变化 OR filesize < offset,覆盖 rename+rebuild 与 copytruncate 两种模式)
|
||||
* - 将读取进度持久化到 NginxLogPosition 表(upsert)
|
||||
* - 将读取进度持久化到 NginxLogPosition 表(upsert,按 node_id 隔离)
|
||||
*
|
||||
* 多节点适配:position 表按 (node_id, file_path) 唯一定位读取进度。
|
||||
* 旧数据(node_id 为空)通过懒加载接管(见 getLastPosition):首个查询到它的本节点原子 UPDATE node_id,
|
||||
* 避免 migration 全表回填(多节点共享 DB 时 migration 只跑一次,只有懒加载能确保每节点各自接管)。
|
||||
*
|
||||
* 不负责:解析(T5 Parser)、聚合(T7 Aggregator)。
|
||||
*
|
||||
* 依赖倒置:内部 model 调用走 `app\admin\model\NginxLogPosition`(App 入口类),
|
||||
* 使用者可在 app/ 重写该 model 拦截行为;本类不直接 `new NginxLogPosition`(静态调用满足多态)。
|
||||
* 节点身份走 `app\common\service\HostService`(App 入口类)。
|
||||
*/
|
||||
class NginxLogReaderServiceBase
|
||||
{
|
||||
@@ -159,15 +165,27 @@ class NginxLogReaderServiceBase
|
||||
/**
|
||||
* 查询文件的上次读取位置.
|
||||
*
|
||||
* 多节点适配:where 条件带 node_id 隔离各节点进度。
|
||||
* 懒加载接管:多节点共享 DB 时 migration 只跑一次,旧记录 node_id 为空,
|
||||
* 首个查询到它的本节点原子 UPDATE node_id 接管,保证每节点各自有独立 position 记录。
|
||||
*
|
||||
* @param string $filePath 日志文件绝对路径
|
||||
*
|
||||
* @return array|null 命中时返回 [inode, offset, last_line_hash, parse_fail_count, parse_fail_samples],无记录返回 null
|
||||
*/
|
||||
public function getLastPosition(string $filePath): ?array
|
||||
{
|
||||
$row = NginxLogPosition::where('file_path', $filePath)->find();
|
||||
$nodeId = HostService::getNodeId();
|
||||
|
||||
$row = NginxLogPosition::where('file_path', $filePath)
|
||||
->where('node_id', $nodeId)
|
||||
->find();
|
||||
if ($row === null) {
|
||||
return null;
|
||||
// 懒加载接管:旧记录 node_id 为空,原子抢占接管到当前节点
|
||||
$row = $this->lazyAdoptPosition($filePath, $nodeId);
|
||||
if ($row === null) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
return [
|
||||
@@ -179,6 +197,39 @@ class NginxLogReaderServiceBase
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* 懒加载接管:把旧的无 node_id 记录原子接管到当前节点.
|
||||
*
|
||||
* 多节点共享 DB 时 migration 只跑一次,无法为每节点预置 position 记录。
|
||||
* 旧记录(node_id='')由首个查询到它的本节点通过原子条件 UPDATE 接管:
|
||||
* UPDATE ... SET node_id=current WHERE node_id='' AND file_path=X
|
||||
* 并发安全:WHERE node_id='' 保证只有一个节点能 affected=1,其他节点查询时已被接管。
|
||||
*
|
||||
* @return \think\Model|null 接管成功返回接管后的记录,无旧记录或被其他节点抢占返回 null
|
||||
*/
|
||||
protected function lazyAdoptPosition(string $filePath, string $nodeId)
|
||||
{
|
||||
$legacy = NginxLogPosition::where('file_path', $filePath)
|
||||
->where('node_id', '')
|
||||
->find();
|
||||
if ($legacy === null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// 原子条件 UPDATE:只有 node_id='' 时才能被接管(防并发多节点同时接管同一记录)
|
||||
$affected = NginxLogPosition::where('id', $legacy->getData('id'))
|
||||
->where('node_id', '')
|
||||
->update(['node_id' => $nodeId]);
|
||||
if ($affected === 0) {
|
||||
// 被其他节点抢先接管,本节点放弃(下次 savePosition 会创建新记录)
|
||||
return null;
|
||||
}
|
||||
|
||||
return NginxLogPosition::where('file_path', $filePath)
|
||||
->where('node_id', $nodeId)
|
||||
->find();
|
||||
}
|
||||
|
||||
/**
|
||||
* 保存读取位置(upsert).
|
||||
*
|
||||
@@ -196,8 +247,11 @@ class NginxLogReaderServiceBase
|
||||
public function savePosition(string $filePath, int $inode, int $offset, ?string $lastLineHash, int $failCount = 0, ?string $failSamples = null): void
|
||||
{
|
||||
$now = time();
|
||||
$nodeId = HostService::getNodeId();
|
||||
|
||||
$existing = NginxLogPosition::where('file_path', $filePath)->find();
|
||||
$existing = NginxLogPosition::where('file_path', $filePath)
|
||||
->where('node_id', $nodeId)
|
||||
->find();
|
||||
if ($existing !== null) {
|
||||
$existing->save([
|
||||
'inode' => $inode,
|
||||
@@ -212,6 +266,7 @@ class NginxLogReaderServiceBase
|
||||
}
|
||||
|
||||
NginxLogPosition::create([
|
||||
'node_id' => $nodeId,
|
||||
'file_path' => $filePath,
|
||||
'inode' => $inode,
|
||||
'offset' => $offset,
|
||||
|
||||
@@ -191,6 +191,7 @@ class NginxLogImportBase extends TimerController
|
||||
{
|
||||
$now = time();
|
||||
return [
|
||||
'node_id' => \app\common\service\HostService::getNodeId(),
|
||||
'file_path' => $filePath,
|
||||
'remote_addr' => $parsed['remote_addr'],
|
||||
'remote_user' => $parsed['remote_user'] ?? '',
|
||||
|
||||
Reference in New Issue
Block a user