ulthon/ulthon_admin
1
0
Fork 0
mirror of https://gitee.com/ulthon/ulthon_admin.git synced 2026-08-30 12:45:32 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(nginx-log): 多节点适配（node_id 全链路 + 全局/按节点聚合 + 节点筛选 + v2.4.0 升级脚本）

Browse Source 
- 6 张表加 node_id 字段 + position/stat 改复合唯一键（migration）
- 6 个 Scheme 同步 node_id 注解 + 唯一键
- Reader 带 node_id 参数 + 懒加载接管（getLastPosition 回退查 node_id=''）
- Aggregator 全局行（node_id=''）+ 按节点循环 insertAggregatesForScope
- Stat 所有查询方法加 where node_id 条件
- Dashboard 控制器/视图加节点筛选下拉框 + AJAX 带 node_id
- AccessLog 列表加节点列 + 采集进度卡片显示节点信息
- import 定时任务 run_type 改 all（每节点采集自己的日志）
- v2.4.0 升级脚本追加 ALTER TABLE（幂等 check-then-alter）
- 菜单调整：去掉 Nginx 顶级菜单 + 读取位置管理，改为系统管理下挂两个子菜单
- ulthon-timer 文档补充 run_type=all 多节点部署注意事项
This commit is contained in:
augushong
2026-08-08 10:33:20 +08:00
parent f1a2e8fd5d
commit 8faa52ce90
21 changed files with 578 additions and 206 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
extend/base/admin/controller/nginx/AccessLogBase.php
View File

@@ -56,6 +56,24 @@ class AccessLogBase extends AdminController
return json($data);
}
// 采集位置信息（合并自原"读取位置管理"独立页面）
$positions = \app\admin\model\NginxLogPosition::order('id', 'desc')->select()->map(function ($item) {
$bytes = (int) $item['offset'];
if ($bytes >= 1048576) {
$item['offset_human'] = number_format($bytes / 1048576, 2) . ' MB';
} elseif ($bytes >= 1024) {
$item['offset_human'] = number_format($bytes / 1024, 2) . ' KB';
} else {
$item['offset_human'] = $bytes . ' B';
}
$item['last_read_time_text'] = $item['last_read_time'] > 0
? date('Y-m-d H:i:s', (int) $item['last_read_time'])
: '-';
return $item;
});
$this->assign('positions', $positions);
return $this->fetch();
}

33
extend/base/admin/controller/system/NginxLogStatBase.php
View File

@@ -45,11 +45,13 @@ class NginxLogStatBase extends AdminController
$period = 'day';
}
$nodeId = (string) $request->param('node_id', '');
// 解析时间范围（YYYYMMDD int 表示）
[$startDate, $endDate] = $this->resolveDateRange($period);
if ($request->isAjax()) {
$data = $this->buildStatData($period, $startDate, $endDate);
$data = $this->buildStatData($period, $startDate, $endDate, $nodeId);
return json([
'code' => 0,
@@ -63,6 +65,11 @@ class NginxLogStatBase extends AdminController
$this->assign('start', $startDate);
$this->assign('end', $endDate);
// 在线节点列表（供视图渲染节点筛选下拉）
$nodes = \app\admin\model\SystemHost::where('status', 1)->order('node_id', 'asc')->select();
$this->assign('nodes', $nodes);
$this->assign('node_id', $nodeId);
return $this->fetch();
}
@@ -140,20 +147,20 @@ class NginxLogStatBase extends AdminController
/**
* 组装完整统计数据
*/
protected function buildStatData(string $period, int $startDate, int $endDate): array
protected function buildStatData(string $period, int $startDate, int $endDate, string $nodeId = ''): array
{
// 时间范围不合法 → 空数据（不崩）
if ($startDate <= 0 || $endDate <= 0 || $startDate > $endDate) {
return $this->emptyData($period);
}
$summary = $this->querySummary($startDate, $endDate);
$trend = $this->queryTrend($period, $startDate, $endDate);
$summary = $this->querySummary($startDate, $endDate, $nodeId);
$trend = $this->queryTrend($period, $startDate, $endDate, $nodeId);
$topN = $this->getTopN();
$topUrls = $this->queryTopByDateRange(NginxStatUrl::class, 'uri', $startDate, $endDate, $topN);
$topReferers = $this->queryTopByDateRange(NginxStatReferer::class, 'referer_domain', $startDate, $endDate, $topN);
$topUas = $this->queryTopByDateRange(NginxStatUa::class, 'ua_name', $startDate, $endDate, $topN);
$topUrls = $this->queryTopByDateRange(NginxStatUrl::class, 'uri', $startDate, $endDate, $topN, $nodeId);
$topReferers = $this->queryTopByDateRange(NginxStatReferer::class, 'referer_domain', $startDate, $endDate, $topN, $nodeId);
$topUas = $this->queryTopByDateRange(NginxStatUa::class, 'ua_name', $startDate, $endDate, $topN, $nodeId);
// 跨周期 UV 估算标注：仅 day 为精确值，week/month/year/custom 均为按日累加的估算值
$uvIsEstimate = $period !== 'day';
@@ -208,7 +215,7 @@ class NginxLogStatBase extends AdminController
* 整体指标：SUM stat_hour
* avg_request_time 按 PV 加权平均（避免简单 AVG 被低 PV 时段拉偏）
*/
protected function querySummary(int $startDate, int $endDate): array
protected function querySummary(int $startDate, int $endDate, string $nodeId = ''): array
{
$sumFields = [
'IFNULL(SUM(pv),0) AS pv',
@@ -223,6 +230,7 @@ class NginxLogStatBase extends AdminController
];
$row = NginxStatHour::where('stat_date', 'between', [$startDate, $endDate])
->where('node_id', $nodeId)
->field(implode(',', $sumFields))
->find();
@@ -252,10 +260,11 @@ class NginxLogStatBase extends AdminController
* - year → 12 个月（按 stat_date DIV 100 分组，补全 1~12 月）
* - custom→ 按 stat_date 分组，补全 start~end 日期序列
*/
protected function queryTrend(string $period, int $startDate, int $endDate): array
protected function queryTrend(string $period, int $startDate, int $endDate, string $nodeId = ''): array
{
if ($period === 'day') {
$rows = NginxStatHour::where('stat_date', 'between', [$startDate, $endDate])
->where('node_id', $nodeId)
->field('stat_hour, IFNULL(SUM(pv),0) AS pv, IFNULL(SUM(uv),0) AS uv')
->group('stat_hour')
->select()
@@ -280,6 +289,7 @@ class NginxLogStatBase extends AdminController
if ($period === 'year') {
$rows = NginxStatHour::where('stat_date', 'between', [$startDate, $endDate])
->where('node_id', $nodeId)
->field('(stat_date DIV 100) AS ym, IFNULL(SUM(pv),0) AS pv, IFNULL(SUM(uv),0) AS uv')
->group('ym')
->order('ym asc')
@@ -307,6 +317,7 @@ class NginxLogStatBase extends AdminController
// week / month / custom → 按 stat_date 分组，补全日期序列
$rows = NginxStatHour::where('stat_date', 'between', [$startDate, $endDate])
->where('node_id', $nodeId)
->field('stat_date, IFNULL(SUM(pv),0) AS pv, IFNULL(SUM(uv),0) AS uv')
->group('stat_date')
->order('stat_date asc')
@@ -340,10 +351,12 @@ class NginxLogStatBase extends AdminController
* @param string $modelClass 模型类（依赖倒置：调 app/admin/model 入口）
* @param string $labelField 分组与展示的字段名
* @param int $topN 取前 N 条；0 不限制
* @param string $nodeId 节点 ID（空字符串=全局聚合行）
*/
protected function queryTopByDateRange(string $modelClass, string $labelField, int $startDate, int $endDate, int $topN): array
protected function queryTopByDateRange(string $modelClass, string $labelField, int $startDate, int $endDate, int $topN, string $nodeId = ''): array
{
$query = $modelClass::where('stat_date', 'between', [$startDate, $endDate])
->where('node_id', $nodeId)
->field($labelField . ' AS label, IFNULL(SUM(pv),0) AS pv, IFNULL(SUM(uv),0) AS uv')
->group($labelField)
->order('pv desc');

26
extend/base/admin/service/adminInitData/SystemMenu.php
View File

@@ -209,20 +209,9 @@ $ul_system_menu = array(
"sort" => 0,
"status" => 1,
),
array(
"id" => 300,
"pid" => 0,
"title" => "Nginx 日志分析",
"icon" => "fa fa-chart-bar",
"href" => "",
"params" => "",
"target" => "_self",
"sort" => 0,
"status" => 1,
),
array(
"id" => 301,
"pid" => 300,
"pid" => 228,
"title" => "访问统计仪表盘",
"icon" => "fa fa-line-chart",
"href" => "system.nginx_log_stat/index",
@@ -233,7 +222,7 @@ $ul_system_menu = array(
),
array(
"id" => 302,
"pid" => 300,
"pid" => 228,
"title" => "原始访问日志",
"icon" => "fa fa-file-text",
"href" => "nginx.access_log/index",
@@ -242,17 +231,6 @@ $ul_system_menu = array(
"sort" => 0,
"status" => 1,
),
array(
"id" => 303,
"pid" => 300,
"title" => "读取位置管理",
"icon" => "fa fa-map-marker",
"href" => "nginx.log_position/index",
"params" => "",
"target" => "_self",
"sort" => 0,
"status" => 1,
),
);

129
extend/base/admin/service/adminUpdateCodeData/v2.4.0.php
View File

@@ -13,11 +13,12 @@
* stat_retention_days 统计聚合保留天数（0=永久）
* topn_default Top N 默认值
* exclude_static 是否排除静态资源访问
* - system_menu 新增 4 条菜单（顶级分组 300 + 3 个子菜单 301/302/303）
* 300 Nginx 日志分析（顶级分组，href 为空）
* - system_menu 新增 2 条菜单（挂到系统管理 pid=228 下）
* 301 访问统计仪表盘 system.nginx_log_stat/index
* 302 原始访问日志 nginx.access_log/index
* 303 读取位置管理 nginx.log_position/index
* - ALTER TABLE：6 张 nginx_log 表新增 node_id 字段 + 唯一键前置 node_id（多节点采集支持）
* 与 migration 20260801125222_nginx_log_add_node_id.php 保持一致，
* 存量库通过 admin:update 幂等执行（check-then-alter）
*
* 设计说明：
* - 幂等 check-then-insert（先查询存在性，不存在才插入）
@@ -82,23 +83,12 @@ class UpdateFunction
/**
* 待新增菜单.
* 字段顺序与 adminInitData/SystemMenu.php 一致：id/pid/title/icon/href/params/target/sort/status.
* id 显式分配 300-303，与 adminInitData 保持一致（顶级 300 + 3 个子菜单）。
* 挂到系统管理（pid=228）下，不创建独立顶级分组。
*/
public $newMenus = [
[
'id' => 300,
'pid' => 0,
'title' => 'Nginx 日志分析',
'icon' => 'fa fa-chart-bar',
'href' => '',
'params' => '',
'target' => '_self',
'sort' => 0,
'status' => 1,
],
[
'id' => 301,
'pid' => 300,
'pid' => 228,
'title' => '访问统计仪表盘',
'icon' => 'fa fa-line-chart',
'href' => 'system.nginx_log_stat/index',
@@ -109,7 +99,7 @@ class UpdateFunction
],
[
'id' => 302,
'pid' => 300,
'pid' => 228,
'title' => '原始访问日志',
'icon' => 'fa fa-file-text',
'href' => 'nginx.access_log/index',
@@ -118,24 +108,13 @@ class UpdateFunction
'sort' => 0,
'status' => 1,
],
[
'id' => 303,
'pid' => 300,
'title' => '读取位置管理',
'icon' => 'fa fa-map-marker',
'href' => 'nginx.log_position/index',
'params' => '',
'target' => '_self',
'sort' => 0,
'status' => 1,
],
];
public function update()
{
$this->output->writeln('更新代码');
$this->output->info('v2.4.0：nginx_log 分析模块存量升级（补齐 4 条 sysconfig + 4 条菜单）');
$this->output->info('v2.4.0：nginx_log 分析模块存量升级（补齐 4 条 sysconfig + 2 条菜单）');
$configTable = config('database.connections.mysql.prefix', 'ul_') . 'system_config';
$menuTable = config('database.connections.mysql.prefix', 'ul_') . 'system_menu';
@@ -190,6 +169,98 @@ class UpdateFunction
$totalSkipped = $configSkipped + $menuSkipped;
$this->output->info("本次新增 {$configAdded} 条配置、{$menuAdded} 条菜单，跳过 {$totalSkipped} 条已存在");
// ==================== node_id 字段 + 唯一键变更（多节点采集支持）====================
// 与 migration 20260801125222_nginx_log_add_node_id.php 保持一致
// 6 张表全部新增 node_id varchar(64) default ''
// position / 4 张 stat 表的唯一键前置 node_id（多节点隔离，避免聚合 INSERT 触发 duplicate key）
// raw 表（nginx_access_log）加 node_id + time_local 普通索引，无唯一键变更
$this->output->info('v2.4.0：nginx_log 6 张表新增 node_id 字段与唯一键调整');
$prefix = config('database.connections.mysql.prefix', 'ul_');
// 表短名 => 索引变更配置
// drop_idx: 要删除的旧索引名（raw 表无）
// add_idx: 新索引 [name, cols, unique]
$nodeIdTables = [
'nginx_log_position' => [
'drop_idx' => 'uniq_file_path',
'add_idx' => ['name' => 'uniq_node_file', 'cols' => ['node_id', 'file_path'], 'unique' => true],
],
'nginx_access_log' => [
'add_idx' => ['name' => 'idx_node_time', 'cols' => ['node_id', 'time_local'], 'unique' => false],
],
'nginx_stat_hour' => [
'drop_idx' => 'uniq_date_hour',
'add_idx' => ['name' => 'uniq_node_date_hour', 'cols' => ['node_id', 'stat_date', 'stat_hour'], 'unique' => true],
],
'nginx_stat_url' => [
'drop_idx' => 'uniq_date_uri',
'add_idx' => ['name' => 'uniq_node_date_uri', 'cols' => ['node_id', 'stat_date', 'uri'], 'unique' => true],
],
'nginx_stat_referer' => [
'drop_idx' => 'uniq_date_referer',
'add_idx' => ['name' => 'uniq_node_date_referer', 'cols' => ['node_id', 'stat_date', 'referer_domain'], 'unique' => true],
],
'nginx_stat_ua' => [
'drop_idx' => 'uniq_date_type_name',
'add_idx' => ['name' => 'uniq_node_date_type_name', 'cols' => ['node_id', 'stat_date', 'ua_type', 'ua_name'], 'unique' => true],
],
];
$colAdded = 0;
$colSkipped = 0;
foreach ($nodeIdTables as $tableShort => $idxCfg) {
$fullTable = $prefix . $tableShort;
// 防御性检查：表是否存在（未运行 migration 时跳过，不报错）
$tableExists = Db::query(
"SELECT COUNT(*) AS cnt FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ?",
[$fullTable]
);
if (empty($tableExists) || (int) $tableExists[0]['cnt'] === 0) {
$this->output->writeln(" - {$fullTable} 表不存在，跳过（请先执行 migrate:run）");
continue;
}
// 幂等 1：检查 node_id 字段是否存在
$colExists = Db::query(
"SELECT COUNT(*) AS cnt FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND COLUMN_NAME = 'node_id'",
[$fullTable]
);
if (!empty($colExists) && (int) $colExists[0]['cnt'] > 0) {
$this->output->writeln(" - {$fullTable} node_id 字段已存在，跳过");
++$colSkipped;
} else {
Db::execute("ALTER TABLE `{$fullTable}` ADD COLUMN `node_id` VARCHAR(64) NOT NULL DEFAULT '' COMMENT '采集节点 ID（空=全局汇总）'");
$this->output->writeln(" - {$fullTable} 新增 node_id 字段完成");
++$colAdded;
}
// 幂等 2：唯一键/索引变更（检查新索引是否存在）
$newIdx = $idxCfg['add_idx']['name'] ?? null;
if ($newIdx !== null) {
$idxExists = Db::query(
"SELECT COUNT(*) AS cnt FROM information_schema.STATISTICS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND INDEX_NAME = ?",
[$fullTable, $newIdx]
);
if (empty($idxExists) || (int) $idxExists[0]['cnt'] === 0) {
// 删除旧索引（position / stat 表有，raw 表无）
if (!empty($idxCfg['drop_idx'])) {
Db::execute("ALTER TABLE `{$fullTable}` DROP INDEX `{$idxCfg['drop_idx']}`");
}
// 新增新索引
$idxType = !empty($idxCfg['add_idx']['unique']) ? 'UNIQUE' : 'INDEX';
$colsStr = implode(', ', array_map(function ($c) {
return "`{$c}`";
}, $idxCfg['add_idx']['cols']));
Db::execute("ALTER TABLE `{$fullTable}` ADD {$idxType} `{$newIdx}` ({$colsStr})");
$this->output->writeln(" - {$fullTable} 索引调整：{$newIdx}");
}
}
}
$this->output->info("node_id 迁移完成（新增字段 {$colAdded} 表，跳过 {$colSkipped} 表已存在）");
$this->output->writeln('更新代码完成');
}
}

8
extend/base/admin/view/system/nginx_log_stat/index.html
View File

@@ -19,6 +19,14 @@
<input type="text" class="layui-input" id="dateEnd" placeholder="结束日期" autocomplete="off" style="width:130px;" readonly>
<button type="button" class="layui-btn layui-btn-sm layui-btn-normal" id="customSearchBtn">查询</button>
</span>
<div class="layui-input-inline" style="margin-left: 15px; width: 180px; vertical-align: middle;">
<select id="node-filter" lay-filter="nodeFilter">
<option value="">全部节点</option>
{volist name="nodes" id="node"}
<option value="{$node.node_id}" {if condition="$node_id eq $node.node_id"}selected{/if}>{$node.node_id}</option>
{/volist}
</select>
</div>
</div>
</div>

19
extend/base/admin/view/system/nginx_log_stat/index.js
View File

@@ -6,11 +6,15 @@
* 后端 URL 不硬编码：用 ua.url('system.nginx_log_stat/index') 由 CONFIG.ADMIN 拼接。
* AJAX 触发方式：URL 带 _ajax=1（不依赖 Accept header，命令行测试友好）。
*/
layui.use(['table', 'laydate', 'util'], function () {
layui.use(['table', 'laydate', 'util', 'form'], function () {
var table = layui.table;
var laydate = layui.laydate;
var util = layui.util;
var form = layui.form;
// 渲染节点下拉框
form.render();
// ── 配置（URL 不硬编码：由 ua.url 拼接 /{ADMIN}/system.nginx_log_stat/index）──
var STAT_INDEX_URL = ua.url('system.nginx_log_stat/index');
@@ -129,6 +133,15 @@ layui.use(['table', 'laydate', 'util'], function () {
loadData({ period: 'custom', start: startVal, end: endVal });
});
// ================================================================
// 节点下拉框切换
// ================================================================
form.on('select(nodeFilter)', function (data) {
// 切换节点时用当前 period 重新加载
loadData({ period: currentPeriod });
});
// ================================================================
// AJAX 加载数据
// ================================================================
@@ -141,6 +154,10 @@ layui.use(['table', 'laydate', 'util'], function () {
function loadData(extra) {
extra = extra || {};
// 当前选中的节点 ID（默认空字符串=全部节点）
var nodeId = $('#node-filter').val() || '';
extra.node_id = nodeId;
var params = $.extend({ _ajax: 1, period: currentPeriod }, extra);
ua.request.get({