mirror of
https://gitee.com/ulthon/ulthon_admin.git
synced 2026-08-30 12:45:32 +08:00
feat(nginx-log): NginxLogAggregator 聚合 service（Base/App 双层，delete-then-insert 幂等）
This commit is contained in:
augushong
229
extend/base/common/service/NginxLogAggregatorBase.php
Normal file
229
extend/base/common/service/NginxLogAggregatorBase.php
Normal file
@@ -0,0 +1,229 @@
|
||||
<?php
|
||||
|
||||
namespace base\common\service;
|
||||
|
||||
use app\admin\model\NginxAccessLog;
|
||||
use app\admin\model\NginxStatHour;
|
||||
use app\admin\model\NginxStatReferer;
|
||||
use app\admin\model\NginxStatUa;
|
||||
use app\admin\model\NginxStatUrl;
|
||||
use think\facade\Db;
|
||||
|
||||
/**
|
||||
* Nginx 访问日志聚合统计 Service（Base 内核）.
|
||||
*
|
||||
* 设计要点：
|
||||
* - 幂等：delete-then-insert 模式（不用 INSERT ON DUPLICATE KEY UPDATE，避免翻倍）
|
||||
* - 事务：Db::startTrans / commit / rollback 包裹所有 DELETE + INSERT
|
||||
* - top N 截断后不保留长尾（明确接受，不在 sysconfig 控制"是否截断"）
|
||||
* - 依赖倒置：内部 use app 入口 model（NginxStatHour 等），不 use Base model
|
||||
* - 只读 raw 表（ul_nginx_access_log），不做 nginx log line 解析（那是 T5 Parser 的事）
|
||||
*/
|
||||
class NginxLogAggregatorBase
|
||||
{
|
||||
/**
|
||||
* 聚合指定日期的指定小时.
|
||||
*
|
||||
* 流程：
|
||||
* 1. stat_hour：DELETE WHERE stat_date=:d AND stat_hour=:h → INSERT 1 行（该小时聚合）
|
||||
* 2. stat_url / stat_referer / stat_ua：DELETE WHERE stat_date=:d（整天重跑）→ INSERT top N 行
|
||||
*
|
||||
* @param int $statDate 日期 YYYYMMDD（如 20260728）
|
||||
* @param int $statHour 小时 0-23
|
||||
* @return array {stat_hour_rows:int, stat_url_rows:int, stat_referer_rows:int, stat_ua_rows:int}
|
||||
*
|
||||
* @throws \InvalidArgumentException 日期/小时非法
|
||||
* @throws \Throwable 事务内任何错误（已 rollback 后 rethrow）
|
||||
*/
|
||||
public function aggregateHour(int $statDate, int $statHour): array
|
||||
{
|
||||
if ($statHour < 0 || $statHour > 23) {
|
||||
throw new \InvalidArgumentException("stat_hour must be 0-23, got: {$statHour}");
|
||||
}
|
||||
|
||||
// 时间窗口
|
||||
$dateStr = $this->formatDateStr($statDate);
|
||||
$dayStart = strtotime("{$dateStr} 00:00:00");
|
||||
if ($dayStart === false) {
|
||||
throw new \InvalidArgumentException("invalid stat_date: {$statDate}");
|
||||
}
|
||||
$dayEnd = $dayStart + 86400;
|
||||
$hourStart = $dayStart + $statHour * 3600;
|
||||
$hourEnd = $hourStart + 3600;
|
||||
|
||||
// topN 配置（topn=0 时不限制）
|
||||
$topn = (int) sysconfig('nginx_log', 'topn_default', 10);
|
||||
|
||||
// 表名（带前缀）。框架默认连接是 main（不是 mysql），按 default 取连接 prefix
|
||||
$conn = (string) config('database.default', 'main');
|
||||
$prefix = (string) config("database.connections.{$conn}.prefix", 'ul_');
|
||||
$rawTable = $prefix . 'nginx_access_log';
|
||||
$hourTable = $prefix . 'nginx_stat_hour';
|
||||
$urlTable = $prefix . 'nginx_stat_url';
|
||||
$refererTable = $prefix . 'nginx_stat_referer';
|
||||
$uaTable = $prefix . 'nginx_stat_ua';
|
||||
|
||||
Db::startTrans();
|
||||
try {
|
||||
// 1. stat_hour：DELETE + INSERT 1 行
|
||||
NginxStatHour::where('stat_date', $statDate)
|
||||
->where('stat_hour', $statHour)
|
||||
->delete();
|
||||
|
||||
$hourSql = "INSERT INTO `{$hourTable}`
|
||||
(stat_date, stat_hour, pv, uv, total_bytes, status_2xx, status_3xx, status_4xx, status_5xx, status_other, avg_request_time)
|
||||
SELECT :stat_date, :stat_hour,
|
||||
COUNT(*) AS pv,
|
||||
COUNT(DISTINCT remote_addr) AS uv,
|
||||
COALESCE(SUM(body_bytes_sent), 0) AS total_bytes,
|
||||
COALESCE(SUM(CASE WHEN status BETWEEN 200 AND 299 THEN 1 ELSE 0 END), 0) AS status_2xx,
|
||||
COALESCE(SUM(CASE WHEN status BETWEEN 300 AND 399 THEN 1 ELSE 0 END), 0) AS status_3xx,
|
||||
COALESCE(SUM(CASE WHEN status BETWEEN 400 AND 499 THEN 1 ELSE 0 END), 0) AS status_4xx,
|
||||
COALESCE(SUM(CASE WHEN status BETWEEN 500 AND 599 THEN 1 ELSE 0 END), 0) AS status_5xx,
|
||||
COALESCE(SUM(CASE WHEN status < 200 OR status > 599 THEN 1 ELSE 0 END), 0) AS status_other,
|
||||
COALESCE(AVG(request_time), 0) AS avg_request_time
|
||||
FROM `{$rawTable}`
|
||||
WHERE time_local >= :hour_start AND time_local < :hour_end
|
||||
HAVING COUNT(*) > 0";
|
||||
|
||||
$statHourRows = (int) Db::execute($hourSql, [
|
||||
'stat_date' => $statDate,
|
||||
'stat_hour' => $statHour,
|
||||
'hour_start' => $hourStart,
|
||||
'hour_end' => $hourEnd,
|
||||
]);
|
||||
|
||||
// 2. stat_url：DELETE（整天）+ INSERT topN
|
||||
NginxStatUrl::where('stat_date', $statDate)->delete();
|
||||
|
||||
$urlLimit = $topn > 0 ? 'LIMIT ' . $topn : '';
|
||||
$urlSql = "INSERT INTO `{$urlTable}`
|
||||
(stat_date, uri, pv, uv, total_bytes, avg_request_time)
|
||||
SELECT :stat_date, uri,
|
||||
COUNT(*) AS pv,
|
||||
COUNT(DISTINCT remote_addr) AS uv,
|
||||
COALESCE(SUM(body_bytes_sent), 0) AS total_bytes,
|
||||
COALESCE(AVG(request_time), 0) AS avg_request_time
|
||||
FROM `{$rawTable}`
|
||||
WHERE time_local >= :day_start AND time_local < :day_end
|
||||
GROUP BY uri
|
||||
ORDER BY pv DESC
|
||||
{$urlLimit}";
|
||||
|
||||
$statUrlRows = (int) Db::execute($urlSql, [
|
||||
'stat_date' => $statDate,
|
||||
'day_start' => $dayStart,
|
||||
'day_end' => $dayEnd,
|
||||
]);
|
||||
|
||||
// 3. stat_referer：DELETE + INSERT topN（从 http_referer 提取 domain）
|
||||
NginxStatReferer::where('stat_date', $statDate)->delete();
|
||||
|
||||
$refererLimit = $topn > 0 ? 'LIMIT ' . $topn : '';
|
||||
$refererSql = "INSERT INTO `{$refererTable}`
|
||||
(stat_date, referer_domain, pv, uv)
|
||||
SELECT :stat_date,
|
||||
CASE
|
||||
WHEN http_referer IS NULL OR http_referer = '' OR http_referer = '-' THEN '-'
|
||||
ELSE SUBSTRING_INDEX(
|
||||
SUBSTRING_INDEX(
|
||||
REPLACE(REPLACE(http_referer, 'https://', ''), 'http://', ''),
|
||||
'/', 1
|
||||
),
|
||||
'?', 1
|
||||
)
|
||||
END AS referer_domain,
|
||||
COUNT(*) AS pv,
|
||||
COUNT(DISTINCT remote_addr) AS uv
|
||||
FROM `{$rawTable}`
|
||||
WHERE time_local >= :day_start AND time_local < :day_end
|
||||
GROUP BY referer_domain
|
||||
ORDER BY pv DESC
|
||||
{$refererLimit}";
|
||||
|
||||
$statRefererRows = (int) Db::execute($refererSql, [
|
||||
'stat_date' => $statDate,
|
||||
'day_start' => $dayStart,
|
||||
'day_end' => $dayEnd,
|
||||
]);
|
||||
|
||||
// 4. stat_ua：DELETE + INSERT topN（从 http_user_agent 分类）
|
||||
NginxStatUa::where('stat_date', $statDate)->delete();
|
||||
|
||||
$uaLimit = $topn > 0 ? 'LIMIT ' . $topn : '';
|
||||
$uaSql = "INSERT INTO `{$uaTable}`
|
||||
(stat_date, ua_type, ua_name, pv, uv)
|
||||
SELECT :stat_date,
|
||||
CASE
|
||||
WHEN http_user_agent LIKE '%bot%'
|
||||
OR http_user_agent LIKE '%spider%'
|
||||
OR http_user_agent LIKE '%crawl%'
|
||||
OR http_user_agent LIKE '%slurp%'
|
||||
OR http_user_agent LIKE '%bingpreview%'
|
||||
OR http_user_agent LIKE '%facebookexternalhit%'
|
||||
OR http_user_agent LIKE '%twitterbot%' THEN 'spider'
|
||||
WHEN http_user_agent IS NULL OR http_user_agent = '' OR http_user_agent = '-' THEN 'unknown'
|
||||
WHEN http_user_agent LIKE '%Mozilla%'
|
||||
OR http_user_agent LIKE '%Chrome%'
|
||||
OR http_user_agent LIKE '%Safari%'
|
||||
OR http_user_agent LIKE '%Firefox%'
|
||||
OR http_user_agent LIKE '%Edg%'
|
||||
OR http_user_agent LIKE '%Opera%'
|
||||
OR http_user_agent LIKE '%MSIE%'
|
||||
OR http_user_agent LIKE '%Trident%' THEN 'browser'
|
||||
ELSE 'unknown'
|
||||
END AS ua_type,
|
||||
CASE
|
||||
WHEN http_user_agent LIKE '%Googlebot%' THEN 'Googlebot'
|
||||
WHEN http_user_agent LIKE '%Baiduspider%' THEN 'Baiduspider'
|
||||
WHEN http_user_agent LIKE '%bingbot%' THEN 'Bingbot'
|
||||
WHEN http_user_agent LIKE '%DuckDuckBot%' THEN 'DuckDuckBot'
|
||||
WHEN http_user_agent LIKE '%YandexBot%' THEN 'YandexBot'
|
||||
WHEN http_user_agent LIKE '%Edg/%' THEN 'Microsoft Edge'
|
||||
WHEN http_user_agent LIKE '%OPR/%' OR http_user_agent LIKE '%Opera%' THEN 'Opera'
|
||||
WHEN http_user_agent LIKE '%Firefox/%' THEN 'Firefox'
|
||||
WHEN http_user_agent LIKE '%Chrome/%' THEN 'Chrome'
|
||||
WHEN http_user_agent LIKE '%Safari/%' THEN 'Safari'
|
||||
WHEN http_user_agent LIKE '%MSIE%' OR http_user_agent LIKE '%Trident%' THEN 'Internet Explorer'
|
||||
WHEN http_user_agent IS NULL OR http_user_agent = '' OR http_user_agent = '-' THEN 'Unknown'
|
||||
ELSE 'Other'
|
||||
END AS ua_name,
|
||||
COUNT(*) AS pv,
|
||||
COUNT(DISTINCT remote_addr) AS uv
|
||||
FROM `{$rawTable}`
|
||||
WHERE time_local >= :day_start AND time_local < :day_end
|
||||
GROUP BY ua_type, ua_name
|
||||
ORDER BY pv DESC
|
||||
{$uaLimit}";
|
||||
|
||||
$statUaRows = (int) Db::execute($uaSql, [
|
||||
'stat_date' => $statDate,
|
||||
'day_start' => $dayStart,
|
||||
'day_end' => $dayEnd,
|
||||
]);
|
||||
|
||||
Db::commit();
|
||||
|
||||
return [
|
||||
'stat_hour_rows' => $statHourRows,
|
||||
'stat_url_rows' => $statUrlRows,
|
||||
'stat_referer_rows' => $statRefererRows,
|
||||
'stat_ua_rows' => $statUaRows,
|
||||
];
|
||||
} catch (\Throwable $e) {
|
||||
Db::rollback();
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 将 int 日期 YYYYMMDD 格式化为 "YYYY-MM-DD".
|
||||
*/
|
||||
protected function formatDateStr(int $statDate): string
|
||||
{
|
||||
$y = intdiv($statDate, 10000);
|
||||
$m = intdiv($statDate % 10000, 100);
|
||||
$d = $statDate % 100;
|
||||
return sprintf('%04d-%02d-%02d', $y, $m, $d);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user