From ceb73c750a0ce7556eee79311d18caaebc17ada9 Mon Sep 17 00:00:00 2001
From: augushong <augushong@163.com>
Date: Tue, 28 Jul 2026 00:52:51 +0800
Subject: [PATCH] =?UTF-8?q?feat(nginx-log):=20NginxLogAggregator=20?=
 =?UTF-8?q?=E8=81=9A=E5=90=88=20service=EF=BC=88Base/App=20=E5=8F=8C?=
 =?UTF-8?q?=E5=B1=82=EF=BC=8Cdelete-then-insert=20=E5=B9=82=E7=AD=89?=
 =?UTF-8?q?=EF=BC=89?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 app/common/service/NginxLogAggregator.php     |  15 ++
 .../common/service/NginxLogAggregatorBase.php | 229 ++++++++++++++++++
 2 files changed, 244 insertions(+)
 create mode 100644 app/common/service/NginxLogAggregator.php
 create mode 100644 extend/base/common/service/NginxLogAggregatorBase.php

diff --git a/app/common/service/NginxLogAggregator.php b/app/common/service/NginxLogAggregator.php
new file mode 100644
index 0000000..2243698
--- /dev/null
+++ b/app/common/service/NginxLogAggregator.php
@@ -0,0 +1,15 @@
+<?php
+
+namespace app\common\service;
+
+use base\common\service\NginxLogAggregatorBase;
+
+/**
+ * Nginx 访问日志聚合统计 Service（业务入口）.
+ *
+ * 继承自 Base 内核，业务侧可在本类中重写 aggregateHour / formatDateStr 等方法
+ * 来定制聚合逻辑（依赖倒置：Base 层只调用 app 入口类）。
+ */
+class NginxLogAggregator extends NginxLogAggregatorBase
+{
+}
diff --git a/extend/base/common/service/NginxLogAggregatorBase.php b/extend/base/common/service/NginxLogAggregatorBase.php
new file mode 100644
index 0000000..b996db2
--- /dev/null
+++ b/extend/base/common/service/NginxLogAggregatorBase.php
@@ -0,0 +1,229 @@
+<?php
+
+namespace base\common\service;
+
+use app\admin\model\NginxAccessLog;
+use app\admin\model\NginxStatHour;
+use app\admin\model\NginxStatReferer;
+use app\admin\model\NginxStatUa;
+use app\admin\model\NginxStatUrl;
+use think\facade\Db;
+
+/**
+ * Nginx 访问日志聚合统计 Service（Base 内核）.
+ *
+ * 设计要点：
+ * - 幂等：delete-then-insert 模式（不用 INSERT ON DUPLICATE KEY UPDATE，避免翻倍）
+ * - 事务：Db::startTrans / commit / rollback 包裹所有 DELETE + INSERT
+ * - top N 截断后不保留长尾（明确接受，不在 sysconfig 控制"是否截断"）
+ * - 依赖倒置：内部 use app 入口 model（NginxStatHour 等），不 use Base model
+ * - 只读 raw 表（ul_nginx_access_log），不做 nginx log line 解析（那是 T5 Parser 的事）
+ */
+class NginxLogAggregatorBase
+{
+    /**
+     * 聚合指定日期的指定小时.
+     *
+     * 流程：
+     * 1. stat_hour：DELETE WHERE stat_date=:d AND stat_hour=:h → INSERT 1 行（该小时聚合）
+     * 2. stat_url / stat_referer / stat_ua：DELETE WHERE stat_date=:d（整天重跑）→ INSERT top N 行
+     *
+     * @param int $statDate 日期 YYYYMMDD（如 20260728）
+     * @param int $statHour 小时 0-23
+     * @return array {stat_hour_rows:int, stat_url_rows:int, stat_referer_rows:int, stat_ua_rows:int}
+     *
+     * @throws \InvalidArgumentException 日期/小时非法
+     * @throws \Throwable 事务内任何错误（已 rollback 后 rethrow）
+     */
+    public function aggregateHour(int $statDate, int $statHour): array
+    {
+        if ($statHour < 0 || $statHour > 23) {
+            throw new \InvalidArgumentException("stat_hour must be 0-23, got: {$statHour}");
+        }
+
+        // 时间窗口
+        $dateStr = $this->formatDateStr($statDate);
+        $dayStart = strtotime("{$dateStr} 00:00:00");
+        if ($dayStart === false) {
+            throw new \InvalidArgumentException("invalid stat_date: {$statDate}");
+        }
+        $dayEnd = $dayStart + 86400;
+        $hourStart = $dayStart + $statHour * 3600;
+        $hourEnd = $hourStart + 3600;
+
+        // topN 配置（topn=0 时不限制）
+        $topn = (int) sysconfig('nginx_log', 'topn_default', 10);
+
+        // 表名（带前缀）。框架默认连接是 main（不是 mysql），按 default 取连接 prefix
+        $conn = (string) config('database.default', 'main');
+        $prefix = (string) config("database.connections.{$conn}.prefix", 'ul_');
+        $rawTable = $prefix . 'nginx_access_log';
+        $hourTable = $prefix . 'nginx_stat_hour';
+        $urlTable = $prefix . 'nginx_stat_url';
+        $refererTable = $prefix . 'nginx_stat_referer';
+        $uaTable = $prefix . 'nginx_stat_ua';
+
+        Db::startTrans();
+        try {
+            // 1. stat_hour：DELETE + INSERT 1 行
+            NginxStatHour::where('stat_date', $statDate)
+                ->where('stat_hour', $statHour)
+                ->delete();
+
+            $hourSql = "INSERT INTO `{$hourTable}`
+                (stat_date, stat_hour, pv, uv, total_bytes, status_2xx, status_3xx, status_4xx, status_5xx, status_other, avg_request_time)
+                SELECT :stat_date, :stat_hour,
+                    COUNT(*) AS pv,
+                    COUNT(DISTINCT remote_addr) AS uv,
+                    COALESCE(SUM(body_bytes_sent), 0) AS total_bytes,
+                    COALESCE(SUM(CASE WHEN status BETWEEN 200 AND 299 THEN 1 ELSE 0 END), 0) AS status_2xx,
+                    COALESCE(SUM(CASE WHEN status BETWEEN 300 AND 399 THEN 1 ELSE 0 END), 0) AS status_3xx,
+                    COALESCE(SUM(CASE WHEN status BETWEEN 400 AND 499 THEN 1 ELSE 0 END), 0) AS status_4xx,
+                    COALESCE(SUM(CASE WHEN status BETWEEN 500 AND 599 THEN 1 ELSE 0 END), 0) AS status_5xx,
+                    COALESCE(SUM(CASE WHEN status < 200 OR status > 599 THEN 1 ELSE 0 END), 0) AS status_other,
+                    COALESCE(AVG(request_time), 0) AS avg_request_time
+                FROM `{$rawTable}`
+                WHERE time_local >= :hour_start AND time_local < :hour_end
+                HAVING COUNT(*) > 0";
+
+            $statHourRows = (int) Db::execute($hourSql, [
+                'stat_date'  => $statDate,
+                'stat_hour'  => $statHour,
+                'hour_start' => $hourStart,
+                'hour_end'   => $hourEnd,
+            ]);
+
+            // 2. stat_url：DELETE（整天）+ INSERT topN
+            NginxStatUrl::where('stat_date', $statDate)->delete();
+
+            $urlLimit = $topn > 0 ? 'LIMIT ' . $topn : '';
+            $urlSql = "INSERT INTO `{$urlTable}`
+                (stat_date, uri, pv, uv, total_bytes, avg_request_time)
+                SELECT :stat_date, uri,
+                    COUNT(*) AS pv,
+                    COUNT(DISTINCT remote_addr) AS uv,
+                    COALESCE(SUM(body_bytes_sent), 0) AS total_bytes,
+                    COALESCE(AVG(request_time), 0) AS avg_request_time
+                FROM `{$rawTable}`
+                WHERE time_local >= :day_start AND time_local < :day_end
+                GROUP BY uri
+                ORDER BY pv DESC
+                {$urlLimit}";
+
+            $statUrlRows = (int) Db::execute($urlSql, [
+                'stat_date'  => $statDate,
+                'day_start'  => $dayStart,
+                'day_end'    => $dayEnd,
+            ]);
+
+            // 3. stat_referer：DELETE + INSERT topN（从 http_referer 提取 domain）
+            NginxStatReferer::where('stat_date', $statDate)->delete();
+
+            $refererLimit = $topn > 0 ? 'LIMIT ' . $topn : '';
+            $refererSql = "INSERT INTO `{$refererTable}`
+                (stat_date, referer_domain, pv, uv)
+                SELECT :stat_date,
+                    CASE
+                        WHEN http_referer IS NULL OR http_referer = '' OR http_referer = '-' THEN '-'
+                        ELSE SUBSTRING_INDEX(
+                            SUBSTRING_INDEX(
+                                REPLACE(REPLACE(http_referer, 'https://', ''), 'http://', ''),
+                                '/', 1
+                            ),
+                            '?', 1
+                        )
+                    END AS referer_domain,
+                    COUNT(*) AS pv,
+                    COUNT(DISTINCT remote_addr) AS uv
+                FROM `{$rawTable}`
+                WHERE time_local >= :day_start AND time_local < :day_end
+                GROUP BY referer_domain
+                ORDER BY pv DESC
+                {$refererLimit}";
+
+            $statRefererRows = (int) Db::execute($refererSql, [
+                'stat_date'  => $statDate,
+                'day_start'  => $dayStart,
+                'day_end'    => $dayEnd,
+            ]);
+
+            // 4. stat_ua：DELETE + INSERT topN（从 http_user_agent 分类）
+            NginxStatUa::where('stat_date', $statDate)->delete();
+
+            $uaLimit = $topn > 0 ? 'LIMIT ' . $topn : '';
+            $uaSql = "INSERT INTO `{$uaTable}`
+                (stat_date, ua_type, ua_name, pv, uv)
+                SELECT :stat_date,
+                    CASE
+                        WHEN http_user_agent LIKE '%bot%'
+                            OR http_user_agent LIKE '%spider%'
+                            OR http_user_agent LIKE '%crawl%'
+                            OR http_user_agent LIKE '%slurp%'
+                            OR http_user_agent LIKE '%bingpreview%'
+                            OR http_user_agent LIKE '%facebookexternalhit%'
+                            OR http_user_agent LIKE '%twitterbot%' THEN 'spider'
+                        WHEN http_user_agent IS NULL OR http_user_agent = '' OR http_user_agent = '-' THEN 'unknown'
+                        WHEN http_user_agent LIKE '%Mozilla%'
+                            OR http_user_agent LIKE '%Chrome%'
+                            OR http_user_agent LIKE '%Safari%'
+                            OR http_user_agent LIKE '%Firefox%'
+                            OR http_user_agent LIKE '%Edg%'
+                            OR http_user_agent LIKE '%Opera%'
+                            OR http_user_agent LIKE '%MSIE%'
+                            OR http_user_agent LIKE '%Trident%' THEN 'browser'
+                        ELSE 'unknown'
+                    END AS ua_type,
+                    CASE
+                        WHEN http_user_agent LIKE '%Googlebot%' THEN 'Googlebot'
+                        WHEN http_user_agent LIKE '%Baiduspider%' THEN 'Baiduspider'
+                        WHEN http_user_agent LIKE '%bingbot%' THEN 'Bingbot'
+                        WHEN http_user_agent LIKE '%DuckDuckBot%' THEN 'DuckDuckBot'
+                        WHEN http_user_agent LIKE '%YandexBot%' THEN 'YandexBot'
+                        WHEN http_user_agent LIKE '%Edg/%' THEN 'Microsoft Edge'
+                        WHEN http_user_agent LIKE '%OPR/%' OR http_user_agent LIKE '%Opera%' THEN 'Opera'
+                        WHEN http_user_agent LIKE '%Firefox/%' THEN 'Firefox'
+                        WHEN http_user_agent LIKE '%Chrome/%' THEN 'Chrome'
+                        WHEN http_user_agent LIKE '%Safari/%' THEN 'Safari'
+                        WHEN http_user_agent LIKE '%MSIE%' OR http_user_agent LIKE '%Trident%' THEN 'Internet Explorer'
+                        WHEN http_user_agent IS NULL OR http_user_agent = '' OR http_user_agent = '-' THEN 'Unknown'
+                        ELSE 'Other'
+                    END AS ua_name,
+                    COUNT(*) AS pv,
+                    COUNT(DISTINCT remote_addr) AS uv
+                FROM `{$rawTable}`
+                WHERE time_local >= :day_start AND time_local < :day_end
+                GROUP BY ua_type, ua_name
+                ORDER BY pv DESC
+                {$uaLimit}";
+
+            $statUaRows = (int) Db::execute($uaSql, [
+                'stat_date'  => $statDate,
+                'day_start'  => $dayStart,
+                'day_end'    => $dayEnd,
+            ]);
+
+            Db::commit();
+
+            return [
+                'stat_hour_rows'    => $statHourRows,
+                'stat_url_rows'     => $statUrlRows,
+                'stat_referer_rows' => $statRefererRows,
+                'stat_ua_rows'      => $statUaRows,
+            ];
+        } catch (\Throwable $e) {
+            Db::rollback();
+            throw $e;
+        }
+    }
+
+    /**
+     * 将 int 日期 YYYYMMDD 格式化为 "YYYY-MM-DD".
+     */
+    protected function formatDateStr(int $statDate): string
+    {
+        $y = intdiv($statDate, 10000);
+        $m = intdiv($statDate % 10000, 100);
+        $d = $statDate % 100;
+        return sprintf('%04d-%02d-%02d', $y, $m, $d);
+    }
+}