feat(tools): XHProf 日志导入三层服务与函数监控物化

- XhprofProfileParserService(Base/App):纯数组解析 jsonl 行,
  meta 字段以真实 fixture 为准;simple_url 缺失兜底与采集端回调同算法;
  watch 正则按 callee 物化(ct/wt_sum/wt_max),非法正则编译计 fail 跳过
- XhprofLogReaderService(Base/App):增量读取,半行缓冲超限整行丢弃
  (不照抄 nginx 8MB 强制切行);EOF 无换行半行不 yield 不推进 offset;
  进度不自动落盘,由导入侧在同一事务内显式 savePosition
- XhprofLogImport 定时任务(Base/App):不受 XHPROF_ENABLE 门控,
  runs-*.jsonl 按文件名序增量导入;run 逐条 insert + detail chunk 20
  insertAll + run_watch insertAll 与 position 同事务落盘;导至 EOF 后
  只删已导完文件
- 修复 T1 缺陷:request_time decimal(12,3) 装不下 10 位时间戳,
  Scheme 与 migration 同步改 decimal(13,3)
- 新增 parser 单元测试(真实 fixture 断言,9 用例 42 断言)
  与 jsonl fixture;phpunit 注册 unit 测试套件
This commit is contained in:
augushong
2026-08-15 06:11:01 +08:00
parent b472df8c1d
commit a43a9a48b9
11 changed files with 1054 additions and 2 deletions

View File

@@ -0,0 +1,286 @@
<?php
declare(strict_types=1);
namespace base\common\service;
use app\admin\model\XhprofLogPosition;
use app\common\service\HostService;
/**
* XHProf jsonl 增量读取 serviceBase 层).
*
* 与 NginxLogReaderServiceBase 的两处语义差异(有意为之,勿"顺手同步"回去):
* 1. 本类不自动 savePositionjsonl 单行很大(实测 ~100KB导入侧要求
* "insertBatch + savePosition 同一事务"消崩溃重复窗口,进度落盘由
* XhprofLogImportBase 在事务内显式调用 savePosition 完成。
* 每次 yield 前先推进 currentOffset消费方任意时刻调 getCurrentPosition()
* 取到的都是"最后一条完整行末尾"的安全偏移。
* EOF 无换行的尾部半行不 yield 不推进 offset下轮写入 \n 后补齐)。
* 2. 超长行不再像 nginx 那样 8MB 强制切行(会把一条 JSON 切成两条坏行),
* 而是整行丢弃并计 dropped缓冲超限时扫描到下一个换行符为止
*
* 其余协议与 nginx 版一致:按 offset 增量读、inode/size 双条件轮转检测归零、
* maxLines 上限、position 按 (node_id, file_path) 唯一定位。
*
* 依赖倒置model 走 app\admin\model\XhprofLogPosition 入口类;节点身份走
* app\common\service\HostService。本表为全新表node_id 默认 'default' 非 NULL
* 无 nginx 版的懒加载接管旧数据问题,故不含 lazyAdopt 逻辑。
*/
class XhprofLogReaderServiceBase
{
/** 单次 fread 的 buffer 大小64KB */
protected const BUFFER_SIZE = 65536;
/** 单行硬上限32MB超过整行丢弃计 dropped。真实采样行 ~100KB32MB 已是异常防御 */
protected const MAX_LINE_BYTES = 33554432;
/** @var int 当前文件 inoderead 期间有效) */
protected $currentInode = 0;
/** @var int 安全偏移:最后一条已 yield 完整行的末尾(含换行符) */
protected $currentOffset = 0;
/** @var string|null 最后一条已 yield 行的 md5 */
protected $currentLineHash = null;
/** @var bool 本轮 read 是否干净到达 EOF尾部无半行超长行丢弃至 EOF 也视为干净收尾) */
protected $reachedEof = false;
/** @var int 本轮 read 丢弃的超长行数 */
protected $droppedLines = 0;
/**
* 增量读取 jsonl 文件yield 完整行.
*
* 读取协议:
* 1. fopen + fstat 取当前 inode/size读 position无则视为新建 offset=0
* 2. 轮转检测inode 不一致 OR filesize < offset → 归零重读
* 3. fseek 到 offset循环 fread 64KB定位已读段内最后 \n仅 yield 完整行
* 4. 每次 yield 前 currentOffset 推进到该行末尾(含 \n保证事务内 savePosition 与已消费行严格对齐
* 5. 尾部半行(无 \n不 yieldcurrentOffset 停在半行起点,下次重读
* 6. 半行缓冲超过 MAX_LINE_BYTES整行丢弃计 dropped扫描至下一个 \n 继续
* 7. yield maxLines 后退出(不自动 savePosition由上层事务内落盘
*
* @param string $filePath jsonl 文件绝对路径
* @param int $maxLines 单次 yield 的最大行数
*
* @return \Generator<string> 每次迭代返回一行(不含换行符)
*
* @throws \RuntimeException 文件无法打开或 stat/fseek/fread 失败时抛出(交由上层 try-catch
*/
public function read(string $filePath, int $maxLines = 100000): \Generator
{
// per-run 状态复位(同一个 reader 实例可串行处理多个文件)
$this->reachedEof = false;
$this->droppedLines = 0;
$this->currentLineHash = null;
$fp = @fopen($filePath, 'rb');
if ($fp === false) {
throw new \RuntimeException("XhprofLogReaderService: 无法打开文件 {$filePath}");
}
try {
clearstatcache(true, $filePath);
$stat = fstat($fp);
if ($stat === false) {
throw new \RuntimeException("XhprofLogReaderService: fstat 失败 {$filePath}");
}
$currentInode = (int) $stat['ino'];
$currentSize = (int) $stat['size'];
$this->currentInode = $currentInode;
// 读 position无则默认 offset=0
$offset = 0;
$position = $this->getLastPosition($filePath);
if ($position !== null) {
$offset = (int) $position['offset'];
// 轮转检测inode 变化同日文件被重建OR size < offset防御性归零
if ((int) $position['inode'] !== $currentInode || $currentSize < $offset) {
$offset = 0;
}
}
$this->currentOffset = $offset;
if ($offset > 0 && fseek($fp, $offset) !== 0) {
throw new \RuntimeException("XhprofLogReaderService: fseek 失败 offset={$offset} file={$filePath}");
}
$yielded = 0;
$buffer = '';
// $bufferOffset 始终等于 buffer[0] 对应的文件字节偏移
$bufferOffset = $offset;
while (true) {
$chunk = fread($fp, self::BUFFER_SIZE);
if ($chunk === false) {
throw new \RuntimeException("XhprofLogReaderService: fread 失败 file={$filePath}");
}
if ($chunk === '') {
// EOFbuffer 为空说明干净收尾;有半行则 currentOffset 已停在最后完整行末尾
$this->reachedEof = ($buffer === '');
return;
}
$buffer .= $chunk;
// 内层循环:处理 buffer 中所有完整行(含末尾 \n 的)
$pos = 0;
while (($nlPos = strpos($buffer, "\n", $pos)) !== false) {
$line = substr($buffer, $pos, $nlPos - $pos);
$pos = $nlPos + 1;
// 先推进安全偏移再 yield消费方在事务内 savePosition 拿到的
// 一定是"已 yield 行末尾",与该事务内入库的数据严格对齐
$this->currentOffset = $bufferOffset + $pos;
$this->currentLineHash = md5($line);
yield $line;
$yielded++;
if ($yielded >= $maxLines) {
return;
}
}
// buffer 中已消费 $pos 字节半行buffer[$pos..])保留
$bufferOffset += $pos;
$buffer = substr($buffer, $pos);
// 超长行防御:整行丢弃(禁止强制切行——会把一条 JSON 切成两条坏行)
if (strlen($buffer) >= self::MAX_LINE_BYTES) {
$this->droppedLines++;
$buffer = '';
// 扫描至下一个换行符,将其后内容作为新 buffer 继续
while (true) {
$chunk = fread($fp, self::BUFFER_SIZE);
if ($chunk === false || $chunk === '') {
// 至 EOF 仍未等到换行符:整个尾部按已丢弃处理,偏移推到文件末尾
$bufferOffset = (int) ftell($fp);
$this->currentOffset = $bufferOffset;
$this->reachedEof = true;
return;
}
$nl = strpos($chunk, "\n");
if ($nl !== false) {
$bufferOffset = (int) ftell($fp) - (strlen($chunk) - $nl - 1);
$this->currentOffset = $bufferOffset;
$buffer = substr($chunk, $nl + 1);
break;
}
}
}
}
} finally {
if (is_resource($fp)) {
fclose($fp);
}
}
}
/**
* 当前安全读取位置read 期间/结束后可调).
*
* @return array{inode: int, offset: int, last_line_hash: string|null}
*/
public function getCurrentPosition(): array
{
return [
'inode' => $this->currentInode,
'offset' => $this->currentOffset,
'last_line_hash' => $this->currentLineHash,
];
}
/**
* 本轮 read 是否干净到达 EOF尾部无半行.
*
* 导入侧据此判断"文件已导完"reachedEof 且 offset >= filesize 才允许 unlink。
*/
public function reachedEof(): bool
{
return $this->reachedEof;
}
/**
* 本轮 read 丢弃的超长行数(由导入侧并入 fails 统计).
*/
public function getDroppedLines(): int
{
return $this->droppedLines;
}
/**
* 查询文件的上次读取位置(按 node_id 隔离).
*
* @return array|null 命中时返回 [inode, offset, last_line_hash, parse_fail_count, parse_fail_samples],无记录返回 null
*/
public function getLastPosition(string $filePath): ?array
{
$nodeId = HostService::getNodeId();
$row = XhprofLogPosition::where('file_path', $filePath)
->where('node_id', $nodeId)
->find();
if ($row === null) {
return null;
}
return [
'inode' => (int) $row->getData('inode'),
'offset' => (int) $row->getData('offset'),
'last_line_hash' => $row->getData('last_line_hash'),
'parse_fail_count' => (int) $row->getData('parse_fail_count'),
'parse_fail_samples' => $row->getData('parse_fail_samples'),
];
}
/**
* 保存读取位置upsert按 node_id 隔离).
*
* 由导入任务在"insertBatch 同一事务"内显式调用(本类不自动保存),
* 保证崩溃时入库与进度要么同时生效、要么同时回滚,消重复导入窗口。
*
* @param string $filePath jsonl 文件绝对路径
* @param int $inode 当前文件 inode
* @param int $offset 新的字节偏移(最后一条完整行末尾)
* @param string|null $lastLineHash 最后一条已入库行 md5
* @param int $failCount 本批解析失败累计次数
* @param string|null $failSamples 解析失败样本json 或 null
*/
public function savePosition(string $filePath, int $inode, int $offset, ?string $lastLineHash, int $failCount = 0, ?string $failSamples = null): void
{
$now = time();
$nodeId = HostService::getNodeId();
$existing = XhprofLogPosition::where('file_path', $filePath)
->where('node_id', $nodeId)
->find();
if ($existing !== null) {
$existing->save([
'inode' => $inode,
'offset' => $offset,
'last_read_time' => $now,
'last_line_hash' => $lastLineHash,
'parse_fail_count' => $failCount,
'parse_fail_samples' => $failSamples,
]);
return;
}
XhprofLogPosition::create([
'node_id' => $nodeId,
'file_path' => $filePath,
'inode' => $inode,
'offset' => $offset,
'last_read_time' => $now,
'last_line_hash' => $lastLineHash,
'parse_fail_count' => $failCount,
'parse_fail_samples' => $failSamples,
'create_time' => $now,
'update_time' => $now,
]);
}
}

View File

@@ -0,0 +1,204 @@
<?php
declare(strict_types=1);
namespace base\common\service;
/**
* XHProf jsonl 行解析器Base 层).
*
* 输入php-profiler FileSaver 落盘的 jsonl 单行 {"profile":{...},"meta":{...}}
* 输出ul_xhprof_run 入库行字段 + 解码后的 profile供 watch 物化)。
*
* 纯数组处理,不依赖 Xhgui\Profiler 等镜像内类库app 侧禁止 use见计划 guardrails
*
* meta 真实结构(以容器内实采 fixture 为准tests/fixtures/xhprof-sample.jsonl
* url = meta.url原生存在通常为 path 形如 '/',无需从 REQUEST_URI 拼)
* simple_url = meta.simple_urlbootstrap 的 profiler.simple_url 回调原生产出;
* 仅缺失/为空时兜底自算,算法与回调保持一致,勿漂移)
* method = meta.SERVER.REQUEST_METHOD
* request_time = meta.request_ts_micro.sec + usec/1e6缺失退化 meta.SERVER.REQUEST_TIME_FLOAT
* wall/cpu/pmu = profile['main()'] 的 wt/cpu/pmumain() 缺失降级 0 并置 degraded由上层计 fail
*
* 业务侧通过 app/common/service/XhprofProfileParserService 覆盖本类方法.
*/
class XhprofProfileParserServiceBase
{
/** run.url 列长ul_xhprof_run.url varchar(500),超长截断) */
protected const URL_MAX_LENGTH = 500;
/** run.simple_url 列长 */
protected const SIMPLE_URL_MAX_LENGTH = 255;
/** run.method 列长 */
protected const METHOD_MAX_LENGTH = 10;
/**
* 解析单行 jsonl.
*
* @param string $line 原始行(不含换行符)
*
* @return array|null 成功返回 run 行字段 + profile空行/非法 JSON/缺 profile 键返回 null。
* main() 缺失时 wall/cpu/memory_peak 降级 0 且 degraded=true行仍可入库
*/
public function parse(string $line): ?array
{
$line = trim($line);
if ($line === '' || $line[0] !== '{') {
return null;
}
$data = json_decode($line, true);
if (!is_array($data) || !isset($data['profile']) || !is_array($data['profile'])) {
return null;
}
$meta = isset($data['meta']) && is_array($data['meta']) ? $data['meta'] : [];
// main() 缺失:降级 0 并标记 degraded导入侧对 degraded 行计 fail 但不丢数据)
$main = $data['profile']['main()'] ?? null;
$degraded = !is_array($main);
$wallTime = $degraded ? 0 : (int) ($main['wt'] ?? 0);
$cpuTime = $degraded ? 0 : (int) ($main['cpu'] ?? 0);
$memoryPeak = $degraded ? 0 : (int) ($main['pmu'] ?? 0);
$url = (string) ($meta['url'] ?? '');
$simpleUrl = (string) ($meta['simple_url'] ?? '');
if ($simpleUrl === '') {
$simpleUrl = $this->buildSimpleUrlFallback($url);
}
return [
'url' => mb_substr($url, 0, self::URL_MAX_LENGTH),
'simple_url' => mb_substr($simpleUrl, 0, self::SIMPLE_URL_MAX_LENGTH),
'method' => mb_substr((string) ($meta['SERVER']['REQUEST_METHOD'] ?? ''), 0, self::METHOD_MAX_LENGTH),
'wall_time' => $wallTime,
'cpu_time' => $cpuTime,
'memory_peak' => $memoryPeak,
'request_time' => $this->parseRequestTime($meta),
'profile' => $data['profile'],
'degraded' => $degraded,
];
}
/**
* simple_url 兜底计算.
*
* 与采集端 bootstrap 的 profiler.simple_url 回调算法完全一致(唯一实现点在采集端,
* 这里仅做缺失兜底path 去 query、纯数字段替换 ':id'、'/' 连接(无前导斜杠)。
*/
public function buildSimpleUrlFallback(string $url): string
{
$path = parse_url($url, PHP_URL_PATH);
if (!is_string($path) || $path === '') {
return '';
}
$parts = explode('/', trim($path, '/'));
foreach ($parts as &$part) {
if ($part !== '' && ctype_digit($part)) {
$part = ':id';
}
}
unset($part);
return implode('/', $parts);
}
/**
* 编译 watch 正则(导入批次调用一次,结果供 materializeWatches 复用,避免逐行重复编译).
*
* @param array $watches [['id' => int, 'regex' => string], ...]
*
* @return array{patterns: array<int, string>, fails: int}
* patterns 以 watch_id 为键id 非法/正则为空/@preg_match 编译失败
* 的条目跳过并计入 fails防非法正则打爆导入任务
*/
public function compileWatchRegexes(array $watches): array
{
$patterns = [];
$fails = 0;
foreach ($watches as $watch) {
$id = (int) ($watch['id'] ?? 0);
$regex = (string) ($watch['regex'] ?? '');
if ($id <= 0 || $regex === '') {
$fails++;
continue;
}
if (@preg_match($regex, '') === false) {
$fails++;
continue;
}
$patterns[$id] = $regex;
}
return ['patterns' => $patterns, 'fails' => $fails];
}
/**
* watch 物化:拆 "caller==>callee" 边,按 callee被调用函数名匹配正则.
*
* 命中边 ct 累加、wt 累加 wt_sum、wt_max 取最大。
* 裸键(如 main())无 ==> 分隔符,不参与物化。
*
* @param array $profile 解码后的 profile 边表
* @param array $patterns compileWatchRegexes 产出的 [watch_id => regex]
*
* @return array<int, array{ct: int, wt_sum: int, wt_max: int}> 以 watch_id 为键的聚合行
*/
public function materializeWatches(array $profile, array $patterns): array
{
if (empty($patterns)) {
return [];
}
$result = [];
foreach ($profile as $key => $metrics) {
$sep = strpos($key, '==>');
if ($sep === false) {
continue;
}
$callee = substr($key, $sep + 3);
$ct = (int) ($metrics['ct'] ?? 0);
$wt = (int) ($metrics['wt'] ?? 0);
foreach ($patterns as $watchId => $regex) {
if (@preg_match($regex, $callee) !== 1) {
continue;
}
if (!isset($result[$watchId])) {
$result[$watchId] = ['ct' => 0, 'wt_sum' => 0, 'wt_max' => 0];
}
$result[$watchId]['ct'] += $ct;
$result[$watchId]['wt_sum'] += $wt;
if ($wt > $result[$watchId]['wt_max']) {
$result[$watchId]['wt_max'] = $wt;
}
}
}
return $result;
}
/**
* request_time优先 meta.request_ts_microsec+usec缺失退化 meta.SERVER.REQUEST_TIME_FLOAT.
*
* @return float 浮点秒round 3 位,与 decimal(12,3) 列精度对齐)
*/
protected function parseRequestTime(array $meta): float
{
$ts = $meta['request_ts_micro'] ?? null;
if (is_array($ts) && isset($ts['sec'])) {
$sec = (int) $ts['sec'];
$usec = (int) ($ts['usec'] ?? 0);
return round($sec + $usec / 1000000, 3);
}
$float = $meta['SERVER']['REQUEST_TIME_FLOAT'] ?? null;
if (is_numeric($float)) {
return round((float) $float, 3);
}
return 0.0;
}
}

View File

@@ -0,0 +1,316 @@
<?php
declare(strict_types=1);
namespace base\tools\controller\timer;
use app\admin\model\XhprofRun;
use app\admin\model\XhprofRunDetail;
use app\admin\model\XhprofRunWatch;
use app\admin\model\XhprofWatch;
use app\common\controller\TimerController;
use think\facade\Db;
use think\facade\Log;
/**
* XHProf jsonl 导入定时任务Base 层).
*
* 流程:
* 1. 扫描 XHPROF_FILE_DIR 下 runs-*.jsonl 按文件名序(旧日期先导)
* 2. 依赖倒置:实例化 app 层 Parser / Readerwatch 正则 per 批次编译一次
* 3. 每文件增量读取position 按 node_id+file_path 定位inode 变化归零)→
* run 逐条 insert 拿自增 id → detail chunk 20 insertAll →
* run_watch 物化聚合 insertAll同一事务内 insertBatch + savePosition消崩溃重复窗口
* 4. 文件导至 EOF 且 position 落盘后 unlink只删已导完文件
*
* 门控约定:本任务不受 XHPROF_ENABLE 门控——开关只管采集端,
* 已落盘的残留文件必须能导完(关闭采集后存量数据照常入库)。
*
* 目录单一来源getenv('XHPROF_FILE_DIR'),与采集端 bootstrap 读同一变量,
* 禁止与 ThinkPHP env() 混用(两处漂移会导致导入读不到文件)。
*
* 业务侧如需定制导入逻辑,重写 app/tools/controller/timer/XhprofLogImport.php 对应方法即可拦截。
*/
class XhprofLogImportBase extends TimerController
{
/**
* 防刷间隔(秒).
* 仅做控制器侧防刷,不影响定时器侧 frequency 节流。
*/
protected $frequency = 60;
/**
* 并发上限 cap单分片id=0。jsonl 是节点本地文件,多节点各自导自己的。
*/
protected $concurrency = 1;
/**
* run 行缓冲上限(同时是 detail insertAll 的 chunk单行实测 ~100KB20 条一批防单条 SQL 过大).
*/
protected const RUN_BATCH = 20;
/**
* 单文件单轮最大读取行数行大2000 行 ≈ 200MB 上界已足够单轮消化;未读完下轮继续).
*/
protected const READ_MAX_LINES = 2000;
/**
* parse_fail_samples 最大记录条数.
*/
protected const FAIL_SAMPLES_MAX = 10;
public function do()
{
// 注意:不受 XHPROF_ENABLE 门控(开关只管采集,残留文件必须能导完)
$dir = $this->fileDir();
$files = glob($dir . '/runs-*.jsonl');
if ($files === false || $files === []) {
return json_encode([
'files' => 0, 'runs' => 0, 'watches' => 0, 'fails' => 0, 'deleted' => 0,
], JSON_UNESCAPED_UNICODE);
}
sort($files); // 按文件名序runs-{Ymd} 字典序即日期序),旧日期先导
// 依赖倒置:使用 app 层入口类(业务侧可重写拦截)
$parser = new \app\common\service\XhprofProfileParserService();
$reader = new \app\common\service\XhprofLogReaderService();
// watch 正则 per 批次编译:本节点启用的规则一次编译,导入全程复用
$watchRows = XhprofWatch::field('id,regex')->where('status', 1)->select()->toArray();
$compiled = $parser->compileWatchRegexes($watchRows);
$patterns = $compiled['patterns'];
$totalFiles = 0;
$totalRuns = 0;
$totalWatches = 0;
$totalFails = $compiled['fails'];
$deleted = 0;
$errors = [];
foreach ($files as $file) {
try {
$stats = $this->processFile($parser, $reader, $file, $patterns);
$totalFiles++;
$totalRuns += $stats['runs'];
$totalWatches += $stats['watches'];
$totalFails += $stats['fails'];
if ($stats['deleted']) {
$deleted++;
}
} catch (\Throwable $e) {
Log::error('xhprof_log_import: 处理文件异常 ' . $file . ' - ' . $e->getMessage());
$errors[] = 'error:' . basename($file) . ':' . mb_substr($e->getMessage(), 0, 200);
}
}
return json_encode([
'files' => $totalFiles,
'runs' => $totalRuns,
'watches' => $totalWatches,
'fails' => $totalFails,
'deleted' => $deleted,
'errors' => $errors,
], JSON_UNESCAPED_UNICODE);
}
/**
* jsonl 落盘目录(与采集端 bootstrap getenv 读同一变量,禁止 ThinkPHP env().
*/
protected function fileDir(): string
{
return getenv('XHPROF_FILE_DIR') ?: '/var/www/html/runtime/xhprof';
}
/**
* 处理单个文件:增量读取、解析、批量入库(事务内 savePosition、导完删除.
*
* @return array{runs: int, watches: int, fails: int, deleted: bool}
*/
protected function processFile(
\app\common\service\XhprofProfileParserService $parser,
\app\common\service\XhprofLogReaderService $reader,
string $file,
array $patterns
): array {
$runs = 0;
$watches = 0;
$fails = 0;
$samples = [];
$buffer = [];
// Reader 是 Generator让其自然走完maxLines / EOF 自然结束,进度由本类事务内落盘)
foreach ($reader->read($file, self::READ_MAX_LINES) as $line) {
$parsed = $parser->parse($line);
if ($parsed === null) {
$fails++;
$this->addFailSample($samples, $line);
continue;
}
if ($parsed['degraded']) {
// main() 缺失:行仍入库(降级 0但计 fail 留痕
$fails++;
$this->addFailSample($samples, $line);
}
$buffer[] = [
'run' => $this->buildRunRow($parsed),
'raw' => $line,
'profile' => $parsed['profile'],
];
if (count($buffer) >= self::RUN_BATCH) {
$flushed = $this->flushBatch($parser, $reader, $file, $buffer, $patterns, $fails, $samples);
$runs += $flushed['runs'];
$watches += $flushed['watches'];
$buffer = [];
}
}
// flush 剩余buffer 为空时也补一次 position 落盘(推进被失败行消费掉的区间)
if (!empty($buffer)) {
$flushed = $this->flushBatch($parser, $reader, $file, $buffer, $patterns, $fails, $samples);
$runs += $flushed['runs'];
$watches += $flushed['watches'];
} else {
$position = $reader->getCurrentPosition();
$reader->savePosition(
$file,
$position['inode'],
$position['offset'],
$position['last_line_hash'],
$fails,
$this->encodeSamples($samples)
);
}
// 超长行丢弃并入 failsReader 整行丢弃计数)
$fails += $reader->getDroppedLines();
// 文件导至 EOF 且 position 落盘后才可删除(只删已导完文件,未导完/尾部半行不删)
$isDeleted = false;
if ($reader->reachedEof()) {
clearstatcache(true, $file);
$position = $reader->getCurrentPosition();
$size = filesize($file);
if ($size !== false && $position['offset'] >= $size) {
$isDeleted = @unlink($file);
if ($isDeleted) {
Log::info('xhprof_log_import: 文件已导完删除 ' . $file);
}
}
}
Log::info("xhprof_log_import: 文件 {$file} 完成 runs={$runs} watches={$watches} fails={$fails}");
return ['runs' => $runs, 'watches' => $watches, 'fails' => $fails, 'deleted' => $isDeleted];
}
/**
* 构造 run 入库行(字段对齐 ul_xhprof_run单位 μs/字节).
*/
protected function buildRunRow(array $parsed): array
{
return [
'node_id' => \app\common\service\HostService::getNodeId(),
'url' => $parsed['url'],
'simple_url' => $parsed['simple_url'],
'method' => $parsed['method'],
'wall_time' => $parsed['wall_time'],
'cpu_time' => $parsed['cpu_time'],
'memory_peak' => $parsed['memory_peak'],
'request_time' => $parsed['request_time'],
'create_time' => time(),
];
}
/**
* 批量入库:同一事务内 run 逐条 insert拿自增 id→ detail/run_watch insertAll → savePosition.
*
* 事务语义:入库与 position 落盘要么同时生效、要么同时回滚——
* 崩溃恢复后从 position 处重读,不会重复导入已提交批次。
*
* @param array $buffer [['run' => array, 'raw' => string, 'profile' => array], ...]
*
* @return array{runs: int, watches: int}
*/
protected function flushBatch(
\app\common\service\XhprofProfileParserService $parser,
\app\common\service\XhprofLogReaderService $reader,
string $file,
array $buffer,
array $patterns,
int $fails,
array $samples
): array {
// 捕获当前安全偏移(最后一条已 yield 完整行末尾),事务内与数据同落盘
$position = $reader->getCurrentPosition();
$samplesJson = $this->encodeSamples($samples);
$runs = 0;
$watches = 0;
Db::transaction(function () use ($parser, $reader, $file, $buffer, $patterns, $position, $fails, $samplesJson, &$runs, &$watches) {
$detailBatch = [];
$watchBatch = [];
$now = time();
foreach ($buffer as $item) {
$runId = (int) XhprofRun::insertGetId($item['run']);
$runs++;
$detailBatch[] = [
'id' => $runId,
'profile_data' => $item['raw'],
];
foreach ($parser->materializeWatches($item['profile'], $patterns) as $watchId => $agg) {
$watchBatch[] = [
'run_id' => $runId,
'watch_id' => $watchId,
'ct' => $agg['ct'],
'wt_sum' => $agg['wt_sum'],
'wt_max' => $agg['wt_max'],
'create_time' => $now,
];
$watches++;
}
}
if (!empty($detailBatch)) {
XhprofRunDetail::insertAll($detailBatch);
}
if (!empty($watchBatch)) {
XhprofRunWatch::insertAll($watchBatch);
}
$reader->savePosition(
$file,
$position['inode'],
$position['offset'],
$position['last_line_hash'],
$fails,
$samplesJson
);
});
return ['runs' => $runs, 'watches' => $watches];
}
/**
* 收集解析失败样本(截断至 500 字符,最多 FAIL_SAMPLES_MAX 条).
*/
protected function addFailSample(array &$samples, string $line): void
{
if (count($samples) >= self::FAIL_SAMPLES_MAX) {
return;
}
$samples[] = mb_substr($line, 0, 500);
}
/**
* 失败样本编码为 json空样本返回 null与 position 列 nullable 对齐).
*/
protected function encodeSamples(array $samples): ?string
{
return empty($samples) ? null : json_encode($samples, JSON_UNESCAPED_UNICODE);
}
}