ulthon/ulthon_admin
1
0
Fork 0
mirror of https://gitee.com/ulthon/ulthon_admin.git synced 2026-08-30 12:45:32 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(log): 新增 DebugLogReader 增量读服务（JSONL 按行 + CSV fgetcsv 多行安全）

Browse Source
This commit is contained in:
augushong
2026-08-16 08:17:52 +08:00
parent 4d8cfc8844
commit c0cae93562
3 changed files with 859 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

329
tests/Unit/DebugLogReaderServiceTest.php Normal file
View File

@@ -0,0 +1,329 @@
<?php
declare(strict_types=1);
namespace tests\Unit;
use app\common\service\DebugLogReaderService;
use PHPUnit\Framework\TestCase;
use think\log\DebugLogToolkit;
/**
* DebugLogReaderService 增量读协议单元测试（JSONL 按行 + CSV fgetcsv 多行安全，不触 DB）.
*
* getLastPosition/savePosition 的 DB 持久化通过文件底部的 StubDebugLogReaderService
* 替换为内存实现（返回 canned position / 记录调用），只测 read/readCsv 的文件协议部分。
*
* 平台限制（Windows 宿主机 PHP）：fstat()['ino'] 恒为 0，轮转用例只能验证
* size<offset 归零路径，无法验证 inode 变化路径——生产 Linux 有真实 inode，
* 该路径与 XhprofLogReader 逐行同款（协议镜像），由代码评审保证。
*
* 用例 2（CSV 引号内换行跨 3 物理行）是本计划的 BLOCKER 回归锚点：遗留 CSV 的
* content 含换行是常态，若实现退化为"按 \n 切行 + str_getcsv"该用例必红。
*/
class DebugLogReaderServiceTest extends TestCase
{
/** 夹具根目录（系统临时目录下，不碰项目 runtime） */
private const TEMP_BASE = 'debug-log-reader-test';
protected function setUp(): void
{
// 写前清理（上一轮残留）
$base = self::baseDir();
if (is_dir($base)) {
self::rrmdir($base);
}
mkdir($base, 0777, true);
}
protected function tearDown(): void
{
// 测后清理
$base = self::baseDir();
if (is_dir($base)) {
self::rrmdir($base);
}
}
private static function baseDir(): string
{
return sys_get_temp_dir() . '/' . self::TEMP_BASE;
}
/**
* 递归删除目录.
*/
private static function rrmdir(string $dir): void
{
$items = scandir($dir);
foreach ($items as $item) {
if ($item === '.' || $item === '..') {
continue;
}
$path = $dir . DIRECTORY_SEPARATOR . $item;
is_dir($path) ? self::rrmdir($path) : unlink($path);
}
rmdir($dir);
}
/**
* 构造与 FIELDS 契约一致的 8 字段行（create_time_title 由时间戳确定性推导）.
*/
private static function makeRow(string $level, string $content, int $createTime): array
{
return [
'level' => $level,
'content' => $content,
'create_time' => $createTime,
'create_time_title' => date('Y-m-d H:i:s', $createTime),
'uid' => 'test-uid',
'app_name' => 'cli',
'controller_name' => '',
'action_name' => '',
];
}
/**
* 用例 1：JSONL 3 条完整行 + 尾部半行 → 产出 3 条、reachedEof=false、
* offset 停在半行起点（半行不产出不消费，下轮补 \n 后重读）.
*/
public function test_read_jsonl_yields_complete_lines_and_stops_at_half_line(): void
{
$file = self::baseDir() . '/20260816.jsonl';
$complete = '';
$expected = [];
for ($i = 1; $i <= 3; ++$i) {
$row = self::makeRow('info', "完整行 {$i}", 1700000000 + $i);
$expected[] = $row;
$complete .= json_encode($row, JSON_UNESCAPED_UNICODE) . "\n";
}
$halfLine = '{"level":"info","content":"尾部半行';
file_put_contents($file, $complete . $halfLine);
$reader = new StubDebugLogReaderService();
$yielded = iterator_to_array($reader->read($file, 100), false);
self::assertCount(3, $yielded, '3 条完整行应全部产出');
foreach ($yielded as $i => $row) {
self::assertSame($expected[$i], $row, "第 {$i} 行 decode 后应与夹具一致");
self::assertSame(DebugLogToolkit::FIELDS, array_keys($row));
}
self::assertFalse($reader->reachedEof(), '存在尾部半行时不得宣告 EOF');
self::assertSame(
strlen($complete),
$reader->getCurrentPosition()['offset'],
'offset 必须停在半行起点（= 完整部分字节数）'
);
self::assertSame(0, $reader->getDroppedLines());
self::assertSame(0, $reader->getParseFails());
}
/**
* 用例 2（BLOCKER 回归锚点）：CSV 表头 + 1 条 content 带换行的记录（引号包裹
* 跨 3 物理行）+ 1 条普通记录 → 产出 2 条完整记录（表头跳过），字段与夹具
* 一致、offset=文件末尾.
*
* 遗留 CSV 的 content 含换行是常态——fgetcsv 有状态解析是唯一正确实现，
* 任何"按 \n 切行"的退化实现都会在此用例红（跨行记录被撕碎/字段数错位）。
*/
public function test_read_csv_multiline_content_as_single_record(): void
{
$file = self::baseDir() . '/20260815.csv';
$header = 'level,content,create_time,create_time_title,uid,app_name,controller_name,action_name';
// 引号内 2 个换行 + CSV 规范的双写引号（"" 解码回 "）→ 逻辑记录跨 3 物理行
$csvEncodedContent = "第一行\nquoted \"\"word\"\" 内嵌双引号\nthird physical line";
$expectedContent = "第一行\nquoted \"word\" 内嵌双引号\nthird physical line";
$row1 = 'info,"' . $csvEncodedContent . '",1755302401,2026-08-16 00:00:01,uid-1,admin,Index,index';
$row2 = 'error,plain single line content,1755302402,2026-08-16 00:00:02,uid-2,tools,Timer,run';
$raw = $header . "\n" . $row1 . "\n" . $row2 . "\n";
file_put_contents($file, $raw);
// 夹具自检：证明这是真正的多物理行夹具（朴素按 \n 切会得到 5 条"行"而非 2 条记录）
self::assertSame(5, substr_count($raw, "\n"), '夹具应有 5 个物理行（表头 1 + 记录 3 + 记录 1）');
$reader = new StubDebugLogReaderService();
$rows = iterator_to_array($reader->readCsv($file, 100), false);
self::assertCount(2, $rows, '表头跳过，产出 2 条数据记录');
$first = $rows[0];
self::assertSame(DebugLogToolkit::FIELDS, array_keys($first));
self::assertSame('info', $first['level']);
self::assertSame($expectedContent, $first['content'], '引号内换行与双写引号必须完整还原（BLOCKER）');
self::assertSame(1755302401, $first['create_time'], 'create_time 数字字符串转 int');
self::assertSame('2026-08-16 00:00:01', $first['create_time_title'], 'create_time_title 保留字符串');
self::assertSame('uid-1', $first['uid']);
self::assertSame('admin', $first['app_name']);
self::assertSame('Index', $first['controller_name']);
self::assertSame('index', $first['action_name']);
self::assertSame('error', $rows[1]['level']);
self::assertSame('plain single line content', $rows[1]['content']);
self::assertSame(1755302402, $rows[1]['create_time']);
self::assertTrue($reader->reachedEof());
self::assertSame(
(int) filesize($file),
$reader->getCurrentPosition()['offset'],
'offset 必须推进到文件末尾（含跨物理行记录的全部字节）'
);
self::assertSame(0, $reader->getParseFails());
}
/**
* 用例 3：轮转——重写更短文件后 offset > size → 归零重读.
*
* Windows 平台限制：fstat()['ino'] 恒为 0，无法构造"inode 变化"路径，
* 只能验证 size<offset 兜底归零（stub 返回原 inode=0 使 inode 检查通过，
* 归零由 size 条件触发）；生产 Linux 两条路径并存，语义与 xhprof 版镜像。
*/
public function test_rotation_offset_beyond_size_resets_to_zero(): void
{
$file = self::baseDir() . '/rot.jsonl';
$row1 = self::makeRow('info', '旧行 1', 1700000001);
$row2 = self::makeRow('info', '旧行 2', 1700000002);
file_put_contents(
$file,
json_encode($row1, JSON_UNESCAPED_UNICODE) . "\n" . json_encode($row2, JSON_UNESCAPED_UNICODE) . "\n"
);
// 第一轮：全量读，拿到 offset（Windows 下 inode 恒 0）
$reader = new StubDebugLogReaderService();
$first = iterator_to_array($reader->read($file, 100), false);
self::assertCount(2, $first);
self::assertTrue($reader->reachedEof());
$position = $reader->getCurrentPosition();
$oldOffset = $position['offset'];
// 续读：从旧 offset 起，无新数据 → 0 条、EOF
$resume = new StubDebugLogReaderService();
$resume->stubPosition = [
'inode' => $position['inode'],
'offset' => $oldOffset,
'last_line_hash' => $position['last_line_hash'],
'parse_fail_count' => 0,
'parse_fail_samples' => null,
];
$again = iterator_to_array($resume->read($file, 100), false);
self::assertCount(0, $again, '续读起点在文件末尾，应无新行');
self::assertTrue($resume->reachedEof());
// 轮转：重写更短的文件，旧 offset > 新 size → 归零重读全量
$newRow = self::makeRow('info', '新文件唯一行', 1700000009);
file_put_contents($file, json_encode($newRow, JSON_UNESCAPED_UNICODE) . "\n");
clearstatcache();
self::assertLessThan(
$oldOffset,
(int) filesize($file),
'前置：新文件 size 必须小于旧 offset（size<offset 归零条件成立）'
);
$rotated = new StubDebugLogReaderService();
$rotated->stubPosition = [
'inode' => $position['inode'], // Windows 恒 0：inode 检查恒过，归零只能靠 size 条件
'offset' => $oldOffset,
'last_line_hash' => 'stale',
'parse_fail_count' => 0,
'parse_fail_samples' => null,
];
$reread = iterator_to_array($rotated->read($file, 100), false);
self::assertCount(1, $reread, '归零后应重读新文件全量');
self::assertSame('新文件唯一行', $reread[0]['content']);
self::assertTrue($rotated->reachedEof());
}
/**
* 用例 4：failure——坏 JSON 行 decodeLine 返 null → 计 parseFails 不产出，
* 坏行仍消费（offset 推进过毒行，不无限重读），样本可取.
*/
public function test_read_jsonl_bad_line_counted_as_fail_not_yielded(): void
{
$file = self::baseDir() . '/bad.jsonl';
$good1 = self::makeRow('info', '好行 1', 1700000001);
$good2 = self::makeRow('info', '好行 2', 1700000002);
$badLine = '{"level":"info","content":"坏 JSON 没闭合';
file_put_contents(
$file,
json_encode($good1, JSON_UNESCAPED_UNICODE) . "\n" . $badLine . "\n" . json_encode($good2, JSON_UNESCAPED_UNICODE) . "\n"
);
$reader = new StubDebugLogReaderService();
$rows = iterator_to_array($reader->read($file, 100), false);
self::assertCount(2, $rows, '坏行不产出，好行全产出');
self::assertSame('好行 1', $rows[0]['content']);
self::assertSame('好行 2', $rows[1]['content']);
self::assertSame(1, $reader->getParseFails(), '坏 JSON 行计 1 次 parseFails');
self::assertNotSame('', (string) $reader->getFailSamples(), '失败样本应非空');
self::assertStringContainsString('坏 JSON', (string) $reader->getFailSamples());
// 坏行已消费：offset 推进到文件末尾，毒行不会被下一轮重读
self::assertSame((int) filesize($file), $reader->getCurrentPosition()['offset']);
self::assertTrue($reader->reachedEof());
}
/**
* 用例 5：CSV 字段数 !== 8 → 计 parseFails 不产出（readCsv 失败路径）.
*/
public function test_read_csv_field_count_mismatch_counted_as_fail(): void
{
$file = self::baseDir() . '/bad.csv';
$header = 'level,content,create_time,create_time_title,uid,app_name,controller_name,action_name';
$good = 'info,ok content,1755302401,2026-08-16 00:00:01,uid-1,admin,Index,index';
$bad = 'info,only,three,fields'; // 4 列 ≠ 8
file_put_contents($file, $header . "\n" . $good . "\n" . $bad . "\n");
$reader = new StubDebugLogReaderService();
$rows = iterator_to_array($reader->readCsv($file, 100), false);
self::assertCount(1, $rows, '字段数不符的行不产出');
self::assertSame('ok content', $rows[0]['content']);
self::assertSame(1, $reader->getParseFails(), 'CSV 坏行计 1 次 parseFails');
self::assertSame((int) filesize($file), $reader->getCurrentPosition()['offset'], '坏行已消费推进到文件末尾');
self::assertTrue($reader->reachedEof());
}
}
/**
* 测试桩：把 position 持久化（DB 依赖）替换为内存实现，只测文件协议部分.
*
* - getLastPosition 返回 canned stubPosition（null 模拟无历史记录/首次读）
* - savePosition 仅记录调用参数，不触 DB（本任务 reader 不自动调 savePosition，
* 事务内显式调用是 T6 的职责）
*/
class StubDebugLogReaderService extends DebugLogReaderService
{
/** @var array|null getLastPosition 的 canned 返回值（null=无历史 position） */
public ?array $stubPosition = null;
/** @var array[] savePosition 调用记录 */
public array $savedPositions = [];
public function getLastPosition(string $filePath): ?array
{
unset($filePath);
return $this->stubPosition;
}
public function savePosition(string $filePath, int $inode, int $offset, ?string $lastLineHash, int $failCount = 0, ?string $failSamples = null): void
{
$this->savedPositions[] = [
'file_path' => $filePath,
'inode' => $inode,
'offset' => $offset,
'last_line_hash' => $lastLineHash,
'fail_count' => $failCount,
'fail_samples' => $failSamples,
];
}
}