Skip to content

E
ezijing-node-server
提交 9e258d66 authored 作者: 王鹏飞's avatar 王鹏飞
浏览文件
操作

chore: 本地日志统计分析

上级 47acf8b6
显示空白字符变更
内嵌 并排
正在显示 包含 519 行增加 和 3 行删除
package-lock.json
浏览文件 @ 9e258d66
......@@ -16,6 +16,7 @@
"dotenv": "^16.4.5",
"express": "^4.21.1",
"mongoose": "^8.8.0",
"mysql2": "^3.20.0",
"pino": "^9.5.0",
"pino-http": "^10.3.0",
"ua-parser-js": "^2.0.6"
......@@ -317,6 +318,16 @@
"dev": true,
"license": "MIT"
},
"node_modules/@types/node": {
"version": "25.5.0",
"resolved": "https://registry.npmjs.org/@types/node/-/node-25.5.0.tgz",
"integrity": "sha512-jp2P3tQMSxWugkCUKLRPVUpGaL5MVFwF8RDuSRztfwgN1wmqJeMSbKlnEtQqU8UrhTmzEmZdu2I6v2dpp7XIxw==",
"license": "MIT",
"peer": true,
"dependencies": {
"undici-types": "~7.18.0"
}
},
"node_modules/@types/webidl-conversions": {
"version": "7.0.3",
"resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-7.0.3.tgz",
......@@ -351,7 +362,6 @@
"integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==",
"dev": true,
"license": "MIT",
"peer": true,
"bin": {
"acorn": "bin/acorn"
},
......@@ -412,6 +422,15 @@
"node": ">=8.0.0"
}
},
"node_modules/aws-ssl-profiles": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/aws-ssl-profiles/-/aws-ssl-profiles-1.1.2.tgz",
"integrity": "sha512-NZKeq9AfyQvEeNlN0zSYAaWrmBffJh3IELMZfRpJVWgrpEbtEpnjvzqBPf+mxoI287JohRDoa+/nsfqqiZmF6g==",
"license": "MIT",
"engines": {
"node": ">= 6.0.0"
}
},
"node_modules/axios": {
"version": "1.13.2",
"resolved": "https://registry.npmjs.org/axios/-/axios-1.13.2.tgz",
......@@ -699,6 +718,15 @@
"node": ">=0.4.0"
}
},
"node_modules/denque": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
"integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==",
"license": "Apache-2.0",
"engines": {
"node": ">=0.10"
}
},
"node_modules/depd": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz",
......@@ -845,7 +873,6 @@
"integrity": "sha512-BhHmn2yNOFA9H9JmmIVKJmd288g9hrVRDkdoIgRCRuSySRUHH7r/DI6aAXW9T1WwUuY3DFgrcaqB+deURBLR5g==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@eslint-community/eslint-utils": "^4.8.0",
"@eslint-community/regexpp": "^4.12.1",
......@@ -1366,6 +1393,15 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/generate-function": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.3.1.tgz",
"integrity": "sha512-eeB5GfMNeevm/GRYq20ShmsaGcmI81kIX2K9XQx5miC8KdHaC6Jm0qQ8ZNeGOi7wYB8OsdxKs+Y2oVuTFuVwKQ==",
"license": "MIT",
"dependencies": {
"is-property": "^1.0.2"
}
},
"node_modules/get-caller-file": {
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
......@@ -1598,6 +1634,12 @@
"node": ">=0.10.0"
}
},
"node_modules/is-property": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz",
"integrity": "sha512-Ks/IoX00TtClbGQr4TWXemAnktAQvYB7HzcCxDGqEZU6oCmb2INHuOoKxbtR+HFkmYWBKv/dOZtGRiAjDhj92g==",
"license": "MIT"
},
"node_modules/is-standalone-pwa": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/is-standalone-pwa/-/is-standalone-pwa-0.1.1.tgz",
......@@ -1723,6 +1765,27 @@
"dev": true,
"license": "MIT"
},
"node_modules/long": {
"version": "5.3.2",
"resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz",
"integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==",
"license": "Apache-2.0"
},
"node_modules/lru.min": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/lru.min/-/lru.min-1.1.4.tgz",
"integrity": "sha512-DqC6n3QQ77zdFpCMASA1a3Jlb64Hv2N2DciFGkO/4L9+q/IpIAuRlKOvCXabtRW6cQf8usbmM6BE/TOPysCdIA==",
"license": "MIT",
"engines": {
"bun": ">=1.0.0",
"deno": ">=1.30.0",
"node": ">=8.0.0"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/wellwelwel"
}
},
"node_modules/math-intrinsics": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz",
......@@ -1952,6 +2015,56 @@
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
},
"node_modules/mysql2": {
"version": "3.20.0",
"resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.20.0.tgz",
"integrity": "sha512-eCLUs7BNbgA6nf/MZXsaBO1SfGs0LtLVrJD3WeWq+jPLDWkSufTD+aGMwykfUVPdZnblaUK1a8G/P63cl9FkKg==",
"license": "MIT",
"dependencies": {
"aws-ssl-profiles": "^1.1.2",
"denque": "^2.1.0",
"generate-function": "^2.3.1",
"iconv-lite": "^0.7.2",
"long": "^5.3.2",
"lru.min": "^1.1.4",
"named-placeholders": "^1.1.6",
"sql-escaper": "^1.3.3"
},
"engines": {
"node": ">= 8.0"
},
"peerDependencies": {
"@types/node": ">= 8"
}
},
"node_modules/mysql2/node_modules/iconv-lite": {
"version": "0.7.2",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz",
"integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==",
"license": "MIT",
"dependencies": {
"safer-buffer": ">= 2.1.2 < 3.0.0"
},
"engines": {
"node": ">=0.10.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/named-placeholders": {
"version": "1.1.6",
"resolved": "https://registry.npmjs.org/named-placeholders/-/named-placeholders-1.1.6.tgz",
"integrity": "sha512-Tz09sEL2EEuv5fFowm419c1+a/jSMiBjI9gHxVLrVdbUkkNUUfjsVYs9pVZu5oCon/kmRh9TfLEObFtkVxmY0w==",
"license": "MIT",
"dependencies": {
"lru.min": "^1.1.0"
},
"engines": {
"node": ">=8.0.0"
}
},
"node_modules/natural-compare": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz",
......@@ -2548,6 +2661,21 @@
"node": ">= 10.x"
}
},
"node_modules/sql-escaper": {
"version": "1.3.3",
"resolved": "https://registry.npmjs.org/sql-escaper/-/sql-escaper-1.3.3.tgz",
"integrity": "sha512-BsTCV265VpTp8tm1wyIm1xqQCS+Q9NHx2Sr+WcnUrgLrQ6yiDIvHYJV5gHxsj1lMBy2zm5twLaZao8Jd+S8JJw==",
"license": "MIT",
"engines": {
"bun": ">=1.0.0",
"deno": ">=2.0.0",
"node": ">=12.0.0"
},
"funding": {
"type": "github",
"url": "https://github.com/mysqljs/sql-escaper?sponsor=1"
}
},
"node_modules/statuses": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz",
......@@ -2676,6 +2804,13 @@
"node": "*"
}
},
"node_modules/undici-types": {
"version": "7.18.2",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz",
"integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==",
"license": "MIT",
"peer": true
},
"node_modules/unpipe": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
......
package.json
浏览文件 @ 9e258d66
......@@ -30,6 +30,7 @@
"dotenv": "^16.4.5",
"express": "^4.21.1",
"mongoose": "^8.8.0",
"mysql2": "^3.20.0",
"pino": "^9.5.0",
"pino-http": "^10.3.0",
"ua-parser-js": "^2.0.6"
......
src/config.js
浏览文件 @ 9e258d66
......@@ -10,6 +10,13 @@ const config = {
process.env.MONGODB_URI ||
'mongodb://log:log%402025@dds-2ze70e3ecb9601941614-pub.mongodb.rds.aliyuncs.com:3717/logs_data',
},
mysql: {
host: process.env.MYSQL_HOST || '172.16.3.199',
port: parseInt(process.env.MYSQL_PORT || '3306', 10) || 3306,
user: process.env.MYSQL_USER || 'root',
password: process.env.MYSQL_PASSWORD || 'Wa*3@m3DcyuK',
database: process.env.MYSQL_DATABASE || 'com_resource',
},
wechat: { apps: {} },
}
......
src/lib/mysql.js 0 → 100644
浏览文件 @ 9e258d66
import mysql from 'mysql2/promise'
import config from '../config.js'
let pool
export const getMysqlPool = () => {
if (!pool) {
pool = mysql.createPool({
host: config.mysql.host,
port: config.mysql.port,
user: config.mysql.user,
password: config.mysql.password,
database: config.mysql.database,
waitForConnections: true,
connectionLimit: 10,
queueLimit: 0,
supportBigNumbers: true,
bigNumberStrings: true,
})
}
return pool
}
src/modules/logs/logs.controller.js
浏览文件 @ 9e258d66
......@@ -88,3 +88,43 @@ export const getStats = async (req, res, next) => {
next(err)
}
}
export const getUserUsage = async (req, res, next) => {
try {
const { appName, startTime, endTime, page, limit, sortBy, sortOrder } = req.query
const result = await logsService.getUserUsage(
{ appName, startTime, endTime },
{ page: parseInt(page) || 1, limit: parseInt(limit) || 20, sortBy, sortOrder }
)
success(res, result)
} catch (err) {
logger.error({ err }, 'Failed to get user usage stats')
next(err)
}
}
export const aggregateStats = async (req, res, next) => {
try {
const { field, value, filters = {}, options = {} } = req.body
if (!field) {
return error(res, 'field is required')
}
logger.info({ field, value, filters, options }, 'Aggregating stats')
const result = await logsService.aggregateStats({
field,
value,
filters,
options,
})
success(res, result)
} catch (err) {
logger.error({ err }, 'Failed to aggregate stats')
next(err)
}
}
src/modules/logs/logs.routes.js
浏览文件 @ 9e258d66
......@@ -7,5 +7,7 @@ router.post('/logs', controller.create)
router.post('/logs/batch', controller.createBatch)
router.get('/logs', controller.query)
router.get('/logs/stats', controller.getStats)
router.get('/logs/user-usage', controller.getUserUsage)
router.post('/logs/aggregate', controller.aggregateStats)
export default router
src/modules/logs/logs.service.js
浏览文件 @ 9e258d66
import Log from './logs.model.js'
import dayjs from 'dayjs'
import { getMysqlPool } from '../../lib/mysql.js'
const TEXT_API_PATTERN = '/api/volcano/api/v3/chat/completions'
const IMAGE_API_PATTERN = '/api/volcano/api/v3/images/generations'
const VIDEO_API_PATTERN = '/api/volcano/api/v3/contents/generations/tasks'
const DIGITAL_HUMAN_API_PATTERNS = [
'/api/digitalhuman/open/v1/video/submit/fast',
'/api/digitalhuman/open/v1/video/submit',
'/api/digitalhuman/open/v1/video/advanced/submit',
]
const ALLOWED_API_PATTERNS = [TEXT_API_PATTERN, IMAGE_API_PATTERN, VIDEO_API_PATTERN, ...DIGITAL_HUMAN_API_PATTERNS]
const escapeRegex = (value) => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
const buildStrictApiRegex = (pattern) => `${escapeRegex(pattern)}(?:$|[?#])`
const buildAllowedApiUrlMatch = () => ({
$in: ALLOWED_API_PATTERNS.map((pattern) => new RegExp(buildStrictApiRegex(pattern), 'i')),
})
const enrichUserUsageList = async (list) => {
if (!list.length) return list
const ssoIds = list.map((item) => item.userId).filter(Boolean)
if (!ssoIds.length) return list
const pool = getMysqlPool()
const [rows] = await pool.query(
`
SELECT
CAST(s.sso_id AS CHAR) AS userId,
s.name AS studentName,
s.mobile AS mobile,
CAST(s.organ_id AS CHAR) AS organId,
c.name AS className
FROM learning_s_edu_student s
LEFT JOIN learning_r_edu_class_student r ON r.student_id = s.id
LEFT JOIN learning_s_edu_class c ON c.id = r.class_id
WHERE s.sso_id IN (?)
`,
[ssoIds]
)
const profileMap = new Map(rows.map((row) => [row.userId, row]))
return list.map((item) => {
const profile = profileMap.get(item.userId)
return {
...item,
studentName: profile?.studentName || null,
mobile: profile?.mobile || null,
organId: profile?.organId || null,
className: profile?.className || null,
}
})
}
export const create = async (logData) => {
const log = new Log(logData)
......@@ -16,6 +73,9 @@ export const query = async (filters = {}, options = {}) => {
const query = {}
query.type = 'api'
query['api.url'] = buildAllowedApiUrlMatch()
if (filters.type) query.type = filters.type
if (filters.level) query.level = filters.level
if (filters.appName) query.appName = filters.appName
......@@ -66,7 +126,7 @@ export const getStats = async (filters = {}) => {
if (filters.appName) baseMatch.appName = filters.appName
// 并行查询所有统计数据
const [todayTotal, yesterdayTotal, todayErrors, yesterdayErrors, todayApps, yesterdayApps, recentLogs] =
const [todayTotal, yesterdayTotal, todayErrors, yesterdayErrors, todayApps, yesterdayApps, todayUsers, yesterdayUsers, recentLogs] =
await Promise.all([
// 今日日志总数
Log.countDocuments({ ...baseMatch, createdAt: { $gte: todayStart } }),
......@@ -84,6 +144,10 @@ export const getStats = async (filters = {}) => {
Log.distinct('appName', { ...baseMatch, createdAt: { $gte: todayStart } }),
// 昨日活跃应用数
Log.distinct('appName', { ...baseMatch, createdAt: { $gte: yesterdayStart, $lt: yesterdayEnd } }),
// 今日活跃用户数
Log.distinct('userId', { ...baseMatch, createdAt: { $gte: todayStart }, userId: { $exists: true, $ne: null } }),
// 昨日活跃用户数
Log.distinct('userId', { ...baseMatch, createdAt: { $gte: yesterdayStart, $lt: yesterdayEnd }, userId: { $exists: true, $ne: null } }),
// 最近日志
Log.find(baseMatch).sort({ createdAt: -1 }).limit(10).lean(),
])
......@@ -110,7 +174,251 @@ export const getStats = async (filters = {}) => {
count: todayApps.length,
change: todayApps.length - yesterdayApps.length,
},
// 活跃用户
activeUsers: {
count: todayUsers.length,
change: todayUsers.length - yesterdayUsers.length,
},
// 最近日志
recentLogs,
}
}
export const getUserUsage = async (filters = {}, options = {}) => {
const { page = 1, limit = 20, sortBy = 'callCount', sortOrder = 'desc' } = options
const skip = (page - 1) * limit
const allowedSortFields = ['callCount', 'textCount', 'imageCount', 'videoCount', 'digitalHumanCount', 'lastCallAt']
const finalSortField = allowedSortFields.includes(sortBy) ? sortBy : 'callCount'
const finalSortOrder = sortOrder === 'asc' ? 1 : -1
const match = {
type: 'api',
userId: { $exists: true, $nin: [null, ''] },
'api.url': buildAllowedApiUrlMatch(),
'api.statusCode': { $gte: 200, $lt: 400 },
}
if (filters.appName) match.appName = filters.appName
if (filters.startTime || filters.endTime) {
match.createdAt = {}
if (filters.startTime) match.createdAt.$gte = filters.startTime
if (filters.endTime) match.createdAt.$lte = filters.endTime
}
const [rawList, totalResult, summaryResult] = await Promise.all([
Log.aggregate([
{ $match: match },
{
$group: {
_id: '$userId',
callCount: { $sum: 1 },
textCount: {
$sum: {
$cond: [{ $regexMatch: { input: '$api.url', regex: buildStrictApiRegex(TEXT_API_PATTERN), options: 'i' } }, 1, 0],
},
},
imageCount: {
$sum: {
$cond: [{ $regexMatch: { input: '$api.url', regex: buildStrictApiRegex(IMAGE_API_PATTERN), options: 'i' } }, 1, 0],
},
},
videoCount: {
$sum: {
$cond: [
{
$regexMatch: {
input: '$api.url',
regex: buildStrictApiRegex(VIDEO_API_PATTERN),
options: 'i',
},
},
1,
0,
],
},
},
digitalHumanCount: {
$sum: {
$cond: [
{
$regexMatch: {
input: '$api.url',
regex: DIGITAL_HUMAN_API_PATTERNS.map(buildStrictApiRegex).join('|'),
options: 'i',
},
},
1,
0,
],
},
},
lastCallAt: { $max: '$createdAt' },
},
},
{ $sort: { [finalSortField]: finalSortOrder, lastCallAt: -1 } },
{ $skip: skip },
{ $limit: limit },
{
$project: {
_id: 0,
userId: '$_id',
callCount: 1,
textCount: 1,
imageCount: 1,
videoCount: 1,
digitalHumanCount: 1,
lastCallAt: 1,
},
},
]),
Log.aggregate([
{ $match: match },
{
$group: {
_id: '$userId',
},
},
{
$count: 'total',
},
]),
Log.aggregate([
{ $match: match },
{
$group: {
_id: null,
callCount: { $sum: 1 },
textCount: {
$sum: {
$cond: [{ $regexMatch: { input: '$api.url', regex: buildStrictApiRegex(TEXT_API_PATTERN), options: 'i' } }, 1, 0],
},
},
imageCount: {
$sum: {
$cond: [{ $regexMatch: { input: '$api.url', regex: buildStrictApiRegex(IMAGE_API_PATTERN), options: 'i' } }, 1, 0],
},
},
videoCount: {
$sum: {
$cond: [
{
$regexMatch: {
input: '$api.url',
regex: buildStrictApiRegex(VIDEO_API_PATTERN),
options: 'i',
},
},
1,
0,
],
},
},
digitalHumanCount: {
$sum: {
$cond: [
{
$regexMatch: {
input: '$api.url',
regex: DIGITAL_HUMAN_API_PATTERNS.map(buildStrictApiRegex).join('|'),
options: 'i',
},
},
1,
0,
],
},
},
},
},
{
$project: {
_id: 0,
callCount: 1,
textCount: 1,
imageCount: 1,
videoCount: 1,
digitalHumanCount: 1,
},
},
]),
])
const total = totalResult[0]?.total || 0
const list = await enrichUserUsageList(rawList)
const summary = summaryResult[0] || {
callCount: 0,
textCount: 0,
imageCount: 0,
videoCount: 0,
digitalHumanCount: 0,
}
return {
list,
summary,
pagination: {
page,
limit,
total,
pages: Math.ceil(total / limit),
},
}
}
/**
* 通用统计接口
* @param {object} params - 统计参数
* @param {string} params.field - 要统计的字段名称（如 'api.url', 'appName', 'userId' 等）
* @param {string} params.value - 字段包含的字符串（用于过滤，可选）
* @param {object} params.filters - 其他过滤条件
* @param {object} params.options - 统计选项
*/
export const aggregateStats = async (params = {}) => {
const { field, value, filters = {}, options = {} } = params
if (!field) {
throw new Error('field is required')
}
// 构建基础匹配条件
const match = {}
// 如果指定了字段值，添加正则匹配
if (value) {
match[field] = { $regex: value, $options: 'i' }
} else {
// 如果没有指定值，确保字段存在
match[field] = { $exists: true, $ne: null }
}
// 添加其他过滤条件
if (filters.appName) match.appName = filters.appName
if (filters.userId) match.userId = filters.userId
if (filters.type) match.type = filters.type
if (filters.level) match.level = filters.level
if (filters.startTime || filters.endTime) {
match.createdAt = {}
if (filters.startTime) match.createdAt.$gte = filters.startTime
if (filters.endTime) match.createdAt.$lte = filters.endTime
}
// 构建聚合管道
const pipeline = [
{ $match: match },
{
$group: {
_id: `$${field}`,
count: { $sum: 1 },
},
},
{
$sort: { count: -1 },
},
{
$limit: options.limit || 50,
},
]
const result = await Log.aggregate(pipeline)
return result
}
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论