Skip to content

Commit 3422b18

Browse files
authored
sync: Monthly version update & fix astbuilder bug (#161)
* sync: update version * sync: fix astbuilder bug
1 parent 249420d commit 3422b18

60 files changed

Lines changed: 5480 additions & 593 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎build-client.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ ROOT_DIR=$(cd "$(dirname "$0")" && pwd)
66
LOGGER_FILE="$ROOT_DIR/src/util/logger.ts"
77
PACKAGE_FILE="$ROOT_DIR/package.json"
88
STARTER_FILE="$ROOT_DIR/src/interface/starter.ts"
9-
SNAPSHOT_DIR=$(mktemp -d "${TMPDIR:-/tmp}/engine-client-build.XXXXXX")
9+
SNAPSHOT_DIR=$(mktemp -d "${TMPDIR:-/tmp}/yasa2-client-build.XXXXXX")
1010
LOGGER_SNAPSHOT="$SNAPSHOT_DIR/logger.ts"
1111
PACKAGE_SNAPSHOT="$SNAPSHOT_DIR/package.json"
1212
STARTER_SNAPSHOT="$SNAPSHOT_DIR/starter.ts"

‎package.json‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,14 +98,17 @@
9898
"test-clone-util": "npx mocha --require tsx/cjs test/rules-basic-handler/test-clone-util.ts",
9999
"test-sink-util-new-expr": "npx mocha --require tsx/cjs test/rules-basic-handler/test-sink-util-new-expr.ts",
100100
"test-sink-util-go-interface-passthrough": "npx mocha --require tsx/cjs test/rules-basic-handler/test-sink-util-go-interface-passthrough.ts",
101+
"test-sink-util-go-qid-fallback": "npx mocha --require tsx/cjs test/rules-basic-handler/test-sink-util-go-qid-fallback.ts",
101102
"test-taint-output-strategy": "npx mocha --require tsx/cjs test/rules-basic-handler/test-taint-output-strategy.ts",
102103
"test-go-dynamic-key-taint": "npx mocha --require tsx/cjs test/rules-basic-handler/test-go-dynamic-key-taint.ts",
103104
"test-go-json-unmarshal-field-taint": "npx mocha --require tsx/cjs test/rules-basic-handler/test-go-json-unmarshal-field-taint.ts",
104105
"test-go-short-var-callback-taint": "npx mocha --require tsx/cjs test/rules-basic-handler/test-go-short-var-callback-taint.ts",
106+
"test-go-callsite-limit-memory-guard": "npx mocha --require tsx/cjs test/go/test-go-callsite-limit-memory-guard.ts",
107+
"test-go-callsite-limit-integration": "npx mocha --require tsx/cjs test/go/test-go-callsite-limit-integration.ts",
105108
"test-php-checker": "npx mocha --require tsx/cjs test/php/test-php-checker.ts",
106109
"test-php": "npx mocha --require tsx/cjs test/php/test-php-benchmark.ts",
107110
"update-php-expect": "YASA_UPDATE_PHP_BENCHMARK_EXPECT=1 npm run test-php && cp test/php/expect/phpbenchmark-expect.json.candidate test/php/expect/phpbenchmark-expect.json && rm test/php/expect/phpbenchmark-expect.json.candidate && npm run test-php",
108-
"test-all": "npm run test-callargs && npm run test-fastapi-backgroundtasks && npm run test-match-field && npm run test-entrypoint-deadline-budget && npm run test-entrypoint-deadline-scheduler && npm run test-clone-util && npm run test-sink-util-new-expr && npm run test-taint-output-strategy && npm run test-go-dynamic-key-taint && npm run test-js && npm run test-java && npm run test-sarif-codeflow-quality && npm run test-go && npm run test-python && npm run test-callchain && npm run test-php",
111+
"test-all": "npm run test-callargs && npm run test-fastapi-backgroundtasks && npm run test-match-field && npm run test-entrypoint-deadline-budget && npm run test-entrypoint-deadline-scheduler && npm run test-clone-util && npm run test-sink-util-new-expr && npm run test-taint-output-strategy && npm run test-go-dynamic-key-taint && npm run test-go-callsite-limit-memory-guard && npm run test-go-callsite-limit-integration && npm run test-sink-util-go-qid-fallback && npm run test-js && npm run test-java && npm run test-sarif-codeflow-quality && npm run test-go && npm run test-python && npm run test-callchain && npm run test-php",
109112
"build": "bash build.sh",
110113
"prepare-better-sqlite3-native": "bash scripts/download-better-sqlite3-native.sh",
111114
"lint": "eslint .",

‎resource/example-rule-config/rule_config_sanitizer_for_java_example.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
"id": "SANITIZER_2",
1717
"sanitizerType": "FunctionCallSanitizer",
1818
"sanitizerScenario": "SANITIZER.VALIDATE_BY_FUNCTIONCALL",
19-
"calleeType": "com.example.security.util.CommandParameterChecker",
19+
"calleeType": "com.alipay.common.security.util.secutils.cmd.CmdParameterChecker",
2020
"fsig": "checkCmdParameter",
2121
"args": [
2222
"0"
@@ -26,7 +26,7 @@
2626
"id": "SANITIZER_3",
2727
"sanitizerType": "FunctionCallSanitizer",
2828
"sanitizerScenario": "SANITIZER.CALLSTACK_HAS_FUNCTIONCALL",
29-
"calleeType": "com.example.security.util.CommandChecker",
29+
"calleeType": "com.alipay.common.security.util.secutils.cmd.AlipayCmdChecker",
3030
"fsig": "hookStart",
3131
"args": [
3232
"0"

‎scripts/regtest-run-case.sh‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
#!/usr/bin/env bash
2+
# 回归用例标准执行脚本(源码形式 tsx + src/main.ts),参数从用例 .config 读取,不自行改参数。
3+
# 用法: ENGINE_DIR=<engine path> scripts/regtest-run-case.sh <case-name> <report-dir> [testcase-root]
4+
# ENGINE_DIR 默认为本脚本所在仓库根(ours)。
5+
# 严格串行:同一 engine 目录禁止并行;调用方负责不并行起任务。
6+
set -uo pipefail
7+
8+
CASE="${1:?case name required}"
9+
REPORT="${2:?report dir required}"
10+
ROOT="${3:-/Users/ariel/code/yasa-reformat/test/yasa2-bench/testcase/python}"
11+
12+
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
13+
ENGINE_DIR="${ENGINE_DIR:-$(cd "$SCRIPT_DIR/.." && pwd)}"
14+
cd "$ENGINE_DIR"
15+
16+
CASE_DIR="$ROOT/$CASE"
17+
# mpc_server 目录内部 case 名是 mcp_server,特例处理
18+
if [[ "$CASE" == "mpc_server" ]]; then
19+
CFG_NAME="mcp_server"
20+
SOURCE_PATH="$CASE_DIR/mcp_server"
21+
else
22+
CFG_NAME="$CASE"
23+
SOURCE_PATH="$CASE_DIR/$CASE"
24+
fi
25+
26+
CONFIG_FILE="$CASE_DIR/$CFG_NAME.config"
27+
[[ -f "$CONFIG_FILE" ]] || { echo "missing $CONFIG_FILE" >&2; exit 2; }
28+
29+
CHECKER_PACKS=$(python -c "import json;d=json.load(open('$CONFIG_FILE'));print(','.join(d['checkerPackIds']))")
30+
ANALYZER=$(python -c "import json;d=json.load(open('$CONFIG_FILE'));print(d.get('analyzer',''))")
31+
LANGUAGE=$(python -c "import json;d=json.load(open('$CONFIG_FILE'));print(d.get('language',''))")
32+
RULE_CONFIG_REL=$(python -c "import json;d=json.load(open('$CONFIG_FILE'));print(d.get('ruleConfigFile',''))")
33+
RULE_CONFIG="$CASE_DIR/${RULE_CONFIG_REL#./}"
34+
35+
NODE_OPTS=$(python -c "import json;d=json.load(open('$CONFIG_FILE'));print(d.get('nodeOptions',''))")
36+
[[ -n "$NODE_OPTS" ]] && export NODE_OPTIONS="$NODE_OPTS"
37+
38+
[[ -d "$SOURCE_PATH" ]] || { echo "missing sourcePath $SOURCE_PATH" >&2; exit 2; }
39+
40+
rm -rf "$REPORT"; mkdir -p "$REPORT"
41+
42+
ARGS=(--sourcePath "$SOURCE_PATH" --checkerPackIds "$CHECKER_PACKS" --language "$LANGUAGE" --ruleConfigFile "$RULE_CONFIG" --report "$REPORT")
43+
[[ -n "$ANALYZER" ]] && ARGS+=(--analyzer "$ANALYZER")
44+
45+
# 留痕:完整命令写进 report/cmd.log
46+
{
47+
echo "# ENGINE_DIR=$ENGINE_DIR"
48+
echo "# HEAD=$(git rev-parse --short HEAD 2>/dev/null)"
49+
echo "# NODE_OPTIONS=${NODE_OPTIONS:-}"
50+
echo "node ./node_modules/tsx/dist/cli.mjs ./src/main.ts \\"
51+
for a in "${ARGS[@]}"; do echo " $a \\"; done
52+
echo
53+
} > "$REPORT/cmd.log"
54+
55+
echo "[regtest] ENGINE=$ENGINE_DIR CASE=$CASE REPORT=$REPORT"
56+
/usr/bin/time -l node ./node_modules/tsx/dist/cli.mjs ./src/main.ts "${ARGS[@]}" 1>/dev/null 2>"$REPORT/time.log"
57+
echo "[regtest] exit=$?"

‎src/checker/common/output/taint-output-strategy.ts‎

Lines changed: 39 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import type { IResultManager } from '../../../engine/analyzer/common/result-manager'
22
import type { IConfig } from '../../../config'
33
import type { TaintFinding } from '../../../engine/analyzer/common/common-types'
4+
import type { SarifLocation } from '../../../engine/analyzer/common/sarif'
45

56
const _ = require('lodash')
67
const path = require('path')
@@ -16,8 +17,7 @@ const logger = require('../../../util/logger')(__filename)
1617
const {
1718
registerDedupFunction,
1819
} = require('../../../engine/analyzer/common/entrypoint/merge-coordinator') as typeof import('../../../engine/analyzer/common/entrypoint/merge-coordinator')
19-
20-
type SarifLocation = { physicalLocation?: Record<string, unknown>; [key: string]: unknown }
20+
const { getJavaFindingSignature } = require('../../taint/java/java-finding-candidate') as typeof import('../../taint/java/java-finding-candidate')
2121

2222
const {
2323
prepareLocation,
@@ -265,7 +265,7 @@ function extractSinkKey(finding: TracePreserveFinding): string | null {
265265
}
266266

267267
type JavaTraceGroup = { call: string; argPass: string }
268-
type JavaFindingProjection = { source: string; sink: string; groups: JavaTraceGroup[] }
268+
type JavaFindingProjection = { source: string; sink: string; sinkAttribute: string; groups: JavaTraceGroup[] }
269269

270270
function traceItemIdentity(item: TaintTraceItem): string | null {
271271
const file = getTraceFileKey(item)
@@ -282,6 +282,9 @@ function getJavaFindingProjection(finding: TracePreserveFinding): JavaFindingPro
282282
const source = extractSourceKey(finding)
283283
const sink = extractSinkKey(finding)
284284
if (!source || source.includes(':-1:-1') || !sink || sink.includes(':-1:-1')) return null
285+
// sinkAttribute 纳入投影键,避免不同 attribute 的同类 finding 被误去重
286+
const attrArr = (finding as any).sinkAttribute as string[] | undefined
287+
const sinkAttribute = Array.isArray(attrArr) ? attrArr.slice().sort().join(',') : ''
285288
const groups: JavaTraceGroup[] = []
286289
for (let i = 0; i < finding.trace.length; i++) {
287290
const item = finding.trace[i]
@@ -293,12 +296,12 @@ function getJavaFindingProjection(finding: TracePreserveFinding): JavaFindingPro
293296
if (call && argPass) groups.push({ call, argPass })
294297
i++
295298
}
296-
return { source, sink, groups }
299+
return { source, sink, sinkAttribute, groups }
297300
}
298301

299302
function getJavaProjectionKey(projection: JavaFindingProjection): string {
300303
const groups = projection.groups.map((group) => `${group.call}${group.argPass}`).join('')
301-
return `${projection.source}${projection.sink}${groups}`
304+
return `${projection.source}${projection.sink}${projection.sinkAttribute}${groups}`
302305
}
303306

304307
const javaProjectionIndexes = new WeakMap<TaintFinding[], Map<string, TaintFinding>>()
@@ -316,15 +319,21 @@ function getJavaProjectionIndex(category: TaintFinding[]): Map<string, TaintFind
316319
}
317320

318321
function isDuplicateJavaFinding(category: TaintFinding[], finding: TaintFinding): boolean {
319-
const projection = getJavaFindingProjection(finding as TracePreserveFinding)
320-
if (!projection) return false
321-
return getJavaProjectionIndex(category).has(getJavaProjectionKey(projection))
322+
const signature = getJavaFindingSignature(finding)
323+
if (!signature) return false
324+
for (const issue of category) {
325+
if (getJavaFindingSignature(issue) === signature) return true
326+
}
327+
return false
322328
}
323329

324330
function rememberJavaFinding(category: TaintFinding[], finding: TaintFinding): void {
331+
const signature = getJavaFindingSignature(finding)
332+
if (!signature) return
333+
// 保持统一 finding 签名索引的建立副作用,实际查重使用同一 helper。
325334
const projection = getJavaFindingProjection(finding as TracePreserveFinding)
326-
if (!projection) return
327-
getJavaProjectionIndex(category).set(getJavaProjectionKey(projection), finding)
335+
if (projection) getJavaProjectionIndex(category).set(getJavaProjectionKey(projection), finding)
336+
void signature
328337
}
329338

330339
function invalidateJavaProjectionIndex(category: TaintFinding[]): void {
@@ -383,7 +392,7 @@ class TaintOutputStrategy extends OutputStrategy {
383392
* @param config
384393
* @param printf
385394
*/
386-
outputFindings(resultManager: IResultManager, outputFilePath: string, config: IConfig, printf: any): void {
395+
outputFindings(resultManager: IResultManager, outputFilePath: string, config: IConfig, printf: any): any {
387396
const outputStartedAt = Date.now()
388397
let reportFilePath
389398
if (resultManager) {
@@ -425,8 +434,10 @@ class TaintOutputStrategy extends OutputStrategy {
425434
logger.info(`[outputFindings] strategy=taintflow phase=write elapsed=${Date.now() - writeStartedAt}ms total=${Date.now() - outputStartedAt}ms`)
426435
// for taint flow checker, output result to console at the same time
427436
logger.info(`report is write to ${reportFilePath}`)
437+
return results
428438
}
429439
}
440+
return undefined
430441
}
431442

432443
/**
@@ -439,14 +450,27 @@ class TaintOutputStrategy extends OutputStrategy {
439450
if (!finding) return false
440451
let category: TaintFinding[] | undefined
441452
try {
442-
category = resultManager?.findings[TaintOutputStrategy.outputStrategyId] as TaintFinding[] | undefined
453+
// 跨入口 dedup:dedupIndex(已 snapshot 的历史 finding)+ findings(当前批次 live finding)
454+
const dedupCategory = resultManager?.dedupIndex?.[TaintOutputStrategy.outputStrategyId] as TaintFinding[] | undefined
455+
const currentCategory = resultManager?.findings?.[TaintOutputStrategy.outputStrategyId] as TaintFinding[] | undefined
456+
if (!dedupCategory && !currentCategory) return true
457+
if (!dedupCategory) {
458+
category = currentCategory
459+
} else if (!currentCategory) {
460+
category = dedupCategory
461+
} else {
462+
category = [...dedupCategory, ...currentCategory]
463+
}
443464
if (!category) return true
444465
if (isDuplicateJavaFinding(category as TaintFinding[], finding)) return false
445466
// 依赖 trace 形态的折叠判据只比较原始链路节点,避免输出辅助节点改变去重结果。
446467
// 非 callstack-only 输出下 trace 通常不含 `_synthetic` 标记,过滤后与原数组等价。
447468
const findingTraceNoSynthetic = filterOutSyntheticSteps(getDedupTrace(finding))
448469
for (let i = 0; i < category.length; i++) {
449470
const issue = category[i]
471+
const issueProjection = getJavaFindingProjection(issue as TracePreserveFinding)
472+
const findingProjection = getJavaFindingProjection(finding as TracePreserveFinding)
473+
if (issueProjection && findingProjection && getJavaProjectionKey(issueProjection) !== getJavaProjectionKey(findingProjection)) continue
450474
if (
451475
issue.line === finding.line &&
452476
isNodeEqual(issue.node, finding.node) &&
@@ -556,7 +580,7 @@ class TaintOutputStrategy extends OutputStrategy {
556580
uri,
557581
snippetText,
558582
nodeHash,
559-
affectedNodeName,
583+
affectedNodeName
560584
),
561585
key
562586
)
@@ -573,7 +597,7 @@ class TaintOutputStrategy extends OutputStrategy {
573597
'egg controller',
574598
item.str,
575599
nodeHash,
576-
affectedNodeName,
600+
affectedNodeName
577601
),
578602
key
579603
)
@@ -605,7 +629,7 @@ class TaintOutputStrategy extends OutputStrategy {
605629
trace,
606630
location,
607631
finding.matchedSanitizerTags,
608-
callstackElements,
632+
callstackElements
609633
)
610634
)
611635
})

‎src/checker/taint/common-kit/sink-util.ts‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -69,8 +69,11 @@ const GO_INTERFACE_PASSTHROUGH_MAX_IMPLEMENTERS = 8
6969
* 注意:本函数只在 Go 路径(由调用方传入 analyzer + declType 才会触发)启用,
7070
* Java 共用的 checkInvocationMatchSink 不受影响。
7171
*
72-
* Go 分析流程把层级信息写入 analyzer.classHierarchyMap,而子类型查询由
73-
* analyzer.typeResolver.findSubTypes 提供,因此 helper 同时读取这两个对象。
72+
* Bug 1 修复(W4 实测):Go 路径下 classHierarchyMap 由 checker triggerAtStartOfAnalyze 调
73+
* `analyzer.typeResolver.findClassHierarchy` 后写入到 `analyzer.classHierarchyMap`(独立对象,
74+
* size 实测 42),而 `analyzer.typeResolver.classHierarchyMap` 始终是默认空 Map(size 0)。
75+
* helper 入参改为接受 analyzer,从 analyzer 读 classHierarchyMap,从 analyzer.typeResolver
76+
* 读 findSubTypes 方法。
7477
*/
7578
function tryMatchSinkGoInterfacePassthrough(
7679
declType: string,
@@ -321,8 +324,8 @@ function matchSinkAtFuncCallWithCalleeType(
321324
}
322325

323326
/**
324-
* 当 fclos.rtype 缺失但 fclos.qid 编码 receiver 类型时(如 Go 全局变量 method call),
325-
* 按 qid 后缀匹配 calleeType 与 fsig,并忽略 qid 中的 `<instance_*>` 标记。
327+
* W14f: 当 fclos.rtype 缺失但 fclos.qid 编码 receiver 类型时(如 Go 全局变量 method call),
328+
* 按 qid 后缀匹配 calleeType 与 fsig。先 strip qid 中的 `<instance_*>` 标记。
326329
* @param qid fclos.qid
327330
* @param calleeType sink rule calleeType(可能带 `*` 前缀)
328331
* @param calleeTypeBase calleeType 去 `*` 后的形式
@@ -379,10 +382,12 @@ function matchEmptyCalleeTypeInstanceMethod(
379382
if (propertyName !== methodName) return false
380383

381384
const cleanQid = typeof fclos.qid === 'string' ? fclos.qid.replace(/<instance_[^>]*>/g, '') : ''
385+
// Go qid 用 `/` 分隔包路径,rule.fsig 用 `.`,统一归一化后再做 endsWith
386+
const normalizedQid = cleanQid.replace(/\//g, '.')
382387
return (
383388
matchReceiverTypeName(AstUtilSinkUtil.prettyPrint(fclos.object?.rtype?.definiteType), typeName) ||
384389
matchReceiverTypeName(AstUtilSinkUtil.prettyPrint(fclos.rtype?.definiteType), typeName) ||
385-
cleanQid.endsWith(`.${rule.fsig}`)
390+
normalizedQid.endsWith(`.${rule.fsig}`)
386391
)
387392
}
388393

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
const QidUnifyUtil = require('../../../util/qid-unify-util') as { qidUnifyByRemoveAngleAndPrefix(value: string): string }
2+
3+
export type CalleeEntryLocation = {
4+
sourcefile: string
5+
start: { line: number; column?: number }
6+
end: { line: number; column?: number }
7+
}
8+
9+
type NodeLocation = {
10+
sourcefile?: string
11+
start?: { line?: number; column?: number }
12+
end?: { line?: number; column?: number }
13+
}
14+
15+
type CalleeEntryNode = {
16+
loc?: NodeLocation
17+
_meta?: { nodehash?: unknown }
18+
id?: { name?: string; loc?: NodeLocation }
19+
body?: { loc?: NodeLocation }
20+
[key: string]: unknown
21+
}
22+
23+
type CalleeEntryFrame = {
24+
fname?: string
25+
qid?: string
26+
ast?: { node?: CalleeEntryNode }
27+
}
28+
29+
export type CalleeEntry = {
30+
file: string
31+
line: number
32+
loc: CalleeEntryLocation
33+
node: CalleeEntryNode
34+
name: string
35+
}
36+
37+
function validLocation(location: NodeLocation | undefined): { file: string; line: number; column?: number } | undefined {
38+
const file = location?.sourcefile
39+
const line = location?.start?.line
40+
if (typeof file !== 'string' || file.length === 0 || typeof line !== 'number') return undefined
41+
return { file, line, column: location?.start?.column }
42+
}
43+
44+
/**
45+
* 统一 callee-entry 展示位置:方法标识符、方法体、函数节点依次回退,并压缩为单行。
46+
* 节点显式复制 frame nodeHash,避免位置包装丢失调用栈身份。
47+
*/
48+
export function getCalleeEntry(frame: unknown): CalleeEntry | undefined {
49+
if (!frame || typeof frame !== 'object') return undefined
50+
const candidate = frame as CalleeEntryFrame
51+
const node = candidate.ast?.node
52+
if (!node) return undefined
53+
const selected = validLocation(node.id?.loc) ?? validLocation(node.body?.loc) ?? validLocation(node.loc)
54+
if (!selected) return undefined
55+
const nodehash = node._meta?.nodehash
56+
const loc: CalleeEntryLocation = {
57+
sourcefile: selected.file,
58+
start: { line: selected.line, column: selected.column ?? 0 },
59+
end: { line: selected.line, column: selected.column ?? 0 },
60+
}
61+
const wrappedNode: CalleeEntryNode = { ...node, loc, _meta: { ...node._meta, nodehash } }
62+
const rawName = node.id?.name || candidate.fname || candidate.qid || '<bridge>'
63+
const name = QidUnifyUtil.qidUnifyByRemoveAngleAndPrefix(rawName) || rawName
64+
return { file: selected.file, line: selected.line, loc, node: wrappedNode, name }
65+
}

0 commit comments

Comments
 (0)