Compare commits
3 Commits
feature/sp
...
feature/sp
| Author | SHA1 | Date | |
|---|---|---|---|
| da85b92a6b | |||
| 83fb627d2f | |||
| 39d5f26c40 |
6
Plans.md
6
Plans.md
@@ -20,13 +20,13 @@
|
|||||||
| 2 | Skill 강제 진입 hook + bypass 감지 + 차단 | [SPRINT-002](.plans/sprints/SPRINT-002-enforcement.md) | cc:완료 [PR#2] |
|
| 2 | Skill 강제 진입 hook + bypass 감지 + 차단 | [SPRINT-002](.plans/sprints/SPRINT-002-enforcement.md) | cc:완료 [PR#2] |
|
||||||
| 3 | Sprint Contract + DoD validator (Zod) | [SPRINT-003](.plans/sprints/SPRINT-003-contract.md) | cc:완료 [PR#3] |
|
| 3 | Sprint Contract + DoD validator (Zod) | [SPRINT-003](.plans/sprints/SPRINT-003-contract.md) | cc:완료 [PR#3] |
|
||||||
| 4 | 4자매 핸드오프 엔진 (상태 전이 기반) | [SPRINT-004](.plans/sprints/SPRINT-004-handoff.md) | cc:완료 [PR#4] |
|
| 4 | 4자매 핸드오프 엔진 (상태 전이 기반) | [SPRINT-004](.plans/sprints/SPRINT-004-handoff.md) | cc:완료 [PR#4] |
|
||||||
| 5 | 재시도 / 타임아웃 / 에스컬레이션 policy | [SPRINT-005](.plans/sprints/SPRINT-005-resilience.md) | cc:WIP |
|
| 5 | 재시도 / 타임아웃 / 에스컬레이션 policy | [SPRINT-005](.plans/sprints/SPRINT-005-resilience.md) | cc:완료 [PR#5] |
|
||||||
| 6 | QA 체크리스트 템플릿 + 다랑이 runtime | [SPRINT-006](.plans/sprints/SPRINT-006-qa.md) | cc:TODO |
|
| 6 | QA 체크리스트 템플릿 + 다랑이 runtime | [SPRINT-006](.plans/sprints/SPRINT-006-qa.md) | cc:WIP |
|
||||||
| 7 | 기존 프로젝트 마이그레이션 (아랑 등) | [SPRINT-007](.plans/sprints/SPRINT-007-migration.md) | cc:TODO |
|
| 7 | 기존 프로젝트 마이그레이션 (아랑 등) | [SPRINT-007](.plans/sprints/SPRINT-007-migration.md) | cc:TODO |
|
||||||
|
|
||||||
## 현재 스프린트
|
## 현재 스프린트
|
||||||
|
|
||||||
**Sprint 005 — 재시도 / 타임아웃 / 에스컬레이션 policy** (`cc:TODO`)
|
**Sprint 006 — QA 체크리스트 템플릿 + 다랑이 runtime** (`cc:TODO`)
|
||||||
|
|
||||||
다음 착수 예정. 상세는 `.plans/sprints/SPRINT-001-skeleton.md` 참조.
|
다음 착수 예정. 상세는 `.plans/sprints/SPRINT-001-skeleton.md` 참조.
|
||||||
|
|
||||||
|
|||||||
50
qa-templates/bugfix-v1.yaml
Normal file
50
qa-templates/bugfix-v1.yaml
Normal file
@@ -0,0 +1,50 @@
|
|||||||
|
template: bugfix-v1
|
||||||
|
version: v1
|
||||||
|
appliesTo: [bugfix]
|
||||||
|
|
||||||
|
requiredChecks:
|
||||||
|
- id: regression-test
|
||||||
|
description: 버그를 재현하는 테스트가 추가됨 (수정 전 fail → 수정 후 pass)
|
||||||
|
kind: manual
|
||||||
|
spec:
|
||||||
|
question: 새로 추가된 regression 테스트가 있는가?
|
||||||
|
guidance: 수정 전 커밋에서 테스트가 실패하는지 확인했는가?
|
||||||
|
blocking: true
|
||||||
|
severity: major
|
||||||
|
|
||||||
|
- id: root-cause-documented
|
||||||
|
description: root cause 기록
|
||||||
|
kind: manual
|
||||||
|
spec:
|
||||||
|
question: 스프린트 문서 또는 커밋 메시지에 root cause 가 명시되었는가?
|
||||||
|
blocking: true
|
||||||
|
severity: major
|
||||||
|
|
||||||
|
- id: no-scope-creep
|
||||||
|
description: 버그 외 리팩터/기능 추가 없음
|
||||||
|
kind: manual
|
||||||
|
spec:
|
||||||
|
question: 이번 커밋이 오직 해당 버그만 수정하는가?
|
||||||
|
guidance: 동반 리팩터/포매팅 변경은 별도 커밋으로 분리되어야 함.
|
||||||
|
blocking: true
|
||||||
|
severity: major
|
||||||
|
|
||||||
|
- id: tests-pass
|
||||||
|
description: 전체 테스트 pass
|
||||||
|
kind: command_success
|
||||||
|
spec:
|
||||||
|
command: pnpm test
|
||||||
|
timeoutMs: 120000
|
||||||
|
expectExitCode: 0
|
||||||
|
blocking: true
|
||||||
|
severity: major
|
||||||
|
|
||||||
|
- id: typecheck
|
||||||
|
description: 타입 체크 pass
|
||||||
|
kind: command_success
|
||||||
|
spec:
|
||||||
|
command: pnpm tsc --noEmit
|
||||||
|
timeoutMs: 60000
|
||||||
|
expectExitCode: 0
|
||||||
|
blocking: true
|
||||||
|
severity: major
|
||||||
66
qa-templates/feature-v1.yaml
Normal file
66
qa-templates/feature-v1.yaml
Normal file
@@ -0,0 +1,66 @@
|
|||||||
|
template: feature-v1
|
||||||
|
version: v1
|
||||||
|
appliesTo: [feature]
|
||||||
|
|
||||||
|
requiredChecks:
|
||||||
|
- id: tests-pass
|
||||||
|
description: 전체 테스트 통과
|
||||||
|
kind: command_success
|
||||||
|
spec:
|
||||||
|
command: pnpm test
|
||||||
|
timeoutMs: 120000
|
||||||
|
expectExitCode: 0
|
||||||
|
blocking: true
|
||||||
|
severity: major
|
||||||
|
|
||||||
|
- id: typecheck
|
||||||
|
description: TypeScript 타입 체크 통과
|
||||||
|
kind: command_success
|
||||||
|
spec:
|
||||||
|
command: pnpm tsc --noEmit
|
||||||
|
timeoutMs: 60000
|
||||||
|
expectExitCode: 0
|
||||||
|
blocking: true
|
||||||
|
severity: major
|
||||||
|
|
||||||
|
- id: no-console-log
|
||||||
|
description: console.* 호출 없음 (pino 사용)
|
||||||
|
kind: manual
|
||||||
|
spec:
|
||||||
|
question: 모든 새 코드가 pino logger 를 사용하고 console.* 직접 호출이 없는가?
|
||||||
|
guidance: grep -rn 'console\.' src/ 로 확인. 테스트 코드는 예외.
|
||||||
|
blocking: true
|
||||||
|
severity: major
|
||||||
|
|
||||||
|
- id: no-any-type
|
||||||
|
description: any 타입 신규 도입 없음
|
||||||
|
kind: manual
|
||||||
|
spec:
|
||||||
|
question: Zod 경계 밖에서 any 타입이 도입되지 않았는가?
|
||||||
|
guidance: 외부 입력은 Zod 검증 후 타입이 확정됨. any 는 절대 금지.
|
||||||
|
blocking: true
|
||||||
|
severity: major
|
||||||
|
|
||||||
|
- id: tests-added
|
||||||
|
description: 새 기능에 대한 테스트가 추가됨
|
||||||
|
kind: manual
|
||||||
|
spec:
|
||||||
|
question: 이번 변경 사항에 대한 단위/통합 테스트가 최소 1개 추가되었는가?
|
||||||
|
blocking: true
|
||||||
|
severity: major
|
||||||
|
|
||||||
|
- id: error-handling
|
||||||
|
description: 주요 에러 경로에 Result / try-catch 적용
|
||||||
|
kind: manual
|
||||||
|
spec:
|
||||||
|
question: 외부 시스템 호출 (네트워크, DB, subprocess) 에러가 적절히 처리되는가?
|
||||||
|
blocking: false
|
||||||
|
severity: minor
|
||||||
|
|
||||||
|
- id: docs-updated
|
||||||
|
description: README / .plans 에 변경 반영
|
||||||
|
kind: manual
|
||||||
|
spec:
|
||||||
|
question: 사용자 관찰 가능한 변경 사항이 README 또는 .plans 에 반영되었는가?
|
||||||
|
blocking: false
|
||||||
|
severity: minor
|
||||||
46
qa-templates/infra-v1.yaml
Normal file
46
qa-templates/infra-v1.yaml
Normal file
@@ -0,0 +1,46 @@
|
|||||||
|
template: infra-v1
|
||||||
|
version: v1
|
||||||
|
appliesTo: [infra, deploy-only]
|
||||||
|
|
||||||
|
requiredChecks:
|
||||||
|
- id: config-validated
|
||||||
|
description: 인프라 설정 파일이 유효한지 확인
|
||||||
|
kind: manual
|
||||||
|
spec:
|
||||||
|
question: 변경된 설정 파일이 파싱/검증을 통과했는가?
|
||||||
|
guidance: docker-compose config, nginx -t, terraform validate 등.
|
||||||
|
blocking: true
|
||||||
|
severity: critical
|
||||||
|
|
||||||
|
- id: secrets-not-leaked
|
||||||
|
description: 시크릿이 리포지토리에 누출되지 않음
|
||||||
|
kind: manual
|
||||||
|
spec:
|
||||||
|
question: 새로 추가된 파일에 토큰/비밀번호가 포함되지 않았는가?
|
||||||
|
guidance: git diff 로 확인. .env 류는 예제만 commit.
|
||||||
|
blocking: true
|
||||||
|
severity: critical
|
||||||
|
|
||||||
|
- id: backward-compatible
|
||||||
|
description: 기존 서비스 호환
|
||||||
|
kind: manual
|
||||||
|
spec:
|
||||||
|
question: 기존에 돌던 서비스가 계속 동작하는가?
|
||||||
|
blocking: true
|
||||||
|
severity: major
|
||||||
|
|
||||||
|
- id: rollback-documented
|
||||||
|
description: 롤백 절차 문서화
|
||||||
|
kind: manual
|
||||||
|
spec:
|
||||||
|
question: 배포 실패 시 복구 절차가 명확한가?
|
||||||
|
blocking: true
|
||||||
|
severity: major
|
||||||
|
|
||||||
|
- id: health-check
|
||||||
|
description: 배포 후 health check 정의
|
||||||
|
kind: manual
|
||||||
|
spec:
|
||||||
|
question: 배포 성공 여부를 자동 판정할 수 있는 health check 가 있는가?
|
||||||
|
blocking: false
|
||||||
|
severity: major
|
||||||
53
qa-templates/migration-v1.yaml
Normal file
53
qa-templates/migration-v1.yaml
Normal file
@@ -0,0 +1,53 @@
|
|||||||
|
template: migration-v1
|
||||||
|
version: v1
|
||||||
|
appliesTo: [migration]
|
||||||
|
|
||||||
|
requiredChecks:
|
||||||
|
- id: migration-script-exists
|
||||||
|
description: 마이그레이션 스크립트 파일 존재
|
||||||
|
kind: manual
|
||||||
|
spec:
|
||||||
|
question: prisma/migrations, SQL, 또는 해당 마이그레이션 스크립트가 존재하는가?
|
||||||
|
blocking: true
|
||||||
|
severity: critical
|
||||||
|
|
||||||
|
- id: rollback-plan
|
||||||
|
description: 롤백 계획 문서화
|
||||||
|
kind: manual
|
||||||
|
spec:
|
||||||
|
question: 롤백 절차가 .plans 또는 커밋 메시지에 문서화되었는가?
|
||||||
|
blocking: true
|
||||||
|
severity: critical
|
||||||
|
|
||||||
|
- id: dry-run-tested
|
||||||
|
description: dry-run 검증 완료
|
||||||
|
kind: manual
|
||||||
|
spec:
|
||||||
|
question: 프로덕션 전 stage/dry-run 환경에서 검증되었는가?
|
||||||
|
blocking: true
|
||||||
|
severity: critical
|
||||||
|
|
||||||
|
- id: data-loss-assessment
|
||||||
|
description: 데이터 손실 가능성 평가
|
||||||
|
kind: manual
|
||||||
|
spec:
|
||||||
|
question: 데이터 손실 리스크가 평가되었고 완화책이 있는가?
|
||||||
|
guidance: DROP / ALTER / NULL 전환 등은 반드시 평가.
|
||||||
|
blocking: true
|
||||||
|
severity: critical
|
||||||
|
|
||||||
|
- id: backup-captured
|
||||||
|
description: 운영 DB 백업 확인
|
||||||
|
kind: manual
|
||||||
|
spec:
|
||||||
|
question: 실행 직전 백업이 생성되었음을 확인했는가?
|
||||||
|
blocking: true
|
||||||
|
severity: critical
|
||||||
|
|
||||||
|
- id: idempotent
|
||||||
|
description: 재실행 안전성
|
||||||
|
kind: manual
|
||||||
|
spec:
|
||||||
|
question: 마이그레이션이 중단 후 재실행에도 안전한가?
|
||||||
|
blocking: false
|
||||||
|
severity: major
|
||||||
50
qa-templates/refactor-v1.yaml
Normal file
50
qa-templates/refactor-v1.yaml
Normal file
@@ -0,0 +1,50 @@
|
|||||||
|
template: refactor-v1
|
||||||
|
version: v1
|
||||||
|
appliesTo: [refactor]
|
||||||
|
|
||||||
|
requiredChecks:
|
||||||
|
- id: tests-pass
|
||||||
|
description: 리팩터 후 모든 테스트 pass
|
||||||
|
kind: command_success
|
||||||
|
spec:
|
||||||
|
command: pnpm test
|
||||||
|
timeoutMs: 120000
|
||||||
|
expectExitCode: 0
|
||||||
|
blocking: true
|
||||||
|
severity: major
|
||||||
|
|
||||||
|
- id: typecheck
|
||||||
|
description: 타입 체크 pass
|
||||||
|
kind: command_success
|
||||||
|
spec:
|
||||||
|
command: pnpm tsc --noEmit
|
||||||
|
timeoutMs: 60000
|
||||||
|
expectExitCode: 0
|
||||||
|
blocking: true
|
||||||
|
severity: major
|
||||||
|
|
||||||
|
- id: no-behavior-change
|
||||||
|
description: 외부 동작 변경 없음 (순수 리팩터)
|
||||||
|
kind: manual
|
||||||
|
spec:
|
||||||
|
question: 사용자 관찰 가능한 동작이 변경되지 않았는가?
|
||||||
|
guidance: 만약 변경되었다면 feature 로 재분류되어야 함.
|
||||||
|
blocking: true
|
||||||
|
severity: major
|
||||||
|
|
||||||
|
- id: tests-still-cover
|
||||||
|
description: 기존 테스트 커버리지 유지
|
||||||
|
kind: manual
|
||||||
|
spec:
|
||||||
|
question: 리팩터로 인해 테스트가 삭제되거나 우회되지 않았는가?
|
||||||
|
blocking: true
|
||||||
|
severity: major
|
||||||
|
|
||||||
|
- id: public-api-compatible
|
||||||
|
description: 공개 API 하위 호환
|
||||||
|
kind: manual
|
||||||
|
spec:
|
||||||
|
question: 공개 export 의 시그니처가 변경되지 않았는가?
|
||||||
|
guidance: 변경되었다면 breaking-change flag 필요.
|
||||||
|
blocking: false
|
||||||
|
severity: minor
|
||||||
54
qa-templates/scaffold-v1.yaml
Normal file
54
qa-templates/scaffold-v1.yaml
Normal file
@@ -0,0 +1,54 @@
|
|||||||
|
template: scaffold-v1
|
||||||
|
version: v1
|
||||||
|
appliesTo: [scaffold]
|
||||||
|
|
||||||
|
requiredChecks:
|
||||||
|
- id: readme-exists
|
||||||
|
description: README.md 가 존재하고 최소 내용 포함
|
||||||
|
kind: file_exists
|
||||||
|
spec:
|
||||||
|
path: README.md
|
||||||
|
blocking: true
|
||||||
|
severity: major
|
||||||
|
|
||||||
|
- id: license-exists
|
||||||
|
description: LICENSE 파일 존재
|
||||||
|
kind: file_exists
|
||||||
|
spec:
|
||||||
|
path: LICENSE
|
||||||
|
blocking: true
|
||||||
|
severity: minor
|
||||||
|
|
||||||
|
- id: gitignore-exists
|
||||||
|
description: .gitignore 존재
|
||||||
|
kind: file_exists
|
||||||
|
spec:
|
||||||
|
path: .gitignore
|
||||||
|
blocking: true
|
||||||
|
severity: major
|
||||||
|
|
||||||
|
- id: package-manager-lockfile
|
||||||
|
description: pnpm-lock.yaml 존재 (npm/yarn lock 금지)
|
||||||
|
kind: file_exists
|
||||||
|
spec:
|
||||||
|
path: pnpm-lock.yaml
|
||||||
|
blocking: true
|
||||||
|
severity: major
|
||||||
|
|
||||||
|
- id: no-npm-lock
|
||||||
|
description: package-lock.json 이 없어야 함 (pnpm 전용)
|
||||||
|
kind: manual
|
||||||
|
spec:
|
||||||
|
question: package-lock.json 이 존재하지 않습니까?
|
||||||
|
guidance: pnpm-lock.yaml 만 사용. package-lock.json 이 있으면 실패.
|
||||||
|
blocking: true
|
||||||
|
severity: major
|
||||||
|
|
||||||
|
- id: tsconfig-strict
|
||||||
|
description: tsconfig.json strict 모드
|
||||||
|
kind: regex_in_file
|
||||||
|
spec:
|
||||||
|
path: tsconfig.json
|
||||||
|
pattern: '"strict"\s*:\s*true'
|
||||||
|
blocking: true
|
||||||
|
severity: major
|
||||||
@@ -19,6 +19,7 @@ const main = defineCommand({
|
|||||||
run: () => import("./run.js").then((m) => m.default),
|
run: () => import("./run.js").then((m) => m.default),
|
||||||
resume: () => import("./resume.js").then((m) => m.default),
|
resume: () => import("./resume.js").then((m) => m.default),
|
||||||
abort: () => import("./abort.js").then((m) => m.default),
|
abort: () => import("./abort.js").then((m) => m.default),
|
||||||
|
qa: () => import("./qa.js").then((m) => m.default),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
107
src/cli/qa.ts
Normal file
107
src/cli/qa.ts
Normal file
@@ -0,0 +1,107 @@
|
|||||||
|
import { defineCommand } from "citty";
|
||||||
|
import { readFile } from "node:fs/promises";
|
||||||
|
import { loadTemplateForType, listTemplates } from "../qa/template.js";
|
||||||
|
import { runQaTemplate, saveQaArtifact } from "../qa/runtime.js";
|
||||||
|
import { QaArtifact } from "../qa/schema.js";
|
||||||
|
import { join } from "node:path";
|
||||||
|
|
||||||
|
const runCmd = defineCommand({
|
||||||
|
meta: { name: "run", description: "Run QA template against current workdir" },
|
||||||
|
args: {
|
||||||
|
type: {
|
||||||
|
type: "positional",
|
||||||
|
description: "Sprint type (scaffold, feature, bugfix, refactor, migration, infra)",
|
||||||
|
required: true,
|
||||||
|
},
|
||||||
|
sprintId: {
|
||||||
|
type: "string",
|
||||||
|
alias: "s",
|
||||||
|
description: "Sprint ID",
|
||||||
|
default: "manual-run",
|
||||||
|
},
|
||||||
|
workdir: {
|
||||||
|
type: "string",
|
||||||
|
alias: "w",
|
||||||
|
description: "Working directory (default: cwd)",
|
||||||
|
default: "",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
async run({ args }) {
|
||||||
|
const workdir = args.workdir || process.cwd();
|
||||||
|
const template = await loadTemplateForType(args.type);
|
||||||
|
const artifact = await runQaTemplate({
|
||||||
|
template,
|
||||||
|
workdir,
|
||||||
|
sprintId: args.sprintId ?? "manual-run",
|
||||||
|
});
|
||||||
|
const filePath = await saveQaArtifact(workdir, artifact);
|
||||||
|
|
||||||
|
console.log(`QA Artifact: ${artifact.artifactId}`);
|
||||||
|
console.log(` template: ${artifact.templateId}`);
|
||||||
|
console.log(` sprintId: ${artifact.sprintId}`);
|
||||||
|
console.log(` verdict: ${artifact.verdict}`);
|
||||||
|
console.log(
|
||||||
|
` summary: ${artifact.summary.passed}/${artifact.summary.total} passed, ${artifact.summary.blockingFailed} blocking failures`,
|
||||||
|
);
|
||||||
|
console.log(` path: ${filePath}`);
|
||||||
|
console.log("");
|
||||||
|
|
||||||
|
for (const c of artifact.checks) {
|
||||||
|
const mark = c.passed ? "✓" : "✗";
|
||||||
|
const msg = c.passed ? c.evidence : c.errorMessage;
|
||||||
|
console.log(` ${mark} [${c.severity}] ${c.id}: ${msg}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
process.exitCode =
|
||||||
|
artifact.verdict === "APPROVE" || artifact.verdict === "APPROVE_WITH_NITS"
|
||||||
|
? 0
|
||||||
|
: 1;
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const showCmd = defineCommand({
|
||||||
|
meta: { name: "show", description: "Show a saved QA artifact" },
|
||||||
|
args: {
|
||||||
|
artifactId: {
|
||||||
|
type: "positional",
|
||||||
|
description: "Artifact ID",
|
||||||
|
required: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
async run({ args }) {
|
||||||
|
const filePath = join(
|
||||||
|
process.cwd(),
|
||||||
|
".rails",
|
||||||
|
"qa-artifacts",
|
||||||
|
`${args.artifactId}.json`,
|
||||||
|
);
|
||||||
|
const raw = await readFile(filePath, "utf8");
|
||||||
|
const artifact = QaArtifact.parse(JSON.parse(raw));
|
||||||
|
console.log(JSON.stringify(artifact, null, 2));
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const listCmd = defineCommand({
|
||||||
|
meta: { name: "templates", description: "List available QA templates" },
|
||||||
|
async run() {
|
||||||
|
const names = await listTemplates();
|
||||||
|
if (names.length === 0) {
|
||||||
|
console.log("No templates found. Check qa-templates/ directory.");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
console.log("Available QA templates:");
|
||||||
|
for (const name of names) console.log(` - ${name}`);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
export default defineCommand({
|
||||||
|
meta: {
|
||||||
|
name: "qa",
|
||||||
|
description: "Run QA templates (reviewer stage)",
|
||||||
|
},
|
||||||
|
subCommands: {
|
||||||
|
run: runCmd,
|
||||||
|
show: showCmd,
|
||||||
|
templates: listCmd,
|
||||||
|
},
|
||||||
|
});
|
||||||
191
src/qa/runtime.ts
Normal file
191
src/qa/runtime.ts
Normal file
@@ -0,0 +1,191 @@
|
|||||||
|
import { ulid } from "ulid";
|
||||||
|
import { writeFile, mkdir } from "node:fs/promises";
|
||||||
|
import { dirname, join } from "node:path";
|
||||||
|
import type { QaTemplate, QaArtifact, QaChecklistResult } from "./schema.js";
|
||||||
|
import { CHECK_HANDLERS } from "../contract/checks/index.js";
|
||||||
|
import { computeVerdict, summarize } from "./verdict.js";
|
||||||
|
import { childLogger } from "../logger.js";
|
||||||
|
|
||||||
|
const log = childLogger({ module: "qa-runtime" });
|
||||||
|
|
||||||
|
export interface QaRunOptions {
|
||||||
|
template: QaTemplate;
|
||||||
|
workdir: string;
|
||||||
|
sprintId: string;
|
||||||
|
contractId?: string;
|
||||||
|
reviewer?: string;
|
||||||
|
reviewRound?: number;
|
||||||
|
env?: Record<string, string>;
|
||||||
|
/**
|
||||||
|
* Optional resolver for manual checks. If not provided, manual checks
|
||||||
|
* are marked as SKIPPED (passed=true) which is the default for Sprint 006.
|
||||||
|
* Sprint 007 or later can plug in an LLM-backed resolver.
|
||||||
|
*/
|
||||||
|
manualResolver?: (check: {
|
||||||
|
id: string;
|
||||||
|
question: string;
|
||||||
|
guidance?: string;
|
||||||
|
}) => Promise<{ passed: boolean; note: string }>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Run a QA template against a working directory.
|
||||||
|
* Returns a structured QaArtifact capturing every check result.
|
||||||
|
*/
|
||||||
|
export async function runQaTemplate(
|
||||||
|
opts: QaRunOptions,
|
||||||
|
): Promise<QaArtifact> {
|
||||||
|
const startedAt = new Date().toISOString();
|
||||||
|
const artifactId = ulid();
|
||||||
|
const env = opts.env ?? (process.env as Record<string, string>);
|
||||||
|
|
||||||
|
const allChecks = [
|
||||||
|
...opts.template.requiredChecks,
|
||||||
|
...opts.template.additionalChecks,
|
||||||
|
];
|
||||||
|
|
||||||
|
const results: QaChecklistResult[] = [];
|
||||||
|
for (const check of allChecks) {
|
||||||
|
const start = Date.now();
|
||||||
|
|
||||||
|
if (check.kind === "manual") {
|
||||||
|
if (opts.manualResolver) {
|
||||||
|
try {
|
||||||
|
const spec = check.spec as { question: string; guidance?: string };
|
||||||
|
const resolved = await opts.manualResolver({
|
||||||
|
id: check.id,
|
||||||
|
question: spec.question,
|
||||||
|
...(spec.guidance !== undefined && { guidance: spec.guidance }),
|
||||||
|
});
|
||||||
|
results.push({
|
||||||
|
id: check.id,
|
||||||
|
kind: check.kind,
|
||||||
|
passed: resolved.passed,
|
||||||
|
severity: check.severity,
|
||||||
|
evidence: resolved.passed ? resolved.note : "",
|
||||||
|
errorMessage: resolved.passed ? "" : resolved.note,
|
||||||
|
reviewerNote: resolved.note,
|
||||||
|
durationMs: Date.now() - start,
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
results.push({
|
||||||
|
id: check.id,
|
||||||
|
kind: check.kind,
|
||||||
|
passed: false,
|
||||||
|
severity: check.severity,
|
||||||
|
evidence: "",
|
||||||
|
errorMessage: `Manual resolver errored: ${err instanceof Error ? err.message : String(err)}`,
|
||||||
|
reviewerNote: "",
|
||||||
|
durationMs: Date.now() - start,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// Default: SKIPPED
|
||||||
|
results.push({
|
||||||
|
id: check.id,
|
||||||
|
kind: check.kind,
|
||||||
|
passed: true,
|
||||||
|
severity: check.severity,
|
||||||
|
evidence: "[SKIPPED — manual, no resolver]",
|
||||||
|
errorMessage: "",
|
||||||
|
reviewerNote: "",
|
||||||
|
durationMs: Date.now() - start,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const handler = CHECK_HANDLERS[check.kind];
|
||||||
|
if (!handler) {
|
||||||
|
results.push({
|
||||||
|
id: check.id,
|
||||||
|
kind: check.kind,
|
||||||
|
passed: false,
|
||||||
|
severity: check.severity,
|
||||||
|
evidence: "",
|
||||||
|
errorMessage: `No handler for kind: ${check.kind}`,
|
||||||
|
reviewerNote: "",
|
||||||
|
durationMs: 0,
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const outcome = await handler(check, { workdir: opts.workdir, env });
|
||||||
|
results.push({
|
||||||
|
id: check.id,
|
||||||
|
kind: check.kind,
|
||||||
|
passed: outcome.passed,
|
||||||
|
severity: check.severity,
|
||||||
|
evidence: outcome.evidence,
|
||||||
|
errorMessage: outcome.errorMessage,
|
||||||
|
reviewerNote: "",
|
||||||
|
durationMs: outcome.durationMs,
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
results.push({
|
||||||
|
id: check.id,
|
||||||
|
kind: check.kind,
|
||||||
|
passed: false,
|
||||||
|
severity: check.severity,
|
||||||
|
evidence: "",
|
||||||
|
errorMessage: `Handler threw: ${err instanceof Error ? err.message : String(err)}`,
|
||||||
|
reviewerNote: "",
|
||||||
|
durationMs: Date.now() - start,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const verdict = computeVerdict({
|
||||||
|
checks: results,
|
||||||
|
prerequisitesPassed: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
const summary = summarize(results);
|
||||||
|
const completedAt = new Date().toISOString();
|
||||||
|
|
||||||
|
const artifact: QaArtifact = {
|
||||||
|
schemaVersion: "v1",
|
||||||
|
artifactId,
|
||||||
|
sprintId: opts.sprintId,
|
||||||
|
contractId: opts.contractId ?? "",
|
||||||
|
templateId: opts.template.template,
|
||||||
|
reviewer: opts.reviewer ?? "darang",
|
||||||
|
reviewRound: opts.reviewRound ?? 1,
|
||||||
|
startedAt,
|
||||||
|
completedAt,
|
||||||
|
checks: results,
|
||||||
|
verdict,
|
||||||
|
summary,
|
||||||
|
};
|
||||||
|
|
||||||
|
log.info(
|
||||||
|
{
|
||||||
|
artifactId,
|
||||||
|
sprintId: opts.sprintId,
|
||||||
|
verdict,
|
||||||
|
...summary,
|
||||||
|
},
|
||||||
|
"QA template run complete",
|
||||||
|
);
|
||||||
|
|
||||||
|
return artifact;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Save a QA artifact to disk. Path: .rails/qa-artifacts/<id>.json
|
||||||
|
*/
|
||||||
|
export async function saveQaArtifact(
|
||||||
|
workdir: string,
|
||||||
|
artifact: QaArtifact,
|
||||||
|
): Promise<string> {
|
||||||
|
const filePath = join(
|
||||||
|
workdir,
|
||||||
|
".rails",
|
||||||
|
"qa-artifacts",
|
||||||
|
`${artifact.artifactId}.json`,
|
||||||
|
);
|
||||||
|
await mkdir(dirname(filePath), { recursive: true });
|
||||||
|
await writeFile(filePath, JSON.stringify(artifact, null, 2), "utf8");
|
||||||
|
return filePath;
|
||||||
|
}
|
||||||
61
src/qa/schema.ts
Normal file
61
src/qa/schema.ts
Normal file
@@ -0,0 +1,61 @@
|
|||||||
|
import { z } from "zod";
|
||||||
|
import { DodCheck } from "../contract/schema.js";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* QA Template — a reusable checklist applied during the review stage,
|
||||||
|
* on top of the sprint contract. Templates are selected by sprint type.
|
||||||
|
*
|
||||||
|
* Unlike contracts (which define "done" for the whole sprint), templates
|
||||||
|
* focus on quality gates the reviewer (darang) must verify.
|
||||||
|
*/
|
||||||
|
export const QaTemplate = z.object({
|
||||||
|
template: z.string().min(1),
|
||||||
|
version: z.string().default("v1"),
|
||||||
|
appliesTo: z.array(z.string()).default([]), // sprint types
|
||||||
|
extends: z.string().optional(), // parent template name
|
||||||
|
requiredChecks: z.array(DodCheck).default([]),
|
||||||
|
additionalChecks: z.array(DodCheck).default([]),
|
||||||
|
});
|
||||||
|
|
||||||
|
export type QaTemplate = z.infer<typeof QaTemplate>;
|
||||||
|
|
||||||
|
export const QaChecklistResult = z.object({
|
||||||
|
id: z.string(),
|
||||||
|
kind: z.string(),
|
||||||
|
passed: z.boolean(),
|
||||||
|
severity: z.enum(["critical", "major", "minor", "recommendation"]),
|
||||||
|
evidence: z.string().default(""),
|
||||||
|
errorMessage: z.string().default(""),
|
||||||
|
reviewerNote: z.string().default(""),
|
||||||
|
durationMs: z.number().default(0),
|
||||||
|
});
|
||||||
|
|
||||||
|
export type QaChecklistResult = z.infer<typeof QaChecklistResult>;
|
||||||
|
|
||||||
|
export const QaArtifact = z.object({
|
||||||
|
schemaVersion: z.literal("v1"),
|
||||||
|
artifactId: z.string(),
|
||||||
|
sprintId: z.string(),
|
||||||
|
contractId: z.string().default(""),
|
||||||
|
templateId: z.string(),
|
||||||
|
reviewer: z.string().default("darang"),
|
||||||
|
reviewRound: z.number().int().min(0).default(1),
|
||||||
|
startedAt: z.string().datetime(),
|
||||||
|
completedAt: z.string().datetime(),
|
||||||
|
checks: z.array(QaChecklistResult),
|
||||||
|
verdict: z.enum([
|
||||||
|
"APPROVE",
|
||||||
|
"APPROVE_WITH_NITS",
|
||||||
|
"REQUEST_CHANGES",
|
||||||
|
"ABORT",
|
||||||
|
]),
|
||||||
|
summary: z.object({
|
||||||
|
total: z.number().int().min(0),
|
||||||
|
passed: z.number().int().min(0),
|
||||||
|
failed: z.number().int().min(0),
|
||||||
|
skipped: z.number().int().min(0),
|
||||||
|
blockingFailed: z.number().int().min(0),
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
export type QaArtifact = z.infer<typeof QaArtifact>;
|
||||||
158
src/qa/template.ts
Normal file
158
src/qa/template.ts
Normal file
@@ -0,0 +1,158 @@
|
|||||||
|
import { readFile, readdir } from "node:fs/promises";
|
||||||
|
import { join, dirname, resolve } from "node:path";
|
||||||
|
import { fileURLToPath } from "node:url";
|
||||||
|
import { parse as parseYaml } from "yaml";
|
||||||
|
import { QaTemplate } from "./schema.js";
|
||||||
|
import type { QaTemplate as Template } from "./schema.js";
|
||||||
|
import { childLogger } from "../logger.js";
|
||||||
|
|
||||||
|
const log = childLogger({ module: "qa-template" });
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Locate the qa-templates directory. Priority:
|
||||||
|
* 1. $RAILS_QA_TEMPLATES_DIR
|
||||||
|
* 2. ./qa-templates (project root)
|
||||||
|
* 3. built-in templates next to dist/
|
||||||
|
*/
|
||||||
|
export function resolveTemplatesDir(cwd: string = process.cwd()): string {
|
||||||
|
const envDir = process.env["RAILS_QA_TEMPLATES_DIR"];
|
||||||
|
if (envDir) return resolve(envDir);
|
||||||
|
|
||||||
|
const projectDir = join(cwd, "qa-templates");
|
||||||
|
return projectDir;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function loadTemplate(
|
||||||
|
nameOrPath: string,
|
||||||
|
templatesDir?: string,
|
||||||
|
): Promise<Template> {
|
||||||
|
const dir = templatesDir ?? resolveTemplatesDir();
|
||||||
|
const candidates = [
|
||||||
|
nameOrPath,
|
||||||
|
join(dir, nameOrPath),
|
||||||
|
join(dir, `${nameOrPath}.yaml`),
|
||||||
|
join(dir, `${nameOrPath}.yml`),
|
||||||
|
];
|
||||||
|
|
||||||
|
for (const candidate of candidates) {
|
||||||
|
try {
|
||||||
|
const raw = await readFile(candidate, "utf8");
|
||||||
|
const parsed = parseYaml(raw) as unknown;
|
||||||
|
return QaTemplate.parse(parsed);
|
||||||
|
} catch (err) {
|
||||||
|
if ((err as NodeJS.ErrnoException).code !== "ENOENT") {
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new Error(
|
||||||
|
`QA template not found: ${nameOrPath} (searched in ${dir})`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Load template by sprint type, following `extends` chain.
|
||||||
|
* Example: sprint type 'feature' → feature-v1.yaml
|
||||||
|
*/
|
||||||
|
export async function loadTemplateForType(
|
||||||
|
sprintType: string,
|
||||||
|
templatesDir?: string,
|
||||||
|
): Promise<Template> {
|
||||||
|
const base = await loadTemplate(`${sprintType}-v1`, templatesDir);
|
||||||
|
return resolveExtends(base, templatesDir);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function resolveExtends(
|
||||||
|
template: Template,
|
||||||
|
templatesDir?: string,
|
||||||
|
seen: Set<string> = new Set(),
|
||||||
|
): Promise<Template> {
|
||||||
|
if (!template.extends) return template;
|
||||||
|
if (seen.has(template.template)) {
|
||||||
|
throw new Error(
|
||||||
|
`Circular extends chain in QA template: ${[...seen].join(" → ")}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
seen.add(template.template);
|
||||||
|
|
||||||
|
const parent = await loadTemplate(template.extends, templatesDir);
|
||||||
|
const resolved = await resolveExtends(parent, templatesDir, seen);
|
||||||
|
|
||||||
|
return {
|
||||||
|
template: template.template,
|
||||||
|
version: template.version,
|
||||||
|
appliesTo: template.appliesTo.length ? template.appliesTo : resolved.appliesTo,
|
||||||
|
extends: template.extends,
|
||||||
|
requiredChecks: [...resolved.requiredChecks, ...template.requiredChecks],
|
||||||
|
additionalChecks: [
|
||||||
|
...resolved.additionalChecks,
|
||||||
|
...template.additionalChecks,
|
||||||
|
],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Merge project-specific overrides (qa-extra.yaml) with a base template.
|
||||||
|
*/
|
||||||
|
export async function loadProjectExtras(
|
||||||
|
projectDir: string,
|
||||||
|
templatesDir?: string,
|
||||||
|
): Promise<Template | null> {
|
||||||
|
const extraPath = join(projectDir, "qa-extra.yaml");
|
||||||
|
try {
|
||||||
|
const raw = await readFile(extraPath, "utf8");
|
||||||
|
const parsed = parseYaml(raw) as unknown;
|
||||||
|
const extra = QaTemplate.parse(parsed);
|
||||||
|
if (extra.extends) {
|
||||||
|
return resolveExtends(extra, templatesDir);
|
||||||
|
}
|
||||||
|
return extra;
|
||||||
|
} catch (err) {
|
||||||
|
if ((err as NodeJS.ErrnoException).code === "ENOENT") return null;
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Combine a base template with a project-extras template.
|
||||||
|
*/
|
||||||
|
export function mergeTemplates(base: Template, extra: Template): Template {
|
||||||
|
return {
|
||||||
|
template: `${base.template}+${extra.template}`,
|
||||||
|
version: base.version,
|
||||||
|
appliesTo: base.appliesTo,
|
||||||
|
requiredChecks: [...base.requiredChecks, ...extra.requiredChecks],
|
||||||
|
additionalChecks: [
|
||||||
|
...base.additionalChecks,
|
||||||
|
...extra.additionalChecks,
|
||||||
|
],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* List all shipped templates in the templates directory.
|
||||||
|
*/
|
||||||
|
export async function listTemplates(
|
||||||
|
templatesDir?: string,
|
||||||
|
): Promise<string[]> {
|
||||||
|
const dir = templatesDir ?? resolveTemplatesDir();
|
||||||
|
try {
|
||||||
|
const files = await readdir(dir);
|
||||||
|
return files
|
||||||
|
.filter((f) => f.endsWith(".yaml") || f.endsWith(".yml"))
|
||||||
|
.map((f) => f.replace(/\.ya?ml$/, ""))
|
||||||
|
.sort();
|
||||||
|
} catch {
|
||||||
|
log.warn({ dir }, "Templates directory not found");
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// For test fixtures and shipped bundle discovery
|
||||||
|
export const BUILTIN_TEMPLATES_DIR = join(
|
||||||
|
dirname(fileURLToPath(import.meta.url)),
|
||||||
|
"..",
|
||||||
|
"..",
|
||||||
|
"qa-templates",
|
||||||
|
);
|
||||||
60
src/qa/verdict.ts
Normal file
60
src/qa/verdict.ts
Normal file
@@ -0,0 +1,60 @@
|
|||||||
|
import type { QaChecklistResult } from "./schema.js";
|
||||||
|
|
||||||
|
export type QaVerdict = "APPROVE" | "APPROVE_WITH_NITS" | "REQUEST_CHANGES" | "ABORT";
|
||||||
|
|
||||||
|
export interface VerdictInput {
|
||||||
|
checks: QaChecklistResult[];
|
||||||
|
prerequisitesPassed: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Apply the Harness verdict rules:
|
||||||
|
* - Any critical/major failure in a blocking check → REQUEST_CHANGES
|
||||||
|
* - Only minor failures → APPROVE_WITH_NITS
|
||||||
|
* - Prerequisites failed → ABORT
|
||||||
|
* - Otherwise → APPROVE
|
||||||
|
*
|
||||||
|
* Minor / recommendation issues NEVER cause REQUEST_CHANGES.
|
||||||
|
* This mirrors the rule documented in .plans/design/qa-template.md.
|
||||||
|
*/
|
||||||
|
export function computeVerdict(input: VerdictInput): QaVerdict {
|
||||||
|
if (!input.prerequisitesPassed) return "ABORT";
|
||||||
|
|
||||||
|
const failed = input.checks.filter((c) => !c.passed);
|
||||||
|
const blockingMajor = failed.filter(
|
||||||
|
(c) => c.severity === "critical" || c.severity === "major",
|
||||||
|
);
|
||||||
|
|
||||||
|
if (blockingMajor.length > 0) return "REQUEST_CHANGES";
|
||||||
|
|
||||||
|
const minorFailed = failed.filter(
|
||||||
|
(c) => c.severity === "minor" || c.severity === "recommendation",
|
||||||
|
);
|
||||||
|
if (minorFailed.length > 0) return "APPROVE_WITH_NITS";
|
||||||
|
|
||||||
|
return "APPROVE";
|
||||||
|
}
|
||||||
|
|
||||||
|
export function summarize(
|
||||||
|
checks: QaChecklistResult[],
|
||||||
|
): {
|
||||||
|
total: number;
|
||||||
|
passed: number;
|
||||||
|
failed: number;
|
||||||
|
skipped: number;
|
||||||
|
blockingFailed: number;
|
||||||
|
} {
|
||||||
|
const total = checks.length;
|
||||||
|
const passed = checks.filter((c) => c.passed).length;
|
||||||
|
const failed = total - passed;
|
||||||
|
const skipped = checks.filter((c) =>
|
||||||
|
c.evidence.toUpperCase().includes("SKIPPED"),
|
||||||
|
).length;
|
||||||
|
const blockingFailed = checks.filter(
|
||||||
|
(c) =>
|
||||||
|
!c.passed &&
|
||||||
|
(c.severity === "critical" || c.severity === "major"),
|
||||||
|
).length;
|
||||||
|
|
||||||
|
return { total, passed, failed, skipped, blockingFailed };
|
||||||
|
}
|
||||||
322
tests/qa.test.ts
Normal file
322
tests/qa.test.ts
Normal file
@@ -0,0 +1,322 @@
|
|||||||
|
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
||||||
|
import { mkdtemp, rm, writeFile, mkdir } from "node:fs/promises";
|
||||||
|
import { join } from "node:path";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { computeVerdict, summarize } from "../src/qa/verdict.js";
|
||||||
|
import {
|
||||||
|
loadTemplate,
|
||||||
|
loadTemplateForType,
|
||||||
|
listTemplates,
|
||||||
|
mergeTemplates,
|
||||||
|
} from "../src/qa/template.js";
|
||||||
|
import { runQaTemplate, saveQaArtifact } from "../src/qa/runtime.js";
|
||||||
|
import type { QaTemplate, QaChecklistResult } from "../src/qa/schema.js";
|
||||||
|
|
||||||
|
const PROJECT_TEMPLATES = join(process.cwd(), "qa-templates");
|
||||||
|
|
||||||
|
let workDir: string;
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
workDir = await mkdtemp(join(tmpdir(), "rails-qa-test-"));
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
await rm(workDir, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("computeVerdict", () => {
|
||||||
|
const makeCheck = (
|
||||||
|
passed: boolean,
|
||||||
|
severity: QaChecklistResult["severity"],
|
||||||
|
): QaChecklistResult => ({
|
||||||
|
id: "test",
|
||||||
|
kind: "manual",
|
||||||
|
passed,
|
||||||
|
severity,
|
||||||
|
evidence: "",
|
||||||
|
errorMessage: "",
|
||||||
|
reviewerNote: "",
|
||||||
|
durationMs: 0,
|
||||||
|
});
|
||||||
|
|
||||||
|
it("APPROVE when all checks pass", () => {
|
||||||
|
expect(
|
||||||
|
computeVerdict({
|
||||||
|
checks: [makeCheck(true, "major"), makeCheck(true, "minor")],
|
||||||
|
prerequisitesPassed: true,
|
||||||
|
}),
|
||||||
|
).toBe("APPROVE");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("REQUEST_CHANGES on any major failure", () => {
|
||||||
|
expect(
|
||||||
|
computeVerdict({
|
||||||
|
checks: [makeCheck(true, "major"), makeCheck(false, "major")],
|
||||||
|
prerequisitesPassed: true,
|
||||||
|
}),
|
||||||
|
).toBe("REQUEST_CHANGES");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("REQUEST_CHANGES on any critical failure", () => {
|
||||||
|
expect(
|
||||||
|
computeVerdict({
|
||||||
|
checks: [makeCheck(false, "critical")],
|
||||||
|
prerequisitesPassed: true,
|
||||||
|
}),
|
||||||
|
).toBe("REQUEST_CHANGES");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("APPROVE_WITH_NITS when only minor issues fail", () => {
|
||||||
|
expect(
|
||||||
|
computeVerdict({
|
||||||
|
checks: [makeCheck(true, "major"), makeCheck(false, "minor")],
|
||||||
|
prerequisitesPassed: true,
|
||||||
|
}),
|
||||||
|
).toBe("APPROVE_WITH_NITS");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("APPROVE_WITH_NITS on recommendation only", () => {
|
||||||
|
expect(
|
||||||
|
computeVerdict({
|
||||||
|
checks: [makeCheck(false, "recommendation")],
|
||||||
|
prerequisitesPassed: true,
|
||||||
|
}),
|
||||||
|
).toBe("APPROVE_WITH_NITS");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("ABORT on prerequisite failure", () => {
|
||||||
|
expect(
|
||||||
|
computeVerdict({
|
||||||
|
checks: [makeCheck(true, "major")],
|
||||||
|
prerequisitesPassed: false,
|
||||||
|
}),
|
||||||
|
).toBe("ABORT");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("NEVER REQUEST_CHANGES for minor-only failures (rule)", () => {
|
||||||
|
const v = computeVerdict({
|
||||||
|
checks: [
|
||||||
|
makeCheck(false, "minor"),
|
||||||
|
makeCheck(false, "minor"),
|
||||||
|
makeCheck(false, "recommendation"),
|
||||||
|
],
|
||||||
|
prerequisitesPassed: true,
|
||||||
|
});
|
||||||
|
expect(v).not.toBe("REQUEST_CHANGES");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("summarize counts blocking failures correctly", () => {
|
||||||
|
const s = summarize([
|
||||||
|
makeCheck(true, "major"),
|
||||||
|
makeCheck(false, "major"),
|
||||||
|
makeCheck(false, "minor"),
|
||||||
|
makeCheck(false, "critical"),
|
||||||
|
]);
|
||||||
|
expect(s.total).toBe(4);
|
||||||
|
expect(s.passed).toBe(1);
|
||||||
|
expect(s.failed).toBe(3);
|
||||||
|
expect(s.blockingFailed).toBe(2); // major + critical
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("template loader", () => {
|
||||||
|
it("lists shipped templates", async () => {
|
||||||
|
const names = await listTemplates(PROJECT_TEMPLATES);
|
||||||
|
expect(names).toContain("scaffold-v1");
|
||||||
|
expect(names).toContain("feature-v1");
|
||||||
|
expect(names).toContain("bugfix-v1");
|
||||||
|
expect(names).toContain("migration-v1");
|
||||||
|
expect(names).toContain("refactor-v1");
|
||||||
|
expect(names).toContain("infra-v1");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("loads scaffold-v1 template", async () => {
|
||||||
|
const t = await loadTemplate("scaffold-v1", PROJECT_TEMPLATES);
|
||||||
|
expect(t.template).toBe("scaffold-v1");
|
||||||
|
expect(t.appliesTo).toContain("scaffold");
|
||||||
|
expect(t.requiredChecks.length).toBeGreaterThan(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("loadTemplateForType maps type → template", async () => {
|
||||||
|
const t = await loadTemplateForType("feature", PROJECT_TEMPLATES);
|
||||||
|
expect(t.template).toBe("feature-v1");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("throws on unknown template", async () => {
|
||||||
|
await expect(
|
||||||
|
loadTemplate("nonexistent", PROJECT_TEMPLATES),
|
||||||
|
).rejects.toThrow(/not found/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("merges templates", async () => {
|
||||||
|
const base: QaTemplate = {
|
||||||
|
template: "base-v1",
|
||||||
|
version: "v1",
|
||||||
|
appliesTo: ["feature"],
|
||||||
|
requiredChecks: [
|
||||||
|
{
|
||||||
|
id: "c1",
|
||||||
|
description: "",
|
||||||
|
kind: "file_exists",
|
||||||
|
spec: { path: "a" },
|
||||||
|
blocking: true,
|
||||||
|
severity: "major",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
additionalChecks: [],
|
||||||
|
};
|
||||||
|
const extra: QaTemplate = {
|
||||||
|
template: "extra-v1",
|
||||||
|
version: "v1",
|
||||||
|
appliesTo: [],
|
||||||
|
requiredChecks: [
|
||||||
|
{
|
||||||
|
id: "c2",
|
||||||
|
description: "",
|
||||||
|
kind: "file_exists",
|
||||||
|
spec: { path: "b" },
|
||||||
|
blocking: true,
|
||||||
|
severity: "major",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
additionalChecks: [],
|
||||||
|
};
|
||||||
|
const merged = mergeTemplates(base, extra);
|
||||||
|
expect(merged.requiredChecks).toHaveLength(2);
|
||||||
|
expect(merged.template).toBe("base-v1+extra-v1");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("runtime", () => {
|
||||||
|
it("passes a file_exists check when file present", async () => {
|
||||||
|
await writeFile(join(workDir, "README.md"), "# test");
|
||||||
|
const template: QaTemplate = {
|
||||||
|
template: "test-v1",
|
||||||
|
version: "v1",
|
||||||
|
appliesTo: ["scaffold"],
|
||||||
|
requiredChecks: [
|
||||||
|
{
|
||||||
|
id: "readme",
|
||||||
|
description: "",
|
||||||
|
kind: "file_exists",
|
||||||
|
spec: { path: "README.md" },
|
||||||
|
blocking: true,
|
||||||
|
severity: "major",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
additionalChecks: [],
|
||||||
|
};
|
||||||
|
const artifact = await runQaTemplate({
|
||||||
|
template,
|
||||||
|
workdir: workDir,
|
||||||
|
sprintId: "S1",
|
||||||
|
});
|
||||||
|
expect(artifact.verdict).toBe("APPROVE");
|
||||||
|
expect(artifact.summary.passed).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fails on missing file", async () => {
|
||||||
|
const template: QaTemplate = {
|
||||||
|
template: "test-v1",
|
||||||
|
version: "v1",
|
||||||
|
appliesTo: ["scaffold"],
|
||||||
|
requiredChecks: [
|
||||||
|
{
|
||||||
|
id: "missing",
|
||||||
|
description: "",
|
||||||
|
kind: "file_exists",
|
||||||
|
spec: { path: "never.txt" },
|
||||||
|
blocking: true,
|
||||||
|
severity: "major",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
additionalChecks: [],
|
||||||
|
};
|
||||||
|
const artifact = await runQaTemplate({
|
||||||
|
template,
|
||||||
|
workdir: workDir,
|
||||||
|
sprintId: "S1",
|
||||||
|
});
|
||||||
|
expect(artifact.verdict).toBe("REQUEST_CHANGES");
|
||||||
|
expect(artifact.summary.blockingFailed).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("manual checks are SKIPPED by default (no resolver)", async () => {
|
||||||
|
const template: QaTemplate = {
|
||||||
|
template: "test-v1",
|
||||||
|
version: "v1",
|
||||||
|
appliesTo: ["feature"],
|
||||||
|
requiredChecks: [
|
||||||
|
{
|
||||||
|
id: "review",
|
||||||
|
description: "",
|
||||||
|
kind: "manual",
|
||||||
|
spec: { question: "Is it good?" },
|
||||||
|
blocking: true,
|
||||||
|
severity: "major",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
additionalChecks: [],
|
||||||
|
};
|
||||||
|
const artifact = await runQaTemplate({
|
||||||
|
template,
|
||||||
|
workdir: workDir,
|
||||||
|
sprintId: "S1",
|
||||||
|
});
|
||||||
|
expect(artifact.verdict).toBe("APPROVE");
|
||||||
|
expect(artifact.checks[0]!.evidence).toContain("SKIPPED");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("manual checks use resolver when provided", async () => {
|
||||||
|
const template: QaTemplate = {
|
||||||
|
template: "test-v1",
|
||||||
|
version: "v1",
|
||||||
|
appliesTo: ["feature"],
|
||||||
|
requiredChecks: [
|
||||||
|
{
|
||||||
|
id: "review",
|
||||||
|
description: "",
|
||||||
|
kind: "manual",
|
||||||
|
spec: { question: "Is it clean?" },
|
||||||
|
blocking: true,
|
||||||
|
severity: "major",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
additionalChecks: [],
|
||||||
|
};
|
||||||
|
const artifact = await runQaTemplate({
|
||||||
|
template,
|
||||||
|
workdir: workDir,
|
||||||
|
sprintId: "S1",
|
||||||
|
manualResolver: async () => ({ passed: false, note: "found a TODO" }),
|
||||||
|
});
|
||||||
|
expect(artifact.verdict).toBe("REQUEST_CHANGES");
|
||||||
|
expect(artifact.checks[0]!.errorMessage).toBe("found a TODO");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("saves artifact to disk", async () => {
|
||||||
|
const template: QaTemplate = {
|
||||||
|
template: "test-v1",
|
||||||
|
version: "v1",
|
||||||
|
appliesTo: ["feature"],
|
||||||
|
requiredChecks: [],
|
||||||
|
additionalChecks: [],
|
||||||
|
};
|
||||||
|
const artifact = await runQaTemplate({
|
||||||
|
template,
|
||||||
|
workdir: workDir,
|
||||||
|
sprintId: "S1",
|
||||||
|
});
|
||||||
|
const path = await saveQaArtifact(workDir, artifact);
|
||||||
|
expect(path).toContain(artifact.artifactId);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("scaffold-v1 on real project", () => {
|
||||||
|
it("loads without error and has expected checks", async () => {
|
||||||
|
const t = await loadTemplate("scaffold-v1", PROJECT_TEMPLATES);
|
||||||
|
const ids = t.requiredChecks.map((c) => c.id);
|
||||||
|
expect(ids).toContain("readme-exists");
|
||||||
|
expect(ids).toContain("tsconfig-strict");
|
||||||
|
});
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user