Sprint 005 전체 구현 — F5 (중간 끊김/타임아웃 무한대기) 해결:
Resilience core:
- src/resilience/backoff.ts — exponential backoff + jitter (base 1s, cap 30s)
+ cancellable sleep
- src/resilience/classifier.ts — error → {retryable, reason}
retryable: timeout, network, rate_limit, transient
non-retryable: permission, config, invariant(ZodError)
휴리스틱: ETIMEDOUT/ECONNREFUSED/429 등 메시지 패턴 감지
+ assertAllowedThinkingTier('xhigh' 금지)
- src/resilience/retry.ts — withRetry 래퍼
non-retryable은 즉시 중단, max retries 초과시 classification 반환
- src/resilience/kill.ts — child process SIGTERM→SIGKILL grace 처리
+ 전역 cleanup handler (SIGINT/SIGTERM)
- src/resilience/escalate.ts — recordEscalation + EscalationNotifier
+ listEscalations / resolveEscalation
Prisma:
- Escalation 모델 추가 (pipelineId, reason, errorCategory, attempts, contextSnapshot)
- Pipeline.escalations 역참조
Runner 통합:
- transport.invoke 를 withRetry 로 래핑
- 실패시 classification 기반으로 자동 escalation 기록 (non-retryable만)
- RunOptions 에 maxRetries / notifier 추가
CLI:
- rails resume <pipeline-id> — escalated → idle 전이
- rails abort <pipeline-id> [-r reason] — 강제 종료
Tests (25 신규, 82 total pass):
- backoff: 기본값/지수/캡/jitter 범위/abort
- classifier: 6 error 클래스 + 3 휴리스틱 + xhigh 금지
- withRetry: 성공/재시도 후 성공/non-retryable 즉시 중단/max 초과/abort
검증: tsc --noEmit ✓ | vitest 82/82 ✓ | build ✓ | CLI ✓
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
224 lines
6.7 KiB
TypeScript
224 lines
6.7 KiB
TypeScript
import { describe, it, expect } from "vitest";
|
|
import { backoffMs, sleep } from "../src/resilience/backoff.js";
|
|
import {
|
|
classifyError,
|
|
assertAllowedThinkingTier,
|
|
TimeoutError,
|
|
NetworkError,
|
|
RateLimitError,
|
|
PermissionError,
|
|
ConfigError,
|
|
} from "../src/resilience/classifier.js";
|
|
import { withRetry } from "../src/resilience/retry.js";
|
|
import { ZodError, z } from "zod";
|
|
|
|
describe("backoffMs", () => {
|
|
it("starts near base for retry 0", () => {
|
|
const ms = backoffMs(0, { base: 1000, max: 30_000, jitter: 0 });
|
|
expect(ms).toBe(1000);
|
|
});
|
|
|
|
it("doubles each retry", () => {
|
|
expect(backoffMs(1, { base: 1000, max: 30_000, jitter: 0 })).toBe(2000);
|
|
expect(backoffMs(2, { base: 1000, max: 30_000, jitter: 0 })).toBe(4000);
|
|
expect(backoffMs(3, { base: 1000, max: 30_000, jitter: 0 })).toBe(8000);
|
|
});
|
|
|
|
it("caps at max", () => {
|
|
expect(backoffMs(10, { base: 1000, max: 30_000, jitter: 0 })).toBe(30_000);
|
|
expect(backoffMs(20, { base: 1000, max: 30_000, jitter: 0 })).toBe(30_000);
|
|
});
|
|
|
|
it("adds jitter within bounds", () => {
|
|
// With jitter 0.3, retry 0 should be in [700, 1300]
|
|
for (let i = 0; i < 50; i++) {
|
|
const ms = backoffMs(0, { base: 1000, max: 30_000, jitter: 0.3 });
|
|
expect(ms).toBeGreaterThanOrEqual(700);
|
|
expect(ms).toBeLessThanOrEqual(1300);
|
|
}
|
|
});
|
|
|
|
it("returns non-negative values", () => {
|
|
for (let i = 0; i < 20; i++) {
|
|
expect(backoffMs(i)).toBeGreaterThanOrEqual(0);
|
|
}
|
|
});
|
|
});
|
|
|
|
describe("sleep", () => {
|
|
it("waits approximately the specified time", async () => {
|
|
const start = Date.now();
|
|
await sleep(50);
|
|
const elapsed = Date.now() - start;
|
|
expect(elapsed).toBeGreaterThanOrEqual(40);
|
|
expect(elapsed).toBeLessThan(200);
|
|
});
|
|
|
|
it("aborts when signal fires", async () => {
|
|
const controller = new AbortController();
|
|
const promise = sleep(5000, controller.signal);
|
|
setTimeout(() => controller.abort(), 10);
|
|
await expect(promise).rejects.toThrow("Aborted");
|
|
});
|
|
});
|
|
|
|
describe("classifyError", () => {
|
|
it("TimeoutError → retryable timeout", () => {
|
|
const r = classifyError(new TimeoutError("timed out"));
|
|
expect(r.retryable).toBe(true);
|
|
expect(r.reason).toBe("timeout");
|
|
});
|
|
|
|
it("NetworkError → retryable network", () => {
|
|
const r = classifyError(new NetworkError("econnrefused"));
|
|
expect(r.retryable).toBe(true);
|
|
expect(r.reason).toBe("network");
|
|
});
|
|
|
|
it("RateLimitError → retryable rate_limit", () => {
|
|
const r = classifyError(new RateLimitError("429 too many"));
|
|
expect(r.retryable).toBe(true);
|
|
expect(r.reason).toBe("rate_limit");
|
|
});
|
|
|
|
it("PermissionError → non-retryable permission", () => {
|
|
const r = classifyError(new PermissionError("EACCES"));
|
|
expect(r.retryable).toBe(false);
|
|
expect(r.reason).toBe("permission");
|
|
});
|
|
|
|
it("ConfigError → non-retryable config", () => {
|
|
const r = classifyError(new ConfigError("bad config"));
|
|
expect(r.retryable).toBe(false);
|
|
expect(r.reason).toBe("config");
|
|
});
|
|
|
|
it("ZodError → non-retryable invariant", () => {
|
|
const schema = z.object({ x: z.number() });
|
|
let zodErr: unknown;
|
|
try {
|
|
schema.parse({ x: "not a number" });
|
|
} catch (e) {
|
|
zodErr = e;
|
|
}
|
|
expect(zodErr).toBeInstanceOf(ZodError);
|
|
const r = classifyError(zodErr);
|
|
expect(r.retryable).toBe(false);
|
|
expect(r.reason).toBe("invariant");
|
|
});
|
|
|
|
it("detects timeout by message heuristic", () => {
|
|
const r = classifyError(new Error("ETIMEDOUT on request"));
|
|
expect(r.retryable).toBe(true);
|
|
expect(r.reason).toBe("timeout");
|
|
});
|
|
|
|
it("detects network error by message", () => {
|
|
const r = classifyError(new Error("ECONNREFUSED"));
|
|
expect(r.retryable).toBe(true);
|
|
expect(r.reason).toBe("network");
|
|
});
|
|
|
|
it("detects rate limit by message", () => {
|
|
const r = classifyError(new Error("429 Rate limit exceeded"));
|
|
expect(r.retryable).toBe(true);
|
|
expect(r.reason).toBe("rate_limit");
|
|
});
|
|
|
|
it("unknown error defaults to retryable transient", () => {
|
|
const r = classifyError(new Error("something weird"));
|
|
expect(r.retryable).toBe(true);
|
|
expect(r.reason).toBe("transient");
|
|
});
|
|
});
|
|
|
|
describe("assertAllowedThinkingTier", () => {
|
|
it("allows high and below", () => {
|
|
expect(() => assertAllowedThinkingTier("high")).not.toThrow();
|
|
expect(() => assertAllowedThinkingTier("medium")).not.toThrow();
|
|
expect(() => assertAllowedThinkingTier("low")).not.toThrow();
|
|
});
|
|
|
|
it("allows undefined", () => {
|
|
expect(() => assertAllowedThinkingTier(undefined)).not.toThrow();
|
|
});
|
|
|
|
it("forbids xhigh", () => {
|
|
expect(() => assertAllowedThinkingTier("xhigh")).toThrow(/forbidden/);
|
|
expect(() => assertAllowedThinkingTier("XHIGH")).toThrow(/forbidden/);
|
|
});
|
|
});
|
|
|
|
describe("withRetry", () => {
|
|
it("succeeds on first attempt", async () => {
|
|
let attempts = 0;
|
|
const result = await withRetry(async () => {
|
|
attempts += 1;
|
|
return "ok";
|
|
});
|
|
expect(result.ok).toBe(true);
|
|
expect(result.value).toBe("ok");
|
|
expect(result.attempts).toBe(1);
|
|
expect(attempts).toBe(1);
|
|
});
|
|
|
|
it("retries retryable errors and eventually succeeds", async () => {
|
|
let attempts = 0;
|
|
const result = await withRetry(
|
|
async () => {
|
|
attempts += 1;
|
|
if (attempts < 3) throw new TimeoutError("not yet");
|
|
return "finally";
|
|
},
|
|
{ maxRetries: 3, baseMs: 1, maxMs: 10 },
|
|
);
|
|
expect(result.ok).toBe(true);
|
|
expect(result.value).toBe("finally");
|
|
expect(result.attempts).toBe(3);
|
|
});
|
|
|
|
it("stops on non-retryable error", async () => {
|
|
let attempts = 0;
|
|
const result = await withRetry(
|
|
async () => {
|
|
attempts += 1;
|
|
throw new PermissionError("no");
|
|
},
|
|
{ maxRetries: 3, baseMs: 1 },
|
|
);
|
|
expect(result.ok).toBe(false);
|
|
expect(result.classification?.retryable).toBe(false);
|
|
expect(attempts).toBe(1);
|
|
});
|
|
|
|
it("gives up after max retries", async () => {
|
|
let attempts = 0;
|
|
const result = await withRetry(
|
|
async () => {
|
|
attempts += 1;
|
|
throw new TimeoutError("never succeeds");
|
|
},
|
|
{ maxRetries: 2, baseMs: 1, maxMs: 10 },
|
|
);
|
|
expect(result.ok).toBe(false);
|
|
expect(result.attempts).toBe(3); // initial + 2 retries
|
|
expect(attempts).toBe(3);
|
|
});
|
|
|
|
it("aborts when signal fires mid-backoff", async () => {
|
|
const controller = new AbortController();
|
|
let attempts = 0;
|
|
const promise = withRetry(
|
|
async () => {
|
|
attempts += 1;
|
|
throw new TimeoutError("slow");
|
|
},
|
|
{ maxRetries: 5, baseMs: 1000, maxMs: 5000, signal: controller.signal },
|
|
);
|
|
setTimeout(() => controller.abort(), 50);
|
|
const result = await promise;
|
|
expect(result.ok).toBe(false);
|
|
expect(result.error?.message).toContain("Aborted");
|
|
});
|
|
});
|