Files
hanarang-rails/tests/resilience.test.ts
이랑이 30e782a32d feat(sprint-005): Resilience — retry + backoff + escalation + xhigh 금지
Sprint 005 전체 구현 — F5 (중간 끊김/타임아웃 무한대기) 해결:

Resilience core:
- src/resilience/backoff.ts — exponential backoff + jitter (base 1s, cap 30s)
  + cancellable sleep
- src/resilience/classifier.ts — error → {retryable, reason}
  retryable: timeout, network, rate_limit, transient
  non-retryable: permission, config, invariant(ZodError)
  휴리스틱: ETIMEDOUT/ECONNREFUSED/429 등 메시지 패턴 감지
  + assertAllowedThinkingTier('xhigh' 금지)
- src/resilience/retry.ts — withRetry 래퍼
  non-retryable은 즉시 중단, max retries 초과시 classification 반환
- src/resilience/kill.ts — child process SIGTERM→SIGKILL grace 처리
  + 전역 cleanup handler (SIGINT/SIGTERM)
- src/resilience/escalate.ts — recordEscalation + EscalationNotifier
  + listEscalations / resolveEscalation

Prisma:
- Escalation 모델 추가 (pipelineId, reason, errorCategory, attempts, contextSnapshot)
- Pipeline.escalations 역참조

Runner 통합:
- transport.invoke 를 withRetry 로 래핑
- 실패시 classification 기반으로 자동 escalation 기록 (non-retryable만)
- RunOptions 에 maxRetries / notifier 추가

CLI:
- rails resume <pipeline-id> — escalated → idle 전이
- rails abort <pipeline-id> [-r reason] — 강제 종료

Tests (25 신규, 82 total pass):
- backoff: 기본값/지수/캡/jitter 범위/abort
- classifier: 6 error 클래스 + 3 휴리스틱 + xhigh 금지
- withRetry: 성공/재시도 후 성공/non-retryable 즉시 중단/max 초과/abort

검증: tsc --noEmit ✓ | vitest 82/82 ✓ | build ✓ | CLI ✓

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-10 15:40:50 +09:00

224 lines
6.7 KiB
TypeScript

import { describe, it, expect } from "vitest";
import { backoffMs, sleep } from "../src/resilience/backoff.js";
import {
classifyError,
assertAllowedThinkingTier,
TimeoutError,
NetworkError,
RateLimitError,
PermissionError,
ConfigError,
} from "../src/resilience/classifier.js";
import { withRetry } from "../src/resilience/retry.js";
import { ZodError, z } from "zod";
describe("backoffMs", () => {
it("starts near base for retry 0", () => {
const ms = backoffMs(0, { base: 1000, max: 30_000, jitter: 0 });
expect(ms).toBe(1000);
});
it("doubles each retry", () => {
expect(backoffMs(1, { base: 1000, max: 30_000, jitter: 0 })).toBe(2000);
expect(backoffMs(2, { base: 1000, max: 30_000, jitter: 0 })).toBe(4000);
expect(backoffMs(3, { base: 1000, max: 30_000, jitter: 0 })).toBe(8000);
});
it("caps at max", () => {
expect(backoffMs(10, { base: 1000, max: 30_000, jitter: 0 })).toBe(30_000);
expect(backoffMs(20, { base: 1000, max: 30_000, jitter: 0 })).toBe(30_000);
});
it("adds jitter within bounds", () => {
// With jitter 0.3, retry 0 should be in [700, 1300]
for (let i = 0; i < 50; i++) {
const ms = backoffMs(0, { base: 1000, max: 30_000, jitter: 0.3 });
expect(ms).toBeGreaterThanOrEqual(700);
expect(ms).toBeLessThanOrEqual(1300);
}
});
it("returns non-negative values", () => {
for (let i = 0; i < 20; i++) {
expect(backoffMs(i)).toBeGreaterThanOrEqual(0);
}
});
});
describe("sleep", () => {
it("waits approximately the specified time", async () => {
const start = Date.now();
await sleep(50);
const elapsed = Date.now() - start;
expect(elapsed).toBeGreaterThanOrEqual(40);
expect(elapsed).toBeLessThan(200);
});
it("aborts when signal fires", async () => {
const controller = new AbortController();
const promise = sleep(5000, controller.signal);
setTimeout(() => controller.abort(), 10);
await expect(promise).rejects.toThrow("Aborted");
});
});
describe("classifyError", () => {
it("TimeoutError → retryable timeout", () => {
const r = classifyError(new TimeoutError("timed out"));
expect(r.retryable).toBe(true);
expect(r.reason).toBe("timeout");
});
it("NetworkError → retryable network", () => {
const r = classifyError(new NetworkError("econnrefused"));
expect(r.retryable).toBe(true);
expect(r.reason).toBe("network");
});
it("RateLimitError → retryable rate_limit", () => {
const r = classifyError(new RateLimitError("429 too many"));
expect(r.retryable).toBe(true);
expect(r.reason).toBe("rate_limit");
});
it("PermissionError → non-retryable permission", () => {
const r = classifyError(new PermissionError("EACCES"));
expect(r.retryable).toBe(false);
expect(r.reason).toBe("permission");
});
it("ConfigError → non-retryable config", () => {
const r = classifyError(new ConfigError("bad config"));
expect(r.retryable).toBe(false);
expect(r.reason).toBe("config");
});
it("ZodError → non-retryable invariant", () => {
const schema = z.object({ x: z.number() });
let zodErr: unknown;
try {
schema.parse({ x: "not a number" });
} catch (e) {
zodErr = e;
}
expect(zodErr).toBeInstanceOf(ZodError);
const r = classifyError(zodErr);
expect(r.retryable).toBe(false);
expect(r.reason).toBe("invariant");
});
it("detects timeout by message heuristic", () => {
const r = classifyError(new Error("ETIMEDOUT on request"));
expect(r.retryable).toBe(true);
expect(r.reason).toBe("timeout");
});
it("detects network error by message", () => {
const r = classifyError(new Error("ECONNREFUSED"));
expect(r.retryable).toBe(true);
expect(r.reason).toBe("network");
});
it("detects rate limit by message", () => {
const r = classifyError(new Error("429 Rate limit exceeded"));
expect(r.retryable).toBe(true);
expect(r.reason).toBe("rate_limit");
});
it("unknown error defaults to retryable transient", () => {
const r = classifyError(new Error("something weird"));
expect(r.retryable).toBe(true);
expect(r.reason).toBe("transient");
});
});
describe("assertAllowedThinkingTier", () => {
it("allows high and below", () => {
expect(() => assertAllowedThinkingTier("high")).not.toThrow();
expect(() => assertAllowedThinkingTier("medium")).not.toThrow();
expect(() => assertAllowedThinkingTier("low")).not.toThrow();
});
it("allows undefined", () => {
expect(() => assertAllowedThinkingTier(undefined)).not.toThrow();
});
it("forbids xhigh", () => {
expect(() => assertAllowedThinkingTier("xhigh")).toThrow(/forbidden/);
expect(() => assertAllowedThinkingTier("XHIGH")).toThrow(/forbidden/);
});
});
describe("withRetry", () => {
it("succeeds on first attempt", async () => {
let attempts = 0;
const result = await withRetry(async () => {
attempts += 1;
return "ok";
});
expect(result.ok).toBe(true);
expect(result.value).toBe("ok");
expect(result.attempts).toBe(1);
expect(attempts).toBe(1);
});
it("retries retryable errors and eventually succeeds", async () => {
let attempts = 0;
const result = await withRetry(
async () => {
attempts += 1;
if (attempts < 3) throw new TimeoutError("not yet");
return "finally";
},
{ maxRetries: 3, baseMs: 1, maxMs: 10 },
);
expect(result.ok).toBe(true);
expect(result.value).toBe("finally");
expect(result.attempts).toBe(3);
});
it("stops on non-retryable error", async () => {
let attempts = 0;
const result = await withRetry(
async () => {
attempts += 1;
throw new PermissionError("no");
},
{ maxRetries: 3, baseMs: 1 },
);
expect(result.ok).toBe(false);
expect(result.classification?.retryable).toBe(false);
expect(attempts).toBe(1);
});
it("gives up after max retries", async () => {
let attempts = 0;
const result = await withRetry(
async () => {
attempts += 1;
throw new TimeoutError("never succeeds");
},
{ maxRetries: 2, baseMs: 1, maxMs: 10 },
);
expect(result.ok).toBe(false);
expect(result.attempts).toBe(3); // initial + 2 retries
expect(attempts).toBe(3);
});
it("aborts when signal fires mid-backoff", async () => {
const controller = new AbortController();
let attempts = 0;
const promise = withRetry(
async () => {
attempts += 1;
throw new TimeoutError("slow");
},
{ maxRetries: 5, baseMs: 1000, maxMs: 5000, signal: controller.signal },
);
setTimeout(() => controller.abort(), 50);
const result = await promise;
expect(result.ok).toBe(false);
expect(result.error?.message).toContain("Aborted");
});
});