fix: multi-line TUI detection, stale scrollback, and false positives

The rate limit detection was fundamentally broken because Claude Code
renders rate limit messages across multiple lines in its TUI (Ink/React),
but all regex patterns required keywords on a single line.

Key fixes:
- Split detection into LIMIT_PATTERNS + RESET_PATTERNS with a 6-line
  sliding window, so multi-line TUI renders are detected correctly
- Capture only last 20 lines instead of 200 to avoid stale scrollback
- Add 30s post-retry cooldown to prevent re-detecting old messages
- Fix max-retries infinite loop (stay in waiting, check clear first)
- Add OSC/DCS sequence stripping for complete ANSI handling
- Parse relative times ("try again in 5 minutes") instead of 5h fallback
- Handle invalid timezones gracefully with fallback
- Fix zombie monitor on pane destruction (exit after 10 consecutive errors)
- Replace setInterval with recursive setTimeout to prevent concurrent ticks
- Restore user shell traps instead of clobbering them
- Remove overly broad /\blimit\b/ pattern that caused false positives

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
CheapestInference
2026-03-28 10:58:16 +01:00
parent 1aad1e0313
commit db9f4f81af
7 changed files with 240 additions and 46 deletions
+32 -9
View File
@@ -13,23 +13,27 @@ export function createMonitorState() {
export async function processOneTick(state, tmuxAdapter, pane, config, isAlive) { export async function processOneTick(state, tmuxAdapter, pane, config, isAlive) {
if (!isAlive()) return 'exit'; if (!isAlive()) return 'exit';
const raw = await tmuxAdapter.capturePane(pane); const raw = await tmuxAdapter.capturePane(pane, 20);
const stripped = stripAnsi(raw); const stripped = stripAnsi(raw);
if (state.status === 'waiting') { if (state.status === 'waiting') {
if (Date.now() < state.waitUntil) return 'waiting'; if (Date.now() < state.waitUntil) return 'waiting';
if (!isAlive()) return 'exit'; if (!isAlive()) return 'exit';
if (state.attempts >= config.maxRetries) {
state.status = 'monitoring';
return 'max-retries';
}
// Use same window size for detection and recovery check to avoid asymmetry // Always check if rate limit cleared FIRST — even when maxRetries
// exhausted, the user (or time passing) may have resolved it.
if (!isRateLimited(stripped, config.customPatterns)) { if (!isRateLimited(stripped, config.customPatterns)) {
state.status = 'monitoring'; state.attempts = 0; state.status = 'monitoring'; state.attempts = 0;
return 'user-continued'; return 'user-continued';
} }
if (state.attempts >= config.maxRetries) {
// Stay in 'waiting' to avoid re-detecting the stale rate limit
// on the next tick and creating an infinite max-retries loop.
state.waitUntil = Date.now() + (config.pollIntervalSeconds * 1000 * 12);
return 'max-retries';
}
const fg = await tmuxAdapter.getPaneCommand(pane); const fg = await tmuxAdapter.getPaneCommand(pane);
if (!CLAUDE_COMMANDS.some(c => fg.toLowerCase().includes(c))) { if (!CLAUDE_COMMANDS.some(c => fg.toLowerCase().includes(c))) {
// Push waitUntil forward to avoid tight-loop polling every tick // Push waitUntil forward to avoid tight-loop polling every tick
@@ -39,7 +43,11 @@ export async function processOneTick(state, tmuxAdapter, pane, config, isAlive)
await tmuxAdapter.sendKeys(pane, config.retryMessage); await tmuxAdapter.sendKeys(pane, config.retryMessage);
state.attempts++; state.attempts++;
state.status = 'monitoring'; // Stay in 'waiting' with a 30s cooldown instead of going to 'monitoring'.
// This gives Claude time to process the retry and produce enough output
// to push the stale rate-limit message out of the recent-lines window.
// Without this, the next tick re-detects the old message and waits 24h.
state.waitUntil = Date.now() + 30_000;
return 'retried'; return 'retried';
} }
@@ -59,6 +67,8 @@ export async function startMonitor(pane, pid) {
const config = await loadConfig(); const config = await loadConfig();
const logger = createLogger(); const logger = createLogger();
const state = createMonitorState(); const state = createMonitorState();
let consecutiveErrors = 0;
const MAX_CONSECUTIVE_ERRORS = 10;
await logger.info(`Monitor started for pane ${pane} (claude PID: ${pid})`); await logger.info(`Monitor started for pane ${pane} (claude PID: ${pid})`);
@@ -68,6 +78,7 @@ export async function startMonitor(pane, pid) {
const loop = async () => { const loop = async () => {
try { try {
const result = await processOneTick(state, tmuxAdapter, pane, config, isAlive); const result = await processOneTick(state, tmuxAdapter, pane, config, isAlive);
consecutiveErrors = 0;
if (result === 'exit') { await logger.info('Claude exited. Monitor shutting down.'); process.exit(0); } if (result === 'exit') { await logger.info('Claude exited. Monitor shutting down.'); process.exit(0); }
if (result === 'waiting' && state.lastRateLimitMessage) { if (result === 'waiting' && state.lastRateLimitMessage) {
@@ -80,12 +91,24 @@ export async function startMonitor(pane, pid) {
if (result === 'max-retries') await logger.warn(`Max retries (${config.maxRetries}) reached. Monitor still active but will not send further retries until rate limit clears.`); if (result === 'max-retries') await logger.warn(`Max retries (${config.maxRetries}) reached. Monitor still active but will not send further retries until rate limit clears.`);
if (result === 'skipped-not-claude') await logger.warn('Foreground is not Claude. Skipping send-keys.'); if (result === 'skipped-not-claude') await logger.warn('Foreground is not Claude. Skipping send-keys.');
} catch (err) { } catch (err) {
consecutiveErrors++;
await logger.error(`Monitor tick error: ${err.message}`).catch(() => {}); await logger.error(`Monitor tick error: ${err.message}`).catch(() => {});
if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) {
await logger.error(`${MAX_CONSECUTIVE_ERRORS} consecutive errors. Pane likely destroyed. Exiting.`).catch(() => {});
process.exit(1);
}
} }
}; };
setInterval(loop, config.pollIntervalSeconds * 1000); // Use recursive setTimeout instead of setInterval to prevent concurrent
loop(); // tick execution when a tick takes longer than the poll interval.
const scheduleNext = () => {
setTimeout(async () => {
await loop();
scheduleNext();
}, config.pollIntervalSeconds * 1000);
};
loop().then(scheduleNext);
} }
// Direct execution: node monitor.js <pane> <pid> // Direct execution: node monitor.js <pane> <pid>
+70 -20
View File
@@ -1,37 +1,87 @@
// Full CSI sequence range per ECMA-48: parameter/intermediate bytes (0x20-0x3f) + final byte (0x40-0x7e) // Full CSI sequence range per ECMA-48: parameter/intermediate bytes (0x20-0x3f) + final byte (0x40-0x7e)
// Covers standard, private-mode (\x1b[?25h), and extended sequences // Covers standard, private-mode (\x1b[?25h), and extended sequences
const ANSI_REGEX = /\x1b\[[\x20-\x3f]*[\x40-\x7e]/g; const CSI_REGEX = /\x1b\[[\x20-\x3f]*[\x40-\x7e]/g;
// OSC sequences: \x1b] ... (terminated by BEL \x07 or ST \x1b\\)
// Covers hyperlinks (\x1b]8;;url\x1b\\), window titles (\x1b]0;title\x07), etc.
const OSC_REGEX = /\x1b\][\s\S]*?(?:\x07|\x1b\\)/g;
// DCS sequences: \x1bP ... ST
const DCS_REGEX = /\x1bP[\s\S]*?(?:\x07|\x1b\\)/g;
// APC, SOS, PM sequences: \x1b[_X^] ... ST
const OTHER_ESC_REGEX = /\x1b[_X^][\s\S]*?(?:\x07|\x1b\\)/g;
export function stripAnsi(text) { export function stripAnsi(text) {
return text.replace(ANSI_REGEX, ''); return text
.replace(OSC_REGEX, '')
.replace(DCS_REGEX, '')
.replace(OTHER_ESC_REGEX, '')
.replace(CSI_REGEX, '');
} }
const DEFAULT_PATTERNS = [ // Claude Code renders rate limits across multiple lines in its TUI, e.g.:
/\d+-hour limit reached/i, // "⚠ You've hit your limit"
/limit reached.*resets?\s/i, // "· resets 3pm (UTC)"
/usage limit.*resets?\s/i, // Detection: find a "limit" line and a "resets" line within 6 lines of each other.
/out of.*usage.*resets?\s/i,
/try again in \d+\s*(hours?|minutes?|h|m)/i, const LIMIT_PATTERNS = [
/rate limit.*resets?\s/i, /(?:hit|exceeded|reached).*(?:your|the)\s*(?:\d+-hour\s+)?limit/i, // "hit/exceeded/reached your limit"
/hit.*(?:your|the)?\s*limit.*resets?\s/i, /\d+-hour limit/i, // "5-hour limit"
/\blimit\b.*resets?\s+(?:at\s+|in[:\s])\s*\d/i, /limit reached/i, // "limit reached"
/usage limit/i, // "usage limit"
/out of.*usage/i, // "out of extra usage"
/rate limit/i, // "rate limit"
/try again in/i, // "try again in X hours" (implies rate limiting)
]; ];
const RESET_PATTERNS = [
/resets?\s+(?:at\s+)?\d{1,2}(?::\d{2})?\s*(?:am|pm)?/i, // "resets 3pm" / "resets at 3:00 PM"
/resets?\s+in[:\s]\s*\d/i, // "resets in: 3 hours"
/try again in \d+\s*(?:hours?|minutes?|h|m)/i, // "try again in 5 hours"
];
const WINDOW = 6;
function hasNearbyMatch(lines, idx, patterns) {
const start = Math.max(0, idx - WINDOW);
const end = Math.min(lines.length, idx + WINDOW + 1);
for (let j = start; j < end; j++) {
if (patterns.some(p => p.test(lines[j]))) return true;
}
return false;
}
export function isRateLimited(text, customPatterns = []) { export function isRateLimited(text, customPatterns = []) {
const stripped = stripAnsi(text); const lines = stripAnsi(text).split('\n');
const patterns = [...DEFAULT_PATTERNS, ...customPatterns.map(p =>
typeof p === 'string' ? new RegExp(p, 'i') : p // Custom patterns: check full text (user controls their own regex)
)]; if (customPatterns.length > 0) {
return patterns.some(pattern => pattern.test(stripped)); const full = lines.join('\n');
const custom = customPatterns.map(p => typeof p === 'string' ? new RegExp(p, 'i') : p);
if (custom.some(p => p.test(full))) return true;
}
// Find a "limit" line with a "resets" line nearby (works for both
// single-line messages and multi-line TUI renders)
for (let i = 0; i < lines.length; i++) {
if (LIMIT_PATTERNS.some(p => p.test(lines[i]))) {
if (hasNearbyMatch(lines, i, RESET_PATTERNS)) return true;
}
}
return false;
} }
export function findRateLimitMessage(text, customPatterns = []) { export function findRateLimitMessage(text, customPatterns = []) {
const lines = stripAnsi(text).split('\n'); const lines = stripAnsi(text).split('\n');
const patterns = [...DEFAULT_PATTERNS, ...customPatterns.map(p =>
typeof p === 'string' ? new RegExp(p, 'i') : p // Return the "resets" line — that's what parseResetTime needs
)];
for (const line of lines) { for (const line of lines) {
if (patterns.some(pattern => pattern.test(line))) return line.trim(); if (RESET_PATTERNS.some(p => p.test(line))) return line.trim();
} }
// Fallback: any "limit" line
for (const line of lines) {
if (LIMIT_PATTERNS.some(p => p.test(line))) return line.trim();
}
return null; return null;
} }
+37 -12
View File
@@ -1,27 +1,52 @@
const RESET_TIME_REGEX = /resets?\s+(?:at\s+)?(\d{1,2})(?::(\d{2}))?\s*(am|pm)?\s*(?:\(([^)]+)\))?/i; const RESET_TIME_REGEX = /resets?\s+(?:at\s+)?(\d{1,2})(?::(\d{2}))?\s*(am|pm)?\s*(?:\(([^)]+)\))?/i;
const RELATIVE_TIME_REGEX = /(?:try again|wait)\s+(?:for\s+)?(?:in\s+)?(\d+)\s*(hours?|minutes?|mins?|h|m)\b/i;
export function parseResetTime(text) { export function parseResetTime(text) {
const match = text.match(RESET_TIME_REGEX); // Try absolute time first: "resets at 3pm (UTC)"
if (!match) return null; const absMatch = text.match(RESET_TIME_REGEX);
if (absMatch) {
let hour = parseInt(absMatch[1], 10);
const minute = absMatch[2] ? parseInt(absMatch[2], 10) : 0;
const ampm = absMatch[3]?.toLowerCase() || null;
const timezone = absMatch[4] || null;
let hour = parseInt(match[1], 10); if (ampm === 'pm' && hour !== 12) hour += 12;
const minute = match[2] ? parseInt(match[2], 10) : 0; if (ampm === 'am' && hour === 12) hour = 0;
const ampm = match[3]?.toLowerCase() || null;
const timezone = match[4] || null;
if (ampm === 'pm' && hour !== 12) hour += 12; const ambiguous = !ampm && hour >= 1 && hour <= 12;
if (ampm === 'am' && hour === 12) hour = 0; return { hour, minute, timezone, ambiguous };
}
// Ambiguous only when no am/pm AND hour is 1-12 (not 0, which is unambiguous 24h midnight) // Try relative time: "try again in 5 minutes" / "wait 2 hours"
const ambiguous = !ampm && hour >= 1 && hour <= 12; const relMatch = text.match(RELATIVE_TIME_REGEX);
if (relMatch) {
const amount = parseInt(relMatch[1], 10);
const unit = relMatch[2].toLowerCase();
const isMinutes = unit.startsWith('m');
const ms = amount * (isMinutes ? 60_000 : 3_600_000);
return { relative: true, waitMs: ms };
}
return { hour, minute, timezone, ambiguous }; return null;
} }
export function calculateWaitMs(parsed, marginSeconds = 60, fallbackHours = 5, now = new Date()) { export function calculateWaitMs(parsed, marginSeconds = 60, fallbackHours = 5, now = new Date()) {
if (!parsed) return (fallbackHours * 3600 + marginSeconds) * 1000; if (!parsed) return (fallbackHours * 3600 + marginSeconds) * 1000;
const tz = parsed.timezone || Intl.DateTimeFormat().resolvedOptions().timeZone; // Handle relative times: "try again in 5 minutes"
if (parsed.relative) {
return parsed.waitMs + marginSeconds * 1000;
}
let tz;
try {
tz = parsed.timezone || Intl.DateTimeFormat().resolvedOptions().timeZone;
// Validate timezone early to avoid cryptic errors later
Intl.DateTimeFormat('en-US', { timeZone: tz });
} catch {
// Invalid timezone (possibly garbled by TUI capture) — use fallback
return (fallbackHours * 3600 + marginSeconds) * 1000;
}
// DST-safe approach: binary search for the correct UTC timestamp // DST-safe approach: binary search for the correct UTC timestamp
// that corresponds to the given hour:minute in the target timezone. // that corresponds to the given hour:minute in the target timezone.
+7 -2
View File
@@ -5,11 +5,16 @@ claude() {
return $? return $?
fi fi
export CLAUDE_AUTO_RETRY_ACTIVE=1 export CLAUDE_AUTO_RETRY_ACTIVE=1
trap 'unset CLAUDE_AUTO_RETRY_ACTIVE' EXIT INT TERM local _car_old_int_trap _car_old_term_trap
_car_old_int_trap=$(trap -p INT)
_car_old_term_trap=$(trap -p TERM)
trap 'unset CLAUDE_AUTO_RETRY_ACTIVE' INT TERM
node "__LAUNCHER_PATH__" "$@" node "__LAUNCHER_PATH__" "$@"
local _car_exit=$? local _car_exit=$?
unset CLAUDE_AUTO_RETRY_ACTIVE unset CLAUDE_AUTO_RETRY_ACTIVE
trap - EXIT INT TERM # Restore previous traps instead of clobbering them
eval "${_car_old_int_trap:-trap - INT}"
eval "${_car_old_term_trap:-trap - TERM}"
return $_car_exit return $_car_exit
} }
# <<< claude-auto-retry <<< # <<< claude-auto-retry <<<
+21 -1
View File
@@ -39,6 +39,15 @@ describe('processOneTick', () => {
assert.equal(await processOneTick(s, t, '%0', DEFAULT_CONFIG, () => true), 'retried'); assert.equal(await processOneTick(s, t, '%0', DEFAULT_CONFIG, () => true), 'retried');
assert.equal(t._sent.length, 1); assert.equal(t._sent.length, 1);
assert.equal(s.attempts, 1); assert.equal(s.attempts, 1);
// Should stay in 'waiting' with a cooldown to let Claude process
assert.equal(s.status, 'waiting');
assert.ok(s.waitUntil > Date.now());
});
it('detects multi-line TUI rate limit', async () => {
const t = mockTmux('⚠ You\'ve hit your limit\n· resets 3pm (UTC)');
const s = createMonitorState();
assert.equal(await processOneTick(s, t, '%0', DEFAULT_CONFIG, () => true), 'waiting');
assert.ok(s.waitUntil > Date.now());
}); });
it('skips when foreground is not claude', async () => { it('skips when foreground is not claude', async () => {
const t = mockTmux('5-hour limit reached - resets 3pm (UTC)', 'vim'); const t = mockTmux('5-hour limit reached - resets 3pm (UTC)', 'vim');
@@ -54,10 +63,21 @@ describe('processOneTick', () => {
assert.equal(await processOneTick(s, t, '%0', DEFAULT_CONFIG, () => true), 'user-continued'); assert.equal(await processOneTick(s, t, '%0', DEFAULT_CONFIG, () => true), 'user-continued');
assert.equal(s.attempts, 0); assert.equal(s.attempts, 0);
}); });
it('stops retrying after max attempts', async () => { it('stops retrying after max attempts and stays in waiting', async () => {
const t = mockTmux('5-hour limit reached - resets 3pm (UTC)'); const t = mockTmux('5-hour limit reached - resets 3pm (UTC)');
const s = createMonitorState(); const s = createMonitorState();
s.waitUntil = Date.now() - 1000; s.status = 'waiting'; s.attempts = 5; s.waitUntil = Date.now() - 1000; s.status = 'waiting'; s.attempts = 5;
assert.equal(await processOneTick(s, t, '%0', DEFAULT_CONFIG, () => true), 'max-retries'); assert.equal(await processOneTick(s, t, '%0', DEFAULT_CONFIG, () => true), 'max-retries');
// Should stay in 'waiting' to avoid re-detection loop
assert.equal(s.status, 'waiting');
assert.ok(s.waitUntil > Date.now());
});
it('resets from max-retries when rate limit clears', async () => {
const t = mockTmux('Claude is working normally');
const s = createMonitorState();
s.waitUntil = Date.now() - 1000; s.status = 'waiting'; s.attempts = 10;
// Rate limit cleared → should detect user-continued before max-retries check
assert.equal(await processOneTick(s, t, '%0', DEFAULT_CONFIG, () => true), 'user-continued');
assert.equal(s.attempts, 0);
}); });
}); });
+50 -2
View File
@@ -57,8 +57,8 @@ describe('isRateLimited', () => {
it('detects "hit the limit resets"', () => { it('detects "hit the limit resets"', () => {
assert.equal(isRateLimited('You hit the limit. Resets at 5pm'), true); assert.equal(isRateLimited('You hit the limit. Resets at 5pm'), true);
}); });
it('detects "limit resets in: 3 hours"', () => { it('detects "usage limit · resets in: 3 hours"', () => {
assert.equal(isRateLimited('limit resets in: 3 hours'), true); assert.equal(isRateLimited('usage limit · resets in: 3 hours'), true);
}); });
}); });
@@ -79,4 +79,52 @@ describe('findRateLimitMessage', () => {
it('returns null when no match', () => { it('returns null when no match', () => {
assert.equal(findRateLimitMessage('normal output\nmore output'), null); assert.equal(findRateLimitMessage('normal output\nmore output'), null);
}); });
it('returns the resets line from multi-line TUI render', () => {
const text = '⚠ You\'ve hit your limit\n· resets 3pm (UTC)';
assert.equal(findRateLimitMessage(text), '· resets 3pm (UTC)');
});
it('returns Resets line when limit and resets on different lines', () => {
const text = '5-hour limit reached\nResets at 3pm (UTC)';
assert.ok(findRateLimitMessage(text).includes('3pm'));
});
});
describe('isRateLimited (multi-line TUI renders)', () => {
it('detects limit + resets on separate lines', () => {
assert.ok(isRateLimited('⚠ You\'ve hit your limit\n· resets 3pm (UTC)'));
});
it('detects box-drawing TUI format', () => {
const text = '╭──────────╮\n│ ⚠ You\'ve hit your limit │\n│ · resets 3pm │\n╰──────────╯';
assert.ok(isRateLimited(text));
});
it('detects 5-hour limit + Resets on separate lines', () => {
assert.ok(isRateLimited('⚠ 5-hour limit reached\nResets at 3pm (UTC)'));
});
it('detects middle-dot separated multi-line', () => {
assert.ok(isRateLimited('⚠ You\'ve hit your 5-hour limit\n· resets 3pm (Asia/Tbilisi)'));
});
it('rejects limit + resets too far apart (>6 lines)', () => {
assert.equal(isRateLimited('hit your limit\n1\n2\n3\n4\n5\n6\n7\nresets 3pm'), false);
});
it('rejects normal output with no rate limit keywords', () => {
assert.equal(isRateLimited('Working on your request\nHere is the code\nDone'), false);
});
});
describe('stripAnsi (OSC sequences)', () => {
it('strips OSC hyperlinks (\\x1b]8;;url\\x1b\\\\)', () => {
const input = '\x1b]8;;https://example.com\x1b\\click here\x1b]8;;\x1b\\';
assert.equal(stripAnsi(input), 'click here');
});
it('strips OSC window title (\\x1b]0;title\\x07)', () => {
assert.equal(stripAnsi('\x1b]0;My Terminal\x07hello'), 'hello');
});
it('strips OSC + CSI mixed sequences', () => {
const input = '\x1b]8;;url\x1b\\\x1b[33m5-hour limit reached - resets 3pm\x1b[0m\x1b]8;;\x1b\\';
assert.equal(stripAnsi(input), '5-hour limit reached - resets 3pm');
});
it('rate limit detection works through OSC hyperlinks', () => {
const input = '\x1b]8;;link\x1b\\5-hour limit reached\x1b]8;;\x1b\\ - resets 3pm';
assert.ok(isRateLimited(input));
});
}); });
+23
View File
@@ -31,6 +31,21 @@ describe('parseResetTime', () => {
it('returns null for unparseable text', () => { it('returns null for unparseable text', () => {
assert.equal(parseResetTime('some random text'), null); assert.equal(parseResetTime('some random text'), null);
}); });
it('parses "try again in 5 minutes" as relative time', () => {
const r = parseResetTime('try again in 5 minutes');
assert.ok(r.relative);
assert.equal(r.waitMs, 5 * 60_000);
});
it('parses "try again in 2 hours" as relative time', () => {
const r = parseResetTime('try again in 2 hours');
assert.ok(r.relative);
assert.equal(r.waitMs, 2 * 3_600_000);
});
it('parses "wait 30 mins" as relative time', () => {
const r = parseResetTime('wait 30 mins');
assert.ok(r.relative);
assert.equal(r.waitMs, 30 * 60_000);
});
}); });
describe('calculateWaitMs', () => { describe('calculateWaitMs', () => {
@@ -59,4 +74,12 @@ describe('calculateWaitMs', () => {
); );
assert.ok(wait > 0 && wait <= 3 * 3600_000); assert.ok(wait > 0 && wait <= 3 * 3600_000);
}); });
it('handles relative time correctly', () => {
const wait = calculateWaitMs({ relative: true, waitMs: 300_000 }, 60, 5);
assert.ok(Math.abs(wait - 360_000) < 2000); // 5 min + 60s margin
});
it('falls back on invalid timezone', () => {
const wait = calculateWaitMs({ hour: 15, minute: 0, timezone: 'Invalid/Zone' }, 60, 5);
assert.ok(Math.abs(wait - (5 * 3600 + 60) * 1000) < 2000); // fallback
});
}); });