Refactor: use warn message when retrying (#21420)

This commit is contained in:
Wang Qi authored and GitHub committed 2026-10-23 20:09:20 +08:00
1 parent 14133cc5f7
commit a55dccee4c
7 files changed
+57 -16

No files matched your search

@@ -15,7 +15,7 @@ func TestRetryFailureReporterCoalescesRepeatedErrors(t *testing.T) {
reporter := RetryFailureReporter{}
err := errors.New("429 rate limit")
if message, ok := reporter.FailureMessage(1, 6, time.Second, err, false); !ok || !strings.Contains(message, "attempt 1/6") {
if message, ok := reporter.FailureMessage(1, 6, time.Second, err, false); !ok || !strings.HasPrefix(message, "[WARN] LLM call failed (attempt 1/6)") {
t.Fatalf("first failure = (%q, %v), want an emitted attempt message", message, ok)
}
if message, ok := reporter.FailureMessage(2, 6, 2*time.Second, err, false); ok || message != "" {
@@ -24,7 +24,7 @@ func TestRetryFailureReporterCoalescesRepeatedErrors(t *testing.T) {
if message, ok := reporter.FailureMessage(3, 6, 4*time.Second, err, false); ok || message != "" {
t.Fatalf("third repeated failure = (%q, %v), want suppressed", message, ok)
}
if message, ok := reporter.FailureMessage(4, 6, 8*time.Second, err, false); !ok || !strings.Contains(message, "3 repeated failures") {
if message, ok := reporter.FailureMessage(4, 6, 8*time.Second, err, false); !ok || !strings.HasPrefix(message, "[WARN] LLM call failed: 3 repeated failures") {
t.Fatalf("fourth failure = (%q, %v), want a coalesced retry update", message, ok)
}
if message, ok := reporter.RecoveryMessage(5); !ok || !strings.Contains(message, "recovered after 4 failures") {
@@ -94,6 +94,10 @@ func TestGenJSONReportsLLMFailure(t *testing.T) {
// budget. Without the override GenJSON sleeps ~62s per failed call.
func TestGenJSONRetryDelayOverride(t *testing.T) {
ctx := WithRetryDelay(context.Background(), time.Millisecond)
var messages []string
ctx = runtime.WithProgressMessageCallback(ctx, func(_, message string) {
messages = append(messages, message)
})
start := time.Now()
_, err := GenJSON(ctx, failingChat{err: errors.New("llm down")}, ChatRequest{})
if err == nil {
@@ -102,6 +106,9 @@ func TestGenJSONRetryDelayOverride(t *testing.T) {
if time.Since(start) > 5*time.Second {
t.Fatalf("override did not shorten retry backoff: %s", time.Since(start))
}
if len(messages) != 3 || !strings.HasPrefix(messages[0], "[WARN]") || !strings.HasPrefix(messages[1], "[WARN]") || !strings.HasPrefix(messages[2], "[ERROR] LLM call failed after 6 attempts") {
t.Fatalf("unexpected retry progress messages: %v", messages)
}
}
func TestCompactErrorRedactsCredentials(t *testing.T) {
@@ -23,18 +23,22 @@ func (r *RetryFailureReporter) FailureMessage(attempt, total int, delay time.Dur
if r == nil {
return "", false
}
level := "[WARN]"
if final {
level = "[ERROR]"
}
compact := CompactError(err)
if compact != r.lastError {
r.lastError = compact
r.failures = 1
return formatRetryFailure(attempt, total, delay, compact), true
return fmt.Sprintf("%s LLM call failed (attempt %d/%d): %s%s", level, attempt, total, compact, retryDelaySuffix(delay)), true
}
r.failures++
if final {
return fmt.Sprintf("[ERROR] LLM call failed after %d attempts (%d repeated failures): %s", attempt, r.failures-1, compact), true
return fmt.Sprintf("%s LLM call failed after %d attempts (%d repeated failures): %s", level, attempt, r.failures-1, compact), true
}
if r.failures > 1 && (r.failures-1)%retryFailureReportEvery == 0 {
return fmt.Sprintf("[ERROR] LLM call failed: %d repeated failures (latest attempt %d/%d): %s%s", r.failures-1, attempt, total, compact, retryDelaySuffix(delay)), true
return fmt.Sprintf("%s LLM call failed: %d repeated failures (latest attempt %d/%d): %s%s", level, r.failures-1, attempt, total, compact, retryDelaySuffix(delay)), true
}
return "", false
}
@@ -51,10 +55,6 @@ func (r *RetryFailureReporter) RecoveryMessage(attempt int) (string, bool) {
return message, true
}
func formatRetryFailure(attempt, total int, delay time.Duration, compact string) string {
return fmt.Sprintf("[ERROR] LLM call failed (attempt %d/%d): %s%s", attempt, total, compact, retryDelaySuffix(delay))
}
func retryDelaySuffix(delay time.Duration) string {
if delay <= 0 {
return ""
@@ -98,6 +98,8 @@ export function DataflowTimeline({ traceList }: DataflowTimelineProps) {
className={cn('break-all', {
'text-state-error':
x.message.startsWith('[ERROR]'),
'text-state-warning':
x.message.startsWith('[WARN]'),
})}
>
{x.message}
@@ -46,10 +46,20 @@ export function ProgressLogPanel({
<span
className={cn(
'size-2 shrink-0 rounded-full',
entry.isError ? 'bg-state-error' : 'bg-state-success',
entry.isError
? 'bg-state-error'
: entry.isWarning
? 'bg-state-warning'
: 'bg-state-success',
)}
/>
<span className="break-all text-sm text-text-primary">
<span
className={cn('break-all text-sm', {
'text-state-error': entry.isError,
'text-state-warning': entry.isWarning && !entry.isError,
'text-text-primary': !entry.isError && !entry.isWarning,
})}
>
{entry.text}
</span>
</div>
@@ -2,6 +2,7 @@ export interface IProgressLogEntry {
key: number;
text: string;
isError: boolean;
isWarning: boolean;
}
export const parseProgressMsg = (progressMsg?: string): IProgressLogEntry[] =>
@@ -12,4 +13,5 @@ export const parseProgressMsg = (progressMsg?: string): IProgressLogEntry[] =>
key: index,
text: line,
isError: line.includes('[ERROR]') || line.startsWith('ERROR:'),
isWarning: line.includes('[WARN]'),
}));
@@ -63,14 +63,27 @@ describe('ProcessLogModal ingestion events', () => {
event_type: 1,
component: '',
phase: 0,
message: 'embedding 40/120 chunks',
message: '[WARN] LLM call failed; retrying in 2s',
},
{
id: 3,
ts: '2026-09-18T10:21:10+08:00',
event_type: 2,
component: 'Compiler',
phase: 0,
message: '[ERROR] LLM call failed after 6 attempts',
},
]);
const times = screen.getAllByTestId('ingestion-event-time');
expect(times).toHaveLength(2);
expect(times).toHaveLength(3);
expect(times[0].textContent).toBe(formatTime(eventTs));
expect(screen.getByText('embedding 40/120 chunks')).toBeInTheDocument();
expect(
screen.getByText('[WARN] LLM call failed; retrying in 2s'),
).toHaveClass('text-state-warning');
expect(
screen.getByText('[ERROR] LLM call failed after 6 attempts'),
).toHaveClass('text-state-error');
});
it('omits the timestamp span when ts is absent', () => {
+9 -2
View File
@@ -79,10 +79,17 @@ export const replaceText = (text: string) => {
const replacedText = reactStringReplace(
nextText,
/(\[ERROR\].+\s)/g,
/(\[(?:ERROR|WARN)\][^\r\n]*)/g,
(match, i) => {
return (
<span key={i} className={'text-red-600'}>
<span
key={i}
className={
match.startsWith('[WARN]')
? 'text-state-warning'
: 'text-state-error'
}
>
{match}
</span>
);