mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-11-11 13:51:01 +08:00
Refactor: use warn message when retrying (#21420)
This commit is contained in:
1 parent
14133cc5f7
commit
a55dccee4c
7 files changed
+57
-16
No files matched your search
@@ -15,7 +15,7 @@ func TestRetryFailureReporterCoalescesRepeatedErrors(t *testing.T) {
|
||||
reporter := RetryFailureReporter{}
|
||||
err := errors.New("429 rate limit")
|
||||
|
||||
if message, ok := reporter.FailureMessage(1, 6, time.Second, err, false); !ok || !strings.Contains(message, "attempt 1/6") {
|
||||
if message, ok := reporter.FailureMessage(1, 6, time.Second, err, false); !ok || !strings.HasPrefix(message, "[WARN] LLM call failed (attempt 1/6)") {
|
||||
t.Fatalf("first failure = (%q, %v), want an emitted attempt message", message, ok)
|
||||
}
|
||||
if message, ok := reporter.FailureMessage(2, 6, 2*time.Second, err, false); ok || message != "" {
|
||||
@@ -24,7 +24,7 @@ func TestRetryFailureReporterCoalescesRepeatedErrors(t *testing.T) {
|
||||
if message, ok := reporter.FailureMessage(3, 6, 4*time.Second, err, false); ok || message != "" {
|
||||
t.Fatalf("third repeated failure = (%q, %v), want suppressed", message, ok)
|
||||
}
|
||||
if message, ok := reporter.FailureMessage(4, 6, 8*time.Second, err, false); !ok || !strings.Contains(message, "3 repeated failures") {
|
||||
if message, ok := reporter.FailureMessage(4, 6, 8*time.Second, err, false); !ok || !strings.HasPrefix(message, "[WARN] LLM call failed: 3 repeated failures") {
|
||||
t.Fatalf("fourth failure = (%q, %v), want a coalesced retry update", message, ok)
|
||||
}
|
||||
if message, ok := reporter.RecoveryMessage(5); !ok || !strings.Contains(message, "recovered after 4 failures") {
|
||||
@@ -94,6 +94,10 @@ func TestGenJSONReportsLLMFailure(t *testing.T) {
|
||||
// budget. Without the override GenJSON sleeps ~62s per failed call.
|
||||
func TestGenJSONRetryDelayOverride(t *testing.T) {
|
||||
ctx := WithRetryDelay(context.Background(), time.Millisecond)
|
||||
var messages []string
|
||||
ctx = runtime.WithProgressMessageCallback(ctx, func(_, message string) {
|
||||
messages = append(messages, message)
|
||||
})
|
||||
start := time.Now()
|
||||
_, err := GenJSON(ctx, failingChat{err: errors.New("llm down")}, ChatRequest{})
|
||||
if err == nil {
|
||||
@@ -102,6 +106,9 @@ func TestGenJSONRetryDelayOverride(t *testing.T) {
|
||||
if time.Since(start) > 5*time.Second {
|
||||
t.Fatalf("override did not shorten retry backoff: %s", time.Since(start))
|
||||
}
|
||||
if len(messages) != 3 || !strings.HasPrefix(messages[0], "[WARN]") || !strings.HasPrefix(messages[1], "[WARN]") || !strings.HasPrefix(messages[2], "[ERROR] LLM call failed after 6 attempts") {
|
||||
t.Fatalf("unexpected retry progress messages: %v", messages)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompactErrorRedactsCredentials(t *testing.T) {
|
||||
|
||||
@@ -23,18 +23,22 @@ func (r *RetryFailureReporter) FailureMessage(attempt, total int, delay time.Dur
|
||||
if r == nil {
|
||||
return "", false
|
||||
}
|
||||
level := "[WARN]"
|
||||
if final {
|
||||
level = "[ERROR]"
|
||||
}
|
||||
compact := CompactError(err)
|
||||
if compact != r.lastError {
|
||||
r.lastError = compact
|
||||
r.failures = 1
|
||||
return formatRetryFailure(attempt, total, delay, compact), true
|
||||
return fmt.Sprintf("%s LLM call failed (attempt %d/%d): %s%s", level, attempt, total, compact, retryDelaySuffix(delay)), true
|
||||
}
|
||||
r.failures++
|
||||
if final {
|
||||
return fmt.Sprintf("[ERROR] LLM call failed after %d attempts (%d repeated failures): %s", attempt, r.failures-1, compact), true
|
||||
return fmt.Sprintf("%s LLM call failed after %d attempts (%d repeated failures): %s", level, attempt, r.failures-1, compact), true
|
||||
}
|
||||
if r.failures > 1 && (r.failures-1)%retryFailureReportEvery == 0 {
|
||||
return fmt.Sprintf("[ERROR] LLM call failed: %d repeated failures (latest attempt %d/%d): %s%s", r.failures-1, attempt, total, compact, retryDelaySuffix(delay)), true
|
||||
return fmt.Sprintf("%s LLM call failed: %d repeated failures (latest attempt %d/%d): %s%s", level, r.failures-1, attempt, total, compact, retryDelaySuffix(delay)), true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
@@ -51,10 +55,6 @@ func (r *RetryFailureReporter) RecoveryMessage(attempt int) (string, bool) {
|
||||
return message, true
|
||||
}
|
||||
|
||||
func formatRetryFailure(attempt, total int, delay time.Duration, compact string) string {
|
||||
return fmt.Sprintf("[ERROR] LLM call failed (attempt %d/%d): %s%s", attempt, total, compact, retryDelaySuffix(delay))
|
||||
}
|
||||
|
||||
func retryDelaySuffix(delay time.Duration) string {
|
||||
if delay <= 0 {
|
||||
return ""
|
||||
|
||||
@@ -98,6 +98,8 @@ export function DataflowTimeline({ traceList }: DataflowTimelineProps) {
|
||||
className={cn('break-all', {
|
||||
'text-state-error':
|
||||
x.message.startsWith('[ERROR]'),
|
||||
'text-state-warning':
|
||||
x.message.startsWith('[WARN]'),
|
||||
})}
|
||||
>
|
||||
{x.message}
|
||||
|
||||
@@ -46,10 +46,20 @@ export function ProgressLogPanel({
|
||||
<span
|
||||
className={cn(
|
||||
'size-2 shrink-0 rounded-full',
|
||||
entry.isError ? 'bg-state-error' : 'bg-state-success',
|
||||
entry.isError
|
||||
? 'bg-state-error'
|
||||
: entry.isWarning
|
||||
? 'bg-state-warning'
|
||||
: 'bg-state-success',
|
||||
)}
|
||||
/>
|
||||
<span className="break-all text-sm text-text-primary">
|
||||
<span
|
||||
className={cn('break-all text-sm', {
|
||||
'text-state-error': entry.isError,
|
||||
'text-state-warning': entry.isWarning && !entry.isError,
|
||||
'text-text-primary': !entry.isError && !entry.isWarning,
|
||||
})}
|
||||
>
|
||||
{entry.text}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
@@ -2,6 +2,7 @@ export interface IProgressLogEntry {
|
||||
key: number;
|
||||
text: string;
|
||||
isError: boolean;
|
||||
isWarning: boolean;
|
||||
}
|
||||
|
||||
export const parseProgressMsg = (progressMsg?: string): IProgressLogEntry[] =>
|
||||
@@ -12,4 +13,5 @@ export const parseProgressMsg = (progressMsg?: string): IProgressLogEntry[] =>
|
||||
key: index,
|
||||
text: line,
|
||||
isError: line.includes('[ERROR]') || line.startsWith('ERROR:'),
|
||||
isWarning: line.includes('[WARN]'),
|
||||
}));
|
||||
@@ -63,14 +63,27 @@ describe('ProcessLogModal ingestion events', () => {
|
||||
event_type: 1,
|
||||
component: '',
|
||||
phase: 0,
|
||||
message: 'embedding 40/120 chunks',
|
||||
message: '[WARN] LLM call failed; retrying in 2s',
|
||||
},
|
||||
{
|
||||
id: 3,
|
||||
ts: '2026-09-18T10:21:10+08:00',
|
||||
event_type: 2,
|
||||
component: 'Compiler',
|
||||
phase: 0,
|
||||
message: '[ERROR] LLM call failed after 6 attempts',
|
||||
},
|
||||
]);
|
||||
|
||||
const times = screen.getAllByTestId('ingestion-event-time');
|
||||
expect(times).toHaveLength(2);
|
||||
expect(times).toHaveLength(3);
|
||||
expect(times[0].textContent).toBe(formatTime(eventTs));
|
||||
expect(screen.getByText('embedding 40/120 chunks')).toBeInTheDocument();
|
||||
expect(
|
||||
screen.getByText('[WARN] LLM call failed; retrying in 2s'),
|
||||
).toHaveClass('text-state-warning');
|
||||
expect(
|
||||
screen.getByText('[ERROR] LLM call failed after 6 attempts'),
|
||||
).toHaveClass('text-state-error');
|
||||
});
|
||||
|
||||
it('omits the timestamp span when ts is absent', () => {
|
||||
|
||||
@@ -79,10 +79,17 @@ export const replaceText = (text: string) => {
|
||||
|
||||
const replacedText = reactStringReplace(
|
||||
nextText,
|
||||
/(\[ERROR\].+\s)/g,
|
||||
/(\[(?:ERROR|WARN)\][^\r\n]*)/g,
|
||||
(match, i) => {
|
||||
return (
|
||||
<span key={i} className={'text-red-600'}>
|
||||
<span
|
||||
key={i}
|
||||
className={
|
||||
match.startsWith('[WARN]')
|
||||
? 'text-state-warning'
|
||||
: 'text-state-error'
|
||||
}
|
||||
>
|
||||
{match}
|
||||
</span>
|
||||
);
|
||||
|
||||
Reference in new issue
Block a user