diff --git a/apps/docs/content/docs/en/blocks/api.mdx b/apps/docs/content/docs/en/blocks/api.mdx index 6bf1d05226..5cdab33785 100644 --- a/apps/docs/content/docs/en/blocks/api.mdx +++ b/apps/docs/content/docs/en/blocks/api.mdx @@ -95,11 +95,17 @@ const apiUrl = `https://api.example.com/users/${userId}/profile`; ### Request Retries -The API block automatically handles: -- Network timeouts with exponential backoff -- Rate limit responses (429 status codes) -- Server errors (5xx status codes) with retry logic -- Connection failures with reconnection attempts +The API block supports **configurable retries** (see the block’s **Advanced** settings): + +- **Retries**: Number of retry attempts (additional tries after the first request) +- **Retry delay (ms)**: Initial delay before retrying (uses exponential backoff) +- **Max retry delay (ms)**: Maximum delay between retries +- **Retry non-idempotent methods**: Allow retries for **POST/PATCH** (may create duplicate requests) + +Retries are attempted for: + +- Network/connection failures and timeouts (with exponential backoff) +- Rate limits (**429**) and server errors (**5xx**) ### Response Validation diff --git a/apps/sim/blocks/blocks/api.ts b/apps/sim/blocks/blocks/api.ts index 0914546212..e12c1422ff 100644 --- a/apps/sim/blocks/blocks/api.ts +++ b/apps/sim/blocks/blocks/api.ts @@ -89,6 +89,38 @@ Example: 'Request timeout in milliseconds (default: 300000 = 5 minutes, max: 600000 = 10 minutes)', mode: 'advanced', }, + { + id: 'retries', + title: 'Retries', + type: 'short-input', + placeholder: '0', + description: + 'Number of retry attempts for timeouts, 429 responses, and 5xx errors (default: 0, no retries)', + mode: 'advanced', + }, + { + id: 'retryDelayMs', + title: 'Retry delay (ms)', + type: 'short-input', + placeholder: '500', + description: 'Initial retry delay in milliseconds (exponential backoff)', + mode: 'advanced', + }, + { + id: 'retryMaxDelayMs', + title: 'Max retry delay (ms)', + type: 'short-input', + placeholder: '30000', + description: 'Maximum delay between retries in milliseconds', + mode: 'advanced', + }, + { + id: 'retryNonIdempotent', + title: 'Retry non-idempotent methods', + type: 'switch', + description: 'Allow retries for POST/PATCH requests (may create duplicate requests)', + mode: 'advanced', + }, ], tools: { access: ['http_request'], @@ -100,6 +132,16 @@ Example: body: { type: 'json', description: 'Request body data' }, params: { type: 'json', description: 'URL query parameters' }, timeout: { type: 'number', description: 'Request timeout in milliseconds' }, + retries: { type: 'number', description: 'Number of retry attempts for retryable failures' }, + retryDelayMs: { type: 'number', description: 'Initial retry delay in milliseconds' }, + retryMaxDelayMs: { + type: 'number', + description: 'Maximum delay between retries in milliseconds', + }, + retryNonIdempotent: { + type: 'boolean', + description: 'Allow retries for non-idempotent methods like POST/PATCH', + }, }, outputs: { data: { type: 'json', description: 'API response data (JSON, text, or other formats)' }, diff --git a/apps/sim/tools/http/request.ts b/apps/sim/tools/http/request.ts index 687ccb46fe..e501b02bdb 100644 --- a/apps/sim/tools/http/request.ts +++ b/apps/sim/tools/http/request.ts @@ -53,6 +53,28 @@ export const requestTool: ToolConfig = { visibility: 'user-only', description: 'Request timeout in milliseconds (default: 300000 = 5 minutes)', }, + retries: { + type: 'number', + visibility: 'hidden', + description: + 'Number of retry attempts for retryable failures (timeouts, 429, 5xx). Default: 0 (no retries).', + }, + retryDelayMs: { + type: 'number', + visibility: 'hidden', + description: 'Initial retry delay in milliseconds (default: 500)', + }, + retryMaxDelayMs: { + type: 'number', + visibility: 'hidden', + description: 'Maximum delay between retries in milliseconds (default: 30000)', + }, + retryNonIdempotent: { + type: 'boolean', + visibility: 'hidden', + description: + 'Allow retries for non-idempotent methods like POST/PATCH (may create duplicate requests).', + }, }, request: { @@ -119,6 +141,14 @@ export const requestTool: ToolConfig = { return undefined }) as (params: RequestParams) => Record | string | FormData | undefined, + + retry: { + enabled: true, + maxRetries: 0, + initialDelayMs: 500, + maxDelayMs: 30000, + retryIdempotentOnly: true, + }, }, transformResponse: async (response: Response) => { diff --git a/apps/sim/tools/http/types.ts b/apps/sim/tools/http/types.ts index 3799fd0ca6..314227f818 100644 --- a/apps/sim/tools/http/types.ts +++ b/apps/sim/tools/http/types.ts @@ -9,6 +9,10 @@ export interface RequestParams { pathParams?: Record formData?: Record timeout?: number + retries?: number + retryDelayMs?: number + retryMaxDelayMs?: number + retryNonIdempotent?: boolean } export interface RequestResponse extends ToolResponse { diff --git a/apps/sim/tools/index.test.ts b/apps/sim/tools/index.test.ts index 9a20977ae8..fe4b446919 100644 --- a/apps/sim/tools/index.test.ts +++ b/apps/sim/tools/index.test.ts @@ -958,4 +958,231 @@ describe('MCP Tool Execution', () => { expect(result.error).toContain('Network error') expect(result.timing).toBeDefined() }) + + describe('Tool request retries', () => { + function makeJsonResponse( + status: number, + body: unknown, + extraHeaders?: Record + ): any { + const headers = new Headers({ 'content-type': 'application/json', ...(extraHeaders ?? {}) }) + return { + ok: status >= 200 && status < 300, + status, + statusText: status >= 200 && status < 300 ? 'OK' : 'Error', + headers, + json: () => Promise.resolve(body), + text: () => Promise.resolve(typeof body === 'string' ? body : JSON.stringify(body)), + arrayBuffer: () => Promise.resolve(new ArrayBuffer(0)), + blob: () => Promise.resolve(new Blob()), + } + } + + it('retries on 5xx responses for http_request', async () => { + global.fetch = Object.assign( + vi + .fn() + .mockResolvedValueOnce(makeJsonResponse(500, { error: 'nope' })) + .mockResolvedValueOnce(makeJsonResponse(200, { ok: true })), + { preconnect: vi.fn() } + ) as typeof fetch + + const result = await executeTool('http_request', { + url: '/api/test', + method: 'GET', + retries: 2, + retryDelayMs: 0, + retryMaxDelayMs: 0, + }) + + expect(global.fetch).toHaveBeenCalledTimes(2) + expect(result.success).toBe(true) + expect((result.output as any).status).toBe(200) + }) + + it('does not retry when retries is not specified (default: 0)', async () => { + global.fetch = Object.assign( + vi.fn().mockResolvedValue(makeJsonResponse(500, { error: 'server error' })), + { preconnect: vi.fn() } + ) as typeof fetch + + const result = await executeTool('http_request', { + url: '/api/test', + method: 'GET', + }) + + expect(global.fetch).toHaveBeenCalledTimes(1) + expect(result.success).toBe(false) + }) + + it('stops retrying after max attempts for http_request', async () => { + global.fetch = Object.assign( + vi.fn().mockResolvedValue(makeJsonResponse(502, { error: 'bad gateway' })), + { preconnect: vi.fn() } + ) as typeof fetch + + const result = await executeTool('http_request', { + url: '/api/test', + method: 'GET', + retries: 2, + retryDelayMs: 0, + retryMaxDelayMs: 0, + }) + + expect(global.fetch).toHaveBeenCalledTimes(3) + expect(result.success).toBe(false) + }) + + it('does not retry on 4xx responses for http_request', async () => { + global.fetch = Object.assign( + vi.fn().mockResolvedValue(makeJsonResponse(400, { error: 'bad request' })), + { preconnect: vi.fn() } + ) as typeof fetch + + const result = await executeTool('http_request', { + url: '/api/test', + method: 'GET', + retries: 5, + retryDelayMs: 0, + retryMaxDelayMs: 0, + }) + + expect(global.fetch).toHaveBeenCalledTimes(1) + expect(result.success).toBe(false) + }) + + it('does not retry POST by default (non-idempotent)', async () => { + global.fetch = Object.assign( + vi + .fn() + .mockResolvedValueOnce(makeJsonResponse(500, { error: 'nope' })) + .mockResolvedValueOnce(makeJsonResponse(200, { ok: true })), + { preconnect: vi.fn() } + ) as typeof fetch + + const result = await executeTool('http_request', { + url: '/api/test', + method: 'POST', + retries: 2, + retryDelayMs: 0, + retryMaxDelayMs: 0, + }) + + expect(global.fetch).toHaveBeenCalledTimes(1) + expect(result.success).toBe(false) + }) + + it('retries POST when retryNonIdempotent is enabled', async () => { + global.fetch = Object.assign( + vi + .fn() + .mockResolvedValueOnce(makeJsonResponse(500, { error: 'nope' })) + .mockResolvedValueOnce(makeJsonResponse(200, { ok: true })), + { preconnect: vi.fn() } + ) as typeof fetch + + const result = await executeTool('http_request', { + url: '/api/test', + method: 'POST', + retries: 1, + retryNonIdempotent: true, + retryDelayMs: 0, + retryMaxDelayMs: 0, + }) + + expect(global.fetch).toHaveBeenCalledTimes(2) + expect(result.success).toBe(true) + expect((result.output as any).status).toBe(200) + }) + + it('retries on timeout errors for http_request', async () => { + const abortError = Object.assign(new Error('Aborted'), { name: 'AbortError' }) + global.fetch = Object.assign( + vi + .fn() + .mockRejectedValueOnce(abortError) + .mockResolvedValueOnce(makeJsonResponse(200, { ok: true })), + { preconnect: vi.fn() } + ) as typeof fetch + + const result = await executeTool('http_request', { + url: '/api/test', + method: 'GET', + retries: 1, + retryDelayMs: 0, + retryMaxDelayMs: 0, + }) + + expect(global.fetch).toHaveBeenCalledTimes(2) + expect(result.success).toBe(true) + }) + + it('skips retry when Retry-After header exceeds maxDelayMs', async () => { + global.fetch = Object.assign( + vi + .fn() + .mockResolvedValueOnce( + makeJsonResponse(429, { error: 'rate limited' }, { 'retry-after': '60' }) + ) + .mockResolvedValueOnce(makeJsonResponse(200, { ok: true })), + { preconnect: vi.fn() } + ) as typeof fetch + + const result = await executeTool('http_request', { + url: '/api/test', + method: 'GET', + retries: 3, + retryMaxDelayMs: 5000, + }) + + expect(global.fetch).toHaveBeenCalledTimes(1) + expect(result.success).toBe(false) + }) + + it('retries when Retry-After header is within maxDelayMs', async () => { + global.fetch = Object.assign( + vi + .fn() + .mockResolvedValueOnce( + makeJsonResponse(429, { error: 'rate limited' }, { 'retry-after': '1' }) + ) + .mockResolvedValueOnce(makeJsonResponse(200, { ok: true })), + { preconnect: vi.fn() } + ) as typeof fetch + + const result = await executeTool('http_request', { + url: '/api/test', + method: 'GET', + retries: 2, + retryMaxDelayMs: 5000, + }) + + expect(global.fetch).toHaveBeenCalledTimes(2) + expect(result.success).toBe(true) + }) + + it('retries on ETIMEDOUT errors for http_request', async () => { + const etimedoutError = Object.assign(new Error('connect ETIMEDOUT 10.0.0.1:443'), { + code: 'ETIMEDOUT', + }) + global.fetch = Object.assign( + vi + .fn() + .mockRejectedValueOnce(etimedoutError) + .mockResolvedValueOnce(makeJsonResponse(200, { ok: true })), + { preconnect: vi.fn() } + ) as typeof fetch + + const result = await executeTool('http_request', { + url: '/api/test', + method: 'GET', + retries: 1, + retryDelayMs: 0, + retryMaxDelayMs: 0, + }) + + expect(global.fetch).toHaveBeenCalledTimes(2) + expect(result.success).toBe(true) + }) + }) }) diff --git a/apps/sim/tools/index.ts b/apps/sim/tools/index.ts index 8cf48a749a..9426d73103 100644 --- a/apps/sim/tools/index.ts +++ b/apps/sim/tools/index.ts @@ -14,7 +14,7 @@ import { resolveSkillContent } from '@/executor/handlers/agent/skills-resolver' import type { ExecutionContext } from '@/executor/types' import type { ErrorInfo } from '@/tools/error-extractors' import { extractErrorMessage } from '@/tools/error-extractors' -import type { OAuthTokenPayload, ToolConfig, ToolResponse } from '@/tools/types' +import type { OAuthTokenPayload, ToolConfig, ToolResponse, ToolRetryConfig } from '@/tools/types' import { formatRequestParams, getTool, @@ -610,6 +610,68 @@ async function addInternalAuthIfNeeded( } } +interface ResolvedRetryConfig { + maxRetries: number + initialDelayMs: number + maxDelayMs: number +} + +function getRetryConfig( + retry: ToolRetryConfig | undefined, + params: Record, + method: string +): ResolvedRetryConfig | null { + if (!retry?.enabled) return null + + const isIdempotent = ['GET', 'HEAD', 'PUT', 'DELETE'].includes(method.toUpperCase()) + if (retry.retryIdempotentOnly && !isIdempotent && !params.retryNonIdempotent) { + return null + } + + const maxRetries = Math.min(10, Math.max(0, Number(params.retries) || retry.maxRetries || 0)) + if (maxRetries === 0) return null + + return { + maxRetries, + initialDelayMs: Number(params.retryDelayMs) || retry.initialDelayMs || 500, + maxDelayMs: Number(params.retryMaxDelayMs) || retry.maxDelayMs || 30000, + } +} + +function isRetryableFailure(error: unknown, status?: number): boolean { + if (status === 429 || (status && status >= 500 && status <= 599)) return true + if (error instanceof Error) { + const code = (error as NodeJS.ErrnoException).code + if (code === 'ETIMEDOUT' || code === 'ECONNRESET' || code === 'ECONNABORTED') { + return true + } + const msg = error.message.toLowerCase() + if (isBodySizeLimitError(msg)) return false + return msg.includes('timeout') || msg.includes('timed out') + } + return false +} + +function calculateBackoff(attempt: number, initialDelayMs: number, maxDelayMs: number): number { + const base = Math.min(initialDelayMs * 2 ** attempt, maxDelayMs) + return Math.round(base / 2 + Math.random() * (base / 2)) +} + +function parseRetryAfterHeader(header: string | null): number { + if (!header) return 0 + const trimmed = header.trim() + if (/^\d+$/.test(trimmed)) { + const seconds = Number.parseInt(trimmed, 10) + return seconds > 0 ? seconds * 1000 : 0 + } + const date = new Date(trimmed) + if (!Number.isNaN(date.getTime())) { + const deltaMs = date.getTime() - Date.now() + return deltaMs > 0 ? deltaMs : 0 + } + return 0 +} + /** * Execute a tool request directly * Internal routes (/api/...) use regular fetch @@ -691,59 +753,123 @@ async function executeToolRequest( headersRecord[key] = value }) - let response: Response + const retryConfig = getRetryConfig(tool.request.retry, params, requestParams.method) + const maxAttempts = retryConfig ? 1 + retryConfig.maxRetries : 1 - if (isInternalRoute) { - const controller = new AbortController() - const timeout = requestParams.timeout || DEFAULT_EXECUTION_TIMEOUT_MS - const timeoutId = setTimeout(() => controller.abort(), timeout) + let response: Response | undefined + let lastError: unknown + + for (let attempt = 0; attempt < maxAttempts; attempt++) { + const isLastAttempt = attempt === maxAttempts - 1 try { - response = await fetch(fullUrl, { - method: requestParams.method, - headers: headers, - body: requestParams.body, - signal: controller.signal, - }) + if (isInternalRoute) { + const controller = new AbortController() + const timeout = requestParams.timeout || DEFAULT_EXECUTION_TIMEOUT_MS + const timeoutId = setTimeout(() => controller.abort(), timeout) + + try { + response = await fetch(fullUrl, { + method: requestParams.method, + headers: headers, + body: requestParams.body, + signal: controller.signal, + }) + } catch (error) { + if (error instanceof Error && error.name === 'AbortError') { + throw new Error(`Request timed out after ${timeout}ms`) + } + throw error + } finally { + clearTimeout(timeoutId) + } + } else { + const urlValidation = await validateUrlWithDNS(fullUrl, 'toolUrl') + if (!urlValidation.isValid) { + throw new Error(`Invalid tool URL: ${urlValidation.error}`) + } + + const secureResponse = await secureFetchWithPinnedIP(fullUrl, urlValidation.resolvedIP!, { + method: requestParams.method, + headers: headersRecord, + body: requestParams.body ?? undefined, + timeout: requestParams.timeout, + }) + + const responseHeaders = new Headers(secureResponse.headers.toRecord()) + const nullBodyStatuses = new Set([101, 204, 205, 304]) + + if (nullBodyStatuses.has(secureResponse.status)) { + response = new Response(null, { + status: secureResponse.status, + statusText: secureResponse.statusText, + headers: responseHeaders, + }) + } else { + const bodyBuffer = await secureResponse.arrayBuffer() + response = new Response(bodyBuffer, { + status: secureResponse.status, + statusText: secureResponse.statusText, + headers: responseHeaders, + }) + } + } } catch (error) { - // Convert AbortError to a timeout error message - if (error instanceof Error && error.name === 'AbortError') { - throw new Error(`Request timed out after ${timeout}ms`) + lastError = error + if (!retryConfig || isLastAttempt || !isRetryableFailure(error)) { + throw error } - throw error - } finally { - clearTimeout(timeoutId) - } - } else { - const urlValidation = await validateUrlWithDNS(fullUrl, 'toolUrl') - if (!urlValidation.isValid) { - throw new Error(`Invalid tool URL: ${urlValidation.error}`) + const delayMs = calculateBackoff( + attempt, + retryConfig.initialDelayMs, + retryConfig.maxDelayMs + ) + logger.warn( + `[${requestId}] Retrying ${toolId} after error (attempt ${attempt + 1}/${maxAttempts})`, + { delayMs } + ) + await new Promise((r) => setTimeout(r, delayMs)) + continue } - const secureResponse = await secureFetchWithPinnedIP(fullUrl, urlValidation.resolvedIP!, { - method: requestParams.method, - headers: headersRecord, - body: requestParams.body ?? undefined, - timeout: requestParams.timeout, - }) + if ( + retryConfig && + !isLastAttempt && + response && + !response.ok && + isRetryableFailure(null, response.status) + ) { + const retryAfterMs = parseRetryAfterHeader(response.headers.get('retry-after')) + if (retryAfterMs > retryConfig.maxDelayMs) { + logger.warn( + `[${requestId}] Retry-After (${retryAfterMs}ms) exceeds maxDelayMs (${retryConfig.maxDelayMs}ms), skipping retry` + ) + break + } + try { + await response.arrayBuffer() + } catch { + // Ignore errors when consuming body + } + const backoffMs = calculateBackoff( + attempt, + retryConfig.initialDelayMs, + retryConfig.maxDelayMs + ) + const delayMs = Math.max(backoffMs, retryAfterMs) + logger.warn( + `[${requestId}] Retrying ${toolId} after HTTP ${response.status} (attempt ${attempt + 1}/${maxAttempts})`, + { delayMs } + ) + await new Promise((r) => setTimeout(r, delayMs)) + continue + } - const responseHeaders = new Headers(secureResponse.headers.toRecord()) - const nullBodyStatuses = new Set([101, 204, 205, 304]) + break + } - if (nullBodyStatuses.has(secureResponse.status)) { - response = new Response(null, { - status: secureResponse.status, - statusText: secureResponse.statusText, - headers: responseHeaders, - }) - } else { - const bodyBuffer = await secureResponse.arrayBuffer() - response = new Response(bodyBuffer, { - status: secureResponse.status, - statusText: secureResponse.statusText, - headers: responseHeaders, - }) - } + if (!response) { + throw lastError ?? new Error(`Request failed for ${toolId}`) } // For non-OK responses, attempt JSON first; if parsing fails, fall back to text diff --git a/apps/sim/tools/types.ts b/apps/sim/tools/types.ts index 296bdacc7c..06322fe660 100644 --- a/apps/sim/tools/types.ts +++ b/apps/sim/tools/types.ts @@ -58,6 +58,14 @@ export interface OAuthConfig { requiredScopes?: string[] // Specific scopes this tool needs (for granular scope validation) } +export interface ToolRetryConfig { + enabled: boolean + maxRetries?: number + initialDelayMs?: number + maxDelayMs?: number + retryIdempotentOnly?: boolean +} + export interface ToolConfig

{ // Basic tool identification id: string @@ -115,6 +123,7 @@ export interface ToolConfig

{ method: HttpMethod | ((params: P) => HttpMethod) headers: (params: P) => Record body?: (params: P) => Record | string | FormData | undefined + retry?: ToolRetryConfig } // Post-processing (optional) - allows additional processing after the initial request