From f208457b634531ab820029f6bd687d77d8f8a120 Mon Sep 17 00:00:00 2001 From: Dangoron Date: Mon, 12 May 2025 14:57:59 +0800 Subject: [PATCH 1/2] feat:add-httpstreamable --- README.md | 68 ++++++++++++++++++++++++++++-- package.json | 2 +- pnpm-lock.yaml | 19 +++++---- src/index.ts | 110 +++++++++++++++++++++++++++++++++++++++++++++---- 4 files changed, 179 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index c140a0a..5d5b709 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,52 @@ env SSE_LOCAL=true FIRECRAWL_API_KEY=fc-YOUR_API_KEY npx -y firecrawl-mcp Use the url: http://localhost:3000/sse +### Running with HTTP Streamable Mode + +To run the server using HTTP Streamable transport: + +```bash +env HTTP_STREAMABLE_SERVER=true FIRECRAWL_API_KEY=fc-YOUR_API_KEY npx -y firecrawl-mcp +``` + +This will start the server on port 8080. Clients can connect to the MCP server using the endpoint: +``` +POST http://localhost:8080/{apiKey}/mcp +``` + +For MCP client integration, use the following configuration: + +```javascript +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'; + +// Create an MCP client +const client = new Client(); + +// Configure HTTP Streamable transport +const transport = new StreamableHTTPClientTransport({ + url: `http://localhost:8080/your-api-key/mcp`, +}); + +// Connect to the server +await client.connect(transport); + +// List available tools +const { tools } = await client.listTools(); +console.log('Available tools:', tools.map(t => t.name)); + +// Call tool example +const result = await client.callTool({ + name: 'firecrawl_scrape', + arguments: { + url: 'https://example.com', + formats: ['markdown'] + } +}); + +console.log('Scrape result:', result); +``` + ### Installing via Smithery (Legacy) To install Firecrawl for Claude Desktop automatically via [Smithery](https://smithery.ai/server/@mendableai/mcp-server-firecrawl): @@ -182,13 +228,17 @@ Optionally, you can add it to a file called `.vscode/mcp.json` in your workspace #### Optional Configuration -##### Retry Configuration - - `FIRECRAWL_RETRY_MAX_ATTEMPTS`: Maximum number of retry attempts (default: 3) - `FIRECRAWL_RETRY_INITIAL_DELAY`: Initial delay in milliseconds before first retry (default: 1000) - `FIRECRAWL_RETRY_MAX_DELAY`: Maximum delay in milliseconds between retries (default: 10000) - `FIRECRAWL_RETRY_BACKOFF_FACTOR`: Exponential backoff multiplier (default: 2) +##### Server Mode Configuration + +- `SSE_LOCAL`: Set to "true" to use Server-Sent Events (SSE) transport (default: false) +- `HTTP_STREAMABLE_SERVER`: Set to "true" to use HTTP Streamable transport (default: false) +- `CLOUD_SERVICE`: Set to "true" for cloud mode operation (default: false) + ##### Credit Usage Monitoring - `FIRECRAWL_CREDIT_WARNING_THRESHOLD`: Credit usage warning threshold (default: 1000) @@ -227,6 +277,18 @@ export FIRECRAWL_RETRY_MAX_ATTEMPTS=10 export FIRECRAWL_RETRY_INITIAL_DELAY=500 # Start with faster retries ``` +For HTTP Streamable server mode: + +```bash +# Run in HTTP Streamable mode +export HTTP_STREAMABLE_SERVER=true +export FIRECRAWL_API_KEY=your-api-key +export PORT=8080 # Optional: customize port (default: 8080) + +# Start the server +npx -y firecrawl-mcp +``` + ### Usage with Claude Desktop Add this to your `claude_desktop_config.json`: @@ -586,4 +648,4 @@ Thanks to MCP.so and Klavis AI for hosting and [@gstarwd](https://github.com/gst ## License -MIT License - see LICENSE file for details +MIT License - see LICENSE file for details \ No newline at end of file diff --git a/package.json b/package.json index 515c90e..03e5a8f 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "license": "MIT", "dependencies": { "@mendable/firecrawl-js": "^1.19.0", - "@modelcontextprotocol/sdk": "^1.4.1", + "@modelcontextprotocol/sdk": "^1.11.1", "dotenv": "^16.4.7", "express": "^5.1.0", "shx": "^0.3.4", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 296b2f1..ce19a4d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12,8 +12,8 @@ importers: specifier: ^1.19.0 version: 1.19.0(ws@8.18.1) '@modelcontextprotocol/sdk': - specifier: ^1.4.1 - version: 1.6.1 + specifier: ^1.11.1 + version: 1.11.1 dotenv: specifier: ^16.4.7 version: 16.4.7 @@ -360,8 +360,8 @@ packages: '@mendable/firecrawl-js@1.19.0': resolution: {integrity: sha512-T0mEBVFyOMQkxLjq7QdXxxtlPJl2tpcl+SpusLSo4rngn/Nv/drJv3krjlN+d1isrCz/PZ6xqU4Sf5LLvuIT2g==} - '@modelcontextprotocol/sdk@1.6.1': - resolution: {integrity: sha512-oxzMzYCkZHMntzuyerehK3fV6A2Kwh5BD6CGEJSVDU2QNEhfLOptf2X7esQgaHZXHZY0oHmMsOtIDLP71UJXgA==} + '@modelcontextprotocol/sdk@1.11.1': + resolution: {integrity: sha512-9LfmxKTb1v+vUS1/emSk1f5ePmTLkb9Le9AxOB5T0XM59EUumwcS45z05h7aiZx3GI0Bl7mjb3FMEglYj+acuQ==} engines: {node: '>=18'} '@nodelib/fs.scandir@2.1.5': @@ -1595,8 +1595,8 @@ packages: resolution: {integrity: sha512-saLsH7WeYYPiD25LDuLRRY/i+6HaPYr6G1OUlN39otzkSTxKnubR9RTxS3/Kk50s1g2JTgFwWQDQyplC5/SHZg==} engines: {node: '>= 6'} - pkce-challenge@4.1.0: - resolution: {integrity: sha512-ZBmhE1C9LcPoH9XZSdwiPtbPHZROwAnMy+kIFQVrnMCxY4Cudlz3gBOpzilgc0jOgRaiT3sIWfpMomW2ar2orQ==} + pkce-challenge@5.0.0: + resolution: {integrity: sha512-ueGLflrrnvwB3xuo/uGob5pd5FN7l0MsLf0Z87o/UQmRtwjvfylfc9MurIxRAWywCYTgrvpXBcqjV4OfCYGCIQ==} engines: {node: '>=16.20.0'} pkg-dir@4.2.0: @@ -2424,14 +2424,15 @@ snapshots: - debug - ws - '@modelcontextprotocol/sdk@1.6.1': + '@modelcontextprotocol/sdk@1.11.1': dependencies: content-type: 1.0.5 cors: 2.8.5 + cross-spawn: 7.0.6 eventsource: 3.0.5 express: 5.1.0 express-rate-limit: 7.5.0(express@5.1.0) - pkce-challenge: 4.1.0 + pkce-challenge: 5.0.0 raw-body: 3.0.0 zod: 3.24.2 zod-to-json-schema: 3.24.3(zod@3.24.2) @@ -3898,7 +3899,7 @@ snapshots: pirates@4.0.6: {} - pkce-challenge@4.1.0: {} + pkce-challenge@5.0.0: {} pkg-dir@4.2.0: dependencies: diff --git a/src/index.ts b/src/index.ts index 5134c1e..3679e47 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,6 +7,7 @@ import { Tool, CallToolRequestSchema, ListToolsRequestSchema, + isInitializeRequest, } from '@modelcontextprotocol/sdk/types.js'; import FirecrawlApp, { type ScrapeParams, @@ -17,6 +18,8 @@ import FirecrawlApp, { import express, { Request, Response } from 'express'; import dotenv from 'dotenv'; +import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js'; +import { randomUUID } from 'node:crypto'; dotenv.config(); @@ -768,7 +771,7 @@ async function withRetry( if (isRateLimit && attempt < CONFIG.retry.maxAttempts) { const delayMs = Math.min( CONFIG.retry.initialDelay * - Math.pow(CONFIG.retry.backoffFactor, attempt - 1), + Math.pow(CONFIG.retry.backoffFactor, attempt - 1), CONFIG.retry.maxDelay ); @@ -973,9 +976,8 @@ Status: ${response.status} Progress: ${response.completed}/${response.total} Credits Used: ${response.creditsUsed} Expires At: ${response.expiresAt} -${ - response.data.length > 0 ? '\nResults:\n' + formatResults(response.data) : '' -}`; +${response.data.length > 0 ? '\nResults:\n' + formatResults(response.data) : '' + }`; return { content: [{ type: 'text', text: trimResponseText(status) }], isError: false, @@ -1255,9 +1257,8 @@ ${result.markdown ? `\nContent:\n${result.markdown}` : ''}` } catch (error) { // Log detailed error information safeLog('error', { - message: `Request failed: ${ - error instanceof Error ? error.message : String(error) - }`, + message: `Request failed: ${error instanceof Error ? error.message : String(error) + }`, tool: request.params.name, arguments: request.params.arguments, timestamp: new Date().toISOString(), @@ -1361,6 +1362,96 @@ async function runSSELocalServer() { } } + +async function runHTTPStreamableServer() { + const app = express(); + app.use(express.json()); + + const transports: { [sessionId: string]: StreamableHTTPServerTransport } = {}; + + // 单一端点处理所有MCP请求 + app.all('/:apiKey/mcp', async (req: Request, res: Response) => { + + try { + const sessionId = req.headers['mcp-session-id'] as string | undefined; + let transport: StreamableHTTPServerTransport; + + if (sessionId && transports[sessionId]) { + transport = transports[sessionId]; + } else if (!sessionId && req.method === 'POST' && isInitializeRequest(req.body)) { + + transport = new StreamableHTTPServerTransport({ + sessionIdGenerator: () => { + const id = randomUUID(); + return id; + }, + onsessioninitialized: (sid: string) => { + transports[sid] = transport; + } + }); + + transport.onclose = () => { + const sid = transport.sessionId; + if (sid && transports[sid]) { + delete transports[sid]; + } + }; + console.log('Creating server instance'); + console.log('Connecting transport to server'); + await server.connect(transport); + + await transport.handleRequest(req, res, req.body); + return; + } else { + res.status(400).json({ + jsonrpc: '2.0', + error: { + code: -32000, + message: 'Invalid or missing session ID', + }, + id: null, + }); + return; + } + + await transport.handleRequest(req, res, req.body); + } catch (error) { + if (!res.headersSent) { + res.status(500).json({ + jsonrpc: '2.0', + error: { + code: -32603, + message: 'Internal server error', + }, + id: null, + }); + } + } + }); + + const PORT = 8080; + const appServer = app.listen(PORT, () => { + console.log(`MCP Streamable HTTP Server listening on port ${PORT}`); + }); + + process.on('SIGINT', async () => { + console.log('Shutting down server...'); + for (const sessionId in transports) { + try { + console.log(`Closing transport for session ${sessionId}`); + await transports[sessionId].close(); + delete transports[sessionId]; + } catch (error) { + console.error(`Error closing transport for session ${sessionId}:`, error); + } + } + appServer.close(() => { + console.log('Server shutdown complete'); + process.exit(0); + }); + }); +} + async function runSSECloudServer() { const transports: { [sessionId: string]: SSEServerTransport } = {}; const app = express(); @@ -1435,6 +1526,11 @@ if (process.env.CLOUD_SERVICE === 'true') { console.error('Fatal error running server:', error); process.exit(1); }); +} else if (process.env.HTTP_STREAMABLE_SERVER === 'true') { + runHTTPStreamableServer().catch((error: any) => { + console.error('Fatal error running server:', error); + process.exit(1); + }); } else { runLocalServer().catch((error: any) => { console.error('Fatal error running server:', error); From 62c1fb3c5c4ee3fbc2989c9e3b3ce17435cfb096 Mon Sep 17 00:00:00 2001 From: Dangoron Date: Wed, 21 May 2025 11:59:44 +0800 Subject: [PATCH 2/2] Update comment: translate to English for single endpoint handling all MCP requests --- src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index.ts b/src/index.ts index 3679e47..76804e3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1369,7 +1369,7 @@ async function runHTTPStreamableServer() { const transports: { [sessionId: string]: StreamableHTTPServerTransport } = {}; - // 单一端点处理所有MCP请求 + // A single endpoint handles all MCP requests. app.all('/:apiKey/mcp', async (req: Request, res: Response) => { try {