diff --git a/package-lock.json b/package-lock.json index 95f187d..7697ebe 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,7 @@ "name": "agent-workflow-builder", "version": "1.0.0", "dependencies": { + "@mendable/firecrawl-js": "^4.4.0", "@types/node": "^16.18.0", "@types/react": "^18.2.0", "@types/react-dom": "^18.2.0", @@ -26,8 +27,7 @@ "typescript": "^4.9.0", "uuid": "^9.0.0", "zustand": "^4.4.1" - }, - "devDependencies": {} + } }, "node_modules/@alloc/quick-lru": { "version": "5.2.0", @@ -2953,6 +2953,21 @@ "integrity": "sha512-Vo+PSpZG2/fmgmiNzYK9qWRh8h/CHrwD0mo1h1DzL4yzHNSfWYujGTYsWGreD000gcgmZ7K4Ys6Tx9TxtsKdDw==", "license": "MIT" }, + "node_modules/@mendable/firecrawl-js": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/@mendable/firecrawl-js/-/firecrawl-js-4.4.0.tgz", + "integrity": "sha512-VJj3t8vbjB8TH08WXE6cmk8ml1qDU05J34Zwgvfs7M5TAQOWwZDr5RIYggUtS2P5whRFPyaYlBbKBGkwImtDVA==", + "license": "MIT", + "dependencies": { + "axios": "^1.12.2", + "typescript-event-target": "^1.1.1", + "zod": "^3.23.8", + "zod-to-json-schema": "^3.23.0" + }, + "engines": { + "node": ">=22.0.0" + } + }, "node_modules/@nicolo-ribaudo/eslint-scope-5-internals": { "version": "5.1.1-v1", "resolved": "https://registry.npmjs.org/@nicolo-ribaudo/eslint-scope-5-internals/-/eslint-scope-5-internals-5.1.1-v1.tgz", @@ -5192,6 +5207,33 @@ "node": ">=4" } }, + "node_modules/axios": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.12.2.tgz", + "integrity": "sha512-vMJzPewAlRyOgxV2dU0Cuz2O8zzzx9VYtbJOaBgXFeLc4IV/Eg50n4LowmehOOR61S8ZMpc2K5Sa7g6A4jfkUw==", + "license": "MIT", + "dependencies": { + "follow-redirects": "^1.15.6", + "form-data": "^4.0.4", + "proxy-from-env": "^1.1.0" + } + }, + "node_modules/axios/node_modules/form-data": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.4.tgz", + "integrity": "sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.2", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/axobject-query": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-4.1.0.tgz", @@ -14027,6 +14069,12 @@ "node": ">= 0.10" } }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "license": "MIT" + }, "node_modules/psl": { "version": "1.15.0", "resolved": "https://registry.npmjs.org/psl/-/psl-1.15.0.tgz", @@ -16817,6 +16865,12 @@ "node": ">=4.2.0" } }, + "node_modules/typescript-event-target": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/typescript-event-target/-/typescript-event-target-1.1.1.tgz", + "integrity": "sha512-dFSOFBKV6uwaloBCCUhxlD3Pr/P1a/tJdcmPrTXCHlEFD3faj0mztjcGn6VBAhQ0/Bdy8K3VWrrqwbt/ffsYsg==", + "license": "MIT" + }, "node_modules/unbox-primitive": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.1.0.tgz", @@ -17976,6 +18030,24 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-to-json-schema": { + "version": "3.24.6", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.24.6.tgz", + "integrity": "sha512-h/z3PKvcTcTetyjl1fkj79MHNEjm+HpD6NXheWjzOekY7kV+lwDYnHw+ivHkijnCSMz1yJaWBD9vu/Fcmk+vEg==", + "license": "ISC", + "peerDependencies": { + "zod": "^3.24.1" + } + }, "node_modules/zustand": { "version": "4.5.7", "resolved": "https://registry.npmjs.org/zustand/-/zustand-4.5.7.tgz", diff --git a/package.json b/package.json index 9437144..7be7271 100644 --- a/package.json +++ b/package.json @@ -4,6 +4,7 @@ "description": "Open-source agent workflow builder using React Flow and AI", "private": true, "dependencies": { + "@mendable/firecrawl-js": "^4.4.0", "@types/node": "^16.18.0", "@types/react": "^18.2.0", "@types/react-dom": "^18.2.0", diff --git a/src/components/NodeConfiguration.tsx b/src/components/NodeConfiguration.tsx index ac84301..f35001a 100644 --- a/src/components/NodeConfiguration.tsx +++ b/src/components/NodeConfiguration.tsx @@ -11,7 +11,7 @@ interface NodeConfigurationProps { const nodeTypeConfigs = { webScraping: { - title: "Web Scraping Configuration", + title: "Web Scraping Configuration (Firecrawl AI)", fields: [ { key: "url", @@ -20,16 +20,47 @@ const nodeTypeConfigs = { placeholder: "https://example.com", }, { - key: "selector", - label: "CSS Selector", + key: "formats", + label: "Output Formats", + type: "select", + options: ["markdown", "html", "text", "summary", "links", "images"], + multiple: true, + }, + { + key: "onlyMainContent", + label: "Only Main Content", + type: "checkbox", + defaultValue: true, + }, + { + key: "includeTags", + label: "Include Tags (comma-separated)", type: "text", - placeholder: ".content", + placeholder: "article, main, .content", + }, + { + key: "excludeTags", + label: "Exclude Tags (comma-separated)", + type: "text", + placeholder: "nav, footer, .ads", }, { key: "maxLength", label: "Max Length", type: "number", - placeholder: "1000", + placeholder: "5000", + }, + { + key: "waitFor", + label: "Wait For (ms)", + type: "number", + placeholder: "2000", + }, + { + key: "timeout", + label: "Timeout (ms)", + type: "number", + placeholder: "30000", }, ], }, @@ -46,10 +77,7 @@ const nodeTypeConfigs = { key: "model", label: "Model", type: "select", - options: [ - "deepseek-chat", - "deepseek-reasoner" - ], + options: ["deepseek-chat", "deepseek-reasoner"], }, { key: "temperature", @@ -117,10 +145,7 @@ const nodeTypeConfigs = { key: "model", label: "Model", type: "select", - options: [ - "deepseek-chat", - "deepseek-reasoner" - ], + options: ["deepseek-chat", "deepseek-reasoner"], }, ], }, @@ -186,7 +211,7 @@ export const NodeConfiguration: React.FC = ({ }; const renderField = (field: any) => { - const value = currentData.config[field.key] || ""; + const value = currentData.config[field.key] || field.defaultValue || ""; switch (field.type) { case "textarea": @@ -200,6 +225,38 @@ export const NodeConfiguration: React.FC = ({ /> ); case "select": + if (field.multiple) { + return ( +
+ {field.options.map((option: string) => ( + + ))} +
+ ); + } return ( ); + case "checkbox": + return ( + + ); case "number": return ( = ({ onClose }) => { const [apiKey, setApiKey] = useState(""); + const [firecrawlApiKey, setFirecrawlApiKey] = useState(""); const [isValidating, setIsValidating] = useState(false); const [validationStatus, setValidationStatus] = useState< "idle" | "success" | "error" @@ -14,8 +15,8 @@ export const OpenAIConfig: React.FC = ({ onClose }) => { const [errorMessage, setErrorMessage] = useState(""); const handleSave = async () => { - if (!apiKey.trim()) { - setErrorMessage("Please enter an API key"); + if (!apiKey.trim() && !firecrawlApiKey.trim()) { + setErrorMessage("Please enter at least one API key"); setValidationStatus("error"); return; } @@ -24,25 +25,48 @@ export const OpenAIConfig: React.FC = ({ onClose }) => { setValidationStatus("idle"); try { - // In a real app, you'd validate the API key with a backend service - // For now, we'll just check if it looks like a valid API key format - if (apiKey.startsWith("sk-") && apiKey.length > 20) { - // Save to localStorage for this session (prioritize DeepSeek) - localStorage.setItem("deepseek_api_key", apiKey); - localStorage.setItem("openai_api_key", apiKey); // Also save as OpenAI for compatibility + let hasValidKey = false; + + // Validate AI API key (DeepSeek/OpenAI) + if (apiKey.trim()) { + if (apiKey.startsWith("sk-") && apiKey.length > 20) { + localStorage.setItem("deepseek_api_key", apiKey); + localStorage.setItem("openai_api_key", apiKey); + hasValidKey = true; + } else { + setErrorMessage( + "Invalid AI API key format. API keys should start with 'sk-'" + ); + setValidationStatus("error"); + setIsValidating(false); + return; + } + } + + // Validate Firecrawl API key + if (firecrawlApiKey.trim()) { + if (firecrawlApiKey.startsWith("fc-") && firecrawlApiKey.length > 20) { + localStorage.setItem("firecrawl_api_key", firecrawlApiKey); + hasValidKey = true; + } else { + setErrorMessage( + "Invalid Firecrawl API key format. API keys should start with 'fc-'" + ); + setValidationStatus("error"); + setIsValidating(false); + return; + } + } + + if (hasValidKey) { setValidationStatus("success"); setTimeout(() => { onClose(); - window.location.reload(); // Reload to pick up the new API key + window.location.reload(); // Reload to pick up the new API keys }, 1500); - } else { - setErrorMessage( - "Invalid API key format. API keys should start with 'sk-'" - ); - setValidationStatus("error"); } } catch (error) { - setErrorMessage("Failed to validate API key"); + setErrorMessage("Failed to validate API keys"); setValidationStatus("error"); } finally { setIsValidating(false); @@ -50,8 +74,8 @@ export const OpenAIConfig: React.FC = ({ onClose }) => { }; const handleTest = async () => { - if (!apiKey.trim()) { - setErrorMessage("Please enter an API key first"); + if (!apiKey.trim() && !firecrawlApiKey.trim()) { + setErrorMessage("Please enter at least one API key to test"); setValidationStatus("error"); return; } @@ -60,34 +84,63 @@ export const OpenAIConfig: React.FC = ({ onClose }) => { setValidationStatus("idle"); try { - // Test the API key by making a simple call to DeepSeek first, then OpenAI - let response = await fetch("https://api.deepseek.com/v1/models", { - headers: { - Authorization: `Bearer ${apiKey}`, - "Content-Type": "application/json", - }, - }); + let aiKeyValid = false; + let firecrawlKeyValid = false; - // If DeepSeek fails, try OpenAI - if (!response.ok) { - response = await fetch("https://api.openai.com/v1/models", { + // Test AI API key if provided + if (apiKey.trim()) { + let response = await fetch("https://api.deepseek.com/v1/models", { headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json", }, }); + + // If DeepSeek fails, try OpenAI + if (!response.ok) { + response = await fetch("https://api.openai.com/v1/models", { + headers: { + Authorization: `Bearer ${apiKey}`, + "Content-Type": "application/json", + }, + }); + } + + aiKeyValid = response.ok; } - if (response.ok) { + // Test Firecrawl API key if provided + if (firecrawlApiKey.trim()) { + try { + const response = await fetch("https://api.firecrawl.dev/v0/scrape", { + method: "POST", + headers: { + Authorization: `Bearer ${firecrawlApiKey}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + url: "https://example.com", + formats: ["markdown"], + }), + }); + firecrawlKeyValid = response.ok; + } catch (error) { + firecrawlKeyValid = false; + } + } + + if (aiKeyValid || firecrawlKeyValid) { setValidationStatus("success"); setErrorMessage(""); } else { - setErrorMessage("API key is invalid or has insufficient permissions"); + setErrorMessage( + "One or more API keys are invalid or have insufficient permissions" + ); setValidationStatus("error"); } } catch (error) { setErrorMessage( - "Failed to test API key. Please check your internet connection." + "Failed to test API keys. Please check your internet connection." ); setValidationStatus("error"); } finally { @@ -102,7 +155,7 @@ export const OpenAIConfig: React.FC = ({ onClose }) => {

- AI API Configuration + API Configuration

- - - 

-
-    
-  
-