Add Firecrawl API integration and enhance web scraping capabilities

- Introduced Firecrawl API support in the workflow builder, allowing users to scrape web content. - Updated NodeConfiguration component to include new fields for Firecrawl settings, such as output formats and content filtering options. - Enhanced OpenAIConfig component to manage both OpenAI and Firecrawl API keys. - Created firecrawlService to handle API interactions and scraping logic. - Modified workflowStore to process web scraping nodes using Firecrawl, improving error handling and fallback responses. - Removed obsolete test-localStorage.html file as part of project cleanup.
2026-07-22 02:01:56 +02:00 · 2025-10-15 01:22:01 -04:00
parent 9c1a0d8661
commit 6244995163
7 changed files with 480 additions and 155 deletions
@@ -8,6 +8,7 @@
      "name": "agent-workflow-builder",
      "version": "1.0.0",
      "dependencies": {
+        "@mendable/firecrawl-js": "^4.4.0",
        "@types/node": "^16.18.0",
        "@types/react": "^18.2.0",
        "@types/react-dom": "^18.2.0",
@@ -26,8 +27,7 @@
        "typescript": "^4.9.0",
        "uuid": "^9.0.0",
        "zustand": "^4.4.1"
-      },
-      "devDependencies": {}
+      }
    },
    "node_modules/@alloc/quick-lru": {
      "version": "5.2.0",
@@ -2953,6 +2953,21 @@
      "integrity": "sha512-Vo+PSpZG2/fmgmiNzYK9qWRh8h/CHrwD0mo1h1DzL4yzHNSfWYujGTYsWGreD000gcgmZ7K4Ys6Tx9TxtsKdDw==",
      "license": "MIT"
    },
+    "node_modules/@mendable/firecrawl-js": {
+      "version": "4.4.0",
+      "resolved": "https://registry.npmjs.org/@mendable/firecrawl-js/-/firecrawl-js-4.4.0.tgz",
+      "integrity": "sha512-VJj3t8vbjB8TH08WXE6cmk8ml1qDU05J34Zwgvfs7M5TAQOWwZDr5RIYggUtS2P5whRFPyaYlBbKBGkwImtDVA==",
+      "license": "MIT",
+      "dependencies": {
+        "axios": "^1.12.2",
+        "typescript-event-target": "^1.1.1",
+        "zod": "^3.23.8",
+        "zod-to-json-schema": "^3.23.0"
+      },
+      "engines": {
+        "node": ">=22.0.0"
+      }
+    },
    "node_modules/@nicolo-ribaudo/eslint-scope-5-internals": {
      "version": "5.1.1-v1",
      "resolved": "https://registry.npmjs.org/@nicolo-ribaudo/eslint-scope-5-internals/-/eslint-scope-5-internals-5.1.1-v1.tgz",
@@ -5192,6 +5207,33 @@
        "node": ">=4"
      }
    },
+    "node_modules/axios": {
+      "version": "1.12.2",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-1.12.2.tgz",
+      "integrity": "sha512-vMJzPewAlRyOgxV2dU0Cuz2O8zzzx9VYtbJOaBgXFeLc4IV/Eg50n4LowmehOOR61S8ZMpc2K5Sa7g6A4jfkUw==",
+      "license": "MIT",
+      "dependencies": {
+        "follow-redirects": "^1.15.6",
+        "form-data": "^4.0.4",
+        "proxy-from-env": "^1.1.0"
+      }
+    },
+    "node_modules/axios/node_modules/form-data": {
+      "version": "4.0.4",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.4.tgz",
+      "integrity": "sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==",
+      "license": "MIT",
+      "dependencies": {
+        "asynckit": "^0.4.0",
+        "combined-stream": "^1.0.8",
+        "es-set-tostringtag": "^2.1.0",
+        "hasown": "^2.0.2",
+        "mime-types": "^2.1.12"
+      },
+      "engines": {
+        "node": ">= 6"
+      }
+    },
    "node_modules/axobject-query": {
      "version": "4.1.0",
      "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-4.1.0.tgz",
@@ -14027,6 +14069,12 @@
        "node": ">= 0.10"
      }
    },
+    "node_modules/proxy-from-env": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
+      "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==",
+      "license": "MIT"
+    },
    "node_modules/psl": {
      "version": "1.15.0",
      "resolved": "https://registry.npmjs.org/psl/-/psl-1.15.0.tgz",
@@ -16817,6 +16865,12 @@
        "node": ">=4.2.0"
      }
    },
+    "node_modules/typescript-event-target": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/typescript-event-target/-/typescript-event-target-1.1.1.tgz",
+      "integrity": "sha512-dFSOFBKV6uwaloBCCUhxlD3Pr/P1a/tJdcmPrTXCHlEFD3faj0mztjcGn6VBAhQ0/Bdy8K3VWrrqwbt/ffsYsg==",
+      "license": "MIT"
+    },
    "node_modules/unbox-primitive": {
      "version": "1.1.0",
      "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.1.0.tgz",
@@ -17976,6 +18030,24 @@
        "url": "https://github.com/sponsors/sindresorhus"
      }
    },
+    "node_modules/zod": {
+      "version": "3.25.76",
+      "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz",
+      "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==",
+      "license": "MIT",
+      "funding": {
+        "url": "https://github.com/sponsors/colinhacks"
+      }
+    },
+    "node_modules/zod-to-json-schema": {
+      "version": "3.24.6",
+      "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.24.6.tgz",
+      "integrity": "sha512-h/z3PKvcTcTetyjl1fkj79MHNEjm+HpD6NXheWjzOekY7kV+lwDYnHw+ivHkijnCSMz1yJaWBD9vu/Fcmk+vEg==",
+      "license": "ISC",
+      "peerDependencies": {
+        "zod": "^3.24.1"
+      }
+    },
    "node_modules/zustand": {
      "version": "4.5.7",
      "resolved": "https://registry.npmjs.org/zustand/-/zustand-4.5.7.tgz",
@@ -4,6 +4,7 @@
  "description": "Open-source agent workflow builder using React Flow and AI",
  "private": true,
  "dependencies": {
+    "@mendable/firecrawl-js": "^4.4.0",
    "@types/node": "^16.18.0",
    "@types/react": "^18.2.0",
    "@types/react-dom": "^18.2.0",
@@ -11,7 +11,7 @@ interface NodeConfigurationProps {

 const nodeTypeConfigs = {
  webScraping: {
-    title: "Web Scraping Configuration",
+    title: "Web Scraping Configuration (Firecrawl AI)",
    fields: [
      {
        key: "url",
@@ -20,16 +20,47 @@ const nodeTypeConfigs = {
        placeholder: "https://example.com",
      },
      {
-        key: "selector",
-        label: "CSS Selector",
+        key: "formats",
+        label: "Output Formats",
+        type: "select",
+        options: ["markdown", "html", "text", "summary", "links", "images"],
+        multiple: true,
+      },
+      {
+        key: "onlyMainContent",
+        label: "Only Main Content",
+        type: "checkbox",
+        defaultValue: true,
+      },
+      {
+        key: "includeTags",
+        label: "Include Tags (comma-separated)",
        type: "text",
-        placeholder: ".content",
+        placeholder: "article, main, .content",
+      },
+      {
+        key: "excludeTags",
+        label: "Exclude Tags (comma-separated)",
+        type: "text",
+        placeholder: "nav, footer, .ads",
      },
      {
        key: "maxLength",
        label: "Max Length",
        type: "number",
-        placeholder: "1000",
+        placeholder: "5000",
+      },
+      {
+        key: "waitFor",
+        label: "Wait For (ms)",
+        type: "number",
+        placeholder: "2000",
+      },
+      {
+        key: "timeout",
+        label: "Timeout (ms)",
+        type: "number",
+        placeholder: "30000",
      },
    ],
  },
@@ -46,10 +77,7 @@ const nodeTypeConfigs = {
        key: "model",
        label: "Model",
        type: "select",
-        options: [
-          "deepseek-chat",
-          "deepseek-reasoner"
-        ],
+        options: ["deepseek-chat", "deepseek-reasoner"],
      },
      {
        key: "temperature",
@@ -117,10 +145,7 @@ const nodeTypeConfigs = {
        key: "model",
        label: "Model",
        type: "select",
-        options: [
-          "deepseek-chat",
-          "deepseek-reasoner"
-        ],
+        options: ["deepseek-chat", "deepseek-reasoner"],
      },
    ],
  },
@@ -186,7 +211,7 @@ export const NodeConfiguration: React.FC<NodeConfigurationProps> = ({
  };

  const renderField = (field: any) => {
-    const value = currentData.config[field.key] || "";
+    const value = currentData.config[field.key] || field.defaultValue || "";

    switch (field.type) {
      case "textarea":
@@ -200,6 +225,38 @@ export const NodeConfiguration: React.FC<NodeConfigurationProps> = ({
          />
        );
      case "select":
+        if (field.multiple) {
+          return (
+            <div className="space-y-2">
+              {field.options.map((option: string) => (
+                <label key={option} className="flex items-center space-x-2">
+                  <input
+                    type="checkbox"
+                    checked={
+                      Array.isArray(value) ? value.includes(option) : false
+                    }
+                    onChange={(e) => {
+                      const currentValues = Array.isArray(value) ? value : [];
+                      if (e.target.checked) {
+                        handleConfigChange(field.key, [
+                          ...currentValues,
+                          option,
+                        ]);
+                      } else {
+                        handleConfigChange(
+                          field.key,
+                          currentValues.filter((v: string) => v !== option)
+                        );
+                      }
+                    }}
+                    className="rounded border-gray-300 text-primary-600 focus:ring-primary-500"
+                  />
+                  <span className="text-sm text-gray-700">{option}</span>
+                </label>
+              ))}
+            </div>
+          );
+        }
        return (
          <select
            value={value}
@@ -214,6 +271,18 @@ export const NodeConfiguration: React.FC<NodeConfigurationProps> = ({
            ))}
          </select>
        );
+      case "checkbox":
+        return (
+          <label className="flex items-center space-x-2">
+            <input
+              type="checkbox"
+              checked={value}
+              onChange={(e) => handleConfigChange(field.key, e.target.checked)}
+              className="rounded border-gray-300 text-primary-600 focus:ring-primary-500"
+            />
+            <span className="text-sm text-gray-700">Enable {field.label}</span>
+          </label>
+        );
      case "number":
        return (
          <input
@@ -7,6 +7,7 @@ interface OpenAIConfigProps {

 export const OpenAIConfig: React.FC<OpenAIConfigProps> = ({ onClose }) => {
  const [apiKey, setApiKey] = useState("");
+  const [firecrawlApiKey, setFirecrawlApiKey] = useState("");
  const [isValidating, setIsValidating] = useState(false);
  const [validationStatus, setValidationStatus] = useState<
    "idle" | "success" | "error"
@@ -14,8 +15,8 @@ export const OpenAIConfig: React.FC<OpenAIConfigProps> = ({ onClose }) => {
  const [errorMessage, setErrorMessage] = useState("");

  const handleSave = async () => {
-    if (!apiKey.trim()) {
-      setErrorMessage("Please enter an API key");
+    if (!apiKey.trim() && !firecrawlApiKey.trim()) {
+      setErrorMessage("Please enter at least one API key");
      setValidationStatus("error");
      return;
    }
@@ -24,25 +25,48 @@ export const OpenAIConfig: React.FC<OpenAIConfigProps> = ({ onClose }) => {
    setValidationStatus("idle");

    try {
-      // In a real app, you'd validate the API key with a backend service
-      // For now, we'll just check if it looks like a valid API key format
-      if (apiKey.startsWith("sk-") && apiKey.length > 20) {
-        // Save to localStorage for this session (prioritize DeepSeek)
-        localStorage.setItem("deepseek_api_key", apiKey);
-        localStorage.setItem("openai_api_key", apiKey); // Also save as OpenAI for compatibility
+      let hasValidKey = false;
+
+      // Validate AI API key (DeepSeek/OpenAI)
+      if (apiKey.trim()) {
+        if (apiKey.startsWith("sk-") && apiKey.length > 20) {
+          localStorage.setItem("deepseek_api_key", apiKey);
+          localStorage.setItem("openai_api_key", apiKey);
+          hasValidKey = true;
+        } else {
+          setErrorMessage(
+            "Invalid AI API key format. API keys should start with 'sk-'"
+          );
+          setValidationStatus("error");
+          setIsValidating(false);
+          return;
+        }
+      }
+
+      // Validate Firecrawl API key
+      if (firecrawlApiKey.trim()) {
+        if (firecrawlApiKey.startsWith("fc-") && firecrawlApiKey.length > 20) {
+          localStorage.setItem("firecrawl_api_key", firecrawlApiKey);
+          hasValidKey = true;
+        } else {
+          setErrorMessage(
+            "Invalid Firecrawl API key format. API keys should start with 'fc-'"
+          );
+          setValidationStatus("error");
+          setIsValidating(false);
+          return;
+        }
+      }
+
+      if (hasValidKey) {
        setValidationStatus("success");
        setTimeout(() => {
          onClose();
-          window.location.reload(); // Reload to pick up the new API key
+          window.location.reload(); // Reload to pick up the new API keys
        }, 1500);
-      } else {
-        setErrorMessage(
-          "Invalid API key format. API keys should start with 'sk-'"
-        );
-        setValidationStatus("error");
      }
    } catch (error) {
-      setErrorMessage("Failed to validate API key");
+      setErrorMessage("Failed to validate API keys");
      setValidationStatus("error");
    } finally {
      setIsValidating(false);
@@ -50,8 +74,8 @@ export const OpenAIConfig: React.FC<OpenAIConfigProps> = ({ onClose }) => {
  };

  const handleTest = async () => {
-    if (!apiKey.trim()) {
-      setErrorMessage("Please enter an API key first");
+    if (!apiKey.trim() && !firecrawlApiKey.trim()) {
+      setErrorMessage("Please enter at least one API key to test");
      setValidationStatus("error");
      return;
    }
@@ -60,34 +84,63 @@ export const OpenAIConfig: React.FC<OpenAIConfigProps> = ({ onClose }) => {
    setValidationStatus("idle");

    try {
-      // Test the API key by making a simple call to DeepSeek first, then OpenAI
-      let response = await fetch("https://api.deepseek.com/v1/models", {
-        headers: {
-          Authorization: `Bearer ${apiKey}`,
-          "Content-Type": "application/json",
-        },
-      });
+      let aiKeyValid = false;
+      let firecrawlKeyValid = false;

-      // If DeepSeek fails, try OpenAI
-      if (!response.ok) {
-        response = await fetch("https://api.openai.com/v1/models", {
+      // Test AI API key if provided
+      if (apiKey.trim()) {
+        let response = await fetch("https://api.deepseek.com/v1/models", {
          headers: {
            Authorization: `Bearer ${apiKey}`,
            "Content-Type": "application/json",
          },
        });
+
+        // If DeepSeek fails, try OpenAI
+        if (!response.ok) {
+          response = await fetch("https://api.openai.com/v1/models", {
+            headers: {
+              Authorization: `Bearer ${apiKey}`,
+              "Content-Type": "application/json",
+            },
+          });
+        }
+
+        aiKeyValid = response.ok;
      }

-      if (response.ok) {
+      // Test Firecrawl API key if provided
+      if (firecrawlApiKey.trim()) {
+        try {
+          const response = await fetch("https://api.firecrawl.dev/v0/scrape", {
+            method: "POST",
+            headers: {
+              Authorization: `Bearer ${firecrawlApiKey}`,
+              "Content-Type": "application/json",
+            },
+            body: JSON.stringify({
+              url: "https://example.com",
+              formats: ["markdown"],
+            }),
+          });
+          firecrawlKeyValid = response.ok;
+        } catch (error) {
+          firecrawlKeyValid = false;
+        }
+      }
+
+      if (aiKeyValid || firecrawlKeyValid) {
        setValidationStatus("success");
        setErrorMessage("");
      } else {
-        setErrorMessage("API key is invalid or has insufficient permissions");
+        setErrorMessage(
+          "One or more API keys are invalid or have insufficient permissions"
+        );
        setValidationStatus("error");
      }
    } catch (error) {
      setErrorMessage(
-        "Failed to test API key. Please check your internet connection."
+        "Failed to test API keys. Please check your internet connection."
      );
      setValidationStatus("error");
    } finally {
@@ -102,7 +155,7 @@ export const OpenAIConfig: React.FC<OpenAIConfigProps> = ({ onClose }) => {
          <div className="flex items-center space-x-2">
            <Key className="w-5 h-5 text-gray-600" />
            <h3 className="text-lg font-semibold text-gray-900">
-              AI API Configuration
+              API Configuration
            </h3>
          </div>
          <button
@@ -116,26 +169,19 @@ export const OpenAIConfig: React.FC<OpenAIConfigProps> = ({ onClose }) => {
        <div className="p-4 space-y-4">
          <div className="bg-blue-50 border border-blue-200 rounded-md p-3">
            <p className="text-sm text-blue-800">
-              To use real AI responses, you need to configure your API key. Get
-              your API key from{" "}
-              <a
-                href="https://platform.deepseek.com/api_keys"
-                target="_blank"
-                rel="noopener noreferrer"
-                className="text-blue-600 underline hover:text-blue-800"
-              >
-                DeepSeek Platform
-              </a>{" "}
-              or{" "}
-              <a
-                href="https://platform.openai.com/api-keys"
-                target="_blank"
-                rel="noopener noreferrer"
-                className="text-blue-600 underline hover:text-blue-800"
-              >
-                OpenAI Platform
-              </a>
+              Configure your API keys to enable full functionality. You need at
+              least one API key:
            </p>
+            <ul className="text-xs text-blue-700 mt-2 space-y-1">
+              <li>
+                • <strong>AI API Key:</strong> For LLM tasks and structured
+                output
+              </li>
+              <li>
+                • <strong>Firecrawl API Key:</strong> For web scraping
+                functionality
+              </li>
+            </ul>
          </div>

          <div>
@@ -149,6 +195,50 @@ export const OpenAIConfig: React.FC<OpenAIConfigProps> = ({ onClose }) => {
              placeholder="sk-... (DeepSeek or OpenAI key)"
              className="w-full p-2 border border-gray-300 rounded-md focus:ring-2 focus:ring-primary-500 focus:border-transparent"
            />
+            <div className="text-xs text-gray-500 mt-1">
+              Get your key from{" "}
+              <a
+                href="https://platform.deepseek.com/api_keys"
+                target="_blank"
+                rel="noopener noreferrer"
+                className="text-blue-600 underline hover:text-blue-800"
+              >
+                DeepSeek
+              </a>{" "}
+              or{" "}
+              <a
+                href="https://platform.openai.com/api-keys"
+                target="_blank"
+                rel="noopener noreferrer"
+                className="text-blue-600 underline hover:text-blue-800"
+              >
+                OpenAI
+              </a>
+            </div>
+          </div>
+
+          <div>
+            <label className="block text-sm font-medium text-gray-700 mb-1">
+              Firecrawl API Key (for Web Scraping)
+            </label>
+            <input
+              type="password"
+              value={firecrawlApiKey}
+              onChange={(e) => setFirecrawlApiKey(e.target.value)}
+              placeholder="fc-... (Firecrawl API key)"
+              className="w-full p-2 border border-gray-300 rounded-md focus:ring-2 focus:ring-primary-500 focus:border-transparent"
+            />
+            <div className="text-xs text-gray-500 mt-1">
+              Get your key from{" "}
+              <a
+                href="https://firecrawl.dev"
+                target="_blank"
+                rel="noopener noreferrer"
+                className="text-blue-600 underline hover:text-blue-800"
+              >
+                Firecrawl
+              </a>
+            </div>
          </div>

          {validationStatus === "error" && (
@@ -0,0 +1,123 @@
+import FirecrawlApp from "@mendable/firecrawl-js";
+
+// Get API key from environment or localStorage
+const getFirecrawlApiKey = (): string => {
+  return (
+    localStorage.getItem("firecrawl_api_key") ||
+    process.env.REACT_APP_FIRECRAWL_API_KEY ||
+    ""
+  );
+};
+
+export interface FirecrawlConfig {
+  url: string;
+  formats?: ("markdown" | "html" | "text" | "summary" | "links" | "images")[];
+  includeTags?: string[];
+  excludeTags?: string[];
+  onlyMainContent?: boolean;
+  maxLength?: number;
+  waitFor?: number;
+  timeout?: number;
+}
+
+export interface FirecrawlResponse {
+  success: boolean;
+  data?: {
+    content: string;
+    markdown?: string;
+    html?: string;
+    metadata?: {
+      title?: string;
+      description?: string;
+      language?: string;
+      sourceURL?: string;
+      createdAt?: string;
+    };
+  };
+  error?: string;
+}
+
+export const scrapeWithFirecrawl = async (
+  config: FirecrawlConfig
+): Promise<FirecrawlResponse> => {
+  try {
+    // Check if API key is configured
+    const apiKey = getFirecrawlApiKey();
+    if (!apiKey || apiKey === "your_firecrawl_api_key_here") {
+      throw new Error(
+        "Firecrawl API key not configured. Please configure your API key in the settings."
+      );
+    }
+
+    // Initialize Firecrawl client with the current API key
+    const firecrawl = new FirecrawlApp({
+      apiKey: apiKey,
+    });
+
+    // Prepare the scrape options according to Firecrawl v2 API
+    const scrapeOptions: any = {
+      formats: config.formats || ["markdown", "html"],
+      onlyMainContent: config.onlyMainContent !== false, // Default to true
+    };
+
+    if (config.includeTags && config.includeTags.length > 0) {
+      scrapeOptions.includeTags = config.includeTags;
+    }
+
+    if (config.excludeTags && config.excludeTags.length > 0) {
+      scrapeOptions.excludeTags = config.excludeTags;
+    }
+
+    if (config.waitFor) {
+      scrapeOptions.waitFor = config.waitFor;
+    }
+
+    if (config.timeout) {
+      scrapeOptions.timeout = config.timeout;
+    }
+
+    // Perform the scrape using the correct method name
+    const scrapeResult = await firecrawl.scrape(config.url, scrapeOptions);
+
+    if (!scrapeResult.success) {
+      throw new Error(scrapeResult.error || "Failed to scrape URL");
+    }
+
+    const data = scrapeResult.data;
+    let content = data.markdown || data.html || data.content || "";
+
+    // Apply max length if specified
+    if (config.maxLength && content.length > config.maxLength) {
+      content = content.substring(0, config.maxLength) + "...";
+    }
+
+    return {
+      success: true,
+      data: {
+        content,
+        markdown: data.markdown,
+        html: data.html,
+        metadata: {
+          title: data.metadata?.title,
+          description: data.metadata?.description,
+          language: data.metadata?.language,
+          sourceURL: config.url,
+          createdAt: new Date().toISOString(),
+        },
+      },
+    };
+  } catch (error) {
+    console.error("Firecrawl API Error:", error);
+
+    // Return a fallback response if API fails
+    return {
+      success: false,
+      error: error instanceof Error ? error.message : "Unknown error occurred",
+    };
+  }
+};
+
+export const getFirecrawlApiKeyStatus = (): boolean => {
+  const apiKey = getFirecrawlApiKey();
+  return !!(apiKey && apiKey !== "your_firecrawl_api_key_here");
+};
@@ -8,6 +8,10 @@ import {
 } from "../types";
 import { v4 as uuidv4 } from "uuid";
 import { callOpenAI, OpenAIConfig } from "../services/openaiService";
+import {
+  scrapeWithFirecrawl,
+  FirecrawlConfig,
+} from "../services/firecrawlService";

 // localStorage key for workflows
 const WORKFLOWS_STORAGE_KEY = "agent-workflow-builder-workflows";
@@ -463,16 +467,63 @@ const processWebScrapingNode = async (
 ) => {
  const config = node.data.config;
  const url = config.url || inputData;
-  const selector = config.selector || "body";
-  const maxLength = config.maxLength || 1000;

-  // In a real app, this would perform actual web scraping
-  const mockScrapedContent = `Scraped content from ${url} using selector "${selector}": ${inputData.substring(
-    0,
-    maxLength
-  )}...`;
+  // Prepare Firecrawl configuration
+  const firecrawlConfig: FirecrawlConfig = {
+    url,
+    formats: config.formats || ["markdown", "html"],
+    onlyMainContent: config.onlyMainContent !== false,
+    maxLength: config.maxLength || 5000,
+    waitFor: config.waitFor || 2000,
+    timeout: config.timeout || 30000,
+  };

-  return { output: mockScrapedContent, url, selector };
+  // Parse include/exclude tags if provided
+  if (config.includeTags) {
+    firecrawlConfig.includeTags = config.includeTags
+      .split(",")
+      .map((tag: string) => tag.trim())
+      .filter((tag: string) => tag.length > 0);
+  }
+
+  if (config.excludeTags) {
+    firecrawlConfig.excludeTags = config.excludeTags
+      .split(",")
+      .map((tag: string) => tag.trim())
+      .filter((tag: string) => tag.length > 0);
+  }
+
+  try {
+    // Use Firecrawl to scrape the URL
+    const result = await scrapeWithFirecrawl(firecrawlConfig);
+
+    if (result.success && result.data) {
+      return {
+        output: result.data.content,
+        url,
+        metadata: result.data.metadata,
+        markdown: result.data.markdown,
+        html: result.data.html,
+        formats: config.formats || ["markdown", "html"],
+        onlyMainContent: config.onlyMainContent !== false,
+      };
+    } else {
+      throw new Error(result.error || "Failed to scrape URL with Firecrawl");
+    }
+  } catch (error) {
+    console.error("Error processing web scraping node:", error);
+
+    // Return fallback response if Firecrawl fails
+    const fallbackContent = `Error scraping ${url}: ${
+      error instanceof Error ? error.message : "Unknown error"
+    }. Please check your Firecrawl API key and try again.`;
+
+    return {
+      output: fallbackContent,
+      url,
+      error: error instanceof Error ? error.message : "Unknown error",
+    };
+  }
 };

 const processEmbeddingNode = async (node: WorkflowNode, inputData: string) => {
@@ -1,81 +0,0 @@
-<!DOCTYPE html>
-<html>
-  <head>
-    <title>Test localStorage Persistence</title>
-  </head>
-  <body>
-    <h1>Test localStorage Persistence</h1>
-    <div id="status"></div>
-    <button onclick="testSave()">Test Save</button>
-    <button onclick="testLoad()">Test Load</button>
-    <button onclick="testClear()">Test Clear</button>
-    <pre id="output"></pre>
-
-    <script>
-      const WORKFLOWS_STORAGE_KEY = "agent-workflow-builder-workflows";
-
-      function testSave() {
-        const testWorkflow = {
-          id: "test-123",
-          name: "Test Workflow",
-          nodes: [],
-          edges: [],
-          createdAt: new Date(),
-          updatedAt: new Date(),
-        };
-
-        try {
-          localStorage.setItem(
-            WORKFLOWS_STORAGE_KEY,
-            JSON.stringify([testWorkflow])
-          );
-          document.getElementById("status").innerHTML =
-            '<span style="color: green;">✓ Saved successfully</span>';
-        } catch (error) {
-          document.getElementById("status").innerHTML =
-            '<span style="color: red;">✗ Save failed: ' +
-            error.message +
-            "</span>";
-        }
-      }
-
-      function testLoad() {
-        try {
-          const stored = localStorage.getItem(WORKFLOWS_STORAGE_KEY);
-          if (stored) {
-            const workflows = JSON.parse(stored);
-            document.getElementById("output").textContent = JSON.stringify(
-              workflows,
-              null,
-              2
-            );
-            document.getElementById("status").innerHTML =
-              '<span style="color: green;">✓ Loaded successfully</span>';
-          } else {
-            document.getElementById("status").innerHTML =
-              '<span style="color: orange;">⚠ No data found</span>';
-          }
-        } catch (error) {
-          document.getElementById("status").innerHTML =
-            '<span style="color: red;">✗ Load failed: ' +
-            error.message +
-            "</span>";
-        }
-      }
-
-      function testClear() {
-        try {
-          localStorage.removeItem(WORKFLOWS_STORAGE_KEY);
-          document.getElementById("status").innerHTML =
-            '<span style="color: green;">✓ Cleared successfully</span>';
-          document.getElementById("output").textContent = "";
-        } catch (error) {
-          document.getElementById("status").innerHTML =
-            '<span style="color: red;">✗ Clear failed: ' +
-            error.message +
-            "</span>";
-        }
-      }
-    </script>
-  </body>
-</html>