{"id":13454156,"url":"https://github.com/mendableai/firecrawl","last_synced_at":"2025-05-12T17:51:39.134Z","repository":{"id":233594049,"uuid":"787076358","full_name":"mendableai/firecrawl","owner":"mendableai","description":"🔥 Turn entire websites into LLM-ready markdown or structured data. Scrape, crawl and extract with a single API.","archived":false,"fork":false,"pushed_at":"2025-05-12T15:46:18.000Z","size":59436,"stargazers_count":37995,"open_issues_count":220,"forks_count":3459,"subscribers_count":196,"default_branch":"main","last_synced_at":"2025-05-12T16:48:09.094Z","etag":null,"topics":["ai","ai-scraping","crawler","data","html-to-markdown","llm","markdown","rag","scraper","scraping","web-crawler","webscraping"],"latest_commit_sha":null,"homepage":"https://firecrawl.dev","language":"TypeScript","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"agpl-3.0","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/mendableai.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":"CONTRIBUTING.md","funding":null,"license":"LICENSE","code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":".github/CODEOWNERS","security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null,"zenodo":null}},"created_at":"2024-04-15T21:02:29.000Z","updated_at":"2025-05-12T16:16:34.000Z","dependencies_parsed_at":"2024-05-15T14:33:34.031Z","dependency_job_id":"82dca70e-86bd-4c48-b403-8be1149ae61e","html_url":"https://github.com/mendableai/firecrawl","commit_stats":{"total_commits":2039,"total_committers":65,"mean_commits":"31.369230769230768","dds":0.5404610102991663,"last_synced_commit":"9a13c1dedee1e1a4383d2a5671b8d97906c98fca"},"previous_names":["mendableai/firecrawl"],"tags_count":15,"template":false,"template_full_name":null,"repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/mendableai%2Ffirecrawl","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/mendableai%2Ffirecrawl/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/mendableai%2Ffirecrawl/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/mendableai%2Ffirecrawl/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/mendableai","download_url":"https://codeload.github.com/mendableai/firecrawl/tar.gz/refs/heads/main","host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":253785012,"owners_count":21963903,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2022-07-04T15:15:14.044Z","host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":["ai","ai-scraping","crawler","data","html-to-markdown","llm","markdown","rag","scraper","scraping","web-crawler","webscraping"],"created_at":"2024-07-31T08:00:51.438Z","updated_at":"2025-05-12T17:51:39.114Z","avatar_url":"https://github.com/mendableai.png","language":"TypeScript","readme":"\u003ch3 align=\"center\"\u003e\n  \u003ca name=\"readme-top\"\u003e\u003c/a\u003e\n  \u003cimg\n    src=\"https://raw.githubusercontent.com/mendableai/firecrawl/main/img/firecrawl_logo.png\"\n    height=\"200\"\n  \u003e\n\u003c/h3\u003e\n\u003cdiv align=\"center\"\u003e\n    \u003ca href=\"https://github.com/mendableai/firecrawl/blob/main/LICENSE\"\u003e\n  \u003cimg src=\"https://img.shields.io/github/license/mendableai/firecrawl\" alt=\"License\"\u003e\n\u003c/a\u003e\n    \u003ca href=\"https://pepy.tech/project/firecrawl-py\"\u003e\n  \u003cimg src=\"https://static.pepy.tech/badge/firecrawl-py\" alt=\"Downloads\"\u003e\n\u003c/a\u003e\n\u003ca href=\"https://GitHub.com/mendableai/firecrawl/graphs/contributors\"\u003e\n  \u003cimg src=\"https://img.shields.io/github/contributors/mendableai/firecrawl.svg\" alt=\"GitHub Contributors\"\u003e\n\u003c/a\u003e\n\u003ca href=\"https://firecrawl.dev\"\u003e\n  \u003cimg src=\"https://img.shields.io/badge/Visit-firecrawl.dev-orange\" alt=\"Visit firecrawl.dev\"\u003e\n\u003c/a\u003e\n\u003c/div\u003e\n\u003cdiv\u003e\n  \u003cp align=\"center\"\u003e\n    \u003ca href=\"https://twitter.com/firecrawl_dev\"\u003e\n      \u003cimg src=\"https://img.shields.io/badge/Follow%20on%20X-000000?style=for-the-badge\u0026logo=x\u0026logoColor=white\" alt=\"Follow on X\" /\u003e\n    \u003c/a\u003e\n    \u003ca href=\"https://www.linkedin.com/company/104100957\"\u003e\n      \u003cimg src=\"https://img.shields.io/badge/Follow%20on%20LinkedIn-0077B5?style=for-the-badge\u0026logo=linkedin\u0026logoColor=white\" alt=\"Follow on LinkedIn\" /\u003e\n    \u003c/a\u003e\n    \u003ca href=\"https://discord.com/invite/gSmWdAkdwd\"\u003e\n      \u003cimg src=\"https://img.shields.io/badge/Join%20our%20Discord-5865F2?style=for-the-badge\u0026logo=discord\u0026logoColor=white\" alt=\"Join our Discord\" /\u003e\n    \u003c/a\u003e\n  \u003c/p\u003e\n\u003c/div\u003e\n\n# 🔥 Firecrawl\n\nEmpower your AI apps with clean data from any website. Featuring advanced scraping, crawling, and data extraction capabilities.\n\n_This repository is in development, and we’re still integrating custom modules into the mono repo. It's not fully ready for self-hosted deployment yet, but you can run it locally._\n\n## What is Firecrawl?\n\n[Firecrawl](https://firecrawl.dev?ref=github) is an API service that takes a URL, crawls it, and converts it into clean markdown or structured data. We crawl all accessible subpages and give you clean data for each. No sitemap required. Check out our [documentation](https://docs.firecrawl.dev).\n\n_Pst. hey, you, join our stargazers :)_\n\n\u003ca href=\"https://github.com/mendableai/firecrawl\"\u003e\n  \u003cimg src=\"https://img.shields.io/github/stars/mendableai/firecrawl.svg?style=social\u0026label=Star\u0026maxAge=2592000\" alt=\"GitHub stars\"\u003e\n\u003c/a\u003e\n\n## How to use it?\n\nWe provide an easy to use API with our hosted version. You can find the playground and documentation [here](https://firecrawl.dev/playground). You can also self host the backend if you'd like.\n\nCheck out the following resources to get started:\n- [x] **API**: [Documentation](https://docs.firecrawl.dev/api-reference/introduction)\n- [x] **SDKs**: [Python](https://docs.firecrawl.dev/sdks/python), [Node](https://docs.firecrawl.dev/sdks/node), [Go](https://docs.firecrawl.dev/sdks/go), [Rust](https://docs.firecrawl.dev/sdks/rust)\n- [x] **LLM Frameworks**: [Langchain (python)](https://python.langchain.com/docs/integrations/document_loaders/firecrawl/), [Langchain (js)](https://js.langchain.com/docs/integrations/document_loaders/web_loaders/firecrawl), [Llama Index](https://docs.llamaindex.ai/en/latest/examples/data_connectors/WebPageDemo/#using-firecrawl-reader), [Crew.ai](https://docs.crewai.com/), [Composio](https://composio.dev/tools/firecrawl/all), [PraisonAI](https://docs.praison.ai/firecrawl/), [Superinterface](https://superinterface.ai/docs/assistants/functions/firecrawl), [Vectorize](https://docs.vectorize.io/integrations/source-connectors/firecrawl)\n- [x] **Low-code Frameworks**: [Dify](https://dify.ai/blog/dify-ai-blog-integrated-with-firecrawl), [Langflow](https://docs.langflow.org/), [Flowise AI](https://docs.flowiseai.com/integrations/langchain/document-loaders/firecrawl), [Cargo](https://docs.getcargo.io/integration/firecrawl), [Pipedream](https://pipedream.com/apps/firecrawl/)\n- [x] **Others**: [Zapier](https://zapier.com/apps/firecrawl/integrations), [Pabbly Connect](https://www.pabbly.com/connect/integrations/firecrawl/)\n- [ ] Want an SDK or Integration? Let us know by opening an issue.\n\nTo run locally, refer to guide [here](https://github.com/mendableai/firecrawl/blob/main/CONTRIBUTING.md).\n\n### API Key\n\nTo use the API, you need to sign up on [Firecrawl](https://firecrawl.dev) and get an API key.\n\n### Features\n\n- [**Scrape**](#scraping): scrapes a URL and get its content in LLM-ready format (markdown, structured data via [LLM Extract](#llm-extraction-beta), screenshot, html)\n- [**Crawl**](#crawling): scrapes all the URLs of a web page and return content in LLM-ready format\n- [**Map**](#map-alpha): input a website and get all the website urls - extremely fast\n- [**Search**](#search): search the web and get full content from results\n- [**Extract**](#extract): get structured data from single page, multiple pages or entire websites with AI.\n\n### Powerful Capabilities\n- **LLM-ready formats**: markdown, structured data, screenshot, HTML, links, metadata\n- **The hard stuff**: proxies, anti-bot mechanisms, dynamic content (js-rendered), output parsing, orchestration\n- **Customizability**: exclude tags, crawl behind auth walls with custom headers, max crawl depth, etc...\n- **Media parsing**: pdfs, docx, images\n- **Reliability first**: designed to get the data you need - no matter how hard it is\n- **Actions**: click, scroll, input, wait and more before extracting data\n- **Batching (New)**: scrape thousands of URLs at the same time with a new async endpoint.\n\nYou can find all of Firecrawl's capabilities and how to use them in our [documentation](https://docs.firecrawl.dev)\n\n### Crawling\n\nUsed to crawl a URL and all accessible subpages. This submits a crawl job and returns a job ID to check the status of the crawl.\n\n```bash\ncurl -X POST https://api.firecrawl.dev/v1/crawl \\\n    -H 'Content-Type: application/json' \\\n    -H 'Authorization: Bearer fc-YOUR_API_KEY' \\\n    -d '{\n      \"url\": \"https://docs.firecrawl.dev\",\n      \"limit\": 10,\n      \"scrapeOptions\": {\n        \"formats\": [\"markdown\", \"html\"]\n      }\n    }'\n```\n\nReturns a crawl job id and the url to check the status of the crawl.\n\n```json\n{\n  \"success\": true,\n  \"id\": \"123-456-789\",\n  \"url\": \"https://api.firecrawl.dev/v1/crawl/123-456-789\"\n}\n```\n\n### Check Crawl Job\n\nUsed to check the status of a crawl job and get its result.\n\n```bash\ncurl -X GET https://api.firecrawl.dev/v1/crawl/123-456-789 \\\n  -H 'Content-Type: application/json' \\\n  -H 'Authorization: Bearer YOUR_API_KEY'\n```\n\n```json\n{\n  \"status\": \"completed\",\n  \"total\": 36,\n  \"creditsUsed\": 36,\n  \"expiresAt\": \"2024-00-00T00:00:00.000Z\",\n  \"data\": [\n    {\n      \"markdown\": \"[Firecrawl Docs home page![light logo](https://mintlify.s3-us-west-1.amazonaws.com/firecrawl/logo/light.svg)!...\",\n      \"html\": \"\u003c!DOCTYPE html\u003e\u003chtml lang=\\\"en\\\" class=\\\"js-focus-visible lg:[--scroll-mt:9.5rem]\\\" data-js-focus-visible=\\\"\\\"\u003e...\",\n      \"metadata\": {\n        \"title\": \"Build a 'Chat with website' using Groq Llama 3 | Firecrawl\",\n        \"language\": \"en\",\n        \"sourceURL\": \"https://docs.firecrawl.dev/learn/rag-llama3\",\n        \"description\": \"Learn how to use Firecrawl, Groq Llama 3, and Langchain to build a 'Chat with your website' bot.\",\n        \"ogLocaleAlternate\": [],\n        \"statusCode\": 200\n      }\n    }\n  ]\n}\n```\n\n### Scraping\n\nUsed to scrape a URL and get its content in the specified formats.\n\n```bash\ncurl -X POST https://api.firecrawl.dev/v1/scrape \\\n    -H 'Content-Type: application/json' \\\n    -H 'Authorization: Bearer YOUR_API_KEY' \\\n    -d '{\n      \"url\": \"https://docs.firecrawl.dev\",\n      \"formats\" : [\"markdown\", \"html\"]\n    }'\n```\n\nResponse:\n\n```json\n{\n  \"success\": true,\n  \"data\": {\n    \"markdown\": \"Launch Week I is here! [See our Day 2 Release 🚀](https://www.firecrawl.dev/blog/launch-week-i-day-2-doubled-rate-limits)[💥 Get 2 months free...\",\n    \"html\": \"\u003c!DOCTYPE html\u003e\u003chtml lang=\\\"en\\\" class=\\\"light\\\" style=\\\"color-scheme: light;\\\"\u003e\u003cbody class=\\\"__variable_36bd41 __variable_d7dc5d font-inter ...\",\n    \"metadata\": {\n      \"title\": \"Home - Firecrawl\",\n      \"description\": \"Firecrawl crawls and converts any website into clean markdown.\",\n      \"language\": \"en\",\n      \"keywords\": \"Firecrawl,Markdown,Data,Mendable,Langchain\",\n      \"robots\": \"follow, index\",\n      \"ogTitle\": \"Firecrawl\",\n      \"ogDescription\": \"Turn any website into LLM-ready data.\",\n      \"ogUrl\": \"https://www.firecrawl.dev/\",\n      \"ogImage\": \"https://www.firecrawl.dev/og.png?123\",\n      \"ogLocaleAlternate\": [],\n      \"ogSiteName\": \"Firecrawl\",\n      \"sourceURL\": \"https://firecrawl.dev\",\n      \"statusCode\": 200\n    }\n  }\n}\n```\n\n### Map\n\nUsed to map a URL and get urls of the website. This returns most links present on the website.\n\n```bash cURL\ncurl -X POST https://api.firecrawl.dev/v1/map \\\n    -H 'Content-Type: application/json' \\\n    -H 'Authorization: Bearer YOUR_API_KEY' \\\n    -d '{\n      \"url\": \"https://firecrawl.dev\"\n    }'\n```\n\nResponse:\n\n```json\n{\n  \"status\": \"success\",\n  \"links\": [\n    \"https://firecrawl.dev\",\n    \"https://www.firecrawl.dev/pricing\",\n    \"https://www.firecrawl.dev/blog\",\n    \"https://www.firecrawl.dev/playground\",\n    \"https://www.firecrawl.dev/smart-crawl\",\n  ]\n}\n```\n\n#### Map with search\n\nMap with `search` param allows you to search for specific urls inside a website.\n\n```bash cURL\ncurl -X POST https://api.firecrawl.dev/v1/map \\\n    -H 'Content-Type: application/json' \\\n    -H 'Authorization: Bearer YOUR_API_KEY' \\\n    -d '{\n      \"url\": \"https://firecrawl.dev\",\n      \"search\": \"docs\"\n    }'\n```\n\nResponse will be an ordered list from the most relevant to the least relevant.\n\n```json\n{\n  \"status\": \"success\",\n  \"links\": [\n    \"https://docs.firecrawl.dev\",\n    \"https://docs.firecrawl.dev/sdks/python\",\n    \"https://docs.firecrawl.dev/learn/rag-llama3\",\n  ]\n}\n```\n\n### Search\n\nSearch the web and get full content from results\n\nFirecrawl’s search API allows you to perform web searches and optionally scrape the search results in one operation.\n\n- Choose specific output formats (markdown, HTML, links, screenshots)\n- Search the web with customizable parameters (language, country, etc.)\n- Optionally retrieve content from search results in various formats\n- Control the number of results and set timeouts\n\n```bash\ncurl -X POST https://api.firecrawl.dev/v1/search \\\n  -H \"Content-Type: application/json\" \\\n  -H \"Authorization: Bearer fc-YOUR_API_KEY\" \\\n  -d '{\n    \"query\": \"what is firecrawl?\",\n    \"limit\": 5\n  }'\n```\n\n#### Response\n\n```json\n{\n  \"success\": true,\n  \"data\": [\n    {\n      \"url\": \"https://firecrawl.dev\",\n      \"title\": \"Firecrawl | Home Page\",\n      \"description\": \"Turn websites into LLM-ready data with Firecrawl\"\n    },\n    {\n      \"url\": \"https://docs.firecrawl.dev\",\n      \"title\": \"Documentation | Firecrawl\",\n      \"description\": \"Learn how to use Firecrawl in your own applications\"\n    }\n  ]\n}\n```\n\n#### With content scraping\n\n```bash\ncurl -X POST https://api.firecrawl.dev/v1/search \\\n  -H \"Content-Type: application/json\" \\\n  -H \"Authorization: Bearer fc-YOUR_API_KEY\" \\\n  -d '{\n    \"query\": \"what is firecrawl?\",\n    \"limit\": 5,\n    \"scrapeOptions\": {\n      \"formats\": [\"markdown\", \"links\"]\n    }\n  }'\n```\n\n### Extract (Beta)\n\nGet structured data from entire websites with a prompt and/or a schema.\n\nYou can extract structured data from one or multiple URLs, including wildcards:\n\nSingle Page:\nExample: https://firecrawl.dev/some-page\n\nMultiple Pages / Full Domain\nExample: https://firecrawl.dev/*\n\nWhen you use /*, Firecrawl will automatically crawl and parse all URLs it can discover in that domain, then extract the requested data.\n\n```bash\ncurl -X POST https://api.firecrawl.dev/v1/extract \\\n    -H 'Content-Type: application/json' \\\n    -H 'Authorization: Bearer YOUR_API_KEY' \\\n    -d '{\n      \"urls\": [\n        \"https://firecrawl.dev/*\", \n        \"https://docs.firecrawl.dev/\", \n        \"https://www.ycombinator.com/companies\"\n      ],\n      \"prompt\": \"Extract the company mission, whether it is open source, and whether it is in Y Combinator from the page.\",\n      \"schema\": {\n        \"type\": \"object\",\n        \"properties\": {\n          \"company_mission\": {\n            \"type\": \"string\"\n          },\n          \"is_open_source\": {\n            \"type\": \"boolean\"\n          },\n          \"is_in_yc\": {\n            \"type\": \"boolean\"\n          }\n        },\n        \"required\": [\n          \"company_mission\",\n          \"is_open_source\",\n          \"is_in_yc\"\n        ]\n      }\n    }'\n```\n\n```json\n{\n  \"success\": true,\n  \"id\": \"44aa536d-f1cb-4706-ab87-ed0386685740\",\n  \"urlTrace\": []\n}\n```\n\nIf you are using the sdks, it will auto pull the response for you:\n\n```json\n{\n  \"success\": true,\n  \"data\": {\n    \"company_mission\": \"Firecrawl is the easiest way to extract data from the web. Developers use us to reliably convert URLs into LLM-ready markdown or structured data with a single API call.\",\n    \"supports_sso\": false,\n    \"is_open_source\": true,\n    \"is_in_yc\": true\n  }\n}\n```\n\n### LLM Extraction (Beta)\n\nUsed to extract structured data from scraped pages.\n\n```bash\ncurl -X POST https://api.firecrawl.dev/v1/scrape \\\n    -H 'Content-Type: application/json' \\\n    -H 'Authorization: Bearer YOUR_API_KEY' \\\n    -d '{\n      \"url\": \"https://www.mendable.ai/\",\n      \"formats\": [\"json\"],\n      \"jsonOptions\": {\n        \"schema\": {\n          \"type\": \"object\",\n          \"properties\": {\n            \"company_mission\": {\n                      \"type\": \"string\"\n            },\n            \"supports_sso\": {\n                      \"type\": \"boolean\"\n            },\n            \"is_open_source\": {\n                      \"type\": \"boolean\"\n            },\n            \"is_in_yc\": {\n                      \"type\": \"boolean\"\n            }\n          },\n          \"required\": [\n            \"company_mission\",\n            \"supports_sso\",\n            \"is_open_source\",\n            \"is_in_yc\"\n          ]\n        }\n      }\n    }'\n```\n\n```json\n{\n  \"success\": true,\n  \"data\": {\n    \"content\": \"Raw Content\",\n    \"metadata\": {\n      \"title\": \"Mendable\",\n      \"description\": \"Mendable allows you to easily build AI chat applications. Ingest, customize, then deploy with one line of code anywhere you want. Brought to you by SideGuide\",\n      \"robots\": \"follow, index\",\n      \"ogTitle\": \"Mendable\",\n      \"ogDescription\": \"Mendable allows you to easily build AI chat applications. Ingest, customize, then deploy with one line of code anywhere you want. Brought to you by SideGuide\",\n      \"ogUrl\": \"https://mendable.ai/\",\n      \"ogImage\": \"https://mendable.ai/mendable_new_og1.png\",\n      \"ogLocaleAlternate\": [],\n      \"ogSiteName\": \"Mendable\",\n      \"sourceURL\": \"https://mendable.ai/\"\n    },\n    \"json\": {\n      \"company_mission\": \"Train a secure AI on your technical resources that answers customer and employee questions so your team doesn't have to\",\n      \"supports_sso\": true,\n      \"is_open_source\": false,\n      \"is_in_yc\": true\n    }\n  }\n}\n```\n\n### Extracting without a schema (New)\n\nYou can now extract without a schema by just passing a `prompt` to the endpoint. The llm chooses the structure of the data.\n\n```bash\ncurl -X POST https://api.firecrawl.dev/v1/scrape \\\n    -H 'Content-Type: application/json' \\\n    -H 'Authorization: Bearer YOUR_API_KEY' \\\n    -d '{\n      \"url\": \"https://docs.firecrawl.dev/\",\n      \"formats\": [\"json\"],\n      \"jsonOptions\": {\n        \"prompt\": \"Extract the company mission from the page.\"\n      }\n    }'\n```\n\n### Interacting with the page with Actions (Cloud-only)\n\nFirecrawl allows you to perform various actions on a web page before scraping its content. This is particularly useful for interacting with dynamic content, navigating through pages, or accessing content that requires user interaction.\n\nHere is an example of how to use actions to navigate to google.com, search for Firecrawl, click on the first result, and take a screenshot.\n\n```bash\ncurl -X POST https://api.firecrawl.dev/v1/scrape \\\n    -H 'Content-Type: application/json' \\\n    -H 'Authorization: Bearer YOUR_API_KEY' \\\n    -d '{\n        \"url\": \"google.com\",\n        \"formats\": [\"markdown\"],\n        \"actions\": [\n            {\"type\": \"wait\", \"milliseconds\": 2000},\n            {\"type\": \"click\", \"selector\": \"textarea[title=\\\"Search\\\"]\"},\n            {\"type\": \"wait\", \"milliseconds\": 2000},\n            {\"type\": \"write\", \"text\": \"firecrawl\"},\n            {\"type\": \"wait\", \"milliseconds\": 2000},\n            {\"type\": \"press\", \"key\": \"ENTER\"},\n            {\"type\": \"wait\", \"milliseconds\": 3000},\n            {\"type\": \"click\", \"selector\": \"h3\"},\n            {\"type\": \"wait\", \"milliseconds\": 3000},\n            {\"type\": \"screenshot\"}\n        ]\n    }'\n```\n\n### Batch Scraping Multiple URLs (New)\n\nYou can now batch scrape multiple URLs at the same time. It is very similar to how the /crawl endpoint works. It submits a batch scrape job and returns a job ID to check the status of the batch scrape.\n\n```bash\ncurl -X POST https://api.firecrawl.dev/v1/batch/scrape \\\n    -H 'Content-Type: application/json' \\\n    -H 'Authorization: Bearer YOUR_API_KEY' \\\n    -d '{\n      \"urls\": [\"https://docs.firecrawl.dev\", \"https://docs.firecrawl.dev/sdks/overview\"],\n      \"formats\" : [\"markdown\", \"html\"]\n    }'\n```\n\n\n\n## Using Python SDK\n\n### Installing Python SDK\n\n```bash\npip install firecrawl-py\n```\n\n### Crawl a website\n\n```python\nfrom firecrawl.firecrawl import FirecrawlApp\n\napp = FirecrawlApp(api_key=\"fc-YOUR_API_KEY\")\n\n# Scrape a website:\nscrape_status = app.scrape_url(\n  'https://firecrawl.dev', \n  formats=[\"markdown\", \"html\"]\n)\nprint(scrape_status)\n\n# Crawl a website:\ncrawl_status = app.crawl_url(\n  'https://firecrawl.dev',\n  limit=100,\n  scrapeOptions'={'formats': ['markdown', 'html']}\n  poll_interval=30\n)\nprint(crawl_status)\n```\n\n### Extracting structured data from a URL\n\nWith LLM extraction, you can easily extract structured data from any URL. We support pydantic schemas to make it easier for you too. Here is how you to use it:\n\n```python\nclass ArticleSchema(BaseModel):\n    title: str\n    points: int \n    by: str\n    commentsURL: str\n\nclass TopArticlesSchema(BaseModel):\n    top: List[ArticleSchema] = Field(..., description=\"Top 5 stories\")\n\njson_config = JsonConfig(schema=TopArticlesSchema.model_json_schema())\n\nllm_extraction_result = app.scrape_url('https://news.ycombinator.com', formats=[\"json\"], json=json_config)\n\nprint(llm_extraction_result.json)\n```\n\n## Using the Node SDK\n\n### Installation\n\nTo install the Firecrawl Node SDK, you can use npm:\n\n```bash\nnpm install @mendable/firecrawl-js\n```\n\n### Usage\n\n1. Get an API key from [firecrawl.dev](https://firecrawl.dev)\n2. Set the API key as an environment variable named `FIRECRAWL_API_KEY` or pass it as a parameter to the `FirecrawlApp` class.\n\n```js\nimport FirecrawlApp, { CrawlParams, CrawlStatusResponse } from '@mendable/firecrawl-js';\n\nconst app = new FirecrawlApp({apiKey: \"fc-YOUR_API_KEY\"});\n\n// Scrape a website\nconst scrapeResponse = await app.scrapeUrl('https://firecrawl.dev', {\n  formats: ['markdown', 'html'],\n});\n\nif (scrapeResponse) {\n  console.log(scrapeResponse)\n}\n\n// Crawl a website\nconst crawlResponse = await app.crawlUrl('https://firecrawl.dev', {\n  limit: 100,\n  scrapeOptions: {\n    formats: ['markdown', 'html'],\n  }\n} satisfies CrawlParams, true, 30) satisfies CrawlStatusResponse;\n\nif (crawlResponse) {\n  console.log(crawlResponse)\n}\n```\n\n\n### Extracting structured data from a URL\n\nWith LLM extraction, you can easily extract structured data from any URL. We support zod schema to make it easier for you too. Here is how to use it:\n\n```js\nimport FirecrawlApp from \"@mendable/firecrawl-js\";\nimport { z } from \"zod\";\n\nconst app = new FirecrawlApp({\n  apiKey: \"fc-YOUR_API_KEY\"\n});\n\n// Define schema to extract contents into\nconst schema = z.object({\n  top: z\n    .array(\n      z.object({\n        title: z.string(),\n        points: z.number(),\n        by: z.string(),\n        commentsURL: z.string(),\n      })\n    )\n    .length(5)\n    .describe(\"Top 5 stories on Hacker News\"),\n});\n\nconst scrapeResult = await app.scrapeUrl(\"https://news.ycombinator.com\", {\n  jsonOptions: { extractionSchema: schema },\n});\n\nconsole.log(scrapeResult.data[\"json\"]);\n```\n\n## Open Source vs Cloud Offering\n\nFirecrawl is open source available under the AGPL-3.0 license. \n\nTo deliver the best possible product, we offer a hosted version of Firecrawl alongside our open-source offering. The cloud solution allows us to continuously innovate and maintain a high-quality, sustainable service for all users.\n\nFirecrawl Cloud is available at [firecrawl.dev](https://firecrawl.dev) and offers a range of features that are not available in the open source version:\n\n![Open Source vs Cloud Offering](https://raw.githubusercontent.com/mendableai/firecrawl/main/img/open-source-cloud.png)\n\n\n## Contributing\n\nWe love contributions! Please read our [contributing guide](CONTRIBUTING.md) before submitting a pull request. If you'd like to self-host, refer to the [self-hosting guide](SELF_HOST.md).\n\n_It is the sole responsibility of the end users to respect websites' policies when scraping, searching and crawling with Firecrawl. Users are advised to adhere to the applicable privacy policies and terms of use of the websites prior to initiating any scraping activities. By default, Firecrawl respects the directives specified in the websites' robots.txt files when crawling. By utilizing Firecrawl, you expressly agree to comply with these conditions._\n\n## Contributors\n\n\u003ca href=\"https://github.com/mendableai/firecrawl/graphs/contributors\"\u003e\n  \u003cimg alt=\"contributors\" src=\"https://contrib.rocks/image?repo=mendableai/firecrawl\"/\u003e\n\u003c/a\u003e\n\n## License Disclaimer\n\nThis project is primarily licensed under the GNU Affero General Public License v3.0 (AGPL-3.0), as specified in the LICENSE file in the root directory of this repository. However, certain components of this project are licensed under the MIT License. Refer to the LICENSE files in these specific directories for details.\n\nPlease note:\n\n- The AGPL-3.0 license applies to all parts of the project unless otherwise specified.\n- The SDKs and some UI components are licensed under the MIT License. Refer to the LICENSE files in these specific directories for details.\n- When using or contributing to this project, ensure you comply with the appropriate license terms for the specific component you are working with.\n\nFor more details on the licensing of specific components, please refer to the LICENSE files in the respective directories or contact the project maintainers.\n\n\n\u003cp align=\"right\" style=\"font-size: 14px; color: #555; margin-top: 20px;\"\u003e\n    \u003ca href=\"#readme-top\" style=\"text-decoration: none; color: #007bff; font-weight: bold;\"\u003e\n        ↑ Back to Top ↑\n    \u003c/a\u003e\n\u003c/p\u003e\n","funding_links":[],"categories":["TypeScript","📥 Data Ingestion \u0026 Parsing","🔥LLM Extraction / Parsing","网络服务","开源工具","Repos","Tools","HarmonyOS","AI Web Scraping \u0026 Automation","ai","Large Language Models (LLMs)","Table of Contents","Data and Web","6. RAG Pipelines","APIs","其他工具"],"sub_categories":["3. The Enterprise / High-Scale Stack (The 1%)","网络爬虫","爬虫","Browser Automation","Windows Manager","Contribute to our Repository","Information Processing","Rust","AI 相关工具"],"project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fmendableai%2Ffirecrawl","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Fmendableai%2Ffirecrawl","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fmendableai%2Ffirecrawl/lists"}