{"id":51528585,"url":"https://github.com/xscriptor/shinobi","last_synced_at":"2026-07-09T00:30:46.189Z","repository":{"id":369140648,"uuid":"1288228136","full_name":"xscriptor/shinobi","owner":"xscriptor","description":null,"archived":false,"fork":false,"pushed_at":"2026-07-03T18:52:03.000Z","size":417,"stargazers_count":0,"open_issues_count":0,"forks_count":0,"subscribers_count":0,"default_branch":"main","last_synced_at":"2026-07-03T20:27:06.408Z","etag":null,"topics":[],"latest_commit_sha":null,"homepage":null,"language":"Rust","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":null,"status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/xscriptor.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":null,"funding":null,"license":null,"code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":"SECURITY.md","support":null,"governance":null,"roadmap":"ROADMAP.md","authors":null,"dei":null,"publiccode":null,"codemeta":null,"zenodo":null,"notice":null,"maintainers":null,"copyright":null,"agents":null,"dco":null,"cla":null}},"created_at":"2026-07-03T11:49:51.000Z","updated_at":"2026-07-03T12:24:16.000Z","dependencies_parsed_at":null,"dependency_job_id":null,"html_url":"https://github.com/xscriptor/shinobi","commit_stats":null,"previous_names":["xscriptor/shinobi"],"tags_count":null,"template":false,"template_full_name":null,"purl":"pkg:github/xscriptor/shinobi","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/xscriptor%2Fshinobi","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/xscriptor%2Fshinobi/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/xscriptor%2Fshinobi/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/xscriptor%2Fshinobi/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/xscriptor","download_url":"https://codeload.github.com/xscriptor/shinobi/tar.gz/refs/heads/main","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/xscriptor%2Fshinobi/sbom","scorecard":null,"host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":286080680,"owners_count":35282897,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2026-05-26T15:22:16.424Z","status":"online","status_checked_at":"2026-07-08T02:00:06.796Z","response_time":61,"last_error":null,"robots_txt_status":"success","robots_txt_updated_at":"2025-07-24T06:49:26.215Z","robots_txt_url":"https://github.com/robots.txt","online":true,"can_crawl_api":true,"host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":[],"created_at":"2026-07-09T00:30:45.708Z","updated_at":"2026-07-09T00:30:46.181Z","avatar_url":"https://github.com/xscriptor.png","language":"Rust","funding_links":[],"categories":[],"sub_categories":[],"readme":"\n\u003ch1 align=\"center\"\u003eShinobi\u003c/h1\u003e\n\n\u003cdiv align=\"center\"\u003e\n\u003cimg src=\"https://raw.githubusercontent.com/xscriptor/xassets/main/xwa/shinobi/shinobi-colors.svg\" width=\"150\"/\u003e \n\u003cp\u003e\u003cem\u003eSilent Web Scraper — Anti-Blocking Download System\u003c/em\u003e\u003c/p\u003e\n\u003c/div\u003e\n\n\n\n\n**Language / Idioma**  \n[English](#) | [Español](./docs/esp/README.md)\n\n\u003cp\u003e\u003cem\u003e\u003ca href=\"https://github.com/xscriptor/xwa\"\u003eXWA\u003c/a\u003e  \u003cstrong\u003esubmodule focused\u003c/strong\u003e on silent web scraping with anti-blocking — under active development\u003c/em\u003e\u003c/p\u003e\n\n\u003chr\u003e\n\n\u003cp\u003e\n\u003ca href=\"#overview\"\u003eOverview\u003c/a\u003e ·\n\u003ca href=\"#capabilities\"\u003eCapabilities\u003c/a\u003e ·\n\u003ca href=\"#anti-blocking-system\"\u003eAnti-Blocking\u003c/a\u003e ·\n\u003ca href=\"#quick-start\"\u003eQuick Start\u003c/a\u003e ·\n\u003ca href=\"#launch-script-recommended\"\u003eLaunch Script\u003c/a\u003e ·\n\u003ca href=\"#web-version-docker-compose\"\u003eDocker Compose\u003c/a\u003e ·\n\u003ca href=\"#standalone-no-docker\"\u003eStandalone\u003c/a\u003e ·\n\u003ca href=\"#execution-modes\"\u003eModes\u003c/a\u003e ·\n\u003ca href=\"#environment-variables\"\u003eEnv Variables\u003c/a\u003e ·\n\u003ca href=\"#related-documents\"\u003eDocs\u003c/a\u003e\n\u003c/p\u003e\n\n\u003chr\u003e\n\n\u003ch2\u003eOverview\u003c/h2\u003e\n\n\u003cp\u003eShinobi is a stealth web scraper with a single web interface. It downloads entire sites — HTML, CSS, JS, images, PDFs — while evading detection through multiple anti-blocking layers.\u003c/p\u003e\n\n\u003ctable\u003e\n  \u003ctr\u003e\n    \u003cth\u003eInterface\u003c/th\u003e\n    \u003cth\u003eDirectory\u003c/th\u003e\n    \u003cth\u003eLanguage\u003c/th\u003e\n    \u003cth\u003eType\u003c/th\u003e\n  \u003c/tr\u003e\n  \u003ctr\u003e\n    \u003ctd\u003e\u003cstrong\u003eShinobi Web\u003c/strong\u003e\u003c/td\u003e\n    \u003ctd\u003e\u003ccode\u003e/\u003c/code\u003e (monorepo root)\u003c/td\u003e\n    \u003ctd\u003eRust (Axum) + Angular 19\u003c/td\u003e\n    \u003ctd\u003eWeb application (standalone or Docker)\u003c/td\u003e\n  \u003c/tr\u003e\n\u003c/table\u003e\n\n\u003ch3\u003eCapabilities\u003c/h3\u003e\n\u003cul\u003e\n  \u003cli\u003e\u003cstrong\u003eRecursive Crawling\u003c/strong\u003e — BFS page discovery with configurable depth and max pages\u003c/li\u003e\n  \u003cli\u003e\u003cstrong\u003eJavaScript Rendering\u003c/strong\u003e — Headless Chromium engine for SPA/React/Vue/Angular sites\u003c/li\u003e\n  \u003cli\u003e\u003cstrong\u003eAsset Download\u003c/strong\u003e — HTML, CSS, JS, images, PDFs, archives, media, fonts\u003c/li\u003e\n  \u003cli\u003e\u003cstrong\u003eFile Type Filtering\u003c/strong\u003e — Select which file extensions to download\u003c/li\u003e\n  \u003cli\u003e\u003cstrong\u003eSame-Domain Scoping\u003c/strong\u003e — Stay within target domain or crawl freely\u003c/li\u003e\n  \u003cli\u003e\u003cstrong\u003eReal-Time Progress\u003c/strong\u003e — SSE live stream with pages scraped, files downloaded, current URL\u003c/li\u003e\n  \u003cli\u003e\u003cstrong\u003eDownloaded File Browser\u003c/strong\u003e — Browse and open downloaded files from the web UI\u003c/li\u003e\n  \u003cli\u003e\u003cstrong\u003eTwo Operation Modes:\u003c/strong\u003e\n    \u003cul\u003e\n      \u003cli\u003e\u003cem\u003eFast Test\u003c/em\u003e — Pure Rust, zero external deps, crawl + download + anti-blocking\u003c/li\u003e\n      \u003cli\u003e\u003cem\u003eDeep Research\u003c/em\u003e — Python sidecar adds structured data extraction (JSON-LD, microdata, Open Graph, RDFa), NLP analysis (summary, entities, keywords), custom CSS selectors, and enriched email/phone extraction\u003c/li\u003e\n    \u003c/ul\u003e\n  \u003c/li\u003e\n\u003c/ul\u003e\n\n\u003ch3\u003eAnti-Blocking System\u003c/h3\u003e\n\u003cul\u003e\n  \u003cli\u003e\u003cstrong\u003eUser-Agent Rotation\u003c/strong\u003e — 15 real browser UAs (Chrome, Firefox, Safari, Edge, mobile)\u003c/li\u003e\n  \u003cli\u003e\u003cstrong\u003eHeader Randomization\u003c/strong\u003e — Accept, Accept-Language, Sec-CH-UA, Sec-Fetch-* per request\u003c/li\u003e\n  \u003cli\u003e\u003cstrong\u003eRequest Delay + Jitter\u003c/strong\u003e — Configurable base delay with random jitter\u003c/li\u003e\n  \u003cli\u003e\u003cstrong\u003eExponential Backoff\u003c/strong\u003e — Retry with jitter on failures (configurable attempts)\u003c/li\u003e\n  \u003cli\u003e\u003cstrong\u003eRate Limit Handling\u003c/strong\u003e — Detects HTTP 429/503, waits, retries with longer backoff\u003c/li\u003e\n  \u003cli\u003e\u003cstrong\u003eProxy Support\u003c/strong\u003e — HTTP/HTTPS/SOCKS5 proxy rotation\u003c/li\u003e\n\u003c/ul\u003e\n\n\u003chr\u003e\n\n\u003ch2\u003eQuick Start\u003c/h2\u003e\n\n\u003ch3\u003eLaunch Script (Recommended)\u003c/h3\u003e\n\u003cpre\u003e\u003ccode\u003e./shinobi.sh --fast       # Rust backend + frontend (Fast Test mode)\n./shinobi.sh --deep        # Rust + Python extractor (Fast Test + Deep Research)\n./shinobi.sh --docker      # Everything via docker-compose\n./shinobi.sh --help        # Full usage help\u003c/code\u003e\u003c/pre\u003e\n\n\u003ch3\u003eWeb Version (Docker Compose)\u003c/h3\u003e\n\u003cpre\u003e\u003ccode\u003edocker compose up -d --build\u003c/code\u003e\u003c/pre\u003e\n\u003cul\u003e\n  \u003cli\u003eWeb UI: \u003ccode\u003ehttp://localhost:8080\u003c/code\u003e\u003c/li\u003e\n  \u003cli\u003eExtractor API: \u003ccode\u003ehttp://localhost:9090\u003c/code\u003e (Deep Research only)\u003c/li\u003e\n\u003c/ul\u003e\n\n\u003ch3\u003eStandalone (No Docker)\u003c/h3\u003e\n\u003cpre\u003e\u003ccode\u003eFast Test mode (Rust only)\ncargo run --release\n\nFull stack (Rust + Python extractor)\n./shinobi.sh --deep\u003c/code\u003e\u003c/pre\u003e\n\u003cp\u003eWeb UI at \u003ccode\u003ehttp://localhost:8080\u003c/code\u003e. Downloaded files stored in \u003ccode\u003e./downloads/\u003c/code\u003e.\u003c/p\u003e\n\n\u003ch3\u003eExecution Modes\u003c/h3\u003e\n\u003ctable\u003e\n  \u003ctr\u003e\u003cth\u003eMode\u003c/th\u003e\u003cth\u003eCommand\u003c/th\u003e\u003cth\u003eDescription\u003c/th\u003e\u003c/tr\u003e\n  \u003ctr\u003e\u003ctd\u003e\u003cstrong\u003eFast Test\u003c/strong\u003e\u003c/td\u003e\u003ctd\u003e\u003ccode\u003e./shinobi.sh --fast\u003c/code\u003e\u003c/td\u003e\u003ctd\u003eCrawling + asset download + anti-blocking. No external dependencies.\u003c/td\u003e\u003c/tr\u003e\n  \u003ctr\u003e\u003ctd\u003e\u003cstrong\u003eDeep Research\u003c/strong\u003e\u003c/td\u003e\u003ctd\u003e\u003ccode\u003e./shinobi.sh --deep\u003c/code\u003e\u003c/td\u003e\u003ctd\u003eAdds Python sidecar for structured data extraction (JSON-LD, microdata, OG), NLP analysis (summary, entities, keywords), and custom CSS selectors.\u003c/td\u003e\u003c/tr\u003e\n  \u003ctr\u003e\u003ctd\u003e\u003cstrong\u003eDocker\u003c/strong\u003e\u003c/td\u003e\u003ctd\u003e\u003ccode\u003e./shinobi.sh --docker\u003c/code\u003e\u003c/td\u003e\u003ctd\u003eBoth services via docker-compose.\u003c/td\u003e\u003c/tr\u003e\n  \u003ctr\u003e\u003ctd\u003e\u003cstrong\u003ePython Only\u003c/strong\u003e\u003c/td\u003e\u003ctd\u003e\u003ccode\u003e./shinobi.sh --python-only\u003c/code\u003e\u003c/td\u003e\u003ctd\u003eExtractor standalone for development.\u003c/td\u003e\u003c/tr\u003e\n\u003c/table\u003e\n\n\u003ch3\u003eEnvironment Variables\u003c/h3\u003e\n\u003ctable\u003e\n  \u003ctr\u003e\u003cth\u003eVariable\u003c/th\u003e\u003cth\u003eDefault\u003c/th\u003e\u003cth\u003eDescription\u003c/th\u003e\u003c/tr\u003e\n  \u003ctr\u003e\u003ctd\u003e\u003ccode\u003ePORT\u003c/code\u003e\u003c/td\u003e\u003ctd\u003e\u003ccode\u003e8080\u003c/code\u003e\u003c/td\u003e\u003ctd\u003eHTTP listen port (Rust)\u003c/td\u003e\u003c/tr\u003e\n  \u003ctr\u003e\u003ctd\u003e\u003ccode\u003eDATA_DIR\u003c/code\u003e\u003c/td\u003e\u003ctd\u003e\u003ccode\u003edownloads\u003c/code\u003e\u003c/td\u003e\u003ctd\u003eDownloaded files directory\u003c/td\u003e\u003c/tr\u003e\n  \u003ctr\u003e\u003ctd\u003e\u003ccode\u003eRUST_LOG\u003c/code\u003e\u003c/td\u003e\u003ctd\u003e\u003ccode\u003eshinobi=info,tower_http=info\u003c/code\u003e\u003c/td\u003e\u003ctd\u003eLogging verbosity\u003c/td\u003e\u003c/tr\u003e\n  \u003ctr\u003e\u003ctd\u003e\u003ccode\u003eEXTRACTOR_URL\u003c/code\u003e\u003c/td\u003e\u003ctd\u003e\u003ccode\u003ehttp://localhost:9090\u003c/code\u003e\u003c/td\u003e\u003ctd\u003ePython extractor endpoint (Deep Research)\u003c/td\u003e\u003c/tr\u003e\n\u003c/table\u003e\n\n\u003chr\u003e\n\n\u003ch2\u003eRelated Documents\u003c/h2\u003e\n\n\u003ctable\u003e\n  \u003ctr\u003e\u003cth\u003eDocument\u003c/th\u003e\u003cth\u003eDescription\u003c/th\u003e\u003c/tr\u003e\n  \u003ctr\u003e\u003ctd\u003e\u003ca href=\"docs/manual.md\"\u003edocs/manual.md\u003c/a\u003e\u003c/td\u003e\u003ctd\u003eDevelopment and production deployment guide\u003c/td\u003e\u003c/tr\u003e\n  \u003ctr\u003e\u003ctd\u003e\u003ca href=\"docs/ui-architecture.md\"\u003edocs/ui-architecture.md\u003c/a\u003e\u003c/td\u003e\u003ctd\u003eFrontend architecture specification\u003c/td\u003e\u003c/tr\u003e\n  \u003ctr\u003e\u003ctd\u003e\u003ca href=\"ROADMAP.md\"\u003eROADMAP.md\u003c/a\u003e\u003c/td\u003e\u003ctd\u003eDevelopment phases and milestones\u003c/td\u003e\u003c/tr\u003e\n  \u003ctr\u003e\u003ctd\u003e\u003ca href=\"docs/project-structure.md\"\u003edocs/project-structure.md\u003c/a\u003e\u003c/td\u003e\u003ctd\u003eDetailed codebase structure with file-by-file breakdown and API reference\u003c/td\u003e\u003c/tr\u003e\n\u003c/table\u003e\n\n\u003chr\u003e\n\n\u003cdiv id=\"x\" align=\"center\"\u003e\n\u003ch2\u003eX\u003c/h2\u003e\n\n\u003ca href=\"https://dev.xscriptor.com\"\u003e\n  \u003cimg src=\"https://xscriptor.github.io/icons/icons/code/product-design/xsvg/verified-filled.svg\" width=\"24\" alt=\"X Web\" /\u003e\n\u003c/a\u003e\n \u0026 \n\u003ca href=\"https://github.com/xscriptor\"\u003e\n  \u003cimg src=\"https://xscriptor.github.io/icons/icons/code/product-design/xsvg/github.svg\" width=\"24\" alt=\"X Github Profile\" /\u003e\n\u003c/a\u003e\n \u0026 \n\u003ca href=\"https://www.xscriptor.com\"\u003e\n  \u003cimg src=\"https://xscriptor.github.io/icons/icons/code/product-design/xsvg/quotes.svg\" width=\"24\" alt=\"Xscriptor web\" /\u003e\n\u003c/a\u003e\n\n\u003c/div\u003e\n","project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fxscriptor%2Fshinobi","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Fxscriptor%2Fshinobi","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fxscriptor%2Fshinobi/lists"}