{"id":18052969,"url":"https://github.com/izumisy/scalable-crawler","last_synced_at":"2026-04-12T07:36:32.859Z","repository":{"id":39866726,"uuid":"239119722","full_name":"IzumiSy/scalable-crawler","owner":"IzumiSy","description":"Scalable crawler, fully-managed by Google Cloud Platrom","archived":false,"fork":false,"pushed_at":"2024-02-06T03:33:22.000Z","size":276,"stargazers_count":2,"open_issues_count":3,"forks_count":0,"subscribers_count":2,"default_branch":"master","last_synced_at":"2025-04-05T07:44:11.233Z","etag":null,"topics":["crawler","docker","gcp","golang","ruby"],"latest_commit_sha":null,"homepage":"","language":"Go","has_issues":false,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":null,"status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/IzumiSy.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":null,"funding":null,"license":null,"code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null}},"created_at":"2020-02-08T11:26:42.000Z","updated_at":"2022-04-15T04:55:59.000Z","dependencies_parsed_at":"2024-12-18T13:32:44.536Z","dependency_job_id":null,"html_url":"https://github.com/IzumiSy/scalable-crawler","commit_stats":null,"previous_names":[],"tags_count":0,"template":false,"template_full_name":null,"purl":"pkg:github/IzumiSy/scalable-crawler","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/IzumiSy%2Fscalable-crawler","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/IzumiSy%2Fscalable-crawler/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/IzumiSy%2Fscalable-crawler/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/IzumiSy%2Fscalable-crawler/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/IzumiSy","download_url":"https://codeload.github.com/IzumiSy/scalable-crawler/tar.gz/refs/heads/master","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/IzumiSy%2Fscalable-crawler/sbom","host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":267668387,"owners_count":24124963,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2022-07-04T15:15:14.044Z","status":"online","status_checked_at":"2025-07-29T02:00:12.549Z","response_time":2574,"last_error":null,"robots_txt_status":"success","robots_txt_updated_at":"2025-07-24T06:49:26.215Z","robots_txt_url":"https://github.com/robots.txt","online":true,"can_crawl_api":true,"host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":["crawler","docker","gcp","golang","ruby"],"created_at":"2024-10-30T23:13:54.376Z","updated_at":"2026-04-12T07:36:27.807Z","avatar_url":"https://github.com/IzumiSy.png","language":"Go","funding_links":[],"categories":[],"sub_categories":[],"readme":"# Scalable Crawler\nCloudNative scalable crawler built on Google Cloud Platform\n\n![architecture](docs/architecture.jpeg)\n\n## Why Cloudbuild?\nIf we just want to run crawler application, there are options such as GAE, Cloud Run, and more, but CloudBuild has the way longer timeout. It can runs application to the extent of 24 hours long at maximum. The longer the better because crawling is the process that sometimes takes long time.\n\n## Setup\n\n### Crawler\n```bash\n$ make build\n$ make push\n```\n\n### Backend\n```bash\n$ make deploy\n```\n\n","project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fizumisy%2Fscalable-crawler","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Fizumisy%2Fscalable-crawler","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fizumisy%2Fscalable-crawler/lists"}