{"id":20332990,"url":"https://github.com/rayspock/go-web-crawler","last_synced_at":"2026-06-10T07:31:50.214Z","repository":{"id":119511643,"uuid":"331138669","full_name":"rayspock/go-web-crawler","owner":"rayspock","description":"A web crawler to fetch all the links from a given website via go routines.","archived":false,"fork":false,"pushed_at":"2023-08-17T14:27:07.000Z","size":11,"stargazers_count":0,"open_issues_count":0,"forks_count":0,"subscribers_count":3,"default_branch":"main","last_synced_at":"2025-03-04T12:46:15.804Z","etag":null,"topics":["concurrency","crawler","golang","goroutine"],"latest_commit_sha":null,"homepage":"","language":"Go","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":null,"status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/rayspock.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":null,"funding":null,"license":null,"code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null}},"created_at":"2021-01-19T23:36:35.000Z","updated_at":"2023-08-17T20:38:12.000Z","dependencies_parsed_at":null,"dependency_job_id":"a42704c2-60c0-472b-a95f-203ce5a785bd","html_url":"https://github.com/rayspock/go-web-crawler","commit_stats":null,"previous_names":[],"tags_count":0,"template":false,"template_full_name":null,"purl":"pkg:github/rayspock/go-web-crawler","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/rayspock%2Fgo-web-crawler","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/rayspock%2Fgo-web-crawler/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/rayspock%2Fgo-web-crawler/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/rayspock%2Fgo-web-crawler/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/rayspock","download_url":"https://codeload.github.com/rayspock/go-web-crawler/tar.gz/refs/heads/main","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/rayspock%2Fgo-web-crawler/sbom","scorecard":null,"host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":286080680,"owners_count":34142638,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2026-05-26T15:22:16.424Z","status":"online","status_checked_at":"2026-06-10T02:00:07.152Z","response_time":89,"last_error":null,"robots_txt_status":"success","robots_txt_updated_at":"2025-07-24T06:49:26.215Z","robots_txt_url":"https://github.com/robots.txt","online":true,"can_crawl_api":true,"host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":["concurrency","crawler","golang","goroutine"],"created_at":"2024-11-14T20:28:58.162Z","updated_at":"2026-06-10T07:31:50.186Z","avatar_url":"https://github.com/rayspock.png","language":"Go","funding_links":[],"categories":[],"sub_categories":[],"readme":"# Web Crawler\n\nA web crawler to fetch all the links from a given website via go routines. \n\n## Getting Started\n\nTo get started with the repository, simply clone or download the code examples and open them in your favorite text\neditor or IDE.\n\nBefore running the examples, make sure you have Go installed on your machine. You can download and install the latest\nversion of Go from the official website at https://golang.org.\n\nTo run the code, simply navigate to the directory containing the code example and run the following command:\n```bash\n# Run the code\n$ go run cmd/main.go -u https://github.com -d 2 -o output.txt\n```\n```bash\n# Parameters\n  -d, --depth int       Maximum of depth to crawl (default 1)\n  -o, --output string   Output file name\n  -u, --url string      Website URL (default \"https://github.com\")\n```\n\n\n## Development \n\n```bash\n# Run the tests\n$ make test\n\n# Generate mock\n$ make generate\n```\n","project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Frayspock%2Fgo-web-crawler","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Frayspock%2Fgo-web-crawler","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Frayspock%2Fgo-web-crawler/lists"}