{"id":16619867,"url":"https://github.com/techguy-bhushan/web-spider","last_synced_at":"2026-03-27T02:27:32.733Z","repository":{"id":73283359,"uuid":"104431878","full_name":"techguy-bhushan/Web-Spider","owner":"techguy-bhushan","description":"multi-threaded webs crawler","archived":false,"fork":false,"pushed_at":"2017-09-23T09:27:13.000Z","size":2,"stargazers_count":3,"open_issues_count":0,"forks_count":2,"subscribers_count":0,"default_branch":"master","last_synced_at":"2025-03-27T19:50:07.549Z","etag":null,"topics":["crawler","python","web-spider"],"latest_commit_sha":null,"homepage":"","language":"Python","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":null,"status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/techguy-bhushan.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":null,"funding":null,"license":null,"code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null}},"created_at":"2017-09-22T04:34:37.000Z","updated_at":"2024-11-23T20:20:23.000Z","dependencies_parsed_at":"2023-06-26T02:00:48.651Z","dependency_job_id":null,"html_url":"https://github.com/techguy-bhushan/Web-Spider","commit_stats":null,"previous_names":[],"tags_count":0,"template":false,"template_full_name":null,"repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/techguy-bhushan%2FWeb-Spider","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/techguy-bhushan%2FWeb-Spider/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/techguy-bhushan%2FWeb-Spider/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/techguy-bhushan%2FWeb-Spider/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/techguy-bhushan","download_url":"https://codeload.github.com/techguy-bhushan/Web-Spider/tar.gz/refs/heads/master","host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":248830389,"owners_count":21168272,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2022-07-04T15:15:14.044Z","host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":["crawler","python","web-spider"],"created_at":"2024-10-12T02:42:59.674Z","updated_at":"2026-03-27T02:27:27.702Z","avatar_url":"https://github.com/techguy-bhushan.png","language":"Python","funding_links":[],"categories":[],"sub_categories":[],"readme":"# Web-Spider\nmulti-threaded webs crawler\n\n# Required in System for run project\n* Python\n\nRun trigger.py, before run this script please edit this script set HOME_PAGE and PROJECT_NAME parameter value\nPROJECT_NAME value is name of directory where all data will store, HOME_PAGE is base url of web site which you want crawler e.x www.w2school.com\n\nIt will collect all links from that web site and store in crawled.txt file\n\n","project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Ftechguy-bhushan%2Fweb-spider","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Ftechguy-bhushan%2Fweb-spider","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Ftechguy-bhushan%2Fweb-spider/lists"}