{"id":4893620,"url":"https://github.com/fumengze/rad_xray","last_synced_at":"2025-09-15T07:31:39.706Z","repository":{"id":242963103,"uuid":"288908374","full_name":"fumengze/rad_xray","owner":"fumengze","description":"基于radium爬虫编写的批量任务处理脚本，使用进程池","archived":false,"fork":false,"pushed_at":"2020-08-20T05:50:26.000Z","size":7766,"stargazers_count":13,"open_issues_count":2,"forks_count":3,"subscribers_count":1,"default_branch":"master","last_synced_at":"2025-01-08T20:33:06.548Z","etag":null,"topics":[],"latest_commit_sha":null,"homepage":null,"language":"Python","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":null,"status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/fumengze.png","metadata":{"files":{"readme":"readme.md","changelog":null,"contributing":null,"funding":null,"license":null,"code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null}},"created_at":"2020-08-20T04:45:20.000Z","updated_at":"2024-03-28T11:28:36.000Z","dependencies_parsed_at":null,"dependency_job_id":"488da569-07a3-4943-bb50-57a3f9d4c034","html_url":"https://github.com/fumengze/rad_xray","commit_stats":null,"previous_names":["fumengze/rad_xray"],"tags_count":0,"template":false,"template_full_name":null,"purl":"pkg:github/fumengze/rad_xray","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/fumengze%2Frad_xray","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/fumengze%2Frad_xray/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/fumengze%2Frad_xray/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/fumengze%2Frad_xray/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/fumengze","download_url":"https://codeload.github.com/fumengze/rad_xray/tar.gz/refs/heads/master","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/fumengze%2Frad_xray/sbom","scorecard":null,"host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":275225204,"owners_count":25427000,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2022-07-04T15:15:14.044Z","status":"online","status_checked_at":"2025-09-15T02:00:09.272Z","response_time":75,"last_error":null,"robots_txt_status":"success","robots_txt_updated_at":"2025-07-24T06:49:26.215Z","robots_txt_url":"https://github.com/robots.txt","online":true,"can_crawl_api":true,"host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":[],"created_at":"2024-02-21T10:39:00.846Z","updated_at":"2025-09-15T07:31:39.696Z","avatar_url":"https://github.com/fumengze.png","language":"Python","funding_links":[],"categories":["Python"],"sub_categories":[],"readme":"基于长亭科技的radium爬虫造了个新轮子\n\n实现功能：\n\n批量爬取链接，可以在cmd中添加radium支持的功能，比如登录后爬取，把流量转发到xray的监听地址中\n详细查看radium爬虫的使用方法：\n\nhttps://github.com/chaitin/rad\n\n使用Pool进程池管理进程，可以设置同时运行几个进程。\n\n使用方法：\n\n1、把需要爬取的域名放入rad.txt中\n\n2、在代码的po = Pool(3)设置进程池的最大数，脚本里写的是最大3个，也就是同时开启3个chrom浏览器进行爬取。\n\n3、爬取完成后会生成crawl.txt，格式为GET http://www.xxx.com/index.php?id=1  POST http://www.xxx.com/index.php\n\n4、--disable-headless参数为前台显示chrom浏览器的操作，不需要可以删除该参数。\n\n5、-http-proxy参数为转发流量到代理地址中，可以无缝对接xray，或考虑做个接口用于处理接收到的流量。\n\n还存在的问题：\n\n1、目前不清楚单个标签页最长的爬取时间是多久，因为没有timeout的参数（这应该是在爬虫内设定的，如果未来也没有会考虑对进程的执行时间做个限制）\n\n2、测试时发现会爬取大量类似的页面，而且对无意义的页面还会爬取，比如遇到伪静态的页面会一直爬取，看到可以在配置文件中通过正则来判断做限制（还要再考虑下怎么写正则的匹配规则）\n","project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Ffumengze%2Frad_xray","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Ffumengze%2Frad_xray","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Ffumengze%2Frad_xray/lists"}