{"id":21978425,"url":"https://github.com/zhaipro/cdn","last_synced_at":"2026-05-05T20:37:22.434Z","repository":{"id":90571512,"uuid":"150608310","full_name":"zhaipro/cdn","owner":"zhaipro","description":"抓取静态网站","archived":false,"fork":false,"pushed_at":"2021-03-08T14:28:29.000Z","size":16,"stargazers_count":0,"open_issues_count":0,"forks_count":0,"subscribers_count":2,"default_branch":"master","last_synced_at":"2025-01-28T03:50:01.997Z","etag":null,"topics":["cdn","spider"],"latest_commit_sha":null,"homepage":null,"language":"Python","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"mit","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/zhaipro.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":null,"funding":null,"license":"LICENSE","code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null}},"created_at":"2018-09-27T15:25:24.000Z","updated_at":"2018-12-01T05:39:12.000Z","dependencies_parsed_at":null,"dependency_job_id":"598173ff-141d-4704-b8ed-82d9cfd66cea","html_url":"https://github.com/zhaipro/cdn","commit_stats":null,"previous_names":[],"tags_count":0,"template":false,"template_full_name":null,"repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/zhaipro%2Fcdn","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/zhaipro%2Fcdn/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/zhaipro%2Fcdn/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/zhaipro%2Fcdn/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/zhaipro","download_url":"https://codeload.github.com/zhaipro/cdn/tar.gz/refs/heads/master","host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":245040188,"owners_count":20551299,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2022-07-04T15:15:14.044Z","host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":["cdn","spider"],"created_at":"2024-11-29T16:24:07.337Z","updated_at":"2026-05-05T20:37:22.402Z","avatar_url":"https://github.com/zhaipro.png","language":"Python","funding_links":[],"categories":[],"sub_categories":[],"readme":"# cdn\n抓取静态网站\n\n## 爬取效果\n```\n$ make run\nscrapy crawl spider -L INFO -o db.sqlite\n2018-09-28 08:02:57 [scrapy.utils.log] INFO: Scrapy 1.5.1 started (bot: crawler)\n...\n2018-09-28 08:34:29 [scrapy.core.engine] INFO: Spider closed (finished)\n$\n$ du -B G db.sqlite\n10G     db.sqlite\n$\n$ sqlite3 db.sqlite 'select count(*) from page;'\n39536\n$\n$ make runserver\nexport FLASK_DEBUG=1 FLASK_APP=app.py; flask run\n...\n127.0.0.1 - - [28/Sep/2018 08:35:26] \"GET / HTTP/1.1\" 200 -\n...\n```\n","project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fzhaipro%2Fcdn","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Fzhaipro%2Fcdn","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fzhaipro%2Fcdn/lists"}