{"id":17323998,"url":"https://github.com/strugee/https-crawler","last_synced_at":"2026-02-18T17:31:00.248Z","repository":{"id":142464437,"uuid":"106209367","full_name":"strugee/https-crawler","owner":"strugee","description":"Build a SQLite database of per-page HTTPS support","archived":false,"fork":false,"pushed_at":"2022-08-30T23:58:35.000Z","size":70,"stargazers_count":1,"open_issues_count":10,"forks_count":0,"subscribers_count":1,"default_branch":"master","last_synced_at":"2025-10-10T08:41:02.042Z","etag":null,"topics":[],"latest_commit_sha":null,"homepage":null,"language":"JavaScript","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"lgpl-3.0","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/strugee.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":null,"funding":null,"license":"COPYING","code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null,"zenodo":null,"notice":null,"maintainers":null,"copyright":null,"agents":null,"dco":null,"cla":null}},"created_at":"2017-10-08T20:56:10.000Z","updated_at":"2023-03-05T03:35:56.000Z","dependencies_parsed_at":null,"dependency_job_id":"b3c187c1-f8fd-484e-b001-b45c825f5043","html_url":"https://github.com/strugee/https-crawler","commit_stats":null,"previous_names":[],"tags_count":0,"template":false,"template_full_name":null,"purl":"pkg:github/strugee/https-crawler","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/strugee%2Fhttps-crawler","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/strugee%2Fhttps-crawler/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/strugee%2Fhttps-crawler/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/strugee%2Fhttps-crawler/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/strugee","download_url":"https://codeload.github.com/strugee/https-crawler/tar.gz/refs/heads/master","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/strugee%2Fhttps-crawler/sbom","scorecard":null,"host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":286080680,"owners_count":29587076,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2026-02-18T16:55:40.614Z","status":"ssl_error","status_checked_at":"2026-02-18T16:55:37.558Z","response_time":162,"last_error":"SSL_connect returned=1 errno=0 peeraddr=140.82.121.6:443 state=error: unexpected eof while reading","robots_txt_status":"success","robots_txt_updated_at":"2025-07-24T06:49:26.215Z","robots_txt_url":"https://github.com/robots.txt","online":false,"can_crawl_api":true,"host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":[],"created_at":"2024-10-15T14:09:46.287Z","updated_at":"2026-02-18T17:30:59.868Z","avatar_url":"https://github.com/strugee.png","language":"JavaScript","funding_links":[],"categories":[],"sub_categories":[],"readme":"# https-crawler\n\nBuild a SQLite database of per-page HTTPS support\n\nOriginally inspired by https://securethe.news/ and built to crawl the University of Rochester's internal service pages, but can be used for any website.\n\n## Why?\n\nI go to the University of Rochester. The University's pages are somewhat inconsistent in their support for HTTPS, which is unfortunate since I access internal services on `*.rochester.edu` all the time. So I built this to comprehensively evaluate their support for HTTPS.\n\nThis crawler is designed to create a comprehensive dataset that can be used for further analysis. It does this on a per-page basis, not per-domain, because sometimes different people are responsible for running different pages under the same (sub)domain, so HTTPS support varies. Also because on other websites administrators will often choose to only protect e.g. login pages with HTTPS, which is a Bad Idea™. So you want to be able to find out about that.\n\nI will probably also build out better analysis tools, eventually.\n\n## Author\n\nAJ Jordan \u003calex@strugee.net\u003e\n\n## License\n\nGPL 3.0 or later\n","project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fstrugee%2Fhttps-crawler","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Fstrugee%2Fhttps-crawler","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fstrugee%2Fhttps-crawler/lists"}