{"id":13822775,"url":"https://github.com/messense/robotparser-rs","last_synced_at":"2026-03-12T06:02:55.912Z","repository":{"id":56508993,"uuid":"38168857","full_name":"messense/robotparser-rs","owner":"messense","description":"robots.txt parser for Rust.","archived":false,"fork":false,"pushed_at":"2022-03-30T02:24:19.000Z","size":20571,"stargazers_count":18,"open_issues_count":3,"forks_count":10,"subscribers_count":3,"default_branch":"master","last_synced_at":"2025-10-19T01:32:42.941Z","etag":null,"topics":["http","robots-parser","rust"],"latest_commit_sha":null,"homepage":"https://crates.io/crates/robotparser","language":"Rust","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"mit","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/messense.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":null,"funding":null,"license":"LICENSE","code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null}},"created_at":"2015-06-27T17:28:41.000Z","updated_at":"2024-12-15T14:14:48.000Z","dependencies_parsed_at":"2022-08-15T20:01:06.222Z","dependency_job_id":null,"html_url":"https://github.com/messense/robotparser-rs","commit_stats":null,"previous_names":[],"tags_count":9,"template":false,"template_full_name":null,"purl":"pkg:github/messense/robotparser-rs","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/messense%2Frobotparser-rs","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/messense%2Frobotparser-rs/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/messense%2Frobotparser-rs/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/messense%2Frobotparser-rs/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/messense","download_url":"https://codeload.github.com/messense/robotparser-rs/tar.gz/refs/heads/master","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/messense%2Frobotparser-rs/sbom","scorecard":{"id":637419,"data":{"date":"2025-08-11","repo":{"name":"github.com/messense/robotparser-rs","commit":"ec7b2fd986b16eb372981bc1679100501d600739"},"scorecard":{"version":"v5.2.1-40-gf6ed084d","commit":"f6ed084d17c9236477efd66e5b258b9d4cc7b389"},"score":2.8,"checks":[{"name":"Packaging","score":-1,"reason":"packaging workflow not detected","details":["Warn: no GitHub/GitLab publishing workflow detected."],"documentation":{"short":"Determines if the project is published as a package that others can easily download, install, easily update, and uninstall.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#packaging"}},{"name":"Dangerous-Workflow","score":10,"reason":"no dangerous workflow patterns detected","details":null,"documentation":{"short":"Determines if the project's GitHub Action workflows avoid dangerous patterns.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#dangerous-workflow"}},{"name":"Code-Review","score":3,"reason":"Found 11/29 approved changesets -- score normalized to 3","details":null,"documentation":{"short":"Determines if the project requires human code review before pull requests (aka merge requests) are merged.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#code-review"}},{"name":"Maintained","score":0,"reason":"0 commit(s) and 0 issue activity found in the last 90 days -- score normalized to 0","details":null,"documentation":{"short":"Determines if the project is \"actively maintained\".","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#maintained"}},{"name":"Binary-Artifacts","score":10,"reason":"no binaries found in the repo","details":null,"documentation":{"short":"Determines if the project has generated executable (binary) artifacts in the source repository.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#binary-artifacts"}},{"name":"Token-Permissions","score":0,"reason":"detected GitHub workflow tokens with excessive permissions","details":["Warn: no topLevel permission defined: .github/workflows/workflow.yml:1","Info: no jobLevel write permissions found"],"documentation":{"short":"Determines if the project's workflows follow the principle of least privilege.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#token-permissions"}},{"name":"Pinned-Dependencies","score":0,"reason":"dependency not pinned by hash detected -- score normalized to 0","details":["Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/workflow.yml:18: update your workflow using https://app.stepsecurity.io/secureworkflow/messense/robotparser-rs/workflow.yml/master?enable=pin","Warn: third-party GitHubAction not pinned by hash: .github/workflows/workflow.yml:19: update your workflow using https://app.stepsecurity.io/secureworkflow/messense/robotparser-rs/workflow.yml/master?enable=pin","Warn: third-party GitHubAction not pinned by hash: .github/workflows/workflow.yml:26: update your workflow using https://app.stepsecurity.io/secureworkflow/messense/robotparser-rs/workflow.yml/master?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/workflow.yml:36: update your workflow using https://app.stepsecurity.io/secureworkflow/messense/robotparser-rs/workflow.yml/master?enable=pin","Warn: third-party GitHubAction not pinned by hash: .github/workflows/workflow.yml:37: update your workflow using https://app.stepsecurity.io/secureworkflow/messense/robotparser-rs/workflow.yml/master?enable=pin","Warn: third-party GitHubAction not pinned by hash: .github/workflows/workflow.yml:43: update your workflow using https://app.stepsecurity.io/secureworkflow/messense/robotparser-rs/workflow.yml/master?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/workflow.yml:52: update your workflow using https://app.stepsecurity.io/secureworkflow/messense/robotparser-rs/workflow.yml/master?enable=pin","Warn: third-party GitHubAction not pinned by hash: .github/workflows/workflow.yml:53: update your workflow using https://app.stepsecurity.io/secureworkflow/messense/robotparser-rs/workflow.yml/master?enable=pin","Warn: third-party GitHubAction not pinned by hash: .github/workflows/workflow.yml:58: update your workflow using https://app.stepsecurity.io/secureworkflow/messense/robotparser-rs/workflow.yml/master?enable=pin","Warn: third-party GitHubAction not pinned by hash: .github/workflows/workflow.yml:60: update your workflow using https://app.stepsecurity.io/secureworkflow/messense/robotparser-rs/workflow.yml/master?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/workflow.yml:64: update your workflow using https://app.stepsecurity.io/secureworkflow/messense/robotparser-rs/workflow.yml/master?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/workflow.yml:74: update your workflow using https://app.stepsecurity.io/secureworkflow/messense/robotparser-rs/workflow.yml/master?enable=pin","Warn: third-party GitHubAction not pinned by hash: .github/workflows/workflow.yml:75: update your workflow using https://app.stepsecurity.io/secureworkflow/messense/robotparser-rs/workflow.yml/master?enable=pin","Warn: third-party GitHubAction not pinned by hash: .github/workflows/workflow.yml:81: update your workflow using https://app.stepsecurity.io/secureworkflow/messense/robotparser-rs/workflow.yml/master?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/workflow.yml:91: update your workflow using https://app.stepsecurity.io/secureworkflow/messense/robotparser-rs/workflow.yml/master?enable=pin","Warn: third-party GitHubAction not pinned by hash: .github/workflows/workflow.yml:92: update your workflow using https://app.stepsecurity.io/secureworkflow/messense/robotparser-rs/workflow.yml/master?enable=pin","Warn: third-party GitHubAction not pinned by hash: .github/workflows/workflow.yml:98: update your workflow using https://app.stepsecurity.io/secureworkflow/messense/robotparser-rs/workflow.yml/master?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/workflow.yml:117: update your workflow using https://app.stepsecurity.io/secureworkflow/messense/robotparser-rs/workflow.yml/master?enable=pin","Warn: third-party GitHubAction not pinned by hash: .github/workflows/workflow.yml:120: update your workflow using https://app.stepsecurity.io/secureworkflow/messense/robotparser-rs/workflow.yml/master?enable=pin","Warn: third-party GitHubAction not pinned by hash: .github/workflows/workflow.yml:127: update your workflow using https://app.stepsecurity.io/secureworkflow/messense/robotparser-rs/workflow.yml/master?enable=pin","Info:   0 out of   7 GitHub-owned GitHubAction dependencies pinned","Info:   0 out of  13 third-party GitHubAction dependencies pinned"],"documentation":{"short":"Determines if the project has declared and pinned the dependencies of its build process.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#pinned-dependencies"}},{"name":"CII-Best-Practices","score":0,"reason":"no effort to earn an OpenSSF best practices badge detected","details":null,"documentation":{"short":"Determines if the project has an OpenSSF (formerly CII) Best Practices Badge.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#cii-best-practices"}},{"name":"Security-Policy","score":0,"reason":"security policy file not detected","details":["Warn: no security policy file detected","Warn: no security file to analyze","Warn: no security file to analyze","Warn: no security file to analyze"],"documentation":{"short":"Determines if the project has published a security policy.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#security-policy"}},{"name":"Fuzzing","score":0,"reason":"project is not fuzzed","details":["Warn: no fuzzer integrations found"],"documentation":{"short":"Determines if the project uses fuzzing.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#fuzzing"}},{"name":"License","score":10,"reason":"license file detected","details":["Info: project has a license file: LICENSE:0","Info: FSF or OSI recognized license: MIT License: LICENSE:0"],"documentation":{"short":"Determines if the project has defined a license.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#license"}},{"name":"Signed-Releases","score":-1,"reason":"no releases found","details":null,"documentation":{"short":"Determines if the project cryptographically signs release artifacts.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#signed-releases"}},{"name":"Branch-Protection","score":0,"reason":"branch protection not enabled on development/release branches","details":["Warn: branch protection not enabled for branch 'master'"],"documentation":{"short":"Determines if the default and release branches are protected with GitHub's branch protection settings.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#branch-protection"}},{"name":"SAST","score":0,"reason":"SAST tool is not run on all commits -- score normalized to 0","details":["Warn: 0 commits out of 12 are checked with a SAST tool"],"documentation":{"short":"Determines if the project uses static code analysis.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#sast"}},{"name":"Vulnerabilities","score":0,"reason":"20 existing vulnerabilities detected","details":["Warn: Project is vulnerable to: RUSTSEC-2022-0078 / GHSA-f85w-wvc7-crwc","Warn: Project is vulnerable to: RUSTSEC-2023-0034 / GHSA-f8vr-r385-rh5r","Warn: Project is vulnerable to: RUSTSEC-2024-0003 / GHSA-8r5v-vm4m-4g25","Warn: Project is vulnerable to: RUSTSEC-2024-0332 / GHSA-q6cp-qfwq-4gcv","Warn: Project is vulnerable to: RUSTSEC-2024-0421 / GHSA-h97m-ww89-6jmq","Warn: Project is vulnerable to: RUSTSEC-2024-0384","Warn: Project is vulnerable to: RUSTSEC-2024-0019 / GHSA-r8w9-5wcg-vfj7","Warn: Project is vulnerable to: RUSTSEC-2023-0022 / GHSA-3gxf-9r58-2ghg","Warn: Project is vulnerable to: RUSTSEC-2023-0024 / GHSA-6hcf-g6gr-hhcr","Warn: Project is vulnerable to: RUSTSEC-2023-0023 / GHSA-9qwg-crg9-m2vc","Warn: Project is vulnerable to: RUSTSEC-2023-0044 / GHSA-xcf7-rvmh-g6q4","Warn: Project is vulnerable to: RUSTSEC-2023-0072 / GHSA-xphf-cx8h-7q9g","Warn: Project is vulnerable to: GHSA-q445-7m23-qrmw","Warn: Project is vulnerable to: RUSTSEC-2024-0357","Warn: Project is vulnerable to: RUSTSEC-2025-0004 / GHSA-rpmj-rpgj-qmpm","Warn: Project is vulnerable to: RUSTSEC-2023-0018 / GHSA-mc8h-8q98-g5hr","Warn: Project is vulnerable to: RUSTSEC-2023-0001 / GHSA-7rrj-xr53-82p7","Warn: Project is vulnerable to: RUSTSEC-2023-0005 / GHSA-4q83-7cq4-p6wg","Warn: Project is vulnerable to: GHSA-rr8g-9fpq-6wmg","Warn: Project is vulnerable to: RUSTSEC-2025-0023"],"documentation":{"short":"Determines if the project has open, known unfixed vulnerabilities.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#vulnerabilities"}}]},"last_synced_at":"2025-08-21T09:36:31.405Z","repository_id":56508993,"created_at":"2025-08-21T09:36:31.405Z","updated_at":"2025-08-21T09:36:31.405Z"},"host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":286080680,"owners_count":30416737,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2026-03-12T04:41:02.746Z","status":"ssl_error","status_checked_at":"2026-03-12T04:40:12.571Z","response_time":114,"last_error":"SSL_connect returned=1 errno=0 peeraddr=140.82.121.6:443 state=error: unexpected eof while reading","robots_txt_status":"success","robots_txt_updated_at":"2025-07-24T06:49:26.215Z","robots_txt_url":"https://github.com/robots.txt","online":false,"can_crawl_api":true,"host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":["http","robots-parser","rust"],"created_at":"2024-08-04T08:02:16.778Z","updated_at":"2026-03-12T06:02:55.887Z","avatar_url":"https://github.com/messense.png","language":"Rust","readme":"# robotparser-rs\n\n[![GitHub Actions](https://github.com/messense/robotparser-rs/workflows/CI/badge.svg)](https://github.com/messense/robotparser-rs/actions?query=workflow%3ACI)\n[![Coverage Status](https://coveralls.io/repos/messense/robotparser-rs/badge.svg)](https://coveralls.io/r/messense/robotparser-rs)\n[![codecov](https://codecov.io/gh/messense/robotparser-rs/branch/master/graph/badge.svg)](https://codecov.io/gh/messense/robotparser-rs)\n[![Crates.io](https://img.shields.io/crates/v/robotparser.svg)](https://crates.io/crates/robotparser)\n[![Dependency status](https://deps.rs/repo/github/messense/robotparser-rs/status.svg)](https://deps.rs/repo/github/messense/robotparser-rs)\n\nrobots.txt parser for Rust.\n\n\n## Installation\n\nAdd it to your ``Cargo.toml``:\n\n```toml\n[dependencies]\nrobotparser = \"0.11\"\n```\n\n\n## Examples\n\n```rust\nuse robotparser::http::RobotsTxtClient;\nuse robotparser::service::RobotsTxtService;\nuse reqwest::Client;\nuse url::Url;\n\nfn main() {\n    let client = Client::new();\n    let robots_txt_url = Url::parse(\"https://www.python.org/robots.txt\").unwrap();\n    let robots_txt = client.fetch_robots_txt(robots_txt_url.origin()).unwrap().get_result();\n    let fetch_url = Url::parse(\"https://www.python.org/robots.txt\").unwrap();\n    assert!(robots_txt.can_fetch(\"*\", \u0026fetch_url));\n}\n```\n\n\n## License\n\nThis work is released under the MIT license. A copy of the license is provided in the [LICENSE](./LICENSE) file.\n","funding_links":[],"categories":["Rust"],"sub_categories":[],"project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fmessense%2Frobotparser-rs","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Fmessense%2Frobotparser-rs","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fmessense%2Frobotparser-rs/lists"}