{"id":37151886,"url":"https://github.com/disclose/diosts","last_synced_at":"2026-01-14T17:56:19.398Z","repository":{"id":48312200,"uuid":"286988990","full_name":"disclose/diosts","owner":"disclose","description":"A Go scraper that validates security.txt files and outputs them in the disclose.io JSON format.","archived":false,"fork":false,"pushed_at":"2025-05-18T10:16:06.000Z","size":73,"stargazers_count":20,"open_issues_count":2,"forks_count":5,"subscribers_count":4,"default_branch":"master","last_synced_at":"2025-05-18T11:26:22.111Z","etag":null,"topics":["golang","json","scraper","security-txt"],"latest_commit_sha":null,"homepage":"","language":"Go","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"mit","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/disclose.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":null,"funding":null,"license":"LICENSE","code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null,"zenodo":null}},"created_at":"2020-08-12T10:51:58.000Z","updated_at":"2025-05-18T10:14:51.000Z","dependencies_parsed_at":"2025-05-18T11:25:30.586Z","dependency_job_id":null,"html_url":"https://github.com/disclose/diosts","commit_stats":null,"previous_names":["disclose/securitytxt-scraper"],"tags_count":5,"template":false,"template_full_name":null,"purl":"pkg:github/disclose/diosts","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/disclose%2Fdiosts","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/disclose%2Fdiosts/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/disclose%2Fdiosts/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/disclose%2Fdiosts/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/disclose","download_url":"https://codeload.github.com/disclose/diosts/tar.gz/refs/heads/master","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/disclose%2Fdiosts/sbom","scorecard":{"id":344515,"data":{"date":"2025-08-11","repo":{"name":"github.com/disclose/diosts","commit":"252e5b9c8375e4d368af04b7b38bdfa0395b0d34"},"scorecard":{"version":"v5.2.1-40-gf6ed084d","commit":"f6ed084d17c9236477efd66e5b258b9d4cc7b389"},"score":1.9,"checks":[{"name":"Dangerous-Workflow","score":-1,"reason":"no workflows found","details":null,"documentation":{"short":"Determines if the project's GitHub Action workflows avoid dangerous patterns.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#dangerous-workflow"}},{"name":"Maintained","score":0,"reason":"1 commit(s) and 0 issue activity found in the last 90 days -- score normalized to 0","details":null,"documentation":{"short":"Determines if the project is \"actively maintained\".","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#maintained"}},{"name":"Code-Review","score":1,"reason":"Found 2/18 approved changesets -- score normalized to 1","details":null,"documentation":{"short":"Determines if the project requires human code review before pull requests (aka merge requests) are merged.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#code-review"}},{"name":"Pinned-Dependencies","score":-1,"reason":"no dependencies found","details":null,"documentation":{"short":"Determines if the project has declared and pinned the dependencies of its build process.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#pinned-dependencies"}},{"name":"Packaging","score":-1,"reason":"packaging workflow not detected","details":["Warn: no GitHub/GitLab publishing workflow detected."],"documentation":{"short":"Determines if the project is published as a package that others can easily download, install, easily update, and uninstall.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#packaging"}},{"name":"Token-Permissions","score":-1,"reason":"No tokens found","details":null,"documentation":{"short":"Determines if the project's workflows follow the principle of least privilege.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#token-permissions"}},{"name":"Binary-Artifacts","score":10,"reason":"no binaries found in the repo","details":null,"documentation":{"short":"Determines if the project has generated executable (binary) artifacts in the source repository.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#binary-artifacts"}},{"name":"CII-Best-Practices","score":0,"reason":"no effort to earn an OpenSSF best practices badge detected","details":null,"documentation":{"short":"Determines if the project has an OpenSSF (formerly CII) Best Practices Badge.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#cii-best-practices"}},{"name":"Security-Policy","score":0,"reason":"security policy file not detected","details":["Warn: no security policy file detected","Warn: no security file to analyze","Warn: no security file to analyze","Warn: no security file to analyze"],"documentation":{"short":"Determines if the project has published a security policy.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#security-policy"}},{"name":"Fuzzing","score":0,"reason":"project is not fuzzed","details":["Warn: no fuzzer integrations found"],"documentation":{"short":"Determines if the project uses fuzzing.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#fuzzing"}},{"name":"License","score":10,"reason":"license file detected","details":["Info: project has a license file: LICENSE:0","Info: FSF or OSI recognized license: MIT License: LICENSE:0"],"documentation":{"short":"Determines if the project has defined a license.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#license"}},{"name":"Signed-Releases","score":-1,"reason":"no releases found","details":null,"documentation":{"short":"Determines if the project cryptographically signs release artifacts.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#signed-releases"}},{"name":"Branch-Protection","score":0,"reason":"branch protection not enabled on development/release branches","details":["Warn: branch protection not enabled for branch 'master'"],"documentation":{"short":"Determines if the default and release branches are protected with GitHub's branch protection settings.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#branch-protection"}},{"name":"SAST","score":0,"reason":"SAST tool is not run on all commits -- score normalized to 0","details":["Warn: 0 commits out of 19 are checked with a SAST tool"],"documentation":{"short":"Determines if the project uses static code analysis.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#sast"}},{"name":"Vulnerabilities","score":0,"reason":"11 existing vulnerabilities detected","details":["Warn: Project is vulnerable to: GO-2022-0209 / GHSA-r5c5-pr8j-pfp7","Warn: Project is vulnerable to: GO-2023-1992 / GHSA-x3jr-pf6g-c48f","Warn: Project is vulnerable to: GO-2022-0229 / GHSA-cjjc-xp8v-855w","Warn: Project is vulnerable to: GO-2020-0012 / GHSA-ffhg-7mh4-33c4","Warn: Project is vulnerable to: GO-2021-0227 / GHSA-3vm4-22fp-5rfm","Warn: Project is vulnerable to: GO-2022-0968 / GHSA-gwc9-m7rh-j2ww","Warn: Project is vulnerable to: GO-2021-0356 / GHSA-8c26-wmh5-6g9v","Warn: Project is vulnerable to: GO-2024-2961","Warn: Project is vulnerable to: GO-2023-2402 / GHSA-45x7-px36-x8w8","Warn: Project is vulnerable to: GO-2024-3321 / GHSA-v778-237x-gjrc","Warn: Project is vulnerable to: GO-2025-3487 / GHSA-hcg3-q754-cr77"],"documentation":{"short":"Determines if the project has open, known unfixed vulnerabilities.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#vulnerabilities"}}]},"last_synced_at":"2025-08-18T06:44:12.297Z","repository_id":48312200,"created_at":"2025-08-18T06:44:12.297Z","updated_at":"2025-08-18T06:44:12.297Z"},"host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":286080680,"owners_count":28429063,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2026-01-14T16:38:47.836Z","status":"ssl_error","status_checked_at":"2026-01-14T16:34:59.695Z","response_time":107,"last_error":"SSL_connect returned=1 errno=0 peeraddr=140.82.121.5:443 state=error: unexpected eof while reading","robots_txt_status":"success","robots_txt_updated_at":"2025-07-24T06:49:26.215Z","robots_txt_url":"https://github.com/robots.txt","online":false,"can_crawl_api":true,"host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":["golang","json","scraper","security-txt"],"created_at":"2026-01-14T17:56:18.673Z","updated_at":"2026-01-14T17:56:19.385Z","avatar_url":"https://github.com/disclose.png","language":"Go","funding_links":[],"categories":[],"sub_categories":[],"readme":"# diosts\n\nThe disclose.io security.txt scraper (`diosts`) takes a list of domains as the input, retrieves and validates the `security.txt` if available and outputs it in the disclose.io JSON format.\n\n## Installation\n\n### Prerequisites: \n- Go 1.13 or newer\n\n### Option 1: Using go install (recommended)\n```bash\n# Install the latest version (v0.2.2)\ngo install github.com/disclose/diosts/cmd/diosts@latest\n\n# The binary will be installed to your $GOPATH/bin directory\n# Make sure this is in your PATH to run diosts from anywhere\n```\n\n### Option 2: From source\n```bash\n# Clone the repository\ngit clone https://github.com/disclose/diosts.git\ncd diosts\n\n# Build the binary\ngo build ./cmd/diosts\n\n# Optional: Install to your $GOPATH/bin\ngo install ./cmd/diosts\n```\n\n## Usage\n```bash\ncat domains.txt | diosts -t \u003cthreads\u003e -n \u003cnon-compliant-output\u003e 2\u003ediosts.log \u003esecuritytxt.json\n```\n\nThis will try and scrape the `security.txt` from the domains listed in `domains.txt`, with `\u003cthreads\u003e` parallel threads (defaults to 8). Logging (with information on each of the domains in the input) will be written to `diosts.log` (because it's output to `stderr`) and a JSON array of retrieved `security.txt` information in disclose.io format will be written to `securitytxt.json`.\n\nThe `-n` or `--non-compliant` flag enables you to output the non-RFC-compliant security.txt files to a separate JSON file for further analysis and processing.\n\nFor each input, the following URIs are tried, in order:\n1. `https://\u003cdomain\u003e/.well-known/security.txt`\n2. `https://\u003cdomain\u003e/security.txt`\n3. `http://\u003cdomain\u003e/.well-known/security.txt`\n4. `http://\u003cdomain\u003e/security.txt`\n\nAny non-fatal violations of the [`security.txt` specification](https://www.rfc-editor.org/rfc/rfc9116) will be logged and tracked in the output.\n\n## Supported Fields\n\nThe tool supports all fields defined in RFC 9116 plus extensions:\n\n| Field | Required | Description |\n|-------|----------|-------------|\n| Contact | Yes | Contact information for reporting security issues |\n| Expires | Yes | Date after which the security.txt file should be considered stale |\n| Encryption | No | Link to encryption key for secure communication |\n| Acknowledgments | No | Link to a page where security researchers are recognized |\n| Policy | No | Link to the security policy |\n| Hiring | No | Link to security-related job positions |\n| Preferred-Languages | No | Languages the security team understands |\n| Canonical | No | The canonical URIs where the security.txt file is located |\n| CSAF | No | Link to the provider-metadata.json of the CSAF (Common Security Advisory Framework) provider |\n\n## RFC 9116 Compliance\n\nThe tool fully supports RFC 9116 compliance checking and will report:\n- Whether a security.txt file is RFC compliant \n- Specific compliance issues found\n- Expires date checking (required field per RFC 9116)\n- Field validation according to the standard\n\n## Notes\n\n### Redirects\n\nAccording to the specifications, a redirect should be followed when retrieving `security.txt`. However:\n\n\u003e When retrieving the file and any resources referenced in the file,\n\u003e researchers should record any redirects since they can lead to a\n\u003e different domain or IP address controlled by an attacker.  Further\n\u003e inspections of such redirects is recommended before using the\n\u003e information contained within the file.\n\nAt this point, we blindly accept redirects within the same organization (e.g., google.com to www.google.com is accepted). Any other redirect is logged as an error, to be dealt with later.\n\n### Canonical\n\nA `security.txt` should contain a `Canonical` field with a URL pointing to the canonical version of the `security.txt`. We should check if we retrieved the `security.txt` from the canonical URL and if not, do so.\n\n### Program name\n\nCurrently, we use the input domain name as program name. This might or might not be correct, especially with redirects and canonical URL entries. To be discussed later.\n","project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fdisclose%2Fdiosts","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Fdisclose%2Fdiosts","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fdisclose%2Fdiosts/lists"}