{"id":44146282,"url":"https://github.com/everdrone/grab","last_synced_at":"2026-02-09T02:15:29.631Z","repository":{"id":56855414,"uuid":"526371721","full_name":"everdrone/grab","owner":"everdrone","description":"Configurable Scraper \u0026 Downloader, Powered by RegExp and Go","archived":false,"fork":false,"pushed_at":"2023-04-16T22:39:26.000Z","size":425,"stargazers_count":63,"open_issues_count":0,"forks_count":2,"subscribers_count":3,"default_branch":"main","last_synced_at":"2024-06-20T15:58:38.854Z","etag":null,"topics":["cli","cli-app","downloader","golang","hcl","scraper","terminal","tool","utility"],"latest_commit_sha":null,"homepage":"","language":"Go","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"mit","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/everdrone.png","metadata":{"files":{"readme":"README.md","changelog":"CHANGELOG.md","contributing":null,"funding":null,"license":"LICENSE","code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null}},"created_at":"2022-08-18T21:01:27.000Z","updated_at":"2024-01-26T13:59:10.000Z","dependencies_parsed_at":"2024-06-20T15:44:53.420Z","dependency_job_id":"b75e7c05-39e3-423b-8c48-fbd3f15c85ce","html_url":"https://github.com/everdrone/grab","commit_stats":null,"previous_names":[],"tags_count":5,"template":false,"template_full_name":null,"purl":"pkg:github/everdrone/grab","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/everdrone%2Fgrab","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/everdrone%2Fgrab/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/everdrone%2Fgrab/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/everdrone%2Fgrab/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/everdrone","download_url":"https://codeload.github.com/everdrone/grab/tar.gz/refs/heads/main","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/everdrone%2Fgrab/sbom","scorecard":null,"host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":286080680,"owners_count":29254310,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2026-02-09T01:52:29.835Z","status":"online","status_checked_at":"2026-02-09T02:00:09.501Z","response_time":56,"last_error":null,"robots_txt_status":"success","robots_txt_updated_at":"2025-07-24T06:49:26.215Z","robots_txt_url":"https://github.com/robots.txt","online":true,"can_crawl_api":true,"host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":["cli","cli-app","downloader","golang","hcl","scraper","terminal","tool","utility"],"created_at":"2026-02-09T02:15:28.908Z","updated_at":"2026-02-09T02:15:29.625Z","avatar_url":"https://github.com/everdrone.png","language":"Go","funding_links":[],"categories":[],"sub_categories":[],"readme":"\u003cdiv align=\"center\"\u003e\n    \u003ch1\u003e\n        \u003cimg width=\"750\" src=\"https://raw.githubusercontent.com/everdrone/grab/main/.github/media/Dark@2x.png#gh-light-mode-only\" alt=\"GRAB\" /\u003e\n        \u003cimg width=\"750\" src=\"https://raw.githubusercontent.com/everdrone/grab/main/.github/media/Light@2x.png#gh-dark-mode-only\" alt=\"GRAB\" /\u003e\n    \u003c/h1\u003e\n    \u003ch3\u003eGreedy, Regex-Aware Binary Downloader\u003c/h3\u003e\n\u003c/div\u003e\n\n\u003cp align=\"center\"\u003e\n\u003ca href=\"https://github.com/everdrone/grab/stargazers\"\u003e\n    \u003cimg src=\"https://img.shields.io/github/stars/everdrone/grab?color=8bd5ca\u0026logo=github\u0026logoColor=d9e0ee\u0026labelColor=1e1d2f\u0026style=for-the-badge\" alt=\"Stargazers\"\u003e\n\u003c/a\u003e\n\u003ca href=\"https://github.com/everdrone/grab/releases/latest\"\u003e\n    \u003cimg src=\"https://img.shields.io/github/v/release/everdrone/grab?color=b7bdf8\u0026logo=data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAxNiAxNiIgd2lkdGg9IjE2IiBoZWlnaHQ9IjE2Ij48cGF0aCBmaWxsPSIjZDllMGVlIiBmaWxsLXJ1bGU9ImV2ZW5vZGQiIGQ9Ik02LjEyMi4zOTJhMS43NSAxLjc1IDAgMDExLjc1NiAwbDUuMjUgMy4wNDVjLjU0LjMxMy44NzIuODkuODcyIDEuNTE0VjcuMjVhLjc1Ljc1IDAgMDEtMS41IDBWNS42NzdMNy43NSA4LjQzMnY2LjM4NGExIDEgMCAwMS0xLjUwMi44NjVMLjg3MiAxMi41NjNBMS43NSAxLjc1IDAgMDEwIDExLjA0OVY0Ljk1MWMwLS42MjQuMzMyLTEuMi44NzItMS41MTRMNi4xMjIuMzkyek03LjEyNSAxLjY5bDQuNjMgMi42ODVMNyA3LjEzMyAyLjI0NSA0LjM3NWw0LjYzLTIuNjg1YS4yNS4yNSAwIDAxLjI1IDB6TTEuNSAxMS4wNDlWNS42NzdsNC43NSAyLjc1NXY1LjUxNmwtNC42MjUtMi42ODNhLjI1LjI1IDAgMDEtLjEyNS0uMjE2em0xMC44MjggMy42ODRhLjc1Ljc1IDAgMTAxLjA4NyAxLjAzNGwyLjM3OC0yLjVhLjc1Ljc1IDAgMDAwLTEuMDM0bC0yLjM3OC0yLjVhLjc1Ljc1IDAgMDAtMS4wODcgMS4wMzRMMTMuNTAxIDEySDEwLjI1YS43NS43NSAwIDAwMCAxLjVoMy4yNTFsLTEuMTczIDEuMjMzeiI+PC9wYXRoPjwvc3ZnPg==\u0026logoColor=d9e0ee\u0026labelColor=1e1d2f\u0026style=for-the-badge\" alt=\"Latest Release\"\u003e\n\u003c/a\u003e\n\u003ca href=\"https://app.codecov.io/gh/everdrone/grab\" target=\"_blank\"\u003e\n    \u003cimg src=\"https://img.shields.io/codecov/c/github/everdrone/grab?color=c6a0f6\u0026logo=codecov\u0026logoColor=d9e0ee\u0026labelColor=1e1d2f\u0026style=for-the-badge\u0026token=NkRjXNdxZI\" alt=\"Codecov\"\u003e\n\u003c/a\u003e\n\u003ca href=\"https://github.com/everdrone/grab/issues\"\u003e\n    \u003cimg src=\"https://img.shields.io/github/issues/everdrone/grab?color=f8bd96\u0026logo=data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAxNiAxNiIgd2lkdGg9IjE2IiBoZWlnaHQ9IjE2Ij48cGF0aCBmaWxsPSIjZDllMGVlIiBmaWxsLXJ1bGU9ImV2ZW5vZGQiIGQ9Ik0xMC41NjEgMS41YS4wMTYuMDE2IDAgMDAtLjAxLjAwNEwzLjI4NiA4LjU3MUEuMjUuMjUgMCAwMDMuNDYyIDlINi43NWEuNzUuNzUgMCAwMS42OTQgMS4wMzRsLTEuNzEzIDQuMTg4IDYuOTgyLTYuNzkzQS4yNS4yNSAwIDAwMTIuNTM4IDdIOS4yNWEuNzUuNzUgMCAwMS0uNjgzLTEuMDZsMi4wMDgtNC40MTguMDAzLS4wMDZhLjAyLjAyIDAgMDAtLjAwNC0uMDA5LjAyLjAyIDAgMDAtLjAwNi0uMDA2TDEwLjU2IDEuNXpNOS41MDQuNDNhMS41MTYgMS41MTYgMCAwMTIuNDM3IDEuNzEzTDEwLjQxNSA1LjVoMi4xMjNjMS41NyAwIDIuMzQ2IDEuOTA5IDEuMjIgMy4wMDRsLTcuMzQgNy4xNDJhMS4yNSAxLjI1IDAgMDEtLjg3MS4zNTRoLS4zMDJhMS4yNSAxLjI1IDAgMDEtMS4xNTctMS43MjNMNS42MzMgMTAuNUgzLjQ2MmMtMS41NyAwLTIuMzQ2LTEuOTA5LTEuMjItMy4wMDRMOS41MDMuNDI5eiI+PC9wYXRoPjwvc3ZnPg==\u0026logoColor=d9e0ee\u0026labelColor=1e1d2f\u0026style=for-the-badge\" alt=\"GitHub issues\"\u003e\n\u003c/a\u003e\n\u003c/p\u003e\n\n# Table of contents\n\n- [Motivation](#why)\n- [Installation](#installation)\n- [Usage](#usage)\n- [Quickstart](#quickstart)\n- [Options](#command-options)\n- [Next steps](#next-steps)\n\n# Why\n\nThis project helps you automate scraping data and downloading assets from the internet. Based on Go's Regular Expression engine and HCL, for ease of use, performance and flexibility.\n\n# Installation\n\nDownload and install the [latest release](https://github.com/everdrone/grab/releases/latest).\n\n# Usage\n\nRun the following command to generate a new configuration file in the current directory.\n\n```\ngrab config generate\n```\n\n\u003e **Note**  \n\u003e Grab's configuration file uses [Hashicorp's HCL](https://github.com/hashicorp/hcl).  \n\u003e You can always refer to their specification for topics not covered by the documentation in this repo.\n\nOnce you're happy with your configuration, you can check if everything is ok by running:\n\n```\ngrab config check\n```\n\nTo scrape and download assets, pass one or more URLs to the `get` subcommand:\n\n```ini\n# single URL\ngrab get https://url.to/scrape/files?from\n\n# list of URLs\ngrab get urls.ini\n\n# at least one of each\ngrab get https://my.url/and urls.ini list.ini\n```\n\n\u003e **Note**  \n\u003e The list of URLs can contain comments, like the `ini` format: all lines starting with `#` and `;` will be ignored.\n\n# Quickstart\n\nThe default configuration, generated with `grab config generate` already works out of the box.\n\n```hcl\nglobal {\n  location = \"/home/yourusername/Downloads/grab\"\n}\n\nsite \"unsplash\" {\n  test = \"unsplash\"\n\n  asset \"image\" {\n    pattern = \"contentUrl\\\":\\\"([^\\\"]+)\\\"\"\n    capture = 1\n\n    transform filename {\n      pattern = \"(?:.+)photos\\\\/(.*)\"\n      replace = \"$${1}.jpg\"\n    }\n  }\n\n  info \"title\" {\n    pattern = \"meta[^\u003e]+property=\\\"og:title\\\"[^\u003e]+content=\\\"(?P\u003ctitle\u003e[^\\\"]+)\\\"\"\n    capture = \"title\"\n  }\n\n  subdirectory {\n    pattern = \"\\\\(@(?P\u003cusername\u003e\\\\w+)\\\\)\"\n    capture = \"username\"\n    from    = body\n  }\n}\n```\n\nFor demonstration purposes, we can already download pictures from [unsplash](https://unsplash.com) by using the following command:\n\n```\ngrab get https://unsplash.com/photos/uOi3lg8fGl4\n```\n\n\u003e **Warning**  \n\u003e Please use this tool responsibly. Don't use this tool for Denial of Service attacks! Don't violate Copyright or intellectual property!\n\nInternally, the program checks checks each URL passed to `get`, if it matches a `test` pattern inside of any `site` block, it will parse find all matches for assets or data defined in `asset` and `info` blocks.\nOnce all the asset URLs are gathered, the download starts.\n\nAfter running the above command, you should have a new `grab` directory in your `~/Downloads` folder, containing subdirectories for each site defined in the configuration. Inside each site directories you will find all the assets extracted from the provided URLs.\n\nThe configuration syntax is based on a few fundamental blocks:\n\n- `global` block defines the main download directory and global network options.\n- `site \u003cname\u003e` blocks group other blocks based on the site URL.\n- `asset \u003cname\u003e` blocks define what to look for from each site and how to download it.\n- `info \u003cname\u003e` blocks define what strings to extract from the page body.\n\nAdditional configuration settings can be specified:\n\n- `network` blocks to pass headers and other network options when making requests.\n- `transform url` blocks to replace the asset URL before downloading.\n- `transform filename` blocks to replace the asset's destination path.\n- `subdirectory` blocks to organize downloads into subdirectories named by strings present in the page body or URL.\n\nFor a more in-depth look into Grab's confguration options, check out [the guide](/docs/guide.md).\n\n# Command Options\n\nTo get help about any command, use the `help` subcommand or the `--help` flag:\n\n```ini\n# to list all available commands:\ngrab help\n\n# to show instructions for a specific subcommand:\ngrab help \u003csubcommand\u003e\n```\n\n### `get`\n\n#### Arguments\n\nAccepts both URLs or path to lists of URLs. Both can be provided at the same time.\n\n```sh\n# grab get \u003curl|file\u003e [url|file...] [options]\n\ngrab get https://example.com/gallery/1 \\\n         https://example.com/gallery/2 \\\n         path/to/list.ini \\\n         other/file.ini -n\n```\n\n#### Options\n\n| Long       | Short | Default | Description                                                                                                                    |\n| ---------- | ----- | ------- | ------------------------------------------------------------------------------------------------------------------------------ |\n| `force`    | `f`   | `false` | To overwrite already existing files                                                                                            |\n| `config`   | `c`   | `nil`   | To specify the path to a configuration file                                                                                    |\n| `strict`   | `s`   | `false` | To stop the program at the first encountered error                                                                             |\n| `dry-run`  | `n`   | `false` | To send requests without writing to the disk                                                                                   |\n| `progress` | `p`   | `false` | To show a progress bar                                                                                                         |\n| `quiet`    | `q`   | `false` | To suppress all output to `stdout` (errors will still be printed to `stderr`).\u003cbr/\u003eThis option takes precedence over `verbose` |\n| `verbose`  | `v`   | `1`     | To set the verbosity level:\u003cbr/\u003e`-v` is 1, `-vv` is 2 and so on...\u003cbr/\u003e`quiet` overrides this option.                          |\n\n## Next steps\n\n- [x] Retries \u0026 Timeout\n- [x] Network options with inheritance\n- [x] URL manipulation\n- [x] Destination manipulation\n- [x] Improve logging\n- [x] Check for updates\n- [ ] Display a progress bar\n- [ ] Add HCL eval context functions\n- [ ] Distribute via various package managers:\n  - [ ] Homebrew\n  - [ ] Apt\n  - [ ] Chocolatey\n  - [ ] Scoop\n- [ ] Scripting language integration\n- [ ] Plugin system\n- [ ] Sequential jobs (like GitHub workflows)\n\n## Credits\n\n- [Catppuccin](https://github.com/catppuccin/) for the color palette\n- [Shields.io](https://github.com/badges/shields) for the badges\n\n## License\n\nDistributed under the [MIT License](/LICENSE).\n","project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Feverdrone%2Fgrab","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Feverdrone%2Fgrab","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Feverdrone%2Fgrab/lists"}