{"id":13582537,"url":"https://github.com/shenwei356/rush","last_synced_at":"2026-01-16T14:40:38.060Z","repository":{"id":41044956,"uuid":"77892243","full_name":"shenwei356/rush","owner":"shenwei356","description":"A cross-platform command-line tool for executing jobs in parallel","archived":false,"fork":false,"pushed_at":"2026-01-15T02:38:52.000Z","size":296,"stargazers_count":1070,"open_issues_count":11,"forks_count":69,"subscribers_count":23,"default_branch":"master","last_synced_at":"2026-01-15T08:40:52.362Z","etag":null,"topics":["bioinformatics","command","cross-platform","execute","golang","parallel","pipeline","shell","windows"],"latest_commit_sha":null,"homepage":"https://github.com/shenwei356/rush","language":"Go","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"mit","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/shenwei356.png","metadata":{"files":{"readme":"README.md","changelog":"CHANGELOG.md","contributing":null,"funding":".github/FUNDING.yml","license":"LICENSE","code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null,"zenodo":null,"notice":null,"maintainers":null,"copyright":null,"agents":null,"dco":null,"cla":null},"funding":{"custom":"http://paypal.me/shenwei356"}},"created_at":"2017-01-03T07:01:03.000Z","updated_at":"2026-01-15T02:34:49.000Z","dependencies_parsed_at":"2024-01-13T00:39:12.435Z","dependency_job_id":"3524a8b4-10a5-4766-8e4c-f89a80fb0ca4","html_url":"https://github.com/shenwei356/rush","commit_stats":{"total_commits":164,"total_committers":7,"mean_commits":"23.428571428571427","dds":"0.21341463414634143","last_synced_commit":"28d997f6410ac4f489aa59a675ad55bd05cee401"},"previous_names":[],"tags_count":37,"template":false,"template_full_name":null,"purl":"pkg:github/shenwei356/rush","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/shenwei356%2Frush","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/shenwei356%2Frush/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/shenwei356%2Frush/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/shenwei356%2Frush/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/shenwei356","download_url":"https://codeload.github.com/shenwei356/rush/tar.gz/refs/heads/master","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/shenwei356%2Frush/sbom","scorecard":{"id":817793,"data":{"date":"2025-08-11","repo":{"name":"github.com/shenwei356/rush","commit":"767b1d9a009d896f25c5bb75788f306616b4f95a"},"scorecard":{"version":"v5.2.1-40-gf6ed084d","commit":"f6ed084d17c9236477efd66e5b258b9d4cc7b389"},"score":2.9,"checks":[{"name":"Code-Review","score":0,"reason":"Found 2/24 approved changesets -- score normalized to 0","details":null,"documentation":{"short":"Determines if the project requires human code review before pull requests (aka merge requests) are merged.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#code-review"}},{"name":"Token-Permissions","score":-1,"reason":"No tokens found","details":null,"documentation":{"short":"Determines if the project's workflows follow the principle of least privilege.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#token-permissions"}},{"name":"Maintained","score":2,"reason":"2 commit(s) and 1 issue activity found in the last 90 days -- score normalized to 2","details":null,"documentation":{"short":"Determines if the project is \"actively maintained\".","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#maintained"}},{"name":"Binary-Artifacts","score":10,"reason":"no binaries found in the repo","details":null,"documentation":{"short":"Determines if the project has generated executable (binary) artifacts in the source repository.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#binary-artifacts"}},{"name":"Packaging","score":-1,"reason":"packaging workflow not detected","details":["Warn: no GitHub/GitLab publishing workflow detected."],"documentation":{"short":"Determines if the project is published as a package that others can easily download, install, easily update, and uninstall.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#packaging"}},{"name":"Dangerous-Workflow","score":-1,"reason":"no workflows found","details":null,"documentation":{"short":"Determines if the project's GitHub Action workflows avoid dangerous patterns.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#dangerous-workflow"}},{"name":"Pinned-Dependencies","score":-1,"reason":"no dependencies found","details":null,"documentation":{"short":"Determines if the project has declared and pinned the dependencies of its build process.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#pinned-dependencies"}},{"name":"CII-Best-Practices","score":0,"reason":"no effort to earn an OpenSSF best practices badge detected","details":null,"documentation":{"short":"Determines if the project has an OpenSSF (formerly CII) Best Practices Badge.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#cii-best-practices"}},{"name":"Security-Policy","score":0,"reason":"security policy file not detected","details":["Warn: no security policy file detected","Warn: no security file to analyze","Warn: no security file to analyze","Warn: no security file to analyze"],"documentation":{"short":"Determines if the project has published a security policy.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#security-policy"}},{"name":"Fuzzing","score":0,"reason":"project is not fuzzed","details":["Warn: no fuzzer integrations found"],"documentation":{"short":"Determines if the project uses fuzzing.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#fuzzing"}},{"name":"License","score":10,"reason":"license file detected","details":["Info: project has a license file: LICENSE:0","Info: FSF or OSI recognized license: MIT License: LICENSE:0"],"documentation":{"short":"Determines if the project has defined a license.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#license"}},{"name":"Signed-Releases","score":0,"reason":"Project has not signed or included provenance with any releases.","details":["Warn: release artifact v0.7.0 not signed: https://api.github.com/repos/shenwei356/rush/releases/225067327","Warn: release artifact v0.6.1 not signed: https://api.github.com/repos/shenwei356/rush/releases/205228257","Warn: release artifact v0.6.0 not signed: https://api.github.com/repos/shenwei356/rush/releases/184815270","Warn: release artifact v0.5.7 not signed: https://api.github.com/repos/shenwei356/rush/releases/183087080","Warn: release artifact v0.5.6 not signed: https://api.github.com/repos/shenwei356/rush/releases/172732541","Warn: release artifact v0.7.0 does not have provenance: https://api.github.com/repos/shenwei356/rush/releases/225067327","Warn: release artifact v0.6.1 does not have provenance: https://api.github.com/repos/shenwei356/rush/releases/205228257","Warn: release artifact v0.6.0 does not have provenance: https://api.github.com/repos/shenwei356/rush/releases/184815270","Warn: release artifact v0.5.7 does not have provenance: https://api.github.com/repos/shenwei356/rush/releases/183087080","Warn: release artifact v0.5.6 does not have provenance: https://api.github.com/repos/shenwei356/rush/releases/172732541"],"documentation":{"short":"Determines if the project cryptographically signs release artifacts.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#signed-releases"}},{"name":"Branch-Protection","score":0,"reason":"branch protection not enabled on development/release branches","details":["Warn: branch protection not enabled for branch 'master'"],"documentation":{"short":"Determines if the default and release branches are protected with GitHub's branch protection settings.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#branch-protection"}},{"name":"SAST","score":0,"reason":"SAST tool is not run on all commits -- score normalized to 0","details":["Warn: 0 commits out of 8 are checked with a SAST tool"],"documentation":{"short":"Determines if the project uses static code analysis.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#sast"}},{"name":"Vulnerabilities","score":10,"reason":"0 existing vulnerabilities detected","details":null,"documentation":{"short":"Determines if the project has open, known unfixed vulnerabilities.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#vulnerabilities"}}]},"last_synced_at":"2025-08-23T14:37:38.451Z","repository_id":41044956,"created_at":"2025-08-23T14:37:38.451Z","updated_at":"2025-08-23T14:37:38.451Z"},"host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":286080680,"owners_count":28479399,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2026-01-16T11:59:17.896Z","status":"ssl_error","status_checked_at":"2026-01-16T11:55:55.838Z","response_time":107,"last_error":"SSL_read: unexpected eof while reading","robots_txt_status":"success","robots_txt_updated_at":"2025-07-24T06:49:26.215Z","robots_txt_url":"https://github.com/robots.txt","online":false,"can_crawl_api":true,"host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":["bioinformatics","command","cross-platform","execute","golang","parallel","pipeline","shell","windows"],"created_at":"2024-08-01T15:02:48.687Z","updated_at":"2026-01-16T14:40:38.054Z","avatar_url":"https://github.com/shenwei356.png","language":"Go","readme":"# rush -- a cross-platform command-line tool for executing jobs in parallel\n\n[![Built with GoLang](https://img.shields.io/badge/powered_by-go-6362c2.svg?style=flat)](https://golang.org)\n[![Go Report Card](https://goreportcard.com/badge/github.com/shenwei356/rush)](https://goreportcard.com/report/github.com/shenwei356/rush)\n[![Cross-platform](https://img.shields.io/badge/platform-any-ec2eb4.svg?style=flat)](#download)\n[![Latest Version](https://img.shields.io/github/release/shenwei356/rush.svg?style=flat?maxAge=86400)](https://github.com/shenwei356/rush/releases)\n[![Github Releases](https://img.shields.io/github/downloads/shenwei356/rush/latest/total.svg?maxAge=3600)](https://github.com/shenwei356/rush/releases)\n\n`rush` is a tool similar to [GNU parallel](https://www.gnu.org/software/parallel/)\n and [gargs](https://github.com/brentp/gargs).\n `rush` borrows some idea from them and has some unique features,\n  e.g.,\n  supporting custom defined variables,\n  resuming multi-line commands,\n  more advanced embeded replacement strings.\n\nThese features make `rush` suitable for easily and flexibly parallelizing\ncomplex workflows in fields like Bioinformatics (see [examples](#examples)).\n\n\n## Table of Contents\n\u003c!-- START doctoc generated TOC please keep comment here to allow auto update --\u003e\n\u003c!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE --\u003e\n\n- [Features](#features)\n- [Performance](#performance)\n- [Installation](#installation)\n- [Usage](#usage)\n- [Examples](#examples)\n- [Special Cases](#special-cases)\n- [Contributors](#contributors)\n- [Acknowledgements](#acknowledgements)\n- [Contact](#contact)\n- [License](#license)\n\n\u003c!-- END doctoc generated TOC please keep comment here to allow auto update --\u003e\n\n\n## Features\n\nMajor:\n\n- Supporting Linux, OS X and **Windows** (not CygWin)!\n- **Avoid mixed line from multiple processes without loss of performance**,\n  e.g. the first half of a line is from one process\n  and the last half of the line is from another process.\n  (`--line-buffer` in GNU parallel)\n- **Timeout** (`-t`). (`--timeout` in GNU parallel)\n- **Retry** (`-r`). (`--retry-failed --joblog` in GNU parallel)\n- **Safe exit after capturing Ctrl-C** (*not perfect*, you may stop it by typing ctrl-c or closing terminal)\n- **Continue** (`-c`). (`--resume --joblog` in GNU parallel,\n  ***\u003cs/\u003esut it does not support multi-line commands, which are common in workflow\u003c/s\u003e***)\n- **`awk -v` like custom defined variables** (`-v`). (***Using Shell variable in GNU parallel***)\n- **Keeping output in order of input** (`-k`). (Same `-k/--keep-order` in GNU parallel)\n- **Exit on first error(s)** (`-e`). (*not perfect*, you may stop it by typing ctrl-c or closing terminal) (`--halt 2` in GNU parallel) \n- **Settable record delimiter** (`-D`, default `\\n`). (`--recstart` and `--recend` in GNU parallel)\n- **Settable records sending to every command** (`-n`, default `1`). (`-n/--max-args` in GNU parallel)\n- **Settable field delimiter** (`-d`, default `\\s+`). (Same `-d/--delimiter` in GNU parallel)\n- **Practical replacement strings** (like GNU parallel):\n    - `{{}}`, `{}` itself\n    - `{{1,}}`, `{1,}`.\n    - `{#}`, job ID. (Same in GNU parallel)\n    - `{}`, full data. (Same in GNU parallel)\n    - `{n}`, `n`th field in delimiter-delimited data. (Same in GNU parallel)\n    - Directory and file\n        - `{/}`, dirname. (`{//}` in GNU parallel)\n        - `{%}`, basename. (`{/}` in GNU parallel)\n        - `{.}`, remove the last file extension. (Same in GNU parallel)\n        - `{:}`, remove all file extensions (***Not directly supported in GNU parallel***)\n        - `{^suffix}`, remove `suffix` (***Not directly supported in GNU parallel***)\n        - `{@regexp}`, capture submatch using regular expression (***Not directly supported in GNU parallel***).\n           There's a limitation here: curly brackets can't be used in the regular expression.\n    - Combinations\n        - `{%.}`, `{%:}`, basename without extension\n        - `{2.}`, `{2/}`, `{2%.}`, manipulate `n`th field\n        - `{file:}`, `{file:^_1}`, remove all extensions of a preset variable (see below)\n- **Preset variable (macro)**, e.g., `rush -v p={^suffix} 'echo {p}_new_suffix'`,\nwhere `{p}` is replaced with `{^suffix}`. (***Using Shell variable in GNU parallel***)\n\nMinor:\n\n- Dry run (`--dry-run`). (Same in GNU parallel)\n- Trim input data (`--trim`). (Same in GNU parallel)\n- Verbose output (`--verbose`). (Same in GNU parallel)\n\n[Differences between rush and GNU parallel](https://www.gnu.org/software/parallel/parallel_alternatives.html#DIFFERENCES-BETWEEN-Rush-AND-GNU-Parallel) on GNU parallel site.\n\n## Performance\n\nPerformance of `rush` is similar to `gargs`, and they are both slightly faster than `parallel` (Perl) and both slower than `Rust parallel` ([discussion](https://github.com/shenwei356/rush/issues/1)).\n\nNote that speed is not the #.1 target, especially for processes that last long.\n\n\n## Installation\n\n`rush` is implemented in [Go](https://golang.org/) programming language,\n executable binary files **for most popular operating systems** are freely available\n  in [release](https://github.com/shenwei356/rush/releases) page.\n\n#### Method 0: Conda\n\n[Install conda](https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html), then run\n\n    conda install -c conda-forge rush\n\nOr use [mamba](https://mamba.readthedocs.io/en/latest/installation/mamba-installation.html), which is faster.\n\n    mamba install -c conda-forge rush\n\n#### Method 1: Download binaries\n\n[rush v0.8.0](https://github.com/shenwei356/rush/releases/tag/v0.8.0)\n[![Github Releases (by Release)](https://img.shields.io/github/downloads/shenwei356/rush/v0.8.0/total.svg)](https://github.com/shenwei356/rush/releases/tag/v0.8.0)\n\n***Tip: run `rush -V` to check update !!!***\n\nOS     |Arch      |File,                                                                                                                           |Download Count\n:------|:---------|:-------------------------------------------------------------------------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------\nLinux  |32-bit    |[rush_linux_386.tar.gz](https://github.com/shenwei356/rush/releases/download/v0.8.0/rush_linux_386.tar.gz)                      |[![Github Releases (by Asset)](https://img.shields.io/github/downloads/shenwei356/rush/latest/rush_linux_386.tar.gz.svg?maxAge=3600)](https://github.com/shenwei356/rush/releases/download/v0.8.0/rush_linux_386.tar.gz)\nLinux  |**64-bit**|[**rush_linux_amd64.tar.gz**](https://github.com/shenwei356/rush/releases/download/v0.8.0/rush_linux_amd64.tar.gz)              |[![Github Releases (by Asset)](https://img.shields.io/github/downloads/shenwei356/rush/latest/rush_linux_amd64.tar.gz.svg?maxAge=3600)](https://github.com/shenwei356/rush/releases/download/v0.8.0/rush_linux_amd64.tar.gz)\nLinux  |**arm64** |[**rush_linux_arm64.tar.gz**](https://github.com/shenwei356/rush/releases/download/v0.8.0/rush_linux_arm64.tar.gz)              |[![Github Releases (by Asset)](https://img.shields.io/github/downloads/shenwei356/rush/latest/rush_linux_arm64.tar.gz.svg?maxAge=3600)](https://github.com/shenwei356/rush/releases/download/v0.8.0/rush_linux_arm64.tar.gz)\nOS X   |**64-bit**|[**rush_darwin_amd64.tar.gz**](https://github.com/shenwei356/rush/releases/download/v0.8.0/rush_darwin_amd64.tar.gz)            |[![Github Releases (by Asset)](https://img.shields.io/github/downloads/shenwei356/rush/latest/rush_darwin_amd64.tar.gz.svg?maxAge=3600)](https://github.com/shenwei356/rush/releases/download/v0.8.0/rush_darwin_amd64.tar.gz)\nOS X   |**arm64** |[**rush_darwin_arm64.tar.gz**](https://github.com/shenwei356/rush/releases/download/v0.8.0/rush_darwin_arm64.tar.gz)            |[![Github Releases (by Asset)](https://img.shields.io/github/downloads/shenwei356/rush/latest/rush_darwin_arm64.tar.gz.svg?maxAge=3600)](https://github.com/shenwei356/rush/releases/download/v0.8.0/rush_darwin_arm64.tar.gz)\nWindows|32-bit    |[rush_windows_386.exe.tar.gz](https://github.com/shenwei356/rush/releases/download/v0.8.0/rush_windows_386.exe.tar.gz)          |[![Github Releases (by Asset)](https://img.shields.io/github/downloads/shenwei356/rush/latest/rush_windows_386.exe.tar.gz.svg?maxAge=3600)](https://github.com/shenwei356/rush/releases/download/v0.8.0/rush_windows_386.exe.tar.gz)\nWindows|**64-bit**|[**rush_windows_amd64.exe.tar.gz**](https://github.com/shenwei356/rush/releases/download/v0.8.0/rush_windows_amd64.exe.tar.gz)  |[![Github Releases (by Asset)](https://img.shields.io/github/downloads/shenwei356/rush/latest/rush_windows_amd64.exe.tar.gz.svg?maxAge=3600)](https://github.com/shenwei356/rush/releases/download/v0.8.0/rush_windows_amd64.exe.tar.gz)\nFreeBSD|32-bit    |[rush_freebsd_386.tar.gz](https://github.com/shenwei356/rush/releases/download/v0.8.0/rush_freebsd_386.tar.gz)                  |[![Github Releases (by Asset)](https://img.shields.io/github/downloads/shenwei356/rush/latest/rush_freebsd_386.tar.gz.svg?maxAge=3600)](https://github.com/shenwei356/rush/releases/download/v0.8.0/rush_freebsd_386.tar.gz)\nFreeBSD|**64-bit**|[**rush_freebsd_amd64.tar.gz**](https://github.com/shenwei356/rush/releases/download/v0.8.0/rush_freebsd_amd64.tar.gz)          |[![Github Releases (by Asset)](https://img.shields.io/github/downloads/shenwei356/rush/latest/rush_freebsd_amd64.tar.gz.svg?maxAge=3600)](https://github.com/shenwei356/rush/releases/download/v0.8.0/rush_freebsd_amd64.tar.gz)\n\n\nJust [download](https://github.com/shenwei356/rush/releases) compressed\nexecutable file of your operating system,\nand decompress it with `tar -zxvf *.tar.gz` command or other tools.\nAnd then:\n\n1. **For Linux-like systems**\n    1. If you have root privilege simply copy it to `/usr/local/bin`:\n\n            sudo cp rush /usr/local/bin/\n\n    1. Or copy to anywhere in the environment variable `PATH`:\n\n            mkdir -p $HOME/bin/; cp rush $HOME/bin/\n\n1. **For windows**, just copy `rush.exe` to `C:\\WINDOWS\\system32`.\n\n#### Method 2: For Go developer\n\n    go install github.com/shenwei356/rush@latest\n    \n#### Method 3: Compiling from source\n\n    # download Go from https://go.dev/dl\n    wget https://go.dev/dl/go1.25.5.linux-amd64.tar.gz\n    \n    tar -zxf go1.25.5.linux-amd64.tar.gz -C $HOME/\n    \n    # or \n    #   echo \"export PATH=$PATH:$HOME/go/bin\" \u003e\u003e ~/.bashrc\n    #   source ~/.bashrc\n    export PATH=$PATH:$HOME/go/bin\n    \n    git clone https://github.com/shenwei356/rush\n    cd rush\n    \n    go build\n    \n    # or statically-linked binary\n    CGO_ENABLED=0 go build -tags netgo -ldflags '-w -s'\n    \n    # or cross compile for other operating systems and architectures\n    CGO_ENABLED=0 GOOS=openbsd GOARCH=amd64 go build -tags netgo -ldflags '-w -s'\n\n\n## Usage\n\n```text\nrush -- a cross-platform command-line tool for executing jobs in parallel\n\nVersion: 0.8.0\n\nAuthor: Wei Shen \u003cshenwei356@gmail.com\u003e\n\nHomepage: https://github.com/shenwei356/rush\n\nInput:\n  - Input could be a list of strings or numbers, e.g., file paths.\n  - Input can be given either from the STDIN or file(s) via the option -i/--infile.\n  - Some options could be used to defined how the input records are parsed:\n    -d, --field-delimiter   field delimiter in records (default \"\\s+\")\n    -D, --record-delimiter  record delimiter (default \"\\n\")\n    -n, --nrecords          number of records sent to a command (default 1)\n    -J, --records-join-sep  record separator for joining multi-records (default \"\\n\")\n    -T, --trim              trim white space (\" \\t\\r\\n\") in input\n\nOutput:\n  - Outputs of all commands are written to STDOUT by default,\n    you can also use -o/--out-file to specify a output file.\n  - Outputs of all commands are random, you can use the flag -k/--keep-order\n    to keep output in order of input.\n  - Outputs of all commands are buffered, you can use the flag -I/--immediate-output\n    to print output immediately and interleaved.\n\nReplacement strings in commands:\n  {}          full data\n  {#}         job ID\n  {n}         nth field in delimiter-delimited data\n  {/}         dirname\n  {%}         basename\n  {.}         remove the last file extension\n  {:}         remove all file extensions.\n  {^suffix}   remove suffix\n  {@regexp}   capture submatch using regular expression.\n              Limitation: curly brackets can't be used in the regexp.\n\n  Escaping curly brackets \"{}\":\n    {{}}        {}\n    {{1}}       {1}\n    {{1,}}      {1,}\n    {{a}}       {a}\n\n  Combinations:\n    {%.}, {%:}            basename without extension\n    {2.}, {2/}, {2%.}     manipulate nth field\n    {file:}, {file:^_1}   remove all extensions of a preset variable (see below)\n\nPreset variable (macro):\n  1. You can pass variables to the command like awk via the option -v. E.g.,\n     $ seq 3 | rush -v p=prefix_ -v s=_suffix 'echo {p}{}{s}'\n     prefix_3_suffix\n     prefix_1_suffix\n     prefix_2_suffix\n  2. A variable name should start with a letter and be followed by letters, digits, or underscores.\n     A regular expression is used to check them: ^[a-zA-Z][A-Za-z0-9_]*$\n  3. The value could also contain replacement strings.\n     # {p} will be replaced with {%:}, which computes the basename and remove all file extensions.\n     $ echo a/b/c.txt.gz | rush -v 'p={%:}' 'echo {p} {p}.csv'\n     c c.csv\n\nUsage:\n  rush [flags] [command] \n\nExamples:\n  1. simple run, quoting is not necessary\n      $ seq 1 10 | rush echo {}\n  2. keep order\n      $ seq 1 10 | rush 'echo {}' -k\n  3. timeout\n      $ seq 1 | rush 'sleep 2; echo {}' -t 1\n  4. retry\n      $ seq 1 | rush 'python script.py' -r 3\n  5. dirname \u0026 basename \u0026 remove suffix\n      $ echo dir/file_1.txt.gz | rush 'echo {/} {%} {^_1.txt.gz}'\n      dir file.txt.gz dir/file\n  6. basename without the last or any extension\n      $ echo dir.d/file.txt.gz | rush 'echo {.} {:} {%.} {%:}'\n      dir.d/file.txt dir.d/file file.txt file\n  7. job ID, combine fields and other replacement strings\n      $ echo 12 file.txt dir/s_1.fq.gz | rush 'echo job {#}: {2} {2.} {3%:^_1}'\n      job 1: file.txt file s\n  8. capture submatch using regular expression\n      $ echo read_1.fq.gz | rush 'echo {@(.+)_\\d}'\n      read\n  9. custom field delimiter\n      $ echo a=b=c | rush 'echo {1} {2} {3}' -d =\n      a b c\n  10. custom record delimiter\n      $ echo a=b=c | rush -D \"=\" -k 'echo {}'\n      a\n      b\n      c\n      $ echo abc | rush -D \"\" -k 'echo {}'\n      a\n      b\n      c\n  11. assign value to variable, like \"awk -v\"\n      # seq 1 | rush 'echo Hello, {fname} {lname}!' -v fname=Wei,lname=Shen\n      $ seq 1 | rush 'echo Hello, {fname} {lname}!' -v fname=Wei -v lname=Shen\n      Hello, Wei Shen!\n\n      # preset variables support extra operations as well.\n      echo read_1.fq.gz | ./rush -v 'p={:^_1}' -v 'f=a.s-10.txt' 'echo {} {p} {f:} {f@s\\-(\\d+)}'\n      read_1.fq.gz read a 10\n  12. preset variable (Macro)\n      # equal to: echo sample_1.fq.gz | rush 'echo {:^_1} {} {:^_1}_2.fq.gz'\n      $ echo sample_1.fq.gz | rush -v p={:^_1} 'echo {p} {} {p}_2.fq.gz'\n      sample sample_1.fq.gz sample_2.fq.gz\n  13. save successful commands to continue in NEXT run\n      $ seq 1 3 | rush 'sleep {}; echo {}' -c -t 2\n      [INFO] ignore cmd #1: sleep 1; echo 1\n      [ERRO] run cmd #1: sleep 2; echo 2: time out\n      [ERRO] run cmd #2: sleep 3; echo 3: time out\n  14. escape special symbols\n      $ seq 1 | rush 'echo -e \"a\\tb\" | awk \"{print $1}\"' -q\n      a\n  15. escape curly brackets \"{}\"\n      $ echo aaa bbb ccc | sed -E \"s/(\\S){3,}/\\1/g\"\n      a b c\n      $ echo 1 | rush 'echo aaa bbb ccc | sed -E \"s/(\\S){{3,}}/\\1/g\"' --dry-run\n      echo aaa bbb ccc | sed -E \"s/(\\S){3,}/\\1/g\"\n  16. run a command with relative paths in Windows, please use backslash as the separator.\n      # \"brename -l -R\" is used to search paths recursively\n      $ brename -l -q -R -i -p \"\\.go$\" | rush \"bin\\app.exe {}\"\n\n  More examples: https://github.com/shenwei356/rush\n\nFlags:\n  -v, --assign strings            assign the value val to the variable var (format: var=val, val also\n                                  supports replacement strings)\n      --cleanup-time int          time to allow child processes to clean up between stop / kill signals\n                                  (unit: seconds, 0 for no time) (default 3) (default 3)\n  -c, --continue                  continue jobs. NOTES: 1) successful commands are saved in file (given\n                                  by flag -C/--succ-cmd-file); 2) if the file does not exist, rush saves\n                                  data so we can continue jobs next time; 3) if the file exists, rush\n                                  ignores jobs in it and update the file\n      --dry-run                   print command but not run\n  -q, --escape                    escape special symbols like $ which you can customize by flag\n                                  -Q/--escape-symbols\n  -Q, --escape-symbols string     symbols to escape (default \"$#\u0026`\")\n      --eta                       show ETA progress bar\n  -d, --field-delimiter string    field delimiter in records, support regular expression (default \"\\\\s+\")\n  -h, --help                      help for rush\n  -I, --immediate-output          print output immediately and interleaved, to aid debugging\n  -i, --infile strings            input data file, multi-values supported\n  -j, --jobs int                  run n jobs in parallel (default value depends on your device) (default 16)\n  -k, --keep-order                keep output in order of input\n      --no-kill-exes strings      exe names to exclude from kill signal, example: mspdbsrv.exe; or use\n                                  all for all exes (default none)\n      --no-stop-exes strings      exe names to exclude from stop signal, example: mspdbsrv.exe; or use\n                                  all for all exes (default none)\n  -n, --nrecords int              number of records sent to a command (default 1)\n  -o, --out-file string           out file (\"-\" for stdout) (default \"-\")\n      --print-retry-output        print output from retry commands (default true)\n      --propagate-exit-status     propagate child exit status up to the exit status of rush (default true)\n  -D, --record-delimiter string   record delimiter (default is \"\\n\") (default \"\\n\")\n  -J, --records-join-sep string   record separator for joining multi-records (default is \"\\n\") (default \"\\n\")\n  -r, --retries int               maximum retries (default 0)\n      --retry-interval int        retry interval (unit: second) (default 0)\n  -e, --stop-on-error             stop child processes on first error (not perfect, you may stop it by\n                                  typing ctrl-c or closing terminal)\n  -C, --succ-cmd-file string      file for saving successful commands (default \"successful_cmds.rush\")\n  -t, --timeout int               timeout of a command (unit: seconds, 0 for no timeout) (default 0)\n  -T, --trim string               trim white space (\" \\t\\r\\n\") in input (available values: \"l\" for left,\n                                  \"r\" for right, \"lr\", \"rl\", \"b\" for both side)\n      --verbose                   print verbose information\n  -V, --version                   print version information and check for update\n\n```\n\n\n## Examples\n\n1. Simple run, quoting is not necessary\n\n        # seq 1 3 | rush 'echo {}'\n        $ seq 1 3 | rush echo {}\n        3\n        1\n        2\n\n1. Read data from file (`-i`)\n\n        $ rush echo {} -i data1.txt -i data2.txt\n\n1. Keep output order (`-k`)\n\n        $ seq 1 3 | rush 'echo {}' -k\n        1\n        2\n        3\n\n1. Timeout (`-t`)\n\n        $ time seq 1 | rush 'sleep 2; echo {}' -t 1\n        [ERRO] run command #1: sleep 2; echo 1: time out\n\n        real    0m1.010s\n        user    0m0.005s\n        sys     0m0.007s\n\n1. Retry (`-r`)\n\n        $ seq 1 | rush 'python unexisted_script.py' -r 1\n        python: can't open file 'unexisted_script.py': [Errno 2] No such file or directory\n        [WARN] wait command: python unexisted_script.py: exit status 2\n        python: can't open file 'unexisted_script.py': [Errno 2] No such file or directory\n        [ERRO] wait command: python unexisted_script.py: exit status 2\n\n1. Input containing `{}` (since v0.7.0)\n\n        $ echo \"a attr{href}\"=\"h4 text{}\" | rush -T b -k -D \"=\" 'echo \"{}\"'\n        a attr{href}\n        h4 text{}\n\n        $ echo -ne \"a{},b{{}},c{d}\" | rush -D , -k \"echo {}\"\n        a{}\n        b{{}}\n        c{d}\n\n1. Output `{}` itself (since v0.7.0)\n\n        $ echo abc | rush 'echo \"{} {{}}\"'\n        abc {}\n\n1. Dirname (`{/}`) and basename (`{%}`) and remove custom suffix (`{^suffix}`)\n\n        $ echo dir/file_1.txt.gz | rush 'echo {/} {%} {^_1.txt.gz}'\n        dir file_1.txt.gz dir/file\n\n1. Get basename, and remove last (`{.}`) or any (`{:}`) extension\n\n        $ echo dir.d/file.txt.gz | rush 'echo {.} {:} {%.} {%:}'\n        dir.d/file.txt dir.d/file file.txt file\n\n1. Job ID, combine fields index and other replacement strings\n\n        $ echo 12 file.txt dir/s_1.fq.gz | rush 'echo job {#}: {2} {2.} {3%:^_1}'\n        job 1: file.txt file s\n\n1. Capture submatch using regular expression (`{@regexp}`)\n\n        $ echo read_1.fq.gz | rush 'echo {@(.+)_\\d}'\n\n1. Custom field delimiter (`-d`)\n\n        $ echo a=b=c | rush 'echo {1} {2} {3}' -d =\n        a b c\n\n1. Send multi-lines to every command (`-n`)\n\n        $ seq 5 | rush -n 2 -k 'echo \"{}\"; echo'\n        1\n        2\n\n        3\n        4\n\n        5\n\n        # Multiple records are joined with separator `\"\\n\"` (`-J/--records-join-sep`)\n        $ seq 5 | rush -n 2 -k 'echo \"{}\"; echo' -J ' '\n        1 2\n\n        3 4\n\n        5\n\n        $ seq 5 | rush -n 2 -k -j 3 'echo {1}'\n        1\n        3\n        5\n\n1. Custom record delimiter (`-D`), note that empty records are not used.\n\n        $ echo a b c d | rush -D \" \" -k 'echo {}'\n        a\n        b\n        c\n        d\n\n        $ echo abcd | rush -D \"\" -k 'echo {}'\n        a\n        b\n        c\n        d\n\n        # FASTA format\n        $ echo -ne \"\u003eseq1\\nactg\\n\u003eseq2\\nAAAA\\n\u003eseq3\\nCCCC\"\n        \u003eseq1\n        actg\n        \u003eseq2\n        AAAA\n        \u003eseq3\n        CCCC\n\n        $ echo -ne \"\u003eseq1\\nactg\\n\u003eseq2\\nAAAA\\n\u003eseq3\\nCCCC\" | rush -D \"\u003e\" 'echo FASTA record {#}: name: {1} sequence: {2}' -k -d \"\\n\"\n        FASTA record 1: name: seq1 sequence: actg\n        FASTA record 2: name: seq2 sequence: AAAA\n        FASTA record 3: name: seq3 sequence: CCCC\n\n1. **Assign value to variable**, like `awk -v` (`-v`)\n\n        $ seq 1  | rush 'echo Hello, {fname} {lname}!' -v fname=Wei -v lname=Shen\n        Hello, Wei Shen!\n\n        $ seq 1  | rush 'echo Hello, {fname} {lname}!' -v fname=Wei,lname=Shen\n        Hello, Wei Shen!\n\n        $ for var in a b; do \\\n        $   seq 1 3 | rush -k -v var=$var 'echo var: {var}, data: {}'; \\\n        $ done\n        var: a, data: 1\n        var: a, data: 2\n        var: a, data: 3\n        var: b, data: 1\n        var: b, data: 2\n        var: b, data: 3\n        \n1. Preset variables support extra operations as well!!!\n\n        $ echo read_1.fq.gz | ./rush -v 'p={:^_1}' -v 'f=a.s-10.txt' 'echo {} {p} {f:} {f@s\\-(\\d+)}'\n        read_1.fq.gz read a 10\n\n1. **Preset variable** (`-v`), avoid repeatedly writing verbose replacement strings\n\n        # naive way\n        $ echo read_1.fq.gz | rush 'echo {:^_1} {:^_1}_2.fq.gz'\n        read read_2.fq.gz\n\n        # macro + removing suffix\n        $ echo read_1.fq.gz | rush -v p='{:^_1}' 'echo {p} {p}_2.fq.gz'\n\n        # macro + regular expression\n        $ echo read_1.fq.gz | rush -v p='{@(.+?)_\\d}' 'echo {p} {p}_2.fq.gz'\n\n1. Escape special symbols\n\n        $ seq 1 | rush 'echo \"I have $100\"'\n        I have 00\n        $ seq 1 | rush 'echo \"I have $100\"' -q\n        I have $100\n        $ seq 1 | rush 'echo \"I have $100\"' -q --dry-run\n        echo \"I have \\$100\"\n\n        $ seq 1 | rush 'echo -e \"a\\tb\" | awk \"{print $1}\"'\n        a       b\n\n        $ seq 1 | rush 'echo -e \"a\\tb\" | awk \"{print $1}\"' -q\n        a\n\n1. Interrupt jobs by `Ctrl-C`, rush will stop unfinished commands and exit.\n\n        $ seq 1 20 | rush 'sleep 1; echo {}'\n        ^C[CRIT] received an interrupt, stopping unfinished commands...\n        [ERRO] wait cmd #7: sleep 1; echo 7: signal: interrupt\n        [ERRO] wait cmd #5: sleep 1; echo 5: signal: killed\n        [ERRO] wait cmd #6: sleep 1; echo 6: signal: killed\n        [ERRO] wait cmd #8: sleep 1; echo 8: signal: killed\n        [ERRO] wait cmd #9: sleep 1; echo 9: signal: killed\n        1\n        3\n        4\n        2\n\n1. Continue/resume jobs (`-c`). When some jobs failed (by execution failure, timeout,\n    or cancelling by user with `Ctrl + C`),\n    please switch flag `-c/--continue` on and run again,\n    so that `rush` can save successful commands and ignore them in **NEXT** run.\n\n        $ seq 1 3 | rush 'sleep {}; echo {}' -t 3 -c\n        1\n        2\n        [ERRO] run cmd #3: sleep 3; echo 3: time out\n\n        # successful commands:\n        $ cat successful_cmds.rush\n        sleep 1; echo 1__CMD__\n        sleep 2; echo 2__CMD__\n\n        # run again\n        $ seq 1 3 | rush 'sleep {}; echo {}' -t 3 -c\n        [INFO] ignore cmd #1: sleep 1; echo 1\n        [INFO] ignore cmd #2: sleep 2; echo 2\n        [ERRO] run cmd #1: sleep 3; echo 3: time out\n\n    Commands of multi-lines (***Not supported in GNU parallel***)\n\n        $ seq 1 3 | rush 'sleep {}; echo {}; \\\n        echo finish {}' -t 3 -c -C finished.rush\n        1\n        finish 1\n        2\n        finish 2\n        [ERRO] run cmd #3: sleep 3; echo 3; \\\n        echo finish 3: time out\n\n        $ cat finished.rush\n        sleep 1; echo 1; \\\n        echo finish 1__CMD__\n        sleep 2; echo 2; \\\n        echo finish 2__CMD__\n\n        # run again\n        $ seq 1 3 | rush 'sleep {}; echo {}; \\\n        echo finish {}' -t 3 -c -C finished.rush\n        [INFO] ignore cmd #1: sleep 1; echo 1; \\\n        echo finish 1\n        [INFO] ignore cmd #2: sleep 2; echo 2; \\\n        echo finish 2\n        [ERRO] run cmd #1: sleep 3; echo 3; \\\n        echo finish 3: time out\n\n    Commands are saved to file (`-C`) right after it finished, so we can view\n    the check finished jobs:\n\n        grep -c __CMD__ successful_cmds.rush\n\n1. A comprehensive example: downloading 1K+ pages given by three URL list files\n   using `phantomjs save_page.js` (some page contents are dynamicly generated by Javascript,\n   so `wget` does not work). Here I set max jobs number (`-j`) as `20`,\n   each job has a max running time (`-t`) of `60` seconds and `3` retry changes\n   (`-r`). Continue flag `-c` is also switched on, so we can continue unfinished\n   jobs. Luckily, it's accomplished in one run :smile:\n\n        $ for f in $(seq 2014 2016); do \\\n        $    /bin/rm -rf $f; mkdir -p $f; \\\n        $    cat $f.html.txt | rush -v d=$f -d = 'phantomjs save_page.js \"{}\" \u003e {d}/{3}.html' -j 20 -t 60 -r 3 -c; \\\n        $ done\n\n1. A bioinformatics example: mapping with `bwa`, and processing result with `samtools`:\n\n        $ tree raw.cluster.clean.mapping\n        raw.cluster.clean.mapping\n        ├── M1\n        │   ├── M1_1.fq.gz -\u003e ../../raw.cluster.clean/M1/M1_1.fq.gz\n        │   ├── M1_2.fq.gz -\u003e ../../raw.cluster.clean/M1/M1_2.fq.gz\n        ...\n\n        $ ref=ref/xxx.fa\n        $ threads=25\n        $ ls -d raw.cluster.clean.mapping/* \\\n            | rush -v ref=$ref -v j=$threads \\\n                'bwa mem -t {j} -M -a {ref} {}/{%}_1.fq.gz {}/{%}_2.fq.gz \u003e {}/{%}.sam; \\\n                samtools view -bS {}/{%}.sam \u003e {}/{%}.bam; \\\n                samtools sort -T {}/{%}.tmp -@ {j} {}/{%}.bam -o {}/{%}.sorted.bam; \\\n                samtools index {}/{%}.sorted.bam; \\\n                samtools flagstat {}/{%}.sorted.bam \u003e {}/{%}.sorted.bam.flagstat; \\\n                /bin/rm {}/{%}.bam {}/{%}.sam;' \\\n                -j 2 --verbose -c -C mapping.rush\n\n    Since `{}/{%}` appears many times, we can use preset variable (macro) to\n    simplify it:\n\n        $ ls -d raw.cluster.clean.mapping/* \\\n            | rush -v ref=$ref -v j=$threads -v p='{}/{%}' \\\n                'bwa mem -t {j} -M -a {ref} {p}_1.fq.gz {p}_2.fq.gz \u003e {p}.sam; \\\n                samtools view -bS {p}.sam \u003e {p}.bam; \\\n                samtools sort -T {p}.tmp -@ {j} {p}.bam -o {p}.sorted.bam; \\\n                samtools index {p}.sorted.bam; \\\n                samtools flagstat {p}.sorted.bam \u003e {p}.sorted.bam.flagstat; \\\n                /bin/rm {p}.bam {p}.sam;' \\\n                -j 2 --verbose -c -C mapping.rush\n\n\n## Special Cases\n\n- Shell `grep` returns exit code `1` when no matches found.\n`rush` thinks it failed to run.\n Please use `grep foo bar || true` instead of `grep foo bar`.\n\n        $ seq 1 | rush 'echo abc | grep 123'\n        [ERRO] wait cmd #1: echo abc | grep 123: exit status 1\n        $ seq 1 | rush 'echo abc | grep 123 || true'\n\n## Contributors\n\nMain contributors:\n\n- [Wei Shen](https://github.com/shenwei356)\n- @bburgin [Brian Burgin](https://github.com/bburgin) for cross-platform process management.\n- @howeyc [Chris Howey](https://github.com/howeyc) for ETA progress bar.\n\n[Others contributors](https://github.com/shenwei356/rush/graphs/contributors)\n\n\n## Acknowledgements\n\nSpecially thank [@brentp](https://github.com/brentp)\nand his [gargs](https://github.com/brentp/gargs), from which `rush` borrows\nsome ideas.\n\nThank [@bburgin](https://github.com/bburgin) for his contribution on improvement\nof child process management.\n\n## Contact\n\n[Create an issue](https://github.com/shenwei356/rush/issues) to report bugs,\npropose new functions or ask for help.\n\n## License\n\n[MIT License](https://github.com/shenwei356/rush/blob/master/LICENSE)\n\n## Starchart\n\n\u003cimg src=\"https://starchart.cc/shenwei356/rush.svg\" alt=\"Stargazers over time\" style=\"max-width: 100%\"\u003e\n","funding_links":["http://paypal.me/shenwei356"],"categories":["Go","Ranked by starred repositories","Go (531)","bioinformatics","终端","Other"],"sub_categories":["资源传输下载"],"project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fshenwei356%2Frush","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Fshenwei356%2Frush","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fshenwei356%2Frush/lists"}