{"id":18417377,"url":"https://github.com/stellar/stellar-etl","last_synced_at":"2026-04-02T12:03:53.420Z","repository":{"id":39582322,"uuid":"275098256","full_name":"stellar/stellar-etl","owner":"stellar","description":"Stellar ETL will enable real-time analytics on the Stellar network","archived":false,"fork":false,"pushed_at":"2026-03-24T22:48:32.000Z","size":34067,"stargazers_count":37,"open_issues_count":36,"forks_count":18,"subscribers_count":17,"default_branch":"master","last_synced_at":"2026-03-25T23:16:21.235Z","etag":null,"topics":["bitcoin","blockchain","data-analysis","ethereum","etl-framework","etl-pipeline","stellar","stellar-lumens","stellar-network"],"latest_commit_sha":null,"homepage":"","language":"Go","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"apache-2.0","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/stellar.png","metadata":{"files":{"readme":"README.md","changelog":"CHANGELOG.md","contributing":"CONTRIBUTING.md","funding":null,"license":"LICENSE","code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":".github/CODEOWNERS","security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null,"zenodo":null,"notice":null,"maintainers":null,"copyright":null,"agents":null,"dco":null,"cla":null}},"created_at":"2020-06-26T07:28:28.000Z","updated_at":"2026-03-24T22:25:04.000Z","dependencies_parsed_at":"2023-10-16T19:40:59.570Z","dependency_job_id":"fb2101b7-d0c5-4df4-9e64-d2eb1d8715fb","html_url":"https://github.com/stellar/stellar-etl","commit_stats":null,"previous_names":[],"tags_count":40,"template":false,"template_full_name":null,"purl":"pkg:github/stellar/stellar-etl","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/stellar%2Fstellar-etl","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/stellar%2Fstellar-etl/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/stellar%2Fstellar-etl/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/stellar%2Fstellar-etl/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/stellar","download_url":"https://codeload.github.com/stellar/stellar-etl/tar.gz/refs/heads/master","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/stellar%2Fstellar-etl/sbom","scorecard":{"id":37594,"data":{"date":"2025-08-04","repo":{"name":"github.com/stellar/stellar-etl","commit":"1bbd0d843390ed81cca613e2de53d896b3f7b050"},"scorecard":{"version":"v5.2.1-28-gc1d103a9","commit":"c1d103a9bb9f635ec7260bf9aa0699466fa4be0e"},"score":6.8,"checks":[{"name":"Maintained","score":9,"reason":"9 commit(s) and 2 issue activity found in the last 90 days -- score normalized to 9","details":null,"documentation":{"short":"Determines if the project is \"actively maintained\".","url":"https://github.com/ossf/scorecard/blob/c1d103a9bb9f635ec7260bf9aa0699466fa4be0e/docs/checks.md#maintained"}},{"name":"Code-Review","score":10,"reason":"all changesets reviewed","details":null,"documentation":{"short":"Determines if the project requires human code review before pull requests (aka merge requests) are merged.","url":"https://github.com/ossf/scorecard/blob/c1d103a9bb9f635ec7260bf9aa0699466fa4be0e/docs/checks.md#code-review"}},{"name":"Dangerous-Workflow","score":10,"reason":"no dangerous workflow patterns detected","details":null,"documentation":{"short":"Determines if the project's GitHub Action workflows avoid dangerous patterns.","url":"https://github.com/ossf/scorecard/blob/c1d103a9bb9f635ec7260bf9aa0699466fa4be0e/docs/checks.md#dangerous-workflow"}},{"name":"CII-Best-Practices","score":0,"reason":"no effort to earn an OpenSSF best practices badge detected","details":null,"documentation":{"short":"Determines if the project has an OpenSSF (formerly CII) Best Practices Badge.","url":"https://github.com/ossf/scorecard/blob/c1d103a9bb9f635ec7260bf9aa0699466fa4be0e/docs/checks.md#cii-best-practices"}},{"name":"Binary-Artifacts","score":10,"reason":"no binaries found in the repo","details":null,"documentation":{"short":"Determines if the project has generated executable (binary) artifacts in the source repository.","url":"https://github.com/ossf/scorecard/blob/c1d103a9bb9f635ec7260bf9aa0699466fa4be0e/docs/checks.md#binary-artifacts"}},{"name":"Token-Permissions","score":0,"reason":"detected GitHub workflow tokens with excessive permissions","details":["Warn: jobLevel 'contents' permission set to 'write': .github/workflows/release.yml:20","Warn: no topLevel permission defined: .github/workflows/build.yml:1","Warn: no topLevel permission defined: .github/workflows/codeql.yml:1","Warn: no topLevel permission defined: .github/workflows/integration-tests.yml:1","Warn: no topLevel permission defined: .github/workflows/internal.yml:1","Warn: no topLevel permission defined: .github/workflows/lint-tests.yml:1","Info: topLevel 'contents' permission set to 'read': .github/workflows/release.yml:13"],"documentation":{"short":"Determines if the project's workflows follow the principle of least privilege.","url":"https://github.com/ossf/scorecard/blob/c1d103a9bb9f635ec7260bf9aa0699466fa4be0e/docs/checks.md#token-permissions"}},{"name":"Fuzzing","score":0,"reason":"project is not fuzzed","details":["Warn: no fuzzer integrations found"],"documentation":{"short":"Determines if the project uses fuzzing.","url":"https://github.com/ossf/scorecard/blob/c1d103a9bb9f635ec7260bf9aa0699466fa4be0e/docs/checks.md#fuzzing"}},{"name":"License","score":10,"reason":"license file detected","details":["Info: project has a license file: LICENSE:0","Info: FSF or OSI recognized license: Apache License 2.0: LICENSE:0"],"documentation":{"short":"Determines if the project has defined a license.","url":"https://github.com/ossf/scorecard/blob/c1d103a9bb9f635ec7260bf9aa0699466fa4be0e/docs/checks.md#license"}},{"name":"Pinned-Dependencies","score":3,"reason":"dependency not pinned by hash detected -- score normalized to 3","details":["Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/build.yml:16: update your workflow using https://app.stepsecurity.io/secureworkflow/stellar/stellar-etl/build.yml/master?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/codeql.yml:33: update your workflow using https://app.stepsecurity.io/secureworkflow/stellar/stellar-etl/codeql.yml/master?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/codeql.yml:37: update your workflow using https://app.stepsecurity.io/secureworkflow/stellar/stellar-etl/codeql.yml/master?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/codeql.yml:43: update your workflow using https://app.stepsecurity.io/secureworkflow/stellar/stellar-etl/codeql.yml/master?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/integration-tests.yml:21: update your workflow using https://app.stepsecurity.io/secureworkflow/stellar/stellar-etl/integration-tests.yml/master?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/internal.yml:16: update your workflow using https://app.stepsecurity.io/secureworkflow/stellar/stellar-etl/internal.yml/master?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/internal.yml:18: update your workflow using https://app.stepsecurity.io/secureworkflow/stellar/stellar-etl/internal.yml/master?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/internal.yml:29: update your workflow using https://app.stepsecurity.io/secureworkflow/stellar/stellar-etl/internal.yml/master?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/internal.yml:31: update your workflow using https://app.stepsecurity.io/secureworkflow/stellar/stellar-etl/internal.yml/master?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/lint-tests.yml:20: update your workflow using https://app.stepsecurity.io/secureworkflow/stellar/stellar-etl/lint-tests.yml/master?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/lint-tests.yml:25: update your workflow using https://app.stepsecurity.io/secureworkflow/stellar/stellar-etl/lint-tests.yml/master?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/release.yml:28: update your workflow using https://app.stepsecurity.io/secureworkflow/stellar/stellar-etl/release.yml/master?enable=pin","Warn: containerImage not pinned by hash: docker/Dockerfile:2","Warn: containerImage not pinned by hash: docker/Dockerfile:19: pin your Docker image by updating ubuntu:22.04 to ubuntu:22.04@sha256:1ec65b2719518e27d4d25f104d93f9fac60dc437f81452302406825c46fcc9cb","Warn: containerImage not pinned by hash: docker/Dockerfile.test:2","Info:   0 out of  12 GitHub-owned GitHubAction dependencies pinned","Info:   4 out of   4 third-party GitHubAction dependencies pinned","Info:   0 out of   3 containerImage dependencies pinned"],"documentation":{"short":"Determines if the project has declared and pinned the dependencies of its build process.","url":"https://github.com/ossf/scorecard/blob/c1d103a9bb9f635ec7260bf9aa0699466fa4be0e/docs/checks.md#pinned-dependencies"}},{"name":"Signed-Releases","score":-1,"reason":"no releases found","details":null,"documentation":{"short":"Determines if the project cryptographically signs release artifacts.","url":"https://github.com/ossf/scorecard/blob/c1d103a9bb9f635ec7260bf9aa0699466fa4be0e/docs/checks.md#signed-releases"}},{"name":"Branch-Protection","score":-1,"reason":"internal error: error during branchesHandler.setup: internal error: githubv4.Query: Resource not accessible by integration","details":null,"documentation":{"short":"Determines if the default and release branches are protected with GitHub's branch protection settings.","url":"https://github.com/ossf/scorecard/blob/c1d103a9bb9f635ec7260bf9aa0699466fa4be0e/docs/checks.md#branch-protection"}},{"name":"Security-Policy","score":9,"reason":"security policy file detected","details":["Info: security policy file detected: github.com/stellar/.github/SECURITY.md:1","Info: Found linked content: github.com/stellar/.github/SECURITY.md:1","Warn: One or no descriptive hints of disclosure, vulnerability, and/or timelines in security policy","Info: Found text in security policy: github.com/stellar/.github/SECURITY.md:1"],"documentation":{"short":"Determines if the project has published a security policy.","url":"https://github.com/ossf/scorecard/blob/c1d103a9bb9f635ec7260bf9aa0699466fa4be0e/docs/checks.md#security-policy"}},{"name":"Packaging","score":10,"reason":"packaging workflow detected","details":["Info: Project packages its releases by way of GitHub Actions.: .github/workflows/build.yml:11"],"documentation":{"short":"Determines if the project is published as a package that others can easily download, install, easily update, and uninstall.","url":"https://github.com/ossf/scorecard/blob/c1d103a9bb9f635ec7260bf9aa0699466fa4be0e/docs/checks.md#packaging"}},{"name":"SAST","score":10,"reason":"SAST tool is run on all commits","details":["Info: SAST configuration detected: CodeQL","Info: all commits (30) are checked with a SAST tool"],"documentation":{"short":"Determines if the project uses static code analysis.","url":"https://github.com/ossf/scorecard/blob/c1d103a9bb9f635ec7260bf9aa0699466fa4be0e/docs/checks.md#sast"}},{"name":"Vulnerabilities","score":3,"reason":"7 existing vulnerabilities detected","details":["Warn: Project is vulnerable to: GO-2022-0635","Warn: Project is vulnerable to: GO-2022-0646","Warn: Project is vulnerable to: GO-2025-3487 / GHSA-hcg3-q754-cr77","Warn: Project is vulnerable to: GO-2024-3333","Warn: Project is vulnerable to: GO-2025-3503 / GHSA-qxp5-gwg8-xv66","Warn: Project is vulnerable to: GO-2025-3595 / GHSA-vvgc-356p-c3xw","Warn: Project is vulnerable to: GO-2025-3488 / GHSA-6v2p-p543-phr9"],"documentation":{"short":"Determines if the project has open, known unfixed vulnerabilities.","url":"https://github.com/ossf/scorecard/blob/c1d103a9bb9f635ec7260bf9aa0699466fa4be0e/docs/checks.md#vulnerabilities"}}]},"last_synced_at":"2025-08-14T20:40:38.487Z","repository_id":39582322,"created_at":"2025-08-14T20:40:38.488Z","updated_at":"2025-08-14T20:40:38.488Z"},"host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":286080680,"owners_count":31305971,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2026-04-02T09:48:21.550Z","status":"ssl_error","status_checked_at":"2026-04-02T09:48:19.196Z","response_time":89,"last_error":"SSL_read: unexpected eof while reading","robots_txt_status":"success","robots_txt_updated_at":"2025-07-24T06:49:26.215Z","robots_txt_url":"https://github.com/robots.txt","online":false,"can_crawl_api":true,"host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":["bitcoin","blockchain","data-analysis","ethereum","etl-framework","etl-pipeline","stellar","stellar-lumens","stellar-network"],"created_at":"2024-11-06T04:09:14.510Z","updated_at":"2026-04-02T12:03:53.413Z","avatar_url":"https://github.com/stellar.png","language":"Go","funding_links":[],"categories":[],"sub_categories":[],"readme":"# **Stellar ETL**\n\n[![Apache 2.0 licensed](https://img.shields.io/badge/license-apache%202.0-blue.svg)](LICENSE)\n[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/stellar/stellar-etl)\n\nThe Stellar-ETL is a data pipeline that allows users to extract data from the history of the Stellar network.\n\n## **Table of Contents**\n\n- [Install](#install)\n- [Command Reference](#command-reference)\n  - [Export Commands](#export-commands)\n    - [export_ledgers](#export_ledgers)\n    - [export_transactions](#export_transactions)\n    - [export_operations](#export_operations)\n    - [export_effects](#export_effects)\n    - [export_assets](#export_assets)\n    - [export_trades](#export_trades)\n    - [export_diagnostic_events](#export_diagnostic_events)\n    - [export_ledger_entry_changes](#export_ledger_entry_changes)\n  - [Utility Commands](#utility-commands)\n    - [get_ledger_range_from_times](#get_ledger_range_from_times)\n- [Schemas](#schemas)\n- [Extensions](#extensions)\n  - [Adding New Commands](#adding-new-commands)\n\n\u003cbr\u003e\n\n---\n\n# Install\n\n## **Docker**\n\n1. Download the latest version of docker [Docker](https://www.docker.com/get-started)\n2. Pull the latest stellar-etl Docker image: `docker pull stellar/stellar-etl:latest`\n3. Run the Docker images with the desired stellar-etl command: `docker run stellar/stellar-etl:latest stellar-etl [etl-command] [etl-command arguments]`\n\n## **Manual Installation**\n\n1. Install Golang v1.23.0 or later: https://golang.org/dl/\n2. Ensure that your Go bin has been added to the PATH env variable: `export PATH=$PATH:$(go env GOPATH)/bin`\n3. If using captive-core, download and install Stellar-Core v20.0.0 or later: https://github.com/stellar/stellar-core/blob/master/INSTALL.md\n4. Run `go install github.com/stellar/stellar-etl/v2@latest` to install the ETL\n5. Run export commands to export information about the legder\n\n## **Manual build for local development**\n\n1. Clone this repo `git clone https://github.com/stellar/stellar-etl`\n2. Build stellar-etl with `go build`\n3. Build the docker image locally with `make docker-build`\n4. Run the docker container in interactive mode to run export commands.\n\n```sh\n$ docker run --platform linux/amd64 -it stellar/stellar-etl:latest /bin/bash\n```\n\n5. Run export commands to export information about the legder\n   Example command to export ledger data\n\n```sh\nroot@71890b878fca:/etl/data# stellar-etl export_ledgers --start-ledger 1000 --end-ledger 500000 --output exported_ledgers.txt\n```\n\n\u003e _*Note:*_ If using the GCS datastore, you can run the following to set GCP credentials to use in your shell\n\n```\ngcloud auth login\ngcloud config set project dev-hubble\ngcloud auth application-default login\n```\n\nAdd following to docker run command to pass gcloud credentials to docker container\n\n```\n-e GOOGLE_APPLICATION_CREDENTIALS=/.config/gcp/credentials.json -v \"$HOME/.config/gcloud/application_default_credentials.json\":/.config/gcp/credentials.json:ro\n```\n\n\u003e _*Note:*_ Instructions for installing gcloud can be found [here](https://cloud.google.com/sdk/docs/install-sdk)\n\n\u003cbr\u003e\n\n## **Running Tests**\n\n### Unit tests\n\n```sh\n# Running all unit tests\ngo test -v -cover ./internal/transform\n\n# Running an individual test\ngo test -v -run ^TestTransformAsset$ ./internal/transform\n```\n\n### Integration tests\n\n```sh\n# Running all integration tests\nmake int-test\n\n# Running all integration tests and update golden files\nmake int-test-update\n\n# Above essentially runs following:\ndocker-compose build\ndocker-compose run \\\n-v $(HOME)/.config/gcloud/application_default_credentials.json:/usr/credential.json:ro \\\n-v $(PWD)/testdata:/usr/src/etl/testdata \\\n-e GOOGLE_APPLICATION_CREDENTIALS=/usr/credential.json \\\nintegration-tests \\\ngo test -v ./cmd -timeout 30m -args -update=true\n\n# Running an individual test\ndocker-compose build\ndocker-compose run \\\n-v $(HOME)/.config/gcloud/application_default_credentials.json:/usr/credential.json:ro \\\n-v $(PWD)/testdata:/usr/src/etl/testdata \\\n-e GOOGLE_APPLICATION_CREDENTIALS=/usr/credential.json \\\nintegration-tests \\\ngo test -v -run ^TestExportAssets$ ./cmd -timeout 30m -args -update=true\n```\n\n---\n\n# **Command Reference**\n\n- [Export Commands](#export-commands)\n  - [export_ledgers](#export_ledgers)\n  - [export_transactions](#export_transactions)\n  - [export_operations](#export_operations)\n  - [export_effects](#export_effects)\n  - [export_assets](#export_assets)\n  - [export_trades](#export_trades)\n  - [export_diagnostic_events](#export_diagnostic_events)\n  - [export_ledger_entry_changes](#export_ledger_entry_changes)\n- [Utility Commands](#utility-commands)\n  - [get_ledger_range_from_times](#get_ledger_range_from_times)\n\nEvery command accepts a `-h` parameter, which provides a help screen containing information about the command, its usage, and its flags.\n\nCommands have the option to read from testnet with the `--testnet` flag, from futurenet with the `--futurenet` flag, and defaults to reading from mainnet without any flags.\n\n\u003e _*NOTE:*_ Adding both flags will default to testnet. Each stellar-etl command can only run from one network at a time.\n\n\u003cbr\u003e\n\n---\n\n## **Export Commands**\n\nThese commands export information using the [Ledger Exporter](https://github.com/stellar/go/blob/master/exp/services/ledgerexporter/README.md) output files within a specified datastore (currently [datastore](https://github.com/stellar/go/tree/master/support/datastore) only supports GCS). This allows users to provide a start and end ledger range. The commands in this category export a list of everything that occurred within the provided range. All of the ranges are inclusive.\n\n\u003e _*NOTE:*_ The datastore must contain the expected compressed LedgerCloseMetaBatch XDR binary files as exported from [Ledger Exporter](https://github.com/stellar/go/blob/master/exp/services/ledgerexporter/README.md#exported-files).\n\n#### Common Flags\n\n| Flag           | Description                                                                                   | Default                 |\n| -------------- | --------------------------------------------------------------------------------------------- | ----------------------- |\n| start-ledger   | The ledger sequence number for the beginning of the export period. Defaults to genesis ledger | 2                       |\n| end-ledger     | The ledger sequence number for the end of the export range                                    | 0                       |\n| strict-export  | If set, transform errors will be fatal                                                        | false                   |\n| testnet        | If set, will connect to Testnet instead of Pubnet                                             | false                   |\n| futurenet      | If set, will connect to Futurenet instead of Pubnet                                           | false                   |\n| extra-fields   | Additional fields to append to output jsons. Used for appending metadata                      | ---                     |\n| captive-core   | If set, run captive core to retrieve data. Otherwise use TxMeta file datastore                | false                   |\n| datastore-path | Datastore bucket path to read txmeta files from                                               | ledger-exporter/ledgers |\n| buffer-size    | Buffer size sets the max limit for the number of txmeta files that can be held in memory      | 1000                    |\n| num-workers    | Number of workers to spawn that read txmeta files from the datastore                          | 5                       |\n| retry-limit    | Datastore GetLedger retry limit                                                               | 3                       |\n| retry-wait     | Time in seconds to wait for GetLedger retry                                                   | 5                       |\n\n\u003e _*NOTE:*_ Using captive-core requires a Stellar Core instance that is v20.0.0 or later. The commands use the Core instance to retrieve information about changes from the ledger. More information about the Stellar ledger information can be found [here](https://developers.stellar.org/network/horizon/api-reference/resources).\n\u003e \u003cbr\u003e As the Stellar network grows, the Stellar Core instance has to catch up on an increasingly large amount of information. This catch-up process can add some overhead to the commands in this category. In order to avoid this overhead, run prefer processing larger ranges instead of many small ones, or use unbounded mode.\n\u003e \u003cbr\u003e\u003cbr\u003e Recommended resources for running captive-core within a KubernetesPod:\n\u003e\n\u003e ```\n\u003e {cpu: 3.5, memory: 20Gi, ephemeral-storage: 12Gi}\n\u003e ```\n\n\u003cbr\u003e\n\n---\n\n### **export_ledgers**\n\n```bash\n\u003e stellar-etl export_ledgers --start-ledger 1000 \\\n--end-ledger 500000 --output exported_ledgers.txt\n```\n\nThis command exports ledgers within the provided range.\n\n\u003cbr\u003e\n\n---\n\n### **export_transactions**\n\n```bash\n\u003e stellar-etl export_transactions --start-ledger 1000 \\\n--end-ledger 500000 --output exported_transactions.txt\n```\n\nThis command exports transactions within the provided range.\n\n\u003cbr\u003e\n\n---\n\n### **export_operations**\n\n```bash\n\u003e stellar-etl export_operations --start-ledger 1000 \\\n--end-ledger 500000 --output exported_operations.txt\n```\n\nThis command exports operations within the provided range.\n\n\u003cbr\u003e\n\n---\n\n### **export_effects**\n\n```bash\n\u003e stellar-etl export_effects --start-ledger 1000 \\\n--end-ledger 500000 --output exported_effects.txt\n```\n\nThis command exports effects within the provided range.\n\n\u003cbr\u003e\n\n---\n\n### **export_assets**\n\n```bash\n\u003e stellar-etl export_assets \\\n--start-ledger 1000 \\\n--end-ledger 500000 --output exported_assets.txt\n```\n\nExports the assets that are created from payment operations over a specified ledger range.\n\n\u003cbr\u003e\n\n---\n\n### **export_trades**\n\n```bash\n\u003e stellar-etl export_trades \\\n--start-ledger 1000 \\\n--end-ledger 500000 --output exported_trades.txt\n```\n\nExports trade data within the specified range to an output file\n\n\u003cbr\u003e\n\n---\n\n### **export_diagnostic_events**\n\n```bash\n\u003e stellar-etl export_diagnostic_events \\\n--start-ledger 1000 \\\n--end-ledger 500000 --output export_diagnostic_events.txt\n```\n\nExports diagnostic events data within the specified range to an output file\n\n\u003cbr\u003e\n\n---\n\n### **export_ledger_entry_changes**\n\n```bash\n\u003e stellar-etl export_ledger_entry_changes --start-ledger 1000 \\\n--end-ledger 500000 --output exported_changes_folder/\n```\n\nThis command exports ledger changes within the provided ledger range. Flags can filter which ledger entry types are exported. If no data type flags are set, then by default all types are exported. If any are set, it is assumed that the others should not be exported. If a data type flag is set to false, and no other data type flags are set to true, all types are exported.\n\n```bash\n# This command exports all data types\n\u003e stellar-etl export_ledger_entry_changes --export-accounts=false --start-ledger 1000 \\\n--end-ledger 500000 --output exported_changes_folder/\n```\n\nChanges are exported in batches of a size defined by the `--batch-size` flag. By default, the batch-size parameter is set to 64 ledgers, which corresponds to a five minute period of time. This batch size is convenient because checkpoint ledgers are created every 64 ledgers. Checkpoint ledgers act as anchoring points for the nodes on the network, so it is beneficial to export in multiples of 64.\n\nThis command has two modes: bounded and unbounded.\n\n#### **Bounded**\n\nIf both a start and end ledger are provided, then the command runs in a bounded mode. This means that once all the ledgers in the range are processed and exported, the command shuts down.\n\n#### **Unbounded (Currently Unsupported)**\n\nIf only a start ledger is provided, then the command runs in an unbounded fashion starting from the provided ledger. In this mode, stellar-etl will block and wait for the next sequentially written ledger file in the datastore. Since the changes are continually exported in batches, this process can be continually run in the background in order to avoid the overhead of closing and starting new stellar-etl instances.\n\nThe following are the ledger entry type flags that can be used to export data:\n\n- export-accounts\n- export-trustlines\n- export-offers\n- export-pools\n- export-balances\n- export-contract-code\n- export-contract-data\n- export-config-settings\n- export-ttl\n\n\u003cbr\u003e\n\n---\n\n## **Utility Commands**\n\nThese commands aid in the usage of [Export Commands](#export-commands).\n\n### **get_ledger_range_from_times**\n\n```bash\n\u003e stellar-etl get_ledger_range_from_times \\\n--start-time 2019-09-13T23:00:00+00:00 \\\n--end-time 2019-09-14T13:35:10+00:00 --output exported_range.txt\n```\n\nThis command takes in a start and end time and converts it to a ledger range. The ledger range that is returned will be the smallest possible ledger range that completely covers the provided time period.\n\n\u003cbr\u003e\n\n---\n\n# Schemas\n\nSee https://github.com/stellar/stellar-etl/blob/master/internal/transform/schema.go for the schemas of the data structures that are outputted by the ETL.\n\n\u003cbr\u003e\n\n---\n\n# Extensions\n\nThis section covers some possible extensions or further work that can be done.\n\n## **Adding New Commands**\n\nIn general, in order to add new commands, you need to add these files:\n\n- `export_new_data_structure.go` in the `cmd` folder\n  - This file can be generated with cobra by calling: `cobra add {command}`\n  - This file will parse flags, create output files, get the transformed data from the input package, and then export the data.\n- `export_new_data_structure_test.go` in the `cmd` folder\n  - This file will contain some tests for the newly added command. The `runCLI` function does most of the heavy lifting. All the tests need is the command arguments to test and the desired output.\n  - Test data should be stored in the `testdata/new_data_structure` folder\n- `new_data_structure.go` in the `internal/input` folder\n  - This file will contain the methods needed to extract the new data structure from wherever it is located. This may be the history archives, the bucket list, a captive core instance, or a datastore.\n  - If working with captive core, the methods need to work in the background. There should be methods that export batches of data and send them to a channel. There should be other methods that read from the channel and transform the data so it can be exported.\n- `new_data_structure.go` in the `internal/transform` folder\n  - This file will contain the methods needed to transform the extracted data into a form that is suitable for BigQuery.\n  - The struct definition for the transformed object should be stored in `schemas.go` in the `internal/transform` folder.\n\nA good number of common methods are already written and stored in the `util` package.\n","project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fstellar%2Fstellar-etl","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Fstellar%2Fstellar-etl","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fstellar%2Fstellar-etl/lists"}