{"id":15108956,"url":"https://github.com/googlecloudplatform/dataflowtemplates","last_synced_at":"2026-04-08T02:01:03.945Z","repository":{"id":37677334,"uuid":"120970441","full_name":"GoogleCloudPlatform/DataflowTemplates","owner":"GoogleCloudPlatform","description":"Cloud Dataflow Google-provided templates for solving in-Cloud data tasks","archived":false,"fork":false,"pushed_at":"2026-04-01T07:14:10.000Z","size":32991,"stargazers_count":1289,"open_issues_count":101,"forks_count":1078,"subscribers_count":95,"default_branch":"main","last_synced_at":"2026-04-01T09:08:01.819Z","etag":null,"topics":["apache-beam","bigquery","bigtable","dataflow-templates","google-cloud-dataflow","google-cloud-spanner","google-cloud-storage"],"latest_commit_sha":null,"homepage":"https://cloud.google.com/dataflow/docs/guides/templates/provided-templates","language":"Java","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"apache-2.0","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/GoogleCloudPlatform.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":"CONTRIBUTING.md","funding":null,"license":"LICENSE","code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":".github/CODEOWNERS","security":"SECURITY.md","support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null,"zenodo":null,"notice":null,"maintainers":null,"copyright":null,"agents":null,"dco":null,"cla":null}},"created_at":"2018-02-10T00:31:37.000Z","updated_at":"2026-03-31T10:59:56.000Z","dependencies_parsed_at":"2024-01-19T20:23:57.995Z","dependency_job_id":"c5e6ad15-960c-4273-aeef-9a0d170fbac0","html_url":"https://github.com/GoogleCloudPlatform/DataflowTemplates","commit_stats":{"total_commits":3448,"total_committers":170,"mean_commits":"20.276470588235295","dds":0.8488979118329466,"last_synced_commit":"8ad9a7b72a95321c0026e50acf8bdac36816fef8"},"previous_names":[],"tags_count":250,"template":false,"template_full_name":null,"purl":"pkg:github/GoogleCloudPlatform/DataflowTemplates","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/GoogleCloudPlatform%2FDataflowTemplates","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/GoogleCloudPlatform%2FDataflowTemplates/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/GoogleCloudPlatform%2FDataflowTemplates/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/GoogleCloudPlatform%2FDataflowTemplates/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/GoogleCloudPlatform","download_url":"https://codeload.github.com/GoogleCloudPlatform/DataflowTemplates/tar.gz/refs/heads/main","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/GoogleCloudPlatform%2FDataflowTemplates/sbom","scorecard":{"id":57653,"data":{"date":"2025-08-14T17:28:58Z","repo":{"name":"github.com/GoogleCloudPlatform/DataflowTemplates","commit":"f8ba8dc7747d78486beb4a19eed720898baf2fb0"},"scorecard":{"version":"v5.2.1","commit":"ab2f6e92482462fe66246d9e32f642855a691dc1"},"score":6.6,"checks":[{"name":"Maintained","score":10,"reason":"30 commit(s) and 1 issue activity found in the last 90 days -- score normalized to 10","details":null,"documentation":{"short":"Determines if the project is \"actively maintained\".","url":"https://github.com/ossf/scorecard/blob/ab2f6e92482462fe66246d9e32f642855a691dc1/docs/checks.md#maintained"}},{"name":"Security-Policy","score":10,"reason":"security policy file detected","details":["Info: security policy file detected: SECURITY.md:1","Info: Found linked content: SECURITY.md:1","Info: Found disclosure, vulnerability, and/or timelines in security policy: SECURITY.md:1","Info: Found text in security policy: SECURITY.md:1"],"documentation":{"short":"Determines if the project has published a security policy.","url":"https://github.com/ossf/scorecard/blob/ab2f6e92482462fe66246d9e32f642855a691dc1/docs/checks.md#security-policy"}},{"name":"Code-Review","score":10,"reason":"all changesets reviewed","details":null,"documentation":{"short":"Determines if the project requires human code review before pull requests (aka merge requests) are merged.","url":"https://github.com/ossf/scorecard/blob/ab2f6e92482462fe66246d9e32f642855a691dc1/docs/checks.md#code-review"}},{"name":"Dependency-Update-Tool","score":10,"reason":"update tool detected","details":["Info: detected update tool: Dependabot: .github/dependabot.yml:1"],"documentation":{"short":"Determines if the project uses a dependency update tool.","url":"https://github.com/ossf/scorecard/blob/ab2f6e92482462fe66246d9e32f642855a691dc1/docs/checks.md#dependency-update-tool"}},{"name":"Dangerous-Workflow","score":10,"reason":"no dangerous workflow patterns detected","details":null,"documentation":{"short":"Determines if the project's GitHub Action workflows avoid dangerous patterns.","url":"https://github.com/ossf/scorecard/blob/ab2f6e92482462fe66246d9e32f642855a691dc1/docs/checks.md#dangerous-workflow"}},{"name":"Token-Permissions","score":0,"reason":"detected GitHub workflow tokens with excessive permissions","details":["Info: jobLevel 'actions' permission set to 'read': .github/workflows/scorecards-analysis.yml:20","Info: jobLevel 'contents' permission set to 'read': .github/workflows/scorecards-analysis.yml:21","Warn: topLevel permissions set to 'write-all': .github/workflows/bigtable-pr.yml:47"],"documentation":{"short":"Determines if the project's workflows follow the principle of least privilege.","url":"https://github.com/ossf/scorecard/blob/ab2f6e92482462fe66246d9e32f642855a691dc1/docs/checks.md#token-permissions"}},{"name":"Packaging","score":-1,"reason":"packaging workflow not detected","details":["Warn: no GitHub/GitLab publishing workflow detected."],"documentation":{"short":"Determines if the project is published as a package that others can easily download, install, easily update, and uninstall.","url":"https://github.com/ossf/scorecard/blob/ab2f6e92482462fe66246d9e32f642855a691dc1/docs/checks.md#packaging"}},{"name":"Binary-Artifacts","score":10,"reason":"no binaries found in the repo","details":null,"documentation":{"short":"Determines if the project has generated executable (binary) artifacts in the source repository.","url":"https://github.com/ossf/scorecard/blob/ab2f6e92482462fe66246d9e32f642855a691dc1/docs/checks.md#binary-artifacts"}},{"name":"CII-Best-Practices","score":0,"reason":"no effort to earn an OpenSSF best practices badge detected","details":null,"documentation":{"short":"Determines if the project has an OpenSSF (formerly CII) Best Practices Badge.","url":"https://github.com/ossf/scorecard/blob/ab2f6e92482462fe66246d9e32f642855a691dc1/docs/checks.md#cii-best-practices"}},{"name":"Signed-Releases","score":-1,"reason":"no releases found","details":null,"documentation":{"short":"Determines if the project cryptographically signs release artifacts.","url":"https://github.com/ossf/scorecard/blob/ab2f6e92482462fe66246d9e32f642855a691dc1/docs/checks.md#signed-releases"}},{"name":"Pinned-Dependencies","score":4,"reason":"dependency not pinned by hash detected -- score normalized to 4","details":["Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/bigtable-pr.yml:110: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/bigtable-pr.yml/main?enable=pin","Warn: third-party GitHubAction not pinned by hash: .github/workflows/bigtable-pr.yml:117: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/bigtable-pr.yml/main?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/bigtable-pr.yml:145: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/bigtable-pr.yml/main?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/bigtable-pr.yml:174: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/bigtable-pr.yml/main?enable=pin","Warn: third-party GitHubAction not pinned by hash: .github/workflows/bigtable-pr.yml:181: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/bigtable-pr.yml/main?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/cleanup-spanner-test-infra.yml:38: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/cleanup-spanner-test-infra.yml/main?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/datastream-pr.yml:114: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/datastream-pr.yml/main?enable=pin","Warn: third-party GitHubAction not pinned by hash: .github/workflows/datastream-pr.yml:121: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/datastream-pr.yml/main?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/datastream-pr.yml:149: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/datastream-pr.yml/main?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/datastream-pr.yml:178: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/datastream-pr.yml/main?enable=pin","Warn: third-party GitHubAction not pinned by hash: .github/workflows/datastream-pr.yml:185: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/datastream-pr.yml/main?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/java-pr.yml:112: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/java-pr.yml/main?enable=pin","Warn: third-party GitHubAction not pinned by hash: .github/workflows/java-pr.yml:119: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/java-pr.yml/main?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/java-pr.yml:148: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/java-pr.yml/main?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/java-pr.yml:180: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/java-pr.yml/main?enable=pin","Warn: third-party GitHubAction not pinned by hash: .github/workflows/java-pr.yml:187: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/java-pr.yml/main?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/kafka-pr.yml:176: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/kafka-pr.yml/main?enable=pin","Warn: third-party GitHubAction not pinned by hash: .github/workflows/kafka-pr.yml:183: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/kafka-pr.yml/main?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/kafka-pr.yml:112: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/kafka-pr.yml/main?enable=pin","Warn: third-party GitHubAction not pinned by hash: .github/workflows/kafka-pr.yml:119: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/kafka-pr.yml/main?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/kafka-pr.yml:147: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/kafka-pr.yml/main?enable=pin","Warn: third-party GitHubAction not pinned by hash: .github/workflows/load-tests.yml:45: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/load-tests.yml/main?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/load-tests.yml:53: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/load-tests.yml/main?enable=pin","Warn: third-party GitHubAction not pinned by hash: .github/workflows/load-tests.yml:60: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/load-tests.yml/main?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/run-it-tests-beam-snapshots.yml:118: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/run-it-tests-beam-snapshots.yml/main?enable=pin","Warn: third-party GitHubAction not pinned by hash: .github/workflows/spanner-load-tests.yml:114: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/spanner-load-tests.yml/main?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/spanner-load-tests.yml:122: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/spanner-load-tests.yml/main?enable=pin","Warn: third-party GitHubAction not pinned by hash: .github/workflows/spanner-load-tests.yml:57: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/spanner-load-tests.yml/main?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/spanner-load-tests.yml:65: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/spanner-load-tests.yml/main?enable=pin","Warn: third-party GitHubAction not pinned by hash: .github/workflows/spanner-load-tests.yml:72: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/spanner-load-tests.yml/main?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/spanner-pr.yml:115: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/spanner-pr.yml/main?enable=pin","Warn: third-party GitHubAction not pinned by hash: .github/workflows/spanner-pr.yml:122: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/spanner-pr.yml/main?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/spanner-pr.yml:150: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/spanner-pr.yml/main?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/spanner-pr.yml:179: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/spanner-pr.yml/main?enable=pin","Warn: third-party GitHubAction not pinned by hash: .github/workflows/spanner-pr.yml:186: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/spanner-pr.yml/main?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/spanner-staging-tests.yml:67: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/spanner-staging-tests.yml/main?enable=pin","Warn: third-party GitHubAction not pinned by hash: .github/workflows/spanner-staging-tests.yml:74: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/spanner-staging-tests.yml/main?enable=pin","Warn: third-party GitHubAction not pinned by hash: .github/workflows/spanner-terraform-validator.yml:29: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/spanner-terraform-validator.yml/main?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/spanner-terraform-validator.yml:31: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/spanner-terraform-validator.yml/main?enable=pin","Warn: third-party GitHubAction not pinned by hash: .github/workflows/spanner-terraform-validator.yml:33: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/spanner-terraform-validator.yml/main?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/stale.yml:22: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/stale.yml/main?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/update-docs.yml:19: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/update-docs.yml/main?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/update-docs.yml:22: update your workflow using https://app.stepsecurity.io/secureworkflow/GoogleCloudPlatform/DataflowTemplates/update-docs.yml/main?enable=pin","Warn: containerImage not pinned by hash: python/src/main/python/streaming-llm/Dockerfile:1: pin your Docker image by updating gcr.io/dataflow-templates-base/python311-template-launcher-base to gcr.io/dataflow-templates-base/python311-template-launcher-base@sha256:ddedaa3fcee843e70ea4151db09ad7a6d71aeadcb93e24b8f64269b453d54c7c","Warn: containerImage not pinned by hash: python/src/main/python/word-count-python/Dockerfile:1: pin your Docker image by updating gcr.io/dataflow-templates-base/python311-template-launcher-base to gcr.io/dataflow-templates-base/python311-template-launcher-base@sha256:ddedaa3fcee843e70ea4151db09ad7a6d71aeadcb93e24b8f64269b453d54c7c","Warn: pipCommand not pinned by hash: .github/workflows/cleanup-spanner-test-infra.yml:43","Warn: pipCommand not pinned by hash: .github/workflows/cleanup-spanner-test-infra.yml:44","Info:  53 out of  78 GitHub-owned GitHubAction dependencies pinned","Info:   5 out of  23 third-party GitHubAction dependencies pinned","Info:   0 out of   2 containerImage dependencies pinned","Info:   5 out of   7 pipCommand dependencies pinned"],"documentation":{"short":"Determines if the project has declared and pinned the dependencies of its build process.","url":"https://github.com/ossf/scorecard/blob/ab2f6e92482462fe66246d9e32f642855a691dc1/docs/checks.md#pinned-dependencies"}},{"name":"SAST","score":4,"reason":"SAST tool is not run on all commits -- score normalized to 4","details":["Warn: 12 commits out of 30 are checked with a SAST tool"],"documentation":{"short":"Determines if the project uses static code analysis.","url":"https://github.com/ossf/scorecard/blob/ab2f6e92482462fe66246d9e32f642855a691dc1/docs/checks.md#sast"}},{"name":"Fuzzing","score":0,"reason":"project is not fuzzed","details":["Warn: no fuzzer integrations found"],"documentation":{"short":"Determines if the project uses fuzzing.","url":"https://github.com/ossf/scorecard/blob/ab2f6e92482462fe66246d9e32f642855a691dc1/docs/checks.md#fuzzing"}},{"name":"License","score":10,"reason":"license file detected","details":["Info: project has a license file: LICENSE:0","Info: FSF or OSI recognized license: Apache License 2.0: LICENSE:0"],"documentation":{"short":"Determines if the project has defined a license.","url":"https://github.com/ossf/scorecard/blob/ab2f6e92482462fe66246d9e32f642855a691dc1/docs/checks.md#license"}},{"name":"Branch-Protection","score":-1,"reason":"internal error: error during branchesHandler.setup: internal error: githubv4.Query: Resource not accessible by integration","details":null,"documentation":{"short":"Determines if the default and release branches are protected with GitHub's branch protection settings.","url":"https://github.com/ossf/scorecard/blob/ab2f6e92482462fe66246d9e32f642855a691dc1/docs/checks.md#branch-protection"}},{"name":"CI-Tests","score":10,"reason":"30 out of 30 merged PRs checked by a CI test -- score normalized to 10","details":null,"documentation":{"short":"Determines if the project runs tests before pull requests are merged.","url":"https://github.com/ossf/scorecard/blob/ab2f6e92482462fe66246d9e32f642855a691dc1/docs/checks.md#ci-tests"}},{"name":"Contributors","score":10,"reason":"project has 9 contributing companies or organizations","details":["Info: found contributions from: GoogleCloudPlatform, epam, esporx-tv, google, googlecloudplatform, googlers, hack-commit-push, mongodb, neo4j"],"documentation":{"short":"Determines if the project has a set of contributors from multiple organizations (e.g., companies).","url":"https://github.com/ossf/scorecard/blob/ab2f6e92482462fe66246d9e32f642855a691dc1/docs/checks.md#contributors"}},{"name":"Vulnerabilities","score":0,"reason":"34 existing vulnerabilities detected","details":["Warn: Project is vulnerable to: GHSA-j288-q9x7-2f5v","Warn: Project is vulnerable to: PYSEC-2013-22 / GHSA-27x4-j476-jp5f","Warn: Project is vulnerable to: PYSEC-2025-49 / GHSA-5rjg-fvgr-3xxf","Warn: Project is vulnerable to: GHSA-cx63-2mw6-8hw5","Warn: Project is vulnerable to: PYSEC-2022-43012 / GHSA-r9hx-vwmv-q579","Warn: Project is vulnerable to: GHSA-9548-qrrj-x5pj","Warn: Project is vulnerable to: GHSA-h95x-26f3-88hr","Warn: Project is vulnerable to: GHSA-8qvm-5x2c-j2w7","Warn: Project is vulnerable to: PYSEC-2024-161","Warn: Project is vulnerable to: GHSA-48p4-8xcf-vxj5","Warn: Project is vulnerable to: GHSA-pq67-6m6q-mj2v","Warn: Project is vulnerable to: GHSA-3749-ghw9-m3mg","Warn: Project is vulnerable to: PYSEC-2022-43015 / GHSA-47fc-vmwq-366v","Warn: Project is vulnerable to: PYSEC-2025-41 / GHSA-53q9-r3pm-6pq6","Warn: Project is vulnerable to: PYSEC-2024-252 / GHSA-5pcm-hx3q-hm94","Warn: Project is vulnerable to: GHSA-887c-mr87-cxwp","Warn: Project is vulnerable to: PYSEC-2024-251 / GHSA-pg7h-5qx3-wjr3","Warn: Project is vulnerable to: PYSEC-2024-250","Warn: Project is vulnerable to: PYSEC-2024-259","Warn: Project is vulnerable to: PYSEC-2023-299 / GHSA-282v-666c-3fvg","Warn: Project is vulnerable to: GHSA-37mw-44qp-f5jm","Warn: Project is vulnerable to: GHSA-37q5-v5qm-c9v8","Warn: Project is vulnerable to: PYSEC-2023-300 / GHSA-3863-2447-669p","Warn: Project is vulnerable to: GHSA-6rvg-6v2m-4j46","Warn: Project is vulnerable to: GHSA-9356-575x-2w9m","Warn: Project is vulnerable to: GHSA-fpwr-67px-3qhx","Warn: Project is vulnerable to: PYSEC-2024-229 / GHSA-hxxf-235m-72v3","Warn: Project is vulnerable to: GHSA-jjph-296x-mrcr","Warn: Project is vulnerable to: GHSA-phhr-52qp-3mj4","Warn: Project is vulnerable to: GHSA-q2wp-rjmx-x6x9","Warn: Project is vulnerable to: PYSEC-2025-40 / GHSA-qq3j-4f4f-9583","Warn: Project is vulnerable to: PYSEC-2024-227 / GHSA-qxrp-vhvm-j765","Warn: Project is vulnerable to: PYSEC-2023-301 / GHSA-v68g-wm8c-6x7j","Warn: Project is vulnerable to: PYSEC-2024-228 / GHSA-wrfc-pvp9-mr9g"],"documentation":{"short":"Determines if the project has open, known unfixed vulnerabilities.","url":"https://github.com/ossf/scorecard/blob/ab2f6e92482462fe66246d9e32f642855a691dc1/docs/checks.md#vulnerabilities"}}]},"last_synced_at":"2025-08-15T01:05:57.316Z","repository_id":37677334,"created_at":"2025-08-15T01:05:57.316Z","updated_at":"2025-08-15T01:05:57.316Z"},"host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":286080680,"owners_count":31536473,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2026-04-07T16:28:08.000Z","status":"online","status_checked_at":"2026-04-08T02:00:06.127Z","response_time":54,"last_error":null,"robots_txt_status":"success","robots_txt_updated_at":"2025-07-24T06:49:26.215Z","robots_txt_url":"https://github.com/robots.txt","online":true,"can_crawl_api":true,"host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":["apache-beam","bigquery","bigtable","dataflow-templates","google-cloud-dataflow","google-cloud-spanner","google-cloud-storage"],"created_at":"2024-09-25T22:44:37.568Z","updated_at":"2026-04-08T02:01:03.937Z","avatar_url":"https://github.com/GoogleCloudPlatform.png","language":"Java","readme":"# Google Cloud Dataflow Template Pipelines\n\nThese Dataflow templates are an effort to solve simple, but large, in-Cloud data\ntasks, including data import/export/backup/restore and bulk API operations,\nwithout a development environment. The technology under the hood which makes\nthese operations possible is the\n[Google Cloud Dataflow](https://cloud.google.com/dataflow/) service combined\nwith a set of [Apache Beam](https://beam.apache.org/) SDK templated pipelines.\n\nGoogle is providing this collection of pre-implemented Dataflow templates as a\nreference and to provide easy customization for developers wanting to extend\ntheir functionality.\n\n[![Open in Cloud Shell](http://gstatic.com/cloudssh/images/open-btn.svg)](https://console.cloud.google.com/cloudshell/editor?cloudshell_git_repo=https%3A%2F%2Fgithub.com%2FGoogleCloudPlatform%2FDataflowTemplates.git)\n\n## Status\n\n[![Bigtable PR](https://github.com/GoogleCloudPlatform/DataflowTemplates/actions/workflows/bigtable-pr.yml/badge.svg?event=schedule\u0026\u0026?branch=main)](https://github.com/GoogleCloudPlatform/DataflowTemplates/actions?query=workflow%3A%22Bigtable+PR%22+branch%3Amain+event%3Aschedule)\n[![Datastream PR](https://github.com/GoogleCloudPlatform/DataflowTemplates/actions/workflows/datastream-pr.yml/badge.svg?event=schedule\u0026\u0026?branch=main)](https://github.com/GoogleCloudPlatform/DataflowTemplates/actions?query=workflow%3A%22Datastream+PR%22+branch%3Amain+event%3Aschedule)\n[![Java PR](https://github.com/GoogleCloudPlatform/DataflowTemplates/actions/workflows/java-pr.yml/badge.svg?event=schedule\u0026\u0026?branch=main)](https://github.com/GoogleCloudPlatform/DataflowTemplates/actions?query=workflow%3A%22Java+PR%22+branch%3Amain+event%3Aschedule)\n[![Kafka PR](https://github.com/GoogleCloudPlatform/DataflowTemplates/actions/workflows/kafka-pr.yml/badge.svg?event=schedule\u0026\u0026?branch=main)](https://github.com/GoogleCloudPlatform/DataflowTemplates/actions?query=workflow%3A%22Kafka+PR%22+branch%3Amain+event%3Aschedule)\n[![Load Tests](https://github.com/GoogleCloudPlatform/DataflowTemplates/actions/workflows/load-tests.yml/badge.svg?event=schedule\u0026\u0026?branch=main)](https://github.com/GoogleCloudPlatform/DataflowTemplates/actions?query=workflow%3A%22Load+Tests%22+branch%3Amain+event%3Aschedule)\n[![Run IT Tests with Beam Snapshots](https://github.com/GoogleCloudPlatform/DataflowTemplates/actions/workflows/run-it-tests-beam-snapshots.yml/badge.svg?event=schedule\u0026\u0026?branch=main)](https://github.com/GoogleCloudPlatform/DataflowTemplates/actions?query=workflow%3A%22Run+IT+Tests+with+Beam+Snapshots%22+branch%3Amain+event%3Aschedule)\n[![Spanner Load Tests](https://github.com/GoogleCloudPlatform/DataflowTemplates/actions/workflows/spanner-load-tests.yml/badge.svg?event=schedule\u0026\u0026?branch=main)](https://github.com/GoogleCloudPlatform/DataflowTemplates/actions?query=workflow%3A%22Spanner+Load+Tests%22+branch%3Amain+event%3Aschedule)\n[![Spanner PR](https://github.com/GoogleCloudPlatform/DataflowTemplates/actions/workflows/spanner-pr.yml/badge.svg?event=schedule\u0026\u0026?branch=main)](https://github.com/GoogleCloudPlatform/DataflowTemplates/actions?query=workflow%3A%22Spanner+PR%22+branch%3Amain+event%3Aschedule)\n\n## Note on Default Branch\n\nAs of November 18, 2021, our default branch is now named `main`. This does not\naffect forks. If you would like your fork and its local clone to reflect these\nchanges you can\nfollow [GitHub's branch renaming guide](https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/managing-branches-in-your-repository/renaming-a-branch).\n\n## Getting Help\n\n- Have a bug or a feature request? Please file [a GitHub issue](https://github.com/GoogleCloudPlatform/DataflowTemplates/issues/new/choose).\n\n## Template Pipelines\n\n- Get Started\n    - [Word Count](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Word_Count\u0026type=code)\n- Process Data Continuously (stream)\n    - [Azure Eventhub to Pubsub](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Azure_Eventhub_to_PubSub\u0026type=code)\n    - [Bigtable Change Streams to HBase Replicator](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Bigtable_Change_Streams_to_HBase\u0026type=code)\n    - [Cloud Bigtable change streams to BigQuery](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Bigtable_Change_Streams_to_BigQuery\u0026type=code)\n    - [Cloud Bigtable change streams to Cloud Storage](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Bigtable_Change_Streams_to_Google_Cloud_Storage\u0026type=code)\n    - [Cloud Spanner change streams to BigQuery](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Spanner_Change_Streams_to_BigQuery\u0026type=code)\n    - [Cloud Spanner change streams to Cloud Storage](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Spanner_Change_Streams_to_Google_Cloud_Storage\u0026type=code)\n    - [Cloud Spanner change streams to Pub/Sub](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Spanner_Change_Streams_to_PubSub\u0026type=code)\n    - [Cloud Storage Text to BigQuery (Stream)](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Stream_GCS_Text_to_BigQuery_Flex\u0026type=code)\n    - [Data Masking/Tokenization from Cloud Storage to BigQuery (using Cloud DLP)](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Stream_DLP_GCS_Text_to_BigQuery_Flex\u0026type=code)\n    - [Datastream to BigQuery](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Cloud_Datastream_to_BigQuery\u0026type=code)\n    - [Datastream to Cloud Spanner](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Cloud_Datastream_to_Spanner\u0026type=code)\n    - [Datastream to SQL](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Cloud_Datastream_to_SQL\u0026type=code)\n    - [JMS to Pubsub](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Jms_to_PubSub\u0026type=code)\n    - [Kafka to BigQuery](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Kafka_to_BigQuery\u0026type=code)\n    - [Kafka to BigQuery (YAML)](https://github.com/GoogleCloudPlatform/DataflowTemplates/blob/main/yaml/README_Kafka_to_BigQuery_Yaml.md)\n    - [Kafka to Cloud Storage](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Kafka_to_GCS\u0026type=code)\n    - [Kafka to Iceberg (YAML)](https://github.com/GoogleCloudPlatform/DataflowTemplates/blob/main/yaml/README_Kafka_To_Iceberg_Yaml.md)\n    - [Kinesis To Pubsub](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Kinesis_To_Pubsub\u0026type=code)\n    - [MongoDB (CDC) to BigQuery](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20MongoDB_to_BigQuery_CDC\u0026type=code)\n    - [Mqtt to Pubsub](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Mqtt_to_PubSub\u0026type=code)\n    - [Ordered change stream buffer to Source DB](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Ordered_Changestream_Buffer_to_Sourcedb\u0026type=code)\n    - [Pub/Sub Avro to BigQuery](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20PubSub_Avro_to_BigQuery\u0026type=code)\n    - [Pub/Sub CDC to Bigquery](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20PubSub_CDC_to_BigQuery\u0026type=code)\n    - [Pub/Sub Proto to BigQuery](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20PubSub_Proto_to_BigQuery\u0026type=code)\n    - [Pub/Sub Subscription or Topic to Text Files on Cloud Storage](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Cloud_PubSub_to_GCS_Text_Flex\u0026type=code)\n    - [Pub/Sub Subscription to BigQuery](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20PubSub_to_BigQuery_Flex\u0026type=code)\n    - [Pub/Sub Topic to BigQuery](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20PubSub_to_BigQuery\u0026type=code)\n    - [Pub/Sub to Avro Files on Cloud Storage](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Cloud_PubSub_to_Avro_Flex\u0026type=code)\n    - [Pub/Sub to BigTable (YAML)](https://github.com/GoogleCloudPlatform/DataflowTemplates/blob/main/yaml/README_PubSub_To_BigTable_Yaml.md)\n    - [Pub/Sub to Datadog](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Cloud_PubSub_to_Datadog\u0026type=code)\n    - [Pub/Sub to Elasticsearch](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20PubSub_to_Elasticsearch\u0026type=code)\n    - [Pub/Sub to JDBC](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Pubsub_to_Jdbc\u0026type=code)\n    - [Pub/Sub to Kafka](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20PubSub_to_Kafka\u0026type=code)\n    - [Pub/Sub to MongoDB](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Cloud_PubSub_to_MongoDB\u0026type=code)\n    - [Pub/Sub to Pub/Sub](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Cloud_PubSub_to_Cloud_PubSub\u0026type=code)\n    - [Pub/Sub to Redis](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Cloud_PubSub_to_Redis\u0026type=code)\n    - [Pub/Sub to Splunk](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Cloud_PubSub_to_Splunk\u0026type=code)\n    - [Pub/Sub to Text Files on Cloud Storage](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Cloud_PubSub_to_GCS_Text\u0026type=code)\n    - [Pubsub to JMS](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Pubsub_to_Jms\u0026type=code)\n    - [Spanner Change Streams to Sink](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Spanner_Change_Streams_to_Sink\u0026type=code)\n    - [Synchronizing CDC data to BigQuery](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Cdc_To_BigQuery_Template\u0026type=code)\n    - [Text Files on Cloud Storage to Pub/Sub](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Stream_GCS_Text_to_Cloud_PubSub\u0026type=code)\n- Process Data in Bulk (batch)\n    - [Any SourceDB to Cloud Spanner](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Sourcedb_To_Spanner\u0026type=code)\n    - [AstraDB to BigQuery](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20AstraDB_To_BigQuery\u0026type=code)\n    - [Avro Files on Cloud Storage to Cloud Bigtable](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20GCS_Avro_to_Cloud_Bigtable\u0026type=code)\n    - [Avro Files on Cloud Storage to Cloud Spanner](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20GCS_Avro_to_Cloud_Spanner\u0026type=code)\n    - [BigQuery export to Parquet (via Storage API)](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20BigQuery_to_Parquet\u0026type=code)\n    - [BigQuery to Bigtable](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20BigQuery_to_Bigtable\u0026type=code)\n    - [BigQuery to Datastore](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Cloud_BigQuery_to_Cloud_Datastore\u0026type=code)\n    - [BigQuery to Elasticsearch](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20BigQuery_to_Elasticsearch\u0026type=code)\n    - [BigQuery to MongoDB](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20BigQuery_to_MongoDB\u0026type=code)\n    - [BigQuery to TensorFlow Records](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Cloud_BigQuery_to_GCS_TensorFlow_Records\u0026type=code)\n    - [Cassandra to Cloud Bigtable](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Cassandra_To_Cloud_Bigtable\u0026type=code)\n    - [Cloud Bigtable to Avro Files in Cloud Storage](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Cloud_Bigtable_to_GCS_Avro\u0026type=code)\n    - [Cloud Bigtable to Parquet Files on Cloud Storage](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Cloud_Bigtable_to_GCS_Parquet\u0026type=code)\n    - [Cloud Bigtable to SequenceFile Files on Cloud Storage](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Cloud_Bigtable_to_GCS_SequenceFile\u0026type=code)\n    - [Cloud Spanner to Avro Files on Cloud Storage](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Cloud_Spanner_to_GCS_Avro\u0026type=code)\n    - [Cloud Spanner to Text Files on Cloud Storage](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Spanner_to_GCS_Text\u0026type=code)\n    - [Cloud Storage To Splunk](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20GCS_To_Splunk\u0026type=code)\n    - [Cloud Storage to Elasticsearch](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20GCS_to_Elasticsearch\u0026type=code)\n    - [Dataplex JDBC Ingestion](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Dataplex_JDBC_Ingestion\u0026type=code)\n    - [Dataplex: Convert Cloud Storage File Format](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Dataplex_File_Format_Conversion\u0026type=code)\n    - [Dataplex: Tier Data from BigQuery to Cloud Storage](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Dataplex_BigQuery_to_GCS\u0026type=code)\n    - [Firestore (Datastore mode) to BigQuery](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Firestore_to_BigQuery_Flex\u0026type=code)\n    - [Firestore (Datastore mode) to Text Files on Cloud Storage](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Firestore_to_GCS_Text\u0026type=code)\n    - [Google Ads to BigQuery](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Google_Ads_to_BigQuery\u0026type=code)\n    - [Google Cloud to Neo4j](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Google_Cloud_to_Neo4j\u0026type=code)\n    - [Iceberg to Postgres (YAML)](https://github.com/GoogleCloudPlatform/DataflowTemplates/blob/main/yaml/README_Iceberg_To_Postgres_Yaml.md)\n    - [JDBC to BigQuery](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Jdbc_to_BigQuery\u0026type=code)\n    - [JDBC to BigQuery with BigQuery Storage API support](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Jdbc_to_BigQuery_Flex\u0026type=code)\n    - [JDBC to Pub/Sub](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Jdbc_to_PubSub\u0026type=code)\n    - [MongoDB to BigQuery](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20MongoDB_to_BigQuery\u0026type=code)\n    - [MySQL to BigQuery](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20MySQL_to_BigQuery\u0026type=code)\n    - [MySQL to Iceberg (YAML)](https://github.com/GoogleCloudPlatform/DataflowTemplates/blob/main/yaml/README_MySql_To_Iceberg_Yaml.md)\n    - [Parquet Files on Cloud Storage to Cloud Bigtable](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20GCS_Parquet_to_Cloud_Bigtable\u0026type=code)\n    - [PostgreSQL to BigQuery](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20PostgreSQL_to_BigQuery\u0026type=code)\n    - [PostgreSQL to Iceberg (YAML)](https://github.com/GoogleCloudPlatform/DataflowTemplates/blob/main/yaml/README_Postgres_To_Iceberg_Yaml.md)\n    - [SQLServer to BigQuery](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20SQLServer_to_BigQuery\u0026type=code)\n    - [SQLServer to Iceberg (YAML)](https://github.com/GoogleCloudPlatform/DataflowTemplates/blob/main/yaml/README_SqlServer_To_Iceberg_Yaml.md)\n    - [SequenceFile Files on Cloud Storage to Cloud Bigtable](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20GCS_SequenceFile_to_Cloud_Bigtable\u0026type=code)\n    - [Text Files on Cloud Storage to BigQuery](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20GCS_Text_to_BigQuery\u0026type=code)\n    - [Text Files on Cloud Storage to BigQuery with BigQuery Storage API support](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20GCS_Text_to_BigQuery_Flex\u0026type=code)\n    - [Text Files on Cloud Storage to Cloud Spanner](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20GCS_Text_to_Cloud_Spanner\u0026type=code)\n    - [Text Files on Cloud Storage to Firestore (Datastore mode)](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20GCS_Text_to_Firestore\u0026type=code)\n- Utilities\n    - [Bulk Compress Files on Cloud Storage](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Bulk_Compress_GCS_Files\u0026type=code)\n    - [Bulk Decompress Files on Cloud Storage](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Bulk_Decompress_GCS_Files\u0026type=code)\n    - [Bulk Delete Entities in Firestore (Datastore mode)](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Firestore_to_Firestore_Delete\u0026type=code)\n    - [Convert file formats between Avro, Parquet \u0026 CSV](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20File_Format_Conversion\u0026type=code)\n    - [Streaming Data Generator](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Streaming_Data_Generator\u0026type=code)\n- Legacy Templates\n    - [Bulk Delete Entities in Datastore [Deprecated]](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Datastore_to_Datastore_Delete\u0026type=code)\n    - [Datastore to Text Files on Cloud Storage [Deprecated]](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20Datastore_to_GCS_Text\u0026type=code)\n    - [Text Files on Cloud Storage to Datastore [Deprecated]](https://github.com/search?q=repo%3AGoogleCloudPlatform%2FDataflowTemplates%20GCS_Text_to_Datastore\u0026type=code)\n\nFor documentation on each template's usage and parameters, please see the\nofficial [docs](https://cloud.google.com/dataflow/docs/templates/provided-templates).\n\n## Using UDFs\n\nUser-defined functions (UDFs) allow you to customize a template's functionality\nby providing a short JavaScript function without having to maintain the entire\ncodebase. This is useful in situations which you'd like to rename fields, filter\nvalues, or even transform data formats before output to the destination. All\nUDFs are executed by providing the payload of the element as a string to the\nJavaScript function. You can then use JavaScript's in-built JSON parser or other\nsystem functions to transform the data prior to the pipeline's output. The\nreturn statement of a UDF specifies the payload to pass forward in the pipeline.\nThis should always return a string value. If no value is returned or the\nfunction returns undefined, the incoming record will be filtered from the\noutput.\n\n### UDF Function Specification\n\n| Template              | UDF Input Type | Input Description                               | UDF Output Type | Output Description                                                                                     |\n|-----------------------|----------------|-------------------------------------------------|-----------------|--------------------------------------------------------------------------------------------------------|\n| Datastore Bulk Delete | String         | A JSON string of the entity                     | String          | A JSON string of the entity to delete; filter entities by returning undefined                          |\n| Datastore to Pub/Sub  | String         | A JSON string of the entity                     | String          | The payload to publish to Pub/Sub                                                                      |\n| Datastore to GCS Text | String         | A JSON string of the entity                     | String          | A single-line within the output file                                                                   |\n| GCS Text to BigQuery  | String         | A single-line within the input file             | String          | A JSON string which matches the destination table's schema                                             |\n| Pub/Sub to BigQuery   | String         | A string representation of the incoming payload | String          | A JSON string which matches the destination table's schema                                             |\n| Pub/Sub to Datastore  | String         | A string representation of the incoming payload | String          | A JSON string of the entity to write to Datastore                                                      |\n| Pub/Sub to Splunk     | String         | A string representation of the incoming payload | String          | The event data to be sent to Splunk HEC events endpoint. Must be a string or a stringified JSON object |\n\n## UDF Examples\n\nFor a comprehensive list of samples, please check our [udf-samples](v2/common/src/main/resources/udf-samples) folder.\n\n### Adding fields\n\n```js\n/**\n * A transform which adds a field to the incoming data.\n * @param {string} inJson\n * @return {string} outJson\n */\nfunction transform(inJson) {\n  var obj = JSON.parse(inJson);\n  obj.dataFeed = \"Real-time Transactions\";\n  obj.dataSource = \"POS\";\n  return JSON.stringify(obj);\n}\n```\n\n### Filtering records\n\n```js\n/**\n * A transform function which only accepts 42 as the answer to life.\n * @param {string} inJson\n * @return {string} outJson\n */\nfunction transform(inJson) {\n  var obj = JSON.parse(inJson);\n  // only output objects which have an answer to life of 42.\n  if (obj.hasOwnProperty('answerToLife') \u0026\u0026 obj.answerToLife === 42) {\n    return JSON.stringify(obj);\n  }\n}\n```\n\n## Contributing\n\nTo contribute to the repository, see [CONTRIBUTING.md](./CONTRIBUTING.md).\n\n## Release Process\n\nTemplates are released in a weekly basis (best-effort) as part of the efforts to\nkeep [Google-provided Templates](https://cloud.google.com/dataflow/docs/guides/templates/provided-templates)\nupdated with latest fixes and improvements.\n\nTo learn more about this process, or how you can stage your own changes, see [Release Process](./release-process.md).\n\n## More Information\n\n* [Dataflow](https://cloud.google.com/dataflow/docs/overview) - general Dataflow documentation.\n* [Dataflow Templates](https://cloud.google.com/dataflow/docs/concepts/dataflow-templates) - basic template concepts.\n* [Google-provided Templates](https://cloud.google.com/dataflow/docs/guides/templates/provided-templates) - official documentation for templates provided by Google (the source code is in this repository).\n* Dataflow Cookbook: [Blog](https://cloud.google.com/blog/products/data-analytics/introducing-dataflow-cookbook), [GitHub Repository](https://github.com/GoogleCloudPlatform/dataflow-cookbook) - pipeline examples and practical solutions to common data processing challenges.\n* [Dataflow Metrics Collector](https://github.com/GoogleCloudPlatform/dataflow-metrics-exporter) -  CLI tool to collect dataflow resource \u0026 execution metrics and export to either BigQuery or Google Cloud Storage. Useful for comparison and visualization of the metrics while benchmarking the dataflow pipelines using various data formats, resource configurations etc\n* [Apache Beam](https://beam.apache.org)\n  - [Overview](https://beam.apache.org/use/beam-overview/)\n  - Quickstart: [Java](https://beam.apache.org/get-started/quickstart-java), [Python](https://beam.apache.org/get-started/quickstart-py), [Go](https://beam.apache.org/get-started/quickstart-go)\n  - [Tour of Beam](https://tour.beam.apache.org/) - an interactive tour with learning topics covering core Beam concepts from simple ones to more advanced ones.\n  - [Beam Playground](https://beam.apache.org/get-started/try-beam-playground/) -  an interactive environment to try out Beam transforms and examples without having to install Apache Beam.\n  - [Beam College](https://beamcollege.dev/) - hands-on training and practical tips, including video recordings of Apache Beam and Dataflow Templates lessons.\n  - [Getting Started with Apache Beam - Quest](https://www.cloudskillsboost.google/course_templates/724) - A 5 lab series that provides a Google Cloud certified badge upon completion.\n","funding_links":[],"categories":[],"sub_categories":[],"project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fgooglecloudplatform%2Fdataflowtemplates","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Fgooglecloudplatform%2Fdataflowtemplates","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fgooglecloudplatform%2Fdataflowtemplates/lists"}