{"id":15208966,"url":"https://github.com/dimajix/flowman","last_synced_at":"2026-01-19T09:04:13.981Z","repository":{"id":38817194,"uuid":"132567372","full_name":"dimajix/flowman","owner":"dimajix","description":"Flowman is an ETL framework powered by Apache Spark. With its declarative approach, Flowman simplifies the development of complex data pipelines.","archived":false,"fork":false,"pushed_at":"2026-01-12T08:19:29.000Z","size":19116,"stargazers_count":97,"open_issues_count":9,"forks_count":19,"subscribers_count":2,"default_branch":"main","last_synced_at":"2026-01-12T18:03:50.225Z","etag":null,"topics":["apache-spark","big-data","bigdata","data-engineering","etl","flowman","hadoop","scala","spark","sql"],"latest_commit_sha":null,"homepage":"https://flowman.io","language":"Scala","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"apache-2.0","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/dimajix.png","metadata":{"files":{"readme":"README.md","changelog":"CHANGELOG.md","contributing":"CONTRIBUTING.md","funding":null,"license":"LICENSE","code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":"SECURITY.md","support":null,"governance":null,"roadmap":null,"authors":"AUTHORS","dei":null,"publiccode":null,"codemeta":null,"zenodo":null,"notice":"NOTICE","maintainers":null,"copyright":null,"agents":null,"dco":null,"cla":null}},"created_at":"2018-05-08T06:56:06.000Z","updated_at":"2026-01-12T08:19:32.000Z","dependencies_parsed_at":"2024-03-15T11:54:55.730Z","dependency_job_id":"33eb51bf-fcca-4681-b1b5-9493f47c8c67","html_url":"https://github.com/dimajix/flowman","commit_stats":{"total_commits":2085,"total_committers":5,"mean_commits":417.0,"dds":0.05179856115107917,"last_synced_commit":"1eff5120f9e2ec8764b9785d9ac10e4b5d0f51b8"},"previous_names":[],"tags_count":76,"template":false,"template_full_name":null,"purl":"pkg:github/dimajix/flowman","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/dimajix%2Fflowman","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/dimajix%2Fflowman/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/dimajix%2Fflowman/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/dimajix%2Fflowman/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/dimajix","download_url":"https://codeload.github.com/dimajix/flowman/tar.gz/refs/heads/main","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/dimajix%2Fflowman/sbom","scorecard":{"id":343161,"data":{"date":"2025-08-11","repo":{"name":"github.com/dimajix/flowman","commit":"de3e0949e64699ffb935bce0d4986fd3947ebf78"},"scorecard":{"version":"v5.2.1-40-gf6ed084d","commit":"f6ed084d17c9236477efd66e5b258b9d4cc7b389"},"score":4.5,"checks":[{"name":"Packaging","score":-1,"reason":"packaging workflow not detected","details":["Warn: no GitHub/GitLab publishing workflow detected."],"documentation":{"short":"Determines if the project is published as a package that others can easily download, install, easily update, and uninstall.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#packaging"}},{"name":"Code-Review","score":0,"reason":"Found 1/13 approved changesets -- score normalized to 0","details":null,"documentation":{"short":"Determines if the project requires human code review before pull requests (aka merge requests) are merged.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#code-review"}},{"name":"Security-Policy","score":10,"reason":"security policy file detected","details":["Info: security policy file detected: SECURITY.md:1","Info: Found linked content: SECURITY.md:1","Info: Found disclosure, vulnerability, and/or timelines in security policy: SECURITY.md:1","Info: Found text in security policy: SECURITY.md:1"],"documentation":{"short":"Determines if the project has published a security policy.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#security-policy"}},{"name":"Dangerous-Workflow","score":10,"reason":"no dangerous workflow patterns detected","details":null,"documentation":{"short":"Determines if the project's GitHub Action workflows avoid dangerous patterns.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#dangerous-workflow"}},{"name":"Token-Permissions","score":0,"reason":"detected GitHub workflow tokens with excessive permissions","details":["Warn: topLevel 'contents' permission set to 'write': .github/workflows/maven.yml:20","Info: no jobLevel write permissions found"],"documentation":{"short":"Determines if the project's workflows follow the principle of least privilege.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#token-permissions"}},{"name":"Maintained","score":10,"reason":"15 commit(s) and 0 issue activity found in the last 90 days -- score normalized to 10","details":null,"documentation":{"short":"Determines if the project is \"actively maintained\".","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#maintained"}},{"name":"CII-Best-Practices","score":0,"reason":"no effort to earn an OpenSSF best practices badge detected","details":null,"documentation":{"short":"Determines if the project has an OpenSSF (formerly CII) Best Practices Badge.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#cii-best-practices"}},{"name":"License","score":10,"reason":"license file detected","details":["Info: project has a license file: LICENSE:0","Info: FSF or OSI recognized license: Apache License 2.0: LICENSE:0"],"documentation":{"short":"Determines if the project has defined a license.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#license"}},{"name":"Fuzzing","score":0,"reason":"project is not fuzzed","details":["Warn: no fuzzer integrations found"],"documentation":{"short":"Determines if the project uses fuzzing.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#fuzzing"}},{"name":"Binary-Artifacts","score":10,"reason":"no binaries found in the repo","details":null,"documentation":{"short":"Determines if the project has generated executable (binary) artifacts in the source repository.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#binary-artifacts"}},{"name":"Pinned-Dependencies","score":0,"reason":"dependency not pinned by hash detected -- score normalized to 0","details":["Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/maven.yml:28: update your workflow using https://app.stepsecurity.io/secureworkflow/dimajix/flowman/maven.yml/main?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/maven.yml:30: update your workflow using https://app.stepsecurity.io/secureworkflow/dimajix/flowman/maven.yml/main?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/maven.yml:36: update your workflow using https://app.stepsecurity.io/secureworkflow/dimajix/flowman/maven.yml/main?enable=pin","Warn: third-party GitHubAction not pinned by hash: .github/workflows/maven.yml:44: update your workflow using https://app.stepsecurity.io/secureworkflow/dimajix/flowman/maven.yml/main?enable=pin","Warn: containerImage not pinned by hash: docker/Dockerfile:1","Warn: containerImage not pinned by hash: docker/Dockerfile:49","Warn: containerImage not pinned by hash: tests/sqlserver/docker/Dockerfile:1: pin your Docker image by updating mcr.microsoft.com/mssql/server:2019-latest to mcr.microsoft.com/mssql/server:2019-latest@sha256:a1159cf154695d265fc6fd5a93020253759b1b726c503ea6c0f32acd2729f2fe","Info:   0 out of   3 GitHub-owned GitHubAction dependencies pinned","Info:   0 out of   1 third-party GitHubAction dependencies pinned","Info:   0 out of   3 containerImage dependencies pinned"],"documentation":{"short":"Determines if the project has declared and pinned the dependencies of its build process.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#pinned-dependencies"}},{"name":"Branch-Protection","score":-1,"reason":"internal error: error during branchesHandler.setup: internal error: githubv4.Query: Resource not accessible by integration","details":null,"documentation":{"short":"Determines if the default and release branches are protected with GitHub's branch protection settings.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#branch-protection"}},{"name":"Signed-Releases","score":-1,"reason":"no releases found","details":null,"documentation":{"short":"Determines if the project cryptographically signs release artifacts.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#signed-releases"}},{"name":"SAST","score":0,"reason":"SAST tool is not run on all commits -- score normalized to 0","details":["Warn: 0 commits out of 18 are checked with a SAST tool"],"documentation":{"short":"Determines if the project uses static code analysis.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#sast"}},{"name":"Vulnerabilities","score":0,"reason":"51 existing vulnerabilities detected","details":["Warn: Project is vulnerable to: GHSA-5mg8-w23w-74h3","Warn: Project is vulnerable to: GHSA-7g45-4rm6-3mm3","Warn: Project is vulnerable to: GHSA-mvr2-9pj6-7w5j","Warn: Project is vulnerable to: GHSA-4jrv-ppp4-jm57","Warn: Project is vulnerable to: GHSA-735f-pc8j-v9w8","Warn: Project is vulnerable to: GHSA-4gg5-vx3j-xwc7","Warn: Project is vulnerable to: GHSA-77rm-9x9h-xj3g","Warn: Project is vulnerable to: GHSA-g5ww-5jh7-63cx","Warn: Project is vulnerable to: GHSA-h4h5-3hr4-j3g2","Warn: Project is vulnerable to: GHSA-wrvw-hg22-4m67","Warn: Project is vulnerable to: GHSA-968p-4wvh-cqc8","Warn: Project is vulnerable to: GHSA-v6h2-p8h4-qcjw","Warn: Project is vulnerable to: GHSA-3xgq-45jj-v275","Warn: Project is vulnerable to: GHSA-952p-6rrq-rcjv","Warn: Project is vulnerable to: GHSA-rhx6-c78j-4q9w","Warn: Project is vulnerable to: GHSA-7fh5-64p2-3v2j","Warn: Project is vulnerable to: GHSA-c2qf-rxjj-qqgw","Warn: Project is vulnerable to: GHSA-5j4c-8p2g-v4jx","Warn: Project is vulnerable to: GHSA-g3ch-rx76-35fx","Warn: Project is vulnerable to: GHSA-4v9v-hfq4-rm2v","Warn: Project is vulnerable to: GHSA-9jgg-88mc-972h","Warn: Project is vulnerable to: GHSA-3h5v-q93c-6h6q","Warn: Project is vulnerable to: GHSA-3f7h-mf4q-vrm4","Warn: Project is vulnerable to: GHSA-gvpg-vgmx-xg6w","Warn: Project is vulnerable to: GHSA-xwmg-2g98-w7v9","Warn: Project is vulnerable to: GHSA-wxr5-93ph-8wr9","Warn: Project is vulnerable to: GHSA-3832-9276-x7gf","Warn: Project is vulnerable to: GHSA-j288-q9x7-2f5v","Warn: Project is vulnerable to: GHSA-cfxw-4h78-h7fw","Warn: Project is vulnerable to: GHSA-389x-839f-4rhx","Warn: Project is vulnerable to: GHSA-xq3w-v528-46rv","Warn: Project is vulnerable to: GHSA-4g8c-wm8x-jfhw","Warn: Project is vulnerable to: GHSA-4265-ccf5-phj5","Warn: Project is vulnerable to: GHSA-4g9r-vxhx-9pgx","Warn: Project is vulnerable to: GHSA-cgwf-w82q-5jrr","Warn: Project is vulnerable to: GHSA-9w38-p64v-xpmv","Warn: Project is vulnerable to: GHSA-xjp4-hw94-mvp5","Warn: Project is vulnerable to: GHSA-rcjc-c4pj-xxrp","Warn: Project is vulnerable to: GHSA-f5fw-25gw-5m92","Warn: Project is vulnerable to: GHSA-c476-j253-5rgq","Warn: Project is vulnerable to: GHSA-p953-3j66-hg45","Warn: Project is vulnerable to: GHSA-2jc4-r94c-rp7h","Warn: Project is vulnerable to: GHSA-g2fg-mr77-6vrm","Warn: Project is vulnerable to: GHSA-rj7p-rfgp-852x","Warn: Project is vulnerable to: GHSA-7286-pgfv-vxvh","Warn: Project is vulnerable to: GHSA-r978-9m6m-6gm6","Warn: Project is vulnerable to: GHSA-c27h-mcmw-48hv","Warn: Project is vulnerable to: GHSA-r6j9-8759-g62w","Warn: Project is vulnerable to: GHSA-cj7v-27pg-wf7q","Warn: Project is vulnerable to: GHSA-hmr7-m48g-48f6","Warn: Project is vulnerable to: GHSA-qh8g-58pp-2wxh"],"documentation":{"short":"Determines if the project has open, known unfixed vulnerabilities.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#vulnerabilities"}}]},"last_synced_at":"2025-08-18T06:25:50.468Z","repository_id":38817194,"created_at":"2025-08-18T06:25:50.468Z","updated_at":"2025-08-18T06:25:50.468Z"},"host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":286080680,"owners_count":28565001,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2026-01-19T08:53:44.001Z","status":"ssl_error","status_checked_at":"2026-01-19T08:52:40.245Z","response_time":67,"last_error":"SSL_connect returned=1 errno=0 peeraddr=140.82.121.6:443 state=error: unexpected eof while reading","robots_txt_status":"success","robots_txt_updated_at":"2025-07-24T06:49:26.215Z","robots_txt_url":"https://github.com/robots.txt","online":false,"can_crawl_api":true,"host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":["apache-spark","big-data","bigdata","data-engineering","etl","flowman","hadoop","scala","spark","sql"],"created_at":"2024-09-28T07:05:09.582Z","updated_at":"2026-01-19T09:04:13.954Z","avatar_url":"https://github.com/dimajix.png","language":"Scala","readme":"# [![Flowman Logo](docs/images/flowman-favicon.png) Flowman](https://flowman.io)\nThe declarative data build tool based on Apache Spark.\n\n[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)\n[![Documentation](https://readthedocs.org/projects/flowman/badge/?version=latest)](https://flowman.readthedocs.io/en/latest/)\n[![Build](https://github.com/dimajix/flowman/actions/workflows/maven.yml/badge.svg)](https://github.com/dimajix/flowman/actions/workflows/maven.yml)\n\n## 🤔 What is Flowman?\n\nFlowman is a *data build tool* based on [Apache Spark](https://spark.apache.org) that simplifies the act of \nimplementing data transformation logic as part of complex data pipelines. Flowman follows a strict \"everything-as-code\"\napproach, where the whole transformation logic is specified in purely declarative YAML files.\nThese describe all details of the data sources, sinks and data transformations. This is much simpler and efficient\nthan writing Spark jobs in Scala or Python. Flowman will take care of all the technical details of a correct and robust \nimplementation and the developers can concentrate on the data transformations themselves.\n\nIn addition to writing and executing data transformations, Flowman can also be used for managing physical data models, \ni.e. Hive or SQL tables. Flowman can create such tables from a specification with the correct schema and also \nautomatically perform migrations. This helps to \nkeep all aspects (like transformations and schema information) in a single place managed by a single tool.\n\n[![Flowman Diagram](docs/images/flowman-overview.png)](https://flowman.io)\n\n\n## 💪 Noteable Features\n\n* Semantics of a build tool like Maven — just for data instead for applications\n* Declarative syntax in YAML files\n* Data model management (Create, Migrate and Destroy Hive tables, JDBC tables or file based storage)\n* Generation of meaningful data model documentation \n* Flexible expression language for parametrizing a project for different environments (DEV, TEST, PROD)\n* Jobs for managing build targets (like copying files or uploading data via sftp)\n* Automatic data dependency management within the execution of individual jobs\n* Meaningful logging output \u0026 rich set of execution metrics\n* Powerful yet simple command line tools\n* Extendable via Plugins\n\n\n## 💾 Supported Data Sources and Sinks\nFlowman supports a wide range of data sources, for example\n* Various cloud blob storages (S3, ABS, ...)\n* Relational databases (Postgres, Azure SQL, MS SQL Server, MariaDB, ...)\n* Hadoop (HDFS \u0026 Hive)\n* Streaming sources (Kafka)\n\nFor file-based sources and sinks, Flowman supports commonly used file formats like CSV, JSON, Parquet and much more. \nThe official documentation provides an overview of\n[supported connectors](https://docs.flowman.io/en/latest/connectors/index.html).\n\n\n## 📚 Documentation\n\nYou can find the official homepage at [Flowman.io](https://flowman.io)\n and a comprehensive documentation at [Read the Docs](https://docs.flowman.io). \n\n\n## 🤓 How do I use Flowman?\n\n### 1. Install Flowman\n\nYou can set up Flowman by following our step-by-step instructions for [local installations](https://docs.flowman.io/en/latest/setup/installation.html)\nor by starting a [Docker container](https://docs.flowman.io/en/latest/setup/docker.html)\n\n### 2. Create a Project\n\nFlowman will provide some example projects in the `examples` subdirectory, which you can use as a starting point.\n\n### 3. Execute the Project\n\nYou can execute the project interactively by starting the [Flowman Shell](https://docs.flowman.io/en/latest/cli/flowshell/index.html)\n\n\n## 🚀 Installation\n\nYou simply grab an appropriate pre-build package at [GitHub](https://github.com/dimajix/flowman/releases),\nor you can use a Docker image, which is available at [Docker Hub](https://hub.docker.com/repository/docker/dimajix/flowman).\nMore details are described in the [Quickstart Guide](QUICKSTART.md) or in the [official Flowman documentation](https://docs.flowman.io/en/latest/setup/index.html).\n\n\n## 🏗 Building\n\nYou can build your own Flowman version via Maven with\n```shell\nmvn clean install\n```\nPlease also read [BUILDING.md](BUILDING.md) for detailed instructions, specifically on build profiles.\n\n\n## 💙 Community\n\n- [Slack](https://join.slack.com/t/flowman-io/shared_invite/zt-168ltudzp-52cCI1S69OMh7sSajtp~7A): Message us on Slack\n\n\n## 😍 Contributing\n\nYou want to contribute to Flowman? Welcome! Please read [CONTRIBUTING.md](CONTRIBUTING.md) to understand how you can \ncontribute to the project.\n\n* [GitHub issue tracker for Flowman](https://github.com/dimajix/flowman/issues/new) to report feature requests or bugs.\n\n\n## 📄 License\n\nThis project is licensed under Apache License 2.0 - see the [LICENSE](LICENSE) file for details.\n","funding_links":[],"categories":["大数据"],"sub_categories":[],"project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fdimajix%2Fflowman","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Fdimajix%2Fflowman","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fdimajix%2Fflowman/lists"}