{"id":33136611,"url":"https://netflix.github.io/genie/","last_synced_at":"2025-11-16T03:00:50.761Z","repository":{"id":40420673,"uuid":"10828921","full_name":"Netflix/genie","owner":"Netflix","description":"Distributed Big Data Orchestration Service","archived":false,"fork":false,"pushed_at":"2025-10-29T00:23:59.000Z","size":228036,"stargazers_count":1756,"open_issues_count":18,"forks_count":373,"subscribers_count":512,"default_branch":"master","last_synced_at":"2025-10-29T02:33:00.084Z","etag":null,"topics":["big-data","bigdata","cloud","configuration","configuration-management","distributed-systems","java","microservice","microservices","netflix-oss","netflixoss","orchestration","spring-boot"],"latest_commit_sha":null,"homepage":"https://netflix.github.io/genie","language":"Java","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"apache-2.0","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/Netflix.png","metadata":{"files":{"readme":"README.md","changelog":"CHANGELOG.md","contributing":null,"funding":null,"license":"LICENSE","code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null,"zenodo":null,"notice":null,"maintainers":null,"copyright":null,"agents":null,"dco":null,"cla":null}},"created_at":"2013-06-20T20:35:56.000Z","updated_at":"2025-10-23T07:12:06.000Z","dependencies_parsed_at":"2022-08-09T20:00:26.256Z","dependency_job_id":"5f103c44-d5b3-45ba-b3a0-b09187116bb0","html_url":"https://github.com/Netflix/genie","commit_stats":{"total_commits":2736,"total_committers":44,"mean_commits":62.18181818181818,"dds":0.5409356725146199,"last_synced_commit":"a117a939874792e4ccbba527f3504a30fea59230"},"previous_names":[],"tags_count":258,"template":false,"template_full_name":null,"purl":"pkg:github/Netflix/genie","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/Netflix%2Fgenie","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/Netflix%2Fgenie/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/Netflix%2Fgenie/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/Netflix%2Fgenie/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/Netflix","download_url":"https://codeload.github.com/Netflix/genie/tar.gz/refs/heads/master","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/Netflix%2Fgenie/sbom","scorecard":{"id":100315,"data":{"date":"2025-08-11","repo":{"name":"github.com/Netflix/genie","commit":"81f36f93b02fe47ee78493039fc1a8dd8c3154eb"},"scorecard":{"version":"v5.2.1-40-gf6ed084d","commit":"f6ed084d17c9236477efd66e5b258b9d4cc7b389"},"score":5.8,"checks":[{"name":"Maintained","score":10,"reason":"17 commit(s) and 0 issue activity found in the last 90 days -- score normalized to 10","details":null,"documentation":{"short":"Determines if the project is \"actively maintained\".","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#maintained"}},{"name":"Packaging","score":-1,"reason":"packaging workflow not detected","details":["Warn: no GitHub/GitLab publishing workflow detected."],"documentation":{"short":"Determines if the project is published as a package that others can easily download, install, easily update, and uninstall.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#packaging"}},{"name":"Dangerous-Workflow","score":10,"reason":"no dangerous workflow patterns detected","details":null,"documentation":{"short":"Determines if the project's GitHub Action workflows avoid dangerous patterns.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#dangerous-workflow"}},{"name":"Code-Review","score":10,"reason":"all changesets reviewed","details":null,"documentation":{"short":"Determines if the project requires human code review before pull requests (aka merge requests) are merged.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#code-review"}},{"name":"CII-Best-Practices","score":0,"reason":"no effort to earn an OpenSSF best practices badge detected","details":null,"documentation":{"short":"Determines if the project has an OpenSSF (formerly CII) Best Practices Badge.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#cii-best-practices"}},{"name":"Token-Permissions","score":0,"reason":"detected GitHub workflow tokens with excessive permissions","details":["Warn: no topLevel permission defined: .github/workflows/genie-build.yml:1","Info: no jobLevel write permissions found"],"documentation":{"short":"Determines if the project's workflows follow the principle of least privilege.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#token-permissions"}},{"name":"Binary-Artifacts","score":9,"reason":"binaries present in source code","details":["Warn: binary detected: gradle/wrapper/gradle-wrapper.jar:1"],"documentation":{"short":"Determines if the project has generated executable (binary) artifacts in the source repository.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#binary-artifacts"}},{"name":"License","score":10,"reason":"license file detected","details":["Info: project has a license file: LICENSE:0","Info: FSF or OSI recognized license: Apache License 2.0: LICENSE:0"],"documentation":{"short":"Determines if the project has defined a license.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#license"}},{"name":"Fuzzing","score":0,"reason":"project is not fuzzed","details":["Warn: no fuzzer integrations found"],"documentation":{"short":"Determines if the project uses fuzzing.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#fuzzing"}},{"name":"Vulnerabilities","score":10,"reason":"0 existing vulnerabilities detected","details":null,"documentation":{"short":"Determines if the project has open, known unfixed vulnerabilities.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#vulnerabilities"}},{"name":"Signed-Releases","score":-1,"reason":"no releases found","details":null,"documentation":{"short":"Determines if the project cryptographically signs release artifacts.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#signed-releases"}},{"name":"Pinned-Dependencies","score":0,"reason":"dependency not pinned by hash detected -- score normalized to 0","details":["Info: Possibly incomplete results: error parsing shell code: invalid parameter name: .github/workflows/genie-build.yml:57","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/genie-build.yml:47: update your workflow using https://app.stepsecurity.io/secureworkflow/Netflix/genie/genie-build.yml/master?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/genie-build.yml:63: update your workflow using https://app.stepsecurity.io/secureworkflow/Netflix/genie/genie-build.yml/master?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/genie-build.yml:66: update your workflow using https://app.stepsecurity.io/secureworkflow/Netflix/genie/genie-build.yml/master?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/genie-build.yml:73: update your workflow using https://app.stepsecurity.io/secureworkflow/Netflix/genie/genie-build.yml/master?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/genie-build.yml:80: update your workflow using https://app.stepsecurity.io/secureworkflow/Netflix/genie/genie-build.yml/master?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/genie-build.yml:87: update your workflow using https://app.stepsecurity.io/secureworkflow/Netflix/genie/genie-build.yml/master?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/genie-build.yml:94: update your workflow using https://app.stepsecurity.io/secureworkflow/Netflix/genie/genie-build.yml/master?enable=pin","Warn: containerImage not pinned by hash: genie-demo/src/main/docker/client/Dockerfile:1: pin your Docker image by updating python:3.10-slim to python:3.10-slim@sha256:81f1cdb3770d54ecfdbddcc52c2125fce674c14a1d976dfd8f65dc0734f9c3c5","Warn: pipCommand not pinned by hash: genie-demo/src/main/docker/client/Dockerfile:5-9","Info:   0 out of   7 GitHub-owned GitHubAction dependencies pinned","Info:   0 out of   1 containerImage dependencies pinned","Info:   0 out of   1 pipCommand dependencies pinned"],"documentation":{"short":"Determines if the project has declared and pinned the dependencies of its build process.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#pinned-dependencies"}},{"name":"Branch-Protection","score":-1,"reason":"internal error: error during branchesHandler.setup: internal error: githubv4.Query: Resource not accessible by integration","details":null,"documentation":{"short":"Determines if the default and release branches are protected with GitHub's branch protection settings.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#branch-protection"}},{"name":"Security-Policy","score":0,"reason":"security policy file not detected","details":["Warn: no security policy file detected","Warn: no security file to analyze","Warn: no security file to analyze","Warn: no security file to analyze"],"documentation":{"short":"Determines if the project has published a security policy.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#security-policy"}},{"name":"SAST","score":0,"reason":"SAST tool is not run on all commits -- score normalized to 0","details":["Warn: 0 commits out of 30 are checked with a SAST tool"],"documentation":{"short":"Determines if the project uses static code analysis.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#sast"}}]},"last_synced_at":"2025-08-15T09:59:32.493Z","repository_id":40420673,"created_at":"2025-08-15T09:59:32.493Z","updated_at":"2025-08-15T09:59:32.493Z"},"host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":284654194,"owners_count":27041729,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2022-07-04T15:15:14.044Z","status":"online","status_checked_at":"2025-11-16T02:00:05.974Z","response_time":65,"last_error":null,"robots_txt_status":"success","robots_txt_updated_at":"2025-07-24T06:49:26.215Z","robots_txt_url":"https://github.com/robots.txt","online":true,"can_crawl_api":true,"host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":["big-data","bigdata","cloud","configuration","configuration-management","distributed-systems","java","microservice","microservices","netflix-oss","netflixoss","orchestration","spring-boot"],"created_at":"2025-11-15T11:00:32.762Z","updated_at":"2025-11-16T03:00:50.753Z","avatar_url":"https://github.com/Netflix.png","language":"Java","funding_links":[],"categories":["DevOps Tools","Tools"],"sub_categories":["Winetricks","Objective-C Tools, Libraries, and Frameworks"],"readme":"# Genie\n\n[![License](https://img.shields.io/github/license/Netflix/genie.svg)](http://www.apache.org/licenses/LICENSE-2.0)\n[![Issues](https://img.shields.io/github/issues/Netflix/genie.svg)](https://github.com/Netflix/genie/issues)\n[![NetflixOSS Lifecycle](https://img.shields.io/osslifecycle/Netflix/genie.svg)]()\n\n## Introduction\n\nGenie is a federated Big Data orchestration and execution engine developed by Netflix.\n\nGenie’s value is best described in terms of the problem it solves.\n\nBig Data infrastructure is complex and ever-evolving.\n\nData consumers (Data Scientists or other applications) need to jump over a lot of hurdles in order to run a simple query:\n - Find, download, install and configure a number of binaries, libraries and tools\n - Point to the correct cluster, using valid configuration and reasonable parameters, some of which are very obscure\n - Manually monitor the query, retrieve its output\n\nWhat works today, may not work tomorrow.\nThe cluster may have moved, the binaries may no longer be compatible, etc.\n\nMultiply this overhead times the number of data consumers, and it adds up to a lot of wasted time (and grief!).\n\nData infrastructure providers face a different set of problems:\n - Users require a lot of help configuring their working setup, which is not easy to debug remotely\n - Infrastructure upgrades and expansion require careful coordination with all users\n\n\nGenie is designed to sit at the boundary of these two worlds, and simplify the lives of people on either side.\n\nA data scientist can “rub the magic lamp” and just say “Genie, run query ‘Q’ using engine SparkSQL against production data”.\nGenie takes care of all the nitty-gritty details. It dynamically assembles the necessary binaries and configurations, execute the job, monitors it, notifies the user of its completion, and makes the output data available for immediate and future use.\n\nProviders of Big data infrastructure work with Genie by making resources available for use (clusters, binaries, etc) and plugging in the magic logic that the user doesn’t need to worry about: which cluster should a given query be routed to? Which version of spark should a given query be executed with? Is this user allowed to access this data? etc.\nMoreover, every job’s details are recorded for later audit or debugging.\n\nGenie is designed from the ground up to be very flexible and customizable.\nFor more details visit the [official documentation](https://netflix.github.io/genie)\n\n## Builds\n\nGenie builds are run on Travis CI [here](https://travis-ci.com/Netflix/genie).\n\n|     Branch     |                                                     Build                                                     |                                                                Coverage (coveralls.io)                                                                 |\n|:--------------:|:-------------------------------------------------------------------------------------------------------------:|:------------------------------------------------------------------------------------------------------------------------------------------------------:|\n| master (4.2.x) | [![Build Status](https://travis-ci.com/Netflix/genie.svg?branch=master)](https://travis-ci.com/Netflix/genie) | [![Coverage Status](https://coveralls.io/repos/github/Netflix/genie/badge.svg?branch=master)](https://coveralls.io/github/Netflix/genie?branch=master) |\n|     4.1.x      | [![Build Status](https://travis-ci.com/Netflix/genie.svg?branch=4.1.x)](https://travis-ci.com/Netflix/genie)  |  [![Coverage Status](https://coveralls.io/repos/github/Netflix/genie/badge.svg?branch=4.1.x)](https://coveralls.io/github/Netflix/genie?branch=4.1.x)  |\n|     4.0.x      | [![Build Status](https://travis-ci.com/Netflix/genie.svg?branch=4.0.x)](https://travis-ci.com/Netflix/genie)  |  [![Coverage Status](https://coveralls.io/repos/github/Netflix/genie/badge.svg?branch=4.0.x)](https://coveralls.io/github/Netflix/genie?branch=4.0.x)  |\n\n## Project structure\n\n### `genie-app`\nSelf-contained Genie service server.\n\n### `genie-agent-app`\nSelf-contained Genie CLI job executor.\n\n### `genie-client`\nGenie client interact with the service via REST API.\n\n### `genie-web`\nThe main server library, can be re-wrapped to inject and override server components.\n\n### `genie-agent`\nThe main agent library, can be re-wrapped to inject and override components.\n\n### `genie-common`, `genie-common-internal`, `genie-common-external`\n\nInternal components libraries shared by the server, agent, and client modules.\n\n### `genie-proto`\n\nProtobuf messages and gRPC services definition shared by server and agent.\nThis is not a public API meant for use by other clients.\n\n### `genie-docs`, `genie-demo`\n\nDocumentation and demo application.\n\n### `genie-test`, `genie-test-web`\n\nTesting classes and utilities shared by other modules.\n\n### `genie-ui`\n\nJavaScript UI to search and visualize jobs, clusters, commands.\n\n### `genie-swagger`\n\nAuto-configuration of [Swagger](https://swagger.io/) via [Spring Fox](https://springfox.github.io/springfox/). Add\nto final deployment artifact of server to enable.\n\n## Artifacts\n\nGenie publishes to [Maven Central](https://search.maven.org/) and [Docker Hub](https://hub.docker.com/r/netflixoss/genie-app/)\n\nRefer to the [demo]() section of the documentations for examples.\nAnd to the [setup]() section for more detailed instructions to set up Genie.\n\n## Python Client\n\nThe [Genie Python client](https://github.com/Netflix/pygenie) is hosted in a different repository.\n\n## Further info\nFor a detailed explanation of Genie architecture, use cases, API documentation, demos, deployment and customization guides, and more, visit the\n[Genie documentation](https://netflix.github.io/genie).\n\n## Contact\n\nTo contact Genie developers with questions and suggestions, please use [GitHub Issues](https://github.com/Netflix/genie/issues)\n","project_url":"https://awesome.ecosyste.ms/api/v1/projects/netflix.github.io%2Fgenie%2F","html_url":"https://awesome.ecosyste.ms/projects/netflix.github.io%2Fgenie%2F","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/netflix.github.io%2Fgenie%2F/lists"}