{"id":28793034,"url":"https://github.com/cdapio/hadoop_cookbook","last_synced_at":"2025-09-07T10:34:36.166Z","repository":{"id":11378111,"uuid":"13817108","full_name":"cdapio/hadoop_cookbook","owner":"cdapio","description":"Cookbook to install Hadoop 2.0+ using Chef","archived":false,"fork":false,"pushed_at":"2023-02-16T17:51:39.000Z","size":1362,"stargazers_count":82,"open_issues_count":1,"forks_count":77,"subscribers_count":78,"default_branch":"develop","last_synced_at":"2025-07-08T15:05:48.394Z","etag":null,"topics":["chef","chef-cookbook","cookbooks","hadoop","hadoop-cookbook","hbase","hive","spark","zookeeper"],"latest_commit_sha":null,"homepage":"http://supermarket.chef.io/cookbooks/hadoop","language":"Ruby","has_issues":false,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"apache-2.0","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/cdapio.png","metadata":{"files":{"readme":"README.md","changelog":"CHANGELOG.md","contributing":"CONTRIBUTING.md","funding":null,"license":"LICENSE","code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null,"zenodo":null}},"created_at":"2013-10-23T22:56:32.000Z","updated_at":"2024-09-09T16:21:07.000Z","dependencies_parsed_at":"2025-07-08T15:05:53.798Z","dependency_job_id":"8f5117ee-d747-42dd-ad5e-72515c314634","html_url":"https://github.com/cdapio/hadoop_cookbook","commit_stats":null,"previous_names":[],"tags_count":74,"template":false,"template_full_name":null,"purl":"pkg:github/cdapio/hadoop_cookbook","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/cdapio%2Fhadoop_cookbook","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/cdapio%2Fhadoop_cookbook/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/cdapio%2Fhadoop_cookbook/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/cdapio%2Fhadoop_cookbook/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/cdapio","download_url":"https://codeload.github.com/cdapio/hadoop_cookbook/tar.gz/refs/heads/develop","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/cdapio%2Fhadoop_cookbook/sbom","scorecard":{"id":269523,"data":{"date":"2025-08-11","repo":{"name":"github.com/cdapio/hadoop_cookbook","commit":"ca8be70f86110b5936f344eded9b640b6f095d47"},"scorecard":{"version":"v5.2.1-40-gf6ed084d","commit":"f6ed084d17c9236477efd66e5b258b9d4cc7b389"},"score":4.2,"checks":[{"name":"Packaging","score":-1,"reason":"packaging workflow not detected","details":["Warn: no GitHub/GitLab publishing workflow detected."],"documentation":{"short":"Determines if the project is published as a package that others can easily download, install, easily update, and uninstall.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#packaging"}},{"name":"Token-Permissions","score":-1,"reason":"No tokens found","details":null,"documentation":{"short":"Determines if the project's workflows follow the principle of least privilege.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#token-permissions"}},{"name":"Maintained","score":0,"reason":"0 commit(s) and 0 issue activity found in the last 90 days -- score normalized to 0","details":null,"documentation":{"short":"Determines if the project is \"actively maintained\".","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#maintained"}},{"name":"Code-Review","score":9,"reason":"Found 12/13 approved changesets -- score normalized to 9","details":null,"documentation":{"short":"Determines if the project requires human code review before pull requests (aka merge requests) are merged.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#code-review"}},{"name":"Dangerous-Workflow","score":-1,"reason":"no workflows found","details":null,"documentation":{"short":"Determines if the project's GitHub Action workflows avoid dangerous patterns.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#dangerous-workflow"}},{"name":"Pinned-Dependencies","score":-1,"reason":"no dependencies found","details":["Info: Possibly incomplete results: error parsing shell code: \u003e must be followed by a word: templates/default/hadoop-init.erb:0"],"documentation":{"short":"Determines if the project has declared and pinned the dependencies of its build process.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#pinned-dependencies"}},{"name":"CII-Best-Practices","score":0,"reason":"no effort to earn an OpenSSF best practices badge detected","details":null,"documentation":{"short":"Determines if the project has an OpenSSF (formerly CII) Best Practices Badge.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#cii-best-practices"}},{"name":"Binary-Artifacts","score":10,"reason":"no binaries found in the repo","details":null,"documentation":{"short":"Determines if the project has generated executable (binary) artifacts in the source repository.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#binary-artifacts"}},{"name":"Fuzzing","score":0,"reason":"project is not fuzzed","details":["Warn: no fuzzer integrations found"],"documentation":{"short":"Determines if the project uses fuzzing.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#fuzzing"}},{"name":"Vulnerabilities","score":10,"reason":"0 existing vulnerabilities detected","details":null,"documentation":{"short":"Determines if the project has open, known unfixed vulnerabilities.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#vulnerabilities"}},{"name":"License","score":10,"reason":"license file detected","details":["Info: project has a license file: LICENSE:0","Info: FSF or OSI recognized license: Apache License 2.0: LICENSE:0"],"documentation":{"short":"Determines if the project has defined a license.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#license"}},{"name":"Security-Policy","score":0,"reason":"security policy file not detected","details":["Warn: no security policy file detected","Warn: no security file to analyze","Warn: no security file to analyze","Warn: no security file to analyze"],"documentation":{"short":"Determines if the project has published a security policy.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#security-policy"}},{"name":"Signed-Releases","score":0,"reason":"Project has not signed or included provenance with any releases.","details":["Warn: release artifact v2.14.0 not signed: https://api.github.com/repos/cdapio/hadoop_cookbook/releases/10860066","Warn: release artifact 2.13.0 not signed: https://api.github.com/repos/cdapio/hadoop_cookbook/releases/8785808","Warn: release artifact v2.12.0 not signed: https://api.github.com/repos/cdapio/hadoop_cookbook/releases/7731409","Warn: release artifact v2.11.5 not signed: https://api.github.com/repos/cdapio/hadoop_cookbook/releases/7322893","Warn: release artifact v2.11.4 not signed: https://api.github.com/repos/cdapio/hadoop_cookbook/releases/7311435","Warn: release artifact v2.14.0 does not have provenance: https://api.github.com/repos/cdapio/hadoop_cookbook/releases/10860066","Warn: release artifact 2.13.0 does not have provenance: https://api.github.com/repos/cdapio/hadoop_cookbook/releases/8785808","Warn: release artifact v2.12.0 does not have provenance: https://api.github.com/repos/cdapio/hadoop_cookbook/releases/7731409","Warn: release artifact v2.11.5 does not have provenance: https://api.github.com/repos/cdapio/hadoop_cookbook/releases/7322893","Warn: release artifact v2.11.4 does not have provenance: https://api.github.com/repos/cdapio/hadoop_cookbook/releases/7311435"],"documentation":{"short":"Determines if the project cryptographically signs release artifacts.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#signed-releases"}},{"name":"Branch-Protection","score":-1,"reason":"internal error: error during branchesHandler.setup: internal error: githubv4.Query: Resource not accessible by integration","details":null,"documentation":{"short":"Determines if the default and release branches are protected with GitHub's branch protection settings.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#branch-protection"}},{"name":"SAST","score":0,"reason":"SAST tool is not run on all commits -- score normalized to 0","details":["Warn: 0 commits out of 30 are checked with a SAST tool"],"documentation":{"short":"Determines if the project uses static code analysis.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#sast"}}]},"last_synced_at":"2025-08-17T12:59:14.593Z","repository_id":11378111,"created_at":"2025-08-17T12:59:14.593Z","updated_at":"2025-08-17T12:59:14.593Z"},"host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":274026692,"owners_count":25209739,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2022-07-04T15:15:14.044Z","status":"online","status_checked_at":"2025-09-07T02:00:09.463Z","response_time":67,"last_error":null,"robots_txt_status":"success","robots_txt_updated_at":"2025-07-24T06:49:26.215Z","robots_txt_url":"https://github.com/robots.txt","online":true,"can_crawl_api":true,"host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":["chef","chef-cookbook","cookbooks","hadoop","hadoop-cookbook","hbase","hive","spark","zookeeper"],"created_at":"2025-06-18T01:04:21.421Z","updated_at":"2025-09-07T10:34:36.116Z","avatar_url":"https://github.com/cdapio.png","language":"Ruby","funding_links":[],"categories":[],"sub_categories":[],"readme":"# hadoop cookbook\n\n[![Cookbook Version](http://img.shields.io/cookbook/v/hadoop.svg)](https://supermarket.chef.io/cookbooks/hadoop)\n[![Apache License 2.0](http://img.shields.io/badge/license-apache%202.0-green.svg)](http://opensource.org/licenses/Apache-2.0)\n[![Build Status](http://img.shields.io/travis/caskdata/hadoop_cookbook.svg)](http://travis-ci.org/caskdata/hadoop_cookbook)\n[![Code Climate](https://codeclimate.com/github/caskdata/hadoop_cookbook/badges/gpa.svg)](https://codeclimate.com/github/caskdata/hadoop_cookbook)\n\n## Requirements\n\nThis cookbook may work on earlier versions, but these are the minimal tested versions.\n\n* Chef 11.4.0+\n* CentOS 6.4+\n* Debian 6.0+\n* Ubuntu 12.04+\n\nThis cookbook assumes that you have a working Java installation. It has been tested using version `1.21.2` of the `java` cookbook, using Oracle JDK 7. If you plan on using Hive with a database other than the embedded Derby, you will need to provide it and set it up prior to starting Hive Metastore service.\n\n## Usage\n\nThis cookbook is designed to be used with a wrapper cookbook or a role with settings for configuring Hadoop. The services should work out of the box on a single host, but little validation is done that you have made a working Hadoop configuration. The cookbook is attribute-driven and is suitable for use via either `chef-client` or `chef-solo` since it does not use any server-based functionality. The cookbook defines service definitions for each Hadoop service, but it does not enable or start them, by default.\n\nFor more information, read the [Wrapping this cookbook](https://github.com/caskdata/hadoop_cookbook/wiki/Wrapping-this-cookbook) wiki entry.\n\n## Attributes\n\nAttributes for this cookbook define the configuration files for Hadoop and its various services. Hadoop configuration files are XML files, with name/value property pairs. The attribute name determines which file the property is placed and the property name. The attribute value is the property value. The attribute `hadoop['core_site']['fs.defaultFS']` will configure a property named `fs.defaultFS` in `core-site.xml` in `hadoop['conf_dir']`. All attribute values are taken as-is and only minimal configuration checking is done on values. It is up to the user to provide a valid configuration for your cluster.\n\nAttribute Tree | File | Location\n-------------- | ---- | --------\nflume['flume_conf'] | flume.conf | `flume['conf_dir']`\nhadoop['capacity_scheduler'] | capacity-scheduler.xml | `hadoop['conf_dir']`\nhadoop['container_executor'] | container-executor.cfg | `hadoop['conf_dir']`\nhadoop['core_site'] | core-site.xml | `hadoop['conf_dir']`\nhadoop['fair_scheduler'] | fair-scheduler.xml | `hadoop['conf_dir']`\nhadoop['hadoop_env'] | hadoop-env.sh | `hadoop['conf_dir']`\nhadoop['hadoop_metrics'] | hadoop-metrics.properties | `hadoop['conf_dir']`\nhadoop['hadoop_policy'] | hadoop-policy.xml | `hadoop['conf_dir']`\nhadoop['hdfs_site'] | hdfs-site.xml | `hadoop['conf_dir']`\nhadoop['log4j'] | log4j.properties | `hadoop['conf_dir']`\nhadoop['mapred_env'] | mapred-env.sh | `hadoop['conf_dir']`\nhadoop['mapred_site'] | mapred-site.xml | `hadoop['conf_dir']`\nhadoop['yarn_env'] | yarn-env.sh | `hadoop['conf_dir']`\nhadoop['yarn_site'] | yarn-site.xml | `hadoop['conf_dir']`\nhbase['hadoop_metrics'] | hadoop-metrics.properties | `hbase['conf_dir']`\nhbase['hbase_env'] | hbase-env.sh | `hbase['conf_dir']`\nhbase['hbase_policy'] | hbase-policy.xml | `hbase['conf_dir']`\nhbase['hbase_site'] | hbase-site.xml | `hbase['conf_dir']`\nhbase['jaas'] | jaas.conf | `hbase['conf_dir']`\nhbase['log4j'] | log4j.properties | `hbase['conf_dir']`\nhive['hive_env'] | hive-env.sh | `hive['conf_dir']`\nhive['hive_site'] | hive-site.xml | `hive['conf_dir']`\nhive['jaas'] | jaas.conf | `hive['conf_dir']`\nhive2['hive_env'] | hive-env.sh | `hive2['conf_dir']`\nhive2['hive_site'] | hive-site.xml | `hive2['conf_dir']`\nhive2['jaas'] | jaas.conf | `hive2['conf_dir']`\noozie['oozie_env'] | oozie-env.sh | `oozie['conf_dir']`\noozie['oozie_site'] | oozie-site.xml | `oozie['conf_dir']`\nspark['log4j'] | log4j.properties | `spark['conf_dir']`\nspark['metrics'] | metrics.properties | `spark['conf_dir']`\nspark['spark_env'] | spark-env.sh | `spark['conf_dir']`\nstorm['storm_env'] | storm-env.sh | `storm['conf_dir']`\nstorm['storm_env'] | storm_env.ini | `storm['conf_dir']`\nstorm['storm_conf'] | storm.yaml | `storm['conf_dir']`\ntez['tez_env'] | tez-env.sh | `tez['conf_dir']`\ntez['tez_site'] | tez-site.xml | `tez['conf_dir']`\nzookeeper['jaas'] | jaas.conf | `zookeeper['conf_dir']`\nzookeeper['log4j'] | log4j.properties | `zookeeper['conf_dir']`\nzookeeper['zoocfg'] | zoo.cfg | `zookeeper['conf_dir']`\n\n### Distribution Attributes\n\n* `hadoop['distribution']` - Specifies which Hadoop distribution to use, currently supported: cdh, hdp, bigtop. Default `hdp`\n* `hadoop['distribution_version']` - Specifies which version of `hadoop['distribution']` to use. Default `2.0` if `hadoop['distribution']` is `hdp`, `5` if `hadoop['distribution']` is `cdh`, and `0.8.0` if `hadoop['distribution']` is `bigtop`.  It can also be set to `develop` when `hadoop['distribution']` is `bigtop` to allow installing from development repos without gpg validation.\n\n#### APT-specific settings\n\n* `hadoop['apt_repo_url']` - Provide an alternate apt installation source location. If you change this attribute, you are expected to provide a path to a working repo for the `hadoop['distribution']` used. Default: `nil`\n* `hadoop['apt_repo_key_url']` - Provide an alternative apt repository key source location. Default `nil`\n\n#### RPM-specific settings\n\n* `hadoop['yum_repo_url']` - Provide an alternate yum installation source location. If you change this attribute, you are expected to provide a path to a working repo for the `hadoop['distribution']` used. Default: `nil`\n* `hadoop['yum_repo_key_url']` - Provide an alternative yum repository key source location. Default `nil`\n\n### Global Configuration Attributes\n\n* `hadoop['conf_dir']` - The directory used inside `/etc/hadoop` and used via the alternatives system. Default `conf.chef`\n* `hbase['conf_dir']` - The directory used inside `/etc/hbase` and used via the alternatives system. Default `conf.chef`\n* `hive['conf_dir']` - The directory used inside `/etc/hive` and used via the alternatives system. Default `conf.chef`\n* `oozie['conf_dir']` - The directory used inside `/etc/oozie` and used via the alternatives system. Default `conf.chef`\n* `tez['conf_dir']` - The directory used inside `/etc/tez` and used via the alternatives system. Default `conf.chef`\n* `spark['conf_dir']` - The directory used inside `/etc/spark` and used via the alternatives system. Default `conf.chef`\n* `storm['conf_dir']` - The directory used inside `/etc/storm` and used via the alternatives system. Default `conf.chef`\n* `zookeeper['conf_dir']` - The directory used inside `/etc/zookeeper` and used via the alternatives system. Default `conf.chef`\n\n### Default Attributes\n\n* `hadoop['core_site']['fs.defaultFS']` - Sets URI to HDFS NameNode. Default `hdfs://localhost`\n* `hadoop['yarn_site']['yarn.resourcemanager.hostname']` - Sets hostname of YARN ResourceManager. Default `localhost`\n* `hive['hive_site']['javax.jdo.option.ConnectionURL']` - Sets JDBC URL. Default `jdbc:derby:;databaseName=/var/lib/hive/metastore/metastore_db;create=true`\n* `hive['hive_site']['javax.jdo.option.ConnectionDriverName']` - Sets JDBC Driver. Default `org.apache.derby.jdbc.EmbeddedDriver`\n\n## Recipes\n\n* `default.rb` - Sets up configuration and `hadoop-client` packages.\n* `hadoop_hdfs_checkconfig` - Ensures the HDFS configuration meets required parameters.\n* `hadoop_hdfs_datanode` - Sets up an HDFS DataNode.\n* `hadoop_hdfs_ha_checkconfig` - Ensures the HDFS configuration meets requirements for High Availability.\n* `hadoop_hdfs_journalnode` - Sets up an HDFS JournalNode.\n* `hadoop_hdfs_namenode` - Sets up an HDFS NameNode.\n* `hadoop_hdfs_secondarynamenode` - Sets up an HDFS Secondary NameNode.\n* `hadoop_hdfs_zkfc` - Sets up HDFS Failover Controller, required for automated NameNode failover.\n* `hadoop_yarn_nodemanager` - Sets up a YARN NodeManager.\n* `hadoop_yarn_proxyserver` - Sets up a YARN Web Proxy.\n* `hadoop_yarn_resourcemanager` - Sets up a YARN ResourceManager.\n* `hbase` - Sets up configuration and `hbase` packages.\n* `hbase_checkconfig` - Ensures the HBase configuration meets required parameters.\n* `hbase_master` - Sets up an HBase Master.\n* `hbase_regionserver` - Sets up an HBase RegionServer.\n* `hbase_rest` - Sets up an HBase REST interface.\n* `hbase_thrift` - Sets up an HBase Thrift interface.\n* `hive` - Sets up configuration and `hive` packages.\n* `hive_metastore` - Sets up Hive Metastore metadata repository.\n* `hive_server` - Sets up a Hive Thrift service.\n* `hive_server2` - Sets up a Hive Thrift service with Kerberos and multi-client concurrency support.\n* `oozie` - Sets up an Oozie server.\n* `oozie_client` - Sets up an Oozie client.\n* `pig` - Installs pig interpreter.\n* `repo` - Sets up package manager repositories for specified `hadoop['distribution']`\n* `spark` - Sets up configuration and `spark-core` packages.\n* `spark_master` - Sets up a Spark Master.\n* `spark_worker` - Sets up a Spark Worker.\n* `storm` - Sets up `storm` package.\n* `storm_nimbus` - Setups a Storm Nimbus server.\n* `storm_supervisor` - Setups a Storm Supervisor server.\n* `storm_ui` - Setups a Storm UI server.\n* `tez` - Sets up configuration  and `tez` packages.\n* `zookeeper` - Sets up `zookeeper` package.\n* `zookeeper_server` - Sets up a ZooKeeper server.\n\n## Author\n\nAuthor:: Cask Data, Inc. (\u003cops@cask.co\u003e)\n\n## Testing\n\nThis cookbook has several ways to test it. It includes code tests, which are done using `foodcritic`, `rubocop`, and `chefspec`.\nIt, also, includes functionality testing, provided by `kitchen`.\n\n```text\nrake chefspec     # Run RSpec code examples\nrake foodcritic   # Foodcritic linter\nrake integration  # Run Test Kitchen integration tests\nrake metadata     # Create metadata.json from metadata.rb\nrake rubocop      # Ruby style guide linter\nrake share        # Share cookbook to community site\n```\n\n## License\n\nLicensed under the Apache License, Version 2.0 (the \"License\"); you may not use this software except in compliance with the License. You may obtain a copy of the License at\n\nhttp://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.\n","project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fcdapio%2Fhadoop_cookbook","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Fcdapio%2Fhadoop_cookbook","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fcdapio%2Fhadoop_cookbook/lists"}