{"id":24805025,"url":"https://github.com/deas/r00ki","last_synced_at":"2026-04-13T17:32:04.588Z","repository":{"id":274089852,"uuid":"905879298","full_name":"deas/r00ki","owner":"deas","description":"R00ki : Taking Rook Ceph from localhost to Prod","archived":false,"fork":false,"pushed_at":"2025-04-28T13:35:24.000Z","size":3185,"stargazers_count":0,"open_issues_count":0,"forks_count":0,"subscribers_count":1,"default_branch":"main","last_synced_at":"2025-08-12T23:35:47.539Z","etag":null,"topics":["ceph","kubernetes","rook"],"latest_commit_sha":null,"homepage":"","language":"Python","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":null,"status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/deas.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":null,"funding":null,"license":null,"code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null,"zenodo":null}},"created_at":"2024-12-19T17:51:06.000Z","updated_at":"2025-04-28T13:34:06.000Z","dependencies_parsed_at":null,"dependency_job_id":"71ff1bf7-8a8d-4a0d-bdc9-9041ab637b7e","html_url":"https://github.com/deas/r00ki","commit_stats":null,"previous_names":["deas/r00ki"],"tags_count":0,"template":false,"template_full_name":null,"purl":"pkg:github/deas/r00ki","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/deas%2Fr00ki","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/deas%2Fr00ki/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/deas%2Fr00ki/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/deas%2Fr00ki/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/deas","download_url":"https://codeload.github.com/deas/r00ki/tar.gz/refs/heads/main","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/deas%2Fr00ki/sbom","scorecard":null,"host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":286080680,"owners_count":31762496,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2026-04-13T15:25:13.801Z","status":"ssl_error","status_checked_at":"2026-04-13T15:25:09.162Z","response_time":93,"last_error":"SSL_read: unexpected eof while reading","robots_txt_status":"success","robots_txt_updated_at":"2025-07-24T06:49:26.215Z","robots_txt_url":"https://github.com/robots.txt","online":false,"can_crawl_api":true,"host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":["ceph","kubernetes","rook"],"created_at":"2025-01-30T07:16:30.527Z","updated_at":"2026-04-13T17:32:04.562Z","avatar_url":"https://github.com/deas.png","language":"Python","funding_links":[],"categories":[],"sub_categories":[],"readme":"\u003c!-- Improved compatibility of back to top link: See: https://github.com/othneildrew/Best-README-Template/pull/73 --\u003e\n\u003ca id=\"readme-top\"\u003e\u003c/a\u003e\n\u003c!-- PROJECT SHIELDS --\u003e\n\u003c!--\n*** I'm using markdown \"reference style\" links for readability.\n*** Reference links are enclosed in brackets [ ] instead of parentheses ( ).\n*** See the bottom of this document for the declaration of the reference variables\n*** for contributors-url, forks-url, etc. This is an optional, concise syntax you may use.\n*** https://www.markdownguide.org/basic-syntax/#reference-style-links\n--\u003e\n\u003c!--\n[![Contributors][contributors-shield]][contributors-url]\n[![Forks][forks-shield]][forks-url]\n[![Stargazers][stars-shield]][stars-url]\n--\u003e\n\u003c!--\n[![Issues][issues-shield]][issues-url]\n[![MIT License][license-shield]][license-url]\n--\u003e\n\n\u003c!-- PROJECT LOGO --\u003e\n\u003cbr /\u003e\n\u003cdiv align=\"center\"\u003e\n  \u003c!--a href=\"https://github.com/deas/r00ki\"\u003e\n    \u003cimg src=\"images/logo.png\" alt=\"Logo\" width=\"80\" height=\"80\"\u003e\n  \u003c/a--\u003e\n\n\u003ch3 align=\"center\"\u003eR00ki : Taking Rook Ceph from localhost to Production 🧪\u003c/h3\u003e\n\n  \u003cp align=\"center\"\u003e\n    \u003c!--project_description\n    \u003cbr /--\u003e\n    \u003c!--a href=\"https://github.com/deas/r00ki\"\u003e\u003cstrong\u003eExplore the docs »\u003c/strong\u003e\u003c/a\u003e\n    \u003cbr /--\u003e\n    \u003cbr /\u003e\n    \u003c!-- a href=\"https://github.com/deas/r00ki\"\u003eView Demo\u003c/a\u003e\n    ·\n    --\u003e\n    \u003ca href=\"https://github.com/deas/r00ki/issues/new?labels=bug\u0026template=bug-report---.md\"\u003eReport Bug\u003c/a\u003e\n    ·\n    \u003ca href=\"https://github.com/deas/r00ki/issues/new?labels=enhancement\u0026template=feature-request---.md\"\u003eRequest Feature\u003c/a\u003e\n  \u003c/p\u003e\n\u003c/div\u003e\n\n\u003c!-- TABLE OF CONTENTS --\u003e\n\u003cdetails\u003e\n  \u003csummary\u003eTable of Contents\u003c/summary\u003e\n  \u003col\u003e\n    \u003cli\u003e\n      \u003ca href=\"#about-the-project\"\u003eAbout The Project\u003c/a\u003e\n      \u003c!--ul\u003e\n        \u003cli\u003e\u003ca href=\"#built-with\"\u003eBuilt With\u003c/a\u003e\u003c/li\u003e\n      \u003c/ul--\u003e\n    \u003c/li\u003e\n    \u003cli\u003e\n      \u003ca href=\"#getting-started\"\u003eGetting Started\u003c/a\u003e\n      \u003cul\u003e\n        \u003cli\u003e\u003ca href=\"#prerequisites\"\u003ePrerequisites\u003c/a\u003e\u003c/li\u003e\n        \u003cli\u003e\u003ca href=\"#usage\"\u003eUsage\u003c/a\u003e\u003c/li\u003e\n      \u003c/ul\u003e\n    \u003c/li\u003e\n    \u003cli\u003e\u003ca href=\"#todo\"\u003eTODO\u003c/a\u003e\u003c/li\u003e\n    \u003cli\u003e\u003ca href=\"#known-issues\"\u003eKnown Issues\u003c/a\u003e\u003c/li\u003e\n    \u003cli\u003e\u003ca href=\"#references\"\u003eReferences\u003c/a\u003e\u003c/li\u003e\n    \u003cli\u003e\u003ca href=\"#license\"\u003eLicense\u003c/a\u003e\u003c/li\u003e\n    \u003c!--\n    \u003cli\u003e\u003ca href=\"#contact\"\u003eContact\u003c/a\u003e\u003c/li\u003e\n    \u003cli\u003e\u003ca href=\"#acknowledgments\"\u003eAcknowledgments\u003c/a\u003e\u003c/li\u003e\n    --\u003e\n  \u003c/ol\u003e\n\u003c/details\u003e\n\n\u003c!-- ABOUT THE PROJECT --\u003e\n## About The Project\n\n\u003c!--\n[![Product Name Screen Shot][product-screenshot]](https://example.com)\n--\u003e\n\u003c!--\nHere's a blank template to get started: To avoid retyping too much info. Do a search and replace with your text editor for the following: `github_username`, `repo_name`, `twitter_handle`, `linkedin_username`, `email_client`, `email`, `project_title`, `project_description`\n\n--\u003e\n\nKubernetes Storage. Rook. The Boss Fight. Still a bit messy. But it works. Most of the time.\n\nThere must be a reason [Red Hat OpenShift Data Foundation](https://docs.redhat.com/en/documentation/red_hat_openshift_data_foundation) is expensive ...\n\nNow seriously: Storage is one of the most critical bits in general. Many workloads are stateful, and not every Kubernetes infrastructure solves the problem nicely. That was where I found myself a few times in the past. We we given virtual machines with basic disks attached - VMware VMDKs in my case. Customers were in demand of ... you name it - everything: RWX-/RWO Volumes, S3, Snapshots, Backup/Recovery - superfast and always available. The code reflects these roots.\n\nDisclaimer: We started by borrowing proven things from the Rook project - adapted them as we went along.\n\n\u003c!-- https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/organizing-information-with-collapsed-sections --\u003e\n\u003cdetails\u003e\n\u003csummary\u003eDemo creating a Minikube cluster and running a few tests 🪄🎩🐰\u003c/summary\u003e\n\n```sh\n make apply-r00ki-aio test-csi-io test-csi-snapshot test-velero\n```\n\n![Demo](./assets/demo.gif)\n\n\u003c/details\u003e\n\n### Goals\n\n- Awesome local first Rook Ceph Dev Experience\n- First Class Observability\n- Fail early and loud (Notifications)\n- Simplicity (yes, really)\n- Composability\n- Target `minikube`, vanilla Kubernetes and Openshift.\n- Add the Rook Ops bits not covered by the Operator\n- Declarative trumps Imperative\n\n### Non Goals\n\n### Decisions\n\n- ArgoCD is great, but `helmfile` appears even better for our use case\n- We aim for first class citizens. For Rook, it's the helm charts, for some operators, its OLM Subscriptions.\n\n### Features\n\nWe cover:\n\n- Single (All in Once Cluster) Deployments targetting `minikube` and Production Kubernetes (including Openshift)\n- Two Cluster Deployments (Service and Consumer) targetting `minikube` and Production Kubernetes (including Openshift)\n- Kube-Prometheus bits all wired up - including alerts\n- Shiny Dashboards (including Grafana)\n- Seamless integration with ArgoCD, specifically [`deas/argcocd-conductor`](https://github.com/deas/argcocd-conductor)\n\n\u003cp align=\"right\"\u003e(\u003ca href=\"#readme-top\"\u003eback to top\u003c/a\u003e)\u003c/p\u003e\n\n\u003c!--\n### Built With\n\n* [![Docker][Docker]][Docker-url]\n* [![Terraform][Terraform]][Terraform-url]\n\n\u003cp align=\"right\"\u003e(\u003ca href=\"#readme-top\"\u003eback to top\u003c/a\u003e)\u003c/p\u003e\n\n--\u003e\n\n\u003c!-- GETTING STARTED --\u003e\n## Getting Started\n\nSome opinions first:\n\n- Ceph is complex\n- Automating Trust Relationships is hard\n\n### Prerequisites\n\n- `make`\n- `minikube`\n- `kubectl`\n- `helmfile`\n\n### Usage\n\nRun\n\n```sh\nmake\n```\n\nshows help for basic tasks and give you an idea where to start.\n\nWe want lifecycle of things (Create/Destroy) to be as fast as possible. We ship support to levarage registry mirrors using pull through.\n\n\u003cp align=\"right\"\u003e(\u003ca href=\"#readme-top\"\u003eback to top\u003c/a\u003e)\u003c/p\u003e\n\n\u003c!-- TODO --\u003e\n## TODO\n\u003c!--\n- [ ] Feature 1\n- [ ] Feature 3\n    - [ ] Nested Feature\n--\u003e\n- Use `dyff` to separate out value files?\n- Separate out Observability, add Logging and Alerting\n- Support for Mon v2\n- Support for TLS/encryption\n- Replace imperative bits by declarative ones\n- Introduce Pentesting - maybe even Chaos Scenarios\n- Improve Observability / Include Alerts\n- Smoketests in CI\n- Cleanup bits aroud `TODO` tags sprinkled across the code\n- Use LVM instead of raw disks/partitions?\n- Performance: How/When do multiple disks per node make sense?\n- Exercise Upgrade/Recreate and Desaster Recovery + build tests\n- Introduce unhappy path tests -likely leveraging Litmus\n- Proper cascaded removal of `CephCluster`?\n- Finding-/cleaning up orphans (volumes or buckets)\n- Go deeper with `nix`/`devenv` - maybe even replace `mise`\n\n\u003cp align=\"right\"\u003e(\u003ca href=\"#readme-top\"\u003eback to top\u003c/a\u003e)\u003c/p\u003e\n\n## Known Issues\n\n- With kvm + minikube, there appears to be a timing issue with helm when used via helmfile. `helm upgrade` sometimes fails due to CRDs unavailable - s. [fix: clear the discovery cache after CRDs are installed](https://github.com/helm/helm/pull/6332)\n- [\"To sum up: the Docker daemon does not currently support multiple registry mirrors ...\"](https://blog.alexellis.io/how-to-configure-multiple-docker-registry-mirrors/) -\u003e `minikube start --registry-mirror=\"http://yourmirror\"`\n- kvm network dns(masq) slow from minikube kubernetes. Times out for s3.\n  Patching coredns gets around the issue.\n- mons on port 3300 (workaround: use port 6789 / `ROOK_EXTERNAL_CEPH_MON_DATA`): `2024-12-16T16:56:02.784+0000 7fd593d1c000 -1 failed for service _ceph-mon._tcp\nmount error: no mds (Metadata Server) is up. The cluster might be laggy, or you may not be authorized\n  Warning  FailedMount  2m25s  kubelet  (combined from similar events): MountVolume.MountDevice failed for volume \"pvc-026c86e8-9ee4-4261-a7e4-083011b80494\" : rpc error: code = Internal desc = an error (exit status 32) occurred while running mount args: [-t ceph 192.168.122.231:3300:/volumes/csi/csi-vol-7072e90c-5d6b-477b-bbab-655b76d0425f/e8d828a3-a1ad-4a22-9b36-7d5bc9fe9026 /var/lib/kubelet/plugins/kubernetes.io/csi/rook-ceph.cephfs.csi.ceph.com/f172f41f387d01c38f46e71a4097304d70c35494e81e1c8a070549de56234790/globalmount -o name=csi-cephfs-node,secretfile=/tmp/csi/keys/keyfile-2436134297,mds_namespace=myfs,_netdev] stderr: unable to get monitor info from DNS SRV with service name: ceph-mon`\n- [Looking up Monitors through DNS](https://docs.ceph.com/en/latest/rados/configuration/mon-lookup-dns/)\n- [OperatorHub Sub Outdated - at 1.1.1](https://operatorhub.io/operator/rook-ceph)\n\n## References\n\n- [Monitor OpenShift Virtualization using user-defined projects and Grafana](https://developers.redhat.com/articles/2024/08/19/monitor-openshift-virtualization-using-user-defined-projects-and-grafana)\n- [How to create a long lived service account token in RHOCP4](https://access.redhat.com/solutions/7025261)\n\u003c!-- CONTRIBUTING --\u003e\n## Contributing\n\nContributions are what make the open source community such an amazing place to learn, inspire, and create. Any contributions you make are **greatly appreciated**.\n\nIf you have a suggestion that would make this better, please fork the repo and create a pull request. You can also simply open an issue with the tag \"enhancement\".\nDon't forget to give the project a star! Thanks again!\n\n1. Fork the Project\n2. Create your Feature Branch (`git checkout -b feature/AmazingFeature`)\n3. Commit your Changes (`git commit -m 'Add some AmazingFeature'`)\n4. Push to the Branch (`git push origin feature/AmazingFeature`)\n5. Open a Pull Request\n\n\u003cp align=\"right\"\u003e(\u003ca href=\"#readme-top\"\u003eback to top\u003c/a\u003e)\u003c/p\u003e\n\u003c!--\n### Top contributors:\n\n\u003ca href=\"https://github.com/deas/r00ki/graphs/contributors\"\u003e\n  \u003cimg src=\"https://contrib.rocks/image?repo=deas/r00ki\" alt=\"contrib.rocks image\" /\u003e\n\u003c/a\u003e\n--\u003e\n\n\u003c!-- LICENSE --\u003e\n## License\n\nDistributed under the MIT License. See `LICENSE.txt` for more information.\n\n\u003cp align=\"right\"\u003e(\u003ca href=\"#readme-top\"\u003eback to top\u003c/a\u003e)\u003c/p\u003e\n\n\u003c!-- CONTACT --\u003e\n\u003c!--\n## Contact\n\nYour Name - [@twitter_handle](https://twitter.com/twitter_handle) - email@email_client.com\n\nProject Link: [https://github.com/deas/r00ki](https://github.com/deas/r00ki)\n\n\u003cp align=\"right\"\u003e(\u003ca href=\"#readme-top\"\u003eback to top\u003c/a\u003e)\u003c/p\u003e\n--\u003e\n\n\u003c!-- ACKNOWLEDGMENTS --\u003e\n\u003c!--\n## Acknowledgments\n\n* []()\n--\u003e\n\n\u003c!-- p align=\"right\"\u003e(\u003ca href=\"#readme-top\"\u003eback to top\u003c/a\u003e)\u003c/p--\u003e\n\n\u003c!-- MARKDOWN LINKS \u0026 IMAGES --\u003e\n\u003c!-- https://www.markdownguide.org/basic-syntax/#reference-style-links --\u003e\n","project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fdeas%2Fr00ki","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Fdeas%2Fr00ki","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fdeas%2Fr00ki/lists"}