{"id":13942078,"url":"https://github.com/ploomber/projects","last_synced_at":"2025-05-06T06:33:33.434Z","repository":{"id":37954489,"uuid":"261592163","full_name":"ploomber/projects","owner":"ploomber","description":"Sample projects using Ploomber.","archived":false,"fork":false,"pushed_at":"2024-01-25T00:58:58.000Z","size":11371,"stargazers_count":86,"open_issues_count":14,"forks_count":25,"subscribers_count":8,"default_branch":"master","last_synced_at":"2025-04-09T09:51:48.096Z","etag":null,"topics":[],"latest_commit_sha":null,"homepage":"","language":"Jupyter Notebook","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"apache-2.0","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/ploomber.png","metadata":{"files":{"readme":"README.ipynb","changelog":null,"contributing":"CONTRIBUTING.md","funding":null,"license":"LICENSE","code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null}},"created_at":"2020-05-05T21:58:11.000Z","updated_at":"2025-02-11T10:39:26.000Z","dependencies_parsed_at":"2024-11-13T17:42:25.432Z","dependency_job_id":null,"html_url":"https://github.com/ploomber/projects","commit_stats":null,"previous_names":[],"tags_count":0,"template":false,"template_full_name":null,"repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/ploomber%2Fprojects","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/ploomber%2Fprojects/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/ploomber%2Fprojects/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/ploomber%2Fprojects/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/ploomber","download_url":"https://codeload.github.com/ploomber/projects/tar.gz/refs/heads/master","host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":252633912,"owners_count":21779944,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2022-07-04T15:15:14.044Z","host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":[],"created_at":"2024-08-08T02:01:41.352Z","updated_at":"2025-05-06T06:33:32.762Z","avatar_url":"https://github.com/ploomber.png","language":"Jupyter Notebook","funding_links":[],"categories":["Jupyter Notebook"],"sub_categories":[],"readme":"{\n \"cells\": [\n  {\n   \"cell_type\": \"markdown\",\n   \"id\": \"6d898d50\",\n   \"metadata\": {\n    \"papermill\": {\n     \"duration\": 0.00677,\n     \"end_time\": \"2022-11-16T14:39:07.742657\",\n     \"exception\": false,\n     \"start_time\": \"2022-11-16T14:39:07.735887\",\n     \"status\": \"completed\"\n    },\n    \"tags\": []\n   },\n   \"source\": [\n    \"# Ploomber sample projects\\n\",\n    \"\\n\",\n    \"![CI](https://github.com/ploomber/projects/workflows/ci/badge.svg)\\n\",\n    \"\\n\",\n    \"\u003cp align=\\\"center\\\"\u003e\\n\",\n    \"  \u003ca href=\\\"https://ploomber.io/community\\\"\u003eJoin our community\u003c/a\u003e\\n\",\n    \"  |\\n\",\n    \"  \u003ca href=\\\"https://share.hsforms.com/1E7Qa_OpcRPi_MV-segFsaAe6c2g\\\"\u003eNewsletter\u003c/a\u003e\\n\",\n    \"  |\\n\",\n    \"  \u003ca href=\\\"https://docs.ploomber.io/\\\"\u003eDocs\u003c/a\u003e\\n\",\n    \"  |\\n\",\n    \"  \u003ca href=\\\"https://twitter.com/intent/user?screen_name=ploomber\\\"\u003eTwitter\u003c/a\u003e\\n\",\n    \"  |\\n\",\n    \"  \u003ca href=\\\"https://ploomber.io/\\\"\u003eBlog\u003c/a\u003e\\n\",\n    \"  |\\n\",\n    \"  \u003ca href=\\\"https://www.youtube.com/channel/UCaIS5BMlmeNQE4-Gn0xTDXQ\\\"\u003eYouTube\u003c/a\u003e\\n\",\n    \"  |\\n\",\n    \"  \u003ca href=\\\"mailto:contact@ploomber.io\\\"\u003eContact us\u003c/a\u003e\\n\",\n    \"\u003c/p\u003e\\n\",\n    \"\\n\",\n    \"This repository contains sample pipelines developed using [Ploomber](https://github.com/ploomber/ploomber).\\n\",\n    \"\\n\",\n    \"**Note:** We recommend you to go through the [first tutorial](https://docs.ploomber.io/en/latest/get-started/first-pipeline.html) to learn the basics of Ploomber.\\n\",\n    \"\\n\",\n    \"## Running examples\"\n   ]\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"id\": \"b7826667\",\n   \"metadata\": {\n    \"papermill\": {\n     \"duration\": 0.002241,\n     \"end_time\": \"2022-11-16T14:39:07.747334\",\n     \"exception\": false,\n     \"start_time\": \"2022-11-16T14:39:07.745093\",\n     \"status\": \"completed\"\n    },\n    \"tags\": []\n   },\n   \"source\": [\n    \"Use Colab:\\n\",\n    \"\\n\",\n    \"\u003cp align=\\\"center\\\"\u003e\\n\",\n    \"    \u003ca href=\\\"https://colab.research.google.com/github/ploomber/projects/blob/master/guides/first-pipeline/colab.ipynb\\\"\u003e\u003cimg src=\\\"https://colab.research.google.com/assets/colab-badge.svg\\\" alt=\\\"Open In Colab\\\"/\u003e\u003c/a\u003e\\n\",\n    \"\u003c/p\u003e\\n\",\n    \"\\n\",\n    \"\\n\",\n    \"Or run locally:\\n\",\n    \"\\n\",\n    \"~~~sh\\n\",\n    \"pip install ploomber\\n\",\n    \"\\n\",\n    \"# list examples\\n\",\n    \"ploomber examples\\n\",\n    \"\\n\",\n    \"# download example with name\\n\",\n    \"ploomber examples --name {name}\\n\",\n    \"\\n\",\n    \"# example\\n\",\n    \"ploomber examples --name templates/mlflow\\n\",\n    \"~~~\\n\",\n    \"\\n\",\n    \"## How to read the examples\\n\",\n    \"\\n\",\n    \"Each example contains a `README.md` file that describes it; a `README.ipynb` is also available with the same contents but in Jupyter notebook format and with command outputs. In addition, files for `pip` (`requirements.txt`) and  `conda` (`environment.yml`) are provided for local execution.\\n\",\n    \"\\n\",\n    \"## Index\\n\",\n    \"\\n\",\n    \"### Templates\\n\",\n    \"\\n\",\n    \"Starting points for common use cases. Use them to ramp up a project quickly.\"\n   ]\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"id\": \"98eed9ff\",\n   \"metadata\": {\n    \"papermill\": {\n     \"duration\": 0.002723,\n     \"end_time\": \"2022-11-16T14:39:07.751775\",\n     \"exception\": false,\n     \"start_time\": \"2022-11-16T14:39:07.749052\",\n     \"status\": \"completed\"\n    },\n    \"tags\": []\n   },\n   \"source\": [\n    \"1. [`templates/etl`](templates/etl/README.ipynb) Download a data file, upload it to a database, process it, and plot with Python and R.\\n\",\n    \"\\n\",\n    \"2. [`templates/exploratory-analysis`](templates/exploratory-analysis/README.ipynb) Sample pipeline that explores penguins data.\\n\",\n    \"\\n\",\n    \"3. [`templates/google-cloud`](templates/google-cloud/README.ipynb) Use Google Cloud and Ploomber to develop a scalable and production-ready pipeline.\\n\",\n    \"\\n\",\n    \"4. [`templates/ml-advanced`](templates/ml-advanced/README.ipynb) ML pipeline using the Python API. Shows how to create a Python package, test it with pytest, and train models in parallel.\\n\",\n    \"\\n\",\n    \"5. [`templates/ml-basic`](templates/ml-basic/README.ipynb) Download data, clean it, generate features and train a model.\\n\",\n    \"\\n\",\n    \"6. [`templates/ml-intermediate`](templates/ml-intermediate/README.ipynb) Training and serving ML pipelines with integration testing to evaluate training data quality.\\n\",\n    \"\\n\",\n    \"7. [`templates/ml-online`](templates/ml-online/README.ipynb) Load data, generate features, train a model, and deploy model with flask.\\n\",\n    \"\\n\",\n    \"8. [`templates/mlflow`](templates/mlflow/README.ipynb) Train a grid of models and log them to MLflow.\\n\",\n    \"\\n\",\n    \"9. [`templates/python-api`](templates/python-api/README.ipynb) Loads, clean, and plot data using the Python API.\\n\",\n    \"\\n\",\n    \"10. [`templates/pytorch`](templates/pytorch/README.ipynb) Using GPUs to train models in Ploomber Cloud.\\n\",\n    \"\\n\",\n    \"11. [`templates/shell`](templates/shell/README.ipynb) Create a pipeline with shell scripts as tasks.\\n\",\n    \"\\n\",\n    \"12. [`templates/spec-api-directory`](templates/spec-api-directory/README.ipynb) Create a pipeline from a directory with scripts (without a pipeline.yaml file).\\n\",\n    \"\\n\",\n    \"13. [`templates/spec-api-r`](templates/spec-api-r/README.ipynb) Load, clean and plot data with R.\\n\",\n    \"\\n\",\n    \"14. [`templates/spec-api-sql`](templates/spec-api-sql/README.ipynb) Use SQL scripts to manipulate data in a database, dump a table, and plot it with Python.\"\n   ]\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"id\": \"3df2e459\",\n   \"metadata\": {\n    \"papermill\": {\n     \"duration\": 0.001936,\n     \"end_time\": \"2022-11-16T14:39:07.755901\",\n     \"exception\": false,\n     \"start_time\": \"2022-11-16T14:39:07.753965\",\n     \"status\": \"completed\"\n    },\n    \"tags\": []\n   },\n   \"source\": [\n    \"### Cookbook\\n\",\n    \"\\n\",\n    \"Short and to-the-point examples showing how to use a specific feature.\"\n   ]\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"id\": \"032ca243\",\n   \"metadata\": {\n    \"papermill\": {\n     \"duration\": 0.001735,\n     \"end_time\": \"2022-11-16T14:39:07.759411\",\n     \"exception\": false,\n     \"start_time\": \"2022-11-16T14:39:07.757676\",\n     \"status\": \"completed\"\n    },\n    \"tags\": []\n   },\n   \"source\": [\n    \"2. [`cookbook/dynamic-params`](cookbook/dynamic-params/README.ipynb) Pipeline parameters whose values are computed at runtime.\\n\",\n    \"\\n\",\n    \"3. [`cookbook/file-client`](cookbook/file-client/README.ipynb) Upload task's products upon execution (local, S3, GCloud storage)\\n\",\n    \"\\n\",\n    \"4. [`cookbook/grid`](cookbook/grid/README.ipynb) An example showing how to create a grid of tasks to train models with different parameters.\\n\",\n    \"\\n\",\n    \"5. [`cookbook/hooks`](cookbook/hooks/README.ipynb) Task hooks\\n\",\n    \"\\n\",\n    \"6. [`cookbook/incremental`](cookbook/incremental/README.ipynb) A pipeline that processes new records from a database and uploads them.\\n\",\n    \"\\n\",\n    \"7. [`cookbook/nested-cv`](cookbook/nested-cv/README.ipynb) Nested cross-validation for model selection and hyperparameter tuning.\\n\",\n    \"\\n\",\n    \"8. [`cookbook/python-load`](cookbook/python-load/README.ipynb) Load pipeline.yaml file in a Python session to customize initialization.\\n\",\n    \"\\n\",\n    \"9. [`cookbook/report-generation`](cookbook/report-generation/README.ipynb) Generating HTML/PDF reports.\\n\",\n    \"\\n\",\n    \"10. [`cookbook/serialization`](cookbook/serialization/README.ipynb) Shows how to use the serializer and unserializer decorators.\\n\",\n    \"\\n\",\n    \"11. [`cookbook/sql-dump`](cookbook/sql-dump/README.ipynb) A minimal example showing how to dump a table from a SQL database.\\n\",\n    \"\\n\",\n    \"13. [`cookbook/variable-number-of-products`](cookbook/variable-number-of-products/README.ipynb) Shows how to create tasks whose number of products depends on runtime conditions.\"\n   ]\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"id\": \"e65493ce\",\n   \"metadata\": {\n    \"papermill\": {\n     \"duration\": 0.002152,\n     \"end_time\": \"2022-11-16T14:39:07.763435\",\n     \"exception\": false,\n     \"start_time\": \"2022-11-16T14:39:07.761283\",\n     \"status\": \"completed\"\n    },\n    \"tags\": []\n   },\n   \"source\": [\n    \"### Guides\\n\",\n    \"\\n\",\n    \"In-depth tutorials for learning.  These are part of the [documentation](https://docs.ploomber.io/en/latest/user-guide/index.html).\"\n   ]\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"id\": \"dd47377d\",\n   \"metadata\": {\n    \"papermill\": {\n     \"duration\": 0.002412,\n     \"end_time\": \"2022-11-16T14:39:07.768693\",\n     \"exception\": false,\n     \"start_time\": \"2022-11-16T14:39:07.766281\",\n     \"status\": \"completed\"\n    },\n    \"tags\": []\n   },\n   \"source\": [\n    \"5. [`guides/cron`](guides/cron/README.ipynb) This guide shows how to schedule Ploomber pipelines using cron.\\n\",\n    \"\\n\",\n    \"6. [`guides/debugging`](guides/debugging/README.ipynb) Tutorial showing techniques for debugging pipelines.\\n\",\n    \"\\n\",\n    \"7. [`guides/first-pipeline`](guides/first-pipeline/README.ipynb) Introductory tutorial to learn the basics of Ploomber.\\n\",\n    \"\\n\",\n    \"8. [`guides/intro-to-ploomber`](guides/intro-to-ploomber/README.ipynb) Introductory tutorial to learn the basics of Ploomber.\\n\",\n    \"\\n\",\n    \"9. [`guides/logging`](guides/logging/README.ipynb) Tutorial showing how to add logging to a pipeline.\\n\",\n    \"\\n\",\n    \"10. [`guides/parametrized`](guides/parametrized/README.ipynb) Tutorial showing how to parametrize pipelines and change parameters from the command-line.\\n\",\n    \"\\n\",\n    \"11. [`guides/refactor`](guides/refactor/README.ipynb) Using Soorgeon to convert a notebook into a Ploomber pipeline.\\n\",\n    \"\\n\",\n    \"12. [`guides/serialization`](guides/serialization/README.ipynb) Tutorial explaining how the serializer and unserializer fields in a pipeline.yaml file work.\\n\",\n    \"\\n\",\n    \"13. [`guides/sql-templating`](guides/sql-templating/README.ipynb) Introductory tutorial teaching how to develop modular SQL pipelines.\\n\",\n    \"\\n\",\n    \"14. [`guides/testing`](guides/testing/README.ipynb) Tutorial showing how to use a task's on_finish hook to test data quality.\\n\",\n    \"\\n\",\n    \"15. [`guides/versioning`](guides/versioning/README.ipynb) A tutorial showing how to version pipeline products.\\n\"\n   ]\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"id\": \"8b5601ce\",\n   \"metadata\": {\n    \"papermill\": {\n     \"duration\": 0.001954,\n     \"end_time\": \"2022-11-16T14:39:07.773003\",\n     \"exception\": false,\n     \"start_time\": \"2022-11-16T14:39:07.771049\",\n     \"status\": \"completed\"\n    },\n    \"tags\": []\n   },\n   \"source\": [\n    \"## Python API\\n\",\n    \"\\n\",\n    \"The simplest way to get started with Ploomber is via the Spec API, which allows you to describe pipelines using a `pipeline.yaml` file, most examples on this repository use the Spec API. However, if you want more flexibility, you may write pipelines with Python.\\n\",\n    \"\\n\",\n    \"The [`templates/python-api/`](templates/python-api) directory contains a project written using the Python API. And the [`python-api-examples/`](python-api-examples) includes some tutorials and more examples.\"\n   ]\n  },\n  {\n   \"cell_type\": \"markdown\",\n   \"id\": \"093cf2ff\",\n   \"metadata\": {\n    \"papermill\": {\n     \"duration\": 0.002256,\n     \"end_time\": \"2022-11-16T14:39:07.777053\",\n     \"exception\": false,\n     \"start_time\": \"2022-11-16T14:39:07.774797\",\n     \"status\": \"completed\"\n    },\n    \"tags\": []\n   },\n   \"source\": [\n    \"## Micro-pipelines\\n\",\n    \"\\n\",\n    \"In Ploomber `0.21`, we introduced a simplified API to write pipelines in a single Jupyter notebook (or `.py`) file. This is a great option for small projects.\\n\",\n    \"\\n\",\n    \"You can find the examples in the [`micro-pipelines/`](micro-pipelines) directory.\"\n   ]\n  }\n ],\n \"metadata\": {\n  \"jupytext\": {\n   \"cell_metadata_filter\": \"-all\",\n   \"main_language\": \"python\",\n   \"notebook_metadata_filter\": \"-all\"\n  },\n  \"kernelspec\": {\n   \"display_name\": \"Python 3 (ipykernel)\",\n   \"language\": \"python\",\n   \"name\": \"python3\"\n  },\n  \"language_info\": {\n   \"codemirror_mode\": {\n    \"name\": \"ipython\",\n    \"version\": 3\n   },\n   \"file_extension\": \".py\",\n   \"mimetype\": \"text/x-python\",\n   \"name\": \"python\",\n   \"nbconvert_exporter\": \"python\",\n   \"pygments_lexer\": \"ipython3\",\n   \"version\": \"3.10.1\"\n  },\n  \"papermill\": {\n   \"default_parameters\": {},\n   \"duration\": 2.004917,\n   \"end_time\": \"2022-11-16T14:39:08.010236\",\n   \"environment_variables\": {},\n   \"exception\": null,\n   \"input_path\": \"README.ipynb\",\n   \"output_path\": \"README.ipynb\",\n   \"parameters\": {},\n   \"start_time\": \"2022-11-16T14:39:06.005319\",\n   \"version\": \"2.4.0\"\n  }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 5\n}\n","project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fploomber%2Fprojects","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Fploomber%2Fprojects","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fploomber%2Fprojects/lists"}