{"id":16484376,"url":"https://github.com/fabricesalvaire/sidita","last_synced_at":"2025-06-23T17:07:06.105Z","repository":{"id":57467038,"uuid":"131419409","full_name":"FabriceSalvaire/sidita","owner":"FabriceSalvaire","description":"A simple distributed task queue for Python 3 asyncio","archived":false,"fork":false,"pushed_at":"2018-05-03T23:14:42.000Z","size":2584,"stargazers_count":0,"open_issues_count":1,"forks_count":0,"subscribers_count":1,"default_branch":"master","last_synced_at":"2025-06-03T21:24:35.035Z","etag":null,"topics":["python","python-library","python3","queue-tasks","queue-workers","queued-jobs","task-manager","task-runner","task-scheduler"],"latest_commit_sha":null,"homepage":"https://fabricesalvaire.github.io/sidita","language":"Python","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"gpl-3.0","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/FabriceSalvaire.png","metadata":{"files":{"readme":"README.html","changelog":null,"contributing":".github/CONTRIBUTING.md","funding":null,"license":"LICENSE.txt","code_of_conduct":".github/CODE_OF_CONDUCT.md","threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null}},"created_at":"2018-04-28T15:03:48.000Z","updated_at":"2018-05-09T20:27:15.000Z","dependencies_parsed_at":"2022-09-10T03:43:44.574Z","dependency_job_id":null,"html_url":"https://github.com/FabriceSalvaire/sidita","commit_stats":null,"previous_names":[],"tags_count":0,"template":false,"template_full_name":null,"purl":"pkg:github/FabriceSalvaire/sidita","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/FabriceSalvaire%2Fsidita","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/FabriceSalvaire%2Fsidita/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/FabriceSalvaire%2Fsidita/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/FabriceSalvaire%2Fsidita/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/FabriceSalvaire","download_url":"https://codeload.github.com/FabriceSalvaire/sidita/tar.gz/refs/heads/master","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/FabriceSalvaire%2Fsidita/sbom","host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":261518948,"owners_count":23171227,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2022-07-04T15:15:14.044Z","host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":["python","python-library","python3","queue-tasks","queue-workers","queued-jobs","task-manager","task-runner","task-scheduler"],"created_at":"2024-10-11T13:16:56.898Z","updated_at":"2025-06-23T17:07:06.040Z","avatar_url":"https://github.com/FabriceSalvaire.png","language":"Python","funding_links":[],"categories":[],"sub_categories":[],"readme":"\u003c?xml version=\"1.0\" encoding=\"utf-8\" ?\u003e\n\u003c!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\"\u003e\n\u003chtml xmlns=\"http://www.w3.org/1999/xhtml\" xml:lang=\"en\" lang=\"en\"\u003e\n\u003chead\u003e\n\u003cmeta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\" /\u003e\n\u003cmeta name=\"generator\" content=\"Docutils 0.13.1: http://docutils.sourceforge.net/\" /\u003e\n\u003ctitle\u003esidita\u003c/title\u003e\n\u003cstyle type=\"text/css\"\u003e\n\n/*\n:Author: David Goodger (goodger@python.org)\n:Id: $Id: html4css1.css 7952 2016-07-26 18:15:59Z milde $\n:Copyright: This stylesheet has been placed in the public domain.\n\nDefault cascading style sheet for the HTML output of Docutils.\n\nSee http://docutils.sf.net/docs/howto/html-stylesheets.html for how to\ncustomize this style sheet.\n*/\n\n/* used to remove borders from tables and images */\n.borderless, table.borderless td, table.borderless th {\n  border: 0 }\n\ntable.borderless td, table.borderless th {\n  /* Override padding for \"table.docutils td\" with \"! important\".\n     The right padding separates the table cells. */\n  padding: 0 0.5em 0 0 ! important }\n\n.first {\n  /* Override more specific margin styles with \"! important\". */\n  margin-top: 0 ! important }\n\n.last, .with-subtitle {\n  margin-bottom: 0 ! important }\n\n.hidden {\n  display: none }\n\n.subscript {\n  vertical-align: sub;\n  font-size: smaller }\n\n.superscript {\n  vertical-align: super;\n  font-size: smaller }\n\na.toc-backref {\n  text-decoration: none ;\n  color: black }\n\nblockquote.epigraph {\n  margin: 2em 5em ; }\n\ndl.docutils dd {\n  margin-bottom: 0.5em }\n\nobject[type=\"image/svg+xml\"], object[type=\"application/x-shockwave-flash\"] {\n  overflow: hidden;\n}\n\n/* Uncomment (and remove this text!) to get bold-faced definition list terms\ndl.docutils dt {\n  font-weight: bold }\n*/\n\ndiv.abstract {\n  margin: 2em 5em }\n\ndiv.abstract p.topic-title {\n  font-weight: bold ;\n  text-align: center }\n\ndiv.admonition, div.attention, div.caution, div.danger, div.error,\ndiv.hint, div.important, div.note, div.tip, div.warning {\n  margin: 2em ;\n  border: medium outset ;\n  padding: 1em }\n\ndiv.admonition p.admonition-title, div.hint p.admonition-title,\ndiv.important p.admonition-title, div.note p.admonition-title,\ndiv.tip p.admonition-title {\n  font-weight: bold ;\n  font-family: sans-serif }\n\ndiv.attention p.admonition-title, div.caution p.admonition-title,\ndiv.danger p.admonition-title, div.error p.admonition-title,\ndiv.warning p.admonition-title, .code .error {\n  color: red ;\n  font-weight: bold ;\n  font-family: sans-serif }\n\n/* Uncomment (and remove this text!) to get reduced vertical space in\n   compound paragraphs.\ndiv.compound .compound-first, div.compound .compound-middle {\n  margin-bottom: 0.5em }\n\ndiv.compound .compound-last, div.compound .compound-middle {\n  margin-top: 0.5em }\n*/\n\ndiv.dedication {\n  margin: 2em 5em ;\n  text-align: center ;\n  font-style: italic }\n\ndiv.dedication p.topic-title {\n  font-weight: bold ;\n  font-style: normal }\n\ndiv.figure {\n  margin-left: 2em ;\n  margin-right: 2em }\n\ndiv.footer, div.header {\n  clear: both;\n  font-size: smaller }\n\ndiv.line-block {\n  display: block ;\n  margin-top: 1em ;\n  margin-bottom: 1em }\n\ndiv.line-block div.line-block {\n  margin-top: 0 ;\n  margin-bottom: 0 ;\n  margin-left: 1.5em }\n\ndiv.sidebar {\n  margin: 0 0 0.5em 1em ;\n  border: medium outset ;\n  padding: 1em ;\n  background-color: #ffffee ;\n  width: 40% ;\n  float: right ;\n  clear: right }\n\ndiv.sidebar p.rubric {\n  font-family: sans-serif ;\n  font-size: medium }\n\ndiv.system-messages {\n  margin: 5em }\n\ndiv.system-messages h1 {\n  color: red }\n\ndiv.system-message {\n  border: medium outset ;\n  padding: 1em }\n\ndiv.system-message p.system-message-title {\n  color: red ;\n  font-weight: bold }\n\ndiv.topic {\n  margin: 2em }\n\nh1.section-subtitle, h2.section-subtitle, h3.section-subtitle,\nh4.section-subtitle, h5.section-subtitle, h6.section-subtitle {\n  margin-top: 0.4em }\n\nh1.title {\n  text-align: center }\n\nh2.subtitle {\n  text-align: center }\n\nhr.docutils {\n  width: 75% }\n\nimg.align-left, .figure.align-left, object.align-left, table.align-left {\n  clear: left ;\n  float: left ;\n  margin-right: 1em }\n\nimg.align-right, .figure.align-right, object.align-right, table.align-right {\n  clear: right ;\n  float: right ;\n  margin-left: 1em }\n\nimg.align-center, .figure.align-center, object.align-center {\n  display: block;\n  margin-left: auto;\n  margin-right: auto;\n}\n\ntable.align-center {\n  margin-left: auto;\n  margin-right: auto;\n}\n\n.align-left {\n  text-align: left }\n\n.align-center {\n  clear: both ;\n  text-align: center }\n\n.align-right {\n  text-align: right }\n\n/* reset inner alignment in figures */\ndiv.align-right {\n  text-align: inherit }\n\n/* div.align-center * { */\n/*   text-align: left } */\n\n.align-top    {\n  vertical-align: top }\n\n.align-middle {\n  vertical-align: middle }\n\n.align-bottom {\n  vertical-align: bottom }\n\nol.simple, ul.simple {\n  margin-bottom: 1em }\n\nol.arabic {\n  list-style: decimal }\n\nol.loweralpha {\n  list-style: lower-alpha }\n\nol.upperalpha {\n  list-style: upper-alpha }\n\nol.lowerroman {\n  list-style: lower-roman }\n\nol.upperroman {\n  list-style: upper-roman }\n\np.attribution {\n  text-align: right ;\n  margin-left: 50% }\n\np.caption {\n  font-style: italic }\n\np.credits {\n  font-style: italic ;\n  font-size: smaller }\n\np.label {\n  white-space: nowrap }\n\np.rubric {\n  font-weight: bold ;\n  font-size: larger ;\n  color: maroon ;\n  text-align: center }\n\np.sidebar-title {\n  font-family: sans-serif ;\n  font-weight: bold ;\n  font-size: larger }\n\np.sidebar-subtitle {\n  font-family: sans-serif ;\n  font-weight: bold }\n\np.topic-title {\n  font-weight: bold }\n\npre.address {\n  margin-bottom: 0 ;\n  margin-top: 0 ;\n  font: inherit }\n\npre.literal-block, pre.doctest-block, pre.math, pre.code {\n  margin-left: 2em ;\n  margin-right: 2em }\n\npre.code .ln { color: grey; } /* line numbers */\npre.code, code { background-color: #eeeeee }\npre.code .comment, code .comment { color: #5C6576 }\npre.code .keyword, code .keyword { color: #3B0D06; font-weight: bold }\npre.code .literal.string, code .literal.string { color: #0C5404 }\npre.code .name.builtin, code .name.builtin { color: #352B84 }\npre.code .deleted, code .deleted { background-color: #DEB0A1}\npre.code .inserted, code .inserted { background-color: #A3D289}\n\nspan.classifier {\n  font-family: sans-serif ;\n  font-style: oblique }\n\nspan.classifier-delimiter {\n  font-family: sans-serif ;\n  font-weight: bold }\n\nspan.interpreted {\n  font-family: sans-serif }\n\nspan.option {\n  white-space: nowrap }\n\nspan.pre {\n  white-space: pre }\n\nspan.problematic {\n  color: red }\n\nspan.section-subtitle {\n  /* font-size relative to parent (h1..h6 element) */\n  font-size: 80% }\n\ntable.citation {\n  border-left: solid 1px gray;\n  margin-left: 1px }\n\ntable.docinfo {\n  margin: 2em 4em }\n\ntable.docutils {\n  margin-top: 0.5em ;\n  margin-bottom: 0.5em }\n\ntable.footnote {\n  border-left: solid 1px black;\n  margin-left: 1px }\n\ntable.docutils td, table.docutils th,\ntable.docinfo td, table.docinfo th {\n  padding-left: 0.5em ;\n  padding-right: 0.5em ;\n  vertical-align: top }\n\ntable.docutils th.field-name, table.docinfo th.docinfo-name {\n  font-weight: bold ;\n  text-align: left ;\n  white-space: nowrap ;\n  padding-left: 0 }\n\n/* \"booktabs\" style (no vertical lines) */\ntable.docutils.booktabs {\n  border: 0px;\n  border-top: 2px solid;\n  border-bottom: 2px solid;\n  border-collapse: collapse;\n}\ntable.docutils.booktabs * {\n  border: 0px;\n}\ntable.docutils.booktabs th {\n  border-bottom: thin solid;\n  text-align: left;\n}\n\nh1 tt.docutils, h2 tt.docutils, h3 tt.docutils,\nh4 tt.docutils, h5 tt.docutils, h6 tt.docutils {\n  font-size: 100% }\n\nul.auto-toc {\n  list-style-type: none }\n\n\u003c/style\u003e\n\u003c/head\u003e\n\u003cbody\u003e\n\u003cdiv class=\"document\" id=\"sidita\"\u003e\n\u003ch1 class=\"title\"\u003esidita\u003c/h1\u003e\n\n\u003c!-- -*- Mode: rst -*- --\u003e\n\u003c!-- -*- Mode: rst -*- --\u003e\n\u003c!-- |siditaUrl|\n|siditaHomePage|_\n|siditaDoc|_\n|sidita@github|_\n|sidita@readthedocs|_\n|sidita@readthedocs-badge|\n|sidita@pypi|_ --\u003e\n\u003c!-- .. _sidita@github: https://github.com/FabriceSalvaire/sidita --\u003e\n\u003c!-- .. _sidita@pypi: https://pypi.python.org/pypi/sidita --\u003e\n\u003c!-- coverage test --\u003e\n\u003c!-- https://img.shields.io/pypi/status/Django.svg --\u003e\n\u003c!-- https://img.shields.io/github/stars/badges/shields.svg?style=social\u0026label=Star --\u003e\n\u003c!-- -*- Mode: rst -*- --\u003e\n\u003cp\u003e\u003ca class=\"reference external\" href=\"https://pypi.python.org/pypi/sidita\"\u003e\u003cobject data=\"https://img.shields.io/pypi/l/sidita.svg\" type=\"image/svg+xml\"\u003esidita license\u003c/object\u003e\u003c/a\u003e\n\u003ca class=\"reference external\" href=\"https://pypi.python.org/pypi/sidita\"\u003e\u003cobject data=\"https://img.shields.io/pypi/pyversions/sidita.svg\" type=\"image/svg+xml\"\u003esidita python version\u003c/object\u003e\u003c/a\u003e\u003c/p\u003e\n\u003cp\u003e\u003ca class=\"reference external\" href=\"https://pypi.python.org/pypi/sidita\"\u003e\u003cobject data=\"https://img.shields.io/pypi/v/sidita.svg\" type=\"image/svg+xml\"\u003esidita last version\u003c/object\u003e\u003c/a\u003e\u003c/p\u003e\n\u003cul class=\"simple\"\u003e\n\u003cli\u003eQuick Link to \u003ca class=\"reference external\" href=\"https://github.com/FabriceSalvaire/sidita/tree/master\"\u003eProduction Branch\u003c/a\u003e\u003c/li\u003e\n\u003cli\u003eQuick Link to \u003ca class=\"reference external\" href=\"https://github.com/FabriceSalvaire/sidita/tree/devel\"\u003eDevel Branch\u003c/a\u003e\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"section\" id=\"overview\"\u003e\n\u003ch1\u003eOverview\u003c/h1\u003e\n\u003cdiv class=\"section\" id=\"what-is-sidita\"\u003e\n\u003ch2\u003eWhat is sidita ?\u003c/h2\u003e\n\u003cp\u003eSidita is a Python module which implements a distributed task queue featuring an intermediate\nsolution between the \u003ca class=\"reference external\" href=\"https://docs.python.org/3.6/library/multiprocessing.html\"\u003emultiprocessing module\u003c/a\u003e and a task scheduler like \u003ca class=\"reference external\" href=\"http://www.celeryproject.org\"\u003eCelery\u003c/a\u003e.\u003c/p\u003e\n\u003cp\u003eThe Sidita use case corresponds to the case where you need to run CPU bound tasks in parallel and\nyou require an immunity to crashes, memory leaks and overruns.  Theses requirements are met by a\nmonitored subprocess implementation.\u003c/p\u003e\n\u003c/div\u003e\n\u003cdiv class=\"section\" id=\"where-is-the-documentation\"\u003e\n\u003ch2\u003eWhere is the Documentation ?\u003c/h2\u003e\n\u003cp\u003eThe documentation is available on the \u003ca class=\"reference external\" href=\"https://fabricesalvaire.github.io/sidita\"\u003esidita Home Page\u003c/a\u003e.\u003c/p\u003e\n\u003c/div\u003e\n\u003cdiv class=\"section\" id=\"what-are-the-main-features\"\u003e\n\u003ch2\u003eWhat are the main features ?\u003c/h2\u003e\n\u003c!-- -*- Mode: rst -*- --\u003e\n\u003c!-- no title here --\u003e\n\u003cul class=\"simple\"\u003e\n\u003cli\u003eMust be a simple solution to circumvent to the GIL limitation : i.e. distribute and run tasks in parallel on local CPU cores\u003c/li\u003e\n\u003cli\u003eMust be a lightweight solution to Celery : no broker\u003c/li\u003e\n\u003cli\u003eShould be portable on main OS like Unix and Windows\u003c/li\u003e\n\u003cli\u003eScheduler is implemented using an \u003ca class=\"reference external\" href=\"https://docs.python.org/3/library/asyncio-eventloops.html\"\u003easyncio event loop\u003c/a\u003e and \u003ca class=\"reference external\" href=\"https://docs.python.org/3/library/asyncio-queue.html\"\u003easyncio queue\u003c/a\u003e\u003c/li\u003e\n\u003cli\u003eScheduler queue can be limited on size so as to prevent a memory overshoot at startup\u003c/li\u003e\n\u003cli\u003eWorkers are spawned using \u003ca class=\"reference external\" href=\"https://docs.python.org/3/library/asyncio-subprocess.html\"\u003easyncio subprocess\u003c/a\u003e and communicate through a pipe (stdin, stdout)\u003c/li\u003e\n\u003cli\u003ePass any pickable object as request and response\u003c/li\u003e\n\u003cli\u003eScheduler monitors workers and restart them when they was killed or they exceeded a memory or timeout threshold\u003c/li\u003e\n\u003cli\u003eProvide basic metrics on workers\u003c/li\u003e\n\u003c/ul\u003e\n\u003cp\u003eTechnical details in brief:\u003c/p\u003e\n\u003cul class=\"simple\"\u003e\n\u003cli\u003escheduler run a producer and N consumers coroutines in an \u003ca class=\"reference external\" href=\"https://docs.python.org/3/library/asyncio.html\"\u003easyncio\u003c/a\u003e event loop\u003c/li\u003e\n\u003cli\u003ethe producer coroutine awaits on \u003ccode\u003equeue.put(task)\u003c/code\u003e ( if a size limit is set )\u003c/li\u003e\n\u003cli\u003eeach consumer coroutine wraps a worker subprocess and await data on the stdout pipe\u003c/li\u003e\n\u003c/ul\u003e\n\u003c!-- -*- Mode: rst -*- --\u003e\n\u003c!-- no title here --\u003e\n\u003cp\u003eTechnical details in brief:\u003c/p\u003e\n\u003cul class=\"simple\"\u003e\n\u003cli\u003escheduler run a producer and N consumers coroutines in an \u003ca class=\"reference external\" href=\"https://docs.python.org/3/library/asyncio.html\"\u003easyncio\u003c/a\u003e event loop\u003c/li\u003e\n\u003cli\u003ethe producer coroutine awaits on \u003ccode\u003equeue.put(task)\u003c/code\u003e ( if a size limit is set )\u003c/li\u003e\n\u003cli\u003eeach consumer coroutine wraps a worker subprocess and await data on the stdout pipe\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/div\u003e\n\u003cdiv class=\"section\" id=\"how-to-install-it\"\u003e\n\u003ch2\u003eHow to install it ?\u003c/h2\u003e\n\u003cp\u003eLook at the \u003ca class=\"reference external\" href=\"https://fabricesalvaire.github.io/sidita/installation.html\"\u003einstallation\u003c/a\u003e section in the documentation.\u003c/p\u003e\n\u003c/div\u003e\n\u003c/div\u003e\n\u003cdiv class=\"section\" id=\"credits\"\u003e\n\u003ch1\u003eCredits\u003c/h1\u003e\n\u003cp\u003eAuthors: \u003ca class=\"reference external\" href=\"http://fabrice-salvaire.fr\"\u003eFabrice Salvaire\u003c/a\u003e\u003c/p\u003e\n\u003c/div\u003e\n\u003cdiv class=\"section\" id=\"news\"\u003e\n\u003ch1\u003eNews\u003c/h1\u003e\n\u003c!-- -*- Mode: rst -*- --\u003e\n\u003c!-- no title here --\u003e\n\u003cdiv class=\"section\" id=\"v0-2018-05-01\"\u003e\n\u003ch2\u003eV0 2018-05-01\u003c/h2\u003e\n\u003cp\u003eStarted project\u003c/p\u003e\n\u003c/div\u003e\n\u003c/div\u003e\n\u003c/div\u003e\n\u003c/body\u003e\n\u003c/html\u003e\n","project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Ffabricesalvaire%2Fsidita","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Ffabricesalvaire%2Fsidita","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Ffabricesalvaire%2Fsidita/lists"}