{"id":17197660,"url":"https://github.com/dmcg/dask-multicloud-poc","last_synced_at":"2026-02-28T16:32:27.503Z","repository":{"id":66725000,"uuid":"505498059","full_name":"dmcg/dask-multicloud-poc","owner":"dmcg","description":null,"archived":false,"fork":false,"pushed_at":"2022-07-11T11:35:27.000Z","size":2533,"stargazers_count":6,"open_issues_count":0,"forks_count":0,"subscribers_count":6,"default_branch":"main","last_synced_at":"2025-10-08T19:35:02.321Z","etag":null,"topics":[],"latest_commit_sha":null,"homepage":null,"language":"Shell","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"other","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/dmcg.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":null,"funding":null,"license":null,"code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null,"zenodo":null}},"created_at":"2022-06-20T15:31:32.000Z","updated_at":"2024-09-26T12:18:25.000Z","dependencies_parsed_at":null,"dependency_job_id":"11ae0633-cadc-4f05-99ea-0fa817ec07b2","html_url":"https://github.com/dmcg/dask-multicloud-poc","commit_stats":null,"previous_names":[],"tags_count":0,"template":false,"template_full_name":null,"purl":"pkg:github/dmcg/dask-multicloud-poc","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/dmcg%2Fdask-multicloud-poc","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/dmcg%2Fdask-multicloud-poc/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/dmcg%2Fdask-multicloud-poc/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/dmcg%2Fdask-multicloud-poc/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/dmcg","download_url":"https://codeload.github.com/dmcg/dask-multicloud-poc/tar.gz/refs/heads/main","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/dmcg%2Fdask-multicloud-poc/sbom","scorecard":null,"host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":286080680,"owners_count":29942859,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2026-02-28T13:49:17.081Z","status":"ssl_error","status_checked_at":"2026-02-28T13:48:50.396Z","response_time":90,"last_error":"SSL_connect returned=1 errno=0 peeraddr=140.82.121.5:443 state=error: unexpected eof while reading","robots_txt_status":"success","robots_txt_updated_at":"2025-07-24T06:49:26.215Z","robots_txt_url":"https://github.com/robots.txt","online":false,"can_crawl_api":true,"host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":[],"created_at":"2024-10-15T01:57:00.527Z","updated_at":"2026-02-28T16:32:27.492Z","avatar_url":"https://github.com/dmcg.png","language":"Shell","funding_links":[],"categories":[],"sub_categories":[],"readme":"# Dask Multicloud Proof of Concept\n\nWe have devised a technique for creating a Dask cluster where worker nodes are \nhosted in different data centres, connected by a mesh VPN that allows the\nscheduler and workers to communicate and exchange results.\n\nA novel (ab)use of Dask resources allows us to run data processing tasks on the\nworkers in the cluster closest to the source data, so that communication between\ndata centres is minimised. If combined with zarr to give access to huge\nhyper-cube datasets in object storage, we believe that the technique could \nrealise the potential of data-proximate distributed computing in the Cloud.\n\nThis repository documents a running proof-of-concept that addresses these problems.\nIt contains\n\n## ./demo/\n\nWe show the working of the system in a Jupyter notebook\n\n[dask-multi-cloud.ipynb](./demo/dask-multi-cloud.ipynb) \n\nand more details in \n\n[dask-multi-cloud-details.ipynb](./demo/dask-multi-cloud-details.ipynb).\n\n## ./build.sh\n\nBuilds the Docker image used to host the scheduler and workers.\n\n## ./docker/\n\nResources for the Docker image.\n\n## ./ansible/\n\nAn example Ansible playbook showing how to commission worker machines.\n\n## ./start-cluster\n\nA script to start a Dask scheduler and a distributed cluster.","project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fdmcg%2Fdask-multicloud-poc","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Fdmcg%2Fdask-multicloud-poc","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fdmcg%2Fdask-multicloud-poc/lists"}