{"id":13998130,"url":"https://github.com/danielbeach/RustForDataPipelines","last_synced_at":"2025-07-23T05:32:38.242Z","repository":{"id":113322565,"uuid":"588777907","full_name":"danielbeach/RustForDataPipelines","owner":"danielbeach","description":"Testing out if Rust can be used for a normal Data Engineering Pipeline.","archived":false,"fork":false,"pushed_at":"2023-03-03T01:07:34.000Z","size":4,"stargazers_count":11,"open_issues_count":0,"forks_count":2,"subscribers_count":3,"default_branch":"main","last_synced_at":"2024-11-17T21:11:57.302Z","etag":null,"topics":[],"latest_commit_sha":null,"homepage":null,"language":"Rust","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":null,"status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/danielbeach.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":null,"funding":null,"license":null,"code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null}},"created_at":"2023-01-14T01:14:06.000Z","updated_at":"2024-08-30T08:44:11.000Z","dependencies_parsed_at":"2023-11-01T03:15:15.096Z","dependency_job_id":null,"html_url":"https://github.com/danielbeach/RustForDataPipelines","commit_stats":null,"previous_names":[],"tags_count":0,"template":false,"template_full_name":null,"repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/danielbeach%2FRustForDataPipelines","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/danielbeach%2FRustForDataPipelines/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/danielbeach%2FRustForDataPipelines/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/danielbeach%2FRustForDataPipelines/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/danielbeach","download_url":"https://codeload.github.com/danielbeach/RustForDataPipelines/tar.gz/refs/heads/main","host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":227245132,"owners_count":17753239,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2022-07-04T15:15:14.044Z","host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":[],"created_at":"2024-08-09T19:01:25.102Z","updated_at":"2024-11-30T00:31:12.347Z","avatar_url":"https://github.com/danielbeach.png","language":"Rust","funding_links":[],"categories":["Rust"],"sub_categories":[],"readme":"# RustForDataPipelines\nTesting out if Rust can be used for a normal Data Engineering Pipeline.\n\nCheck out the full blog post here. \nhttps://www.confessionsofadataguy.com/using-rust-to-write-a-data-pipeline-thoughts-musings/\n\nThe idea was to try things like HTTP, File Systems, and Database calls (Postgres) out with Rust.\n\nThese are normal data engineering functions, is it too verbose to write data pipelines in Rust?\nThe data flow is as follows\n- download `.zip` file.\n- unpack the `.zip` file into `.csv`, mess with files.\n- connect Rust to `postgres`\n- Process CSV file into `postgres`\n- Run some `sql` and push some `rows`\n","project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fdanielbeach%2FRustForDataPipelines","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Fdanielbeach%2FRustForDataPipelines","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fdanielbeach%2FRustForDataPipelines/lists"}