{"id":14982346,"url":"https://github.com/longnguyen010203/spark-processing-aws","last_synced_at":"2026-03-17T00:35:44.880Z","repository":{"id":244986436,"uuid":"816923148","full_name":"longNguyen010203/Spark-Processing-AWS","owner":"longNguyen010203","description":"👷🌇 Set up and build a big data processing pipeline with Apache Spark, 📦 AWS services (S3, EMR, EC2, IAM, VPC, Redshift) Terraform to setup the infrastructure and Integration Airflow to automate workflows🥊","archived":false,"fork":false,"pushed_at":"2024-07-12T10:20:19.000Z","size":1034,"stargazers_count":2,"open_issues_count":0,"forks_count":0,"subscribers_count":1,"default_branch":"main","last_synced_at":"2025-02-14T10:41:50.378Z","etag":null,"topics":["apache-airflow","apache-spark","aws","aws-ec2","aws-s3","aws-services","cloud-computing","data-pipeline","emr-cluster","iam","pyspark","redshift","spark-cluster","spark-master","spark-worker","terraform"],"latest_commit_sha":null,"homepage":"","language":"Python","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"apache-2.0","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/longNguyen010203.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":null,"funding":null,"license":"LICENSE","code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null}},"created_at":"2024-06-18T16:52:12.000Z","updated_at":"2024-09-10T01:49:53.000Z","dependencies_parsed_at":"2024-07-12T11:34:36.908Z","dependency_job_id":null,"html_url":"https://github.com/longNguyen010203/Spark-Processing-AWS","commit_stats":{"total_commits":85,"total_committers":1,"mean_commits":85.0,"dds":0.0,"last_synced_commit":"21d431e95d6038409e4c2ab713ba66c1b414bed0"},"previous_names":["longnguyen010203/spark-processing-aws"],"tags_count":0,"template":false,"template_full_name":null,"repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/longNguyen010203%2FSpark-Processing-AWS","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/longNguyen010203%2FSpark-Processing-AWS/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/longNguyen010203%2FSpark-Processing-AWS/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/longNguyen010203%2FSpark-Processing-AWS/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/longNguyen010203","download_url":"https://codeload.github.com/longNguyen010203/Spark-Processing-AWS/tar.gz/refs/heads/main","host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":243023599,"owners_count":20223448,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2022-07-04T15:15:14.044Z","host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":["apache-airflow","apache-spark","aws","aws-ec2","aws-s3","aws-services","cloud-computing","data-pipeline","emr-cluster","iam","pyspark","redshift","spark-cluster","spark-master","spark-worker","terraform"],"created_at":"2024-09-24T14:05:14.433Z","updated_at":"2025-12-25T00:13:49.509Z","avatar_url":"https://github.com/longNguyen010203.png","language":"Python","funding_links":[],"categories":[],"sub_categories":[],"readme":"# 👷 Spark-Processing-AWS\nIn this project, I set up and build a big data processing pipeline using Apache Spark integrated with various AWS services, including S3, EMR, EC2, VPC, IAM, and Redshift and Terraform to setup the infrastructure\n\n## 🔦 About Project\n\u003cimg src=\"./images/design_pipeline.png\"\u003e\n\n## 📦 Technologies\n - `S3`\n - `EMR`\n - `EC2`\n - `Airflow`\n - `Redshift`\n - `Terraform`\n - `Spark`\n - `VPC`\n - `IAM`\n\n## 🦄 Features\n## 👩🏽‍🍳 The Process\n## 📚 What I Learned\n## 💭 How can it be improved?\n## 🚦 Running the Project\n","project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Flongnguyen010203%2Fspark-processing-aws","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Flongnguyen010203%2Fspark-processing-aws","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Flongnguyen010203%2Fspark-processing-aws/lists"}