{"id":17661813,"url":"https://github.com/18520339/web-scraping-with-scrapy","last_synced_at":"2025-03-30T11:26:14.192Z","repository":{"id":56558144,"uuid":"275875499","full_name":"18520339/web-scraping-with-scrapy","owner":"18520339","description":"Python web scraping with Scrapy","archived":false,"fork":false,"pushed_at":"2020-10-31T17:17:12.000Z","size":491,"stargazers_count":1,"open_issues_count":0,"forks_count":0,"subscribers_count":1,"default_branch":"master","last_synced_at":"2025-03-25T23:47:10.343Z","etag":null,"topics":["scrapy","web-crawling","web-scraping"],"latest_commit_sha":null,"homepage":"https://www.youtube.com/playlist?list=PLhTjy8cBISEqkN-5Ku_kXG4QW33sxQo0t","language":"Python","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":null,"status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/18520339.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":null,"funding":null,"license":null,"code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null}},"created_at":"2020-06-29T16:55:00.000Z","updated_at":"2022-08-17T20:13:39.000Z","dependencies_parsed_at":"2022-08-15T20:51:01.199Z","dependency_job_id":null,"html_url":"https://github.com/18520339/web-scraping-with-scrapy","commit_stats":null,"previous_names":[],"tags_count":0,"template":false,"template_full_name":null,"repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/18520339%2Fweb-scraping-with-scrapy","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/18520339%2Fweb-scraping-with-scrapy/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/18520339%2Fweb-scraping-with-scrapy/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/18520339%2Fweb-scraping-with-scrapy/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/18520339","download_url":"https://codeload.github.com/18520339/web-scraping-with-scrapy/tar.gz/refs/heads/master","host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":246310233,"owners_count":20756898,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2022-07-04T15:15:14.044Z","host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":["scrapy","web-crawling","web-scraping"],"created_at":"2024-10-23T17:42:41.374Z","updated_at":"2025-03-30T11:26:14.163Z","avatar_url":"https://github.com/18520339.png","language":"Python","funding_links":[],"categories":[],"sub_categories":[],"readme":"# Python web scraping with Scrapy\n\u003e Demo: https://www.youtube.com/watch?v=ysyskgjsPI0\u0026t=1m45s\n\n## Features:\n+ Download Images\n+ Store data in many kinds of database: SQLite, MySQL, MongDB\n+ Store data in many formats: csv, json, xml\n+ Using User Agent, Proxy\n\n## Installation:\n1. run `pip install -r requirements.txt`\n2. Install and connect SQLite, MySQL, MongDB\n\n## Usage:\n+ for .csv: `scrapy crawl amazon -o \"store by formats\"/products.csv`\n+ for .json: `scrapy crawl amazon -o \"store by formats\"/products.json`\n+ for .xml: `scrapy crawl amazon -o \"store by formats\"/products.xml`\n","project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2F18520339%2Fweb-scraping-with-scrapy","html_url":"https://awesome.ecosyste.ms/projects/github.com%2F18520339%2Fweb-scraping-with-scrapy","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2F18520339%2Fweb-scraping-with-scrapy/lists"}