{"id":16542513,"url":"https://github.com/zhanymkanov/marketplace_parser","last_synced_at":"2026-05-26T23:33:27.314Z","repository":{"id":111804238,"uuid":"223702442","full_name":"zhanymkanov/marketplace_parser","owner":"zhanymkanov","description":"Products and Reviews Crawler","archived":false,"fork":false,"pushed_at":"2020-07-27T08:12:21.000Z","size":2770,"stargazers_count":1,"open_issues_count":0,"forks_count":2,"subscribers_count":1,"default_branch":"master","last_synced_at":"2025-12-04T16:54:30.313Z","etag":null,"topics":["crawler","python","scrapy"],"latest_commit_sha":null,"homepage":"","language":"Python","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"cc0-1.0","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/zhanymkanov.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":null,"funding":null,"license":"LICENSE","code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null}},"created_at":"2019-11-24T06:30:07.000Z","updated_at":"2023-10-13T09:37:54.000Z","dependencies_parsed_at":null,"dependency_job_id":"29d667cc-9026-4646-9fd5-d3354fa5f25e","html_url":"https://github.com/zhanymkanov/marketplace_parser","commit_stats":null,"previous_names":[],"tags_count":0,"template":false,"template_full_name":null,"purl":"pkg:github/zhanymkanov/marketplace_parser","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/zhanymkanov%2Fmarketplace_parser","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/zhanymkanov%2Fmarketplace_parser/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/zhanymkanov%2Fmarketplace_parser/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/zhanymkanov%2Fmarketplace_parser/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/zhanymkanov","download_url":"https://codeload.github.com/zhanymkanov/marketplace_parser/tar.gz/refs/heads/master","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/zhanymkanov%2Fmarketplace_parser/sbom","scorecard":null,"host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":286080680,"owners_count":33543973,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2026-05-26T15:22:16.424Z","status":"ssl_error","status_checked_at":"2026-05-26T15:22:15.568Z","response_time":63,"last_error":"SSL_connect returned=1 errno=0 peeraddr=140.82.121.5:443 state=error: unexpected eof while reading","robots_txt_status":"success","robots_txt_updated_at":"2025-07-24T06:49:26.215Z","robots_txt_url":"https://github.com/robots.txt","online":false,"can_crawl_api":true,"host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":["crawler","python","scrapy"],"created_at":"2024-10-11T18:57:47.001Z","updated_at":"2026-05-26T23:33:27.297Z","avatar_url":"https://github.com/zhanymkanov.png","language":"Python","funding_links":[],"categories":[],"sub_categories":[],"readme":"# About\nMarketplace Web-Crawler\n\n# How it works\n## Parser steps\n1. Use Products API to get JSON List of the products\n2. Use Products JSON List to crawl Products Specifications from HTML pages\n3. Use Products JSON List to request Reviews API for every product\n4. Clean the collected JSON files\n5. Extract valuable information from Product Specifications\n6. Dump data into the database\n\n### Comment on API access\n\u003ci\u003e\n  Although API is not private, it is nor public.\n\n  I had to do some stuff with my outgoing traffic to find out its endpoints. \n\n  Therefore, I think it is not tethical to put it online.\n\u003c/i\u003e\n\n## Installation\n### Prerequisites\n1. Python 3.8+\n2. Docker - optional\n\n### Installation steps\n1. Get the project\n```\ngit clone https://github.com/zhanymkanov/reviews_parser\n```\n2a. Install the packages without docker\n```\npip install -r requirements/base.txt\n```\n2b. Install the packages with docker\n```\ndocker-compose up -d --build\n```\n","project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fzhanymkanov%2Fmarketplace_parser","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Fzhanymkanov%2Fmarketplace_parser","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fzhanymkanov%2Fmarketplace_parser/lists"}