{"id":22678953,"url":"https://github.com/ammirsm/data-grabber-cnn-twitter","last_synced_at":"2026-05-02T13:32:35.069Z","repository":{"id":37061379,"uuid":"220649382","full_name":"ammirsm/data-grabber-cnn-twitter","owner":"ammirsm","description":"Basic setup to get data from twitter and CNN with a keyword.","archived":false,"fork":false,"pushed_at":"2022-11-04T19:28:11.000Z","size":706,"stargazers_count":1,"open_issues_count":4,"forks_count":1,"subscribers_count":1,"default_branch":"master","last_synced_at":"2025-03-29T13:13:24.925Z","etag":null,"topics":["cnn","crawler","django","scrapyd","twitter"],"latest_commit_sha":null,"homepage":"","language":"JavaScript","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"mit","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/ammirsm.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":null,"funding":null,"license":"LICENSE","code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null}},"created_at":"2019-11-09T13:53:24.000Z","updated_at":"2023-03-05T03:31:54.000Z","dependencies_parsed_at":"2023-01-20T12:33:46.662Z","dependency_job_id":null,"html_url":"https://github.com/ammirsm/data-grabber-cnn-twitter","commit_stats":null,"previous_names":[],"tags_count":0,"template":false,"template_full_name":null,"purl":"pkg:github/ammirsm/data-grabber-cnn-twitter","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/ammirsm%2Fdata-grabber-cnn-twitter","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/ammirsm%2Fdata-grabber-cnn-twitter/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/ammirsm%2Fdata-grabber-cnn-twitter/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/ammirsm%2Fdata-grabber-cnn-twitter/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/ammirsm","download_url":"https://codeload.github.com/ammirsm/data-grabber-cnn-twitter/tar.gz/refs/heads/master","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/ammirsm%2Fdata-grabber-cnn-twitter/sbom","scorecard":null,"host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":286080680,"owners_count":32536576,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2026-05-02T12:25:33.646Z","status":"ssl_error","status_checked_at":"2026-05-02T12:24:51.733Z","response_time":132,"last_error":"SSL_read: unexpected eof while reading","robots_txt_status":"success","robots_txt_updated_at":"2025-07-24T06:49:26.215Z","robots_txt_url":"https://github.com/robots.txt","online":false,"can_crawl_api":true,"host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":["cnn","crawler","django","scrapyd","twitter"],"created_at":"2024-12-09T18:18:17.054Z","updated_at":"2026-05-02T13:32:35.049Z","avatar_url":"https://github.com/ammirsm.png","language":"JavaScript","funding_links":[],"categories":[],"sub_categories":[],"readme":"# CNN and Twitter Crawler for getting clear data\nBasic setup to get data from twitter and CNN with a keyword.\n\n* A crawler that crawls the latest 25 articles about Trump from CNN.com and his latest tweets\n* A simple website that displays the titles of the crawled information\n* A convenient way of displaying the information after I click on one of the titles\n* Word Cloud of lastests news and tweet\n\n\n## Setup\n1 - Install requirements\n````\n$ pip install -r requirements.txt\n````\n2 - Configure the database\n````\n$ python manage.py migrate\n````\n## Start the project\nIn order to start this project you will need to have running Django and Scrapyd at the same time.\n\nIn order to run Django\n````\n$ python manage.py runserver\n````\nIn order to run Scrapyd\n````\n$ cd scrapy_app\n$ scrapyd\n````\n\nAt this point you will be able to send job request to Scrapyd. This project is setup with a demo spider from the oficial tutorial of scrapy. To run it you must send a http request to Scrapyd with the job info\n\nProject contains two spiders, 'icrawler' for crawling CNN and 'twitter' for crawling twitter.\n\n````\ncurl http://localhost:6800/schedule.json -d project=default -d spider=icrawler\n````\n\nThe crawled data will be automatically be saved in the Django models\n\nAn accessiblity have implemented in frontend for running these crawlers.","project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fammirsm%2Fdata-grabber-cnn-twitter","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Fammirsm%2Fdata-grabber-cnn-twitter","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fammirsm%2Fdata-grabber-cnn-twitter/lists"}