{"id":14978822,"url":"https://github.com/voliveirajr/seleniumcrawler","last_synced_at":"2025-10-11T21:10:05.299Z","repository":{"id":81345690,"uuid":"12383821","full_name":"voliveirajr/seleniumcrawler","owner":"voliveirajr","description":"An example using Selenium webdrivers for python and Scrapy framework to create a web scraper to crawl an ASP site","archived":false,"fork":false,"pushed_at":"2019-02-28T14:18:19.000Z","size":21,"stargazers_count":127,"open_issues_count":0,"forks_count":45,"subscribers_count":9,"default_branch":"master","last_synced_at":"2025-04-08T19:52:36.047Z","etag":null,"topics":["asp-net","python","scraper","scraping","scraping-websites","scrapper","scrapy","selenium","selenium-webdriver","webcrawler","webcrawling"],"latest_commit_sha":null,"homepage":null,"language":"Python","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"gpl-3.0","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/voliveirajr.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":null,"funding":null,"license":"LICENSE","code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null}},"created_at":"2013-08-26T16:18:50.000Z","updated_at":"2025-03-18T06:31:16.000Z","dependencies_parsed_at":null,"dependency_job_id":"f4120992-62ef-4c1a-ae23-c72f4306637b","html_url":"https://github.com/voliveirajr/seleniumcrawler","commit_stats":null,"previous_names":[],"tags_count":0,"template":false,"template_full_name":null,"purl":"pkg:github/voliveirajr/seleniumcrawler","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/voliveirajr%2Fseleniumcrawler","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/voliveirajr%2Fseleniumcrawler/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/voliveirajr%2Fseleniumcrawler/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/voliveirajr%2Fseleniumcrawler/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/voliveirajr","download_url":"https://codeload.github.com/voliveirajr/seleniumcrawler/tar.gz/refs/heads/master","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/voliveirajr%2Fseleniumcrawler/sbom","scorecard":null,"host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":279008731,"owners_count":26084494,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2022-07-04T15:15:14.044Z","status":"online","status_checked_at":"2025-10-11T02:00:06.511Z","response_time":55,"last_error":null,"robots_txt_status":"success","robots_txt_updated_at":"2025-07-24T06:49:26.215Z","robots_txt_url":"https://github.com/robots.txt","online":true,"can_crawl_api":true,"host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":["asp-net","python","scraper","scraping","scraping-websites","scrapper","scrapy","selenium","selenium-webdriver","webcrawler","webcrawling"],"created_at":"2024-09-24T13:58:28.403Z","updated_at":"2025-10-11T21:10:05.294Z","avatar_url":"https://github.com/voliveirajr.png","language":"Python","funding_links":[],"categories":[],"sub_categories":[],"readme":"seleniumcrawler\n===============\n\nThis is a Webcrawler based on Scrapy and Selenium frameworks\n\nThis spider crawls thru the directferries.com website in order to generate a json file with all tickets available for one \nof these directions Dublin-Liverpool / Liverpool-Dublin with departure tomorrow and returning in 3 days.\n\nHOW TO EXECUTE:\n\nIs required an environment with the following requirements installed:\n-Python 2.7\n-Scrapy 0.18\n-Selenium web-drivers\n\nTo execute the crawler the following command should be executed from the project path\n\nscrapy crawl crawlermate_selenium -a category=[dublin or liverpool] -o [filename] -t json\n\nfor an example, to generate tickets for dublin to liverpool on items.json file you should execute\nscrapy crawl crawlermate_selenium -a category=dublin -o items.json -t json\n\nReferences:\n\nhttp://docs.seleniumhq.org/\nhttp://http://scrapy.org/\n","project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fvoliveirajr%2Fseleniumcrawler","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Fvoliveirajr%2Fseleniumcrawler","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fvoliveirajr%2Fseleniumcrawler/lists"}