{"id":31790742,"url":"https://github.com/slava-vishnyakov/grucrawler","last_synced_at":"2026-07-10T08:31:50.205Z","repository":{"id":23465731,"uuid":"26829958","full_name":"slava-vishnyakov/grucrawler","owner":"slava-vishnyakov","description":"Simple Ruby crawler","archived":false,"fork":false,"pushed_at":"2014-11-18T23:14:18.000Z","size":196,"stargazers_count":0,"open_issues_count":0,"forks_count":0,"subscribers_count":0,"default_branch":"master","last_synced_at":"2026-05-24T13:07:11.648Z","etag":null,"topics":["crawler","ruby"],"latest_commit_sha":null,"homepage":null,"language":"Ruby","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"mit","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/slava-vishnyakov.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":null,"funding":null,"license":"LICENSE.txt","code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null}},"created_at":"2014-11-18T21:06:39.000Z","updated_at":"2014-11-18T23:14:10.000Z","dependencies_parsed_at":"2022-07-15T23:46:15.539Z","dependency_job_id":null,"html_url":"https://github.com/slava-vishnyakov/grucrawler","commit_stats":null,"previous_names":[],"tags_count":4,"template":false,"template_full_name":null,"purl":"pkg:github/slava-vishnyakov/grucrawler","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/slava-vishnyakov%2Fgrucrawler","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/slava-vishnyakov%2Fgrucrawler/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/slava-vishnyakov%2Fgrucrawler/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/slava-vishnyakov%2Fgrucrawler/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/slava-vishnyakov","download_url":"https://codeload.github.com/slava-vishnyakov/grucrawler/tar.gz/refs/heads/master","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/slava-vishnyakov%2Fgrucrawler/sbom","scorecard":null,"host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":286080680,"owners_count":35326962,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2026-05-26T15:22:16.424Z","status":"online","status_checked_at":"2026-07-10T02:00:06.465Z","response_time":60,"last_error":null,"robots_txt_status":"success","robots_txt_updated_at":"2025-07-24T06:49:26.215Z","robots_txt_url":"https://github.com/robots.txt","online":true,"can_crawl_api":true,"host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":["crawler","ruby"],"created_at":"2025-10-10T16:29:21.941Z","updated_at":"2026-07-10T08:31:50.185Z","avatar_url":"https://github.com/slava-vishnyakov.png","language":"Ruby","funding_links":[],"categories":[],"sub_categories":[],"readme":"# Grucrawler\n\n```ruby\nrequire 'grucrawler'\nrequire 'colorize'\n\nclass ItalianCrawler\n  def options\n    {\n        visit_urls_only_once: true,\n        follow_redirects: true,\n        concurrency: 5,\n        domain_wait: 20, # seconds between visits to the same domain\n        max_page_size: 1000000\n    }\n  end\n\n  def on_init(crawler)\n    @crawler = crawler\n  end\n\n  def on_page_received(typhoeus_response, nokogiri_html)\n    puts \"GOT #{typhoeus_response.effective_url.green}\"\n\n    # typhoeus_response.body\n    # typhoeus_response.request.url\n    # typhoeus_response.effective_url\n    # nokogiri_html.css('a').each |a| { puts a.text; }\n  end\n\n  def follow_link(target_url, typhoeus_response, nokogiri_html)\n    return false if target_url.match(/\\.(jpg|png|js|css|pdf|exe|dmg|zip|doc|rtf|rar|swf|bmp|swf|mp3|wav|mp4|mpg|flv|wma)$/)\n\n    return true if target_url.include? '.it'\n\n    false\n  end\n\n  def debug(message)\n    #puts message.blue\n  end\n\n  def log_info(message)\n    puts message.yellow\n  end\n\n  def log_error(typhoeus_response, exception)\n    puts exception.to_s.red\n  end\nend\n\nc = GruCrawler.new(ItalianCrawler.new)\n# c.reset() # deletes all memory of all events - useful for restarting crawl\nc.add_url('http://www.oneworlditaliano.com/english/italian/news-in-italian.htm')\nc.run()\n```\n\n\n\n## Installation\n\n[!] Requires local Redis at the moment.\n\nAdd this line to your application's Gemfile:\n\n    gem 'grucrawler'\n\nAnd then execute:\n\n    $ bundle\n\nOr install it yourself as:\n\n    $ gem install grucrawler\n\n\n## Contributing\n\n1. Fork it ( https://github.com/[my-github-username]/grucrawler/fork )\n2. Create your feature branch (`git checkout -b my-new-feature`)\n3. Commit your changes (`git commit -am 'Add some feature'`)\n4. Push to the branch (`git push origin my-new-feature`)\n5. Create a new Pull Request\n","project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fslava-vishnyakov%2Fgrucrawler","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Fslava-vishnyakov%2Fgrucrawler","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fslava-vishnyakov%2Fgrucrawler/lists"}