{"id":13462958,"url":"https://github.com/documentcloud/cloud-crowd","last_synced_at":"2025-04-12T20:42:55.678Z","repository":{"id":647532,"uuid":"289802","full_name":"documentcloud/cloud-crowd","owner":"documentcloud","description":"Parallel Processing for the Rest of Us","archived":false,"fork":false,"pushed_at":"2023-01-20T09:47:10.000Z","size":1007,"stargazers_count":853,"open_issues_count":21,"forks_count":85,"subscribers_count":28,"default_branch":"master","last_synced_at":"2025-04-04T00:07:54.408Z","etag":null,"topics":[],"latest_commit_sha":null,"homepage":"https://github.com/documentcloud/cloud-crowd/wiki","language":"Ruby","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"mit","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/documentcloud.png","metadata":{"files":{"readme":"README","changelog":null,"contributing":null,"funding":null,"license":"LICENSE","code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null}},"created_at":"2009-08-27T12:42:56.000Z","updated_at":"2025-02-19T19:47:00.000Z","dependencies_parsed_at":"2023-02-12T01:16:04.654Z","dependency_job_id":null,"html_url":"https://github.com/documentcloud/cloud-crowd","commit_stats":null,"previous_names":[],"tags_count":21,"template":false,"template_full_name":null,"repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/documentcloud%2Fcloud-crowd","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/documentcloud%2Fcloud-crowd/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/documentcloud%2Fcloud-crowd/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/documentcloud%2Fcloud-crowd/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/documentcloud","download_url":"https://codeload.github.com/documentcloud/cloud-crowd/tar.gz/refs/heads/master","host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":248631674,"owners_count":21136554,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2022-07-04T15:15:14.044Z","host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":[],"created_at":"2024-07-31T13:00:42.336Z","updated_at":"2025-04-12T20:42:55.647Z","avatar_url":"https://github.com/documentcloud.png","language":"Ruby","readme":"=                                                                               \n           _  _                                                                \n          ( `   )_                                                             \n         (    )    `)                                                          \n       (_   (_ .  _) _)                                                        \n                                      _                                        \n                                     (  )                                      \n      _ .                         ( `  ) . )                                   \n    (  _ )_                      (_, _(  ,_)_)                                 \n  (_  _(_ ,)                                                                   \n                                                                               \n           _  _               ___ _             _  ___                   _     \n          ( `   )_           / __| |___ _  _ __| |/ __|_ _ _____ __ ____| |    \n         (    )    `)       | (__| / _ \\ || / _` | (__| '_/ _ \\ V  V / _` |    \n       (_   (_ .  _) _)      \\___|_\\___/\\_,_\\__,_|\\___|_| \\___/\\_/\\_/\\__,_|    \n                                                                               \n                                                     _                         \n                                                    (  )                       \n                  _, _ .                         ( `  ) . )                    \n                 ( (  _ )_                      (_, _(  ,_)_)                  \n               (_(_  _(_ ,)                                                    \n                                                                               \n                                                                               \n                                                                               \n  ~ CloudCrowd ~\n\n    * Parallel processing for the rest of us\n    * Write your scripts in Ruby\n    * Works with Amazon EC2 and S3\n    * split -\u003e process -\u003e merge\n    * As easy as `gem install cloud-crowd`\n\n    Well-suited for:\n    \n    * Generating or resizing images.\n    * Encoding video.\n    * Running text extraction or OCR on PDFs.\n    * Migrating a large file set or database.\n    * Web scraping.\n    \n    \n  ~ Documentation ~\n  \n    Wiki: https://github.com/documentcloud/cloud-crowd/wiki\n    Rdoc: http://www.rubydoc.info/github/documentcloud/cloud-crowd\n  \n  \n  ~ Getting started ~\n  \n    # Install the gem.\n    \n      \u003e\u003e sudo gem install cloud-crowd\n    \n    # Install the CloudCrowd configuration files to a location of your choosing.\n    \n      \u003e\u003e crowd install ~/config/cloud-crowd\n    \n    # Now, you can use the full complement of `crowd` commands from inside of\n    # this configuration directory. To see the available commands:\n    \n      \u003e\u003e crowd --help\n    \n    # Edit the configuration files to your satisfaction, add AWS credentials, \n    # and then load the CloudCrowd schema into your configured database.\n    \n      \u003e\u003e cd ~/config/cloud-crowd\n      \u003e\u003e mate config.yml\n      \u003e\u003e mate database.yml\n      \u003e\u003e [create the database you just configured...]\n      \u003e\u003e crowd load_schema\n    \n    # Write your actions, and install them into the 'actions' subdirectory.\n    # CloudCrowd comes with a few default actions as an example.\n    \n    # To launch the central server (make sure that you include its location\n    # in config.yml):\n    \n      \u003e\u003e crowd server\n    \n    # The configuration folder also includes 'config.ru', which can be used by\n     # any Rack-compliant webserver to run your central server.\n    \n    # Then, to launch a node of workers:\n    \n      \u003e\u003e crowd node\n    \n    # To spin up remote nodes, install the 'cloud-crowd' gem and copy over\n    # your configuration directory. Run `crowd node`, and the remote machines\n    # will register with the central server, becoming available for processing.\n    \n    # At this point you can visit your Operations Center at localhost:9173 to \n    # view all of your nodes, ready for action.\n  \n","funding_links":[],"categories":["Background Processing","Ruby"],"sub_categories":["Background Jobs"],"project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fdocumentcloud%2Fcloud-crowd","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Fdocumentcloud%2Fcloud-crowd","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fdocumentcloud%2Fcloud-crowd/lists"}