{"id":23817882,"url":"https://github.com/lsongdev/node-crawler","last_synced_at":"2025-02-21T21:57:28.776Z","repository":{"id":68773915,"uuid":"119159141","full_name":"lsongdev/node-crawler","owner":"lsongdev","description":"simple crawler","archived":false,"fork":false,"pushed_at":"2018-01-27T11:41:12.000Z","size":4,"stargazers_count":0,"open_issues_count":0,"forks_count":0,"subscribers_count":3,"default_branch":"master","last_synced_at":"2025-02-13T22:23:45.036Z","etag":null,"topics":["crawler","node-crawler"],"latest_commit_sha":null,"homepage":"","language":"JavaScript","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"isc","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/lsongdev.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":null,"funding":null,"license":"LICENSE","code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null},"funding":{"github":"song940","patreon":"song940","open_collective":"song940","ko_fi":null,"tidelift":null,"community_bridge":null,"liberapay":null,"issuehunt":null,"otechie":null,"custom":"https://git.io/fjRcB"}},"created_at":"2018-01-27T11:40:27.000Z","updated_at":"2019-11-05T02:28:53.000Z","dependencies_parsed_at":"2023-02-23T00:16:10.313Z","dependency_job_id":null,"html_url":"https://github.com/lsongdev/node-crawler","commit_stats":null,"previous_names":["lsongdev/node-crawler"],"tags_count":0,"template":false,"template_full_name":null,"repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/lsongdev%2Fnode-crawler","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/lsongdev%2Fnode-crawler/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/lsongdev%2Fnode-crawler/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/lsongdev%2Fnode-crawler/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/lsongdev","download_url":"https://codeload.github.com/lsongdev/node-crawler/tar.gz/refs/heads/master","host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":240093155,"owners_count":19746780,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2022-07-04T15:15:14.044Z","host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":["crawler","node-crawler"],"created_at":"2025-01-02T05:48:59.565Z","updated_at":"2025-02-21T21:57:28.752Z","avatar_url":"https://github.com/lsongdev.png","language":"JavaScript","readme":"## node-crawler\n\n\u003e simple crawler\n\n### Installation\n\n```bash\n$ npm install @song940/crawler --save\n```\n\n### Example\n\n```js\nconst Crawler = require('@song940/crawler');\n\nconst engine = new Crawler({ \n    url: 'https://news.ycombinator.com'\n});\n\nengine.parse = async ($, commit) =\u003e {\n    $('table.itemlist tr.athing').each((i, row) =\u003e {\n        const title = $('td.title', row).text();\n        commit({ title });\n    });\n};\n\nengine.on('commit', posts =\u003e {\n    console.log(posts);\n});\n\nfor(var i=1;i\u003c100;i++){\n    engine.push(`${engine.url}/news?p=${i}`);\n}\n\nengine.on('end', () =\u003e {\n    console.log('all job done');\n});\n\nengine.start();\n\n```\n\n### Contributing\n- Fork this Repo first\n- Clone your Repo\n- Install dependencies by `$ npm install`\n- Checkout a feature branch\n- Feel free to add your features\n- Make sure your features are fully tested\n- Publish your local branch, Open a pull request\n- Enjoy hacking \u003c3\n\n### ISC\n\nThis work is licensed under the [ISC license](./LICENSE).\n\n---","funding_links":["https://github.com/sponsors/song940","https://patreon.com/song940","https://opencollective.com/song940","https://git.io/fjRcB"],"categories":[],"sub_categories":[],"project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Flsongdev%2Fnode-crawler","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Flsongdev%2Fnode-crawler","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Flsongdev%2Fnode-crawler/lists"}