{"id":13447604,"url":"https://github.com/tawsbob/puppeteer-infinite-scroll","last_synced_at":"2026-01-17T17:40:04.064Z","repository":{"id":57332057,"uuid":"136332677","full_name":"tawsbob/puppeteer-infinite-scroll","owner":"tawsbob","description":"aabstraction to help scrape data from sites with infinite scroll","archived":false,"fork":false,"pushed_at":"2020-03-03T14:50:30.000Z","size":9,"stargazers_count":9,"open_issues_count":0,"forks_count":2,"subscribers_count":0,"default_branch":"master","last_synced_at":"2025-10-02T06:24:53.095Z","etag":null,"topics":[],"latest_commit_sha":null,"homepage":null,"language":"JavaScript","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"mit","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/tawsbob.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":null,"funding":null,"license":"LICENSE","code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null}},"created_at":"2018-06-06T13:28:24.000Z","updated_at":"2021-02-19T01:58:51.000Z","dependencies_parsed_at":"2022-09-21T03:43:12.794Z","dependency_job_id":null,"html_url":"https://github.com/tawsbob/puppeteer-infinite-scroll","commit_stats":null,"previous_names":[],"tags_count":0,"template":false,"template_full_name":null,"purl":"pkg:github/tawsbob/puppeteer-infinite-scroll","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/tawsbob%2Fpuppeteer-infinite-scroll","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/tawsbob%2Fpuppeteer-infinite-scroll/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/tawsbob%2Fpuppeteer-infinite-scroll/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/tawsbob%2Fpuppeteer-infinite-scroll/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/tawsbob","download_url":"https://codeload.github.com/tawsbob/puppeteer-infinite-scroll/tar.gz/refs/heads/master","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/tawsbob%2Fpuppeteer-infinite-scroll/sbom","scorecard":{"id":869781,"data":{"date":"2025-08-11","repo":{"name":"github.com/tawsbob/puppeteer-infinite-scroll","commit":"a44b7c010bea570306f743ccdf04e0be83040579"},"scorecard":{"version":"v5.2.1-40-gf6ed084d","commit":"f6ed084d17c9236477efd66e5b258b9d4cc7b389"},"score":2.1,"checks":[{"name":"Token-Permissions","score":-1,"reason":"No tokens found","details":null,"documentation":{"short":"Determines if the project's workflows follow the principle of least privilege.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#token-permissions"}},{"name":"Binary-Artifacts","score":10,"reason":"no binaries found in the repo","details":null,"documentation":{"short":"Determines if the project has generated executable (binary) artifacts in the source repository.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#binary-artifacts"}},{"name":"Packaging","score":-1,"reason":"packaging workflow not detected","details":["Warn: no GitHub/GitLab publishing workflow detected."],"documentation":{"short":"Determines if the project is published as a package that others can easily download, install, easily update, and uninstall.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#packaging"}},{"name":"Dangerous-Workflow","score":-1,"reason":"no workflows found","details":null,"documentation":{"short":"Determines if the project's GitHub Action workflows avoid dangerous patterns.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#dangerous-workflow"}},{"name":"Pinned-Dependencies","score":-1,"reason":"no dependencies found","details":null,"documentation":{"short":"Determines if the project has declared and pinned the dependencies of its build process.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#pinned-dependencies"}},{"name":"SAST","score":0,"reason":"no SAST tool detected","details":["Warn: no pull requests merged into dev branch"],"documentation":{"short":"Determines if the project uses static code analysis.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#sast"}},{"name":"Code-Review","score":0,"reason":"Found 0/6 approved changesets -- score normalized to 0","details":null,"documentation":{"short":"Determines if the project requires human code review before pull requests (aka merge requests) are merged.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#code-review"}},{"name":"Maintained","score":0,"reason":"0 commit(s) and 0 issue activity found in the last 90 days -- score normalized to 0","details":null,"documentation":{"short":"Determines if the project is \"actively maintained\".","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#maintained"}},{"name":"CII-Best-Practices","score":0,"reason":"no effort to earn an OpenSSF best practices badge detected","details":null,"documentation":{"short":"Determines if the project has an OpenSSF (formerly CII) Best Practices Badge.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#cii-best-practices"}},{"name":"Security-Policy","score":0,"reason":"security policy file not detected","details":["Warn: no security policy file detected","Warn: no security file to analyze","Warn: no security file to analyze","Warn: no security file to analyze"],"documentation":{"short":"Determines if the project has published a security policy.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#security-policy"}},{"name":"Fuzzing","score":0,"reason":"project is not fuzzed","details":["Warn: no fuzzer integrations found"],"documentation":{"short":"Determines if the project uses fuzzing.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#fuzzing"}},{"name":"License","score":10,"reason":"license file detected","details":["Info: project has a license file: LICENSE:0","Info: FSF or OSI recognized license: MIT License: LICENSE:0"],"documentation":{"short":"Determines if the project has defined a license.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#license"}},{"name":"Branch-Protection","score":0,"reason":"branch protection not enabled on development/release branches","details":["Warn: branch protection not enabled for branch 'master'"],"documentation":{"short":"Determines if the default and release branches are protected with GitHub's branch protection settings.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#branch-protection"}},{"name":"Signed-Releases","score":-1,"reason":"no releases found","details":null,"documentation":{"short":"Determines if the project cryptographically signs release artifacts.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#signed-releases"}},{"name":"Vulnerabilities","score":3,"reason":"7 existing vulnerabilities detected","details":["Warn: Project is vulnerable to: GHSA-v6h2-p8h4-qcjw","Warn: Project is vulnerable to: GHSA-pc5p-h8pf-mvwp","Warn: Project is vulnerable to: GHSA-f8q6-p94x-37v3","Warn: Project is vulnerable to: GHSA-vh95-rmgr-6w4m","Warn: Project is vulnerable to: GHSA-xvch-5gv4-984h","Warn: Project is vulnerable to: GHSA-c2gp-86p4-5935","Warn: Project is vulnerable to: GHSA-3h5v-q93c-6h6q"],"documentation":{"short":"Determines if the project has open, known unfixed vulnerabilities.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#vulnerabilities"}}]},"last_synced_at":"2025-08-24T03:48:08.701Z","repository_id":57332057,"created_at":"2025-08-24T03:48:08.701Z","updated_at":"2025-08-24T03:48:08.701Z"},"host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":286080680,"owners_count":28513904,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2026-01-17T13:38:16.342Z","status":"ssl_error","status_checked_at":"2026-01-17T13:37:44.060Z","response_time":85,"last_error":"SSL_read: unexpected eof while reading","robots_txt_status":"success","robots_txt_updated_at":"2025-07-24T06:49:26.215Z","robots_txt_url":"https://github.com/robots.txt","online":false,"can_crawl_api":true,"host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":[],"created_at":"2024-07-31T05:01:22.194Z","updated_at":"2026-01-17T17:40:04.001Z","avatar_url":"https://github.com/tawsbob.png","language":"JavaScript","funding_links":[],"categories":["JavaScript"],"sub_categories":[],"readme":"# puppeter-infinite-scroll\nJust a helper to scrape data in sites that use infinete scroll.\n```\nnpm install puppeter-infinite-scroll\n```\n\n# The problem\nin most of solution that I found use a timing to scroll down the webpage and evaluate what you need, but if the request or network slow down and take more time than defined in the code and then the scraper just fail.\n\n# See working\n```\nnpm run test\n```\n\n# How use?\n```javascript\nconst puppeteerInfiniteScroll = require('./src/puppeter-infinite-scroll')\n\n;(async ()=\u003e{\ntry {\n  const browser = new puppeteerInfiniteScroll()\n  await browser.start()\n  await browser.open({\n    url: 'https://medium.com/search?q=python',\n    endpoint: 'https://medium.com/search/posts?q',\n    loadImages: false,\n    onResponse: (res)=\u003e{\n      //console.log(res)\n    },\n    onScroll: ()=\u003e{\n      console.log(`onScroll ${browser.scrollCount}`)\n\n    }\n  })\n} catch (e) {\n  console.error(e)\n}\n\n})()\n```\n\n# async browser.start() = puppeteer.lauch(opts)\n\n```javascript\n    //params(opts)\n    //default: { headless: false, devtools: true }\n    await browser.start()\n```\n# async browser.open()\nthis method create a new page.  setViewport({ width: 1280, height: 926 }), setRequestInterception(true)\n```javascript\n    //params(opts)\n    //default: { url, onResponse, onScroll, loadImages = true, endpoint }\n    //url = 'https://medium.com/search?q=python' - url of the page to be loaded\n    //endpoint = 'https://medium.com/search?q=python' - endpoint wich load content to page\n    //loadImages = true - if you need to prevent to load images set to false\n    //onResponse = (response)=\u003e{ } - if you need do something with request object\n    //onScroll = ()=\u003e{} - trigged after every scroll\n\n    await browser.open({\n    url: 'https://medium.com/search?q=python',\n    endpoint: 'https://medium.com/search/posts?q',\n    loadImages: false,\n    onResponse: (res)=\u003e{\n      //console.log(res)\n    },\n    onScroll: ()=\u003e{\n      console.log(`onScroll ${browser.scrollCount}`)\n    }\n  })\n```\n","project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Ftawsbob%2Fpuppeteer-infinite-scroll","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Ftawsbob%2Fpuppeteer-infinite-scroll","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Ftawsbob%2Fpuppeteer-infinite-scroll/lists"}