{"id":51292537,"url":"https://github.com/nermalcat69/lightweight-tweet-scraper","last_synced_at":"2026-06-30T11:32:19.675Z","repository":{"id":306768053,"uuid":"1027158618","full_name":"nermalcat69/lightweight-tweet-scraper","owner":"nermalcat69","description":"Scrape Tweets while Scrolling","archived":false,"fork":false,"pushed_at":"2025-08-20T15:15:59.000Z","size":24,"stargazers_count":82,"open_issues_count":0,"forks_count":15,"subscribers_count":0,"default_branch":"main","last_synced_at":"2026-06-20T20:27:45.470Z","etag":null,"topics":["coldran","scraping","tweet-scrape-scroll","tweet-scraping","tweets"],"latest_commit_sha":null,"homepage":"","language":"JavaScript","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"mit","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/nermalcat69.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":null,"funding":null,"license":"LICENSE","code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null,"zenodo":null,"notice":null,"maintainers":null,"copyright":null,"agents":null,"dco":null,"cla":null}},"created_at":"2025-07-27T12:46:25.000Z","updated_at":"2026-06-08T14:27:45.000Z","dependencies_parsed_at":"2025-08-05T10:36:33.596Z","dependency_job_id":null,"html_url":"https://github.com/nermalcat69/lightweight-tweet-scraper","commit_stats":null,"previous_names":["nermalcat69/tweet-scrape-scroll","coldranai/lightweight-tweet-scraper","nermalcat69/lightweight-tweet-scraper"],"tags_count":0,"template":false,"template_full_name":null,"purl":"pkg:github/nermalcat69/lightweight-tweet-scraper","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/nermalcat69%2Flightweight-tweet-scraper","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/nermalcat69%2Flightweight-tweet-scraper/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/nermalcat69%2Flightweight-tweet-scraper/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/nermalcat69%2Flightweight-tweet-scraper/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/nermalcat69","download_url":"https://codeload.github.com/nermalcat69/lightweight-tweet-scraper/tar.gz/refs/heads/main","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/nermalcat69%2Flightweight-tweet-scraper/sbom","scorecard":null,"host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":286080680,"owners_count":34965642,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2026-05-26T15:22:16.424Z","status":"online","status_checked_at":"2026-06-30T02:00:05.919Z","response_time":92,"last_error":null,"robots_txt_status":"success","robots_txt_updated_at":"2025-07-24T06:49:26.215Z","robots_txt_url":"https://github.com/robots.txt","online":true,"can_crawl_api":true,"host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":["coldran","scraping","tweet-scrape-scroll","tweet-scraping","tweets"],"created_at":"2026-06-30T11:32:19.014Z","updated_at":"2026-06-30T11:32:19.650Z","avatar_url":"https://github.com/nermalcat69.png","language":"JavaScript","funding_links":[],"categories":[],"sub_categories":[],"readme":"# Scrape Tweets while Scrolling\n\nScrape Tweets while Scrolling\n\n\n\n\n1. Go to Chrome \n\n\n2. Go to X.com\n\n\n3. Open your Browser Console and Paste\n\n```js\n(() =\u003e {\n  const scraped = new Set();\n  const results = [];\n\n  const extractTweets = () =\u003e {\n    const articles = document.querySelectorAll(\"article\");\n\n    articles.forEach((article) =\u003e {\n      const textEl = article.querySelector('div[data-testid=\"tweetText\"]');\n      const userEl = article.querySelector('div[dir=\"ltr\"] \u003e span');\n\n      const statGroup = article.querySelector('div[role=\"group\"]');\n      if (!statGroup) return;\n\n      let replies = null, reposts = null, likes = null, views = null;\n\n      const statElements = statGroup.querySelectorAll('[aria-label]');\n      statElements.forEach((el) =\u003e {\n        const label = el.getAttribute(\"aria-label\")?.toLowerCase() || \"\";\n        const match = label.match(/([\\d.,Kk]+)/);\n        const value = match ? match[1].replace(/,/g, \"\") : null;\n\n        if (label.includes(\"reply\")) replies = value;\n        else if (label.includes(\"repost\")) reposts = value;\n        else if (label.includes(\"like\")) likes = value;\n        else if (label.includes(\"view\")) views = value;\n      });\n\n      const text = textEl?.innerText?.trim();\n      const username = userEl?.innerText?.trim();\n\n      if (text \u0026\u0026 username) {\n        const id = `${username}::${text}`;\n        if (!scraped.has(id)) {\n          scraped.add(id);\n          results.push({ username, text, replies, reposts, likes, views });\n          console.log(`@${username} — 💬 ${replies} 🔁 ${reposts} ❤️ ${likes} 👁️ ${views}\\n\u003e ${text}`);\n        }\n      }\n    });\n  };\n\n  extractTweets();\n\n  const observer = new MutationObserver(() =\u003e {\n    extractTweets();\n  });\n\n  observer.observe(document.body, { childList: true, subtree: true });\n\n  console.log(\"Scraper is live... just keep scrolling!\");\n  console.log(\"Use `downloadTweets()` to save as json.\");\n\n  window.downloadTweets = () =\u003e {\n    const blob = new Blob([JSON.stringify(results, null, 2)], { type: \"application/json\" });\n    const url = URL.createObjectURL(blob);\n    const a = document.createElement(\"a\");\n    a.href = url;\n    a.download = \"tweets_with_stats.json\";\n    a.click();\n    URL.revokeObjectURL(url);\n    const message = `Downloaded ${results.length} tweets as tweets_with_stats.json`;\n    console.log(message);\n    return message;\n  };\n\n})();\n```\n\nVoila you're done\n\ndownload via thia\n```js\ndownloadTweets()\n```\n\n\nvery random but this the graphql endpoint\n\n\u003cdetails\u003e\n  \u003csummary\u003eX Graphql Endpoint\u003c/summary\u003e\n\n```bash\nhttps://x.com/i/api/graphql/0uQE4rvNofAr4pboHOZWVA/UserTweets?variables={\n  \"userId\": \"1654221044503408640\",\n  \"count\": 20,\n  \"includePromotedContent\": true,\n  \"withQuickPromoteEligibilityTweetFields\": true,\n  \"withVoice\": true\n}\u0026features={\n  \"rweb_video_screen_enabled\": false,\n  \"payments_enabled\": false,\n  \"profile_label_improvements_pcf_label_in_post_enabled\": true,\n  \"rweb_tipjar_consumption_enabled\": true,\n  \"verified_phone_label_enabled\": true,\n  \"creator_subscriptions_tweet_preview_api_enabled\": true,\n  \"responsive_web_graphql_timeline_navigation_enabled\": true,\n  \"responsive_web_graphql_skip_user_profile_image_extensions_enabled\": false,\n  \"premium_content_api_read_enabled\": false,\n  \"communities_web_enable_tweet_community_results_fetch\": true,\n  \"c9s_tweet_anatomy_moderator_badge_enabled\": true,\n  \"responsive_web_grok_analyze_button_fetch_trends_enabled\": false,\n  \"responsive_web_grok_analyze_post_followups_enabled\": true,\n  \"responsive_web_jetfuel_frame\": true,\n  \"responsive_web_grok_share_attachment_enabled\": true,\n  \"articles_preview_enabled\": true,\n  \"responsive_web_edit_tweet_api_enabled\": true,\n  \"graphql_is_translatable_rweb_tweet_is_translatable_enabled\": true,\n  \"view_counts_everywhere_api_enabled\": true,\n  \"longform_notetweets_consumption_enabled\": true,\n  \"responsive_web_twitter_article_tweet_consumption_enabled\": true,\n  \"tweet_awards_web_tipping_enabled\": false,\n  \"responsive_web_grok_show_grok_translated_post\": false,\n  \"responsive_web_grok_analysis_button_from_backend\": true,\n  \"creator_subscriptions_quote_tweet_preview_enabled\": false,\n  \"freedom_of_speech_not_reach_fetch_enabled\": true,\n  \"standardized_nudges_misinfo\": true,\n  \"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled\": true,\n  \"longform_notetweets_rich_text_read_enabled\": true,\n  \"longform_notetweets_inline_media_enabled\": true,\n  \"responsive_web_grok_image_annotation_enabled\": true,\n  \"responsive_web_grok_community_note_auto_translation_is_enabled\": false,\n  \"responsive_web_enhance_cards_enabled\": false\n}\u0026fieldToggles={\n  \"withArticlePlainText\": false\n}\n```\n\n```bash\ncurl 'https://x.com/i/api/graphql/0uQE4rvNofAr4pboHOZWVA/UserTweets?variables=...' \\\n  -H 'authorization: Bearer AAAAAAAAAAAAAAAAANRegergerAAAAAnNwIzUejRCOuH5...' \\\n  -H 'x-csrf-token: \u003cyour ct0 token\u003e' \\\n  -H 'cookie: auth_token=...; ct0=...' \\\n  -H 'x-twitter-auth-type: OAuth2Session' \\\n  -H 'x-twitter-active-user: yes'\n```\n\n\u003c/details\u003e\n\nyou can do whatever the heck u want wit this info and pls use your web console it's love\n\nalsooooo\n\nMost likely i'm banned from twitter for this basic thing or maybe just winning + freedom of speech is a joke and flawed with their own standards.\n\nBut again use this for educational purposes only and don't misuse this but one of my main reason to build this is to replicate a persona of my fav twitter creators and write tweets like them :3\n\nWait are you lazy? You need Auto Scroll\n\n### Auto Scroll with Batch Scraping\n\n1. first step to start scraping\n```js\n(() =\u003e {\n  window.currentChunk = [];\n  const scraped = new Set();\n  let chunk = 1;\n  const CHUNK_SIZE = 100;\n\n  const saveChunk = () =\u003e {\n    const blob = new Blob([JSON.stringify(window.currentChunk, null, 2)], { type: \"application/json\" });\n    const a = document.createElement(\"a\");\n    a.href = URL.createObjectURL(blob);\n    a.download = `tweets_${chunk++}.json`;\n    a.click();\n    URL.revokeObjectURL(a.href);\n    console.log(`💾 Saved ${CHUNK_SIZE} tweets as tweets_${chunk - 1}.json`);\n    window.currentChunk = []; // 🔥 delete them from memory!\n  };\n\n  const extractTweets = () =\u003e {\n    const articles = document.querySelectorAll(\"article\");\n    articles.forEach((article) =\u003e {\n      const textEl = article.querySelector('div[data-testid=\"tweetText\"]');\n      const userEl = article.querySelector('div[dir=\"ltr\"] \u003e span');\n      const statGroup = article.querySelector('div[role=\"group\"]');\n      if (!textEl || !userEl || !statGroup) return;\n\n      let replies = null, reposts = null, likes = null, views = null;\n      statGroup.querySelectorAll('[aria-label]').forEach((el) =\u003e {\n        const label = el.getAttribute(\"aria-label\")?.toLowerCase() || \"\";\n        const value = label.match(/([\\d.,Kk]+)/)?.[1]?.replace(/,/g, \"\") || null;\n        if (label.includes(\"reply\")) replies = value;\n        else if (label.includes(\"repost\")) reposts = value;\n        else if (label.includes(\"like\")) likes = value;\n        else if (label.includes(\"view\")) views = value;\n      });\n\n      const text = textEl?.innerText?.trim();\n      const username = userEl?.innerText?.trim();\n      const id = `${username}::${text}`;\n      if (text \u0026\u0026 username \u0026\u0026 !scraped.has(id)) {\n        window.currentChunk.push({ username, text, replies, reposts, likes, views });\n        scraped.add(id);\n        console.log(`[${window.currentChunk.length}] @${username}: ${text}`);\n        if (window.currentChunk.length \u003e= CHUNK_SIZE) saveChunk();\n      }\n    });\n  };\n\n  const observer = new MutationObserver(() =\u003e extractTweets());\n  observer.observe(document.body, { childList: true, subtree: true });\n\n  window.scrollInterval = setInterval(() =\u003e window.scrollBy(0, 1000), 1500);\n\n  window.stopScroll = () =\u003e {\n    clearInterval(window.scrollInterval);\n    if (window.currentChunk.length \u003e 0) {\n      const blob = new Blob([JSON.stringify(window.currentChunk, null, 2)], { type: \"application/json\" });\n      const a = document.createElement(\"a\");\n      a.href = URL.createObjectURL(blob);\n      a.download = `tweets_final_${window.currentChunk.length}.json`;\n      a.click();\n      URL.revokeObjectURL(a.href);\n      console.log(\"🛑 Final partial chunk saved.\");\n    } else {\n      console.log(\"🛑 Stopped. No tweets left to save.\");\n    }\n  };\n\n  console.log(\"🚀 Scraper started. Will auto-save every 100 tweets and flush memory each time.\");\n})();\n```\n\n2. Stop Scroll\n```js\nstopScroll();\n```\n\nthis will download all the tweets saved\n\n\n\n\n4. Cleanup (Reset Everything) ~ optional\n\n```js\n  delete window.currentChunk;\n  delete window.scrollInterval;\n  delete window.stopScroll;\n```\n\n\n\n\n## Want the original tweet and the username in the scrapper data \n\u003e Try this \n```js\n\n(() =\u003e {\n  window.currentChunk = [];\n  const scraped = new Set();\n  let chunk = 1;\n  const CHUNK_SIZE = 100;\n  \n  const saveChunk = () =\u003e {\n    const blob = new Blob([JSON.stringify(window.currentChunk, null, 2)], { type: \"application/json\" });\n    const a = document.createElement(\"a\");\n    a.href = URL.createObjectURL(blob);\n    a.download = `tweets_${chunk++}.json`;\n    a.click();\n    URL.revokeObjectURL(a.href);\n    console.log(`💾 Saved ${CHUNK_SIZE} tweets as tweets_${chunk - 1}.json`);\n    window.currentChunk = []; // 🔥 delete them from memory!\n  };\n  \n  const extractTweetId = (article) =\u003e {\n    // Method 1: Try to find a link with tweet ID pattern\n    const tweetLink = article.querySelector('a[href*=\"/status/\"]');\n    if (tweetLink) {\n      const href = tweetLink.getAttribute('href');\n      const match = href.match(/\\/status\\/(\\d+)/);\n      if (match) return match[1];\n    }\n    \n    // Method 2: Try to find time element with datetime attribute\n    const timeEl = article.querySelector('time');\n    if (timeEl) {\n      const nearestLink = timeEl.closest('a') || timeEl.parentElement?.querySelector('a');\n      if (nearestLink) {\n        const href = nearestLink.getAttribute('href');\n        const match = href?.match(/\\/status\\/(\\d+)/);\n        if (match) return match[1];\n      }\n    }\n    \n    // Method 3: Search all links in the article for status pattern\n    const allLinks = article.querySelectorAll('a[href]');\n    for (const link of allLinks) {\n      const href = link.getAttribute('href');\n      const match = href?.match(/\\/status\\/(\\d+)/);\n      if (match) return match[1];\n    }\n    \n    return null;\n  };\n  \n  const extractUsername = (article) =\u003e {\n    // Method 1: Try to extract from any link that contains a username pattern\n    const links = article.querySelectorAll('a[href]');\n    for (const link of links) {\n      const href = link.getAttribute('href');\n      // Look for pattern like /username or /username/status/...\n      const match = href?.match(/^\\/([^\\/]+)(?:\\/|$)/);\n      if (match \u0026\u0026 match[1] \u0026\u0026 !match[1].includes('status') \u0026\u0026 !match[1].includes('search') \u0026\u0026 !match[1].includes('home')) {\n        return match[1];\n      }\n    }\n    \n    // Method 2: Look for elements that might contain @username\n    const spanElements = article.querySelectorAll('span');\n    for (const span of spanElements) {\n      const text = span.innerText?.trim();\n      if (text \u0026\u0026 text.startsWith('@')) {\n        return text.substring(1); // Remove @ symbol\n      }\n    }\n    \n    // Method 3: Try to find username in data attributes or other patterns\n    const userLinks = article.querySelectorAll('a[href*=\"/\"]');\n    for (const link of userLinks) {\n      const href = link.getAttribute('href');\n      if (href?.startsWith('/') \u0026\u0026 !href.includes('/status/') \u0026\u0026 !href.includes('/search') \u0026\u0026 !href.includes('/home')) {\n        const username = href.substring(1).split('/')[0];\n        if (username \u0026\u0026 username.length \u003e 0 \u0026\u0026 !username.includes('?')) {\n          return username;\n        }\n      }\n    }\n    \n    return null;\n  };\n  \n  const extractDisplayName = (article) =\u003e {\n    // Try to find the display name (full name)\n    const nameSelectors = [\n      'div[dir=\"ltr\"] \u003e span',\n      'a[role=\"link\"] span',\n      'div[data-testid=\"User-Name\"] span'\n    ];\n    \n    for (const selector of nameSelectors) {\n      const element = article.querySelector(selector);\n      if (element \u0026\u0026 element.innerText?.trim()) {\n        const text = element.innerText.trim();\n        // Make sure it's not a username (doesn't start with @)\n        if (!text.startsWith('@')) {\n          return text;\n        }\n      }\n    }\n    \n    return null;\n  };\n  \n  const extractTweets = () =\u003e {\n    const articles = document.querySelectorAll(\"article\");\n    articles.forEach((article) =\u003e {\n      const textEl = article.querySelector('div[data-testid=\"tweetText\"]');\n      const statGroup = article.querySelector('div[role=\"group\"]');\n      \n      if (!textEl || !statGroup) return;\n      \n      // Extract engagement stats\n      let replies = null, reposts = null, likes = null, views = null;\n      statGroup.querySelectorAll('[aria-label]').forEach((el) =\u003e {\n        const label = el.getAttribute(\"aria-label\")?.toLowerCase() || \"\";\n        const value = label.match(/([\\d.,Kk]+)/)?.[1]?.replace(/,/g, \"\") || null;\n        if (label.includes(\"reply\")) replies = value;\n        else if (label.includes(\"repost\")) reposts = value;\n        else if (label.includes(\"like\")) likes = value;\n        else if (label.includes(\"view\")) views = value;\n      });\n      \n      // Extract basic info\n      const text = textEl?.innerText?.trim();\n      const username = extractUsername(article);\n      const displayName = extractDisplayName(article);\n      const tweetId = extractTweetId(article);\n      \n      // Create tweet URL if we have the ID and username\n      let tweetUrl = null;\n      if (tweetId \u0026\u0026 username) {\n        tweetUrl = `https://x.com/${username}/status/${tweetId}`;\n      }\n      \n      const id = `${username}::${text}`;\n      \n      if (text \u0026\u0026 username \u0026\u0026 !scraped.has(id)) {\n        const tweetData = {\n          username,\n          displayName,\n          text,\n          replies,\n          reposts,\n          likes,\n          views,\n          tweetId,\n          tweetUrl\n        };\n        \n        window.currentChunk.push(tweetData);\n        scraped.add(id);\n        console.log(`[${window.currentChunk.length}] @${username} (${displayName}): ${text}`);\n        if (tweetUrl) console.log(`   🔗 ${tweetUrl}`);\n        \n        if (window.currentChunk.length \u003e= CHUNK_SIZE) saveChunk();\n      }\n    });\n  };\n  \n  const observer = new MutationObserver(() =\u003e extractTweets());\n  observer.observe(document.body, { childList: true, subtree: true });\n  \n  window.scrollInterval = setInterval(() =\u003e window.scrollBy(0, 1000), 1500);\n  \n  window.stopScroll = () =\u003e {\n    clearInterval(window.scrollInterval);\n    observer.disconnect(); // Stop observing when done\n    \n    if (window.currentChunk.length \u003e 0) {\n      const blob = new Blob([JSON.stringify(window.currentChunk, null, 2)], { type: \"application/json\" });\n      const a = document.createElement(\"a\");\n      a.href = URL.createObjectURL(blob);\n      a.download = `tweets_final_${window.currentChunk.length}.json`;\n      a.click();\n      URL.revokeObjectURL(a.href);\n      console.log(\"🛑 Final partial chunk saved.\");\n    } else {\n      console.log(\"🛑 Stopped. No tweets left to save.\");\n    }\n  };\n  \n  console.log(\"🚀 Enhanced scraper started. Will auto-save every 100 tweets with tweet URLs!\");\n  console.log(\"📝 Each tweet now includes: username, displayName, text, engagement stats, tweetId, and tweetUrl\");\n  console.log(\"⏹️ Call window.stopScroll() to stop and save remaining tweets\");\n})();\n```","project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fnermalcat69%2Flightweight-tweet-scraper","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Fnermalcat69%2Flightweight-tweet-scraper","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fnermalcat69%2Flightweight-tweet-scraper/lists"}