{"id":13474139,"url":"https://github.com/youyudehexie/simple-scrapy","last_synced_at":"2025-03-26T21:30:39.802Z","repository":{"id":10703843,"uuid":"12949332","full_name":"youyudehexie/simple-scrapy","owner":"youyudehexie","description":"simple-scrapy","archived":false,"fork":false,"pushed_at":"2014-07-06T12:59:54.000Z","size":474,"stargazers_count":42,"open_issues_count":3,"forks_count":22,"subscribers_count":7,"default_branch":"master","last_synced_at":"2024-08-01T16:35:00.053Z","etag":null,"topics":[],"latest_commit_sha":null,"homepage":null,"language":"JavaScript","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":null,"status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/youyudehexie.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":null,"funding":null,"license":null,"code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null}},"created_at":"2013-09-19T13:33:04.000Z","updated_at":"2024-01-04T15:55:25.000Z","dependencies_parsed_at":"2022-08-30T10:00:20.234Z","dependency_job_id":null,"html_url":"https://github.com/youyudehexie/simple-scrapy","commit_stats":null,"previous_names":[],"tags_count":0,"template":false,"template_full_name":null,"repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/youyudehexie%2Fsimple-scrapy","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/youyudehexie%2Fsimple-scrapy/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/youyudehexie%2Fsimple-scrapy/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/youyudehexie%2Fsimple-scrapy/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/youyudehexie","download_url":"https://codeload.github.com/youyudehexie/simple-scrapy/tar.gz/refs/heads/master","host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":222169239,"owners_count":16942668,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2022-07-04T15:15:14.044Z","host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":[],"created_at":"2024-07-31T16:01:09.810Z","updated_at":"2024-10-30T05:30:38.979Z","avatar_url":"https://github.com/youyudehexie.png","language":"JavaScript","funding_links":[],"categories":["JavaScript"],"sub_categories":[],"readme":"simple-scrapy\n=============\n\n#详解Node.js API系列 Http模块(2) CNodejs爬虫实现\n\n##简单爬虫设计\n\n    var http = require('http');\n    http.get(\"http://cnodejs.org/\", function(res) {\n    \tvar size = 0;\n    \tvar chunks = [];\n      res.on('data', function(chunk){\n      \tsize += chunk.length;\n      \tchunks.push(chunk);\n      });\n      res.on('end', function(){\n      \tvar data = Buffer.concat(chunks, size);\n      \tconsole.log(data.toString())\n      });\n    }).on('error', function(e) {\n      console.log(\"Got error: \" + e.message);\n    });\n    \n###http.get(options, callback)\n\n- http://cnodejs.org/ 爬行目标地址。\n- res.on('data') 监听data事件。\n- res.on('end') 数据获取完毕事件。\n- Buffer.concat(chunks, size); 连接多次data的buff。\n- data.toString() 将data二进制数据转换成utf-8的字符串，如果页面是GBK的时候，请使用iconv模块进行转换，原生Node.js不支持GBK。\n\n##设计目标\n\n- 制定爬虫的url规则\n- 分析页面信息\n- 清洗没用数据\n- 存储有用数据\n\n##制定爬虫的url规则\n\n观察 http://cnodejs.org/ 的url规则，http://cnodejs.org/?page=页数， 根据规则，不难想出处理的思路，首先获取用迭代器模式是最方便的，首先，获取单个page页里面的单个page路径，通过路径爬取page的页面内容。采用迭代器模式是最方便的，next()做page的页面索引，hasNext()判断page页是否超出了有效范围,超出范围则停止索引，下面是伪代码\n\n    var Urls = function(start_url){\n      this.start_url = start_url; //base url\n      this.page = 0;  //url page\n      this.targetPage = ''; //topic page\n    }\n    Urls.prototype.next = function(){\n      var data;\n      if (!this.hasNext()) {\n          return null;\n      }\n      this.page += 1;\n      data = request.get(this.targetPage)  //get topic page\n      return data;\n    }\n    Urls.prototype.hasNext = function(){\n      //http://cnodejs.org/p=[1,2,3,4]\n      var url = this.start_url + this.page;\n      // if get page success from url,return ture,or return false\n      // get topic page url\n    }\n    // main     \n    var urls = new Urls();\n    while(urls.hasNext()){\n      console.log(urls.next());\n    }\n    \n##分析页面数据\n\n分析页面的过程，主要工作是分析页面的元素提取出目标的内容，例如正文和评论等。这里我们需要采用cheerio的第三方库,该模块采取类似Jquery方式的DOM选择器，通过DOM选择器来实现信息提取。\n\n    npm install cheerio\n\n项目地址：https://github.com/MatthewMueller/cheerio\n\n官方demo例子\n\n    var cheerio = require('cheerio'),\n        $ = cheerio.load('\u003ch2 class=\"title\"\u003eHello world\u003c/h2\u003e');\n    \n    $('h2.title').text('Hello there!');\n    $('h2').addClass('welcome');\n    \n    $.html();\n    //=\u003e \u003ch2 class=\"title welcome\"\u003eHello there!\u003c/h2\u003e\n    \n提取cnodejs的topics 链接\n\n    $ = cheerio.load(data); //data是的页面数据\n    topics = $('.cell .topic_wrap a')\n    for(var i=0; i \u003c topics.length; i++){\n    console.log(topics[i].attribs['href'])\n    \n    result：\n    \n    /topic/52386d26101e574521a12ccd\n    /topic/5232cd39101e57452106ce5a\n    /topic/52390cdb101e574521b1e252\n    /topic/521b1dcabee8d3cb128c56dd\n    /topic/5238c6d2101e574521aaca13\n    /topic/52380b4e101e57452193617c\n    \n内容信息提取\n\n提取condejs帖子内容和标题\n\n    $ = cheerio.load(data);  \n    var topic = $('.inner.topic');  \n    console.log(topic.children('h3').text()) //标题\n    var content = topic.children('.topic_content').text()\n    console.log(content);   //文章内容\n    \n##清洗没用的数据\n\n由于爬取的内容，可能带有html标签或者表情方面的信息，可能跟目标内容不符合，通过这个环节来过滤，这里向大家推荐一个模块 validator,该模块可以过滤xss攻击，字符串里面的空格，判断内容的属性等，详细可以到项目地址学习 https://github.com/chriso/node-validator\n\n安装\n\n    npm install validator\n\ndemo例子\n\n    var check = require('validator').check,\n        sanitize = require('validator').sanitize\n    //Validate\n    check('test@email.com').len(6, 64).isEmail();        //Methods are chainable\n    check('abc').isInt();                                //Throws 'Invalid integer'\n    check('abc', 'Please enter a number').isInt();       //Throws 'Please enter a number'\n    check('abcdefghijklmnopzrtsuvqxyz').is(/^[a-z]+$/);\n    //Set a message per validator\n    check('foo', {\n        isNumeric: 'This is not a number',\n        contains: 'The value doesn\\'t have a 0 in it'\n    }).isNumeric().contains('0');\n    //Referencing validator args from the message\n    check('foo', 'The message needs to be between %1 and %2 characters long (you passed \"%0\")').len(2, 6);\n    //Sanitize / Filter\n    var int = sanitize('0123').toInt();                  //123\n    var bool = sanitize('true').toBoolean();             //true\n    var str = sanitize(' \\t\\r hello \\n').trim();         //'hello'\n    var str = sanitize('aaaaaaaaab').ltrim('a');         //'b'\n    var str = sanitize(large_input_str).xss();\n    var str = sanitize('\u0026lt;a\u0026gt;').entityDecode();      //'\u003ca\u003e'\n    \n过滤刚才爬取的内容，主要是过滤空格\n\n    var topic = $('.inner.topic');  \n    title = topic.children('h3').text() //标题\n    sanitize(title).trim()\n\n##存储有用数据\n\n咸鱼白菜各有所需，对于游泳的数据，可以存成文本，也可以存到数据库，本次实例，为了足够精简，所以不采用数据库存储，采用文本的方式记录和json的格式记录数据。\n\n一个爬虫的流程完成了，我们来重新看看实现代码\n\nvi url.js\n\n    var http = require('http');\n    var cheerio = require('cheerio');\n    var sanitize = require('validator').sanitize;\n    var async = require('async');\n    var fs = require('fs');\n    \n    var BASE_URL = 'http://cnodejs.org'\n    var scrapy = {}\n    /**\n     * Get page from url.\n     *\n     * Examples:\n     *\n     *     scrapy.get('http://www.baidu.com', cb);\n     *     // =\u003e 'baidu page html\n     * \n     * @interface\n     * @param {String} url:ex http://www.baidu.com\n     * @param {Function} cb\n     * @private\n     */\n    scrapy.get = function(url, cb){\n      http.get(url, function(res) {\n    \n        var size = 0;\n        var chunks = [];\n    \n        res.on('data', function(chunk){\n          size += chunk.length;\n          chunks.push(chunk);\n        });\n    \n        res.on('end', function(){\n          var data = Buffer.concat(chunks, size);\n          cb(null, data);\n        });\n    \n      }).on('error', function(e) {\n        cb(e, null);\n      });\n    }\n    \n    var Urls = function(startUrl){\n      this.startUrl = startUrl;\n      this.page = 0;\n      this.homePage = '';\n    }\n    \n    Urls.prototype.next = function(cb){\n      var self = this;\n    \n      this.hasNext(function(err, bRet){\n        if(!bRet){\n          return null;\n        }\n    \n        self.homeParse(function(err, topics){\n          self.page += 1;\n          cb(null, topics);\n        })\n      })\n    }\n    \n    Urls.prototype.hasNext = function(cb){\n      var self = this;\n      var url = this.startUrl + this.page;\n    \n      scrapy.get(url, function(err, data){\n        var html = data.toString();\n        $ = cheerio.load(html);\n        self.homePage = $('.cell .topic_wrap a');\n    \n        if(self.homePage.length === 0){\n          return cb(null, false);\n        }\n        return cb(null, true);\n    \n      });\n    };\n    \n    Urls.prototype.homeParse = function(cb){\n      var self = this;\n      var topics = [];\n    \n      async.filter(self.homePage, function(i, cb){\n        var url = BASE_URL + self.homePage[i].attribs['href']\n        scrapy.get(url, function(err, topic){\n          topics.push(topic.toString());\n          cb(null);\n        })\n        \n      },function(err){\n        cb(err, topics);\n      });\n    }\n    \n    Urls.prototype.parseTopic = function(html){\n      $ = cheerio.load(html);\n      var topic = $('.inner.topic');\n      var item = {};\n      item.title = sanitize(topic.children('h3').text()).trim();\n      item.content = sanitize(topic.children('.topic_content').text()).trim();\n    \n      return item;\n    };\n    \n    Urls.prototype.Pipeline = function(items){\n    \n      var result = JSON.stringify(items);\n      fs.writeFileSync('result.txt', result)\n    \n    }\n    \n    exports = module.exports = Urls\n    \nvi app.js\n    \n    var Urls = require('./lib/url.js');\n    var async = require('async');\n    \n    \n    var startUrl = 'http://cnodejs.org/?page='\n    \n    var urls = new Urls(startUrl);\n    \n    urls.next(function(err, topics){\n      var self = this;\n      var items = [];\n      async.filter(topics, function(topic, cb){\n        items.push(urls.parseTopic(topic));\n        cb(null);\n      },function(err){\n        urls.Pipeline(items);\n      })\n    })\n    \n完整项目地址：https://github.com/youyudehexie/simple-scrapy\n\n\n\n\n\n    \n\n    \n\n\n\n\n\n\n\n","project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fyouyudehexie%2Fsimple-scrapy","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Fyouyudehexie%2Fsimple-scrapy","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fyouyudehexie%2Fsimple-scrapy/lists"}