接着这篇文章[js高手之路]Node.js+jade抓取博客所有文章生成静态html文件继续,在这篇文章中实现了采集与静态文件的生成,在实际的采集项目中, 应该是先入库再选择性的生成静态文件。
那么我选择的数据库是mongodb,为什么用这个数据库,因为这个数据库是基于集合,数据的操作基本是json,与dom模块cheerio具有非常大的亲和力,cheerio处理过滤出来的数据,可以直接插入mongodb,不需要经过任何的处理,非常的便捷,当然跟node.js的亲和力那就不用说了,更重要的是,性能很棒。这篇文章我就不具体写mongodb的基本用法,到时候会另起文章从0开始写mongodb基本常用用法.先看下入库的效果与生成静态文件的效果:
我在这个阶段,把爬虫分离成2个模块,采集入库( crawler.js ), 生成静态文件(makeHtml.js).
crawler.js:
- var http = require('http');
- var cheerio = require('cheerio');
- var mongoose = require('mongoose');
- mongoose.Promise = global.Promise;
- var DB_URL = 'mongodb://localhost:27017/crawler';
- var aList = []; //博客文章列表信息
- var aUrl = []; //博客所有的文章url
- var db = mongoose.createConnection(DB_URL);
- db.on('connected',
- function(err) {
- if (err) {
- console.log(err);
- } else {
- console.log('db connected success');
- }
- });
- var Schema = mongoose.Schema;
- var arcSchema = new Schema({
- id: Number,
- //文章id
- title: String,
- //文章标题
- url: String,
- //文章链接
- body: String,
- //文章内容
- entry: String,
- //摘要
- listTime: Date //发布时间
- });
- var Article = db.model('Article', arcSchema);
- function saveArticle(arcInfo) {
- var arcModel = new Article(arcInfo);
- arcModel.save(function(err, result) {
- if (err) {
- console.log(err);
- } else {
- console.log(`$ {
- arcInfo['title']
- }插入成功`);
- }
- });
- }
- function filterArticle(html) {
- var $ = cheerio.load(html);
- var arcDetail = {};
- var title = $("#cb_post_title_url").text();
- var href = $("#cb_post_title_url").attr("href");
- var re = /\/(\d+)\.html/;
- var id = href.match(re)[1];
- var body = $("#cnblogs_post_body").html();
- return {
- id: id,
- title: title,
- url: href,
- body: body
- };
- }
- function crawlerArc(url) {
- var html = '';
- var str = '';
- var arcDetail = {};
- http.get(url,
- function(res) {
- res.on('data',
- function(chunk) {
- html += chunk;
- });
- res.on('end',
- function() {
- arcDetail = filterArticle(html);
- saveArticle(arcDetail);
- if (aUrl.length) {
- setTimeout(function() {
- if (aUrl.length) {
- crawlerArc(aUrl.shift());
- }
- },
- 100);
- } else {
- console.log('采集任务完成');
- return;
- }
- });
- });
- }
- function filterHtml(html) {
- var $ = cheerio.load(html);
- var arcList = [];
- var aPost = $("#content").find(".post-list-item");
- aPost.each(function() {
- var ele = $(this);
- var title = ele.find("h2 a").text();
- var url = ele.find("h2 a").attr("href");
- ele.find(".c_b_p_desc a").remove();
- var entry = ele.find(".c_b_p_desc").text();
- ele.find("small a").remove();
- var listTime = ele.find("small").text();
- var re = /\d{4}-\d{2}-\d{2}\s*\d{2}[:]\d{2}/;
- listTime = listTime.match(re)[0];
- arcList.push({
- title: title,
- url: url,
- entry: entry,
- listTime: listTime
- });
- });
- return arcList;
- }
- function nextPage(html) {
- var $ = cheerio.load(html);
- var nextUrl = $("#pager a:last-child").attr('href');
- if (!nextUrl) return getArcUrl(aList);
- var curPage = $("#pager .current").text();
- if (!curPage) curPage = 1;
- var nextPage = nextUrl.substring(nextUrl.indexOf('=') + 1);
- if (curPage < nextPage) crawler(nextUrl);
- }
- function crawler(url) {
- http.get(url,
- function(res) {
- var html = '';
- res.on('data',
- function(chunk) {
- html += chunk;
- });
- res.on('end',
- function() {
- aList.push(filterHtml(html));
- nextPage(html);
- });
- });
- }
- function getArcUrl(arcList) {
- for (var key in arcList) {
- for (var k in arcList[key]) {
- aUrl.push(arcList[key][k]['url']);
- }
- }
- crawlerArc(aUrl.shift());
- }
- var url = 'http://www.cnblogs.com/ghostwu/';
- crawler(url);
- 其他的核心模块没有怎么改动,主要增加了数据库连接,数据库创建,集合创建( 集合相当于关系型数据库中的表 ),Schema( 相当于关系型数据库的表结构 ).
- mongoose操作数据库( save:插入数据 ).分离了文件生成模块.
- makeHtml.js文件
- var fs = require('fs');
- var jade = require('jade');
- var mongoose = require('mongoose');
- mongoose.Promise = global.Promise;
- var DB_URL = 'mongodb://localhost:27017/crawler';
- var allArc = [];
- var count = 0;
- var db = mongoose.createConnection(DB_URL);
- db.on('connected', function (err) {
- if (err) {
- console.log(err);
- } else {
- console.log('db connected success');
- }
- });
- var Schema = mongoose.Schema;
- var arcSchema = new Schema({
- id: Number, //文章id
- title: String, //文章标题
- url: String, //文章链接
- body: String, //文章内容
- entry: String, //摘要
- listTime: Date //发布时间
- });
- var Article = db.model('Article', arcSchema);
- function makeHtml(arcDetail) {
- str = jade.renderFile('./views/layout.jade', arcDetail);
- ++count;
- fs.writeFile('./html/' + count + '.html', str, function (err) {
- if (err) {
- console.log(err);
- }
- console.log( `${arcDetail['id']}.html创建成功` + count );
- if ( allArc.length ){
- setTimeout( function(){
- makeHtml( allArc.shift() );
- }, 100 );
- }
- });
- }
- function getAllArc(){
- Article.find( {}, function( err, arcs ){
- allArc = arcs;
- makeHtml( allArc.shift() );
- } ).sort( { 'id' : 1 } );
- }
- getAllArc();