- var mysqlpool=require('../modules/mysqlpool')();
- var SimpleQuene=require('../modules/SimpleQuene');
- var hash=require('../modules/hash');
- var moment=require('moment');
- var fs=require('fs');
- var tempfilepath='E:/1.txt';
- var limit=200000;
- var queue=new SimpleQuene(limit);
- var nowTime=moment().format('YYYY-MM-DD h:mm:ss');
- console.time('init data');
- for (var i =0;i<limit;i++){
- url='http://www.baidu.com'+i;
- urlhash=hash.fnv1a(url);
- queue.in({
- url:url,
- hash:urlhash,
- title:'标题是什么'+i,
- time:nowTime
- });
- }
- console.timeEnd('init data');
- writeDate();
- function writeDate(){
- console.time('fs');
- var writer=fs.createWriteStream(tempfilepath);
- while(!queue.isEmpty()){
- var temp=queue.out();
- writer.write(temp.url+','+temp.hash+','+(temp.title || '""')+','+(temp.time || '""')+"\\n");
- };
- writer.end(function(){
- console.timeEnd('fs');
- save();
- });
- }
- function save(){
- console.time('sql');
- var sql="load data infile '"+tempfilepath+"' replace into table spider_page FIELDS TERMINATED BY ',' LINES TERMINATED BY '\\n' (url,urlhash,title,time)" ;
- mysqlpool.query(sql,function(err,result){
- console.timeEnd('sql');
- console.log(result.affectedRows+' rows have been insert');
- });
- }
- //该片段来自于http://www.codesnippet.cn/detail/0212201411151.html
来源: http://www.codesnippet.cn/detail/0212201411151.html