Patch 1

.gitignore

@@ -0,0 +1,2 @@
+node_modules
+npm-debug.log

README.md

@@ -1,2 +1,14 @@
-# todojs
-personal blog
+# nodejs 爬虫
+地址： 127.0.0.1/map
+路由：
+```js
+app.post('/mapsearch', function(req,res){
+    var url = req.body.url;
+    supers(url);
+    res.send({tit:'ok'})
+  })
+```
+爬虫核心代码：
+```bash
+data/superagentDemo.js
+```

app.js

@@ -0,0 +1,60 @@
+var express = require('express');
+var path = require('path');
+var favicon = require('serve-favicon');
+var logger = require('morgan');
+var cookieParser = require('cookie-parser');
+var bodyParser = require('body-parser');
+
+var routes = require('./routes/index');
+
+var app = express();
+
+// view engine setup
+app.set('views', path.join(__dirname, 'views'));
+app.set('view engine', 'ejs');
+
+// uncomment after placing your favicon in /public
+//app.use(favicon(path.join(__dirname, 'public', 'favicon.ico')));
+app.use(logger('dev'));
+app.use(bodyParser.json());
+app.use(bodyParser.urlencoded({ extended: false }));
+app.use(cookieParser());
+app.use(express.static(path.join(__dirname, 'public')));
+
+routes(app);
+// app.use('/users', users);
+
+// catch 404 and forward to error handler
+app.use(function(req, res, next) {
+  var err = new Error('Not Found');
+  err.status = 404;
+  next(err);
+});
+
+// error handlers
+
+// development error handler
+// will print stacktrace
+if (app.get('env') === 'development') {
+  app.use(function(err, req, res, next) {
+    res.status(err.status || 500);
+    res.render('error', {
+      message: err.message,
+      error: err
+    });
+  });
+}
+
+// production error handler
+// no stacktraces leaked to user
+app.use(function(err, req, res, next) {
+  res.status(err.status || 500);
+  res.render('error', {
+    message: err.message,
+    error: {}
+  });
+});
+
+
+
+module.exports = app;

bin/www

@@ -0,0 +1,90 @@
+#!/usr/bin/env node
+
+/**
+ * Module dependencies.
+ */
+
+var app = require('../app');
+var debug = require('debug')('todojs:server');
+var http = require('http');
+
+/**
+ * Get port from environment and store in Express.
+ */
+
+var port = normalizePort(process.env.PORT || '3000');
+app.set('port', port);
+
+/**
+ * Create HTTP server.
+ */
+
+var server = http.createServer(app);
+
+/**
+ * Listen on provided port, on all network interfaces.
+ */
+
+server.listen(port);
+server.on('error', onError);
+server.on('listening', onListening);
+
+/**
+ * Normalize a port into a number, string, or false.
+ */
+
+function normalizePort(val) {
+  var port = parseInt(val, 10);
+
+  if (isNaN(port)) {
+    // named pipe
+    return val;
+  }
+
+  if (port >= 0) {
+    // port number
+    return port;
+  }
+
+  return false;
+}
+
+/**
+ * Event listener for HTTP server "error" event.
+ */
+
+function onError(error) {
+  if (error.syscall !== 'listen') {
+    throw error;
+  }
+
+  var bind = typeof port === 'string'
+    ? 'Pipe ' + port
+    : 'Port ' + port;
+
+  // handle specific listen errors with friendly messages
+  switch (error.code) {
+    case 'EACCES':
+      console.error(bind + ' requires elevated privileges');
+      process.exit(1);
+      break;
+    case 'EADDRINUSE':
+      console.error(bind + ' is already in use');
+      process.exit(1);
+      break;
+    default:
+      throw error;
+  }
+}
+
+/**
+ * Event listener for HTTP server "listening" event.
+ */
+
+function onListening() {
+  var addr = server.address();
+  var bind = typeof addr === 'string'
+    ? 'pipe ' + addr
+    : 'port ' + addr.port;
+  debug('Listening on ' + bind);
+}

data/phantomjsDemo.js

@@ -0,0 +1,30 @@
+function phmap(url, res){
+  var sitepage = null;
+  var phInstance = null;
+  phantom.create()
+      .then(instance => {
+          phInstance = instance;
+          return instance.createPage();
+      })
+      .then(page => {
+          sitepage = page;
+          return page.open(url);
+      })
+      .then(status => {
+          console.log(status);
+          return sitepage.property('content');
+      })
+      .then(content => {
+          sitepage.viewpoetSize = { width: 400, height: 200};
+          sitepage.zoomFactor = 0.25;
+          sitepage.render('C:/Users/codedreamfy/vagrant/devwork/todojs/public/img/data/ca.png');
+          res.send({'url': '/img/data/ca.png'})
+          sitepage.close();
+          phInstance.exit();
+      })
+      .catch(error => {
+          console.log(error);
+          phInstance.exit();
+      });
+}
+module.exports = phmap

data/superagentDemo.js

@@ -0,0 +1,68 @@
+var superagent = require('superagent');
+var cheerio = require('cheerio');
+var async = require('async');
+
+function asyncCall(url){
+
+  var concurrencyCount = 0; // 并发连接数的计数器
+  var itemList = [];
+  var commit = [];
+  fetch(url, function(list){
+    async.mapLimit(list, 26, function(item, callback){
+      concurrencyCount++;
+      var promise2 = new Promise(function(resolve, reject){
+        superagent
+          .get(item.href)
+          .end(function(err,res){
+            if(err){
+              reject(item)
+            }
+            if( res.status == 200){
+              console.log(concurrencyCount," --正在连接：",item.tit)
+              if(!!res.text){
+                var $ = cheerio.load(res.text);
+                var c = $('a[href*="http://"]')
+                // console.log(c.length)
+                resolve(c)
+              }
+            }
+          })
+      })
+      promise2.then(function(d){
+        callback(null, d.length)
+      }, function(){
+        callback(null, "Error");
+      })
+    }, function(err, results){ console.log(results)})
+  })
+
+  function fetch(url, cb){
+    var items = [];
+    var promise = new Promise(function(resolve, reject){
+      superagent.get(url).end(function(err, sres){
+        if(err){
+          return console.error(err);
+        }
+        var $ = cheerio.load(sres.text);
+        $('a[href!=""][href!="javascript:;"][href*="http://"]').map(function(i, e){
+          var _this = $(e);
+          var tit = _this.text();
+          var hf = _this.attr('href');
+          if(hf.indexOf('.exe'))
+          items[i] = { 'tit' : tit, 'href' : hf, 'active': false};
+        })
+        resolve(items);
+      })
+    })
+
+    promise.then(function(e){
+      itemList = e;
+      !!cb && cb(itemList)
+    }, function(err){
+      console.log("error:", err)
+    })
+
+  }
+}
+
+module.exports = asyncCall

demo.js

@@ -0,0 +1,18 @@
+var http = require('http');
+for (var i = 0; i < 1000; i++) {
+  console.log(i);
+  var req = http.get("http://localhost:8000",function(res) {
+    resultdata = '';
+    res.on('data',function(chunk) {
+      resultdata += chunk;
+    });
+
+    res.on('end',function() {
+      console.log('return ' + resultdata);
+    });
+  });
+
+  req.on('error',function(err) {
+    console.log('problem with request: ' + err.message);
+  });
+}

gulpfile.js

@@ -0,0 +1,50 @@
+var gulp = require("gulp");
+var postcss = require("postcss");
+var gulp_postcss = require("gulp-postcss");
+var babel = require("gulp-babel");
+var mqpacker = require("css-mqpacker");
+var csswring = require("csswring");
+var autoprefixer = require("autoprefixer");
+var rename = require("gulp-rename");
+var cssimport = require("postcss-import");
+var cssnested = require("postcss-nested");
+var order = require("gulp-order");
+var concat = require("gulp-concat");
+
+gulp.task('postcss', function(){
+  var processors = [
+    cssimport,
+    cssnested,
+    mqpacker,
+    autoprefixer,
+    csswring({ removeAllcomments: true})
+  ];
+  return gulp.src('./public/css/main.css')
+          .pipe(gulp_postcss(processors))
+          .on('error', errorHandler)
+          .pipe(rename({suffix: ".min"}))
+          .pipe(gulp.dest('./public/css'))
+});
+
+gulp.task('babel', function(){
+  return gulp.src('./public/js/src/*.js')
+          .pipe(babel({
+            presets: ['es2015']
+          }))
+          .pipe(concat("main.js"))
+          .pipe(gulp.dest('./public/js/dest'))
+
+});
+gulp.task('watch', function(){
+  gulp.watch('./public/css/src/**/*.css',['postcss']);
+  gulp.watch('./public/js/src/**/*.js',['babel']);
+});
+
+gulp.task('default', ['postcss', 'babel', 'watch']);
+
+function errorHandler(error){
+  console.log(error.message);
+  console.log(error.fileName);
+  console.log('line', error.line, 'column:', error.column);
+  this.emit('end');
+}

package.json

@@ -0,0 +1,37 @@
+{
+  "name": "todojs",
+  "version": "0.0.0",
+  "private": true,
+  "scripts": {
+    "start": "node ./bin/www"
+  },
+  "dependencies": {
+    "async": "^2.0.0-rc.5",
+    "body-parser": "~1.13.2",
+    "cheerio": "^0.20.0",
+    "cookie-parser": "~1.3.5",
+    "debug": "~2.2.0",
+    "ejs": "~2.3.3",
+    "express": "~4.13.1",
+    "morgan": "~1.6.1",
+    "serve-favicon": "~2.3.0",
+    "superagent": "^2.0.0"
+  },
+  "devDependencies": {
+    "autoprefixer": "^6.2.3",
+    "babel-preset-es2015": "^6.9.0",
+    "css-mqpacker": "^4.0.0",
+    "csswring": "^4.1.1",
+    "gulp": "^3.9.1",
+    "gulp-babel": "^6.1.2",
+    "gulp-concat": "^2.6.0",
+    "gulp-order": "^1.1.1",
+    "gulp-postcss": "^6.0.1",
+    "gulp-rename": "^1.2.2",
+    "gulp-util": "^3.0.7",
+    "nodemon": "^1.9.2",
+    "postcss": "^5.0.14",
+    "postcss-import": "^7.1.3",
+    "postcss-nested": "^1.0.0"
+  }
+}