node-crawler/rr.js at master · 13W/node-crawler · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
var fs = require('fs'),
    url = require('url'),
    path = require('path'),
    mkdirp = require('mkdirp');

exports['pdf-giant.com'] = {
    index           :   function( self, $) { // http://pdf-giant.com/
        $("div.mybox").each(function(i, e) {
            self.Queue.push({
                uri : $(e).find('h1>a').attr('href'),
                rule: 'parse'
            });
        });
/*
        $("div.navigation a:contains('Next')").each(function(i, e) {
            self.Queue.push({
                uri : $(e).attr('href'),
                rule: 'index'
            });
        });
*/
        self.Done();
    },
    parse           :   function(self, $) { // http://pdf-giant.com/money/27524-belarus-may-2013.html
//        console.log(self);
        var result = self.result,
            ctx = $('#dle-content'),
            h1 = ctx.find('h1').html(),
            img = ctx.find('img').attr('src'),
            links = ctx.find('div>a[rel="nofollow"]').map(function(i, e) {return $(e).attr('href')}),
            speedlink = $("#dle-speedbar").text(),
            category = $("#dle-speedbar a:last-child").text(),
            props = $('div[id*="news-id"]>div').text().split(' | '),
            text = $('div[id*="news-id"]'),
            imgDir = path.join(process.cwd(), 'downloads' , path.dirname(url.parse(img || '').pathname)),
            imgPath = path.resolve(imgDir, path.basename(img));
//        text.find('div').remove();
        text = text.text();
        result[h1] = {
            name: h1,
            url: self.referer,
            image: img,
            links: links,
            speedlink: speedlink,
            category: category,
            description: text,
            props: {
                type: props[1],
                pages: props[2],
                size: props[3]
            }
        };
        mkdirp.sync(imgDir);
        if (img && !fs.existsSync(imgPath))
            self.Queue.push({
                uri: url.resolve(self.referer, img),
                require: 'binary',
                filename: imgPath
            });
        self.Done();
    },
    _callback       :   function(result) {
        require('fs').writeFileSync('./pdf-giant.json', JSON.stringify(result, null, 2, true));
        console.inspect(arguments);
    }
};