crawler/BmpCode.java at master · peanut7379/crawler · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
public void startProxy() {
  proxy = new BrowserMobProxyServer();
        proxy.setTrustAllServers(true);
        /*proxy.setMitmManager(ImpersonatingMitmManager.builder().trustAllServers(true).build());*/
        proxy.start(0);
        proxyPort = proxy.getPort();

        logger.info("Proxy started @port {}", proxyPort);

        RequestFilter filter = new RequestFilter() {
            @Override
            public HttpResponse filterRequest(HttpRequest request, HttpMessageContents contents, HttpMessageInfo messageInfo) {

                logger.info("[DEBUG] originalUrl:" + messageInfo.getOriginalUrl());
                if (messageInfo.getOriginalUrl().contains("https://live.direct.ly//rest/lives/v1/discovery/channels/latest?viewedId=")) {

                    logger.info(messageInfo.getOriginalUrl());

                    logger.info("[DEBUG]X-Request-ID:" + messageInfo.getOriginalRequest().headers().get("X-Request-ID"));
                    logger.info("[DEBUG]X-Request-Info5:" + messageInfo.getOriginalRequest().headers().get("X-Request-Info5"));
                    logger.info("[DEBUG]X-Request-Sign5:" + messageInfo.getOriginalRequest().headers().get("X-Request-Sign5"));
                    logger.info("[DEBUG]Authorization:" + messageInfo.getOriginalRequest().headers().get("Authorization"));
                    logger.info("[DEBUG]build:" + messageInfo.getOriginalRequest().headers().get("build"));

                    //AccessConfidential ac = new AccessConfidential(
                    //        messageInfo.getOriginalRequest().headers().get("X-Request-ID"),
                    //        messageInfo.getOriginalRequest().headers().get("X-Request-Info5"),
                    //        messageInfo.getOriginalRequest().headers().get("X-Request-Sign5"),
                    //        messageInfo.getOriginalRequest().headers().get("Authorization"),
                    //        messageInfo.getOriginalRequest().headers().get("build"),
                    //        0L
                    //);

                    try {
                        //TODO:
                        // ac.insert();
                        logger.info("TODO:");
                    } catch (Exception e) {
                        logger.error("Error insert AccessConfidential, ", e);
                    }
                }

                return null;
            }
        };

        proxy.addFirstHttpFilterFactory(new RequestFilterAdapter.FilterSource(filter, 16777216));

        proxy.addResponseFilter(new ResponseFilter() {
            @Override
            public void filterResponse(HttpResponse response, HttpMessageContents contents, HttpMessageInfo messageInfo) {

                if (messageInfo.getOriginalUrl().contains("https://live.direct.ly//rest/lives/v1/discovery/channels/latest?viewedId=")) {

                    String uid = contents.getTextContents().replaceAll("^.+?\"id\":", "").replaceAll(",.+?$", "");

                    try {

                        if (!Crawler.userIds.contains(uid) && !Crawler.userIdQueueSet.contains(uid)) {
                            Crawler.userIdQueue.put(uid);
                            Crawler.userIdQueueSet.add(uid);
                        }

                    } catch (InterruptedException e) {
                        e.printStackTrace();
                    }
                }
            }
        });

    }