-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathparse.ts
More file actions
33 lines (26 loc) · 1015 Bytes
/
parse.ts
File metadata and controls
33 lines (26 loc) · 1015 Bytes
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
import puppeteer from 'puppeteer';
import 'dotenv/config'
let shouldVisitArchive = true
const isArchiveLink = link => link === <string>process.env.SUPPLIER_LINK
async function returnLinksOnCategories(page, categorySelector: String): Promise<string[]> {
return await page.$$eval(categorySelector, links =>
links.map( link => link.href)
)
}
async function parse() {
const browser = await puppeteer.launch({ headless:false, args: ["--disable-setuid-sandbox"], 'ignoreHTTPSErrors': true });
const page = await browser.newPage();
await page.goto(<string>process.env.SUPPLIER_LINK);
await page.waitForSelector(<string>process.env.CATEGORY_NODE);
const links = await returnLinksOnCategories(page, <string>process.env.CATEGORY_NODE)
links.forEach(async link => {
if(isArchiveLink(link) && shouldVisitArchive) {
shouldVisitArchive = false
// visit link
}
// const secondPage = await browser.newPage()
// secondPage.goto(link)
})
// await browser.close();
}
parse()