-
Notifications
You must be signed in to change notification settings - Fork 0
/
index.js
58 lines (46 loc) · 1.79 KB
/
index.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
const program = require('commander');
const ppt = require('puppeteer');
const Scraper = require('./lib/Scraper');
const IssueManager = require('./lib/IssueManager');
const UserManager = require('./lib/UserManager');
const thisYear = new Date().getFullYear().toString();
program
.usage('-y year')
.option('-y, --year [value]', 'set target year to scrape, like 2018. Default value is This year', thisYear)
.option('-v, --view', 'set this option when you want to see GUI')
.option('-f, --force', 'force start from scracth')
.parse(process.argv);
const targetYear = program.year || thisYear;
const isHeadless = !program.view;
// scrape mode, scrape from scratch or resume from last time
const isForce = program.force? true: false;
const puppenomist = async () => {
const userManager = new UserManager();
await userManager.askAccount();
const browser = await ppt.launch({ headless: isHeadless });
let page = await browser.newPage();
await page.setViewport({ width: 1200, height: 1000 });
await userManager.login(page).catch((error) => {
console.log(error.message);
});
if(!userManager.isLogined) {
console.log("terminating...");
await browser.close();
process.exit(1);
}
const scrapeMode = IssueManager.decideMode(isForce, targetYear);
const issueMgr = new IssueManager(scrapeMode, targetYear);
await issueMgr.selectTarget(page).catch((err) => {throw new Error(err.message)});
for (link of issueMgr.issueLinks) {
issueMgr.setStarted(link);
const scraper = new Scraper(page, link);
await scraper.extractArticleLinks();
await scraper.scrape();
issueMgr.setDone(link);
}
await browser.close();
};
(async () => {
await puppenomist();
})();
module.exports = puppenomist;