forked from surprisetalk/blogs.hn
-
Notifications
You must be signed in to change notification settings - Fork 0
/
fetch.js
70 lines (64 loc) · 1.92 KB
/
fetch.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
const fetch = require("node-fetch");
const axios = require("axios");
const cheerio = require("cheerio");
const links = process.argv.slice(2);
(async () => {
const blogs = [];
for (const link of links) {
try {
const res = await axios.get(link),
$ = cheerio.load(res.data),
title = $("title").first().text() ?? undefined,
keywords = $('meta[name="keywords"]').attr("content") ?? undefined,
desc = $('meta[name="description"]').attr("content") ?? undefined;
let about, now, feed, news, hn;
$("link, a, button").each((i, x) => {
let url = new URL($(x).attr("href"), link);
if (url.pathname === "/about") about = url.href;
if (url.pathname === "/now") now = url.href;
if (
url.pathname.match(
/(\/index.xml|\/rss\.xml|\/feed\.xml|\/rss|\/feed|\.atom)$/
)
)
feed = url.href;
});
try {
const alg = await fetch(
`https://hn.algolia.com/api/v1/search` +
`?tags=story` +
`&restrictSearchableAttributes=url` +
`&query=${encodeURIComponent(link.replace(/^https?:\/\//, ""))}`
);
const data = await alg.json();
hn = data?.hits
?.map(hit => ({
created_at: hit.created_at,
title: hit.title,
url: hit.url,
points: hit.points,
comments: hit.num_comments,
id: hit.objectID,
}))
?.filter(hit => hit.points >= 10 || hit.comments >= 5);
if (!hn.length) hn = undefined;
} catch (err) {
console.error(`Error: ${err}`);
}
blogs.push({
url: link,
title,
about,
now,
feed,
news,
keywords,
desc,
hn,
});
} catch (error) {
console.error(`Error: ${error} for url: ${link}`);
}
}
console.log(JSON.stringify(blogs, null, 2));
})();