-
Notifications
You must be signed in to change notification settings - Fork 445
/
index.js
68 lines (59 loc) · 1.69 KB
/
index.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
// Rename CustomExtractor
// to fit your publication
// (e.g., NYTimesExtractor)
export const NewYorkerExtractor = {
domain: 'www.newyorker.com',
title: {
selectors: [
'h1[class^="content-header"]',
'h1[class^="ArticleHeader__hed"]',
'h1[class*="ContentHeaderHed"]',
['meta[name="og:title"]', 'value'],
],
},
author: {
selectors: [
'article header div[class^="BylinesWrapper"]',
['meta[name="article:author"]', 'value'],
'div[class^="ArticleContributors"] a[rel="author"]',
'article header div[class*="Byline__multipleContributors"]',
],
},
content: {
selectors: [
'.article__body',
'article.article.main-content',
'main[class^="Layout__content"]',
],
// Is there anything in the content you selected that needs transformed
// before it's consumable content? E.g., unusual lazy loaded images
transforms: {
'.caption__text': 'figcaption',
'.caption__credit': 'figcaption',
},
// Is there anything that is in the result that shouldn't be?
// The clean selectors will remove anything that matches from
// the result
clean: ['footer[class^="ArticleFooter__footer"]', 'aside'],
},
date_published: {
selectors: [
['meta[name="article:published_time"]', 'value'],
'time.content-header__publish-date',
['meta[name="pubdate"]', 'value'],
],
timezone: 'America/New_York',
},
lead_image_url: {
selectors: [['meta[name="og:image"]', 'value']],
},
dek: {
selectors: [
'div[class^="ContentHeaderDek"]',
'div.content-header__dek',
'h2[class^="ArticleHeader__dek"]',
],
},
next_page_url: null,
excerpt: null,
};