-
Notifications
You must be signed in to change notification settings - Fork 0
/
scrapeAPI.js
87 lines (77 loc) · 3.97 KB
/
scrapeAPI.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
const fetch = require('node-fetch');
const cheerio = require('cheerio');
let classInfo = [];
async function fetchData() {
let url = 'https://globalsearch.cuny.edu/CFGlobalSearchTool/CFSearchToolController';
// initial fetch to set the initial search parameters and get the Set-Cookie header
let initialOptions = {
method: 'POST',
headers: {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language': 'en-US,en;q=0.9',
'Cache-Control': 'max-age=0',
'Connection': 'keep-alive',
'Content-Type': 'application/x-www-form-urlencoded',
'Origin': 'https://globalsearch.cuny.edu',
'Referer': 'https://globalsearch.cuny.edu/CFGlobalSearchTool/CFSearchToolController',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'same-origin',
'Sec-Fetch-User': '?1',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'sec-ch-ua': '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Windows"'
},
body: 'selectedInstName=Hunter+College+%7C+&inst_selection=HTR01&selectedTermName=2024+Spring+Term&term_value=1242&next_btn=Next'
};
let initialResponse = await fetch(url, initialOptions);
let cookie = initialResponse.headers.get('set-cookie'); // Extract the cookie from the Set-Cookie header
// second fetch to set the additional search parameters and perform the search
let searchOptions = {
method: 'POST',
headers: {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language': 'en-US,en;q=0.9',
'Cache-Control': 'max-age=0',
'Connection': 'keep-alive',
'Content-Type': 'application/x-www-form-urlencoded',
'Cookie': cookie,
'Origin': 'https://globalsearch.cuny.edu',
'Referer': 'https://globalsearch.cuny.edu/CFGlobalSearchTool/CFSearchToolController',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'same-origin',
'Sec-Fetch-User': '?1',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'sec-ch-ua': '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Windows"'
},
body: 'selectedSubjectName=Computer+Science&subject_name=CMSC&selectedCCareerName=Undergraduate&courseCareer=UGRD&selectedCAttrName=&courseAttr=&selectedCAttrVName=&courseAttValue=&selectedReqDName=&reqDesignation=&open_class=O&selectedSessionName=&class_session=&selectedModeInsName=&meetingStart=LT&selectedMeetingStartName=less+than&meetingStartText=&AndMeetingStartText=&meetingEnd=LE&selectedMeetingEndName=less+than+or+equal+to&meetingEndText=&AndMeetingEndText=&daysOfWeek=I&selectedDaysOfWeekName=include+only+these+days&instructor=B&selectedInstructorName=begins+with&instructorName=&search_btn_search=Search'
};
let response = await fetch(url, searchOptions);
let htmlString = await response.text();
let $ = cheerio.load(htmlString);
let rows = $('.classinfo tbody tr');
rows.each((i, row) => {
let cells = $(row).find('td');
// parse class data
let classData = {
Class: $(cells[0]).text(),
Section: $(cells[1]).text(),
'Days & Times': $(cells[2]).text(),
Room: $(cells[3]).text(),
Instructor: $(cells[4]).text(),
'Instruction Mode': $(cells[5]).text(),
'Meeting Dates': $(cells[6]).text(),
Status: "Open✅",
'Course Topic': $(cells[8]).text()
};
classInfo.push(classData);
});
return classInfo;
}
module.exports = fetchData;