diff --git a/package-lock.json b/package-lock.json index eec15d7..770ab80 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "dependencies": { "cheerio": "^1.0.0-rc.12", "iconv-lite": "^0.6.3", + "qs": "^6.11.0", "slugify": "^1.6.5", "undici": "^5.8.0" }, @@ -19,6 +20,7 @@ "@types/iconv-lite": "^0.0.1", "@types/jest": "^28.1.6", "@types/node": "18.6.1", + "@types/qs": "^6.9.7", "@typescript-eslint/eslint-plugin": "^5.30.7", "@typescript-eslint/parser": "^5.30.7", "eslint": "^8.20.0", @@ -1196,6 +1198,12 @@ "integrity": "sha512-ymZk3LEC/fsut+/Q5qejp6R9O1rMxz3XaRHDV6kX8MrGAhOSPqVARbDi+EZvInBpw+BnCX3TD240byVkOfQsHg==", "dev": true }, + "node_modules/@types/qs": { + "version": "6.9.7", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz", + "integrity": "sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==", + "dev": true + }, "node_modules/@types/stack-utils": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.1.tgz", @@ -1693,6 +1701,18 @@ "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "dev": true }, + "node_modules/call-bind": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", + "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "dependencies": { + "function-bind": "^1.1.1", + "get-intrinsic": "^1.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -2551,8 +2571,7 @@ "node_modules/function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", - "dev": true + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" }, "node_modules/functional-red-black-tree": { "version": "1.0.1", @@ -2578,6 +2597,19 @@ "node": "6.* || 8.* || >= 10.*" } }, + "node_modules/get-intrinsic": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.2.tgz", + "integrity": "sha512-Jfm3OyCxHh9DJyc28qGk+JmfkpO41A4XkneDSujN9MDXrm4oDKdHvndhZ2dN94+ERNfkYJWDclW6k2L/ZGHjXA==", + "dependencies": { + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/get-package-type": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", @@ -2676,7 +2708,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", - "dev": true, "dependencies": { "function-bind": "^1.1.1" }, @@ -2693,6 +2724,17 @@ "node": ">=8" } }, + "node_modules/has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/html-escaper": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", @@ -3797,6 +3839,14 @@ "url": "https://github.com/fb55/nth-check?sponsor=1" } }, + "node_modules/object-inspect": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.2.tgz", + "integrity": "sha512-z+cPxW0QGUp0mcqcsgQyLVRDoXFQbXOwBaqyF7VIgI4TWNQsDHrBpUQslRmIfAoYWdYzs6UlKJtB2XJpTaNSpQ==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -4084,6 +4134,20 @@ "node": ">=6" } }, + "node_modules/qs": { + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", + "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", + "dependencies": { + "side-channel": "^1.0.4" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", @@ -4282,6 +4346,19 @@ "node": ">=8" } }, + "node_modules/side-channel": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", + "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", + "dependencies": { + "call-bind": "^1.0.0", + "get-intrinsic": "^1.0.2", + "object-inspect": "^1.9.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/signal-exit": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", @@ -5793,6 +5870,12 @@ "integrity": "sha512-ymZk3LEC/fsut+/Q5qejp6R9O1rMxz3XaRHDV6kX8MrGAhOSPqVARbDi+EZvInBpw+BnCX3TD240byVkOfQsHg==", "dev": true }, + "@types/qs": { + "version": "6.9.7", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz", + "integrity": "sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==", + "dev": true + }, "@types/stack-utils": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.1.tgz", @@ -6121,6 +6204,15 @@ "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "dev": true }, + "call-bind": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", + "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "requires": { + "function-bind": "^1.1.1", + "get-intrinsic": "^1.0.2" + } + }, "callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -6757,8 +6849,7 @@ "function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", - "dev": true + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" }, "functional-red-black-tree": { "version": "1.0.1", @@ -6778,6 +6869,16 @@ "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", "dev": true }, + "get-intrinsic": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.2.tgz", + "integrity": "sha512-Jfm3OyCxHh9DJyc28qGk+JmfkpO41A4XkneDSujN9MDXrm4oDKdHvndhZ2dN94+ERNfkYJWDclW6k2L/ZGHjXA==", + "requires": { + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.3" + } + }, "get-package-type": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", @@ -6846,7 +6947,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", - "dev": true, "requires": { "function-bind": "^1.1.1" } @@ -6857,6 +6957,11 @@ "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", "dev": true }, + "has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==" + }, "html-escaper": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", @@ -7700,6 +7805,11 @@ "boolbase": "^1.0.0" } }, + "object-inspect": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.2.tgz", + "integrity": "sha512-z+cPxW0QGUp0mcqcsgQyLVRDoXFQbXOwBaqyF7VIgI4TWNQsDHrBpUQslRmIfAoYWdYzs6UlKJtB2XJpTaNSpQ==" + }, "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -7902,6 +8012,14 @@ "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", "dev": true }, + "qs": { + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", + "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", + "requires": { + "side-channel": "^1.0.4" + } + }, "queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", @@ -8025,6 +8143,16 @@ "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", "dev": true }, + "side-channel": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", + "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", + "requires": { + "call-bind": "^1.0.0", + "get-intrinsic": "^1.0.2", + "object-inspect": "^1.9.0" + } + }, "signal-exit": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", diff --git a/package.json b/package.json index 12082cd..f9dd327 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "@types/iconv-lite": "^0.0.1", "@types/jest": "^28.1.6", "@types/node": "18.6.1", + "@types/qs": "^6.9.7", "@typescript-eslint/eslint-plugin": "^5.30.7", "@typescript-eslint/parser": "^5.30.7", "eslint": "^8.20.0", @@ -54,6 +55,7 @@ "dependencies": { "cheerio": "^1.0.0-rc.12", "iconv-lite": "^0.6.3", + "qs": "^6.11.0", "slugify": "^1.6.5", "undici": "^5.8.0" } diff --git a/src/__tests__/index.test.ts b/src/__tests__/index.test.ts index 78af757..dd1eb96 100644 --- a/src/__tests__/index.test.ts +++ b/src/__tests__/index.test.ts @@ -1,6 +1,32 @@ import pensador from '../index'; -it('Works', async () => { - const result = await pensador({ term: 'Elon Musk', max: 5 }); - expect(result.searchTerm).toBe('frases_de_elon_musk'); +describe('Pensador', () => { + it('Works', async () => { + const result = await pensador({ term: 'Elon Musk', max: 5 }); + expect(result.searchTerm).toBe('elon_musk'); + }); + it('Should return phrases', async () => { + const result = await pensador({ term: 'Elon Musk', max: 5 }); + expect(result.total).toBeGreaterThan(0); + }); + it('Should return an error when the term is not provided', async () => { + try { + await pensador({ + max: 5, + term: '', + }); + } catch (err: any) { + expect(err.message).toBe('The term is required'); + } + }); + it('Should return an error when the max is not provided', async () => { + try { + await pensador({ + max: 0, + term: 'Elon Musk', + }); + } catch (err: any) { + expect(err.message).toBe('The max is required'); + } + }); }); diff --git a/src/constants/index.ts b/src/constants/index.ts index 696fbdd..e704e2b 100644 --- a/src/constants/index.ts +++ b/src/constants/index.ts @@ -1,3 +1,3 @@ -const BASE_URL = 'https://www.pensador.com/'; +const BASE_URL = 'https://www.pensador.com'; export { BASE_URL }; diff --git a/src/index.ts b/src/index.ts index 5c44845..849cb97 100644 --- a/src/index.ts +++ b/src/index.ts @@ -10,7 +10,7 @@ export default async (options: IOptions) => { _throw('A search term must be defined'); } - const searchTerm = slugify(`frases de ${options.term}`, { + const searchTerm = slugify(`${options.term}`, { replacement: '_', remove: /[*+~.()'"!:@]/g, lower: true, diff --git a/src/services/fetchPage.ts b/src/services/fetchPage.ts index d2a0ab1..d1587b8 100644 --- a/src/services/fetchPage.ts +++ b/src/services/fetchPage.ts @@ -1,13 +1,27 @@ import iconv from 'iconv-lite'; import { request } from 'undici'; +import qs from 'qs'; +import { isSuccessfulRequest } from '../utils/isSuccessfulRequest'; -async function fetchPage(searchTerm: string, current = 1, baseUrl: string) { - return new Promise((resolve, reject) => { - return request(`${baseUrl}/${searchTerm}/${current}`) - .then((res) => res.body.arrayBuffer()) - .then((arrayBuffer) => iconv.decode(Buffer.from(arrayBuffer), 'utf-8').toString()) - .then((body) => resolve(body)) - .catch((err) => reject(err)); +export async function fetchPage(searchTerm: string, current = 1, baseUrl: string) { + const params = qs.stringify({ + q: searchTerm, + p: current, }); + try { + const res = await request(`${baseUrl}/busca.php?${params}`); + if (!isSuccessfulRequest(res.statusCode)) { + throw new Error(`Error fetching ${searchTerm}, status code: ${res.statusCode}`); + } + const arrayBuffer = await res.body.arrayBuffer(); + const body = iconv.decode(Buffer.from(arrayBuffer), 'utf-8').toString(); + return body; + } catch (error) { + if (error instanceof Error) { + throw error; + } + throw new Error(`Error fetching ${searchTerm}`); + } } + export default fetchPage; diff --git a/src/utils/_throw.ts b/src/utils/_throw.ts index 325e0b7..1656b48 100644 --- a/src/utils/_throw.ts +++ b/src/utils/_throw.ts @@ -1,3 +1,9 @@ -export default function _throw(m: string | string[]) { - throw m; +export default function _throw(message: string | string[] | Error) { + if (Array.isArray(message)) { + throw new Error(message.join('\n')); + } + if (message instanceof Error) { + throw message.message; + } + throw message; } diff --git a/src/utils/extractHTML.ts b/src/utils/extractHTML.ts index 19e9235..fa55053 100644 --- a/src/utils/extractHTML.ts +++ b/src/utils/extractHTML.ts @@ -2,32 +2,30 @@ import cheerio from 'cheerio'; import { IPhrases } from '../typescript'; async function extractHTML(htmlContent: string) { - return new Promise((resolve, reject) => { - try { - const phrases: IPhrases[] = []; - const $ = cheerio.load(htmlContent); - $('.thought-card').each(function (i, e) { - void i; - void e; - phrases.push({ - author: $(this).find('a').first().text(), - text: $(this).find('p').first().text().replace(/\n/g, ''), - }); + try { + const phrases: IPhrases[] = []; + const $ = cheerio.load(htmlContent); + $('.thought-card').each(function (i, e) { + void i; + void e; + phrases.push({ + author: $(this).find('a').first().text(), + text: $(this).find('p').first().text().replace(/\n/g, ''), }); + }); - let next = false; - $('#paginacao').each(function (i, e) { - void i; - void e; - if ($(this).find('.nav').last().text().includes('xima')) { - next = true; - } - }); + let next = false; + $('#paginacao').each(function (i, e) { + void i; + void e; + if ($(this).find('.nav').last().text().includes('xima')) { + next = true; + } + }); - resolve({ phrases, next }); - } catch (err) { - reject(err); - } - }); + return Promise.resolve({ phrases, next }); + } catch (err) { + return Promise.reject(err); + } } export default extractHTML; diff --git a/src/utils/isSuccessfulRequest.ts b/src/utils/isSuccessfulRequest.ts new file mode 100644 index 0000000..b2e2125 --- /dev/null +++ b/src/utils/isSuccessfulRequest.ts @@ -0,0 +1,3 @@ +export function isSuccessfulRequest(status: number) { + return status >= 200 && status <= 399; +}