diff --git a/etc/fetchers/__mocks__/wbw-database.html b/etc/fetchers/__mocks__/wbw-database.html new file mode 100644 index 000000000..8befa4bcc --- /dev/null +++ b/etc/fetchers/__mocks__/wbw-database.html @@ -0,0 +1,131 @@ + + + + + + + + [Database Utama] WargaBantuWarga - Google Drive + + + + +
+
+ [Database Utama] WargaBantuWarga +
+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
KebutuhanKeteranganLokasiPenyediaKontakAlamatLinkTambahan InformasiTerakhir UpdateBentuk VerifikasiKetersediaan
Kontak pentingCall CenterBanda AcehAceh Tanggap Covid-19(0651) 7319111, (0651) 22118 + https://covid19.acehprov.go.id/ +
Kontak pentingKontak PentingAcehDinas Kesehatan Provinsi Aceh(0651) 22421 + Jl. Tgk. Syech Mudawali No 6 Fax 34005 + + dinkes.acehprov.go.id +
Kontak pentingKontak PentingAceh +
+ Badan Penanggulangan Bencana Aceh (BPBA) +
+ 		(0651) 34783 + Jalan Teungku Daud Beureueh No. 18 Kuta Alam Banda Aceh Kota 23121 +
+
+ + diff --git a/etc/fetchers/__mocks__/wbw-database.json b/etc/fetchers/__mocks__/wbw-database.json new file mode 100644 index 000000000..a0c059dd5 --- /dev/null +++ b/etc/fetchers/__mocks__/wbw-database.json @@ -0,0 +1,57 @@ +[ + { + "id": "1011184764", + "name": "Aceh", + "slug": "aceh", + "data": [ + { + "id": "0", + "kebutuhan": "Kontak penting", + "keterangan": "Call Center", + "lokasi": "Banda Aceh", + "penyedia": "Aceh Tanggap Covid-19", + "kontak": "(0651) 7319111, (0651) 22118", + "slug": "kontak-penting-call-center-banda-aceh-aceh-tanggap-covid-19-0651-7319111-0651-22118", + "alamat": "", + "link": "https://covid19.acehprov.go.id/", + "tambahan_informasi": "", + "terakhir_update": "", + "verifikasi": 0, + "bentuk_verifikasi": "", + "ketersediaan": "" + }, + { + "id": "1", + "kebutuhan": "Kontak penting", + "keterangan": "Kontak Penting", + "lokasi": "Aceh", + "penyedia": "Dinas Kesehatan Provinsi Aceh", + "kontak": "(0651) 22421", + "slug": "kontak-penting-kontak-penting-aceh-dinas-kesehatan-provinsi-aceh-0651-22421", + "alamat": "Jl. Tgk. Syech Mudawali No 6 Fax 34005", + "link": "dinkes.acehprov.go.id", + "tambahan_informasi": "", + "terakhir_update": "", + "verifikasi": 0, + "bentuk_verifikasi": "", + "ketersediaan": "" + }, + { + "id": "2", + "kebutuhan": "Kontak penting", + "keterangan": "Kontak Penting", + "lokasi": "Aceh", + "penyedia": "Badan Penanggulangan Bencana Aceh (BPBA)", + "kontak": "(0651) 34783", + "slug": "kontak-penting-kontak-penting-aceh-badan-penanggulangan-bencana-aceh-bpba-0651-34783", + "alamat": "Jalan Teungku Daud Beureueh No. 18 Kuta Alam Banda Aceh Kota 23121", + "link": "", + "tambahan_informasi": "", + "terakhir_update": "", + "verifikasi": 0, + "bentuk_verifikasi": "", + "ketersediaan": "" + } + ] + } +] diff --git a/etc/fetchers/__tests__/fetch-database.test.ts b/etc/fetchers/__tests__/fetch-database.test.ts new file mode 100644 index 000000000..a52a2319c --- /dev/null +++ b/etc/fetchers/__tests__/fetch-database.test.ts @@ -0,0 +1,43 @@ +import fs from "fs"; +import path from "path"; +import fetchMock from "jest-fetch-mock"; +import { fetchDatabase } from "../fetch-database"; + +describe("fetchDatabase", () => { + const writeFileSyncSpy = jest.spyOn(fs, "writeFileSync"); + + beforeEach(() => { + fetchMock.resetMocks(); + writeFileSyncSpy.mockImplementation(() => {}); + }); + + afterEach(() => { + writeFileSyncSpy.mockRestore(); + }); + + it("fetches database from https://kcov.id/wbw-database correctly", async () => { + fetchMock.mockResponseOnce( + fs.readFileSync( + path.resolve(__dirname, "../__mocks__/wbw-database.html"), + "utf-8", + ), + ); + await fetchDatabase(); + + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(fetchMock).toHaveBeenCalledWith("https://kcov.id/wbw-database"); + + expect(writeFileSyncSpy).toHaveBeenCalledTimes(1); + expect(writeFileSyncSpy).toHaveBeenCalledWith( + path.resolve(__dirname, "../../../data/wbw-database.json"), + JSON.stringify( + JSON.parse( + fs.readFileSync( + path.resolve(__dirname, "../__mocks__/wbw-database.json"), + "utf-8", + ), + ), + ), + ); + }); +}); diff --git a/etc/fetchers/fetch-database.ts b/etc/fetchers/fetch-database.ts new file mode 100644 index 000000000..6185edbe5 --- /dev/null +++ b/etc/fetchers/fetch-database.ts @@ -0,0 +1,71 @@ +import fs from "fs"; +import path from "path"; +import cheerio from "cheerio"; +import fetch from "cross-fetch"; +import { allIsEmptyString, getKebabCase } from "../../lib/string-utils"; +import { contactReducer } from "./utils"; + +export async function fetchDatabase() { + const source = await fetch("https://kcov.id/wbw-database"); + const $ = cheerio.load(await source.text()); + + const colMap: Record = {}; + + const sheetList = $("#sheet-menu > li") + .map((_, li) => { + const sheetId = ($(li).attr("id") as string).replace("sheet-button-", ""); + const sheetName = $(li).text(); + const sheetColumns = $(`#${sheetId} tbody > tr:nth-child(1)`) + .find("td") + .map((colIndex, td) => { + colMap[colIndex] = $(td).text(); + return { + name: $(td).text(), + index: colIndex, + }; + }) + .toArray() + .filter((col) => col.name.trim().length !== 0); + const sheetRows = $(`#${sheetId} tbody > tr`) + .map((rowIndex, tr) => { + if (rowIndex === 0) { + return []; + } + return [ + $(tr) + .find("td") + .map((colIndex, td) => { + if (colMap[colIndex]) { + // Kebutuhan, Keterangan, Lokasi, & Penyedia aren't supposed to be linked + if (colIndex < 5) { + return $(td).text().trim(); + } else { + return ($(td).html() as string).trim(); + } + } + return ""; + }) + .toArray(), + ]; + }) + .toArray() + .filter((row) => !allIsEmptyString(row)); + + return { + id: sheetId, + name: sheetName, + slug: getKebabCase(sheetName), + data: sheetRows.map((row, rowIndex) => { + return sheetColumns.reduce(contactReducer(row), { + id: rowIndex.toString(), + }); + }), + }; + }) + .toArray(); + + fs.writeFileSync( + path.resolve(__dirname, "../../data/wbw-database.json"), + JSON.stringify(sheetList), + ); +} diff --git a/etc/fetchers/fetch-wbw.ts b/etc/fetchers/fetch-wbw.ts index 076a21104..f39204275 100644 --- a/etc/fetchers/fetch-wbw.ts +++ b/etc/fetchers/fetch-wbw.ts @@ -1,6 +1,7 @@ import chalk from "chalk"; import ora from "ora"; import { toSecond } from "../../lib/string-utils"; +import { fetchDatabase } from "./fetch-database"; import { fetchFaqSheets } from "./fetch-faq-sheets"; import { fetchSheets } from "./fetch-sheets"; @@ -18,6 +19,16 @@ import { fetchSheets } from "./fetch-sheets"; chalk.red(err); }); + fetchDatabase() + .then(() => { + const end = `${toSecond(process.hrtime(start))} seconds`; + spinner.succeed(`Fetching Database done in ${chalk.greenBright(end)}`); + spinner.start(`${chalk.yellowBright("Fetching next data...")}`); + }) + .catch((err) => { + chalk.red(err); + }); + fetchSheets() .then(() => { const end = `${toSecond(process.hrtime(start))} seconds`; diff --git a/jest.config.js b/jest.config.js index 8d103ed21..5b21ac025 100644 --- a/jest.config.js +++ b/jest.config.js @@ -4,13 +4,14 @@ module.exports = { "./(components|etc|lib|pages)/**/*.(ts|tsx|js|jsx)", "!./(components|etc|lib|pages)/**/__tests__/**/*.test.(ts|tsx|js|jsx)", "!./(components|etc|lib|pages)/**/__mocks__/**/*.(ts|tsx|js|jsx)", + "!./etc/fetchers/fetch-wbw.ts", ], coverageThreshold: { global: { - statements: 83, - branches: 74, - functions: 81, - lines: 83, + statements: 84, + branches: 76, + functions: 82, + lines: 84, }, }, transform: { diff --git a/package.json b/package.json index a9fe1cfbc..36b40bc79 100644 --- a/package.json +++ b/package.json @@ -90,6 +90,7 @@ "identity-obj-proxy": "^3.0.0", "is-ci-cli": "2.2.0", "jest": "27.0.6", + "jest-fetch-mock": "3.0.3", "jest-watch-select-projects": "2.0.0", "jest-watch-typeahead": "0.6.4", "lint-staged": "11.0.1", diff --git a/test/jest-common.js b/test/jest-common.js index 7c2767883..509350f7b 100644 --- a/test/jest-common.js +++ b/test/jest-common.js @@ -1,6 +1,7 @@ const path = require("path"); module.exports = { + automock: false, rootDir: path.join(__dirname, ".."), moduleNameMapper: { "\\.css$": "identity-obj-proxy", diff --git a/test/jest.server.js b/test/jest.server.js index 09a5ac4cb..26ade06a5 100644 --- a/test/jest.server.js +++ b/test/jest.server.js @@ -2,6 +2,7 @@ module.exports = { ...require("./jest-common"), displayName: "server", testEnvironment: "jest-environment-node", + setupFiles: ["/test/setup.server.js"], testMatch: [ "**/(etc|lib)/**/__tests__/*.test.(ts|tsx|js|jsx)", "!**/lib/__tests__/gtm.test.ts", diff --git a/test/setup.server.js b/test/setup.server.js new file mode 100644 index 000000000..c496a18e5 --- /dev/null +++ b/test/setup.server.js @@ -0,0 +1,4 @@ +import fetchMock from "jest-fetch-mock"; + +// eslint-disable-next-line no-undef +jest.setMock("cross-fetch", fetchMock); diff --git a/yarn.lock b/yarn.lock index bf45ed043..16e2fc731 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5063,7 +5063,7 @@ cross-env@^7.0.3: dependencies: cross-spawn "^7.0.1" -cross-fetch@^3.1.4: +cross-fetch@^3.0.4, cross-fetch@^3.1.4: version "3.1.4" resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-3.1.4.tgz#9723f3a3a247bf8b89039f3a380a9244e8fa2f39" integrity sha512-1eAtFWdIubi6T4XPy6ei9iUFoKpUkIF971QLN8lIvvvwueI65+Nw5haMNKUwfJxabqlIIDODJKGrQ66gxC0PbQ== @@ -8685,6 +8685,14 @@ jest-environment-node@^27.0.6: jest-mock "^27.0.6" jest-util "^27.0.6" +jest-fetch-mock@3.0.3: + version "3.0.3" + resolved "https://registry.yarnpkg.com/jest-fetch-mock/-/jest-fetch-mock-3.0.3.tgz#31749c456ae27b8919d69824f1c2bd85fe0a1f3b" + integrity sha512-Ux1nWprtLrdrH4XwE7O7InRY6psIi3GOsqNESJgMJ+M5cv4A8Lh7SN9d2V2kKRZ8ebAfcd1LNyZguAOb6JiDqw== + dependencies: + cross-fetch "^3.0.4" + promise-polyfill "^8.1.3" + jest-get-type@^26.3.0: version "26.3.0" resolved "https://registry.yarnpkg.com/jest-get-type/-/jest-get-type-26.3.0.tgz#e97dc3c3f53c2b406ca7afaed4493b1d099199e0" @@ -11522,6 +11530,11 @@ progress@^2.0.0: resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8" integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA== +promise-polyfill@^8.1.3: + version "8.2.0" + resolved "https://registry.yarnpkg.com/promise-polyfill/-/promise-polyfill-8.2.0.tgz#367394726da7561457aba2133c9ceefbd6267da0" + integrity sha512-k/TC0mIcPVF6yHhUvwAp7cvL6I2fFV7TzF1DuGPI8mBh4QQazf36xCKEHKTZKRysEoTQoQdKyP25J8MPJp7j5g== + prompts@^2.0.1, prompts@^2.2.1: version "2.4.1" resolved "https://registry.yarnpkg.com/prompts/-/prompts-2.4.1.tgz#befd3b1195ba052f9fd2fde8a486c4e82ee77f61"