kawalcovid19 · kodiakhq · Sep 6, 2021 · Aug 31, 2021 · Sep 1, 2021 · Sep 1, 2021
diff --git a/etc/fetchers/__mocks__/wbw-database.html b/etc/fetchers/__mocks__/wbw-database.html
@@ -0,0 +1,131 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta name="google" content="notranslate" />
+    <meta
+      name="viewport"
+      content="target-densitydpi=device-dpi,user-scalable=1,minimum-scale=1,maximum-scale=2.5,initial-scale=1,width=device-width"
+    />
+    <meta http-equiv="X-UA-Compatible" content="IE=edge;" />
+    <meta name="referrer" content="origin" />
+    <title>[Database Utama] WargaBantuWarga - Google Drive</title>
+    <link
+      rel="shortcut icon"
+      href="//ssl.gstatic.com/docs/spreadsheets/favicon3.ico"
+    />
+    <link
+      href="/static/spreadsheets2/client/css/2244948090-waffle_k_ltr.css"
+      type="text/css"
+      rel="stylesheet"
+      nonce="lyd3b6OfDAGkvIym1NvIqw"
+    />
+  </head>
+  <body class="docs-gm">
+    <div id="top-bar">
+      <div id="doc-title">
+        <span class="name">[Database Utama] WargaBantuWarga</span>
+      </div>
+      <ul id="sheet-menu">
+        <li id="sheet-button-1011184764"><a href="#">Aceh</a></li>
+      </ul>
+    </div>
+    <div id="1011184764">
+      <table>
+        <tbody>
+          <tr>
+            <td class="s14" dir="ltr">Kebutuhan</td>
+            <td class="freezebar-cell"></td>
+            <td class="s14" dir="ltr">Keterangan</td>
+            <td class="s14" dir="ltr">Lokasi</td>
+            <td class="s14" dir="ltr">Penyedia</td>
+            <td class="s15" dir="ltr">Kontak</td>
+            <td class="s15" dir="ltr">Alamat</td>
+            <td class="s14" dir="ltr">Link</td>
+            <td class="s15" dir="ltr">Tambahan Informasi</td>
+            <td class="s14" dir="ltr">Terakhir Update</td>
+            <td class="s14" dir="ltr">Bentuk Verifikasi</td>
+            <td class="s14" dir="ltr">Ketersediaan</td>
+          </tr>
+          <tr>
+            <td class="freezebar-cell"></td>
+            <td class="freezebar-cell"></td>
+            <td class="freezebar-cell"></td>
+            <td class="freezebar-cell"></td>
+            <td class="freezebar-cell"></td>
+            <td class="freezebar-cell"></td>
+            <td class="freezebar-cell"></td>
+            <td class="freezebar-cell"></td>
+            <td class="freezebar-cell"></td>
+            <td class="freezebar-cell"></td>
+            <td class="freezebar-cell"></td>
+            <td class="freezebar-cell"></td>
+          </tr>
+          <tr>
+            <td class="s22" dir="ltr">Kontak penting</td>
+            <td class="freezebar-cell"></td>
+            <td class="s22" dir="ltr">Call Center</td>
+            <td class="s17" dir="ltr">Banda Aceh</td>
+            <td class="s17" dir="ltr">Aceh Tanggap Covid-19</td>
+            <td class="s17" dir="ltr">(0651) 7319111, (0651) 22118</td>
+            <td class="s22"></td>
+            <td class="s19" dir="ltr">
+              <a
+                target="_blank"
+                rel="noreferrer"
+                href="https://www.google.com/url?q=https://covid19.acehprov.go.id/&amp;sa=D&amp;source=editors&amp;ust=1630948208168000&amp;usg=AOvVaw11EOHZZMec9BOJMXRwdhPu"
+                >https://covid19.acehprov.go.id/</a
+              >
+            </td>
+            <td class="s16" dir="ltr"></td>
+            <td class="s27"></td>
+            <td class="s17"></td>
+            <td class="s16" dir="ltr"></td>
+          </tr>
+          <tr>
+            <td class="s17" dir="ltr">Kontak penting</td>
+            <td class="freezebar-cell"></td>
+            <td class="s22" dir="ltr">Kontak Penting</td>
+            <td class="s17" dir="ltr">Aceh</td>
+            <td class="s22" dir="ltr">Dinas Kesehatan Provinsi Aceh</td>
+            <td class="s22" dir="ltr">(0651) 22421</td>
+            <td class="s17" dir="ltr">
+              Jl. Tgk. Syech Mudawali No 6 Fax 34005
+            </td>
+            <td class="s19" dir="ltr">
+              <a
+                target="_blank"
+                rel="noreferrer"
+                href="https://www.google.com/url?q=http://dinkes.acehprov.go.id/&amp;sa=D&amp;source=editors&amp;ust=1630948208168000&amp;usg=AOvVaw3cGbgyw7nvM_KFBb0zTv6y"
+                >dinkes.acehprov.go.id</a
+              >
+            </td>
+            <td class="s16"></td>
+            <td class="s27"></td>
+            <td class="s17"></td>
+            <td class="s16" dir="ltr"></td>
+          </tr>
+          <tr>
+            <td class="s17" dir="ltr">Kontak penting</td>
+            <td class="freezebar-cell"></td>
+            <td class="s22" dir="ltr">Kontak Penting</td>
+            <td class="s17" dir="ltr">Aceh</td>
+            <td class="s22 softmerge" dir="ltr">
+              <div class="softmerge-inner" style="width: 286px; left: -1px">
+                Badan Penanggulangan Bencana Aceh (BPBA)
+              </div>
+            </td>
+            <td class="s17" dir="ltr">(0651) 34783</td>
+            <td class="s17" dir="ltr">
+              Jalan Teungku Daud Beureueh No. 18 Kuta Alam Banda Aceh Kota 23121
+            </td>
+            <td class="s16"></td>
+            <td class="s16"></td>
+            <td class="s27"></td>
+            <td class="s17"></td>
+            <td class="s16" dir="ltr"></td>
+          </tr>
+        </tbody>
+      </table>
+    </div>
+  </body>
+</html>
diff --git a/etc/fetchers/__mocks__/wbw-database.json b/etc/fetchers/__mocks__/wbw-database.json
@@ -0,0 +1,57 @@
+[
+  {
+    "id": "1011184764",
+    "name": "Aceh",
+    "slug": "aceh",
+    "data": [
+      {
+        "id": "0",
+        "kebutuhan": "Kontak penting",
+        "keterangan": "Call Center",
+        "lokasi": "Banda Aceh",
+        "penyedia": "Aceh Tanggap Covid-19",
+        "kontak": "(0651) 7319111, (0651) 22118",
+        "slug": "kontak-penting-call-center-banda-aceh-aceh-tanggap-covid-19-0651-7319111-0651-22118",
+        "alamat": "",
+        "link": "<a target=\"_blank\" rel=\"noreferrer\" href=\"https://covid19.acehprov.go.id/\">https://covid19.acehprov.go.id/</a>",
+        "tambahan_informasi": "",
+        "terakhir_update": "",
+        "verifikasi": 0,
+        "bentuk_verifikasi": "",
+        "ketersediaan": ""
+      },
+      {
+        "id": "1",
+        "kebutuhan": "Kontak penting",
+        "keterangan": "Kontak Penting",
+        "lokasi": "Aceh",
+        "penyedia": "Dinas Kesehatan Provinsi Aceh",
+        "kontak": "(0651) 22421",
+        "slug": "kontak-penting-kontak-penting-aceh-dinas-kesehatan-provinsi-aceh-0651-22421",
+        "alamat": "Jl. Tgk. Syech Mudawali No 6 Fax 34005",
+        "link": "<a target=\"_blank\" rel=\"noreferrer\" href=\"http://dinkes.acehprov.go.id/\">dinkes.acehprov.go.id</a>",
+        "tambahan_informasi": "",
+        "terakhir_update": "",
+        "verifikasi": 0,
+        "bentuk_verifikasi": "",
+        "ketersediaan": ""
+      },
+      {
+        "id": "2",
+        "kebutuhan": "Kontak penting",
+        "keterangan": "Kontak Penting",
+        "lokasi": "Aceh",
+        "penyedia": "Badan Penanggulangan Bencana Aceh (BPBA)",
+        "kontak": "(0651) 34783",
+        "slug": "kontak-penting-kontak-penting-aceh-badan-penanggulangan-bencana-aceh-bpba-0651-34783",
+        "alamat": "Jalan Teungku Daud Beureueh No. 18 Kuta Alam Banda Aceh Kota 23121",
+        "link": "",
+        "tambahan_informasi": "",
+        "terakhir_update": "",
+        "verifikasi": 0,
+        "bentuk_verifikasi": "",
+        "ketersediaan": ""
+      }
+    ]
+  }
+]
diff --git a/etc/fetchers/__tests__/fetch-database.test.ts b/etc/fetchers/__tests__/fetch-database.test.ts
@@ -0,0 +1,43 @@
+import fs from "fs";
+import path from "path";
+import fetchMock from "jest-fetch-mock";
+import { fetchDatabase } from "../fetch-database";
+
+describe("fetchDatabase", () => {
+  const writeFileSyncSpy = jest.spyOn(fs, "writeFileSync");
+
+  beforeEach(() => {
+    fetchMock.resetMocks();
+    writeFileSyncSpy.mockImplementation(() => {});
+  });
+
+  afterEach(() => {
+    writeFileSyncSpy.mockRestore();
+  });
+
+  it("fetches database from https://kcov.id/wbw-database correctly", async () => {
+    fetchMock.mockResponseOnce(
+      fs.readFileSync(
+        path.resolve(__dirname, "../__mocks__/wbw-database.html"),
+        "utf-8",
+      ),
+    );
+    await fetchDatabase();
+
+    expect(fetchMock).toHaveBeenCalledTimes(1);
+    expect(fetchMock).toHaveBeenCalledWith("https://kcov.id/wbw-database");
+
+    expect(writeFileSyncSpy).toHaveBeenCalledTimes(1);
+    expect(writeFileSyncSpy).toHaveBeenCalledWith(
+      path.resolve(__dirname, "../../../data/wbw-database.json"),
+      JSON.stringify(
+        JSON.parse(
+          fs.readFileSync(
+            path.resolve(__dirname, "../__mocks__/wbw-database.json"),
+            "utf-8",
+          ),
+        ),
+      ),
+    );
+  });
+});
diff --git a/etc/fetchers/fetch-database.ts b/etc/fetchers/fetch-database.ts
@@ -0,0 +1,71 @@
+import fs from "fs";
+import path from "path";
+import cheerio from "cheerio";
+import fetch from "cross-fetch";
+import { allIsEmptyString, getKebabCase } from "../../lib/string-utils";
+import { contactReducer } from "./utils";
+
+export async function fetchDatabase() {
+  const source = await fetch("https://kcov.id/wbw-database");
+  const $ = cheerio.load(await source.text());
+
+  const colMap: Record<string, string> = {};
+
+  const sheetList = $("#sheet-menu > li")
+    .map((_, li) => {
+      const sheetId = ($(li).attr("id") as string).replace("sheet-button-", "");
+      const sheetName = $(li).text();
+      const sheetColumns = $(`#${sheetId} tbody > tr:nth-child(1)`)
+        .find("td")
+        .map((colIndex, td) => {
+          colMap[colIndex] = $(td).text();
+          return {
+            name: $(td).text(),
+            index: colIndex,
+          };
+        })
+        .toArray()
+        .filter((col) => col.name.trim().length !== 0);
+      const sheetRows = $(`#${sheetId} tbody > tr`)
+        .map((rowIndex, tr) => {
+          if (rowIndex === 0) {
+            return [];
+          }
+          return [
+            $(tr)
+              .find("td")
+              .map((colIndex, td) => {
+                if (colMap[colIndex]) {
+                  // Kebutuhan, Keterangan, Lokasi, & Penyedia aren't supposed to be linked
+                  if (colIndex < 5) {
+                    return $(td).text().trim();
+                  } else {
+                    return ($(td).html() as string).trim();
+                  }
+                }
+                return "";
+              })
+              .toArray(),
+          ];
+        })
+        .toArray()
+        .filter((row) => !allIsEmptyString(row));
+
+      return {
+        id: sheetId,
+        name: sheetName,
+        slug: getKebabCase(sheetName),
+        data: sheetRows.map((row, rowIndex) => {
+          return sheetColumns.reduce(contactReducer(row), {
+            id: rowIndex.toString(),
+          });
+        }),
+      };
+    })
+    .toArray();
+
+  fs.writeFileSync(
+    path.resolve(__dirname, "../../data/wbw-database.json"),
+    JSON.stringify(sheetList),
+  );
+}
diff --git a/etc/fetchers/fetch-wbw.ts b/etc/fetchers/fetch-wbw.ts
@@ -1,6 +1,7 @@
 import chalk from "chalk";
 import ora from "ora";
 import { toSecond } from "../../lib/string-utils";
+import { fetchDatabase } from "./fetch-database";
 import { fetchFaqSheets } from "./fetch-faq-sheets";
 import { fetchSheets } from "./fetch-sheets";
 
@@ -18,6 +19,16 @@ import { fetchSheets } from "./fetch-sheets";
       chalk.red(err);
     });
 
+  fetchDatabase()
+    .then(() => {
+      const end = `${toSecond(process.hrtime(start))} seconds`;
+      spinner.succeed(`Fetching Database done in ${chalk.greenBright(end)}`);
+      spinner.start(`${chalk.yellowBright("Fetching next data...")}`);
+    })
+    .catch((err) => {
+      chalk.red(err);
+    });
+
   fetchSheets()
     .then(() => {
       const end = `${toSecond(process.hrtime(start))} seconds`;

diff --git a/jest.config.js b/jest.config.js
@@ -4,13 +4,14 @@ module.exports = {
     "./(components|etc|lib|pages)/**/*.(ts|tsx|js|jsx)",
     "!./(components|etc|lib|pages)/**/__tests__/**/*.test.(ts|tsx|js|jsx)",
     "!./(components|etc|lib|pages)/**/__mocks__/**/*.(ts|tsx|js|jsx)",
+    "!./etc/fetchers/fetch-wbw.ts",
   ],
   coverageThreshold: {
     global: {
-      statements: 83,
-      branches: 74,
-      functions: 81,
-      lines: 83,
+      statements: 84,
+      branches: 76,
+      functions: 82,
+      lines: 84,
     },
   },
   transform: {

diff --git a/package.json b/package.json
@@ -90,6 +90,7 @@
     "identity-obj-proxy": "^3.0.0",
     "is-ci-cli": "2.2.0",
     "jest": "27.0.6",
+    "jest-fetch-mock": "3.0.3",
     "jest-watch-select-projects": "2.0.0",
     "jest-watch-typeahead": "0.6.4",
     "lint-staged": "11.0.1",

diff --git a/test/jest-common.js b/test/jest-common.js
@@ -1,6 +1,7 @@
 const path = require("path");
 
 module.exports = {
+  automock: false,
   rootDir: path.join(__dirname, ".."),
   moduleNameMapper: {
     "\\.css$": "identity-obj-proxy",

diff --git a/test/jest.server.js b/test/jest.server.js
@@ -2,6 +2,7 @@ module.exports = {
   ...require("./jest-common"),
   displayName: "server",
   testEnvironment: "jest-environment-node",
+  setupFiles: ["<rootDir>/test/setup.server.js"],
   testMatch: [
     "**/(etc|lib)/**/__tests__/*.test.(ts|tsx|js|jsx)",
     "!**/lib/__tests__/gtm.test.ts",

diff --git a/test/setup.server.js b/test/setup.server.js
@@ -0,0 +1,4 @@
+import fetchMock from "jest-fetch-mock";
+
+// eslint-disable-next-line no-undef
+jest.setMock("cross-fetch", fetchMock);