From f08984698d6e6c1e881466a92dabf7b1d3c664bb Mon Sep 17 00:00:00 2001
From: Jan Potoms <2109932+Janpot@users.noreply.github.com>
Date: Tue, 13 Sep 2022 17:11:22 +0200
Subject: [PATCH] Put Toolpad user in charge of response parsing (#987)

* automatically parse all fetch responses based on content sniffing

* Put user in charge of parsing bodies
---
 packages/toolpad-app/package.json             |  2 -
 .../toolpadDataSources/rest/BodyEditor.tsx    | 11 ++++-
 .../src/toolpadDataSources/rest/client.tsx    | 38 ++++++++++++++-
 .../src/toolpadDataSources/rest/server.ts     | 23 ++++-----
 .../src/toolpadDataSources/rest/types.ts      | 47 +++++++++++++++++++
 yarn.lock                                     |  5 --
 6 files changed, 103 insertions(+), 23 deletions(-)

diff --git a/packages/toolpad-app/package.json b/packages/toolpad-app/package.json
index a0d8fbe3879..c5fa94fccab 100644
--- a/packages/toolpad-app/package.json
+++ b/packages/toolpad-app/package.json
@@ -101,7 +101,6 @@
     "ts-node": "^10.9.1",
     "typescript": "^4.8.2",
     "web-streams-polyfill": "^3.2.1",
-    "whatwg-mimetype": "^3.0.0",
     "whatwg-url": "^11.0.0"
   },
   "devDependencies": {
@@ -119,7 +118,6 @@
     "@types/react-dom": "^18.0.6",
     "@types/react-inspector": "^4.0.2",
     "@types/serialize-javascript": "^5.0.2",
-    "@types/whatwg-mimetype": "^2.1.1",
     "@types/whatwg-url": "^11.0.0",
     "ajv": "^8.11.0",
     "eslint": "8.23.0",
diff --git a/packages/toolpad-app/src/toolpadDataSources/rest/BodyEditor.tsx b/packages/toolpad-app/src/toolpadDataSources/rest/BodyEditor.tsx
index 852d81c6168..c56a65c3713 100644
--- a/packages/toolpad-app/src/toolpadDataSources/rest/BodyEditor.tsx
+++ b/packages/toolpad-app/src/toolpadDataSources/rest/BodyEditor.tsx
@@ -118,6 +118,8 @@ function RawBodyEditor({
           <React.Fragment>
             <TextField
               select
+              label="content-type"
+              sx={{ width: 200 }}
               value={value?.contentType.value}
               onChange={handleContentTypeChange}
               disabled={disabled}
@@ -227,7 +229,14 @@ export default function BodyEditor({
   const renderToolbar = React.useCallback<RenderBodyToolbar>(
     ({ actions } = {}) => (
       <BodyEditorToolbar>
-        <TextField select value={activeTab} onChange={handleTabChange} disabled={disabled}>
+        <TextField
+          label="body"
+          sx={{ width: 200 }}
+          select
+          value={activeTab}
+          onChange={handleTabChange}
+          disabled={disabled}
+        >
           <MenuItem value="raw">raw</MenuItem>
           <MenuItem value="urlEncoded">x-www-form-urlencoded</MenuItem>
         </TextField>
diff --git a/packages/toolpad-app/src/toolpadDataSources/rest/client.tsx b/packages/toolpad-app/src/toolpadDataSources/rest/client.tsx
index 4f48dd0e8a2..f9d74135c10 100644
--- a/packages/toolpad-app/src/toolpadDataSources/rest/client.tsx
+++ b/packages/toolpad-app/src/toolpadDataSources/rest/client.tsx
@@ -14,7 +14,14 @@ import {
 import { Controller, useForm } from 'react-hook-form';
 import { TabContext, TabList } from '@mui/lab';
 import { ClientDataSource, ConnectionEditorProps, QueryEditorProps } from '../../types';
-import { FetchPrivateQuery, FetchQuery, FetchResult, RestConnectionParams, Body } from './types';
+import {
+  FetchPrivateQuery,
+  FetchQuery,
+  FetchResult,
+  RestConnectionParams,
+  Body,
+  ResponseType,
+} from './types';
 import { getAuthenticationHeaders, parseBaseUrl } from './shared';
 import BindableEditor, {
   RenderControlParams,
@@ -232,6 +239,16 @@ function QueryEditor({
     }));
   }, []);
 
+  const handleResponseTypeChange = React.useCallback(
+    (event: React.ChangeEvent<HTMLInputElement>) => {
+      setInput((existing) => ({
+        ...existing,
+        query: { ...existing.query, response: { kind: event.target.value } as ResponseType },
+      }));
+    },
+    [],
+  );
+
   const paramsEditorLiveValue = useEvaluateLiveBindingEntries({
     input: input.params,
     globalScope,
@@ -325,6 +342,7 @@ function QueryEditor({
                       <Tab label="URL query" value="urlQuery" />
                       <Tab label="Body" value="body" />
                       <Tab label="Headers" value="headers" />
+                      <Tab label="Response" value="response" />
                       <Tab label="Transform" value="transform" />
                     </TabList>
                   </Box>
@@ -352,6 +370,24 @@ function QueryEditor({
                       liveValue={liveHeaders}
                     />
                   </TabPanel>
+                  <TabPanel disableGutters value="response">
+                    <TextField
+                      select
+                      label="response type"
+                      sx={{ width: 200, mt: 1 }}
+                      value={input.query.response?.kind || 'json'}
+                      onChange={handleResponseTypeChange}
+                    >
+                      <MenuItem value="raw">raw</MenuItem>
+                      <MenuItem value="json">JSON</MenuItem>
+                      <MenuItem value="csv" disabled>
+                        🚧 CSV
+                      </MenuItem>
+                      <MenuItem value="xml" disabled>
+                        🚧 XML
+                      </MenuItem>
+                    </TextField>
+                  </TabPanel>
                   <TabPanel disableGutters value="transform">
                     <TransformInput
                       value={input.query.transform ?? 'return data;'}
diff --git a/packages/toolpad-app/src/toolpadDataSources/rest/server.ts b/packages/toolpad-app/src/toolpadDataSources/rest/server.ts
index 68b31607db3..8420f9e77e4 100644
--- a/packages/toolpad-app/src/toolpadDataSources/rest/server.ts
+++ b/packages/toolpad-app/src/toolpadDataSources/rest/server.ts
@@ -1,6 +1,5 @@
 import { BindableAttrEntries, BindableAttrValue, BindableAttrValues } from '@mui/toolpad-core';
 import fetch, { Headers, RequestInit, Response } from 'node-fetch';
-import MIMEType from 'whatwg-mimetype';
 import { withHarInstrumentation, createHarLog } from '../../server/har';
 import { ServerDataSource, ApiResult } from '../../types';
 import {
@@ -113,18 +112,14 @@ async function resolveBody(body: Body, boundValues: Record<string, string>) {
   }
 }
 
-function isJSON(mimeType: MIMEType): boolean {
-  // See https://mimesniff.spec.whatwg.org/#json-mime-type
-  const essence = `${mimeType.type}/${mimeType.subtype}`;
-  return (
-    essence === 'text/json' || essence === 'application/json' || mimeType.subtype.endsWith('+json')
-  );
-}
-
-async function readData(res: Response): Promise<any> {
-  const contentType = res.headers.get('content-type');
-  const mimeType = contentType ? new MIMEType(contentType) : null;
-  return mimeType && isJSON(mimeType) ? res.json() : res.text();
+async function readData(res: Response, fetchQuery: FetchQuery): Promise<any> {
+  if (!fetchQuery.response || fetchQuery.response?.kind === 'json') {
+    return res.json();
+  }
+  if (fetchQuery.response?.kind === 'raw') {
+    return res.text();
+  }
+  throw new Error(`Unsupported response type "${fetchQuery.response.kind}"`);
 }
 
 async function execBase(
@@ -183,7 +178,7 @@ async function execBase(
       throw new Error(`HTTP ${res.status}`);
     }
 
-    untransformedData = await readData(res);
+    untransformedData = await readData(res, fetchQuery);
     data = untransformedData;
 
     if (fetchQuery.transformEnabled && fetchQuery.transform) {
diff --git a/packages/toolpad-app/src/toolpadDataSources/rest/types.ts b/packages/toolpad-app/src/toolpadDataSources/rest/types.ts
index 6835c666010..c79fe2d2ff4 100644
--- a/packages/toolpad-app/src/toolpadDataSources/rest/types.ts
+++ b/packages/toolpad-app/src/toolpadDataSources/rest/types.ts
@@ -44,14 +44,61 @@ export type UrlEncodedBody = {
 
 export type Body = RawBody | UrlEncodedBody;
 
+export type RawResponseType = {
+  kind: 'raw';
+};
+
+export type JsonResponseType = {
+  kind: 'json';
+};
+
+export type CsvResponseType = {
+  kind: 'csv';
+  /**
+   * First row contains headers
+   */
+  headers: boolean;
+};
+
+export type XmlResponseType = {
+  kind: 'xml';
+};
+
+export type ResponseType = RawResponseType | JsonResponseType | CsvResponseType | XmlResponseType;
+
 export interface FetchQuery {
+  /**
+   * The URL of the rquest.
+   */
   readonly url: BindableAttrValue<string>;
+  /**
+   * The request method.
+   */
   readonly method: string;
+  /**
+   * Extra request headers.
+   */
   readonly headers: [string, BindableAttrValue<string>][];
+  /**
+   * Extra url query parameters.
+   */
   readonly searchParams?: [string, BindableAttrValue<string>][];
+  /**
+   * The request body.
+   */
   readonly body?: Body;
+  /**
+   * Run a custom transformer on the response.
+   */
   readonly transformEnabled?: boolean;
+  /**
+   * The custom transformer to run when enabled.
+   */
   readonly transform?: string;
+  /**
+   * How to parse the response.
+   */
+  readonly response?: ResponseType;
 }
 
 export type FetchParams = {
diff --git a/yarn.lock b/yarn.lock
index cabeb34f2e3..ce915907245 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2951,11 +2951,6 @@
   resolved "https://registry.yarnpkg.com/@types/webidl-conversions/-/webidl-conversions-7.0.0.tgz#2b8e60e33906459219aa587e9d1a612ae994cfe7"
   integrity sha512-xTE1E+YF4aWPJJeUzaZI5DRntlkY3+BCVJi0axFptnjGmAoWxkyREIh/XMrfxVLejwQxMCfDXdICo0VLxThrog==
 
-"@types/whatwg-mimetype@^2.1.1":
-  version "2.1.1"
-  resolved "https://registry.yarnpkg.com/@types/whatwg-mimetype/-/whatwg-mimetype-2.1.1.tgz#1b7b7aecaa3695209fd2f3a808dd5729973a52fa"
-  integrity sha512-ojnf89qt5AWnqsjyPqMLN8uVaxm5x23vxNQ1me6EPBOdJe1YYuIZUzg809MZUG8UU6HKhkr6ah4fi2WUvD0DFw==
-
 "@types/whatwg-url@^11.0.0":
   version "11.0.0"
   resolved "https://registry.yarnpkg.com/@types/whatwg-url/-/whatwg-url-11.0.0.tgz#5c42518a163a6867e14235223a1a558143bccbab"