From 523419df11ccff68b5f99c2d2b37d0c7985a796c Mon Sep 17 00:00:00 2001 From: Alicia Sykes Date: Wed, 20 Mar 2024 20:43:51 +0000 Subject: [PATCH 1/7] Adds timeout functionality --- api/_common/middleware.js | 61 +++++++++++++++++++++++++++++++++------ 1 file changed, 52 insertions(+), 9 deletions(-) diff --git a/api/_common/middleware.js b/api/_common/middleware.js index a80e16ec..b463c67f 100644 --- a/api/_common/middleware.js +++ b/api/_common/middleware.js @@ -2,13 +2,46 @@ const normalizeUrl = (url) => { return url.startsWith('http') ? url : `https://${url}`; }; + +// If present, set a shorter timeout for API requests +const TIMEOUT = parseInt(process.env.API_TIMEOUT_LIMIT, 10) || 60000; + +// If present, set CORS allowed origins for responses +const ALLOWED_ORIGINS = process.env.API_CORS_ORIGIN || '*'; + +// Set the platform currently being used +let PLATFORM = 'NETLIFY'; +if (process.env.PLATFORM) { PLATFORM = process.env.PLATFORM.toUpperCase(); } +else if (process.env.VERCEL) { PLATFORM = 'VERCEL'; } +else if (process.env.WC_SERVER) { PLATFORM = 'NODE'; } + +// Define the headers to be returned with each response const headers = { - 'Access-Control-Allow-Origin': process.env.API_CORS_ORIGIN || '*', + 'Access-Control-Allow-Origin': ALLOWED_ORIGINS, 'Access-Control-Allow-Credentials': true, 'Content-Type': 'application/json;charset=UTF-8', }; + +// A middleware function used by all API routes on all platforms const commonMiddleware = (handler) => { + + // Create a timeout promise, to throw an error if a request takes too long + const createTimeoutPromise = (timeoutMs) => { + return new Promise((_, reject) => { + setTimeout(() => { + const howToResolve = 'You can re-trigger this request, by clicking "Retry"\n' + + 'If you\'re running your own instance of Web Check, then you can ' + + 'resolve this issue, by increasing the timeout limit in the ' + + '`API_TIMEOUT_LIMIT` environmental variable to a higher value (in milliseconds). \n\n' + + `The public instance currently has a lower timeout of ${timeoutMs}ms ` + + 'in order to keep running costs affordable, so that Web Check can ' + + 'remain freely available for everyone.'; + reject(new Error(`Request timed-out after ${timeoutMs} ms.\n\n${howToResolve}`)); + }, timeoutMs); + }); + }; + // Vercel const vercelHandler = async (request, response) => { const queryParams = request.query || {}; @@ -21,7 +54,12 @@ const commonMiddleware = (handler) => { const url = normalizeUrl(rawUrl); try { - const handlerResponse = await handler(url, request); + // Race the handler against the timeout + const handlerResponse = await Promise.race([ + handler(url, request), + createTimeoutPromise(TIMEOUT) + ]); + if (handlerResponse.body && handlerResponse.statusCode) { response.status(handlerResponse.statusCode).json(handlerResponse.body); } else { @@ -30,7 +68,11 @@ const commonMiddleware = (handler) => { ); } } catch (error) { - response.status(500).json({ error: error.message }); + let errorCode = 500; + if (error.message.includes('timed-out')) { + errorCode = 408; + } + response.status(errorCode).json({ error: error.message }); } }; @@ -51,7 +93,12 @@ const commonMiddleware = (handler) => { const url = normalizeUrl(rawUrl); try { - const handlerResponse = await handler(url, event, context); + // Race the handler against the timeout + const handlerResponse = await Promise.race([ + handler(url, event, context), + createTimeoutPromise(TIMEOUT) + ]); + if (handlerResponse.body && handlerResponse.statusCode) { callback(null, handlerResponse); } else { @@ -71,11 +118,7 @@ const commonMiddleware = (handler) => { }; // The format of the handlers varies between platforms - // E.g. Netlify + AWS expect Lambda functions, but Vercel or Node needs standard handler - const platformEnv = (process.env.PLATFORM || '').toUpperCase(); // Has user set platform manually? - - const nativeMode = (['VERCEL', 'NODE'].includes(platformEnv) || process.env.VERCEL || process.env.WC_SERVER); - + const nativeMode = (['VERCEL', 'NODE'].includes(PLATFORM)); return nativeMode ? vercelHandler : netlifyHandler; }; From c764bbfcd458e47de9165be750a4eb22e384a091 Mon Sep 17 00:00:00 2001 From: Alicia Sykes Date: Wed, 20 Mar 2024 20:44:37 +0000 Subject: [PATCH 2/7] If response is a timeout, set type to timeout --- src/hooks/motherOfAllHooks.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/hooks/motherOfAllHooks.ts b/src/hooks/motherOfAllHooks.ts index e8071313..76603e5e 100644 --- a/src/hooks/motherOfAllHooks.ts +++ b/src/hooks/motherOfAllHooks.ts @@ -35,16 +35,21 @@ const useMotherOfAllHooks = (params: UseIpAddressProps(); // Fire off the HTTP fetch request, then set results and update loading / error state + const doTheFetch = () => { return fetchRequest() .then((res: any) => { if (!res) { // No response :( - updateLoadingJobs(jobId, 'error', res.error || 'No response', reset); + updateLoadingJobs(jobId, 'error', 'No response', reset); } else if (res.error) { // Response returned an error message - updateLoadingJobs(jobId, 'error', res.error, reset); + if (res.error.includes("timed-out")) { // Specific handling for timeout errors + updateLoadingJobs(jobId, 'timed-out', res.error, reset); + } else { + updateLoadingJobs(jobId, 'error', res.error, reset); + } } else if (res.skipped) { // Response returned a skipped message updateLoadingJobs(jobId, 'skipped', res.skipped, reset); - } else { // Yay, everything went to plan :) + } else { // Yay, everything went to plan :) setResult(res); updateLoadingJobs(jobId, 'success', '', undefined, res); } From 7a8e694abcfcb0af4e3f6839d5b5c1c50db1dcf7 Mon Sep 17 00:00:00 2001 From: Alicia Sykes Date: Wed, 20 Mar 2024 20:45:13 +0000 Subject: [PATCH 3/7] Logging for timeout info, and adds key to map --- src/pages/Results.tsx | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/pages/Results.tsx b/src/pages/Results.tsx index 31902870..a74c4a81 100644 --- a/src/pages/Results.tsx +++ b/src/pages/Results.tsx @@ -208,12 +208,24 @@ const Results = (): JSX.Element => { console.log( `%cFetch Error - ${job}%c\n\n${timeString}%c The ${job} job failed ` +`after ${timeTaken}ms, with the following error:%c\n${error}`, - `background: ${colors.danger}; padding: 4px 8px; font-size: 16px;`, + `background: ${colors.danger}; color:${colors.background}; padding: 4px 8px; font-size: 16px;`, `font-weight: bold; color: ${colors.danger};`, `color: ${colors.danger};`, `color: ${colors.warning};`, ); } + + if (newState === 'timed-out') { + console.log( + `%cFetch Timeout - ${job}%c\n\n${timeString}%c The ${job} job timed out ` + +`after ${timeTaken}ms, with the following error:%c\n${error}`, + `background: ${colors.info}; color:${colors.background}; padding: 4px 8px; font-size: 16px;`, + `font-weight: bold; color: ${colors.info};`, + `color: ${colors.info};`, + `color: ${colors.warning};`, + ); + } + return newJobs; }); }); @@ -225,8 +237,9 @@ const Results = (): JSX.Element => { .then(data => resolve(data)) .catch(error => resolve( { error: `Failed to get a valid response 😒\n` - + `This is likely due the target not exposing the required data, ` - + `or limitations in how Netlify executes lambda functions, such as the 10-sec timeout.\n\n` + + 'This is likely due the target not exposing the required data, ' + + 'or limitations in imposed by the infrastructure this instance ' + + 'of Web Check is running on.\n\n' + `Error info:\n${error}`} )); }); @@ -910,7 +923,7 @@ const Results = (): JSX.Element => { && title.toLowerCase().includes(searchTerm.toLowerCase()) && (result && !result.error); return show ? ( - + Date: Wed, 20 Mar 2024 21:19:58 +0000 Subject: [PATCH 4/7] Update timeout message for sitemap --- api/sitemap.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/api/sitemap.js b/api/sitemap.js index 8d593e4e..4d6c01d3 100644 --- a/api/sitemap.js +++ b/api/sitemap.js @@ -6,15 +6,17 @@ const xml2js = require('xml2js'); const handler = async (url) => { let sitemapUrl = `${url}/sitemap.xml`; + const hardTimeOut = 5000; + try { // Try to fetch sitemap directly let sitemapRes; try { - sitemapRes = await axios.get(sitemapUrl, { timeout: 5000 }); + sitemapRes = await axios.get(sitemapUrl, { timeout: hardTimeOut }); } catch (error) { if (error.response && error.response.status === 404) { // If sitemap not found, try to fetch it from robots.txt - const robotsRes = await axios.get(`${url}/robots.txt`, { timeout: 5000 }); + const robotsRes = await axios.get(`${url}/robots.txt`, { timeout: hardTimeOut }); const robotsTxt = robotsRes.data.split('\n'); for (let line of robotsTxt) { @@ -28,7 +30,7 @@ const handler = async (url) => { return { skipped: 'No sitemap found' }; } - sitemapRes = await axios.get(sitemapUrl, { timeout: 5000 }); + sitemapRes = await axios.get(sitemapUrl, { timeout: hardTimeOut }); } else { throw error; // If other error, throw it } @@ -40,7 +42,7 @@ const handler = async (url) => { return sitemap; } catch (error) { if (error.code === 'ECONNABORTED') { - return { error: 'Request timed out after 5000ms' }; + return { error: `Request timed-out after ${hardTimeOut}ms` }; } else { return { error: error.message }; } From d43a05a0ed5f1f37b9899ce4d97488d5786f7a63 Mon Sep 17 00:00:00 2001 From: Alicia Sykes Date: Wed, 20 Mar 2024 21:20:23 +0000 Subject: [PATCH 5/7] Show timeout seperatley --- src/components/misc/ProgressBar.tsx | 92 +++++++++++++++++------------ 1 file changed, 55 insertions(+), 37 deletions(-) diff --git a/src/components/misc/ProgressBar.tsx b/src/components/misc/ProgressBar.tsx index b6d2c16f..51578bd7 100644 --- a/src/components/misc/ProgressBar.tsx +++ b/src/components/misc/ProgressBar.tsx @@ -224,6 +224,52 @@ const jobNames = [ 'carbon', ] as const; +interface JobListItemProps { + job: LoadingJob; + showJobDocs: (name: string) => void; + showErrorModal: (name: string, state: LoadingState, timeTaken: number | undefined, error: string, isInfo?: boolean) => void; + barColors: Record; +} + +const getStatusEmoji = (state: LoadingState): string => { + switch (state) { + case 'success': + return 'βœ…'; + case 'loading': + return 'πŸ”„'; + case 'error': + return '❌'; + case 'timed-out': + return '⏸️'; + case 'skipped': + return '⏭️'; + default: + return '❓'; + } +}; + +const JobListItem: React.FC = ({ job, showJobDocs, showErrorModal, barColors }) => { + const { name, state, timeTaken, retry, error } = job; + const actionButton = retry && state !== 'success' && state !== 'loading' ? + ↻ Retry : null; + + const showModalButton = error && ['error', 'timed-out', 'skipped'].includes(state) && + showErrorModal(name, state, timeTaken, error, state === 'skipped')}> + {state === 'timed-out' ? 'β–  Show Timeout Reason' : 'β–  Show Error'} + ; + + return ( +
  • + showJobDocs(name)}>{getStatusEmoji(state)} {name} + ({state}). + {timeTaken && state !== 'loading' ? ` Took ${timeTaken} ms` : ''} + {actionButton} + {showModalButton} +
    • + ); +}; + + export const initialJobs = jobNames.map((job: string) => { return { name: job, @@ -239,9 +285,9 @@ export const calculateLoadingStatePercentages = (loadingJobs: LoadingJob[]): Rec const stateCount: Record = { 'success': 0, 'loading': 0, - 'skipped': 0, - 'error': 0, 'timed-out': 0, + 'error': 0, + 'skipped': 0, }; // Count the number of each state @@ -253,9 +299,9 @@ export const calculateLoadingStatePercentages = (loadingJobs: LoadingJob[]): Rec const statePercentage: Record = { 'success': (stateCount['success'] / totalJobs) * 100, 'loading': (stateCount['loading'] / totalJobs) * 100, - 'skipped': (stateCount['skipped'] / totalJobs) * 100, - 'error': (stateCount['error'] / totalJobs) * 100, 'timed-out': (stateCount['timed-out'] / totalJobs) * 100, + 'error': (stateCount['error'] / totalJobs) * 100, + 'skipped': (stateCount['skipped'] / totalJobs) * 100, }; return statePercentage; @@ -353,26 +399,9 @@ const ProgressLoader = (props: { loadStatus: LoadingJob[], showModal: (err: Reac const barColors: Record = { 'success': isDone ? makeBarColor(colors.primary) : makeBarColor(colors.success), 'loading': makeBarColor(colors.info), - 'skipped': makeBarColor(colors.warning), 'error': makeBarColor(colors.danger), - 'timed-out': makeBarColor(colors.neutral), - }; - - const getStatusEmoji = (state: LoadingState): string => { - switch (state) { - case 'success': - return 'βœ…'; - case 'loading': - return 'πŸ”„'; - case 'skipped': - return '⏭️'; - case 'error': - return '❌'; - case 'timed-out': - return '⏸️'; - default: - return '❓'; - } + 'timed-out': makeBarColor(colors.warning), + 'skipped': makeBarColor(colors.neutral), }; const showErrorModal = (name: string, state: LoadingState, timeTaken: number | undefined, error: string, isInfo?: boolean) => { @@ -416,20 +445,9 @@ const ProgressLoader = (props: { loadStatus: LoadingJob[], showModal: (err: Reac
    Show Details
      - { - loadStatus.map(({ name, state, timeTaken, retry, error }: LoadingJob) => { - return ( -
    • - props.showJobDocs(name)}>{getStatusEmoji(state)} {name} - ({state}). - {(timeTaken && state !== 'loading') ? ` Took ${timeTaken} ms` : '' } - { (retry && state !== 'success' && state !== 'loading') && ↻ Retry } - { (error && state === 'error') && showErrorModal(name, state, timeTaken, error)}>β–  Show Error } - { (error && state === 'skipped') && showErrorModal(name, state, timeTaken, error, true)}>β–  Show Reason } -
      • - ); - }) - } + {loadStatus.map((job: LoadingJob) => ( + + ))}
    { loadStatus.filter((val: LoadingJob) => val.state === 'error').length > 0 &&

    From 2304aaf17cf712d3cf031a1563439d025f82acd9 Mon Sep 17 00:00:00 2001 From: Alicia Sykes Date: Wed, 20 Mar 2024 21:20:41 +0000 Subject: [PATCH 6/7] Adds unique keys to itteratbles, remove react warn --- src/components/Form/Row.tsx | 4 ++-- src/components/Results/BlockLists.tsx | 6 ++++-- src/components/Results/DnsServer.tsx | 4 ++-- src/components/Results/ServerStatus.tsx | 2 +- src/components/Results/TlsCipherSuites.tsx | 2 +- src/components/Results/TlsClientSupport.tsx | 11 +++++++++-- src/components/Results/TlsIssueAnalysis.tsx | 8 +++++++- src/components/misc/ErrorBoundary.tsx | 1 + 8 files changed, 27 insertions(+), 11 deletions(-) diff --git a/src/components/Form/Row.tsx b/src/components/Form/Row.tsx index 96a38a5a..15724fe8 100644 --- a/src/components/Form/Row.tsx +++ b/src/components/Form/Row.tsx @@ -159,7 +159,7 @@ export const ExpandableRow = (props: RowProps) => { { rowList?.map((row: RowProps, index: number) => { return ( - {row.lbl} + {row.lbl} copyToClipboard(row.val)}> {formatValue(row.val)} @@ -199,7 +199,7 @@ const Row = (props: RowProps) => { if (children) return {children}; return ( - { lbl && {lbl} } + { lbl && {lbl} } copyToClipboard(val)}> {formatValue(val)} diff --git a/src/components/Results/BlockLists.tsx b/src/components/Results/BlockLists.tsx index ef107685..b6cf5726 100644 --- a/src/components/Results/BlockLists.tsx +++ b/src/components/Results/BlockLists.tsx @@ -6,11 +6,13 @@ const BlockListsCard = (props: {data: any, title: string, actionButtons: any }): const blockLists = props.data.blocklists; return ( - { blockLists.map((blocklist: any) => ( + { blockLists.map((blocklist: any, blockIndex: number) => ( + val={blocklist.isBlocked ? '❌ Blocked' : 'βœ… Not Blocked'} + key={`blocklist-${blockIndex}-${blocklist.serverIp}`} + /> ))} ); diff --git a/src/components/Results/DnsServer.tsx b/src/components/Results/DnsServer.tsx index 683de735..2a3395e1 100644 --- a/src/components/Results/DnsServer.tsx +++ b/src/components/Results/DnsServer.tsx @@ -18,12 +18,12 @@ const DnsServerCard = (props: {data: any, title: string, actionButtons: any }): return ( {dnsSecurity.dns.map((dns: any, index: number) => { - return (<> + return (

    { dnsSecurity.dns.length > 1 && DNS Server #{index+1} } { dns.hostname && } - ); +
    ); })} {dnsSecurity.dns.length > 0 && ( * DoH Support is determined by the DNS server's response to a DoH query. diff --git a/src/components/Results/ServerStatus.tsx b/src/components/Results/ServerStatus.tsx index f23b1934..dc7ba011 100644 --- a/src/components/Results/ServerStatus.tsx +++ b/src/components/Results/ServerStatus.tsx @@ -13,7 +13,7 @@ span.val { const ServerStatusCard = (props: { data: any, title: string, actionButtons: any }): JSX.Element => { const serverStatus = props.data; return ( - + Is Up? { serverStatus.isUp ? βœ… Online : ❌ Offline} diff --git a/src/components/Results/TlsCipherSuites.tsx b/src/components/Results/TlsCipherSuites.tsx index 59fe13a9..fcf42663 100644 --- a/src/components/Results/TlsCipherSuites.tsx +++ b/src/components/Results/TlsCipherSuites.tsx @@ -52,7 +52,7 @@ const TlsCard = (props: {data: any, title: string, actionButtons: any }): JSX.El { cipherSuites.length && cipherSuites.map((cipherSuite: any, index: number) => { return ( - + ); })} { !cipherSuites.length && ( diff --git a/src/components/Results/TlsClientSupport.tsx b/src/components/Results/TlsClientSupport.tsx index 8f1c6828..7682fa9f 100644 --- a/src/components/Results/TlsClientSupport.tsx +++ b/src/components/Results/TlsClientSupport.tsx @@ -59,8 +59,15 @@ const TlsCard = (props: {data: any, title: string, actionButtons: any }): JSX.El const scanId = props.data?.id; return ( - {clientSupport.map((support: any) => { - return () + {clientSupport.map((support: any, index: number) => { + return ( + + ) })} { !clientSupport.length && (
    diff --git a/src/components/Results/TlsIssueAnalysis.tsx b/src/components/Results/TlsIssueAnalysis.tsx index 0926344d..ddee3e40 100644 --- a/src/components/Results/TlsIssueAnalysis.tsx +++ b/src/components/Results/TlsIssueAnalysis.tsx @@ -99,7 +99,13 @@ const TlsCard = (props: {data: any, title: string, actionButtons: any }): JSX.El { tlsResults.length > 0 && tlsResults.map((row: any, index: number) => { return ( - + ); })} diff --git a/src/components/misc/ErrorBoundary.tsx b/src/components/misc/ErrorBoundary.tsx index b404756f..f032159f 100644 --- a/src/components/misc/ErrorBoundary.tsx +++ b/src/components/misc/ErrorBoundary.tsx @@ -7,6 +7,7 @@ import colors from 'styles/colors'; interface Props { children: ReactNode; title?: string; + key?: string; } interface State { From ff656967293639ef73f900a792c37521c0322848 Mon Sep 17 00:00:00 2001 From: Alicia Sykes Date: Wed, 20 Mar 2024 21:43:04 +0000 Subject: [PATCH 7/7] Bump to V 1.1.2 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 86af95ef..2efe12c1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "web-check", - "version": "1.1.0", + "version": "1.1.2", "private": false, "description": "All-in-one OSINT tool for analyzing any website", "repository": "github:lissy93/web-check",