Enhance Airbyte Integration with Connection Checks and Conditional Features

docker-compose.yml

@@ -100,6 +100,7 @@ services:
       - AIRBYTE_PASSWORD=${AIRBYTE_PASSWORD:-password}
       - AIRBYTE_CLIENT_ID=${AIRBYTE_CLIENT_ID}
       - AIRBYTE_CLIENT_SECRET=${AIRBYTE_CLIENT_SECRET}
+      - NEXT_PUBLIC_IS_AIRBYTE_ENABLED=false
       - GCS_BUCKET_NAME=${GCS_BUCKET_NAME}
       - GCS_BUCKET_LOCATION=${GCS_BUCKET_LOCATION}
       - STRIPE_FREE_PLAN_PRICE_ID=price_1P0zlRDxQ9GZKzvoYUAzWMSv

webapp/.env.example

@@ -36,6 +36,7 @@ AIRBYTE_USERNAME=airbyte
 AIRBYTE_PASSWORD=password
 AIRBYTE_CLIENT_ID=
 AIRBYTE_CLIENT_SECRET=
+NEXT_PUBLIC_IS_AIRBYTE_ENABLED=false
 NEXT_PUBLIC_GCS_BUCKET_NAME_PRIVATE=agentcloud-bucket-dev
 NEXT_PUBLIC_GCS_BUCKET_NAME=agentcloud-public-dev
 GCS_BUCKET_LOCATION=australia-southeast1

webapp/src/api.ts

@@ -704,6 +704,17 @@ export function getDatasourceSchema(body, dispatch, errorCallback, router) {
 	);
 }//@TEST
 
+export function checkAirbyteConnection(body, dispatch, errorCallback, router) {
+	return ApiCall(
+		`/${body.resourceSlug}/airbyte/connection`,
+		'GET',
+		null,
+		dispatch,
+		errorCallback,
+		router
+	);
+}
+
 //Temp datasource stuff
 export function uploadDatasourceFileTemp(body, dispatch, errorCallback, router) {
 	return ApiCall(

webapp/src/components/DatasourceTable.tsx

@@ -13,6 +13,7 @@ import DatasourceStatusIndicator from 'components/DatasourceStatusIndicator'; //
 import DevBadge from 'components/DevBadge';
 import { useAccountContext } from 'context/account';
 import { useNotificationContext } from 'context/notifications';
+import cn from 'lib/cn';
 import Link from 'next/link';
 import { useRouter } from 'next/router';
 import { Fragment, useReducer, useState } from 'react';
@@ -22,10 +23,12 @@ import submittingReducer from 'utils/submittingreducer';
 
 export default function DatasourceTable({
 	datasources,
-	fetchDatasources
+	fetchDatasources,
+	isAirbyteEnabled
 }: {
 	datasources: any[];
 	fetchDatasources?: any;
+	isAirbyteEnabled?: boolean;
 }) {
 	const [notificationContext, refreshNotificationContext]: any = useNotificationContext();
 	const [accountContext]: any = useAccountContext();
@@ -37,6 +40,12 @@ export default function DatasourceTable({
 	const [deletingMap, setDeletingMap] = useState({});
 	const [confirmClose, setConfirmClose] = useState(false);
 
+	const goToDatasourcePage = (id: string) => {
+		if (isAirbyteEnabled) {
+			router.push(`/${resourceSlug}/datasource/${id}`);
+		}
+	};
+
 	async function deleteDatasource(datasourceId) {
 		setDeleting({ [datasourceId]: true });
 		try {
@@ -152,13 +161,14 @@ export default function DatasourceTable({
 							return (
 								<tr
 									key={datasource._id}
-									className={`cursor-pointer hover:bg-gray-50 dark:hover:bg-slate-700 dark:text-white dark:!border-slate-700 transition-all opacity-1 duration-700 ${deletingMap[datasource._id] ? 'bg-red-400' : 'cursor-pointer hover:bg-gray-50'}`}
+									className={cn(
+										`hover:bg-gray-50 dark:hover:bg-slate-700 dark:text-white dark:!border-slate-700 transition-all opacity-1 duration-700 ${deletingMap[datasource._id] ? 'bg-red-400' : ' hover:bg-gray-50'}`,
+										{ 'cursor-pointer': isAirbyteEnabled }
+									)}
 									style={{ borderColor: deletingMap[datasource._id] ? 'red' : '' }}
+									onClick={() => goToDatasourcePage(datasource._id)}
 								>
-									<td
-										className='px-6 py-3 whitespace-nowrap flex items-center'
-										onClick={() => router.push(`/${resourceSlug}/datasource/${datasource._id}`)}
-									>
+									<td className='px-6 py-3 whitespace-nowrap flex items-center'>
 										<img
 											src={`https://connectors.airbyte.com/files/metadata/airbyte/source-${datasource.sourceType}/latest/icon.svg`}
 											className='w-6 me-1.5'
@@ -168,34 +178,22 @@ export default function DatasourceTable({
 										</span>
 										<DevBadge value={datasource?._id} />
 									</td>
-									<td
-										className='px-6 py-3 whitespace-nowrap'
-										onClick={() => router.push(`/${resourceSlug}/datasource/${datasource._id}`)}
-									>
+									<td className='px-6 py-3 whitespace-nowrap'>
 										<div className='flex items-center'>
 											<div className='text-sm font-medium text-gray-900 dark:text-white'>
 												{datasource.name}
 											</div>
 										</div>
 									</td>
-									<td
-										className='px-6 py-3 whitespace-nowrap'
-										onClick={() => router.push(`/${resourceSlug}/datasource/${datasource._id}`)}
-									>
+									<td className='px-6 py-3 whitespace-nowrap'>
 										<DatasourceStatusIndicator datasource={datasource} />
 									</td>
-									<td
-										className='px-6 py-3 whitespace-nowrap'
-										onClick={() => router.push(`/${resourceSlug}/datasource/${datasource._id}`)}
-									>
+									<td className='px-6 py-3 whitespace-nowrap'>
 										<span className='px-2 inline-flex text-sm leading-5 rounded-full capitalize'>
 											{datasource?.connectionSettings?.scheduleType || '-'}
 										</span>
 									</td>
-									<td
-										className='px-6 py-3 whitespace-nowrap'
-										onClick={() => router.push(`/${resourceSlug}/datasource/${datasource._id}`)}
-									>
+									<td className='px-6 py-3 whitespace-nowrap'>
 										<div className='text-sm text-gray-900 dark:text-white' suppressHydrationWarning>
 											{datasource.sourceType === 'file'
 												? 'N/A'
@@ -204,10 +202,7 @@ export default function DatasourceTable({
 													: 'Never'}
 										</div>
 									</td>
-									<td
-										className='px-6 py-3 whitespace-nowrap'
-										onClick={() => router.push(`/${resourceSlug}/datasource/${datasource._id}`)}
-									>
+									<td className='px-6 py-3 whitespace-nowrap'>
 										<span
 											suppressHydrationWarning
 											className='text-sm text-gray-900 dark:text-white'
@@ -217,7 +212,8 @@ export default function DatasourceTable({
 									</td>
 									<td className='px-6 py-5 whitespace-nowrap text-right text-sm font-medium flex justify-end space-x-5 items-center'>
 										<button
-											onClick={() => {
+											onClick={e => {
+												e.stopPropagation();
 												// if (datasource.status !== DatasourceStatus.READY) {
 												// 	setConfirmClose(datasource._id);
 												// } else {
@@ -227,7 +223,8 @@ export default function DatasourceTable({
 											disabled={
 												syncing[datasource._id] ||
 												deleting[datasource._id] ||
-												datasource.status !== DatasourceStatus.READY
+												datasource.status !== DatasourceStatus.READY ||
+												!isAirbyteEnabled
 											}
 											className='rounded-md disabled:bg-slate-400 bg-indigo-600 px-2 -my-1 py-1 text-sm font-semibold text-white shadow-sm hover:bg-indigo-500 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-indigo-600 dark:text-white'
 										>

webapp/src/controllers/airbyte.ts

@@ -14,6 +14,7 @@ import {
 } from 'db/datasource';
 import { addNotification } from 'db/notification';
 import debug from 'debug';
+import * as airbyteSetup from 'lib/airbyte/setup';
 import posthog from 'lib/posthog';
 import { chainValidations } from 'lib/utils/validationutils';
 import toObjectId from 'misc/toobjectid';
@@ -339,3 +340,19 @@ export async function handleSuccessfulEmbeddingWebhook(req, res, next) {
 
 	return dynamicResponse(req, res, 200, {});
 }
+
+export async function checkAirbyteConnection(req, res, next) {
+	const status = await airbyteSetup.checkAirbyteStatus();
+
+	let isEnabled = process.env.NEXT_PUBLIC_IS_AIRBYTE_ENABLED === 'true';
+
+	if (status && !isEnabled) {
+		isEnabled = await airbyteSetup.init();
+	}
+
+	if (!status) {
+		process.env.NEXT_PUBLIC_IS_AIRBYTE_ENABLED = 'false';
+	}
+
+	return dynamicResponse(req, res, 201, { isEnabled });
+}

webapp/src/lib/airbyte/setup.ts

@@ -1,19 +1,16 @@
+import * as dns from 'node:dns';
+import * as util from 'node:util';
+
 import debug from 'debug';
 import dotenv from 'dotenv';
 import fs from 'fs';
-import getGoogleCredentials from 'misc/getgooglecredentials';
 import fetch from 'node-fetch'; // Ensure node-fetch is installed or use a compatible fetch API
-import path from 'path';
-const { GoogleAuth } = require('google-auth-library');
-import * as dns from 'node:dns';
-import * as util from 'node:util';
 const lookup = util.promisify(dns.lookup);
 
 import * as process from 'node:process';
 
 import getAirbyteApi, { AirbyteApiType, getAirbyteAuthToken } from 'airbyte/api';
 import SecretProviderFactory from 'lib/secret';
-import { AIRBYTE_OAUTH_PROVIDERS } from 'struct/oauth';
 
 import getAirbyteInternalApi from './internal';
 
@@ -86,6 +83,30 @@ async function deleteDestination(destinationId: string) {
 	);
 }
 
+export async function checkAirbyteStatus() {
+	try {
+		const response = await fetch(`${process.env.AIRBYTE_API_URL}/api/v1/health`, {
+			method: 'GET'
+		});
+		if (response?.status !== 200) {
+			return false;
+		}
+		const workspaces = await fetch(`${process.env.AIRBYTE_API_URL}/api/v1/workspaces`, {
+			method: 'GET',
+			headers: {
+				accept: 'application/json',
+				authorization: `Bearer ${await getAirbyteAuthToken()}`
+			}
+		});
+		if (response?.status !== 200) {
+			return false;
+		}
+		return true;
+	} catch (error) {
+		console.log('error', error);
+	}
+}
+
 async function getDestinationConfiguration(provider: string) {
 	if (provider === 'rabbitmq') {
 		log(`RabbitMQ HOST: ${process.env.AIRBYTE_RABBITMQ_HOST}`);
@@ -267,7 +288,10 @@ export async function init() {
 		// for (let provider in AIRBYTE_OAUTH_PROVIDERS) {
 		// 	overrideOauthCreds(airbyteAdminWorkspaceId, provider.toLowerCase());
 		// }
+		process.env.NEXT_PUBLIC_IS_AIRBYTE_ENABLED = 'true';
+		return true;
 	} catch (error) {
+		process.env.NEXT_PUBLIC_IS_AIRBYTE_ENABLED = 'false';
 		logerror('Error during Airbyte configuration:', error);
 	}
 }

webapp/src/pages/[resourceSlug]/datasource/[datasourceId]/index.tsx

@@ -35,6 +35,7 @@ export default function Datasource(props) {
 	const router = useRouter();
 	const { resourceSlug, datasourceId } = router.query;
 	const [state, dispatch] = useState(props);
+	const [airbyteState, setAirbyteState] = useState(null);
 	const [jobsList, setJobsList] = useState(null);
 	const [tab, setTab] = useState(0);
 	const [schemaDiscoverState, setSchemaDiscoverState] = useState(null);
@@ -79,6 +80,7 @@ export default function Datasource(props) {
 			setError,
 			router
 		);
+		await API.checkAirbyteConnection({ resourceSlug }, setAirbyteState, setError, router);
 	}
 
 	async function fetchJobsList() {
@@ -264,7 +266,11 @@ export default function Datasource(props) {
 							</button>
 							<button
 								onClick={e => updateStreams(e, true)}
-								disabled={submitting['updateStreamssync'] || submitting['updateStreams']}
+								disabled={
+									submitting['updateStreamssync'] ||
+									submitting['updateStreams'] ||
+									!airbyteState?.isEnabled
+								}
 								type='submit'
 								className='rounded-md disabled:bg-slate-400 bg-indigo-600 px-3 py-2 text-sm font-semibold text-white shadow-sm hover:bg-indigo-500 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-indigo-600'
 							>

webapp/src/pages/[resourceSlug]/datasources.tsx

@@ -4,26 +4,21 @@ import CreateDatasourceForm from 'components/CreateDatasourceForm';
 import CreateDatasourceModal from 'components/CreateDatasourceModal';
 import DatasourceFileTable from 'components/DatasourceFileTable';
 import DatasourceTable from 'components/DatasourceTable';
-import ErrorAlert from 'components/ErrorAlert';
-import NewButtonSection from 'components/NewButtonSection';
 import PageTitleWithNewButton from 'components/PageTitleWithNewButton';
 import Spinner from 'components/Spinner';
 import { useAccountContext } from 'context/account';
 import { useSocketContext } from 'context/socket';
 import Head from 'next/head';
-import Link from 'next/link';
 import { useRouter } from 'next/router';
 import React, { useEffect, useState } from 'react';
-import { pricingMatrix } from 'struct/billing';
-import { DatasourceStatus } from 'struct/datasource';
+import { toast } from 'react-toastify';
 import { NotificationType, WebhookType } from 'struct/notification';
 
 export default function Datasources(props) {
 	const [accountContext, refreshAccountContext]: any = useAccountContext();
 	const [, notificationTrigger]: any = useSocketContext();
 
 	const { account, teamName } = accountContext as any;
-	const { stripePlan } = account?.stripe || {};
 	const router = useRouter();
 	const { resourceSlug } = router.query;
 	const [state, dispatch] = useState(props);
@@ -32,9 +27,12 @@ export default function Datasources(props) {
 	const filteredDatasources = datasources?.filter(x => !x.hidden);
 	const [open, setOpen] = useState(false);
 	const [spec, setSpec] = useState(null);
+	const [airbyteState, setAirbyteState] = useState(null);
+	const [airbyteLoading, setAirbyteLoading] = useState(false);
 
 	async function fetchDatasources(silent = false) {
 		await API.getDatasources({ resourceSlug }, dispatch, setError, router);
+		await API.checkAirbyteConnection({ resourceSlug }, setAirbyteState, setError, router);
 	}
 
 	useEffect(() => {
@@ -100,12 +98,47 @@ export default function Datasources(props) {
 
 			<span className='py-8 h-1'></span>
 
-			<PageTitleWithNewButton
-				list={filteredDatasources}
-				title='Data Connections'
-				buttonText='New Connection'
-				onClick={() => setOpen(true)}
-			/>
+			{airbyteState?.isEnabled && (
+				<PageTitleWithNewButton
+					list={filteredDatasources}
+					title='Data Connections'
+					buttonText='New Connection'
+					onClick={async () => {
+						if (airbyteState?.isEnabled) {
+							setOpen(true);
+						} else {
+							await API.checkAirbyteConnection({ resourceSlug }, setAirbyteState, setError, router);
+						}
+					}}
+				/>
+			)}
+
+			{!airbyteState?.isEnabled && (
+				<button
+					type='button'
+					onClick={async () => {
+						setAirbyteLoading(true);
+						await API.checkAirbyteConnection(
+							{ resourceSlug },
+							({ isEnabled }) => {
+								if (isEnabled) {
+									toast.success('Airbyte is enabled');
+								} else {
+									toast.error('Failed to enable Airbyte');
+								}
+								setAirbyteState({ isEnabled });
+							},
+							setError,
+							router
+						);
+						setAirbyteLoading(false);
+					}}
+					className='inline-flex items-center rounded-md bg-indigo-500 px-3 py-2 text-sm font-semibold text-white shadow-sm hover:bg-indigo-500 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-indigo-600 disabled:bg-gray-300 disabled:text-gray-700 disabled:cursor-not-allowed ml-auto mb-2'
+					disabled={airbyteLoading}
+				>
+					{airbyteLoading ? 'Enabling Airbyte...' : 'Enable Airbyte'}
+				</button>
+			)}
 
 			<CreateDatasourceModal
 				open={open}
@@ -120,6 +153,7 @@ export default function Datasources(props) {
 			<DatasourceTable
 				datasources={filteredDatasources.filter(d => d?.sourceType !== 'file')}
 				fetchDatasources={fetchDatasources}
+				isAirbyteEnabled={airbyteState?.isEnabled}
 			/>
 		</>
 	);

webapp/src/router.ts

@@ -330,6 +330,7 @@ export default function router(server, app) {
 	teamRouter.get('/airbyte/specification', airbyteProxyController.specificationJson);
 	teamRouter.get('/airbyte/schema', airbyteProxyController.discoverSchemaApi);
 	teamRouter.get('/airbyte/jobs', airbyteProxyController.listJobsApi);
+	teamRouter.get('/airbyte/connection', airbyteProxyController.checkAirbyteConnection);
 
 	//sessions
 	teamRouter.get(