Add openapi code generation script and api versioning (#235)

## Problem

We need a way to generate code from updated openapi specs

## Solution

- Copy & modify overall approach used in python repository for similar
purpose
- Create new `codegen/` directory to be the home for code generation
stuff
  - Use git submodules to get information from `apis` repo
- Build using the `typescript-fetch` generator in the v7 openapi
generator cli. This is the same as we have used in the past, to keep the
diff somewhat small.
- Build data and control planes separately and copy outputs into
`src/pinecone-generated-ts-fetch` directory in source. This has been the
traditional home for these generated files.
- Since we're no longer mashing specs together ahead of code generation,
I need to adjust import paths to find either
`pinecone-generated-ts-fetch/control` or
`pinecone-generated-ts-fetch/data` as appropriate
- In addition to the default generated output, write a constant called
`X_PINECONE_API_VERSION` in each generated module. This will be used
when building header configuration with `X-Pinecone-Api-Version` for
each request.

## Type of Change

- [x] Infrastructure change (CI configs, etc)

## Test Plan

`npm run generate:openapi`

.gitignore

@@ -2,6 +2,7 @@
 .env
 
 # Generated files
+build/
 coverage/
 dist/
 docs/
@@ -13,4 +14,4 @@ ts-compilation-test/lib/
 # Clutter
 .vscode
 .DS_Store
-scratch
+scratch

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "codegen/apis"]
+	path = codegen/apis
+	url = git@github.com:pinecone-io/apis.git

.prettierignore

@@ -1,3 +1,3 @@
 dist
 node_modules
-src/pinecone-generated-ts-fetch
+codegen

codegen/build-oas.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+
+set -eux -o pipefail
+
+version=$1 # e.g. 2024-07
+modules=("control" "data")
+
+destination="src/pinecone-generated-ts-fetch"
+build_dir="build"
+
+update_apis_repo() {
+	echo "Updating apis repo"
+	pushd codegen/apis
+		git fetch
+		git pull
+		just build
+	popd
+}
+
+verify_spec_version() {
+	local version=$1
+	echo "Verifying spec version $version exists in apis repo"
+	if [ -z "$version" ]; then
+		echo "Version is required"
+		exit 1
+	fi
+
+	verify_directory_exists "codegen/apis/_build/${version}"
+}
+
+verify_file_exists() {
+	local filename=$1
+	if [ ! -f "$filename" ]; then
+		echo "File does not exist at $filename"
+		exit 1
+	fi
+}
+
+verify_directory_exists() {
+	local directory=$1
+	if [ ! -d "$directory" ]; then
+		echo "Directory does not exist at $directory"
+		exit 1
+	fi
+}
+
+generate_client() {
+	local module_name=$1
+
+	oas_file="codegen/apis/_build/${version}/${module_name}_${version}.oas.yaml"
+
+	verify_file_exists $oas_file
+
+	# Cleanup previous build files
+	echo "Cleaning up previous build files"
+	rm -rf "${build_dir}"
+
+	# Generate client module
+	docker run --rm -v $(pwd):/workspace openapitools/openapi-generator-cli:v7.0.0 generate \
+		--input-spec "/workspace/$oas_file" \
+		--generator-name typescript-fetch \
+		--output "/workspace/${build_dir}"
+
+	# Copy the generated module to the correct location
+	rm -rf "${destination}/${module_name}"
+	mkdir -p "${destination}/${module_name}"
+	cp -r ${build_dir}/* "${destination}/${module_name}"
+
+	echo "export const X_PINECONE_API_VERSION = '${version}';" > ${destination}/${module_name}/api_version.ts
+	echo "export * from './api_version';" >> ${destination}/${module_name}/index.ts
+}
+
+update_apis_repo
+verify_spec_version $version
+
+rm -rf "${destination}"
+mkdir -p "${destination}"
+
+for module in "${modules[@]}"; do
+	generate_client $module
+done

package.json

@@ -16,6 +16,7 @@
     "build": "rm -rf dist/ && tsc",
     "version": "jq --null-input --arg version $npm_package_version '{\"name\": \"@pinecone-database/pinecone\", \"version\": $version}' > src/version.json",
     "docs:build": "typedoc --plugin ./assets/docs-theme.mjs",
+    "generate:openapi": "./codegen/build-oas.sh 2024-07 && npm run build && npm run format",
     "format": "prettier --write .",
     "lint": "eslint src/ --ext .ts",
     "repl": "npm run build && node utils/replInit.ts",

src/control/__tests__/configureIndex.test.ts

@@ -1,9 +1,9 @@
 import { configureIndex } from '../configureIndex';
-import { ManageIndexesApi } from '../../pinecone-generated-ts-fetch';
+import { ManageIndexesApi } from '../../pinecone-generated-ts-fetch/control';
 import type {
   ConfigureIndexOperationRequest,
   IndexModel,
-} from '../../pinecone-generated-ts-fetch';
+} from '../../pinecone-generated-ts-fetch/control';
 
 describe('configureIndex', () => {
   test('calls the openapi configure endpoint', async () => {

src/control/__tests__/configureIndex.validation.test.ts

@@ -6,13 +6,13 @@
 
 import { configureIndex } from '../configureIndex';
 import { PineconeArgumentError } from '../../errors';
-import { ManageIndexesApi } from '../../pinecone-generated-ts-fetch';
+import { ManageIndexesApi } from '../../pinecone-generated-ts-fetch/control';
 
 describe('configureIndex argument validations', () => {
   let MIA: ManageIndexesApi;
   beforeEach(() => {
     MIA = { configureIndex: jest.fn() };
-    jest.mock('../../pinecone-generated-ts-fetch', () => ({
+    jest.mock('../../pinecone-generated-ts-fetch/control', () => ({
       IndexOperationsApi: MIA,
     }));
   });

src/control/__tests__/createCollection.test.ts

@@ -1,11 +1,11 @@
 import { createCollection } from '../createCollection';
 import { PineconeArgumentError } from '../../errors';
-import { ManageIndexesApi } from '../../pinecone-generated-ts-fetch';
+import { ManageIndexesApi } from '../../pinecone-generated-ts-fetch/control';
 import type {
   CollectionModel,
   CreateCollectionOperationRequest,
   IndexList,
-} from '../../pinecone-generated-ts-fetch';
+} from '../../pinecone-generated-ts-fetch/control';
 
 const setOpenAPIResponse = (fakeCreateCollectionResponse) => {
   const fakeCreateCollection: (

src/control/__tests__/createIndex.test.ts

@@ -1,10 +1,10 @@
 import { createIndex } from '../createIndex';
-import { ManageIndexesApi } from '../../pinecone-generated-ts-fetch';
+import { ManageIndexesApi } from '../../pinecone-generated-ts-fetch/control';
 import type {
   CreateIndexOperationRequest,
   DescribeIndexRequest,
   IndexModel,
-} from '../../pinecone-generated-ts-fetch';
+} from '../../pinecone-generated-ts-fetch/control';
 
 // describeIndexResponse can either be a single response, or an array of responses for testing polling scenarios
 const setupCreateIndexResponse = (

src/control/__tests__/createIndex.validation.test.ts

@@ -6,13 +6,13 @@
 
 import { createIndex } from '../createIndex';
 import { PineconeArgumentError } from '../../errors';
-import { ManageIndexesApi } from '../../pinecone-generated-ts-fetch';
+import { ManageIndexesApi } from '../../pinecone-generated-ts-fetch/control';
 
 describe('createIndex argument validations', () => {
   let MIA: ManageIndexesApi;
   beforeEach(() => {
     MIA = { createIndex: jest.fn() };
-    jest.mock('../../pinecone-generated-ts-fetch', () => ({
+    jest.mock('../../pinecone-generated-ts-fetch/control', () => ({
       ManageIndexesApi: MIA,
     }));
   });

src/control/__tests__/deleteCollection.test.ts

@@ -1,18 +1,17 @@
 import { deleteCollection } from '../deleteCollection';
 import { PineconeArgumentError } from '../../errors';
-import { ManageIndexesApi } from '../../pinecone-generated-ts-fetch';
+import { ManageIndexesApi } from '../../pinecone-generated-ts-fetch/control';
 import type {
   DeleteCollectionRequest,
   CollectionList,
-} from '../../pinecone-generated-ts-fetch';
+} from '../../pinecone-generated-ts-fetch/control';
 
 const setupMocks = (
   deleteResponse,
   listCollectionResponse = () => Promise.resolve([])
 ) => {
-  const fakeDeleteCollection: (
-    req: DeleteCollectionRequest
-  ) => Promise<string> = jest.fn().mockImplementation(deleteResponse);
+  const fakeDeleteCollection: (req: DeleteCollectionRequest) => Promise<void> =
+    jest.fn().mockImplementation(deleteResponse);
   const fakeListCollections: () => Promise<CollectionList> = jest
     .fn()
     .mockImplementation(listCollectionResponse);

src/control/__tests__/describeCollection.test.ts

@@ -1,11 +1,11 @@
 import { describeCollection } from '../describeCollection';
 import { PineconeArgumentError } from '../../errors';
-import { ManageIndexesApi } from '../../pinecone-generated-ts-fetch';
+import { ManageIndexesApi } from '../../pinecone-generated-ts-fetch/control';
 import type {
   DescribeCollectionRequest,
   CollectionList,
   CollectionModel,
-} from '../../pinecone-generated-ts-fetch';
+} from '../../pinecone-generated-ts-fetch/control';
 
 const setupMocks = (
   describeResponse,

src/control/__tests__/indexOperationsBuilder.test.ts

@@ -1,8 +1,8 @@
 import { indexOperationsBuilder } from '../indexOperationsBuilder';
-import { Configuration } from '../../pinecone-generated-ts-fetch';
+import { Configuration } from '../../pinecone-generated-ts-fetch/control';
 
-jest.mock('../../pinecone-generated-ts-fetch', () => ({
-  ...jest.requireActual('../../pinecone-generated-ts-fetch'),
+jest.mock('../../pinecone-generated-ts-fetch/control', () => ({
+  ...jest.requireActual('../../pinecone-generated-ts-fetch/control'),
   Configuration: jest.fn(),
 }));
 

src/control/configureIndex.ts

@@ -2,7 +2,7 @@ import {
   ManageIndexesApi,
   IndexModel,
   ConfigureIndexRequestSpecPod,
-} from '../pinecone-generated-ts-fetch';
+} from '../pinecone-generated-ts-fetch/control';
 import { PineconeArgumentError } from '../errors';
 import { buildValidator } from '../validator';
 import type { IndexName } from './types';

src/control/createCollection.ts

@@ -2,7 +2,7 @@ import {
   CollectionModel,
   CreateCollectionRequest,
   ManageIndexesApi,
-} from '../pinecone-generated-ts-fetch';
+} from '../pinecone-generated-ts-fetch/control';
 import { buildConfigValidator } from '../validator';
 import { CollectionNameSchema, IndexNameSchema } from './types';
 import { Type } from '@sinclair/typebox';

src/control/createIndex.ts

@@ -3,7 +3,7 @@ import {
   IndexModel,
   ManageIndexesApi,
   CreateIndexRequestMetricEnum,
-} from '../pinecone-generated-ts-fetch';
+} from '../pinecone-generated-ts-fetch/control';
 import { buildConfigValidator } from '../validator';
 import { debugLog } from '../utils';
 import { handleApiError } from '../errors';

src/control/deleteCollection.ts

@@ -1,4 +1,4 @@
-import { ManageIndexesApi } from '../pinecone-generated-ts-fetch';
+import { ManageIndexesApi } from '../pinecone-generated-ts-fetch/control';
 import { buildConfigValidator } from '../validator';
 import { CollectionNameSchema } from './types';
 import type { CollectionName } from './types';

src/control/deleteIndex.ts

@@ -1,4 +1,4 @@
-import { ManageIndexesApi } from '../pinecone-generated-ts-fetch';
+import { ManageIndexesApi } from '../pinecone-generated-ts-fetch/control';
 import { buildConfigValidator } from '../validator';
 import { IndexName, IndexNameSchema } from './types';
 

src/control/describeCollection.ts

@@ -1,7 +1,7 @@
 import {
   ManageIndexesApi,
   CollectionModel,
-} from '../pinecone-generated-ts-fetch';
+} from '../pinecone-generated-ts-fetch/control';
 import { buildConfigValidator } from '../validator';
 import { CollectionNameSchema } from './types';
 import type { CollectionName } from './types';

src/control/describeIndex.ts

@@ -1,5 +1,8 @@
 import { buildConfigValidator } from '../validator';
-import { IndexModel, ManageIndexesApi } from '../pinecone-generated-ts-fetch';
+import {
+  IndexModel,
+  ManageIndexesApi,
+} from '../pinecone-generated-ts-fetch/control';
 import { IndexNameSchema } from './types';
 import type { IndexName } from './types';
 

src/control/indexOperationsBuilder.ts

@@ -1,7 +1,8 @@
 import {
   ManageIndexesApi,
   Configuration,
-} from '../pinecone-generated-ts-fetch';
+  X_PINECONE_API_VERSION,
+} from '../pinecone-generated-ts-fetch/control';
 import {
   queryParamsStringify,
   buildUserAgent,
@@ -10,7 +11,7 @@ import {
 } from '../utils';
 import { middleware } from '../utils/middleware';
 import type { PineconeConfiguration } from '../data/types';
-import type { ConfigurationParameters as IndexOperationsApiConfigurationParameters } from '../pinecone-generated-ts-fetch';
+import type { ConfigurationParameters as IndexOperationsApiConfigurationParameters } from '../pinecone-generated-ts-fetch/control';
 
 export const indexOperationsBuilder = (
   config: PineconeConfiguration
@@ -25,6 +26,7 @@ export const indexOperationsBuilder = (
     queryParamsStringify,
     headers: {
       'User-Agent': buildUserAgent(config),
+      'X-Pinecone-Api-Version': X_PINECONE_API_VERSION,
       ...headers,
     },
     fetchApi: getFetch(config),

src/control/listCollections.ts

@@ -1,7 +1,7 @@
 import {
   ManageIndexesApi,
   CollectionList,
-} from '../pinecone-generated-ts-fetch';
+} from '../pinecone-generated-ts-fetch/control';
 
 export const listCollections = (api: ManageIndexesApi) => {
   return async (): Promise<CollectionList> => {

src/control/listIndexes.ts

@@ -1,4 +1,7 @@
-import { ManageIndexesApi, IndexList } from '../pinecone-generated-ts-fetch';
+import {
+  ManageIndexesApi,
+  IndexList,
+} from '../pinecone-generated-ts-fetch/control';
 
 export const listIndexes = (api: ManageIndexesApi) => {
   return async (): Promise<IndexList> => {

src/data/__tests__/dataOperationsProvider.test.ts

@@ -1,9 +1,9 @@
 import { DataOperationsProvider } from '../dataOperationsProvider';
 import { IndexHostSingleton } from '../indexHostSingleton';
-import { Configuration } from '../../pinecone-generated-ts-fetch';
+import { Configuration } from '../../pinecone-generated-ts-fetch/data';
 
-jest.mock('../../pinecone-generated-ts-fetch', () => ({
-  ...jest.requireActual('../../pinecone-generated-ts-fetch'),
+jest.mock('../../pinecone-generated-ts-fetch/data', () => ({
+  ...jest.requireActual('../../pinecone-generated-ts-fetch/data'),
   Configuration: jest.fn(),
 }));
 

src/data/__tests__/deleteOne.test.ts

@@ -2,7 +2,7 @@ import { deleteOne } from '../deleteOne';
 import type {
   DeleteOperationRequest,
   DataPlaneApi,
-} from '../../pinecone-generated-ts-fetch';
+} from '../../pinecone-generated-ts-fetch/data';
 import { DataOperationsProvider } from '../dataOperationsProvider';
 
 const setupDeleteResponse = (response, isSuccess) => {

src/data/__tests__/describeIndexStats.test.ts

@@ -1,7 +1,7 @@
 import { describeIndexStats } from '../describeIndexStats';
-import { DataPlaneApi } from '../../pinecone-generated-ts-fetch';
+import { DataPlaneApi } from '../../pinecone-generated-ts-fetch/data';
 import { DataOperationsProvider } from '../dataOperationsProvider';
-import type { DescribeIndexStatsOperationRequest } from '../../pinecone-generated-ts-fetch';
+import type { DescribeIndexStatsOperationRequest } from '../../pinecone-generated-ts-fetch/data';
 
 const setupResponse = (response, isSuccess) => {
   const fakeDescribeIndexStats: (

src/data/__tests__/fetch.test.ts

@@ -1,10 +1,10 @@
 import { FetchCommand } from '../fetch';
-import { DataPlaneApi } from '../../pinecone-generated-ts-fetch';
+import { DataPlaneApi } from '../../pinecone-generated-ts-fetch/data';
 import { DataOperationsProvider } from '../dataOperationsProvider';
 import type {
   FetchRequest,
   FetchResponse,
-} from '../../pinecone-generated-ts-fetch';
+} from '../../pinecone-generated-ts-fetch/data';
 
 const setupResponse = (response, isSuccess) => {
   const fakeFetch: (req: FetchRequest) => Promise<FetchResponse> = jest

src/data/__tests__/indexHostSingleton.test.ts

@@ -173,11 +173,6 @@ describe('IndexHostSingleton', () => {
     expect(mockDescribeIndex).toHaveBeenNthCalledWith(1, 'index-1');
     expect(mockDescribeIndex).toHaveBeenNthCalledWith(2, 'index-2');
 
-    console.log(
-      'mockIndexOperationsBuilder',
-      JSON.stringify(mockIndexOperationsBuilder.mock)
-    );
-
     expect(mockIndexOperationsBuilder).toHaveBeenCalledTimes(2);
     expect(mockIndexOperationsBuilder).toHaveBeenNthCalledWith(
       1,