Skip to content

Commit

Permalink
Remove old repositories deletion after migration ends
Browse files Browse the repository at this point in the history
  • Loading branch information
Zvonimir authored and Zvonimir committed Dec 19, 2024
1 parent a488c44 commit c082d52
Show file tree
Hide file tree
Showing 2 changed files with 1 addition and 179 deletions.
128 changes: 0 additions & 128 deletions v8-data-migration/triple-store-utils.js
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import { setTimeout } from 'timers/promises';
import axios from 'axios';
import graphdb from 'graphdb';
import {
OT_BLAZEGRAPH,
OT_FUSEKI,
Expand Down Expand Up @@ -29,8 +28,6 @@ import {
} from './validation.js';
import logger from './logger.js';

const { server, http } = graphdb;

export function getTripleStoreData(tripleStoreConfig) {
// Validation
validateTripleStoreConfig(tripleStoreConfig);
Expand Down Expand Up @@ -255,59 +252,6 @@ export async function ensureConnections(tripleStoreRepositories, tripleStoreImpl
await Promise.all(ensureConnectionPromises);
}

export async function repositoryExists(
tripleStoreRepositories,
repository,
tripleStoreImplementation,
) {
// Validation
validateTripleStoreRepositories(tripleStoreRepositories);
validateRepository(repository);
validateTripleStoreImplementation(tripleStoreImplementation);

const { url, name } = tripleStoreRepositories[repository];
switch (tripleStoreImplementation) {
case OT_BLAZEGRAPH:
try {
await axios.get(`${url}/blazegraph/namespace/${name}/properties`, {
params: {
'describe-each-named-graph': 'false',
},
headers: {
Accept: 'application/ld+json',
},
});
return true;
} catch (error) {
if (error.response && error.response.status === 404) {
// Expected error: GraphDB is up but has not created node0 repository
// Ot-node will create repo in initialization
return false;
}
logger.error(
`Error while getting ${repository} repositories. Error: ${error.message}`,
);

return false;
}
case OT_FUSEKI:
try {
const response = await axios.get(`${url}/$/datasets`);

return response.data.datasets.filter((dataset) => dataset['ds.name'] === `/${name}`)
.length;
} catch (error) {
logger.error(
`Error while getting ${repository} repositories. Error: ${error.message}`,
);

return false;
}
default:
throw new Error(`Invalid triple store repository name: ${repository}`);
}
}

// blazegraph only
function hasUnicodeCodePoints(input) {
const unicodeRegex = /(?<!\\)\\U([a-fA-F0-9]{8})/g;
Expand Down Expand Up @@ -526,78 +470,6 @@ export async function getAssertionFromV6TripleStore(
};
}

export async function deleteRepository(
tripleStoreRepositories,
tripleStoreImplementation,
repository,
) {
// Validation
validateTripleStoreRepositories(tripleStoreRepositories);
validateTripleStoreImplementation(tripleStoreImplementation);
validateRepository(repository);

const { url, name } = tripleStoreRepositories[repository];
logger.info(
`Deleting ${tripleStoreImplementation} triple store repository: ${repository} with name: ${name}`,
);

switch (tripleStoreImplementation) {
case OT_BLAZEGRAPH: {
if (
await repositoryExists(
tripleStoreRepositories,
repository,
tripleStoreImplementation,
)
) {
await axios
.delete(`${url}/blazegraph/namespace/${name}`, {})
.catch((e) =>
logger.error(
`Error while deleting ${tripleStoreImplementation} triple store repository: ${repository} with name: ${name}. Error: ${e.message}`,
),
);
}
break;
}
case OT_GRAPHDB: {
const serverConfig = new server.ServerClientConfig(url)
.setTimeout(40000)
.setHeaders({
Accept: http.RDFMimeType.N_QUADS,
})
.setKeepAlive(true);
const s = new server.GraphDBServerClient(serverConfig);
s.deleteRepository(name).catch((e) =>
logger.error(
`Error while deleting ${tripleStoreImplementation} triple store repository: ${repository} with name: ${name}. Error: ${e.message}`,
),
);
break;
}
case OT_FUSEKI: {
if (
await repositoryExists(
tripleStoreRepositories,
repository,
tripleStoreImplementation,
)
) {
await axios
.delete(`${url}/$/datasets/${name}`, {})
.catch((e) =>
logger.error(
`Error while deleting ${tripleStoreImplementation} triple store repository: ${repository} with name: ${name}. Error: ${e.message}`,
),
);
}
break;
}
default:
logger.error(`Unknown triple store implementation: ${tripleStoreImplementation}`);
}
}

export function processContent(str) {
return str
.split('\n')
Expand Down
52 changes: 1 addition & 51 deletions v8-data-migration/v8-data-migration.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import { createRequire } from 'module';
import dotenv from 'dotenv';
import axios from 'axios';
import {
DKG_REPOSITORY,
BATCH_SIZE,
ENV_PATH,
BLOCKCHAINS,
Expand All @@ -22,10 +21,8 @@ import {
getTokenIdsToProcessCount,
} from './v8-data-migration-utils.js';
import {
repositoryExists,
getAssertionFromV6TripleStore,
insertAssertionsIntoV8UnifiedRepository,
deleteRepository,
getTripleStoreData,
initializeRepositories,
initializeContexts,
Expand All @@ -39,7 +36,6 @@ import {
validateTokenId,
validateTripleStoreRepositories,
validateTripleStoreImplementation,
validateTripleStoreConfig,
validateBatchData,
} from './validation.js';
import logger from './logger.js';
Expand Down Expand Up @@ -149,48 +145,6 @@ async function processAndInsertNewerAssertions(
}
}

async function deleteV6TripleStoreRepositories(tripleStoreConfig, tripleStoreImplementation) {
// Validation
validateTripleStoreConfig(tripleStoreConfig);
validateTripleStoreImplementation(tripleStoreImplementation);

// Delete old repositories
const oldTripleStoreRepositories =
tripleStoreConfig.implementation[tripleStoreImplementation].config.repositories;
for (const repository in oldTripleStoreRepositories) {
if (repository === DKG_REPOSITORY) {
continue;
}

logger.info(`Deleting repository: ${repository}`);

let deleted = false;
while (!deleted) {
// eslint-disable-next-line no-await-in-loop
await deleteRepository(
oldTripleStoreRepositories,
tripleStoreImplementation,
repository,
);
// eslint-disable-next-line no-await-in-loop
if (
await repositoryExists(
oldTripleStoreRepositories,
repository,
tripleStoreImplementation,
)
) {
logger.error(
`Something went wrong. Repository ${repository} still exists after deletion. Retrying deletion...`,
);
} else {
deleted = true;
}
}
logger.info(`Repository ${repository} deleted successfully`);
}
}

async function processAndInsertAssertions(
v6Assertions,
tripleStoreRepositories,
Expand Down Expand Up @@ -354,7 +308,7 @@ async function main() {

if (!blockchainName) {
throw new Error(
`Blockchain ${blockchain} not found. Make sure you have the correct blockchain data and .env file.`,
`Blockchain ${blockchain} not found. Make sure you have the correct blockchain ID and correct NODE_ENV in .env file.`,
);
}

Expand Down Expand Up @@ -506,10 +460,6 @@ async function main() {
logger.timeEnd('BLOCKCHAIN ASSERRTION GET AND TRIPLE STORE INSERT');
}

logger.time('DELETE V6 TRIPLE STORE REPOSITORIES');
await deleteV6TripleStoreRepositories(tripleStoreConfig, tripleStoreImplementation);
logger.timeEnd('DELETE V6 TRIPLE STORE REPOSITORIES');

// REMOTE
markMigrationAsSuccessfull();
// REMOTE END
Expand Down

0 comments on commit c082d52

Please sign in to comment.