Update v1.12.0

.env.example

@@ -1,5 +1,6 @@
 # SPDX-FileCopyrightText: 2021 - 2022 Netherlands eScience Center
 # SPDX-FileCopyrightText: 2021 - 2022 dv4all
+# SPDX-FileCopyrightText: 2022 Helmholtz Centre for Environmental Research (UFZ)
 #
 # SPDX-License-Identifier: CC-BY-4.0
 #
@@ -25,6 +26,10 @@ COMPOSE_PROJECT_NAME="rsd"
 # ---- PUBLIC ENV VARIABLES -------------
 
 # postgresql
+# consumed by services: backend
+POSTGRES_DB_HOST=database
+# consumed by services: backend
+POSTGRES_DB_HOST_PORT=5432
 # consumed by services: database, backend
 POSTGRES_DB=rsd-db
 # consumed by services: database

CITATION.cff

@@ -75,5 +75,5 @@ keywords:
   - Software Impact
   - Software Reuse
 license: Apache-2.0
-version: v1.11.1
-date-released: '2022-11-18'
+version: v1.12.0
+date-released: '2022-11-30'

data-generation/main.js

@@ -261,7 +261,9 @@ function generateSoftwareForSoftware(ids) {
 	return result;
 }
 
-function generateProjects(amount=50) {
+async function generateProjects(amount=50) {
+	const imageIds = await downloadAndGetImages(faker.image.cats, amount);
+
 	const result = [];
 
 	for (let index = 0; index < amount; index++) {
@@ -278,6 +280,7 @@ function generateProjects(amount=50) {
 			grant_id: faker.helpers.replaceSymbols('******'),
 			image_caption: faker.animal.cat(),
 			image_contain: !!faker.helpers.maybe(() => true, {probability: 0.5}),
+			image_id: imageIds[index] ?? null,
 			is_published: !!faker.helpers.maybe(() => true, {probability: 0.8}),
 		});
 	}
@@ -286,7 +289,7 @@ function generateProjects(amount=50) {
 }
 
 async function generateContributors(ids, amount=100) {
-	const base64Images = await downloadImagesAsBase64(faker.image.avatar, amount);
+	const imageIds = await downloadAndGetImages(faker.image.avatar, amount);
 
 	const result = [];
 
@@ -300,8 +303,7 @@ async function generateContributors(ids, amount=100) {
 			affiliation: faker.company.name(),
 			role: faker.name.jobTitle(),
 			orcid: faker.helpers.replaceSymbolWithNumber('####-####-####-####'),
-			avatar_data: base64Images[index],
-			avatar_mime_type: 'image/jpeg',
+			avatar_id: imageIds[index] ?? null,
 		});
 	}
 
@@ -317,23 +319,6 @@ async function generateTeamMembers(ids, amount=100) {
 	return result;
 }
 
-async function generateImagesForProjects(ids) {
-	const base64Images = await downloadImagesAsBase64(faker.image.cats, ids.length);
-
-	const result = [];
-
-	for (let index = 0; index < ids.length; index++) {
-		if (base64Images[index] === null) continue;
-		result.push({
-			project: ids[index],
-			data: base64Images[index],
-			mime_type: 'image/jpeg',
-		});
-	}
-
-	return result;
-}
-
 function generateUrlsForProjects(ids) {
 	const result = [];
 
@@ -352,7 +337,9 @@ function generateUrlsForProjects(ids) {
 	return result;
 }
 
-function generateOrganisations(amount=50) {
+async function generateOrganisations(amount=50) {
+	const imageIds = await downloadAndGetImages(faker.image.business, amount);
+
 	const result = [];
 
 	for (let index = 0; index < amount; index++) {
@@ -367,23 +354,7 @@ function generateOrganisations(amount=50) {
 			ror_id: faker.helpers.replaceSymbols('https://ror.org/********'),
 			website: faker.internet.url(),
 			is_tenant: !!faker.helpers.maybe(() => true, {probability: 0.3}),
-		});
-	}
-
-	return result;
-}
-
-async function generateLogosForOrganisations(ids) {
-	const base64Images = await downloadImagesAsBase64(faker.image.business, ids.length);
-
-	const result = [];
-
-	for (let index = 0; index < ids.length; index++) {
-		if (base64Images[index] === null) continue;
-		result.push({
-			organisation: ids[index],
-			data: base64Images[index],
-			mime_type: 'image/jpeg',
+			logo_id: imageIds[index] ?? null,
 		});
 	}
 
@@ -442,7 +413,11 @@ function createJWT() {
 }
 
 const token = createJWT();
-const headers = {'Content-Type': 'application/json', 'Authorization': 'bearer ' + token, 'Prefer': 'return=representation'}
+const headers = {
+		'Content-Type': 'application/json',
+		'Authorization': 'bearer ' + token,
+		'Prefer': 'return=representation,resolution=ignore-duplicates'
+	}
 const backendUrl = process.env.POSTGREST_URL || 'http://localhost/api/v1';
 
 async function postToBackend(endpoint, body) {
@@ -461,12 +436,13 @@ async function getFromBackend(endpoint) {
 	return response;
 }
 
-async function downloadImagesAsBase64(urlGenerator, amount) {
-	const imagePromises = [];
+// returns the IDs of the images after they have been posted to the database
+async function downloadAndGetImages(urlGenerator, amount) {
+	const imageAsBase64Promises = [];
 	const timeOuts = [];
 	for (let index = 0; index < amount; index++) {
 		const url = urlGenerator();
-		imagePromises.push(
+		imageAsBase64Promises.push(
 			Promise.race([
 				fetch(url)
 					.then(resp => {clearTimeout(timeOuts[index]); return resp.arrayBuffer()})
@@ -477,7 +453,16 @@ async function downloadImagesAsBase64(urlGenerator, amount) {
 			])
 		);
 	}
-	return await Promise.all(imagePromises);
+	const imagesAsBase64 = await Promise.all(imageAsBase64Promises);
+
+	const imagesWithoutNulls = imagesAsBase64
+		.filter(img => img !== null)
+		.map(base64 => {return {data: base64, mime_type: 'image/jpeg'}});
+
+	const resp = await postToBackend('/image?select=id', imagesWithoutNulls);
+	const idsAsObjects = await resp.json();
+	const ids = idsAsObjects.map(idAsObject => idAsObject.id);
+	return ids
 }
 
 let idsMentions, idsKeywords, idsResearchDomains;
@@ -508,24 +493,22 @@ const softwarePromise = postToBackend('/software', generateSofware())
 		postToBackend('/mention_for_software', generateMentionsForEntity(idsSoftware, idsMentions, 'software'));
 		postToBackend('/software_for_software', generateSoftwareForSoftware(idsSoftware));
 	});
-const projectPromise = postToBackend('/project', generateProjects())
+const projectPromise = postToBackend('/project', await generateProjects())
 	.then(resp => resp.json())
 	.then(async pjArray => {
 		idsProjects = pjArray.map(sw => sw['id']);
 		postToBackend('/team_member', await generateTeamMembers(idsProjects));
-		postToBackend('/image_for_project', await generateImagesForProjects(idsProjects));
 		postToBackend('/url_for_project', generateUrlsForProjects(idsProjects));
 		postToBackend('/keyword_for_project', generateKeywordsForEntity(idsProjects, idsKeywords, 'project'));
 		postToBackend('/output_for_project', generateMentionsForEntity(idsProjects, idsMentions, 'project'));
 		postToBackend('/impact_for_project', generateMentionsForEntity(idsProjects, idsMentions, 'project'));
 		postToBackend('/research_domain_for_project', generateResearchDomainsForProjects(idsProjects, idsResearchDomains));
 		postToBackend('/project_for_project', generateSoftwareForSoftware(idsProjects));
 	});
-const organisationPromise = postToBackend('/organisation', generateOrganisations())
+const organisationPromise = postToBackend('/organisation', await generateOrganisations())
 	.then(resp => resp.json())
 	.then(async orgArray => {
 		idsOrganisations = orgArray.map(org => org['id']);
-		postToBackend('/logo_for_organisation', await generateLogosForOrganisations(idsOrganisations));
 	});
 await postToBackend('/meta_pages', generateMetaPages()).then(() => console.log('meta pages done'));
 

database/002-create-image-table.sql

@@ -0,0 +1,74 @@
+-- SPDX-FileCopyrightText: 2022 Dusan Mijatovic (dv4all)
+-- SPDX-FileCopyrightText: 2022 Ewan Cahen (Netherlands eScience Center) <e.cahen@esciencecenter.nl>
+-- SPDX-FileCopyrightText: 2022 Netherlands eScience Center
+-- SPDX-FileCopyrightText: 2022 dv4all
+--
+-- SPDX-License-Identifier: Apache-2.0
+
+-- create extension pgcrypto to encrypt base64 content
+CREATE EXTENSION IF NOT EXISTS pgcrypto;
+
+CREATE TABLE image (
+	id VARCHAR(40) PRIMARY KEY,
+	data VARCHAR(2750000) NOT NULL,
+	mime_type VARCHAR(100) NOT NULL,
+	created_at TIMESTAMPTZ NOT NULL
+);
+
+CREATE FUNCTION sanitise_insert_image() RETURNS TRIGGER LANGUAGE plpgsql AS
+$$
+BEGIN
+-- create SHA-1 id based on provided data content
+	NEW.id = ENCODE(DIGEST(NEW.data, 'sha1'), 'hex');
+	NEW.created_at = LOCALTIMESTAMP;
+	return NEW;
+END
+$$;
+
+CREATE TRIGGER sanitise_insert_image BEFORE INSERT ON image FOR EACH ROW EXECUTE PROCEDURE sanitise_insert_image();
+
+CREATE FUNCTION sanitise_update_image() RETURNS TRIGGER LANGUAGE plpgsql AS
+$$
+BEGIN
+-- create SHA-1 id based on provided data content
+	NEW.id = ENCODE(DIGEST(NEW.data, 'sha1'), 'hex');
+	return NEW;
+END
+$$;
+
+CREATE TRIGGER sanitise_update_image BEFORE UPDATE ON image FOR EACH ROW EXECUTE PROCEDURE sanitise_update_image();
+
+
+-- ----------------------------------------
+-- RPC to get image by id => sha-1 of data
+-- ----------------------------------------
+
+CREATE FUNCTION get_image(uid VARCHAR(40)) RETURNS BYTEA STABLE LANGUAGE plpgsql AS
+$$
+DECLARE headers TEXT;
+DECLARE blob BYTEA;
+
+BEGIN
+	SELECT format(
+		'[{"Content-Type": "%s"},'
+		'{"Content-Disposition": "inline; filename=\"%s\""},'
+		'{"Cache-Control": "max-age=259200"}]',
+		mime_type,
+		uid)
+	FROM image WHERE id = uid INTO headers;
+
+	PERFORM set_config('response.headers', headers, TRUE);
+
+	SELECT decode(image.data, 'base64') FROM image WHERE id = uid INTO blob;
+
+	IF FOUND
+		THEN RETURN(blob);
+	ELSE RAISE SQLSTATE 'PT404'
+		USING
+			message = 'NOT FOUND',
+			detail = 'File not found',
+			hint = format('%s seems to be an invalid file id', image_id);
+	END IF;
+END
+$$;
+

database/003-create-relations-for-software.sql → database/004-create-relations-for-software.sql

@@ -75,8 +75,7 @@ CREATE TABLE contributor (
 	role VARCHAR(200),
 	orcid VARCHAR(19) CHECK (orcid ~ '^\d{4}-\d{4}-\d{4}-\d{3}[0-9X]$'),
 	position INTEGER,
-	avatar_data VARCHAR(2750000),
-	avatar_mime_type VARCHAR(100),
+	avatar_id VARCHAR(40) REFERENCES image(id),	
 	created_at TIMESTAMPTZ NOT NULL,
 	updated_at TIMESTAMPTZ NOT NULL
 );
@@ -106,37 +105,6 @@ $$;
 
 CREATE TRIGGER sanitise_update_contributor BEFORE UPDATE ON contributor FOR EACH ROW EXECUTE PROCEDURE sanitise_update_contributor();
 
-
-CREATE FUNCTION get_contributor_image(id UUID) RETURNS BYTEA STABLE LANGUAGE plpgsql AS
-$$
-DECLARE headers TEXT;
-DECLARE blob BYTEA;
-
-BEGIN
-	SELECT format(
-		'[{"Content-Type": "%s"},'
-		'{"Content-Disposition": "inline; filename=\"%s\""},'
-		'{"Cache-Control": "max-age=259200"}]',
-		contributor.avatar_mime_type,
-		contributor.id)
-	FROM contributor WHERE contributor.id = get_contributor_image.id INTO headers;
-
-	PERFORM set_config('response.headers', headers, TRUE);
-
-	SELECT decode(contributor.avatar_data, 'base64') FROM contributor WHERE contributor.id = get_contributor_image.id INTO blob;
-
-	IF FOUND
-		THEN RETURN(blob);
-	ELSE RAISE SQLSTATE 'PT404'
-		USING
-			message = 'NOT FOUND',
-			detail = 'File not found',
-			hint = format('%s seems to be an invalid file id', get_contributor_image.id);
-	END IF;
-END
-$$;
-
-
 CREATE TABLE testimonial (
 	id UUID DEFAULT gen_random_uuid() PRIMARY KEY,
 	software UUID REFERENCES software (id) NOT NULL,

database/004-create-project-table.sql → database/005-create-project-table.sql

@@ -16,6 +16,7 @@ CREATE TABLE project (
 	grant_id VARCHAR(50),
 	image_caption VARCHAR(500),
 	image_contain BOOLEAN DEFAULT FALSE NOT NULL,
+	image_id VARCHAR(40) REFERENCES image(id),
 	is_published BOOLEAN DEFAULT FALSE NOT NULL,
 	created_at TIMESTAMPTZ NOT NULL,
 	updated_at TIMESTAMPTZ NOT NULL
@@ -80,42 +81,3 @@ END
 $$;
 
 CREATE TRIGGER sanitise_update_url_for_project BEFORE UPDATE ON url_for_project FOR EACH ROW EXECUTE PROCEDURE sanitise_update_url_for_project();
-
-
-CREATE TABLE image_for_project (
-	project UUID REFERENCES project (id) PRIMARY KEY,
-	data VARCHAR(2750000) NOT NULL,
-	mime_type VARCHAR(100) NOT NULL
-);
-
-
-CREATE FUNCTION get_project_image(id UUID) RETURNS BYTEA STABLE LANGUAGE plpgsql AS
-$$
-DECLARE headers TEXT;
-DECLARE blob BYTEA;
-DECLARE project_slug VARCHAR;
-
-BEGIN
-	SELECT slug FROM project WHERE project.id = get_project_image.id INTO project_slug;
-	SELECT format(
-		'[{"Content-Type": "%s"},'
-		'{"Content-Disposition": "inline; filename=\"%s\""},'
-		'{"Cache-Control": "max-age=259200"}]',
-		mime_type,
-		project_slug)
-	FROM image_for_project WHERE project = id INTO headers;
-
-	PERFORM set_config('response.headers', headers, TRUE);
-
-	SELECT decode(image_for_project.data, 'base64') FROM image_for_project WHERE image_for_project.project = get_project_image.id INTO blob;
-
-	IF FOUND
-		THEN RETURN(blob);
-	ELSE RAISE SQLSTATE 'PT404'
-		USING
-			message = 'NOT FOUND',
-			detail = 'File not found',
-			hint = format('%s seems to be an invalid file id', get_project_image.id);
-	END IF;
-END
-$$;