From 4c83e9b6b4e37308ca3b4d77d7de22fe946d1daa Mon Sep 17 00:00:00 2001 From: Ewan Cahen Date: Wed, 26 Oct 2022 15:39:19 +0200 Subject: [PATCH 01/16] fix: adapt DataCite scraper to schema version 4.4 --- .../scraper/doi/DataciteMentionRepository.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/DataciteMentionRepository.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/DataciteMentionRepository.java index 16298eec6..e063ec613 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/DataciteMentionRepository.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/DataciteMentionRepository.java @@ -55,20 +55,33 @@ public class DataciteMentionRepository implements MentionRepository { private static final Map dataciteTextTypeMap; static { -// https://schema.datacite.org/meta/kernel-4.3/ +// https://schema.datacite.org/meta/kernel-4.4/ dataciteTypeMap = new HashMap<>(); dataciteTypeMap.put("Audiovisual", MentionType.presentation); + dataciteTypeMap.put("Book", MentionType.book); + dataciteTypeMap.put("BookChapter", MentionType.bookSection); dataciteTypeMap.put("Collection", MentionType.other); + dataciteTypeMap.put("ComputationalNotebook", MentionType.computerProgram); + dataciteTypeMap.put("ConferencePaper", MentionType.conferencePaper); + dataciteTypeMap.put("ConferenceProceeding", MentionType.conferencePaper); dataciteTypeMap.put("DataPaper", MentionType.other); dataciteTypeMap.put("Dataset", MentionType.dataset); + dataciteTypeMap.put("Dissertation", MentionType.thesis); dataciteTypeMap.put("Event", MentionType.other); dataciteTypeMap.put("Image", MentionType.other); dataciteTypeMap.put("InteractiveResource", MentionType.other); + dataciteTypeMap.put("Journal", MentionType.other); + dataciteTypeMap.put("JournalArticle", MentionType.journalArticle); dataciteTypeMap.put("Model", MentionType.other); + dataciteTypeMap.put("OutputManagementPlan", MentionType.other); + dataciteTypeMap.put("PeerReview", MentionType.other); + dataciteTypeMap.put("Preprint", MentionType.other); dataciteTypeMap.put("PhysicalObject", MentionType.other); + dataciteTypeMap.put("Report", MentionType.report); dataciteTypeMap.put("Service", MentionType.other); dataciteTypeMap.put("Software", MentionType.computerProgram); dataciteTypeMap.put("Sound", MentionType.other); + dataciteTypeMap.put("Standard", MentionType.other); // dataciteTypeMap.put("Text", MentionType.other); dataciteTypeMap.put("Workflow", MentionType.other); dataciteTypeMap.put("Other", MentionType.other); From 9c1c023bd45c6ba5a7be8fc566b3888964435427 Mon Sep 17 00:00:00 2001 From: Ewan Cahen Date: Wed, 26 Oct 2022 15:41:31 +0200 Subject: [PATCH 02/16] fix: add workshop as mention type --- database/008-create-mention-table.sql | 1 + frontend/components/mention/config.ts | 12 ++++++++++-- .../rsd/scraper/doi/DataciteMentionRepository.java | 2 +- .../esciencecenter/rsd/scraper/doi/MentionType.java | 1 + 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/database/008-create-mention-table.sql b/database/008-create-mention-table.sql index 27a2f2b71..2dc16adb6 100644 --- a/database/008-create-mention-table.sql +++ b/database/008-create-mention-table.sql @@ -22,6 +22,7 @@ CREATE TYPE mention_type AS ENUM ( 'thesis', 'videoRecording', 'webpage', + 'workshop', 'other' ); diff --git a/frontend/components/mention/config.ts b/frontend/components/mention/config.ts index 696ddb092..71c27034c 100644 --- a/frontend/components/mention/config.ts +++ b/frontend/components/mention/config.ts @@ -1,4 +1,6 @@ // SPDX-FileCopyrightText: 2022 Dusan Mijatovic (dv4all) +// SPDX-FileCopyrightText: 2022 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 Netherlands eScience Center // SPDX-FileCopyrightText: 2022 dv4all // // SPDX-License-Identifier: Apache-2.0 @@ -190,7 +192,7 @@ export const mentionType = { interview: { key: 'interview', plural: 'Interviews', - singular: 'Interviews', + singular: 'Interview', manual: true }, journalArticle: { @@ -220,7 +222,7 @@ export const mentionType = { report: { key: 'report', plural: 'Reports', - singular: 'Reports', + singular: 'Report', manual: true }, thesis: { @@ -241,6 +243,12 @@ export const mentionType = { singular: 'Webpage', manual: true }, + workshop: { + key: 'workshop', + plural: 'Workshops', + singular: 'Workshop', + manual: true + }, other: { key: 'other', plural: 'Other', diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/DataciteMentionRepository.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/DataciteMentionRepository.java index e063ec613..f998d277f 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/DataciteMentionRepository.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/DataciteMentionRepository.java @@ -67,7 +67,7 @@ public class DataciteMentionRepository implements MentionRepository { dataciteTypeMap.put("DataPaper", MentionType.other); dataciteTypeMap.put("Dataset", MentionType.dataset); dataciteTypeMap.put("Dissertation", MentionType.thesis); - dataciteTypeMap.put("Event", MentionType.other); + dataciteTypeMap.put("Event", MentionType.workshop); dataciteTypeMap.put("Image", MentionType.other); dataciteTypeMap.put("InteractiveResource", MentionType.other); dataciteTypeMap.put("Journal", MentionType.other); diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MentionType.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MentionType.java index fff5c79b3..f35548835 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MentionType.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/MentionType.java @@ -22,5 +22,6 @@ public enum MentionType { thesis, videoRecording, webpage, + workshop, other } From 1f193dce9caeb76c712946202e8d38cc2daf43d0 Mon Sep 17 00:00:00 2001 From: Ewan Cahen Date: Wed, 26 Oct 2022 17:05:44 +0200 Subject: [PATCH 03/16] fix: add note field to mention table --- database/008-create-mention-table.sql | 1 + frontend/components/mention/EditMentionModal.tsx | 16 +++++++++++++++- frontend/components/mention/config.ts | 11 +++++++++++ frontend/types/Mention.ts | 5 ++++- frontend/utils/editMentions.ts | 5 ++++- frontend/utils/getCrossref.ts | 5 ++++- frontend/utils/getDataCite.ts | 5 ++++- 7 files changed, 43 insertions(+), 5 deletions(-) diff --git a/database/008-create-mention-table.sql b/database/008-create-mention-table.sql index 2dc16adb6..2ea6bfc32 100644 --- a/database/008-create-mention-table.sql +++ b/database/008-create-mention-table.sql @@ -38,6 +38,7 @@ CREATE TABLE mention ( image_url VARCHAR(500) CHECK (image_url ~ '^https?://'), mention_type mention_type NOT NULL, source VARCHAR(50) NOT NULL, + note VARCHAR(500), scraped_at TIMESTAMPTZ, created_at TIMESTAMPTZ NOT NULL, updated_at TIMESTAMPTZ NOT NULL diff --git a/frontend/components/mention/EditMentionModal.tsx b/frontend/components/mention/EditMentionModal.tsx index a4e404f75..704cc1939 100644 --- a/frontend/components/mention/EditMentionModal.tsx +++ b/frontend/components/mention/EditMentionModal.tsx @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: 2022 Christian Meeßen (GFZ) -// SPDX-FileCopyrightText: 2022 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences // SPDX-FileCopyrightText: 2022 Dusan Mijatovic (dv4all) +// SPDX-FileCopyrightText: 2022 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +// SPDX-FileCopyrightText: 2022 Netherlands eScience Center // SPDX-FileCopyrightText: 2022 dv4all // // SPDX-License-Identifier: Apache-2.0 @@ -213,6 +215,18 @@ export default function EditMentionModal({open, onCancel, onSubmit, item, pos, t }} rules={config.url.validation} /> +
{formData.mention_type === 'highlight' ? +// SPDX-FileCopyrightText: 2022 Netherlands eScience Center // SPDX-FileCopyrightText: 2022 dv4all // // SPDX-License-Identifier: Apache-2.0 @@ -30,6 +32,7 @@ export type MentionItemProps = { // is_featured?: boolean mention_type: MentionTypeKeys | null source: string + note: string | null } export type MentionByType = { @@ -47,4 +50,4 @@ export type MentionForProject = MentionItemProps & { impact_for_project?: any[] } -export const mentionColumns ='id,doi,url,title,authors,publisher,publication_year,page,image_url,mention_type,source' +export const mentionColumns ='id,doi,url,title,authors,publisher,publication_year,page,image_url,mention_type,source,note' diff --git a/frontend/utils/editMentions.ts b/frontend/utils/editMentions.ts index 5766a5c14..85048cb0d 100644 --- a/frontend/utils/editMentions.ts +++ b/frontend/utils/editMentions.ts @@ -1,4 +1,6 @@ // SPDX-FileCopyrightText: 2022 Dusan Mijatovic (dv4all) +// SPDX-FileCopyrightText: 2022 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 Netherlands eScience Center // SPDX-FileCopyrightText: 2022 dv4all // // SPDX-License-Identifier: Apache-2.0 @@ -303,7 +305,8 @@ export function newMentionItem(title?: string) { // url to external image image_url: null, mention_type: null, - source: 'manual' + source: 'manual', + note: null } return newItem } diff --git a/frontend/utils/getCrossref.ts b/frontend/utils/getCrossref.ts index d81640a53..b87eb81ed 100644 --- a/frontend/utils/getCrossref.ts +++ b/frontend/utils/getCrossref.ts @@ -1,4 +1,6 @@ // SPDX-FileCopyrightText: 2022 Dusan Mijatovic (dv4all) +// SPDX-FileCopyrightText: 2022 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 Netherlands eScience Center // SPDX-FileCopyrightText: 2022 dv4all // // SPDX-License-Identifier: Apache-2.0 @@ -57,7 +59,8 @@ export function crossrefItemToMentionItem(item: CrossrefSelectItem) { page: item.page ?? null, image_url: null, mention_type: crossrefToRsdType(item.type), - source: 'Crossref' + source: 'Crossref', + note: null } // debugger return mention diff --git a/frontend/utils/getDataCite.ts b/frontend/utils/getDataCite.ts index 7621bd1bb..8d08b251c 100644 --- a/frontend/utils/getDataCite.ts +++ b/frontend/utils/getDataCite.ts @@ -1,4 +1,6 @@ // SPDX-FileCopyrightText: 2022 Dusan Mijatovic (dv4all) +// SPDX-FileCopyrightText: 2022 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2022 Netherlands eScience Center // SPDX-FileCopyrightText: 2022 dv4all // // SPDX-License-Identifier: Apache-2.0 @@ -138,7 +140,8 @@ export function dataCiteGraphQLItemToMentionItem(item: WorkResponse) { page: null, image_url: null, mention_type: dataciteToRsdType(item), - source: 'DataCite' + source: 'DataCite', + note: null } return mention } From cefea70f1c30379da7695e7b4db771d9f59b980e Mon Sep 17 00:00:00 2001 From: "Dusan Mijatovic (PC2020)" Date: Wed, 26 Oct 2022 18:23:30 +0200 Subject: [PATCH 04/16] feat: show note in page edit and page view components --- .../components/mention/EditMentionModal.tsx | 35 ++++++++++--------- .../components/mention/MentionItemBase.tsx | 2 ++ frontend/components/mention/MentionNote.tsx | 7 ++++ .../components/mention/MentionViewItem.tsx | 2 ++ 4 files changed, 30 insertions(+), 16 deletions(-) create mode 100644 frontend/components/mention/MentionNote.tsx diff --git a/frontend/components/mention/EditMentionModal.tsx b/frontend/components/mention/EditMentionModal.tsx index 704cc1939..651cd929c 100644 --- a/frontend/components/mention/EditMentionModal.tsx +++ b/frontend/components/mention/EditMentionModal.tsx @@ -215,6 +215,25 @@ export default function EditMentionModal({open, onCancel, onSubmit, item, pos, t }} rules={config.url.validation} /> + {formData.mention_type === 'highlight' ? + <> +
+ + + :null + } +
-
- {formData.mention_type === 'highlight' ? - - :null - } {/* Validate entered information */} Please double check the data because this entry cannot be edited after it has been created. diff --git a/frontend/components/mention/MentionItemBase.tsx b/frontend/components/mention/MentionItemBase.tsx index cd2c831b7..88175af0b 100644 --- a/frontend/components/mention/MentionItemBase.tsx +++ b/frontend/components/mention/MentionItemBase.tsx @@ -8,6 +8,7 @@ import {ReactNode} from 'react' import {MentionItemProps} from '~/types/Mention' import MentionAuthors from './MentionAuthors' import MentionDoi from './MentionDoi' +import MentionNote from './MentionNote' import MentionPublisherItem from './MentionPublisherItem' export type MentionItemRole = 'list'|'find'|'view' @@ -69,6 +70,7 @@ export default function MentionItemBase({item,pos,nav,type,role='find'}:MentionI className="text-sm" role={role} /> + ) } diff --git a/frontend/components/mention/MentionNote.tsx b/frontend/components/mention/MentionNote.tsx new file mode 100644 index 000000000..da3b416ec --- /dev/null +++ b/frontend/components/mention/MentionNote.tsx @@ -0,0 +1,7 @@ + +export default function MentionNote({note}: { note: string | null }) { + if (note) { + return
{note}
+ } + return null +} diff --git a/frontend/components/mention/MentionViewItem.tsx b/frontend/components/mention/MentionViewItem.tsx index 7e279bba8..91c4c22f6 100644 --- a/frontend/components/mention/MentionViewItem.tsx +++ b/frontend/components/mention/MentionViewItem.tsx @@ -11,6 +11,7 @@ import MentionAuthors from './MentionAuthors' import MentionPublisherItem from './MentionPublisherItem' import MentionDoi from './MentionDoi' import {MentionTitle} from './MentionItemBase' +import MentionNote from './MentionNote' export default function MentionViewItem({item, pos}: {item: MentionItemProps, pos:number}) { @@ -39,6 +40,7 @@ export default function MentionViewItem({item, pos}: {item: MentionItemProps, po doi={item?.doi} className="text-sm" /> +
{item?.url ? : null} From abd24649bae709439d82cdba788d9054f526c7ba Mon Sep 17 00:00:00 2001 From: "Dusan Mijatovic (PC2020)" Date: Wed, 26 Oct 2022 18:25:30 +0200 Subject: [PATCH 05/16] chore: use next script tag to inject matomo script after initial load. Less priority, faster page load time (at least that's the promise). --- frontend/pages/_document.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frontend/pages/_document.tsx b/frontend/pages/_document.tsx index a11c87fe8..7473b347c 100644 --- a/frontend/pages/_document.tsx +++ b/frontend/pages/_document.tsx @@ -8,6 +8,7 @@ // SPDX-License-Identifier: Apache-2.0 import * as React from 'react' +import Script from 'next/script' import Document, {Html, Head, Main, NextScript, DocumentInitialProps} from 'next/document' import createEmotionServer from '@emotion/server/create-instance' import createEmotionCache from '../styles/createEmotionCache' @@ -61,7 +62,9 @@ export default class MyDocument extends Document{ */ this.matomoUrl !== undefined && this.matomoUrl.length !== 0 && this.matomoId !== undefined && this.matomoId.length !== 0 && - } From bcffbf9de1f89f3167295f3f4dba66312e56fc1f Mon Sep 17 00:00:00 2001 From: "Dusan Mijatovic (PC2020)" Date: Thu, 27 Oct 2022 15:36:12 +0200 Subject: [PATCH 11/16] chore: fix classification datacite typos en remove console log --- frontend/pages/_document.tsx | 1 - frontend/utils/getDataCite.ts | 1 + .../rsd/scraper/doi/DataciteMentionRepository.java | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/pages/_document.tsx b/frontend/pages/_document.tsx index 7d2d1a8d5..e952d6463 100644 --- a/frontend/pages/_document.tsx +++ b/frontend/pages/_document.tsx @@ -81,7 +81,6 @@ export default class MyDocument extends Document{ var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; g.async=true; g.src=u+'piwik.js'; s.parentNode.insertBefore(g,s); })(); - console.log('matomo loaded with nonce...') `} } diff --git a/frontend/utils/getDataCite.ts b/frontend/utils/getDataCite.ts index d79103e90..a3daab4ab 100644 --- a/frontend/utils/getDataCite.ts +++ b/frontend/utils/getDataCite.ts @@ -282,6 +282,7 @@ function rsdTypeFromResourceType(resourceType: string) { case 'newspaper-article': case 'newspaper article': return 'newspaperArticle' + case 'audiovisual': case 'poster': case 'presentation': return 'presentation' diff --git a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/DataciteMentionRepository.java b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/DataciteMentionRepository.java index f998d277f..75f35bcd9 100644 --- a/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/DataciteMentionRepository.java +++ b/scrapers/src/main/java/nl/esciencecenter/rsd/scraper/doi/DataciteMentionRepository.java @@ -70,7 +70,7 @@ public class DataciteMentionRepository implements MentionRepository { dataciteTypeMap.put("Event", MentionType.workshop); dataciteTypeMap.put("Image", MentionType.other); dataciteTypeMap.put("InteractiveResource", MentionType.other); - dataciteTypeMap.put("Journal", MentionType.other); + dataciteTypeMap.put("Journal", MentionType.journalArticle); dataciteTypeMap.put("JournalArticle", MentionType.journalArticle); dataciteTypeMap.put("Model", MentionType.other); dataciteTypeMap.put("OutputManagementPlan", MentionType.other); From b0c5400755dcb12dc02e01155b520e3763134391 Mon Sep 17 00:00:00 2001 From: Ewan Cahen Date: Thu, 27 Oct 2022 16:30:42 +0200 Subject: [PATCH 12/16] fix: scraper uses note field and workshop type for mention --- data-generation/main.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/data-generation/main.js b/data-generation/main.js index c7f165190..d84231512 100644 --- a/data-generation/main.js +++ b/data-generation/main.js @@ -37,6 +37,7 @@ function generateMentions(amountExtra = 10) { 'thesis', 'videoRecording', 'webpage', + 'workshop', 'other', ]; @@ -55,6 +56,7 @@ function generateMentions(amountExtra = 10) { image_url: null, mention_type: faker.helpers.arrayElement(mentionTypes), source: 'faker', + note: faker.helpers.maybe(() => faker.company.catchPhrase(), 0.3) ?? null }); } @@ -70,6 +72,7 @@ function generateMentions(amountExtra = 10) { image_url: null, mention_type: faker.helpers.arrayElement(mentionTypes), source: 'faker', + note: faker.helpers.maybe(() => faker.company.catchPhrase(), 0.3) ?? null }); } From c55856b3b46db1cd65e72771eb6fd6aaeb77512a Mon Sep 17 00:00:00 2001 From: "Dusan Mijatovic (PC2020)" Date: Fri, 28 Oct 2022 11:29:44 +0200 Subject: [PATCH 13/16] fix: cropped cookie banner on mobile --- frontend/components/cookies/CookieConsentMatomo.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/components/cookies/CookieConsentMatomo.tsx b/frontend/components/cookies/CookieConsentMatomo.tsx index cc8e38a61..253421663 100644 --- a/frontend/components/cookies/CookieConsentMatomo.tsx +++ b/frontend/components/cookies/CookieConsentMatomo.tsx @@ -52,8 +52,8 @@ export default function CookieConsentMatomo({matomo, route}: CookieConsentMatomo className="fixed bottom-0 right-0 animated animatedFadeInUp fadeInUp" data-testid="cookie-consent-matomo" > -
-
+
+
From d3a764e0af67395e30c2f2d089c47879f432b7e3 Mon Sep 17 00:00:00 2001 From: jmaassen Date: Fri, 28 Oct 2022 12:12:56 +0000 Subject: [PATCH 14/16] chore(release): update citation file --- CITATION.cff | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CITATION.cff b/CITATION.cff index 35fd60a9e..9d03408dc 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -75,5 +75,5 @@ keywords: - Software Impact - Software Reuse license: Apache-2.0 -version: v1.8.0 -date-released: '2022-10-14' +version: v1.9.0 +date-released: '2022-10-28' From 56883df34865482776bc0c0f91adaa11b95d994a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Mee=C3=9Fen?= <14222414+cmeessen@users.noreply.github.com> Date: Fri, 28 Oct 2022 15:09:47 +0200 Subject: [PATCH 15/16] feat: autoremove initial-spotlights container --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 91a396b16..c21de26c8 100644 --- a/Makefile +++ b/Makefile @@ -62,7 +62,7 @@ data: docker-compose down spotlights: - docker-compose run initial-spotlights + docker-compose run --rm initial-spotlights # Helper commands # - From 9955049b3256d9bae7f311629e6b8a793c43da1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Mee=C3=9Fen?= <14222414+cmeessen@users.noreply.github.com> Date: Fri, 28 Oct 2022 15:10:01 +0200 Subject: [PATCH 16/16] feat: always pull spotlight migration image --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 52016dd7d..7788561da 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -239,6 +239,7 @@ services: initial-spotlights: image: ghcr.io/hifis-net/rsd-spotlight-migration:latest + pull_policy: always environment: - POSTGREST_URL - PGRST_JWT_SECRET