diff --git a/site/gatsby-site/config.js b/site/gatsby-site/config.js index ae693dd658..d63f01a0db 100644 --- a/site/gatsby-site/config.js +++ b/site/gatsby-site/config.js @@ -201,6 +201,9 @@ const config = { rollbar: { token: process.env.GATSBY_ROLLBAR_TOKEN, }, + discover: { + taxa: ['CSETv0', 'CSETv1', 'GMF'], + }, cloudflareR2: { accountId: process.env.CLOUDFLARE_R2_ACCOUNT_ID, accessKeyId: process.env.CLOUDFLARE_R2_ACCESS_KEY_ID, diff --git a/site/gatsby-site/cypress/e2e/integration/discover.cy.js b/site/gatsby-site/cypress/e2e/integration/discover.cy.js index 5c6591a53d..db97d76102 100644 --- a/site/gatsby-site/cypress/e2e/integration/discover.cy.js +++ b/site/gatsby-site/cypress/e2e/integration/discover.cy.js @@ -546,4 +546,32 @@ describe('The Discover app', () => { cy.get('div[data-cy="hits-container"]').children().should('have.length.at.least', 8); } ); + + conditionalIt( + !Cypress.env('isEmptyEnvironment'), + 'Search using the classifications filter', + () => { + cy.visit(url); + + cy.waitForStableDOM(); + + cy.get('[data-cy=expand-filters]').click(); + + cy.contains('button', 'Classifications').click(); + + cy.get('[data-cy="search"] input').type('Buenos Aires'); + + cy.get('[data-cy="attributes"] [data-cy="Named Entities"]').contains('Buenos Aires').click(); + + cy.waitForStableDOM(); + + cy.url().should('include', 'classifications=CSETv0%3ANamed%20Entities%3ABuenos%20Aires'); + + cy.get('[data-cy="selected-refinements"]') + .contains('CSETv0 : Named Entities : Buenos Aires') + .should('be.visible'); + + cy.get('div[data-cy="hits-container"]').children().should('have.length.at.least', 1); + } + ); }); diff --git a/site/gatsby-site/cypress/e2e/integration/incidents/history.cy.js b/site/gatsby-site/cypress/e2e/integration/incidents/history.cy.js index 9cae38a674..2016d3be4e 100644 --- a/site/gatsby-site/cypress/e2e/integration/incidents/history.cy.js +++ b/site/gatsby-site/cypress/e2e/integration/incidents/history.cy.js @@ -149,16 +149,12 @@ describe('Incidents', () => { it('Should display an error message if no Incident ID is provided', () => { cy.visit('/incidents/history?incident_id='); - cy.waitForStableDOM(); - cy.contains('Invalid Incident ID').should('exist'); }); it('Should display an error message if an invalid Incident ID is provided', () => { cy.visit('/incidents/history?incident_id=xxx'); - cy.waitForStableDOM(); - cy.contains('Invalid Incident ID').should('exist'); }); @@ -172,7 +168,6 @@ describe('Incidents', () => { cy.url().should('include', '/cite/10'); }); - // wasn't able to get this one passing on github runners conditionalIt.skip( !Cypress.env('isEmptyEnvironment'), 'Should refresh Report history if the user go back on the browser', @@ -217,9 +212,7 @@ describe('Incidents', () => { cy.go('forward'); - cy.waitForStableDOM(); - - cy.wait(['@FindIncidentHistory', '@FindEntities'], { timeout: 30000 }); + cy.wait(['@FindIncidentHistory', '@FindEntities']); } ); diff --git a/site/gatsby-site/cypress/e2e/integration/integrity.cy.js b/site/gatsby-site/cypress/e2e/integration/integrity.cy.js index 2acb7091f8..528e778696 100644 --- a/site/gatsby-site/cypress/e2e/integration/integrity.cy.js +++ b/site/gatsby-site/cypress/e2e/integration/integrity.cy.js @@ -49,7 +49,7 @@ describe('Integrity', () => { } ); - it.skip( + it( `is_incident_report should be true for reports assigned to incidents and vice versa`, { requestTimeout: 60000, defaultCommandTimeout: 60000, responseTimeout: 60000 }, () => { @@ -85,7 +85,7 @@ describe('Integrity', () => { it( `Classifications should be linked to one and only one incident`, - { requestTimeout: 30000, defaultCommandTimeout: 30000, responseTimeout: 30000 }, + { requestTimeout: 60000, defaultCommandTimeout: 60000, responseTimeout: 60000 }, () => { cy.query({ query: gql` diff --git a/site/gatsby-site/cypress/e2e/integration/pages.cy.js b/site/gatsby-site/cypress/e2e/integration/pages.cy.js index b180e7e78f..7f2ae0a867 100644 --- a/site/gatsby-site/cypress/e2e/integration/pages.cy.js +++ b/site/gatsby-site/cypress/e2e/integration/pages.cy.js @@ -54,7 +54,7 @@ describe('Pages', () => { paths.forEach((path) => { languages.forEach(({ code }) => { - it(`/${code}${path} Should not have errors`, () => { + it(`/${code}${path} Should not have errors`, { defaultCommandTimeout: 30000 }, () => { const canonicalPath = switchLocalizedPath({ newLang: code, path }); cy.visit(canonicalPath, { @@ -118,44 +118,50 @@ describe('Pages', () => { `[rel="alternate"][hrefLang="${language.hrefLang}"][href="${alternateUrl}"]` ).should('exist'); } + }); - cy.get('body') - .then(($body) => { - if ($body.find('[data-cy="cloudinary-image-wrapper"]').length) { - return true; - } - return false; - }) - .then((selectorExists) => { - if (selectorExists) { - cy.get('[data-cy="cloudinary-image-wrapper"]').each(($el) => { - cy.wrap($el) - .scrollIntoView() - .find('[data-cy="cloudinary-image"]') - .should('have.attr', 'src') - .then(($src) => { - cy.request({ - url: $src.toString(), - failOnStatusCode: false, - }).then((resp) => { - if (resp.status !== 200) { - // If image is failing, check if cloudinary image is hidden - cy.wrap($el) - .find('[data-cy="cloudinary-image"]') - .should('have.class', 'hidden'); - // Check if placeholder image is displayed instead - cy.wrap($el) - .find('[data-cy="cloudinary-image-placeholder"]') - .should('not.have.class', 'hidden'); - } + it.skip( + `/${code}${path} Should load images properly`, + { defaultCommandTimeout: 30000 }, + () => { + cy.get('body') + .then(($body) => { + if ($body.find('[data-cy="cloudinary-image-wrapper"]').length) { + return true; + } + return false; + }) + .then((selectorExists) => { + if (selectorExists) { + cy.get('[data-cy="cloudinary-image-wrapper"]').each(($el) => { + cy.wrap($el) + .scrollIntoView() + .find('[data-cy="cloudinary-image"]') + .should('have.attr', 'src') + .then(($src) => { + cy.request({ + url: $src.toString(), + failOnStatusCode: false, + }).then((resp) => { + if (resp.status !== 200) { + // If image is failing, check if cloudinary image is hidden + cy.wrap($el) + .find('[data-cy="cloudinary-image"]') + .should('have.class', 'hidden'); + // Check if placeholder image is displayed instead + cy.wrap($el) + .find('[data-cy="cloudinary-image-placeholder"]') + .should('not.have.class', 'hidden'); + } + }); }); - }); - }); - } else { - cy.log(`No images found on page, skipping image test for path ${path}`); - } - }); - }); + }); + } else { + cy.log(`No images found on page, skipping image test for path ${path}`); + } + }); + } + ); }); }); }); diff --git a/site/gatsby-site/cypress/e2e/unit/AlgoliaUpdater.cy.js b/site/gatsby-site/cypress/e2e/unit/AlgoliaUpdater.cy.js index 4eecab3ec1..ffb0626d16 100644 --- a/site/gatsby-site/cypress/e2e/unit/AlgoliaUpdater.cy.js +++ b/site/gatsby-site/cypress/e2e/unit/AlgoliaUpdater.cy.js @@ -59,19 +59,41 @@ const reports = [ title: 'Report 23 title', url: 'https://url.com/stuff', }, + + // this report hast no parent incidents + { + _id: new ObjectID('60dd465f80935bc89e6f9b07'), + authors: ['Test User'], + date_downloaded: '2019-04-13', + date_modified: '2020-06-14', + date_published: '2015-05-19', + date_submitted: '2019-06-01', + description: 'Description of report 40', + epoch_date_downloaded: 1555113600, + epoch_date_modified: 1592092800, + epoch_date_published: 1431993600, + epoch_date_submitted: 1559347200, + image_url: 'http://url.com', + cloudinary_id: 'http://cloudinary.com', + language: 'es', + report_number: 40, + source_domain: 'blogs.wsj.com', + submitters: ['Roman Yampolskiy'], + tags: [], + text: 'Report 40 **text**', + plain_text: 'Report 40 text', + title: 'Report 40 title', + url: 'https://url.com/stuff', + }, ]; const classifications = [ { _id: '60dd465f80935bc89e6f9b00', incidents: [1], + reports: [], namespace: 'CSETv0', attributes: [ - // { short_name: 'Annotator', value_json: '"1"' }, - // { short_name: 'Annotation Status', value_json: '"6. Complete and final"' }, - // { short_name: 'Reviewer', value_json: '"5"' }, - // { short_name: 'Quality Control', value_json: 'false' }, - // { short_name: 'Full Description', value_json: '"On December 5, 2018, a robot punctured."' }, { short_name: 'Named Entities', value_json: '["Amazon"]' }, { short_name: 'Harm Type', @@ -83,7 +105,8 @@ const classifications = [ }, { _id: '60dd465f80935bc89e6f9b01', - incidents: [1], + incidents: [], + reports: [], namespace: 'SHOULD NOT BE INCLUDED', attributes: [{ short_name: 'Something', value_json: '"Great"' }], notes: 'Nothing to see here', @@ -265,7 +288,11 @@ describe('Algolia', () => { }); cy.wrap(updater.run()).then(() => { - expect(mongoClient.connect.callCount).to.eq(4); + expect(mongoClient.connect.callCount).to.eq(1); + + // english + + expect(enIndex.replaceAllObjects.getCalls().length).eq(1); expect(enIndex.replaceAllObjects.getCall(0).args[0].length).eq(2); @@ -290,11 +317,16 @@ describe('Algolia', () => { incident_id: 1, epoch_incident_date: 1592092800, incident_date: '2020-06-14', + namespaces: ['CSETv0'], classifications: [ 'CSETv0:Named Entities:Amazon', 'CSETv0:Harm Type:Harm to physical health/safety', 'CSETv0:Harm Type:Harm to physical property', ], + CSETv0: { + 'Named Entities': ['Amazon'], + 'Harm Type': ['Harm to physical health/safety', 'Harm to physical property'], + }, }); expect(enIndex.replaceAllObjects.getCall(0).args[0][1]).to.deep.nested.include({ @@ -318,13 +350,29 @@ describe('Algolia', () => { incident_id: 1, incident_date: '2020-06-14', epoch_incident_date: 1592092800, + namespaces: ['CSETv0'], classifications: [ 'CSETv0:Named Entities:Amazon', 'CSETv0:Harm Type:Harm to physical health/safety', 'CSETv0:Harm Type:Harm to physical property', ], + CSETv0: { + 'Named Entities': ['Amazon'], + 'Harm Type': ['Harm to physical health/safety', 'Harm to physical property'], + }, + }); + + expect(enIndex.deleteBy.getCall(0).args[0]).deep.eq({ + filters: 'incident_id = 247', }); + ``; + // spanish + + expect(esIndex.replaceAllObjects.getCalls().length).eq(1); + + expect(esIndex.replaceAllObjects.getCall(0).args[0].length).eq(2); + expect(esIndex.replaceAllObjects.getCall(0).args[0][0]).to.deep.nested.include({ authors: ['Alistair Barr'], description: 'Description of report 1', @@ -346,11 +394,16 @@ describe('Algolia', () => { incident_id: 1, incident_date: '2020-06-14', epoch_incident_date: 1592092800, + namespaces: ['CSETv0'], classifications: [ 'CSETv0:Named Entities:Amazon', 'CSETv0:Harm Type:Harm to physical health/safety', 'CSETv0:Harm Type:Harm to physical property', ], + CSETv0: { + 'Named Entities': ['Amazon'], + 'Harm Type': ['Harm to physical health/safety', 'Harm to physical property'], + }, featured: 0, }); @@ -375,23 +428,24 @@ describe('Algolia', () => { incident_id: 1, incident_date: '2020-06-14', epoch_incident_date: 1592092800, + namespaces: ['CSETv0'], classifications: [ 'CSETv0:Named Entities:Amazon', 'CSETv0:Harm Type:Harm to physical health/safety', 'CSETv0:Harm Type:Harm to physical property', ], + CSETv0: { + 'Named Entities': ['Amazon'], + 'Harm Type': ['Harm to physical health/safety', 'Harm to physical property'], + }, featured: 2, }); - expect(enIndex.deleteBy.getCall(0).args[0]).deep.eq({ - filters: 'incident_id = 247', - }); - expect(esIndex.deleteBy.getCall(0).args[0]).deep.eq({ filters: 'incident_id = 247', }); - expect(mongoClient.close.callCount).to.eq(4); + expect(mongoClient.close.callCount).to.eq(1); }); }); }); diff --git a/site/gatsby-site/cypress/e2e/unit/discover/routing.cy.js b/site/gatsby-site/cypress/e2e/unit/discover/routing.cy.js index 518f13dfdd..aa41c27837 100644 --- a/site/gatsby-site/cypress/e2e/unit/discover/routing.cy.js +++ b/site/gatsby-site/cypress/e2e/unit/discover/routing.cy.js @@ -23,7 +23,7 @@ describe('Discover routing', () => { expect(state.refinementList).to.deep.eq({ source_domain: ['theguardian.com'], authors: ['Christopher Knaus', 'Sam Levin'], - classifications: ['CSETv0:Intent:Accident'], + 'CSETv0.Intent': ['Accident'], is_incident_report: ['true'], }); expect(state.range).to.deep.eq({ @@ -40,6 +40,7 @@ describe('Discover routing', () => { indexName, locale: 'en', queryConfig, + taxa: ['CSETv0', 'CSETv1'], }); expect('?' + resultURL).to.eq(location.search); diff --git a/site/gatsby-site/src/components/discover/Discover.js b/site/gatsby-site/src/components/discover/Discover.js index 775c008c08..f62fa393e9 100644 --- a/site/gatsby-site/src/components/discover/Discover.js +++ b/site/gatsby-site/src/components/discover/Discover.js @@ -57,6 +57,8 @@ export default function Discover() { return null; } + const taxa = config.discover.taxa; + return ( { return window.location; }, - parseURL: ({ location }) => parseURL({ location, indexName, queryConfig }), - createURL: ({ routeState }) => createURL({ indexName, locale, queryConfig, routeState }), + parseURL: ({ location }) => parseURL({ location, indexName, queryConfig, taxa }), + createURL: ({ routeState }) => + createURL({ indexName, locale, queryConfig, routeState, taxa }), push: (url) => { navigate(`?${url}`); }, diff --git a/site/gatsby-site/src/components/discover/Filter.js b/site/gatsby-site/src/components/discover/Filter.js index 5a8cca2f30..d70c57f65a 100644 --- a/site/gatsby-site/src/components/discover/Filter.js +++ b/site/gatsby-site/src/components/discover/Filter.js @@ -125,7 +125,7 @@ const FilterOverlay = React.forwardRef(function Container(
diff --git a/site/gatsby-site/src/components/discover/Filters.js b/site/gatsby-site/src/components/discover/Filters.js index 7a8987dd19..43ff61efd1 100644 --- a/site/gatsby-site/src/components/discover/Filters.js +++ b/site/gatsby-site/src/components/discover/Filters.js @@ -2,10 +2,26 @@ import React from 'react'; import REFINEMENT_LISTS from 'components/discover/REFINEMENT_LISTS'; import Filter from './Filter'; import { useMenuContext } from 'contexts/MenuContext'; +import { graphql, useStaticQuery } from 'gatsby'; function Filters() { const { isCollapsed } = useMenuContext(); + const { + taxa: { nodes: taxa }, + } = useStaticQuery(graphql` + query FiltersTaxaQuery { + taxa: allMongodbAiidprodTaxa(filter: { namespace: { in: ["CSETv1", "CSETv0", "GMF"] } }) { + nodes { + namespace + field_list { + short_name + } + } + } + } + `); + return (
{REFINEMENT_LISTS.map((list) => ( @@ -16,7 +32,7 @@ function Filters() { } px-1 ${list.hidden ? 'hidden' : ''}`} data-cy={list.attribute} > - +
))}
diff --git a/site/gatsby-site/src/components/discover/OptionsModal.js b/site/gatsby-site/src/components/discover/OptionsModal.js index 94e984f282..02a601310a 100644 --- a/site/gatsby-site/src/components/discover/OptionsModal.js +++ b/site/gatsby-site/src/components/discover/OptionsModal.js @@ -8,6 +8,7 @@ import DisplayModeSwitch from './DisplayModeSwitch'; import Button from 'elements/Button'; import DisplayOptions from './DisplayOptions'; import { Accordion, Modal } from 'flowbite-react'; +import { graphql, useStaticQuery } from 'gatsby'; import { useRange, useRefinementList } from 'react-instantsearch'; const VirtualRefinementList = ({ attribute }) => { @@ -24,6 +25,7 @@ const VirtualRange = ({ attribute }) => { const componentsMap = { refinement: VirtualRefinementList, + classifications: VirtualRefinementList, range: VirtualRange, }; @@ -44,6 +46,21 @@ function OptionsModal() { const handleClose = () => setShowModal(false); + const { + taxa: { nodes: taxa }, + } = useStaticQuery(graphql` + query FiltersTaxaQuery { + taxa: allMongodbAiidprodTaxa(filter: { namespace: { in: ["CSETv1", "CSETv0", "GMF"] } }) { + nodes { + namespace + field_list { + short_name + } + } + } + } + `); + return (
@@ -74,7 +91,12 @@ function OptionsModal() {
{REFINEMENT_LISTS.map((list) => ( - + ))}
diff --git a/site/gatsby-site/src/components/discover/REFINEMENT_LISTS.js b/site/gatsby-site/src/components/discover/REFINEMENT_LISTS.js index 04d65a2e40..17a48e7191 100644 --- a/site/gatsby-site/src/components/discover/REFINEMENT_LISTS.js +++ b/site/gatsby-site/src/components/discover/REFINEMENT_LISTS.js @@ -16,7 +16,8 @@ const REFINEMENT_LISTS = [ label: 'Classifications', faIcon: faNewspaper, faClasses: 'far fa-newspaper', - type: 'refinement', + type: 'classifications', + width: 420, // algolia specific showMore: true, diff --git a/site/gatsby-site/src/components/discover/createURL.js b/site/gatsby-site/src/components/discover/createURL.js index 8806477fa9..49ca492009 100644 --- a/site/gatsby-site/src/components/discover/createURL.js +++ b/site/gatsby-site/src/components/discover/createURL.js @@ -40,48 +40,63 @@ const convertRangeToQueryString = (range) => { return result; }; -const getQueryFromState = (searchState, locale) => { +const getQueryFromState = ({ state, locale, taxa }) => { let query = {}; - if (searchState.query !== '') { - query.s = searchState.query; + if (state.query !== '') { + query.s = state.query; } - if (searchState.refinementList) { + if (state.refinementList) { + const classifications = []; + + const other = {}; + + for (const [key, values] of Object.entries(state.refinementList)) { + const [namespace, ...attribute] = key.split('.'); + + if (taxa.includes(namespace)) { + for (const value of values) { + classifications.push(`${namespace}:${attribute}:${value}`); + } + } else { + other[key] = values; + } + } + query = { ...query, - ...convertArrayToString(removeEmptyAttributes(searchState.refinementList)), + ...convertArrayToString(removeEmptyAttributes({ classifications, ...other })), }; } - if (searchState.range) { + if (state.range) { query = { ...query, - ...convertRangeToQueryString(removeEmptyAttributes(searchState.range)), + ...convertRangeToQueryString(removeEmptyAttributes(state.range)), }; } - if (searchState.sortBy) { + if (state.sortBy) { query.sortBy = - SORTING_LIST.find((s) => s[`value_${locale}`] === searchState.sortBy)?.name || - searchState.sortBy; + SORTING_LIST.find((s) => s[`value_${locale}`] === state.sortBy)?.name || state.sortBy; } - if (searchState.configure?.distinct === true) { + if (state.configure?.distinct === true) { query.hideDuplicates = 1; } - if (searchState.page) { - query.page = searchState.page; + if (state.page) { + query.page = state.page; } return query; }; -export default function ({ routeState, indexName, locale, queryConfig }) { +export default function ({ routeState, indexName, locale, queryConfig, taxa }) { const state = routeState[indexName] || {}; - const query = getQueryFromState(state, locale); + const query = getQueryFromState({ state, locale, taxa }); const encoded = encodeQueryParams(queryConfig, query); diff --git a/site/gatsby-site/src/components/discover/filterTypes/Classifications.js b/site/gatsby-site/src/components/discover/filterTypes/Classifications.js new file mode 100644 index 0000000000..837523484c --- /dev/null +++ b/site/gatsby-site/src/components/discover/filterTypes/Classifications.js @@ -0,0 +1,259 @@ +import TextInputGroup from 'components/forms/TextInputGroup'; +import React, { useCallback, useEffect, useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import { useRefinementList, useInstantSearch } from 'react-instantsearch'; +import debounce from 'lodash/debounce'; +import { faClose } from '@fortawesome/free-solid-svg-icons'; +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; + +/* eslint-disable jsx-a11y/click-events-have-key-events */ + +function Attribute({ name, refinement, searchResults }) { + const [collapsed, setCollapsed] = useState(true); + + useEffect(() => { + if (searchResults?.length > 0) { + setCollapsed(false); + } + }, [searchResults]); + + if (refinement.items.length === 0) { + return null; + } + + return ( +
+
setCollapsed((c) => !c)} + > + {name} ({refinement.items.length}){' '} +
{collapsed ? <>+ : <>-}
+
+ + {!collapsed && ( +
+ {refinement.items + .filter( + (item) => + searchResults == null || + searchResults.map((r) => r.value.split(':')[2]).includes(item.value) + ) + .map((item) => { + return ( +
refinement.refine(item.value)} + > + {item.label} ({item.count}){' '} +
+ ); + })} +
+ )} +
+ ); +} + +function Namespace({ taxonomy, refinement, searchResults }) { + const [collapsed, setCollapsed] = useState(true); + + const { namespace } = taxonomy; + + const refinements = {}; + + for (const field of taxonomy.field_list) { + const key = `${namespace}.${field.short_name}`; + + const refinement = useRefinementList({ attribute: key, limit: 999 }); + + refinements[field.short_name] = refinement; + } + + useEffect(() => { + if (searchResults?.length > 0) { + setCollapsed(false); + } + }, [searchResults]); + + return ( +
+
setCollapsed((c) => !c)} + > + {namespace} ({refinement.count}){' '} +
{collapsed ? <>+ : <>-}
+
+ {!collapsed && ( +
+ {Object.keys(refinements) + .filter( + (name) => + searchResults == null || + searchResults.map((r) => r.value.split(':')[1]).includes(name) + ) + .map((name) => { + const refinement = refinements[name]; + + return ( + + ); + })} +
+ )} +
+ ); +} + +function EmptyNamespace({ taxonomy }) { + const { namespace } = taxonomy; + + return ( +
+
{namespace} (0)
+
+ ); +} + +function SelectedRefinement({ attribute }) { + const { items, refine } = useRefinementList({ attribute, limit: 100 }); + + const [namespace, name] = attribute.split('.'); + + return items + .filter((item) => item.isRefined) + .map((item) => ( +
refine(item.value)} + data-cy={item.value} + > +
+ {namespace} : {name} : {item.label} +
+ +
+ )); +} + +function Search({ setSearchResults }) { + const { t } = useTranslation(); + + const { + items: searchItems, + searchForItems, + isFromSearch, + } = useRefinementList({ attribute: 'classifications' }); + + const [query, setQuery] = useState(''); + + const debouncedSearchForItems = useCallback( + debounce((query) => { + searchForItems(query); + }, 1000), + [] + ); + + useEffect(() => { + if (query.length > 2) { + debouncedSearchForItems(query); + } + + if (query.length == 0) { + setSearchResults(null); + } + }, [query]); + + useEffect(() => { + if (isFromSearch) { + setSearchResults(searchItems); + } + }, [searchItems, isFromSearch]); + + return ( +
+ setQuery(e.currentTarget.value)} + /> +
+ ); +} + +export default function Classifications({ taxa }) { + const { indexUiState } = useInstantSearch(); + + const { items: namespaces } = useRefinementList({ attribute: 'namespaces' }); + + const selectedAttributes = Object.keys(indexUiState.refinementList).filter((key) => { + return taxa.map((t) => t.namespace).some((n) => key.startsWith(`${n}.`)); + }); + + const [searchResults, setSearchResults] = useState(null); + + return ( +
+
+ {selectedAttributes.map((attribute) => { + return ; + })} +
+ + + + {searchResults == null || searchResults.length > 0 ? ( +
+ {taxa.map((taxonomy) => { + const refinement = namespaces.find((r) => r.value === taxonomy.namespace); + + if (!refinement) { + return ; + } + + return ( + + ); + })} +
+ ) : ( +
No results
+ )} +
+ ); +} + +export const touchedCount = ({ searchState }) => { + return Object.keys(searchState.refinementList).filter((key) => key.split('.').length > 1).length; +}; diff --git a/site/gatsby-site/src/components/discover/filterTypes/index.js b/site/gatsby-site/src/components/discover/filterTypes/index.js index 02bb1b5418..fee199d6da 100644 --- a/site/gatsby-site/src/components/discover/filterTypes/index.js +++ b/site/gatsby-site/src/components/discover/filterTypes/index.js @@ -1,4 +1,5 @@ import * as range from './RangeInput'; import * as refinement from './RefinementList'; +import * as classifications from './Classifications'; -export default { range, refinement }; +export default { range, refinement, classifications }; diff --git a/site/gatsby-site/src/components/discover/parseURL.js b/site/gatsby-site/src/components/discover/parseURL.js index d83906c076..00e5e82c05 100644 --- a/site/gatsby-site/src/components/discover/parseURL.js +++ b/site/gatsby-site/src/components/discover/parseURL.js @@ -1,8 +1,8 @@ import { parse } from 'query-string'; import { decodeQueryParams } from 'use-query-params'; -const convertStringToRefinement = (obj) => { - const stringKeys = [ +const parseRefinements = ({ query }) => { + const refinementKeys = [ 'source_domain', 'authors', 'submitters', @@ -14,21 +14,33 @@ const convertStringToRefinement = (obj) => { 'language', ]; - let newObj = {}; - - for (const attr in obj) { - if (stringKeys.includes(attr) && obj[attr] !== undefined) { - // TODO: we have to make the other namespaces available here too - if (obj[attr].indexOf('CSET') >= 0) { - // The facet separator is double pipe sign - "||" - newObj[attr] = obj[attr].split('||CSET').map((i) => 'CSET' + i); - newObj[attr][0] = newObj[attr][0].substr(4); - } else { - newObj[attr] = obj[attr].split('||'); + const result = {}; + + for (const [key, value] of Object.entries(query)) { + if (value) { + if (refinementKeys.includes(key)) { + if (key == 'classifications') { + const facets = value.split('||'); + + for (const facet of facets) { + const [namespace, attribute, ...value] = facet.split(':'); + + const refinementKey = `${namespace}.${attribute}`; + + if (!result[refinementKey]) { + result[refinementKey] = []; + } + + result[refinementKey].push(...value); + } + } else { + result[key] = value.split('||'); + } } } } - return newObj; + + return result; }; const convertStringToRange = (query) => { @@ -50,7 +62,7 @@ const generateSearchState = ({ query }) => { page: query.page, query: query.s ?? '', refinementList: { - ...convertStringToRefinement(query), + ...parseRefinements({ query }), }, range: { ...convertStringToRange(query), diff --git a/site/gatsby-site/src/utils/AlgoliaUpdater.js b/site/gatsby-site/src/utils/AlgoliaUpdater.js index 134127d2fe..d9d54386ca 100644 --- a/site/gatsby-site/src/utils/AlgoliaUpdater.js +++ b/site/gatsby-site/src/utils/AlgoliaUpdater.js @@ -6,6 +6,8 @@ const config = require('../../config'); const { isCompleteReport } = require('./variants'); +const { merge } = require('lodash'); + const subset = !!process.env.ALGOLIA_SUBSET; const truncate = (doc) => { @@ -44,8 +46,25 @@ const includedCSETAttributes = [ 'Technology Purveyor', ]; -const getClassificationArray = ({ classification, taxonomy }) => { - const result = []; +/** + * + * @returns classifications in both tree and list format: + * + * { + * tree: { + * CSETv0: { + * 'Sector of Deployment': 'Defense', + * 'Problem Nature': 'Cybersecurity', + * } + * }, + * list: ['CSETv0:Sector of Deployment:Defense', 'CSETv0:Sector of Deployment:Healthcare'], + * } + */ + +const getClassificationObject = ({ classification, taxonomy }) => { + const attributes = {}; + + const list = []; if (classification.attributes) { for (const attribute of classification.attributes) { @@ -61,20 +80,23 @@ const getClassificationArray = ({ classification, taxonomy }) => { ) { const value = JSON.parse(attribute.value_json); + attributes[attribute.short_name] = value; + const values = Array.isArray(value) ? value : [value]; for (const v of values) { - if (v == '' || typeof v === 'object') continue; - result.push(`${classification.namespace}:${attribute.short_name}:${v}`); + if (v != '' && typeof v != 'object') { + list.push(`${classification.namespace}:${attribute.short_name}:${v}`); + } } } } } - return result; + return { tree: { [classification.namespace]: attributes }, list }; }; -const reportToEntry = ({ incident = null, report }) => { +const reportToEntry = ({ incident = null, report, classifications = [{ list: [], tree: [] }] }) => { let featuredValue = 0; if (config?.header?.search?.featured) { @@ -102,6 +124,7 @@ const reportToEntry = ({ incident = null, report }) => { source_domain: report.source_domain, submitters: report.submitters, title: report.title, + name: report.title, url: report.url, tags: report.tags, editor_notes: report.editor_notes, @@ -112,8 +135,16 @@ const reportToEntry = ({ incident = null, report }) => { featured: featuredValue, flag: report.flag, is_incident_report: report.is_incident_report, + namespaces: classifications.map((c) => { + return Object.keys(c.tree)[0]; + }), + classifications: classifications.map((c) => c.list).flat(), }; + for (const classification of classifications) { + merge(entry, classification.tree); + } + if (incident) { entry.incident_id = incident.incident_id; entry.incident_date = incident.date; @@ -142,30 +173,36 @@ class AlgoliaUpdater { } generateIndexEntries = async ({ reports, incidents, classifications, taxa }) => { - const classificationsHash = {}; - - for (const classification of classifications) { - for (const incident_id of classification.incidents) { - const taxonomy = taxa.find((t) => t.namespace == classification.namespace); - - if (!classificationsHash[incident_id]) { - classificationsHash[incident_id] = getClassificationArray({ classification, taxonomy }); - } - } - } - const downloadData = []; for (const incident of incidents) { + const incidentClassifications = classifications + .filter((c) => c.incidents.includes(incident.incident_id)) + .map((classification) => + getClassificationObject({ + classification, + taxonomy: taxa.find((t) => t.namespace == classification.namespace), + }) + ); + for (const report_number of incident.reports) { if (reports.some((r) => r.report_number == report_number)) { const report = reports.find((r) => r.report_number == report_number) || {}; - const entry = reportToEntry({ incident, report }); - - if (classificationsHash[entry.incident_id]) { - entry.classifications = classificationsHash[entry.incident_id]; - } + const reportClassifications = classifications + .filter((c) => c.reports.includes(report.report_number)) + .map((classification) => + getClassificationObject({ + classification, + taxonomy: taxa.find((t) => t.namespace == classification.namespace), + }) + ); + + const entry = reportToEntry({ + incident, + report, + classifications: [...incidentClassifications, ...reportClassifications], + }); downloadData.push(entry); } @@ -306,6 +343,10 @@ class AlgoliaUpdater { await index.setSettings( { ...algoliaSettings, + attributesForFaceting: [ + ...algoliaSettings.attributesForFaceting, + ...config.discover.taxa, + ], indexLanguages: [language], queryLanguages: [language], replicas: [ @@ -382,8 +423,6 @@ class AlgoliaUpdater { }; deleteDuplicates = async ({ language }) => { - await this.mongoClient.connect(); - const indexName = `instant_search-${language}`; const index = await this.algoliaClient.initIndex(indexName); @@ -398,13 +437,9 @@ class AlgoliaUpdater { await index.deleteBy({ filters }); } - - await this.mongoClient.close(); }; async generateIndex({ language }) { - await this.mongoClient.connect(); - const classifications = await this.getClassifications(); const taxa = await this.getTaxa(); @@ -420,12 +455,12 @@ class AlgoliaUpdater { taxa, }); - await this.mongoClient.close(); - return entries; } async run() { + await this.mongoClient.connect(); + for (let { code: language } of this.languages) { const entries = await this.generateIndex({ language }); @@ -437,6 +472,8 @@ class AlgoliaUpdater { await this.deleteDuplicates({ language }); } + + await this.mongoClient.close(); } } diff --git a/site/gatsby-site/src/utils/algoliaSettings.js b/site/gatsby-site/src/utils/algoliaSettings.js index d22583f78b..f6dc705f0d 100644 --- a/site/gatsby-site/src/utils/algoliaSettings.js +++ b/site/gatsby-site/src/utils/algoliaSettings.js @@ -28,6 +28,7 @@ module.exports = { 'is_incident_report', 'searchable(tags)', 'searchable(language)', + 'namespaces', ], attributesToSnippet: ['description:200', 'text:15'], attributesToHighlight: ['description', 'text', 'title', 'incident_title', 'incident_description'],