} - The status and data of the retrieved resource metadata.
+ */
+const getResourceMetaDataJob = async (job) => {
+ const {
+ resourceId, resourceUrl,
+ } = job.data;
+
+ try {
+ // Determine if this is an ECLKC resource.
+ const isEclkc = resourceUrl.includes('eclkc.ohs.acf.hhs.gov');
+
+ let statusCode;
+ let mimeType;
+ let title = null;
+
+ // Get the MIME type and status code of the resource.
+ // eslint-disable-next-line prefer-const
+ ({ mimeType, statusCode } = await getMimeType(resourceUrl));
+
+ // Check if the MIME type is unparsable.
+ if (mimeType && unparsableMimeTypes.includes(mimeType.toLowerCase().replace(/\s+/g, ''))) {
+ auditLogger.error(`Resource Queue: Warning, unable to process resource '${resourceUrl}', received status code '${statusCode}'.`);
+ return {
+ status: httpCodes.NO_CONTENT,
+ data: { url: resourceUrl },
+ };
+ }
+
+ // Check if the status code indicates an error.
+ if (statusCode !== httpCodes.OK) {
+ auditLogger.error(`Resource Queue: Warning, unable to retrieve resource '${resourceUrl}', received status code '${statusCode}'.`);
+ return { status: statusCode || 500, data: { url: resourceUrl } };
+ }
+
+ // If it is an ECLKC resource, get the metadata values.
+ if (isEclkc) {
+ ({ title, statusCode } = await getMetadataValues(resourceUrl));
+ if (statusCode !== httpCodes.OK) {
+ auditLogger.error(`Resource Queue: Warning, unable to retrieve metadata or resource TITLE for resource '${resourceUrl}', received status code '${statusCode || 500}'.`);
+ return { status: statusCode || 500, data: { url: resourceUrl } };
+ }
+ } else {
+ // If it is not an ECLKC resource, scrape the page title.
+ ({ title, statusCode } = await getPageScrapeValues(resourceUrl));
+ if (statusCode !== httpCodes.OK) {
+ auditLogger.error(`Resource Queue: Warning, unable to retrieve resource TITLE for resource '${resourceUrl}', received status code '${statusCode || 500}'.`);
+ return { status: statusCode || 500, data: { url: resourceUrl } };
+ }
+ }
+ logger.info(`Resource Queue: Successfully retrieved resource metadata for resource '${resourceUrl}'`);
+ return { status: httpCodes.OK, data: { url: resourceUrl } };
+ } catch (error) {
+ auditLogger.error(`Resource Queue: Unable to retrieve metadata or title for Resource (ID: ${resourceId} URL: ${resourceUrl}), please make sure this is a valid address:`, error);
+ return { status: httpCodes.NOT_FOUND, data: { url: resourceUrl } };
+ }
+};
+
+export {
+ getResourceMetaDataJob,
+};
diff --git a/src/lib/resource.test.js b/src/lib/resource.test.js
index db12f0924d..c48c4c4b09 100644
--- a/src/lib/resource.test.js
+++ b/src/lib/resource.test.js
@@ -1,568 +1,588 @@
-/* eslint-disable no-useless-escape */
-import axios from 'axios';
-import { expect } from '@playwright/test';
-import { auditLogger } from '../logger';
-import { getResourceMetaDataJob } from './resource';
-import db, { Resource } from '../models';
-
-jest.mock('../logger');
-jest.mock('bull');
-
-const urlReturn = `
-
-
-
-Head Start | ECLKC
-
-test
-
-
-`;
-
-const urlMissingTitle = `
-
-
-
-test
-
-
-`;
-
-const metadata = {
- created: [{ value: '2020-04-21T15:20:23+00:00' }],
- changed: [{ value: '2023-05-26T18:57:15+00:00' }],
- title: [{ value: 'Head Start Heals Campaign' }],
- field_taxonomy_national_centers: [{ target_type: 'taxonomy_term' }],
- field_taxonomy_topic: [{ target_type: 'taxonomy_term' }],
- langcode: [{ value: 'en' }],
- field_context: [{ value: '
+Head Start | ECLKC
+
+test
+
+
+`;
+
+const urlMissingTitle = `
+
+
+
+test
+
+
+`;
+
+const metadata = {
+ created: [{ value: '2020-04-21T15:20:23+00:00' }],
+ changed: [{ value: '2023-05-26T18:57:15+00:00' }],
+ title: [{ value: 'Head Start Heals Campaign' }],
+ field_taxonomy_national_centers: [{ target_type: 'taxonomy_term' }],
+ field_taxonomy_topic: [{ target_type: 'taxonomy_term' }],
+ langcode: [{ value: 'en' }],
+ field_context: [{ value: '
{
const splitPipe = (str: string) => str.split('\n').map((s) => s.trim()).filter(Boolean);
const mappings: Record = {
- Audience: 'audience',
- Creator: 'creator',
- 'Edit Title': 'eventName',
+ Audience: 'eventIntendedAudience',
+ 'IST/Creator': 'creator',
'Event Title': 'eventName',
- 'Event Duration/#NC Days of Support': 'eventDuration',
- 'Event Duration/# NC Days of Support': 'eventDuration',
+ 'Event Duration': 'trainingType',
+ 'Event Duration/#NC Days of Support': 'trainingType',
+ 'Event Duration/# NC Days of Support': 'trainingType',
'Event ID': 'eventId',
'Overall Vision/Goal for the PD Event': 'vision',
+ 'Vision/Goal/Outcomes for the PD Event': 'vision',
'Reason for Activity': 'reasons',
+ 'Reason(s) for PD': 'reasons',
'Target Population(s)': 'targetPopulations',
'Event Organizer - Type of Event': 'eventOrganizer',
'IST Name:': 'istName',
@@ -536,26 +544,63 @@ const mapLineToData = (line: Record) => {
return data;
};
-const checkUserExists = async (creator: string) => {
+const checkUserExists = async (userWhere: UserWhereOptions) => {
+ const user = await db.User.findOne({
+ where: userWhere,
+ include: [
+ {
+ model: db.Permission,
+ as: 'permissions',
+ },
+ {
+ model: db.NationalCenter,
+ as: 'nationalCenters',
+ },
+ ],
+ });
+
+ if (!user) {
+ throw new Error(`User with ${
+ Object.keys(userWhere).map((key) => `${key}: ${userWhere[key]}`).join('AND ')
+ } does not exist`);
+ }
+ return user;
+};
+
+const checkUserExistsByNationalCenter = async (identifier: string) => {
const user = await db.User.findOne({
- where: { email: creator },
+ attributes: ['id', 'name'],
include: [
{
model: db.Permission,
as: 'permissions',
},
{
+ attributes: ['name'],
model: db.NationalCenter,
as: 'nationalCenters',
+ where: {
+ name: identifier,
+ },
+ required: true,
},
],
});
- if (!user) throw new Error(`User ${creator} does not exist`);
+
+ if (!user) throw new Error(`User associated with National Center: ${identifier} does not exist`);
return user;
};
+const checkUserExistsByName = async (name: string) => checkUserExists({
+ name: {
+ [Op.iLike]: name,
+ },
+});
+const checkUserExistsByEmail = async (email: string) => checkUserExists({ email });
+
const checkEventExists = async (eventId: string) => {
const event = await db.EventReportPilot.findOne({
+ attributes: ['id'],
where: {
id: {
[Op.in]: sequelize.literal(
@@ -564,6 +609,7 @@ const checkEventExists = async (eventId: string) => {
},
},
});
+
if (event) throw new Error(`Event ${eventId} already exists`);
};
@@ -581,11 +627,14 @@ export async function csvImport(buffer: Buffer) {
const eventId = cleanLine['Event ID'];
// If the eventId doesn't start with the prefix R and two numbers, it's invalid.
- if (!eventId.match(/^R\d{2}/i)) {
+ const match = eventId.match(/^R\d{2}/i);
+ if (match === null) {
skipped.push(`Invalid "Event ID" format expected R##-TR-#### received ${eventId}`);
return false;
}
+ await checkEventExists(eventId);
+
// Validate audience else skip.
if (!EVENT_AUDIENCE.includes(cleanLine.Audience)) {
skipped.push(`Value "${cleanLine.Audience}" is invalid for column "Audience". Must be of one of ${EVENT_AUDIENCE.join(', ')}: ${eventId}`);
@@ -594,10 +643,15 @@ export async function csvImport(buffer: Buffer) {
const regionId = Number(eventId.split('-')[0].replace(/\D/g, '').replace(/^0+/, ''));
- const creator = cleanLine.Creator;
+ const creator = cleanLine['IST/Creator'] || cleanLine.Creator;
+ if (!creator) {
+ errors.push(`No creator listed on import for ${eventId}`);
+ return false;
+ }
let owner;
if (creator) {
- owner = await checkUserExists(creator);
+ owner = await checkUserExistsByEmail(creator);
+
const policy = new EventReport(owner, {
regionId,
});
@@ -608,33 +662,70 @@ export async function csvImport(buffer: Buffer) {
}
}
- await checkEventExists(eventId);
+ const collaborators = [];
+ const pocs = [];
+
+ if (cleanLine['Designated Region POC for Event/Request']) {
+ const pocNames = cleanLine['Designated Region POC for Event/Request'].split('/').map((name) => name.trim());
+ // eslint-disable-next-line no-restricted-syntax
+ for await (const pocName of pocNames) {
+ const poc = await checkUserExistsByName(pocName);
+ const policy = new EventReport(poc, {
+ regionId,
+ });
+
+ if (!policy.hasPocInRegion()) {
+ errors.push(`User ${pocName} does not have POC permission in region ${regionId}`);
+ return false;
+ }
+ pocs.push(poc.id);
+ }
+ }
- const data = mapLineToData(cleanLine);
+ if (cleanLine['National Centers']) {
+ const nationalCenterNames = cleanLine['National Centers'].split('\n').map((name) => name.trim());
+ // eslint-disable-next-line no-restricted-syntax
+ for await (const center of nationalCenterNames) {
+ const collaborator = await checkUserExistsByNationalCenter(center);
+ const policy = new EventReport(collaborator, {
+ regionId,
+ });
+
+ if (!policy.canWriteInRegion()) {
+ errors.push(`User ${collaborator.name} does not have permission to write in region ${regionId}`);
+ return false;
+ }
+ collaborators.push(collaborator.id);
+ }
+ }
+
+ if (!collaborators.length) {
+ errors.push(`No collaborators found for ${eventId}`);
+ return false;
+ }
- data.goals = []; // shape: { grantId: number, goalId: number, sessionId: number }[]
- data.goal = '';
+ const data = mapLineToData(cleanLine);
// Reasons, remove duplicates and invalid values.
- data.reasons = [...new Set(data.reasons as string[])];
- data.reasons = (data.reasons as string[]).filter((reason) => REASONS.includes(reason));
+ data.reasons = [...new Set(data.reasons as string[])].filter((reason) => REASONS.includes(reason));
// Target Populations, remove duplicates and invalid values.
- data.targetPopulations = [...new Set(data.targetPopulations as string[])];
- data.targetPopulations = (data.targetPopulations as string[]).filter((target) => TARGET_POPULATIONS.includes(target));
+ data.targetPopulations = [...new Set(data.targetPopulations as string[])].filter((target) => [...TARGET_POPULATIONS, ...EVENT_TARGET_POPULATIONS].includes(target));
await db.EventReportPilot.create({
- collaboratorIds: [],
+ collaboratorIds: collaborators,
ownerId: owner.id,
regionId,
+ pocIds: pocs,
data: sequelize.cast(JSON.stringify(data), 'jsonb'),
imported: sequelize.cast(JSON.stringify(cleanLine), 'jsonb'),
});
return true;
} catch (error) {
- if (error.message.startsWith('User')) {
- errors.push(error.message);
+ const message = (error.message || '').replace(/\/t/g, '');
+ if (message.startsWith('User')) {
+ errors.push(message);
} else if (error.message.startsWith('Event')) {
skipped.push(line['Event ID']);
}
diff --git a/yarn-audit-known-issues b/yarn-audit-known-issues
index 49bae093ea..83ed067419 100644
--- a/yarn-audit-known-issues
+++ b/yarn-audit-known-issues
@@ -1,4 +1,5 @@
{"type":"auditAdvisory","data":{"resolution":{"id":1096366,"path":"email-templates>preview-email>mailparser>nodemailer","dev":false,"optional":false,"bundled":false},"advisory":{"findings":[{"version":"6.7.3","paths":["email-templates>preview-email>mailparser>nodemailer"]}],"metadata":null,"vulnerable_versions":"<=6.9.8","module_name":"nodemailer","severity":"moderate","github_advisory_id":"GHSA-9h6g-pr28-7cqp","cves":[],"access":"public","patched_versions":">=6.9.9","cvss":{"score":5.3,"vectorString":"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:L"},"updated":"2024-02-01T17:58:50.000Z","recommendation":"Upgrade to version 6.9.9 or later","cwe":["CWE-1333"],"found_by":null,"deleted":null,"id":1096366,"references":"- https://github.com/nodemailer/nodemailer/security/advisories/GHSA-9h6g-pr28-7cqp\n- https://gist.github.com/francoatmega/890dd5053375333e40c6fdbcc8c58df6\n- https://gist.github.com/francoatmega/9aab042b0b24968d7b7039818e8b2698\n- https://github.com/nodemailer/nodemailer/commit/dd8f5e8a4ddc99992e31df76bcff9c590035cd4a\n- https://github.com/advisories/GHSA-9h6g-pr28-7cqp","created":"2024-01-31T22:42:54.000Z","reported_by":null,"title":"nodemailer ReDoS when trying to send a specially crafted email","npm_advisory_id":null,"overview":"### Summary\nA ReDoS vulnerability occurs when nodemailer tries to parse img files with the parameter `attachDataUrls` set, causing the stuck of event loop. \nAnother flaw was found when nodemailer tries to parse an attachments with a embedded file, causing the stuck of event loop. \n\n### Details\n\nRegex: /^data:((?:[^;]*;)*(?:[^,]*)),(.*)$/\n\nPath: compile -> getAttachments -> _processDataUrl\n\nRegex: /(]* src\\s*=[\\s\"']*)(data:([^;]+);[^\"'>\\s]+)/\n\nPath: _convertDataImages\n\n### PoC\n\nhttps://gist.github.com/francoatmega/890dd5053375333e40c6fdbcc8c58df6\nhttps://gist.github.com/francoatmega/9aab042b0b24968d7b7039818e8b2698\n\n### Impact\n\nReDoS causes the event loop to stuck a specially crafted evil email can cause this problem.\n","url":"https://github.com/advisories/GHSA-9h6g-pr28-7cqp"}}}
+{"type":"auditAdvisory","data":{"resolution":{"id":1098307,"path":"newrelic>@newrelic/security-agent>@aws-sdk/client-lambda>@aws-sdk/client-sts>fast-xml-parser","dev":false,"optional":false,"bundled":false},"advisory":{"findings":[{"version":"4.2.5","paths":["newrelic>@newrelic/security-agent>@aws-sdk/client-lambda>@aws-sdk/client-sts>fast-xml-parser"]}],"metadata":null,"vulnerable_versions":"<4.4.1","module_name":"fast-xml-parser","severity":"high","github_advisory_id":"GHSA-mpg4-rc92-vx8v","cves":["CVE-2024-41818"],"access":"public","patched_versions":">=4.4.1","cvss":{"score":7.5,"vectorString":"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H"},"updated":"2024-07-29T19:47:38.000Z","recommendation":"Upgrade to version 4.4.1 or later","cwe":["CWE-400"],"found_by":null,"deleted":null,"id":1098307,"references":"- https://github.com/NaturalIntelligence/fast-xml-parser/security/advisories/GHSA-mpg4-rc92-vx8v\n- https://github.com/NaturalIntelligence/fast-xml-parser/commit/d0bfe8a3a2813a185f39591bbef222212d856164\n- https://github.com/NaturalIntelligence/fast-xml-parser/blob/master/src/v5/valueParsers/currency.js#L10\n- https://nvd.nist.gov/vuln/detail/CVE-2024-41818\n- https://github.com/advisories/GHSA-mpg4-rc92-vx8v","created":"2024-07-29T17:46:16.000Z","reported_by":null,"title":"fast-xml-parser vulnerable to ReDOS at currency parsing","npm_advisory_id":null,"overview":"### Summary\nA ReDOS exists on currency.js was discovered by Gauss Security Labs R&D team.\n\n### Details\nhttps://github.com/NaturalIntelligence/fast-xml-parser/blob/master/src/v5/valueParsers/currency.js#L10\ncontains a vulnerable regex \n\n### PoC\npass the following string '\\t'.repeat(13337) + '.'\n\n### Impact\nDenial of service during currency parsing in experimental version 5 of fast-xml-parser-library\n\nhttps://gauss-security.com","url":"https://github.com/advisories/GHSA-mpg4-rc92-vx8v"}}}
{"type":"auditAdvisory","data":{"resolution":{"id":1096410,"path":"xml2json>hoek","dev":false,"bundled":false,"optional":false},"advisory":{"findings":[{"version":"4.2.1","paths":["xml2json>hoek"]},{"version":"5.0.4","paths":["xml2json>joi>hoek"]},{"version":"6.1.3","paths":["xml2json>joi>topo>hoek"]}],"metadata":null,"vulnerable_versions":"<=6.1.3","module_name":"hoek","severity":"high","github_advisory_id":"GHSA-c429-5p7v-vgjp","cves":["CVE-2020-36604"],"access":"public","patched_versions":"<0.0.0","cvss":{"score":8.1,"vectorString":"CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H"},"updated":"2024-02-07T18:59:37.000Z","recommendation":"None","cwe":["CWE-1321"],"found_by":null,"deleted":null,"id":1096410,"references":"- https://nvd.nist.gov/vuln/detail/CVE-2020-36604\n- https://github.com/hapijs/hoek/issues/352\n- https://github.com/hapijs/hoek/commit/4d0804bc6135ad72afdc5e1ec002b935b2f5216a\n- https://github.com/hapijs/hoek/commit/948baf98634a5c206875b67d11368f133034fa90\n- https://github.com/advisories/GHSA-c429-5p7v-vgjp","created":"2022-09-25T00:00:27.000Z","reported_by":null,"title":"hoek subject to prototype pollution via the clone function.","npm_advisory_id":null,"overview":"hoek versions prior to 8.5.1, and 9.x prior to 9.0.3 are vulnerable to prototype pollution in the clone function. If an object with the __proto__ key is passed to clone() the key is converted to a prototype. This issue has been patched in version 9.0.3, and backported to 8.5.1. ","url":"https://github.com/advisories/GHSA-c429-5p7v-vgjp"}}}
{"type":"auditAdvisory","data":{"resolution":{"id":1096410,"path":"xml2json>joi>hoek","dev":false,"bundled":false,"optional":false},"advisory":{"findings":[{"version":"4.2.1","paths":["xml2json>hoek"]},{"version":"5.0.4","paths":["xml2json>joi>hoek"]},{"version":"6.1.3","paths":["xml2json>joi>topo>hoek"]}],"metadata":null,"vulnerable_versions":"<=6.1.3","module_name":"hoek","severity":"high","github_advisory_id":"GHSA-c429-5p7v-vgjp","cves":["CVE-2020-36604"],"access":"public","patched_versions":"<0.0.0","cvss":{"score":8.1,"vectorString":"CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H"},"updated":"2024-02-07T18:59:37.000Z","recommendation":"None","cwe":["CWE-1321"],"found_by":null,"deleted":null,"id":1096410,"references":"- https://nvd.nist.gov/vuln/detail/CVE-2020-36604\n- https://github.com/hapijs/hoek/issues/352\n- https://github.com/hapijs/hoek/commit/4d0804bc6135ad72afdc5e1ec002b935b2f5216a\n- https://github.com/hapijs/hoek/commit/948baf98634a5c206875b67d11368f133034fa90\n- https://github.com/advisories/GHSA-c429-5p7v-vgjp","created":"2022-09-25T00:00:27.000Z","reported_by":null,"title":"hoek subject to prototype pollution via the clone function.","npm_advisory_id":null,"overview":"hoek versions prior to 8.5.1, and 9.x prior to 9.0.3 are vulnerable to prototype pollution in the clone function. If an object with the __proto__ key is passed to clone() the key is converted to a prototype. This issue has been patched in version 9.0.3, and backported to 8.5.1. ","url":"https://github.com/advisories/GHSA-c429-5p7v-vgjp"}}}
{"type":"auditAdvisory","data":{"resolution":{"id":1096410,"path":"xml2json>joi>topo>hoek","dev":false,"bundled":false,"optional":false},"advisory":{"findings":[{"version":"4.2.1","paths":["xml2json>hoek"]},{"version":"5.0.4","paths":["xml2json>joi>hoek"]},{"version":"6.1.3","paths":["xml2json>joi>topo>hoek"]}],"metadata":null,"vulnerable_versions":"<=6.1.3","module_name":"hoek","severity":"high","github_advisory_id":"GHSA-c429-5p7v-vgjp","cves":["CVE-2020-36604"],"access":"public","patched_versions":"<0.0.0","cvss":{"score":8.1,"vectorString":"CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H"},"updated":"2024-02-07T18:59:37.000Z","recommendation":"None","cwe":["CWE-1321"],"found_by":null,"deleted":null,"id":1096410,"references":"- https://nvd.nist.gov/vuln/detail/CVE-2020-36604\n- https://github.com/hapijs/hoek/issues/352\n- https://github.com/hapijs/hoek/commit/4d0804bc6135ad72afdc5e1ec002b935b2f5216a\n- https://github.com/hapijs/hoek/commit/948baf98634a5c206875b67d11368f133034fa90\n- https://github.com/advisories/GHSA-c429-5p7v-vgjp","created":"2022-09-25T00:00:27.000Z","reported_by":null,"title":"hoek subject to prototype pollution via the clone function.","npm_advisory_id":null,"overview":"hoek versions prior to 8.5.1, and 9.x prior to 9.0.3 are vulnerable to prototype pollution in the clone function. If an object with the __proto__ key is passed to clone() the key is converted to a prototype. This issue has been patched in version 9.0.3, and backported to 8.5.1. ","url":"https://github.com/advisories/GHSA-c429-5p7v-vgjp"}}}