From b5a615d5e45bae7f753e0463857654bc290c2d69 Mon Sep 17 00:00:00 2001 From: Beau Dacious Date: Fri, 14 Dec 2018 15:02:13 -0800 Subject: [PATCH 01/14] use node references where applicable --- nodes.js | 68 +++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 65 insertions(+), 3 deletions(-) diff --git a/nodes.js b/nodes.js index 27e8649..26d351d 100644 --- a/nodes.js +++ b/nodes.js @@ -1,7 +1,8 @@ const createNodeHelpers = require('gatsby-node-helpers').default; const { - createNodeFactory + createNodeFactory, + generateNodeId } = createNodeHelpers({ typePrefix: 'Ghost' }); @@ -11,7 +12,68 @@ const PAGE = 'Page'; const TAG = 'Tag'; const AUTHOR = 'Author'; -module.exports.PostNode = createNodeFactory(POST); -module.exports.PageNode = createNodeFactory(PAGE); +const replacePropWithNodeRef = (node, prop, refId) => { + node[`${prop}___NODE`] = + typeof refId === 'function' ? refId(node[prop]) : refId; + delete node[prop]; +}; + +module.exports.PostNode = createNodeFactory(POST, (node) => { + if (node.primary_tag) { + replacePropWithNodeRef( + node, + 'primary_tag', + generateNodeId(TAG, node.primary_tag.id) + ); + } + if (node.primary_author) { + replacePropWithNodeRef( + node, + 'primary_author', + generateNodeId(AUTHOR, node.primary_author.id) + ); + } + if (node.tags) { + replacePropWithNodeRef( + node, + 'tags', + tags => tags.map(t => generateNodeId(TAG, t.id)) + ); + } + if (node.authors) { + replacePropWithNodeRef( + node, + 'authors', + authors => authors.map(a => generateNodeId(AUTHOR, a.id)) + ); + } + return node; +}); + +module.exports.PageNode = createNodeFactory(PAGE, (node) => { + if (node.primary_author) { + replacePropWithNodeRef( + node, + 'primary_author', + generateNodeId(AUTHOR, node.primary_author.id) + ); + } + if (node.tags) { + replacePropWithNodeRef( + node, + 'tags', + tags => tags.map(t => generateNodeId(TAG, t.id)) + ); + } + if (node.authors) { + replacePropWithNodeRef( + node, + 'authors', + authors => authors.map(a => generateNodeId(AUTHOR, a.id)) + ); + } + return node; +}); + module.exports.TagNode = createNodeFactory(TAG); module.exports.AuthorNode = createNodeFactory(AUTHOR); From 5521efe26b5ec644aed4fdfe0bf712151c893048 Mon Sep 17 00:00:00 2001 From: Beau Dacious Date: Sat, 15 Dec 2018 12:57:51 -0800 Subject: [PATCH 02/14] allow async arrow functions for eslint --- .eslintrc.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.eslintrc.js b/.eslintrc.js index 6a5eab5..1d421c3 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -1,4 +1,7 @@ module.exports = { + parserOptions: { + ecmaVersion: 8, + }, plugins: ['ghost'], extends: [ 'plugin:ghost/node', From 2aafc47efbd35ce25945fdf5f7f1bd196ee47d85 Mon Sep 17 00:00:00 2001 From: Beau Dacious Date: Sat, 15 Dec 2018 13:01:07 -0800 Subject: [PATCH 03/14] fetch from Ghost /tags and /users endpoints This will allow for working with tags and users in Gatsby that do not yet have posts associated with them. --- api.js | 64 ++++++++++++++++++++++++++++++++++++++++---------- gatsby-node.js | 41 ++++++++++++++++---------------- 2 files changed, 71 insertions(+), 34 deletions(-) diff --git a/api.js b/api.js index c93c954..e40b3a4 100644 --- a/api.js +++ b/api.js @@ -3,7 +3,7 @@ const qs = require('qs'); const printError = (...args) => console.error('\n', ...args); // eslint-disable-line no-console -module.exports.fetchAllPosts = (options) => { +const validatePluginOptions = (options) => { if (!options.clientId || !options.clientSecret || !options.apiUrl) { printError('Plugin Configuration Missing: gatsby-source-ghost requires your apiUrl, clientId and clientSecret'); process.exit(1); @@ -17,24 +17,62 @@ module.exports.fetchAllPosts = (options) => { if (options.apiUrl.substring(0, 8) !== 'https://') { printError('Ghost apiUrl should be served over HTTPS, are you sure you want:', options.apiUrl, '?'); } +}; - const baseApiUrl = `${options.apiUrl}/ghost/api/v0.1`; - const postApiOptions = { +const buildApiConfigFromOptions = options => ({ + baseApiUrl: `${options.apiUrl}/ghost/api/v0.1`, + baseApiOptions: { client_id: options.clientId, client_secret: options.clientSecret, + absolute_urls: true, + limit: 'all' + } +}); + +const get = (url, successCallback) => axios.get(url) + .then(successCallback) + .catch((err) => { + printError('Error:', err); + printError('Unable to fetch data from your Ghost API. Perhaps your credentials or apiUrl are incorrect?'); + process.exit(1); + }); + +module.exports.fetchAllPosts = (options) => { + validatePluginOptions(options); + + const {baseApiUrl, baseApiOptions} = buildApiConfigFromOptions(options); + const postApiOptions = Object.assign({}, baseApiOptions, { include: 'authors,tags', filter: 'page:[true,false]', - formats: 'plaintext,html', + formats: 'plaintext,html' + }); + const postsApiUrl = `${baseApiUrl}/posts/?${qs.stringify(postApiOptions)}`; + + return get(postsApiUrl, res => res.data.posts); +}; + +module.exports.fetchAllTags = (options) => { + validatePluginOptions(options); + + const {baseApiUrl, baseApiOptions} = buildApiConfigFromOptions(options); + const postApiOptions = Object.assign({}, baseApiOptions, { absolute_urls: true, limit: 'all' - }; - const postsApiUrl = `${baseApiUrl}/posts/?${qs.stringify(postApiOptions)}`; + }); + const tagsApiUrl = `${baseApiUrl}/tags/?${qs.stringify(postApiOptions)}`; + + return get(tagsApiUrl, res => res.data.tags); +}; + +module.exports.fetchAllUsers = (options) => { + validatePluginOptions(options); + + const {baseApiUrl, baseApiOptions} = buildApiConfigFromOptions(options); + const postApiOptions = Object.assign({}, baseApiOptions, { + absolute_urls: true, + limit: 'all' + }); + const usersApiUrl = `${baseApiUrl}/users/?${qs.stringify(postApiOptions)}`; - return axios.get(postsApiUrl) - .then(res => res.data.posts) - .catch((err) => { - printError('Error:', err); - printError('Unable to fetch data from your Ghost API. Perhaps your credentials or apiUrl are incorrect?'); - process.exit(1); - }); + return get(usersApiUrl, res => res.data.users); }; diff --git a/gatsby-node.js b/gatsby-node.js index ebb55c4..02d027e 100644 --- a/gatsby-node.js +++ b/gatsby-node.js @@ -1,30 +1,29 @@ const GhostAPI = require('./api'); const {PostNode, PageNode, TagNode, AuthorNode} = require('./nodes'); -exports.sourceNodes = ({boundActionCreators}, configOptions) => { +exports.sourceNodes = async ({boundActionCreators}, configOptions) => { const {createNode} = boundActionCreators; - return GhostAPI - .fetchAllPosts(configOptions) - .then((posts) => { - posts.forEach((post) => { - if (post.page) { - createNode(PageNode(post)); - } else { - createNode(PostNode(post)); - } - if (post.tags) { - post.tags.forEach((tag) => { - createNode(TagNode(tag)); - }); - } + return Promise.all([ + GhostAPI.fetchAllPosts(configOptions), + GhostAPI.fetchAllTags(configOptions), + GhostAPI.fetchAllUsers(configOptions) + ]).then(([posts, tags, users]) => { + posts.filter(p => p.page).forEach((page) => { + createNode(PageNode(page)); + }); + + posts.filter(p => !p.page).forEach((post) => { + createNode(PostNode(post)); + }); + + tags.forEach((tag) => { + createNode(TagNode(tag)); + }); - if (post.authors) { - post.authors.forEach((author) => { - createNode(AuthorNode(author)); - }); - } - }); + users.forEach((user) => { + createNode(AuthorNode(user)); }); + }); }; From c7f06716536d23d97400ca7794b5510c0dd6e33b Mon Sep 17 00:00:00 2001 From: Beau Dacious Date: Sun, 16 Dec 2018 17:29:51 -0800 Subject: [PATCH 04/14] create backreferences during node creation --- gatsby-node.js | 33 ++++++------- nodes.js | 127 ++++++++++++++++++++++++++----------------------- 2 files changed, 84 insertions(+), 76 deletions(-) diff --git a/gatsby-node.js b/gatsby-node.js index 02d027e..95bcfdc 100644 --- a/gatsby-node.js +++ b/gatsby-node.js @@ -1,29 +1,30 @@ const GhostAPI = require('./api'); -const {PostNode, PageNode, TagNode, AuthorNode} = require('./nodes'); - -exports.sourceNodes = async ({boundActionCreators}, configOptions) => { - const {createNode} = boundActionCreators; +const {createNodeFactories} = require('./nodes'); +exports.sourceNodes = async ({actions}, configOptions) => { + const {createNode} = actions; return Promise.all([ GhostAPI.fetchAllPosts(configOptions), GhostAPI.fetchAllTags(configOptions), GhostAPI.fetchAllUsers(configOptions) ]).then(([posts, tags, users]) => { - posts.filter(p => p.page).forEach((page) => { - createNode(PageNode(page)); - }); + const { + buildPostNode, + buildPageNode, + buildTagNode, + buildAuthorNode + } = createNodeFactories({posts, tags, users}); - posts.filter(p => !p.page).forEach((post) => { - createNode(PostNode(post)); - }); + posts + .filter(p => !p.page) + .forEach(post => createNode(buildPostNode(post))); - tags.forEach((tag) => { - createNode(TagNode(tag)); - }); + posts + .filter(p => p.page) + .forEach(page => createNode(buildPageNode(page))); - users.forEach((user) => { - createNode(AuthorNode(user)); - }); + tags.forEach(tag => createNode(buildTagNode(tag))); + users.forEach(user => createNode(buildAuthorNode(user))); }); }; diff --git a/nodes.js b/nodes.js index 26d351d..8d168ca 100644 --- a/nodes.js +++ b/nodes.js @@ -12,68 +12,75 @@ const PAGE = 'Page'; const TAG = 'Tag'; const AUTHOR = 'Author'; -const replacePropWithNodeRef = (node, prop, refId) => { - node[`${prop}___NODE`] = - typeof refId === 'function' ? refId(node[prop]) : refId; - delete node[prop]; -}; +function mapPostToTags(post, tags) { + const postHasTags = post.tags && Array.isArray(post.tags) && post.tags.length; -module.exports.PostNode = createNodeFactory(POST, (node) => { - if (node.primary_tag) { - replacePropWithNodeRef( - node, - 'primary_tag', - generateNodeId(TAG, node.primary_tag.id) - ); - } - if (node.primary_author) { - replacePropWithNodeRef( - node, - 'primary_author', - generateNodeId(AUTHOR, node.primary_author.id) - ); - } - if (node.tags) { - replacePropWithNodeRef( - node, - 'tags', - tags => tags.map(t => generateNodeId(TAG, t.id)) - ); - } - if (node.authors) { - replacePropWithNodeRef( - node, - 'authors', - authors => authors.map(a => generateNodeId(AUTHOR, a.id)) - ); - } - return node; -}); + if (postHasTags) { + // replace tags with links to their nodes + post.tags___NODE = post.tags.map(t => generateNodeId(TAG, t.id)); -module.exports.PageNode = createNodeFactory(PAGE, (node) => { - if (node.primary_author) { - replacePropWithNodeRef( - node, - 'primary_author', - generateNodeId(AUTHOR, node.primary_author.id) - ); - } - if (node.tags) { - replacePropWithNodeRef( - node, - 'tags', - tags => tags.map(t => generateNodeId(TAG, t.id)) - ); + // add a backreference for this post to the tags + post.tags.forEach(({id: tagId}) => { + const tag = tags.find(t => t.id === tagId); + if (!tag.posts___NODE) { + tag.posts___NODE = []; + } + tag.posts___NODE.push(post.id); + }); + + // replace primary_tag with a link to the tag node + if (post.primary_tag) { + post.primary_tag___NODE = generateNodeId(TAG, post.primary_tag.id); + } + + delete post.tags; + delete post.primary_tag; } - if (node.authors) { - replacePropWithNodeRef( - node, - 'authors', - authors => authors.map(a => generateNodeId(AUTHOR, a.id)) - ); +} + +function mapPostToUsers(post, users) { + const postHasAuthors = post.authors && Array.isArray(post.authors) && post.authors.length; + + if (postHasAuthors) { + // replace authors with links to their (user) nodes + post.authors___NODE = post.authors.map(a => generateNodeId(AUTHOR, a.id)); + + // add a backreference for this post to the user + post.authors.forEach(({id: authorId}) => { + const user = users.find(u => u.id === authorId); + if (!user.posts___NODE) { + user.posts___NODE = []; + } + user.posts___NODE.push(post.id); + }); + + // replace primary_author with a link to the user node + if (post.primary_author) { + post.primary_author___NODE = generateNodeId(AUTHOR, post.primary_author.id); + } + + delete post.authors; + delete post.primary_author; } - return node; -}); +} + +module.exports.createNodeFactories = ({tags, users}) => { + const postNodeMiddleware = (node) => { + mapPostToTags(node, tags); + mapPostToUsers(node, users); + return node; + }; + + const buildPostNode = createNodeFactory(POST, postNodeMiddleware); + const buildPageNode = createNodeFactory(PAGE, postNodeMiddleware); + const buildTagNode = createNodeFactory(TAG); + const buildAuthorNode = createNodeFactory(AUTHOR); + + return { + buildPostNode, + buildPageNode, + buildTagNode, + buildAuthorNode + }; +}; -module.exports.TagNode = createNodeFactory(TAG); -module.exports.AuthorNode = createNodeFactory(AUTHOR); From 9e29735e22630926166aa2325237476888296735 Mon Sep 17 00:00:00 2001 From: Beau Dacious Date: Sun, 16 Dec 2018 18:43:36 -0800 Subject: [PATCH 05/14] rewrite api functions to share more code --- api.js | 88 ++++++++++++++++++++++++++++++++-------------------------- 1 file changed, 48 insertions(+), 40 deletions(-) diff --git a/api.js b/api.js index e40b3a4..203a65d 100644 --- a/api.js +++ b/api.js @@ -3,76 +3,84 @@ const qs = require('qs'); const printError = (...args) => console.error('\n', ...args); // eslint-disable-line no-console -const validatePluginOptions = (options) => { - if (!options.clientId || !options.clientSecret || !options.apiUrl) { +const validateOptions = ({clientId, clientSecret, apiUrl}) => { + if (!clientId || !clientSecret || !apiUrl) { printError('Plugin Configuration Missing: gatsby-source-ghost requires your apiUrl, clientId and clientSecret'); process.exit(1); } - if (options.apiUrl.substring(0, 4) !== 'http') { + if (apiUrl.substring(0, 4) !== 'http') { printError('Ghost apiUrl requires a protocol, E.g. https://.ghost.io'); process.exit(1); } - if (options.apiUrl.substring(0, 8) !== 'https://') { - printError('Ghost apiUrl should be served over HTTPS, are you sure you want:', options.apiUrl, '?'); + if (apiUrl.substring(0, 8) !== 'https://') { + printError('Ghost apiUrl should be served over HTTPS, are you sure you want:', apiUrl, '?'); } }; -const buildApiConfigFromOptions = options => ({ - baseApiUrl: `${options.apiUrl}/ghost/api/v0.1`, - baseApiOptions: { - client_id: options.clientId, - client_secret: options.clientSecret, +const exitOnApiError = (err) => { + printError('Error:', err); + printError('Unable to fetch data from your Ghost API. Perhaps your credentials or apiUrl are incorrect?'); + process.exit(1); +}; + +const createApiHelpers = ({clientId, clientSecret, apiUrl}) => { + const baseApiUrl = `${apiUrl}/ghost/api/v0.1`; + + const baseApiParams = { + client_id: clientId, + client_secret: clientSecret, absolute_urls: true, limit: 'all' - } -}); + }; -const get = (url, successCallback) => axios.get(url) - .then(successCallback) - .catch((err) => { - printError('Error:', err); - printError('Unable to fetch data from your Ghost API. Perhaps your credentials or apiUrl are incorrect?'); - process.exit(1); - }); + const extendParams = params => Object.assign({}, baseApiParams, params); + + const buildApiUrl = (endpoint, params = {}) => { + const query = qs.stringify(extendParams(params)); + return `${baseApiUrl}/${endpoint}/?${query}`; + }; + + return { + extendParams, + buildApiUrl + }; +}; module.exports.fetchAllPosts = (options) => { - validatePluginOptions(options); + validateOptions(options); - const {baseApiUrl, baseApiOptions} = buildApiConfigFromOptions(options); - const postApiOptions = Object.assign({}, baseApiOptions, { + const {buildApiUrl} = createApiHelpers(options); + const postsUrl = buildApiUrl('posts', { include: 'authors,tags', filter: 'page:[true,false]', formats: 'plaintext,html' }); - const postsApiUrl = `${baseApiUrl}/posts/?${qs.stringify(postApiOptions)}`; - return get(postsApiUrl, res => res.data.posts); + return axios.get(postsUrl) + .then(res => res.data.posts) + .catch(exitOnApiError); }; module.exports.fetchAllTags = (options) => { - validatePluginOptions(options); + validateOptions(options); - const {baseApiUrl, baseApiOptions} = buildApiConfigFromOptions(options); - const postApiOptions = Object.assign({}, baseApiOptions, { - absolute_urls: true, - limit: 'all' - }); - const tagsApiUrl = `${baseApiUrl}/tags/?${qs.stringify(postApiOptions)}`; + const {buildApiUrl} = createApiHelpers(options); + const tagsUrl = buildApiUrl('tags'); - return get(tagsApiUrl, res => res.data.tags); + return axios.get(tagsUrl) + .then(res => res.data.tags) + .catch(exitOnApiError); }; module.exports.fetchAllUsers = (options) => { - validatePluginOptions(options); + validateOptions(options); - const {baseApiUrl, baseApiOptions} = buildApiConfigFromOptions(options); - const postApiOptions = Object.assign({}, baseApiOptions, { - absolute_urls: true, - limit: 'all' - }); - const usersApiUrl = `${baseApiUrl}/users/?${qs.stringify(postApiOptions)}`; + const {buildApiUrl} = createApiHelpers(options); + const usersUrl = buildApiUrl('users'); - return get(usersApiUrl, res => res.data.users); + return axios.get(usersUrl) + .then(res => res.data.users) + .catch(exitOnApiError); }; From a74abade0e02ed4f88b7feff8b128eb7a7917544 Mon Sep 17 00:00:00 2001 From: Beau Dacious Date: Sat, 15 Dec 2018 10:35:54 -0800 Subject: [PATCH 06/14] use sinon object as sandbox From the docs: > Since `sinon@5.0.0`, the `sinon` object is a default sandbox. Unless > you have a very advanced setup or need a special configuration, you > probably want to just use that one. https://sinonjs.org/releases/v7.2.2/sandbox/#default-sandbox --- test/gatsby-node.test.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/gatsby-node.test.js b/test/gatsby-node.test.js index 2d8024c..08de743 100644 --- a/test/gatsby-node.test.js +++ b/test/gatsby-node.test.js @@ -1,7 +1,6 @@ // Switch these lines once there are useful utils // const testUtils = require('./utils'); require('./utils'); -const sandbox = sinon.createSandbox(); // Thing we are testing const gatsbyNode = require('../gatsby-node'); @@ -9,14 +8,14 @@ const GhostAPI = require('../api'); describe('Basic Functionality ', function () { afterEach(() => { - sandbox.restore(); + sinon.restore(); }); it('Gatsby Node does roughly the right thing', function (done) { - const createNode = sandbox.stub(); + const createNode = sinon.stub(); // Pass in some fake data - sandbox.stub(GhostAPI, 'fetchAllPosts').resolves([ + sinon.stub(GhostAPI, 'fetchAllPosts').resolves([ {slug: 'welcome-to-ghost', page: false, tags: [ {slug: 'getting-started'}, {slug: 'hash-feature-img'} From c5c9f434baac84c6c442cb179924b003e170d6af Mon Sep 17 00:00:00 2001 From: Beau Dacious Date: Sun, 16 Dec 2018 19:02:27 -0800 Subject: [PATCH 07/14] update main test so that it passes --- test/gatsby-node.test.js | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/test/gatsby-node.test.js b/test/gatsby-node.test.js index 08de743..066003b 100644 --- a/test/gatsby-node.test.js +++ b/test/gatsby-node.test.js @@ -26,8 +26,18 @@ describe('Basic Functionality ', function () { {slug: 'about', page: true} ]); + sinon.stub(GhostAPI, 'fetchAllTags').resolves([ + {slug: 'getting-started'}, + {slug: 'hash-feature-img'} + ]); + + sinon.stub(GhostAPI, 'fetchAllUsers').resolves([ + {name: 'Ghost Writer'}, + {name: 'Ghost Author'} + ]); + gatsbyNode - .sourceNodes({boundActionCreators: {createNode}}, {}) + .sourceNodes({actions: {createNode}}, {}) .then(() => { createNode.callCount.should.eql(6); @@ -35,11 +45,11 @@ describe('Basic Functionality ', function () { // Check that we get the right type of node created getArg(0).internal.should.have.property('type', 'GhostPost'); - getArg(1).internal.should.have.property('type', 'GhostTag'); + getArg(1).internal.should.have.property('type', 'GhostPage'); getArg(2).internal.should.have.property('type', 'GhostTag'); - getArg(3).internal.should.have.property('type', 'GhostAuthor'); + getArg(3).internal.should.have.property('type', 'GhostTag'); getArg(4).internal.should.have.property('type', 'GhostAuthor'); - getArg(5).internal.should.have.property('type', 'GhostPage'); + getArg(5).internal.should.have.property('type', 'GhostAuthor'); done(); }) From 96b78d8362dfbbc40ac0366c1d5ce8c8c2418552 Mon Sep 17 00:00:00 2001 From: Beau Dacious Date: Mon, 17 Dec 2018 12:12:42 -0800 Subject: [PATCH 08/14] remove GhostPage type --- gatsby-node.js | 10 +--------- nodes.js | 3 --- test/gatsby-node.test.js | 2 +- 3 files changed, 2 insertions(+), 13 deletions(-) diff --git a/gatsby-node.js b/gatsby-node.js index 95bcfdc..03dd82e 100644 --- a/gatsby-node.js +++ b/gatsby-node.js @@ -11,19 +11,11 @@ exports.sourceNodes = async ({actions}, configOptions) => { ]).then(([posts, tags, users]) => { const { buildPostNode, - buildPageNode, buildTagNode, buildAuthorNode } = createNodeFactories({posts, tags, users}); - posts - .filter(p => !p.page) - .forEach(post => createNode(buildPostNode(post))); - - posts - .filter(p => p.page) - .forEach(page => createNode(buildPageNode(page))); - + posts.forEach(post => createNode(buildPostNode(post))); tags.forEach(tag => createNode(buildTagNode(tag))); users.forEach(user => createNode(buildAuthorNode(user))); }); diff --git a/nodes.js b/nodes.js index 8d168ca..be1cd52 100644 --- a/nodes.js +++ b/nodes.js @@ -8,7 +8,6 @@ const { }); const POST = 'Post'; -const PAGE = 'Page'; const TAG = 'Tag'; const AUTHOR = 'Author'; @@ -72,13 +71,11 @@ module.exports.createNodeFactories = ({tags, users}) => { }; const buildPostNode = createNodeFactory(POST, postNodeMiddleware); - const buildPageNode = createNodeFactory(PAGE, postNodeMiddleware); const buildTagNode = createNodeFactory(TAG); const buildAuthorNode = createNodeFactory(AUTHOR); return { buildPostNode, - buildPageNode, buildTagNode, buildAuthorNode }; diff --git a/test/gatsby-node.test.js b/test/gatsby-node.test.js index 066003b..dfcef1c 100644 --- a/test/gatsby-node.test.js +++ b/test/gatsby-node.test.js @@ -45,7 +45,7 @@ describe('Basic Functionality ', function () { // Check that we get the right type of node created getArg(0).internal.should.have.property('type', 'GhostPost'); - getArg(1).internal.should.have.property('type', 'GhostPage'); + getArg(1).internal.should.have.property('type', 'GhostPost'); getArg(2).internal.should.have.property('type', 'GhostTag'); getArg(3).internal.should.have.property('type', 'GhostTag'); getArg(4).internal.should.have.property('type', 'GhostAuthor'); From e15ae5a009620dd0a2ec4f1e8cdea6c96b0ce4b2 Mon Sep 17 00:00:00 2001 From: Beau Dacious Date: Mon, 17 Dec 2018 14:48:53 -0800 Subject: [PATCH 09/14] remove empty line at eof --- nodes.js | 1 - 1 file changed, 1 deletion(-) diff --git a/nodes.js b/nodes.js index be1cd52..fc83ded 100644 --- a/nodes.js +++ b/nodes.js @@ -80,4 +80,3 @@ module.exports.createNodeFactories = ({tags, users}) => { buildAuthorNode }; }; - From 2e9117f86b8b97003e126bca73c8d57413d9c2c6 Mon Sep 17 00:00:00 2001 From: Beau Dacious Date: Tue, 18 Dec 2018 14:08:06 -0800 Subject: [PATCH 10/14] convert image urls to media nodes w/ local files --- gatsby-node.js | 42 +++++++++++++++++-------- lib.js | 9 ++++++ nodes.js | 85 +++++++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 118 insertions(+), 18 deletions(-) create mode 100644 lib.js diff --git a/gatsby-node.js b/gatsby-node.js index 03dd82e..95bd09f 100644 --- a/gatsby-node.js +++ b/gatsby-node.js @@ -1,22 +1,38 @@ const GhostAPI = require('./api'); const {createNodeFactories} = require('./nodes'); +const {getImagesFromApiResults} = require('./lib'); -exports.sourceNodes = async ({actions}, configOptions) => { - const {createNode} = actions; +exports.sourceNodes = async ({actions, createNodeId, store, cache}, configOptions) => { + const {createNode, touchNode} = actions; + const imageArgs = {createNode, createNodeId, touchNode, store, cache}; - return Promise.all([ + const [posts, tags, users] = await Promise.all([ GhostAPI.fetchAllPosts(configOptions), GhostAPI.fetchAllTags(configOptions), GhostAPI.fetchAllUsers(configOptions) - ]).then(([posts, tags, users]) => { - const { - buildPostNode, - buildTagNode, - buildAuthorNode - } = createNodeFactories({posts, tags, users}); + ]); - posts.forEach(post => createNode(buildPostNode(post))); - tags.forEach(tag => createNode(buildTagNode(tag))); - users.forEach(user => createNode(buildAuthorNode(user))); - }); + const { + buildPostNode, + buildTagNode, + buildAuthorNode, + buildMediaNode + } = createNodeFactories({posts, tags, users}, imageArgs); + + for (const post of posts) { + createNode(await buildPostNode(post)); + } + + for (const tag of tags) { + createNode(buildTagNode(tag)); + } + + for (const user of users) { + createNode(buildAuthorNode(user)); + } + + const images = getImagesFromApiResults([posts, tags, users]); + for (const image of images) { + createNode(await buildMediaNode(image)); + } }; diff --git a/lib.js b/lib.js new file mode 100644 index 0000000..1b88980 --- /dev/null +++ b/lib.js @@ -0,0 +1,9 @@ +module.exports.getImagesFromApiResults = results => results + .reduce((acc, entities) => acc.concat(entities)) + .reduce((acc, entity) => acc.concat([ + entity.feature_image, + entity.cover_image, + entity.profile_image + ]), []) + .filter(url => !!url) + .map(url => ({id: url, src: url})); diff --git a/nodes.js b/nodes.js index fc83ded..95501d1 100644 --- a/nodes.js +++ b/nodes.js @@ -1,15 +1,48 @@ const createNodeHelpers = require('gatsby-node-helpers').default; +const {createRemoteFileNode} = require('gatsby-source-filesystem'); +const TYPE_PREFIX = 'Ghost'; const { createNodeFactory, generateNodeId } = createNodeHelpers({ - typePrefix: 'Ghost' + typePrefix: TYPE_PREFIX }); const POST = 'Post'; const TAG = 'Tag'; const AUTHOR = 'Author'; +const MEDIA = 'Media'; + +async function downloadImageAndCreateFileNode( + {url}, + {createNode, createNodeId, touchNode, store, cache}, +) { + let fileNodeID; + + const mediaDataCacheKey = `${TYPE_PREFIX}__Media__${url}`; + const cacheMediaData = await cache.get(mediaDataCacheKey); + + if (cacheMediaData) { + fileNodeID = cacheMediaData.fileNodeID; + touchNode({nodeId: fileNodeID}); + return fileNodeID; + } + + const fileNode = await createRemoteFileNode({ + url, + store, + cache, + createNode, + createNodeId + }); + + if (fileNode) { + fileNodeID = fileNode.id; + await cache.set(mediaDataCacheKey, {fileNodeID}); + return fileNodeID; + } +} function mapPostToTags(post, tags) { const postHasTags = post.tags && Array.isArray(post.tags) && post.tags.length; @@ -63,20 +96,62 @@ function mapPostToUsers(post, users) { } } -module.exports.createNodeFactories = ({tags, users}) => { +async function mapImagesToMedia(node) { + if (node.feature_image) { + node.feature_image___NODE = generateNodeId(MEDIA, node.feature_image); + delete node.feature_image; + } + + if (node.profile_image) { + node.profile_image___NODE = generateNodeId(MEDIA, node.profile_image); + delete node.profile_image; + } + + if (node.cover_image) { + node.cover_image___NODE = generateNodeId(MEDIA, node.cover_image); + delete node.cover_image; + } +} + +async function createLocalFileFromMedia(node, imageArgs) { + node.localFile___NODE = await downloadImageAndCreateFileNode( + {url: node.src.split('?')[0]}, + imageArgs + ); +} + +module.exports.createNodeFactories = ({tags, users}, imageArgs) => { const postNodeMiddleware = (node) => { mapPostToTags(node, tags); mapPostToUsers(node, users); + mapImagesToMedia(node); + return node; + }; + + const tagNodeMiddleware = (node) => { + mapImagesToMedia(node); + return node; + }; + + const authorNodeMiddleware = (node) => { + mapImagesToMedia(node); + return node; + }; + + const mediaNodeMiddleware = async (node) => { + await createLocalFileFromMedia(node, imageArgs); return node; }; const buildPostNode = createNodeFactory(POST, postNodeMiddleware); - const buildTagNode = createNodeFactory(TAG); - const buildAuthorNode = createNodeFactory(AUTHOR); + const buildTagNode = createNodeFactory(TAG, tagNodeMiddleware); + const buildAuthorNode = createNodeFactory(AUTHOR, authorNodeMiddleware); + const buildMediaNode = createNodeFactory(MEDIA, mediaNodeMiddleware); return { buildPostNode, buildTagNode, - buildAuthorNode + buildAuthorNode, + buildMediaNode }; }; From 48bb89b2fb28fd53ea64e975aef20e3a32941b93 Mon Sep 17 00:00:00 2001 From: Beau Dacious Date: Wed, 19 Dec 2018 10:58:02 -0800 Subject: [PATCH 11/14] add gatsby-source-filesystem as peer dependency --- package.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/package.json b/package.json index d07c3ce..611970a 100644 --- a/package.json +++ b/package.json @@ -30,5 +30,8 @@ "bluebird": "^3.5.1", "gatsby-node-helpers": "^0.3.0", "qs": "^6.5.2" + }, + "peerDependencies": { + "gatsby-source-filesystem": "^2.0.0" } } From 9d9407065565166e0fa2def210018d1c319392d0 Mon Sep 17 00:00:00 2001 From: Beau Dacious Date: Wed, 19 Dec 2018 11:16:33 -0800 Subject: [PATCH 12/14] set gatsby-source-filesystem as dependency --- package.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/package.json b/package.json index 611970a..dbc7918 100644 --- a/package.json +++ b/package.json @@ -29,9 +29,7 @@ "axios": "^0.18.0", "bluebird": "^3.5.1", "gatsby-node-helpers": "^0.3.0", + "gatsby-source-filesystem": "^2.0.0", "qs": "^6.5.2" - }, - "peerDependencies": { - "gatsby-source-filesystem": "^2.0.0" } } From 7519e5b51608fc049d36ecbd9196264edabdcc52 Mon Sep 17 00:00:00 2001 From: Niko Simonson Date: Fri, 21 Dec 2018 14:51:24 -0800 Subject: [PATCH 13/14] Add og_image and twitter_image as GhostMedia nodes --- lib.js | 15 ++++++++++----- nodes.js | 31 ++++++++++++++++++++++--------- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/lib.js b/lib.js index 1b88980..c62db9f 100644 --- a/lib.js +++ b/lib.js @@ -1,9 +1,14 @@ module.exports.getImagesFromApiResults = results => results .reduce((acc, entities) => acc.concat(entities)) - .reduce((acc, entity) => acc.concat([ - entity.feature_image, - entity.cover_image, - entity.profile_image - ]), []) + .reduce( + (acc, entity) => acc.concat([ + entity.feature_image, + entity.cover_image, + entity.profile_image, + entity.og_image, + entity.twitter_image + ]), + [] + ) .filter(url => !!url) .map(url => ({id: url, src: url})); diff --git a/nodes.js b/nodes.js index 95501d1..70e0de4 100644 --- a/nodes.js +++ b/nodes.js @@ -2,10 +2,7 @@ const createNodeHelpers = require('gatsby-node-helpers').default; const {createRemoteFileNode} = require('gatsby-source-filesystem'); const TYPE_PREFIX = 'Ghost'; -const { - createNodeFactory, - generateNodeId -} = createNodeHelpers({ +const {createNodeFactory, generateNodeId} = createNodeHelpers({ typePrefix: TYPE_PREFIX }); @@ -16,7 +13,7 @@ const MEDIA = 'Media'; async function downloadImageAndCreateFileNode( {url}, - {createNode, createNodeId, touchNode, store, cache}, + {createNode, createNodeId, touchNode, store, cache} ) { let fileNodeID; @@ -45,7 +42,8 @@ async function downloadImageAndCreateFileNode( } function mapPostToTags(post, tags) { - const postHasTags = post.tags && Array.isArray(post.tags) && post.tags.length; + const postHasTags = + post.tags && Array.isArray(post.tags) && post.tags.length; if (postHasTags) { // replace tags with links to their nodes @@ -71,11 +69,13 @@ function mapPostToTags(post, tags) { } function mapPostToUsers(post, users) { - const postHasAuthors = post.authors && Array.isArray(post.authors) && post.authors.length; + const postHasAuthors = + post.authors && Array.isArray(post.authors) && post.authors.length; if (postHasAuthors) { // replace authors with links to their (user) nodes - post.authors___NODE = post.authors.map(a => generateNodeId(AUTHOR, a.id)); + post.authors___NODE = post.authors.map(a => generateNodeId(AUTHOR, a.id) + ); // add a backreference for this post to the user post.authors.forEach(({id: authorId}) => { @@ -88,7 +88,10 @@ function mapPostToUsers(post, users) { // replace primary_author with a link to the user node if (post.primary_author) { - post.primary_author___NODE = generateNodeId(AUTHOR, post.primary_author.id); + post.primary_author___NODE = generateNodeId( + AUTHOR, + post.primary_author.id + ); } delete post.authors; @@ -111,6 +114,16 @@ async function mapImagesToMedia(node) { node.cover_image___NODE = generateNodeId(MEDIA, node.cover_image); delete node.cover_image; } + + if (node.og_image) { + node.og_image___NODE = generateNodeId(MEDIA, node.og_image); + delete node.og_image; + } + + if (node.twitter_image) { + node.twitter_image___NODE = generateNodeId(MEDIA, node.twitter_image); + delete node.twitter_image; + } } async function createLocalFileFromMedia(node, imageArgs) { From 7ac51d4671995c0c712d1f0b84911e9b03f3428f Mon Sep 17 00:00:00 2001 From: Beau Dacious Date: Mon, 14 Jan 2019 10:50:35 -0800 Subject: [PATCH 14/14] move image processing up in code to make it harder to miss --- gatsby-node.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gatsby-node.js b/gatsby-node.js index 881fc94..e7bdaee 100644 --- a/gatsby-node.js +++ b/gatsby-node.js @@ -38,6 +38,11 @@ exports.sourceNodes = async ({actions, createNodeId, store, cache}, configOption buildMediaNode } = createNodeFactories({posts, pages, tags, users}, imageArgs); + const images = getImagesFromApiResults([posts, pages, tags, users]); + for (const image of images) { + createNode(await buildMediaNode(image)); + } + for (const post of posts) { createNode(await buildPostNode(post)); } @@ -53,9 +58,4 @@ exports.sourceNodes = async ({actions, createNodeId, store, cache}, configOption for (const user of users) { createNode(buildAuthorNode(user)); } - - const images = getImagesFromApiResults([posts, pages, tags, users]); - for (const image of images) { - createNode(await buildMediaNode(image)); - } };