From 202278feaf33b82823459bbd227cd47ec9b8e43b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Berthou?= Date: Thu, 4 Mar 2021 17:12:00 +0100 Subject: [PATCH] feat(matomo): add monthly download stats --- package.json | 1 + src/app.ts | 6 +-- src/config.ts | 2 + src/github/github-service.ts | 2 +- src/matomo/matomo-config.ts | 1 + src/matomo/matomo-service.ts | 44 +++++++++++------- src/matomo/matomo-types.ts | 1 + src/matomo/matomo-utils.ts | 86 +++++++++++++++++++++++++++++------- src/utils/date.test.ts | 16 +++++++ src/utils/date.ts | 19 ++++++++ src/utils/fp-util.ts | 7 +++ tsconfig.json | 1 + yarn.lock | 5 +++ 13 files changed, 156 insertions(+), 35 deletions(-) create mode 100644 src/utils/date.test.ts create mode 100644 src/utils/date.ts create mode 100644 src/utils/fp-util.ts diff --git a/package.json b/package.json index 2653dcca..e1247c9d 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "@types/jest": "^26.0.20", "axios": "^0.21.0", "cors": "^2.8.5", + "date-fns": "^2.18.0", "dotenv": "^8.2.0", "express": "^4.17.1", "lodash": "^4.17.20", diff --git a/src/app.ts b/src/app.ts index b8aedcac..b235d8ce 100644 --- a/src/app.ts +++ b/src/app.ts @@ -4,14 +4,12 @@ import { flatten } from "lodash/fp"; import packageJson from "../package.json"; import { createCache } from "./caching/caching-service"; -import { corsOrigins, port } from "./config"; +import { cacheTTL, corsOrigins, port } from "./config"; import { getGitHubData } from "./github/github-service"; import { matomoConfig } from "./matomo/matomo-config"; import { getMultiSiteMatomoData } from "./matomo/matomo-service"; import { getYoutubeData } from "./youtube/youtube-service"; -const CACHE_TTL = 10 * 60 * 1000; - const app = express(); app.use( @@ -35,7 +33,7 @@ const statsCache = createCache( getYoutubeData(), getGitHubData(), ]).then(flatten), - CACHE_TTL + cacheTTL ); app.get("/statistics", (req, res) => { diff --git a/src/config.ts b/src/config.ts index 73c021d4..37afab8d 100644 --- a/src/config.ts +++ b/src/config.ts @@ -11,3 +11,5 @@ export const youtubeChannelId: string = process.env.YOUTUBE_CHANNEL_ID ?? ""; export const githubApiUrl: string = process.env.GITHUB_API_URL ?? ""; export const githubApiKey: string = process.env.GITHUB_API_KEY ?? ""; export const corsOrigins: string = process.env.CORS_ORIGINS ?? ""; +export const cacheTTL: number = + parseInt(process.env.CACHE_TTL ?? "") || 10 * 60 * 1000; diff --git a/src/github/github-service.ts b/src/github/github-service.ts index 63829922..e909dd89 100644 --- a/src/github/github-service.ts +++ b/src/github/github-service.ts @@ -16,6 +16,6 @@ export const getGitHubData = async (): Promise => .then(filterWikiItem) .then(convertGitHubDataToApiData) .catch((err) => { - console.log(err); + console.error(err); return []; }); diff --git a/src/matomo/matomo-config.ts b/src/matomo/matomo-config.ts index a3aa616b..60f46c63 100644 --- a/src/matomo/matomo-config.ts +++ b/src/matomo/matomo-config.ts @@ -25,6 +25,7 @@ export const matomoConfig: MatomoSiteConfig[] = [ ], events: ["download", "appDownload"], idSite: 20, + monthlyEvents: ["download", "appDownload"], visits: true, }, ]; diff --git a/src/matomo/matomo-service.ts b/src/matomo/matomo-service.ts index 43eda291..c6029256 100644 --- a/src/matomo/matomo-service.ts +++ b/src/matomo/matomo-service.ts @@ -1,29 +1,43 @@ import axios from "axios"; -import { flatten } from "lodash/fp"; +import { compose, flatten } from "lodash/fp"; +import querystring from "querystring"; import type { ArchifiltreCountStatistic } from "../api-types"; import { matomoToken, matomoUrl } from "../config"; +import { liftPromise } from "../utils/fp-util"; import type { MatomoEventCategory, MatomoSiteConfig } from "./matomo-types"; import { createMatomoDataSanitizer, getBulkRequestParamsFromConfig, } from "./matomo-utils"; -const getBulkMatomoData = async ( - config: MatomoSiteConfig -): Promise => - axios - .get(matomoUrl, { - params: { - format: "JSON", - method: "API.getBulkRequest", - module: "API", - // eslint-disable-next-line @typescript-eslint/naming-convention - token_auth: matomoToken, - ...getBulkRequestParamsFromConfig(config), - }, +type BulkRequestData = { + data: MatomoEventCategory[][]; +}; + +const makeBulkRequest = async ( + params: Record +): Promise => + axios.post( + matomoUrl, + querystring.stringify({ + format: "JSON", + method: "API.getBulkRequest", + module: "API", + // eslint-disable-next-line @typescript-eslint/naming-convention + token_auth: matomoToken, + ...params, }) - .then(({ data }: { data: MatomoEventCategory[][] }) => data); + ); + +const formatResult = ({ data }: BulkRequestData): MatomoEventCategory[][] => + data; + +const getBulkMatomoData = compose( + liftPromise(formatResult), + makeBulkRequest, + getBulkRequestParamsFromConfig +); export const getMatomoData = async ( config: MatomoSiteConfig diff --git a/src/matomo/matomo-types.ts b/src/matomo/matomo-types.ts index 8c995176..f203f5ef 100644 --- a/src/matomo/matomo-types.ts +++ b/src/matomo/matomo-types.ts @@ -19,5 +19,6 @@ export type MatomoSiteConfig = { idSite: number; events?: MatomoEventConfig[]; actions?: MatomoActionConfigObject[]; + monthlyEvents?: MatomoEventConfig[]; visits?: boolean; }; diff --git a/src/matomo/matomo-utils.ts b/src/matomo/matomo-utils.ts index 25e54eb0..8a90ff47 100644 --- a/src/matomo/matomo-utils.ts +++ b/src/matomo/matomo-utils.ts @@ -1,7 +1,10 @@ -import { flatten } from "lodash"; +import { format, parseISO } from "date-fns/fp"; +import { isString } from "lodash"; +import { compose, map } from "lodash/fp"; import * as querystring from "querystring"; import type { ArchifiltreCountStatistic } from "../api-types"; +import { getLastMonthsRanges } from "../utils/date"; import type { MatomoActionConfigObject, MatomoEventCategory, @@ -10,9 +13,12 @@ import type { MatomoSiteConfig, } from "./matomo-types"; +const MONTHS_REQUESTED = 12; + type CreateMatomoEventCategoryMethodParams = { config: MatomoEventConfig; idSite: number; + date?: [string, string]; }; const sanitizeMatomoEventConfig = ( @@ -25,9 +31,10 @@ const sanitizeMatomoEventConfig = ( : config; const createMatomoRequestBaseParams = ( - idSite: number + idSite: number, + date: [string, string] = ["2020-01-01", "today"] ): Record => ({ - date: "2020-01-01,today", + date: date.join(","), idSite, period: "range", }); @@ -35,13 +42,19 @@ const createMatomoRequestBaseParams = ( const createMatomoEventCategoryMethod = ({ config, idSite, + date, }: CreateMatomoEventCategoryMethodParams) => { const { label } = sanitizeMatomoEventConfig(config); - return querystring.stringify({ - ...createMatomoRequestBaseParams(idSite), - label, - method: "Events.getCategory", - }); + return querystring.stringify( + { + ...createMatomoRequestBaseParams(idSite, date), + label, + method: "Events.getCategory", + }, + "&", + "=", + { encodeURIComponent: (val) => val } + ); }; type CreateMatomoEventActionMethodParams = { @@ -67,9 +80,23 @@ const createMatomoVisitMethod = (idSite: number): string => type RequestParams = Record; +type CreateMonthlyEvenMethodParams = { + config: MatomoEventConfig; + idSite: number; +}; + +const createMonthlyEventMethod = ({ + config, + idSite, +}: CreateMonthlyEvenMethodParams) => + getLastMonthsRanges(MONTHS_REQUESTED)(new Date()).map((dateRange) => + createMatomoEventCategoryMethod({ config, date: dateRange, idSite }) + ); + export const getBulkRequestParamsFromConfig = ({ events = [], actions = [], + monthlyEvents = [], visits = false, idSite, }: MatomoSiteConfig): RequestParams => @@ -80,6 +107,9 @@ export const getBulkRequestParamsFromConfig = ({ ...actions.map((config) => createMatomoEventActionMethod({ config, idSite }) ), + ...monthlyEvents.flatMap((config) => + createMonthlyEventMethod({ config, idSite }) + ), ...(visits ? [createMatomoVisitMethod(idSite)] : []), ].reduce( (urlParams, urlParam, index) => ({ @@ -95,6 +125,32 @@ const formatEventsOrActionsResponse = () => ( // eslint-disable-next-line @typescript-eslint/naming-convention eventCategories.map(({ label, nb_events }) => ({ label, value: nb_events })); +const getConfigLabel = (config: MatomoEventConfig) => + isString(config) ? config : config.label; + +type ResultFormatter = ( + eventCategories: MatomoEventCategory[] +) => ArchifiltreCountStatistic[]; + +const formatResultDate = compose(format("y-MM"), parseISO); + +const formatMonthlyApiResult = (config: MatomoEventConfig, date: string) => ({ + value, +}: ArchifiltreCountStatistic) => ({ + label: `${getConfigLabel(config)}:${formatResultDate(date)}`, + value, +}); + +const formatMonthlyEvents = (config: MatomoEventConfig): ResultFormatter[] => + getLastMonthsRanges(MONTHS_REQUESTED)(new Date()) + .map((date): [string, ResultFormatter] => [ + date[0], + formatEventsOrActionsResponse(), + ]) + .map(([date, formatter]) => + compose(map(formatMonthlyApiResult(config, date)), formatter) + ); + const formatVisitsResponse = () => ( value: number ): ArchifiltreCountStatistic => ({ label: "visitsCount", value }); @@ -102,15 +158,15 @@ const formatVisitsResponse = () => ( export const createMatomoDataSanitizer = ({ events = [], actions = [], + monthlyEvents = [], visits = false, }: MatomoSiteConfig) => ( // eslint-disable-next-line @typescript-eslint/no-explicit-any matomoApiResponse: any[] ): ArchifiltreCountStatistic[] => - flatten( - [ - ...events.map(formatEventsOrActionsResponse), - ...actions.map(formatEventsOrActionsResponse), - ...(visits ? [formatVisitsResponse()] : []), - ].map((formatter, index) => formatter(matomoApiResponse[index])) - ); + [ + ...events.map(formatEventsOrActionsResponse), + ...actions.map(formatEventsOrActionsResponse), + ...monthlyEvents.flatMap(formatMonthlyEvents), + ...(visits ? [formatVisitsResponse()] : []), + ].flatMap((formatter, index) => formatter(matomoApiResponse[index])); diff --git a/src/utils/date.test.ts b/src/utils/date.test.ts new file mode 100644 index 00000000..68fb2894 --- /dev/null +++ b/src/utils/date.test.ts @@ -0,0 +1,16 @@ +import { parseISO } from "date-fns"; + +import { getLastMonthsRanges } from "./date"; + +describe("date", () => { + describe("getLastMonthsRanges", () => { + it("should compute the last months ranges", () => { + const startDate = parseISO("2021-02-10"); + expect(getLastMonthsRanges(3)(startDate)).toEqual([ + ["2021-02-01", "2021-02-10"], + ["2021-01-01", "2021-01-31"], + ["2020-12-01", "2020-12-31"], + ]); + }); + }); +}); diff --git a/src/utils/date.ts b/src/utils/date.ts new file mode 100644 index 00000000..8cb5c877 --- /dev/null +++ b/src/utils/date.ts @@ -0,0 +1,19 @@ +import { endOfMonth, formatISO, startOfMonth, subMonths } from "date-fns"; +import { range } from "lodash"; + +export const getLastMonthsRanges = (monthCount: number) => ( + now: Date +): [string, string][] => + range(0, monthCount) + .map((index) => subMonths(now, index)) + .map((date, index): [Date, Date] => [ + startOfMonth(date), + index === 0 ? date : endOfMonth(date), + ]) + .map( + (dates): [string, string] => + dates.map((date) => formatISO(date, { representation: "date" })) as [ + string, + string + ] + ); diff --git a/src/utils/fp-util.ts b/src/utils/fp-util.ts new file mode 100644 index 00000000..5c845fe5 --- /dev/null +++ b/src/utils/fp-util.ts @@ -0,0 +1,7 @@ +/** + * Lift method for promise + * @param method + */ +export const liftPromise = ( + method: (input: TInput) => TOutput +) => async (promise: Promise): Promise => promise.then(method); diff --git a/tsconfig.json b/tsconfig.json index 10585e63..25d7ce55 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,6 +1,7 @@ { "compilerOptions": { "target": "es5", + "lib": ["es2019"], "module": "commonjs", "allowJs": false, "outDir": "./dist", diff --git a/yarn.lock b/yarn.lock index c8aa0007..e16300dc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1565,6 +1565,11 @@ data-urls@^2.0.0: whatwg-mimetype "^2.3.0" whatwg-url "^8.0.0" +date-fns@^2.18.0: + version "2.18.0" + resolved "https://registry.yarnpkg.com/date-fns/-/date-fns-2.18.0.tgz#08e50aee300ad0d2c5e054e3f0d10d8f9cdfe09e" + integrity sha512-NYyAg4wRmGVU4miKq5ivRACOODdZRY3q5WLmOJSq8djyzftYphU7dTHLcEtLqEvfqMKQ0jVv91P4BAwIjsXIcw== + debug@2.6.9, debug@^2.2.0, debug@^2.3.3, debug@^2.6.9: version "2.6.9" resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f"