diff --git a/definitions/output/core_web_vitals/technologies.js b/definitions/output/core_web_vitals/technologies.js index 6c72775..688219a 100644 --- a/definitions/output/core_web_vitals/technologies.js +++ b/definitions/output/core_web_vitals/technologies.js @@ -15,11 +15,19 @@ publish('technologies', { DELETE FROM ${ctx.self()} WHERE date = '${pastMonth}'; -CREATE TEMP FUNCTION IS_GOOD(good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( +CREATE TEMP FUNCTION IS_GOOD( + good FLOAT64, + needs_improvement FLOAT64, + poor FLOAT64 +) RETURNS BOOL AS ( SAFE_DIVIDE(good, good + needs_improvement + poor) >= 0.75 ); -CREATE TEMP FUNCTION IS_NON_ZERO(good FLOAT64, needs_improvement FLOAT64, poor FLOAT64) RETURNS BOOL AS ( +CREATE TEMP FUNCTION IS_NON_ZERO( + good FLOAT64, + needs_improvement FLOAT64, + poor FLOAT64 +) RETURNS BOOL AS ( good + needs_improvement + poor > 0 ); `).query(ctx => ` @@ -28,8 +36,7 @@ WITH geo_summary AS ( CAST(REGEXP_REPLACE(CAST(yyyymm AS STRING), r'(\\d{4})(\\d{2})', r'\\1-\\2-01') AS DATE) AS date, * EXCEPT (country_code), \`chrome-ux-report\`.experimental.GET_COUNTRY(country_code) AS geo - FROM - ${ctx.ref('chrome-ux-report', 'materialized', 'country_summary')} + FROM ${ctx.ref('chrome-ux-report', 'materialized', 'country_summary')} WHERE yyyymm = CAST(FORMAT_DATE('%Y%m', '${pastMonth}') AS INT64) AND device IN ('desktop', 'phone') @@ -37,8 +44,7 @@ UNION ALL SELECT * EXCEPT (yyyymmdd, p75_fid_origin, p75_cls_origin, p75_lcp_origin, p75_inp_origin), 'ALL' AS geo - FROM - ${ctx.ref('chrome-ux-report', 'materialized', 'device_summary')} + FROM ${ctx.ref('chrome-ux-report', 'materialized', 'device_summary')} WHERE date = '${pastMonth}' AND device IN ('desktop', 'phone') @@ -81,11 +87,9 @@ crux AS ( IS_GOOD(fast_ttfb, avg_ttfb, slow_ttfb) AS good_ttfb, IS_NON_ZERO(fast_inp, avg_inp, slow_inp) AS any_inp, IS_GOOD(fast_inp, avg_inp, slow_inp) AS good_inp - FROM - geo_summary, + FROM geo_summary, UNNEST([1000, 10000, 100000, 1000000, 10000000, 100000000]) AS _rank - WHERE - rank <= _rank + WHERE rank <= _rank ), technologies AS ( @@ -93,8 +97,7 @@ technologies AS ( technology.technology AS app, client, page AS url - FROM - ${ctx.ref('crawl', 'pages')}, + FROM ${ctx.ref('crawl', 'pages')}, UNNEST(technologies) AS technology WHERE date = '${pastMonth}' @@ -106,8 +109,7 @@ UNION ALL 'ALL' AS app, client, page AS url - FROM - ${ctx.ref('crawl', 'pages')} + FROM ${ctx.ref('crawl', 'pages')} WHERE date = '${pastMonth}' ${constants.devRankFilter} @@ -117,21 +119,18 @@ categories AS ( SELECT technology.technology AS app, ARRAY_TO_STRING(ARRAY_AGG(DISTINCT category IGNORE NULLS ORDER BY category), ', ') AS category - FROM - ${ctx.ref('crawl', 'pages')}, + FROM ${ctx.ref('crawl', 'pages')}, UNNEST(technologies) AS technology, UNNEST(technology.categories) AS category WHERE date = '${pastMonth}' ${constants.devRankFilter} - GROUP BY - app + GROUP BY app UNION ALL SELECT 'ALL' AS app, ARRAY_TO_STRING(ARRAY_AGG(DISTINCT category IGNORE NULLS ORDER BY category), ', ') AS category - FROM - ${ctx.ref('crawl', 'pages')}, + FROM ${ctx.ref('crawl', 'pages')}, UNNEST(technologies) AS technology, UNNEST(technology.categories) AS category WHERE @@ -153,8 +152,7 @@ summary_stats AS ( SAFE.FLOAT64(lighthouse.categories.performance.score) AS performance, SAFE.FLOAT64(lighthouse.categories.pwa.score) AS pwa, SAFE.FLOAT64(lighthouse.categories.seo.score) AS seo - FROM - ${ctx.ref('crawl', 'pages')} + FROM ${ctx.ref('crawl', 'pages')} WHERE date = '${pastMonth}' ${constants.devRankFilter} @@ -174,16 +172,11 @@ lab_data AS ( AVG(performance) AS performance, AVG(pwa) AS pwa, AVG(seo) AS seo - FROM - summary_stats - JOIN - technologies - USING - (client, url) - JOIN - categories - USING - (app) + FROM summary_stats + JOIN technologies + USING (client, url) + JOIN categories + USING (app) GROUP BY client, root_page_url, @@ -232,10 +225,8 @@ SELECT SAFE_CAST(APPROX_QUANTILES(bytesJS, 1000)[OFFSET(500)] AS INT64) AS median_bytes_js, SAFE_CAST(APPROX_QUANTILES(bytesImg, 1000)[OFFSET(500)] AS INT64) AS median_bytes_image -FROM - lab_data -JOIN - crux +FROM lab_data +JOIN crux USING (client, root_page_url) GROUP BY diff --git a/definitions/output/tech_reports/adoption.js b/definitions/output/tech_reports/adoption.js index 8e9f3b8..fce5818 100644 --- a/definitions/output/tech_reports/adoption.js +++ b/definitions/output/tech_reports/adoption.js @@ -28,7 +28,7 @@ return Object.fromEntries( '''; SELECT - STRING(DATE(date)) as date, + date, app AS technology, rank, geo, @@ -38,7 +38,6 @@ SELECT ))) AS adoption FROM ${ctx.ref('core_web_vitals', 'technologies')} WHERE date = '${pastMonth}' - ${constants.devRankFilter} GROUP BY date, app, diff --git a/definitions/output/tech_reports/categories.js b/definitions/output/tech_reports/categories.js index 977a0e1..6d981b5 100644 --- a/definitions/output/tech_reports/categories.js +++ b/definitions/output/tech_reports/categories.js @@ -2,19 +2,25 @@ const pastMonth = constants.fnPastMonth(constants.currentMonth) publish('technologies', { schema: 'tech_reports', + type: 'table', tags: ['cwv_tech_report'] }).query(ctx => ` -WITH categories AS ( +WITH pages AS ( SELECT - category, - COUNT(DISTINCT root_page) AS origins - FROM ${ctx.ref('crawl', 'pages')}, - UNNEST(technologies) AS t, - UNNEST(t.categories) AS category + root_page, + technologies + FROM ${ctx.ref('crawl', 'pages')} WHERE date = '${pastMonth}' AND client = 'mobile' ${constants.devRankFilter} +),categories AS ( + SELECT + category, + COUNT(DISTINCT root_page) AS origins + FROM pages, + UNNEST(technologies) AS t, + UNNEST(t.categories) AS category GROUP BY category ), technologies AS ( @@ -22,13 +28,9 @@ technologies AS ( category, technology, COUNT(DISTINCT root_page) AS origins - FROM ${ctx.ref('crawl', 'pages')}, + FROM pages, UNNEST(technologies) AS t, UNNEST(t.categories) AS category - WHERE - date = '${pastMonth}' AND - client = 'mobile' - ${constants.devRankFilter} GROUP BY category, technology diff --git a/definitions/output/tech_reports/core_web_vitals.js b/definitions/output/tech_reports/core_web_vitals.js index 81e1cfe..ba278a6 100644 --- a/definitions/output/tech_reports/core_web_vitals.js +++ b/definitions/output/tech_reports/core_web_vitals.js @@ -69,7 +69,7 @@ return Object.values(vitals); '''; SELECT - STRING(DATE(date)) as date, + date, app AS technology, rank, geo, @@ -92,7 +92,6 @@ SELECT ))) AS vitals FROM ${ctx.ref('core_web_vitals', 'technologies')} WHERE date = '${pastMonth}' - ${constants.devRankFilter} GROUP BY date, app, diff --git a/definitions/output/tech_reports/lighthouse.js b/definitions/output/tech_reports/lighthouse.js index b73ae09..9c2e8ce 100644 --- a/definitions/output/tech_reports/lighthouse.js +++ b/definitions/output/tech_reports/lighthouse.js @@ -54,7 +54,7 @@ return Object.values(lighthouse); '''; SELECT - STRING(DATE(date)) as date, + date, app AS technology, rank, geo, @@ -68,7 +68,6 @@ SELECT ))) AS lighthouse FROM ${ctx.ref('core_web_vitals', 'technologies')} WHERE date = '${pastMonth}' - ${constants.devRankFilter} GROUP BY date, app, diff --git a/definitions/output/tech_reports/page_weight.js b/definitions/output/tech_reports/page_weight.js index 6d95567..9c8d84b 100644 --- a/definitions/output/tech_reports/page_weight.js +++ b/definitions/output/tech_reports/page_weight.js @@ -46,7 +46,7 @@ return Object.values(pageWeight) '''; SELECT - STRING(DATE(date)) as date, + date, app AS technology, rank, geo, @@ -58,7 +58,6 @@ SELECT ))) AS pageWeight FROM ${ctx.ref('core_web_vitals', 'technologies')} WHERE date = '${pastMonth}' - ${constants.devRankFilter} GROUP BY date, app, diff --git a/definitions/output/tech_reports/technologies.js b/definitions/output/tech_reports/technologies.js index ecbfb5a..3580fcc 100644 --- a/definitions/output/tech_reports/technologies.js +++ b/definitions/output/tech_reports/technologies.js @@ -2,6 +2,7 @@ const pastMonth = constants.fnPastMonth(constants.currentMonth) publish('technologies', { schema: 'tech_reports', + type: 'table', tags: ['cwv_tech_report'] }).query(ctx => ` SELECT