From 78257e183172f6871896526652368ddfb943bdac Mon Sep 17 00:00:00 2001 From: Patrick Meenan Date: Tue, 11 Oct 2022 14:49:57 -0700 Subject: [PATCH] Fix extraction of background image URLs. Also: * added support for other types of non-src images * fixed edge-side processing of the LCP events * Moved the edge-processing to before the debug log gets collected (so logs can be included) For https://github.com/WPO-Foundation/webpagetest/issues/2455 --- internal/process_test.py | 8 +++- internal/support/chrome/inject.js | 67 ++++++++++++++++--------------- internal/support/trace_parser.py | 2 + internal/webpagetest.py | 12 +++--- 4 files changed, 50 insertions(+), 39 deletions(-) diff --git a/internal/process_test.py b/internal/process_test.py index 4c7bfb845..95fe1d4ab 100644 --- a/internal/process_test.py +++ b/internal/process_test.py @@ -352,7 +352,7 @@ def process_chrome_timings(self): if 'name' in event and 'ts' in event and 'args' in event and 'frame' in event['args'] and \ event['args']['frame'] in main_frames and \ (event['ts'] >= start_time or 'value' in event['args']) and \ - event['name'].lower().find('largest') and \ + event['name'].lower().find('largest') >= 0 and \ 'data' in event['args'] and 'size' in event['args']['data']: name = event['name'] if name not in largest or event['args']['data']['size'] > largest[name]['args']['data']['size']: @@ -379,6 +379,10 @@ def process_chrome_timings(self): paint_event['element'] = event['args']['data']['element'] if 'type' in event['args']['data']: paint_event['type'] = event['args']['data']['type'] + if 'imageUrl' in event['args']['data'] and len(event['args']['data']['imageUrl']): + paint_event['imageUrl'] = event['args']['data']['imageUrl'] + if 'url' in event['args']['data'] and len(event['args']['data']['url']): + paint_event['url'] = event['args']['data']['url'] if 'largestPaints' not in page_data: page_data['largestPaints'] = [] page_data['largestPaints'].append(paint_event) @@ -489,6 +493,8 @@ def process_chrome_timings(self): if matches: page_data['LargestContentfulPaintType'] = 'background-image' page_data['LargestContentfulPaintImageURL'] = matches.group(1) + if 'imageUrl' in paint_event: + page_data['LargestContentfulPaintImageURL'] = paint_event['imageUrl'] elif 'largestPaints' in page_data: for paint_event in page_data['largestPaints']: if paint_event['event'] == 'LargestTextPaint' and paint_event['time'] == event['time']: diff --git a/internal/support/chrome/inject.js b/internal/support/chrome/inject.js index 7429cea8f..2b601dbd5 100644 --- a/internal/support/chrome/inject.js +++ b/internal/support/chrome/inject.js @@ -1,37 +1,40 @@ new PerformanceObserver((entryList) => { for (const entry of entryList.getEntries()) { - try { - let event = { - name: entry.name, - entryType: entry.entryType, - startTime: entry['startTime'], - size: entry['size'], - url: entry['url'], - id: entry['id'], - loadTime: entry['loadTime'], - renderTime: entry['renderTime'], - }; - if (entry['element']) { - event['element'] = { - nodeName: entry.element['nodeName'], - boundingRect: entry.element.getBoundingClientRect(), - outerHTML: entry.element.outerHTML, - } - if (entry.element['src']) { - event.element['src'] = entry.element.src; - } - if (entry.element['currentSrc']) { - event.element['currentSrc'] = entry.element.currentSrc; - } - try { - let style = window.getComputedStyle(object[key]); - if (style.backgroundImage && style.backgroundImage != 'none') { - contents[key]['background-image'] = style.backgroundImage; - } - } catch (err) { - } - } - console.debug('wptagent_message:' + JSON.stringify({'name': 'perfentry', 'data': event})); + try { + let event = { + name: entry.name, + entryType: entry.entryType, + startTime: entry['startTime'], + size: entry['size'], + url: entry['url'], + id: entry['id'], + loadTime: entry['loadTime'], + renderTime: entry['renderTime'], + }; + if (entry['element']) { + event['element'] = { + nodeName: entry.element['nodeName'], + boundingRect: entry.element.getBoundingClientRect(), + outerHTML: entry.element.outerHTML, + } + if (entry.element['src']) { + event.element['src'] = entry.element.src; + } + if (entry.element['currentSrc']) { + event.element['currentSrc'] = entry.element.currentSrc; + } + try { + let style = window.getComputedStyle(entry.element); + if (style.backgroundImage && style.backgroundImage != 'none') { + event.element['background-image'] = style.backgroundImage; + } + if (style.content && style.content != 'none') { + event.element['content'] = style.content; + } + } catch (err) { + } + } + console.debug('wptagent_message:' + JSON.stringify({'name': 'perfentry', 'data': event})); } catch (err) { } } diff --git a/internal/support/trace_parser.py b/internal/support/trace_parser.py index e3acd02e2..c149c059d 100644 --- a/internal/support/trace_parser.py +++ b/internal/support/trace_parser.py @@ -392,6 +392,8 @@ def post_process_user_timing(self, dom_tree, performance_timing): for perf_entry in performance_timing: if 'entryType' in perf_entry and perf_entry['entryType'] == 'largest-contentful-paint' and 'size' in perf_entry and perf_entry['size'] == event['args']['data']['size'] and 'consumed' not in perf_entry: perf_entry['consumed'] = True + if 'url' in perf_entry and len(perf_entry['url']) and 'url' not in event['args']['data']: + event['args']['data']['url'] = perf_entry['url'] if 'element' in perf_entry: event['args']['data']['element'] = perf_entry['element'] elif 'score' in event['args']['data'] and event['name'].startswith('LayoutShift'): diff --git a/internal/webpagetest.py b/internal/webpagetest.py index b934e1ebe..6853e1455 100644 --- a/internal/webpagetest.py +++ b/internal/webpagetest.py @@ -1566,6 +1566,12 @@ def upload_task_result(self, task): self.update_browser_viewport(task) if task['run'] == 1 and not task['cached']: self.collect_crux_data(task) + # Post-process the given test run + try: + from internal.process_test import ProcessTest + ProcessTest(self.options, self.job, task) + except Exception: + logging.exception('Error post-processing test') # Stop logging to the file if self.log_handler is not None: try: @@ -1586,12 +1592,6 @@ def upload_task_result(self, task): if self.job['warmup'] > 0: logging.debug('Discarding warmup run') else: - # Post-process the given test run - try: - from internal.process_test import ProcessTest - ProcessTest(self.options, self.job, task) - except Exception: - logging.exception('Error post-processing test') # Continue with the upload if 'page_data' in task and 'fullyLoadedCPUpct' in task['page_data']: self.cpu_pct = task['page_data']['fullyLoadedCPUpct']