Merge pull request #500 from AutomationSolutionz/dom

\t \n removal ignored in extensions and chatbot
AutomationSolutionz · Sep 9, 2024 · 17a8a3e · 17a8a3e
2 parents 95c0e74 + dde668f
commit 17a8a3e
Show file tree

Hide file tree

Showing 6 changed files with 76 additions and 36 deletions.
diff --git a/Apps/Web/AI_Recorder_2/dist/background/back_reocrder.js b/Apps/Web/AI_Recorder_2/dist/background/back_reocrder.js
@@ -4,6 +4,7 @@ fetch("./data.json")
     .then(Response => Response.json())
     .then(data => {
         metaData = data;
+        console.log('metaData', metaData)
     });
 
 const browserAppData = chrome || browser;
@@ -95,8 +96,11 @@ async function fetchAIData(id, command, value, url, document){
         validate_full_text_by_ai = true;
     }
 
-    document = document.replace(/[^\x00-\x7F]/g, '');
-    document = document.replace(/[\x00-\x1F\x7F]/g, ''); // Optionally remove control characters (0-31 and 127 in ASCII)
+    //The following code removes non-unicode characters (except \x09 and \x0A that are \t and \n), that causes issue in lxml tree
+    // Optionally remove control characters (0-31 and 127 in ASCII) except 9,10 \t and \n
+    document = document.replace(/[\x00-\x08\x0B-\x1F\x7F]/g, ''); 
+    // console.log('document', document)
+
     var dataj = {
         "page_src": document,
         "action_name": command,

diff --git a/Apps/Web/AI_Recorder_2/dist/content/rec_handlers.js b/Apps/Web/AI_Recorder_2/dist/content/rec_handlers.js
@@ -13,7 +13,7 @@ var preventClick = false;
 var enterTarget = null;
 var enterValue = null;
 var tabCheck = null;
-
+var contextMenuFired = false;
 function print_event(event) {
     var print = true;
     let ignore = ['mouseover', 'mouseout', 'focus', 'focusout', 'blur', 'DOMNodeInserted', 'scroll']
@@ -599,13 +599,21 @@ const asCheckbox = function(target) {
         return null;
     return ['checkbox', 'radio'].includes(target.type) ? target : null;
 }
-const _shouldIgnoreMouseEvent = function(event) {
+const _shouldIgnoreMouseEvent = async function(event) {
     const target = event.target;
     const nodeName = target.nodeName;
     if (/* nodeName === 'SELECT' || */ nodeName === 'OPTION')
         return true;
     if (nodeName === 'INPUT' && ['date', 'range'].includes(target.type))
         return true;
+    if (event.button == 2){ // Ignoring right mouse click if that opens contextMenu
+        return await new Promise((resolve) => {
+            setTimeout(()=>{
+                resolve(contextMenuFired);
+                contextMenuFired = false;
+            }, 0)
+        })
+    }
     return false;
 }
 
@@ -639,12 +647,12 @@ const _shouldGenerateKeyPressFor = function(event) {
     return true;
 }
 
-const onClick = function(event) {
+const onClick = async function(event) {
     // print_event(event)
     const target = event.target
     if (isRangeInput(target))
         return;
-    if (_shouldIgnoreMouseEvent(event))
+    if (await _shouldIgnoreMouseEvent(event))
         return;
 
     const checkbox = asCheckbox(target);
@@ -658,7 +666,14 @@ const onClick = function(event) {
         return;
     }
 
-    if (event.detail === 2){
+    if (event.button == 2){
+        recorder.record({
+            name: 'context click',
+            xpath: recorder.getXpath(target),
+            value: '',
+        });
+    }
+    else if (event.detail === 2){
         recorder.record({
             name: 'double click',
             xpath: recorder.getXpath(target),
@@ -762,6 +777,7 @@ const onKeyDown = function(event) {
 
 var myPort = browserAppData.runtime.connect();
 const onContextMenu = function (event){
+    contextMenuFired = true;
     print_event(event)
     myPort = browserAppData.runtime.connect();
     const target = event.target;
@@ -784,27 +800,27 @@ function consumeEvent(e) {
     e.stopImmediatePropagation();
   }
 
-const onMouseDown = function(event) {
+const onMouseDown = async function(event) {
     print_event(event)
-    if (_shouldIgnoreMouseEvent(event))
-        return;
 }
 
-const onMouseUp = function(event) {
+const onMouseUp = async function(event) {
     print_event(event)
     /* select-2 jquery elements only dispath mousedown-up event and not click event.
     Codegen record it by event.stopPropagation() then record then continue propagation somehow
     We are not able to do it right now. So, we are simply recording click on mouseup */
-    onClick(event)      
-    if (_shouldIgnoreMouseEvent(event))
-        return;
+    onClick(event)
+}
+
+const onAuxClick = async function(event) {
+    print_event(event)
 }
 
 Recorder.prototype.newEventListeners = [
     {name: 'mouseup', handler: onMouseUp, capture: true},
     {name: 'mousedown', handler: onMouseDown, capture: true},
     // {name: 'click', handler: onClick, capture: true},
-    {name: 'auxclick', handler: onClick, capture: true},
+    {name: 'auxclick', handler: onAuxClick, capture: true},
     {name: 'input', handler: onInput, capture: true},
     {name: 'keydown', handler: onKeyDown, capture: true},
     {name: 'contextmenu', handler: onContextMenu, capture: true},

diff --git a/Apps/Web/AI_Recorder_2/public/background/back_reocrder.js b/Apps/Web/AI_Recorder_2/public/background/back_reocrder.js
@@ -4,6 +4,7 @@ fetch("./data.json")
     .then(Response => Response.json())
     .then(data => {
         metaData = data;
+        console.log('metaData', metaData)
     });
 
 const browserAppData = chrome || browser;
@@ -95,8 +96,11 @@ async function fetchAIData(id, command, value, url, document){
         validate_full_text_by_ai = true;
     }
 
-    document = document.replace(/[^\x00-\x7F]/g, '');
-    document = document.replace(/[\x00-\x1F\x7F]/g, ''); // Optionally remove control characters (0-31 and 127 in ASCII)
+    //The following code removes non-unicode characters (except \x09 and \x0A that are \t and \n), that causes issue in lxml tree
+    // Optionally remove control characters (0-31 and 127 in ASCII) except 9,10 \t and \n
+    document = document.replace(/[\x00-\x08\x0B-\x1F\x7F]/g, ''); 
+    // console.log('document', document)
+
     var dataj = {
         "page_src": document,
         "action_name": command,

diff --git a/Apps/Web/AI_Recorder_2/public/content/rec_handlers.js b/Apps/Web/AI_Recorder_2/public/content/rec_handlers.js
@@ -13,7 +13,7 @@ var preventClick = false;
 var enterTarget = null;
 var enterValue = null;
 var tabCheck = null;
-
+var contextMenuFired = false;
 function print_event(event) {
     var print = true;
     let ignore = ['mouseover', 'mouseout', 'focus', 'focusout', 'blur', 'DOMNodeInserted', 'scroll']
@@ -599,13 +599,21 @@ const asCheckbox = function(target) {
         return null;
     return ['checkbox', 'radio'].includes(target.type) ? target : null;
 }
-const _shouldIgnoreMouseEvent = function(event) {
+const _shouldIgnoreMouseEvent = async function(event) {
     const target = event.target;
     const nodeName = target.nodeName;
     if (/* nodeName === 'SELECT' || */ nodeName === 'OPTION')
         return true;
     if (nodeName === 'INPUT' && ['date', 'range'].includes(target.type))
         return true;
+    if (event.button == 2){ // Ignoring right mouse click if that opens contextMenu
+        return await new Promise((resolve) => {
+            setTimeout(()=>{
+                resolve(contextMenuFired);
+                contextMenuFired = false;
+            }, 0)
+        })
+    }
     return false;
 }
 
@@ -639,12 +647,12 @@ const _shouldGenerateKeyPressFor = function(event) {
     return true;
 }
 
-const onClick = function(event) {
+const onClick = async function(event) {
     // print_event(event)
     const target = event.target
     if (isRangeInput(target))
         return;
-    if (_shouldIgnoreMouseEvent(event))
+    if (await _shouldIgnoreMouseEvent(event))
         return;
 
     const checkbox = asCheckbox(target);
@@ -658,7 +666,14 @@ const onClick = function(event) {
         return;
     }
 
-    if (event.detail === 2){
+    if (event.button == 2){
+        recorder.record({
+            name: 'context click',
+            xpath: recorder.getXpath(target),
+            value: '',
+        });
+    }
+    else if (event.detail === 2){
         recorder.record({
             name: 'double click',
             xpath: recorder.getXpath(target),
@@ -762,6 +777,7 @@ const onKeyDown = function(event) {
 
 var myPort = browserAppData.runtime.connect();
 const onContextMenu = function (event){
+    contextMenuFired = true;
     print_event(event)
     myPort = browserAppData.runtime.connect();
     const target = event.target;
@@ -784,27 +800,27 @@ function consumeEvent(e) {
     e.stopImmediatePropagation();
   }
 
-const onMouseDown = function(event) {
+const onMouseDown = async function(event) {
     print_event(event)
-    if (_shouldIgnoreMouseEvent(event))
-        return;
 }
 
-const onMouseUp = function(event) {
+const onMouseUp = async function(event) {
     print_event(event)
     /* select-2 jquery elements only dispath mousedown-up event and not click event.
     Codegen record it by event.stopPropagation() then record then continue propagation somehow
     We are not able to do it right now. So, we are simply recording click on mouseup */
-    onClick(event)      
-    if (_shouldIgnoreMouseEvent(event))
-        return;
+    onClick(event)
+}
+
+const onAuxClick = async function(event) {
+    print_event(event)
 }
 
 Recorder.prototype.newEventListeners = [
     {name: 'mouseup', handler: onMouseUp, capture: true},
     {name: 'mousedown', handler: onMouseDown, capture: true},
     // {name: 'click', handler: onClick, capture: true},
-    {name: 'auxclick', handler: onClick, capture: true},
+    {name: 'auxclick', handler: onAuxClick, capture: true},
     {name: 'input', handler: onInput, capture: true},
     {name: 'keydown', handler: onKeyDown, capture: true},
     {name: 'contextmenu', handler: onContextMenu, capture: true},

diff --git a/Apps/Web/aiplugin/inspect.js b/Apps/Web/aiplugin/inspect.js
@@ -141,11 +141,9 @@ class Inspector {
 								var server_url = result.url;
 								var api_key = result.key;
 
-
-								// send data to zeuz server directly
-
-								refinedHtml = refinedHtml.replace(/[^\x00-\x7F]/g, '');
-								refinedHtml = refinedHtml.replace(/[\x00-\x1F\x7F]/g, '');
+								//The following code removes non-unicode characters (except \x09 and \x0A that are \t and \n), that causes issue in lxml tree
+								// Optionally remove control characters (0-31 and 127 in ASCII) except 9,10 \t and \n
+								refinedHtml = refinedHtml.replace(/[\x00-\x08\x0B-\x1F\x7F]/g, '');
 								var data = JSON.stringify({
 									"page_src": refinedHtml,
 									"action_type": "selenium"

diff --git a/Framework/MainDriverApi.py b/Framework/MainDriverApi.py
@@ -1248,7 +1248,9 @@ def send_dom_variables():
                     
                     // AI model works better on indented dom, so not removing indentation.
                     // var result = html.outerHTML.replace(/\s+/g, ' ').replace(/>\s+</g, '><');
-                    var result = html.outerHTML;
+
+                    //The following code removes non-unicode characters except newline and tab
+                    var result = html.outerHTML.replace(/[\x00-\x08\x0B-\x1F\x7F]/g, '');
                     return result;""")
             except selenium.common.exceptions.JavascriptException:
                 CommonUtil.Exception_Handler(sys.exc_info())