Update pyvirtualdisplay

README.md

@@ -82,8 +82,8 @@ Quick Start
 
 Once installed, it is very easy to run a quick test of OpenWPM. Check out
 `demo.py` for an example. This will use the default setting specified in
-`openwpm/default_manager_params.json` and
-`openwpm/default_browser_params.json`, with the exception of the changes
+`openwpm/config.py::ManagerParams` and
+`openwpm/config.py::BrowserParams`, with the exception of the changes
 specified in `demo.py`.
 
 More information on the instrumentation and configuration parameters is given

docs/Configuration.md

@@ -11,7 +11,7 @@ To load the default configuration create instances of `config::ManagerParams` an
 from openwpm.config import BrowserParams, ManagerParams
 
 manager_params = ManagerParams(num_browsers=5)
-browser_params = [BrowserParams() for _ in range(num_browsers)]
+browser_params = [BrowserParams() for _ in range(manager_params.num_browsers)]
 ```
 
 where `manager_params` is of type `class<ManagerParams>` and `browser_params` is a length 5 list
@@ -20,11 +20,16 @@ of configurations of `class<BrowserParams>`.
 ####Validations:
 To validate `browser_params` and `manager_params`, we have two methods each for type of params, `config::validate_browser_params` and `config::validate_manager_params`. For example:
 ```python
-from openwpm.config import validate_browser_params, validate_manager_params
+from openwpm.config import (
+  validate_browser_params, 
+  validate_manager_params,
+  validate_crawl_configs,
+)
 
 for bp in browser_params:
   validate_browser_params(bp)
 validate_manager_params(manager_params)
+validate_crawl_configs(manager_params, browser_params)
 ```
 **NOTE**: If any validations fail, we raise `ConfigError`
 

docs/Platform-Architecture.md

@@ -6,17 +6,33 @@ The user-facing component of the OpenWPM platform is the Task Manager. The Task
 
 ## Instantiating a Task Manager
 
-All automation code is contained within the `openwpm` folder; the Task Manager code is contained in `openwpm/TaskManager.py`.
+All automation code is contained within the `openwpm` folder; the Task Manager code is contained in `openwpm/task_manager.py`.
 
 Task Managers can be instantiated in the following way:
+```python
+from opemwpm.task_manager import TaskManager
+from openwpm.config import (
+    BrowserParams,
+    ManagerParams,
+)
 
-`manager = TaskManager.TaskManager(manager_params, browser_params)`
+number_of_browser = 5    #  Number of browsers to spawn
+
+# Instantiating Browser and Manager Params with default values.
+manager_params = ManagerParams(num_browsers = number_of_browsers)
+browser_params = [BrowserParams() for bp in range(manager_params.num_browsers)]
+
+# These instances can be used to modify default values of both browser and manager params.
+manager_params.data_directory = '~/Documents'
+manager_params.database_name = 'custom_name.sqlite'
+
+for i in range(len(browser_params)):
+    browser_params[i].display_mode = 'headless'  # all 5 browsers will spawn in headless mode
+
+# Instantiating TaskManager
+manager = TaskManager(manager_params, browser_params)
 
-You can call
-```python
-manager_params, browser_params = TaskManager.load_default_params(number_of_browsers_to_spawn)
 ```
-to get the default parameters.
 
 To learn more about the `manager_params` and `browser_params` have a look at [Configuration.md](Configuration.md)
 

docs/Using_OpenWPM.md

@@ -14,7 +14,7 @@ Users can change the settings for task manager and individual browsers so that,
 from openwpm.config import BrowserParams, ManagerParams
 
 manager_params = ManagerParams(num_browsers=5)
-browser_params = [BrowserParams() for _ in range(num_browsers)]
+browser_params = [BrowserParams() for _ in range(manager_params.num_browsers)]
 ```
 
 #### Loading Custom Browser or Manager configs

environment.yaml

@@ -22,7 +22,7 @@ dependencies:
 - pytest-cov=2.10.1
 - pytest=6.1.2
 - python=3.8.6
-- pyvirtualdisplay=0.2.5
+- pyvirtualdisplay=1.3.2
 - redis-py=3.5.3
 - s3fs=0.4.0
 - selenium=3.141.0

openwpm/browser_manager.py

@@ -433,7 +433,7 @@ def BrowserManager(
     logger = logging.getLogger("openwpm")
     try:
         # Start Xvfb (if necessary), webdriver, and browser
-        driver, prof_folder = deploy_firefox.deploy_firefox(
+        driver, prof_folder, display = deploy_firefox.deploy_firefox(
             status_queue, browser_params, manager_params, crash_recovery
         )
         if prof_folder[-1] != "/":
@@ -497,6 +497,8 @@ def BrowserManager(
                 if driver.profile and not os.path.isdir(driver.profile.path):
                     driver.profile = None
                 driver.quit()
+                if browser_params.display_mode == "xvfb":
+                    display.stop()
                 status_queue.put("OK")
                 return
 

openwpm/deploy_browsers/deploy_firefox.py

@@ -60,6 +60,7 @@ def deploy_firefox(status_queue, browser_params, manager_params, crash_recovery)
     display_mode = browser_params.display_mode
     display_pid = None
     display_port = None
+    display = None
     if display_mode == "headless":
         fo.set_headless(True)
         fo.add_argument("--width={}".format(DEFAULT_SCREEN_RES[0]))
@@ -68,7 +69,7 @@ def deploy_firefox(status_queue, browser_params, manager_params, crash_recovery)
         try:
             display = Display(visible=0, size=DEFAULT_SCREEN_RES)
             display.start()
-            display_pid, display_port = display.pid, display.cmd_param[-1][1:]
+            display_pid, display_port = display.pid, display.display
         except EasyProcessError:
             raise RuntimeError(
                 "Xvfb could not be started. \
@@ -157,4 +158,4 @@ def deploy_firefox(status_queue, browser_params, manager_params, crash_recovery)
 
     status_queue.put(("STATUS", "Browser Launched", int(pid)))
 
-    return driver, driver.capabilities["moz:profile"]
+    return driver, driver.capabilities["moz:profile"], display

test/manual_test.py

@@ -89,7 +89,7 @@ def start_webdriver(
         Set to True to load browser_params
     browser_params_file : string
         Specify the browser_params.json to load.
-        If None, default_params will be loaded.
+        If None, default params form openwpm/config.py::BrowserParams will be loaded.
 
     Returns
     -------

test/test_mp_logger.py

@@ -176,7 +176,9 @@ def test_child_process_logging(self, tmpdir):
         child_process = Process(target=child_proc_logging_exception())
         child_process.daemon = True
         child_process.start()
-        openwpm_logger.close()
+        time.sleep(2)
         child_process.join()
+        print("Child processes joined...")
+        openwpm_logger.close()
         log_content = self.get_logfile_contents(log_file)
         assert "I'm logging an exception" in log_content

test/test_xvfb_browser.py

@@ -0,0 +1,26 @@
+from functools import partial
+from typing import List
+
+from openwpm.command_sequence import CommandSequence
+from openwpm.task_manager import TaskManager
+
+from .openwpmtest import OpenWPMTest
+from .utilities import BASE_TEST_URL
+
+
+class TestXVFBDisplay(OpenWPMTest):
+    """Test the XVFB display option to see if it runs and shuts down properly after a command is given"""
+
+    def get_config(self, data_dir=""):
+        return self.get_test_config(data_dir, display_mode="xvfb")
+
+    def test_display_shutdown(self):
+        manager_params, browser_params = self.get_config()
+        TEST_SITE = BASE_TEST_URL + "/test_pages/simple_a.html"
+        manager = TaskManager(manager_params, browser_params)
+
+        sequence = CommandSequence(TEST_SITE)
+        sequence.get()
+        sequence.save_screenshot()
+        manager.execute_command_sequence(sequence)
+        manager.close()