From e6ba7036857e8417f730b911e431b2e23641325a Mon Sep 17 00:00:00 2001
From: Dave Eargle <dave@daveeargle.com>
Date: Wed, 19 May 2021 22:28:48 -0600
Subject: [PATCH] Feature: live and sandbox quals (#505); simplify backwards
 compat logic

* allow separate qualifications for sandbox and live mode
* simplify backwards compat logic

`prefer_this` if set, should override the value of `over_this`, and the
code should usually access the `over_this` key.
---
 CHANGELOG.md                                  |  4 +
 doc/settings.rst                              | 98 +++++++++++++++++++
 psiturk/amt_services_wrapper.py               | 13 ++-
 .../default_configs/local_config_defaults.txt | 70 ++++++++++++-
 psiturk/example/config.txt.sample             | 68 ++++++++++++-
 psiturk/experiment.py                         |  2 +-
 psiturk/psiturk_config.py                     | 55 +++++++++--
 7 files changed, 288 insertions(+), 22 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index aca80824..ead7ca96 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+### Added
+- Separate config settings for mturk qualifications for `live` and `sandbox` modes (#505)
+
 ## [3.2.0]
 ### Added
 - Add custom MTurk qualification support (#493)
diff --git a/doc/settings.rst b/doc/settings.rst
index 5ca3a46c..d541382d 100644
--- a/doc/settings.rst
+++ b/doc/settings.rst
@@ -209,6 +209,8 @@ Note that this option does not affect the behavior when a participant starts
 the experiment but the quits or refreshes the page. In those cases, they will
 still be locked out, regardless of the setting of `allow_repeats`.
 
+
+
 .. _require_quals:
 
 require_quals
@@ -219,6 +221,10 @@ perform your task.
 
 :Type: comma-delimited ``string``
 
+If set, applies in both ``live`` and ``sandbox`` modes. Overrides ``require_quals_live`` and ``require_quals_sandbox``.
+
+Deprecated. Use ``require_quals_live`` and ``require_quals_sandbox`` instead.
+
 You may need to ensure that workers have some requisite skill or pass some
 previous screening factors, such as language proficiency or having already
 completed one of your tasks.  AMT uses custom qualification types to perform
@@ -233,6 +239,29 @@ and `Best practices for managing workers in follow-up surveys <https://blog.mtur
 for additional details on custom qualifications.
 
 
+require_quals_live
+~~~~~~~~~~~~~~~~~~
+
+A list of custom qualifications that participants must possess to
+perform your task.
+
+:Type: comma-delimited ``string``
+
+Will only be used during ``live`` mode. Is overridden by ``require_quals``, if set.
+
+
+require_quals_sandbox
+~~~~~~~~~~~~~~~~~~~~~
+
+A list of custom qualifications that participants must possess to
+perform your task.
+
+:Type: comma-delimited ``string``
+
+Will only be used during ``sandbox`` mode. Is overridden by ``require_quals``, if set.
+
+
+
 .. _block_quals:
 
 block_quals
@@ -243,11 +272,41 @@ perform your task.
 
 :Type: comma-delimited ``string``
 
+If set, applies in both ``live`` and ``sandbox`` modes. Overrides ``block_quals_live`` and ``block_quals_sandbox``.
+
+Deprecated. Use ``block_quals_live`` and ``block_quals_sandbox`` instead.
+
 When you add a custom qualification to ``block_quals``, MTurk
 workers with that qualification already set will neither see your ad nor be able
 to accept your HIT. This is the recommended way of excluding participants who
 have performed other HITs for you from participating in your new HIT.
 
+
+
+block_quals_live
+~~~~~~~~~~~~~~~~
+
+Will only be used during ``live`` mode. Is overridden by ``block_quals``, if set.
+
+A list of custom qualifications that participants must not possess to
+perform your task.
+
+:Type: comma-delimited ``string``
+
+
+
+block_quals_sandbox
+~~~~~~~~~~~~~~~~~~~
+
+Will only be used during ``sandbox`` mode. Is overridden by ``block_quals``, if set.
+
+A list of custom qualifications that participants must not possess to
+perform your task.
+
+:Type: comma-delimited ``string``
+
+
+
 .. _advanced_quals:
 
 advanced_quals_path
@@ -256,6 +315,10 @@ advanced_quals_path
 A path to a custom JSON qualifications file, where you can define your own
 MTurk qualification requirements, as seen in `advanced_quals.json.sample`__
 
+If set, applies in both ``live`` and ``sandbox`` modes. Overrides ``advanced_quals_path_live`` and ``advanced_quals_path_sandbox``.
+
+Deprecated. Use ``advanced_quals_path_live`` and ``advanced_quals_path_sandbox`` instead.
+
 __ https://raw.githubusercontent.com/NYUCCL/psiTurk/master/psiturk/example/advanced_quals.json.sample
 
 :type: ``path``
@@ -265,6 +328,41 @@ Example::
     advanced_quals_path = ./advanced_quals.json
 
 
+
+advanced_quals_path_live
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+A path to a custom JSON qualifications file, where you can define your own
+MTurk qualification requirements, as seen in `advanced_quals.json.sample`__
+
+:type: ``path``
+
+Will only be used during ``live`` mode. Is overridden by ``advanced_quals_path``, if set.
+
+__ https://raw.githubusercontent.com/NYUCCL/psiTurk/master/psiturk/example/advanced_quals.json.sample
+
+
+advanced_quals_path_sandbox
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A path to a custom JSON qualifications file, where you can define your own
+MTurk qualification requirements, as seen in `advanced_quals.json.sample`__
+
+:type: ``path``
+
+Will only be used during ``sandbox`` mode. Is overridden by ``advanced_quals_path``, if set.
+
+__ https://raw.githubusercontent.com/NYUCCL/psiTurk/master/psiturk/example/advanced_quals.json.sample
+
+
+
+
+
+
+
+
+
+
 .. _hit_configuration_ad_url:
 
 Hit Configuration -- Ad Url
diff --git a/psiturk/amt_services_wrapper.py b/psiturk/amt_services_wrapper.py
index 31b62626..5c75de69 100644
--- a/psiturk/amt_services_wrapper.py
+++ b/psiturk/amt_services_wrapper.py
@@ -818,16 +818,19 @@ def _generate_hit_config(self, ad_url, num_workers, reward, duration, require_qu
         if block_qualification_ids is None:
             block_qualification_ids = []
 
-        require_quals = self.config.get('HIT Configuration', 'require_quals', fallback=None)
+        key = f'require_quals_{self.mode}'
+        require_quals = self.config.get('HIT Configuration', key, fallback=None)
         if require_quals:
             require_qualification_ids.extend(require_quals.split(','))
 
-        block_quals = self.config.get('HIT Configuration', 'block_quals', fallback=None)
+        key = f'block_quals_{self.mode}'
+        block_quals = self.config.get('HIT Configuration', key, fallback=None)
         if block_quals:
             block_qualification_ids.extend(block_quals.split(','))
-        
-        advanced_quals_path = self.config.get('HIT Configuration', 'advanced_quals_path', fallback=None)
-        advanced_qualifications = []
+
+        key = f'advanced_quals_path_{self.mode}'
+        advanced_quals_path = self.config.get('HIT Configuration', key, fallback=None)
+        advanced_qualifications = [] # default
         if advanced_quals_path:
             with open(advanced_quals_path) as f:
                 advanced_qualifications = json.load(f)
diff --git a/psiturk/default_configs/local_config_defaults.txt b/psiturk/default_configs/local_config_defaults.txt
index 1d26635c..316e7bf3 100644
--- a/psiturk/default_configs/local_config_defaults.txt
+++ b/psiturk/default_configs/local_config_defaults.txt
@@ -38,22 +38,84 @@ approve_requirement = 95
 number_hits_approved = 0
 
 # Whether a worker must have the master worker qualification to take this hit.
-# Note that there is a commission surcharge for using this qualification
+# Note that there is a commission surcharge for using this qualification.
+# Applies in both `live` and `sandbox` modes
 require_master_workers = false
 
 # qualification_ids, commma-delimited a worker is required to have in order to
 # take this HIT
+#
+# If set, applies in both `live` and `sandbox` modes. Overrides `require_quals_{mode}`.
+#
+# Deprecated. Use `require_quals_live` and `require_quals_sandbox` instead.
 require_quals =
 
+
+# qualification_ids, commma-delimited a worker is required to have in order to
+# take this HIT
+#
+# Will only be used during `live` mode. Is overridden by `require_quals`, if set.
+require_quals_live =
+
+
+# qualification_ids, commma-delimited a worker is required to have in order to
+# take this HIT
+#
+# Will only be used during `sandbox` mode. Is overridden by `require_quals`, if set.
+require_quals_sandbox =
+
+
 # qualification_ids, commma-delimited, that will disqualify a worker from
 # accepting this HIT
+#
+# If set, applies in both `live` and `sandbox` modes. Overrides `block_quals_{mode}`.
+#
+# Deprecated. Use `block_quals_live` and `block_quals_sandbox` instead.
 block_quals =
 
+
+# qualification_ids, commma-delimited, that will disqualify a worker from
+# accepting this HIT
+#
+# Will only be used during `live` mode. Is overridden by `block_quals`, if set.
+block_quals_live =
+
+
+# qualification_ids, commma-delimited, that will disqualify a worker from
+# accepting this HIT
+#
+# Will only be used during `sandbox` mode. Is overridden by `block_quals`, if set.
+block_quals_sandbox =
+
+
 # A path to a custom JSON qualifications file, where you can define your own
 # MTurk qualification requirements, as seen in advanced_quals.json.sample
 # Example:
 #    ;advanced_quals_path = ./advanced_quals.json
-advanced_quals_path = 
+#
+# If set, applies in both `live` and `sandbox` modes. Overrides `advanced_quals_path_{mode}`.
+#
+# Deprecated. Use `advanced_quals_path_live` and `advanced_quals_path_sandbox` instead.
+advanced_quals_path =
+
+
+# A path to a custom JSON qualifications file, where you can define your own
+# MTurk qualification requirements, as seen in advanced_quals.json.sample
+# Example:
+#    ;advanced_quals_path = ./advanced_quals.json
+#
+# Will only be used during `live` mode. Is overridden by `advanced_quals_path`, if set.
+advanced_quals_path_live =
+
+
+# A path to a custom JSON qualifications file, where you can define your own
+# MTurk qualification requirements, as seen in advanced_quals.json.sample
+# Example:
+#    ;advanced_quals_path = ./advanced_quals.json
+#
+# Will only be used during `sandbox` mode. Is overridden by `advanced_quals_path`, if set.
+advanced_quals_path_sandbox =
+
 
 ## Hit Configuration - Ad Url ##################################################
 # Config settings for constructing the task's "landing page"
@@ -109,10 +171,10 @@ port = 22362
 # If ON_CLOUD env var is set, then these both default to stdout (-)
 #
 accesslog = /dev/null
-errorlog = server.log
+errorlog =
 # For backwards compatibility, `logfile` is synonymous with `errorlog`. If
 # both are set, `errorlog` will be preferred over `logfile`.
-;logfile = server.log
+logfile = server.log
 
 # Log level for the psiturk gunicorn server
 loglevel = 2
diff --git a/psiturk/example/config.txt.sample b/psiturk/example/config.txt.sample
index c22e3fc7..37069d54 100644
--- a/psiturk/example/config.txt.sample
+++ b/psiturk/example/config.txt.sample
@@ -39,22 +39,84 @@
 ;number_hits_approved = 0
 
 # Whether a worker must have the master worker qualification to take this hit.
-# Note that there is a commission surcharge for using this qualification
+# Note that there is a commission surcharge for using this qualification.
+# Applies in both `live` and `sandbox` modes
 ;require_master_workers = false
 
 # qualification_ids, commma-delimited a worker is required to have in order to
 # take this HIT
+#
+# If set, applies in both `live` and `sandbox` modes. Overrides `require_quals_{mode}`.
+#
+# Deprecated. Use `require_quals_live` and `require_quals_sandbox` instead.
 ;require_quals =
 
+
+# qualification_ids, commma-delimited a worker is required to have in order to
+# take this HIT
+#
+# Will only be used during `live` mode. Is overridden by `require_quals`, if set.
+;require_quals_live =
+
+
+# qualification_ids, commma-delimited a worker is required to have in order to
+# take this HIT
+#
+# Will only be used during `sandbox` mode. Is overridden by `require_quals`, if set.
+;require_quals_sandbox =
+
+
 # qualification_ids, commma-delimited, that will disqualify a worker from
 # accepting this HIT
+#
+# If set, applies in both `live` and `sandbox` modes. Overrides `block_quals_{mode}`.
+#
+# Deprecated. Use `block_quals_live` and `block_quals_sandbox` instead.
 ;block_quals =
 
+
+# qualification_ids, commma-delimited, that will disqualify a worker from
+# accepting this HIT
+#
+# Will only be used during `live` mode. Is overridden by `block_quals`, if set.
+;block_quals_live =
+
+
+# qualification_ids, commma-delimited, that will disqualify a worker from
+# accepting this HIT
+#
+# Will only be used during `sandbox` mode. Is overridden by `block_quals`, if set.
+;block_quals_sandbox =
+
+
 # A path to a custom JSON qualifications file, where you can define your own
 # MTurk qualification requirements, as seen in advanced_quals.json.sample
 # Example:
 #    ;advanced_quals_path = ./advanced_quals.json
-;advanced_quals_path = 
+#
+# If set, applies in both `live` and `sandbox` modes. Overrides `advanced_quals_path_{mode}`.
+#
+# Deprecated. Use `advanced_quals_path_live` and `advanced_quals_path_sandbox` instead.
+;advanced_quals_path =
+
+
+# A path to a custom JSON qualifications file, where you can define your own
+# MTurk qualification requirements, as seen in advanced_quals.json.sample
+# Example:
+#    ;advanced_quals_path = ./advanced_quals.json
+#
+# Will only be used during `live` mode. Is overridden by `advanced_quals_path`, if set.
+;advanced_quals_path_live =
+
+
+# A path to a custom JSON qualifications file, where you can define your own
+# MTurk qualification requirements, as seen in advanced_quals.json.sample
+# Example:
+#    ;advanced_quals_path = ./advanced_quals.json
+#
+# Will only be used during `sandbox` mode. Is overridden by `advanced_quals_path`, if set.
+;advanced_quals_path_sandbox =
+
 
 ## Hit Configuration - Ad Url ##################################################
 # Config settings for constructing the task's "landing page"
@@ -110,7 +172,7 @@
 # If ON_CLOUD env var is set, then these both default to stdout (-)
 #
 ;accesslog = /dev/null
-;errorlog = server.log
+;errorlog =
 # For backwards compatibility, `logfile` is synonymous with `errorlog`. If
 # both are set, `errorlog` will be preferred over `logfile`.
 ;logfile = server.log
diff --git a/psiturk/experiment.py b/psiturk/experiment.py
index a6904f62..5fcb9bab 100644
--- a/psiturk/experiment.py
+++ b/psiturk/experiment.py
@@ -37,7 +37,7 @@
 logging.CRITICAL]
 LOG_LEVEL = LOG_LEVELS[CONFIG.getint('Server Parameters', 'loglevel')]
 
-logfile = CONFIG.get("Server Parameters", "errorlog")
+logfile = CONFIG.get("Server Parameters", "logfile")
 if logfile != '-':
     file_path = os.path.join(os.getcwd(), logfile)
     logging.basicConfig(filename=file_path, format='%(asctime)s %(message)s',
diff --git a/psiturk/psiturk_config.py b/psiturk/psiturk_config.py
index bba37cc0..f003a743 100644
--- a/psiturk/psiturk_config.py
+++ b/psiturk/psiturk_config.py
@@ -57,18 +57,54 @@ def load_config(self):
 
         # backwards compatibility
         backwards_compatibilities = [
+            # logging
             {
                 'in_section': 'Server Parameters',
                 'prefer_this': 'errorlog',
                 'over_this': 'logfile'
-            }
+            },
+            # require_quals
+            {
+                'in_section': 'HIT Configuration',
+                'prefer_this': 'require_quals',
+                'over_this': 'require_quals_live'
+            },
+            {
+                'in_section': 'HIT Configuration',
+                'prefer_this': 'require_quals',
+                'over_this': 'require_quals_sandbox'
+            },
+            # block_quals
+            {
+                'in_section': 'HIT Configuration',
+                'prefer_this': 'block_quals',
+                'over_this': 'block_quals_live'
+            },
+            {
+                'in_section': 'HIT Configuration',
+                'prefer_this': 'block_quals',
+                'over_this': 'block_quals_sandbox'
+            },
+            # advanced_quals_path
+            {
+                'in_section': 'HIT Configuration',
+                'prefer_this': 'advanced_quals_path',
+                'over_this': 'advanced_quals_path_live'
+            },
+            {
+                'in_section': 'HIT Configuration',
+                'prefer_this': 'advanced_quals_path',
+                'over_this': 'advanced_quals_path_sandbox'
+            },
         ]
         for bc in backwards_compatibilities:
-            if (not self.has_option(bc['in_section'], bc['prefer_this'])) and self.has_option(bc['in_section'], bc['over_this']):
-                self.set(bc['in_section'], bc['prefer_this'], self.get(bc['in_section'], bc['over_this']))
-            env_key = f'PSITURK_{bc["over_this"].upper()}'
+            env_key = f'PSITURK_{bc["prefer_this"].upper()}'
             if env_key in os.environ:
-                self.set(bc['in_section'], bc['prefer_this'], os.environ.get(env_key))
+                self.set(bc['in_section'], bc['over_this'], os.environ.get(env_key))
+            else:
+                preferred = self.get('HIT Configuration', bc['prefer_this'], fallback=None)
+                if preferred:
+                    self.set(bc['in_section'], bc['over_this'], preferred)
 
         # prefer environment
         these_as_they_are = [
@@ -76,7 +112,7 @@ def load_config(self):
             'DATABASE_URL',
             'AWS_ACCESS_KEY_ID',
             'AWS_SECRET_ACCESS_KEY'
-            ]  
+            ]
         for section in self.sections():
             for config_var in self[section]:
                 config_var_upper = config_var.upper()
@@ -93,9 +129,6 @@ def load_config(self):
                 if config_val_env_override:
                     self.set(section, config_var, config_val_env_override)
 
-        # Also set bc's in reverse direction, for code compatibility
-        for bc in backwards_compatibilities:
-            self.set(bc['in_section'], bc['over_this'], self.get(bc['in_section'], bc['prefer_this']))
 
         # heroku files are ephemeral.
         # Error if we're trying to use a file as the db
@@ -105,6 +138,7 @@ def load_config(self):
             if ('localhost' in database_url) or ('sqlite' in database_url):
                 raise EphemeralContainerDBError(database_url)
 
+
     def get_ad_url(self):
         """Get ad url."""
         if self.has_option('HIT Configuration', 'ad_url'):
@@ -121,3 +155,6 @@ def get_ad_url(self):
             ad_url_port = self.get('HIT Configuration', 'ad_url_port')
             ad_url_route = self.get('HIT Configuration', 'ad_url_route')
             return f"{ad_url_protocol}://{ad_url_domain}:{ad_url_port}/{ad_url_route}"
+
+    def get_require_quals(self):
+        pass