Feature: live and sandbox quals

CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+### Added
+- Separate config settings for mturk qualifications for `live` and `sandbox` modes (#505)
+
 ## [3.2.0]
 ### Added
 - Add custom MTurk qualification support (#493)

doc/settings.rst

@@ -209,6 +209,8 @@ Note that this option does not affect the behavior when a participant starts
 the experiment but the quits or refreshes the page. In those cases, they will
 still be locked out, regardless of the setting of `allow_repeats`.
 
+
+
 .. _require_quals:
 
 require_quals
@@ -219,6 +221,10 @@ perform your task.
 
 :Type: comma-delimited ``string``
 
+If set, applies in both ``live`` and ``sandbox`` modes. Overrides ``require_quals_live`` and ``require_quals_sandbox``.
+
+Deprecated. Use ``require_quals_live`` and ``require_quals_sandbox`` instead.
+
 You may need to ensure that workers have some requisite skill or pass some
 previous screening factors, such as language proficiency or having already
 completed one of your tasks.  AMT uses custom qualification types to perform
@@ -233,6 +239,29 @@ and `Best practices for managing workers in follow-up surveys <https://blog.mtur
 for additional details on custom qualifications.
 
 
+require_quals_live
+~~~~~~~~~~~~~~~~~~
+
+A list of custom qualifications that participants must possess to
+perform your task.
+
+:Type: comma-delimited ``string``
+
+Will only be used during ``live`` mode. Is overridden by ``require_quals``, if set.
+
+
+require_quals_sandbox
+~~~~~~~~~~~~~~~~~~~~~
+
+A list of custom qualifications that participants must possess to
+perform your task.
+
+:Type: comma-delimited ``string``
+
+Will only be used during ``sandbox`` mode. Is overridden by ``require_quals``, if set.
+
+
+
 .. _block_quals:
 
 block_quals
@@ -243,11 +272,41 @@ perform your task.
 
 :Type: comma-delimited ``string``
 
+If set, applies in both ``live`` and ``sandbox`` modes. Overrides ``block_quals_live`` and ``block_quals_sandbox``.
+
+Deprecated. Use ``block_quals_live`` and ``block_quals_sandbox`` instead.
+
 When you add a custom qualification to ``block_quals``, MTurk
 workers with that qualification already set will neither see your ad nor be able
 to accept your HIT. This is the recommended way of excluding participants who
 have performed other HITs for you from participating in your new HIT.
 
+
+
+block_quals_live
+~~~~~~~~~~~~~~~~
+
+Will only be used during ``live`` mode. Is overridden by ``block_quals``, if set.
+
+A list of custom qualifications that participants must not possess to
+perform your task.
+
+:Type: comma-delimited ``string``
+
+
+
+block_quals_sandbox
+~~~~~~~~~~~~~~~~~~~
+
+Will only be used during ``sandbox`` mode. Is overridden by ``block_quals``, if set.
+
+A list of custom qualifications that participants must not possess to
+perform your task.
+
+:Type: comma-delimited ``string``
+
+
+
 .. _advanced_quals:
 
 advanced_quals_path
@@ -256,6 +315,10 @@ advanced_quals_path
 A path to a custom JSON qualifications file, where you can define your own
 MTurk qualification requirements, as seen in `advanced_quals.json.sample`__
 
+If set, applies in both ``live`` and ``sandbox`` modes. Overrides ``advanced_quals_path_live`` and ``advanced_quals_path_sandbox``.
+
+Deprecated. Use ``advanced_quals_path_live`` and ``advanced_quals_path_sandbox`` instead.
+
 __ https://raw.githubusercontent.com/NYUCCL/psiTurk/master/psiturk/example/advanced_quals.json.sample
 
 :type: ``path``
@@ -265,6 +328,41 @@ Example::
     advanced_quals_path = ./advanced_quals.json
 
 
+
+advanced_quals_path_live
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+A path to a custom JSON qualifications file, where you can define your own
+MTurk qualification requirements, as seen in `advanced_quals.json.sample`__
+
+:type: ``path``
+
+Will only be used during ``live`` mode. Is overridden by ``advanced_quals_path``, if set.
+
+__ https://raw.githubusercontent.com/NYUCCL/psiTurk/master/psiturk/example/advanced_quals.json.sample
+
+
+advanced_quals_path_sandbox
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A path to a custom JSON qualifications file, where you can define your own
+MTurk qualification requirements, as seen in `advanced_quals.json.sample`__
+
+:type: ``path``
+
+Will only be used during ``sandbox`` mode. Is overridden by ``advanced_quals_path``, if set.
+
+__ https://raw.githubusercontent.com/NYUCCL/psiTurk/master/psiturk/example/advanced_quals.json.sample
+
+
+
+
+
+
+
+
+
+
 .. _hit_configuration_ad_url:
 
 Hit Configuration -- Ad Url

psiturk/amt_services_wrapper.py

@@ -818,16 +818,19 @@ def _generate_hit_config(self, ad_url, num_workers, reward, duration, require_qu
         if block_qualification_ids is None:
             block_qualification_ids = []
 
-        require_quals = self.config.get('HIT Configuration', 'require_quals', fallback=None)
+        key = f'require_quals_{self.mode}'
+        require_quals = self.config.get('HIT Configuration', key, fallback=None)
         if require_quals:
             require_qualification_ids.extend(require_quals.split(','))
 
-        block_quals = self.config.get('HIT Configuration', 'block_quals', fallback=None)
+        key = f'block_quals_{self.mode}'
+        block_quals = self.config.get('HIT Configuration', key, fallback=None)
         if block_quals:
             block_qualification_ids.extend(block_quals.split(','))
-
-        advanced_quals_path = self.config.get('HIT Configuration', 'advanced_quals_path', fallback=None)
-        advanced_qualifications = []
+
+        key = f'advanced_quals_path_{self.mode}'
+        advanced_quals_path = self.config.get('HIT Configuration', key, fallback=None)
+        advanced_qualifications = [] # default
         if advanced_quals_path:
             with open(advanced_quals_path) as f:
                 advanced_qualifications = json.load(f)

psiturk/default_configs/local_config_defaults.txt

@@ -38,22 +38,84 @@ approve_requirement = 95
 number_hits_approved = 0
 
 # Whether a worker must have the master worker qualification to take this hit.
-# Note that there is a commission surcharge for using this qualification
+# Note that there is a commission surcharge for using this qualification.
+# Applies in both `live` and `sandbox` modes
 require_master_workers = false
 
 # qualification_ids, commma-delimited a worker is required to have in order to
 # take this HIT
+#
+# If set, applies in both `live` and `sandbox` modes. Overrides `require_quals_{mode}`.
+#
+# Deprecated. Use `require_quals_live` and `require_quals_sandbox` instead.
 require_quals =
 
+
+# qualification_ids, commma-delimited a worker is required to have in order to
+# take this HIT
+#
+# Will only be used during `live` mode. Is overridden by `require_quals`, if set.
+require_quals_live =
+
+
+# qualification_ids, commma-delimited a worker is required to have in order to
+# take this HIT
+#
+# Will only be used during `sandbox` mode. Is overridden by `require_quals`, if set.
+require_quals_sandbox =
+
+
 # qualification_ids, commma-delimited, that will disqualify a worker from
 # accepting this HIT
+#
+# If set, applies in both `live` and `sandbox` modes. Overrides `block_quals_{mode}`.
+#
+# Deprecated. Use `block_quals_live` and `block_quals_sandbox` instead.
 block_quals =
 
+
+# qualification_ids, commma-delimited, that will disqualify a worker from
+# accepting this HIT
+#
+# Will only be used during `live` mode. Is overridden by `block_quals`, if set.
+block_quals_live =
+
+
+# qualification_ids, commma-delimited, that will disqualify a worker from
+# accepting this HIT
+#
+# Will only be used during `sandbox` mode. Is overridden by `block_quals`, if set.
+block_quals_sandbox =
+
+
 # A path to a custom JSON qualifications file, where you can define your own
 # MTurk qualification requirements, as seen in advanced_quals.json.sample
 # Example:
 #    ;advanced_quals_path = ./advanced_quals.json
-advanced_quals_path = 
+#
+# If set, applies in both `live` and `sandbox` modes. Overrides `advanced_quals_path_{mode}`.
+#
+# Deprecated. Use `advanced_quals_path_live` and `advanced_quals_path_sandbox` instead.
+advanced_quals_path =
+
+
+# A path to a custom JSON qualifications file, where you can define your own
+# MTurk qualification requirements, as seen in advanced_quals.json.sample
+# Example:
+#    ;advanced_quals_path = ./advanced_quals.json
+#
+# Will only be used during `live` mode. Is overridden by `advanced_quals_path`, if set.
+advanced_quals_path_live =
+
+
+# A path to a custom JSON qualifications file, where you can define your own
+# MTurk qualification requirements, as seen in advanced_quals.json.sample
+# Example:
+#    ;advanced_quals_path = ./advanced_quals.json
+#
+# Will only be used during `sandbox` mode. Is overridden by `advanced_quals_path`, if set.
+advanced_quals_path_sandbox =
+
 
 ## Hit Configuration - Ad Url ##################################################
 # Config settings for constructing the task's "landing page"
@@ -109,10 +171,10 @@ port = 22362
 # If ON_CLOUD env var is set, then these both default to stdout (-)
 #
 accesslog = /dev/null
-errorlog = server.log
+errorlog =
 # For backwards compatibility, `logfile` is synonymous with `errorlog`. If
 # both are set, `errorlog` will be preferred over `logfile`.
-;logfile = server.log
+logfile = server.log
 
 # Log level for the psiturk gunicorn server
 loglevel = 2

psiturk/example/config.txt.sample

@@ -39,22 +39,84 @@
 ;number_hits_approved = 0
 
 # Whether a worker must have the master worker qualification to take this hit.
-# Note that there is a commission surcharge for using this qualification
+# Note that there is a commission surcharge for using this qualification.
+# Applies in both `live` and `sandbox` modes
 ;require_master_workers = false
 
 # qualification_ids, commma-delimited a worker is required to have in order to
 # take this HIT
+#
+# If set, applies in both `live` and `sandbox` modes. Overrides `require_quals_{mode}`.
+#
+# Deprecated. Use `require_quals_live` and `require_quals_sandbox` instead.
 ;require_quals =
 
+
+# qualification_ids, commma-delimited a worker is required to have in order to
+# take this HIT
+#
+# Will only be used during `live` mode. Is overridden by `require_quals`, if set.
+;require_quals_live =
+
+
+# qualification_ids, commma-delimited a worker is required to have in order to
+# take this HIT
+#
+# Will only be used during `sandbox` mode. Is overridden by `require_quals`, if set.
+;require_quals_sandbox =
+
+
 # qualification_ids, commma-delimited, that will disqualify a worker from
 # accepting this HIT
+#
+# If set, applies in both `live` and `sandbox` modes. Overrides `block_quals_{mode}`.
+#
+# Deprecated. Use `block_quals_live` and `block_quals_sandbox` instead.
 ;block_quals =
 
+
+# qualification_ids, commma-delimited, that will disqualify a worker from
+# accepting this HIT
+#
+# Will only be used during `live` mode. Is overridden by `block_quals`, if set.
+;block_quals_live =
+
+
+# qualification_ids, commma-delimited, that will disqualify a worker from
+# accepting this HIT
+#
+# Will only be used during `sandbox` mode. Is overridden by `block_quals`, if set.
+;block_quals_sandbox =
+
+
 # A path to a custom JSON qualifications file, where you can define your own
 # MTurk qualification requirements, as seen in advanced_quals.json.sample
 # Example:
 #    ;advanced_quals_path = ./advanced_quals.json
-;advanced_quals_path = 
+#
+# If set, applies in both `live` and `sandbox` modes. Overrides `advanced_quals_path_{mode}`.
+#
+# Deprecated. Use `advanced_quals_path_live` and `advanced_quals_path_sandbox` instead.
+;advanced_quals_path =
+
+
+# A path to a custom JSON qualifications file, where you can define your own
+# MTurk qualification requirements, as seen in advanced_quals.json.sample
+# Example:
+#    ;advanced_quals_path = ./advanced_quals.json
+#
+# Will only be used during `live` mode. Is overridden by `advanced_quals_path`, if set.
+;advanced_quals_path_live =
+
+
+# A path to a custom JSON qualifications file, where you can define your own
+# MTurk qualification requirements, as seen in advanced_quals.json.sample
+# Example:
+#    ;advanced_quals_path = ./advanced_quals.json
+#
+# Will only be used during `sandbox` mode. Is overridden by `advanced_quals_path`, if set.
+;advanced_quals_path_sandbox =
+
 
 ## Hit Configuration - Ad Url ##################################################
 # Config settings for constructing the task's "landing page"
@@ -110,7 +172,7 @@
 # If ON_CLOUD env var is set, then these both default to stdout (-)
 #
 ;accesslog = /dev/null
-;errorlog = server.log
+;errorlog =
 # For backwards compatibility, `logfile` is synonymous with `errorlog`. If
 # both are set, `errorlog` will be preferred over `logfile`.
 ;logfile = server.log

psiturk/experiment.py

@@ -37,7 +37,7 @@
 logging.CRITICAL]
 LOG_LEVEL = LOG_LEVELS[CONFIG.getint('Server Parameters', 'loglevel')]
 
-logfile = CONFIG.get("Server Parameters", "errorlog")
+logfile = CONFIG.get("Server Parameters", "logfile")
 if logfile != '-':
     file_path = os.path.join(os.getcwd(), logfile)
     logging.basicConfig(filename=file_path, format='%(asctime)s %(message)s',