use corpus webarchives config instead of global one when deploying (#372

)
medialab · Jun 7, 2021 · 86d6123 · 86d6123
1 parent b5cc2d9
commit 86d6123
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/hyphe_backend/crawler/deploy.py b/hyphe_backend/crawler/deploy.py
@@ -41,6 +41,7 @@
 if corpus_conf:
     corpus_conf = corpus_conf["options"]
     config["phantom"].update(corpus_conf["phantom"])
+    config["webarchives"].update(corpus_conf["webarchives"])
     if corpus_conf["proxy"]["host"]:
         config["mongo-scrapy"]["proxy_host"] = corpus_conf["proxy"]["host"]
     if corpus_conf["proxy"]["port"]: