Move storage module up and deduplicate MediaStore tests (#192)

* Move storage module higher up Signed-off-by: Olga Bulat <obulat@gmail.com> * Deduplicate MediaStore tests Signed-off-by: Olga Bulat <obulat@gmail.com> * Use constants Signed-off-by: Olga Bulat <obulat@gmail.com> * Parametrize image tests Co-authored-by: Madison Swain-Bowden <bowdenm@spu.edu> * Pluralize table name lookup dictionary Signed-off-by: Olga Bulat <obulat@gmail.com> * Fix lint error Signed-off-by: Olga Bulat <obulat@gmail.com> Co-authored-by: Madison Swain-Bowden <bowdenm@spu.edu>
WordPress · Sep 30, 2021 · 7af6564 · 7af6564
1 parent 793d67a
commit 7af6564
Show file tree

Hide file tree

Showing 39 changed files with 351 additions and 845 deletions.
diff --git a/openverse_catalog/dags/common/storage/test_image.py b/openverse_catalog/dags/common/storage/test_image.py
diff --git a/openverse_catalog/dags/provider_api_scripts/brooklyn_museum.py b/openverse_catalog/dags/provider_api_scripts/brooklyn_museum.py
@@ -4,7 +4,7 @@
 import lxml.html as html
 from common.licenses.licenses import get_license_info
 from common.requester import DelayedRequester
-from common.storage.image import ImageStore
+from storage.image import ImageStore
 from util.loader import provider_details as prov
 
 

diff --git a/openverse_catalog/dags/provider_api_scripts/cleveland_museum_of_art.py b/openverse_catalog/dags/provider_api_scripts/cleveland_museum_of_art.py
@@ -2,7 +2,7 @@
 
 from common.licenses.licenses import get_license_info
 from common.requester import DelayedRequester
-from common.storage.image import ImageStore
+from storage.image import ImageStore
 from util.loader import provider_details as prov
 
 

diff --git a/openverse_catalog/dags/provider_api_scripts/europeana.py b/openverse_catalog/dags/provider_api_scripts/europeana.py
@@ -16,7 +16,7 @@
 
 from common.licenses.licenses import get_license_info
 from common.requester import DelayedRequester
-from common.storage.image import ImageStore
+from storage.image import ImageStore
 from util.loader import provider_details as prov
 
 

diff --git a/openverse_catalog/dags/provider_api_scripts/finnish_museums.py b/openverse_catalog/dags/provider_api_scripts/finnish_museums.py
@@ -2,7 +2,7 @@
 
 from common.licenses.licenses import get_license_info
 from common.requester import DelayedRequester
-from common.storage.image import ImageStore
+from storage.image import ImageStore
 from util.loader import provider_details as prov
 
 

diff --git a/openverse_catalog/dags/provider_api_scripts/flickr.py b/openverse_catalog/dags/provider_api_scripts/flickr.py
@@ -18,7 +18,7 @@
 import lxml.html as html
 from common.licenses.licenses import get_license_info
 from common.requester import DelayedRequester
-from common.storage.image import ImageStore
+from storage.image import ImageStore
 from util.loader import provider_details as prov
 
 

diff --git a/openverse_catalog/dags/provider_api_scripts/jamendo.py b/openverse_catalog/dags/provider_api_scripts/jamendo.py
@@ -21,8 +21,8 @@
 
 from common.licenses.licenses import get_license_info
 from common.requester import DelayedRequester
-from common.storage.audio import AudioStore
 from common.urls import rewrite_redirected_url
+from storage.audio import AudioStore
 from util.loader import provider_details as prov
 
 

diff --git a/openverse_catalog/dags/provider_api_scripts/metropolitan_museum_of_art.py b/openverse_catalog/dags/provider_api_scripts/metropolitan_museum_of_art.py
@@ -15,7 +15,7 @@
 
 from common.licenses.licenses import get_license_info
 from common.requester import DelayedRequester
-from common.storage.image import ImageStore
+from storage.image import ImageStore
 
 
 DELAY = 1.0  # time delay (in seconds)

diff --git a/openverse_catalog/dags/provider_api_scripts/museum_victoria.py b/openverse_catalog/dags/provider_api_scripts/museum_victoria.py
@@ -2,7 +2,7 @@
 
 from common.licenses.licenses import get_license_info
 from common.requester import DelayedRequester
-from common.storage.image import ImageStore
+from storage.image import ImageStore
 from util.loader import provider_details as prov
 
 

diff --git a/openverse_catalog/dags/provider_api_scripts/nypl.py b/openverse_catalog/dags/provider_api_scripts/nypl.py
@@ -4,7 +4,7 @@
 
 from common.licenses.licenses import get_license_info
 from common.requester import DelayedRequester
-from common.storage.image import ImageStore
+from storage.image import ImageStore
 from util.loader import provider_details as prov
 
 

diff --git a/openverse_catalog/dags/provider_api_scripts/phylopic.py b/openverse_catalog/dags/provider_api_scripts/phylopic.py
@@ -15,7 +15,7 @@
 
 from common.licenses.licenses import get_license_info
 from common.requester import DelayedRequester
-from common.storage.image import ImageStore
+from storage.image import ImageStore
 
 
 logging.basicConfig(

diff --git a/openverse_catalog/dags/provider_api_scripts/raw_pixel.py b/openverse_catalog/dags/provider_api_scripts/raw_pixel.py
@@ -4,7 +4,7 @@
 import requests
 from common.licenses.licenses import get_license_info
 from common.requester import DelayedRequester
-from common.storage.image import ImageStore
+from storage.image import ImageStore
 from util.loader import provider_details as prov
 
 

diff --git a/openverse_catalog/dags/provider_api_scripts/science_museum.py b/openverse_catalog/dags/provider_api_scripts/science_museum.py
@@ -2,7 +2,7 @@
 
 from common.licenses.licenses import get_license_info
 from common.requester import DelayedRequester
-from common.storage.image import ImageStore
+from storage.image import ImageStore
 from util.loader import provider_details as prov
 
 

diff --git a/openverse_catalog/dags/provider_api_scripts/smithsonian.py b/openverse_catalog/dags/provider_api_scripts/smithsonian.py
@@ -15,7 +15,7 @@
 
 from common.licenses.licenses import LicenseInfo
 from common.requester import DelayedRequester
-from common.storage.image import ImageStore
+from storage.image import ImageStore
 from util.loader import provider_details as prov
 
 

diff --git a/openverse_catalog/dags/provider_api_scripts/staten_museum.py b/openverse_catalog/dags/provider_api_scripts/staten_museum.py
@@ -2,7 +2,7 @@
 
 from common.licenses.licenses import get_license_info
 from common.requester import DelayedRequester
-from common.storage.image import ImageStore
+from storage.image import ImageStore
 from util.loader import provider_details as prov
 
 

diff --git a/openverse_catalog/dags/provider_api_scripts/stocksnap.py b/openverse_catalog/dags/provider_api_scripts/stocksnap.py
@@ -15,7 +15,7 @@
 
 from common.licenses.licenses import get_license_info
 from common.requester import DelayedRequester
-from common.storage.image import ImageStore
+from storage.image import ImageStore
 from util.loader import provider_details as prov
 
 

diff --git a/openverse_catalog/dags/provider_api_scripts/test_cleveland_museum_of_art.py b/openverse_catalog/dags/provider_api_scripts/test_cleveland_museum_of_art.py
@@ -6,7 +6,7 @@
 import cleveland_museum_of_art as clm
 import requests
 from common.licenses.licenses import LicenseInfo
-from common.storage.image import MockImageStore
+from storage.image import MockImageStore
 
 
 _license_info = (

diff --git a/openverse_catalog/dags/provider_api_scripts/test_raw_pixel.py b/openverse_catalog/dags/provider_api_scripts/test_raw_pixel.py
@@ -5,7 +5,7 @@
 
 import raw_pixel as rwp
 from common.licenses.licenses import LicenseInfo
-from common.storage.image import MockImageStore
+from storage.image import MockImageStore
 
 
 _license_info = (

diff --git a/openverse_catalog/dags/provider_api_scripts/test_science_museum.py b/openverse_catalog/dags/provider_api_scripts/test_science_museum.py
@@ -6,7 +6,7 @@
 import requests
 import science_museum as sm
 from common.licenses.licenses import LicenseInfo
-from common.storage.image import MockImageStore
+from storage.image import MockImageStore
 
 
 _license_info = (

diff --git a/openverse_catalog/dags/provider_api_scripts/walters_art_museum.py b/openverse_catalog/dags/provider_api_scripts/walters_art_museum.py
@@ -14,7 +14,7 @@
 import os
 
 from common.requester import DelayedRequester
-from common.storage.image import ImageStore
+from storage.image import ImageStore
 from util.loader import provider_details as prov
 
 

diff --git a/openverse_catalog/dags/provider_api_scripts/wikimedia_commons.py b/openverse_catalog/dags/provider_api_scripts/wikimedia_commons.py
@@ -20,7 +20,7 @@
 import lxml.html as html
 from common.licenses.licenses import get_license_info
 from common.requester import DelayedRequester
-from common.storage.image import ImageStore
+from storage.image import ImageStore
 from util.loader import provider_details as prov
 
 

diff --git a/...e_catalog/dags/common/storage/__init__.py → openverse_catalog/dags/storage/__init__.py b/...e_catalog/dags/common/storage/__init__.py → openverse_catalog/dags/storage/__init__.py
diff --git a/...erse_catalog/dags/common/storage/audio.py → openverse_catalog/dags/storage/audio.py b/...erse_catalog/dags/common/storage/audio.py → openverse_catalog/dags/storage/audio.py
@@ -3,8 +3,8 @@
 from typing import Dict, Optional, Union
 
 from common.licenses.licenses import LicenseInfo
-from common.storage import columns
-from common.storage.media import MediaStore
+from storage import columns
+from storage.media import MediaStore
 
 
 logger = logging.getLogger(__name__)

diff --git a/..._catalog/dags/util/loader/column_names.py → ...erse_catalog/dags/storage/column_names.py b/..._catalog/dags/util/loader/column_names.py → ...erse_catalog/dags/storage/column_names.py
diff --git a/...se_catalog/dags/common/storage/columns.py → openverse_catalog/dags/storage/columns.py b/...se_catalog/dags/common/storage/columns.py → openverse_catalog/dags/storage/columns.py
diff --git a/...erse_catalog/dags/common/storage/image.py → openverse_catalog/dags/storage/image.py b/...erse_catalog/dags/common/storage/image.py → openverse_catalog/dags/storage/image.py
@@ -3,8 +3,8 @@
 from typing import Dict, Optional, Union
 
 from common.licenses.licenses import LicenseInfo
-from common.storage import columns
-from common.storage.media import MediaStore
+from storage import columns
+from storage.media import MediaStore
 
 
 logger = logging.getLogger(__name__)

diff --git a/...erse_catalog/dags/common/storage/media.py → openverse_catalog/dags/storage/media.py b/...erse_catalog/dags/common/storage/media.py → openverse_catalog/dags/storage/media.py
@@ -5,7 +5,7 @@
 from typing import Optional, Union
 
 from common.licenses.licenses import is_valid_license_info
-from common.storage import util
+from storage import util
 
 
 logger = logging.getLogger(__name__)

diff --git a/...catalog/dags/common/storage/test_audio.py → openverse_catalog/dags/storage/test_audio.py b/...catalog/dags/common/storage/test_audio.py → openverse_catalog/dags/storage/test_audio.py
@@ -3,7 +3,7 @@
 
 import pytest
 from common.licenses.licenses import LicenseInfo
-from common.storage import audio
+from storage import audio
 
 
 logging.basicConfig(

diff --git a/...talog/dags/common/storage/test_columns.py → ...erse_catalog/dags/storage/test_columns.py b/...talog/dags/common/storage/test_columns.py → ...erse_catalog/dags/storage/test_columns.py
@@ -2,7 +2,7 @@
 import string
 
 import tldextract
-from common.storage import columns
+from storage import columns
 
 
 logging.basicConfig(