Support ISO 3166 locale codes

network-api/networkapi/mozfest/migrations/0002_auto_20200708_2215.py

@@ -0,0 +1,53 @@
+# Generated by Django 2.2.13 on 2020-07-08 22:15
+
+from django.db import migrations, models
+import networkapi.wagtailpages.pagemodels.blog.blog_category
+import wagtail.core.blocks
+import wagtail.core.blocks.static_block
+import wagtail.core.fields
+import wagtail.images.blocks
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('mozfest', '0001_to_0015'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='mozfesthomepage',
+            name='banner_guide_text_fy_NL',
+            field=models.CharField(blank=True, help_text='A banner paragraph specific to the homepage', max_length=1000, null=True),
+        ),
+        migrations.AddField(
+            model_name='mozfesthomepage',
+            name='banner_heading_fy_NL',
+            field=models.CharField(blank=True, help_text='A banner heading specific to the homepage', max_length=250, null=True),
+        ),
+        migrations.AddField(
+            model_name='mozfesthomepage',
+            name='banner_video_url_fy_NL',
+            field=models.URLField(blank=True, help_text='The video to play when users click "watch video"', max_length=2048, null=True),
+        ),
+        migrations.AddField(
+            model_name='mozfesthomepage',
+            name='cta_button_label_fy_NL',
+            field=models.CharField(blank=True, help_text='Label text for the CTA button in the primary nav bar', max_length=250, null=True),
+        ),
+        migrations.AddField(
+            model_name='mozfestprimarypage',
+            name='body_fy_NL',
+            field=wagtail.core.fields.StreamField([('paragraph', wagtail.core.blocks.RichTextBlock(features=['bold', 'italic', 'large', 'h2', 'h3', 'h4', 'h5', 'ol', 'ul', 'link', 'hr'])), ('image', wagtail.core.blocks.StructBlock([('image', wagtail.images.blocks.ImageChooserBlock()), ('altText', wagtail.core.blocks.CharBlock(help_text='Image description (for screen readers).', required=True)), ('caption', wagtail.core.blocks.CharBlock(required=False)), ('captionURL', wagtail.core.blocks.CharBlock(help_text='Optional URL that this caption should link out to.', required=False))])), ('image_text', wagtail.core.blocks.StructBlock([('image', wagtail.images.blocks.ImageChooserBlock()), ('altText', wagtail.core.blocks.CharBlock(help_text='Image description (for screen readers).', required=True)), ('text', wagtail.core.blocks.RichTextBlock(features=['bold', 'italic', 'h2', 'h3', 'h4', 'h5', 'h6', 'ol', 'ul', 'link'])), ('url', wagtail.core.blocks.CharBlock(help_text='Optional URL that this image should link out to.', required=False)), ('top_divider', wagtail.core.blocks.BooleanBlock(help_text='Optional divider above content block.', required=False)), ('bottom_divider', wagtail.core.blocks.BooleanBlock(help_text='Optional divider below content block.', required=False))])), ('image_text_mini', wagtail.core.blocks.StructBlock([('image', wagtail.images.blocks.ImageChooserBlock()), ('altText', wagtail.core.blocks.CharBlock(help_text='Image description (for screen readers).', required=True)), ('text', wagtail.core.blocks.RichTextBlock(features=['bold', 'italic', 'link']))])), ('image_grid', wagtail.core.blocks.StructBlock([('grid_items', wagtail.core.blocks.ListBlock(wagtail.core.blocks.StructBlock([('image', wagtail.images.blocks.ImageChooserBlock()), ('caption', wagtail.core.blocks.CharBlock(help_text='Please remember to properly attribute any images we use.', required=False)), ('url', wagtail.core.blocks.CharBlock(help_text='Optional URL that this figure should link out to.', required=False)), ('square_image', wagtail.core.blocks.BooleanBlock(default=True, help_text='If left checked, the image will be cropped to be square.', required=False))])))])), ('video', wagtail.core.blocks.StructBlock([('url', wagtail.core.blocks.CharBlock(help_text='To make sure a video will embed properly, go to your YouTube video and click “Share,” then “Embed,” and then copy and paste the provided URL only. For example: https://www.youtube.com/embed/3FIVXBawyQw')), ('caption', wagtail.core.blocks.CharBlock(required=False)), ('captionURL', wagtail.core.blocks.CharBlock(help_text='Optional URL for caption to link to.', required=False))])), ('iframe', wagtail.core.blocks.StructBlock([('url', wagtail.core.blocks.CharBlock(help_text='Please note that only URLs from white-listed domains will work.')), ('caption', wagtail.core.blocks.CharBlock(required=False)), ('captionURL', wagtail.core.blocks.CharBlock(help_text='Optional URL that this caption should link out to.', required=False))])), ('linkbutton', wagtail.core.blocks.StructBlock([('label', wagtail.core.blocks.CharBlock()), ('URL', wagtail.core.blocks.CharBlock()), ('styling', wagtail.core.blocks.ChoiceBlock(choices=[('btn-primary', 'Primary button'), ('btn-secondary', 'Secondary button')]))])), ('spacer', wagtail.core.blocks.StructBlock([('size', wagtail.core.blocks.ChoiceBlock(choices=[('1', 'quarter spacing'), ('2', 'half spacing'), ('3', 'single spacing'), ('4', 'one and a half spacing'), ('5', 'triple spacing')]))])), ('quote', wagtail.core.blocks.StructBlock([('quotes', wagtail.core.blocks.ListBlock(wagtail.core.blocks.StructBlock([('quote', wagtail.core.blocks.CharBlock()), ('attribution', wagtail.core.blocks.CharBlock())])))])), ('pulse_listing', wagtail.core.blocks.StructBlock([('search_terms', wagtail.core.blocks.CharBlock(help_text='Test your search at mozillapulse.org/search', label='Search', required=False)), ('max_number_of_results', wagtail.core.blocks.IntegerBlock(default=6, help_text='Choose 1-12. If you want visitors to see more, link to a search or tag on Pulse.', max_value=12, min_value=0, required=True)), ('only_featured_entries', wagtail.core.blocks.BooleanBlock(default=False, help_text='Featured items are selected by Pulse moderators.', label='Display only featured entries', required=False)), ('newest_first', wagtail.core.blocks.ChoiceBlock(choices=[('True', 'Show newer entries first'), ('False', 'Show older entries first')], label='Sort')), ('advanced_filter_header', wagtail.core.blocks.static_block.StaticBlock(admin_text='-------- ADVANCED FILTERS: OPTIONS TO DISPLAY FEWER, MORE TARGETED RESULTS. --------', label=' ')), ('issues', wagtail.core.blocks.ChoiceBlock(choices=[('all', 'All'), ('Decentralization', 'Decentralization'), ('Digital Inclusion', 'Digital Inclusion'), ('Online Privacy & Security', 'Online Privacy & Security'), ('Open Innovation', 'Open Innovation'), ('Web Literacy', 'Web Literacy')])), ('help', wagtail.core.blocks.ChoiceBlock(choices=[('all', 'All'), ('Attend', 'Attend'), ('Create content', 'Create content'), ('Code', 'Code'), ('Design', 'Design'), ('Fundraise', 'Fundraise'), ('Join community', 'Join community'), ('Localize & translate', 'Localize & translate'), ('Mentor', 'Mentor'), ('Plan & organize', 'Plan & organize'), ('Promote', 'Promote'), ('Take action', 'Take action'), ('Test & feedback', 'Test & feedback'), ('Write documentation', 'Write documentation')], label='Type of help needed')), ('direct_link', wagtail.core.blocks.BooleanBlock(default=False, help_text='Checked: user goes to project link. Unchecked: user goes to pulse entry', label='Direct link', required=False))])), ('profile_listing', wagtail.core.blocks.StructBlock([('max_number_of_results', wagtail.core.blocks.IntegerBlock(default=12, help_text='Pick up to 48 profiles.', max_value=48, min_value=1, required=True)), ('advanced_filter_header', wagtail.core.blocks.static_block.StaticBlock(admin_text='-------- ADVANCED FILTERS: OPTIONS TO DISPLAY FEWER, MORE TARGETED RESULTS. --------', label=' ')), ('profile_type', wagtail.core.blocks.CharBlock(default='', help_text='Example: Fellow.', required=False)), ('program_type', wagtail.core.blocks.CharBlock(default='', help_text='Example: Tech Policy.', required=False)), ('year', wagtail.core.blocks.CharBlock(default='', required=False))])), ('profile_by_id', wagtail.core.blocks.StructBlock([('ids', wagtail.core.blocks.CharBlock(help_text='Show profiles for pulse users with specific profile ids (mozillapulse.org/profile/[##]). For multiple profiles, specify a comma separated list (e.g. 85,105,332).', label='Profile by ID'))])), ('profile_directory', wagtail.core.blocks.StructBlock([('max_number_of_results', wagtail.core.blocks.IntegerBlock(default=12, help_text='Pick up to 48 profiles.', max_value=48, min_value=1, required=True)), ('advanced_filter_header', wagtail.core.blocks.static_block.StaticBlock(admin_text='-------- ADVANCED FILTERS: OPTIONS TO DISPLAY FEWER, MORE TARGETED RESULTS. --------', label=' ')), ('profile_type', wagtail.core.blocks.CharBlock(default='', help_text='Example: Fellow.', required=False)), ('program_type', wagtail.core.blocks.CharBlock(default='', help_text='Example: Tech Policy.', required=False)), ('year', wagtail.core.blocks.CharBlock(default='', required=False)), ('filter_values', wagtail.core.blocks.CharBlock(default='2019,2018,2017,2016,2015,2014,2013', help_text='Example: 2019,2018,2017,2016,2015,2014,2013', required=True))])), ('recent_blog_entries', wagtail.core.blocks.StructBlock([('title', wagtail.core.blocks.CharBlock(required=True)), ('tag_filter', wagtail.core.blocks.CharBlock(help_text='Test this filter at foundation.mozilla.org/blog/tags/', label='Filter by Tag', required=False)), ('category_filter', wagtail.core.blocks.ChoiceBlock(choices=networkapi.wagtailpages.pagemodels.blog.blog_category.BlogPageCategory.get_categories, help_text='Test this filter at foundation.mozilla.org/blog/category/', label='Filter by Category', required=False)), ('top_divider', wagtail.core.blocks.BooleanBlock(help_text='Optional divider above content block.', required=False)), ('bottom_divider', wagtail.core.blocks.BooleanBlock(help_text='Optional divider below content block.', required=False))])), ('airtable', wagtail.core.blocks.StructBlock([('url', wagtail.core.blocks.URLBlock(help_text="Copied from the Airtable embed code. The word 'embed' will be in the url")), ('height', wagtail.core.blocks.IntegerBlock(default=533, help_text='The height of the view on a desktop, usually copied from the Airtable embed code'))])), ('typeform', wagtail.core.blocks.StructBlock([('url', wagtail.core.blocks.URLBlock(help_text='The URL of the published Typeform')), ('button_type', wagtail.core.blocks.ChoiceBlock(choices=[('btn-primary', 'Primary button'), ('btn-secondary', 'Secondary button')])), ('button_text', wagtail.core.blocks.CharBlock(required=True))]))], null=True),
+        ),
+        migrations.AddField(
+            model_name='mozfestprimarypage',
+            name='header_fy_NL',
+            field=models.CharField(blank=True, max_length=250, null=True),
+        ),
+        migrations.AddField(
+            model_name='mozfestprimarypage',
+            name='intro_fy_NL',
+            field=wagtail.core.fields.RichTextField(blank=True, help_text='Page intro content', null=True),
+        ),
+    ]

network-api/networkapi/settings.py

@@ -403,6 +403,7 @@
     ('pt', gettext_lazy('Portuguese')),
     ('es', gettext_lazy('Spanish')),
     ('fr', gettext_lazy('French')),
+    ('fy-NL', gettext_lazy('Frisian')),
     ('nl', gettext_lazy('Dutch')),
     ('pl', gettext_lazy('Polish')),
 )

network-api/networkapi/templates/fragments/language_switcher.html

@@ -7,12 +7,12 @@
     <select name="language" id="language-switcher" class="mt-3 mt-md-0 ml-md-3 w-100 form-control" onchange="this.form.submit()">
       {% get_current_language as LANGUAGE_CODE %}
       {% get_available_languages as LANGUAGES %}
-      {% get_language_info_list for LANGUAGES as languages %}
-      {% for language in languages %}
-      <option value="{{ language.code }}"{% if language.code == LANGUAGE_CODE %} selected{% endif %}>
-        {{ language.name_local | capfirst }}
-      </option>
-    {% endfor %}
+      {% for CODE, NAME in LANGUAGES %}
+        {% get_language_info for CODE as lang %}
+        <option value="{{ CODE }}"{% if CODE == LANGUAGE_CODE %} selected{% endif %}>
+          {{ lang.name_local | capfirst }}
+        </option>
+      {% endfor %}
     </select>
   </div>
 </form>

network-api/networkapi/tests.py

@@ -3,12 +3,19 @@
 from django.contrib.auth.models import User, Group
 from django.core.management import call_command
 from django.test import TestCase, RequestFactory
+from django.urls import reverse
+from django.utils import translation
+from django.utils.translation.trans_real import (
+    to_language as django_to_language,
+    parse_accept_lang_header as django_parse_accept_lang_header
+)
 from unittest.mock import MagicMock
 
 from wagtail_factories import SiteFactory
 
 from networkapi.utility.redirects import redirect_to_default_cms_site
 from networkapi.utility.middleware import ReferrerMiddleware, XRobotsTagMiddleware
+from networkapi.wagtailpages import language_code_to_iso_3166, parse_accept_lang_header, to_language
 
 
 class ReferrerMiddlewareTests(TestCase):
@@ -134,16 +141,55 @@ def test_PNI_homepage_redirect_to_foundation_site(self):
             fetch_redirect_response=False
         )
 
-    class XRobotsTagMiddlewareTest(TestCase):
-        def test_returns_response(self):
-            xrobotstag_middleware = XRobotsTagMiddleware('response')
-            self.assertEqual(xrobotstag_middleware.get_response, 'response')
 
-        def test_sends_x_robots_tag(self):
-            """
-            Ensure that the middleware assigns an X-Robots-Tag to the response
-            """
+class WagtailPagesTestCase(TestCase):
 
-            xrobotstag_middleware = XRobotsTagMiddleware(MagicMock())
-            response = xrobotstag_middleware(MagicMock())
-            response.__setitem__.assert_called_with('X-Robots-Tag', 'noindex')
+    def test_get_language_code_to_iso_3166(self):
+        self.assertEqual(language_code_to_iso_3166('en-gb'), 'en-GB')
+        self.assertEqual(language_code_to_iso_3166('en-us'), 'en-US')
+        self.assertEqual(language_code_to_iso_3166('fr'), 'fr')
+
+    def test_to_language(self):
+        self.assertEqual(to_language('en_US'), 'en-US')
+
+    def test_parse_accept_lang_header_returns_iso_3166_language(self):
+        self.assertEqual(
+            parse_accept_lang_header('en-GB,en;q=0.5'),
+            (('en-GB', 1.0), ('en', 0.5)),
+        )
+
+
+class WagtailPagesIntegrationTestCase(TestCase):
+
+    """
+    Test that our overrides to Django translation functions work.
+    """
+    def test_to_language(self):
+        self.assertEqual(django_to_language('fy_NL'), 'fy-NL')
+
+    def test_parse_accept_lang_header_returns_iso_3166_language(self):
+        self.assertEqual(
+            django_parse_accept_lang_header('fy-NL,fy;q=0.5'),
+            (('fy-NL', 1.0), ('fy', 0.5)),
+        )
+
+    def test_reverse_produces_correct_url_prefix(self):
+        translation.activate('fy-NL')
+        url = reverse('buyersguide-home')
+        self.assertTrue(url.startswith('/fy-NL/'))
+        translation.deactivate()
+
+
+class XRobotsTagMiddlewareTest(TestCase):
+    def test_returns_response(self):
+        xrobotstag_middleware = XRobotsTagMiddleware('response')
+        self.assertEqual(xrobotstag_middleware.get_response, 'response')
+
+    def test_sends_x_robots_tag(self):
+        """
+        Ensure that the middleware assigns an X-Robots-Tag to the response
+        """
+
+        xrobotstag_middleware = XRobotsTagMiddleware(MagicMock())
+        response = xrobotstag_middleware(MagicMock())
+        response.__setitem__.assert_called_with('X-Robots-Tag', 'noindex')

network-api/networkapi/wagtailpages/__init__.py

@@ -1,5 +1,9 @@
+import django
+import functools
+
 from django.urls import reverse
 from django.utils.translation import ugettext_lazy as _
+from django.utils.translation.trans_real import accept_language_re
 
 from wagtail.admin.menu import MenuItem
 from wagtail.core import hooks
@@ -16,3 +20,54 @@ def register_howto_menu_item():
         _('How Do I Wagtail'), reverse('how-do-i-wagtail'),
         name='howdoIwagtail', classnames='icon icon-help', order=900
     )
+
+
+# WARNING: this is not necessarily a good idea, but is the only way to override
+# Django's default behaviour of requiring language codes to be lowercased.
+# We have to modify the core Django method, because we have no way to replace
+# all the core functionality that relies on this - e.g., url resolvers that
+# the Django admin and third party apps use.
+# A fix upstream has been asked in https://code.djangoproject.com/ticket/31795
+
+
+def language_code_to_iso_3166(language):
+    """Turn a language name (en-us) into an ISO 3166 format (en-US)."""
+    language, _, country = language.lower().partition('-')
+    if country:
+        return f'{language}-{country.upper()}'
+    return language
+
+
+def to_language(locale):
+    """Turn a locale name (en_US) into a language name (en-US)."""
+    return locale.replace('_', '-')
+
+
+@functools.lru_cache(maxsize=1000)
+def parse_accept_lang_header(lang_string):
+    """
+    Parse the lang_string, which is the body of an HTTP Accept-Language
+    header, and return a tuple of (lang, q-value), ordered by 'q' values.
+    Return an empty tuple if there are any format errors in lang_string.
+    """
+    result = []
+    pieces = accept_language_re.split(lang_string.lower())
+    if pieces[-1]:
+        return ()
+    for i in range(0, len(pieces) - 1, 3):
+        first, lang, priority = pieces[i:i + 3]
+        if first:
+            return ()
+        if priority:
+            priority = float(priority)
+        else:
+            priority = 1.0
+        result.append((language_code_to_iso_3166(lang), priority))
+    result.sort(key=lambda k: k[1], reverse=True)
+    return tuple(result)
+
+
+# Replace some functions in django.utils.translation.trans_real with our own
+# versions that support a language in the form en-US instead of en-us.
+django.utils.translation.trans_real.to_language = to_language
+django.utils.translation.trans_real.parse_accept_lang_header = parse_accept_lang_header