Merge pull request #357 from datamade/viewable-reports

Viewable reports

lametro/models.py

@@ -78,6 +78,34 @@ def get_last_action_date(self):
     def topics(self):
         return [s.subject for s in self.subjects.all()]
 
+    @property 
+    def is_viewable(self):
+        '''
+        Sometimes, a Bill may be imported to Councilmatic, though it should not be visible to the public.
+        This issue summarizes why that might happen: https://github.com/datamade/la-metro-councilmatic/issues/345#issuecomment-421184826
+        Metro staff devised three checks for knowing when to hide or show a report:
+
+        (1) Is the view restricted, i.e., is `MatterRestrictViewViaWeb` set to True in the Legistar API? We skip these bills further upstream. https://github.com/opencivicdata/scrapers-us-municipal/pull/251
+
+        (2) Does the Bill have a classification of "Board Box"? Then, show it.
+
+        (3) Is the Bill on a published agenda, i.e., an event with the status of "passed" or "cancelled"? Then, show it.
+
+        This property coms into play when filtering the SearchQuerySet object in  LAMetroCouncilmaticSearchForm (views.py).
+        Note: we filter the sqs object, rather than remove "hidden" bills from the Solr index.
+        This strategy minimizes complexity (e.g., attempting to implement an LAMetroBillManager), 
+        and this strategy avoids making adjustments to the `data_integrity` script (https://github.com/datamade/django-councilmatic/blob/master/councilmatic_core/management/commands/data_integrity.py)
+        '''
+        if self.bill_type == "Board Box":
+            return True
+
+        events_with_bill = Event.objects.filter(agenda_items__bill_id=self.ocd_id)
+        passed_events = [event for event in events_with_bill if (event.status == 'passed' or event.status == 'cancelled')]
+        if passed_events:
+            return True
+
+        return False
+
 class LAMetroPost(Post):
 
     class Meta:

lametro/search_indexes.py

@@ -10,6 +10,7 @@
 class LAMetroBillIndex(BillIndex, indexes.Indexable):
     topics = indexes.MultiValueField(faceted=True)
     attachment_text = indexes.CharField()
+    viewable = indexes.BooleanField()
 
     def get_model(self):
         return LAMetroBill
@@ -44,3 +45,6 @@ def prepare_topics(self, obj):
 
     def prepare_attachment_text(self, obj):
         return ' '.join(d.full_text for d in obj.documents.all() if d.full_text)
+
+    def prepare_viewable(self, obj):
+        return obj.is_viewable

lametro/views.py

@@ -687,6 +687,8 @@ def search(self):
             # Don't auto-escape my query! https://django-haystack.readthedocs.io/en/v2.4.1/searchqueryset_api.html#SearchQuerySet.filter
             sqs = sqs.filter_or(attachment_text=Raw(self.cleaned_data['q']))
 
+        sqs = sqs.filter(viewable=True)
+
         return sqs
 
 

solr_scripts/schema.xml

@@ -201,6 +201,8 @@
 
     <field name="identifier" type="text_en" indexed="true" stored="true" multiValued="false" />
 
+    <field name="viewable" type="boolean" indexed="true" stored="false" multiValued="false" />
+
   </fields>
 
   <!-- field to use to determine and enforce document uniqueness. -->

tests/conftest.py

@@ -6,9 +6,8 @@
 
 from django.core.management import call_command
 
-from councilmatic_core.models import EventDocument, Bill, Membership
-from lametro.models import LAMetroPerson, LAMetroEvent, LAMetroOrganization
-
+from councilmatic_core.models import EventDocument, Bill, EventAgendaItem, Membership
+from lametro.models import LAMetroPerson, LAMetroEvent, LAMetroBill, LAMetroOrganization
 
 def get_uid_chunk(uid=None):
     '''
@@ -37,7 +36,7 @@ def build(self, **kwargs):
 
             bill_info.update(kwargs)
 
-            bill = Bill.objects.create(**bill_info)
+            bill = LAMetroBill.objects.create(**bill_info)
             bill.save()
 
             return bill
@@ -68,6 +67,29 @@ def build(self, **kwargs):
 
     return EventFactory()
 
+@pytest.fixture
+@pytest.mark.django_db
+def event_agenda_item(db, event):
+    class EventAgendaItemFactory():
+        def build(self, **kwargs):
+            named_event = event.build()
+
+            event_agenda_item_info = {
+                'event_id': named_event.ocd_id,
+                'updated_at': '2017-05-27 11:10:46.574-05',
+                'order': 1,
+            }
+
+            event_agenda_item_info.update(kwargs)
+
+            event_agenda_item = EventAgendaItem.objects.create(**event_agenda_item_info)
+            event_agenda_item.save()
+
+            return event_agenda_item
+
+    return EventAgendaItemFactory()
+
+
 @pytest.fixture
 @pytest.mark.django_db
 def event_document(db):

tests/test_bills.py

@@ -2,7 +2,7 @@
 
 from django.core.urlresolvers import reverse
 
-from councilmatic_core.models import Bill, RelatedBill
+from councilmatic_core.models import Bill, RelatedBill, Event
 from lametro.utils import format_full_text, parse_subject
 
 # This collection of tests checks the functionality of Bill-specific views, helper functions, and relations.
@@ -59,5 +59,35 @@ def test_format_full_text(bill, text, subject):
 
     assert format_full_text(full_text) == subject
 
+@pytest.mark.parametrize('bill_type,event_status,assertion', [
+        ('Board Box', 'passed', True),
+        ('Resolution', 'passed', True),
+        ('Resolution', 'cancelled', True),
+        ('Resolution', 'confirmed', False),
+    ])
+def test_viewable_bill(bill, 
+                       event_agenda_item, 
+                       bill_type, 
+                       event_status, 
+                       assertion):
+    bill_info = {
+        'bill_type': bill_type,
+    }
+    bill = bill.build(**bill_info)
+    bill.refresh_from_db()
+
+    event_agenda_item_info = {
+        'bill_id': bill.ocd_id,
+    }
+    item = event_agenda_item.build(**event_agenda_item_info)
+    item.refresh_from_db()
+
+    event = Event.objects.get(ocd_id=item.event_id)
+    event.status = event_status
+    event.save()
+    event.refresh_from_db()
+
+    assert bill.is_viewable == assertion
+