Merge pull request #617 from datamade/patch/hcg/revert-revert-bill-fi…

…lter

Revise bill display logic

README.md

@@ -75,6 +75,29 @@ docker-compose run --rm app python manage.py update_index
 When the command exits, your search index has been filled. (You can view the
 Solr admin panel at http://localhost:8987/solr.)
 
+## Running arbitrary scrapes
+Occasionally, after a while without running an event scrape, you may find that your local app is broken. If this happens, make sure you have events in your database that are scheduled for the future, as the app queries for upcoming events in order to render the landing page.
+
+1. Make sure there are future events scheduled in Legistar. Go to the [LA Metro Legistar page](https://metro.legistar.com/Calendar.aspx) and open up the time filter for "All Years".
+
+2. If you notice that there are future events in Legistar, run a small windowed event scrape:
+
+```bash
+docker-compose run --rm scrapers pupa update lametro events window=0.05 --rpm=0
+```
+
+This will bring in a few upcoming events, and your app will be able to load the landing page.
+
+## Scraping specific bill
+It's sometimes helpful to make sure you have a specific bill in your database for debugging. Here's how you can scrape a bill you need:
+
+1. Go to the Legistar Web API at the following URL: http://webapi.legistar.com/v1/metro/matters/?$filter=MatterFile%20eq%20%27<bill_identifier>%27 and find the `<MatterId>` of the bill. The identifier should be in XXXX-XXXX format, and the `MatterId` is a 4 digit number.
+
+2. Run the following command in your shell:
+```bash
+docker-compose run --rm scrapers pupa update lametro bills matter_ids=<bill_matter_id> --rpm=0
+```
+
 ## Making changes to the Solr schema
 
 Did you make a change to the schema file that Solr uses to make its magic (`solr_configs/conf/schema.xml`)? Did you add a new field or adjust how Solr indexes data? If so, you need to take a few steps – locally and on the server.

lametro/models.py

@@ -54,31 +54,48 @@ def get_queryset(self):
         N.b., the scrapers contain logic for populating the restrict_view field:
         https://github.com/opencivicdata/scrapers-us-municipal/blob/master/lametro/bills.py
 
-        (2) Does the Bill have a classification of "Board Box"? Then, show it.
+        (2) Does the Bill have a classification of "Board Box" or "Board
+        Correspondence"? Then, show it.
 
         (3) Is the Bill on a published agenda, i.e., an event with the
         status of "passed" or "cancelled"? Then, show it.
 
-        NOTE! A single bill can appear on multiple event agendas.
-        We thus call 'distinct' on the below query, otherwise
-        the queryset would contain duplicate bills.
+        (4) Sometimes motions are made during meetings that were not submitted
+        in advance, i.e., they do not appear on the published agenda. They will
+        be entered as matter history, which we translate to bill actions. Does
+        the bill have any associated actions? Then, show it.
+        https://github.com/datamade/la-metro-councilmatic/issues/477
+
+        NOTE! A single bill can appear on multiple event agendas. We thus call
+        'distinct' on the below query, otherwise the queryset would contain
+        duplicate bills.
 
         WARNING! Be sure to use LAMetroBill, rather than the base Bill class,
         when getting bill querysets. Otherwise restricted view bills
         may slip through the crevices of Councilmatic display logic.
         '''
         qs = super().get_queryset()
 
+        on_published_agenda = (
+            Q(eventrelatedentity__agenda_item__event__status='passed') |
+            Q(eventrelatedentity__agenda_item__event__status='cancelled')
+        )
+        is_board_box = Q(board_box=True)
+        has_minutes_history = (
+            Q(actions__isnull=False) &
+            Q(extras__local_classification='Motion / Motion Response')
+        )
+
         qs = qs.exclude(
             extras__restrict_view=True
         ).annotate(board_box=Case(
-            When(extras__local_classification='Board Box', then=True),
+            When(extras__local_classification__in=('Board Box', 'Board Correspondence'), then=True),
             When(classification__contains=['Board Box'], then=True),
+            When(classification__contains=['Board Correspondence'], then=True),
             default=False,
             output_field=models.BooleanField()
-        )).filter(Q(eventrelatedentity__agenda_item__event__status='passed') | \
-                  Q(eventrelatedentity__agenda_item__event__status='cancelled') | \
-                  Q(board_box=True)
+        )).filter(
+            on_published_agenda | is_board_box | has_minutes_history
         ).distinct()
 
         return qs
@@ -190,14 +207,13 @@ def actions_and_agendas(self):
                 participants__name=action.organization,
                 start_time__date=action.date)
 
-
             action_dict = {
                 'date': action.date_dt,
                 'description': action.description,
                 'event': event,
                 'organization': action.organization
             }
-            
+
             data.append(action_dict)
 
         for event in events:

tests/conftest.py

@@ -12,7 +12,7 @@
     EventRelatedEntity,
     )
 from opencivicdata.core.models import Jurisdiction, Division
-from opencivicdata.legislative.models import EventDocument
+from opencivicdata.legislative.models import EventDocument, BillAction
 from councilmatic_core.models import Bill, Membership
 from lametro.models import LAMetroPerson, LAMetroEvent, LAMetroBill, \
     LAMetroOrganization, LAMetroSubject
@@ -53,6 +53,30 @@ def build(self, **kwargs):
 
     return BillFactory()
 
+
+@pytest.fixture
+@pytest.mark.django_db
+def bill_action(db, bill, metro_organization):
+    class BillActionFactory():
+        def build(self, **kwargs):
+            bill_action_info = {
+                'organization': metro_organization.build(),
+                'description': 'test action',
+                'date': '2019-11-09',
+                'order': 999,
+            }
+
+            bill_action_info.update(kwargs)
+
+            if not bill_action_info.get('bill'):
+                bill_action_info['bill'] = bill.build()
+
+            bill_action = BillAction.objects.create(**bill_action_info)
+
+            return bill_action
+
+    return BillActionFactory()
+
 @pytest.fixture
 @pytest.mark.django_db
 def division(db):

tests/test_bills.py

@@ -66,18 +66,23 @@ def test_format_full_text(bill, text, subject):
 
     assert format_full_text(full_text) == subject
 
-@pytest.mark.parametrize('restrict_view,bill_type,event_status,is_public', [
-        (True, 'Board Box', 'passed', False),
-        (False, 'Board Box', 'passed', True),
-        (False, 'Resolution', 'passed', True),
-        (False, 'Resolution', 'cancelled', True),
-        (False, 'Resolution', 'confirmed', False),
+@pytest.mark.parametrize('restrict_view,bill_type,event_status,has_action,is_public', [
+        (True, 'Board Box', 'passed', False, False),  # private bill
+        (False, 'Board Box', 'passed', False, True),  # board box
+        (False, 'Board Correspondence', 'passed', False, True),  # board correspondence
+        (False, 'Resolution', 'passed', False, True),  # on published agenda
+        (False, 'Resolution', 'cancelled', False, True),  # on published agenda
+        (False, 'Resolution', 'confirmed', False, False),  # not on published agenda
+        (False, 'Resolution', 'passed', True, True),  # has matter history
+        (False, 'Test', 'test', False, False),  # not private, but does not meet conditions for display
     ])
 def test_bill_manager(bill,
+                      bill_action,
                       event_related_entity,
                       restrict_view,
                       bill_type,
                       event_status,
+                      has_action,
                       is_public):
     '''
     Tests if the LAMetroBillManager properly filters public and private bills.
@@ -92,6 +97,9 @@ def test_bill_manager(bill,
 
     some_bill = bill.build(**bill_info)
 
+    if has_action:
+        bill_action.build(bill=some_bill)
+
     event_related_entity_info = {
         'bill': some_bill,
     }