Merge pull request #2694 from data-for-change/dev

merge dev into master

alembic/versions/97740b33407d_add_numeric_indexes.py

@@ -0,0 +1,36 @@
+"""add numeric indexes
+
+Revision ID: 97740b33407d
+Revises: 53d0b00fb750
+Create Date: 2024-08-24 11:19:33.674396
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '97740b33407d'
+down_revision = '53d0b00fb750'
+branch_labels = None
+depends_on = None
+
+from alembic import op
+# import sqlalchemy as sa
+
+
+# pylint: disable=E1101
+def upgrade():
+    for table in ['markers', 'markers_hebrew']:
+        for field in ['yishuv_symbol', 'street1', 'street2']:
+            op.create_index(f'ix_{table}_{field}', table, [field], unique=False)
+    for table in ['markers_hebrew']:
+        for field in ['yishuv_name', 'street1_hebrew', 'street2_hebrew']:
+            op.drop_index(f'ix_{table}_{field}', table_name=table)
+
+
+# pylint: disable=E1101
+def downgrade():
+    for table in ['markers', 'markers_hebrew']:
+        for field in ['yishuv_symbol', 'street1', 'street2']:
+            op.drop_index(f'ix_{table}_{field}', table_name=table)
+    for table in ['markers_hebrew']:
+        for field in ['yishuv_name', 'street1_hebrew', 'street2_hebrew']:
+            op.create_index(f'ix_{table}_{field}', table, [field], unique=False)

alembic/versions/mn9px8cacn24_add_house_number.py

@@ -0,0 +1,31 @@
+"""add house number
+
+Revision ID: mn9px8cacn24
+Revises: 97740b33407d
+Create Date: 2024-06-16 15:05:30.522542
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'mn9px8cacn24'
+down_revision = '97740b33407d'
+branch_labels = None
+depends_on = None
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('involved_markers_hebrew', sa.Column('house_number', sa.Integer(), nullable=True))
+    op.add_column('vehicles_markers_hebrew', sa.Column('house_number', sa.Integer(), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('vehicles_markers_hebrew', 'house_number')
+    op.drop_column('involved_markers_hebrew', 'house_number')
+    # ### end Alembic commands ###

anyway/db_views.py

@@ -638,6 +638,7 @@ def create_involved_hebrew_markers_hebrew_view(self):
             AccidentMarkerView.street1_hebrew,
             AccidentMarkerView.street2,
             AccidentMarkerView.street2_hebrew,
+            AccidentMarkerView.house_number,
             AccidentMarkerView.non_urban_intersection,
             AccidentMarkerView.non_urban_intersection_hebrew,
             AccidentMarkerView.non_urban_intersection_by_junction_number,
@@ -836,6 +837,7 @@ def create_vehicles_markers_hebrew_view(self):
             AccidentMarkerView.street1_hebrew,
             AccidentMarkerView.street2,
             AccidentMarkerView.street2_hebrew,
+            AccidentMarkerView.house_number,
             AccidentMarkerView.non_urban_intersection,
             AccidentMarkerView.non_urban_intersection_hebrew,
             AccidentMarkerView.non_urban_intersection_by_junction_number,

anyway/models.py

@@ -1231,7 +1231,7 @@ def get_street_by_street_name(yishuv_symbol: int, name: str) -> int:
         )
         if res is None:
             raise ValueError(f"{name}: could not find street in yishuv:{yishuv_symbol}")
-        return res
+        return res.street
 
     @staticmethod
     def get_streets_by_yishuv(yishuv_symbol: int) -> List[dict]:
@@ -2532,6 +2532,7 @@ class InvolvedMarkerView(Base):
     street1_hebrew = Column(Text(), index=True)
     street2 = Column(Integer())
     street2_hebrew = Column(Text(), index=True)
+    house_number = Column(Integer())
     non_urban_intersection = Column(Integer())
     non_urban_intersection_hebrew = Column(Text())
     non_urban_intersection_by_junction_number = Column(Text())
@@ -2729,6 +2730,7 @@ class VehicleMarkerView(Base):
     street1_hebrew = Column(Text(), index=True)
     street2 = Column(Integer())
     street2_hebrew = Column(Text(), index=True)
+    house_number = Column(Integer())
     non_urban_intersection = Column(Integer())
     non_urban_intersection_hebrew = Column(Text())
     non_urban_intersection_by_junction_number = Column(Text())

anyway/parsers/cbs/executor.py

@@ -695,9 +695,10 @@ def get_files(directory):
                     streets_map[key] = [
                         {
                             field_names.street_sign: x[field_names.street_sign],
-                            field_names.street_name: x[field_names.street_name],
+                            field_names.street_name: str(x[field_names.street_name]),
                         }
-                        for _, x in settlement.iterrows() if isinstance(x[field_names.street_name], str)
+                        for _, x in settlement.iterrows() if isinstance(x[field_names.street_name], str) \
+                            or ((isinstance(x[field_names.street_name], int) or isinstance(x[field_names.street_name], float)) and x[field_names.street_name] > 0)
                     ]
 
                 output_files_dict[name] = streets_map

anyway/parsers/cities.py

@@ -53,6 +53,9 @@ def get_city_data_chunks(self, url: str, chunk_size: int) -> Iterable[List[Dict[
                 # "napa": item[NAPA],
                 # "municipal_stance": item[MUNICIPAL_STANCE],
             }
+            if city_entry["yishuv_symbol"] == 0:
+                city_entry["heb_name"] = None
+                city_entry["eng_name"] = None
             chunk.append(city_entry)
             if len(chunk) == chunk_size:
                 yield chunk