Merge branch 'master' into improve_config_flow

.pre-commit-config.yaml

@@ -17,7 +17,7 @@ repos:
     hooks:
       - id: codespell
         args:
-          - --ignore-words-list=hass,alot,datas,dof,dur,farenheit,hist,iff,ines,ist,lightsensor,mut,nd,pres,referer,ser,serie,te,technik,ue,uint,visability,wan,wanna,withing,Adresse,termine,adresse,oder,alle,assistent,hart,marz,worthing,linz,celle,vor
+          - --ignore-words-list=hass,alot,datas,dof,dur,farenheit,hist,iff,ines,ist,lightsensor,mut,nd,pres,referer,ser,serie,te,technik,ue,uint,visability,wan,wanna,withing,Adresse,termine,adresse,oder,alle,assistent,hart,marz,worthing,linz,celle,vor,leibnitz
           - --skip="./.*,*.csv,*.json"
           - --quiet-level=2
         exclude_types: [csv, json]

.vscode/launch.json

@@ -0,0 +1,57 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Test All Sources",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "${workspaceFolder}/custom_components/waste_collection_schedule/waste_collection_schedule/test/test_sources.py",
+            "console": "integratedTerminal",
+            "args": [
+                "-t"
+            ]
+        },
+        {
+            "name": "Test Current Source (.py)",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "${workspaceFolder}/custom_components/waste_collection_schedule/waste_collection_schedule/test/test_sources.py",
+            "console": "integratedTerminal",
+            "args": [
+                "-s",
+                "${fileBasenameNoExtension}",
+                "-l",
+                "-i",
+                "-t"
+            ]
+        },
+        {
+            "name": "Test All ICS Sources",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "${workspaceFolder}/custom_components/waste_collection_schedule/waste_collection_schedule/test/test_sources.py",
+            "console": "integratedTerminal",
+            "args": [
+                "-I",
+                "-t"
+            ]
+        },
+        {
+            "name": "Test Current ICS Source (.yaml)",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "${workspaceFolder}/custom_components/waste_collection_schedule/waste_collection_schedule/test/test_sources.py",
+            "console": "integratedTerminal",
+            "args": [
+                "-y",
+                "${fileBasenameNoExtension}",
+                "-l",
+                "-i",
+                "-t"
+            ]
+        }
+    ]
+}

README.md

@@ -260,6 +260,7 @@ Waste collection schedules in the following formats and countries are supported.
 - [Lackendorf](/doc/source/citiesapps_com.md) / lackendorf.at
 - [Langau](/doc/source/citiesapps_com.md) / langau.at
 - [Langenrohr](/doc/source/citiesapps_com.md) / langenrohr.gv.at
+- [Leibnitz](/doc/source/citiesapps_com.md) / leibnitz.at
 - [Leithaprodersdorf](/doc/source/citiesapps_com.md) / leithaprodersdorf.at
 - [Lendorf](/doc/ics/muellapp_com.md) / muellapp.com
 - [Leoben](/doc/ics/muellapp_com.md) / muellapp.com
@@ -510,6 +511,12 @@ Waste collection schedules in the following formats and countries are supported.
 - [RenoWeb](/doc/source/renoweb_dk.md) / renoweb.dk
 </details>
 
+<details>
+<summary>Finland</summary>
+
+- [Kiertokapula Finland](/doc/source/kiertokapula_fi.md) / kiertokapula.fi
+</details>
+
 <details>
 <summary>France</summary>
 
@@ -708,6 +715,7 @@ Waste collection schedules in the following formats and countries are supported.
 - [Heidelberg](/doc/ics/gipsprojekt_de.md) / heidelberg.de
 - [Heilbronn Entsorgungsbetriebe](/doc/source/heilbronn_de.md) / heilbronn.de
 - [Heinz-Entsorgung (Landkreis Freising)](/doc/ics/heinz_entsorgung_de.md) / heinz-entsorgung.de
+- [Herten (durth-roos.de)](/doc/ics/herten_de.md) / herten.de
 - [Hohenlohekreis](/doc/source/app_abfallplus_de.md) / Abfall+ App: hokwaste
 - [Holtgast (MyMuell App)](/doc/source/jumomind_de.md) / mymuell.de
 - [HubertSchmid Recycling und Umweltschutz GmbH](/doc/source/api_hubert_schmid_de.md) / hschmid24.de/BlaueTonne
@@ -772,6 +780,7 @@ Waste collection schedules in the following formats and countries are supported.
 - [Kreis Vechta](/doc/source/app_abfallplus_de.md) / Abfall+ App: awvapp
 - [Kreis Viersen](/doc/source/abfallnavi_de.md) / kreis-viersen.de
 - [Kreis Vorpommern-Rügen](/doc/source/app_abfallplus_de.md) / Abfall+ App: abfallappvorue
+- [Kreis Waldshut](/doc/source/app_abfallplus_de.md) / Abfall+ App: abfallwecker
 - [Kreis Weißenburg-Gunzenhausen](/doc/source/app_abfallplus_de.md) / Abfall+ App: abfallappwug
 - [Kreis Wesermarsch](/doc/source/app_abfallplus_de.md) / Abfall+ App: abfallappgib
 - [Kreis Würzburg](/doc/source/app_abfallplus_de.md) / Abfall+ App: teamorange
@@ -854,6 +863,7 @@ Waste collection schedules in the following formats and countries are supported.
 - [Landratsamt Bodenseekreis](/doc/ics/bodenseekreis_de.md) / bodenseekreis.de
 - [Landratsamt Dachau](/doc/source/awido_de.md) / landratsamt-dachau.de
 - [Landratsamt Main-Tauber-Kreis](/doc/source/c_trace_de.md) / main-tauber-kreis.de
+- [Landratsamt Regensburg](/doc/source/awido_de.md) / landkreis-regensburg.de
 - [Landratsamt Traunstein](/doc/source/abfall_io.md) / traunstein.com
 - [Landratsamt Unterallgäu](/doc/source/abfall_io.md) / landratsamt-unterallgaeu.de
 - [Landshut](/doc/source/app_abfallplus_de.md) / Abfall+ App: abfallappla
@@ -972,11 +982,10 @@ Waste collection schedules in the following formats and countries are supported.
 - [VIVO Landkreis Miesbach](/doc/source/abfall_io.md) / vivowarngau.de
 - [Volkmarsen (MyMuell App)](/doc/source/jumomind_de.md) / mymuell.de
 - [Vöhringen (MyMuell App)](/doc/source/jumomind_de.md) / mymuell.de
-- [Waldshut](/doc/source/app_abfallplus_de.md) / Abfall+ App: abfallwecker
 - [Waldshut](/doc/source/app_abfallplus_de.md) / Abfall+ App: unterallgaeu
 - [WBO Wirtschaftsbetriebe Oberhausen](/doc/source/abfallnavi_de.md) / wbo-online.de
 - [Wegberg (MyMuell App)](/doc/source/jumomind_de.md) / mymuell.de
-- [Wermelskirchen](/doc/source/wermelskirchen_de.md) / wermelskirchen.de
+- [Wermelskirchen (Service Down)](/doc/source/wermelskirchen_de.md) / wermelskirchen.de
 - [Westerholt (MyMuell App)](/doc/source/jumomind_de.md) / mymuell.de
 - [Westerwaldkreis](/doc/source/app_abfallplus_de.md) / Abfall+ App: wabapp
 - [WGV Recycling GmbH](/doc/source/awido_de.md) / wgv-quarzbichl.de
@@ -1203,18 +1212,22 @@ Waste collection schedules in the following formats and countries are supported.
 - [BCP Council](/doc/source/bcp_gov_uk.md) / bcpcouncil.gov.uk
 - [Bedford Borough Council](/doc/source/bedford_gov_uk.md) / bedford.gov.uk
 - [Binzone](/doc/source/binzone_uk.md) / southoxon.gov.uk
+- [Birmingham City Council](/doc/source/birmingham_gov_uk.md) / birmingham.gov.uk
 - [Blackburn with Darwen Borough Council](/doc/source/blackburn_gov_uk.md) / blackburn.gov.uk
 - [Blackpool Council](/doc/source/blackpool_gov_uk.md) / blackpool.gov.uk
 - [Borough Council of King's Lynn & West Norfolk](/doc/source/west_norfolk_gov_uk.md) / west-norfolk.gov.uk
+- [Borough of Broxbourne Council](/doc/source/broxbourne_gov_uk.md) / broxbourne.gov.uk
 - [Bracknell Forest Council](/doc/source/bracknell_forest_gov_uk.md) / selfservice.mybfc.bracknell-forest.gov.uk
 - [Bradford Metropolitan District Council](/doc/source/bradford_gov_uk.md) / bradford.gov.uk
 - [Braintree District Council](/doc/source/braintree_gov_uk.md) / braintree.gov.uk
 - [Breckland Council](/doc/source/breckland_gov_uk.md) / breckland.gov.uk/mybreckland
 - [Bristol City Council](/doc/source/bristol_gov_uk.md) / bristol.gov.uk
 - [Broadland District Council](/doc/source/south_norfolk_and_broadland_gov_uk.md) / area.southnorfolkandbroadland.gov.uk
+- [Bromsgrove City Council](/doc/source/bromsgrove_gov_uk.md) / bromsgrove.gov.uk
 - [Broxtowe Borough Council](/doc/source/broxtowe_gov_uk.md) / broxtowe.gov.uk
 - [Buckinghamshire Waste Collection - Former Chiltern, South Bucks or Wycombe areas](/doc/source/chiltern_gov_uk.md) / chiltern.gov.uk
 - [Burnley Council](/doc/source/burnley_gov_uk.md) / burnley.gov.uk
+- [Bury Council](/doc/source/bury_gov_uk.md) / bury.gov.uk
 - [Cambridge City Council](/doc/source/cambridge_gov_uk.md) / cambridge.gov.uk
 - [Canterbury City Council](/doc/source/canterbury_gov_uk.md) / canterbury.gov.uk
 - [Cardiff Council](/doc/source/cardiff_gov_uk.md) / cardiff.gov.uk
@@ -1236,6 +1249,7 @@ Waste collection schedules in the following formats and countries are supported.
 - [Derby City Council](/doc/source/derby_gov_uk.md) / derby.gov.uk
 - [Dudley Metropolitan Borough Council](/doc/source/dudley_gov_uk.md) / dudley.gov.uk
 - [Durham County Council](/doc/source/durham_gov_uk.md) / durham.gov.uk
+- [East Ayrshire Council](/doc/source/east_ayrshire_gov_uk.md) / east-ayrshire.gov.uk
 - [East Cambridgeshire District Council](/doc/source/eastcambs_gov_uk.md) / eastcambs.gov.uk
 - [East Devon District Council](/doc/source/eastdevon_gov_uk.md) / eastdevon.gov.uk
 - [East Herts Council](/doc/source/eastherts_gov_uk.md) / eastherts.gov.uk
@@ -1255,6 +1269,7 @@ Waste collection schedules in the following formats and countries are supported.
 - [Flintshire](/doc/source/flintshire_gov_uk.md) / flintshire.gov.uk
 - [Fylde Council](/doc/source/fylde_gov_uk.md) / fylde.gov.uk
 - [Gateshead Council](/doc/source/gateshead_gov_uk.md) / gateshead.gov.uk
+- [Gedling Borough Council (unofficial)](/doc/ics/gedling_gov_uk.md) / github.com/jamesmacwhite/gedling-borough-council-bin-calendars
 - [Glasgow City Council](/doc/source/glasgow_gov_uk.md) / glasgow.gov.uk
 - [Guildford Borough Council](/doc/source/guildford_gov_uk.md) / guildford.gov.uk
 - [Gwynedd](/doc/source/gwynedd_gov_uk.md) / gwynedd.gov.uk
@@ -1296,6 +1311,7 @@ Waste collection schedules in the following formats and countries are supported.
 - [Newcastle City Council](/doc/source/newcastle_gov_uk.md) / community.newcastle.gov.uk
 - [Newcastle Under Lyme Borough Council](/doc/source/newcastle_staffs_gov_uk.md) / newcastle-staffs.gov.uk
 - [Newport City Council](/doc/source/newport_gov_uk.md) / newport.gov.uk
+- [North Ayrshire Council](/doc/source/north_ayrshire_gov_uk.md) / north-ayrshire.gov.uk
 - [North Herts Council](/doc/source/northherts_gov_uk.md) / north-herts.gov.uk
 - [North Kesteven District Council](/doc/source/north_kesteven_org_uk.md) / n-kesteven.org.uk
 - [North Lincolnshire Council](/doc/source/northlincs_gov_uk.md) / northlincs.gov.uk
@@ -1313,6 +1329,7 @@ Waste collection schedules in the following formats and countries are supported.
 - [Reading Council](/doc/source/reading_gov_uk.md) / reading.gov.uk
 - [Redbridge Council](/doc/source/redbridge_gov_uk.md) / redbridge.gov.uk
 - [Reigate & Banstead Borough Council](/doc/source/reigatebanstead_gov_uk.md) / reigate-banstead.gov.uk
+- [Renfrewshire Council](/doc/source/renfrewshire_gov_uk.md) / renfrewshire.gov.uk
 - [Rhondda Cynon Taf County Borough Council](/doc/source/rctcbc_gov_uk.md) / rctcbc.gov.uk
 - [Richmondshire District Council](/doc/source/richmondshire_gov_uk.md) / richmondshire.gov.uk
 - [Rotherham Metropolitan Borough Council](/doc/source/rotherham_gov_uk.md) / rotherham.gov.uk
@@ -1383,6 +1400,7 @@ Waste collection schedules in the following formats and countries are supported.
 <summary>United States of America</summary>
 
 - [Albuquerque, New Mexico, USA](/doc/source/recyclecoach_com.md) / recyclecoach.com/cities/usa-nm-city-of-albuquerque
+- [City of Austin, TX](/doc/ics/recollect.md) / austintexas.gov
 - [City of Bloomington](/doc/ics/recollect.md) / bloomington.in.gov
 - [City of Cambridge](/doc/ics/recollect.md) / cambridgema.gov
 - [City of Gastonia, NC](/doc/ics/recollect.md) / gastonianc.gov

custom_components/waste_collection_schedule/config_flow.py

@@ -304,7 +304,7 @@ async def async_step_user(self, info):
 
         return self.async_show_form(step_id="user", data_schema=SCHEMA)
 
-    # Step 2: User selects country
+    # Step 2: User selects source
     async def async_step_source(self, info=None):
         sources = self._sources[self._country]
         sources_options = [SelectOptionDict(value="", label="")] + [

custom_components/waste_collection_schedule/sensor.py

@@ -126,7 +126,17 @@ async def async_setup_platform(hass, config, async_add_entities, discovery_info=
     source_index = config[CONF_SOURCE_INDEX]
     if not isinstance(source_index, list):
         source_index = [source_index]
-    aggregator = CollectionAggregator([api.get_shell(i) for i in source_index])
+
+    shells = []
+    for i in source_index:
+        shell = api.get_shell(i)
+        if shell is None:
+            raise ValueError(
+                f"source_index {i} out of range (0-{len(api.shells) - 1}) please check your sensor configuration"
+            )
+        shells.append(shell)
+
+    aggregator = CollectionAggregator(shells)
 
     entities = []
 

custom_components/waste_collection_schedule/waste_collection_schedule/collection.py

@@ -34,6 +34,10 @@ def picture(self):
     def set_picture(self, picture: str):
         self["picture"] = picture
 
+    def set_date(self, date: datetime.date):
+        self._date = date
+        self["date"] = date.isoformat()
+
 
 class Collection(CollectionBase):
     def __init__(

custom_components/waste_collection_schedule/waste_collection_schedule/service/AppAbfallplusDe.py

@@ -153,7 +153,7 @@
     "de.abfallwecker": [
         "Rottweil",
         "Tuttlingen",
-        "Waldshut",
+        "Kreis Waldshut",
         "Prignitz",
         "Nordsachsen",
     ],
@@ -352,6 +352,7 @@ def random_hex(length: int = 1) -> str:
 API_BASE = "https://app.abfallplus.de/{}"
 API_ASSISTANT = API_BASE.format("assistent/{}")  # ignore: E501
 USER_AGENT = "{}/9.1.0.0 iOS/17.5 Device/iPhone Screen/1170x2532"
+ABFALLARTEN_H2_SKIP = ["Sondermüll"]
 
 
 def extract_onclicks(
@@ -435,17 +436,12 @@ def _request(
         method="post",
         headers=None,
     ):
-        if headers:
-            headers["User-Agent"] = USER_AGENT.format(
-                MAP_APP_USERAGENTS.get(self._app_id, "%")
-            )
+        if headers is None:
+            headers = {}
 
-        else:
-            headers = {
-                "User-Agent": USER_AGENT.format(
-                    MAP_APP_USERAGENTS.get(self._app_id, "%")
-                )
-            }
+        headers["User-Agent"] = USER_AGENT.format(
+            MAP_APP_USERAGENTS.get(self._app_id, "%")
+        )
 
         if method not in ("get", "post"):
             raise Exception(f"Method {method} not supported.")
@@ -778,6 +774,25 @@ def select_all_waste_types(self):
         r.raise_for_status()
         soup = BeautifulSoup(r.text, features="html.parser")
         self._f_id_abfallart = []
+        for to_skip in ABFALLARTEN_H2_SKIP:
+            to_skip_element = soup.find("h2", text=to_skip)
+            div_to_skip = (
+                to_skip_element.find_parent("div") if to_skip_element else None
+            )
+            if div_to_skip:
+                for input in to_skip_element.find_parent("div").find_all(
+                    "input", {"name": "f_id_abfallart[]"}
+                ):
+                    if compare(input.text, self._region_search, remove_space=True):
+                        id = input.attrs["id"].split("_")[-1]
+                        self._f_id_abfallart.append(input.attrs["value"])
+                        self._needs_subtitle.append(id)
+                        if id.isdigit():
+                            self._needs_subtitle.append(str(int(id) - 1))
+                        break
+                # remove sondermuell h2 from soup
+                div_to_skip.decompose()
+
         for input in soup.find_all("input", {"name": "f_id_abfallart[]"}):
             if input.attrs["value"] == "0":
                 if "id" not in input.attrs:
@@ -790,6 +805,7 @@ def select_all_waste_types(self):
                 continue
 
             self._f_id_abfallart.append(input.attrs["value"])
+        self._f_id_abfallart = list(set(self._f_id_abfallart))
         self._needs_subtitle = list(set(self._needs_subtitle))
 
     def validate(self):

custom_components/waste_collection_schedule/waste_collection_schedule/service/CitiesAppsCom.py

@@ -236,6 +236,7 @@
     {"title": "Lackendorf", "url": "https://www.lackendorf.at", "country": "at"},
     {"title": "Langau", "url": "http://www.langau.at", "country": "at"},
     {"title": "Langenrohr", "url": "https://www.langenrohr.gv.at", "country": "at"},
+    {"title": "Leibnitz", "url": "https://www.leibnitz.at", "country": "at"},
     {
         "title": "Leithaprodersdorf",
         "url": "http://www.leithaprodersdorf.at",

custom_components/waste_collection_schedule/waste_collection_schedule/service/Samiljo_se_wastetype_searcher.py

@@ -3,8 +3,6 @@
 import time
 import requests
 from bs4 import BeautifulSoup
-#import threading
-import sys
 from requests.exceptions import HTTPError
 from http import HTTPStatus
 import argparse

custom_components/waste_collection_schedule/waste_collection_schedule/source/aberdeenshire_gov_uk.py

@@ -1,5 +1,3 @@
-import requests
-
 from bs4 import BeautifulSoup
 from datetime import datetime
 from waste_collection_schedule import Collection  # type: ignore[attr-defined]

custom_components/waste_collection_schedule/waste_collection_schedule/source/abfallwirtschaft_vechta_de.py

@@ -28,7 +28,6 @@
     "Glass": "mdi:bottle-soda",
     "Bioabfall": "mdi:leaf",
     "Altpapier": "mdi:package-variant",
-    "Altpapier": "mdi:package-variant",
     "Altpapier Siemer": "mdi:package-variant",
     "Altpapier Pamo": "mdi:package-variant",
     "Gelbe Tonne": "mdi:recycle",

custom_components/waste_collection_schedule/waste_collection_schedule/source/alw_wf_de.py

@@ -1,5 +1,6 @@
 import datetime
 import json
+import urllib3
 
 import pytz
 import requests
@@ -9,9 +10,9 @@
 # Using verify=False works, but is not ideal. The following links may provide a better way of dealing with this:
 # https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings
 # https://urllib3.readthedocs.io/en/1.26.x/user-guide.html#ssl
-# These two lines areused to suppress the InsecureRequestWarning when using verify=False
-import urllib3
-urllib3.disable_warnings()
+# This line suppresses the InsecureRequestWarning when using verify=False
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
 
 TITLE = "Abfallwirtschaft Landkreis Wolfenbüttel"
 DESCRIPTION = "Source for ALW Wolfenbüttel."

custom_components/waste_collection_schedule/waste_collection_schedule/source/app_abfallplus_de.py

@@ -7,6 +7,11 @@
 DESCRIPTION = "Source for Apps by Abfall+."
 URL = "https://www.abfallplus.de/"
 TEST_CASES = {
+    "de.k4systems.abfallappnf Ahrenviöl alle Straßen": {
+        "app_id": "de.k4systems.abfallappnf",
+        "city": "Ahrenviöl",
+        "strasse": "Alle Straßen",
+    },
     "de.albagroup.app Braunschweig Hauptstraße 7A  ": {
         "app_id": "de.albagroup.app",
         "city": "Braunschweig",

custom_components/waste_collection_schedule/waste_collection_schedule/source/ashford_gov_uk.py

@@ -108,11 +108,17 @@ def fetch(self):
             date_soup = bin_text.find(
                 "span", id=re.compile(r"CollectionDayLookup2_Label_\w*_Date")
             )
-            if not date_soup or " " not in date_soup.text.strip():
+            if not date_soup or (
+                " " not in date_soup.text.strip()
+                and date_soup.text.strip().lower() != "today"
+            ):
                 continue
             date_str: str = date_soup.text.strip()
             try:
-                date = datetime.strptime(date_str.split(" ")[1], "%d/%m/%Y").date()
+                if date_soup.text.strip().lower() == "today":
+                    date = datetime.now().date()
+                else:
+                    date = datetime.strptime(date_str.split(" ")[1], "%d/%m/%Y").date()
 
             except ValueError:
                 continue

custom_components/waste_collection_schedule/waste_collection_schedule/source/awb_es_de.py

@@ -44,7 +44,7 @@ def fetch(self):
                 ics_urls.append(href)
 
         if not ics_urls:
-            raise Exception(f"ics url not found")
+            raise Exception("ics url not found")
 
         entries = []
         for ics_url in ics_urls: