feat: adding manual refresh

[robbrad]

No description provided.

[codecov]

❌ 52 Tests Failed:

Tests completed	Failed	Passed	Skipped
291	52	239	0

View the top 3 failed tests by shortest run time

uk_bin_collection.tests.step_defs.test_validate_council::test_scenario_outline[NorthKestevenDistrictCouncil]

Stack Traces | 0.14s run time

fixturefunc = <function validate_output_step at 0x7faa6999b100>
request = <FixtureRequest for <Function test_scenario_outline[NorthKestevenDistrictCouncil]>>
kwargs = {'context': <test_validate_council.Context object at 0x7faa7e0977d0>}

    def call_fixture_func(
        fixturefunc: _FixtureFunc[FixtureValue], request: FixtureRequest, kwargs
    ) -> FixtureValue:
        if is_generator(fixturefunc):
            fixturefunc = cast(
                Callable[..., Generator[FixtureValue, None, None]], fixturefunc
            )
            generator = fixturefunc(**kwargs)
            try:
                fixture_result = next(generator)
            except StopIteration:
                raise ValueError(f"{request.fixturename} did not yield a value") from None
            finalizer = functools.partial(_teardown_yield_fixture, fixturefunc, generator)
            request.addfinalizer(finalizer)
        else:
            fixturefunc = cast(Callable[..., FixtureValue], fixturefunc)
>           fixture_result = fixturefunc(**kwargs)

../../../..../pypoetry/virtualenvs/uk-bin-collection-EwS6Gn8s-py3.12/lib/python3.12.../site-packages/_pytest/fixtures.py:898: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
.../tests/step_defs/test_validate_council.py:114: in validate_output_step
    assert file_handler.validate_json_schema(
.../step_defs/step_helpers/file_handler.py:40: in validate_json_schema
    validate(instance=json_data, schema=schema)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

instance = {'bins': []}
schema = {'$ref': '#/definitions/BinData', '$schema': 'http://json-schema.org/draft-06/schema#', 'definitions': {'Bin': {'addit...ems': {'$ref': '#/definitions/Bin'}, 'minItems': 1, 'type': 'array'}}, 'required': ['bins'], 'title': 'BinData', ...}}}
cls = <class 'jsonschema.validators.Draft6Validator'>, args = (), kwargs = {}
validator = Draft6Validator(schema={'$ref': '#/definitions/BinData', '$schema': 'http://json-...ft-06/schema#', 'definitions': {'B...nitions/Bin'}, 'minItems': 1, 'type': 'array'}}, 'required': ['bins'], 'title': 'BinData', ...}}}, format_checker=None)
error = <ValidationError: '[] should be non-empty'>

    def validate(instance, schema, cls=None, *args, **kwargs):  # noqa: D417
        """
        Validate an instance under the given schema.
    
            >>> validate([2, 3, 4], {"maxItems": 2})
            Traceback (most recent call last):
                ...
            ValidationError: [2, 3, 4] is too long
    
        :func:`~jsonschema.validators.validate` will first verify that the
        provided schema is itself valid, since not doing so can lead to less
        obvious error messages and fail in less obvious or consistent ways.
    
        If you know you have a valid schema already, especially
        if you intend to validate multiple instances with
        the same schema, you likely would prefer using the
        `jsonschema.protocols.Validator.validate` method directly on a
        specific validator (e.g. ``Draft202012Validator.validate``).
    
    
        Arguments:
    
            instance:
    
                The instance to validate
    
            schema:
    
                The schema to validate with
    
            cls (jsonschema.protocols.Validator):
    
                The class that will be used to validate the instance.
    
        If the ``cls`` argument is not provided, two things will happen
        in accordance with the specification. First, if the schema has a
        :kw:`$schema` keyword containing a known meta-schema [#]_ then the
        proper validator will be used. The specification recommends that
        all schemas contain :kw:`$schema` properties for this reason. If no
        :kw:`$schema` property is found, the default validator class is the
        latest released draft.
    
        Any other provided positional and keyword arguments will be passed
        on when instantiating the ``cls``.
    
        Raises:
    
            `jsonschema.exceptions.ValidationError`:
    
                if the instance is invalid
    
            `jsonschema.exceptions.SchemaError`:
    
                if the schema itself is invalid
    
        .. rubric:: Footnotes
        .. [#] known by a validator registered with
            `jsonschema.validators.validates`
    
        """
        if cls is None:
            cls = validator_for(schema)
    
        cls.check_schema(schema)
        validator = cls(schema, *args, **kwargs)
        error = exceptions.best_match(validator.iter_errors(instance))
        if error is not None:
>           raise error
E           jsonschema.exceptions.ValidationError: [] should be non-empty
E           
E           Failed validating 'minItems' in schema['properties']['bins']:
E               {'type': 'array', 'items': {'$ref': '#/definitions/Bin'}, 'minItems': 1}
E           
E           On instance['bins']:
E               []

../../../..../pypoetry/virtualenvs/uk-bin-collection-EwS6Gn8s-py3.12/lib/python3.12.../site-packages/jsonschema/validators.py:1332: ValidationError

uk_bin_collection.tests.step_defs.test_validate_council::test_scenario_outline[CastlepointDistrictCouncil]

Stack Traces | 0.521s run time

fixturefunc = <function scrape_step at 0x7faa6999b600>
request = <FixtureRequest for <Function test_scenario_outline[CastlepointDistrictCouncil]>>
kwargs = {'context': <test_validate_council.Context object at 0x7faa7e0977d0>, 'headless_mode': 'True', 'local_browser': 'False', 'selenium_url': 'http://localhost:4444'}

    def call_fixture_func(
        fixturefunc: _FixtureFunc[FixtureValue], request: FixtureRequest, kwargs
    ) -> FixtureValue:
        if is_generator(fixturefunc):
            fixturefunc = cast(
                Callable[..., Generator[FixtureValue, None, None]], fixturefunc
            )
            generator = fixturefunc(**kwargs)
            try:
                fixture_result = next(generator)
            except StopIteration:
                raise ValueError(f"{request.fixturename} did not yield a value") from None
            finalizer = functools.partial(_teardown_yield_fixture, fixturefunc, generator)
            request.addfinalizer(finalizer)
        else:
            fixturefunc = cast(Callable[..., FixtureValue], fixturefunc)
>           fixture_result = fixturefunc(**kwargs)

../../../..../pypoetry/virtualenvs/uk-bin-collection-EwS6Gn8s-py3.12/lib/python3.12.../site-packages/_pytest/fixtures.py:898: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
.../tests/step_defs/test_validate_council.py:101: in scrape_step
    context.parse_result = CollectData.run()
uk_bin_collection/uk_bin_collection/collect_data.py:101: in run
    return self.client_code(
uk_bin_collection/uk_bin_collection/collect_data.py:121: in client_code
    return get_bin_data_class.template_method(address_url, **kwargs)
uk_bin_collection/uk_bin_collection/get_bin_data.py:61: in template_method
    bin_data_dict = self.get_and_parse_data(this_url, **kwargs)
uk_bin_collection/uk_bin_collection/get_bin_data.py:84: in get_and_parse_data
    bin_data_dict = self.parse_data("", url=address_url, **kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <CastlepointDistrictCouncil.CouncilClass object at 0x7faa68cb7830>
page = ''
kwargs = {'council_module_str': 'CastlepointDistrictCouncil', 'dev_mode': False, 'headless': True, 'local_browser': False, ...}
uprn = '4525'
post_url = 'https://apps.castlepoint.gov.uk/cpapps/index.cfm?fa=wastecalendar.displayDetails'
post_header_str = 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/... Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.134 Safari/537.36 '
post_headers = {'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,applicati...b3;q=0.9', 'Accept-Encoding': 'gzip, deflate, br', 'Accept-Language': 'en-GB;q=0.8', 'Cache-Control': 'max-age=0', ...}
form_data = {'roadID': '4525'}, post_response = <Response [200]>
soup = <html lang="en" xml:lang="en" xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>Connection denied by Geolocation</ti... administrator for assistance.</p>
</div>
<div class="band">WatchGuard Technologies, Inc.</div>
</div>
</body>
</html>


    def parse_data(self, page: str, **kwargs) -> dict:
        # Disable the SSL warnings that otherwise break everything
        requests.packages.urllib3.disable_warnings()
        try:
            requests.packages.urllib3.contrib.pyopenssl.util.ssl_.DEFAULT_CIPHERS += (
                ":HIGH:!DH:!aNULL"
            )
        except AttributeError:
            pass
    
        # UPRN is street id here
        uprn = kwargs.get("uprn")
        check_uprn(uprn)
    
        post_url = "https://apps.castlepoint.gov.uk/cpapps/index.cfm?fa=wastecalendar.displayDetails"
        post_header_str = (
            "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,"
            "image/apng,"
            "*/*;q=0.8,application/signed-exchange;v=b3;q=0.9|Accept-Encoding: gzip, deflate, "
            "br|Accept-Language: en-GB;q=0.8|Cache-Control: max-age=0|Connection: "
            "keep-alive|Content-Length: "
            "11|Content-Type: application/x-www-form-urlencoded|Host: apps.castlepoint.gov.uk|Origin: "
            "https://apps.castlepoint.gov.uk|Referer: "
            "https://apps.castlepoint.gov.uk/cpapps/index.cfm?fa=wastecalendar|Sec-Fetch-Dest: "
            "document|Sec-Fetch-Mode: navigate|Sec-Fetch-Site: same-origin|Sec-Fetch-User: ?1|Sec-GPC: "
            "1|Upgrade-Insecure-Requests: 1|User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
            "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.134 Safari/537.36 "
        )
    
        post_headers = parse_header(post_header_str)
        form_data = {"roadID": uprn}
        post_response = requests.post(
            post_url, headers=post_headers, data=form_data, verify=False
        )
    
        # Make a BS4 object
        soup = BeautifulSoup(post_response.text, features="html.parser")
        soup.prettify()
    
        data = {"bins": []}
        collection_tuple = []
    
        for i in range(1, 3):
>           calendar = soup.select(
                f"#wasteCalendarContainer > div:nth-child(2) > div:nth-child({i}) > div"
            )[0]
E           IndexError: list index out of range

.../uk_bin_collection/councils/CastlepointDistrictCouncil.py:57: IndexError

uk_bin_collection.tests.step_defs.test_validate_council::test_scenario_outline[CopelandBoroughCouncil]

Stack Traces | 0.781s run time

fixturefunc = <function scrape_step at 0x7fbd1090ae80>
request = <FixtureRequest for <Function test_scenario_outline[CopelandBoroughCouncil]>>
kwargs = {'context': <test_validate_council.Context object at 0x7fbd2458c350>, 'headless_mode': 'True', 'local_browser': 'False', 'selenium_url': 'http://localhost:4444'}

    def call_fixture_func(
        fixturefunc: _FixtureFunc[FixtureValue], request: FixtureRequest, kwargs
    ) -> FixtureValue:
        if is_generator(fixturefunc):
            fixturefunc = cast(
                Callable[..., Generator[FixtureValue, None, None]], fixturefunc
            )
            generator = fixturefunc(**kwargs)
            try:
                fixture_result = next(generator)
            except StopIteration:
                raise ValueError(f"{request.fixturename} did not yield a value") from None
            finalizer = functools.partial(_teardown_yield_fixture, fixturefunc, generator)
            request.addfinalizer(finalizer)
        else:
            fixturefunc = cast(Callable[..., FixtureValue], fixturefunc)
>           fixture_result = fixturefunc(**kwargs)

../../../..../pypoetry/virtualenvs/uk-bin-collection-EwS6Gn8s-py3.12/lib/python3.12.../site-packages/_pytest/fixtures.py:898: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
.../tests/step_defs/test_validate_council.py:101: in scrape_step
    context.parse_result = CollectData.run()
uk_bin_collection/uk_bin_collection/collect_data.py:101: in run
    return self.client_code(
uk_bin_collection/uk_bin_collection/collect_data.py:121: in client_code
    return get_bin_data_class.template_method(address_url, **kwargs)
uk_bin_collection/uk_bin_collection/get_bin_data.py:61: in template_method
    bin_data_dict = self.get_and_parse_data(this_url, **kwargs)
uk_bin_collection/uk_bin_collection/get_bin_data.py:82: in get_and_parse_data
    bin_data_dict = self.parse_data(page, url=address_url, **kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <CopelandBoroughCouncil.CouncilClass object at 0x7fbd0ff63cb0>
page = <Response [200]>
kwargs = {'council_module_str': 'CopelandBoroughCouncil', 'dev_mode': False, 'headless': True, 'local_browser': False, ...}
uprn = '100110734613', council = 'CPL'
headers = {'Content-Type': 'text/xml; charset=UTF-8', 'Referer': 'https://collections-copeland.azurewebsites.net/calendar.html',...nt': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36'}
post_data = '<?xml version="1.0" encoding="utf-8"?><soap:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd=...council>CPL</council><UPRN>100110734613</UPRN><from>Chtml</from></getRoundCalendarForUPRN></soap:Body></soap:Envelope>'
response = <Response [403]>

    def parse_data(self, page: str, **kwargs) -> dict:
        uprn = kwargs.get("uprn")
        check_uprn(uprn)
        council = "CPL"
    
        # Make SOAP request
        headers = {
            "Content-Type": "text/xml; charset=UTF-8",
            "Referer": "https://collections-copeland.azurewebsites.net/calendar.html",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36",
        }
        requests.packages.urllib3.disable_warnings()
        post_data = (
            '<?xml version="1.0" encoding="utf-8"?>'
            '<soap:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/">'
            '<soap:Body><getRoundCalendarForUPRN xmlns="http://webaspx-collections.azurewebsites.net/">'
            "<council>" + council + "</council><UPRN>" + uprn + "</UPRN>"
            "<from>Chtml</from></getRoundCalendarForUPRN></soap:Body></soap:Envelope>"
        )
        response = requests.post(
            "https://collections-copeland.azurewebsites.net/WSCollExternal.asmx",
            headers=headers,
            data=post_data,
        )
    
        if response.status_code != 200:
>           raise ValueError("No bin data found for provided UPRN.")
E           ValueError: No bin data found for provided UPRN.

.../uk_bin_collection/councils/CopelandBoroughCouncil.py:42: ValueError

To view more test analytics, go to the Test Analytics Dashboard
📢 Thoughts on this report? Let us know!