Merge branch 'pythonfluente:main' into add-gh-actions

capitulos/code/20-executors/demo_executor_map.py

@@ -0,0 +1,31 @@
+"""
+Experiment with ``ThreadPoolExecutor.map``
+"""
+# tag::EXECUTOR_MAP[]
+from time import sleep, strftime
+from concurrent import futures
+
+def display(*args):  # <1>
+    print(strftime('[%H:%M:%S]'), end=' ')
+    print(*args)
+
+def loiter(n):  # <2>
+    msg = '{}loiter({}): doing nothing for {}s...'
+    display(msg.format('\t'*n, n, n))
+    sleep(n)
+    msg = '{}loiter({}): done.'
+    display(msg.format('\t'*n, n))
+    return n * 10  # <3>
+
+def main():
+    display('Script starting.')
+    executor = futures.ThreadPoolExecutor(max_workers=3)  # <4>
+    results = executor.map(loiter, range(5))  # <5>
+    display('results:', results)  # <6>
+    display('Waiting for individual results:')
+    for i, result in enumerate(results):  # <7>
+        display(f'result {i}: {result}')
+
+if __name__ == '__main__':
+    main()
+# end::EXECUTOR_MAP[]

capitulos/code/20-executors/getflags/.gitignore

@@ -0,0 +1,2 @@
+flags/
+downloaded/

capitulos/code/20-executors/getflags/README.adoc

@@ -0,0 +1,122 @@
+= Experimenting with the `flags2*` examples
+
+The `flags2*` examples enhance the `flags*` examples with error handling and reporting.
+Therefore, we need a server that generates errors and delays to experiment with them.
+
+The main reason for these instructions is to document how to configure one such server
+in your machine, and how to tell the `flags2*` clients to access it.
+The other reason is to alert of an installation step that MacOS users sometimes overlook.
+
+Contents:
+
+* <<server_setup>>
+* <<client_setup>>
+* <<macos_certificates>>
+
+[[server_setup]]
+== Setting up test servers
+
+If you don't already have a local HTTP server for testing,
+here are the steps to experiment with the `flags2*` examples
+using just the Python ≥ 3.9 distribution:
+
+. Clone or download the https://github.com/fluentpython/example-code-2e[_Fluent Python 2e_ code repository] (this repo!).
+. Open your shell and go to the _20-futures/getflags/_ directory of your local copy of the repository (this directory!)
+. Unzip the _flags.zip_ file, creating a _flags_ directory at _20-futures/getflags/flags/_.
+. Open a second shell, go to the _20-futures/getflags/_ directory and run `python3 -m http.server`. This will start a `ThreadingHTTPServer` listening to port 8000, serving the local files. If you open the URL http://localhost:8000/flags/[http://localhost:8000/flags/] with your browser, you'll see a long list of directories named with two-letter country codes from `ad/` to `zw/`.
+. Now you can go back to the first shell and run the _flags2*.py_ examples with the default `--server LOCAL` option.
+. To test with the `--server DELAY` option, go to _20-futures/getflags/_ and run `python3 slow_server.py`. This binds to port 8001 by default. It will add a random delay of .5s to 5s before each response.
+. To test with the `--server ERROR` option, go to _20-futures/getflags/_ and run `python3 slow_server.py 8002 --error-rate .25`.
+Each request will have a 25% probability of getting a
+https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/418[418 I'm a teapot] response,
+and all responses will be delayed .5s.
+
+I wrote _slow_server.py_ reusing code from Python's
+https://github.com/python/cpython/blob/917eca700aa341f8544ace43b75d41b477e98b72/Lib/http/server.py[`http.server`] standard library module,
+which "is not recommended for production"—according to the
+https://docs.python.org/3/library/http.server.html[documentation].
+
+[NOTE]
+====
+This is a simple testing environment that does not require any external libraries or
+tools—apart from the libraries used in the `flags2*` scripts themselves, as discussed in the book.
+
+For a more robust testing environment, I recommend configuring
+https://www.nginx.com/[NGINX] and
+https://github.com/shopify/toxiproxy[Toxiproxy] with equivalent parameters.
+====
+
+[[client_setup]]
+== Running a `flags2*` script
+
+The `flags2*` examples provide a command-line interface.
+All three scripts accept the same options,
+and you can see them by running any of the scripts with the `-h` option:
+
+[[flags2_help_demo]]
+.Help screen for the scripts in the flags2 series
+====
+[source, text]
+----
+$ python3 flags2_threadpool.py -h
+usage: flags2_threadpool.py [-h] [-a] [-e] [-l N] [-m CONCURRENT] [-s LABEL]
+                            [-v]
+                            [CC [CC ...]]
+
+Download flags for country codes. Default: top 20 countries by population.
+
+positional arguments:
+  CC                    country code or 1st letter (eg. B for BA...BZ)
+
+optional arguments:
+  -h, --help            show this help message and exit
+  -a, --all             get all available flags (AD to ZW)
+  -e, --every           get flags for every possible code (AA...ZZ)
+  -l N, --limit N       limit to N first codes
+  -m CONCURRENT, --max_req CONCURRENT
+                        maximum concurrent requests (default=30)
+  -s LABEL, --server LABEL
+                        Server to hit; one of DELAY, ERROR, LOCAL, REMOTE
+                        (default=LOCAL)
+  -v, --verbose         output detailed progress info
+
+----
+====
+
+All arguments are optional. The most important arguments are discussed next.
+
+One option you can't ignore is `-s/--server`: it lets you choose which HTTP server and base URL will be used in the test.
+You can pass one of four labels to determine where the script will look for the flags (the labels are case-insensitive):
+
+`LOCAL`:: Use `http://localhost:8000/flags`; this is the default.
+You should configure a local HTTP server to answer at port 8000. See <<server_setup>> for instructions.
+Feel free to hit this as hard as you can. It's your machine!
+
+`REMOTE`:: Use `http://fluentpython.com/data/flags`; that is a public website owned by me, hosted on a shared server.
+Please do not hit it with too many concurrent requests.
+The `fluentpython.com` domain is handled by the http://www.cloudflare.com/[Cloudflare] CDN (Content Delivery Network)
+so you may notice that the first downloads are slower, but they get faster when the CDN cache warms
+up.footnote:[Before configuring Cloudflare, I got HTTP 503 errors--Service Temporarily Unavailable--when
+testing the scripts with a few dozen concurrent requests on my inexpensive shared host account. Now those errors are gone.]
+
+`DELAY`:: Use `http://localhost:8001/flags`; a server delaying HTTP responses should be listening to port 8001.
+I wrote _slow_server.py_ to make it easier to experiment. See <<server_setup>> for instructions.
+
+`ERROR`:: Use `http://localhost:8002/flags`; a server introducing HTTP errors and delaying responses should be installed at port 8002.
+Running _slow_server.py_ is an easy way to do it. See <<server_setup>>.
+
+[[macos_certificates]]
+== Install SSL Certificates (for MacOS)
+
+On Macos, depending on how you installed Python you may need to manually run a command
+after Python's installer finishes, to install the SSL certificates Python uses to make HTTPS connections.
+
+Using the Finder, open the `Python 3.X` folder inside `/Applications` folder
+and double-click "Install Certificates" or "Install Certificates.command".
+
+Using the terminal, you can type for example:
+
+[source, text]
+----
+$ open /Applications/Python 3.10/"Install Certificates.command"
+----

capitulos/code/20-executors/getflags/country_codes.txt

@@ -0,0 +1,8 @@
+AD AE AF AG AL AM AO AR AT AU AZ BA BB BD BE BF BG BH BI BJ BN BO BR BS BT
+BW BY BZ CA CD CF CG CH CI CL CM CN CO CR CU CV CY CZ DE DJ DK DM DZ EC EE
+EG ER ES ET FI FJ FM FR GA GB GD GE GH GM GN GQ GR GT GW GY HN HR HT HU ID
+IE IL IN IQ IR IS IT JM JO JP KE KG KH KI KM KN KP KR KW KZ LA LB LC LI LK
+LR LS LT LU LV LY MA MC MD ME MG MH MK ML MM MN MR MT MU MV MW MX MY MZ NA
+NE NG NI NL NO NP NR NZ OM PA PE PG PH PK PL PT PW PY QA RO RS RU RW SA SB
+SC SD SE SG SI SK SL SM SN SO SR SS ST SV SY SZ TD TG TH TJ TL TM TN TO TR
+TT TV TW TZ UA UG US UY UZ VA VC VE VN VU WS YE ZA ZM ZW

capitulos/code/20-executors/getflags/flags.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+
+"""Download flags of top 20 countries by population
+
+Sequential version
+
+Sample runs (first with new domain, so no caching ever)::
+
+    $ ./flags.py
+    BD BR CD CN DE EG ET FR ID IN IR JP MX NG PH PK RU TR US VN
+    20 downloads in 26.21s
+    $ ./flags.py
+    BD BR CD CN DE EG ET FR ID IN IR JP MX NG PH PK RU TR US VN
+    20 downloads in 14.57s
+
+
+"""
+
+# tag::FLAGS_PY[]
+import time
+from pathlib import Path
+from typing import Callable
+
+import httpx  # <1>
+
+POP20_CC = ('CN IN US ID BR PK NG BD RU JP '
+            'MX PH VN ET EG DE IR TR CD FR').split()  # <2>
+
+BASE_URL = 'https://www.fluentpython.com/data/flags'  # <3>
+DEST_DIR = Path('downloaded')                         # <4>
+
+def save_flag(img: bytes, filename: str) -> None:     # <5>
+    (DEST_DIR / filename).write_bytes(img)
+
+def get_flag(cc: str) -> bytes:  # <6>
+    url = f'{BASE_URL}/{cc}/{cc}.gif'.lower()
+    resp = httpx.get(url, timeout=6.1,       # <7>
+                     follow_redirects=True)  # <8>
+    resp.raise_for_status()  # <9>
+    return resp.content
+
+def download_many(cc_list: list[str]) -> int:  # <10>
+    for cc in sorted(cc_list):                 # <11>
+        image = get_flag(cc)
+        save_flag(image, f'{cc}.gif')
+        print(cc, end=' ', flush=True)         # <12>
+    return len(cc_list)
+
+def main(downloader: Callable[[list[str]], int]) -> None:  # <13>
+    DEST_DIR.mkdir(exist_ok=True)                          # <14>
+    t0 = time.perf_counter()                               # <15>
+    count = downloader(POP20_CC)
+    elapsed = time.perf_counter() - t0
+    print(f'\n{count} downloads in {elapsed:.2f}s')
+
+if __name__ == '__main__':
+    main(download_many)     # <16>
+# end::FLAGS_PY[]

capitulos/code/20-executors/getflags/flags2_asyncio.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+
+"""Download flags of countries (with error handling).
+
+asyncio async/await version
+
+"""
+# tag::FLAGS2_ASYNCIO_TOP[]
+import asyncio
+from collections import Counter
+from http import HTTPStatus
+from pathlib import Path
+
+import httpx
+import tqdm  # type: ignore
+
+from flags2_common import main, DownloadStatus, save_flag
+
+# low concurrency default to avoid errors from remote site,
+# such as 503 - Service Temporarily Unavailable
+DEFAULT_CONCUR_REQ = 5
+MAX_CONCUR_REQ = 1000
+
+async def get_flag(client: httpx.AsyncClient,  # <1>
+                   base_url: str,
+                   cc: str) -> bytes:
+    url = f'{base_url}/{cc}/{cc}.gif'.lower()
+    resp = await client.get(url, timeout=3.1, follow_redirects=True)   # <2>
+    resp.raise_for_status()
+    return resp.content
+
+async def download_one(client: httpx.AsyncClient,
+                       cc: str,
+                       base_url: str,
+                       semaphore: asyncio.Semaphore,
+                       verbose: bool) -> DownloadStatus:
+    try:
+        async with semaphore:  # <3>
+            image = await get_flag(client, base_url, cc)
+    except httpx.HTTPStatusError as exc:  # <4>
+        res = exc.response
+        if res.status_code == HTTPStatus.NOT_FOUND:
+            status = DownloadStatus.NOT_FOUND
+            msg = f'not found: {res.url}'
+        else:
+            raise
+    else:
+        await asyncio.to_thread(save_flag, image, f'{cc}.gif')  # <5>
+        status = DownloadStatus.OK
+        msg = 'OK'
+    if verbose and msg:
+        print(cc, msg)
+    return status
+# end::FLAGS2_ASYNCIO_TOP[]
+
+# tag::FLAGS2_ASYNCIO_START[]
+async def supervisor(cc_list: list[str],
+                     base_url: str,
+                     verbose: bool,
+                     concur_req: int) -> Counter[DownloadStatus]:  # <1>
+    counter: Counter[DownloadStatus] = Counter()
+    semaphore = asyncio.Semaphore(concur_req)  # <2>
+    async with httpx.AsyncClient() as client:
+        to_do = [download_one(client, cc, base_url, semaphore, verbose)
+                 for cc in sorted(cc_list)]  # <3>
+        to_do_iter = asyncio.as_completed(to_do)  # <4>
+        if not verbose:
+            to_do_iter = tqdm.tqdm(to_do_iter, total=len(cc_list))  # <5>
+        error: httpx.HTTPError | None = None  # <6>
+        for coro in to_do_iter:  # <7>
+            try:
+                status = await coro  # <8>
+            except httpx.HTTPStatusError as exc:
+                error_msg = 'HTTP error {resp.status_code} - {resp.reason_phrase}'
+                error_msg = error_msg.format(resp=exc.response)
+                error = exc  # <9>
+            except httpx.RequestError as exc:
+                error_msg = f'{exc} {type(exc)}'.strip()
+                error = exc  # <10>
+            except KeyboardInterrupt:
+                break
+
+            if error:
+                status = DownloadStatus.ERROR  # <11>
+                if verbose:
+                    url = str(error.request.url)  # <12>
+                    cc = Path(url).stem.upper()   # <13>
+                    print(f'{cc} error: {error_msg}')
+            counter[status] += 1
+
+    return counter
+
+def download_many(cc_list: list[str],
+                  base_url: str,
+                  verbose: bool,
+                  concur_req: int) -> Counter[DownloadStatus]:
+    coro = supervisor(cc_list, base_url, verbose, concur_req)
+    counts = asyncio.run(coro)  # <14>
+
+    return counts
+
+if __name__ == '__main__':
+    main(download_many, DEFAULT_CONCUR_REQ, MAX_CONCUR_REQ)
+# end::FLAGS2_ASYNCIO_START[]