From 91d85354e6da31cb478bc6019d20acc727d94650 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Sj=C3=B6gren?= Date: Thu, 10 Oct 2024 19:06:59 +0000 Subject: [PATCH 1/3] update readme MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Thomas Sjögren --- README.md | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 324dadf..28e2817 100644 --- a/README.md +++ b/README.md @@ -68,15 +68,16 @@ sites: - "stackoverflow.com" - "xkcd.com" user_agents: - - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36" - - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36" - - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.2 Safari/605.1.15" - - "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 YaBrowser/19.6.2.599 Yowser/2.5 Safari/537.36" - - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36 Edg/108.0.1462.54" - - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:108.0) Gecko/20100101 Firefox/108.0" - - "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko" - - "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36" - - "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:66.0) Gecko/20100101 Firefox/66.0" + - "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Mobile Safari/537.36" + - "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Mobile Safari/537.36" + - "Mozilla/5.0 (Linux; Android 14; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.6668.71 Mobile Safari/537.36" + - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36" + - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Safari/605.1.15" + - "Mozilla/5.0 (Macintosh; Intel Mac OS X 11.6; rv:92.0) Gecko/20100101 Firefox/92.0" + - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36" + - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36" + - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36" + - "Mozilla/5.0 (iPhone; CPU iPhone OS 17_6_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Mobile/15E148 Safari/604.1" ``` The above configuration will make `tymely` randomly choose a site and user agent @@ -86,11 +87,11 @@ from the lists presented in the file, and verbosely return any information. $ ./tymely.py -c tymely.yaml -t Verbose mode enabled Configuration file: tymely.yaml -{'verbose': 1, 'sites': ['cloudflare.com', 'duckduckgo.com', 'github.com', 'proton.me', 'stackoverflow.com', 'xkcd.com'], 'user_agents': ['Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.2 Safari/605.1.15', 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 YaBrowser/19.6.2.599 Yowser/2.5 Safari/537.36', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36 Edg/108.0.1462.54', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:108.0) Gecko/20100101 Firefox/108.0', 'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko', 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36', 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:66.0) Gecko/20100101 Firefox/66.0']} -URL: proton.me -User agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:108.0) Gecko/20100101 Firefox/108.0 -Response headers: {'date': 'Wed, 28 Dec 2022 15:38:52 GMT', 'set-cookie': 'Session-Id=Y6xjDCybqTYU-tqqBvhDmwAAABo; Domain=proton.me; Path=/; HttpOnly; Secure; Max-Age=7776000, Tag=default; Path=/; Secure; Max-Age=7776000', 'last-modified': 'Wed, 28 Dec 2022 10:24:00 GMT', 'accept-ranges': 'bytes', 'cache-control': 'max-age=0, no-cache, no-store, must-revalidate', 'expires': 'Wed, 11 Jan 1984 05:00:00 GMT', 'vary': 'Accept-Encoding', 'content-encoding': 'gzip', 'pragma': 'no-cache', 'content-length': '21937', 'content-type': 'text/html; charset=utf-8', 'content-security-policy-report-only': "default-src 'self'; media-src https://static.zdassets.com; connect-src 'self' wss: https://protonmail.zendesk.com https://ekr.zdassets.com blob: https://account.proton.me https://reports.proton.me https://*.algolia.net https://*.algolianet.com; script-src 'self' blob: 'unsafe-eval' 'unsafe-inline' https://static.zdassets.com; style-src 'self' 'unsafe-inline'; img-src 'self' data: blob: https:; object-src 'self' data: blob:; frame-src 'self' data: blob: https://www.youtube-nocookie.com; child-src 'self' data: blob:; report-uri https://reports.proton.me/reports/csp; frame-ancestors 'self';", 'strict-transport-security': 'max-age=31536000; includeSubDomains; preload', 'expect-ct': 'max-age=2592000, enforce, report-uri="https://reports.protonmail.com/reports/tls"', 'public-key-pins-report-only': 'pin-sha256="CT56BhOTmj5ZIPgb/xD5mH8rY3BLo/MlhP7oPyJUEDo="; report-uri="https://reports.protonmail.com/reports/tls"', 'x-frame-options': 'sameorigin', 'x-content-type-options': 'nosniff', 'x-xss-protection': '0', 'referrer-policy': 'strict-origin-when-cross-origin', 'x-permitted-cross-domain-policies': 'none', 'onion-location': 'https://protonmailrmez3lotccipshtkleegetolb73fuirgj7r4o4vfu7ozyd.onion/'} -Wed, 28 Dec 2022 15:38:52 GMT from proton.me returned but not set +{'verbose': 1, 'sites': ['cloudflare.com', 'duckduckgo.com', 'github.com', 'proton.me', 'stackoverflow.com', 'xkcd.com'], 'user_agents': ['Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Mobile Safari/537.36', 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Mobile Safari/537.36', 'Mozilla/5.0 (Linux; Android 14; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.6668.71 Mobile Safari/537.36', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Safari/605.1.15', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11.6; rv:92.0) Gecko/20100101 Firefox/92.0', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36', 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_6_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Mobile/15E148 Safari/604.1']} +URL: duckduckgo.com +User agent: Mozilla/5.0 (iPhone; CPU iPhone OS 17_6_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Mobile/15E148 Safari/604.1 +Response headers: {'Server': 'nginx', 'Date': 'Thu, 10 Oct 2024 19:05:43 GMT', 'Content-Type': 'text/html; charset=UTF-8', 'Content-Length': '11191', 'Connection': 'keep-alive', 'Vary': 'Accept-Encoding', 'ETag': '"6706e8f7-2bb7"', 'Content-Encoding': 'gzip', 'Strict-Transport-Security': 'max-age=31536000', 'Permissions-Policy': 'interest-cohort=()', 'Content-Security-Policy': "default-src 'none' ; connect-src https://duckduckgo.com https://*.duckduckgo.com https://duckduckgogg42xjoc72x3sjasowoarfbgcmvfimaftt6twagswzczad.onion/ https://spreadprivacy.com ; manifest-src https://duckduckgo.com https://*.duckduckgo.com https://duckduckgogg42xjoc72x3sjasowoarfbgcmvfimaftt6twagswzczad.onion/ https://spreadprivacy.com ; media-src https://duckduckgo.com https://*.duckduckgo.com https://duckduckgogg42xjoc72x3sjasowoarfbgcmvfimaftt6twagswzczad.onion/ https://spreadprivacy.com ; script-src blob: https://duckduckgo.com https://*.duckduckgo.com https://duckduckgogg42xjoc72x3sjasowoarfbgcmvfimaftt6twagswzczad.onion/ https://spreadprivacy.com 'unsafe-inline' 'unsafe-eval' ; font-src data: https://duckduckgo.com https://*.duckduckgo.com https://duckduckgogg42xjoc72x3sjasowoarfbgcmvfimaftt6twagswzczad.onion/ https://spreadprivacy.com ; img-src data: https://duckduckgo.com https://*.duckduckgo.com https://duckduckgogg42xjoc72x3sjasowoarfbgcmvfimaftt6twagswzczad.onion/ https://spreadprivacy.com ; style-src https://duckduckgo.com https://*.duckduckgo.com https://duckduckgogg42xjoc72x3sjasowoarfbgcmvfimaftt6twagswzczad.onion/ https://spreadprivacy.com 'unsafe-inline' ; object-src 'none' ; worker-src blob: ; child-src blob: https://duckduckgo.com https://*.duckduckgo.com https://duckduckgogg42xjoc72x3sjasowoarfbgcmvfimaftt6twagswzczad.onion/ https://spreadprivacy.com ; frame-src blob: https://duckduckgo.com https://*.duckduckgo.com https://duckduckgogg42xjoc72x3sjasowoarfbgcmvfimaftt6twagswzczad.onion/ https://spreadprivacy.com ; form-action https://duckduckgo.com https://*.duckduckgo.com https://duckduckgogg42xjoc72x3sjasowoarfbgcmvfimaftt6twagswzczad.onion/ https://spreadprivacy.com ; frame-ancestors 'self' https://html.duckduckgo.com; base-uri 'self' ; block-all-mixed-content ;", 'X-Frame-Options': 'SAMEORIGIN', 'X-XSS-Protection': '1;mode=block', 'X-Content-Type-Options': 'nosniff', 'Referrer-Policy': 'origin', 'Expect-CT': 'max-age=0', 'Expires': 'Thu, 10 Oct 2024 19:05:42 GMT', 'Cache-Control': 'no-cache'} +Thu, 10 Oct 2024 19:05:43 GMT from duckduckgo.com returned but not set ``` ### Related documentation From 7f7f539141edf9317da3d134787f595b2ee098c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Sj=C3=B6gren?= Date: Thu, 10 Oct 2024 19:07:19 +0000 Subject: [PATCH 2/3] python linting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Thomas Sjögren --- tymely/tymely.py | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/tymely/tymely.py b/tymely/tymely.py index 11cb611..44529e4 100755 --- a/tymely/tymely.py +++ b/tymely/tymely.py @@ -1,13 +1,14 @@ #!/usr/bin/env python3 +# ruff: noqa: T201 """tymely fetches HTTP-date over HTTPS and sets the system time.""" import argparse import datetime -import os -import random +import secrets import shutil import subprocess # nosec B404,S404 import sys +from pathlib import Path import certifi import requests @@ -16,10 +17,8 @@ __version__ = "0.1.0" -def arguments(): - """Parse command line arguments using argparse module and return the parsed - arguments. - """ +def arguments() -> any: + """Parse command line arguments and return the arguments.""" parser = argparse.ArgumentParser( description="tymely fetches HTTP-date over HTTPS and sets the system time", epilog="version: " + __version__, @@ -40,17 +39,18 @@ def arguments(): return parser.parse_args() -def config(args): +def config(args: any) -> any: """Read and parse the configuration file specified in the command line arguments. + Return the configuration as a dictionary. """ try: if args.config: - if not os.path.isfile(args.config): + if not Path(args.config).is_file(): print(args.config, "can't be found.") sys.exit(1) else: - with open(args.config, encoding="utf-8") as args_file: + with Path.open(args.config, encoding="utf-8") as args_file: conf = yaml.safe_load(args_file) else: conf = {"verbose": 0} @@ -68,22 +68,23 @@ def config(args): return conf -def get_site_and_agent(conf): +def get_site_and_agent(conf: str) -> str: """Choose a site and user agent string from the configuration dictionary. + Return the site URL and user agent string. """ user_agent = None if conf.get("sites"): url = conf.get("sites", False) - url = random.SystemRandom().choice(url) + url = secrets.choice(url) else: url = "duckduckgo.com" tymely_agent = "tymely/" + __version__ user_agent = conf.get("user_agents", tymely_agent) if user_agent != tymely_agent: - user_agent = random.SystemRandom().choice(conf["user_agents"]) + user_agent = secrets.choice(conf["user_agents"]) if conf.get("verbose", 0): print("URL:", url, file=sys.stdout) @@ -92,10 +93,8 @@ def get_site_and_agent(conf): return url, user_agent -def main(): - """Main function that fetches the current date over HTTPS, sets the system time - if not in test mode, and prints the date if in test mode. - """ +def main() -> None: + """Fetch the current date over HTTPS, sets the system time if not in test mode.""" args = arguments() conf = config(args) url, user_agent = get_site_and_agent(conf) From 3d8266867d31da480d4ce485614fa2389e5fc921 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Sj=C3=B6gren?= Date: Thu, 10 Oct 2024 19:07:30 +0000 Subject: [PATCH 3/3] update user agents MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Thomas Sjögren --- tymely/tymely.yaml | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/tymely/tymely.yaml b/tymely/tymely.yaml index ff162c9..d7a8f0a 100644 --- a/tymely/tymely.yaml +++ b/tymely/tymely.yaml @@ -8,15 +8,13 @@ sites: - "stackoverflow.com" - "xkcd.com" user_agents: - - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36" - - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36" - - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.2 Safari/605.1.15" - - "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 YaBrowser/19.6.2.599 Yowser/2.5 Safari/537.36" - - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36 Edg/108.0.1462.54" - - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36 Edg/112.0.1722.39" - - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36" - - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:108.0) Gecko/20100101 Firefox/108.0" - - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/112.0" - - "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko" - - "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36" - - "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:66.0) Gecko/20100101 Firefox/66.0" + - "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Mobile Safari/537.36" + - "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Mobile Safari/537.36" + - "Mozilla/5.0 (Linux; Android 14; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.6668.71 Mobile Safari/537.36" + - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36" + - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Safari/605.1.15" + - "Mozilla/5.0 (Macintosh; Intel Mac OS X 11.6; rv:92.0) Gecko/20100101 Firefox/92.0" + - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36" + - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36" + - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36" + - "Mozilla/5.0 (iPhone; CPU iPhone OS 17_6_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Mobile/15E148 Safari/604.1"