Merge pull request #292 from fvaleye/fix/ci-dependencies-issues

fix(dependencies): fix dependencies CI issues

.github/workflows/build.yml

@@ -24,7 +24,9 @@ jobs:
       - name: Check Python
         run: |
           make init
-          make style
+          pip install pre-commit
+          pre-commit install
+          make format
 
       - name: Check Data
         run:  make check-data

.pre-commit-config.yaml

@@ -11,15 +11,9 @@ repos:
     hooks:
       - id: poetry-check
       - id: poetry-lock
-  - repo: https://github.com/timothycrosley/isort
-    rev: '5.12.0'
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.0.290
     hooks:
-      - id: isort
-        args: [ --check-only, --diff ]
-        types: [ python ]
-  - repo: https://github.com/psf/black
-    rev: '24.1.1'
-    hooks:
-      - id: black
-        args: [ --check, --diff ]
-        types: [ python ]
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix]
+      - id: ruff-format

Makefile

@@ -8,15 +8,7 @@ init: ## Init the requirements
 .PHONY: format
 format: ## Format the code
 	$(info --- 🐍 Check Python format ---)
-	@poetry run black .
-	@poetry run isort .
-
-.PHONY: style
-style: ## Run style
-	$(info --- 🐍 Style Python ---)
-	@poetry run isort --diff --check-only .
-	@poetry run black --check .
-	@poetry run mypy
+	pre-commit run -a
 
 .PHONY: security
 security: ## Run security checks

docs/source/_ext/edit_on_github.py

@@ -25,7 +25,7 @@ def html_page_context(app, pagename, templatename, context, doctree):
         return
 
     if not app.config.edit_on_github_project:
-        warnings.warn("edit_on_github_project not specified")
+        warnings.warn("edit_on_github_project not specified", stacklevel=1)
         return
 
     path = os.path.relpath(doctree.get("source"), app.builder.srcdir)
@@ -39,9 +39,7 @@ def html_page_context(app, pagename, templatename, context, doctree):
     context["display_github"] = True
     context["github_user"] = app.config.edit_on_github_project.split("/")[0]
     context["github_repo"] = app.config.edit_on_github_project.split("/")[1]
-    context["github_version"] = (
-        f"{app.config.edit_on_github_branch}/{app.config.page_source_prefix}/"
-    )
+    context["github_version"] = f"{app.config.edit_on_github_branch}/{app.config.page_source_prefix}/"
 
 
 def setup(app):

pyproject.toml

@@ -16,26 +16,25 @@ include = ["tracarbon/py.typed"]
 [tool.poetry.dependencies]
 python = "^3.8"
 loguru = ">=0.6,<0.8"
-aiohttp = "^3.8.4"
+aiohttp = "^3.9.3"
 aiocache = "^0.12.1"
-aiofiles = "^23.1.0"
-psutil = "^5.9.4"
-ujson = "^5.8.0"
+aiofiles = "^23.2.0"
+psutil = "^5.9.8"
+ujson = "^5.9.0"
 msgpack = "^1.0.4"
 pydantic = ">=1.10.7,<3.0.0"
 typer = ">=0.7,<0.10"
 ec2-metadata = "^2.13.0"
 python-dotenv = ">=0.21,<1.1"
-asyncer = ">=0.0.2,<0.0.5"
+asyncer = "^0.0.5"
 datadog = {version = ">=0.44,<0.49", optional = true}
 prometheus-client = {version = ">=0.16,<0.21", optional = true}
 kubernetes = {version = ">=26.1,<30.0", optional = true}
 
 [tool.poetry.dev-dependencies]
 mypy = "^1.8"
-black = "^24.1.1"
-isort = "^5.13.2"
-pytest = "^7.4.4"
+ruff = "^0.3.0"
+pytest = "^8.0.2"
 pytest-mock = "^3.12.0"
 pytest-asyncio = "^0.23.5"
 pytest-cov = "^4.1.0"
@@ -86,16 +85,20 @@ strict_equality = true
 strict_optional = false
 
 [tool.bandit]
-skips = ["B404", "B607", "B602"]
+skips = ["B404", "B607", "B602", "B603"]
 exclude_dirs = ["tests", "scripts"]
 
-[tool.isort]
-profile = "black"
-src_paths = ["tracarbon", "tests"]
+[tool.ruff]
+fix = true
+select = ["I", "S", "B"]
+line-length = 120
+ignore = ["B023", "S603"]
 
-[tool.black]
-include = '\.pyi?$'
-exclude = "venv"
+[tool.ruff.per-file-ignores]  # Don’t apply ruff rules to our tests
+"**/tests/*" = ["S"]
+
+[tool.ruff.isort]
+force-single-line = true
 
 [tool.pytest.ini_options]
 addopts = "--cov=tracarbon -v --asyncio-mode=auto"

scripts/check_data.py

@@ -1,11 +1,20 @@
 import urllib.request
+from urllib.parse import urlparse
+
+
+def is_valid_url(url: str) -> bool:
+    parsed_url = urlparse(url)
+    return parsed_url.scheme in ["http", "https"]
 
 
 def check_content_length(url: str, expected_content_length: str) -> bool:
-    site = urllib.request.urlopen(url)
-    assert (
-        site.getheader("Content-Length") == expected_content_length
-    ), f"This url content changed {url}"
+    if not is_valid_url(url):
+        raise ValueError(f"Invalid or unsafe URL scheme for URL: {url}")
+
+    site = urllib.request.urlopen(url)  # noqa: S310
+    if site.getheader("Content-Length") != expected_content_length:
+        raise ValueError(f"This url content changed {url}")
+    return True
 
 
 if __name__ == "__main__":
@@ -24,6 +33,4 @@ def check_content_length(url: str, expected_content_length: str) -> bool:
         },
     ]
     for url in urls:
-        check_content_length(
-            url=url["url"], expected_content_length=url["content_length"]
-        )
+        check_content_length(url=url["url"], expected_content_length=url["content_length"])

tests/carbon_emissions/test_carbon_emissions.py

@@ -1,14 +1,12 @@
 import pytest
 
-from tracarbon import (
-    CarbonEmission,
-    CarbonUsage,
-    CarbonUsageUnit,
-    EnergyUsage,
-    LinuxEnergyConsumption,
-    MacEnergyConsumption,
-    UsageType,
-)
+from tracarbon import CarbonEmission
+from tracarbon import CarbonUsage
+from tracarbon import CarbonUsageUnit
+from tracarbon import EnergyUsage
+from tracarbon import LinuxEnergyConsumption
+from tracarbon import MacEnergyConsumption
+from tracarbon import UsageType
 from tracarbon.locations import Country
 
 
@@ -32,9 +30,7 @@ async def test_carbon_emission_should_run_to_convert_watt_hours_to_co2g_on_mac(m
     )
     name_alpha_iso_2 = "fr"
     mocker.patch.object(Country, "get_latest_co2g_kwh", return_value=co2g_per_kwh)
-    mocker.patch.object(
-        MacEnergyConsumption, "get_energy_usage", return_value=energy_usage
-    )
+    mocker.patch.object(MacEnergyConsumption, "get_energy_usage", return_value=energy_usage)
     carbon_emission = CarbonEmission(
         location=Country(name=name_alpha_iso_2, co2g_kwh=co2g_per_kwh),
     )
@@ -56,9 +52,7 @@ async def test_carbon_emission_should_run_to_convert_watt_hours_to_co2g_on_linux
     name_alpha_iso_2 = "fr"
     energy_usage = EnergyUsage(host_energy_usage=60.0)
     mocker.patch.object(Country, "get_latest_co2g_kwh", return_value=co2g_per_kwh)
-    mocker.patch.object(
-        LinuxEnergyConsumption, "get_energy_usage", return_value=energy_usage
-    )
+    mocker.patch.object(LinuxEnergyConsumption, "get_energy_usage", return_value=energy_usage)
     carbon_emission = CarbonEmission(
         location=Country(name=name_alpha_iso_2, co2g_kwh=co2g_per_kwh),
     )
@@ -83,26 +77,14 @@ def test_carbon_usage_with_type_and_conversion():
 
     assert carbon_usage.get_carbon_usage_on_type(UsageType.HOST) == host_carbon_usage
     assert carbon_usage.get_carbon_usage_on_type(UsageType.CPU) == cpu_carbon_usage
-    assert (
-        carbon_usage.get_carbon_usage_on_type(UsageType.MEMORY) == memory_carbon_usage
-    )
+    assert carbon_usage.get_carbon_usage_on_type(UsageType.MEMORY) == memory_carbon_usage
     assert carbon_usage.get_carbon_usage_on_type(UsageType.GPU) == gpu_carbon_usage
     assert carbon_usage.unit == CarbonUsageUnit.CO2_G
 
     carbon_usage.convert_unit(CarbonUsageUnit.CO2_MG)
 
-    assert (
-        carbon_usage.get_carbon_usage_on_type(UsageType.HOST)
-        == host_carbon_usage * 1000
-    )
-    assert (
-        carbon_usage.get_carbon_usage_on_type(UsageType.CPU) == cpu_carbon_usage * 1000
-    )
-    assert (
-        carbon_usage.get_carbon_usage_on_type(UsageType.MEMORY)
-        == memory_carbon_usage * 1000
-    )
-    assert (
-        carbon_usage.get_carbon_usage_on_type(UsageType.GPU) == gpu_carbon_usage * 1000
-    )
+    assert carbon_usage.get_carbon_usage_on_type(UsageType.HOST) == host_carbon_usage * 1000
+    assert carbon_usage.get_carbon_usage_on_type(UsageType.CPU) == cpu_carbon_usage * 1000
+    assert carbon_usage.get_carbon_usage_on_type(UsageType.MEMORY) == memory_carbon_usage * 1000
+    assert carbon_usage.get_carbon_usage_on_type(UsageType.GPU) == gpu_carbon_usage * 1000
     assert carbon_usage.unit == CarbonUsageUnit.CO2_MG

tests/cli/test_cli.py

@@ -1,10 +1,16 @@
 import pytest
 from kubernetes import config
 
-from tracarbon import Country, EnergyUsage, Kubernetes, MacEnergyConsumption
-from tracarbon.cli import get_exporter, run_metrics
-from tracarbon.exporters import DatadogExporter, StdoutExporter
-from tracarbon.hardwares import Container, Pod
+from tracarbon import Country
+from tracarbon import EnergyUsage
+from tracarbon import Kubernetes
+from tracarbon import MacEnergyConsumption
+from tracarbon.cli import get_exporter
+from tracarbon.cli import run_metrics
+from tracarbon.exporters import DatadogExporter
+from tracarbon.exporters import StdoutExporter
+from tracarbon.hardwares import Container
+from tracarbon.hardwares import Pod
 
 
 def test_get_exporter_by_name():
@@ -32,9 +38,7 @@ def test_run_metrics_should_be_ok(mocker, caplog):
     )
     energy_usage = EnergyUsage(host_energy_usage=60.0)
     mocker.patch.object(config, "load_kube_config", return_value=None)
-    mocker.patch.object(
-        MacEnergyConsumption, "get_energy_usage", return_value=energy_usage
-    )
+    mocker.patch.object(MacEnergyConsumption, "get_energy_usage", return_value=energy_usage)
 
     run_metrics(exporter_name=exporter, running=False)
 
@@ -44,19 +48,15 @@ def test_run_metrics_should_be_ok(mocker, caplog):
     assert "units:watts" in caplog.text
 
     energy_usage = EnergyUsage(cpu_energy_usage=15.0, memory_energy_usage=12.0)
-    mocker.patch.object(
-        MacEnergyConsumption, "get_energy_usage", return_value=energy_usage
-    )
+    mocker.patch.object(MacEnergyConsumption, "get_energy_usage", return_value=energy_usage)
     mocker.patch.object(
         Kubernetes,
         "get_pods_usage",
         return_value=[
             Pod(
                 name="pod_name",
                 namespace="default",
-                containers=[
-                    Container(name="container_name", cpu_usage="1", memory_usage=2)
-                ],
+                containers=[Container(name="container_name", cpu_usage="1", memory_usage=2)],
             )
         ],
     )

tests/exporters/test_exporter.py

@@ -3,8 +3,11 @@
 import psutil
 import pytest
 
-from tracarbon import Country, MetricGenerator
-from tracarbon.exporters import Metric, StdoutExporter, Tag
+from tracarbon import Country
+from tracarbon import MetricGenerator
+from tracarbon.exporters import Metric
+from tracarbon.exporters import StdoutExporter
+from tracarbon.exporters import Tag
 
 
 def test_exporters_should_run_and_print_the_metrics(mocker, caplog):
@@ -32,10 +35,7 @@ async def get_memory_usage() -> float:
     assert memory_metric.name in caplog.text
     assert str(memory_metric.value) in caplog.text
     assert str(memory_metric.tags) in caplog.text
-    assert (
-        exporter.metric_report["test_metric_1"].exporter_name
-        == StdoutExporter.get_name()
-    )
+    assert exporter.metric_report["test_metric_1"].exporter_name == StdoutExporter.get_name()
     assert exporter.metric_report["test_metric_1"].metric == memory_metric
     assert exporter.metric_report["test_metric_1"].total > 0
     assert exporter.metric_report["test_metric_1"].average > 0

tests/exporters/test_json_exporter.py

@@ -4,8 +4,11 @@
 import psutil
 import ujson
 
-from tracarbon import Country, MetricGenerator
-from tracarbon.exporters import JSONExporter, Metric, Tag
+from tracarbon import Country
+from tracarbon import MetricGenerator
+from tracarbon.exporters import JSONExporter
+from tracarbon.exporters import Metric
+from tracarbon.exporters import Tag
 
 
 def test_json_exporter_should_write_well_formatted_metrics_in_json_file(mocker, tmpdir):
@@ -48,9 +51,7 @@ async def get_memory_usage() -> float:
     )
 
     metric_generators = [MetricGenerator(metrics=[memory_metric])]
-    exporter = JSONExporter(
-        quit=True, metric_generators=metric_generators, path=str(test_json_file)
-    )
+    exporter = JSONExporter(quit=True, metric_generators=metric_generators, path=str(test_json_file))
     exporter.start(interval_in_seconds=interval_in_seconds)
     exporter.stop()
 
@@ -61,9 +62,7 @@ async def get_memory_usage() -> float:
     with open(test_json_file, "r") as file:
         assert ujson.load(file) == expected
 
-    assert (
-        exporter.metric_report["test_metric_1"].exporter_name == JSONExporter.get_name()
-    )
+    assert exporter.metric_report["test_metric_1"].exporter_name == JSONExporter.get_name()
     assert exporter.metric_report["test_metric_1"].metric == memory_metric
     assert exporter.metric_report["test_metric_1"].total > 0
     assert exporter.metric_report["test_metric_1"].average > 0

tests/exporters/test_prometheus_exporter.py

@@ -2,8 +2,11 @@
 
 import psutil
 
-from tracarbon import Country, MetricGenerator
-from tracarbon.exporters import Metric, PrometheusExporter, Tag
+from tracarbon import Country
+from tracarbon import MetricGenerator
+from tracarbon.exporters import Metric
+from tracarbon.exporters import PrometheusExporter
+from tracarbon.exporters import Tag
 
 
 def test_prometheus_exporter(mocker):
@@ -27,19 +30,12 @@ async def get_memory_usage() -> float:
         tags=[Tag(key="test", value="tags")],
     )
     metric_generators = [MetricGenerator(metrics=[memory_metric])]
-    exporter = PrometheusExporter(
-        quit=True, metric_generators=metric_generators, metric_prefix_name="tracarbon"
-    )
+    exporter = PrometheusExporter(quit=True, metric_generators=metric_generators, metric_prefix_name="tracarbon")
     exporter.start(interval_in_seconds=interval_in_seconds)
     exporter.stop()
 
-    assert (
-        str(exporter.prometheus_metrics["tracarbon_test_metric_1"]) == expected_metric_1
-    )
-    assert (
-        exporter.metric_report["test_metric_1"].exporter_name
-        == PrometheusExporter.get_name()
-    )
+    assert str(exporter.prometheus_metrics["tracarbon_test_metric_1"]) == expected_metric_1
+    assert exporter.metric_report["test_metric_1"].exporter_name == PrometheusExporter.get_name()
     assert exporter.metric_report["test_metric_1"].metric == memory_metric
     assert exporter.metric_report["test_metric_1"].total > 0
     assert exporter.metric_report["test_metric_1"].average > 0