Skip to content
This repository has been archived by the owner on Jan 25, 2024. It is now read-only.

Commit

Permalink
feat(json): Implement export to JSON file (#132)
Browse files Browse the repository at this point in the history
  • Loading branch information
vvatelot authored May 5, 2022
1 parent 7b646cc commit 2f91d82
Show file tree
Hide file tree
Showing 5 changed files with 144 additions and 17 deletions.
72 changes: 65 additions & 7 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,26 @@ Processing [####################################] 100%
</details>
### Export to JSON file
By default, the results are exported to a CSV file. But, you can specify to export the results to a JSON file.
```bash
ecoindex-cli analyze --url http://www.ecoindex.fr --export-format json
```
<details><summary>Result</summary>
```bash
📁️ Urls recorded in file `input/www.ecoindex.fr.csv`
There are 1 url(s), do you want to process? [Y/n]:
1 urls for 1 window size with 2 maximum workers
Processing [####################################] 100%
🙌️ File /tmp/ecoindex-cli/output/www.ecoindex.fr/2022-03-05_215320/results.json written !
```
</details>
### Multiple url analysis
```bash
Expand Down Expand Up @@ -255,26 +275,64 @@ ecoindex-cli report "/tmp/ecoindex-cli/output/www.ecoindex.fr/2021-05-06_191355/
## Results example
The result of the analysis is a CSV file which can be easily used for further analysis:
The result of the analysis is a CSV or JSON file which can be easily used for further analysis:
### CSV example
```csv
size,nodes,requests,grade,score,ges,water,url,date,resolution,page_type
119.095,45,8,A,89,1.22,1.83,http://www.ecoindex.fr,2021-04-20 16:45:28.570179,"1920,1080",
769.252,730,94,D,41,2.18,3.27,https://www.greenit.fr/,2021-04-20 16:45:32.199242,"1920,1080",website
width,height,url,size,nodes,requests,grade,score,ges,water,date,page_type
1920,1080,http://www.ecoindex.fr,521.54,45,68,B,75.0,1.5,2.25,2022-05-03 22:28:49.280479,
1920,1080,https://www.greenit.fr,1374.641,666,167,E,32.0,2.36,3.54,2022-05-03 22:28:51.176216,website
```
### JSON example
```json
[
{
"width": 1920,
"height": 1080,
"url": "http://www.ecoindex.fr",
"size": 521.54,
"nodes": 45,
"requests": 68,
"grade": "B",
"score": 75.0,
"ges": 1.5,
"water": 2.25,
"date": "2022-05-03 22:25:01.016749",
"page_type": null
},
{
"width": 1920,
"height": 1080,
"url": "https://www.greenit.fr",
"size": 1163.386,
"nodes": 666,
"requests": 148,
"grade": "E",
"score": 34.0,
"ges": 2.32,
"water": 3.48,
"date": "2022-05-03 22:25:04.516676",
"page_type": "website"
}
]
```
Where:
### Fields description
- `width` is the screen width used for the page analysis (in pixels)
- `height` is the screen height used for the page analysis (in pixels)
- `url` is the analysed page url
- `size` is the size of the page and of the downloaded elements of the page in KB
- `nodes` is the number of the DOM elements in the page
- `requests` is the number of external requests made by the page
- `grade` is the corresponding ecoindex grade of the page (from A to G)
- `score` is the corresponding ecoindex score of the page (0 to 100)
- `ges` is the equivalent of greenhouse gases emission (in `gCO2e`) of the page
- `water`is the equivalent water consumption (in `cl`) of the page
- `url` is the analysed page url
- `date` is the datetime of the page analysis
- `resolution` is the screen resolution used for the page analysis (`width,height`)
- `page_type` is the type of the page, based ton the [opengraph type tag](https://ogp.me/#types)
## Testing
Expand Down
12 changes: 10 additions & 2 deletions ecoindex_cli/cli/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
get_window_sizes_from_args,
)
from ecoindex_cli.cli.helper import run_page_analysis
from ecoindex_cli.enums import ExportFormat
from ecoindex_cli.files import write_results_to_file, write_urls_to_file
from ecoindex_cli.logger import Logger
from ecoindex_cli.report.report import generate_report
Expand Down Expand Up @@ -57,6 +58,11 @@ def analyze(
default=None,
help="You can define the number of workers to use for the analysis. Default is the number of cpu cores",
),
export_format: Optional[ExportFormat] = Option(
default=ExportFormat.csv.value,
help="You can export the results in json or csv. Default is csv",
case_sensitive=False,
),
):
"""
Make an ecoindex analysis of given webpages or website. You
Expand Down Expand Up @@ -163,14 +169,16 @@ def analyze(
output_folder = (
f"/tmp/ecoindex-cli/output/{file_prefix}/{time_now.strftime('%Y-%d-%m_%H%M%S')}"
)
output_filename = f"{output_folder}/results.csv"
output_filename = f"{output_folder}/results.{export_format.value}"

if output_file:
output_filename = output_file
output_folder = dirname(output_filename)

Path(output_folder).mkdir(parents=True, exist_ok=True)
write_results_to_file(filename=output_filename, results=results)
write_results_to_file(
filename=output_filename, results=results, export_format=export_format
)
secho(f"🙌️ File {output_filename} written !", fg=colors.GREEN)
if html_report:
generate_report(
Expand Down
6 changes: 6 additions & 0 deletions ecoindex_cli/enums.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
from enum import Enum


class ExportFormat(Enum):
csv = "csv"
json = "json"
62 changes: 54 additions & 8 deletions ecoindex_cli/files.py
Original file line number Diff line number Diff line change
@@ -1,25 +1,71 @@
from abc import ABC, abstractmethod
from csv import DictWriter
from json import dump
from os import makedirs
from os.path import exists
from typing import List
from typing import List, Optional

from ecoindex_scraper.models import Result

from ecoindex_cli.enums import ExportFormat


def create_folder(path: str) -> None:
if not exists(path):
makedirs(path)


def write_results_to_file(filename: str, results: List[Result]) -> None:
headers = results[0].__dict__
class File(ABC):
def __init__(
self,
filename: str,
results: List[Result],
export_format: Optional[ExportFormat] = ExportFormat.csv,
):
self.filename = filename
self.results = results
self.export_format = export_format

@abstractmethod
def write(self) -> None:
pass


class CsvFile(File):
def write(self) -> None:
headers = self.results[0].__dict__

with open(self.filename, "w") as fp:
writer = DictWriter(fp, fieldnames=headers)

writer.writeheader()
for ecoindex in self.results:
writer.writerow(ecoindex.__dict__)


class JsonFile(File):
def write(self) -> None:
with open(self.filename, "w") as fp:
dump(
obj=[ecoindex.__dict__ for ecoindex in self.results],
fp=fp,
indent=4,
default=str,
)


with open(filename, "w") as fp:
writer = DictWriter(fp, fieldnames=headers)
def write_results_to_file(
filename: str,
results: List[Result],
export_format: Optional[ExportFormat] = ExportFormat.csv,
) -> None:
print(export_format)
if export_format == ExportFormat.csv:
file = CsvFile(filename=filename, results=results, export_format=export_format)
elif export_format == ExportFormat.json:
file = JsonFile(filename=filename, results=results, export_format=export_format)

writer.writeheader()
for ecoindex in results:
writer.writerow(ecoindex.__dict__)
file.write()


def write_urls_to_file(file_prefix: str, urls: List[str]) -> None:
Expand Down
9 changes: 9 additions & 0 deletions tests/ecoindex_cli/cli/test_app.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,3 +91,12 @@ def test_no_interaction():
result = runner.invoke(app=app, args=["analyze", "--recursive", "--no-interaction"])
assert "[Y/n]" not in result.stdout
assert result.exit_code == 1


def test_unauthorized_export_format():
result = runner.invoke(app=app, args=["analyze", "--export-format", "txt"])
assert result.exit_code == 2
assert (
"Error: Invalid value for '--export-format': 'txt' is not one of 'csv', 'json'."
in result.stdout
)

0 comments on commit 2f91d82

Please sign in to comment.