refactor: add James planes test script

sdr-enthusiasts · Mar 5, 2023 · 6dba3dc · 6dba3dc
1 parent 81855c5
commit 6dba3dc
Show file tree

Hide file tree

Showing 4 changed files with 169 additions and 0 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,9 @@
 .DS_Store
+__pycache__
+*.pyc
+
+# Local databases
+james-planes-new-items.csv
 
 # IDE
 .vscode

diff --git a/scripts/README.md b/scripts/README.md
@@ -2,6 +2,7 @@
 
 This folder contains several scripts used in the GitHub actions:
 
+- `check_james_planes`: A script that can be used to check if the planes mentioned in https://github.com/sdr-enthusiasts/plane-alert-db/issues/24 are already in the main databases.
 - `check_main_databases`: A script that is used to check whether the main databases are correctly formatted.
 - `create_db_derivatives`: A script that can be used to create the derivative databases based on the `plane-alert-db.csv`, `plane_images.txt` and `blacklist.txt` files.
 - `create_images_reference`: A tiny little script that I used to create the new `plane_images.txt` file. This file will be removed when we are sure the file of the new image is correct.

diff --git a/scripts/check_james_planes.py b/scripts/check_james_planes.py
@@ -0,0 +1,43 @@
+"""Check whether the interesting planes given by James in 
+https://github.com/sdr-enthusiasts/plane-alert-db/issues/24 are already in the main
+databases.
+"""
+
+import pandas as pd
+import logging
+from james_planes import james_planes_df
+
+logging.basicConfig(
+    format="%(asctime)s %(levelname)-8s [%(name)s] %(message)s", level=logging.INFO
+)
+
+if __name__ == "__main__":
+
+    logging.info("Reading 'plane-alert-db' csv file...")
+    main_df = pd.read_csv("plane-alert-db.csv")
+    logging.info("'plane-alert-db' csv file read successfully.")
+
+    logging.info("Reading the 'plane-alert-twitter-blocked' csv file...")
+    twitter_blocker_df = pd.read_csv("plane-alert-twitter-blocked.csv")
+    logging.info("'plane-alert-twitter-blocked' csv file read successfully.")
+
+    logging.info("Reading 'plane-alert-ukraine' csv file...")
+    ukraine_df = pd.read_csv("plane-alert-ukraine.csv")
+    logging.info("'plane-alert-ukraine' csv file read successfully.")
+
+    logging.info("Concatenating all the dataframes...")
+    main_df = pd.concat([main_df, twitter_blocker_df, ukraine_df])
+    logging.info("Concatenation done.")
+
+    logging.info(
+        f"Get new items in 'james-planes' database ({james_planes_df.shape[0]})..."
+    )
+    new_items = james_planes_df[~james_planes_df["$ICAO"].isin(main_df["$ICAO"])]
+
+    james_planes_df[~james_planes_df["$ICAO"].isin(main_df["$ICAO"])]["$ICAO"].iloc[0]
+    if new_items.empty:
+        logging.info("No new items found.")
+    else:
+        logging.info(f"New items found ({new_items.shape[0]}).")
+        logging.info(new_items)
+        new_items.to_csv("james-planes-new-items.csv", index=False)
diff --git a/scripts/james_planes.py b/scripts/james_planes.py
@@ -0,0 +1,120 @@
+"""A list of interesting planes given by James in
+https://github.com/sdr-enthusiasts/plane-alert-db/issues/24. 
+
+It looks like James got this list from the RadarAtlas extension using the CRX viewer
+extension.
+""" ""
+
+import pandas as pd
+
+intChineseMil = {}
+interestingDirtboxes = {}
+interestingTelevangelists = {}
+interestingPeople = {}
+interestingEntertainmentIndustry = {}
+interestingCompanies = {}
+interestingAthletes = {}
+interestingAthleticOrgs = {}
+interestingAircraft = {}
+interestingRutan = {}
+interestingArea51 = {}
+intSEC = {}
+interestingGovernment = {}
+interestingNASA = {}
+interestingDrones = {}
+interestingNorthKorea = {}
+interestingHeadsOfState = {}
+interestingMusk = {}
+intUkraineMil = {}
+intEco = {}
+c37s = {}
+intRussianMil = {}
+russianOligs = {}
+interestingWealthiestUkrainians = {}
+intAutoRacing = {}
+intPutin = {}
+int50StatesBillionaires = {}
+israeliAirForce = {}
+interestingNATO = {}
+interestingVolga = {}
+interestingPaintJobs = {}
+interestingHomeland = {}
+interestingUN = {}
+intPharma = {}
+intTechBillionaires = {}
+intFirearms = {}
+intOil = {}
+intUltimateJetCharter = {}
+intVertolSystems = {}
+intNuclear = {}
+intTopAces = {}
+intHurricaneHunters = {}
+intTelecom = {}
+intFinanciers = {}
+intSuadiAirForce = {}
+
+# Create a combined dataframe with the constant name as the category column.
+categories = [
+    "intChineseMil",
+    "interestingDirtboxes",
+    "interestingTelevangelists",
+    "interestingPeople",
+    "interestingEntertainmentIndustry",
+    "interestingCompanies",
+    "interestingAthletes",
+    "interestingAthleticOrgs",
+    "interestingAircraft",
+    "interestingRutan",
+    "interestingArea51",
+    "intSEC",
+    "interestingGovernment",
+    "interestingNASA",
+    "interestingDrones",
+    "interestingNorthKorea",
+    "interestingHeadsOfState",
+    "interestingMusk",
+    "intUkraineMil",
+    "intEco",
+    "c37s",
+    "intRussianMil",
+    "russianOligs",
+    "interestingWealthiestUkrainians",
+    "intAutoRacing",
+    "intPutin",
+    "int50StatesBillionaires",
+    "israeliAirForce",
+    "interestingNATO",
+    "interestingVolga",
+    "interestingPaintJobs",
+    "interestingHomeland",
+    "interestingUN",
+    "intPharma",
+    "intTechBillionaires",
+    "intFirearms",
+    "intOil",
+    "intUltimateJetCharter",
+    "intVertolSystems",
+    "intNuclear",
+    "intTopAces",
+    "intHurricaneHunters",
+    "intTelecom",
+    "intFinanciers",
+    "intSuadiAirForce",
+]
+james_planes_df = pd.DataFrame()
+for category in categories:
+    james_planes_df = pd.concat(
+        [
+            james_planes_df,
+            pd.DataFrame(
+                {
+                    "$ICAO": list(eval(category).keys()),
+                    "name": list(eval(category).values()),
+                    "category": category,
+                }
+            ),
+        ]
+    )
+
+# Create a new column with the ICAO in uppercase.
+james_planes_df["ICAO"] = james_planes_df["$ICAO"].str.upper()