From b8f362f115f223f9cb89f3ae1fbcd091469b2df7 Mon Sep 17 00:00:00 2001 From: Benjamin Wilson Date: Sat, 21 May 2022 22:02:16 -0400 Subject: [PATCH] Add additional 3D object detection submission details. (#63) * Add submission format. * Updates. * Updates. * Update tutorial. * Updates. * Updates. * Updates. * Updates. * Updates. * Updates. * Updates. --- .../evaluation/detection/SUBMISSION_FORMAT.md | 46 +++++++++++++++++++ tutorials/object_detection_evaluation.ipynb | 32 ++++++++++++- 2 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 src/av2/evaluation/detection/SUBMISSION_FORMAT.md diff --git a/src/av2/evaluation/detection/SUBMISSION_FORMAT.md b/src/av2/evaluation/detection/SUBMISSION_FORMAT.md new file mode 100644 index 00000000..38312295 --- /dev/null +++ b/src/av2/evaluation/detection/SUBMISSION_FORMAT.md @@ -0,0 +1,46 @@ +# 3D Object Detection Submission Format + +The evaluation expects the following fields within a `pandas.DataFrame`: + +- `tx_m`: x-component of the object translation in the egovehicle reference frame. +- `ty_m`: y-component of the object translation in the egovehicle reference frame. +- `tz_m`: z-component of the object translation in the egovehicle reference frame. +- `length_m`: Object extent along the x-axis in meters. +- `width_m`: Object extent along the y-axis in meters. +- `height_m`: Object extent along the z-axis in meters. +- `qw`: Real quaternion coefficient. +- `qx`: First quaternion coefficient. +- `qy`: Second quaternion coefficient. +- `qz`: Third quaternion coefficient. +- `score`: Object confidence. +- `log_id`: Log id associated with the detection. +- `timestamp_ns`: Timestamp associated with the detection. +- `category`: Object category. + +An example looks like this: + +```python +# These detections are only for example purposes. + +display(detections) # Detections is type `pd.DataFrame` + tx_m ty_m tz_m length_m width_m height_m qw qx qy qz score log_id timestamp_ns category +0 -162.932968 1.720428 0.039064 1.596262 0.772320 1.153996 0.125843 0.0 0.0 0.992050 0.127634 b0116f1c-f88f-3c09-b4bf-fc3c8ebeda56 315968193659921000 WHEELCHAIR +1 -120.362213 19.875946 -0.382618 1.441901 0.593825 1.199819 0.802836 0.0 0.0 0.596200 0.126565 b0116f1c-f88f-3c09-b4bf-fc3c8ebeda56 315968193659921000 BICYCLE +... +14000000 10.182907 29.489899 0.662969 9.166531 1.761454 1.615999 0.023469 0.0 0.0 -0.999725 0.322177 b2d9d8a5-847b-3c3b-aed1-c414319d20af 315978610360111000 REGULAR_VEHICLE + +detections.columns +Index(['tx_m', 'ty_m', 'tz_m', 'length_m', 'width_m', 'height_m', 'qw', 'qx', + 'qy', 'qz', 'score', 'log_id', 'timestamp_ns', 'category'], + dtype='object') +``` + +We need to export the above dataframe for submission. This can be done by: + +```python +import pandas as pd + +detections.to_feather("detections.feather") +``` + +Lastly, submit this file to the competition leaderboard. \ No newline at end of file diff --git a/tutorials/object_detection_evaluation.ipynb b/tutorials/object_detection_evaluation.ipynb index 8100d788..4a4284de 100644 --- a/tutorials/object_detection_evaluation.ipynb +++ b/tutorials/object_detection_evaluation.ipynb @@ -61,7 +61,24 @@ "source": [ "## Preparing detections for submission.\n", "\n", - "Our submission server expects a `.feather` file that contains a list of detections for the entire test set." + "The evaluation expects the following 14 fields within a `pandas.DataFrame`:\n", + "\n", + "- `tx_m`: x-component of the object translation in the egovehicle reference frame.\n", + "- `ty_m`: y-component of the object translation in the egovehicle reference frame.\n", + "- `tz_m`: z-component of the object translation in the egovehicle reference frame.\n", + "- `length_m`: Object extent along the x-axis in meters.\n", + "- `width_m`: Object extent along the y-axis in meters.\n", + "- `height_m`: Object extent along the z-axis in meters.\n", + "- `qw`: Real quaternion coefficient.\n", + "- `qx`: First quaternion coefficient.\n", + "- `qy`: Second quaternion coefficient.\n", + "- `qz`: Third quaternion coefficient.\n", + "- `score`: Object confidence.\n", + "- `log_id`: Log id associated with the detection.\n", + "- `timestamp_ns`: Timestamp associated with the detection.\n", + "- `category`: Object category.\n", + "\n", + "Additional details can be found in [SUBMISSION_FORMAT.md](../src/av2/evaluation/detection/SUBMISSION_FORMAT.md)." ] }, { @@ -85,6 +102,17 @@ "source": [ "display(metrics)" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, if you would like to submit to the evaluation server, you just need to export your detections into a `.feather` file. This can be done by:\n", + "\n", + "```python\n", + "dts.to_feather(\"detections.feather\")\n", + "```" + ] } ], "metadata": { @@ -106,7 +134,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.2" + "version": "3.9.12" }, "orig_nbformat": 4 },