From cc1b52eab902e39e61d4d0216b06b4b9800e3017 Mon Sep 17 00:00:00 2001 From: Yining Li Date: Fri, 2 Jul 2021 16:28:40 +0800 Subject: [PATCH] Add webcam demo (#729) This PR adds a webcam demo tool with the following features: 1. Read video stream from the webcam or offline video file 2. Async model inference (detection + human pose + animal pose) and video I/O 3. Optimized visualization functions of bbox, object label, and pose 4. Apply special effects (sunglasses or bug-eye) 5. Show statistic information, e.g., FPS, CPU usage. 6. Optionally, save out the video --- demo/README.md | 8 + demo/docs/webcam_demo.md | 49 ++ .../cascade_rcnn_x101_64x4d_fpn_1class.py | 14 +- .../cascade_rcnn_x101_64x4d_fpn_coco.py | 14 +- .../faster_rcnn_r50_fpn_1class.py | 14 +- .../faster_rcnn_r50_fpn_coco.py | 14 +- .../yolov3_d53_320_273e_coco.py | 140 +++++ ...or_faster-rcnn_r50_fpn_4e_mot17-private.py | 6 +- demo/resources/sunglasses.jpg | Bin 0 -> 10744 bytes demo/webcam_demo.py | 567 ++++++++++++++++++ mmpose/apis/inference.py | 6 +- mmpose/core/visualization/__init__.py | 8 +- mmpose/core/visualization/effects.py | 110 ++++ mmpose/core/visualization/image.py | 76 +++ mmpose/models/detectors/pose_lifter.py | 5 +- mmpose/models/detectors/top_down.py | 17 +- mmpose/utils/__init__.py | 3 +- mmpose/utils/timer.py | 92 +++ tests/test_regularization.py | 2 +- tests/test_utils.py | 27 +- tests/test_visualization.py | 51 +- 21 files changed, 1175 insertions(+), 48 deletions(-) create mode 100644 demo/docs/webcam_demo.md create mode 100644 demo/mmdetection_cfg/yolov3_d53_320_273e_coco.py create mode 100644 demo/resources/sunglasses.jpg create mode 100644 demo/webcam_demo.py create mode 100644 mmpose/core/visualization/effects.py create mode 100644 mmpose/utils/timer.py diff --git a/demo/README.md b/demo/README.md index f824bbf0cf6..60ecbc33987 100644 --- a/demo/README.md +++ b/demo/README.md @@ -65,3 +65,11 @@ This page provides tutorials about running demos. Please click the caption for m [3D hand_pose demo](docs/3d_hand_demo.md) + +
+ +
+
+ +[Webcam demo](docs/webcam_demo.md) +
diff --git a/demo/docs/webcam_demo.md b/demo/docs/webcam_demo.md new file mode 100644 index 00000000000..17db79752f9 --- /dev/null +++ b/demo/docs/webcam_demo.md @@ -0,0 +1,49 @@ +## Webcam Demo + +We provide a webcam demo tool which integrartes detection and 2D pose estimation for humans and animals. You can simply run the following command: + +```python +python demo/webcam_demo.py +``` + +It will launch a window to display the webcam video steam with detection and pose estimation results: + +
+
+
+ +### Usage Tips + +- **Which model is used in the demo tool?** + + Please check the following default arguments in the script. You can also choose other models from the [MMDetection Model Zoo](https://github.com/open-mmlab/mmdetection/blob/master/docs/model_zoo.md) and [MMPose Model Zoo](https://mmpose.readthedocs.io/en/latest/modelzoo.html#) or use your own models. + + | Model | Arguments | + | :--: | :-- | + | Detection | `--det-config`, `--det-checkpoint` | + | Human Pose | `--human-pose-config`, `--human-pose-checkpoint` | + | Animal Pose | `--animal-pose-config`, `--animal-pose-checkpoint` | + +- **Can this tool run without GPU?** + + Yes, you can set `--device=cpu` and the model inference will be performed on CPU. Of course, this may cause a low inference FPS compared to using GPU devices. + +- **Why there is time delay between the pose visualization and the video?** + + The video I/O and model inference are running asynchronously and the latter usually takes more time for a single frame. To allevidate the time delay, you can: + + 1. set `--display-delay=MILLISECONDS` to defer the video stream, according to the inference delay shown at the top left corner. Or, + + 2. set `--synchronous-mode` to force video stream being aligned with inference results. This may reduce the video display FPS. + +- **Can this tool process video files?** + + Yes. You can set `--cam_id=VIDEO_FILE_PATH` to run the demo tool in offline mode on a video file. Note that `--synchronous-mode` should be set in this case. + +- **How to enable/disable the special effects?** + + The special effects can be enabled/disabled at launch time by setting arguments like `--bugeye`, `--sunglasses`, *etc*. You can also toggle the effects by keyboard shorcuts like `b`, `s` when the tool starts. + +- **What if my computer doesn't have a camera?** + + You can use a smart phone as a webcam with apps like [Camo](https://reincubate.com/camo/) or [DroidCam](https://www.dev47apps.com/). diff --git a/demo/mmdetection_cfg/cascade_rcnn_x101_64x4d_fpn_1class.py b/demo/mmdetection_cfg/cascade_rcnn_x101_64x4d_fpn_1class.py index f45ad10cb0a..4e60b6b7397 100644 --- a/demo/mmdetection_cfg/cascade_rcnn_x101_64x4d_fpn_1class.py +++ b/demo/mmdetection_cfg/cascade_rcnn_x101_64x4d_fpn_1class.py @@ -206,7 +206,7 @@ max_per_img=100))) dataset_type = 'CocoDataset' -data_root = 'data/coco/' +data_root = 'data/coco' img_norm_cfg = dict( mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) train_pipeline = [ @@ -239,17 +239,17 @@ workers_per_gpu=2, train=dict( type=dataset_type, - ann_file=data_root + 'annotations/instances_train2017.json', - img_prefix=data_root + 'train2017/', + ann_file=f'{data_root}/annotations/instances_train2017.json', + img_prefix=f'{data_root}/train2017/', pipeline=train_pipeline), val=dict( type=dataset_type, - ann_file=data_root + 'annotations/instances_val2017.json', - img_prefix=data_root + 'val2017/', + ann_file=f'{data_root}/annotations/instances_val2017.json', + img_prefix=f'{data_root}/val2017/', pipeline=test_pipeline), test=dict( type=dataset_type, - ann_file=data_root + 'annotations/instances_val2017.json', - img_prefix=data_root + 'val2017/', + ann_file=f'{data_root}/annotations/instances_val2017.json', + img_prefix=f'{data_root}/val2017/', pipeline=test_pipeline)) evaluation = dict(interval=1, metric='bbox') diff --git a/demo/mmdetection_cfg/cascade_rcnn_x101_64x4d_fpn_coco.py b/demo/mmdetection_cfg/cascade_rcnn_x101_64x4d_fpn_coco.py index 58b96e62e10..f91bd0d105b 100644 --- a/demo/mmdetection_cfg/cascade_rcnn_x101_64x4d_fpn_coco.py +++ b/demo/mmdetection_cfg/cascade_rcnn_x101_64x4d_fpn_coco.py @@ -207,7 +207,7 @@ max_per_img=100))) dataset_type = 'CocoDataset' -data_root = 'data/coco/' +data_root = 'data/coco' img_norm_cfg = dict( mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) train_pipeline = [ @@ -240,17 +240,17 @@ workers_per_gpu=2, train=dict( type=dataset_type, - ann_file=data_root + 'annotations/instances_train2017.json', - img_prefix=data_root + 'train2017/', + ann_file=f'{data_root}/annotations/instances_train2017.json', + img_prefix=f'{data_root}/train2017/', pipeline=train_pipeline), val=dict( type=dataset_type, - ann_file=data_root + 'annotations/instances_val2017.json', - img_prefix=data_root + 'val2017/', + ann_file=f'{data_root}/annotations/instances_val2017.json', + img_prefix=f'{data_root}/val2017/', pipeline=test_pipeline), test=dict( type=dataset_type, - ann_file=data_root + 'annotations/instances_val2017.json', - img_prefix=data_root + 'val2017/', + ann_file=f'{data_root}/annotations/instances_val2017.json', + img_prefix=f'{data_root}/val2017/', pipeline=test_pipeline)) evaluation = dict(interval=1, metric='bbox') diff --git a/demo/mmdetection_cfg/faster_rcnn_r50_fpn_1class.py b/demo/mmdetection_cfg/faster_rcnn_r50_fpn_1class.py index 8eddf232b78..ee54f5b66bd 100644 --- a/demo/mmdetection_cfg/faster_rcnn_r50_fpn_1class.py +++ b/demo/mmdetection_cfg/faster_rcnn_r50_fpn_1class.py @@ -133,7 +133,7 @@ )) dataset_type = 'CocoDataset' -data_root = 'data/coco/' +data_root = 'data/coco' img_norm_cfg = dict( mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) train_pipeline = [ @@ -166,17 +166,17 @@ workers_per_gpu=2, train=dict( type=dataset_type, - ann_file=data_root + 'annotations/instances_train2017.json', - img_prefix=data_root + 'train2017/', + ann_file=f'{data_root}/annotations/instances_train2017.json', + img_prefix=f'{data_root}/train2017/', pipeline=train_pipeline), val=dict( type=dataset_type, - ann_file=data_root + 'annotations/instances_val2017.json', - img_prefix=data_root + 'val2017/', + ann_file=f'{data_root}/annotations/instances_val2017.json', + img_prefix=f'{data_root}/val2017/', pipeline=test_pipeline), test=dict( type=dataset_type, - ann_file=data_root + 'annotations/instances_val2017.json', - img_prefix=data_root + 'val2017/', + ann_file=f'{data_root}/annotations/instances_val2017.json', + img_prefix=f'{data_root}/val2017/', pipeline=test_pipeline)) evaluation = dict(interval=1, metric='bbox') diff --git a/demo/mmdetection_cfg/faster_rcnn_r50_fpn_coco.py b/demo/mmdetection_cfg/faster_rcnn_r50_fpn_coco.py index d5c17df89a8..a9ad9528b22 100644 --- a/demo/mmdetection_cfg/faster_rcnn_r50_fpn_coco.py +++ b/demo/mmdetection_cfg/faster_rcnn_r50_fpn_coco.py @@ -133,7 +133,7 @@ )) dataset_type = 'CocoDataset' -data_root = 'data/coco/' +data_root = 'data/coco' img_norm_cfg = dict( mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) train_pipeline = [ @@ -166,17 +166,17 @@ workers_per_gpu=2, train=dict( type=dataset_type, - ann_file=data_root + 'annotations/instances_train2017.json', - img_prefix=data_root + 'train2017/', + ann_file=f'{data_root}/annotations/instances_train2017.json', + img_prefix=f'{data_root}/train2017/', pipeline=train_pipeline), val=dict( type=dataset_type, - ann_file=data_root + 'annotations/instances_val2017.json', - img_prefix=data_root + 'val2017/', + ann_file=f'{data_root}/annotations/instances_val2017.json', + img_prefix=f'{data_root}/val2017/', pipeline=test_pipeline), test=dict( type=dataset_type, - ann_file=data_root + 'annotations/instances_val2017.json', - img_prefix=data_root + 'val2017/', + ann_file=f'{data_root}/annotations/instances_val2017.json', + img_prefix=f'{data_root}/val2017/', pipeline=test_pipeline)) evaluation = dict(interval=1, metric='bbox') diff --git a/demo/mmdetection_cfg/yolov3_d53_320_273e_coco.py b/demo/mmdetection_cfg/yolov3_d53_320_273e_coco.py new file mode 100644 index 00000000000..d7e9cca1eb3 --- /dev/null +++ b/demo/mmdetection_cfg/yolov3_d53_320_273e_coco.py @@ -0,0 +1,140 @@ +# model settings +model = dict( + type='YOLOV3', + pretrained='open-mmlab://darknet53', + backbone=dict(type='Darknet', depth=53, out_indices=(3, 4, 5)), + neck=dict( + type='YOLOV3Neck', + num_scales=3, + in_channels=[1024, 512, 256], + out_channels=[512, 256, 128]), + bbox_head=dict( + type='YOLOV3Head', + num_classes=80, + in_channels=[512, 256, 128], + out_channels=[1024, 512, 256], + anchor_generator=dict( + type='YOLOAnchorGenerator', + base_sizes=[[(116, 90), (156, 198), (373, 326)], + [(30, 61), (62, 45), (59, 119)], + [(10, 13), (16, 30), (33, 23)]], + strides=[32, 16, 8]), + bbox_coder=dict(type='YOLOBBoxCoder'), + featmap_strides=[32, 16, 8], + loss_cls=dict( + type='CrossEntropyLoss', + use_sigmoid=True, + loss_weight=1.0, + reduction='sum'), + loss_conf=dict( + type='CrossEntropyLoss', + use_sigmoid=True, + loss_weight=1.0, + reduction='sum'), + loss_xy=dict( + type='CrossEntropyLoss', + use_sigmoid=True, + loss_weight=2.0, + reduction='sum'), + loss_wh=dict(type='MSELoss', loss_weight=2.0, reduction='sum')), + # training and testing settings + train_cfg=dict( + assigner=dict( + type='GridAssigner', + pos_iou_thr=0.5, + neg_iou_thr=0.5, + min_pos_iou=0)), + test_cfg=dict( + nms_pre=1000, + min_bbox_size=0, + score_thr=0.05, + conf_thr=0.005, + nms=dict(type='nms', iou_threshold=0.45), + max_per_img=100)) +# dataset settings +dataset_type = 'CocoDataset' +data_root = 'data/coco' +img_norm_cfg = dict(mean=[0, 0, 0], std=[255., 255., 255.], to_rgb=True) +train_pipeline = [ + dict(type='LoadImageFromFile', to_float32=True), + dict(type='LoadAnnotations', with_bbox=True), + dict(type='PhotoMetricDistortion'), + dict( + type='Expand', + mean=img_norm_cfg['mean'], + to_rgb=img_norm_cfg['to_rgb'], + ratio_range=(1, 2)), + dict( + type='MinIoURandomCrop', + min_ious=(0.4, 0.5, 0.6, 0.7, 0.8, 0.9), + min_crop_size=0.3), + dict(type='Resize', img_scale=(320, 320), keep_ratio=True), + dict(type='RandomFlip', flip_ratio=0.5), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size_divisor=32), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_bboxes', 'gt_labels']) +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=(320, 320), + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size_divisor=32), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img']) + ]) +] +data = dict( + samples_per_gpu=8, + workers_per_gpu=4, + train=dict( + type=dataset_type, + ann_file=f'{data_root}/annotations/instances_train2017.json', + img_prefix=f'{data_root}/train2017/', + pipeline=train_pipeline), + val=dict( + type=dataset_type, + ann_file=f'{data_root}/annotations/instances_val2017.json', + img_prefix=f'{data_root}/val2017/', + pipeline=test_pipeline), + test=dict( + type=dataset_type, + ann_file=f'{data_root}/annotations/instances_val2017.json', + img_prefix=f'{data_root}/val2017/', + pipeline=test_pipeline)) +# optimizer +optimizer = dict(type='SGD', lr=0.001, momentum=0.9, weight_decay=0.0005) +optimizer_config = dict(grad_clip=dict(max_norm=35, norm_type=2)) +# learning policy +lr_config = dict( + policy='step', + warmup='linear', + warmup_iters=2000, # same as burn-in in darknet + warmup_ratio=0.1, + step=[218, 246]) +# runtime settings +runner = dict(type='EpochBasedRunner', max_epochs=273) +evaluation = dict(interval=1, metric=['bbox']) + +checkpoint_config = dict(interval=1) +# yapf:disable +log_config = dict( + interval=50, + hooks=[ + dict(type='TextLoggerHook'), + # dict(type='TensorboardLoggerHook') + ]) +# yapf:enable +custom_hooks = [dict(type='NumClassCheckHook')] + +dist_params = dict(backend='nccl') +log_level = 'INFO' +load_from = None +resume_from = None +workflow = [('train', 1)] diff --git a/demo/mmtracking_cfg/tracktor_faster-rcnn_r50_fpn_4e_mot17-private.py b/demo/mmtracking_cfg/tracktor_faster-rcnn_r50_fpn_4e_mot17-private.py index af31ccaf181..2973d16e873 100644 --- a/demo/mmtracking_cfg/tracktor_faster-rcnn_r50_fpn_4e_mot17-private.py +++ b/demo/mmtracking_cfg/tracktor_faster-rcnn_r50_fpn_4e_mot17-private.py @@ -202,7 +202,7 @@ std=[58.395, 57.12, 57.375], to_rgb=True), dict(type='Pad', size_divisor=32), - dict(type='ImageToTensor', keys=['img']), + dict(type='DefaultFormatBundle', keys=['img']), dict(type='VideoCollect', keys=['img']) ]) ] @@ -272,7 +272,7 @@ std=[58.395, 57.12, 57.375], to_rgb=True), dict(type='Pad', size_divisor=32), - dict(type='ImageToTensor', keys=['img']), + dict(type='DefaultFormatBundle', keys=['img']), dict(type='VideoCollect', keys=['img']) ]) ]), @@ -296,7 +296,7 @@ std=[58.395, 57.12, 57.375], to_rgb=True), dict(type='Pad', size_divisor=32), - dict(type='ImageToTensor', keys=['img']), + dict(type='DefaultFormatBundle', keys=['img']), dict(type='VideoCollect', keys=['img']) ]) ])) diff --git a/demo/resources/sunglasses.jpg b/demo/resources/sunglasses.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5d3cee870232cb35415e3ae71ea07e9fbb45dfdf GIT binary patch literal 10744 zcmeHsc|6qZ_xDIBYh+89%1$VRgh{d|*+a}tc1c3^nJghhNV3i?WJ{BEY|Uif2{E<| zW6M5c8N+P9>HhwH-|s)qAJ6mr`#jgY&h?qkHgnE(z0Y;dd0(_C+5+I5vA&T$fPn!3 zV4!~hv{`^I;1t83_NPDfr=4c}(=(ku&3KxbiJAEyVL5w-ndK}CGxHg?GiO=#$WA8?wRiRbcl9p>}q&Ma5_d6lBl z3eSk$scz%57{!SzyF8CR%gQexctJ=)Qc7AzR^^7On)*!*-Mf1F28Kq)mR8m_wh!&> zAG^A_dw6%{@D3H?SE+(H{GsN zf98PYk6jF>g6YD@eVXa=b!HwNa~5a+^H-Fj&hXwzE39riE2eCL<8ygF%E~XUvM7Q7 zW7(Vy01EkIqBP+D%OdqEWDEm9GT#MbN4kc% z?!Hx6^azUhw9-@Yy(taFbfcbKTa3@zvWP`S`?V@{rF;b_ixGx2K=drNpUNL9ugSEC;URvq9?8;FHSu^t1K?yn7=m_xg}~lG7g~XQes<_Z z%y%f)dL^0T^M>U-teiVRZtI$p2p)1%N?18@SG9s5S264s1(HF&$@i?V7qmcpga@xu!#5*1C+{vko}c{2fBC_y2#Yr4MB z0G|1FXEzBZCJ#XeRwjujLfTw-@re!V5@22~`LbWvH4yH&2X>sMk&KTAEU0 z7o#@&C%kM*R~PwFdv*vSuX6pgI}E|2HV)ZKG9YB1lnDM zmuoXXFyI>0_GN$0G~7NmsMWr8>TIEpw6DT6syRt&K!yhR^5k&Ou9O+CYQ~~@U&vT4 zRL)3I75fYM0(85VY_L!uR#yI(|JQ(UecRaFt#>~iAX!@JiPG?F?K$a>i2!N%MIwn5 z3F)T+Y9l*m3_3xtC-*jR%i6DsF1NJUP&YqAuNqLBm23x~+;l92o#v82w1Z!&1_)=+Hhf&wIlxOK`UJj~%t<$4G3}?n^6<9qrB~!I@ zaoC99&1wpA02cDvF2j;g3-zc5#RQSKD4#{Gvs`l|=H$Ib(N(Mw1L)<6EkhNw?E&iP zcIfM7%HN94EFY)6Y-|>wm_{9VRpFw*`3c){jc14Na>-ZFSO_%fME98%i?>y!)&7G- z@UhA`nQNw54mY{>2LlRIbtpOp7(n|2macRR_?yNP4k@*N`Sy`aLK^7?RNjJWqW#D^ zts9tK6&hgAZ%?!5;J;ykpfu5u0c?6_i*?GR0hkmhO$Y8AUZlGl!v<#cGxj4iK#|K6 zwA&);ABY#$G?daIZpZf1+esu!Hxe>@g(CMJ5i4xO6c*%VLY+^CCd(7^r~Hb|m?mCS z-f7i5{tLZmK?6jQnDiiSA&OMCHoQ$=8stg@3yxa+O{C7(I9%-Tl#W)oUk8*ZkJ^G` zvr^DKND;i^tCA<78AGm!_kA?LsaMJIR0?|Jal|bwnf2*I4~cmW6P#HtP8)R#zoFCGN1t@wR=A{LSz3p%y-2Ue-@%CK0tSJS*~$<1|!|1v++Y!9NeWB z%(usfBPPom+^-XMZB`j4WuGVrT!`&?VEuJMOkJylzZh0ztE0zjBX!%Bfn)+1195`f z{9YkId=QDbA;QB|hj2N$yBu2E1E(wm+b+4jaxuEZWMO8!ySblAWgA(#*gc^w5&=ql z;Sh*G{gg@;6`3wfQG)(rlLTRI9JGpMSd0+k;#e!ad3^h4XJq=|`BnScik%AL@b*WS z69<2bZ|F!|+#ZsV{)BdLB5XeHqyfCZL^DB8er&pf3nw|Ir zMO{{3h~4{{QW1t83=R&qJyLSQJnB;!YYKJG98j)aPB#;W*{9TMx?#7^e|O?X;^Px` zgyiJ48Lw_7HC6ur+)6hDgym4--`zs*>%Dr`KK@2sqAIh_(e(&YRZBdtGTa!SxGy+z z(LQKnXhx>t@M)jEoL+EEqDk*~MNap(!jKm~FvJ-9y6TrfIeRNEI>hQzND#4#6dYlQ zcimg30l1vOTszYTkT)9OUJmx_n;P1aL_JF|w$iAnY?s3zz)|IlY+bYMnZ zmqhAb$Fniqxi4t^v}lJiu{Vl15+ghnJ<(ok-0-i=L8pj-YO*vbq7^bHH-RsRF$4sq zfUG{m*VE|`o-Odv3up9Fe?GFV?i>N}rh{3rF9zm05>fzYa}3To$m6*X98Zge2R=W zXn>EvY6O{|1#8fj>jPzg{CI&zUc+tOmS`wTkNdfmH6HL*!RiHPTl`TkgU>en3=Pos z0P2hqm?~KYiaa?uavPZSCaCd`=|^%B39yQ4(L-XTUT70KF5*U>i%4LJu^Gff923l+ zL#kR;5W*wgs0W<+TJz4Icj+Uc;=a)f3ky>?KET1v6*3XhLS>C1aS-ox?Me|hJ<2q` zSj-w$H#Vi~{#G@pi?n-r1b+>E0wZ4K>P$_ZM2nuHz~tc(z{P1DlYH|Pw(@!Sr)6;m znH}yV;OAN3T3n3w zkdma&8kamF_)edAhT&NMO7`okgU^k$jBey&!Ga-{Z4J8Sc)!A;8 zU(jxE;+j!jDuCe>zpp)fSkyS-_pqaQx@mEa)3JKE>-71Q;;8QD+PwJGH?7uqM#tPv zi8vqA z<02a_tn}hO`rQmB^OpGg;rl1QGinjDLc80!B+cJXM&WOq2zndu`FT*wU3Xx^O>cDn zLG`ahFDf0?ZNSb++(+231=N_hnn*OBzvNL5Y%B!Qv$I@}oe(nsQGgQ_w(%1>p94O* z=Mw3qNah8%IahOayfKfjbnFIo=d+M`9lj4j#~>mby;>GnNooi@-SZ3ha4lh%tVaX9 z>>;O`Q7^^Ezd%fs&;T(~es>LKs zyC8^3e!gNsrgzTDe?O@$15YAZKDJ1Q#1vRU$j%b0IpiF2s@O&P5`euzN!}QTSeQnR zB|#HGrh$>+ckq0Fqj1!EuSkaV8~OeH@pub82l!x>02m#i7SRlg52|co%OoNAgE`Kp z5_#VG%piHGzc4(SQkwSAPHEdtp^!@|#b|@1TSxpVyB{7BBIciCZ_MzljuV7{SGkN(o++Vu&zg0>tKvlqIV)=c; z_2dti%LKG40StTDi_aO#eov{#px1!pbC0{#k}4wfh|sHBle>P-WjYr$#VSpOW1+1= znlwP8+`)5H2C+r5;{Uly5&>AJ7t*N_Gi|F&f~sQfcZU46F;5@*{F z#MyF9W4ghD5TO61TN zu$wf12WA>gh#05JMO>);Hm<0$Z86_tp15z3zThn*?Ys)SmR}S10AgAhjZtI-z5^rl zD^7y9D50LIsR(b?A8L(UtMrg_8BIqrkOuHIo6#j?<94zZxsnUCeGSaU#Cddl_}ERV=0oRH}G1sdQLzz&tZijpK)V>yunEhHYKPs$@sP>glAKmo%W{3aA zYYu(`oLSW4eZcTP;shsMh|qz3rvZdjNN1(wJiahGKhIvM4dCv9f8g~PO|1&mbHq(n zNk(C{K#s2xzh!P&7K$XQJCiFB3SA=%sU-HO?O@XRmt|in+Rcya ziw^G(g%%wQCo-IN6_Pu7L<6XvN|Y97NcUu1^qW<~vQyQGi)9ZDZu)4kv$463Fl1%l z*NSNG&`#8n>zCbq9prU`2H@Th+8VUm^>fSb+i@HKu31$c7hUff5lO8~$e6V7!;kOS z1J^;ED^@j{NgOXfeJ;Oq{kWv^@agvP*N=&E34}X|(tp$c^&jCv&;ZVn)`!69li^Ve z$oc2fzrcTi2YaY@;S(65q%HY8j1Wori1HZH#Ui?WZvDEyZ$6=LHSqW6LmBz*yfh}! zzxH~6$+%0c%{8*^?FAeQ&;X|ovtbL}CFg-nSi9;2Rl*wCXKVK*u%UGOGD$fk*a{cl zh61YQmu~D`kg9**FZU|td7zh7LMzDGA&K<(Hu&d7CTOHwH!iQS(){0?ar2~=~zp9+FnkRsdp>3C_{6;tg z;R|tmAt_RKyMf9VP!w6m9@?eoruR4{HsNf~Q=bDRjoEJZDx+{e*G$I)*2+E}C41a7mcclbsn4JMDd~YBN(FeBRwkndV186rYR>RZXSy=MlIB7oc97Y;57)RFFfe2yLibBQDM1NEGF&;>M zFH2A4$n-4ed&t)K7?EC*EbqUd0mf2adwxtLyHclFr9Zy>w^-??b$Tj2Tws_RvpH%X zI!iXgD|S#t@BShTxz9+6Cb~KH;4Xk(dFkg>TE9`>nE{86Bz5YZeO2xhen-JMsqzja=`E& zNa1=qQRhNJ$Ni=bb3eQdE6Z$vb^b-Y5d$7L^y+_OXZe1;{s_2^JkQPU#GfDHt6r|C z|0Oz6`t$J1kN=^d7`IZ1QpI}O>>JP1j>x8}+R^j#sN1N8cN+IJ5ZmhuflUab$1lAt zHF~4sZ0|SCJc|@87sVpEC@}~79yrd6>5U|(ssQPuHY*xnTJGR6^~|?IRXEA3qwI9= z7+Db-X!a?hF@QdtD0@myE3#^_OP>Z<>qPB&wm>}as*-ZRr5Y25$CqhfPb=tBX(Pdhk62j`J-iP}*Mgag=tl#zLcNvi27Db*4h zST2H+bCf_zo|PJt8KHk~s)g|viOwd;WnapV6cMcC8WgGmrx`0KdF`Zb8{X%;R4Nd7 zXx2-|v>p?$`R#usTyaZr0fATRITO_9JcNvQlOL}Z4o%jq1A_Cu(g4p}y^#}?IPD9l zXzF<&-v&QPQPbdB<8j=XhDV=u@dJ$nf3SJmaYQXUp`c((>u7J5UIkYl$0w0>*yE$B zRbP}lhD*}`^s55DIQ{Zl3eaFzYBvyP2p58kin7!b;dpTO`4G!HKOgh_CcGfraTu6N zHA=zq3(i)g3w~W=Fs{ba08?y)C!l)Z_Vte;EEe@%Dz~l#De_z!*6ulL@mM@?U%1$A zLvxS-pB->;@R9S}+S(oybiSRY$HQdx;2JN(cYuH5i6W0WM&NC~AUpC=y=zm6oYa8m zYp0D}M}iXq93OIhF$LW9tTJa6_q?NRuU)|FzC*)|*IxI!ctKSQSsv66ngvnM@-3N7 zW4<4O`KY`h?S6c}?E~E%U52r}dZBLTHm}VX4+$N*oXQklD02s1v^_8h^6?G#mB?Rd zHf8U%cvy0~e?y=3ZOEmR>)-?^Vs~p2>NYa-*a63x>pl8AxY*n2;^Vx?6qio@vzsf8 zA+Jh7Rq4ICNXYFT$_c)LWJ~x> zbf>Bi?`hhW&hED2mb;f$E=W#)tx4@Tvo_5UVGX|3eKVqlO!!o=MLFEZ(5JSEloczi z!%m9&iViN<373J+!niP;;CvDtOkf2_9a2Gl$B?If9OODO-mU%dIqq^zeaaaFl60ZLZ@I$&>KXq9 zmLDK@%sC=;l_NbQzDr(MJ{84OCtX2j95cAuQID23l4KWT7RM(l0@3quq9l}$XkXG< zpwJ^cnRi$T%E?6SdEF)1)0w@MB~k+2X8?Ps*?>h8vJd5*W)N;~a(1}6=|^$nIF?aj z$G5lfl|fUw1h^Z;x_Lt9PQkN?g?p%4#I`9G75~HTL*TUH5Hl@pjlJ-|3 zAKxIsVzNJ5cCOTf&lE!svZ33k*JQ*&O0Szl%b z3AQbZZX4?o=TTAC@BQ87DEL>oh>l2wdXG$jUEY1K@JvOCePUK0l>=Bv1a$>AX2r^w z=b<9wjq}hB5}qcbUG_AqjzQ(sT^TkgZu1>?N;4Cv_n(v9Lg^?Z8^LW z-Qv3Kb09Q4hBm89Z14VcCGO0mx{J;$J@c1;zW}h?!l*;gQ(F+05bnUuDBU4d0 z@v=YuX{^nQ=AuyfvJi};& zI{(6iRIa5kG72MJS=-u!T!f6EVL~@%MRH106k_ZJz}YIl*~A?~FD@&gNG2-FXbE>u`-sVQgy|8{T;UE`!4W3K`EYv}3`-mANkR93c$LHXU0+wd^ z7v>O;@lx6JgXOKacWHnv_=oTul2RElw&t z_rCq)mqc91%*~H)-5ej%5!A-Qcf3!!9U+D!D&u93+H1seFI*fMEA>0}dD^+X*5qVG zAGI^^pO~2<0tC;s)q@{*BI>}eri#28ja1mv>zTa@Ta9y`T6uO+M8q!Kc9_namVR#m zrf8q`)tEEjuXb+sy|PYZzz^PoHWC3R=Smy64vu)-0z+~oho(=^f%K*93O!T#`_#7>f&?*C>J8+E zgOBYw{#qPapWrT*sHi6k2#2#tZU ze{7hjYWESUWUI0i`_Kmz2jQ4DspB^yJy_h72lo?9;m`AnDYorMKioZXWjCi>Nk_zG zn|Bi4Q?cu9LA5q{-^2t^`|1U`;AU7gusw??Ly`^=SX`MvWCZyZ8u*vp)n|P)civwK z<}dyk^$K#A+73SDM)?T!_G69EE*Kp8)B{mG9Qt)5A21aUWHbq9gs;J?KVJy)>wqo-oaV&e7`+T*t|~KZ~Fo7DK$+vjq>p z+b%Md_KckI`sC=*X5Ki#BR()vQa!hZ>4VOPi}i=w&RB~~R8<{*es9|_I>bKuF6Y7n zrRKEgV$m~V(HvpZ{C)|&0p1TcZnbUydV(Hu41F3w1MuQ@xac4}c9JsuX=C=;wLUqk z$mn?cC-ueVTIx&&K`Wz2pq8vhbU5u*F_85$u%?ZFyQMc4S0lQxp!O&xA44Z(a28;q z)MN`uA!KX`YtS(y=>vqDGutB3TYw-VJK0GSe^kS`O^U;rVsLy;)@vmaQ=y|CtzIk!65B zs7#V;bl{nFyffMieAaQp24$jYzL3gHoSCvCqS|L5r9!D`epNiCu$}i8z-3>~n^RIi z9$Ln_$`M)*9~PK46g`Wf9`pPKvBovRIsNguC*M5#pkeD&R)gJ8@R-zL>m(g7H;LXxV@8@uYJz0s)|Iq z4yrD5zED`xdWS}@kk#JehSZYaDszyz8KW@yEG$VGPZh_jy@|L}PRV}tY4LtoxU5l= zS-%-JeJ4FRyoyVEOy!L&c*3_%i_5fNkFXaQH#RmYMyE47ne?nzKBNN}(@i1t@Ykk- za_*%8T;Xv_AxYhn)u0qjBOdLq8@iFv4Equ}*A^DY!v^d`;aE#EN&;vh2g(L={eZ<6 zG-*O4@U>m>lch1|rgjABSfmV}W!ByQI`ZpuRir#P&7V%SpBuTfNB)L#hK4!ZoOR7I ztU8=PzLtkR!)3H<iK}L2aCeR`j>F5zowPd!HtrDT!0@a^Zz2k&QB^@vUdIfS&$v*=}p?6Va1 zI!`Sn9^b-xY_;H17b9v+DngtbNW4-)@85m$Y&iMK_#FEECE4fjA|Z~FuB{CjIGObV z3uv+gcs&_=S2yV!mD9aBK9H?Hmp0rzv(%ip?@bB5w%8i~y8N@_+2R$#!noN4eQTZN z7c3r!d7bC8xNWvIXsxF4Y<$|<)L7IU%4_TTvuGNawS4ZG9^Z);HZ>N^*6O{jme|Tr zJl~HA!}{>6!8km8K6FJbfZvWjUGgp z6bC6WOa}>z6>q@~?bflVmcAQDO*&MTc0M2dABQmR4Rg;E=1l+aU}YCMpNIAa0b9K8 z$hC3QNEiI-Um=-Fhl?ciitS?5hV}plzzC1(^Nrw!xAK^fmC@ zL3R}q=p~@@#Wl@1-^CS#-{#)Q2DbYDaMPK4jtY`W$2~QXO^>Kss)Tx*9cD}`_5q`9 zyZph4dDdqJjxsAe)5mxhiF@t08g?)aeVd2DWNsM2wuxwsi{4#YOF9_muU&fh!^=z7 zd1375j_&pvXjg0Wiq__~Y|7;1WcTDPIXTb8IKg&Zc0)e%+ut9s02sdWNkFANnvi1J zLlsMvergD}8nA@hS`%fdeePDw;=SlN+qg-UiCn6emYdYZEW{y@o{zR>{66*H%ouCV zPRz%UBK>CE?rW7K9}?S?(8U3{UVg*UH*pt49-jN2(v9S&lzZh5lIRbY=JY z;`Aq%wT_vo(9bcVql5HBp5T@NKg}t(Yb-b07m}pDcryHnx$@1CL)S@Z=}6EhOu2JT zq$3X;XBA+~T7tabT>`}Q*}-pwYnQ{4wN>f#lqF;i$<`PMj={{HlWG6;JEy*mzWH=l zp$6h9mj?Q3yPZ=T%0lqMzt~z5Gkgxx&Lm~)L1GNLuTrno8}lxcW=v>+ryk1>Q;T3{ zf0l=-SByI5#*jk=mICmW9UvV2?rdq7lI%pXg z3ramXNE^X}u)%TUTTeUvQo7twjo?RL#8UU(qBpGhdXn)ziT>R668B*_Nq+R&?m0N> z_X@^fhpBDIgj=@0wL~S0Y9xpve>iqnNF{S05e$juS2f^Cfw{Xv8`saxdnC96-E0Vm zWZ7#pfV-|lq)yTAW>9=!qY8*?U3F5`W;C{;wl>NoQ9^`1yR2F-D^+aU-0g;a@~j9S zX2!}+9Dje~A6QO1L7H@ga&e9l?A(QTvz)9lAT{$~)pmNdag9Ll|4n+b&I~6N5tB?{ zx_IUc-$=KB3nsa@difugSWegpp6i%OJccgO5%@<>_|+MBY+ddylebd(yhr>epqu*< z;;j;$iEvjIiL}%U3w;`pvwV0HXI)}?SUY-B1shWC9B%ZxjFr_MQ0}->;SeA`bCA^q z%rIDicfyF5vwF6fjiiXq_*)&2D~Ly1DaUSK8~yGnh^OwqwM`AlrJjY70*Ute4yc#G z2t)0yX7?hm=YnBT!X>C{9FU1oaCi)Tw4dSYt?!rSCeb%eCH%or$NG;CAPbLN=gN$$ zfYL*BjyR*6eoGR>-m}F=a<7}eP05LHY*FtU6XqHz37S9q-?&m~yz3W(nZjM~aNG>5 zuVaN-agIR@eF$NRTUAT^P<8u!R{zLt`o0uGY`B9WOuV0OGTCL5zLh7iL=S{)Kql%C z40qRzImGe-0uS1jTMESU{`l&@xP4AjRWII4(znJK*KudopwFvY^<+OJgTahksFoQ4o7~b1jjYr tidQ#a|Gg5(cZ+N>(GSLEv_P_Ie!n%> 0: + with pose_result_queue_mutex: + _result = pose_result_queue.popleft() + _ts_input, t_info, pose_results_list = _result + + _ts = time.time() + if ts_inference is not None: + fps_inference = 1.0 / (_ts - ts_inference) + ts_inference = _ts + t_delay_inference = (_ts - _ts_input) * 1000 + + # visualize detection and pose results + if pose_results_list is not None: + for model_info, pose_results in zip(pose_model_list, + pose_results_list): + pose_model = model_info['model'] + bbox_color = model_info['bbox_color'] + + dataset_name = pose_model.cfg.data['test']['type'] + + # show pose results + if args.show_pose: + img = vis_pose_result( + pose_model, + img, + pose_results, + radius=4, + thickness=2, + dataset=dataset_name, + kpt_score_thr=args.kpt_thr, + bbox_color=bbox_color) + + # sunglasses effect + if args.sunglasses: + if dataset_name == 'TopDownCocoDataset': + left_eye_idx = 1 + right_eye_idx = 2 + elif dataset_name == 'AnimalPoseDataset': + left_eye_idx = 0 + right_eye_idx = 1 + else: + raise ValueError( + 'Sunglasses effect does not support' + f'{dataset_name}') + if sunglasses_img is None: + # The image attributes to: + # https://www.vecteezy.com/free-vector/glass + # Glass Vectors by Vecteezy + sunglasses_img = cv2.imread( + 'demo/resources/sunglasses.jpg') + img = apply_sunglasses_effect(img, pose_results, + sunglasses_img, + left_eye_idx, + right_eye_idx) + # bug-eye effect + if args.bugeye: + if dataset_name == 'TopDownCocoDataset': + left_eye_idx = 1 + right_eye_idx = 2 + elif dataset_name == 'AnimalPoseDataset': + left_eye_idx = 0 + right_eye_idx = 1 + else: + raise ValueError('Bug-eye effect does not support' + f'{dataset_name}') + img = apply_bugeye_effect(img, pose_results, + left_eye_idx, right_eye_idx) + + # delay control + if args.display_delay > 0: + t_sleep = args.display_delay * 0.001 - (time.time() - ts_input) + if t_sleep > 0: + time.sleep(t_sleep) + t_delay = (time.time() - ts_input) * 1000 + + # show time information + t_info_display = stop_watch.report_strings() # display fps + t_info_display.append(f'Inference FPS: {fps_inference:>5.1f}') + t_info_display.append(f'Delay: {t_delay:>3.0f}') + t_info_display.append( + f'Inference Delay: {t_delay_inference:>3.0f}') + t_info_str = ' | '.join(t_info_display + t_info) + cv2.putText(img, t_info_str, (20, 20), cv2.FONT_HERSHEY_DUPLEX, + 0.3, text_color, 1) + # collect system information + sys_info = [ + f'RES: {img.shape[1]}x{img.shape[0]}', + f'Buffer: {frame_buffer.qsize()}/{frame_buffer.maxsize}' + ] + if psutil_proc is not None: + sys_info += [ + f'CPU: {psutil_proc.cpu_percent():.1f}%', + f'MEM: {psutil_proc.memory_percent():.1f}%' + ] + sys_info_str = ' | '.join(sys_info) + cv2.putText(img, sys_info_str, (20, 40), cv2.FONT_HERSHEY_DUPLEX, + 0.3, text_color, 1) + + # save the output video frame + if args.out_video_file is not None: + if vid_out is None: + fourcc = cv2.VideoWriter_fourcc(*'mp4v') + fps = args.out_video_fps + frame_size = (img.shape[1], img.shape[0]) + vid_out = cv2.VideoWriter(args.out_video_file, fourcc, fps, + frame_size) + + vid_out.write(img) + + # display + cv2.imshow('mmpose webcam demo', img) + keyboard_input = cv2.waitKey(1) + if keyboard_input in (27, ord('q'), ord('Q')): + break + elif keyboard_input == ord('s'): + args.sunglasses = not args.sunglasses + elif keyboard_input == ord('b'): + args.bugeye = not args.bugeye + elif keyboard_input == ord('v'): + args.show_pose = not args.show_pose + + cv2.destroyAllWindows() + if vid_out is not None: + vid_out.release() + event_exit.set() + + +def main(): + global args + global frame_buffer + global input_queue, input_queue_mutex + global det_result_queue, det_result_queue_mutex + global pose_result_queue, pose_result_queue_mutex + global det_model, pose_model_list, pose_history_list + global event_exit, event_inference_done + + args = parse_args() + + assert has_mmdet, 'Please install mmdet to run the demo.' + assert args.det_config is not None + assert args.det_checkpoint is not None + + # build detection model + det_model = init_detector( + args.det_config, args.det_checkpoint, device=args.device.lower()) + + # build pose models + pose_model_list = [] + if args.enable_human_pose: + pose_model = init_pose_model( + args.human_pose_config, + args.human_pose_checkpoint, + device=args.device.lower()) + model_info = { + 'name': 'HumanPose', + 'model': pose_model, + 'cat_ids': args.human_det_ids, + 'bbox_color': (148, 139, 255), + } + pose_model_list.append(model_info) + if args.enable_animal_pose: + pose_model = init_pose_model( + args.animal_pose_config, + args.animal_pose_checkpoint, + device=args.device.lower()) + model_info = { + 'name': 'AnimalPose', + 'model': pose_model, + 'cat_ids': args.animal_det_ids, + 'bbox_color': 'cyan', + } + pose_model_list.append(model_info) + + # store pose history for pose tracking + pose_history_list = [] + for _ in range(len(pose_model_list)): + pose_history_list.append({'pose_results_last': [], 'next_id': 0}) + + # frame buffer + if args.buffer_size > 0: + buffer_size = args.buffer_size + else: + # infer buffer size from the display delay time + # assume that the maximum video fps is 30 + buffer_size = round(30 * (1 + max(args.display_delay, 0) / 1000.)) + frame_buffer = Queue(maxsize=buffer_size) + + # queue of input frames + # element: (timestamp, frame) + input_queue = deque(maxlen=1) + input_queue_mutex = Lock() + + # queue of detection results + # element: tuple(timestamp, frame, time_info, det_results) + det_result_queue = deque(maxlen=1) + det_result_queue_mutex = Lock() + + # queue of detection/pose results + # element: (timestamp, time_info, pose_results_list) + pose_result_queue = deque(maxlen=1) + pose_result_queue_mutex = Lock() + + try: + event_exit = Event() + event_inference_done = Event() + t_input = Thread(target=read_camera, args=()) + t_det = Thread(target=inference_detection, args=(), daemon=True) + t_pose = Thread(target=inference_pose, args=(), daemon=True) + + t_input.start() + t_det.start() + t_pose.start() + + # run display in the main thread + display() + # join the input thread (non-daemon) + t_input.join() + + except KeyboardInterrupt: + pass + + +if __name__ == '__main__': + main() diff --git a/mmpose/apis/inference.py b/mmpose/apis/inference.py index 550c4e3cd66..b6668070ad9 100644 --- a/mmpose/apis/inference.py +++ b/mmpose/apis/inference.py @@ -520,6 +520,7 @@ def vis_pose_result(model, radius=4, thickness=1, kpt_score_thr=0.3, + bbox_color='green', dataset='TopDownCocoDataset', show=False, out_file=None): @@ -740,8 +741,8 @@ def vis_pose_result(model, [14, 18], [7, 11], [11, 15], [15, 19], [7, 12], [12, 16], [16, 20]] - pose_limb_color = palette[[0] * 20] - pose_kpt_color = palette[[0] * 20] + pose_kpt_color = palette[[15] * 5 + [0] * 7 + [9] * 8] + pose_limb_color = palette[[15] * 5 + [0] * 3 + [0, 9, 9] * 4] else: raise NotImplementedError() @@ -755,6 +756,7 @@ def vis_pose_result(model, pose_kpt_color=pose_kpt_color, pose_limb_color=pose_limb_color, kpt_score_thr=kpt_score_thr, + bbox_color=bbox_color, show=show, out_file=out_file) diff --git a/mmpose/core/visualization/__init__.py b/mmpose/core/visualization/__init__.py index d5e0f781701..eda8d48f9d1 100644 --- a/mmpose/core/visualization/__init__.py +++ b/mmpose/core/visualization/__init__.py @@ -1,3 +1,7 @@ -from .image import imshow_keypoints, imshow_keypoints_3d +from .effects import apply_bugeye_effect, apply_sunglasses_effect +from .image import imshow_bboxes, imshow_keypoints, imshow_keypoints_3d -__all__ = ['imshow_keypoints', 'imshow_keypoints_3d'] +__all__ = [ + 'imshow_keypoints', 'imshow_keypoints_3d', 'imshow_bboxes', + 'apply_bugeye_effect', 'apply_sunglasses_effect' +] diff --git a/mmpose/core/visualization/effects.py b/mmpose/core/visualization/effects.py new file mode 100644 index 00000000000..8711f1b52d3 --- /dev/null +++ b/mmpose/core/visualization/effects.py @@ -0,0 +1,110 @@ +import cv2 +import numpy as np + + +def apply_bugeye_effect(img, + pose_results, + left_eye_index, + right_eye_index, + kpt_thr=0.5): + """Apply bug-eye effect. + + Args: + img (np.ndarray): Image data. + pose_results (list[dict]): The pose estimation results containing: + - "bbox" ([K, 4(or 5)]): detection bbox in + [x1, y1, x2, y2, (score)] + - "keypoints" ([K,3]): keypoint detection result in [x, y, score] + left_eye_index (int): Keypoint index of left eye + right_eye_index (int): Keypoint index of right eye + kpt_thr (float): The score threshold of required keypoints. + """ + + xx, yy = np.meshgrid(np.arange(img.shape[1]), np.arange(img.shape[0])) + xx = xx.astype(np.float32) + yy = yy.astype(np.float32) + + for pose in pose_results: + bbox = pose['bbox'] + kpts = pose['keypoints'] + + if kpts[left_eye_index, 2] < kpt_thr or kpts[right_eye_index, + 2] < kpt_thr: + continue + + kpt_leye = kpts[left_eye_index, :2] + kpt_reye = kpts[right_eye_index, :2] + for xc, yc in [kpt_leye, kpt_reye]: + + # distortion parameters + k1 = 0.001 + epe = 1e-5 + + scale = (bbox[2] - bbox[0])**2 + (bbox[3] - bbox[1])**2 + r2 = ((xx - xc)**2 + (yy - yc)**2) + r2 = (r2 + epe) / scale # normalized by bbox scale + + xx = (xx - xc) / (1 + k1 / r2) + xc + yy = (yy - yc) / (1 + k1 / r2) + yc + + img = cv2.remap( + img, + xx, + yy, + interpolation=cv2.INTER_AREA, + borderMode=cv2.BORDER_REPLICATE) + return img + + +def apply_sunglasses_effect(img, + pose_results, + sunglasses_img, + left_eye_index, + right_eye_index, + kpt_thr=0.5): + """Apply sunglasses effect. + + Args: + img (np.ndarray): Image data. + pose_results (list[dict]): The pose estimation results containing: + - "keypoints" ([K,3]): keypoint detection result in [x, y, score] + sunglasses_img (np.ndarray): Sunglasses image with white background. + left_eye_index (int): Keypoint index of left eye + right_eye_index (int): Keypoint index of right eye + kpt_thr (float): The score threshold of required keypoints. + """ + + hm, wm = sunglasses_img.shape[:2] + # anchor points in the sunglasses mask + pts_src = np.array([[0.3 * wm, 0.3 * hm], [0.3 * wm, 0.7 * hm], + [0.7 * wm, 0.3 * hm], [0.7 * wm, 0.7 * hm]], + dtype=np.float32) + + for pose in pose_results: + kpts = pose['keypoints'] + + if kpts[left_eye_index, 2] < kpt_thr or kpts[right_eye_index, + 2] < kpt_thr: + continue + + kpt_leye = kpts[left_eye_index, :2] + kpt_reye = kpts[right_eye_index, :2] + # orthogonal vector to the left-to-right eyes + vo = 0.5 * (kpt_reye - kpt_leye)[::-1] * [-1, 1] + + # anchor points in the image by eye positions + pts_tar = np.vstack( + [kpt_reye + vo, kpt_reye - vo, kpt_leye + vo, kpt_leye - vo]) + + h_mat, _ = cv2.findHomography(pts_src, pts_tar) + patch = cv2.warpPerspective( + sunglasses_img, + h_mat, + dsize=(img.shape[1], img.shape[0]), + borderValue=(255, 255, 255)) + # mask the white background area in the patch with a threshold 200 + mask = cv2.cvtColor(patch, cv2.COLOR_BGR2GRAY) + mask = (mask < 200).astype(np.uint8) + img = cv2.copyTo(patch, mask, img) + + return img diff --git a/mmpose/core/visualization/image.py b/mmpose/core/visualization/image.py index ba9ddc4e95c..09bbacc79ba 100644 --- a/mmpose/core/visualization/image.py +++ b/mmpose/core/visualization/image.py @@ -6,6 +6,82 @@ from matplotlib import pyplot as plt +def imshow_bboxes(img, + bboxes, + labels=None, + colors='green', + text_color='white', + thickness=1, + font_scale=0.5, + show=True, + win_name='', + wait_time=0, + out_file=None): + """Draw bboxes with labels (optional) on an image. This is a wrapper of + mmcv.imshow_bboxes. + + Args: + img (str or ndarray): The image to be displayed. + bboxes (ndarray): ndarray of shape (k, 4), each row is a bbox in + format [x1, y1, x2, y2]. + labels (str or list[str], optional): labels of each bbox. + colors (list[str or tuple or :obj:`Color`]): A list of colors. + text_color (str or tuple or :obj:`Color`): Color of texts. + thickness (int): Thickness of lines. + font_scale (float): Font scales of texts. + show (bool): Whether to show the image. + win_name (str): The window name. + wait_time (int): Value of waitKey param. + out_file (str, optional): The filename to write the image. + + Returns: + ndarray: The image with bboxes drawn on it. + """ + + # adapt to mmcv.imshow_bboxes input format + bboxes = np.split(bboxes, bboxes.shape[0], axis=0) + if not isinstance(colors, list): + colors = [colors for _ in range(len(bboxes))] + colors = [mmcv.color_val(c) for c in colors] + assert len(bboxes) == len(colors) + + img = mmcv.imshow_bboxes( + img, + bboxes, + colors, + top_k=-1, + thickness=thickness, + show=False, + out_file=None) + + if labels is not None: + if not isinstance(labels, list): + labels = [labels for _ in range(len(bboxes))] + assert len(labels) == len(bboxes) + + for bbox, label, color in zip(bboxes, labels, colors): + bbox_int = bbox[0, :4].astype(np.int32) + # roughly estimate the proper font size + text_size, text_baseline = cv2.getTextSize(label, + cv2.FONT_HERSHEY_DUPLEX, + font_scale, thickness) + text_x1 = bbox_int[0] + text_y1 = max(0, bbox_int[1] - text_size[1] - text_baseline) + text_x2 = bbox_int[0] + text_size[0] + text_y2 = text_y1 + text_size[1] + text_baseline + cv2.rectangle(img, (text_x1, text_y1), (text_x2, text_y2), color, + cv2.FILLED) + cv2.putText(img, label, (text_x1, text_y2 - text_baseline), + cv2.FONT_HERSHEY_DUPLEX, font_scale, + mmcv.color_val(text_color), thickness) + + if show: + mmcv.imshow(img, win_name, wait_time) + if out_file is not None: + mmcv.imwrite(img, out_file) + return img + + def imshow_keypoints(img, pose_result, skeleton=None, diff --git a/mmpose/models/detectors/pose_lifter.py b/mmpose/models/detectors/pose_lifter.py index 81ca2362686..effc405607e 100644 --- a/mmpose/models/detectors/pose_lifter.py +++ b/mmpose/models/detectors/pose_lifter.py @@ -3,7 +3,7 @@ import mmcv import numpy as np -from mmpose.core import imshow_keypoints, imshow_keypoints_3d +from mmpose.core import imshow_bboxes, imshow_keypoints, imshow_keypoints_3d from .. import builder from ..builder import POSENETS from .base import BasePose @@ -345,11 +345,10 @@ def show_result(self, if len(bbox_result) > 0: bboxes = np.vstack(bbox_result) - mmcv.imshow_bboxes( + imshow_bboxes( img, bboxes, colors='green', - top_k=-1, thickness=thickness, show=False) if len(pose_input_2d) > 0: diff --git a/mmpose/models/detectors/top_down.py b/mmpose/models/detectors/top_down.py index 7134df29787..8aa26bb9e72 100644 --- a/mmpose/models/detectors/top_down.py +++ b/mmpose/models/detectors/top_down.py @@ -5,7 +5,7 @@ from mmcv.image import imwrite from mmcv.visualization.image import imshow -from mmpose.core import imshow_keypoints +from mmpose.core import imshow_bboxes, imshow_keypoints from .. import builder from ..builder import POSENETS from .base import BasePose @@ -222,10 +222,11 @@ def show_result(self, bbox_color='green', pose_kpt_color=None, pose_limb_color=None, - text_color=(255, 0, 0), + text_color='white', radius=4, thickness=1, font_scale=0.5, + bbox_thickness=1, win_name='', show=False, show_keypoint_weight=False, @@ -264,7 +265,6 @@ def show_result(self, img = mmcv.imread(img) img = img.copy() - img_h, img_w, _ = img.shape bbox_result = [] pose_result = [] @@ -274,13 +274,18 @@ def show_result(self, if len(bbox_result) > 0: bboxes = np.vstack(bbox_result) + labels = None + if 'label' in result[0]: + labels = [res['label'] for res in result] # draw bounding boxes - mmcv.imshow_bboxes( + imshow_bboxes( img, bboxes, + labels=labels, colors=bbox_color, - top_k=-1, - thickness=thickness, + text_color=text_color, + thickness=bbox_thickness, + font_scale=font_scale, show=False) imshow_keypoints(img, pose_result, skeleton, kpt_score_thr, diff --git a/mmpose/utils/__init__.py b/mmpose/utils/__init__.py index ac489e2dbbc..860a2d4a574 100644 --- a/mmpose/utils/__init__.py +++ b/mmpose/utils/__init__.py @@ -1,4 +1,5 @@ from .collect_env import collect_env from .logger import get_root_logger +from .timer import StopWatch -__all__ = ['get_root_logger', 'collect_env'] +__all__ = ['get_root_logger', 'collect_env', 'StopWatch'] diff --git a/mmpose/utils/timer.py b/mmpose/utils/timer.py new file mode 100644 index 00000000000..2e499dcbd2f --- /dev/null +++ b/mmpose/utils/timer.py @@ -0,0 +1,92 @@ +from collections import defaultdict + +import numpy as np +from mmcv import Timer + + +class StopWatch: + r"""A helper class to measure FPS and detailed time consuming of each phase + in a video processing loop or similar scenarios. + + Args: + window (int): The sliding window size to calculate the running average + of the time consuming. + + Example:: + >>> stop_watch = StopWatch(window=10) + >>> while True: + ... with stop_watch.timeit('total'): + ... sleep(1) + ... # 'timeit' support nested use + ... with stop_watch.timeit('phase1'): + ... sleep(1) + ... with stop_watch.timeit('phase2'): + ... sleep(2) + ... sleep(2) + ... report = stop_watch.report() + report = {'total': 6., 'phase1': 1., 'phase2': 2.} + + """ + + def __init__(self, window=1): + self._record = defaultdict(list) + self._timer_stack = [] + self.window = window + + def timeit(self, timer_name='_FPS_'): + """Timing a code snippet with an assigned name. + + Args: + timer_name (str): The unique name of the interested code snippet to + handle multiple timers and generate reports. Note that '_FPS_' + is a special key that the measurement will be in `fps` instead + of `millisecond`. Also see `report` and `report_strings`. + Default: '_FPS_'. + Note: + This function should always be used in a `with` statement, as shown + in the example. + """ + self._timer_stack.append((timer_name, Timer())) + return self + + def __enter__(self): + pass + + def __exit__(self, exc_type, exc_value, trackback): + timer_name, timer = self._timer_stack.pop() + self._record[timer_name].append(timer.since_start()) + self._record[timer_name] = self._record[timer_name][-self.window:] + + def report(self): + """Report timing information. + + Returns: + dict: The key is the timer name and the value is the corresponding + average time consuming. + """ + result = { + name: np.mean(vals) * 1000. + for name, vals in self._record.items() + } + return result + + def report_strings(self): + """Report timing information in texture strings. + + Returns: + list(str): Each element is the information string of a timed event, + in format of '{timer_name}: {time_in_ms}'. Specially, if + timer_name is '_FPS_', the result will be converted to + fps. + """ + result = self.report() + strings = [] + if '_FPS_' in result: + fps = 1000. / result.pop('_FPS_') + strings.append(f'FPS: {fps:>5.1f}') + strings += [f'{name}: {val:>3.0f}' for name, val in result.items()] + return strings + + def reset(self): + self._record = defaultdict(list) + self._timer_stack = [] diff --git a/tests/test_regularization.py b/tests/test_regularization.py index b23b86f1c74..3de10aceef7 100644 --- a/tests/test_regularization.py +++ b/tests/test_regularization.py @@ -15,4 +15,4 @@ def test_weight_norm_clip(): _ = module(x) weight_norm = module.weight.norm().item() - np.testing.assert_allclose(weight_norm, 1.0, rtol=1e-6) + np.testing.assert_almost_equal(weight_norm, 1.0, decimal=6) diff --git a/tests/test_utils.py b/tests/test_utils.py index 7057dbedb04..67fa096cb8f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,10 +1,13 @@ +import time + import cv2 import mmcv +import numpy as np import torch import torchvision import mmpose -from mmpose.utils import collect_env +from mmpose.utils import StopWatch, collect_env def test_collect_env(): @@ -15,3 +18,25 @@ def test_collect_env(): assert env_info['MMCV'] == mmcv.__version__ assert '+' in env_info['MMPose'] assert mmpose.__version__ in env_info['MMPose'] + + +def test_stopwatch(): + window_size = 5 + test_loop = 10 + outer_time = 100 + inner_time = 100 + + stop_watch = StopWatch(window=window_size) + for _ in range(test_loop): + with stop_watch.timeit(): + time.sleep(outer_time / 1000.) + with stop_watch.timeit('inner'): + time.sleep(inner_time / 1000.) + + report = stop_watch.report() + _ = stop_watch.report_strings() + + np.testing.assert_allclose( + report['_FPS_'], outer_time + inner_time, rtol=0.01) + + np.testing.assert_allclose(report['inner'], inner_time, rtol=0.01) diff --git a/tests/test_visualization.py b/tests/test_visualization.py index 8403713adff..927afaafb48 100644 --- a/tests/test_visualization.py +++ b/tests/test_visualization.py @@ -1,6 +1,10 @@ +import tempfile + +import mmcv import numpy as np -from mmpose.core import imshow_keypoints, imshow_keypoints_3d +from mmpose.core import (apply_bugeye_effect, apply_sunglasses_effect, + imshow_bboxes, imshow_keypoints, imshow_keypoints_3d) def test_imshow_keypoints(): @@ -29,3 +33,48 @@ def test_imshow_keypoints(): pose_kpt_color=pose_kpt_color, pose_limb_color=pose_limb_color, vis_height=400) + + +def test_imshow_bbox(): + img = np.zeros((100, 100, 3), dtype=np.uint8) + bboxes = np.array([[10, 10, 30, 30], [10, 50, 30, 80]], dtype=np.float32) + labels = ['label 1', 'label 2'] + colors = ['red', 'green'] + + with tempfile.TemporaryDirectory() as tmpdir: + _ = imshow_bboxes( + img, + bboxes, + labels=labels, + colors=colors, + show=False, + out_file=f'{tmpdir}/out.png') + + +def test_effects(): + img = np.zeros((100, 100, 3), dtype=np.uint8) + kpts = np.array([[10., 10., 0.8], [20., 10., 0.8]], dtype=np.float32) + bbox = np.array([0, 0, 50, 50], dtype=np.float32) + pose_results = [dict(bbox=bbox, keypoints=kpts)] + # sunglasses + sunglasses_img = mmcv.imread('demo/resources/sunglasses.jpg') + _ = apply_sunglasses_effect( + img, + pose_results, + sunglasses_img, + left_eye_index=1, + right_eye_index=0, + kpt_thr=0.5) + _ = apply_sunglasses_effect( + img, + pose_results, + sunglasses_img, + left_eye_index=1, + right_eye_index=0, + kpt_thr=0.9) + + # bug-eye + _ = apply_bugeye_effect( + img, pose_results, left_eye_index=1, right_eye_index=0, kpt_thr=0.5) + _ = apply_bugeye_effect( + img, pose_results, left_eye_index=1, right_eye_index=0, kpt_thr=0.9)