Face Landmark Detection

[hafedh-trimeche]

Hi,

Would you please integrate Face Landmark Detection into your invaluable library?

• https://docs.opencv.org/4.x/d2/d42/tutorial_face_landmark_detection_in_an_image.html
• https://docs.opencv.org/4.6.0/dc/d63/classcv_1_1face_1_1FacemarkLBF.html
• https://github.com/spmallick/learnopencv/tree/master/FacialLandmarkDetection

Best regards.

[gidesa]

Hello, I am sorry, the face landmark is another extra module, so it's not in the official binaries.
Maybe in future I will try to compile the extra modules DLL, and add a specialized wrapper for it.
Regards

[hafedh-trimeche]

Hello,

This code is used to compare these faces from the same person (ME) [Images resized to 160x160]

function TOpenCVInstance.RecognizeFace(RefFace:TBytes;Gallery:array of TBytes;out CosineScore,NormDistance:Double):Integer;
label
  Clear;
const
  Color3Channels  = COLOR_BGR2RGB;
  CosineThreshold = 0.363;
  L2NormThreshold = 1.128;
var
  OK           : Boolean;
  Photo        : TBytes;
  LWidth       : Integer;
  LHeight      : Integer;
  FaceMat      : PCvMat_t;
  GalleryMat   : PCvMat_t;
  iLabel       ,
  CosineLabel  ,
  NormL2Label  : Integer;
  MaxCosine    ,
  MinL2        ,
  Value        : Double;
var
  ProfileID : string;
begin
  ProfileID := StartProfiler(ClassName,'RecognizeFace');
  CosineLabel  := -1;
  NormL2Label  := -1;
  FaceMat      := nil;
  GalleryMat   := nil;
  CosineScore  := -1;
  NormDistance := -1;
  MaxCosine    := 0;
  MinL2        := MaxWord;
  if RefFace=nil then goto Clear;
  Photo := RefFace;
  ExtractPhoto(Photo,RefFace);
  FaceMat := LoadMat(RefFace);
  if IsEmpty(FaceMat) then goto Clear;
  for iLabel:=0 to Length(Gallery)-1 do
  begin
    Photo := Gallery[iLabel];
    ExtractPhoto(Photo,Gallery[iLabel]);
  end;
  LWidth  := pCvMatGetWidth(FaceMat);
  LHeight := pCvMatGetHeight(FaceMat);
  OK      := True;
  try
    if (LWidth<>FaceImageWidth) or (LHeight<>FaceImageHeight) then
    begin
      pCvresize(FaceMat,FaceMat,FaceInputSize,0,0,Ord(INTER_RESIZE));
    end;
    pCvcvtColor(FaceMat,FaceMat,Ord(Color3Channels),3);
    pCvFaceRecognizerSFfeature(FRecognizer,FaceMat,FWorkFeatures);
    pCvMatCopy(FWorkFeatures,FFaceFeatures);
  except
    OK := False;
  end;
  if (not OK) then goto Clear;
  for iLabel:=0 to Length(Gallery)-1 do
  begin
    GalleryMat := LoadMat(Gallery[iLabel]);
    if IsEmpty(GalleryMat) then Continue;
    LWidth  := pCvMatGetWidth(GalleryMat);
    LHeight := pCvMatGetHeight(GalleryMat);
    OK      := True;
    try
      if (LWidth<>FaceImageWidth) or (LHeight<>FaceImageHeight) then
      begin
        pCvresize(GalleryMat,GalleryMat,FaceInputSize,0,0,Ord(INTER_RESIZE));
      end;
      pCvcvtColor(GalleryMat,GalleryMat,Ord(Color3Channels),3);
      pCvFaceRecognizerSFfeature(FRecognizer,GalleryMat,FWorkFeatures);
      pCvMatCopy(FWorkFeatures,FGalleryFeatures);
    except
      OK := False;
    end;
    if (not OK) then Continue;
    Value := pCvFaceRecognizerSFmatch(FRecognizer,FFaceFeatures,FGalleryFeatures,Ord(FR_COSINE));
    if Value>MaxCosine then MaxCosine := Value;
    OK := (Value>=CosineThreshold) and (Value>=MaxCosine);
    if OK then
    begin
      CosineScore := Value;
      CosineLabel := iLabel;
    end;
    Value := pCvFaceRecognizerSFmatch(FRecognizer,FFaceFeatures,FGalleryFeatures,Ord(FR_NORM_L2));
    if Value<MinL2 then MinL2 := Value;
    OK := (Value<=L2NormThreshold) and (Value<=MinL2);
    if OK then
    begin
      NormDistance := Value;
      NormL2Label  := iLabel;
    end;
  end;
Clear:
  if CosineScore<0 then CosineScore := MaxCosine;
  CosineScore := 100*(1-CosineScore);
  if NormDistance<0 then NormDistance := MinL2;
  if CosineLabel=-1 then Result := NormL2Label
                    else Result := CosineLabel;
  if Assigned(GalleryMat) then pCvMatDelete(GalleryMat);
  if Assigned(FaceMat)    then pCvMatDelete(FaceMat);
  StopProfiler(ProfileID);
end;

but

Cosine dissimilarity ( (1-Score)*100) is too high: 53.45620
Norm Distance: 1.03398

Would you please advise me how to achieve best similarity scroe?

• Input image size (Resolution)
• Which features to include
• Should input images be resized

and if possible test the code by yourself.

Thank you again!

[gidesa]

Hello, I have tested your images with my code (the example project1), not having all your source.
I extract the faces by th DNN net, not by the haar algorithm. The images need to be in RGB (16 million colors), so I saved your images from gray to RGB. No resize, no other manipulation. Then I create the faces db with createfacesDB program.
At end in test program the cosine similarity is good, 0,6.
Attached faces db data and screenshot of result.
facesImg.zip

Regards

[hafedh-trimeche]

Hi,

Thank you for your prompt response.

The face images should be provided by an external module. So, only recognition should be used.

The conversion to RGB 16 is made by:
pCvcvtColor(GalleryMat,GalleryMat,Ord(Color3Channels),3);
Is it possible to only use Recognition Model (without Detection one) because of faces are already detected and Photo portrait not provided?

The Detection code used:

function TOpenCVInstance.ExtractPhoto(const Bytes:TBytes;out Face:TBytes;const ImageType:TImageType):TBytes;
label
  Clear;
const
  WidthRatio         = 1/8;
  HeightRatio        = 1/2.54;
  {$IFDEF UseFaceResizing}
  FaceHeightAddRatio = 2.3/15;
  FaceWidthAddPixels = 4;
  {$ELSE}
  FaceHeightAddRatio = 0;
  FaceWidthAddPixels = 0;
  {$ENDIF}
var
  FromBytes : Boolean;
  Mat       : PCvMat_t;
  ToGray    : PCvMat_t;
  nFaces    : Integer;
  CropRect  : TRectangle;
  Rect      : TRectangle;
  FaceRect  : PCvRect_t;
  CvRect    : CvRectS;
  i         : Integer;
  Area      : Extended;
var
  ProfileID : string;
begin
  ProfileID := StartProfiler(ClassName,'ExtractPhoto');
  Result    := Bytes;
  ToGray    := nil;
  Face      := Bytes;
  FromBytes := False;
  if Bytes=nil then Mat := FMat else
  begin
    Mat       := Load(Bytes);
    FromBytes := True;
  end;
  if IsEmpty(Mat) then goto Clear;
  ToGray  := GrayMat(Mat);
  try
    pCvCascadeClassifierdetectMultiScale(FaceCascade,ToGray,FFaces,FaceCascadeScaleFactorMax,2,FaceCascadeFlags,FSize_30_30,FSizeEmpty);
  except
  end;
  nFaces := pCvVectorRectLength(FFaces);
  if nFaces<=0 then
  begin
    try
      pCvCascadeClassifierdetectMultiScale(FaceCascade,ToGray,FFaces,FaceCascadeScaleFactorMin,2,FaceCascadeFlags,FSize_30_30,FSizeEmpty);
    except
    end;
  end;
  nFaces := pCvVectorRectLength(FFaces);
  Area   := -1;
  if nfaces>0 then
  begin
    InitRecord(Rect,SizeOf(Rect));
    for  i:=0 to nFaces-1 do
    begin
      FaceRect := pCvVectorRectGet(FFaces,i);
      pCvRectToStruct(FaceRect,@CvRect);
      if CvRect.width*CvRect.height>Area then
      begin
        Area        := CvRect.width*CvRect.height;
        Rect.Left   := Max(0,CvRect.x);
        Rect.Top    := Max(0,CvRect.y);
        Rect.Width  := CvRect.width;
        Rect.Height := CvRect.height;
      end;
    end;
  end;
  if Area>0 then
  begin
    CropRect.Left   := Max(0,Rect.Left-FaceWidthAddPixels);
    CropRect.Top    := Max(0,Rect.Top-Round(FaceHeightAddRatio*Rect.Height));
    CropRect.Width  := Rect.Width+(FaceWidthAddPixels*2);
    CropRect.Height := Rect.Height+Round(FaceHeightAddRatio*Rect.Height*2);
    Face            := Crop(ToGray,CropRect,ImageType);
    ////////////////////////////////////////////
    CropRect.Left   := Round(Rect.Left-(Rect.Width*WidthRatio));
    CropRect.Top    := Round(Rect.Top-(Rect.Height*HeightRatio));
    CropRect.Width  := Round(Rect.Width+(2*Rect.Width*WidthRatio));
    CropRect.Height := Round(Rect.Height+(2*Rect.Height*HeightRatio));
    Result          := Crop(Mat,CropRect,ImageType);
  end;
Clear:
  if Assigned(Mat) and FromBytes then pCvMatDelete(Mat);
  if Assigned(ToGray) then pCvMatDelete(ToGray);
  StopProfiler(ProfileID);
end;

Thank you so much.

[hafedh-trimeche]

Hi,

The Code modified by introducing pCvFaceRecognizerSFalignCrop and the score became 07.04583% of Dissimilarity.

function TOpenCVInstance.RecognizeFace(RefImage:TBytes;Gallery:array of TBytes;out CosineScore,NormDistance:Double):Integer;
label
  Clear;
const
  Color3Channels  = COLOR_BGR2RGB;
  CosineThreshold = 0.363;
  L2NormThreshold = 1.128;
var
  OK                    : Boolean;
  Face                  : TBytes;
  RefImageMat           : PCvMat_t;
  FaceMat               : PCvMat_t;
  AlignedFaceMat        : PCvMat_t;
  GalleryImageMat       : PCvMat_t;
  GalleryFaceMat        : PCvMat_t;
  AlignedGalleryFaceMat : PCvMat_t;
  iLabel                ,
  CosineLabel           ,
  NormL2Label           : Integer;
  MaxCosine             ,
  MinL2                 ,
  Value                 : Double;
var
  ProfileID : string;
begin
  ProfileID       := StartProfiler(ClassName,'RecognizeFace');
  CosineLabel     := -1;
  NormL2Label     := -1;
  RefImageMat     := nil;
  FaceMat         := nil;
  AlignedFaceMat  := nil;
  CosineScore     := -1;
  NormDistance    := -1;
  MaxCosine       := 0;
  MinL2           := MaxWord;
  if RefImage=nil then goto Clear;
  RefImageMat := LoadMat(RefImage);
  if IsEmpty(RefImageMat) then goto Clear;
  ExtractPhoto(RefImage,Face);
  FaceMat := LoadMat(Face);
  if IsEmpty(FaceMat) then goto Clear;
  AlignedFaceMat := pCvMatCreateEmpty;
  pCvFaceRecognizerSFalignCrop(FRecognizer,RefImageMat,pCvMatGetRow(FaceMat,0),AlignedFaceMat);
  pCvcvtColor(AlignedFaceMat,AlignedFaceMat,Ord(Color3Channels),3);
  pCvFaceRecognizerSFfeature(FRecognizer,AlignedFaceMat,FWorkFeatures);
  pCvMatCopy(FWorkFeatures,FFaceFeatures);
  for iLabel:=0 to Length(Gallery)-1 do
  begin
    GalleryImageMat := LoadMat(Gallery[iLabel]);
    if IsEmpty(GalleryImageMat) then Continue;
    ExtractPhoto(Gallery[iLabel],Face);
    GalleryFaceMat := LoadMat(Face);
    if IsEmpty(GalleryFaceMat) then
    begin
      pCvMatDelete(GalleryImageMat);
      Continue;
    end;
    AlignedGalleryFaceMat := pCvMatCreateEmpty;
    pCvFaceRecognizerSFalignCrop(FRecognizer,GalleryImageMat,pCvMatGetRow(GalleryFaceMat,0),AlignedGalleryFaceMat);
    pCvcvtColor(AlignedGalleryFaceMat,AlignedGalleryFaceMat,Ord(Color3Channels),3);
    pCvFaceRecognizerSFfeature(FRecognizer,AlignedGalleryFaceMat,FWorkFeatures);
    pCvMatCopy(FWorkFeatures,FGalleryFeatures);
    pCvMatDelete(AlignedGalleryFaceMat);
    pCvMatDelete(GalleryImageMat);
    pCvMatDelete(GalleryFaceMat);
    ////////////////////////////////////////////////////////////////////////////////////////////
    Value := pCvFaceRecognizerSFmatch(FRecognizer,FFaceFeatures,FGalleryFeatures,Ord(FR_COSINE));
    if Value>MaxCosine then MaxCosine := Value;
    OK := (Value>=CosineThreshold) and (Value>=MaxCosine);
    if OK then
    begin
      CosineScore := Value;
      CosineLabel := iLabel;
    end;
    ////////////////////////////////////////////////////////////////////////////////////////////
    Value := pCvFaceRecognizerSFmatch(FRecognizer,FFaceFeatures,FGalleryFeatures,Ord(FR_NORM_L2));
    if Value<MinL2 then MinL2 := Value;
    OK := (Value<=L2NormThreshold) and (Value<=MinL2);
    if OK then
    begin
      NormDistance := Value;
      NormL2Label  := iLabel;
    end;
  end;
Clear:
  if CosineScore<0 then CosineScore := MaxCosine;
  CosineScore := 100*(1-CosineScore);
  if NormDistance<0 then NormDistance := MinL2;
  if CosineLabel=-1 then Result := NormL2Label
                    else Result := CosineLabel;
  if Assigned(FaceMat)        then pCvMatDelete(FaceMat);
  if Assigned(AlignedFaceMat) then pCvMatDelete(AlignedFaceMat);
  if Assigned(RefImageMat)    then pCvMatDelete(RefImageMat);
  StopProfiler(ProfileID);
end;

Should Aligned/Cropped Mat be an image content? Exported one is empty bu has a structure of an image.

Best regards.

[gidesa]

Hello, I tested to save aligned image, adding this simple code in my program:
var
cstr: CvString_t;
............

pCvFaceRecognizerSFalignCrop(................., AlignedImg);
cstr.pstr:=PAnsiChar('d:\temp\prova.jpg');
pCvimwrite(@cstr, alignedImg);

The saved image is a valid jpg image, exactly the face cropped from original image and straightened up.
You can try the same.
It could be that the haar cascade function is not very good to extract faces, as expected by face recognizer.
Try to compare the coefficients that I sent you in faces db file (calculated on the first image) to same coefficients
calculated on same image from your program.
Regards

[hafedh-trimeche]

Hello,

I implemented Face Detection as you advised using FaceDetectorYN.

FaceBox is imported/exported as Json using:

class function TOpenCVInstance.SaveMat(Mat:PCvMat_t):string;
var
  Row     ,
  Column  : Integer;
  OcvMat  : TOcvMat;
begin
  if IsEmpty(Mat) then Exit('');
  OcvMat.Columns  := pCvMatGetWidth(Mat);
  OcvMat.Rows     := pCvMatGetHeight(Mat);
  OcvMat.&Type    := pCvMatGetType(Mat);
  OcvMat.Depth    := pCvMatGetDepth(Mat);
  OcvMat.Channels := pCvMatGetChannels(Mat);
  SetLength(OcvMat.Data,OcvMat.Rows,OcvMat.Columns);
  for Row:=0 to OcvMat.Rows-1 do
  begin
    for Column:=0 to OcvMat.Columns-1 do OcvMat.Data[Row,Column] := pCvMatGetFloat(Mat,Row,Column);
  end;
  Result := Serializer.Marshal(OcvMat);
end;

class function TOpenCVInstance.LoadMat(MatJson:string):PCvMat_t;
var
  OcvMat  : TOcvMat;
  Row     ,
  Column  : Integer;
begin
  if (not Serializer.Unmarshal(MatJson,OcvMat)) then Exit(pCvMatCreateEmpty);
  Result := pCvMat2dCreate(OcvMat.Rows,OcvMat.Columns,OcvMat.&Type);
  for Row:=0 to OcvMat.Rows-1 do
  begin
    for Column:=0 to OcvMat.Columns-1 do pCvMatSetFloat(Result,Row,Column,OcvMat.Data[Row,Column]);
  end;
end;

and re-implemented the Face Detection/Recognition inspired from your code:

function TOpenCVInstance.ExtractPhoto(const Image:TBytes;out FaceBox:string;const ImageType:TImageType=DefaultGraphicFormat):TBytes;
label
  Clear;
var
  FaceDetectorPtr : PCvPtr_FaceDetectorYN;
  FaceDetector    : PCvFaceDetectorYN_t;
  ImageMat        : PCvMat_t;
  Model           : CvString_t;
  InputSize       : PCvSize_t;
  LWidth          ,
  LHeight         : Integer;
  DetectMat       : PCvMat_t;
  Detected        : Integer;
  Area            : Extended;
  Rect            : TRectangle;
  i               : Integer;
var
  ProfileID : string;
begin
  ProfileID       := StartProfiler(ClassName,'ExtractPhoto');
  Result          := nil;
  FaceBox         := '';
  FaceDetectorPtr := nil;
  FaceDetector    := nil;
  DetectMat       := nil;
  if Image=nil then ImageMat := pCvMatClone(FMat)
               else ImageMat := Load(Image);
  if IsEmpty(ImageMat) then goto Clear;
  pCvcvtColor(ImageMat,ImageMat,Ord(Color3Channels),3);
  Model.pstr      := PAnsiChar(AnsiString(LibrarySearchPath+FaceDetectModel));
  FaceDetectorPtr := pCvFaceDetectorYN_create(@Model,pCvStringEmpty,FSize_320_320,FScoreThreshold,FNmsThreshold,FTopK);
  FaceDetector    := pCvPtr_FaceDetectorYNConvert(FaceDetectorPtr);
  LWidth          := pCvMatGetWidth(ImageMat);
  LHeight         := pCvMatGetHeight(ImageMat);
  InputSize       := CvSize_(LWidth,LHeight);
  pCvFaceDetectorYNsetInputSize(FaceDetector,InputSize);
  pCvFaceDetectorYNsetScoreThreshold(FaceDetector,FScoreThreshold);
  pCvFaceDetectorYNsetNMSThreshold(FaceDetector,FNmsThreshold);
  pCvFaceDetectorYNsetTopK(FaceDetector,FTopK);
  pCvSizeDelete(InputSize);

  DetectMat := pCvMatCreateEmpty;
  pCvFaceDetectorYNdetect(FaceDetector,ImageMat,DetectMat);
  Detected := pCvMatGetHeight(DetectMat);
  Area := -1;
  for i:=0 to Detected-1 do
  begin
    LWidth  := Round(pCvMatGetFloat(DetectMat,i,2));
    LHeight := Round(pCvMatGetFloat(DetectMat,i,3));
    if LWidth*LHeight>Area then
    begin
      Area        := LWidth*LHeight;
      Rect.Left   := Max(0,Round(pCvMatGetFloat(DetectMat,i,0)));
      Rect.Top    := Max(0,Round(pCvMatGetFloat(DetectMat,i,1)));
      Rect.Width  := LWidth;
      Rect.Height := LHeight;
    end;
  end;
  if Area>0 then
  begin
    FaceBox     := SaveMat(DetectMat);
    Rect.Left   := Round(Rect.Left-(Rect.Width*PhotoWidthRatio));
    Rect.Top    := Round(Rect.Top-(Rect.Height*PhotoHeightRatio));
    Rect.Width  := Round(Rect.Width+(2*Rect.Width*PhotoWidthRatio));
    Rect.Height := Round(Rect.Height+(2*Rect.Height*PhotoHeightRatio));
    Result      := Crop(ImageMat,Rect,ImageType);
  end;
Clear:
  if Assigned(FaceDetectorPtr) then pCvPtr_FaceDetectorYNDelete(FaceDetectorPtr,FaceDetector);
  if Assigned(DetectMat)       then pCvMatDelete(DetectMat);
  if Assigned(ImageMat)        then pCvMatDelete(ImageMat);
  StopProfiler(ProfileID);
end;

function TOpenCVInstance.RecognizeFace(RefImage:TBytes;Gallery:array of TBytes;out CosineScore,NormDistance:Double):Integer;
label
  Clear;
var
  OK                    : Boolean;
  Face                  : string;
  FaceBox               : PCvMat_t;
  RefImageMat           : PCvMat_t;
  AlignedFaceMat        : PCvMat_t;
  GalleryImageMat       : PCvMat_t;
  AlignedGalleryFaceMat : PCvMat_t;
  iLabel                ,
  CosineLabel           ,
  NormL2Label           : Integer;
  MaxCosine             ,
  MinL2                 ,
  Value                 : Double;
var
  ProfileID : string;
begin
  ProfileID       := StartProfiler(ClassName,'RecognizeFace');
  CosineLabel     := -1;
  NormL2Label     := -1;
  RefImageMat     := nil;
  AlignedFaceMat  := nil;
  CosineScore     := -1;
  NormDistance    := -1;
  MaxCosine       := 0;
  MinL2           := MaxWord;
  if RefImage=nil then goto Clear;
  RefImageMat := LoadMat(RefImage);
  if IsEmpty(RefImageMat) then goto Clear;
  ExtractPhoto(RefImage,Face);
  FaceBox := LoadMat(Face);
  if IsEmpty(FaceBox) then goto Clear;
  AlignedFaceMat := pCvMatCreateEmpty;
  pCvFaceRecognizerSFalignCrop(FRecognizer,RefImageMat,pCvMatGetRow(FaceBox,0),AlignedFaceMat);
  pCvMatDelete(FaceBox);
  pCvcvtColor(AlignedFaceMat,AlignedFaceMat,Ord(Color3Channels),3);
  pCvFaceRecognizerSFfeature(FRecognizer,AlignedFaceMat,FWorkFeatures);
  pCvMatCopy(FWorkFeatures,FFaceFeatures);
  for iLabel:=0 to Length(Gallery)-1 do
  begin
    GalleryImageMat := LoadMat(Gallery[iLabel]);
    if IsEmpty(GalleryImageMat) then Continue;
    ExtractPhoto(Gallery[iLabel],Face);
    FaceBox := LoadMat(Face);
    if IsEmpty(FaceBox) then
    begin
      pCvMatDelete(FaceBox);
      Continue;
    end;
    AlignedGalleryFaceMat := pCvMatCreateEmpty;
    pCvFaceRecognizerSFalignCrop(FRecognizer,GalleryImageMat,pCvMatGetRow(FaceBox,0),AlignedGalleryFaceMat);
    pCvcvtColor(AlignedGalleryFaceMat,AlignedGalleryFaceMat,Ord(Color3Channels),3);
    pCvFaceRecognizerSFfeature(FRecognizer,AlignedGalleryFaceMat,FWorkFeatures);
    pCvMatCopy(FWorkFeatures,FGalleryFeatures);
    pCvMatDelete(AlignedGalleryFaceMat);
    pCvMatDelete(GalleryImageMat);
    pCvMatDelete(FaceBox);
    ////////////////////////////////////////////////////////////////////////////////////////////
    Value := pCvFaceRecognizerSFmatch(FRecognizer,FFaceFeatures,FGalleryFeatures,Ord(FR_COSINE));
    if Value>MaxCosine then MaxCosine := Value;
    OK := (Value>=CosineThreshold) and (Value>=MaxCosine);
    if OK then
    begin
      CosineScore := Value;
      CosineLabel := iLabel;
    end;
    ////////////////////////////////////////////////////////////////////////////////////////////
    Value := pCvFaceRecognizerSFmatch(FRecognizer,FFaceFeatures,FGalleryFeatures,Ord(FR_NORM_L2));
    if Value<MinL2 then MinL2 := Value;
    OK := (Value<=L2NormThreshold) and (Value<=MinL2);
    if OK then
    begin
      NormDistance := Value;
      NormL2Label  := iLabel;
    end;
  end;
Clear:
  if CosineScore<0 then CosineScore := MaxCosine;
  CosineScore := 100*(1-CosineScore);
  if NormDistance<0 then NormDistance := MinL2;
  if CosineLabel=-1 then Result := NormL2Label
                    else Result := CosineLabel;
  if Assigned(AlignedFaceMat) then pCvMatDelete(AlignedFaceMat);
  if Assigned(RefImageMat)    then pCvMatDelete(RefImageMat);
  StopProfiler(ProfileID);
end;

finally, the score became close to yours (100-38.80430) = (0.6*100).

Which Cosine Score and Norm Distance are adequate to settle Similarity?

Many Thanks.

[hafedh-trimeche]

Hi,

Please note these modifications related to a Multi-Face recognition (Row Index):

type
  TOcvMat=
  record
    Columns  : Integer;
    Rows     : Integer;
    &Type    : Integer;
    Depth    : Integer;
    Channels : Integer;
    Data     : array of array of Single;
    RowIdx   : Integer;
  end;

class function TOpenCVInstance.SaveMat(Mat:PCvMat_t;RowIdx:Integer):string;
var
  Row     ,
  Column  : Integer;
  OcvMat  : TOcvMat;
begin
  if IsEmpty(Mat) then Exit('');
  OcvMat.Columns  := pCvMatGetWidth(Mat);
  OcvMat.Rows     := pCvMatGetHeight(Mat);
  OcvMat.&Type    := pCvMatGetType(Mat);
  OcvMat.Depth    := pCvMatGetDepth(Mat);
  OcvMat.Channels := pCvMatGetChannels(Mat);
  OcvMat.RowIdx   := RowIdx;
  SetLength(OcvMat.Data,OcvMat.Rows,OcvMat.Columns);
  for Row:=0 to OcvMat.Rows-1 do
  begin
    for Column:=0 to OcvMat.Columns-1 do OcvMat.Data[Row,Column] := pCvMatGetFloat(Mat,Row,Column);
  end;
  Result := Serializer.Marshal(OcvMat);
end;

class function TOpenCVInstance.LoadMat(MatJson:string;out RowIdx:Integer):PCvMat_t;
var
  OcvMat  : TOcvMat;
  Row     ,
  Column  : Integer;
begin
  RowIdx := -1;
  if (not Serializer.Unmarshal(MatJson,OcvMat)) then Exit(pCvMatCreateEmpty);
  RowIdx := OcvMat.RowIdx;
  Result := pCvMat2dCreate(OcvMat.Rows,OcvMat.Columns,OcvMat.&Type);
  for Row:=0 to OcvMat.Rows-1 do
  begin
    for Column:=0 to OcvMat.Columns-1 do pCvMatSetFloat(Result,Row,Column,OcvMat.Data[Row,Column]);
  end;
end;

Best regards.

[gidesa]

The recommended values from Opencv docs is: cosine >= 0.363 and L2 normal =< 1.128
Of course more cosine is close to 1, better is recognition. Same, more L2 normal is close to 0, better.
If you compare a face to itself should obtain cosine=1 and L2=0.
Regards

[hafedh-trimeche]

Hi,

Please find a function to load image from memory (saved into TBytes) blob:

class function TOpenCVInstance.LoadMat(Bytes:TBytes;cvCommands:CVCommandSet):PCvMat_t;
var
  MatDims : array[0..3] of Integer;
var
  ProfileID : string;
begin
  Result := nil;
  if Bytes=nil then Exit;
  ProfileID  := StartProfiler(ClassName,'Load');
  MatDims[0] := Length(Bytes);
  try
    Result := pCvMatCreate(1,@MatDims,CV_64F,UInt64(@Bytes[0]));
    if (cvGray in cvCommands) then Result := pCvimdecode(Result,Ord(IMREAD_GRAYSCALE)) else
                                   Result := pCvimdecode(Result,Ord(IMREAD_UNCHANGED)) ;
  except
  end;
  if Assigned(Result) then
  begin
    try
      if (cvFlipHorizontal in cvCommands) then pcvFlip(Result,Result,1);
      if (cvFlipVertical in cvCommands)   then pcvFlip(Result,Result,0);
      if (cvFlipBoth in cvCommands)       then pcvFlip(Result,Result,-1);
      if (cvEqualize in cvCommands)       then pCvequalizeHist(Result,Result);
      if (cvBinarize in cvCommands)       then pCvthreshold(Result,Result,0,255,CV_THRESH_BINARY or CV_THRESH_OTSU);
    except
      Result := nil;
    end;
  end;
  StopProfiler(ProfileID);
end;

Best regards.

[hafedh-trimeche]

Hi,

Decreasing ScoreThreshold to 0.8 helped to detect face on blurred image.

Best Regards.

[hafedh-trimeche]

Hi,

Regarding this note from https://github.com/ShiqiYu/libfacedetection

Please note that OpenCV DNN does not support the latest version of YuNet with dynamic input shape. Please ensure you have the exact same input shape as the one in the ONNX model to run latest YuNet with OpenCV DNN.
I implemented this Face Detecting function:

type
  TCNNLandmark =
  record
    x ,
    y : SHORT;
  end;

  TCNNFace =
   record
    score ,
    x     ,
    y     ,
    w     ,
    h     : SHORT;
    lm    : array[0..5-1] of TCNNLandmark;
  end;

  TCNNFaces =
  record
    num_faces : Integer;
    faces     : array[0..1024-1] of TCNNFace;
  end;
var
  facedetect_cnn : function(result_buffer  : Pointer; //buffer memory for storing face detection results, !!its size must be 0x20000 Bytes!!
                            rgb_image_data : Pointer; //input image, it must be BGR (three channels) insteed of RGB image!
                            width          ,
                            height         ,
                            step           :Integer):Pointer;cdecl=nil;

function TOpenCVInstance.ExtractPhotoCNN(const Image:TBytes;out FaceBox:TOcvMat;const ImageType:TImageType):TBytes;
label
  Clear;
var
  LibHandle   : TModuleHandle;
  ImageMat    ,
  WorkMat     : PCvMat_t;
  ImageWidth  ,
  LWidth      ,
  ImageHeight ,
  LHeight     : Integer;
  DataPtr     : UInt64;
  Channels    : Integer;
  CNNFaces    : TCNNFaces;
  Area        : Extended;
  RowIdx      : Integer;
  CvRect      : CvRectS;
  i           : Integer;
begin
  Result := nil;
  InitRecord(FaceBox,SizeOf(FaceBox));
  if @facedetect_cnn=nil then
  begin
    DllLoad(LibHandle,'facedetection');
    @facedetect_cnn := DllProcAddress(LibHandle,'?facedetect_cnn@@YAPEAHPEAE0HHH@Z');
  end;
  if Image=nil then
  begin
    if Assigned(FMat) then ImageMat := pCvMatClone(FMat)
                      else ImageMat := pCvMatCreateEmpty;
  end
  else ImageMat := Load(Image);
  if IsEmpty(ImageMat) then
  begin
    pCvMatDelete(ImageMat);
    Exit;
  end;
  InitRecord(CNNFaces,SizeOf(CNNFaces));
  To24BitColor(ImageMat,ImageMat);
  Channels    := pCvMatGetChannels(ImageMat);
  DataPtr     := pCvMatGetData(ImageMat);
  ImageWidth  := pCvMatGetWidth(ImageMat);
  ImageHeight := pCvMatGetHeight(ImageMat);
  facedetect_cnn(@CNNFaces,Pointer(DataPtr),ImageWidth,ImageHeight,ImageWidth*Channels);
  RowIdx := -1;
  Area   := -1;
  for i:=0 to CNNFaces.num_faces-1 do
  begin
    LWidth  := CNNFaces.faces[i].w;
    LHeight := CNNFaces.faces[i].h;
    if LWidth*LHeight>Area then
    begin
      RowIdx        := i;
      Area          := LWidth*LHeight;
      CvRect.x      := CNNFaces.faces[i].x;
      CvRect.y      := CNNFaces.faces[i].y;
      CvRect.width  := LWidth;
      CvRect.height := LHeight;
    end;
  end;
  if Area>0 then
  begin
    FaceBox.Columns  := 15;
    FaceBox.Rows     := 1;
    FaceBox.&Type    := 5;
    FaceBox.Depth    := 5;
    FaceBox.Channels := 1;
    FaceBox.RowIdx   := RowIdx;
    SetLength(FaceBox.Data,CNNFaces.num_faces,15);
  {
    0-1   : x, y of bbox top left corner
    2-3   : width, height of bbox
    4-5   : x, y of right eye (blue point in the example image)
    6-7   : x, y of left eye (red point in the example image)
    8-9   : x, y of nose tip (green point in the example image)
    10-11 : x, y of right corner of mouth (pink point in the example image)
    12-13 : x, y of left corner of mouth (yellow point in the example image)
    14    : face score
  }
    for i:=0 to CNNFaces.num_faces-1 do
    begin
      FaceBox.Data[i,00] := CNNFaces.faces[i].x;
      FaceBox.Data[i,01] := CNNFaces.faces[i].y;
      FaceBox.Data[i,02] := CNNFaces.faces[i].w;
      FaceBox.Data[i,03] := CNNFaces.faces[i].h;
      FaceBox.Data[i,04] := CNNFaces.faces[i].lm[0].x;
      FaceBox.Data[i,05] := CNNFaces.faces[i].lm[0].y;
      FaceBox.Data[i,06] := CNNFaces.faces[i].lm[1].x;
      FaceBox.Data[i,07] := CNNFaces.faces[i].lm[1].y;
      FaceBox.Data[i,08] := CNNFaces.faces[i].lm[2].x;
      FaceBox.Data[i,09] := CNNFaces.faces[i].lm[2].y;
      FaceBox.Data[i,10] := CNNFaces.faces[i].lm[3].x;
      FaceBox.Data[i,11] := CNNFaces.faces[i].lm[3].y;
      FaceBox.Data[i,12] := CNNFaces.faces[i].lm[4].x;
      FaceBox.Data[i,13] := CNNFaces.faces[i].lm[4].y;
      FaceBox.Data[i,14] := CNNFaces.faces[i].score/100;
    end;
//    if (not FaceOnly) then
    begin
      CvRect.x      := Round(CvRect.x-(CvRect.width*PhotoWidthRatio));
      CvRect.y      := Round(CvRect.y-(CvRect.height*PhotoHeightRatio));
      CvRect.width  := Round(CvRect.width+(2*CvRect.width*PhotoWidthRatio));
      CvRect.height := Round(CvRect.height+(2*CvRect.height*PhotoHeightRatio));
      if CvRect.x<0 then CvRect.x := 0;
      if CvRect.y<0 then CvRect.y := 0;
      if CvRect.x+CvRect.width>ImageWidth then CvRect.width := ImageWidth-CvRect.x;
      if CvRect.y+CvRect.height>ImageHeight then CvRect.height := ImageHeight-CvRect.y;
    end;
    WorkMat := pCvMatROI(ImageMat,@CvRect);
    Result  := Convert(WorkMat,ImageType);
    pCvMatDelete(WorkMat);
  end;
  pCvMatDelete(ImageMat);
end;

Would you please confirm that OpenCV DNN does not support the latest version of YuNet with dynamic input shape?

Best regards.

[hafedh-trimeche]

Hi,

Please find comparative tests using LibFaceDetection & OpenCV.
Cosine Score & Norm Distance are different.

*********** LibFaceDetection ***********
Ref Photo      : C:\_FACES_\DB\Photo\50200794552198.jpg
Cmp Photo      : C:\_FACES_\DB\Photo\45750889719987.jpg
OpenCV CompareFaces: 000098.55459|000001.40396
/////////////////////////////////////////////////////////////////////////
Ref Photo      : D:\Developer\Resources\OpenCV\_FACES_\Samples\FaceMasculine.png
Cmp Photo      : C:\_FACES_\DB\Photo\45750889719987.jpg
OpenCV CompareFaces: 000091.34933|000001.35166
/////////////////////////////////////////////////////////////////////////
Ref Photo      : D:\Developer\Resources\ePassport\Images\passports\c.png
Cmp Photo      : C:\_FACES_\DB\Photo\45750889719987.jpg
OpenCV CompareFaces: 000033.04907|000000.81301

*********** OpenCV ***********
Ref Photo      : C:\_FACES_\DB\Photo\50200794552198.jpg
Cmp Photo      : C:\_FACES_\DB\Photo\45750889719987.jpg
OpenCV CompareFaces: 000088.31275|000001.32901
/////////////////////////////////////////////////////////////////////////
Ref Photo      : D:\Developer\Resources\OpenCV\_FACES_\Samples\FaceMasculine.png
Cmp Photo      : C:\_FACES_\DB\Photo\45750889719987.jpg
OpenCV CompareFaces: 000100.00000|000001.41761
/////////////////////////////////////////////////////////////////////////
Ref Photo      : D:\Developer\Resources\ePassport\Images\passports\-c.png
Cmp Photo      : C:\_FACES_\DB\Photo\45750889719987.jpg
OpenCV CompareFaces: 000044.63427|000000.94482

Face Box is built from libfacedetection and passed to the PCvFaceRecognizerSF_t instance without using PCvPtr_FaceDetectorYN.
Is there a away to compute Cosine Score & Norm Distance using the Face Box Data?

Best regards.

[gidesa]

Hello, attached a simple code to read and write a Mat image in an array of bytes.
mat2bytes.zip
It's good that the blurred face is recognized with a reduced score of 0.8. Obviously this enhance the probabilities to have false positives.
About Opencv and Onnx, at url
opencv/opencv_zoo#44
is confirmed the limitation of Opencv with ONNX DNN format.
But a fix was done in april 2022. Opencv 4.6 was released in june 2022. So Opencv should be ok to load the last version of Yunet network. Indeed I am using an input image with wathever dimensions, it's the Opencv code, that load the image in DNN net, that take care of dimensions.
So I think that today there is no real difference from Opencv and libfacedetection.

[hafedh-trimeche]

Hi,

Please note that this code should be replaced (mat2bytes)

  FOR j := 0 TO iHeight-1   DO
     BEGIN
        offset := longint(iData) + iWidthStep * j;
        dataByte := pbytearray( offset);
        CopyMemory(@byteimg[j,0], dataByte, iWidthStep);
     END;

because of 32/64 bit adressing longint >> NativeInt

FOR j := 0 TO iHeight-1   DO
   BEGIN
      offset := NativeInt(iData) + iWidthStep * j;
      dataByte := pbytearray( offset);
      CopyMemory(@byteimg[j,0], dataByte, iWidthStep);
   END;

Best regards.

[hafedh-trimeche]

Hello,

Please find my implementation for exporting image to Bytes:

function TOpenCVInstance.Convert(Mat:PCvMat_t;const ImageType:TImageType):TBytes;
var
  Extension : string;
  Ext       : CvString_t;
  Buf       : PCvvector_uchar;
  Params    : PCvvector_int;
  Size      : Integer;
var
  ProfileID : string;
begin
  Result := nil;
  if Mat=nil then Mat := FMat;
  if IsEmpty(Mat) then Exit;
  ProfileID := StartProfiler(ClassName,'Convert');
  Extension := StringLower(ImageTypes[ImageType]);
  if Extension='' then Extension := 'bmp';
  Extension := '.'+Extension;
  Params    := pCvVectorintCreate(4);
  pCvVectorintSet(Params,0,Ord(IMWRITE_PNG_COMPRESSION));
  pCvVectorintSet(Params,1,9);
  pCvVectorintSet(Params,2,Ord(IMWRITE_JPEG_QUALITY));
  pCvVectorintSet(Params,3,100);
  Ext.pstr := PAnsiChar(RawByteString(Extension));
  Size     := Round(pCvMatGetWidth(Mat)*pCvMatGetHeight(Mat));
  Buf      := pCvVectorucharCreate(Size);
  try
    if pCvimencode(@Ext,Mat,Buf,Params) then
    begin
      Size := pCvVectorucharLength(Buf);
      if Size>0 then
      begin
        SetLength(Result,Size);
        pCvVectorucharToArray(Buf,@Result[0]);
      end;
    end;
  except
  end;
  pCvVectorucharDelete(Buf);
  pCvVectorintDelete(Params);
  StopProfiler(ProfileID);
end;

Best regards.

[gidesa]

Hello, your function is very good.
Only the size of vector in bytes should consider the number of channels of image. A 3 channels BGR has 3 bytes for every pixel, a 4 channels BGRA has 4 bytes. A gray image has only 1 byte per pixel.
So exact size is width * height * channels number .

[hafedh-trimeche]

Hi,

Even the Size is less than expected, pCvimencode will increase it to fit the one needed.

Allocating the exact size not possible due to the output format (would not be predicted). The Size would be set to at least the original one.

Size also would be set to 0. Setting to maximum size avoids memory reallocation.

function TOpenCVInstance.Convert(Mat:PCvMat_t;const ImageType:TImageType):TBytes;
var
  Extension : string;
  Ext       : CvString_t;
  Buf       : PCvvector_uchar;
  Params    : PCvvector_int;
  Size      : Integer;
var
  ProfileID : string;
begin
  Result := nil;
  if Mat=nil then Mat := FMat;
  if IsEmpty(Mat) then Exit;
  ProfileID := StartProfiler(ClassName,'Convert');
  Extension := StringLower(ImageTypes[ImageType]);
  if Extension='' then Extension := 'bmp';
  Extension := '.'+Extension;
  Params    := pCvVectorintCreate(4);
  pCvVectorintSet(Params,0,Ord(IMWRITE_PNG_COMPRESSION));
  pCvVectorintSet(Params,1,9);
  pCvVectorintSet(Params,2,Ord(IMWRITE_JPEG_QUALITY));
  pCvVectorintSet(Params,3,100);
  Ext.pstr := PAnsiChar(RawByteString(Extension));
  Size     := Round(pCvMatGetWidth(Mat)*pCvMatGetHeight(Mat)*pCvMatGetChannels(Mat));
  Buf      := pCvVectorucharCreate(Size);
  try
    if pCvimencode(@Ext,Mat,Buf,Params) then
    begin
      Size := pCvVectorucharLength(Buf);
      if Size>0 then
      begin
        SetLength(Result,Size);
        pCvVectorucharToArray(Buf,@Result[0]);
      end;
    end;
  except
  end;
  pCvVectorucharDelete(Buf);
  pCvVectorintDelete(Params);
  StopProfiler(ProfileID);
end;

Best regards.

[gidesa]

Hello, pCvimencode is similar to pCvimwrite, so it saves an image in memory array instead that on disk.
If you want equal block of bytes for every pixel (for example one byte/pixel for gray image) you must use BMP format. Other formtas are compressed ones (jpg, png, etc).
My code example use a different technique, copying the internal pCvMat image memory. This memory always represents a pixel with a block of bytes, same as BMP (indeed the from Bmp/to BMP functions use just this approach).
Last thing, pCvimencode requires a PCvMat as input, not a vector. So you cannot reload directly the bytes in a PCvMat variable.

https://docs.opencv.org/3.4/d4/da8/group__imgcodecs.html