Skip to content

Commit

Permalink
Merge pull request #407 from JeromeMartinez/AVI_Hash
Browse files Browse the repository at this point in the history 
Hash check for AVI/v210
  • Loading branch information
@JeromeMartinez
JeromeMartinez authored Mar 31, 2023
2 parents a5d8882 + b7a27e3 commit 42fe853
Show file tree
Hide file tree
Showing 5 changed files with 163 additions and 79 deletions.
6 changes: 4 additions & 2 deletions Source/Lib/Compressed/Matroska/Matroska.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -728,7 +728,8 @@ void matroska::Segment_Attachments_AttachedFile_FileData_RawCookedSegment_FileHa

array<uint8_t, 16> Hash;
memcpy(Hash.data(), Buffer.Data() + Buffer_Offset, Hash.size());
//Hashes_FromRAWcooked->FromHashFile(ReversibilityData->Data(reversibility::element::FileName, 0), Hash); //TODO: correclty manage hashes
if (NoOutputCheck) // TODO: remove check when hash and output check at same time is supported for Compound output
Hashes_FromRAWcooked->FromHashFile(ReversibilityData->Data(reversibility::element::FileName, 0), Hash); //TODO: correclty manage hashes
}

//---------------------------------------------------------------------------
Expand Down Expand Up @@ -1213,7 +1214,8 @@ void matroska::End()
FrameWriter_Template->Mode[frame_writer::IsNotEnd] = false;
raw_frame RawFrame;
RawFrame.FrameProcess = FrameWriter_Template;
//RawFrame.Process(); //TODO: correclty manage hashes
if (NoOutputCheck) // TODO: remove check when hash and output check at same time is supported for Compound output
RawFrame.Process();
}
}

Expand Down
222 changes: 153 additions & 69 deletions Source/Lib/Utils/FileIO/FileWriter.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,11 +64,9 @@ bool CheckFile_Compare(size_t& Offset, const filemap& File, const buffer_base& B
//---------------------------------------------------------------------------
frame_writer::~frame_writer()
{
if (!Compound)
{
delete (MD5_CTX*)MD5;
delete Output;
}
if (Compound)
return;
delete Output;
}

//---------------------------------------------------------------------------
Expand All @@ -84,7 +82,6 @@ void frame_writer::FrameCall(raw_frame* RawFrame)
Mode[frame_writer::IsNotBegin] = true;
Mode[frame_writer::IsNotEnd] = true;
}
MD5 = Compound->MD5;
}
else
{
Expand Down Expand Up @@ -115,6 +112,21 @@ void frame_writer::FrameCall(raw_frame* RawFrame)
}
}

// Check hash operation
if (((M->Hashes && M->Hashes->HashFiles_Count()) || M->Hashes_FromRAWcooked || M->Hashes_FromAttachments) && (!Compound || Mode[NoOutputCheck])) // TODO: remove last check when hash and output check at same time is supported for Compound output
{
bool IsNew = !Output->MD5;
if (IsNew)
Output->MD5 = new MD5_CTX;

if (!Compound || IsNew)
{
MD5_Init((MD5_CTX*)Output->MD5);
if (Compound)
MD5_Update((MD5_CTX*)Output->MD5, Compound->Input.Data(), (unsigned long)Compound->Positions[0].Input_Offset);
}
}

Output->Offset = 0;
if (!Mode[NoOutputCheck] && !Output->Write.IsOpen())
{
Expand All @@ -128,53 +140,30 @@ void frame_writer::FrameCall(raw_frame* RawFrame)
return;
}
SizeOnDisk = Output->Read.Size();
if (Compound)
{
// Do we need to write or check?
if (Output->Offset == (size_t)-1)
return; // File is flagged as already with wrong data

if (!Mode[NoOutputCheck])
if (Compound && !Mode[NoOutputCheck])
{
if (CheckFile_Compare(Output->Offset, Output->Read, buffer_view(Compound->Input.Data(), Compound->Positions[0].Input_Offset))) // Output->Write.Write(Compound->Input.Data(), Compound->Positions[0].Input_Offset);
{
if (CheckFile_Compare(Output->Offset, Output->Read, buffer_view(Compound->Input.Data(), Compound->Positions[0].Input_Offset))) // Output->Write.Write(Compound->Input.Data(), Compound->Positions[0].Input_Offset);
{
if (Errors)
Errors->Error(IO_FileChecker, error::type::Undecodable, (error::generic::code)filechecker_issue::undecodable::FileComparison, OutputFileName);
Output->Offset = (size_t)-1;
return;
}
if (Errors)
Errors->Error(IO_FileChecker, error::type::Undecodable, (error::generic::code)filechecker_issue::undecodable::FileComparison, OutputFileName);
Output->Offset = (size_t)-1;
return;
}
Mode[frame_writer::IsNotBegin] = true;
Mode[frame_writer::IsNotEnd] = true;
}
}
else
{
SizeOnDisk = (size_t)-1;
if (Compound)
{
if (!Mode[NoOutputCheck])
Output->Write.Write(Compound->Input.Data(), Compound->Positions[0].Input_Offset);
Mode[frame_writer::IsNotBegin] = true;
Mode[frame_writer::IsNotEnd] = true;
}

if (Compound && !Mode[NoOutputCheck])
Output->Write.Write(Compound->Input.Data(), Compound->Positions[0].Input_Offset);
}

if (Compound)
{
// Check hash operation
if (M->Hashes || M->Hashes_FromRAWcooked || M->Hashes_FromAttachments)
{
if (!MD5)
{
if (!Compound->MD5)
Compound->MD5 = new MD5_CTX;
MD5 = Compound->MD5;
}
MD5_Init((MD5_CTX*)MD5);
MD5_Update((MD5_CTX*)MD5, Compound->Input.Data(), (unsigned long)Compound->Positions[0].Input_Offset);
Mode[frame_writer::IsNotBegin] = true;
Mode[frame_writer::IsNotEnd] = true;
}
Mode[frame_writer::IsNotBegin] = true;
Mode[frame_writer::IsNotEnd] = true;
}
}

Expand All @@ -183,23 +172,16 @@ void frame_writer::FrameCall(raw_frame* RawFrame)
return; // File is flagged as already with wrong data

// Check hash operation
if (M->Hashes || M->Hashes_FromRAWcooked || M->Hashes_FromAttachments)
if (Output->MD5)
{
if (!Mode[IsNotBegin])
{
if (!MD5)
MD5 = new MD5_CTX;
MD5_Init((MD5_CTX*)MD5);
}

CheckMD5(RawFrame);

if (!Mode[IsNotEnd])
{
md5 MD5_Result;
MD5_Final(MD5_Result.data(), (MD5_CTX*)MD5);
MD5_Final(MD5_Result.data(), (MD5_CTX*)Output->MD5);

if (M->Hashes)
if (M->Hashes && M->Hashes->HashFiles_Count())
M->Hashes->FromFile(OutputFileName, MD5_Result);
if (M->Hashes_FromRAWcooked)
M->Hashes_FromRAWcooked->FromFile(OutputFileName, MD5_Result);
Expand Down Expand Up @@ -483,8 +465,6 @@ bool frame_writer::CheckFile(raw_frame* RawFrame)
{
if (Compound)
{
size_t Offset_Current = Output->Offset;

buffer_view ToWrite;
uint16_t Index;
size_t* Positions_StreamOffsetTemp;
Expand All @@ -506,12 +486,6 @@ bool frame_writer::CheckFile(raw_frame* RawFrame)
}
size_t& Positions_StreamOffset = *Positions_StreamOffsetTemp;
size_t& AdditionalBytes = *AdditionalBytesTemp;

// Save
auto Positions_StreamOffset_Save = Positions_StreamOffset;
auto AdditionalBytes_Save = AdditionalBytes;
auto Positions_Offset_InFileWritten_Save = Compound->Positions_Offset_InFileWritten;

size_t SizeToWrite = ToWrite.Size();
size_t SizeWritten = 0;
while (SizeToWrite)
Expand Down Expand Up @@ -555,12 +529,12 @@ bool frame_writer::CheckFile(raw_frame* RawFrame)
if (AdditionalBytes)
{
auto& Buffer_Before = Compound->Positions[Compound->Positions_Offset_InFileWritten].Buffer;
if (CheckFile_Compare(Offset_Current, Output->Read, buffer_view(Buffer_Before->Data(), AdditionalBytes))) // Output->Write.Write(Buffer_Before->Data(), AdditionalBytes);
if (CheckFile_Compare(Output->Offset, Output->Read, buffer_view(Buffer_Before->Data(), AdditionalBytes))) // Output->Write.Write(Buffer_Before->Data(), AdditionalBytes);
return true;
delete Buffer_Before;
Buffer_Before = nullptr;
}
if (CheckFile_Compare(Offset_Current, Output->Read, buffer_view(ToWrite.Data() + SizeWritten, SizeOfFrame))) // Output->Write.Write(ToWrite.Data() + SizeWritten, SizeOfFrame))
if (CheckFile_Compare(Output->Offset, Output->Read, buffer_view(ToWrite.Data() + SizeWritten, SizeOfFrame))) // Output->Write.Write(ToWrite.Data() + SizeWritten, SizeOfFrame))
return true;
SizeWritten += SizeOfFrame;
AdditionalBytes = 0;
Expand All @@ -576,7 +550,7 @@ bool frame_writer::CheckFile(raw_frame* RawFrame)
auto& Buffer_Before = Compound->Positions[Compound->Positions_Offset_InFileWritten].Buffer;
if (Buffer_Before && Buffer_Before->Size() != 0)
{
if (CheckFile_Compare(Offset_Current, Output->Read, *Buffer_Before)) // Output->Write.Write(Buffer_Before->Data(), Buffer_Before->Size());
if (CheckFile_Compare(Output->Offset, Output->Read, *Buffer_Before)) // Output->Write.Write(Buffer_Before->Data(), Buffer_Before->Size());
return true;
delete Buffer_Before;
Buffer_Before = nullptr;
Expand All @@ -593,13 +567,12 @@ bool frame_writer::CheckFile(raw_frame* RawFrame)
Input_Offset = Compound->Positions[Compound->Positions_Offset_InFileWritten].Input_Offset;
}
auto Size = Input_Offset - Compound->Positions[Compound->Positions_Offset_InFileWritten - 1].Input_Offset;
if (CheckFile_Compare(Offset_Current, Output->Read, buffer_view(Compound->Input.Data() + Input_Offset - Size, Size))) // Output->Write.Write(Compound->Input.Data() + Input_Offset - Size, Size);
if (CheckFile_Compare(Output->Offset, Output->Read, buffer_view(Compound->Input.Data() + Input_Offset - Size, Size))) // Output->Write.Write(Compound->Input.Data() + Input_Offset - Size, Size);
return true;
} while (Compound->Positions_Offset_InFileWritten < Compound->Positions.size() && Compound->Positions_Offset_InFileWritten < Compound->Positions_Offset_Video && Compound->Positions_Offset_InFileWritten < Compound->Positions_Offset_Audio);
}
}

Output->Offset = Offset_Current;
return false;
}

Expand All @@ -622,22 +595,133 @@ bool frame_writer::CheckFile(raw_frame* RawFrame)
//---------------------------------------------------------------------------
bool frame_writer::CheckMD5(raw_frame* RawFrame)
{
if (Compound)
{
buffer_view ToWrite;
uint16_t Index;
size_t* Positions_StreamOffsetTemp;
size_t* AdditionalBytesTemp;
size_t AdditionalBytesTemp_Fake = 0;
if (RawFrame->Buffer().Size() == 0) //TODO
{
if (RawFrame->Planes().empty())
return false;
ToWrite = RawFrame->Plane(0)->Buffer();
Index = 0;
Positions_StreamOffsetTemp = &Compound->Positions_Offset_Video;
AdditionalBytesTemp = &AdditionalBytesTemp_Fake;
}
else
{
ToWrite = RawFrame->Buffer();
Index = 1;
Positions_StreamOffsetTemp = &Compound->Positions_Offset_Audio;
AdditionalBytesTemp = &Compound->Positions_Offset_Audio_AdditionalBytes;
}
size_t& Positions_StreamOffset = *Positions_StreamOffsetTemp;
size_t& AdditionalBytes = *AdditionalBytesTemp;
size_t SizeToWrite = ToWrite.Size();
size_t SizeWritten = 0;
while (SizeToWrite)
{
// Find the next frame
while (Compound->Positions[Positions_StreamOffset].Index != Index)
Positions_StreamOffset++;

auto SizeOfFrame = Compound->Positions[Positions_StreamOffset].Size - AdditionalBytes;
if (SizeToWrite < SizeOfFrame || Positions_StreamOffset != Compound->Positions_Offset_InFileWritten)
{
// Store the content for later
if (!Compound->Positions[Positions_StreamOffset].Buffer)
{
Compound->Positions[Positions_StreamOffset].Buffer = new buffer();
Compound->Positions[Positions_StreamOffset].Buffer->Create(SizeOfFrame);
}
auto SizeToWriteInThisFrame = SizeToWrite >= SizeOfFrame ? SizeOfFrame : SizeToWrite;
memcpy(Compound->Positions[Positions_StreamOffset].Buffer->Data() + AdditionalBytes, ToWrite.Data() + SizeWritten, SizeToWriteInThisFrame);
SizeWritten += SizeToWriteInThisFrame;

if (SizeToWrite < SizeOfFrame)
{
// Still need the frame
AdditionalBytes += SizeToWrite;
SizeToWrite = 0;
}
else
{
AdditionalBytes = 0;
SizeToWrite -= SizeOfFrame;

// Find the next frame
Positions_StreamOffset++;
while (Positions_StreamOffset < Compound->Positions.size() && Compound->Positions[Positions_StreamOffset].Index != Index)
Positions_StreamOffset++;
}
}
else
{
if (AdditionalBytes)
{
auto& Buffer_Before = Compound->Positions[Compound->Positions_Offset_InFileWritten].Buffer;
MD5_Update((MD5_CTX*)Output->MD5, Buffer_Before->Data(), (unsigned long)AdditionalBytes); // Output->Write.Write(Buffer_Before->Data(), AdditionalBytes);
delete Buffer_Before;
Buffer_Before = nullptr;
}
MD5_Update((MD5_CTX*)Output->MD5, ToWrite.Data() + SizeWritten, (unsigned long)SizeOfFrame); // Output->Write.Write(ToWrite.Data() + SizeWritten, SizeOfFrame))
SizeWritten += SizeOfFrame;
AdditionalBytes = 0;
SizeToWrite -= SizeOfFrame;

// Find the next frame
Positions_StreamOffset++;
while (Positions_StreamOffset < Compound->Positions.size() && Compound->Positions[Positions_StreamOffset].Index != Index)
Positions_StreamOffset++;

do
{
auto& Buffer_Before = Compound->Positions[Compound->Positions_Offset_InFileWritten].Buffer;
if (Buffer_Before && Buffer_Before->Size() != 0)
{
MD5_Update((MD5_CTX*)Output->MD5, Buffer_Before->Data(), (unsigned long)Buffer_Before->Size()); // Output->Write.Write(Buffer_Before->Data(), Buffer_Before->Size());
delete Buffer_Before;
Buffer_Before = nullptr;
}

uint64_t Input_Offset;
Compound->Positions_Offset_InFileWritten++;
if (Compound->Positions_Offset_InFileWritten == Compound->Positions.size())
{
Input_Offset = Compound->Input.Size();
}
else
{
Input_Offset = Compound->Positions[Compound->Positions_Offset_InFileWritten].Input_Offset;
}
auto Size = Input_Offset - Compound->Positions[Compound->Positions_Offset_InFileWritten - 1].Input_Offset;
MD5_Update((MD5_CTX*)Output->MD5, Compound->Input.Data() + Input_Offset - Size, (unsigned long)Size); // Output->Write.Write(Compound->Input.Data() + Input_Offset - Size, Size);
} while (Compound->Positions_Offset_InFileWritten < Compound->Positions.size() && Compound->Positions_Offset_InFileWritten < Compound->Positions_Offset_Video && Compound->Positions_Offset_InFileWritten < Compound->Positions_Offset_Audio);
}
}

return false;
}

const auto Pre = RawFrame->Pre();
if (Pre.Size())
MD5_Update((MD5_CTX*)MD5, Pre.Data(), (unsigned long)Pre.Size());
MD5_Update((MD5_CTX*)Output->MD5, Pre.Data(), (unsigned long)Pre.Size());
const auto Buffer = RawFrame->Buffer();
if (Buffer.Size())
MD5_Update((MD5_CTX*)MD5, Buffer.Data(), (unsigned long)Buffer.Size());
MD5_Update((MD5_CTX*)Output->MD5, Buffer.Data(), (unsigned long)Buffer.Size());
for (const auto& Plane : RawFrame->Planes())
if (Plane)
{
const auto& Buffer = Plane->Buffer();
if (Buffer.Size())
MD5_Update((MD5_CTX*)MD5, Buffer.Data(), (unsigned long)Buffer.Size());
MD5_Update((MD5_CTX*)Output->MD5, Buffer.Data(), (unsigned long)Buffer.Size());
}
const auto Post = RawFrame->Post();
if (Post.Size())
MD5_Update((MD5_CTX*)MD5, Post.Data(), (unsigned long)Post.Size());
MD5_Update((MD5_CTX*)Output->MD5, Post.Data(), (unsigned long)Post.Size());

return false;
}
Expand Down
7 changes: 2 additions & 5 deletions Source/Lib/Utils/FileIO/FileWriter.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,7 @@ class frame_writer : public raw_frame_process
UserMode(UserMode_Soure),
Ask_Callback(Ask_Callback_Source),
M(M_Source),
Errors(Errors_Source),
MD5(nullptr)
Errors(Errors_Source)
{
}
frame_writer(const frame_writer& Source) :
Expand All @@ -44,8 +43,7 @@ class frame_writer : public raw_frame_process
Ask_Callback(Source.Ask_Callback),
M(Source.M),
Errors(Source.Errors),
SizeOnDisk(Source.SizeOnDisk),
MD5(Source.MD5)
SizeOnDisk(Source.SizeOnDisk)
{
}
frame_writer(const frame_writer* Source) :
Expand Down Expand Up @@ -82,7 +80,6 @@ class frame_writer : public raw_frame_process
matroska* M;
errors* Errors;
size_t SizeOnDisk;
void* MD5;
};

#endif
2 changes: 1 addition & 1 deletion Source/Lib/Utils/FileIO/Input_Base.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -315,7 +315,7 @@ void input_base_uncompressed::RegisterAsAttachment()
}

//---------------------------------------------------------------------------
input_base_uncompressed_compound::~input_base_uncompressed_compound()
file_output::~file_output()
{
delete (MD5_CTX*)MD5;
}
Expand Down
Loading

0 comments on commit 42fe853

Please sign in to comment.