From 7b1f1993241969cc5cca46c9bee5979a87327926 Mon Sep 17 00:00:00 2001
From: Franco Corbelli <77727889+fcorbelli@users.noreply.github.com>
Date: Mon, 20 Dec 2021 17:50:10 +0100
Subject: [PATCH] Add files via upload

---
 zpaqfranz.cpp | 355 +++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 305 insertions(+), 50 deletions(-)

diff --git a/zpaqfranz.cpp b/zpaqfranz.cpp
index d108e33..6810918 100644
--- a/zpaqfranz.cpp
+++ b/zpaqfranz.cpp
@@ -9,7 +9,7 @@
 ///////// I apologize with the authors, it's not a foolish attempt to take over their jobs
 
 
-#define ZPAQ_VERSION "54.9-experimental"
+#define ZPAQ_VERSION "54.10-experimental"
 
 #if defined(_WIN64)
 #define ZSFX_VERSION "SFX64 v52.15,"
@@ -8743,6 +8743,9 @@ int64_t g_zerotime		=0;
 int64_t g_bytescanned	=0;
 int64_t g_filescanned	=0;
 int64_t g_worked		=0;
+int64_t g_fwritten		=0;
+int64_t g_fexpected		=0;
+
 
 uint64_t minsize;
 uint64_t maxsize;
@@ -20266,6 +20269,13 @@ size_t fread(void* ptr, size_t size, size_t nobj, FP fp) {
 size_t fwrite(const void* ptr, size_t size, size_t nobj, FP fp) {
   DWORD r=0;
   WriteFile(fp, ptr, size*nobj, &r, NULL);
+  
+  /// quick-and-dirty check (media full)
+  g_fexpected+=(size*nobj);
+  g_fwritten+=r;
+  if (flagdebug)
+		if ((size*nobj)!=r)
+			printf("20277: expected bytes != written (media full?)  %s vs  %s\n",migliaia(size*nobj),migliaia2(r));
   if (size>1) r/=size;
   return r;
 }
@@ -22761,7 +22771,7 @@ class Jidac
 	void 	addfile(bool i_checkifselected,DTMap& i_edt,string filename, int64_t edate, int64_t esize,int64_t eattr);          // add external file to dt
 	int64_t franzparallelscandir(bool i_flaghash,bool i_recursive,bool i_forcedir);
 	
-	bool 	equal(DTMap::const_iterator p, const char* filename, uint32_t &o_crc32);// compare file contents with p
+	bool 	equal(DTMap::const_iterator p, const char* filename, uint32_t &o_crc32,string i_myhashtype,string i_myhash,string& o_hash);// compare file contents with p
 	
 	void 	write715attr(libzpaq::StringBuffer& i_sb, uint64_t i_data, unsigned int i_quanti);
 	void 	writefranzattr(libzpaq::StringBuffer& i_sb, uint64_t i_data, unsigned int i_quanti,string i_filename,uint32_t i_crc32fromfragments,uint32_t i_crc32,string i_thehash);
@@ -25826,6 +25836,7 @@ void help_b(bool i_usage,bool i_example)
 	{
 		moreprint("CMD   b (benchmark and CPU stresser)");
 		moreprint("DIFF:               Rough benchmarking of hash-checksum");
+		moreprint("DIFF:               franzomips (fake index) showed if no switch selected");
 		moreprint("DIFF:               By default test ALL for 5 seconds with 400.000 bytes");
 		moreprint("DIFF:               NOTE: THIS IS THE MAXIMUM PERFORMANCES, not the real one!");
 		moreprint("DIFF: -verbose      Verbose output");
@@ -26052,6 +26063,7 @@ void help_t(bool i_usage,bool i_example)
 		moreprint("CMD   t (test)");
 		moreprint("PLUS:               Check that all block are OK, and the CRC-32s of the individual ");
 		moreprint("                    files corresponds to what would be generated by actually extracting.");
+		moreprint("PLUS: -checksum     Enable hash checksums");
 		moreprint("PLUS: -verify       Do a filesystem post-check: STORED CRC==DECOMPRESSED==FROM FILE.");
 		moreprint("PLUS: -find pippo   For path-rework of verify");
 		moreprint("PLUS: -replace plu  For path-rework of verify (find and replace)");
@@ -26067,6 +26079,8 @@ void help_t(bool i_usage,bool i_example)
 		moreprint("Compare against filesystem:          t z:\\1.zpaq -verify");
 		moreprint("Real paranoid: extract all           t z:\\1.zpaq -to z:\\knb -paranoid");
 		moreprint("Fast-SHA1 (nz the source dir):       t z:\\1.zpaq c:\\nz");
+		moreprint("Cnk-SHA1+hash (nz the source dir):   t z:\\1.zpaq c:\\nz -checksum");
+		
 	}
 }
 void help_v(bool i_usage,bool i_example)
@@ -29382,10 +29396,12 @@ void Jidac::writefranzattr(libzpaq::StringBuffer& i_sb, uint64_t i_data, unsigne
 // If filename is 0 then return true if it is possible to compare.
 // In the meantime calc the crc32 of the entire file
 
-bool Jidac::equal(DTMap::const_iterator p, const char* filename,uint32_t &o_crc32) 
+bool Jidac::equal(DTMap::const_iterator p, const char* filename,uint32_t &o_crc32,string i_myhashtype,string i_myhash,string& o_hash) 
 {
 	o_crc32=0;
-
+	o_hash="";
+	if (i_myhash=="FAKE")
+		o_hash="FAKE";
   // test if all fragment sizes and hashes exist
 	if (filename==0) 
 	{
@@ -29433,6 +29449,17 @@ bool Jidac::equal(DTMap::const_iterator p, const char* filename,uint32_t &o_crc3
 	int64_t 	done				=0;
 	int 		ultimapercentuale	=-1;
 
+	uint64_t myseed = 0;
+	XXHash64 myhash(myseed);
+	SHA3 sha3;
+	MD5 md5;
+	blake3_hasher hasher;
+	blake3_hasher_init(&hasher);
+	franzSHA256 mysha256;
+	XXH3_state_t state128;
+    (void)XXH3_128bits_reset(&state128);
+	libzpaq::SHA1 mysha1;
+	
 	int64_t timestart=mtime();
 	for (unsigned i=0; i<p->second.ptr.size(); ++i) 
 	{
@@ -29456,6 +29483,7 @@ bool Jidac::equal(DTMap::const_iterator p, const char* filename,uint32_t &o_crc3
 					ultimapercentuale=percentuale;
 				}
 		}
+
 			
 		for (int j=0; j<ht[f].usize;) 
 		{
@@ -29469,22 +29497,45 @@ bool Jidac::equal(DTMap::const_iterator p, const char* filename,uint32_t &o_crc3
 			done+=r;
 			g_worked+=r;
 			
+			if (flagchecksum)
+			{
+				if (i_myhashtype=="XXHASH64")
+					myhash.add(buf,r);
+
+				if (i_myhashtype=="SHA-3")
+					sha3.add(buf,r);
+			
+				if (i_myhashtype=="MD5")
+					md5.add(buf,r);
+			
+				if (i_myhashtype=="BLAKE3")
+					blake3_hasher_update(&hasher,buf,r);
+		
+				if (i_myhashtype=="SHA-256")
+					mysha256.update((const uint8_t*)buf,r);
+			
+				if (i_myhashtype=="XXH3")
+					(void)XXH3_128bits_update(&state128,buf,r);
+				
+				if (i_myhashtype=="SHA-1")
+					mysha1.write(buf,r);
+			}
+		
 			if (r!=n) 
 			{
-		//		if (flagshow)
-			//		printf("\n");
-					
 				return fclose(in), false;
 			}
 			sha1.write(buf, n);
 			j+=n;
 		}
+		
+		
 		if (memcmp(sha1.result(), ht[f].sha1, 20)!=0) 
 		{	
-			//if (flagshow)
-				//printf("\n");
 			return fclose(in), false;
 		}
+		
+	
 	}
 	//if (flagshow)
 		//printf("\n");
@@ -29492,6 +29543,52 @@ bool Jidac::equal(DTMap::const_iterator p, const char* filename,uint32_t &o_crc3
 	if (fread(buf, 1, BUFSIZE, in)!=0) 
 		return fclose(in), false;
 	fclose(in);
+	
+
+	if (flagchecksum)
+	{
+		if (i_myhashtype=="XXHASH64")
+		{
+			char temp[33];
+			sprintf(temp,"%016llX",(unsigned long long)myhash.hash());
+			o_hash=temp;
+		}
+		else
+		if (i_myhashtype=="SHA-3")
+			o_hash= sha3.getHash();
+		else
+		if (i_myhashtype=="MD5")
+			o_hash=md5.getHash();
+		else
+		if (i_myhashtype=="BLAKE3")
+		{
+			uint8_t output[BLAKE3_OUT_LEN];
+			blake3_hasher_finalize(&hasher, output, BLAKE3_OUT_LEN);
+			o_hash=binarytohex((const unsigned char*)output,BLAKE3_OUT_LEN);	
+		}
+		else
+		if (i_myhashtype=="SHA-256")
+			o_hash=mysha256.gethex();
+		else
+		if (i_myhashtype=="XXH3")
+		{
+			XXH128_hash_t myhash=XXH3_128bits_digest(&state128);
+			char buf[33];
+			sprintf(buf,"%016llX%016llX",(unsigned long long)myhash.high64,(unsigned long long)myhash.low64);
+			o_hash=buf;
+		}
+		else
+		if (i_myhashtype=="SHA-1")
+		{
+			char sha1result[20];
+			memcpy(sha1result, mysha1.result(), 20);
+			o_hash=binarytohex((const unsigned char*)sha1result,20);
+		}
+		else
+			printf("\n29573: WARNING unknown hasher algo %s\n",i_myhashtype.c_str());
+		
+	}
+	
 	return true;
 }
 
@@ -30082,8 +30179,10 @@ int Jidac::testverify()
 		}
 	
 	int64_t usize=0;
-	unsigned matches=0, mismatches=0, internal=0, external=0;
+	unsigned matches=0, mismatches=0, internal=0, external=0,hashmatches=0;
 	uint32_t crc32fromfile;
+	int64_t	hashchecked=0;
+	int64_t	tobetested=0;
 	
 	vector<string> risultati;
 	vector<bool> risultati_utf8;
@@ -30113,24 +30212,61 @@ int Jidac::testverify()
 		if (p->second.data=='-' && fi+1<filelist.size() && filelist[fi+1]->second.data=='+') 
 		{
 			DTMap::iterator p1=filelist[fi+1];
-				
-			if (equal(p, p1->first.c_str(),crc32fromfile)) //crc32 of second parameters
+	
+			string myhashtype="";
+			string myhash="";
+			string mycrc32="";
+			string hashfromfile="";
+			
+			if (!isdirectory(p1->first))
 			{
-				if (!isdirectory(p1->first))
-				{
-					string myhashtype="";
-					string myhash="";
-					string mycrc32="";
-				
 					decode_franz_block(false,p->second.franz_block,
 					myhashtype,
 					myhash,
 					mycrc32);
-					
-					///printf("File ciucciato %s  %s\n",mycrc32.c_str(),p->first.c_str());
-					
+					tobetested+=p->second.size;
+			}
+/*
+			if (myhashtype!="")
+				printf("30133:  %s\n",myhashtype.c_str());
+	*/		
+	
+			if (equal(p, p1->first.c_str(),crc32fromfile,myhashtype,myhash,hashfromfile)) //crc32 of second parameters
+			{
+				if (!isdirectory(p1->first))
+				{
+					if (hashfromfile!="")
+					{
+						if (flagdebug)
+							printf("Check hash for %s\n",p->first.c_str());
+						hashchecked+=p->second.size;
+			
+						if (hashfromfile!=myhash)
+						{
+							
+							p->second.data='#';
+							sprintf(linebuffer,"\nHASH %s NOT MATCH STORED %s vs FROM FILE %s ",myhashtype.c_str(),myhash.c_str(),hashfromfile.c_str());
+							risultati.push_back(linebuffer);
+							risultati_utf8.push_back(false);
+							
+							sprintf(linebuffer,"%s\n",p1->first.c_str());
+							risultati.push_back(linebuffer);
+							risultati_utf8.push_back(true);
+						}
+						else
+						{	
+							hashmatches++;
+							p->second.data='=';
+							++fi;
+						}
+		
+					}
+					else
 					if (mycrc32!="")
 					{
+						if (flagdebug)
+							printf("Check CRC-32 for %s\n",p->first.c_str());
+						
 						uint32_t crc32stored=crchex2int(mycrc32.c_str());
 						if (crc32stored!=crc32fromfile)
 						{
@@ -30152,6 +30288,9 @@ int Jidac::testverify()
 					}
 					else
 					{
+						if (flagdebug)
+							printf("No zpaqfranz check for %s\n",p->first.c_str());
+						
 						/// crc-32 is not stored in the archive, cannot tell anything
 						p->second.data='=';
 						++fi;
@@ -30218,7 +30357,19 @@ int Jidac::testverify()
 		}
 
 	if (matches)
-		printf("%08d =same\n",matches);
+		printf("%08d = same\n",matches);
+	
+	if (hashmatches)
+	{
+		printf("%08d ==very same (hash checked) %19s\n",hashmatches,migliaia(hashchecked));
+		printf("               to be tested by hash %19s\n",migliaia(tobetested));
+		if (hashchecked!=tobetested)
+		{
+			printf("****** WARN: something strange hash checking    *******\n");
+			printf("******       some file(s) stored without hash?  *******\n");
+			printf("******       hash does not match?               *******\n");
+		}
+	}
 	if (mismatches)
 	{
 		printf("%08d #different\n",mismatches);
@@ -31678,8 +31829,8 @@ int Jidac::extract()
 		{
 			string fn=rename(p->first);
 			const bool isdir=isdirectory(p->first);
-	  	  
-			if ((repack=="") && !flagtest && flagforce && !isdir && equal(p, fn.c_str(),crc32fromfile)) 
+			string dummy="";
+			if ((repack=="") && !flagtest && flagforce && !isdir && equal(p, fn.c_str(),crc32fromfile,"","",dummy)) 
 			{
 				// identical
 				if (flagverbose)
@@ -33010,6 +33161,56 @@ bool isfilesequal(string i_source,string i_destination,bool i_flagfast=false)
 
 int Jidac::benchmark()
 {
+	
+	vector<string> 	array_cpu;
+	vector<float> 	array_single;
+	vector<float> 	array_multi;
+	
+	array_cpu	.push_back("Atom N2800      (phy)  4");
+	array_multi	.push_back(113);
+	array_single.push_back(407);
+	
+	array_cpu	.push_back("Xeon E3 1245 V2 (vir)  4");
+	array_multi	.push_back(819);
+	array_single.push_back(2415);
+	
+	array_cpu	.push_back("i5-6200U        (phy)  2");
+	array_multi	.push_back(352);
+	array_single.push_back(1903);
+	
+	array_cpu	.push_back("Xeon E5 2620 V4 (phy)  8");
+	array_multi	.push_back(1520);
+	array_single.push_back(1848);
+	
+	array_cpu	.push_back("Xeon E5 2630 V4 (phy) 10");
+	array_multi	.push_back(1596);
+	array_single.push_back(1552);
+	
+	array_cpu	.push_back("Xeon D-1541     (vir)  8");
+	array_multi	.push_back(1619);
+	array_single.push_back(2031);
+
+	array_cpu	.push_back("i5-3570         (phy)  4");
+	array_multi	.push_back(852);
+	array_single.push_back(2956);
+	
+	array_cpu	.push_back("i7-4790K        (phy)  4");
+	array_multi	.push_back(1700);
+	array_single.push_back(3265);
+	
+	array_cpu	.push_back("i7-8700K        (phy)  6");
+	array_multi	.push_back(1836);
+	array_single.push_back(3358);
+	
+	array_cpu	.push_back("i9-10900        (phy) 10");
+	array_multi	.push_back(3049);
+	array_single.push_back(3705);
+	
+	array_cpu	.push_back("AMD-5950X       (phy) 16");
+	array_multi	.push_back(5843);
+	array_single.push_back(4798);
+	
+	
 	int			chunksize=100000;
 	int			timelimit=5;
 	
@@ -33084,9 +33285,9 @@ int Jidac::benchmark()
 	}
 
 	if (all)
-		if (thehashes.size()!=1)
+		if ((thehashes.size()!=1) && (thehashes.size()!=12))
 		{
-			printf("When benchmarking with -all (multithread), exactly ONE algo\n");
+			printf("When benchmarking with -all (multithread), ONE algo or nothing (all)\n");
 			free(buffer32bit);
 			return 1;
 		}
@@ -33101,31 +33302,28 @@ int Jidac::benchmark()
 	printf("Time limit %10d s  (-n X)\n",timelimit);
 	printf("Chunks of  %12s  (-minsize Y)\n",tohuman(chunksize*4));
 	printf("\n");
-
+	
+	
+	double			franzomips=0;
+	
 	if (all)
 	{
-		if (thehashes.size()!=1)
-		{
-			printf("When benchmarking with -all (multithread), only ONE algo\n");
-			return 1;
-		}
 		int mythreads=howmanythreads;
-	
-		if (!all)
-			mythreads=1;
-		
 		tparametribenchmark	myblock;
+
 		for (int i=0;i<mythreads;i++)
 		{
 			myblock.tnumber=i;
 			myblock.chunksize=chunksize;
 			myblock.buffer32bit=buffer32bit;
-			myblock.speed=0;
 			myblock.timelimit=timelimit;
-			myblock.runningalgo=thehashes[0];
+			if (thehashes.size()==12)
+				myblock.runningalgo="SHA-3";
+			else
+				myblock.runningalgo=thehashes[0];
 			vettoreparametribenchmark.push_back(myblock);
-		}
-	
+		}	
+
 		int rc;
 		pthread_t* threads = new pthread_t[mythreads];
 
@@ -33136,7 +33334,7 @@ int Jidac::benchmark()
 		pthread_attr_init(&attr);
 		pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
 
-		printf("\nCreating %d hashing thread(s) with %s\n\n",mythreads,mygetalgo().c_str());
+///		printf("\nCreating %d hashing thread(s) with %s\n\n",mythreads,mygetalgo().c_str());
 
 		setupConsole();
 		printf("\033[2J"); //cls
@@ -33164,16 +33362,32 @@ int Jidac::benchmark()
 			}
 		}
 		
-
 		setupConsole();
 		printf("\033[2J"); //cls
 		printf("\033[%d;0H",(int)1);
 		restoreConsole();
-		printf("Final results\n\n");
+		
+		string 			risultato;
+		s_benchmark		block;
+		
+		
 		double total_speed=0;
 		for (int i=0;i<mythreads;i++)
+		{
 			total_speed+=vettoreparametribenchmark[i].speed;
+			franzomips+=vettoreparametribenchmark[i].speed;
+		}
+		
 		printf("Total speed %s /s\n",tohuman(total_speed));
+
+		if (thehashes.size()==12)
+		{
+			franzomips/=1000000;
+			
+			for (int i=0;i<array_cpu.size();i++)
+				array_multi[i]=franzomips/array_multi[i]*100.0;
+		}
+
 		delete [] threads;
 	}
 	else
@@ -33190,17 +33404,48 @@ int Jidac::benchmark()
 			block.speed=speed;
 			block.risultati=risultato;
 			vettorerisultati.push_back(block);
+			if (thehashes[i]!="BLAKE3") // no HW implementation, please
+				franzomips+=speed;
+		}
+		/// magic 3500 single 4875 (5885 multi
+		if (thehashes.size()==12)
+		{
+			franzomips/=thehashes.size()-1;
+			franzomips/=1000000;
+			
+			for (int i=0;i<array_cpu.size();i++)
+				array_single[i]=franzomips/array_single[i]*100.0;
 		}
-
 		std::sort(vettorerisultati.begin(), vettorerisultati.end(), compare_s_benchmark);
-
-		printf("Results\n\n");
+		printf("\nResults:\n\n");
+		
 		for (unsigned int i=0;i<vettorerisultati.size();i++)
 			printf("%s\n",vettorerisultati[i].risultati.c_str());
 	}
 	free(buffer32bit);
+	///sysctl hw.model hw.machine hw.ncpu
+	if (thehashes.size()==12)
+	{			
+		if (all)
+			printf("\nfranzomips multi thread index %s ",migliaia(franzomips*7.0));
+		else
+			printf("\nfranzomips single thread index %s ",migliaia(franzomips));
+			
+		printf("(quick CPU check, raw %s)\n",migliaia(franzomips));
+			
+		for (int i=0;i<array_cpu.size();i++)
+			if (all)
+				printf("%s %10.2f %%\n",array_cpu[i].c_str(),array_multi[i]);
+			else
+				printf("%s %10.2f %%\n",array_cpu[i].c_str(),array_single[i]);			
+	}
 	return 0;
 }
+/*
+46184
+
+Single Thread Rating: 3499
+*/
 
 int Jidac::summa() 
 {
@@ -33795,11 +34040,21 @@ void my_handler(int s)
 	}
   
 	fflush(stdout);
+	if (g_fwritten!=g_fexpected)
+	{
+		printf("\n");
+		printbar('*');
+		printf("Something STRANGE happened. Archive seems corrupt. Media full?\n");
+		printf("WRITTEN BYTES  %19s\n",migliaia(g_fwritten));
+		printf("EXPECTED       %19s\n",migliaia(g_fexpected));
+		printbar('*');
+		errorcode=2;
+	  }
+
 	fprintf(stderr, "\n%1.3f seconds (%s)  %s\n", (mtime()-g_start)/1000.0,timetohuman((mtime()-g_start)/1000.0).c_str(),
       errorcode>1 ? "(with errors)" :
       errorcode>0 ? "(with warnings)" : "(all OK)");
 	  
-		
 	if (errorcode==2)
 	{
 		if (flagdebug)
@@ -34982,9 +35237,6 @@ int Jidac::consolidate(string i_archive)
 	
 }
 
-
-
-
 int Jidac::test() 
 {
 	getpasswordifempty();
@@ -34992,6 +35244,9 @@ int Jidac::test()
 	if (files.size()>0)
 	{
 		/// zpaqfranz t z:\1.zpaq k:\sorgente
+		if (flagchecksum)
+		printf("SHA-1-chunked verify+HASH checksum\n");
+		else
 		printf("SHA-1-chunked verify\n");
 		return list();
 	}