diff --git a/zpaqfranz.cpp b/zpaqfranz.cpp index fd8b9cc..36067c7 100644 --- a/zpaqfranz.cpp +++ b/zpaqfranz.cpp @@ -28,7 +28,7 @@ EXPERIMENTAL BUILD: the source is a mess, strongly in development MIT License -Copyright (c) 2021-2022 Franco Corbelli +Copyright (c) 2021-2023 Franco Corbelli Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation @@ -52,8 +52,23 @@ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -#define ZPAQ_VERSION "56.4j" -#define ZPAQ_DATE " (30 Dec 2022)" // cannot use __DATE__ on Debian! +#define ZPAQ_VERSION "57.1g" +#define ZPAQ_DATE " (24 Jan 2023)" // cannot use __DATE__ on Debian! + + +/// optional align for malloc (sparc64) via -DALIGNMALLOC +#define STR(a) #a +#define XSTR(a) STR(a) + +#ifdef ALIGNMALLOC +#define MALLOC_ALIGN 4 +#endif + +#ifdef MALLOC_ALIGN + #define TEXT_ALIGN "." XSTR(MALLOC_ALIGN) +#else + #define TEXT_ALIGN "" +#endif #if defined(_WIN64) #define ZSFX_VERSION "SFX64 v55.1," @@ -720,25 +735,178 @@ Credits and copyrights and licenses and links This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Written by Paul Rubin, David MacKenzie, and Stuart Kemp. - + +20 [Apache License 2.0] HighWay64 hasher https://github.com/google/highwayhash + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + 1. Definitions. + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + Greetings 0 Of course Dr. Matt Mahoney http://mattmahoney.net - 1 Thanks to JFLarvoire for usefun (yes, usefun) informations https://github.com/JFLarvoire/SysToolsLib/blob/master/C/MsvcLibX/src/readlink.c - 2 Thanks to Bulat Ziganshin for contribution on Slicing-by-16 for crc32 - 3 Thanks to SeDD user of the encode.ru forum for SFX debugging - 4 Thanks to Aki user of forums.debian.net for some Debian's packager help - 5 Thanks to https://github.com/dertuxmalwieder for testing on various Unixes and OpenBSD port - 6 Thanks to Felix Palmen for great help on FreeBSD "packaging" - 7 Thanks to https://github.com/omar-polo for a merged-unmerged-hardcoded NOJIT fix - 8 Thanks to https://github.com/Piqlet for non-x86 help - 9 Thanks to https://github.com/osmano807 for non-x86 help -10 Thanks to Stephen Kitt for supporting Debian "packaging" -11 Thanks to Niels Ferguson for the Twofish implementation -12 Thanks to Newcastle University for some winsock related issues, Master Degree, Game Engineering -13 Thanks to https://github.com/akumiszcza for OneDrive issue -14 Thanks to https://github.com/ratay for help help fix, longpath -15 Thanks to https://github.com/graphixillusion for "lost" -vss + 1 Thanks to JFLarvoire for usefun (yes, usefun) informations https://github.com/JFLarvoire/SysToolsLib/blob/master/C/MsvcLibX/src/readlink.c + 2 Thanks to Bulat Ziganshin for contribution on Slicing-by-16 for crc32 + 3 Thanks to SeDD user of the encode.ru forum for SFX debugging + 4 Thanks to Aki user of forums.debian.net for some Debian's packager help + 5 Thanks to https://github.com/dertuxmalwieder for testing on various Unixes and OpenBSD port + 6 Thanks to Felix Palmen for great help on FreeBSD "packaging" + 7 Thanks to https://github.com/omar-polo for a merged-unmerged-hardcoded NOJIT fix + 8 Thanks to https://github.com/Piqlet for non-x86 help + 9 Thanks to https://github.com/osmano807 for non-x86 help +10 Thanks to Stephen Kitt for supporting Debian "packaging" +11 Thanks to Niels Ferguson for the Twofish implementation +12 Thanks to Newcastle University for some winsock related issues, Master Degree, Game Engineering +13 Thanks to https://github.com/akumiszcza for OneDrive issue +14 Thanks to https://github.com/ratay for help help fix, longpath +15 Thanks to https://github.com/graphixillusion for "lost" -vss +16 Thanks to https://discuss.haiku-os.org/u/PulkoMandy for Haiku Almost "universal" (minimal) Makefile. Beware /usr/ LOCAL /bin! @@ -781,7 +949,7 @@ wget https://github.com/fcorbelli/zpaqfranz/raw/main/NONWINDOWS/Makefile make install clean ``` -Dirtiest (!) no SSL certificate (very old systems) +Dirtiest (!) no SSL certificate (very old systems), nightly build ``` wget http://www.francocorbelli.it/zpaqfranz.cpp -O zpaqfranz.cpp ``` @@ -827,6 +995,8 @@ DEFINEs at compile-time -DESX // Yes, zpaqfranz run (kind of) on ESXi too :-) +-DALIGNMALLOC // Force malloc to be aligned at something (sparc64) + -DSERVER // Enable the cloudpaq client (for Windows) HIDDEN GEMS @@ -985,6 +1155,14 @@ ESXi (gcc 3.4.6) Note: not fully developed ( extract() with minimum RAM need to be implemented ) g++ -O3 -DESX zpaqfranz.cpp -o zpaqfranz6 -pthread -static -s +sparc64 (not tested) +try +-DALIGNMALLOC (+ other switches) + +Haiku R1/beta4, 64 bit (gcc 11.2.0), hrev56721 +Not very tested +g++ -O3 -Dunix zpaqfranz.cpp -o zpaqfranz -pthread -static + Beware of #definitions g++ -dM -E - < /dev/null @@ -994,12 +1172,29 @@ sometimes __sun, sometimes not */ +/* + zpaqfranz can use a backwards compatible archive format, using various "packages" + Each FRANZOFFSET occupies, for each archived file, a certain space, + upto ~500 bytes, but they are still compressed, so there is less waste + + V1 (little overhead) is the default + V2 (up to zpaqfranz 56) is for extended hashes, like sha-3 + V3 (from zpaqfranz 57+) is variable-sized in two + "flavours": 190 bytes and 550. + The shorter allows very long hashes (e.g. whirlpool), + the bigger also contains a TAR-like format for storing + symlinks (and other information) +*/ - -#define FRANZOFFSET 50 -#define FRANZOFFSETSHA256 76 +#define FRANZOFFSETV1 50 +#define FRANZOFFSETV2 76 +#define FRANZOFFSETV3 550 // 190 + 360 posix #define FRANZMAXPATH 240 +/* + FRANZO_SOMETHING can be used inside zpaq archive + ALGO_SOMETHING cannot (just for the sum command) +*/ #define FRANZO_NONE 0 #define FRANZO_CRC_32 1 #define FRANZO_XXHASH64 2 @@ -1010,6 +1205,30 @@ sometimes __sun, sometimes not #define FRANZO_SHA3 7 #define FRANZO_MD5 8 #define FRANZO_WINHASH64 9 +#define FRANZO_WHIRLPOOL 10 +#define FRANZO_HIGHWAY64 11 +#define FRANZO_HIGHWAY128 12 +#define FRANZO_HIGHWAY256 13 + +#define ALGO_CRC32 FRANZO_CRC_32 +#define ALGO_XXHASH64 FRANZO_XXHASH64 +#define ALGO_SHA1 FRANZO_SHA_1 +#define ALGO_SHA256 FRANZO_SHA_256 +#define ALGO_XXH3 FRANZO_XXH3 +#define ALGO_BLAKE3 FRANZO_BLAKE3 +#define ALGO_SHA3 FRANZO_SHA3 +#define ALGO_MD5 FRANZO_MD5 +#define ALGO_WINHASH64 FRANZO_WINHASH64 +#define ALGO_WHIRLPOOL FRANZO_WHIRLPOOL +#define ALGO_HIGHWAY64 FRANZO_HIGHWAY64 +#define ALGO_HIGHWAY128 FRANZO_HIGHWAY128 +#define ALGO_HIGHWAY256 FRANZO_HIGHWAY256 + +#define ALGO_CRC32C 100 +#define ALGO_WYHASH 101 +#define ALGO_NILSIMSA 103 +#define ALGO_ENTROPY 104 + #define _FILE_OFFSET_BITS 64 // In Linux make sizeof(off_t) == 8. Define BEFORE including windows.h!!! #ifndef UNICODE @@ -1046,6 +1265,7 @@ sometimes __sun, sometimes not #include #include #include +/// #include #include #include #include @@ -1057,7 +1277,12 @@ sometimes __sun, sometimes not #include #include #include +#ifndef __HAIKU__ #include +#endif +#ifdef __HAIKU__ + #include +#endif #include #include #include @@ -1086,6 +1311,7 @@ sometimes __sun, sometimes not #include #include #include +/// #include #include #include #include @@ -1108,6 +1334,152 @@ string g_cloudkey =""; int64_t g_enter =0; #endif + +/// OK we need a fix for 64-byte-align problem on some Linux compiler +/* +https://github.com/embeddedartistry/embedded-resources/blob/master/examples/c/malloc_aligned.c +*/ +#ifndef align_up +#define align_up(num, align) \ + (((num) + ((align) - 1)) & ~((align) - 1)) +#endif +//Convenience macro for memalign, the linux API +///#define memalign(align, size) aligned_malloc(align, size) +//Number of bytes we're using for storing the aligned pointer offset +typedef uint16_t myoffset_t; +#define PTR_OFFSET_SZ sizeof(myoffset_t) +/** +* aligned_malloc takes in the requested alignment and size +* We will call malloc with extra bytes for our header and the offset +* required to guarantee the desired alignment. +*/ +void * aligned_malloc(size_t align, size_t size) +{ + void * ptr = NULL; + //We want it to be a power of two since align_up operates on powers of two + assert((align & (align - 1)) == 0); + if(align && size) + { + /* + * We know we have to fit an offset value + * We also allocate extra bytes to ensure we can meet the alignment + */ + uint32_t hdr_size = PTR_OFFSET_SZ + (align - 1); + void * p = malloc(size + hdr_size); + if(p) + { + /* + * Add the offset size to malloc's pointer (we will always store that) + * Then align the resulting value to the arget alignment + */ + ptr = (void *) align_up(((uintptr_t)p + PTR_OFFSET_SZ), align); + //Calculate the offset and store it behind our aligned pointer + *((myoffset_t *)ptr - 1) = (myoffset_t)((uintptr_t)ptr - (uintptr_t)p); + } // else NULL, could not malloc + } //else NULL, invalid arguments + return ptr; +} +/** +* aligned_free works like free(), but we work backwards from the returned +* pointer to find the correct offset and pointer location to return to free() +* Note that it is VERY BAD to call free() on an aligned_malloc() pointer. +*/ +void aligned_free(void * ptr) +{ + assert(ptr); + /* + * Walk backwards from the passed-in pointer to get the pointer offset + * We convert to an offset_t pointer and rely on pointer math to get the data + */ + myoffset_t offset = *((myoffset_t *)ptr - 1); + /* + * Once we have the offset, we can get our original pointer and call free + */ + void * p = (void *)((uint8_t *)ptr - offset); + free(p); +} + +/* + Let's try to force align everywhere, on some kind of "choosy" CPUs + I do not have such kind of iron, just a try... +*/ +static void* franz_malloc(size_t i_size) +{ + assert(i_size>0); +#ifdef MALLOC_ALIGN + return aligned_malloc(MALLOC_ALIGN,i_size); +#else + return malloc(i_size); +#endif +} + +static void franz_free(void* i_mem) +{ + assert (i_mem!=NULL); +#ifdef MALLOC_ALIGN + aligned_free(i_mem); +#else + free(i_mem); +#endif +} + +static void* franz_extend(void* i_mem,size_t i_size,size_t i_oldsize) +{ +#ifdef MALLOC_ALIGN + void *new_mem; + if (i_size==0) + { +///#ifdef DEBUG + printf("01200: realloc i_size == 0\n"); +///#endif + franz_free(i_mem); + return NULL; + } + if (!i_mem) + { +///#ifdef DEBUG + printf("01205: realloc from empty, allocating %08d\n",i_size); +///#endif + return franz_malloc(i_size); + } + if (i_oldsize0) + i_oldsize++; /// compiler be quiet! + return realloc(i_mem,i_size); +#endif +} + + + + + + + + + + + + + + + /// very quick and very dirty output inline char * migliaia(int64_t n) { @@ -2191,7 +2563,7 @@ class StringBuffer: public libzpaq::Reader, public libzpaq::Writer { assert(!al==!p); if (a<=al) return; unsigned char* q=0; - if (a>0) q=(unsigned char*)(p ? realloc(p, a) : malloc(a)); + if (a>0) q=(unsigned char*)(p ? franz_extend(p,a,al) : franz_malloc(a)); if (a>0 && !q) error("Out of memory"); p=q; al=a; @@ -2217,14 +2589,14 @@ class StringBuffer: public libzpaq::Reader, public libzpaq::Writer { // Set output limit void setLimit(size_t n) {limit=n;} // Free memory - ~StringBuffer() {if (p) free(p);} + ~StringBuffer() {if (p) franz_free(p);} // Return number of bytes written. size_t size() const {return wpos;} // Return number of bytes left to read size_t remaining() const {return wpos-rpos;} // Reset size to 0 and free memory. void reset() { - if (p) free(p); + if (p) franz_free(p); p=0; al=rpos=wpos=0; } @@ -7827,8 +8199,8 @@ divsufsort(const unsigned char *T, int *SA, int n) { else if(n == 0) { return 0; } else if(n == 1) { SA[0] = 0; return 0; } else if(n == 2) { m = (T[0] < T[1]); SA[m ^ 1] = 0, SA[m] = 1; return 0; } - bucket_A = (int *)malloc(BUCKET_A_SIZE * sizeof(int)); - bucket_B = (int *)malloc(BUCKET_B_SIZE * sizeof(int)); + bucket_A = (int *)franz_malloc(BUCKET_A_SIZE * sizeof(int)); + bucket_B = (int *)franz_malloc(BUCKET_B_SIZE * sizeof(int)); /* Suffixsort. */ if((bucket_A != NULL) && (bucket_B != NULL)) { m = sort_typeBstar(T, SA, bucket_A, bucket_B, n); @@ -7836,8 +8208,8 @@ divsufsort(const unsigned char *T, int *SA, int n) { } else { err = -2; } - free(bucket_B); - free(bucket_A); + franz_free(bucket_B); + franz_free(bucket_A); return err; } // End divsufsort.c @@ -9100,26 +9472,6 @@ struct hash_autocheck hash_autocheck() {ok="";calculated="";}; }; typedef map MAPPAAUTOCHECK; -enum ealgoritmi { ALGO_SHA1,ALGO_CRC32C,ALGO_CRC32,ALGO_XXH3,ALGO_SHA256,ALGO_WYHASH,ALGO_XXHASH64,ALGO_BLAKE3,ALGO_WHIRLPOOL,ALGO_MD5,ALGO_SHA3,ALGO_NILSIMSA,ALGO_ENTROPY,ALGO_WINHASH64,ALGO_LAST }; -typedef std::map algoritmi; -const algoritmi::value_type rawData[] = -{ - algoritmi::value_type(ALGO_SHA1,"sha1"), - algoritmi::value_type(ALGO_CRC32,"crc32"), - algoritmi::value_type(ALGO_CRC32C,"crc32c"), - algoritmi::value_type(ALGO_XXH3,"xxh3"), - algoritmi::value_type(ALGO_XXHASH64,"xxhash"), - algoritmi::value_type(ALGO_SHA256,"sha256"), - algoritmi::value_type(ALGO_WYHASH,"wyhash"), - algoritmi::value_type(ALGO_BLAKE3,"blake3"), - algoritmi::value_type(ALGO_WHIRLPOOL,"whirlpool"), - algoritmi::value_type(ALGO_MD5,"md5"), - algoritmi::value_type(ALGO_MD5,"sha3"), - algoritmi::value_type(ALGO_NILSIMSA,"nilsimsa"), - algoritmi::value_type(ALGO_ENTROPY,"entropy"), -}; -const int numElems = sizeof rawData / sizeof rawData[0]; -algoritmi myalgoritmi(rawData, rawData + numElems); struct s_crc32block { string filename; @@ -9194,86 +9546,108 @@ int64_t g_dateto =0; string g_until; int g_rangefrom =0; int g_rangeto =0; -bool flagdonotforcexls; -bool flagfilelist; -bool flag715; -bool flagzero; -bool flagforcezfs; -bool flagspace; -bool flagdebug; -bool flagutc; -bool flagnoeta; -bool flagpakka; -bool flagvss; -bool flagverbose; -bool flagverify; -bool flagkill; -bool flagchecksum; -bool flagutf; -bool flagflat; -bool flagparanoid; -bool flagfix255; -bool flagfixcase; + + +bool flagnochecksum; +bool flagcrc32; +bool flagxxhash64; +bool flagsha1; +bool flagsha256; +bool flagxxh3; +bool flagmd5; +bool flagblake3; +bool flagsha3; +bool flagwhirlpool; +bool flaghighway64; +bool flaghighway128; +bool flaghighway256; +bool flagcrc32c; +bool flagwyhash; +bool flagnilsimsa; +bool flagentropy; +bool flagwindate; + + +bool flag715; +bool flagappend; +bool flagbig; +bool flagchecksum; +bool flagchecktxt; +bool flagcomment; +bool flagdebug; +bool flagdesc; +bool flagdonotforcexls; +bool flagfilelist; +bool flagfix255; +bool flagfixeml; +bool flagflat; +bool flagforce; +bool flagforcewindows; +bool flagforcezfs; +bool flagfrugal; +bool flaghashdeep; +bool flagkill; +bool flagmm; +bool flagnoattributes; +bool flagnodedup; +bool flagnoeta; +bool flagnopath; +bool flagnoqnap; +bool flagnorecursion; +bool flagnosort; +bool flagpakka; +bool flagparanoid; +bool flagramdisk; +bool flagrename; +bool flagsilent; +bool flagskipzfs; +bool flagspace; +bool flagssd; +bool flagstat; +bool flagstdin; +bool flagstdout; +bool flagstore; +bool flagtar; +bool flagtest; +bool flagtouch; +bool flagutc; +bool flagutf; +bool flagverbose; +bool flagverify; +bool flagvss; +bool flagzero; + +bool flagfixcase; + #ifdef _WIN32 -bool flaglongpath; -bool flagfixreserved; -bool flagfindzpaq; -bool flagwindate; -bool flagopen; -bool flagdd; -bool flagimage; -#endif -bool flagfixeml; +bool flagdd; +bool flagfindzpaq; +bool flagfixreserved; +bool flagimage; +bool flaglongpath; +bool flagopen; +#endif + +bool flaghw; + bool flagbarraod; bool flagbarraon; bool flagbarraos; -bool flagcrc32c; -bool flagentropy; -bool flagsha1; -bool flagxxh3; -bool flagcrc32; -bool flagsha256; -bool flagwyhash; -bool flagxxhash64; -bool flagblake3; -bool flagwhirlpool; -bool flagmd5; -bool flagnilsimsa; -bool flagsha3; -bool flagmm; -bool flagappend; -bool flaghw; -bool flagdesc; -bool flagnodedup; -bool flagtar; -bool flagramdisk; -bool flagssd; -bool flaghashdeep; -bool flagbig; -bool flagchecktxt; -bool flagstdin; -bool flagstdout; -bool flagtouch; -bool flagstat; -bool flagfrugal; -bool flagsilent; -bool flagnorecursion; -bool flagrename; + + int g_franzotype; string g_optional; string orderby; vector g_theorderby; -MAPPACHECK g_mychecks; #ifdef SERVER bool flagserver; #endif int g_ioBUFSIZE=4096; +int g_thechosenhash; +string g_thechosenhash_str; -/* - Section: hashers -*/ void seppuku() { if (g_output_handle) @@ -9292,6 +9666,244 @@ void myprintf(const char *fmt, ...) fprintf(g_output_handle,"%s",buffer); va_end(args); } + +typedef map MAPPAFLAGS; +typedef map HELPFLAGS; + +/* +struct franz_flag +{ + string name; + string help; + string scope; + bool *theflag; + franz_flag(string i_name, string i_help, bool* i_flag) + { + if (i_name=="") + { + printf("48111: GURU i_name empty\n"); + seppuku(); + } + if (i_flag==NULL) + { + printf("48107: GURU i_flag NULL\n"); + seppuku(); + } + name=i_name; + help=i_help; + theflag=i_flag; + }; +}; +*/ + +class franz_flags +{ + public: + MAPPAFLAGS mappaflags; + HELPFLAGS helpflags; + HELPFLAGS helpflagsscope; + + + bool get(string i_name) + { + MAPPAFLAGS::iterator p=mappaflags.find(i_name); + if (p==mappaflags.end()) + { + printf("42098: guru doing getflag %s \n",i_name.c_str()); + seppuku(); + } + if (p->second==NULL) + { + printf("42913: guru empty pointer flag%s\n",i_name.c_str()); + seppuku(); + } + return (*p->second); + } + bool istrue(string i_name) + { + return ( get(i_name)==true); + } + bool isfalse(string i_name) + { + return ( get(i_name)==false); + } + + bool exists(string i_name) + { + return (mappaflags.find(i_name)!=mappaflags.end()); + } + void set(string i_name, bool i_value) + { + + MAPPAFLAGS::iterator p=mappaflags.find(i_name); + if (p==mappaflags.end()) + { + } + if (p==mappaflags.end()) + { + printf("42923: GURU doing setflag %s \n",i_name.c_str()); + seppuku(); + } + if (p->second==NULL) + { + printf("42928: guru empty pointer flag%s\n",i_name.c_str()); + seppuku(); + } + + (*p->second)=i_value; + } + void settrue(string i_name) + { + set(i_name,true); + } + void setfalse(string i_name) + { + set(i_name,false); + } + string gethelp(string i_name) + { + HELPFLAGS::iterator p=helpflags.find(i_name); + if (p!=helpflags.end()) + return p->second; + return ""; + } + bool* getbool(string i_name) + { + + MAPPAFLAGS::iterator p=mappaflags.find(i_name); + if (p!=mappaflags.end()) + return p->second; + + return NULL; + } + + + void debugga() + { + printf("48149: array franz flag size %s\n",migliaia(mappaflags.size())); + + for (MAPPAFLAGS::iterator p=mappaflags.begin(); p!=mappaflags.end(); ++p) + { + printf("48150: %-20s %d ",p->first.c_str(),(int)*p->second); + HELPFLAGS::iterator a=helpflags.find(p->first); + if (p!=mappaflags.end()) + printf(" <<%s>>",a->second.c_str()); + printf("\n"); + } + } + + string compact() + { + + string risultato=""; + for (MAPPAFLAGS::iterator p=mappaflags.begin(); p!=mappaflags.end(); ++p) + if (*p->second) + risultato+=p->first+' '; + return risultato; + } + + void add(bool* i_thebool,string i_name,string i_help,string i_helpscope) + { + if (i_name=="") + { + printf("48125: GURU i_name empty\n"); + seppuku(); + } + if (i_thebool==NULL) + { + printf("09778: GURU thebool NULL\n"); + seppuku(); + } + + MAPPAFLAGS::iterator p=mappaflags.find(i_name); + if (p==mappaflags.end()) + { + *(i_thebool)=false; + mappaflags.insert(std::pair(i_name, i_thebool)); + if (i_help!="") + helpflags.insert(std::pair(i_name, i_help)); + if (i_helpscope!="") + helpflags.insert(std::pair(i_name, i_helpscope)); + } + } + +}; + +franz_flags g_programflags; + +/* + Section: hashers +*/ + +bool isdirectory(string i_filename) +{ + if (i_filename.length()==0) + return false; + else + return + i_filename[i_filename.size()-1]=='/'; +} + +typedef string (*finalize_function)(void*); // type for conciseness + +struct tipohash +{ + string hashname; + string switchname; + string hashdescription; + string franzocode; + string hasherror; + int hashlen; + bool* switchflag; + bool flagiszpaq; + bool flagisbenchmark; + int checkedok; + int checkedfailed; + int checkednotfound; + int64_t checksize; + + finalize_function ffinalize; + + tipohash(string i_hashname,int i_hashlen, string i_hashdescription,bool i_iszpaq,string i_switchname,bool* i_switchflag,finalize_function i_finalize,string i_franzcode): switchflag(NULL),flagiszpaq(i_iszpaq),flagisbenchmark(false), + checkedok(0),checkedfailed(0),checkednotfound(0),checksize(0) + { + hashname =i_hashname; + hashdescription =i_hashdescription; + switchname =i_switchname; + switchflag =i_switchflag; + ffinalize =i_finalize; + hashlen =i_hashlen; + franzocode =""; + if (i_franzcode.size()==2) + franzocode=std::string()+ i_franzcode[0]+i_franzcode[1]; + hasherror="!ERROR!"; + while (hasherror.size()<(unsigned int)i_hashlen) + hasherror+=' '; + + } + void resetcheckstat() + { + checkedok=0; + checkedfailed=0; + checkednotfound=0; + checksize=0; + } + +}; + +///typedef std::unordered_map MAPPATIPOHASH; +typedef std::map MAPPATIPOHASH; + +MAPPATIPOHASH g_mappatipohash; + +tipohash* franz_get_hash(string i_hashstring) +{ + for (MAPPATIPOHASH::iterator p=g_mappatipohash.begin(); p!=g_mappatipohash.end(); ++p) + if (i_hashstring==p->second.hashname) + return &p->second; + return NULL; +} + ///https://patorjk.com/software/taag/#p=display&f=Graffiti&t=Type%20Something%20 ///banner font void bigguru() @@ -9360,116 +9972,58 @@ string binarytohex(const unsigned char* i_risultato,const int i_lunghezza) } return risultato; } +// no to_string on old gcc +string myto_string(int64_t i_number) +{ + char buffer[100]; + snprintf(buffer,sizeof(buffer),"%lld",(long long)i_number); + string risultato=buffer; // explicit converting + return risultato; +} + string decodefranzoffset(int franzotype) { if (franzotype==FRANZO_NONE) return "NOTHING (LIKE 7.15)"; if (franzotype==FRANZO_CRC_32) return "CRC-32"; - if (franzotype==FRANZO_XXHASH64) - return "XXHASH64+CRC-32"; - if (franzotype==FRANZO_WINHASH64) - return "WINHASH64+CRC-32"; - if (franzotype==FRANZO_SHA_1) - return "SHA-1+CRC-32"; - if (franzotype==FRANZO_SHA_256) - return "SHA-256+CRC-32"; - if (franzotype==FRANZO_XXH3) - return "XXH3+CRC-32"; - if (franzotype==FRANZO_BLAKE3) - return "BLAKE3+CRC-32"; - if (franzotype==FRANZO_SHA3) - return "SHA-3+CRC-32"; - if (franzotype==FRANZO_MD5) - return "MD5+CRC-32"; -/* - string temp="16839: franzotype strange "+franzotype; - perror(temp.c_str()); - */ - return "BYPASSWARNING"; -} -string mygetalgo() -{ - if (flagblake3) - return "BLAKE3"; - else - if (flagxxhash64) - return "XXHASH64"; - else -#ifdef _WIN32 - if (flagwindate) - return "WINHASH64"; - else -#endif - if (flagwyhash) - return "WYHASH"; - else - if (flagcrc32) - return "CRC-32"; - else - if (flagentropy) - return "ENTROPY"; - else - if (flagcrc32c) - return "CRC-32C"; - else - if (flagxxh3) - return "XXH3"; - else - if (flagsha256) - return "SHA-256"; - else - if (flagwhirlpool) - return "WHIRLPOOL"; - else - if (flagmd5) - return "MD5"; - else - if (flagnilsimsa) - return "NILSIMSA"; - else - if (flagsha3) - return "SHA-3"; - else - return "SHA-1"; + + MAPPATIPOHASH::iterator p=g_mappatipohash.find(franzotype); + if (p==g_mappatipohash.end()) + { + string temp="16839: franzotype strange "+myto_string(franzotype); + perror(temp.c_str()); + } + if (!p->second.flagiszpaq) + { + string temp="09603: franzotype does not seems iszpaq "+myto_string(franzotype); + perror(temp.c_str()); + } + + return p->second.hashname+"+CRC-32"; } -int flag2algo() +string decodefranzocode(int franzotype) { - if (flagblake3) - return ALGO_BLAKE3; - else - if (flagxxhash64) - return ALGO_XXHASH64; - else - if (flagwyhash) - return ALGO_WYHASH; - else - if (flagwhirlpool) - return ALGO_WHIRLPOOL; - else - if (flagmd5) - return ALGO_MD5; - else - if (flagnilsimsa) - return ALGO_NILSIMSA; - else - if (flagsha3) - return ALGO_SHA3; - else - if (flagcrc32) - return ALGO_CRC32; - else - if (flagcrc32c) - return ALGO_CRC32C; - else - if (flagxxh3) - return ALGO_XXH3; - else - if (flagsha256) - return ALGO_SHA256; - else - return ALGO_SHA1; + if (franzotype==FRANZO_NONE) + return ""; + if (franzotype==FRANZO_CRC_32) + return ""; + + MAPPATIPOHASH::iterator p=g_mappatipohash.find(franzotype); + if (p==g_mappatipohash.end()) + { + string temp="09620: franzotype strange "+myto_string(franzotype); + perror(temp.c_str()); + } + if (!p->second.flagiszpaq) + { + string temp="09625: franzotype does not seems iszpaq "+myto_string(franzotype); + perror(temp.c_str()); + } + + return p->second.franzocode; } + string emptyalgo(const string i_string) { if (i_string=="BLAKE3") @@ -9508,6 +10062,15 @@ string emptyalgo(const string i_string) if (i_string=="NILSIMSA") return "VUOTO"; else + if (i_string=="HIGHWAY64") + return "92943A34E3447FAA"; + else + if (i_string=="HIGHWAY128") + return "D39FAE4F2222AD3611D8E8215B1B01F2"; + else + if (i_string=="HIGHWAY256") + return "D6EB09DBE820BC5A8050209986CEDE586E78E6815864E99DC22811C2FCFE001B"; + else if (i_string=="SHA-3") return "A7FFC6F8BF1ED76651C14756A061D662F580FF4DE43B49FA82D80A4B80F8434A"; else @@ -9708,6 +10271,335 @@ static int __score(int num) { return __POPC[num]; } + + +/// This is a (minor) rework of https://github.com/google/highwayhash/tree/master/c +/* +By Jan Wassenberg jan.wassenberg@gmail.com and Jyrki Alakuijala jyrki.alakuijala@gmail.com + +This is not an official Google product. +*/ + + +/*////////////////////////////////////////////////////////////////////////////*/ +/* Low-level API, use for implementing streams etc... */ +/*////////////////////////////////////////////////////////////////////////////*/ + +typedef struct { + uint64_t v0[4]; + uint64_t v1[4]; + uint64_t mul0[4]; + uint64_t mul1[4]; +} HighwayHashState; + +/* Initializes state with given key */ +static void HighwayHashReset(const uint64_t key[4], HighwayHashState* state); +/* Takes a packet of 32 bytes */ +void HighwayHashUpdatePacket(const uint8_t* packet, HighwayHashState* state); +/* Adds the final 1..31 bytes, do not use if 0 remain */ +void HighwayHashUpdateRemainder(const uint8_t* bytes, const size_t size_mod32, + HighwayHashState* state); +/* Compute final hash value. Makes state invalid. */ +static uint64_t HighwayHashFinalize64(HighwayHashState* state); +static void HighwayHashFinalize128(HighwayHashState* state, uint64_t hash[2]); +static void HighwayHashFinalize256(HighwayHashState* state, uint64_t hash[4]); + +/*////////////////////////////////////////////////////////////////////////////*/ +/* Non-cat API: single call on full data */ +/*////////////////////////////////////////////////////////////////////////////*/ + +uint64_t HighwayHash64(const uint8_t* data, size_t size, const uint64_t key[4]); + +void HighwayHash128(const uint8_t* data, size_t size, + const uint64_t key[4], uint64_t hash[2]); + +void HighwayHash256(const uint8_t* data, size_t size, + const uint64_t key[4], uint64_t hash[4]); + +/*////////////////////////////////////////////////////////////////////////////*/ +/* Cat API: allows appending with multiple calls */ +/*////////////////////////////////////////////////////////////////////////////*/ + +typedef struct { + HighwayHashState state; + uint8_t packet[32]; + int num; +} HighwayHashCat; + +/* Allocates new state for a new streaming hash computation */ +void HighwayHashCatStart(const uint64_t key[4], HighwayHashCat* state); + +void HighwayHashCatAppend(const uint8_t* bytes, size_t num, + HighwayHashCat* state); + +/* Computes final hash value */ +uint64_t HighwayHashCatFinish64(const HighwayHashCat* state); +void HighwayHashCatFinish128(const HighwayHashCat* state, uint64_t hash[2]); +void HighwayHashCatFinish256(const HighwayHashCat* state, uint64_t hash[4]); + + + +/* +This code is compatible with C90 with the additional requirement of +supporting uint64_t. +*/ + +/*////////////////////////////////////////////////////////////////////////////*/ +/* Internal implementation */ +/*////////////////////////////////////////////////////////////////////////////*/ + +void HighwayHashReset(const uint64_t key[4], HighwayHashState* state) { + state->mul0[0] = 0xdbe6d5d5fe4cce2full; + state->mul0[1] = 0xa4093822299f31d0ull; + state->mul0[2] = 0x13198a2e03707344ull; + state->mul0[3] = 0x243f6a8885a308d3ull; + state->mul1[0] = 0x3bd39e10cb0ef593ull; + state->mul1[1] = 0xc0acf169b5f18a8cull; + state->mul1[2] = 0xbe5466cf34e90c6cull; + state->mul1[3] = 0x452821e638d01377ull; + state->v0[0] = state->mul0[0] ^ key[0]; + state->v0[1] = state->mul0[1] ^ key[1]; + state->v0[2] = state->mul0[2] ^ key[2]; + state->v0[3] = state->mul0[3] ^ key[3]; + state->v1[0] = state->mul1[0] ^ ((key[0] >> 32) | (key[0] << 32)); + state->v1[1] = state->mul1[1] ^ ((key[1] >> 32) | (key[1] << 32)); + state->v1[2] = state->mul1[2] ^ ((key[2] >> 32) | (key[2] << 32)); + state->v1[3] = state->mul1[3] ^ ((key[3] >> 32) | (key[3] << 32)); +} + +static void ZipperMergeAndAdd(const uint64_t v1, const uint64_t v0, + uint64_t* add1, uint64_t* add0) { + *add0 += (((v0 & 0xff000000ull) | (v1 & 0xff00000000ull)) >> 24) | + (((v0 & 0xff0000000000ull) | (v1 & 0xff000000000000ull)) >> 16) | + (v0 & 0xff0000ull) | ((v0 & 0xff00ull) << 32) | + ((v1 & 0xff00000000000000ull) >> 8) | (v0 << 56); + *add1 += (((v1 & 0xff000000ull) | (v0 & 0xff00000000ull)) >> 24) | + (v1 & 0xff0000ull) | ((v1 & 0xff0000000000ull) >> 16) | + ((v1 & 0xff00ull) << 24) | ((v0 & 0xff000000000000ull) >> 8) | + ((v1 & 0xffull) << 48) | (v0 & 0xff00000000000000ull); +} + +static void Update(const uint64_t lanes[4], HighwayHashState* state) { + int i; + for (i = 0; i < 4; ++i) { + state->v1[i] += state->mul0[i] + lanes[i]; + state->mul0[i] ^= (state->v1[i] & 0xffffffff) * (state->v0[i] >> 32); + state->v0[i] += state->mul1[i]; + state->mul1[i] ^= (state->v0[i] & 0xffffffff) * (state->v1[i] >> 32); + } + ZipperMergeAndAdd(state->v1[1], state->v1[0], &state->v0[1], &state->v0[0]); + ZipperMergeAndAdd(state->v1[3], state->v1[2], &state->v0[3], &state->v0[2]); + ZipperMergeAndAdd(state->v0[1], state->v0[0], &state->v1[1], &state->v1[0]); + ZipperMergeAndAdd(state->v0[3], state->v0[2], &state->v1[3], &state->v1[2]); +} + +static uint64_t Read64(const uint8_t* src) { + return (uint64_t)src[0] | ((uint64_t)src[1] << 8) | + ((uint64_t)src[2] << 16) | ((uint64_t)src[3] << 24) | + ((uint64_t)src[4] << 32) | ((uint64_t)src[5] << 40) | + ((uint64_t)src[6] << 48) | ((uint64_t)src[7] << 56); +} + +void HighwayHashUpdatePacket(const uint8_t* packet, HighwayHashState* state) { + uint64_t lanes[4]; + lanes[0] = Read64(packet + 0); + lanes[1] = Read64(packet + 8); + lanes[2] = Read64(packet + 16); + lanes[3] = Read64(packet + 24); + Update(lanes, state); +} + +static void Rotate32By(uint64_t count, uint64_t lanes[4]) { + int i; + for (i = 0; i < 4; ++i) { + uint32_t half0 = lanes[i] & 0xffffffff; + uint32_t half1 = (lanes[i] >> 32); + lanes[i] = (half0 << count) | (half0 >> (32 - count)); + lanes[i] |= (uint64_t)((half1 << count) | (half1 >> (32 - count))) << 32; + } +} + +void HighwayHashUpdateRemainder(const uint8_t* bytes, const size_t size_mod32, + HighwayHashState* state) { + int i; + const size_t size_mod4 = size_mod32 & 3; + const uint8_t* remainder = bytes + (size_mod32 & ~3); + uint8_t packet[32] = {0}; + for (i = 0; i < 4; ++i) { + state->v0[i] += ((uint64_t)size_mod32 << 32) + size_mod32; + } + Rotate32By(size_mod32, state->v1); + for (i = 0; i < remainder - bytes; i++) { + packet[i] = bytes[i]; + } + if (size_mod32 & 16) { + for (i = 0; i < 4; i++) { + packet[28 + i] = remainder[i + size_mod4 - 4]; + } + } else { + if (size_mod4) { + packet[16 + 0] = remainder[0]; + packet[16 + 1] = remainder[size_mod4 >> 1]; + packet[16 + 2] = remainder[size_mod4 - 1]; + } + } + HighwayHashUpdatePacket(packet, state); +} + +static void Permute(const uint64_t v[4], uint64_t* permuted) { + permuted[0] = (v[2] >> 32) | (v[2] << 32); + permuted[1] = (v[3] >> 32) | (v[3] << 32); + permuted[2] = (v[0] >> 32) | (v[0] << 32); + permuted[3] = (v[1] >> 32) | (v[1] << 32); +} + +void PermuteAndUpdate(HighwayHashState* state) { + uint64_t permuted[4]; + Permute(state->v0, permuted); + Update(permuted, state); +} + +static void ModularReduction(uint64_t a3_unmasked, uint64_t a2, uint64_t a1, + uint64_t a0, uint64_t* m1, uint64_t* m0) { + uint64_t a3 = a3_unmasked & 0x3FFFFFFFFFFFFFFFull; + *m1 = a1 ^ ((a3 << 1) | (a2 >> 63)) ^ ((a3 << 2) | (a2 >> 62)); + *m0 = a0 ^ (a2 << 1) ^ (a2 << 2); +} + +static uint64_t HighwayHashFinalize64(HighwayHashState* state) { + int i; + for (i = 0; i < 4; i++) { + PermuteAndUpdate(state); + } + + return state->v0[0] + state->v1[0] + state->mul0[0] + state->mul1[0]; +} + +static void HighwayHashFinalize128(HighwayHashState* state, uint64_t hash[2]) { + int i; + + for (i = 0; i < 6; i++) { + PermuteAndUpdate(state); + } + + hash[0] = state->v0[0] + state->mul0[0] + state->v1[2] + state->mul1[2]; + hash[1] = state->v0[1] + state->mul0[1] + state->v1[3] + state->mul1[3]; +} + +static void HighwayHashFinalize256(HighwayHashState* state, uint64_t hash[4]) { + int i; + /* We anticipate that 256-bit hashing will be mostly used with long messages + because storing and using the 256-bit hash (in contrast to 128-bit) + carries a larger additional constant cost by itself. Doing extra rounds + here hardly increases the per-byte cost of long messages. */ + for (i = 0; i < 10; i++) { + PermuteAndUpdate(state); + } + ModularReduction(state->v1[1] + state->mul1[1], state->v1[0] + state->mul1[0], + state->v0[1] + state->mul0[1], state->v0[0] + state->mul0[0], + &hash[1], &hash[0]); + ModularReduction(state->v1[3] + state->mul1[3], state->v1[2] + state->mul1[2], + state->v0[3] + state->mul0[3], state->v0[2] + state->mul0[2], + &hash[3], &hash[2]); +} + +/*////////////////////////////////////////////////////////////////////////////*/ +/* Non-cat API: single call on full data */ +/*////////////////////////////////////////////////////////////////////////////*/ + +static void ProcessAll(const uint8_t* data, size_t size, const uint64_t key[4], + HighwayHashState* state) { + size_t i; + HighwayHashReset(key, state); + for (i = 0; i + 32 <= size; i += 32) { + HighwayHashUpdatePacket(data + i, state); + } + if ((size & 31) != 0) HighwayHashUpdateRemainder(data + i, size & 31, state); +} + +uint64_t HighwayHash64(const uint8_t* data, size_t size, + const uint64_t key[4]) { + HighwayHashState state; + ProcessAll(data, size, key, &state); + return HighwayHashFinalize64(&state); +} + +void HighwayHash128(const uint8_t* data, size_t size, + const uint64_t key[4], uint64_t hash[2]) { + HighwayHashState state; + ProcessAll(data, size, key, &state); + HighwayHashFinalize128(&state, hash); +} + +void HighwayHash256(const uint8_t* data, size_t size, + const uint64_t key[4], uint64_t hash[4]) { + HighwayHashState state; + ProcessAll(data, size, key, &state); + HighwayHashFinalize256(&state, hash); +} + +/*////////////////////////////////////////////////////////////////////////////*/ +/* Cat API: allows appending with multiple calls */ +/*////////////////////////////////////////////////////////////////////////////*/ + +void HighwayHashCatStart(const uint64_t key[4], HighwayHashCat* state) { + HighwayHashReset(key, &state->state); + state->num = 0; +} + +void HighwayHashCatAppend(const uint8_t* bytes, size_t num, + HighwayHashCat* state) { + size_t i; + if (state->num != 0) { + size_t num_add = num > (32u - state->num) ? (32u - state->num) : num; + for (i = 0; i < num_add; i++) { + state->packet[state->num + i] = bytes[i]; + } + state->num += num_add; + num -= num_add; + bytes += num_add; + if (state->num == 32) { + HighwayHashUpdatePacket(state->packet, &state->state); + state->num = 0; + } + } + while (num >= 32) { + HighwayHashUpdatePacket(bytes, &state->state); + num -= 32; + bytes += 32; + } + for (i = 0; i < num; i++) { + state->packet[state->num] = bytes[i]; + state->num++; + } +} + +uint64_t HighwayHashCatFinish64(const HighwayHashCat* state) { + HighwayHashState copy = state->state; + if (state->num) { + HighwayHashUpdateRemainder(state->packet, state->num, ©); + } + return HighwayHashFinalize64(©); +} + +void HighwayHashCatFinish128(const HighwayHashCat* state, uint64_t hash[2]) { + HighwayHashState copy = state->state; + if (state->num) { + HighwayHashUpdateRemainder(state->packet, state->num, ©); + } + HighwayHashFinalize128(©, hash); +} + +void HighwayHashCatFinish256(const HighwayHashCat* state, uint64_t hash[4]) { + HighwayHashState copy = state->state; + if (state->num) { + HighwayHashUpdateRemainder(state->packet, state->num, ©); + } + HighwayHashFinalize256(©, hash); +} + + + /* This implementation is a minor reworked version of https://github.com/System-Glitch/SHA256 @@ -13773,8 +14665,8 @@ XXH_PUBLIC_API XXH128_hash_t XXH128(const void* data, size_t len, XXH64_hash_t s */ -static void* XXH_malloc(size_t s) { return malloc(s); } -static void XXH_free(void* p) { free(p); } +static void* XXH_malloc(size_t s) { return franz_malloc(s); } +static void XXH_free(void* p) { franz_free(p); } /*! and for memcpy() */ static void* XXH_memcpy(void* dest, const void* src, size_t size) @@ -17256,14 +18148,6 @@ class Semaphore { HANDLE h; // Windows semaphore }; #endif -bool isdirectory(string i_filename) -{ - if (i_filename.length()==0) - return false; - else - return - i_filename[i_filename.size()-1]=='/'; -} @@ -17994,6 +18878,17 @@ void moreprint(const char* i_stringa) } } } +void helpprint(string i_testo,string i_help) +{ + char linea[1000]; + while (i_testo.size()<25) + i_testo+=' '; + i_testo.resize(25); + string thehelp=g_programflags.gethelp(i_help); + snprintf(linea,sizeof(linea),"%s%s",i_testo.c_str(),thehelp.c_str()); + moreprint(linea); +} + void morebar(const char i_carattere) { int twidth=terminalwidth(); @@ -18244,69 +19139,7 @@ struct s_fileandsize bool filenotfound; s_fileandsize(): size(0),attr(0),date(0),data(-1),isdir(false),flaghashstored(false),hashok(false),filenotfound(false) {filename="";hashhex="";hashtype="";writtenfilename="";} }; -/// OK we need a fix for 64-byte-align problem on some Linux compiler -/* -https://github.com/embeddedartistry/embedded-resources/blob/master/examples/c/malloc_aligned.c -*/ -#ifndef align_up -#define align_up(num, align) \ - (((num) + ((align) - 1)) & ~((align) - 1)) -#endif -//Convenience macro for memalign, the linux API -///#define memalign(align, size) aligned_malloc(align, size) -//Number of bytes we're using for storing the aligned pointer offset -typedef uint16_t myoffset_t; -#define PTR_OFFSET_SZ sizeof(myoffset_t) -/** -* aligned_malloc takes in the requested alignment and size -* We will call malloc with extra bytes for our header and the offset -* required to guarantee the desired alignment. -*/ -void * aligned_malloc(size_t align, size_t size) -{ - void * ptr = NULL; - //We want it to be a power of two since align_up operates on powers of two - assert((align & (align - 1)) == 0); - if(align && size) - { - /* - * We know we have to fit an offset value - * We also allocate extra bytes to ensure we can meet the alignment - */ - uint32_t hdr_size = PTR_OFFSET_SZ + (align - 1); - void * p = malloc(size + hdr_size); - if(p) - { - /* - * Add the offset size to malloc's pointer (we will always store that) - * Then align the resulting value to the arget alignment - */ - ptr = (void *) align_up(((uintptr_t)p + PTR_OFFSET_SZ), align); - //Calculate the offset and store it behind our aligned pointer - *((myoffset_t *)ptr - 1) = (myoffset_t)((uintptr_t)ptr - (uintptr_t)p); - } // else NULL, could not malloc - } //else NULL, invalid arguments - return ptr; -} -/** -* aligned_free works like free(), but we work backwards from the returned -* pointer to find the correct offset and pointer location to return to free() -* Note that it is VERY BAD to call free() on an aligned_malloc() pointer. -*/ -void aligned_free(void * ptr) -{ - assert(ptr); - /* - * Walk backwards from the passed-in pointer to get the pointer offset - * We convert to an offset_t pointer and rely on pointer math to get the data - */ - myoffset_t offset = *((myoffset_t *)ptr - 1); - /* - * Once we have the offset, we can get our original pointer and call free - */ - void * p = (void *)((uint8_t *)ptr - offset); - free(p); -} + const std::string WHITESPACE = " \n\r\t\f\v"; std::string myltrim(const std::string &s) { @@ -18657,58 +19490,13 @@ string stringtoupper(string i_stringa) i_stringa[i]=toupper(i_stringa[i]); return i_stringa; } -int string2algo(string i_string) -{ - i_string=stringtoupper(i_string); - if (i_string=="BLAKE3") - return ALGO_BLAKE3; - else - if (i_string=="XXHASH64") - return ALGO_XXHASH64; - else - if (i_string=="WINHASH64") - return ALGO_WINHASH64; - else - if (i_string=="WYHASH") - return ALGO_WYHASH; - else - if (i_string=="CRC-32") - return ALGO_CRC32; - else - if (i_string=="ENTROPY") - return ALGO_ENTROPY; - else - if (i_string=="CRC-32C") - return ALGO_CRC32C; - else - if (i_string=="XXH3") - return ALGO_XXH3; - else - if (i_string=="SHA-256") - return ALGO_SHA256; - if (i_string=="SHA256") - return ALGO_SHA256; - else - if (i_string=="WHIRLPOOL") - return ALGO_WHIRLPOOL; - else - if (i_string=="MD5") - return ALGO_MD5; - else - if (i_string=="NILSIMSA") - return ALGO_NILSIMSA; - else - if (i_string=="SHA-3") - return ALGO_SHA3; - else - if (i_string=="SHA-1") - return ALGO_SHA1; - else - if (i_string=="SHA1") - return ALGO_SHA1; - else - return ALGO_SHA1; -} + + + + + + + int myposi(string i_substring,string i_string) { @@ -18844,7 +19632,7 @@ int64_t myatoll(const char * i_str) risultato*=1000000000; else if (toupper(thestring[0])=='T') - risultato*=1000000000000L; + risultato*=1000000000000LL; } if (thestring.size()==2) { @@ -18858,7 +19646,7 @@ int64_t myatoll(const char * i_str) risultato*=1073741824; else if (thestring=="TB") - risultato*=1099511627776L; + risultato*=1099511627776LL; } if (flagdebug) myprintf("18667: final from %s to %s (%s)\n",i_str,migliaia(risultato),tohuman(risultato)); @@ -18877,6 +19665,7 @@ int mystoi(string i_string) } int64_t encodestringdate(string i_date,bool i_flagfrom) { + i_date=format_datetime(i_date); string purged; for (unsigned int i=0;i0) { @@ -32948,6 +33743,7 @@ string relativetolongpath(string i_filename) printUTF8(risultato.c_str()); myprintf("\n"); } + franz_free(buf); return risultato; } } @@ -32985,6 +33781,11 @@ string getuname() } // Guess number of cores. In 32 bit mode, max is 2. int numberOfProcessors() { +#ifdef __HAIKU__ + system_info haikuinfo; + get_system_info(&haikuinfo); + return haikuinfo.cpu_count; +#endif #ifdef ESX return 1; #endif @@ -34471,23 +35272,48 @@ InputArchive::InputArchive(const char* filename, const char* password): -void int64toarray(int64_t i_number, char* i_buffer) +int inttoarray(int64_t i_number, char* i_buffer, int i_size) { if (i_buffer==NULL) { myprintf("29412: GURU buffer NULL!\n"); seppuku(); + return 0; } - ///printf("Numero da scrivere %016llX\n",i_number); - for (int i=0;i<8;i++) + if (i_size<=0) + { + myprintf("35003: GURU i_size not positive!\n"); + seppuku(); + return 0; + } + + for (int i=0;i>=8; } + return i_size; } +int32_t arraytoint32(const char* i_buffer) +{ + if (i_buffer==NULL) + { + myprintf("35024: i_buffer is NULL\n"); + seppuku(); + return -1; + } + int64_t risultato=0; + for (int i=3;i>=0;i--) + { + uint8_t numerino=*(i_buffer+i); + risultato=risultato+numerino; + if (i>0) + risultato<<=8; + } + return risultato; +} int64_t arraytoint64(const char* i_buffer) { @@ -36094,14 +36920,14 @@ class franzpacket XXHash64 myhash(myseed); myhash.add(buffer+FRANZOPACKET_HEADERSIZE,bytes); bufferhash =myhash.hash(); - int64toarray(okhash, buffer+ 0); - int64toarray(buffersize, buffer+ 8); - char8toarray(type, buffer+16); - int64toarray(bufferhash, buffer+24); - int64toarray(bytes, buffer+32); - int64toarray(number1, buffer+40); - int64toarray(number2, buffer+48); - int64toarray(number3, buffer+56); + inttoarray(okhash, buffer+ 0,8); + inttoarray(buffersize, buffer+ 8,8); + char8toarray(type, buffer+16,8); + inttoarray(bufferhash, buffer+24,8); + inttoarray(bytes, buffer+32,8); + inttoarray(number1, buffer+40,8); + inttoarray(number2, buffer+48,8); + inttoarray(number3, buffer+56,8); /* for (int i=0;i<16*chunks;i++) @@ -36511,7 +37337,7 @@ class franzfs franzfs(){data=NULL;position=0;} bool init(int64_t i_size) { - data=(char*)malloc(i_size); + data=(char*)franz_malloc(i_size); if (data==NULL) { myprintf("35250: GURU on malloc\n"); @@ -36536,7 +37362,7 @@ class franzfs myprintf("23006: GURU on filesize\n"); return false; } - free(data); + franz_free(data); g_ramdisksize-=filesize; filesize=0; if (flagdebug) @@ -36569,9 +37395,9 @@ struct DT // if you get some warning here, update your compiler! string hexhash; // for new functions, not for add string hashtype; // for paranoid() /* - new FRANZOFFSET + new FRANZOFFSETV1 */ - char franz_block[FRANZOFFSETSHA256]; + char franz_block[FRANZOFFSETV3]; int franz_block_size; uint32_t file_crc32; int64_t hashedsize; @@ -36587,13 +37413,24 @@ struct DT // if you get some warning here, update your compiler! SHA3 *pfile_sha3; MD5 *pfile_md5; blake3_hasher *pfile_blake3; + NESSIEstruct *pfile_whirlpool; + HighwayHashCat *pfile_highway64; + + franzfs *pramfile; - DT(): date(0), size(0), attr(0), data(0),creationdate(0),accessdate(0),written(-1),isordered(false),franz_block_size(FRANZOFFSETSHA256),file_crc32(0),hashedsize(0),chunk(-1),filefix(0),expectedsize(0),forceadd(false) + DT(): date(0), size(0), attr(0), data(0),creationdate(0),accessdate(0),written(-1),isordered(false),franz_block_size(FRANZOFFSETV3),file_crc32(0),hashedsize(0),chunk(-1),filefix(0),expectedsize(0),forceadd(false) { memset(franz_block,0,sizeof(franz_block)); hexhash =""; hashtype=""; outputname=""; + pfile_highway64=NULL; + if ((g_franzotype==FRANZO_HIGHWAY64) || (g_franzotype==FRANZO_HIGHWAY128) || (g_franzotype==FRANZO_HIGHWAY256)) + { + pfile_highway64=new HighwayHashCat; + uint64_t key[4] = {1, 2, 3, 4}; + HighwayHashCatStart(key,pfile_highway64); + } pfile_xxhash64=NULL; if ((g_franzotype==FRANZO_XXHASH64) || (g_franzotype==FRANZO_WINHASH64)) pfile_xxhash64=new XXHash64(0); @@ -36610,6 +37447,12 @@ struct DT // if you get some warning here, update your compiler! if (g_franzotype==FRANZO_SHA3) pfile_sha3=new SHA3; /// beware of time and space, but now we are sure to maintain 64 byte alignment + pfile_whirlpool=NULL; + if (g_franzotype==FRANZO_WHIRLPOOL) + { + pfile_whirlpool=new NESSIEstruct; + NESSIEinit(pfile_whirlpool); + } pfile_xxh3=NULL; if (g_franzotype==FRANZO_XXH3) { @@ -36619,7 +37462,7 @@ struct DT // if you get some warning here, update your compiler! pfile_blake3=NULL; if (g_franzotype==FRANZO_BLAKE3) { - pfile_blake3=(blake3_hasher*)malloc(sizeof(blake3_hasher)); + pfile_blake3=(blake3_hasher*)franz_malloc(sizeof(blake3_hasher)); blake3_hasher_init(pfile_blake3); } pramfile=NULL; @@ -36677,6 +37520,7 @@ class Jidac friend struct ExtractJob; void usageall (string i_command); // verbose help private: + string fullcommandline; vector results; // warning and errors string zpaqfranzexename; @@ -36725,7 +37569,6 @@ class Jidac bool flagforcewindows; // force alterate data stream $DATA$, system volume information bool flagnopath; // do not store path bool flagnosort; // do not sort files std::sort(vf.begin(), vf.end(), compareFilename); - bool flagnochecksum; // disable (franzotype 0) uint32_t filelength; uint32_t dirlength; string searchfrom; // search something in path @@ -36770,6 +37613,9 @@ class Jidac int purgersync(); // Purge rsync temporary file int sfx(); // Write autoextract module int dd(); + int zfsbackup(); + int zfsrestore(); + int zfsreceive(); int dircompare(bool i_flagonlysize,bool i_flagrobocopy); int benchmark(); int autotest(); @@ -36785,46 +37631,689 @@ class Jidac void examples (string i_command); // some examples void join_split(); // Support functions - void reset(); - void printsanitizeflags(); - string rename(string name); // rename from -to - int64_t read_archive(const char* arc, int *errors=0, int i_myappend=0,bool i_quiet=false); // read arc - bool isselected(const char* filename, bool rn,int64_t i_size);// files, -only, -not - void scandir(bool i_checkifselected,DTMap& i_edt,string filename, bool i_recursive=true); // scan dirs to dt - void addfile(bool i_checkifselected,DTMap& i_edt,string filename, int64_t edate, int64_t esize,int64_t eattr,int64_t i_creationdate,int64_t i_accessdate); // add external file to dt - int64_t franzparallelscandir(bool i_flaghash,bool i_recursive,bool i_forcedir); - bool equal(DTMap::const_iterator p, const char* filename, uint32_t &o_crc32,string i_myhashtype,string i_myhash,string& o_hash);// compare file contents with p - void write715attr(libzpaq::StringBuffer& i_sb, uint64_t i_data, unsigned int i_quanti); - void writefranzattr(libzpaq::StringBuffer& i_sb, uint64_t i_data, unsigned int i_quanti,string i_filename,uint32_t i_crc32fromfragments,uint32_t i_crc32,string i_thehash,int64_t i_creationdate,int64_t i_accessdate); - int enumeratecomments(); - int searchcomments(string i_testo,vector &filelist); - string zfs_get_snaplist(string i_header,string i_footer,vector& o_array_primachiocciola,vector& o_array_dopochiocciola); - string sanitizzanomefile(string i_filename,int i_filelength,int& io_collisioni,MAPPAFILEHASH& io_mappacollisioni); - void getpasswordifempty(); - string getpasswordblind(); - string getpassword(); - int writesfxmodule(string i_filename); - int decompress_sfx_to_file(FILE* i_outfile); - int extractqueue2(int i_chunk,int i_chunksize); - int multiverify(vector & i_arrayfilename); - bool removetempdirifempty(string i_folder,bool i_deleteifsizezero); - void handleflaglongpath(); - bool getfoldersize(string i_folder,uint64_t& o_totalsize,uint32_t & o_totalfile,uint32_t& o_totalfolder,uint32_t& o_longfiles); - uint64_t hashastringa(string& i_string); - uint32_t casekollision(DTMap& i_dtmap,vector& o_collisions,bool i_fix); - void changedtmapkey(string i_oldkey,string i_newkey); - int removeemptydirs(string i_folder,bool i_kill); - int writeresource(string i_filename,bool i_force,const char* i_mime64); - int decompress_mime64_to_file(FILE* i_outfile,const char* i_mime64); - int hashselect(); - string win32_extract_resource(const string i_folder,const string i_filename,const char* i_mime64); - string prepare_dd(); - + void reset(); + void printsanitizeflags(); + string rename(string name); // rename from -to + int64_t read_archive(const char* arc, int *errors=0, int i_myappend=0,bool i_quiet=false); // read arc + bool isselected(const char* filename, bool rn,int64_t i_size);// files, -only, -not + void scandir(bool i_checkifselected,DTMap& i_edt,string filename, bool i_recursive=true); // scan dirs to dt + void addfile(bool i_checkifselected,DTMap& i_edt,string filename, int64_t edate, int64_t esize,int64_t eattr,int64_t i_creationdate,int64_t i_accessdate); // add external file to dt + int64_t franzparallelscandir(bool i_flaghash,bool i_recursive,bool i_forcedir); + bool equal(DTMap::const_iterator p, const char* filename, uint32_t &o_crc32,string i_myhashtype,string i_myhash,string& o_hash);// compare file contents with p + void write715attr(libzpaq::StringBuffer& i_sb, uint64_t i_data, unsigned int i_quanti); + void writefranzattr(libzpaq::StringBuffer& i_sb, uint64_t i_data, unsigned int i_quanti,string i_filename,uint32_t i_crc32fromfragments,uint32_t i_crc32,string i_thehash,int64_t i_creationdate,int64_t i_accessdate,struct franz_posix* i_posix); + int enumeratecomments(); + int searchcomments(string i_testo,vector &filelist); + string zfs_get_snaplist(string i_header,string i_footer,vector& o_array_primachiocciola,vector& o_array_dopochiocciola); + string sanitizzanomefile(string i_filename,int i_filelength,int& io_collisioni,MAPPAFILEHASH& io_mappacollisioni); + void getpasswordifempty(); + string getpasswordblind(); + string getpassword(); + int writesfxmodule(string i_filename); + int decompress_sfx_to_file(FILE* i_outfile); + int extractqueue2(int i_chunk,int i_chunksize); + int multiverify(vector & i_arrayfilename); + bool removetempdirifempty(string i_folder,bool i_deleteifsizezero); + void handleflaglongpath(); + bool getfoldersize(string i_folder,uint64_t& o_totalsize,uint32_t & o_totalfile,uint32_t& o_totalfolder,uint32_t& o_longfiles); + uint64_t hashastringa(string& i_string); + uint32_t casekollision(DTMap& i_dtmap,vector& o_collisions,bool i_fix); + void changedtmapkey(string i_oldkey,string i_newkey); + int removeemptydirs(string i_folder,bool i_kill); + int writeresource(string i_filename,bool i_force,const char* i_mime64); + int decompress_mime64_to_file(FILE* i_outfile,const char* i_mime64); + int hashselect(); + string win32_extract_resource(const string i_folder,const string i_filename,const char* i_mime64); + string prepare_dd(); #ifdef SERVER - int sendtocloudpaq(const int64_t i_extimated,const int i_version); + int sendtocloudpaq(const int64_t i_extimated,const int i_version); #endif + bool cli_onlyfiles(string i_opt,string i_string,char i_command,int argc,const char** argv, int* i_i); + bool cli_onlyfilename(string i_opt,string i_string,string i_alias,string& o_thefile,int argc,const char** argv, int* i_i,bool* i_theflag); + bool cli_getdate(string i_opt,string i_string,int argc,const char** argv, int* i_i,int64_t* o_date); + bool cli_getint(string i_opt,string i_string,int argc,const char** argv, int* i_i,int i_default,int* o_thenumber); + bool cli_getint64(string i_opt,string i_string,int argc,const char** argv, int* i_i,int64_t i_default,int64_t* o_thenumber); + + bool cli_getuint(string i_opt,string i_string,int argc,const char** argv, int* i_i,unsigned int i_default,unsigned int* o_thenumber); + bool cli_getuint64(string i_opt,string i_string,int argc,const char** argv, int* i_i,uint64_t i_default,uint64_t* o_thenumber); + + + + + + }; +#ifdef unix +std::string exec(const char* cmd) +{ +#ifdef ANCIENT + return ""; +#else + std::array buffer; + std::string result; + std::unique_ptr pipe(popen(cmd, "r"), pclose); + while (fgets(buffer.data(), buffer.size(), pipe.get()) != nullptr) + result += buffer.data(); + return result; +#endif +} +#else +// a debug function +string exec(const char* cmd) +{ + myprintf("Debug function (Windows does not have ZFS!)\n"); + if (cmd==NULL) // compiler be quiet + return "NULL"; + return ""; +} +#endif + +int Jidac::zfsreceive() +{ + myprintf("zfsreceive\n"); + if (files.size()!=2) + { + myprintf("zfsreceive examples (count %d)\n",files.size()); + myprintf("[1]: zfsreceive /pippo.zpaq rpool/restored -out thescript.sh\n"); + myprintf("[1]: zfsreceive /pippo.zpaq rpool/restored -out thescript.sh -to mymark\n"); + return 2; + } + + if (g_output=="") + { + myprintf("37423: you must specify an -out thescript.sh\n"); + return 2; + } + if (tofiles.size()>1) + { + myprintf("37431: only a single -to allowed (zfsmark)\n"); + return 2; + } + + string myoutput=g_output; + + string themark ="franco"; + if (tofiles.size()==1) + { + string newmark=""; + string temp=tofiles[0]; + for (unsigned int i=0;i>"); + printUTF8(archive.c_str()); + myprintf("\n"); + return 1; + } + + string thefolder =files[0]; + string thepool =files[1]; + /// test pool for / + int howmanyslash=0; + int howmanyat =0; + for (unsigned int i=0;i0) + { + myprintf("53217: a dataset |%s| cannot have @\n",thepool.c_str()); + return 2; + } + + getpasswordifempty(); + + int errors=0; + command='l'; + g_optional="versum"; //force isselected + flagstdout=true; + + read_archive(archive.c_str(),&errors,1); /// AND NOW THE MAGIC ONE! + vector filename; + for (DTMap::iterator a=dt.begin(); a!=dt.end(); ++a) + { + string temp=a->first.c_str(); + string estensione=prendiestensione(temp); + if (estensione=="zfs") + { + if (a->second.isordered) + filename.push_back(a->first.c_str()); + else + { + myprintf("53249: GURU file is not ordered <<%s>> \n",a->first.c_str()); + return 2; + } + } + } + char nomeatteso[20]; + + for (unsigned int i=0;i abort\n",i,snomeatteso.c_str(),filename[i].c_str()); + return 2; + } + } + + FILE* batch=fopen(myoutput.c_str(), "w"); + if (batch==NULL) + { + myprintf("53273: cannot write on script %s\n",myoutput.c_str()); + return 2; + } + string pv=""; +#ifndef _WIN32 + if (fileexists("/usr/local/bin/pv")) + pv="/usr/local/bin/pv|"; + else + if (fileexists("/usr/bin/pv")) + pv="/usr/bin/pv|"; + else + if (fileexists("/bin/pv")) + pv="/bin/pv|"; +#endif + for (unsigned int i=0;i>\n"); + return 0; +} +int Jidac::zfsrestore() +{ + myprintf("zfsrestore\n"); + if (files.size()!=2) + { + myprintf("zfsrestore examples (count %d)\n",files.size()); + myprintf("[1]: zfsrestore /restoredzfsfolder rpool/restored -out thescript.sh\n"); + myprintf("[2]: zfsrestore /restoredzfsfonder rpool/restored -out thescript.sh -to mymark\n"); + return 2; + } + if (g_output=="") + { + myprintf("37429: you must specify an -out thescript.sh\n"); + return 2; + } + if (tofiles.size()>1) + { + myprintf("37434: only a single -to allowed (zfsmark)\n"); + return 2; + } + + string myoutput=g_output; + + string themark ="franco"; + if (tofiles.size()==1) + { + string newmark=""; + string temp=tofiles[0]; + for (unsigned int i=0;i0) + { + myprintf("53044: a pool |%s| cannot have @\n",thepool.c_str()); + return 2; + } + files.resize(1); + flagforcezfs=true; + DTMap myblock; + g_arraybytescanned.push_back(0); + g_arrayfilescanned.push_back(0); + myprintf("Searching files <<%s>>\n",files[0].c_str()); + edt.clear(); + scandir(false,edt,files[0].c_str(),false); + vector filename; + for (DTMap::iterator a=edt.begin(); a!=edt.end(); ++a) + { + string temp=a->first.c_str(); + string estensione=prendiestensione(temp); + if (estensione=="zfs") + filename.push_back(a->first.c_str()); + } + if (filename.size()==0) + { + myprintf("53074: no .zfs files founded in archive\n"); + return 2; + } + char nomeatteso[20]; + for (unsigned int i=0;i abort\n",i,snomeatteso.c_str(),filename[i].c_str()); + return 2; + } + } + FILE* batch=fopen(myoutput.c_str(), "w"); + if (batch==NULL) + { + myprintf("53088: cannot write on script %s\n",myoutput.c_str()); + return 2; + } + string pv=""; +#ifndef _WIN32 + if (fileexists("/usr/local/bin/pv")) + pv="/usr/local/bin/pv|"; + else + if (fileexists("/usr/bin/pv")) + pv="/usr/bin/pv|"; + else + if (fileexists("/bin/pv")) + pv="/bin/pv|"; +#endif + + for (unsigned int i=0;i>\n"); + return 0; +} +int Jidac::zfsbackup() +{ + myprintf("zfsrestore\n"); + + if (tofiles.size()>1) + { + myprintf("37434: only a single -to allowed (zfsmark)\n"); + return 2; + } + string estensione=prendiestensione(archive); + if (estensione!="zpaq") + { + myprintf("53522: please use .zpaq extension on archive name <<%s>>"); + printUTF8(archive.c_str()); + myprintf("\n"); + return 1; + } + if (files.size()!=1) + { + myprintf("37426: please use 1 single pool/dataset\n"); + return 1; + } + string thepool =files[0]; + if (isdirectory(thepool)) + { + myprintf("53537: A pool-dataset |%s| cannot end with a / \n",thepool.c_str()); + myprintf("53537: Good examples tank tank/d\n",thepool.c_str()); + return 2; + } + string themark ="zpaqfranz"; + if (tofiles.size()==1) + { + string newmark=""; + string temp=tofiles[0]; + for (unsigned int i=0;i1) + { + myprintf("53549: A pool-dataset |%s| must have exactly 0 or 1 / instead of %d\n",thepool.c_str(),howmanyslash); + return 2; + } + if (howmanyat>0) + { + myprintf("52984: A pool-dataset |%s| cannot have @\n",thepool.c_str()); + return 2; + } + if (!fileexists(archive)) + { + myprintf("53015: The archive does not exist, checking for snapshots\n"); + vector array_primachiocciola; + vector array_dopochiocciola; + string risul=zfs_get_snaplist(thepool,themark,array_primachiocciola,array_dopochiocciola); + if (array_primachiocciola.size()!=0) + { + myprintf("53022: Archive does not exists, but we got %d snapshots => abort\n",array_primachiocciola.size()); + return 2; + } + char buffer[1000]; + snprintf(buffer,sizeof(buffer),"%s%08d",themark.c_str(),1); + + string thesnap=buffer; + string fullsnap=thepool+"@"+thesnap; + myprintf("53586: The snap |%s|\n",thesnap.c_str()); + myprintf("53587: Fullsnap |%s|\n",fullsnap.c_str()); + myprintf("53030: Exename |%s|\n",zpaqfranzexename.c_str()); + myprintf("53031: Now preparing a brand-new archive and snapshot (starting from 00000001)\n"); + string filebatch="/tmp/newbackup_zfs_"+purgedpool+".sh"; + filebatch=nomefileseesistegia(filebatch); + myprintf("53033: CREATING script %s\n",filebatch.c_str()); + FILE* batch=fopen(filebatch.c_str(), "w"); + if (batch==NULL) + { + myprintf("53037: cannot write on script %s\n",filebatch.c_str()); + return 2; + } + fprintf(batch,"zfs snapshot %s\n",fullsnap.c_str()); + string hashofile=""; + if (flaghashdeep) + { + string hashdeepposition=""; +#ifndef _WIN32 + if (fileexists("/usr/local/bin/hashdeep")) + hashdeepposition="/usr/local/bin/hashdeep"; + else + if (fileexists("/usr/bin/pv")) + hashdeepposition="/usr/bin/hashdeep"; + else + if (fileexists("/bin/pv")) + hashdeepposition="/bin/hashdeep"; +#endif + myprintf("53607: hashdeep <<%s>>\n",hashdeepposition.c_str()); + if (hashdeepposition=="") + { + myprintf("53612: Cannot locate hashdeep, abort!\n"); + fclose(batch); + return 2; + } + string zfsfolder="/"+thepool+"/.zfs/snapshot/"+thesnap+"/"; + myprintf("53616: zfsfolder <<%s>>\n",zfsfolder.c_str()); + hashofile="/tmp/hasho_"+purgedpool+"_"+thesnap+".txt"; + hashofile=nomefileseesistegia(hashofile.c_str()); + myprintf("53603: Hash-o-file <<%s>>\n",hashofile.c_str()); + fprintf(batch,"/sbin/zfs set snapdir=hidden %s\n",thepool.c_str()); + fprintf(batch,"%s -c md5 -r %s >%s\n", + hashdeepposition.c_str(), + zfsfolder.c_str(), + hashofile.c_str()); + } + fprintf(batch,"zfs send %s@%s00000001 |%s a %s 00000001.zfs -stdin\n",thepool.c_str(),themark.c_str(),fullzpaqexename.c_str(),archive.c_str()); + if (flaghashdeep) + fprintf(batch,"%s a %s %s\n",fullzpaqexename.c_str(),archive.c_str(),hashofile.c_str()); + fclose(batch); +#ifndef _WIN32 + if (chmod(filebatch.c_str(),0700)!=0) + { + myprintf("53046: error on chmod 700 on <<%s>>\n",filebatch.c_str()); + return 2; + } + myprintf("53049: RUNNING script %s\n",filebatch.c_str()); + string runresult=exec(filebatch.c_str()); + if (mypos("(all OK)",runresult)!=0) + { + myprintf("53054: The run seems OK, filesize %s\n",migliaia(prendidimensionefile(archive.c_str()))); + if (flagkill) + { + myprintf("53649: Deleting <<%s>>\n",filebatch.c_str()); + delete_file(filebatch.c_str()); + } + + return 0; + } + else + { + myprintf("53059: GURU, something seems wrong, filesize %s\n",migliaia(prendidimensionefile(archive.c_str()))); + return 2; + } +#endif + return 0; + } + +// archive does exists + myprintf("52995: The archive exist, checking latest snapshot\n"); + getpasswordifempty(); + int errors=0; + command='l'; + g_optional="versum"; //force isselected +#ifndef _WIN32 + int64_t oldsize=prendidimensionefile(archive.c_str()); +#endif + read_archive(archive.c_str(),&errors,1); /// AND NOW THE MAGIC ONE! + vector filename; + for (DTMap::iterator a=dt.begin(); a!=dt.end(); ++a) + { + string temp=a->first.c_str(); + string estensione=prendiestensione(temp); + if (estensione=="zfs") + filename.push_back(a->first.c_str()); + } + char nomeatteso[40]; + for (unsigned int i=0;i abort\n",i,snomeatteso.c_str(),filename[i].c_str()); + } + snprintf(nomeatteso,sizeof(nomeatteso),"%08lld",(long long)filename.size()); + string snomeatteso=nomeatteso; + + string expectedsnapshot=themark+snomeatteso; + string fullexpectedsnapshot=thepool+'@'+themark+snomeatteso; + myprintf("53035: Searching for snapshot |%s|\n",fullexpectedsnapshot.c_str()); + + vector array_primachiocciola; + vector array_dopochiocciola; + string risul=zfs_get_snaplist(thepool,themark,array_primachiocciola,array_dopochiocciola); + + if (array_primachiocciola.size()==0) + { + myprintf("53051: Cannot find the expected snapshot |%s|\n",fullexpectedsnapshot.c_str()); + return 2; + } + int foundindex=-1; + for (unsigned int i=0;i %s\n",i,array_dopochiocciola[i].c_str()); + if (expectedsnapshot==array_dopochiocciola[i]) + { + myprintf("53054: The expected snapshot exists (this is good) index %d |%s|\n",i,expectedsnapshot.c_str()); + foundindex=i; + break; + } + } + if ((foundindex+1)!=(int)array_primachiocciola.size()) + { + myprintf("53075: The founded index %d is < snapshot size %d, cowardly abort (not the LAST snapshot)\n",foundindex,array_primachiocciola.size()); + return 2; + } + + string ultimo =array_dopochiocciola[foundindex]; + string solocifre =""; + for (unsigned int i=0;i>\n",hashdeepposition.c_str()); + if (hashdeepposition=="") + { + myprintf("53758: cannot locate hashdeep, abort!\n"); + fclose(batch); + return 2; + } + string thesnap=themark+nuovonome; + string zfsfolder="/"+thepool+"/.zfs/snapshot/"+thesnap+"/"; + myprintf("53763: Zfsfolder <<%s>>\n",zfsfolder.c_str()); + hashofile="/tmp/hasho_"+purgedpool+"_"+thesnap+".txt"; + hashofile=nomefileseesistegia(hashofile.c_str()); + myprintf("53766: Hashofile <<%s>>\n",hashofile.c_str()); + fprintf(batch,"/sbin/zfs set snapdir=hidden %s\n",thepool.c_str()); + fprintf(batch,"%s -c md5 -r %s >%s\n", + hashdeepposition.c_str(), + zfsfolder.c_str(), + hashofile.c_str()); + } + fprintf(batch,"zfs send -i %s %s |%s a %s %s.zfs -stdin\n",fullexpectedsnapshot.c_str(),nuovonomefull.c_str(),fullzpaqexename.c_str(),archive.c_str(),nuovonome.c_str()); + if (flaghashdeep) + fprintf(batch,"%s a %s %s\n",fullzpaqexename.c_str(),archive.c_str(),hashofile.c_str()); + if (flagkill) + fprintf(batch,"zfs destroy %s\n",fullexpectedsnapshot.c_str()); + fclose(batch); + +#ifndef _WIN32 + + if (chmod(filebatch.c_str(),0700)!=0) + { + myprintf("53127: Error on chmod 700 on %s\n",filebatch.c_str()); + return 2; + } + myprintf("53130: RUNNING script %s\n",filebatch.c_str()); + string runresult=exec(filebatch.c_str()); +// dirrrrty + if (mypos("(all OK)",runresult)!=0) + { + int64_t newsize=prendidimensionefile(archive.c_str()); + myprintf("53185: Seems OK, filesize %s => %s (%s)\n",migliaia(oldsize),migliaia2(newsize),migliaia3(myabs(newsize,oldsize))); + if (flagkill) + { + myprintf("53812: Deleting <<%s>>\n",filebatch.c_str()); + delete_file(filebatch.c_str()); + } + return 0; + } + else + { + myprintf("53190: GURU, something seems wrong, filesize %s\n",migliaia(prendidimensionefile(archive.c_str()))); + return 2; + } +#endif + return 0; +} /// detecting case collision: xxhash64 is more than enough (fast and compact) uint64_t Jidac::hashastringa(string& i_string) { @@ -36989,7 +38478,7 @@ void Jidac::reset() int Jidac::paranoid() //////////////////////////////////////////////////////////////////////////// ///////// This is a merge of unzpaq206.cpp, patched by me to become unz.cpp -///////// Now support FRANZOFFSET256 +///////// Now support FRANZOFFSET { #ifdef _WIN32 #ifndef _WIN64 @@ -38957,8 +40446,8 @@ struct unzDT { // if you get some warning here, update your compiler! // unzdt YYYYMMDDHHMMSS or 0 if deleted uint64_t attr; // first 8 bytes, LSB first std::vector ptr; // fragment IDs - char sha1hex[FRANZOFFSETSHA256]; // 1+32+32 (unzSHA256)+ zero - char sha1decompressedhex[FRANZOFFSETSHA256]; // 1+32+32 (unzSHA256)+ zero + char sha1hex[FRANZOFFSETV3+1]; // 1+32+32 (unzSHA256)+ zero + char sha1decompressedhex[FRANZOFFSETV3+1]; // 1+32+32 (unzSHA256)+ zero std::string sha1fromfile; // 1+32+32 (unzSHA256)+ zero unzDT(): date(0), attr(0) {sha1hex[0]=0x0;sha1decompressedhex[0]=0x0;sha1fromfile="";} }; @@ -38978,7 +40467,8 @@ bool unzcompareprimo(unzDTMap::iterator i_primo, unzDTMap::iterator i_secondo) /// yes, I can do incomprensible code too :) /// decode a franzoblock, discriminating on length of the block, to get crc32 and hash, and hashtype -int decode_franz_block(const bool i_isdirectory,const char* i_franz_block,string& o_hashtype,string& o_hashvalue,string& o_crc32value,int64_t& o_creationtime, int64_t& o_accesstime) +int decode_franz_block +(const bool i_isdirectory,const char* i_franz_block,string& o_hashtype,string& o_hashvalue,string& o_crc32value,int64_t& o_creationtime, int64_t& o_accesstime,bool& o_isordered,int& o_version,franz_posix* o_posix) { int risultato; o_hashtype =""; @@ -38986,6 +40476,11 @@ int decode_franz_block(const bool i_isdirectory,const char* i_franz_block,string o_crc32value =""; o_accesstime =0; o_creationtime =0; + o_isordered =false; + o_version =0; + if (o_posix) // compiler be quiet, please + o_posix =NULL; + if (i_franz_block==NULL) return -1; @@ -39120,6 +40615,67 @@ int decode_franz_block(const bool i_isdirectory,const char* i_franz_block,string o_crc32value=i_franz_block+67; } } + +/// this is the FRANZOBLOCKV3 (upto 128 bytes+atime+ctime+vnumber+isordered) and the V4 (V3+posix structure) + if ((i_franz_block[0]=='3') || (i_franz_block[0]=='4')) + { + bool onlyheader=(i_franz_block[0]=='3'); + + if (flagdebug) + { + if (onlyheader) + myprintf("39665: decode a FRANZOFFSETV3 only header\n"); + else + myprintf("39662: decode a FRANZOFFSETV3 with posix\n"); + } + int expectedfranzotype=i_franz_block[1]-'A'; + MAPPATIPOHASH::iterator p=g_mappatipohash.find(expectedfranzotype); + if (p==g_mappatipohash.end()) + { + myprintf("39675: GURU cannot find expectedfranzotype %d\n",expectedfranzotype); + seppuku(); + return 0; + } + string hashname =p->second.hashname; + int hashlen =p->second.hashlen; + if (flagdebug) + myprintf("39670: expectedfranzotype %d => |%-15s| len %03d\n",expectedfranzotype,hashname.c_str(),hashlen); + + risultato =expectedfranzotype; + o_hashtype =hashname; + o_hashvalue =i_franz_block+2; + + int offsetcrc= 2+hashlen+2; + if (i_isdirectory) + o_crc32value=" "; + else + o_crc32value=i_franz_block+offsetcrc; + + o_isordered=arraytoint32(i_franz_block+offsetcrc+8+2); + if (flagdebug) + myprintf("39724: Ordered stream %08d\n",o_isordered); + + o_version=arraytoint32(i_franz_block+offsetcrc+8+2+4); + if (flagdebug) + myprintf("39728: Version number %08d\n",o_version); + + o_creationtime =arraytoint64(i_franz_block+offsetcrc+8+2+4+4); + o_accesstime =arraytoint64(i_franz_block+offsetcrc+8+2+4+4+8); + + if (flagdebug) + { + myprintf("39734: cdate %s\n",migliaia(o_creationtime)); + myprintf("39736: adate %s\n",migliaia(o_accesstime)); + } + + if (!onlyheader) + { + int64_t offsetposix=offsetcrc+8+2+4+4+8+(8*4); // 8*4= future expansion + if (flagdebug) + myprintf("39742: Taking posix too @ offset position %08d\n",offsetposix); + o_posix=(franz_posix*) (i_franz_block+offsetposix); + } + } return risultato; } ///https://gist.github.com/0x3f00/90edbec0c04616d0b8c21586762bf1ac @@ -39205,6 +40761,9 @@ string Jidac::sanitizzanomefile(string i_filename,int i_filelength,int& io_colli string mycrc32 =""; int64_t myaccesstime=0; int64_t mycreationtime=0; + bool myisordered=false; + int myversion=0; + franz_posix* myposix=NULL; if (p!=dt.end()) { decode_franz_block(isdirectory(i_filename),p->second.franz_block, @@ -39212,7 +40771,10 @@ string Jidac::sanitizzanomefile(string i_filename,int i_filelength,int& io_colli myhash, mycrc32, mycreationtime, - myaccesstime); + myaccesstime, + myisordered, + myversion, + myposix); if (myhashtype!="") if (myhash!="") { @@ -39741,28 +41303,40 @@ void help_orderby() } void help_printhash(bool i_flagadd) { - moreprint("+ : -crc32 Old but ubiquitous, superfast"); - moreprint("+ : -xxh3 One of the fastest (128 bit), strong"); - moreprint("+ : -xxhash Very fast (64 bit), low CPU"); - moreprint("+ : -sha1 Fair speed, very reliable, some collisions known"); - moreprint("+ : -sha256 CPU intensive, one of the most reliable. Legal proof in EU"); - moreprint("+ : -sha3 Latest NIST standard, very, very strong"); - moreprint("+ : -blake3 Fast, CPU intensive (on Win64 HW acceleration), very reliable"); - moreprint("+ : -md5 Very common, widespread usage"); + char linea[200]; + for (MAPPATIPOHASH::iterator p=g_mappatipohash.begin(); p!=g_mappatipohash.end(); ++p) + if (p->second.flagiszpaq) + { + snprintf(linea,sizeof(linea),"+ : %-13s %s",p->second.switchname.c_str(),p->second.hashdescription.c_str()); + moreprint(linea); + } if (!i_flagadd) { - moreprint("+ : -crc32c 'Castagnoli', HW accelerated"); - moreprint("+ : -wyhash Maybe the fastest, limited 'strength'"); - moreprint("+ : -whirlpool Slow but very reliable"); - moreprint("+ : -nilsimsa Look for similarities"); - moreprint("+ : -entropy Quick-and-dirty entropy estimator"); + for (MAPPATIPOHASH::iterator p=g_mappatipohash.begin(); p!=g_mappatipohash.end(); ++p) + if (!p->second.flagiszpaq) + { + snprintf(linea,sizeof(linea),"+ : %-13s %s",p->second.switchname.c_str(),p->second.hashdescription.c_str()); + moreprint(linea); + } } } void help_printhashline(bool i_flagadd) { - moreprint("+ : -crc32 -xxhash -md5 -sha1 -xxh3 -blake3 -sha256 -sha3"); + + string risultato="+ : "; + for (MAPPATIPOHASH::iterator p=g_mappatipohash.begin(); p!=g_mappatipohash.end(); ++p) + if (p->second.flagiszpaq) + risultato+=p->second.switchname+" "; + moreprint(risultato.c_str()); + if (!i_flagadd) - moreprint("+ : -crc32c -wyhash -whirlpool -nilsimsa -entropy"); + { + risultato="+ : "; + for (MAPPATIPOHASH::iterator p=g_mappatipohash.begin(); p!=g_mappatipohash.end(); ++p) + if (!p->second.flagiszpaq) + risultato+=p->second.switchname+" "; + moreprint(risultato.c_str()); + } } void help_b(bool i_usage,bool i_example) { @@ -39947,7 +41521,7 @@ void help_a(bool i_usage,bool i_example) moreprint("+ : -debug -zero -kill Add 0-byte long file (debugging)"); moreprint("+ : -nochecksum Disable zpaqfranz additional checks (faster, less sure)"); moreprint("+ : -nodedup Turn off deduplicator"); - moreprint("+ : -tar Tar mode: no deduplication, no compression"); + moreprint("+ : -store Store mode: no deduplication, no compression"); moreprint("+ : -touch Force 'touch' on date (converting 7.15 to zpaqfranz)"); moreprint("+ : -norecursion Do not recurse into folders (default: YES)"); #ifdef _WIN32 @@ -40523,12 +42097,6 @@ void help_zfs(bool i_usage,bool i_example) moreprint("CMD zfsadd Freeze into an archive a selection of snapshots"); moreprint("+ : -force Do the freezing (wet run)"); } - if ((g_optional=="zfsbackup") || (g_optional=="")) - moreprint("CMD zfsbackup Backup via incremental zfs snapshots"); - if ((g_optional=="zfsrestore") || (g_optional=="")) - moreprint("CMD zfsrestore Restore via incremental zfs snapshots"); - if ((g_optional=="zfsreceive") || (g_optional=="")) - moreprint("CMD zfsreceive Receive incremental zfs snapshots"); moreprint("+ : -output pi.sh Write a script-ready on pi.sh (alias -out)"); moreprint("+ : -pakka Remove first line in output"); } @@ -40556,23 +42124,61 @@ void help_zfs(bool i_usage,bool i_example) moreprint("zfspurge \"tank\" \"--7d\""); moreprint("zfspurge \"tank\" \"--7d\" -output mygoodscript.sh"); } - if ((g_optional=="zfsbackup") || (g_optional=="")) - { + } +} + +void help_zfsbackup(bool i_usage,bool i_example) +{ + if (i_usage) + { + moreprint("CMD zfsbackup Archiving incremental zfs snapshots"); + moreprint("+ : for long-term storage of backups (aka: anti-ransom)"); + moreprint("+ : Usually root privileges required"); + } + if (i_usage && i_example) + moreprint(" Examples:"); + if (i_example) + { moreprint("zfsbackup /temporaneo/incrementale.zpaq tank/d"); moreprint("zfsbackup /temporaneo/incrementale.zpaq tank -kill"); moreprint("zfsbackup /temporaneo/incrementale.zpaq tank/d -hashdeep"); - } - if ((g_optional=="zfsrestore") || (g_optional=="")) - { + } +} + +void help_zfsrestore(bool i_usage,bool i_example) +{ + if (i_usage) + { + moreprint("CMD zfsrestore Prepare a script for restoring"); + moreprint("+ : from zfs backups"); + } + if (i_usage && i_example) + moreprint(" Examples:"); + if (i_example) + { moreprint("zfsrestore /restoredzfsfolder rpool/restored -out thescript.sh"); moreprint("zfsrestore /restoredzfsfonder rpool/restored -out thescript.sh -to mymark"); - } - if ((g_optional=="zfsreceive") || (g_optional=="")) - { + } +} + +void help_zfsreceive(bool i_usage,bool i_example) +{ + if (i_usage) + { + moreprint("CMD zfsreceive Prepare a script for restoring"); + moreprint("+ : from zpaq archive"); + } + if (i_usage && i_example) + moreprint(" Examples:"); + if (i_example) + { moreprint("zfsreceive /tmp/ordinato.zpaq rpool/restored -out thescript.sh"); - } + moreprint("zfsreceive /tmp/ordinato.zpaq rpool/restored -out thescript.sh -to mymark"); } } + + + void help_z(bool i_usage,bool i_example) { if (i_usage) @@ -40728,7 +42334,7 @@ void help_dir(bool i_usage,bool i_example) { if (i_usage) { - moreprint("CMD dir (yes, dir as in Windows)."); + moreprint("CMD dir (yes, dir as in Windows, even better)."); moreprint("+ : I really hate ls: does not show the cumulative filesize (!)"); moreprint(" 'Mini clone' of Windows's dir command, with the main switches"); moreprint("+ : /s Recursive (note: Win-switch must be BEFORE 'normal' switch"); @@ -40749,6 +42355,7 @@ void help_dir(bool i_usage,bool i_example) moreprint("100 biggest dup. files in c:\\z?: dir c:\\z /s -crc32 -n 100"); moreprint("Show -md5 checksum, recurse by size: dir z:\\cb /s /os -checksum -md5"); moreprint("Like dir: dir z:\\cb\\*.avi"); + moreprint("Better than dir: dir c:\\*.cpp /s /os -n 100"); } } void help_k(bool i_usage,bool i_example) @@ -40883,7 +42490,7 @@ void help_franzswitches(bool i_usage,bool i_example) moreprint("+ : -filelist Store the add() list in VFILE-l-filelist.txt"); moreprint("+ : -nodedup Disabling deduplication"); help_orderby(); - moreprint("+ : -tar tar mode: turn off deduplication and compression"); + moreprint("+ : -store Store mode: turn off deduplication and compression"); moreprint("+ : -norecursion Do not recurse into folders (default: YES)"); #ifdef _WIN32 moreprint("+ : -longpath add/extract on Windows filenames longer than 255"); @@ -40958,6 +42565,10 @@ void Jidac::load_help_map() help_map.insert(std::pair("isopen", help_isopen)); help_map.insert(std::pair("autotest", help_autotest)); help_map.insert(std::pair("versum", help_versum)); + help_map.insert(std::pair("zfsbackup", help_zfsbackup)); + help_map.insert(std::pair("zfsrestore", help_zfsrestore)); + help_map.insert(std::pair("zfsreceive", help_zfsreceive)); + #if defined(_WIN32) help_map.insert(std::pair("sfx", help_sfx)); help_map.insert(std::pair("rd", help_rd)); @@ -41330,6 +42941,9 @@ void decoderange(string i_range) myprintf("26450: Rangefrom %d\n",g_rangefrom); myprintf("26451: Rangeto %d\n",g_rangeto); } + + + if (g_rangeto FROM (%d)\n",g_rangeto,g_rangefrom); @@ -41348,6 +42962,11 @@ void decoderange(string i_range) myprintf("26457: cannot decode range |%s| you should use something like 2:4 or 2: or :3\n",i_range.c_str()); seppuku(); } + if ((!flagpakka) && (!flagsilent) && (!flagstdout)) + { + myprintf("franz:%-21s %21s\n","rangefrom (version)",migliaia(g_rangefrom)); + myprintf("franz:%-21s %21s\n","rangeto (version)",migliaia(g_rangeto)); + } } /* @@ -41385,33 +43004,419 @@ string my_realpath(std::string const& i_path) #include #endif + +string finalizza_none(void* p_hash) +{ + if (p_hash==NULL) + return ""; + return ""; +} + +string finalizza_sha3(void* p_hash) +{ + if (p_hash==NULL) + return ""; + SHA3* casted=(SHA3*)p_hash; + return (*casted).getHash(); +} +string finalizza_xxhash64(void* p_hash) +{ + if (p_hash==NULL) + return ""; + XXHash64* casted=(XXHash64*)p_hash; + char temp[40]; + snprintf(temp,sizeof(temp),"%016llX",(unsigned long long)(*casted).hash()); + return temp; + // return binarytohex((const unsigned char*)(*casted).hash(),8); +} +string finalizza_sha1(void* p_hash) +{ + if (p_hash==NULL) + return ""; + libzpaq::SHA1* casted=(libzpaq::SHA1*)p_hash; + char sha1result[20]; + memcpy(sha1result, (*casted).result(), 20); + return binarytohex((const unsigned char*)sha1result,20); +} +string finalizza_sha256(void* p_hash) +{ + if (p_hash==NULL) + return ""; + libzpaq::SHA256* casted=(libzpaq::SHA256*)p_hash; + char sha256result[32]; + memcpy(sha256result, (*casted).result(), 32); + return binarytohex((const unsigned char*)sha256result,32); +} +string finalizza_xxh3(void* p_hash) +{ + if (p_hash==NULL) + return ""; + XXH3_state_t* casted=(XXH3_state_t*)p_hash; + char temp[40]; + XXH128_hash_t myhash=XXH3_128bits_digest(casted); + snprintf(temp,sizeof(temp),"%016llX%016llX",(unsigned long long)myhash.high64,(unsigned long long)myhash.low64); + return temp; +} +string finalizza_md5(void* p_hash) +{ + if (p_hash==NULL) + return ""; + MD5* casted=(MD5*)p_hash; + return (*casted).getHash(); +} +string finalizza_blake3(void* p_hash) +{ + if (p_hash==NULL) + return ""; + blake3_hasher * casted=(blake3_hasher *)p_hash; + uint8_t output[BLAKE3_OUT_LEN]; + blake3_hasher_finalize(casted,output,BLAKE3_OUT_LEN); + return binarytohex((const unsigned char*)output,BLAKE3_OUT_LEN); +} +string finalizza_whirlpool(void* p_hash) +{ + if (p_hash==NULL) + return ""; + NESSIEstruct * casted=(NESSIEstruct *)p_hash; + unsigned char output[512]={0}; + NESSIEfinalize(casted,output); + return binarytohex(output,64); +} +string finalizza_highway64(void* p_hash) +{ + if (p_hash==NULL) + return ""; + HighwayHashCat * casted=(HighwayHashCat *)p_hash; + uint64_t hash=HighwayHashCatFinish64(casted); + char temp[40]; + snprintf(temp,sizeof(temp),"%016llX",(unsigned long long)hash); + return temp; + + ///return binarytohex((const unsigned char*)&hash,8); +} +string finalizza_highway128(void* p_hash) +{ + if (p_hash==NULL) + return ""; + HighwayHashCat * casted=(HighwayHashCat *)p_hash; + uint64_t hash[2]; + HighwayHashCatFinish128(casted,hash); + return binarytohex((const unsigned char*)&hash,16); +} +string finalizza_highway256(void* p_hash) +{ + if (p_hash==NULL) + return ""; + HighwayHashCat * casted=(HighwayHashCat *)p_hash; + uint64_t hash[4]; + HighwayHashCatFinish256(casted,hash); + return binarytohex((const unsigned char*)&hash,32); +} + +string finalizza_crc32(void* p_hash) +{ + if (p_hash==NULL) + return ""; + uint32_t* casted=(uint32_t*)p_hash; + char temp[10]; + snprintf(temp,sizeof(temp),"%08X",(*casted)); + return temp; +} +string finalizza_crc32c(void* p_hash) +{ + if (p_hash==NULL) + return ""; + uint32_t* casted=(uint32_t*)p_hash; + char temp[10]; + snprintf(temp,sizeof(temp),"%08X",(*casted)); + return temp; +} +string finalizza_wyhash(void* p_hash) +{ + if (p_hash==NULL) + return ""; + return ""; +} +string finalizza_nilsimsa(void* p_hash) +{ + if (p_hash==NULL) + return ""; + return ""; +} +string finalizza_entropy(void* p_hash) +{ + if (p_hash==NULL) + return ""; + return ""; +} +bool Jidac::cli_onlyfiles(string i_opt,string i_string,char i_command,int argc,const char** argv, int* i_i) +{ + if ((i_opt==i_string)) + { + command=i_command; + while (++(*i_i)=1) + { + string mytimestamp=argv[(*i_i)+1]; + (*i_i)++; + ///format_datetime + + int64_t newdate=encodestringdate(mytimestamp,true); + if (newdate!=-1) + { + myprintf("franz: %s change from %s => %s\n",i_string.c_str(),dateToString(flagutc,*o_date).c_str(),dateToString(flagutc,newdate).c_str()); + (*o_date)=newdate; + if ((!flagpakka) && (!flagsilent) && (!flagstdout)) + myprintf("franz:%-21s <<%s>>\n",i_string.c_str(),dateToString(true,newdate).c_str()); + } + } + return false; +} + +bool Jidac::cli_onlyfilename(string i_opt,string i_string,string i_alias,string& o_thefile,int argc,const char** argv, int* i_i,bool* i_theflag) +{ + if (i_alias=="") + i_alias="@N3b12vc$fjvn4ba"; + + if ((i_opt==i_string) || (i_opt==i_alias)) + { + if (o_thefile=="") + { + if (++(*i_i)>\n",i_string.c_str(),o_thefile.c_str()); + + if (i_theflag!=NULL) + (*i_theflag)=true; + } + else + (*i_i)--; + } + return true; + } + return false; +} + + + + // Parse the command line. Return 1 if error else 0. int Jidac::doCommand(int argc, const char** argv) { + bool flagdummy=false; + + g_programflags.add(&flag715, "-715", "Runs just about like zpaq 7.15", "a;"); + g_programflags.add(&flagappend, "-append", "Append-only (antiransomware, slow)", "a;"); + g_programflags.add(&flagbig, "-big", "Big", "all;"); + g_programflags.add(&flagchecksum, "-checksum", "Do checksums", ""); + g_programflags.add(&flagchecktxt, "-checktxt", "Checktxt", ""); + g_programflags.add(&flagcomment, "-comment", "Comment version", ""); + g_programflags.add(&flagdebug, "-debug", "Activate debug mode", ""); + g_programflags.add(&flagdesc, "-desc", "Orderby desc", ""); + g_programflags.add(&flagdonotforcexls, "-xls", "Do NOT force adding of XLS/PPT (default: NO)", "a;"); + g_programflags.add(&flagdummy, "-debug -zero -kill", "Add 0-byte long file (debugging)", ""); + g_programflags.add(&flagdummy, "-debug -zero", "Add files but zero-filled (debugging)", ""); + g_programflags.add(&flagfilelist, "-filelist", "Create a filelist .txt", ""); + g_programflags.add(&flagfix255, "-fix255", "Fix 255", ""); + g_programflags.add(&flagfixeml, "-fixeml", "Fix eml filenames", ""); + g_programflags.add(&flagflat, "-flat", "Flat filenames", ""); + g_programflags.add(&flagforce, "-force", "Force", ""); + g_programflags.add(&flagforcewindows, "-forcewindows", "Store ADS stuff (default: NO)", "a;"); + g_programflags.add(&flagforcezfs, "-forcezfs", "Enforce using .zfs", ""); + g_programflags.add(&flagfrugal, "-frugal", "Frugal backup", ""); + g_programflags.add(&flaghashdeep, "-hashdeep", "Hashdeep", ""); + g_programflags.add(&flagkill, "-kill", "Kill", ""); + g_programflags.add(&flagmm, "-mm", "Memory mapped", ""); + g_programflags.add(&flagnoattributes, "-noattributes", "Nessun attributo", ""); + g_programflags.add(&flagnodedup, "-nodedup", "Turn off deduplicator", ""); + g_programflags.add(&flagnoeta, "-noeta", "Do not show ETA", ""); + g_programflags.add(&flagnopath, "-nopath", "Do not store path", ""); + g_programflags.add(&flagnoqnap, "-noqnap", "Exclude QNAP snap & trash", "a;"); + g_programflags.add(&flagnorecursion, "-norecursion", "Do not recurse into folders (default: YES)", ""); + g_programflags.add(&flagnosort, "-nosort", "Do not sort file", ""); + g_programflags.add(&flagpakka, "-pakka", "New output", ""); + g_programflags.add(&flagparanoid, "-paranoid", "Paranoid", ""); + g_programflags.add(&flagramdisk, "-ramdisk", "ramdisk", ""); + g_programflags.add(&flagrename, "-rename", "Rename", ""); + g_programflags.add(&flagsilent, "-silent", "Silent", ""); + g_programflags.add(&flagskipzfs, "-zfs", "Do NOT ignore .zfs (default: YES)", ""); //flagskipzfs + g_programflags.add(&flagspace, "-space", "Do not check space/writeability", ""); + g_programflags.add(&flagssd, "-ssd", "SSD", ""); + g_programflags.add(&flagstat, "-stat", "Statistics", ""); + g_programflags.add(&flagstdin, "-stdin", "stdin", ""); + g_programflags.add(&flagstdout, "-stdout", "stdout", ""); + g_programflags.add(&flagstore, "-store", "Store mode: no deduplication, no compression", ""); + g_programflags.add(&flagtar, "-tar", "TAR mode (store posix)", ""); + g_programflags.add(&flagtest, "-test", "Only do test", ""); + g_programflags.add(&flagtouch, "-touch", "Force 'touch' on date (7.15 to zpaqfranz)", ""); + g_programflags.add(&flagutc, "-utc", "Use UTC time", ""); + g_programflags.add(&flagutf, "-utf", "UTF-8", ""); + g_programflags.add(&flagverbose, "-verbose", "Verbose output", ""); + g_programflags.add(&flagverify, "-verify", "Verify", ""); + g_programflags.add(&flagvss, "-vss", "Enable Volume Shadow Copies", "a;"); + g_programflags.add(&flagzero, "-zero", "Flag zero", ""); + + g_programflags.add(&g_sfxflagforce, "-sfxforce", "Sfx force", ""); + g_programflags.add(&g_sfxflagall, "-sfxall", "Sfx all", ""); + + g_programflags.add(&flagbarraod, "/od", "Order by date", "dir;"); + g_programflags.add(&flagbarraon, "/on", "Order by name", "dir;"); + g_programflags.add(&flagbarraos, "/os", "Order by size", "dir;"); + +#ifdef _WIN32 + g_programflags.add(&flagdd, "-dd", "dd", ""); + g_programflags.add(&flagfindzpaq, "-findzpaq", "Search .zpaq in every drive letter (USB device)", ""); + g_programflags.add(&flagfixcase, "-fixcase", "Fix CAse", ""); + g_programflags.add(&flagfixreserved, "-fixreserverd", "fixreserverd", ""); + g_programflags.add(&flagimage, "-image", "Drive image", ""); + g_programflags.add(&flaglongpath, "-longpath", "Longpath", ""); + g_programflags.add(&flagopen, "-open", "open", ""); +#endif + + g_programflags.add(&flaghw, "-hw", "Use HW SHA1", "a;x;"); + + for (int i=0; i(FRANZO_NONE , tipohash("NONE", 0,"Disable checksums", true ,"-nochecksum", &flagnochecksum, &finalizza_none, ""))); + g_mappatipohash.insert(std::pair(FRANZO_CRC_32 , tipohash("CRC-32", 8,"Old but ubiquitous, superfast, not very strong", true ,"-crc32", &flagcrc32, &finalizza_crc32, ""))); + g_mappatipohash.insert(std::pair(FRANZO_XXHASH64 , tipohash("XXHASH64", 16,"Very fast, low CPU usage, zpaqfranz's default", true ,"-xxhash", &flagxxhash64, &finalizza_xxhash64, ""))); + g_mappatipohash.insert(std::pair(FRANZO_SHA_1 , tipohash("SHA-1", 40,"Fair speed, very reliable, some collisions known", true ,"-sha1", &flagsha1, &finalizza_sha1, "08"))); + g_mappatipohash.insert(std::pair(FRANZO_SHA_256 , tipohash("SHA-256", 64,"CPU intensive, one of the most reliable. Legal proof in EU", true ,"-sha256", &flagsha256, &finalizza_sha256, "04"))); + g_mappatipohash.insert(std::pair(FRANZO_XXH3 , tipohash("XXH3", 32,"Very fast an dstrong, zpaqfranz default for file comparing", true ,"-xxh3", &flagxxh3, &finalizza_xxh3, "02"))); + g_mappatipohash.insert(std::pair(FRANZO_MD5 , tipohash("MD5", 32,"Common, fast, hashdeep compatible. Not cryptographic good", true ,"-md5", &flagmd5, &finalizza_md5, "01"))); + g_mappatipohash.insert(std::pair(FRANZO_BLAKE3 , tipohash("BLAKE3", 64,"Fast, CPU intensive (on Win64 HW acceleration), reliable", true ,"-blake3", &flagblake3, &finalizza_blake3, "03"))); + g_mappatipohash.insert(std::pair(FRANZO_SHA3 , tipohash("SHA-3", 64,"Latest NIST standard, very strong", true ,"-sha3", &flagsha3, &finalizza_sha3, "09"))); + g_mappatipohash.insert(std::pair(FRANZO_WHIRLPOOL , tipohash("WHIRLPOOL", 128,"Slow but very reliable", true ,"-whirlpool", &flagwhirlpool, &finalizza_whirlpool, "06"))); + g_mappatipohash.insert(std::pair(FRANZO_HIGHWAY64 , tipohash("HIGHWAY64", 16,"Experimental Google's HighwayHash, straight C, 64 bit", true ,"-highway64", &flaghighway64, &finalizza_highway64, "11"))); + g_mappatipohash.insert(std::pair(FRANZO_HIGHWAY128, tipohash("HIGHWAY128", 32,"Experimental Google's HighwayHash, straight C, 128 bit", true ,"-highway128", &flaghighway128, &finalizza_highway128, "12"))); + g_mappatipohash.insert(std::pair(FRANZO_HIGHWAY256, tipohash("HIGHWAY256", 64,"Experimental Google's HighwayHash, straight C, 256 bit", true ,"-highway256", &flaghighway256, &finalizza_highway256, "13"))); + g_mappatipohash.insert(std::pair(ALGO_CRC32C , tipohash("CRC-32C", 8,"'Castagnoli' variation of CRC-32, HW accelerated", false ,"-crc32c", &flagcrc32c, &finalizza_crc32c, ""))); + g_mappatipohash.insert(std::pair(ALGO_WYHASH , tipohash("WYHASH", 16,"Maybe the fastest, limited 'strength'", false ,"-wyhash", &flagwyhash, &finalizza_wyhash, ""))); + g_mappatipohash.insert(std::pair(ALGO_NILSIMSA , tipohash("NILSIMSA", 128,"Look for similarities, not differences", false ,"-nilsimsa", &flagnilsimsa, &finalizza_nilsimsa, ""))); + g_mappatipohash.insert(std::pair(ALGO_ENTROPY , tipohash("ENTROPY", 6,"Quick-and-dirty entropy estimator", false ,"-entropy", &flagentropy, &finalizza_entropy, ""))); + +#ifdef _WIN32 + g_mappatipohash.insert(std::pair(FRANZO_WINHASH64,tipohash("WINHASH64", 16,"Same as xxhash64 (with Windows' dates)", true ,"-windate", &flagwindate, &finalizza_xxhash64, "10"))); +#endif // Initialize options to default values // Why some variables out of Jidac? Because of pthread: does not like very much C++ objects // quick and dirty. g_ioBUFSIZE =4096; //just like zpaq - + g_thechosenhash =ALGO_SHA1; //ALGO_SHA1; + g_thechosenhash_str ="SHA-1"; fullcommandline =""; plainpassword =""; g_franzsnap ="c:/franzsnap"; g_vss_shadow =""; - g_franzotype =2; //by default take CRC-32 AND XXHASH64 + g_franzotype =FRANZO_XXHASH64; //by default take CRC-32 AND XXHASH64 + g_255 =0; // errors on longer than 255 chars g_sfx =""; g_sfxto =""; g_sfxnot =""; g_sfxonly =""; g_sfxuntil =""; - g_sfxflagforce =false; - g_sfxflagall =false; g_freeze =""; g_archive =""; g_indexname =""; g_output_handle =NULL; g_output =""; g_exec_error =""; - g_255 =0; // errors on longer than 255 chars g_exec_warn =""; g_exec_ok =""; g_exec =""; @@ -41419,78 +43424,6 @@ int Jidac::doCommand(int argc, const char** argv) g_copy =""; g_until =""; command =0; - flagforce =false; - flagzero =false; - flagnoattributes =false; - flagbarraod =false; - flagbarraon =false; - flagbarraos =false; - flagdebug =false; - flagnodedup =false; - flagtar =false; - flagramdisk =false; - flagssd =false; - flaghashdeep =false; - flagbig =false; - flagchecktxt =false; - flagstdin =false; - flagstdout =false; - flagnorecursion =false; - flagsilent =false; - flagtouch =false; - flagrename =false; - flagfrugal =false; - flagstat =false; - flagdesc =false; - flagtest =false; - flagskipzfs =false; - flagverbose =false; - flagnoqnap =false; - flagforcewindows =false; - flagnopath =false; - flagnoeta =false; - flagpakka =false; - flagvss =false; - flagnosort =false; - flagchecksum =false; - flagnochecksum =false; - flagcrc32c =false; - flagentropy =false; - flagsha1 =false; - flagverify =false; - flagkill =false; - flagutf =false; - flagfix255 =false; -#ifdef _WIN32 - flagfixcase =false; - flagfixreserved =false; - flaglongpath =false; - flagfindzpaq =false; - flagwindate =false; - flagopen =false; - flagdd =false; - flagimage =false; -#endif - flagfixeml =false; - flagflat =false; - flagparanoid =false; - flagxxh3 =false; - flagcrc32 =false; - flagsha256 =false; - flagwyhash =false; - flagwhirlpool =false; - flagmd5 =false; - flagsha3 =false; - flagblake3 =false; - flagxxhash64 =false; - flagdonotforcexls =false; - flagcomment =false; - flag715 =false; - flagfilelist =false; - flagmm =false; - flagappend =false; - flagutc =false; - flaghw =false; fragment =6; minsize =0; maxsize =0; @@ -41517,10 +43450,9 @@ int Jidac::doCommand(int argc, const char** argv) date =0; if (argc>1) for (int i=1; i0) { zpaqfranzexename=argv[0]; @@ -41529,6 +43461,9 @@ int Jidac::doCommand(int argc, const char** argv) #endif } + + + #ifdef _WIN32 wchar_t myexepath[_MAX_PATH]; GetModuleFileName(NULL,myexepath,_MAX_PATH); @@ -41595,33 +43530,21 @@ int Jidac::doCommand(int argc, const char** argv) return dir(); } #endif + + if (flagdebug) printf("FULL exename <<%s>>\n",fullzpaqexename.c_str()); - /// if we have a -pakka, set the flag early - for (int i=0; itm_year+1900)*10000000000LL+(t->tm_mon+1)*100000000LL +t->tm_mday*1000000+t->tm_hour*10000+t->tm_min*100+t->tm_sec; g_optional=""; -// Get optional options + + /// a bit of refactoring. getting, early, the hasher + string dummyhash="-dummyhash"; + int howmanyhasher=0; + for (int i=1; isecond.switchname)) + if ((p->second.switchflag)!=NULL) + { + *(p->second.switchflag)=true; + argv[i]=dummyhash.c_str(); + howmanyhasher++; + g_thechosenhash=p->first; + g_thechosenhash_str=p->second.hashname; + } + if (howmanyhasher>1) + { + myprintf("\n42418: ABORT: you cannot choose more than ONE hasher, but you picked %d\n",howmanyhasher); + for (MAPPATIPOHASH::iterator p=g_mappatipohash.begin(); p!=g_mappatipohash.end(); ++p) + if (p->second.switchname!="") + myprintf("%s ",p->second.switchname.c_str()); + myprintf("\n"); + seppuku(); + return 0; + } + if (flagdebug) + myprintf("42993: The chosen algo %d %s\n",g_thechosenhash,g_thechosenhash_str.c_str()); + +// Get optional options. the hasher turned to -dummyhash + + for (int i=1; iopt %s\n",opt.c_str()); - if (opt=="-715") - { - flag715=true; - } - else - if (opt=="/od") - { - flagbarraod=true; - //i++; - } - else - if (opt=="/on") - { - flagbarraon=true; - ///i++; - } - else - if (opt=="/os") + + if (opt=="-dummyhash") { - flagbarraos=true; - ///i++; + /// taken by hash selection } else if ((opt=="benchmark") || (opt=="b")) - { command='b'; - } else if ((opt=="autotest")) - { command='8'; - } else if ((opt=="pause")) - { command='0'; - } else - if ((opt=="trim")) - { - command='4'; - while (++i=1) - { - searchfrom=argv[i+1]; - i++; - } - } - else if (opt=="-orderby") - { - if (i!=argc-1) // not the last parameter - if (orderby=="" && strlen(argv[i+1])>=1) - { - orderby=argv[i+1]; - i++; - } - } - else if ((opt=="-findhash") || (opt=="-searchhash")) - { - if (i!=argc-1) // not the last parameter - if (searchhash=="" && strlen(argv[i+1])>=1) - { - searchhash=argv[i+1]; - i++; - } - } - else if (opt=="-replace") - { - if (i!=argc-1) // not the last parameter - if (replaceto=="" && strlen(argv[i+1])>=1) - { - replaceto=argv[i+1]; - i++; - } - } - else if ((opt=="-exec_error") || (opt=="-exec_err")) - { - if (g_exec_error=="") - { - if (++i=1) - { - string mytimestamp=argv[i+1]; - i++; - int64_t newdate=encodestringdate(mytimestamp,true); - if (newdate!=-1) - { - myprintf("franz: -timestamp change from %s => %s\n",dateToString(flagutc,date).c_str(),dateToString(flagutc,newdate).c_str()); - date=newdate; - } - } - } - else if (opt=="-datefrom") - { - g_datefrom=0; - if (i+1=1) - { - string mytimestamp=argv[i+1]; - i++; - int64_t newdate=encodestringdate(mytimestamp,true); - if (newdate!=-1) - g_datefrom=newdate; - } - } - else if (opt=="-dateto") - { - g_dateto=0; - if (i+1=1) - { - string mytimestamp=argv[i+1]; - i++; - int64_t newdate=encodestringdate(mytimestamp,false); - if (newdate!=-1) - g_dateto=newdate; - } - } - else if (opt=="-force" || opt=="-f") flagforce =true; - else if (opt=="-sfxforce") g_sfxflagforce =true; - else if (opt=="-sfxall") g_sfxflagall =true; - else if (opt=="-zero") flagzero =true; - else if (opt=="-noattributes") flagnoattributes =true; - else if (opt=="-test") flagtest =true; - else if (opt=="-zfs") flagskipzfs =true; - else if (opt=="-forcezfs") flagforcezfs =true; - else if (opt=="-space") flagspace =true; - else if (opt=="-filelist") flagfilelist =true; - else if (opt=="-xls") flagdonotforcexls =true; - else if (opt=="-verbose") flagverbose =true; - else if (opt=="-v") flagverbose =true; - else if (opt=="-debug") flagdebug =true; -#ifdef HWSHA1 - else if (opt=="-hw") flaghw =true; -#endif - else if (opt=="-nodedup") flagnodedup =true; - else if (opt=="-tar") flagtar =true; - else if (opt=="-ramdisk") flagramdisk =true; - else if (opt=="-ssd") flagssd =true; - else if (opt=="-hashdeep") flaghashdeep =true; - else if (opt=="-big") flagbig =true; - else if (opt=="-stdin") flagstdin =true; - else if (opt=="-stdout") flagstdout =true; - else if (opt=="-norecursion") flagnorecursion =true; - else if (opt=="-silent") flagsilent =true; - else if (opt=="-touch") flagtouch =true; - else if (opt=="-rename") flagrename =true; - else if (opt=="-frugal") flagfrugal =true; - else if (opt=="-stat") flagstat =true; - else if (opt=="-desc") flagdesc =true; - else if (opt=="-noqnap") flagnoqnap =true; - else if (opt=="-nopath") flagnopath =true; - else if (opt=="-nosort") flagnosort =true; - else if (opt=="-checksum") flagchecksum =true; - else if (opt=="-nochecksum") flagnochecksum =true; - else if (opt=="-crc32c") flagcrc32c =true; - else if (opt=="-entropy") flagentropy =true; - else if (opt=="-sha1") flagsha1 =true;// stub: by default flagsha1 - else if (opt=="-xxh3") flagxxh3 =true; - else if (opt=="-sha256") flagsha256 =true; - else if (opt=="-blake3") flagblake3 =true; - else if (opt=="-mm") flagmm =true; - else if (opt=="-append") flagappend =true; - else if (opt=="-utc") flagutc =true; - else if (opt=="-whirlpool") flagwhirlpool =true; - else if (opt=="-md5") flagmd5 =true; - else if (opt=="-sha3") flagsha3 =true; - else if (opt=="-nilsimsa") flagnilsimsa =true; - else if (opt=="-wyhash") - { - flagwyhash=true; - myprintf("\n****** WARNING: WYHASH IMPLEMENTATION IS JUST FOR TEST *******\n\n"); - } - else if (opt=="-xxhash") flagxxhash64 =true; - else if (opt=="-crc32") flagcrc32 =true; - else if (opt=="-verify") flagverify =true; - else if (opt=="-kill") flagkill =true; - else if (opt=="-utf") flagutf =true; - else if (opt=="-utf8") flagutf =true;//alias - else if (opt=="-fix255") flagfix255 =true; -#ifdef _WIN32 - else if (opt=="-image") flagimage =true; - else if (opt=="-dd") flagdd =true; - else if (opt=="-open") flagopen =true; - else if (opt=="-fixcase") flagfixcase =true; - else if (opt=="-fixreserved") flagfixreserved =true; - else if (opt=="-longpath") flaglongpath =true; - else if (opt=="-findzpaq") flagfindzpaq =true; - else if (opt=="-windate") flagwindate =true; -#endif - else if (opt=="-fixeml") flagfixeml =true; - else if (opt=="-flat") flagflat =true; - else if (opt=="-paranoid") flagparanoid =true; - else if (opt=="-noeta") flagnoeta =true; - else if (opt=="-pakka") flagpakka =true; - else if (opt=="-vss") flagvss =true; - else if (opt=="-forcewindows") flagforcewindows =true; + + else + if (cli_onlyfilename(opt,"-find","-search",searchfrom,argc,argv,&i,NULL)); + else + if (cli_onlyfilename(opt,"-replace","",replaceto,argc,argv,&i,NULL)); + else + if (cli_onlyfilename(opt,"-orderby","",orderby,argc,argv,&i,NULL)); + else + if (cli_onlyfilename(opt,"-findhash","-searchhash",searchhash,argc,argv,&i,NULL)); + else + if (cli_onlyfilename(opt,"-exec_error","-exec_err",g_exec_error,argc,argv,&i,NULL)); + else + if (cli_onlyfilename(opt,"-exec_warn","-exec_warn",g_exec_warn,argc,argv,&i,NULL)); + else + if (cli_onlyfilename(opt,"-exec_ok","",g_exec_ok,argc,argv,&i,NULL)); + else + if (cli_onlyfilename(opt,"-copy","",g_copy,argc,argv,&i,NULL)); + else + if (cli_onlyfilename(opt,"-freeze","",g_freeze,argc,argv,&i,NULL)); + else + if (cli_onlyfilename(opt,"-output","-out",g_output,argc,argv,&i,NULL)); + else + if (cli_onlyfilename(opt,"-sfx","",g_sfx,argc,argv,&i,NULL)); + else + if (cli_onlyfilename(opt,"-sfxto","",g_sfxto,argc,argv,&i,NULL)); + else + if (cli_onlyfilename(opt,"-exec","",g_exec,argc,argv,&i,NULL)); + else + if (cli_onlyfilename(opt,"-exec","",g_exec,argc,argv,&i,NULL)); + else + if (cli_onlyfilename(opt,"-comment","",versioncomment,argc,argv,&i,&flagcomment)); + else + if (cli_onlyfilename(opt,"-checktxt","",checktxt,argc,argv,&i,&flagchecktxt)); + else + if (cli_getdate(opt,"-timestamp",argc,argv,&i,&date)); + else + if (cli_getdate(opt,"-datefrom",argc,argv,&i,&g_datefrom)); + else + if (cli_getdate(opt,"-dateto",argc,argv,&i,&g_dateto)); + + else if (opt=="-not") { // read notfiles while (++i0) - franzparameters+="-datefrom "+dateToString(true,g_datefrom)+" "; - if (g_dateto>0) - franzparameters+="-dateto "+dateToString(true,g_dateto)+" "; - if (flag715) - franzparameters+="mode -715 activated " ; - if (flagfilelist) - franzparameters+="-filelist " ; - if (flagnoqnap) - franzparameters+="Exclude QNAP snap & trash (-noqnap) "; - if (flagforcewindows) - franzparameters+="Force Windows stuff (-forcewindows) "; - if (flagnopath) - franzparameters+="Do not store path (-nopath) "; - if (flagzero) - franzparameters+="Zero file "; - if (flagnosort) - franzparameters+="-nosort "; - if (flagdonotforcexls) - franzparameters+="Do not force XLS/PPT (-xls) "; - if (flagspace) - franzparameters+="Do not check free space/writeability "; - if (flagverbose) - franzparameters+="-verbose "; - if (flagchecksum) - franzparameters+="-checksum "; - if (flagnochecksum) - franzparameters+="NO checksum (-nochecksum) "; - if (flagcrc32c) - franzparameters+="-crc32c "; - if (flagentropy) - franzparameters+="-entropy "; - if (flagsha1) - franzparameters+="-sha1 "; - if (flagcrc32) - franzparameters+="-crc32 "; - if (flagxxh3) - franzparameters+="-xxh3 "; - if (flagsha256) - franzparameters+="-sha256 "; - if (flagwyhash) - franzparameters+="-wyhash "; - if (flagwhirlpool) - franzparameters+="-whirlpool "; - if (flagnilsimsa) - franzparameters+="-nilsimsa "; - if (flagmd5) - franzparameters+="-md5 "; - if (flagsha3) - franzparameters+="-sha3 "; - if (flagblake3) - franzparameters+="-blake3 "; - if (flagxxhash64) - franzparameters+="xxhash64 (-xxhash) "; - if (flagmm) - franzparameters+="memory mapped file (-mm) "; - if (flagappend) - franzparameters+="append (-append) "; - if (flagutc) - franzparameters+="UTC instead of local time (-utc) "; - if (flagverify) - franzparameters+="-verify "; - if (flagkill) - franzparameters+="do a wet run! (-kill) "; - if (flagutf) - franzparameters+="flagutf (-utf / -utf8) "; - if (flagfix255) - franzparameters+="-fix255 "; - if (flagfixcase) - flagstat=true; -#ifdef _WIN32 - if (flagfixcase) - franzparameters+="-fixcase "; - if (flagfixreserved) - franzparameters+="-fixreserved "; - if (flaglongpath) - franzparameters+="Long path (on Windows) "; - if (flagfindzpaq) - franzparameters+="Find zpaq archive (on Windows) "; - if (flagwindate) - franzparameters+="winhash64 (-windate) "; -#endif - if (flagfixeml) - franzparameters+="Fix eml filenames (-fixeml) "; - if (flagflat) - franzparameters+="Flat filenames (-flat) "; - if (flagparanoid) - franzparameters+="-paranoid "; - if (flagnodedup) - franzparameters+="-nodedup "; - if (flagtar) - franzparameters+="TAR mode "; - if (flagramdisk) - franzparameters+="-ramdisk "; - if (flagssd) - franzparameters+="-ssd "; - if (flaghashdeep) - franzparameters+="-hashdeep "; - if (flagbig) - franzparameters+="-big "; - if (flagstdin) - franzparameters+="-stdin "; - if (flagstdout) - franzparameters+="-stdout "; - if (flagnorecursion) - franzparameters+="-norecursion "; - if (flagtouch) - franzparameters+="-touch "; - if (flagrename) - franzparameters+="-rename "; -#ifdef _WIN32 - if (flagimage) - franzparameters+="-image "; - if (flagdd) - franzparameters+="-dd "; - if (flagopen) - franzparameters+="-open "; -#endif - if (flagfrugal) - franzparameters+="-frugal "; - if (flagstat) - franzparameters+="-stat "; - if (flagdebug) - franzparameters+="DEBUG very verbose (-debug) "; - if (flagdesc) - if (orderby!="") - franzparameters+="desc sort "; -#ifdef HWSHA1 - if (flaghw) - { - franzparameters+="SHA1 HW "; - Sha1Prepare(true); - } - else - Sha1Prepare(false); -#endif - if (flagcomment) - franzparameters+="-comment <<"+versioncomment+">> "; - - if (summary>0) - franzparameters+="-summary "; - char temp[100]; - if (minsize) - { - snprintf(temp,sizeof(temp),"minsize %s ",migliaia((int64_t)minsize)); - franzparameters+=temp; - } - if (maxsize) - { - snprintf(temp,sizeof(temp),"maxsize %s ",migliaia((int64_t)maxsize)); - franzparameters+=temp; - } - if (dirlength) - { - snprintf(temp,sizeof(temp),"dirlength %s ",migliaia((int64_t)dirlength)); - franzparameters+=temp; - } - if (filelength) - { - snprintf(temp,sizeof(temp),"filelength %s ",migliaia((int64_t)filelength)); - franzparameters+=temp; - } - if (menoenne) - { - snprintf(temp,sizeof(temp),"limit (-n -limit) %u ",menoenne); - franzparameters+=temp; - } -#ifdef _WIN32 - if (flagvss) - { - franzparameters+=" VSS Volume Shadow Copy (-vss) "; - if (!isadmin()) - { - myprintf("\nImpossible to run VSS: admin rights required\n"); - return 2; - } - } -#endif - if (franzparameters!="") - myprintf("franz:%s\n",franzparameters.c_str()); - - if (flagchecktxt) - if (checktxt=="") - { - string percorso=extractfilepath(archive); - string nome=prendinomefileebasta(archive); - checktxt=percorso+nome+".txt"; - } +/* + Check from "weird" parameters +*/ - if (checktxt!="") - myprintf("franz:checktxt <<%s>>\n",checktxt .c_str()); - if (orderby!="") - myprintf("franz:orderby <<%s>>\n",orderby.c_str()); - if (searchhash!="") - myprintf("franz:find hash <<%s>>\n",searchhash.c_str()); - if (searchfrom!="") - myprintf("franz:find <<%s>>\n",searchfrom.c_str()); - if (replaceto!="") - myprintf("franz:replace <<%s>>\n",replaceto.c_str()); - if (g_exec_error!="") - myprintf("franz:exec_error <<%s>>\n",g_exec_error.c_str()); - if (g_exec_warn!="") - myprintf("franz:exec_warn <<%s>>\n",g_exec_warn.c_str()); - if (g_exec_ok!="") - myprintf("franz:exec_ok <<%s>>\n",g_exec_ok.c_str()); - if (g_exec!="") - myprintf("franz:exec <<%s>>\n",g_exec.c_str()); - if (g_copy!="") - myprintf("franz:copy <<%s>>\n",g_copy.c_str()); - if (g_output!="") - { - myprintf("franz:output <<%s>>\n",g_output.c_str()); - open_output(g_output); - } - if (g_sfx!="") - myprintf("franz:sfx <<%s>>\n",g_sfx.c_str()); - if (g_sfxto!="") - myprintf("franz:sfxto <<%s>>\n",g_sfxto.c_str()); - if (g_rangefrom>0) - myprintf("franz:rangefrom %08d (version)\n",g_rangefrom); - if (g_rangeto>0) - myprintf("franz:rangeto %08d (version)\n",g_rangeto); -#ifdef SERVER - if (flagserver) - { - myprintf("franz:zpaqfranz-server %s\n",g_server.c_str()); - myprintf("franz:server port %s\n",g_port.c_str()); - int64_t myport=myatoll(g_port.c_str()); - if (myport>65353) - { - myprintf("27952: server port must be [0..65353]\n"); - return 2; - } - } -#endif - if (g_ioBUFSIZE!=4096) - myprintf("franz: io buffer size %s\n",migliaia(g_ioBUFSIZE)); - } if (g_rangefrom>0) { if (g_until!="") @@ -42711,10 +44116,18 @@ int Jidac::doCommand(int argc, const char** argv) { myprintf("27626: -range needs -all\n"); return 2; - } - + } } + #ifdef _WIN32 + if (flagvss) + { + if (!isadmin()) + { + myprintf("\nImpossible to run VSS: admin rights required\n"); + return 2; + } + } if (flagimage && flagstdin) { myprintf("29886: -image incompatible with -stdin\n"); @@ -42736,9 +44149,67 @@ int Jidac::doCommand(int argc, const char** argv) myprintf("\nImpossible to image: admin rights required\n"); return 2; } +#endif +/* + fix some defaults +*/ + if (flagchecktxt) + if (checktxt=="") + { + string percorso=extractfilepath(archive); + string nome=prendinomefileebasta(archive); + checktxt=percorso+nome+".txt"; + } + + if (g_output!="") + open_output(g_output); + + if (flagforcezfs) + flagskipzfs=false; // win over skip + +#ifdef HWSHA1 + if (flaghw) + Sha1Prepare(true); + else + Sha1Prepare(false); #endif + +// write franzs' parameters (if not very briefly) + if ((!flagpakka) && (!flagstdout)) + { + string franzparameters=""; + + for (MAPPATIPOHASH::iterator p=g_mappatipohash.begin(); p!=g_mappatipohash.end(); ++p) + if ((p->second.switchflag)!=NULL) + if (*(p->second.switchflag)) + franzparameters+=p->second.switchname+" "; + + string attivati=g_programflags.compact(); + franzparameters+=attivati; + + if (summary>0) + franzparameters+="-summary "; + + if (franzparameters!="") + myprintf("franz:%s\n",franzparameters.c_str()); + +#ifdef SERVER + if (flagserver) + { + myprintf("franz:zpaqfranz-server %s\n",g_server.c_str()); + myprintf("franz:server port %s\n",g_port.c_str()); + int64_t myport=myatoll(g_port.c_str()); + if (myport>65353) + { + myprintf("27952: server port must be [0..65353]\n"); + return 2; + } + } +#endif + } + if (howmanythreads<1) howmanythreads=numberOfProcessors(); @@ -42776,6 +44247,7 @@ int Jidac::doCommand(int argc, const char** argv) /// if (!findFirstStreamW || !findNextStreamW) ///broken XP parser myprintf("Alternate streams not supported in Windows XP.\n"); #endif + if (flag715) { flagutc =true; @@ -42844,7 +44316,7 @@ int Jidac::doCommand(int argc, const char** argv) // Execute command if ( (command=='a') && files.size()>0) // enforce: we do not want to change anything when adding { - if (flagtar) + if (flagstore) { flagnodedup=true; method='0'; @@ -42853,48 +44325,28 @@ int Jidac::doCommand(int argc, const char** argv) flagfixeml =false; flagfix255 =false; flagutf =false; - flagfixcase =false; #ifdef _WIN32 + flagfixcase =false; flagfixreserved =false; #endif flagflat =false; - ///flagparanoid =false; + + g_franzotype=FRANZO_XXHASH64; // by default 2 (CRC32+XXHASH64) + + for (MAPPATIPOHASH::iterator p=g_mappatipohash.begin(); p!=g_mappatipohash.end(); ++p) + if (p->second.switchflag!=NULL) + if (*(p->second.switchflag)) + if (p->second.flagiszpaq) + g_franzotype=p->first; + if (flag715 || flagnochecksum) g_franzotype=FRANZO_NONE; - else - if (flagcrc32) - g_franzotype=FRANZO_CRC_32; - else - if (flagsha1 || flagchecksum) //backward compatibility 51 - g_franzotype=FRANZO_SHA_1; //SHA1 - else - if (flagsha256) - g_franzotype=FRANZO_SHA_256; - else - if (flagxxh3) - g_franzotype=FRANZO_XXH3; - else - if (flagblake3) - g_franzotype=FRANZO_BLAKE3; - else - if (flagsha3) - g_franzotype=FRANZO_SHA3; - else - if (flagmd5) - g_franzotype=FRANZO_MD5; - else -#ifdef _WIN32 - if (flagwindate) - g_franzotype=FRANZO_WINHASH64; - else -#endif - g_franzotype=FRANZO_XXHASH64; // by default 2 (CRC32+XXHASH64) + if (flagappend) return append(); else return add(); } -/// j,o,u,y else if (command=='1') return summa(); else if (command=='2') return dir(); else if (command=='3') return purgersync(); @@ -42935,6 +44387,8 @@ int Jidac::doCommand(int argc, const char** argv) else if (command=='x') return extract(); else if (command=='y') return sfx(); else if (command=='z') return zero(); + else if (command=='$') return zfsbackup(); + else if (command=='%') return zfsrestore(); else usage(); return 0; } @@ -43550,9 +45004,9 @@ int64_t Jidac::read_archive(const char* arc, int *errors, int i_myappend,bool i_ if (s+na>end || na>65535) error("attr too long"); if (!i_isinfo) - if (na>FRANZOFFSET) //Get FRANZOFFSET + if (na>FRANZOFFSETV1) //Get FRANZOFFSETV1 { - assert((na-8)0); // I do not like empty() @@ -44606,12 +46092,12 @@ void Jidac::writefranzattr(libzpaq::StringBuffer& i_sb, uint64_t i_data, unsigne } if (g_franzotype==FRANZO_CRC_32) /// store only CRC-32 { - char mybuffer[FRANZOFFSET]={0}; + char mybuffer[FRANZOFFSETV1]={0}; snprintf(mybuffer+41,sizeof(mybuffer)-41,"%08X",writtencrc); - puti(i_sb, 8+FRANZOFFSET, 4); // 8+FRANZOFFSET block + puti(i_sb, 8+FRANZOFFSETV1, 4); // 8+FRANZOFFSETV1 block puti(i_sb, i_data, i_quanti); puti(i_sb, 0, (8 - i_quanti)); // pad with zeros (for 7.15 little bug) - i_sb.write(mybuffer,FRANZOFFSET); + i_sb.write(mybuffer,FRANZOFFSETV1); if (flagdebug) myprintf("Mode1: CRC32 by frag <<%s>> %s\n",mybuffer+41,i_filename.c_str()); } @@ -44619,13 +46105,13 @@ void Jidac::writefranzattr(libzpaq::StringBuffer& i_sb, uint64_t i_data, unsigne if (g_franzotype==FRANZO_XXHASH64) { assert(i_thehash.length()==32); - char mybuffer[FRANZOFFSET]={0}; + char mybuffer[FRANZOFFSETV1]={0}; snprintf(mybuffer+8,sizeof(mybuffer)-8, "%s",i_thehash.c_str()); snprintf(mybuffer+41,sizeof(mybuffer)-41,"%08X",writtencrc); - puti(i_sb, 8+FRANZOFFSET, 4); + puti(i_sb, 8+FRANZOFFSETV1, 4); puti(i_sb, i_data, i_quanti); puti(i_sb, 0, (8 - i_quanti)); // pad with zeros (for 7.15 little bug) - i_sb.write(mybuffer,FRANZOFFSET); + i_sb.write(mybuffer,FRANZOFFSETV1); /// please note the dirty trick: start by +10 if (flagdebug) myprintf("Mode2: XXHASH64: <<%s>> CRC32 <<%s>> %s\n",mybuffer+8,mybuffer+41,i_filename.c_str()); @@ -44633,14 +46119,14 @@ void Jidac::writefranzattr(libzpaq::StringBuffer& i_sb, uint64_t i_data, unsigne else if (g_franzotype==FRANZO_SHA_1) //3= 51 (SHA1-0-CRC32) { - char mybuffer[FRANZOFFSETSHA256]={0}; + char mybuffer[FRANZOFFSETV2]={0}; snprintf(mybuffer,sizeof(mybuffer), "08"); //<<< look at this snprintf(mybuffer+2,sizeof(mybuffer)-2, "%s",i_thehash.c_str()); snprintf(mybuffer+2+64+1,sizeof(mybuffer)-2-64-1,"%08X",writtencrc); - puti(i_sb, 8+FRANZOFFSETSHA256, 4); // 8+FRANZOFFSET block + puti(i_sb, 8+FRANZOFFSETV2, 4); // 8+FRANZOFFSETV1 block puti(i_sb, i_data, i_quanti); puti(i_sb, 0, (8 - i_quanti)); // pad with zeros (for 7.15 little bug) - i_sb.write(mybuffer,FRANZOFFSETSHA256); ///please note the dirty trick: start by +8 + i_sb.write(mybuffer,FRANZOFFSETV2); ///please note the dirty trick: start by +8 if (flagdebug) myprintf("Model8: SHA1 <<%s>> CRC32 <<%s>> %s\n",mybuffer+2,mybuffer+67,i_filename.c_str()); } @@ -44648,14 +46134,14 @@ void Jidac::writefranzattr(libzpaq::StringBuffer& i_sb, uint64_t i_data, unsigne if (g_franzotype==FRANZO_SHA_256) ///4= 52 (01-SHA256-0-CRC32) { assert(i_thehash.length()==64); - char mybuffer[FRANZOFFSETSHA256]={0}; + char mybuffer[FRANZOFFSETV2]={0}; snprintf(mybuffer,sizeof(mybuffer), "04"); //<<< look at this snprintf(mybuffer+2,sizeof(mybuffer)-2, "%s",i_thehash.c_str()); snprintf(mybuffer+2+64+1,sizeof(mybuffer)-2-64-1,"%08X",writtencrc); - puti(i_sb, 8+FRANZOFFSETSHA256, 4); // 8+FRANZOFFSET block + puti(i_sb, 8+FRANZOFFSETV2, 4); // 8+FRANZOFFSETV1 block puti(i_sb, i_data, i_quanti); puti(i_sb, 0, (8 - i_quanti)); // pad with zeros (for 7.15 little bug) - i_sb.write(mybuffer,FRANZOFFSETSHA256); + i_sb.write(mybuffer,FRANZOFFSETV2); /// please note the dirty trick: start by +8 if (flagdebug) myprintf("Mode4: SHA256 <<%s>> CRC32 <<%s>> %s\n",mybuffer+2,mybuffer+67,i_filename.c_str()); @@ -44664,14 +46150,14 @@ void Jidac::writefranzattr(libzpaq::StringBuffer& i_sb, uint64_t i_data, unsigne if (g_franzotype==FRANZO_SHA3) ///2= 52 (01-SHA3-0-CRC32) { assert(i_thehash.length()==64); - char mybuffer[FRANZOFFSETSHA256]={0}; + char mybuffer[FRANZOFFSETV2]={0}; snprintf(mybuffer,sizeof(mybuffer), "02"); //<<< look at this snprintf(mybuffer+2,sizeof(mybuffer)-2, "%s",i_thehash.c_str()); snprintf(mybuffer+2+64+1,sizeof(mybuffer)-2-64-1,"%08X",writtencrc); - puti(i_sb, 8+FRANZOFFSETSHA256, 4); // 8+FRANZOFFSET block + puti(i_sb, 8+FRANZOFFSETV2, 4); // 8+FRANZOFFSETV1 block puti(i_sb, i_data, i_quanti); puti(i_sb, 0, (8 - i_quanti)); // pad with zeros (for 7.15 little bug) - i_sb.write(mybuffer,FRANZOFFSETSHA256); ///please note the dirty trick: start by +8 + i_sb.write(mybuffer,FRANZOFFSETV2); ///please note the dirty trick: start by +8 if (flagdebug) myprintf("Mode7: SHA3 <<%s>> CRC32 <<%s>> %s\n",mybuffer+2,mybuffer+67,i_filename.c_str()); } @@ -44679,18 +46165,129 @@ void Jidac::writefranzattr(libzpaq::StringBuffer& i_sb, uint64_t i_data, unsigne if (g_franzotype==FRANZO_MD5) ///1= 52 (01-MD5-0-CRC32) { assert(i_thehash.length()==32); - char mybuffer[FRANZOFFSETSHA256]={0}; + char mybuffer[FRANZOFFSETV2]={0}; snprintf(mybuffer,sizeof(mybuffer), "01"); //<<< look at this snprintf(mybuffer+2,sizeof(mybuffer)-2, "%s",i_thehash.c_str()); snprintf(mybuffer+2+64+1,sizeof(mybuffer)-2-64-1,"%08X",writtencrc); - puti(i_sb, 8+FRANZOFFSETSHA256, 4); // 8+FRANZOFFSET block + puti(i_sb, 8+FRANZOFFSETV2, 4); // 8+FRANZOFFSETV1 block puti(i_sb, i_data, i_quanti); puti(i_sb, 0, (8 - i_quanti)); // pad with zeros (for 7.15 little bug) - i_sb.write(mybuffer,FRANZOFFSETSHA256); ///please note the dirty trick: start by +8 + i_sb.write(mybuffer,FRANZOFFSETV2); ///please note the dirty trick: start by +8 if (flagdebug) myprintf("Mode8: MD5 <<%s>> CRC32 <<%s>> %s\n",mybuffer+2,mybuffer+67,i_filename.c_str()); } else + if ((g_franzotype==FRANZO_WHIRLPOOL) || (g_franzotype==FRANZO_HIGHWAY64) || (g_franzotype==FRANZO_HIGHWAY128) || (g_franzotype==FRANZO_HIGHWAY256)) ///1= 52 (01-MD5-0-CRC32) + { + + /// [0][1] + 0 2bytes long FRANZO_CODE + /// + 2 128 bytes hash + /// +2+128+1 8 bytes CRC32 + /// 8 bytes creationdate (binary) + /// 8 bytes accessdate (binary) + + char mybuffer[FRANZOFFSETV3]={0}; + + if (flagdebug) + { + myprintf("\n"); + myprintf("45487: thehash %s\n",i_thehash.c_str()); + myprintf("45487: writtencrc %08X\n",writtencrc); + } + + string mycode; + if (flagtar) + mycode=std::string()+"4"+char(g_franzotype+'A'); + else + mycode=std::string()+"3"+char(g_franzotype+'A'); + if (flagdebug) + myprintf("45492: MYCODE |%s|\n",mycode.c_str()); + if (mycode.size()!=2) + { + myprintf("45495: GURU mycode must be 2 chars!\n"); + seppuku(); + return; + } + + int offsetcrc =0; + int offset =0; + snprintf(mybuffer+offset,sizeof(mybuffer)-offset,"%s",mycode.c_str()); //two bytes + offset+=mycode.size(); + if (flagdebug) + myprintf("45503: Offset1 (after mycode) %3d\n",offset); + + snprintf(mybuffer+offset,sizeof(mybuffer)-offset,"%s",i_thehash.c_str()); + offset+=i_thehash.size()+2; + if (flagdebug) + myprintf("45507: Offset2 (after hash) %3d\n",offset); + + offsetcrc=offset; + snprintf(mybuffer+offset,sizeof(mybuffer)-offset,"%08X",writtencrc); + offset+=8+2; + + if (flagdebug) + myprintf("45512: Offset3 (after CRC) %3d\n",offset); + + uint32_t orderedstream=0; + if (flagstdin) + orderedstream=1; + offset+=inttoarray(orderedstream ,mybuffer+offset,4); + + if (flagdebug) + myprintf("45512: Offset4 (after ordered)%3d\n",offset); + + // catch "holes" into multipart + uint32_t versionnumber=1440408; + offset+=inttoarray(versionnumber ,mybuffer+offset,4); + if (flagdebug) + myprintf("45524: Offset5 (after version)%3d\n",offset); + + ///i_creationdate=987654321L; + ///i_accessdate=554433229944L; + + offset+=inttoarray(i_creationdate ,mybuffer+offset,8); + + if (flagdebug) + myprintf("45524: Offset6 (after cdate) %3d\n",offset); + + offset+=inttoarray(i_accessdate ,mybuffer+offset,8); + + if (flagdebug) + myprintf("45531: Offset6 (after adate) %3d\n",offset); + + + uint64_t azero=0; + for (int i=0;i<3;i++) + offset+=inttoarray(azero ,mybuffer+offset,8); + + if (flagdebug) + myprintf("45537: Offset7 (after spare) %3d\n",offset); + + int dascrivere=offset; + if (flagtar) + if (i_posix!=NULL) + { + if (flagdebug) + myprintf("45543: flagtar and posix not null, use all FRANZOFFSETV3\n"); + dascrivere=FRANZOFFSETV3; + } + if (flagdebug) + myprintf("45546: To write %08d header %08d + sizeof %08d\n",dascrivere,offset,sizeof(franz_posix)); + if (dascrivere>FRANZOFFSETV3) + { + myprintf("45542: GURU dascrivere %08d > FRANZOFFSETV3 %08d\n",dascrivere,FRANZOFFSETV3); + seppuku(); + return; + } + puti(i_sb, 8+dascrivere, 4); // 8+FRANZOFFSETV1 block + puti(i_sb, i_data, i_quanti); + puti(i_sb, 0, (8 - i_quanti)); // pad with zeros (for 7.15 little bug) + i_sb.write(mybuffer,dascrivere); ///please note the dirty trick: start by +8 + if (flagdebug) + myprintf("45557: Mode |%s| %s <<%s>> CRC32 <<%s>> %s\n",mycode.c_str(),decodefranzoffset(g_franzotype).c_str(),mybuffer+2,mybuffer+offsetcrc,i_filename.c_str()); + + } + else if (g_franzotype==FRANZO_WINHASH64) ///10= 52 (10-XXHASH64-0-CRC32-0-creationwrite) { /// lpCreationTime, lpLastAccessTime, @@ -44704,16 +46301,16 @@ void Jidac::writefranzattr(libzpaq::StringBuffer& i_sb, uint64_t i_data, unsigne ///printf("FRANZO WINNNNNNNNNN ENCODE\n"); assert(i_thehash.length()==32); - char mybuffer[FRANZOFFSETSHA256]={0}; + char mybuffer[FRANZOFFSETV2]={0}; snprintf(mybuffer,sizeof(mybuffer), "10"); //<<< look at this snprintf(mybuffer+2,sizeof(mybuffer)-2, "%s",i_thehash.c_str()); - int64toarray(i_creationdate ,mybuffer+2+31+1); - int64toarray(i_accessdate ,mybuffer+2+31+1+8); + inttoarray(i_creationdate ,mybuffer+2+31+1,8); + inttoarray(i_accessdate ,mybuffer+2+31+1+8,8); snprintf(mybuffer+2+64+1,sizeof(mybuffer)-2-64-1,"%08X",writtencrc); - puti(i_sb, 8+FRANZOFFSETSHA256, 4); // 8+FRANZOFFSET block + puti(i_sb, 8+FRANZOFFSETV2, 4); // 8+FRANZOFFSETV1 block puti(i_sb, i_data, i_quanti); puti(i_sb, 0, (8 - i_quanti)); // pad with zeros (for 7.15 little bug) - i_sb.write(mybuffer,FRANZOFFSETSHA256); ///please note the dirty trick: start by +8 + i_sb.write(mybuffer,FRANZOFFSETV2); ///please note the dirty trick: start by +8 if (flagdebug) myprintf("Mode10: WINHASH64 <<%s>> CRC32 <<%s>> %s\n",mybuffer+2,mybuffer+67,i_filename.c_str()); @@ -44722,14 +46319,14 @@ void Jidac::writefranzattr(libzpaq::StringBuffer& i_sb, uint64_t i_data, unsigne if (g_franzotype==FRANZO_BLAKE3) ///3= 52 (01-BLAKE3-0-CRC32) { assert(i_thehash.length()==64); - char mybuffer[FRANZOFFSETSHA256]={0}; + char mybuffer[FRANZOFFSETV2]={0}; snprintf(mybuffer,sizeof(mybuffer), "03"); //<<< look at this snprintf(mybuffer+2,sizeof(mybuffer)-2, "%s",i_thehash.c_str()); snprintf(mybuffer+2+64+1,sizeof(mybuffer)-2-64-1,"%08X",writtencrc); - puti(i_sb, 8+FRANZOFFSETSHA256, 4); // 8+FRANZOFFSET block + puti(i_sb, 8+FRANZOFFSETV2, 4); // 8+FRANZOFFSETV1 block puti(i_sb, i_data, i_quanti); puti(i_sb, 0, (8 - i_quanti)); // pad with zeros (for 7.15 little bug) - i_sb.write(mybuffer,FRANZOFFSETSHA256); ///please note the dirty trick: start by +8 + i_sb.write(mybuffer,FRANZOFFSETV2); ///please note the dirty trick: start by +8 if (flagdebug) myprintf("Mode6: BLAKE3 <<%s>> CRC32 <<%s>> %s\n",mybuffer+2,mybuffer+67,i_filename.c_str()); } @@ -44737,14 +46334,14 @@ void Jidac::writefranzattr(libzpaq::StringBuffer& i_sb, uint64_t i_data, unsigne if (g_franzotype==FRANZO_XXH3) ///5= 52 (00XXH3-0-CRC32) { assert(i_thehash.length()==32); - char mybuffer[FRANZOFFSETSHA256]={0}; + char mybuffer[FRANZOFFSETV2]={0}; snprintf(mybuffer,sizeof(mybuffer), "09"); //<<< look at this snprintf(mybuffer+2,sizeof(mybuffer)-2, "%s",i_thehash.c_str()); snprintf(mybuffer+2+64+1,sizeof(mybuffer)-2-64-1,"%08X",writtencrc); - puti(i_sb, 8+FRANZOFFSETSHA256, 4); // 8+FRANZOFFSET block + puti(i_sb, 8+FRANZOFFSETV2, 4); // 8+FRANZOFFSETV1 block puti(i_sb, i_data, i_quanti); puti(i_sb, 0, (8 - i_quanti)); // pad with zeros (for 7.15 little bug) - i_sb.write(mybuffer,FRANZOFFSETSHA256); ///please note the dirty trick: start by +8 + i_sb.write(mybuffer,FRANZOFFSETV2); ///please note the dirty trick: start by +8 /// please note the dirty trick: start by +8 if (flagdebug) myprintf("Mode5: XXH3: <<%s>> CRC32 <<%s>> %s\n",mybuffer+2,mybuffer+67,i_filename.c_str()); @@ -44753,178 +46350,7 @@ void Jidac::writefranzattr(libzpaq::StringBuffer& i_sb, uint64_t i_data, unsigne else perror("22144: unknown franzotype"); } -/////////////////////////////// extract /////////////////////////////// -// Return true if the internal file p -// and external file contents are equal or neither exists. -// If filename is 0 then return true if it is possible to compare. -// In the meantime calc the crc32 of the entire file -bool Jidac::equal(DTMap::const_iterator p, const char* filename,uint32_t &o_crc32,string i_myhashtype,string i_myhash,string& o_hash) -{ - o_crc32=0; - o_hash=""; - if (i_myhash=="FAKE") - o_hash="FAKE"; - // test if all fragment sizes and hashes exist - if (filename==0) - { - static const char zero[20]={0}; - for (unsigned i=0; isecond.ptr.size(); ++i) - { - unsigned j=p->second.ptr[i]; - if (j<1 || j>=ht.size() || ht[j].usize<0 || !memcmp(ht[j].sha1, zero, 20)) - return false; - } - return true; - } - // internal or neither file exists - if (p->second.date==0) - return !exists(filename); - // directories always match - if (p->first!="" && isdirectory(p->first)) - return exists(filename); - // compare sizes - FP in=fopen(filename, RB); - if (in==FPNULL) - return false; - if (in==NULL) - return false; - fseeko(in, 0, SEEK_END); - if (ftello(in)!=p->second.size) - return fclose(in), false; - // compare hashes chunk by chunk. - fseeko(in, 0, SEEK_SET); - libzpaq::SHA1 sha1; - const int BUFSIZE=4096; - char buf[BUFSIZE]; - bool flagshow=false; - if (!flagnoeta) - if (flagverbose || (p->second.size>100000000)) //only 100MB+ files - flagshow=true; - if (flagshow) - myprintf("\n"); - int64_t done =0; - int ultimapercentuale =-1; - uint64_t myseed = 0; - XXHash64 myhash(myseed); - SHA3 sha3; - MD5 md5; - blake3_hasher hasher; - blake3_hasher_init(&hasher); - franzSHA256 mysha256; - XXH3_state_t state128; - (void)XXH3_128bits_reset(&state128); - libzpaq::SHA1 mysha1; - int64_t timestart=mtime(); - for (unsigned i=0; isecond.ptr.size(); ++i) - { - unsigned f=p->second.ptr[i]; - if (f<1 || f>=ht.size() || ht[f].usize<0) - return fclose(in), false; - double percentuale=0; - if (flagshow) - { - percentuale=100.0*((double)i/(double)p->second.ptr.size()); - int proper=(int)percentuale; - if (percentuale>0) - if (proper!=ultimapercentuale) - { - if (proper==99) - proper=100; - double tempo=(mtime()-timestart+1)/1000.0; - int myspeed=(int)(done/tempo); - myprintf("SHA1 %03d %% (%12s) @ %12s/s\r",proper,tohuman(done),tohuman2(myspeed)); - ultimapercentuale=(int)percentuale; - } - } - for (int j=0; jBUFSIZE) - n=BUFSIZE; - int r=fread(buf, 1, n, in); - o_crc32=crc32_16bytes(buf,r,o_crc32); - done+=r; - g_worked+=r; - if (flagchecksum) - { - if (i_myhashtype=="XXHASH64") - myhash.add(buf,r); - if (i_myhashtype=="WINHASH64") - myhash.add(buf,r); - if (i_myhashtype=="SHA-3") - sha3.add(buf,r); - if (i_myhashtype=="MD5") - md5.add(buf,r); - if (i_myhashtype=="BLAKE3") - blake3_hasher_update(&hasher,buf,r); - if (i_myhashtype=="SHA-256") - mysha256.update((const uint8_t*)buf,r); - if (i_myhashtype=="XXH3") - (void)XXH3_128bits_update(&state128,buf,r); - if (i_myhashtype=="SHA-1") - mysha1.write(buf,r); - } - if (r!=n) - { - return fclose(in), false; - } - sha1.write(buf, n); - j+=n; - } - if (memcmp(sha1.result(), ht[f].sha1, 20)!=0) - { - return fclose(in), false; - } - } - //if (flagshow) - //myprintf("\n"); - if (fread(buf, 1, BUFSIZE, in)!=0) - return fclose(in), false; - fclose(in); - if (flagchecksum) - { - if ((i_myhashtype=="XXHASH64") ||(i_myhashtype=="WINHASH64")) - { - char temp[33]; - snprintf(temp,sizeof(temp),"%016llX",(unsigned long long)myhash.hash()); - o_hash=temp; - } - else - if (i_myhashtype=="SHA-3") - o_hash= sha3.getHash(); - else - if (i_myhashtype=="MD5") - o_hash=md5.getHash(); - else - if (i_myhashtype=="BLAKE3") - { - uint8_t output[BLAKE3_OUT_LEN]; - blake3_hasher_finalize(&hasher, output, BLAKE3_OUT_LEN); - o_hash=binarytohex((const unsigned char*)output,BLAKE3_OUT_LEN); - } - else - if (i_myhashtype=="SHA-256") - o_hash=mysha256.gethex(); - else - if (i_myhashtype=="XXH3") - { - XXH128_hash_t myhash=XXH3_128bits_digest(&state128); - char buf[33]; - snprintf(buf,sizeof(buf),"%016llX%016llX",(unsigned long long)myhash.high64,(unsigned long long)myhash.low64); - o_hash=buf; - } - else - if (i_myhashtype=="SHA-1") - { - char sha1result[20]; - memcpy(sha1result, mysha1.result(), 20); - o_hash=binarytohex((const unsigned char*)sha1result,20); - } - else - myprintf("\n29573: WARNING unknown hasher algo %s\n",i_myhashtype.c_str()); - } - return true; -} + // An extract job is a set of blocks with at least one file pointing to them. // Blocks are extracted in separate threads, set READY -> WORKING. // A block is extracted to memory up to the last fragment that has a file @@ -45837,6 +47263,9 @@ int Jidac::testverify() string mycrc32=""; int64_t mycreationtime=0; int64_t myaccesstime=0; + bool myisordered=false; + int myversion=0; + franz_posix* myposix=NULL; string hashfromfile=""; if (!isdirectory(p1->first)) @@ -45846,13 +47275,14 @@ int Jidac::testverify() myhash, mycrc32, mycreationtime, - myaccesstime); + myaccesstime, + myisordered, + myversion, + myposix); tobetested+=p->second.size; } -/* - if (myhashtype!="") - myprintf("30133: %s\n",myhashtype.c_str()); - */ + if (flagdebug) + myprintf("30133: |%s| %s\n",myhashtype.c_str(),p->first.c_str()); if (equal(p, p1->first.c_str(),crc32fromfile,myhashtype,myhash,hashfromfile)) //crc32 of second parameters { if (!isdirectory(p1->first)) @@ -45967,9 +47397,9 @@ int Jidac::testverify() myprintf(" to be tested by hash %19s\n",migliaia(tobetested)); if (hashchecked!=tobetested) { - myprintf("****** WARN: something strange hash checking *******\n"); + myprintf("****** WARN: something strange (hash checking) *******\n"); myprintf("****** some file(s) stored without hash? *******\n"); - myprintf("****** hash does not match? *******\n"); + myprintf("****** hash does not match? unknown hash? *******\n"); } } if (mismatches) @@ -45988,8 +47418,12 @@ int Jidac::testverify() myerror=true; } myprintf("Total different file size: %s bytes\n",migliaia(usize)); - if (myerror) + if (myerror) + { + myprintf("47452: use -verbose to get more details\n"); return 2; + } + else return 0; } @@ -46226,6 +47660,9 @@ int Jidac::list() string mycrc32 =""; int64_t mycreationtime =0; int64_t myaccesstime =0; + bool myisordered=false; + int myversion=0; + franz_posix* myposix=NULL; #ifdef _WIN32 if (flagchecksum || flagwindate) #else @@ -46238,7 +47675,10 @@ int Jidac::list() myhash, mycrc32, mycreationtime, - myaccesstime); + myaccesstime, + myisordered, + myversion, + myposix); if (!flagchecksum) { franzotypedetected =0; @@ -46322,305 +47762,6 @@ int Jidac::list() return 0; } -/////////// HASHING-checksumming functions -string sha3_calc_file(const char * i_filename,bool i_flagcalccrc32,uint32_t& o_crc32,const int64_t i_inizio,const int64_t i_totali,int64_t& io_lavorati,int64_t& o_thefilesize) -{ - o_thefilesize=0; - o_crc32=0; - FILE* myfile = freadopen(i_filename); - if(myfile==NULL ) - return ""; - int64_t lunghezza =prendidimensionehandle(myfile); - int64_t letti =0; -#ifdef ANCIENT - const int BUFSIZE =16384; -#else - const int BUFSIZE =65536*8; -#endif - unsigned char unzBuf[BUFSIZE]; - int n=BUFSIZE; - SHA3 sha3; - while (1) - { - int r=fread(unzBuf, 1, n, myfile); - sha3.add(unzBuf,r); - if (i_flagcalccrc32) - o_crc32=crc32_16bytes(unzBuf,r,o_crc32); - io_lavorati+=r; - letti+=r; - if (r!=n) - break; - if ((flagnoeta==false) && (i_inizio>0)) - myavanzamentoby1sec(io_lavorati,i_totali,i_inizio,false); - } - fclose(myfile); - string risultato = sha3.getHash(); - if (lunghezza!=letti) - { - myprintf("\n29884: *** CORRUPTED FILE DETECTED! expected %s readed %s bytes *** ",migliaia(lunghezza),migliaia2(letti)); - printUTF8(i_filename); - myprintf("\n"); - } - o_thefilesize=letti; - return risultato; -} -string md5_calc_file(const char * i_filename,bool i_flagcalccrc32,uint32_t& o_crc32,const int64_t i_inizio,const int64_t i_totali,int64_t& io_lavorati,int64_t& o_thefilesize) -{ - o_thefilesize=0; - o_crc32=0; - FILE* myfile = freadopen(i_filename); - if(myfile==NULL ) - return ""; - int64_t lunghezza =prendidimensionehandle(myfile); - int64_t letti =0; -#ifdef ANCIENT - const int BUFSIZE =16384; -#else - const int BUFSIZE =65536*8; -#endif - unsigned char unzBuf[BUFSIZE]; - int n=BUFSIZE; - MD5 md5; - while (1) - { - int r=fread(unzBuf, 1, n, myfile); - md5.add(unzBuf,r); - if (i_flagcalccrc32) - o_crc32=crc32_16bytes(unzBuf,r,o_crc32); - io_lavorati+=r; - letti+=r; - if (r!=n) - break; - if ((flagnoeta==false) && (i_inizio>0)) - myavanzamentoby1sec(io_lavorati,i_totali,i_inizio,false); - } - fclose(myfile); - string risultato = md5.getHash(); - if (lunghezza!=letti) - { - myprintf("\n29926: *** CORRUPTED FILE DETECTED! expected %s readed %s bytes *** ",migliaia(lunghezza),migliaia2(letti)); - printUTF8(i_filename); - myprintf("\n"); - } - o_thefilesize=letti; - return risultato; -} -string whirlpool_calc_file(const char * i_filename,bool i_flagcalccrc32,uint32_t& o_crc32,const int64_t i_inizio,const int64_t i_totali,int64_t& io_lavorati,int64_t& o_thefilesize) -{ - o_thefilesize=0; - o_crc32=0; - FILE* myfile = freadopen(i_filename); - if(myfile==NULL ) - return ""; - int64_t lunghezza =prendidimensionehandle(myfile); - int64_t letti =0; -#ifdef ANCIENT - const int BUFSIZE =16384; -#else - const int BUFSIZE =65536*8; -#endif - unsigned char unzBuf[BUFSIZE]; - int n=BUFSIZE; - NESSIEstruct hasher; - NESSIEinit(&hasher); - while (1) - { - int r=fread(unzBuf, 1, n, myfile); - NESSIEadd(unzBuf,r*8,&hasher); - if (i_flagcalccrc32) - o_crc32=crc32_16bytes(unzBuf,r,o_crc32); - io_lavorati+=r; - letti+=r; - if (r!=n) - break; - if ((flagnoeta==false) && (i_inizio>0)) - myavanzamentoby1sec(io_lavorati,i_totali,i_inizio,false); - } - fclose(myfile); - unsigned char buffer[512]={0}; - NESSIEfinalize(&hasher,buffer); - if (lunghezza!=letti) - { - myprintf("\n29970: *** CORRUPTED FILE DETECTED! expected %s readed %s bytes *** ",migliaia(lunghezza),migliaia2(letti)); - printUTF8(i_filename); - myprintf("\n"); - } - o_thefilesize=letti; - return binarytohex(buffer,64); -} -string sha1_calc_file(const char * i_filename,bool i_flagcalccrc32,uint32_t& o_crc32,const int64_t i_inizio,const int64_t i_totali,int64_t& io_lavorati,int64_t& o_thefilesize) -{ - o_thefilesize=0; - o_crc32=0; - FILE* myfile = freadopen(i_filename); - if(myfile==NULL ) - return ""; - int64_t lunghezza =prendidimensionehandle(myfile); - int64_t letti =0; -#ifdef ANCIENT - const int BUFSIZE =16384; -#else - const int BUFSIZE =65536*8; -#endif - char unzBuf[BUFSIZE]; - int n=BUFSIZE; - libzpaq::SHA1 sha1; - while (1) - { - int r=fread(unzBuf, 1, n, myfile); - sha1.write(unzBuf,r); - if (i_flagcalccrc32) - o_crc32=crc32_16bytes(unzBuf,r,o_crc32); - io_lavorati+=r; - letti+=r; - if (r!=n) - break; - if ((flagnoeta==false) && (i_inizio>0)) - myavanzamentoby1sec(io_lavorati,i_totali,i_inizio,false); - } - fclose(myfile); - char sha1result[20]; - memcpy(sha1result, sha1.result(), 20); - if (lunghezza!=letti) - { - myprintf("\n30056: *** CORRUPTED FILE DETECTED! expected %s readed %s bytes *** ",migliaia(lunghezza),migliaia2(letti)); - printUTF8(i_filename); - myprintf("\n"); - } - o_thefilesize=letti; - return binarytohex((const unsigned char*)sha1result,20); -} -string fileentropy(const char * i_filename,const int64_t i_inizio,const int64_t i_totali,int64_t& io_lavorati,int64_t& o_thefilesize) -{ - o_thefilesize=0; - FILE* myfile = freadopen(i_filename); - if(myfile==NULL ) - return ""; - int64_t lunghezza =prendidimensionehandle(myfile); - int64_t letti =0; -#ifdef ANCIENT - const int BUFSIZE =16384; -#else - const int BUFSIZE =65536*8; -#endif - unsigned char buf[BUFSIZE]; - int n=BUFSIZE; - size_t f[256] = { 0 }; - while (1) - { - int r=fread(buf, 1, n, myfile); - io_lavorati+=r; - letti+=r; - for(int i=0;i0) - for(size_t i=0;i<256; i++) - if (f[i]) - { - double prob=(double)f[i]/byteletti; - entropy += prob * log2(prob); - } - char buffer[32]; - if (entropy) - snprintf(buffer, sizeof(buffer), "%8.4f", (float)-entropy); - else - snprintf(buffer, sizeof(buffer), "%8.4f", (float)0); ///quick "fix" for -0.0000 - return buffer; -} -string crc32_calc_file(const char * i_filename,const int64_t i_inizio,const int64_t i_totali,int64_t& io_lavorati,int64_t& o_thefilesize) -{ - o_thefilesize=0; - FILE* myfile = freadopen(i_filename); - if(myfile==NULL ) - return ""; - int64_t lunghezza =prendidimensionehandle(myfile); - int64_t letti =0; -#ifdef ANCIENT - char data[16384]; -#else - char data[65536*16]; -#endif - uint32_t crc=0; - int got=0; - while ((got=fread(data,sizeof(char),sizeof(data),myfile)) > 0) - { - crc=crc32_16bytes (data, got, crc); - io_lavorati+=got; - letti+=got; - if ((flagnoeta==false) && (i_inizio>0)) - myavanzamentoby1sec(io_lavorati,i_totali,i_inizio,false); - } - fclose(myfile); - char temp[9]; - snprintf(temp,sizeof(temp),"%08X",crc); - if (lunghezza!=letti) - { - myprintf("\n30090: *** CORRUPTED FILE DETECTED! expected %s readed %s bytes *** ",migliaia(lunghezza),migliaia2(letti)); - printUTF8(i_filename); - myprintf("\n"); - } - o_thefilesize=letti; - return temp; -} -std::string xxhash64_calc_file(const char * i_filename,bool i_flagcalccrc32,uint32_t& o_crc32,const int64_t i_inizio,const int64_t i_totali,int64_t& io_lavorati,int64_t& o_thefilesize) -{ - o_thefilesize=0; - o_crc32=0; - FILE* myfile = freadopen(i_filename); - if(myfile==NULL ) - return ""; - int64_t lunghezza =prendidimensionehandle(myfile); - int64_t letti =0; -#ifdef ANCIENT - const int BUFSIZE =16384; -#else - const int BUFSIZE =65536*8; -#endif - char unzBuf[BUFSIZE]; - int n=BUFSIZE; - - uint64_t myseed = 0; - XXHash64 myhash(myseed); - while (1) - { - int r=fread(unzBuf, 1, n, myfile); - myhash.add(unzBuf,r); - if (i_flagcalccrc32) - o_crc32=crc32_16bytes(unzBuf,r,o_crc32); - io_lavorati+=r; - letti+=r; - if (r!=n) - break; - if ((flagnoeta==false) && (i_inizio>0)) - myavanzamentoby1sec(io_lavorati,i_totali,i_inizio,false); - } - fclose(myfile); - char temp[33]; - snprintf(temp,sizeof(temp),"%016llX",(unsigned long long)myhash.hash()); - if (lunghezza!=letti) - { - myprintf("\n30134: *** CORRUPTED FILE DETECTED! expected %s readed %s bytes *** ",migliaia(lunghezza),migliaia2(letti)); - printUTF8(i_filename); - myprintf("\n"); - } - o_thefilesize=letti; - return temp; -} // WARNING: I am not sure AT ALL that in this mode a streamed-chunked-wyhash is computable // Just a test, with memory mapped file string wyhash_calc_file(const char * i_filename,bool i_flagcalccrc32,uint32_t& o_crc32,const int64_t i_inizio,const int64_t i_totali,int64_t& io_lavorati,int64_t& o_thefilesize) @@ -46719,11 +47860,12 @@ string wyhash_calc_file(const char * i_filename,bool i_flagcalccrc32,uint32_t& o return risultato; #endif } + string mm_hash_calc_file(int i_algo,const char * i_filename,bool i_flagcalccrc32,uint32_t& o_crc32,const int64_t i_inizio,const int64_t i_totali,int64_t& io_lavorati,int64_t& o_thefilesize) { o_thefilesize=0; o_crc32=0; - char buffer[100]; + char buffer[256]; string risultato ="26304:ERROR"; int64_t lunghezza =prendidimensionefile(i_filename); char* data =NULL; @@ -46916,271 +48058,873 @@ string mm_hash_calc_file(int i_algo,const char * i_filename,bool i_flagcalccrc32 return risultato; } -string nilsimsa_calc_file(const char * i_filename,bool i_flagcalccrc32,uint32_t& o_crc32,const int64_t i_inizio,const int64_t i_totali,int64_t& io_lavorati,int64_t& o_thefilesize) + + + + +class franz_do_hash { - return mm_hash_calc_file(ALGO_NILSIMSA,i_filename,i_flagcalccrc32,o_crc32,i_inizio,i_totali,io_lavorati,o_thefilesize); -} -/// take sha256 -string sha256_calc_file(const char * i_filename,bool i_flagcalccrc32,uint32_t& o_crc32,const int64_t i_inizio,const int64_t i_totali,int64_t& io_lavorati,int64_t& o_thefilesize) +/* + do you think this is kind of "weird" C/C++/Delphi? + Well, we need to compile on gcc 3.4 + therefore no C++20, no C++14, no... nothing + + franz_do_hash hashfrombuffer(par->algo[i]); + hashfrombuffer.init(); + hashfrombuffer.update(par->data[i],par->filesize[i]); + hashstringato=hashfrombuffer.finalize(); + + + franz_do_hash dummy(tmpalgo[i]); + string risu=dummy.filehash(work,false,par->inizio,par->dimensione); + +*/ + + private: + bool isinit; + bool isfinalized; + XXHash64 *p_xxhash64; + libzpaq::SHA1 *p_sha1; + libzpaq::SHA256 *p_sha256; + SHA3 *p_sha3; + XXH3_state_t *p_xxh3; + MD5 *p_md5; + blake3_hasher *p_blake3; + NESSIEstruct *p_whihasher; + HighwayHashCat *p_highway64state; + uint32_t *p_crc32; + uint32_t *p_crc32c; + + finalize_function ffinalize; + + public: + string i_hashtype2; + int ihashtype; + bool flagcalccrc32; + + bool i_filehash; // if false, do a RAM hash + + string i_filename; + bool i_flagmm; + int64_t i_inizio; + int64_t i_totali; + + int i_chunksize; + uint32_t* i_buffer8bit; + + string o_status; + string o_hexhash; + int64_t o_thefilesize; + uint32_t o_crc32; + string o_hexcrc32; + uint32_t o_crc32c; + string o_hexcrc32c; + + void init(); + void update(char *i_buffer,const int i_buflen); + string finalize(); + + franz_do_hash(string i_type): + ihashtype(-1), + flagcalccrc32(false), + i_filehash(true), + i_flagmm(false), + i_inizio(0), + i_totali(0), + i_chunksize(0), + i_buffer8bit(NULL), + o_thefilesize(0), + o_crc32(0), + o_crc32c(0) + { + ffinalize =NULL; + if (i_type=="") + { +/// myprintf("48414: i_type empty\n"); +/// seppuku(); + return; + } + + + for (MAPPATIPOHASH::iterator p=g_mappatipohash.begin(); p!=g_mappatipohash.end(); ++p) + if (i_type==(p->second.hashname)) + { + ffinalize=p->second.ffinalize; + ihashtype=p->first; + } + if (ffinalize==NULL) + { + myprintf("47322: GURU cannot find finalize for %s\n",i_type.c_str()); + seppuku(); + return; + } + i_hashtype2 =i_type; + o_status =""; + o_hexhash =""; + o_hexcrc32 =""; + o_hexcrc32c =""; + isinit =false; + isfinalized =false; + } + + franz_do_hash(int i_franzotype): + ihashtype(-1), + flagcalccrc32(false), + i_filehash(true), + i_flagmm(false), + i_inizio(0), + i_totali(0), + i_chunksize(0), + i_buffer8bit(NULL), + o_thefilesize(0), + o_crc32(0), + o_crc32c(0) + { + MAPPATIPOHASH::iterator p=g_mappatipohash.find(i_franzotype); + if (p==g_mappatipohash.end()) + { + myprintf("62683: GURU, cannot convert int %d to ASCII!\n",i_franzotype); + seppuku(); + return; + } + + i_hashtype2 =p->second.hashname; + ihashtype =i_franzotype; + o_status =""; + o_hexhash =""; + o_hexcrc32 =""; + o_hexcrc32c =""; + isinit =false; + isfinalized =false; + } + + string filehash(string i_filename,bool i_flagcalccrc32,int64_t i_inizio,int64_t i_totali); +}; + + +void franz_do_hash::init() { - o_thefilesize=0; - o_crc32=0; - FILE* myfile = freadopen(i_filename); - if(myfile==NULL ) - return ""; - int64_t lunghezza =prendidimensionehandle(myfile); - int64_t letti =0; -#ifdef ANCIENT - const int BUFSIZE =16384; -#else - const int BUFSIZE =65536*8; -#endif - unsigned char buf[BUFSIZE]; - int n=BUFSIZE; - franzSHA256 mysha256; - while (1) + p_crc32 =NULL; + p_crc32c =NULL; + p_xxhash64 =NULL; + p_sha1 =NULL; + p_sha256 =NULL; + p_sha3 =NULL; + p_xxh3 =NULL; + p_md5 =NULL; + p_blake3 =NULL; + p_whihasher =NULL; + p_highway64state=NULL; + if (ihashtype==-1) { - int r=fread(buf, 1, n, myfile); - mysha256.update(buf,r); - if (i_flagcalccrc32) - o_crc32=crc32_16bytes(buf,r,o_crc32); - io_lavorati+=r; - letti+=r; - if (r!=n) - break; - if (flagnoeta==false) - myavanzamentoby1sec(io_lavorati,i_totali,i_inizio,false); + myprintf("47299: GURU hashtype empty\n"); + seppuku(); + return; } - fclose(myfile); - if (lunghezza!=letti) + + if (i_hashtype2=="NONE") + return; + + if ((ihashtype==ALGO_XXHASH64) || (ihashtype==ALGO_WINHASH64)) + p_xxhash64=new XXHash64(0); + else + if (ihashtype==ALGO_SHA1) + p_sha1=new libzpaq::SHA1; + else + if (ihashtype==ALGO_SHA256) + p_sha256=new libzpaq::SHA256; + else + if (ihashtype==ALGO_XXH3) { - myprintf("\n30410: *** CORRUPTED FILE DETECTED! expected %s readed %s bytes *** ",migliaia(lunghezza),migliaia2(letti)); - printUTF8(i_filename); - myprintf("\n"); + p_xxh3=(XXH3_state_t*)aligned_malloc(64, sizeof(XXH3_state_t)); + (void)XXH3_128bits_reset(p_xxh3); } - o_thefilesize=letti; - return mysha256.gethex(); -} -string xxhash_calc_file(const char * i_filename,bool i_flagcalccrc32,uint32_t& o_crc32,const int64_t i_inizio,const int64_t i_totali,int64_t& io_lavorati,int64_t& o_thefilesize) -{ - o_thefilesize=0; - o_crc32=0; - FILE* inFile = freadopen(i_filename); - if (inFile==NULL) - return ""; - int64_t lunghezza =prendidimensionehandle(inFile); - int64_t letti =0; - size_t const blockSize = 65536; - unsigned char buffer[blockSize]; - size_t readSize; - XXH3_state_t state128; - (void)XXH3_128bits_reset(&state128); - while ((readSize = fread(buffer, 1, blockSize, inFile)) > 0) + else + if (ihashtype==ALGO_MD5) + p_md5=new MD5; + else + if (ihashtype==ALGO_SHA3) + p_sha3=new SHA3; + else + if (ihashtype==ALGO_BLAKE3) { - (void)XXH3_128bits_update(&state128, buffer, readSize); - if (i_flagcalccrc32) - o_crc32=crc32_16bytes(buffer,readSize,o_crc32); - io_lavorati+=readSize; - letti+=readSize; - if (flagnoeta==false) - if (i_inizio>=0) - myavanzamentoby1sec(io_lavorati,i_totali,i_inizio,false); + p_blake3=(blake3_hasher*)franz_malloc(sizeof(blake3_hasher)); + blake3_hasher_init(p_blake3); } - fclose(inFile); - XXH128_hash_t myhash=XXH3_128bits_digest(&state128); - char buf[33]; - snprintf(buf,sizeof(buf),"%016llX%016llX",(unsigned long long)myhash.high64,(unsigned long long)myhash.low64); - string risultato=buf; - if (lunghezza!=letti) + else + if (ihashtype==ALGO_WHIRLPOOL) { - myprintf("\n30463: *** CORRUPTED FILE DETECTED! expected %s readed %s bytes *** ",migliaia(lunghezza),migliaia2(letti)); - printUTF8(i_filename); - myprintf("\n"); + p_whihasher=new NESSIEstruct; + NESSIEinit(p_whihasher); } - o_thefilesize=letti; - return risultato; -} -string blake3_calc_file(const char * i_filename,bool i_flagcalccrc32,uint32_t& o_crc32,const int64_t i_inizio,const int64_t i_totali,int64_t& io_lavorati,int64_t& o_thefilesize) -{ - o_crc32=0; - o_thefilesize=0; - FILE* inFile = freadopen(i_filename); - if (inFile==NULL) - return ""; - int64_t lunghezza =prendidimensionehandle(inFile); - int64_t letti =0; - blake3_hasher hasher; - blake3_hasher_init(&hasher); - unsigned char buffer[65536]; - ssize_t readSize; - while ((readSize = fread(buffer, 1, sizeof(buffer), inFile)) > 0) + else + if ((ihashtype==ALGO_HIGHWAY64) || (ihashtype==ALGO_HIGHWAY128) || (ihashtype==ALGO_HIGHWAY256)) { - blake3_hasher_update(&hasher, buffer, readSize); - if (i_flagcalccrc32) - o_crc32=crc32_16bytes(buffer,readSize,o_crc32); - io_lavorati+=readSize; - letti+=readSize; - if (flagnoeta==false) - myavanzamentoby1sec(io_lavorati,i_totali,i_inizio,false); + /// printf("--------------------- chiamo\n"); + p_highway64state=new HighwayHashCat; + uint64_t key[4] = {1, 2, 3, 4}; + HighwayHashCatStart(key,p_highway64state); } - fclose(inFile); - uint8_t output[BLAKE3_OUT_LEN]; - blake3_hasher_finalize(&hasher, output, BLAKE3_OUT_LEN); - if (lunghezza!=letti) + else + if (ihashtype==ALGO_CRC32) { - myprintf("\n30505: *** CORRUPTED FILE DETECTED! expected %s readed %s bytes *** ",migliaia(lunghezza),migliaia2(letti)); - printUTF8(i_filename); - myprintf("\n"); } - o_thefilesize=letti; - return binarytohex((const unsigned char*)output,BLAKE3_OUT_LEN); -} -string crc32c_calc_file(const char * i_filename,const int64_t i_inizio,const int64_t i_totali,int64_t& io_lavorati,int64_t& o_thefilesize) -{ - o_thefilesize=0; - FILE* myfile = freadopen(i_filename); - if( myfile==NULL ) - return 0; - int64_t lunghezza =prendidimensionehandle(myfile); - int64_t letti =0; -#ifdef ANCIENT - const unsigned int BUFSIZE=16384; -#else - const unsigned int BUFSIZE=262144*3; -#endif - char buffer[BUFSIZE]; - ssize_t got; - size_t off, n; - uint32_t crc=0; - while ((got = fread(buffer, sizeof(char),BUFSIZE,myfile)) > 0) - { - off = 0; - do - { - n = (size_t)got - off; - if (n > BUFSIZE) - n = BUFSIZE; - crc = crc32c(crc, (const unsigned char*)buffer + off, n); - off += n; - } while (off < (size_t)got); - io_lavorati+=got; - letti+=got; - if (flagnoeta==false) - if (i_totali) - myavanzamentoby1sec(io_lavorati,i_totali,i_inizio,false); - } - fclose(myfile); - char buf[9]; - snprintf(buf,sizeof(buf),"%08X",crc); - string risultato=buf; - if (lunghezza!=letti) + else + if (ihashtype==ALGO_CRC32C) { - myprintf("\n30556: *** CORRUPTED FILE DETECTED! expected %s readed %s bytes *** ",migliaia(lunghezza),migliaia2(letti)); - printUTF8(i_filename); - myprintf("\n"); } - o_thefilesize=letti; - return risultato; + else + { + myprintf("47282: GURU hashtype unknown |%s| %d\n",i_hashtype2.c_str(),ihashtype); + seppuku(); + return; + } + isinit =true; + isfinalized =false; } -string hash_calc_file(int i_algo,const char * i_filename,bool i_flagcalccrc32,uint32_t& o_crc32,const int64_t i_inizio,const int64_t i_totali,int64_t& io_lavorati,int64_t& o_thefilesize) + +string franz_do_hash::finalize() { - o_thefilesize=0; - o_crc32=0; - if (flagmm) + if (ihashtype==-1) { - /// printbar('='); - ///myprintf("USING MEMORY MAPPED!!!\n"); - ///printbar('='); - return mm_hash_calc_file(i_algo,i_filename,i_flagcalccrc32,o_crc32,i_inizio,i_totali,io_lavorati,o_thefilesize); + myprintf("47302: GURU hashtype empty\n"); + seppuku(); + return "47302: GURU hashtype empty"; } - - if (i_algo==ALGO_BLAKE3) - return blake3_calc_file(i_filename,i_flagcalccrc32,o_crc32,i_inizio,i_totali,io_lavorati,o_thefilesize); + if (!isinit) + { + myprintf("47308: GURU not init!\n"); + seppuku(); + return "47308: GURU not init!"; + } + if (isfinalized) + { + myprintf("47314: already finalized!\n"); + seppuku(); + return "47314: already finalized!"; + } + if ((ihashtype==ALGO_XXHASH64) || (ihashtype==ALGO_WINHASH64)) + o_hexhash=ffinalize(p_xxhash64); else - if (i_algo==ALGO_NILSIMSA) // memory mapped file - return nilsimsa_calc_file(i_filename,i_flagcalccrc32,o_crc32,i_inizio,i_totali,io_lavorati,o_thefilesize); + if (ihashtype==ALGO_SHA1) + o_hexhash=ffinalize(p_sha1); else - if (i_algo==ALGO_WYHASH) // memory mapped file - return wyhash_calc_file(i_filename,i_flagcalccrc32,o_crc32,i_inizio,i_totali,io_lavorati,o_thefilesize); + if (ihashtype==ALGO_SHA256) + o_hexhash=ffinalize(p_sha256); else - if (i_algo==ALGO_WHIRLPOOL) - return whirlpool_calc_file(i_filename,i_flagcalccrc32,o_crc32,i_inizio,i_totali,io_lavorati,o_thefilesize); + if (ihashtype==ALGO_XXH3) + o_hexhash=ffinalize(p_xxh3); else - if (i_algo==ALGO_MD5) - return md5_calc_file(i_filename,i_flagcalccrc32,o_crc32,i_inizio,i_totali,io_lavorati,o_thefilesize); + if (ihashtype==ALGO_MD5) + o_hexhash=ffinalize(p_md5); else - if (i_algo==ALGO_SHA3) - return sha3_calc_file(i_filename,i_flagcalccrc32,o_crc32,i_inizio,i_totali,io_lavorati,o_thefilesize); + if (ihashtype==ALGO_SHA3) + o_hexhash=ffinalize(p_sha3); else - if (i_algo==ALGO_XXHASH64) - return xxhash64_calc_file(i_filename,i_flagcalccrc32,o_crc32,i_inizio,i_totali,io_lavorati,o_thefilesize); + if (ihashtype==ALGO_BLAKE3) + o_hexhash=ffinalize(p_blake3); else - if (i_algo==ALGO_SHA1) - return sha1_calc_file(i_filename,i_flagcalccrc32,o_crc32,i_inizio,i_totali,io_lavorati,o_thefilesize); + if (ihashtype==ALGO_WHIRLPOOL) + o_hexhash=ffinalize(p_whihasher); else - if (i_algo==ALGO_CRC32C) - return crc32c_calc_file(i_filename,i_inizio,i_totali,io_lavorati,o_thefilesize); + if (ihashtype==ALGO_HIGHWAY64) + o_hexhash=ffinalize(p_highway64state); else - if (i_algo==ALGO_CRC32) - return crc32_calc_file(i_filename,i_inizio,i_totali,io_lavorati,o_thefilesize); + if (ihashtype==ALGO_HIGHWAY128) + o_hexhash=ffinalize(p_highway64state); else - if (i_algo==ALGO_ENTROPY) - return fileentropy(i_filename,i_inizio,i_totali,io_lavorati,o_thefilesize); + if (ihashtype==ALGO_HIGHWAY256) + o_hexhash=ffinalize(p_highway64state); else - if (i_algo==ALGO_XXH3) - return xxhash_calc_file(i_filename,i_flagcalccrc32,o_crc32,i_inizio,i_totali,io_lavorati,o_thefilesize); + if (ihashtype==ALGO_CRC32) + { + o_hexhash =ffinalize(p_crc32); + o_hexcrc32 =o_hexhash; + } else - if (i_algo==ALGO_SHA256) - return sha256_calc_file(i_filename,i_flagcalccrc32,o_crc32,i_inizio,i_totali,io_lavorati,o_thefilesize); + if (ihashtype==ALGO_CRC32C) + o_hexhash =ffinalize(p_crc32c); else - error("GURU 13480 algo ??"); - return ""; + { + myprintf("47453: GURU hashtype unknown |%s|\n",i_hashtype2.c_str()); + seppuku(); + return "47453: GURU hashtype unknown |%s|"; + } + isinit =false; + isfinalized =true; + return o_hexhash; + } -string shash_calc_file(string i_algo,const char * i_filename,bool i_flagcalccrc32,uint32_t& o_crc32,const int64_t i_inizio,const int64_t i_totali,int64_t& io_lavorati,int64_t& o_thefilesize) + + + +void franz_do_hash::update(char *i_buffer,const int i_buflen) { - o_crc32=0; - if (i_algo=="BLAKE3") - return blake3_calc_file(i_filename,i_flagcalccrc32,o_crc32,i_inizio,i_totali,io_lavorati,o_thefilesize); - else - if (i_algo=="WYHASH") // memory mapped file - return wyhash_calc_file(i_filename,i_flagcalccrc32,o_crc32,i_inizio,i_totali,io_lavorati,o_thefilesize); +// (*p_xxhash64).add(i_buffer,i_buflen); + ///return; + /* + if (!isinit) + { + myprintf("47298: GURU hash not init\n"); + seppuku(); + return; + } + + if (isfinalized) + { + myprintf("47130: GURU hash finalized, cannot update\n"); + seppuku(); + return; + } +*/ + if ((ihashtype==ALGO_XXHASH64) || (ihashtype==ALGO_WINHASH64)) + { + /* + if (p_xxhash64==NULL) + { + myprintf("47307: guru pointer null %s\n",i_hashtype2.c_str()); + seppuku(); + return; + } + */ + (*p_xxhash64).add(i_buffer,i_buflen); + return; + } else - if (i_algo=="WHIRLPOOL") - return whirlpool_calc_file(i_filename,i_flagcalccrc32,o_crc32,i_inizio,i_totali,io_lavorati,o_thefilesize); + if (ihashtype==ALGO_SHA1) + { + /* + if (p_sha1==NULL) + { + myprintf("47318: guru pointer null %s\n",i_hashtype2.c_str()); + seppuku(); + return; + } + */ + (*p_sha1).write(i_buffer,i_buflen); + return; + } else - if (i_algo=="MD5") - return md5_calc_file(i_filename,i_flagcalccrc32,o_crc32,i_inizio,i_totali,io_lavorati,o_thefilesize); + if (ihashtype==ALGO_SHA256) + { + /* + if (p_sha256==NULL) + { + myprintf("47329: guru pointer null %s\n",i_hashtype2.c_str()); + seppuku(); + return; + } + */ + for (int i=0;i>\n"); + o_status="62703: CANNOT HASH A FOLDER"; + return ""; + } + + char temp[128]; + + + if (ihashtype==ALGO_NILSIMSA) + { + o_hexhash=mm_hash_calc_file(ALGO_NILSIMSA,i_filename.c_str(),i_flagcalccrc32,o_crc32,i_inizio,i_totali,g_dimensione,o_thefilesize); + return o_hexhash; + } + + if (ihashtype==ALGO_WYHASH) + { + o_hexhash=mm_hash_calc_file(ALGO_WYHASH,i_filename.c_str(),i_flagcalccrc32,o_crc32,i_inizio,i_totali,g_dimensione,o_thefilesize); + return o_hexhash; + } + + FILE* myfilez = freadopen(i_filename.c_str()); + if (myfilez==NULL) + { + +#ifdef _WIN32 + int err=GetLastError(); +#else + int err=1; +#endif + myprintf("62697: error kind %d opening <<",err); + printUTF8(i_filename.c_str()); + myprintf(">>\n"); + + o_status="62697: ERROR OPENING FILE"; + return ""; + } + + int64_t lunghezza =prendidimensionehandle(myfilez); + int64_t letti =0; + +#ifdef __HAIKU__ + const int BUFSIZE =16384; +#else +#ifdef ANCIENT + const int BUFSIZE =16384; +#else + const int BUFSIZE =65536*8; +#endif +#endif + + unsigned char unzBuf[BUFSIZE]; + int n=BUFSIZE; + + uint64_t myseed = 0; + XXHash64 myhash(myseed); + libzpaq::SHA1 sha1; + MD5 md5; + franzSHA256 mysha256; + + XXH3_state_t state128; + if (ihashtype==ALGO_XXH3) + (void)XXH3_128bits_reset(&state128); + + blake3_hasher hasher; + if (ihashtype==ALGO_BLAKE3) + blake3_hasher_init(&hasher); + + SHA3 sha3; + + NESSIEstruct whihasher; + if (ihashtype==ALGO_WHIRLPOOL) + NESSIEinit(&whihasher); + + size_t f[256] = { 0 }; + + HighwayHashCat state; + uint64_t hash; + uint64_t key[4] = {1, 2, 3, 4}; + if ((ihashtype==ALGO_HIGHWAY64) || (ihashtype==ALGO_HIGHWAY128) || (ihashtype==ALGO_HIGHWAY256)) + HighwayHashCatStart(key, &state); + + if (ihashtype==ALGO_CRC32) + i_flagcalccrc32=true; + + if (ihashtype==ALGO_CRC32C) + { + ssize_t got; + size_t off, n; + uint32_t crc=0; + while ((got = fread(unzBuf,sizeof(char),BUFSIZE,myfilez)) > 0) + { + off=0; + do + { + n = (size_t)got - off; + if (n > BUFSIZE) + n = BUFSIZE; + crc=crc32c(crc, (const unsigned char*)unzBuf+off,n); + off += n; + } while (off < (size_t)got); + + if (i_flagcalccrc32) + o_crc32=crc32_16bytes(unzBuf,got,o_crc32); + + g_dimensione+=got; + letti+=got; + + if ((flagnoeta==false) && (i_inizio>0) && (i_totali>0)) + myavanzamentoby1sec(g_dimensione,i_totali,i_inizio,false); + } + snprintf(temp,sizeof(temp),"%08X",crc); + o_hexhash=temp; + } else - error("GURU 31223 algo ??"); - return ""; + while (1) + { + int r=fread(unzBuf, 1, n, myfilez); + + if (ihashtype==ALGO_XXHASH64) + myhash.add(unzBuf,r); + else + if (ihashtype==ALGO_SHA1) + sha1.write((const char*)unzBuf,r); + else + if (ihashtype==ALGO_SHA256) + mysha256.update(unzBuf,r); + else + if (ihashtype==ALGO_XXH3) + (void)XXH3_128bits_update(&state128,unzBuf,r); + else + if (ihashtype==ALGO_BLAKE3) + blake3_hasher_update(&hasher,unzBuf,r); + else + if (ihashtype==ALGO_MD5) + md5.add(unzBuf,r); + else + if (ihashtype==ALGO_SHA3) + sha3.add(unzBuf,r); + else + if (ihashtype==ALGO_WHIRLPOOL) + NESSIEadd(unzBuf,r*8,&whihasher); + else + if (ihashtype==ALGO_ENTROPY) + { + for(int i=0;i0) && (i_totali>0)) + myavanzamentoby1sec(g_dimensione,i_totali,i_inizio,false); + } + if (myfilez!=NULL) + fclose(myfilez); + + if (i_flagcalccrc32) + { + snprintf(temp,sizeof(temp),"%08X",o_crc32); + o_hexcrc32 =temp; + } + + if (ihashtype==ALGO_CRC32) + { + snprintf(temp,sizeof(temp),"%08X",o_crc32); + o_hexhash =temp; + } + if (ihashtype==ALGO_XXHASH64) + { + snprintf(temp,sizeof(temp),"%016llX",(unsigned long long)myhash.hash()); + o_hexhash =temp; + } + if (ihashtype==ALGO_SHA1) + { + char sha1result[20]; + memcpy(sha1result, sha1.result(), 20); + o_hexhash=binarytohex((const unsigned char*)sha1result,20); + } + if (ihashtype==ALGO_SHA256) + o_hexhash=mysha256.gethex(); + if (ihashtype==ALGO_XXH3) + { + XXH128_hash_t myhash=XXH3_128bits_digest(&state128); + snprintf(temp,sizeof(temp),"%016llX%016llX",(unsigned long long)myhash.high64,(unsigned long long)myhash.low64); + o_hexhash=temp; + } + if (ihashtype==ALGO_BLAKE3) + { + uint8_t output[BLAKE3_OUT_LEN]; + blake3_hasher_finalize(&hasher, output, BLAKE3_OUT_LEN); + o_hexhash=binarytohex((const unsigned char*)output,BLAKE3_OUT_LEN); + } + if (ihashtype==ALGO_MD5) + o_hexhash=md5.getHash(); + if (ihashtype==ALGO_SHA3) + o_hexhash=sha3.getHash(); + if (ihashtype==ALGO_WHIRLPOOL) + { + unsigned char output[512]={0}; + NESSIEfinalize(&whihasher,output); + o_hexhash=binarytohex(output,64); + } + if (ihashtype==ALGO_ENTROPY) + { + double entropy=0.0; + double byteletti=letti; + if (letti>0) + for(size_t i=0;i<256; i++) + if (f[i]) + { + double prob=(double)f[i]/byteletti; + entropy += prob * log2(prob); + } + if (entropy) + snprintf(temp,sizeof(temp), "%8.4f", (float)-entropy); + else + snprintf(temp,sizeof(temp), "%8.4f", (float)0); ///quick "fix" for -0.0000 + o_hexhash=temp; + } + + if (ihashtype==ALGO_HIGHWAY64) + { + hash=HighwayHashCatFinish64(&state); + snprintf(temp,sizeof(temp),"%016llX",(unsigned long long)hash); + o_hexhash=temp; + } + if (ihashtype==ALGO_HIGHWAY128) + { + uint64_t hash[2]; + HighwayHashCatFinish128(&state,hash); + o_hexhash=binarytohex((const unsigned char*)hash,16); + } + if (ihashtype==ALGO_HIGHWAY256) + { + uint64_t hash[4]; + HighwayHashCatFinish256(&state,hash); + o_hexhash=binarytohex((const unsigned char*)hash,32); + } + + if (lunghezza!=letti) + { + myprintf("\n62748: *** CORRUPTED FILE DETECTED! expected %s readed %s bytes *** ",migliaia(lunghezza),migliaia2(letti)); + printUTF8(i_filename.c_str()); + myprintf("\n"); + } + o_thefilesize=letti; + return o_hexhash; } + + + + +/////////////////////////////// extract /////////////////////////////// +// Return true if the internal file p +// and external file contents are equal or neither exists. +// If filename is 0 then return true if it is possible to compare. +// In the meantime calc the crc32 of the entire file +bool Jidac::equal(DTMap::const_iterator p, const char* filename,uint32_t &o_crc32,string i_myhashtype,string i_myhash,string& o_hash) +{ + o_crc32=0; + o_hash=""; + if (i_myhash=="FAKE") + o_hash="FAKE"; + // test if all fragment sizes and hashes exist + if (filename==0) + { + static const char zero[20]={0}; + for (unsigned i=0; isecond.ptr.size(); ++i) + { + unsigned j=p->second.ptr[i]; + if (j<1 || j>=ht.size() || ht[j].usize<0 || !memcmp(ht[j].sha1, zero, 20)) + return false; + } + return true; + } + // internal or neither file exists + if (p->second.date==0) + return !exists(filename); + // directories always match + if (p->first!="" && isdirectory(p->first)) + return exists(filename); + // compare sizes + FP in=fopen(filename, RB); + if (in==FPNULL) + return false; + if (in==NULL) + return false; + fseeko(in, 0, SEEK_END); + if (ftello(in)!=p->second.size) + return fclose(in), false; + // compare hashes chunk by chunk. + fseeko(in, 0, SEEK_SET); + libzpaq::SHA1 sha1; + const int BUFSIZE=4096; + char buf[BUFSIZE]; + bool flagshow=false; + if (!flagnoeta) + if (flagverbose || (p->second.size>100000000)) //only 100MB+ files + flagshow=true; + if (flagshow) + myprintf("\n"); + int64_t done =0; + int ultimapercentuale =-1; + + /* + if (flagdebug) + myprintf("49125: franz_do_hash\n"); + */ + if (i_myhashtype=="") // a seppuku if myhashtype is "" + i_myhashtype="NONE"; + + franz_do_hash hashfrombuffer(i_myhashtype); + hashfrombuffer.init(); + + int64_t timestart=mtime(); + for (unsigned i=0; isecond.ptr.size(); ++i) + { + unsigned f=p->second.ptr[i]; + if (f<1 || f>=ht.size() || ht[f].usize<0) + return fclose(in), false; + double percentuale=0; + if (flagshow) + { + percentuale=100.0*((double)i/(double)p->second.ptr.size()); + int proper=(int)percentuale; + if (percentuale>0) + if (proper!=ultimapercentuale) + { + if (proper==99) + proper=100; + double tempo=(mtime()-timestart+1)/1000.0; + int myspeed=(int)(done/tempo); + myprintf("SHA1 %03d %% (%12s) @ %12s/s\r",proper,tohuman(done),tohuman2(myspeed)); + ultimapercentuale=(int)percentuale; + } + } + for (int j=0; jBUFSIZE) + n=BUFSIZE; + int r=fread(buf, 1, n, in); + o_crc32=crc32_16bytes(buf,r,o_crc32); + done+=r; + g_worked+=r; + if (flagchecksum) + hashfrombuffer.update(buf,r); + if (r!=n) + { + return fclose(in), false; + } + sha1.write(buf, n); + j+=n; + } + if (memcmp(sha1.result(), ht[f].sha1, 20)!=0) + { + return fclose(in), false; + } + } + if (fread(buf, 1, BUFSIZE, in)!=0) + return fclose(in), false; + fclose(in); + + if (flagchecksum) + o_hash=hashfrombuffer.finalize(); + return true; +} + bool iszpaqfranzvirtualfile(const string& i_filename) { if (mypos("VCOMMENT ",i_filename)>-1) @@ -47404,77 +49148,16 @@ void * scriviramtodisk(void *t) if (par->algo[i]=="") par->o_filesnotchecked++; else - if ((par->algo[i]=="XXHASH64") || (par->algo[i]=="WINHASH64")) { - uint64_t myseed = 0; - XXHash64 myhash64(myseed); - myhash64.add(par->data[i],par->filesize[i]); - snprintf(buffer,sizeof(buffer),"%016llX",(unsigned long long)myhash64.hash()); - hashstringato=buffer; - par->o_hashedsize+=par->filesize[i]; - } - else - if (par->algo[i]=="SHA-1") - { - libzpaq::SHA1 sha1; - sha1.write(par->data[i],par->filesize[i]); - char sha1result[20]; - memcpy(sha1result, sha1.result(), 20); - hashstringato=binarytohex((const unsigned char*)sha1result,20); - par->o_hashedsize+=par->filesize[i]; - } - else - if (par->algo[i]=="SHA-256") - { - libzpaq::SHA256 sha256; - for (int64_t k=0;ifilesize[i];k++) - sha256.put(*(par->data[i]+k)); - char sha256result[32]; - memcpy(sha256result, sha256.result(), 32); - hashstringato=binarytohex((const unsigned char*)sha256result,32); - par->o_hashedsize+=par->filesize[i]; - } - else - if (par->algo[i]=="BLAKE3") - { - blake3_hasher hasher; - blake3_hasher_init(&hasher); - blake3_hasher_update(&hasher,par->data[i],par->filesize[i]); - uint8_t output[BLAKE3_OUT_LEN]; - blake3_hasher_finalize(&hasher, output, BLAKE3_OUT_LEN); - hashstringato=binarytohex((const unsigned char*)output,BLAKE3_OUT_LEN); - par->o_hashedsize+=par->filesize[i]; - } - else - if (par->algo[i]=="SHA-3") - { - SHA3 sha3; - sha3.add(par->data[i],par->filesize[i]); - hashstringato=sha3.getHash(); - par->o_hashedsize+=par->filesize[i]; - } - else - if (par->algo[i]=="MD5") - { - MD5 md5; - md5.add(par->data[i],par->filesize[i]); - hashstringato=md5.getHash(); - par->o_hashedsize+=par->filesize[i]; - } - else - if (par->algo[i]=="XXH3") - { - XXH3_state_t state128; - (void)XXH3_128bits_reset(&state128); - (void)XXH3_128bits_update(&state128,par->data[i],par->filesize[i]); - XXH128_hash_t myhash=XXH3_128bits_digest(&state128); - char buffer[33]; - snprintf(buffer,sizeof(buffer),"%016llX%016llX",(unsigned long long)myhash.high64,(unsigned long long)myhash.low64); - hashstringato=buffer; + ///if (flagdebug) + ///myprintf("49404: franz_do_hash\n"); + + franz_do_hash hashfrombuffer(par->algo[i]); + hashfrombuffer.init(); + hashfrombuffer.update(par->data[i],par->filesize[i]); + hashstringato=hashfrombuffer.finalize(); par->o_hashedsize+=par->filesize[i]; } - else - error("33964: unknown myhashtype"); if (flagdebug) myprintf("33966: INFO hash stored %s %s from ram|%s| %s\n",par->algo[i].c_str(),par->filecrc[i].c_str(),hashstringato.c_str(),par->filenameondisk[i].c_str()); @@ -47496,6 +49179,7 @@ void * scriviramtodisk(void *t) if (!par->flagtest) { int64_t startfilesystem=mtime(); + ///myprintf("49443: makepath %s\n",par->filenameondisk[i]); makepath(par->filenameondisk[i], par->filedate[i],par->fileattr[i]); FP myfile=fopen(par->filenameondisk[i].c_str(), WB); par->o_timefilesystem+=(mtime()-startfilesystem); @@ -47971,6 +49655,7 @@ int Jidac::extract() if (isdir) real_dirs++; string dummy=""; + ///myprintf("49911\n"); if ((!flagstdout) && (repack=="") && !flagtest && flagforce && !isdir && equal(p, fn.c_str(),crc32fromfile,"","",dummy)) { // identical @@ -48314,6 +49999,8 @@ int Jidac::extract() myprintf("Done %8sK / %s\r",migliaia(fatti/1000),migliaia2(totaledt)); string fn=rename(p->first); fn=makelongpath(fn); + ///myprintf("50263: makepath %s\n",fn.c_str()); + makepath(fn); if (!isdirectory(fn)) { @@ -48415,6 +50102,8 @@ int Jidac::extract() if (!flagtest) { ///myprintf(); + ///myprintf("50366: makepath %s\n",outname.c_str()); + makepath(outname); ///myprintf(); outf=fopen(outname.c_str(), WB); @@ -48477,17 +50166,46 @@ int Jidac::extract() join(tid[i]); ///myprintf("Z7\n"); // Create empty directories and set file dates and attributes - if (!flagtest) { - for (DTMap::reverse_iterator p=dt.rbegin(); p!=dt.rend(); ++p) { - if (p->second.data>=0 && p->second.date && p->first!="") { - string s=rename(p->first); - if (p->first[p->first.size()-1]=='/') - makepath(s, p->second.date, p->second.attr); - else if ((p->second.attr&0x1ff)=='w'+256) // read-only? - close(s.c_str(), 0, p->second.attr); - } - } - } + if (!flagtest) + for (DTMap::reverse_iterator p=dt.rbegin(); p!=dt.rend(); ++p) + if (p->second.data>=0 && p->second.date && p->first!="") + { + string s=rename(p->first); + if (p->first[p->first.size()-1]=='/') + { + bool createpath=true; + if (flagdebug) + myprintf("50436: makepath %s %d %d\n",s.c_str(),g_rangefrom,g_rangeto); + if (all) + if (g_rangefrom>0) + { + /// do not make empty folder extracting by range + int posizionebarra=0; + for (int i=s.size()-2; i>0; i--) + if (s[i]=='/') + { + posizionebarra=i; + break; + } + if (posizionebarra>0) + { + string lastpath=""; + for (unsigned int i=posizionebarra+1;i=g_rangefrom) && (vernumber<=g_rangeto)); + } + } + if (createpath) + makepath(s, p->second.date, p->second.attr); + } + else + if ((p->second.attr&0x1ff)=='w'+256) // read-only? + close(s.c_str(), 0, p->second.attr); + } + + if (flagramdisk) { /// extract @@ -48649,6 +50367,8 @@ int Jidac::extract() int uguali=0; int diversi=0; int64_t lavorati=0; + g_dimensione=0; + for (DTMap::iterator p=dt.begin(); p!=dt.end(); ++p) if (p->second.date && p->first!="") { @@ -48664,17 +50384,11 @@ int Jidac::extract() if (p->second.hashtype!="") if (fileexists(fn)) // no fakefile-dirs { - uint32_t dummycrc; - int64_t thefilesize=0; - string hashfromfile=shash_calc_file( - p->second.hashtype, - fn.c_str(), - false, /// do not want CRC-32 - dummycrc, - inizio, /// start time - dimensionetotale, /// total bytes - lavorati,thefilesize); /// done so far - ///hashfromfile="dummy"; + ///if (flagdebug) + ///myprintf("50609: franz_do_hash\n"); + + franz_do_hash dummy(p->second.hashtype); + string hashfromfile=dummy.filehash(fn.c_str(),false,inizio,dimensionetotale); if (hashfromfile==p->second.hexhash) { uguali++; @@ -48748,12 +50462,18 @@ int Jidac::extract() string mycrc32 =""; int64_t mycreationtime =0; int64_t myaccesstime =0; + bool myisordered=false; + int myversion=0; + franz_posix* myposix=NULL; decode_franz_block(false,p->second.franz_block, myhashtype, myhash, mycrc32, mycreationtime, - myaccesstime); + myaccesstime, + myisordered, + myversion, + myposix); if (mycreationtime>0) if (wintouch(fn,0,mycreationtime)) @@ -48922,14 +50642,6 @@ int Jidac::enumeratecomments() } return 0; } -// no to_string on old gcc -string myto_string(int64_t i_number) -{ - char buffer[100]; - snprintf(buffer,sizeof(buffer),"%lld",(long long)i_number); - string risultato=buffer; // explicit converting - return risultato; -} int Jidac::kill() { getpasswordifempty(); @@ -49072,18 +50784,19 @@ void * scansionahash(void *t) vector& tmphashcalculated = par->o_hashcalculated; vector& tmpsizegetted = par->o_sizegetted; + g_dimensione=0; for (unsigned int i=0;iinizio,par->dimensione,g_dimensione,thefilesize); + if (flagdebug) + myprintf("51014: franz_do_hash\n"); + + franz_do_hash dummy(tmpalgo[i]); + string risu=dummy.filehash(work,false,par->inizio,par->dimensione); + ///,g_dimensione,thefilesize); tmphashcalculated.push_back(risu); - tmpsizegetted.push_back(thefilesize); + tmpsizegetted.push_back(dummy.o_thefilesize); /// tmphashcalculated.push_back(hash_calc_file(string2algo(tmpalgo[i]),tmpfilestobehashed[i].c_str(),false,dummycrc,par->inizio,par->dimensione,g_dimensione)); } pthread_exit(NULL); @@ -49091,7 +50804,12 @@ void * scansionahash(void *t) } bool ischecksum() { - return (flagcrc32 || flagcrc32c || flagxxh3 || flagxxhash64 || flagsha1 || flagsha256 || flagblake3 || flagwyhash || flagwhirlpool || flagmd5 || flagnilsimsa || flagsha3 || flagentropy); + for (MAPPATIPOHASH::iterator p=g_mappatipohash.begin(); p!=g_mappatipohash.end(); ++p) + if (p->second.switchflag!=NULL) + if (p->second.switchflag!= (&flagnochecksum)) + if (*(p->second.switchflag)) + return true; + return false; } int Jidac::deduplicate() { @@ -49158,6 +50876,11 @@ string do_benchmark(int i_tnumber,int i_timelimit,string i_runningalgo,int i_chu uint32_t crcc = 0; uint64_t myseed = 0; XXHash64 myhash(myseed); + + HighwayHashCat state; + uint64_t key[4] = {1, 2, 3, 4}; + HighwayHashCatStart(key, &state); + int i=0; while (1) { @@ -49199,6 +50922,17 @@ string do_benchmark(int i_tnumber,int i_timelimit,string i_runningalgo,int i_chu else if (i_runningalgo=="XXHASH64") myhash.add((const unsigned char*)buffer32bit,buffersize); + else + if ((i_runningalgo=="HIGHWAY64") || (i_runningalgo=="HIGHWAY128") || (i_runningalgo=="HIGHWAY256")) + HighwayHashCatAppend((const uint8_t*)buffer32bit,buffersize,&state); +/* + else + { + myprintf("49891: GURU unknown algo %s\n",i_runningalgo.c_str()); + seppuku(); + return 0; + } +*/ totalhashtime=mtime()-starttutto-randtime; if (totalhashtime==0) totalhashtime=1; @@ -49468,11 +51202,16 @@ string filecopy(bool i_singlefile,bool i_append,const string i_infile,const stri XXH128_hash_t myhash=XXH3_128bits_digest(&state128); char risultato[33]; snprintf(risultato,sizeof(risultato),"%016llX%016llX",(unsigned long long)myhash.high64,(unsigned long long)myhash.low64); - uint32_t dummycrc32; int64_t startverify=mtime(); - int64_t io_lavorati=0; - int64_t thefilesize=0; - string hashreloaded=xxhash_calc_file(filedefinitivo.c_str(),false,dummycrc32,startverify,larghezzain,io_lavorati,thefilesize); + + g_dimensione=0; + ///if (flagdebug) + ///myprintf("51429: franz_do_hash\n"); + + franz_do_hash dummy("XXH3"); + string hashreloaded=dummy.filehash(filedefinitivo,false,startverify,larghezzain); + + if (flagverbose || flagdebug) { myprintf("Expected XXH3 hash of the output file %s\n",risultato); @@ -49998,7 +51737,29 @@ int Jidac::mycopy() - Solaris 11 - OmniOS - MacOS 11 + - Haiku R1/beta4 */ +#ifdef __HAIKU__ +int64_t internal_getramdisksize() +{ + if (flagdebug) + myprintf("Haiku pagesize %s!\n",migliaia(B_PAGE_SIZE)); + system_info haikuinfo; + get_system_info(&haikuinfo); + /* + uint64_t totali=haikuinfo.max_pages*B_PAGE_SIZE; + uint64_t usate=haikuinfo.used_pages*B_PAGE_SIZE; + + myprintf("Totali %s\n",migliaia(totali)); + myprintf("Usate %s\n",migliaia(usate)); + + myprintf("cpu %d\n",haikuinfo.cpu_count); + myprintf("free %s\n",migliaia(haikuinfo.free_memory)); + */ + return haikuinfo.free_memory; +} +#endif + #ifdef _WIN32 int64_t internal_getramdisksize() { @@ -50279,23 +52040,32 @@ int Jidac::autotest() chunksize=menoenne; flagchanged=true; } - uint8_t *buffer8bit=(uint8_t*)malloc(chunksize*sizeof(uint8_t)); + uint8_t *buffer8bit=(uint8_t*)franz_malloc(chunksize*sizeof(uint8_t)); if (buffer8bit==0) { myprintf("32374: GURU cannot alloc the buffer8bit\n"); return 2; } // why this strange thing? Because we want to debug on BIG ENDIAN, of course - uint32_t *buffer32bit = (uint32_t*)malloc(chunksize*sizeof(uint32_t)); + uint32_t *buffer32bit = (uint32_t*)franz_malloc(chunksize*sizeof(uint32_t)); if (buffer32bit==0) { myprintf("32394: GURU cannot alloc the buffer32bit\n"); - free(buffer8bit); + franz_free(buffer8bit); return 2; } int64_t startcalc=mtime(); MAPPAAUTOCHECK myautocheck_map; hash_autocheck myblock; + myblock.ok="93EA42CC35E379AD67927C4F3B7AB78EA038A7FD67D7B245975752FDF319B0A8C40E4F4AD30D9069CBB96777AE2CA6ED23A5FEB1DCF30487401317887E97584A"; + myautocheck_map.insert(std::pair("WHIRLPOOL",myblock)); + myblock.ok="B99C2AD7BDE24144"; + myautocheck_map.insert(std::pair("HIGHWAY64",myblock)); + myblock.ok="794503BA1B12CF8848907AAF0F62A8C5"; + myautocheck_map.insert(std::pair("HIGHWAY128",myblock)); + myblock.ok="6F9B44DF54E86972F2873FEFEBB06BD7E29CF9C7DB71096FF0270D72726088EF"; + myautocheck_map.insert(std::pair("HIGHWAY256",myblock)); + myblock.ok="5AF0D4BB454A7D349CE97EFAFFD93CAC3B200DF166BEB9BFE2EC1838F35396AD"; myautocheck_map.insert(std::pair("BLAKE3",myblock)); myblock.ok="0D26839A"; @@ -50341,6 +52111,9 @@ int Jidac::autotest() XXH64_state_t* state = XXH64_createState(); assert(state != NULL && "Out of memory!"); XXH64_reset(state,seme); + HighwayHashCat highway64state; + uint64_t key[4] = {1, 2, 3, 4}; + HighwayHashCatStart(key,&highway64state); char buffer[256]; int64_t total_hashed=0; string outfolder=""; @@ -50361,15 +52134,15 @@ int Jidac::autotest() if (mypos(" ",outfolder)>=0) { myprintf("33131: -to folder cannot contain spaces\n"); - free(buffer8bit); - free(buffer32bit); + franz_free(buffer8bit); + franz_free(buffer32bit); return 2; } if (!saggiascrivibilitacartella(outfolder)) { myprintf("33607: cannot write test data into -to\n"); - free(buffer8bit); - free(buffer32bit); + franz_free(buffer8bit); + franz_free(buffer32bit); return 2; } uint64_t tofoldersize =0; @@ -50382,8 +52155,8 @@ int Jidac::autotest() myprintf("33617: WARNING the folder <<"); printUTF8(outfolder.c_str()); myprintf(">> Must be empty. Quit\n\n\n"); - free(buffer8bit); - free(buffer32bit); + franz_free(buffer8bit); + franz_free(buffer32bit); return 1; } ///warning: string literal of length 198756 exceeds maximum length 65536 that C++ compilers are required to suppor @@ -50402,19 +52175,20 @@ int Jidac::autotest() const string extract_test=extract1+extract2+extract3+extract4; size_t the_file_len=mimesize(extract_test.c_str()); char* the_file; - the_file=(char*)malloc(the_file_len); + the_file=(char*)franz_malloc(the_file_len); if (the_file==NULL) { myprintf("33652: error in malloc\n"); - free(buffer8bit); - free(buffer32bit); + franz_free(buffer8bit); + franz_free(buffer32bit); return 2; } if (!mime2binary(extract_test.c_str(),(unsigned char *)the_file,the_file_len)) { myprintf("33657: mime decoder kaputt!\n"); - free(buffer8bit); - free(buffer32bit); + franz_free(the_file); + franz_free(buffer8bit); + franz_free(buffer32bit); return 2; } outfile=outfolder+"sha256.zpaq"; @@ -50430,8 +52204,9 @@ int Jidac::autotest() if (outFile==NULL) { myprintf("33639 :CANNOT OPEN outfile %s\n",outfile.c_str()); - free(buffer8bit); - free(buffer32bit); + franz_free(the_file); + franz_free(buffer8bit); + franz_free(buffer32bit); return 2; } unsigned int scritti=fwrite(the_file,1,the_file_len,outFile); @@ -50439,8 +52214,9 @@ int Jidac::autotest() if (scritti!=the_file_len) { myprintf("33666: error extracting file .zpaq\n"); - free(buffer8bit); - free(buffer32bit); + franz_free(the_file); + franz_free(buffer8bit); + franz_free(buffer32bit); return 2; } @@ -50515,7 +52291,7 @@ int Jidac::autotest() chunk++; snprintf(mynomefile,sizeof(mynomefile),"%s%02d/%04d/%05d.dat",outfolder.c_str(),i,k,chunk); makepath(mynomefile); - if (chunk %1000==0) + if (chunk %100==0) myprintf("Test file %s\r",mynomefile); FILE* myfile=fopen(mynomefile, "wb"); if (myfile==NULL) @@ -50526,6 +52302,7 @@ int Jidac::autotest() fwrite(buffer8bit, 1, chunksize, myfile); fclose(myfile); } + XXH3_128bits_update(&state128, buffer8bit, chunksize); for (int64_t l=0;l(unsigned int)chunksize) { myprintf("34124: -n too big, max %s\n",migliaia(sizeof(chunksize))); - free(buffer8bit); - free(buffer32bit); + franz_free(buffer8bit); + franz_free(buffer32bit); return 2; } flagverbose=true; @@ -50579,10 +52364,37 @@ int Jidac::autotest() for (int64_t l=0;lsecond.calculated=buffer; + + + a=myautocheck_map.find("HIGHWAY128"); + if (a==myautocheck_map.end()) + error("guru 52286 HIGHWAY128"); + uint64_t hash_128[2]; + HighwayHashCatFinish128(&highway64state,hash_128); + a->second.calculated=binarytohex((const unsigned char*)hash_128,16); + + + a=myautocheck_map.find("HIGHWAY256"); + if (a==myautocheck_map.end()) + error("guru 52286 HIGHWAY256"); + uint64_t hash_256[4]; + HighwayHashCatFinish256(&highway64state,hash_256); + a->second.calculated=binarytohex((const unsigned char*)hash_256,32); + + + a=myautocheck_map.find("SHA-3"); if (a==myautocheck_map.end()) error("guru 32469 SHA-3"); @@ -50683,8 +52495,8 @@ int Jidac::autotest() if (remove(filebatch.c_str())!=0) { myprintf("33764: Highlander batch %s\n", filebatch.c_str()); - free(buffer8bit); - free(buffer32bit); + franz_free(buffer8bit); + franz_free(buffer32bit); return 1; } string filezpaq=outfolder+"testme.zpaq"; @@ -50712,9 +52524,9 @@ int Jidac::autotest() fprintf(batch,"\"%s\" a \"%s\" \"%s04\" -sha3 -comment \"sha3\" \r\n",myzpaqexe.c_str(),filezpaq.c_str(),outfolder.c_str()); fprintf(batch,"\"%s\" a \"%s\" \"%s05\" -blake3 -comment \"blake3\" \r\n",myzpaqexe.c_str(),filezpaq.c_str(),outfolder.c_str()); fprintf(batch,"\"%s\" a \"%s\" \"%s06\" -md5 -comment \"md5\" \r\n",myzpaqexe.c_str(),filezpaq.c_str(),outfolder.c_str()); - fprintf(batch,"\"%s\" a \"%s\" \"%s07\" -xxhash -comment \"xxhash_2\" \r\n",myzpaqexe.c_str(),filezpaq.c_str(),outfolder.c_str()); - fprintf(batch,"\"%s\" a \"%s\" \"%s08\" -xxh3 -comment \"xxh3_2\" \r\n",myzpaqexe.c_str(),filezpaq.c_str(),outfolder.c_str()); - fprintf(batch,"\"%s\" a \"%s\" \"%s09\" -blake3 -comment \"blake3_2\" \r\n",myzpaqexe.c_str(),filezpaq.c_str(),outfolder.c_str()); + fprintf(batch,"\"%s\" a \"%s\" \"%s07\" -whirlpool -comment \"whirlpool\" \r\n",myzpaqexe.c_str(),filezpaq.c_str(),outfolder.c_str()); + fprintf(batch,"\"%s\" a \"%s\" \"%s08\" -highway64 -comment \"high64\" \r\n",myzpaqexe.c_str(),filezpaq.c_str(),outfolder.c_str()); + fprintf(batch,"\"%s\" a \"%s\" \"%s09\" -highway128 -comment \"high128\" \r\n",myzpaqexe.c_str(),filezpaq.c_str(),outfolder.c_str()); fprintf(batch,"\"%s\" x \"%s\" \"%s00\\0770\\00771.dat\" -to \"%sverifica\"\r\n",myzpaqexe.c_str(),filezpaq.c_str(),outfolder.c_str(),outfolder.c_str()); fprintf(batch,"\"%s\" sum \"%sverifica\" -sha256\r\n",myzpaqexe.c_str(),outfolder.c_str()); if (!flagchanged) @@ -50754,7 +52566,7 @@ int Jidac::autotest() fprintf(batch,"echo \"(1) Sorry, no full test on Debian allowed. Download and compile source from github\"\n"); else { - fprintf(batch,"%s x \"%ssha256.zpaq\" -to \"%ste/\"\n",myzpaqexe.c_str(),outfolder.c_str(),outfolder.c_str()); + fprintf(batch,"%s x \"%ssha256.zpaq\" -to \"%ste/\" -space\n",myzpaqexe.c_str(),outfolder.c_str(),outfolder.c_str()); fprintf(batch,"%s sum \"%ste\" -sha256 -rename -force\n",myzpaqexe.c_str(),outfolder.c_str()); fprintf(batch,"echo \"(1) You should read NOT renamed files 256\"\n"); fprintf(batch,"%s pause -pakka\n",myzpaqexe.c_str()); @@ -50766,9 +52578,9 @@ int Jidac::autotest() fprintf(batch,"%s a \"%s\" \"%s04\" -sha3 -comment \"sha3\"r\n",myzpaqexe.c_str(),filezpaq.c_str(),outfolder.c_str()); fprintf(batch,"%s a \"%s\" \"%s05\" -blake3 -comment \"blake3\"\n",myzpaqexe.c_str(),filezpaq.c_str(),outfolder.c_str()); fprintf(batch,"%s a \"%s\" \"%s06\" -md5 -comment \"md5\"\n",myzpaqexe.c_str(),filezpaq.c_str(),outfolder.c_str()); - fprintf(batch,"%s a \"%s\" \"%s07\" -xxhash -comment \"xxhash_2\"\n",myzpaqexe.c_str(),filezpaq.c_str(),outfolder.c_str()); - fprintf(batch,"%s a \"%s\" \"%s08\" -xxh3 -comment \"xxh3_2\"\n",myzpaqexe.c_str(),filezpaq.c_str(),outfolder.c_str()); - fprintf(batch,"%s a \"%s\" \"%s09\" -blake3 -comment \"blake3_2\"\n",myzpaqexe.c_str(),filezpaq.c_str(),outfolder.c_str()); + fprintf(batch,"%s a \"%s\" \"%s07\" -whirlpool -comment \"whirlpool\"\n",myzpaqexe.c_str(),filezpaq.c_str(),outfolder.c_str()); + fprintf(batch,"%s a \"%s\" \"%s08\" -highway64 -comment \"high64\"\n",myzpaqexe.c_str(),filezpaq.c_str(),outfolder.c_str()); + fprintf(batch,"%s a \"%s\" \"%s09\" -highway128 -comment \"high128\"\n",myzpaqexe.c_str(),filezpaq.c_str(),outfolder.c_str()); fprintf(batch,"%s x \"%s\" \"%s00/0770/00771.dat\" -to \"%sverifica\" -space\n",myzpaqexe.c_str(),filezpaq.c_str(),outfolder.c_str(),outfolder.c_str()); fprintf(batch,"%s sum \"%sverifica\" -sha256\n",myzpaqexe.c_str(),outfolder.c_str()); if (!flagchanged) @@ -50800,8 +52612,8 @@ int Jidac::autotest() #endif myprintf("\n\nThe test batchfile is: %s\n",filebatch.c_str()); } - free(buffer8bit); - free(buffer32bit); + franz_free(buffer8bit); + franz_free(buffer32bit); if (allok) return 0; else @@ -51006,23 +52818,6 @@ void my_handler(int s) g_sse42=true; #endif - struct hash_check dummycheck; - dummycheck.algotype=ALGO_SHA1; - g_mychecks.insert(std::pair("SHA-1",dummycheck)); - dummycheck.algotype=ALGO_SHA256; - g_mychecks.insert(std::pair("SHA-256",dummycheck)); - dummycheck.algotype=ALGO_XXHASH64; - g_mychecks.insert(std::pair("XXHASH64",dummycheck)); - dummycheck.algotype=ALGO_XXH3; - g_mychecks.insert(std::pair("XXH3",dummycheck)); - dummycheck.algotype=ALGO_BLAKE3; - g_mychecks.insert(std::pair("BLAKE3",dummycheck)); - dummycheck.algotype=ALGO_SHA3; - g_mychecks.insert(std::pair("SHA-3",dummycheck)); - dummycheck.algotype=ALGO_MD5; - g_mychecks.insert(std::pair("MD5",dummycheck)); - dummycheck.algotype=ALGO_NILSIMSA; - g_mychecks.insert(std::pair("NILSIMSA",dummycheck)); int errorcode=0; try { @@ -51695,9 +53490,9 @@ int unz(const char * archive,const char * key) if (na>65535) unzerror("attr size > 65535"); ///myprintf("2 NA VALE %d\n",na); - if (na>FRANZOFFSET) // houston we have a FRANZBLOCK? + if (na>FRANZOFFSETV1) // houston we have a FRANZBLOCK? { - assert((na-8)::iterator p; int64_t hashed=0; int64_t iniziocalcolo=mtime(); + int unknownfranzo=0; + for (p=unzdt.begin(); p!=unzdt.end(); ++p) mappadt.push_back(p); std::sort(mappadt.begin(), mappadt.end(),unzcompareprimo); @@ -51783,13 +53580,19 @@ int unz(const char * archive,const char * key) string mycrc32=""; int64_t mycreationtime=0; int64_t myaccesstime=0; + bool myisordered=false; + int myversion=0; + franz_posix* myposix=NULL; int unzfranzotype=decode_franz_block(isdirectory(p->first),p->second.sha1hex, myhashtype, hashstoredinzpaq, mycrc32, mycreationtime, - myaccesstime); + myaccesstime, + myisordered, + myversion, + myposix); if ((unzfranzotype==FRANZO_XXHASH64) ||(unzfranzotype==FRANZO_WINHASH64)) { uint64_t myseed = 0; @@ -51801,7 +53604,7 @@ int unz(const char * archive,const char * key) myhash.add(&frag[f.ptr[i]][j],1); hashed++; } - snprintf(p->second.sha1decompressedhex,FRANZOFFSETSHA256,"%016llX",(unsigned long long)myhash.hash()); + snprintf(p->second.sha1decompressedhex,FRANZOFFSETV3,"%016llX",(unsigned long long)myhash.hash()); } else if (unzfranzotype==FRANZO_SHA_1) @@ -51817,7 +53620,7 @@ int unz(const char * archive,const char * key) char sha1result[20]; memcpy(sha1result, mysha1.result(), 20); for (int j=0; j <= 19; j++) - snprintf(p->second.sha1decompressedhex+j*2,FRANZOFFSETSHA256-j*2,"%02X", (unsigned char)sha1result[j]); + snprintf(p->second.sha1decompressedhex+j*2,FRANZOFFSETV3-j*2,"%02X", (unsigned char)sha1result[j]); p->second.sha1decompressedhex[40]=0x0; } else @@ -51833,8 +53636,10 @@ int unz(const char * archive,const char * key) } char sha256result[32]; memcpy(sha256result, mysha256.result(), 32); + /// sha256result[43]='C'; + for (int j=0; j <= 31; j++) - snprintf(p->second.sha1decompressedhex+j*2,FRANZOFFSETSHA256-j*2,"%02X", (unsigned char)sha256result[j]); + snprintf(p->second.sha1decompressedhex+j*2,FRANZOFFSETV3-j*2,"%02X", (unsigned char)sha256result[j]); p->second.sha1decompressedhex[64]=0x0; } else @@ -51852,7 +53657,7 @@ int unz(const char * archive,const char * key) uint8_t output[BLAKE3_OUT_LEN]; blake3_hasher_finalize(&hasher, output, BLAKE3_OUT_LEN); for (int j=0; j < BLAKE3_OUT_LEN; j++) - snprintf(p->second.sha1decompressedhex+j*2,FRANZOFFSETSHA256-j*2,"%02X", (unsigned char)output[j]); + snprintf(p->second.sha1decompressedhex+j*2,FRANZOFFSETV3-j*2,"%02X", (unsigned char)output[j]); p->second.sha1decompressedhex[64]=0x0; } else @@ -51866,7 +53671,7 @@ int unz(const char * archive,const char * key) hasher.add(&frag[f.ptr[i]][j],1); hashed++; } - snprintf(p->second.sha1decompressedhex,FRANZOFFSETSHA256,"%s",hasher.getHash().c_str()); + snprintf(p->second.sha1decompressedhex,FRANZOFFSETV3,"%s",hasher.getHash().c_str()); p->second.sha1decompressedhex[64]=0x0; } else @@ -51881,7 +53686,7 @@ int unz(const char * archive,const char * key) hashed++; } memset(p->second.sha1decompressedhex,0,64); - snprintf(p->second.sha1decompressedhex,FRANZOFFSETSHA256,"%s",hasher.getHash().c_str()); + snprintf(p->second.sha1decompressedhex,FRANZOFFSETV3,"%s",hasher.getHash().c_str()); } else if (unzfranzotype==FRANZO_XXH3) @@ -51896,10 +53701,69 @@ int unz(const char * archive,const char * key) hashed++; } XXH128_hash_t myhash=XXH3_128bits_digest(&state128); - snprintf(p->second.sha1decompressedhex,FRANZOFFSETSHA256,"%016llX%016llX",(unsigned long long)myhash.high64,(unsigned long long)myhash.low64); + snprintf(p->second.sha1decompressedhex,FRANZOFFSETV3,"%016llX%016llX",(unsigned long long)myhash.high64,(unsigned long long)myhash.low64); + } + else + if (unzfranzotype==FRANZO_WHIRLPOOL) + { + NESSIEstruct hasher; + NESSIEinit(&hasher); + + for (uint32_t i=0; isecond.sha1decompressedhex,sizeof(p->second.sha1decompressedhex),"%s",nessie.c_str()); + } else - error("25904: unknown unzfranzotype"); + if ((unzfranzotype==FRANZO_HIGHWAY64) || (unzfranzotype==FRANZO_HIGHWAY128) || (unzfranzotype==FRANZO_HIGHWAY256)) + { + HighwayHashCat highway64state; + uint64_t key[4] = {1, 2, 3, 4}; + HighwayHashCatStart(key,&highway64state); + + + for (uint32_t i=0; isecond.sha1decompressedhex,sizeof(p->second.sha1decompressedhex),"%016llX",(unsigned long long)hash); + } + if (unzfranzotype==FRANZO_HIGHWAY128) + { + uint64_t hash[2]; + HighwayHashCatFinish128(&highway64state,hash); + string temp=binarytohex((const unsigned char*)hash,16); + snprintf(p->second.sha1decompressedhex,sizeof(p->second.sha1decompressedhex),"%s",temp.c_str()); + } + if (unzfranzotype==FRANZO_HIGHWAY256) + { + uint64_t hash[4]; + HighwayHashCatFinish256(&highway64state,hash); + string temp=binarytohex((const unsigned char*)hash,32); + snprintf(p->second.sha1decompressedhex,sizeof(p->second.sha1decompressedhex),"%s",temp.c_str()); + } + + } + else + { + myprintf("\n25904: WARN UNKNOWN UNZFRANZOTYPE %d (maybe newer zpaqfranz archive?)\n",unzfranzotype); + unknownfranzo++; + } vf.push_back(p); if (flagverbose) myprintf("File <<%-20s>> %08u of %08lld (%20s) %1.3f %s\n",myhashtype.c_str(),i+1,(long long)mappadt.size(),migliaia(size),(mtime()-startrecalc)/1000.0,fn.c_str()); @@ -51907,6 +53771,10 @@ int unz(const char * archive,const char * key) } } } + if (unknownfranzo>0) + { + myprintf("\n53651: *** WARNING UNKNOWN FRANZOTYPE %s. Maybe the archive is done with a newer zpaqfranz version?\n",migliaia(unknownfranzo)); + } std::sort(vf.begin(), vf.end(), unzcomparesha1hex); int64_t finecalcolo=mtime(); myprintf("Calc time (hash rebuild from fragments) %f hashed bytes %s\n",(finecalcolo-iniziocalcolo)*0.001,migliaia(hashed)); @@ -51923,8 +53791,9 @@ int unz(const char * archive,const char * key) unsigned int status_filenotfound=0; unsigned int status_1=0; unsigned int status_2=0; - int64_t rihashati=0; - + ///int64_t rihashati=0; + g_dimensione=0; + std::string hashfromfile; int64_t starthash=mtime(); for (unsigned i=0; ifirst),p->second.sha1hex, myhashtype, hashstoredinzpaq, - mycrc32,mycreationtime,myaccesstime); + mycrc32,mycreationtime,myaccesstime,myisordered,myversion,myposix); + if (myfranzo==-1) - error("33351: archive use an unknown hasher!"); + error("33351: archive use an unknown hasher (maybe a more advanced zpaqfraz)!"); string unzfranzostring=""; if (flagverify || flagparanoid) { - MAPPACHECK::iterator a=g_mychecks.find(myhashtype); - if (a!=g_mychecks.end()) + MAPPATIPOHASH::iterator a=g_mappatipohash.end(); + for (MAPPATIPOHASH::iterator p=g_mappatipohash.begin(); p!=g_mappatipohash.end(); ++p) + if (myhashtype==p->second.hashname) + { + a=p; + break; + } + + if (a!=g_mappatipohash.end()) { - unzfranzostring=decodefranzoffset(a->second.algotype); + unzfranzostring=a->second.hashname; myreplace(unzfranzostring,"+CRC-32",""); if (!fileexists(p->first.c_str())) { @@ -51961,16 +53841,17 @@ int unz(const char * archive,const char * key) } else { - uint32_t dummycrc; - int64_t thefilesize=0; - ///inizio totali lavorati - hashfromfile=hash_calc_file( - a->second.algotype, - p->first.c_str(), - true, /// <--- look at this - dummycrc, - starthash,hashed,rihashati,thefilesize); - } + ///if (flagdebug) + ///myprintf("54051: franz_do_hash\n"); + + franz_do_hash dummy(a->first); + hashfromfile=dummy.filehash( + p->first.c_str(),true,starthash,hashed); + } + } + else + { + myprintf("53126: Cannot find hasher %s\n",myhashtype.c_str()); } } std::string hashdecompresso=p->second.sha1decompressedhex; @@ -52217,11 +54098,15 @@ int Jidac::consolidate(string i_archive) XXH128_hash_t myhash=XXH3_128bits_digest(&state128); char risultato[33]; snprintf(risultato,sizeof(risultato),"%016llX%016llX",(unsigned long long)myhash.high64,(unsigned long long)myhash.low64); - uint32_t dummycrc32; int64_t startverify=mtime(); - int64_t io_lavorati=0; - int64_t thefilesize=0; - string hashreloaded=xxhash_calc_file(outfile.c_str(),false,dummycrc32,startverify,total_size,io_lavorati,thefilesize); + g_dimensione=0; + ///if (flagdebug) + ///myprintf("54310: franz_do_hash\n"); + + franz_do_hash dummy("XXH3"); + string hashreloaded=dummy.filehash(outfile,false,startverify,total_size); + + ///string hashreloaded=xxhash_calc_file(outfile.c_str(),false,dummycrc32,startverify,total_size,io_lavorati,thefilesize); myprintf("Expected XXH3 hash of the output file %s\n",risultato); myprintf("Calculated XXH3 hash of the output file %s\n",hashreloaded.c_str()); if (hashreloaded!=risultato) @@ -52434,13 +54319,16 @@ int Jidac::test() string mycrc32=""; int64_t mycreationtime=0; int64_t myaccesstime=0; + bool myisordered=false; + int myversion=0; + franz_posix* myposix=NULL; if (p != dt.end()) { decode_franz_block(isdirectory(it.filename),p->second.franz_block, myhashtype, myhash, - mycrc32,mycreationtime,myaccesstime); + mycrc32,mycreationtime,myaccesstime,myisordered,myversion,myposix); crc32stored=crchex2int(mycrc32.c_str()); } if (flagdebug) @@ -52462,23 +54350,26 @@ int Jidac::test() } currentcrc32=crc32_combine(currentcrc32, it.crc32,it.crc32size); lavorati+=it.crc32size; - while (g_crc32[i].filename==g_crc32[i+1].filename) - { - if ((g_crc32[i].crc32start+g_crc32[i].crc32size) != (g_crc32[i+1].crc32start)) + + if ((i+1) buffer; - ///myprintf("K2\n"); - std::string result; - //myprintf("K3\n"); - std::unique_ptr pipe(popen(cmd, "r"), pclose); - ///myprintf("K4\n"); - - while (fgets(buffer.data(), buffer.size(), pipe.get()) != nullptr) - result += buffer.data(); - ///myprintf("K5\n"); - - return result; -#endif -} -#else -// a debug function -string exec(const char* cmd) -{ - myprintf("Debug function (Windows does not have ZFS!)\n"); - if (cmd==NULL) // compiler be quiet - return "NULL"; - return ""; -} -#endif int pos(const std::string &i_stringa,const std::string& i_cerca) { return (i_stringa.find(i_cerca)); @@ -53263,570 +55127,6 @@ int Jidac::zfs(string command) return 1; } - if (command=="zfsreceive") - { - if ((files.size()!=4) && (files.size()!=6)) - { - myprintf("zfsreceive examples (count %d)\n",files.size()); - myprintf("[1]: zfsreceive /pippo.zpaq rpool/restored -out thescript.sh\n"); - return 2; - } - string myoutput=heuristic_output_search(files); - if (myoutput=="") - { - myprintf("53187: you really need a -out\n"); - return 2; - } - archive =files[0]; - string estensione=prendiestensione(archive); - if (estensione!="zpaq") - { - myprintf("53194: please use .zpaq extension on archive name <<%s>>"); - printUTF8(archive.c_str()); - myprintf("\n"); - return 1; - } - - string themark ="franco"; - string myto=heuristic_to_search(files); - if (myto!="") - themark=myto; - string thefolder =files[0]; - string thepool =files[1]; - /// test pool for / - int howmanyslash=0; - int howmanyat =0; - for (unsigned int i=0;i0) - { - myprintf("53217: a pool |%s| cannot have @\n",thepool.c_str()); - return 2; - } - - getpasswordifempty(); - - int errors=0; - command='l'; - g_optional="versum"; //force isselected - flagstdout=true; - - read_archive(archive.c_str(),&errors,1); /// AND NOW THE MAGIC ONE! - vector filename; - for (DTMap::iterator a=dt.begin(); a!=dt.end(); ++a) - { - string temp=a->first.c_str(); - string estensione=prendiestensione(temp); - if (estensione=="zfs") - { - if (a->second.isordered) - filename.push_back(a->first.c_str()); - else - { - myprintf("53249: GURU file is not ordered <<%s>> \n",a->first.c_str()); - return 2; - } - } - } - char nomeatteso[20]; - - for (unsigned int i=0;i abort\n",i,snomeatteso.c_str(),filename[i].c_str()); - return 2; - } - } - - FILE* batch=fopen(myoutput.c_str(), "w"); - if (batch==NULL) - { - myprintf("53273: cannot write on script %s\n",myoutput.c_str()); - return 2; - } - string pv=""; -#ifndef _WIN32 - if (fileexists("/usr/local/bin/pv")) - pv="/usr/local/bin/pv|"; - else - if (fileexists("/usr/bin/pv")) - pv="/usr/bin/pv|"; - else - if (fileexists("/bin/pv")) - pv="/bin/pv|"; -#endif - - - for (unsigned int i=0;i>\n"); - - return 0; - } - else - if (command=="zfsrestore") - { - if ((files.size()!=4) && (files.size()!=6)) - { - myprintf("zfsrestore examples (count %d)\n",files.size()); - myprintf("[1]: zfsrestore /restoredzfsfolder rpool/restored -out thescript.sh\n"); - myprintf("[2]: zfsrestore /restoredzfsfonder rpool/restored -out thescript.sh -to mymark\n"); - return 2; - } - string myoutput=heuristic_output_search(files); - if (myoutput=="") - { - myprintf("52996: you really need a -out\n"); - return 2; - } - - string themark ="franco"; - string myto=heuristic_to_search(files); - if (myto!="") - themark=myto; - string thefolder =files[0]; - string thepool =files[1]; - /// test pool for / - int howmanyslash=0; - int howmanyat =0; - for (unsigned int i=0;i0) - { - myprintf("53044: a pool |%s| cannot have @\n",thepool.c_str()); - return 2; - } - files.resize(1); - flagforcezfs=true; - DTMap myblock; - g_arraybytescanned.push_back(0); - g_arrayfilescanned.push_back(0); - myprintf("Searching files <<%s>>\n",files[0].c_str()); - edt.clear(); - scandir(false,edt,files[0].c_str(),false); - vector filename; - for (DTMap::iterator a=edt.begin(); a!=edt.end(); ++a) - { - string temp=a->first.c_str(); - string estensione=prendiestensione(temp); - if (estensione=="zfs") - filename.push_back(a->first.c_str()); - } - if (filename.size()==0) - { - myprintf("53074: no .zfs files founded in archive\n"); - return 2; - } - char nomeatteso[20]; - for (unsigned int i=0;i abort\n",i,snomeatteso.c_str(),filename[i].c_str()); - return 2; - } - } - FILE* batch=fopen(myoutput.c_str(), "w"); - if (batch==NULL) - { - myprintf("53088: cannot write on script %s\n",myoutput.c_str()); - return 2; - } - string pv=""; -#ifndef _WIN32 - if (fileexists("/usr/local/bin/pv")) - pv="/usr/local/bin/pv|"; - else - if (fileexists("/usr/bin/pv")) - pv="/usr/bin/pv|"; - else - if (fileexists("/bin/pv")) - pv="/bin/pv|"; -#endif - - for (unsigned int i=0;i>\n"); - return 0; - } - else - if (command=="zfsbackup") - { - archive =files[0]; - string estensione=prendiestensione(archive); - if (estensione!="zpaq") - { - myprintf("53522: please use .zpaq extension on archive name <<%s>>"); - printUTF8(archive.c_str()); - myprintf("\n"); - return 1; - } - string thepool =files[1]; - if (isdirectory(thepool)) - { - myprintf("53537: A pool-dataset |%s| cannot end with a / \n",thepool.c_str()); - myprintf("53537: Good examples tank tank/d\n",thepool.c_str()); - return 2; - } - string themark ="zpaqfranz"; - int howmanyslash=0; - int howmanyat =0; - for (unsigned int i=0;i1) - { - myprintf("53549: A pool-dataset |%s| must have exactly 0 or 1 / instead of %d\n",thepool.c_str(),howmanyslash); - return 2; - } - if (howmanyat>0) - { - myprintf("52984: A pool-dataset |%s| cannot have @\n",thepool.c_str()); - return 2; - } - if (!fileexists(archive)) - { - myprintf("53015: The archive does not exist, checking for snapshots\n"); - vector array_primachiocciola; - vector array_dopochiocciola; - string risul=zfs_get_snaplist(thepool,themark,array_primachiocciola,array_dopochiocciola); - if (array_primachiocciola.size()!=0) - { - myprintf("53022: Archive does not exists, but we got %d snapshots => abort\n",array_primachiocciola.size()); - return 2; - } - char buffer[1000]; - snprintf(buffer,sizeof(buffer),"%s%08d",themark.c_str(),1); - - string thesnap=buffer; - string fullsnap=thepool+"@"+thesnap; - myprintf("53586: The snap |%s|\n",thesnap.c_str()); - myprintf("53587: Fullsnap |%s|\n",fullsnap.c_str()); - myprintf("53030: Exename |%s|\n",zpaqfranzexename.c_str()); - myprintf("53031: Now preparing a brand-new archive and snapshot (starting from 00000001)\n"); - string filebatch="/tmp/newbackup_zfs_"+purgedpool+".sh"; - filebatch=nomefileseesistegia(filebatch); - myprintf("53033: CREATING script %s\n",filebatch.c_str()); - FILE* batch=fopen(filebatch.c_str(), "w"); - if (batch==NULL) - { - myprintf("53037: cannot write on script %s\n",filebatch.c_str()); - return 2; - } - fprintf(batch,"zfs snapshot %s\n",fullsnap.c_str()); - string hashofile=""; - if (flaghashdeep) - { - string hashdeepposition=""; -#ifndef _WIN32 - if (fileexists("/usr/local/bin/hashdeep")) - hashdeepposition="/usr/local/bin/hashdeep"; - else - if (fileexists("/usr/bin/pv")) - hashdeepposition="/usr/bin/hashdeep"; - else - if (fileexists("/bin/pv")) - hashdeepposition="/bin/hashdeep"; -#endif - myprintf("53607: hashdeep <<%s>>\n",hashdeepposition.c_str()); - if (hashdeepposition=="") - { - myprintf("53612: Cannot locate hashdeep, abort!\n"); - fclose(batch); - return 2; - } - string zfsfolder="/"+thepool+"/.zfs/snapshot/"+thesnap+"/"; - myprintf("53616: zfsfolder <<%s>>\n",zfsfolder.c_str()); - hashofile="/tmp/hasho_"+purgedpool+"_"+thesnap+".txt"; - hashofile=nomefileseesistegia(hashofile.c_str()); - myprintf("53603: Hash-o-file <<%s>>\n",hashofile.c_str()); - fprintf(batch,"/sbin/zfs set snapdir=hidden %s\n",thepool.c_str()); - /* - fprintf(batch,"%s -c md5 -r %s |%s a %s elenco.txt -stdin\n", - hashdeepposition.c_str(), - zfsfolder.c_str(), - zpaqfranzexename.c_str(), - hashofile.c_str()); - */ - fprintf(batch,"%s -c md5 -r %s >%s\n", - hashdeepposition.c_str(), - zfsfolder.c_str(), - hashofile.c_str()); - } - fprintf(batch,"zfs send %s@%s00000001 |%s a %s 00000001.zfs -stdin\n",thepool.c_str(),themark.c_str(),fullzpaqexename.c_str(),archive.c_str()); - if (flaghashdeep) - fprintf(batch,"%s a %s %s\n",fullzpaqexename.c_str(),archive.c_str(),hashofile.c_str()); - fclose(batch); -#ifndef _WIN32 - if (chmod(filebatch.c_str(),0700)!=0) - { - myprintf("53046: error on chmod 700 on <<%s>>\n",filebatch.c_str()); - return 2; - } - myprintf("53049: RUNNING script %s\n",filebatch.c_str()); - string runresult=exec(filebatch.c_str()); - if (mypos("(all OK)",runresult)!=0) - { - myprintf("53054: The run seems OK, filesize %s\n",migliaia(prendidimensionefile(archive.c_str()))); - if (flagkill) - { - myprintf("53649: Deleting <<%s>>\n",filebatch.c_str()); - delete_file(filebatch.c_str()); - } - - return 0; - } - else - { - myprintf("53059: GURU, something seems wrong, filesize %s\n",migliaia(prendidimensionefile(archive.c_str()))); - return 2; - } -#endif - return 0; - } - else - { - myprintf("52995: The archive exist, checking latest snapshot\n"); - getpasswordifempty(); - int errors=0; - command='l'; - g_optional="versum"; //force isselected -#ifndef _WIN32 - int64_t oldsize=prendidimensionefile(archive.c_str()); -#endif - read_archive(archive.c_str(),&errors,1); /// AND NOW THE MAGIC ONE! - vector filename; - for (DTMap::iterator a=dt.begin(); a!=dt.end(); ++a) - { - string temp=a->first.c_str(); - string estensione=prendiestensione(temp); - if (estensione=="zfs") - filename.push_back(a->first.c_str()); - } - char nomeatteso[20]; - for (unsigned int i=0;i abort\n",i,snomeatteso.c_str(),filename[i].c_str()); - } - snprintf(nomeatteso,sizeof(nomeatteso),"%08lld",(long long)filename.size()); - string snomeatteso=nomeatteso; - - string expectedsnapshot=themark+snomeatteso; - string fullexpectedsnapshot=thepool+'@'+themark+snomeatteso; - myprintf("53035: Searching for snapshot |%s|\n",fullexpectedsnapshot.c_str()); - - vector array_primachiocciola; - vector array_dopochiocciola; - string risul=zfs_get_snaplist(thepool,themark,array_primachiocciola,array_dopochiocciola); - - if (array_primachiocciola.size()==0) - { - myprintf("53051: Cannot find the expected snapshot |%s|\n",fullexpectedsnapshot.c_str()); - return 2; - } - int foundindex=-1; - for (unsigned int i=0;i %s\n",i,array_dopochiocciola[i].c_str()); - if (expectedsnapshot==array_dopochiocciola[i]) - { - myprintf("53054: The expected snapshot exists (this is good) index %d |%s|\n",i,expectedsnapshot.c_str()); - foundindex=i; - break; - } - } - if ((foundindex+1)!=(int)array_primachiocciola.size()) - { - myprintf("53075: The founded index %d is < snapshot size %d, cowardly abort (not the LAST snapshot)\n",foundindex,array_primachiocciola.size()); - return 2; - } - - string ultimo=array_dopochiocciola[foundindex]; - string solocifre=""; - for (unsigned int i=0;i>\n",hashdeepposition.c_str()); - if (hashdeepposition=="") - { - myprintf("53758: cannot locate hashdeep, abort!\n"); - fclose(batch); - return 2; - } - string thesnap=themark+nuovonome; - - string zfsfolder="/"+thepool+"/.zfs/snapshot/"+thesnap+"/"; - myprintf("53763: Zfsfolder <<%s>>\n",zfsfolder.c_str()); - hashofile="/tmp/hasho_"+purgedpool+"_"+thesnap+".txt"; - hashofile=nomefileseesistegia(hashofile.c_str()); - myprintf("53766: Hashofile <<%s>>\n",hashofile.c_str()); - fprintf(batch,"/sbin/zfs set snapdir=hidden %s\n",thepool.c_str()); - - fprintf(batch,"%s -c md5 -r %s >%s\n", - hashdeepposition.c_str(), - zfsfolder.c_str(), - hashofile.c_str()); - } - - fprintf(batch,"zfs send -i %s %s |%s a %s %s.zfs -stdin\n",fullexpectedsnapshot.c_str(),nuovonomefull.c_str(),fullzpaqexename.c_str(),archive.c_str(),nuovonome.c_str()); - if (flaghashdeep) - fprintf(batch,"%s a %s %s\n",fullzpaqexename.c_str(),archive.c_str(),hashofile.c_str()); - if (flagkill) - fprintf(batch,"zfs destroy %s\n",fullexpectedsnapshot.c_str()); - fclose(batch); - -#ifndef _WIN32 - - if (chmod(filebatch.c_str(),0700)!=0) - { - myprintf("53127: Error on chmod 700 on %s\n",filebatch.c_str()); - return 2; - } - myprintf("53130: RUNNING script %s\n",filebatch.c_str()); - string runresult=exec(filebatch.c_str()); - ///string runresult="(all OK)"; - - if (mypos("(all OK)",runresult)!=0) - { - int64_t newsize=prendidimensionefile(archive.c_str()); - myprintf("53185: Seems OK, filesize %s => %s (%s)\n",migliaia(oldsize),migliaia2(newsize),migliaia3(myabs(newsize,oldsize))); - if (flagkill) - { - myprintf("53812: Deleting <<%s>>\n",filebatch.c_str()); - delete_file(filebatch.c_str()); - } - return 0; - } - else - { - myprintf("53190: GURU, something seems wrong, filesize %s\n",migliaia(prendidimensionefile(archive.c_str()))); - return 2; - } -#endif - return 0; - } - - } - else if ((command=="zfspurge") || (command=="zfslist") || (command=="zfsadd")) { string header =files[0];//"tank/d@2021"; @@ -54398,7 +55698,7 @@ int Jidac::dircompare(bool i_flagonlysize,bool i_flagrobocopy) { if (p->second.hexhash!=cerca->second.hexhash) { - myprintf("%s different %s (0) vs %s (%d) ",mygetalgo().c_str(),p->second.hexhash.c_str(),cerca->second.hexhash.c_str(),i); + myprintf("%s different %s (0) vs %s (%d) ",g_thechosenhash_str.c_str(),p->second.hexhash.c_str(),cerca->second.hexhash.c_str(),i); printUTF8(filename0.c_str()); myprintf("\n"); flagerror=true; @@ -54426,7 +55726,7 @@ int Jidac::dircompare(bool i_flagonlysize,bool i_flagrobocopy) if (!flagerror) { if (flagchecksum) - myprintf("NO diff FOR SURE in slave dirs (checksum check %s)\n",mygetalgo().c_str()); + myprintf("NO diff FOR SURE in slave dirs (checksum check %s)\n",g_thechosenhash_str.c_str()); else myprintf("NO diff in slave dirs (fast check, only size)\n"); } @@ -54662,7 +55962,7 @@ int Jidac::fillami() } } myprintf("%d chunks of (%s) will be written\n",chunks,tohuman(chunksize*sizeof(uint32_t))); - uint32_t *buffer32bit = (uint32_t*)malloc(chunksize*sizeof(uint32_t)); + uint32_t *buffer32bit = (uint32_t*)franz_malloc(chunksize*sizeof(uint32_t)); if (buffer32bit==0) { myprintf("28628: GURU cannot alloc the buffer\n"); @@ -54741,7 +56041,7 @@ int Jidac::fillami() } int64_t timetutto=mtime()-starttutto; myprintf("Total time %f rnd %f hash %f write %f\n",timetutto/1000.0,totalrandtime/1000.0,totalhashtime/1000.0,totaliotime/1000.0); - free(buffer32bit); + franz_free(buffer32bit); if (chunkfilename.size()!=chunkhash.size()) { myprintf("Guru 23925: filename size != hash size\n"); @@ -54771,11 +56071,13 @@ int Jidac::fillami() { string filename=chunkfilename[i]; myprintf("Reading chunk %05d ",i); - uint32_t dummycrc32; - int64_t thefilesize=0; - ///uint64_t starthash=mtime(); - ///inizio totali - string filehash=xxhash_calc_file(filename.c_str(),false,dummycrc32,-1,-1,lavorati,thefilesize); + g_dimensione=0; + ///if (flagdebug) + ///myprintf("56279: franz_do_hash\n"); + + franz_do_hash dummy("XXH3"); + string filehash=dummy.filehash(filename,false,-1,-1); + uint64_t hashspeed=(uint64_t)(lavorati/((mtime()-startverify+1)/1000.0)); myprintf(" (%12s/s) ",tohuman(hashspeed)); bool flagerrore=(filehash!=chunkhash[i]); @@ -54842,6 +56144,24 @@ int Jidac::dir() ///cartella+='/'; if (cartella==".") cartella+='/'; + + int posstella=mypos("*",cartella); + if (posstella>=0) + { + string parteiniziale =cartella.substr(0,posstella); + string partefinale =cartella.substr(posstella,cartella.size()); + if (flagdebug) + { + myprintf("56153: start |%s|\n",parteiniziale.c_str()); + myprintf("56154: end |%s|\n",partefinale.c_str()); + } + cartella=parteiniziale; + onlyfiles.clear(); + onlyfiles.push_back(partefinale); + } + + if (cartella=="") + cartella="./"; #ifdef unix if (flagdebug) myprintf("38802: folder before %s\n",cartella.c_str()); @@ -54873,6 +56193,9 @@ int Jidac::dir() barraod |=(stringcomparei(files[i],"/od")); barraa |=(stringcomparei(files[i],"/a")); } + + + myprintf("==== Scanning dir <<"); printUTF8(cartella.c_str()); myprintf(">> "); @@ -54884,17 +56207,20 @@ int Jidac::dir() myprintf(" /od"); if (barraa) myprintf(" /a"); + + if (onlyfiles.size()>0) + myprintf(" pattern %s ",onlyfiles[0].c_str()); + myprintf("\n"); bool flagduplicati =false; int64_t total_size =0; int quantifiles =0; int quantedirectory =0; - int64_t dummylavorati =0; int quantihash =0; int64_t hash_calcolati =0; uint64_t iniziohash =0; uint64_t finehash =0; - flagduplicati=ischecksum(); //(flagcrc32 || flagcrc32c || flagxxh3 || flagxxhash64 || flagsha1 || flagsha256 || flagmd5 || flagblake3 || flagxxhash64 || flagsha3); + flagduplicati=ischecksum(); if (flagchecksum) flagduplicati=false; g_bytescanned =0; @@ -54988,6 +56314,8 @@ int Jidac::dir() int64_t ultimapercentuale=0; int64_t rapporto; int64_t percentuale; + g_dimensione=0; + for (unsigned int i=0;ifirst; - uint32_t dummycrc=0; - int64_t thefilesize=0; - string thehash=hash_calc_file(ALGO_XXHASH64,filename.c_str(),false,dummycrc,starthashsort,total_size,hashed,thefilesize); + g_dimensione=0; + ///if (flagdebug) + ///myprintf("57948: franz_do_hash\n"); + + franz_do_hash dummy(ALGO_XXHASH64); + string thehash=dummy.filehash(filename,false,starthashsort,total_size); + + ///string thehash=hash_calc_file(ALGO_XXHASH64,filename.c_str(),false,dummycrc,starthashsort,total_size,hashed,thefilesize); if (thehash!="") p->second.hexhash=thehash; else @@ -56917,7 +58261,13 @@ int Jidac::add() (*p->second.pfile_sha3).add(buf,buflen); if (g_franzotype==FRANZO_MD5) if (p->second.pfile_md5) - (*p->second.pfile_md5).add(buf,buflen); + (*p->second.pfile_md5).add(buf,buflen); + if (g_franzotype==FRANZO_WHIRLPOOL) + if (p->second.pfile_whirlpool) + NESSIEadd64((const unsigned char*)buf,buflen*8,p->second.pfile_whirlpool);/// <-- whirlpool works on BITS, so *8 + if ((g_franzotype==FRANZO_HIGHWAY64) || (g_franzotype==FRANZO_HIGHWAY128) || (g_franzotype==FRANZO_HIGHWAY256)) + if (p->second.pfile_highway64) + HighwayHashCatAppend((const unsigned char*)buf,buflen,p->second.pfile_highway64); } for (int j=0;jsecond.pfile_md5) (*p->second.pfile_md5).add(buf,buflen); + if (g_franzotype==FRANZO_WHIRLPOOL) + if (p->second.pfile_whirlpool) + NESSIEadd64((const unsigned char*)buf,buflen*8,p->second.pfile_whirlpool);/// <-- whirlpool works on BITS, so *8 + if ((g_franzotype==FRANZO_HIGHWAY64) || (g_franzotype==FRANZO_HIGHWAY128) || (g_franzotype==FRANZO_HIGHWAY256)) + if (p->second.pfile_highway64) + HighwayHashCatAppend((const unsigned char*)buf,buflen,p->second.pfile_highway64); } if (bufptr>=buflen) @@ -57345,7 +58701,7 @@ int Jidac::add() if (searchfrom!="") if (replaceto!="") replace(filename,searchfrom,replaceto); -/// by using FRANZOFFSET we need to cut down the attr during this compare +/// by using FRANZOFFSETV1 we need to cut down the attr during this compare DTMap::iterator a=dt.find(filename); if (p->second.date && (a==dt.end() // new file || a->second.date!=p->second.date // date change @@ -57372,9 +58728,11 @@ int Jidac::add() puti(is, p->second.date, 8); is.write(filename.c_str(), filename.size());//strlen(filename.c_str())); is.put(0); - string hashtobewritten=""; - string hasherror=""; + string hashtobewritten =""; + string hasherror =""; + string hashname =""; + ///if (!isdirectory(p->first.c_str())) winhash! if (!(flagdebug && flagzero && flagkill)) @@ -57382,106 +58740,62 @@ int Jidac::add() /// zpaqfranz: write hashes and/or CRC-32 if (g_franzotype==FRANZO_CRC_32) /// store only CRC-32 { - hashtobewritten=""; if (flagdebug) myprintf("Mode1: CRC32 by frag <<%08X>> for %s\n",currentcrc32,p->first.c_str()); } else if (g_franzotype==FRANZO_XXHASH64) - { - char temp[17]={0}; - snprintf(temp,sizeof(temp),"%016llX",(unsigned long long)(*p->second.pfile_xxhash64).hash()); - hashtobewritten=temp; - if (flagdebug) - myprintf("Model2: XXHASH64 %s %s\n",hashtobewritten.c_str(),p->first.c_str()); - p->second.hashtype="XXHASH64"; - p->second.hexhash=hashtobewritten; - hasherror="!ERROR! "; - } + hashtobewritten=finalizza_xxhash64(p->second.pfile_xxhash64); else if (g_franzotype==FRANZO_WINHASH64) - { - char temp[17]={0}; - snprintf(temp,sizeof(temp),"%016llX",(unsigned long long)(*p->second.pfile_xxhash64).hash()); - hashtobewritten=temp; - if (flagdebug) - myprintf("Model10: WINHASH64 %s %s\n",hashtobewritten.c_str(),p->first.c_str()); - p->second.hashtype="WINHASH64"; - p->second.hexhash=hashtobewritten; - hasherror="!ERROR! "; - } + hashtobewritten=finalizza_xxhash64(p->second.pfile_xxhash64); else if (g_franzotype==FRANZO_SHA_1) - { - char sha1result[21]={0}; - memcpy(sha1result, (*p->second.pfile_sha1).result(), 20); - hashtobewritten=binarytohex((const unsigned char*)sha1result,20); - if (flagdebug) - myprintf("Mode3: SHA1 %s %s\n",hashtobewritten.c_str(),p->first.c_str()); - p->second.hashtype="SHA-1"; - p->second.hexhash=hashtobewritten; - hasherror="!ERROR! "; - } + hashtobewritten=finalizza_sha1(p->second.pfile_sha1); else if (g_franzotype==FRANZO_SHA_256) - { - char sha256result[33]={0}; - memcpy(sha256result, (*p->second.pfile_sha256).result(), 32); - hashtobewritten=binarytohex((const unsigned char*)sha256result,32); - if (flagdebug) - myprintf("Mode4: SHA256 %s %s\n",hashtobewritten.c_str(),p->first.c_str()); - p->second.hashtype="SHA-256"; - p->second.hexhash=hashtobewritten; - hasherror="!ERROR! "; - } + hashtobewritten=finalizza_sha256(p->second.pfile_sha256); + else + if (g_franzotype==FRANZO_WHIRLPOOL) + hashtobewritten=finalizza_whirlpool(p->second.pfile_whirlpool); + else + if (g_franzotype==FRANZO_HIGHWAY64) + hashtobewritten=finalizza_highway64(p->second.pfile_highway64); + else + if (g_franzotype==FRANZO_HIGHWAY128) + hashtobewritten=finalizza_highway128(p->second.pfile_highway64); + else + if (g_franzotype==FRANZO_HIGHWAY256) + hashtobewritten=finalizza_highway256(p->second.pfile_highway64); else if (g_franzotype==FRANZO_BLAKE3) - { - uint8_t output[BLAKE3_OUT_LEN]; - blake3_hasher_finalize(p->second.pfile_blake3, output, BLAKE3_OUT_LEN); - hashtobewritten=binarytohex((const unsigned char*)output,BLAKE3_OUT_LEN); - if (flagdebug) - myprintf("Mode6: BLAKE3 %s %s\n",hashtobewritten.c_str(),p->first.c_str()); - p->second.hashtype="BLAKE3"; - p->second.hexhash=hashtobewritten; - hasherror="!ERROR! "; - } + hashtobewritten=finalizza_blake3(p->second.pfile_blake3); else if (g_franzotype==FRANZO_SHA3) - { - hashtobewritten=(*p->second.pfile_sha3).getHash(); - if (flagdebug) - myprintf("Mode7: SHA3 %s %s\n",hashtobewritten.c_str(),p->first.c_str()); - p->second.hashtype="SHA-3"; - p->second.hexhash=hashtobewritten; - hasherror="!ERROR! "; - } + hashtobewritten=finalizza_sha3(p->second.pfile_sha3); else if (g_franzotype==FRANZO_MD5) - { - hashtobewritten=(*p->second.pfile_md5).getHash(); - if (flagdebug) - myprintf("Mode8: MD5 %s %s\n",hashtobewritten.c_str(),p->first.c_str()); - p->second.hashtype="MD5"; - p->second.hexhash=hashtobewritten; - hasherror="!ERROR! "; - } + hashtobewritten=finalizza_md5(p->second.pfile_md5); else if (g_franzotype==FRANZO_XXH3) + hashtobewritten=finalizza_xxh3(p->second.pfile_xxh3); + + MAPPATIPOHASH::iterator a=g_mappatipohash.find(g_franzotype); + if (a!=g_mappatipohash.end()) { - char xxh3result[33]={0}; - XXH128_hash_t myhash=XXH3_128bits_digest(p->second.pfile_xxh3); - snprintf(xxh3result,sizeof(xxh3result),"%016llX%016llX",(unsigned long long)myhash.high64,(unsigned long long)myhash.low64); - hashtobewritten=xxh3result; - if (flagdebug) - myprintf("Model5: XXH3 %s %s\n",hashtobewritten.c_str(),p->first.c_str()); - p->second.hashtype="XXH3"; - p->second.hexhash=hashtobewritten; - hasherror="!ERROR! "; + hashname =a->second.hashname; + hasherror =a->second.hasherror; + p->second.hashtype=hashname; } + + ///myprintf("|%s|\n",hasherror.c_str()); + if (flagdebug) + myprintf("58739: |%s|: |%s| <<%s>>\n",hashname.c_str(),hashtobewritten.c_str(),p->first.c_str()); + /// intercept (some) strange things, enforcing a fake hash (will be reported on verify of course) if (hashtobewritten!="") { + p->second.hexhash=hashtobewritten; if ((p->second.size!=p->second.hashedsize) && (p->second.hashedsize==0)) { if (!flagvss && (!flagstdin)) @@ -57509,10 +58823,10 @@ int Jidac::add() } } if ((p->second.attr&255)=='u') - writefranzattr(is,p->second.attr,3,filename,currentcrc32,p->second.file_crc32,hashtobewritten,0,0); + writefranzattr(is,p->second.attr,3,filename,currentcrc32,p->second.file_crc32,hashtobewritten,0,0,NULL); else if ((p->second.attr&255)=='w') - writefranzattr(is,p->second.attr,5,filename,currentcrc32,p->second.file_crc32,hashtobewritten,p->second.creationdate,p->second.accessdate); + writefranzattr(is,p->second.attr,5,filename,currentcrc32,p->second.file_crc32,hashtobewritten,p->second.creationdate,p->second.accessdate,NULL); else puti(is, 0, 4); // no attributes if (a==dt.end() || p->second.data) @@ -57824,16 +59138,17 @@ int Jidac::add() if (checktxt!="") { myprintf("Creating XXH3 check txt on %s\n",checktxt.c_str()); - uint32_t dummycrc32; int64_t startverify=mtime(); - int64_t io_lavorati=0; int64_t larghezzain =prendidimensionefile(g_archive.c_str()); - int64_t thefilesize=0; - string - hashreloaded=xxhash_calc_file(g_archive.c_str(), - false,dummycrc32, - startverify, - larghezzain,io_lavorati,thefilesize); + + g_dimensione=0; + ///if (flagdebug) + ///myprintf("59324: franz_do_hash\n"); + + franz_do_hash dummy("XXH3"); + string hashreloaded=dummy.filehash(g_archive,false,startverify,larghezzain); + + if (hashreloaded=="") errors=2; else @@ -58013,7 +59328,19 @@ int Jidac::dd() int Jidac::benchmark() { - + /* + string testa1="ABCDEFGHIJK"; + string testa2="LMNOPQRSTUVWXYZ"; + + franz_do_hash prova("SHA3"); + prova.init(); + prova.update((char*)testa1.c_str(),testa1.size()); + prova.update((char*)testa2.c_str(),testa2.size()); + prova.finalize(); + string risu=prova.o_hexhash; + printf("|%s|\n",risu.c_str()); + return 0; + */ /* string tutto=extract_tutto; int lunghezza=tutto.size(); @@ -58087,7 +59414,7 @@ int Jidac::benchmark() array_cpu .push_back("i7-8700K (phy) 6"); array_multi .push_back(1836); array_single.push_back(3358); - array_cpu .push_back("AMD-Ryzen 7 3700X(phy) 8"); + array_cpu .push_back("AMD-Ryzen 7 3700X(phy) 8"); array_multi .push_back(2066); array_single.push_back(3256); array_cpu .push_back("i9-9900K (phy) 8"); @@ -58132,24 +59459,25 @@ int Jidac::benchmark() return 1; } } - uint32_t *buffer32bit = (uint32_t*)malloc(chunksize*sizeof(uint32_t)); + uint32_t *buffer32bit = (uint32_t*)franz_malloc(chunksize*sizeof(uint32_t)); if (buffer32bit==0) { myprintf("21202: GURU cannot alloc the buffer32bit\n"); return 1; } - uint8_t *buffer8bit=(uint8_t*)malloc(chunksize*sizeof(uint8_t)); + uint8_t *buffer8bit=(uint8_t*)franz_malloc(chunksize*sizeof(uint8_t)); if (buffer8bit==0) { myprintf("34608: GURU cannot alloc the buffer8bit\n"); - free(buffer32bit); + franz_free(buffer32bit); return 2; } vector thehashes; string hashes=""; if (!ischecksum()) - hashes="XXHASH64;XXH3;SHA-1;SHA-256;BLAKE3;CRC-32;CRC-32C;WYHASH;WHIRLPOOL;MD5;SHA-3;NILSIMSA"; + hashes="XXHASH64;XXH3;SHA-1;SHA-256;BLAKE3;CRC-32;CRC-32C;WYHASH;WHIRLPOOL;MD5;SHA-3;NILSIMSA;HIGHWAY64"; + else { if (flagxxhash64) @@ -58172,23 +59500,26 @@ int Jidac::benchmark() hashes+="WHIRLPOOL;"; if (flagmd5) hashes+="MD5;"; + if (flaghighway64) + hashes+="HIGHWAY64;"; if (flagnilsimsa) hashes+="NILSIMSA;"; if (flagsha3) hashes+="SHA-3;"; + } explode(hashes,';',thehashes); if (thehashes.size()==0) { myprintf("27144: strange hash selection\n"); - free(buffer32bit); + franz_free(buffer32bit); return 1; } if (all) - if ((thehashes.size()!=1) && (thehashes.size()!=12)) + if ((thehashes.size()!=1) && (thehashes.size()!=13)) { myprintf("When benchmarking with -all (multithread), ONE algo or nothing (all)\n"); - free(buffer32bit); + franz_free(buffer32bit); return 1; } vector vettorerisultati; @@ -58212,7 +59543,7 @@ int Jidac::benchmark() myblock.chunksize=chunksize; myblock.buffer32bit=buffer32bit; myblock.timelimit=timelimit; - if (thehashes.size()==12) + if (thehashes.size()==13) myblock.runningalgo="SHA-3"; else myblock.runningalgo=thehashes[0]; @@ -58266,7 +59597,7 @@ int Jidac::benchmark() franzomips+=vettoreparametribenchmark[i].speed; } myprintf("Total speed %s /s\n",tohuman(total_speed)); - if (thehashes.size()==12) + if (thehashes.size()==13) { franzomips/=1000000; for (unsigned int i=0;ifirst)) { startscan=mtime(); - uint32_t dummycrc; - int64_t thefilesize=0; s_stringpair mypair; mypair.first="dummy"; - mypair.second=hash_calc_file(flag2algo(),p->first.c_str(),false,dummycrc,-1,-1,g_dimensione,thefilesize); + ///if (flagdebug) + ///myprintf("60046: franz_do_hash\n"); + + franz_do_hash dummy(g_thechosenhash); + mypair.second=dummy.filehash(p->first.c_str(),false,-1,-1); vec.push_back(mypair); ///vec.push_back(make_pair("dummy",hash_calc_file(flag2algo(),p->first.c_str(),false,dummycrc,-1,-1,g_dimensione))); total_size+=p->second.size; @@ -58543,7 +59885,7 @@ int Jidac::summa() int64_t startprint=mtime(); char sha256result[32]; memcpy(sha256result, sha256.result(), 32); - myprintf("MAGIC_%s: ",mygetalgo().c_str()); + myprintf("MAGIC_%s: ",g_thechosenhash_str.c_str()); for (int j=0; j <= 31; j++) myprintf("%02X", (unsigned char)sha256result[j]); myprintf(" "); @@ -58555,13 +59897,15 @@ int Jidac::summa() } vec.clear(); startscan=mtime(); - uint32_t dummycrc; for (unsigned int i=0;i filestobepurged; for (DTMap::iterator p=edt.begin(); p!=edt.end(); ++p) @@ -58694,14 +60038,17 @@ int Jidac::summa() myprintf("No multithread: Found (%s) => %s bytes (%s) / %s files in %f\n",tohuman2(scannedsize),migliaia(total_size),tohuman(total_size),migliaia2(quantifiles),scantime); if ((flagverbose) && (!flaghashdeep)) myprintf("Total files %s\n",migliaia(myfiles.size())); - int al=string2algo(mygetalgo()); iniziohash=mtime(); + g_dimensione=0; + for (unsigned int i=0;isecond.size)); @@ -58927,7 +60274,7 @@ int Jidac::summa() /// check if, for some reason, we cannot read a file (ex. corrupted, ACL etc) if ((p->second.size)>0) { - if (vec[i].second==emptyalgo(mygetalgo().c_str())) + if (vec[i].second==emptyalgo(g_thechosenhash_str.c_str())) { myprintf("\n\n"); myprintf("32609: **** GURU ERROR non zero-length file get empty hash!\n"); @@ -59083,7 +60430,7 @@ int Jidac::decompress_sfx_to_file(FILE* i_outfile) /// put down the zsfx.exe size_t zsfx_exe_len=mimesize(zsfx_mime64); char* zsfx_exe; - zsfx_exe=(char*)malloc(zsfx_exe_len); + zsfx_exe=(char*)franz_malloc(zsfx_exe_len); if (zsfx_exe==NULL) { myprintf("87420: error in malloc zsfx\n"); @@ -59097,7 +60444,7 @@ int Jidac::decompress_sfx_to_file(FILE* i_outfile) Mymemreader in (( unsigned char *)zsfx_exe,zsfx_exe_len); Myfilewriter out (i_outfile,zsfx_exe_len,true);//yes, SHA-256 libzpaq::decompress(&in,&out); - free(zsfx_exe); + franz_free(zsfx_exe); printbar(' '); myprintf("\r"); if (out.thehash256.gethex()!=zsfx_hash) @@ -59299,7 +60646,7 @@ int Jidac::decompress_mime64_to_file(FILE* i_outfile,const char* i_mime64) return 0; size_t the_exe_len=mimesize(i_mime64); char* the_exe; - the_exe=(char*)malloc(the_exe_len); + the_exe=(char*)franz_malloc(the_exe_len); if (the_exe==NULL) { myprintf("45953: error in malloc\n"); @@ -59314,7 +60661,7 @@ int Jidac::decompress_mime64_to_file(FILE* i_outfile,const char* i_mime64) Mymemreader in (( unsigned char *)the_exe,the_exe_len); Myfilewriter out (i_outfile,the_exe_len); libzpaq::decompress(&in,&out); - free(the_exe); + franz_free(the_exe); return out.written; #else if (i_mime64) // fake test for compiler warning @@ -59342,7 +60689,7 @@ int Jidac::writeresource(const string i_filename,bool i_force,const char* i_mime } size_t the_res_len=mimesize(i_mime64); char* the_res; - the_res=(char*)malloc(the_res_len); + the_res=(char*)franz_malloc(the_res_len); if (the_res==NULL) { myprintf("44972: error in malloc\n"); @@ -59359,7 +60706,7 @@ int Jidac::writeresource(const string i_filename,bool i_force,const char* i_mime if (outFile==NULL) { myprintf("46011 :CANNOT OPEN outfile %s\n",outfile.c_str()); - free(the_res); + franz_free(the_res); return 2; } size_t written=fwrite(the_res,1,the_res_len,outFile); @@ -59367,7 +60714,7 @@ int Jidac::writeresource(const string i_filename,bool i_force,const char* i_mime { myprintf("44994: guru extracting resource\n"); fclose(outFile); - free(the_res); + franz_free(the_res); seppuku(); } if (outFile!=NULL) @@ -59425,14 +60772,17 @@ int Jidac::verify(bool i_readfile) string mycrc32 =""; int64_t mycreationtime=0; int64_t myaccesstime=0; + bool myisordered=false; + int myversion=0; + franz_posix* myposix=NULL; decode_franz_block(false,p->second.franz_block, myhashtype, myhash, - mycrc32,mycreationtime,myaccesstime); + mycrc32,mycreationtime,myaccesstime,myisordered,myversion,myposix); if (myhashtype!="") { - MAPPACHECK::iterator a=g_mychecks.find(myhashtype); - if (a!=g_mychecks.end()) + tipohash* thehash=franz_get_hash(myhashtype); + if (thehash!=NULL) { if ((searchfrom!="") && (replaceto!="")) replace(finalfile,searchfrom,replaceto); @@ -59440,6 +60790,7 @@ int Jidac::verify(bool i_readfile) myfilesoriginal.push_back(p->first); myfilehash.push_back(myhash); myalgo.push_back(myhashtype); + ///printf("myhashtupppp %s %s\n",myhashtype.c_str(),myhash.c_str()); } else { @@ -59489,15 +60840,17 @@ int Jidac::verify(bool i_readfile) { ///monothread (ESXi) int64_t inizio=mtime(); + g_dimensione=0; + for (unsigned int i=0;isecond.resetcheckstat(); + for(unsigned int i=0;isecond.checkednotfound++; + thehash->checkednotfound++; } else { @@ -59565,12 +60923,12 @@ int Jidac::verify(bool i_readfile) { if (flagdebug) myprintf("GOOD %s: STORED == FROM FILE %s\n",myhashtype.c_str(),finalfile.c_str()); - a->second.checkedok++; + thehash->checkedok++; /// Getting the filesize is slow (very slow on network), then waste some RAM /// a->second.checksize+=prendidimensionefile(finalfile.c_str()); DTMap::iterator p=dt.find(vettoreparametrihash[i].originalfilenames[j]); if (p!=dt.end()) - a->second.checksize+=p->second.size; + thehash->checksize+=p->second.size; else { if (flagdebug) @@ -59585,7 +60943,7 @@ int Jidac::verify(bool i_readfile) printUTF8(finalfile.c_str()); myprintf("\n"); } - a->second.checkedfailed++; + thehash->checkedfailed++; } } } @@ -59603,25 +60961,27 @@ int Jidac::verify(bool i_readfile) bool outsomething=(nohashfound>0); if (nohashfound>0) risultato=1; - for (MAPPACHECK::iterator p=g_mychecks.begin(); p!=g_mychecks.end(); ++p) + + for (MAPPATIPOHASH::iterator p=g_mappatipohash.begin(); p!=g_mappatipohash.end(); ++p) outsomething |= (p->second.checkedok+p->second.checkedfailed+p->second.checkednotfound); + if (outsomething) { printbar('-'); int64_t byteshashed=0; - for (MAPPACHECK::iterator p=g_mychecks.begin(); p!=g_mychecks.end(); ++p) + for (MAPPATIPOHASH::iterator p=g_mappatipohash.begin(); p!=g_mappatipohash.end(); ++p) { byteshashed+=p->second.checksize; if (p->second.checkedok) - myprintf("OK %8s : %08d of %08d (%12s hash check against file on disk)\n",p->first.c_str(),p->second.checkedok,tobechecked,tohuman(p->second.checksize)); + myprintf("OK %12s : %08d of %08d (%12s hash check against file on disk)\n",p->second.hashname.c_str(),p->second.checkedok,tobechecked,tohuman(p->second.checksize)); if (p->second.checkedfailed) { - myprintf("FAIL %8s : %08d of %08d ( FAILED hash check against file on disk)\n",p->first.c_str(),p->second.checkedfailed,tobechecked); + myprintf("FAIL %12s : %08d of %08d ( FAILED hash check against file on disk)\n",p->second.hashname.c_str(),p->second.checkedfailed,tobechecked); risultato=2; } if (p->second.checkednotfound) { - myprintf("WARN %8s : %08d of %08d (file not found, cannot check hash)\n",p->first.c_str(),p->second.checkednotfound,tobechecked); + myprintf("WARN %12s : %08d of %08d (file not found, cannot check hash)\n",p->second.hashname.c_str(),p->second.checkednotfound,tobechecked); if (risultato!=2) risultato=1; } @@ -59757,12 +61117,15 @@ int Jidac::extractw() string mycrc32 =""; int64_t mycreationtime=0; int64_t myaccesstime=0; + bool myisordered=false; + int myversion=0; + franz_posix* myposix=NULL; decode_franz_block(false,p->second.franz_block, myhashtype, myhash, - mycrc32,mycreationtime,myaccesstime); - MAPPACHECK::iterator a=g_mychecks.find(myhashtype); + mycrc32,mycreationtime,myaccesstime,myisordered,myversion,myposix); + tipohash* thehash=franz_get_hash(myhashtype); s_fileandsize myblock; myblock.filename =p->first; myblock.size =p->second.size; @@ -59770,7 +61133,7 @@ int Jidac::extractw() myblock.date =p->second.date; myblock.data =p->second.data; myblock.isdir =isdirectory(p->first); - if (a!=g_mychecks.end()) + if (thehash!=NULL) { myblock.hashtype=myhashtype; myblock.hashhex=myhash; @@ -60020,12 +61383,10 @@ int Jidac::multiverify(vector & i_arrayfilename) return 2; } g_dimensione=0; - for (MAPPACHECK::iterator p=g_mychecks.begin(); p!=g_mychecks.end(); ++p) - { - p->second.checkedok =0; - p->second.checkedfailed =0; - p->second.checkednotfound =0; - } + + for (MAPPATIPOHASH::iterator p=g_mappatipohash.begin(); p!=g_mappatipohash.end(); ++p) + p->second.resetcheckstat(); + uint64_t hashtotali =0; int nohashfound =0; vector myfiles; @@ -60108,7 +61469,8 @@ int Jidac::multiverify(vector & i_arrayfilename) { string finalfile =vettoreparametrihash[i].filestobehashed[j]; string myhashtype =vettoreparametrihash[i].algo[j]; - MAPPACHECK::iterator a =g_mychecks.find(myhashtype); + + tipohash* thehash=franz_get_hash(myhashtype); if (myhashtype=="") { nohashfound++; @@ -60117,7 +61479,7 @@ int Jidac::multiverify(vector & i_arrayfilename) } else { - if (a!=g_mychecks.end()) + if (thehash!=NULL) { if (vettoreparametrihash[i].o_hashcalculated[j]=="") { @@ -60127,7 +61489,7 @@ int Jidac::multiverify(vector & i_arrayfilename) printUTF8(finalfile.c_str()); myprintf("\n"); } - a->second.checkednotfound++; + thehash->checkednotfound++; } else { @@ -60135,12 +61497,12 @@ int Jidac::multiverify(vector & i_arrayfilename) { if (flagdebug) myprintf("GOOD %s: STORED == FROM FILE %s\n",myhashtype.c_str(),finalfile.c_str()); - a->second.checkedok++; + thehash->checkedok++; /// Getting the filesize is slow (very slow on network), then waste some RAM /// a->second.checksize+=prendidimensionefile(finalfile.c_str()); DTMap::iterator p=dt.find(vettoreparametrihash[i].originalfilenames[j]); if (p!=dt.end()) - a->second.checksize+=p->second.size; + thehash->checksize+=p->second.size; else { if (flagdebug) @@ -60155,7 +61517,7 @@ int Jidac::multiverify(vector & i_arrayfilename) printUTF8(finalfile.c_str()); myprintf("\n"); } - a->second.checkedfailed++; + thehash->checkedfailed++; } } } @@ -60172,8 +61534,9 @@ int Jidac::multiverify(vector & i_arrayfilename) bool outsomething=(nohashfound>0); if (nohashfound>0) risultato=1; - for (MAPPACHECK::iterator p=g_mychecks.begin(); p!=g_mychecks.end(); ++p) + for (MAPPATIPOHASH::iterator p=g_mappatipohash.begin(); p!=g_mappatipohash.end(); ++p) outsomething |= (p->second.checkedfailed+p->second.checkednotfound); + if (flagverbose) outsomething=true; if (outsomething) @@ -60182,20 +61545,20 @@ int Jidac::multiverify(vector & i_arrayfilename) myprintf("\n"); printbar('-'); int64_t byteshashed=0; - for (MAPPACHECK::iterator p=g_mychecks.begin(); p!=g_mychecks.end(); ++p) + for (MAPPATIPOHASH::iterator p=g_mappatipohash.begin(); p!=g_mappatipohash.end(); ++p) { byteshashed+=p->second.checksize; if (p->second.checkedok) if (flagverbose) - myprintf("OK %8s : %08d of %08d (%12s hash check against file on disk)\n",p->first.c_str(),p->second.checkedok,(int)myfiles.size(),tohuman(p->second.checksize)); + myprintf("OK %8s : %08d of %08d (%12s hash check against file on disk)\n",p->second.hashname.c_str(),p->second.checkedok,(int)myfiles.size(),tohuman(p->second.checksize)); if (p->second.checkedfailed) { - myprintf("FAIL %8s : %08d of %08d ( FAILED hash check against file on disk) ***\n",p->first.c_str(),p->second.checkedfailed,myfiles.size()); + myprintf("FAIL %8s : %08d of %08d ( FAILED hash check against file on disk) ***\n",p->second.hashname.c_str(),p->second.checkedfailed,myfiles.size()); risultato=2; } if (p->second.checkednotfound) { - myprintf("WARN %8s : %08d of %08d (file not found, cannot check hash)\n",p->first.c_str(),p->second.checkednotfound,myfiles.size()); + myprintf("WARN %8s : %08d of %08d (file not found, cannot check hash)\n",p->second.hashname.c_str(),p->second.checkednotfound,myfiles.size()); if (risultato!=2) risultato=1; } @@ -60446,13 +61809,15 @@ int Jidac::extractqueue2(int i_chunk,int i_chunksize) string mycrc32 =""; int64_t mycreationtime=0; int64_t myaccesstime=0; - + bool myisordered=false; + int myversion=0; + franz_posix* myposix=NULL; if (!isdirectory(finalfile)) { decode_franz_block(false,p->second.franz_block, myhashtype, myhash, - mycrc32,mycreationtime,myaccesstime); + mycrc32,mycreationtime,myaccesstime,myisordered,myversion,myposix); myfilesize.push_back((*p->second.pramfile).filesize); expected+=(*p->second.pramfile).filesize; mydata.push_back((*p->second.pramfile).data); @@ -60971,13 +62336,16 @@ int Jidac::hashselect() string mycrc32 =""; int64_t mycreationtime=0; int64_t myaccesstime=0; + bool myisordered=false; + int myversion=0; + franz_posix* myposix=NULL; if (a!=dt.end()) { decode_franz_block(isdirectory(filename),a->second.franz_block, myhashtype, myhash, - mycrc32,mycreationtime,myaccesstime); + mycrc32,mycreationtime,myaccesstime,myisordered,myversion,myposix); if (myhashtype!="") if (myhash!="") { @@ -61614,14 +62982,17 @@ int Jidac::versum() string mycrc32 =""; int64_t myaccesstime =0; int64_t mycreationtime =0; - + bool myisordered=false; + int myversion=0; + franz_posix* myposix=NULL; + decode_franz_block(false, //not a dir p->second.franz_block, myhashtype, myhash, mycrc32, mycreationtime, - myaccesstime); + myaccesstime,myisordered,myversion,myposix); if (myhashtype!="") { if (myhash!="") @@ -61681,18 +63052,20 @@ int Jidac::versum() if (mythreads==1) { int64_t inizio =mtime(); + g_dimensione=0; + myprintf("60253: Total files %s (%s)\n",migliaia(myfiles.size()),tohuman(total_size)); for (unsigned int i=0;i