From d7a748c54dd42bee2cf4fea0602d1936b24f82c3 Mon Sep 17 00:00:00 2001 From: Pierre Lehnen Date: Tue, 21 Jan 2020 18:00:53 -0300 Subject: [PATCH 1/2] Add partial support for zip64 (larger number of entries) --- headers/mainHeader.js | 40 +++++++++++++++++++++++++++++----------- util/constants.js | 20 ++++++++++++++++++++ util/utils.js | 9 +++++++++ zipFile.js | 29 +++++++++++++++++++++++++---- 4 files changed, 83 insertions(+), 15 deletions(-) diff --git a/headers/mainHeader.js b/headers/mainHeader.js index 3a21d93..ce1f461 100644 --- a/headers/mainHeader.js +++ b/headers/mainHeader.js @@ -31,19 +31,37 @@ module.exports = function () { loadFromBinary : function(/*Buffer*/data) { // data should be 22 bytes and start with "PK 05 06" - if (data.length !== Constants.ENDHDR || data.readUInt32LE(0) !== Constants.ENDSIG) + // or be 56+ bytes and start with "PK 06 06" for Zip64 + if ((data.length !== Constants.ENDHDR || data.readUInt32LE(0) !== Constants.ENDSIG) && + (data.length < Constants.ZIP64HDR || data.readUInt32LE(0) !== Constants.ZIP64SIG)) { + throw Utils.Errors.INVALID_END; + } + + if (data.readUInt32LE(0) === Constants.ENDSIG) { + // number of entries on this volume + _volumeEntries = data.readUInt16LE(Constants.ENDSUB); + // total number of entries + _totalEntries = data.readUInt16LE(Constants.ENDTOT); + // central directory size in bytes + _size = data.readUInt32LE(Constants.ENDSIZ); + // offset of first CEN header + _offset = data.readUInt32LE(Constants.ENDOFF); + // zip file comment length + _commentLength = data.readUInt16LE(Constants.ENDCOM); + } else { + // number of entries on this volume + _volumeEntries = Utils.readBigUInt64LE(data, Constants.ZIP64SUB); + // total number of entries + _totalEntries = Utils.readBigUInt64LE(data, Constants.ZIP64TOT); + // central directory size in bytes + _size = Utils.readBigUInt64LE(data, Constants.ZIP64SIZ); + // offset of first CEN header + _offset = Utils.readBigUInt64LE(data, Constants.ZIP64OFF); + + _commentLength = 0; + } - // number of entries on this volume - _volumeEntries = data.readUInt16LE(Constants.ENDSUB); - // total number of entries - _totalEntries = data.readUInt16LE(Constants.ENDTOT); - // central directory size in bytes - _size = data.readUInt32LE(Constants.ENDSIZ); - // offset of first CEN header - _offset = data.readUInt32LE(Constants.ENDOFF); - // zip file comment length - _commentLength = data.readUInt16LE(Constants.ENDCOM); }, toBinary : function() { diff --git a/util/constants.js b/util/constants.js index ea8ecb0..96a914d 100644 --- a/util/constants.js +++ b/util/constants.js @@ -47,6 +47,26 @@ module.exports = { ENDOFF : 16, // offset of first CEN header ENDCOM : 20, // zip file comment length + END64HDR : 20, // zip64 END header size + END64SIG : 0x07064b50, // zip64 Locator signature, "PK\006\007" + END64START : 4, // number of the disk with the start of the zip64 + END64OFF : 8, // relative offset of the zip64 end of central directory + END64NUMDISKS : 16, // total number of disks + + ZIP64SIG : 0x06064b50, // zip64 signature, "PK\006\006" + ZIP64HDR : 56, // zip64 record minimum size + ZIP64LEAD : 12, // leading bytes at the start of the record, not counted by the value stored in ZIP64SIZE + ZIP64SIZE : 4, // zip64 size of the central directory record + ZIP64VEM : 12, // zip64 version made by + ZIP64VER : 14, // zip64 version needed to extract + ZIP64DSK : 16, // zip64 number of this disk + ZIP64DSKDIR : 20, // number of the disk with the start of the record directory + ZIP64SUB : 24, // number of entries on this disk + ZIP64TOT : 32, // total number of entries + ZIP64SIZB : 40, // zip64 central directory size in bytes + ZIP64OFF : 48, // offset of start of central directory with respect to the starting disk number + ZIP64EXTRA : 56, // extensible data sector + /* Compression methods */ STORED : 0, // no compression SHRUNK : 1, // shrunk diff --git a/util/utils.js b/util/utils.js index e9def57..a1c1b5c 100644 --- a/util/utils.js +++ b/util/utils.js @@ -48,6 +48,13 @@ module.exports = (function() { return files; } + function readBigUInt64LE(/*Buffer*/buffer, /*int*/index) { + var slice = Buffer.from(buffer.slice(index, index + 8)); + slice.swap64(); + + return parseInt(`0x${ slice.toString('hex') }`); + } + return { makeDir : function(/*String*/path) { mkdirSync(path); @@ -203,6 +210,8 @@ module.exports = (function() { } }, + readBigUInt64LE, + Constants : Constants, Errors : Errors } diff --git a/zipFile.js b/zipFile.js index c3026b4..3a62b93 100644 --- a/zipFile.js +++ b/zipFile.js @@ -52,22 +52,43 @@ module.exports = function (/*String|Buffer*/input, /*Number*/inputType) { function readMainHeader() { var i = inBuffer.length - Utils.Constants.ENDHDR, // END header size - n = Math.max(0, i - 0xFFFF), // 0xFFFF is the max zip file comment length - endOffset = -1; // Start offset of the END header + max = Math.max(0, i - 0xFFFF), // 0xFFFF is the max zip file comment length + n = max, + endStart = inBuffer.length, + endOffset = -1, // Start offset of the END header + commentEnd = 0; for (i; i >= n; i--) { if (inBuffer[i] !== 0x50) continue; // quick check that the byte is 'P' if (inBuffer.readUInt32LE(i) === Utils.Constants.ENDSIG) { // "PK\005\006" endOffset = i; + commentEnd = i; + endStart = i + Utils.Constants.ENDHDR; + // We already found a regular signature, let's look just a bit further to check if there's any zip64 signature + n = i - Utils.Constants.END64HDR; + continue; + } + + if (inBuffer.readUInt32LE(i) === Utils.Constants.END64SIG) { + // Found a zip64 signature, let's continue reading the whole zip64 record + n = max; + continue; + } + + if (inBuffer.readUInt32LE(i) == Utils.Constants.ZIP64SIG) { + // Found the zip64 record, let's determine it's size + endOffset = i; + endStart = i + Utils.readBigUInt64LE(inBuffer, i + Utils.Constants.ZIP64SIZE) + Utils.Constants.ZIP64LEAD; break; } } + if (!~endOffset) throw Utils.Errors.INVALID_FORMAT; - mainHeader.loadFromBinary(inBuffer.slice(endOffset, endOffset + Utils.Constants.ENDHDR)); + mainHeader.loadFromBinary(inBuffer.slice(endOffset, endStart)); if (mainHeader.commentLength) { - _comment = inBuffer.slice(endOffset + Utils.Constants.ENDHDR); + _comment = inBuffer.slice(commentEnd + Utils.Constants.ENDHDR); } readEntries(); } From 0a4e8ca600cabe9ba91571e26ed804eca6a29e1e Mon Sep 17 00:00:00 2001 From: Pierre Lehnen Date: Tue, 21 Jan 2020 18:28:23 -0300 Subject: [PATCH 2/2] Iterate over entries without storing their metadata --- adm-zip.js | 8 +++++++ zipFile.js | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/adm-zip.js b/adm-zip.js index 0602374..9312fce 100644 --- a/adm-zip.js +++ b/adm-zip.js @@ -339,6 +339,14 @@ module.exports = function (/*String*/input) { return getEntry(name); }, + getEntryCount: function() { + return _zip.getEntryCount(); + }, + + forEach: function(callback) { + return _zip.forEach(callback); + }, + /** * Extracts the given entry to the given targetPath * If the entry is a directory inside the archive, the entire directory and it's subdirectories will be extracted diff --git a/zipFile.js b/zipFile.js index 3a62b93..0e9c771 100644 --- a/zipFile.js +++ b/zipFile.js @@ -9,7 +9,8 @@ module.exports = function (/*String|Buffer*/input, /*Number*/inputType) { filename = "", fs = Utils.FileSystem.require(), inBuffer = null, - mainHeader = new Headers.MainHeader(); + mainHeader = new Headers.MainHeader(), + loadedEntries = false; if (inputType === Utils.Constants.FILE) { // is a filename @@ -22,9 +23,28 @@ module.exports = function (/*String|Buffer*/input, /*Number*/inputType) { readMainHeader(); } else { // none. is a new file + loadedEntries = true; + } + + function iterateEntries(callback) { + const totalEntries = mainHeader.diskEntries; // total number of entries + let index = mainHeader.offset; // offset of first CEN header + + for (let i = 0; i < totalEntries; i++) { + let tmp = index; + const entry = new ZipEntry(inBuffer); + + entry.header = inBuffer.slice(tmp, tmp += Utils.Constants.CENHDR); + entry.entryName = inBuffer.slice(tmp, tmp += entry.header.fileNameLength); + + index += entry.header.entryHeaderSize; + + callback(entry); + } } function readEntries() { + loadedEntries = true; entryTable = {}; entryList = new Array(mainHeader.diskEntries); // total number of entries var index = mainHeader.offset; // offset of first CEN header @@ -90,7 +110,7 @@ module.exports = function (/*String|Buffer*/input, /*Number*/inputType) { if (mainHeader.commentLength) { _comment = inBuffer.slice(commentEnd + Utils.Constants.ENDHDR); } - readEntries(); + // readEntries(); } return { @@ -99,6 +119,9 @@ module.exports = function (/*String|Buffer*/input, /*Number*/inputType) { * @return Array */ get entries() { + if (!loadedEntries) { + readEntries(); + } return entryList; }, @@ -114,6 +137,23 @@ module.exports = function (/*String|Buffer*/input, /*Number*/inputType) { _comment = val; }, + getEntryCount: function() { + if (!loadedEntries) { + return mainHeader.diskEntries; + } + + return entryList.length; + }, + + forEach: function(callback) { + if (!loadedEntries) { + iterateEntries(callback); + return; + } + + entryList.forEach(callback); + }, + /** * Returns a reference to the entry with the given name or null if entry is inexistent * @@ -121,6 +161,9 @@ module.exports = function (/*String|Buffer*/input, /*Number*/inputType) { * @return ZipEntry */ getEntry: function (/*String*/entryName) { + if (!loadedEntries) { + readEntries(); + } return entryTable[entryName] || null; }, @@ -130,6 +173,9 @@ module.exports = function (/*String|Buffer*/input, /*Number*/inputType) { * @param entry */ setEntry: function (/*ZipEntry*/entry) { + if (!loadedEntries) { + readEntries(); + } entryList.push(entry); entryTable[entry.entryName] = entry; mainHeader.totalEntries = entryList.length; @@ -142,6 +188,9 @@ module.exports = function (/*String|Buffer*/input, /*Number*/inputType) { * @param entryName */ deleteEntry: function (/*String*/entryName) { + if (!loadedEntries) { + readEntries(); + } var entry = entryTable[entryName]; if (entry && entry.isDirectory) { var _self = this; @@ -163,6 +212,9 @@ module.exports = function (/*String|Buffer*/input, /*Number*/inputType) { * @return Array */ getEntryChildren: function (/*ZipEntry*/entry) { + if (!loadedEntries) { + readEntries(); + } if (entry.isDirectory) { var list = [], name = entry.entryName, @@ -184,6 +236,9 @@ module.exports = function (/*String|Buffer*/input, /*Number*/inputType) { * @return Buffer */ compressToBuffer: function () { + if (!loadedEntries) { + readEntries(); + } if (entryList.length > 1) { entryList.sort(function (a, b) { var nameA = a.entryName.toLowerCase(); @@ -258,6 +313,9 @@ module.exports = function (/*String|Buffer*/input, /*Number*/inputType) { }, toAsyncBuffer: function (/*Function*/onSuccess, /*Function*/onFail, /*Function*/onItemStart, /*Function*/onItemEnd) { + if (!loadedEntries) { + readEntries(); + } if (entryList.length > 1) { entryList.sort(function (a, b) { var nameA = a.entryName.toLowerCase();