From 94cb8bdfea6257d7bc67f72ed80a790c2b5dae3a Mon Sep 17 00:00:00 2001 From: Samuel Attard Date: Thu, 9 Sep 2021 11:06:13 -0700 Subject: [PATCH] feat: add file hashes to asar header (#221) * feat: add file hashes to asar header * feat: add getRawHeader method to public API * chore: fix lint * chore: update docs * refactor: use integrity instead of hash pairs * feat: add block hashes * fix: ensure executables are extracted with executable permission * fix: ensure symlinks are not deeply resolved when packaging * chore: update test files * chore: remove DS_Store * perf: generate block hashes as we parse the stream * docs: update README with new options * revert * chore: update per feedback --- README.md | 30 ++++++++- lib/asar.js | 7 ++ lib/crawlfs.js | 10 +++ lib/disk.js | 2 +- lib/filesystem.js | 3 + lib/index.d.ts | 24 +++++++ lib/integrity.js | 62 ++++++++++++++++++ test/expected/packthis-all-unpacked.asar | Bin 348 -> 1588 bytes test/expected/packthis-transformed.asar | Bin 530 -> 1774 bytes test/expected/packthis-unicode-path.asar | Bin 124 -> 536 bytes test/expected/packthis-unpack-dir-glob.asar | Bin 460 -> 1700 bytes .../packthis-unpack-dir-globstar.asar | Bin 478 -> 1718 bytes test/expected/packthis-unpack-dir.asar | Bin 298 -> 1334 bytes test/expected/packthis-unpack.asar | Bin 285 -> 1317 bytes test/expected/packthis-without-hidden.asar | Bin 467 -> 1503 bytes test/expected/packthis.asar | Bin 530 -> 1774 bytes test/util/compareFiles.js | 3 + 17 files changed, 137 insertions(+), 4 deletions(-) create mode 100644 lib/integrity.js diff --git a/README.md b/README.md index 6dbd11c..e05497f 100644 --- a/README.md +++ b/README.md @@ -153,12 +153,24 @@ Structure of `header` is something like this: "ls": { "offset": "0", "size": 100, - "executable": true + "executable": true, + "integrity": { + "algorithm": "SHA256", + "hash": "...", + "blockSize": 1024, + "blocks": ["...", "..."] + } }, "cd": { "offset": "100", "size": 100, - "executable": true + "executable": true, + "integrity": { + "algorithm": "SHA256", + "hash": "...", + "blockSize": 1024, + "blocks": ["...", "..."] + } } } } @@ -168,7 +180,13 @@ Structure of `header` is something like this: "files": { "hosts": { "offset": "200", - "size": 32 + "size": 32, + "integrity": { + "algorithm": "SHA256", + "hash": "...", + "blockSize": 1024, + "blocks": ["...", "..."] + } } } } @@ -187,6 +205,12 @@ precisely represent UINT64 in JavaScript `Number`. `size` is a JavaScript because file size in Node.js is represented as `Number` and it is not safe to convert `Number` to UINT64. +`integrity` is an object consisting of a few keys: +* A hashing `algorithm`, currently only `SHA256` is supported. +* A hex encoded `hash` value representing the hash of the entire file. +* An array of hex encoded hashes for the `blocks` of the file. i.e. for a blockSize of 4KB this array contains the hash of every block if you split the file into N 4KB blocks. +* A integer value `blockSize` representing the size in bytes of each block in the `blocks` hashes above + [pickle]: https://chromium.googlesource.com/chromium/src/+/master/base/pickle.h [node-pickle]: https://www.npmjs.org/package/chromium-pickle [grunt-asar]: https://github.com/bwin/grunt-asar diff --git a/lib/asar.js b/lib/asar.js index c902e6a..050e1a7 100644 --- a/lib/asar.js +++ b/lib/asar.js @@ -157,6 +157,10 @@ module.exports.statFile = function (archive, filename, followLinks) { return filesystem.getFile(filename, followLinks) } +module.exports.getRawHeader = function (archive) { + return disk.readArchiveHeaderSync(archive) +} + module.exports.listPackage = function (archive, options) { return disk.readFilesystemSync(archive).listFiles(options) } @@ -199,6 +203,9 @@ module.exports.extractAll = function (archive, dest) { // it's a file, extract it const content = disk.readFileSync(filesystem, filename, file) fs.writeFileSync(destFilename, content) + if (file.executable) { + fs.chmodSync(destFilename, '755') + } } } } diff --git a/lib/crawlfs.js b/lib/crawlfs.js index cc4a796..a26c3eb 100644 --- a/lib/crawlfs.js +++ b/lib/crawlfs.js @@ -20,11 +20,21 @@ module.exports = async function (dir, options) { const metadata = {} const crawled = await glob(dir, options) const results = await Promise.all(crawled.map(async filename => [filename, await determineFileType(filename)])) + const links = [] const filenames = results.map(([filename, type]) => { if (type) { metadata[filename] = type + if (type.type === 'link') links.push(filename) } return filename + }).filter((filename) => { + // Newer glob can return files inside symlinked directories, to avoid + // those appearing in archives we need to manually exclude theme here + const exactLinkIndex = links.findIndex(link => filename === link) + return links.every((link, index) => { + if (index === exactLinkIndex) return true + return !filename.startsWith(link) + }) }) return [filenames, metadata] } diff --git a/lib/disk.js b/lib/disk.js index 72a3949..34569a4 100644 --- a/lib/disk.js +++ b/lib/disk.js @@ -76,7 +76,7 @@ module.exports.readArchiveHeaderSync = function (archive) { const headerPickle = pickle.createFromBuffer(headerBuf) const header = headerPickle.createIterator().readString() - return { header: JSON.parse(header), headerSize: size } + return { headerString: header, header: JSON.parse(header), headerSize: size } } module.exports.readFilesystemSync = function (archive) { diff --git a/lib/filesystem.js b/lib/filesystem.js index 2a6579d..85e1eb0 100644 --- a/lib/filesystem.js +++ b/lib/filesystem.js @@ -5,6 +5,7 @@ const os = require('os') const path = require('path') const { promisify } = require('util') const stream = require('stream') +const getFileIntegrity = require('./integrity') const UINT32_MAX = 2 ** 32 - 1 @@ -57,6 +58,7 @@ class Filesystem { if (shouldUnpack || dirNode.unpacked) { node.size = file.stat.size node.unpacked = true + node.integrity = await getFileIntegrity(p) return Promise.resolve() } @@ -86,6 +88,7 @@ class Filesystem { node.size = size node.offset = this.offset.toString() + node.integrity = await getFileIntegrity(p) if (process.platform !== 'win32' && (file.stat.mode & 0o100)) { node.executable = true } diff --git a/lib/index.d.ts b/lib/index.d.ts index a95ba9a..b3790ec 100644 --- a/lib/index.d.ts +++ b/lib/index.d.ts @@ -44,6 +44,29 @@ export type InputMetadata = { } }; +export type DirectoryRecord = { + files: Record; +}; + +export type FileRecord = { + offset: string; + size: number; + executable?: boolean; + integrity: { + hash: string; + algorithm: 'SHA256'; + blocks: string[]; + blockSize: number; + }; +} + +export type ArchiveHeader = { + // The JSON parsed header string + header: DirectoryRecord; + headerString: string; + headerSize: number; +} + export function createPackage(src: string, dest: string): Promise; export function createPackageWithOptions( src: string, @@ -59,6 +82,7 @@ export function createPackageFromFiles( ): Promise; export function statFile(archive: string, filename: string, followLinks?: boolean): Metadata; +export function getRawHeader(archive: string): ArchiveHeader; export function listPackage(archive: string, options?: ListOptions): string[]; export function extractFile(archive: string, filename: string): Buffer; export function extractAll(archive: string, dest: string): void; diff --git a/lib/integrity.js b/lib/integrity.js new file mode 100644 index 0000000..6fabee4 --- /dev/null +++ b/lib/integrity.js @@ -0,0 +1,62 @@ +const crypto = require('crypto') +const fs = require('fs') +const stream = require('stream') +const { promisify } = require('util') + +const ALGORITHM = 'SHA256' +// 4MB default block size +const BLOCK_SIZE = 4 * 1024 * 1024 + +const pipeline = promisify(stream.pipeline) + +function hashBlock (block) { + return crypto.createHash(ALGORITHM).update(block).digest('hex') +} + +async function getFileIntegrity (path) { + const fileHash = crypto.createHash(ALGORITHM) + + const blocks = [] + let currentBlockSize = 0 + let currentBlock = [] + + await pipeline( + fs.createReadStream(path), + new stream.PassThrough({ + decodeStrings: false, + transform (_chunk, encoding, callback) { + fileHash.update(_chunk) + + function handleChunk (chunk) { + const diffToSlice = Math.min(BLOCK_SIZE - currentBlockSize, chunk.byteLength) + currentBlockSize += diffToSlice + currentBlock.push(chunk.slice(0, diffToSlice)) + if (currentBlockSize === BLOCK_SIZE) { + blocks.push(hashBlock(Buffer.concat(currentBlock))) + currentBlock = [] + currentBlockSize = 0 + } + if (diffToSlice < chunk.byteLength) { + handleChunk(chunk.slice(diffToSlice)) + } + } + handleChunk(_chunk) + callback() + }, + flush (callback) { + blocks.push(hashBlock(Buffer.concat(currentBlock))) + currentBlock = [] + callback() + } + }) + ) + + return { + algorithm: ALGORITHM, + hash: fileHash.digest('hex'), + blockSize: BLOCK_SIZE, + blocks: blocks + } +} + +module.exports = getFileIntegrity diff --git a/test/expected/packthis-all-unpacked.asar b/test/expected/packthis-all-unpacked.asar index f699be095056799e4297c16dbfb5e73035ea2dbd..f809530da995810dd178bebc63f9f7fd24153fcb 100644 GIT binary patch literal 1588 zcmbu9J#QN^42C@v{UO(-gJF;o^+9jlx@RmpP!ee$;y8xw08NAZ_v#EIsN2EA4Z?|p zaMb(or9Ledi{Hgrw+?IMOui+r zNZ?D0C3#{G(K{nm4Io~EpoV@n_dk{0bA5ccT!oOy16_0d+CSzO`1$_69NKuqNy@)E zKj=&wyyY?n3pqgI<}C-8nTjOyPJ2RYYMGH$9R=*bin)-X0OnYmPZO|~-CZoR3v?`3 zFzzlGw>s=Me;Fpb8IGxyz?HDtB8ESEvRUteWXv zG_wm_95Wfh-x|WtP%SZbzXx_e&KWntt}AzO)+$-a0wkS;nV|_JWzL$TB}?LxsK2Xv zc7dxQ^yzSZ{r{1G8QRqss6>vlRjhq3#EMbP%+U%vngkLmMdHGqst^ZeEU6Dq$syd; zKD)q2$Kkf)IH%==!YLQc*(@gs$wThb?g=6GNQ(j_D$!3SajMnrI4C%c&JhU^86!A5KZ z!A8+CrIjEEVi&Nmvq@_$`V#~Z&kZ4FNwJ14=5_A5bLRWLb7pE;meq!-qxS1$mTl$O zT20-2aw~5%TQR0~wQiIT%gWSkJxci`w-b2>v75-2vwR|Nwac`!(JD79AGf;Fp?PzS zJ}%Br3006s({-D)j=)&7P6HM?X#ozjm0ZiHW#Yi3kA>F~tYn202q`7SoOH_Zq{!FM zPJXo$?l1101GojrVBbb(zB71$yN@4FV1p^__M&Ja?5H^ef&>P3V5fO;U(J`H@HOUmHQ9Gesl6^rB z9AIDlw61HC%Vf9sv3X8}N1gMkOC0f!zT7;6jkRzQUhc!C0~w-SN_tE}Z- zQ=xdHRA1-=2RP81_3F)g1wR&Q;Ld8}lQD=!JEbKvRM-+YD`=HGTIZP6JPFQt(u#Ab zKs&>gr#sU3MLlqUJ;CV-SZcPq^-A$NjsGtHdZ(Ql&u(0w8$EOJ47rUi%+K7UainS^ zC#b)&cKI6(L*1Km)7iVP<3DLpTba5xm1XZ=oc-|n4nObjhej15enVnEkNQ*J?QZLN;d)1{jqH?DiScEU*Vb}bM9 E04b)K@c;k- delta 33 jcmbQiQp3l>z`#%f#05Z{#W0c2ezFXs274_CFfafBV^9XC diff --git a/test/expected/packthis-unpack-dir-glob.asar b/test/expected/packthis-unpack-dir-glob.asar index c0b55c9abd175d14d71ab3c066b40a40e02bd8ac..fd27b9d3fd534f6ec8a2fed0b99be6470e90ad15 100644 GIT binary patch literal 1700 zcmb`HJ#QN^42C@v9r80Y8-*m2`ta7Rd&Z)JBvP@FG%oB*>o~}NFV8m89tzY6905A; zc*K!>@RA>9v)S(BY&Pg!>wPkPbthZ1crrUTJ|FIXly)m@7Q^O2=6^nqaqiQw8^);Z zYAPDv!`I@W?8mNeXoMua_mSE8<$jkixmko z6;GZ!3fm^7ClRh8`(CXTK${VP`7Z+QO$fM()Y_}6Ks*97H7~w%4x9j7@*!*UoU=yRQe2_p zx>Zowv!Xgb9v3G#>0m#FaGaJ26Z#L6{1#b6EVW_OuBlC3eEvDR0Qi?c=dgCo^M z^$w}noUO_SEcKXuBkwzLlFqthDTy-cwF^@f53cc9E*B?w;W8T=AC?=DtI_J~ht1Jg L{XBZMtKHRavb^~q delta 56 zcmZ3&dxn{hg@J)#0V4y$93Y;`IFT=4vM!^?WEUp0$)3z6lYLlRCnvCaa@Ils1H1&Z41hfp9r75OjY2=9M0xAhJ!8>95~8@?@O*mwUfR8|S?sq@GXMK|h;yIz zhy4(>TOGwLX#P+5>Txf3+tvPgh$`#5jjopuW?^o>eD)a4+$_s(sqa;wTTUG?LE}Qj zw^RWkWAW&@gRpHV6 z@S|A#`(Z(fQoIY&I7UywGbSpLF;s#cpa5{Eg6sn#dWX`SWE4=Z7nVjHk#_@#ZV#XWl2j7oUf>IkK<$w(+_h4;zWGU8GY7Dxjs@su+&B?1% z(S_EddIppV(Wp9;6Pzf$>X@4&T-DHzYj6X>cuhtd?xmvKf0dN8MSF-twWN9nSZvN# wg$fpX%D$oZ9hs!FHew`EX1#V{tm1=fJj!Kqf(w_~aPj?eD{^y^Xmz;x1KUyo_W%F@ delta 71 zcmdnSdykoqg@J+L2qOc-0U+MVIFT=4vM!^?WEUp0$)3z^ljB&zC%3SA%G5#tP;FXf SPHIU;QEDm|h*g|f!36-XM->|Y diff --git a/test/expected/packthis-unpack-dir.asar b/test/expected/packthis-unpack-dir.asar index 02cbd14331e1cc27eed62604267ba5a4d87b84ba..51d6439b42570c8e7ed90064122b3e2b43e46aef 100644 GIT binary patch literal 1334 zcmbtUJ#QN^4E4~C-8ys(ZZ-^3qCV)YTlb7b2YryX5yv*{3(z#^e=jc$(i$C{!UqVV zc)BCsdwO~|o6SDGpUpnn_+aB{(9Kp(!}4k9n}m5uh9%F3z3NFU zHNY?*HoH+)$IbXSZ7JK;-s1JWy}bSM*@b93*5$OeH90`y?llLOnQBSuomfIz>Y0&6 z90g2{s=AP&0_NDdPrmlpv$H?d?Y`aJKDR;0OoB#8{o2hAyz}3XDH+I!kR9|8T4j2G@&jcXR=F*C4O;s*Zb9Diu<~ITU{51G0P?s1(p}-EvIpt1hxUzxM-biaMK*g9)Rn!7Wn6u=l zNmFsDh`)&X>H;qZLhcXa<79<%?C_Tp*qV8>+~O-VBF9-9HZw)hs?;)b)XI(~bc8~Y zxU#2Khyyd$WX`EMgp0thF7PThoI^&_`_=vbg_*E$%2je!&8dXsA)C)VAw*kARW1cM qp*RGT6cBx&R5FGt$<>RPuP*Q`w%v`B&^P;?ET#Z%+TJd>-S`W|0eERxGKC~UHd|b6P6Cg|PEO|=8B}a3lEul5`Nl;XY zJx1bU|GVzh6{yT4$tu>?0@;djO(MaH5Rdd2l>^3t4^pIuWiBEtOMbjelHRtobcW~a-5J7w%;W} zc9YoJ483G-H_2MttmsdLV!*2T;I+IeVfxAL80iQQ~; zaNOL&hwJqkmnp7j+Fldqz$pWhiKr#l2Aow<@PI<_D$|CU5I9JtIJn5Arj9b5`b;R{ zvNY2izvMPM{^9!ZZvZnOD4b~c_xDB*aQ`V@mBI!7p9{R&Z8!e8z%)POgBivL?}2is zOqzMa9d|4Wn`YEVCMgw}21-MUz4bOmZ&DS&cya8rm3_b4X40)G#N0&(#RGw57B8$!J-i{Miwnq!5AHM8dX%5D;vZ>=#c}QBtL9* z^POVb_oLH3bQlIb2uD2tH#&Oe@d()HB6w{~8lB-#@Bs^uf^kNGw_v0(%qcu4PD?ou zedGYA`KJBQtUuvHQ3JC^X&1DPQ7bEzpc>D$07g@s1s9FA)F>7>qbw-Nn2?~XX3_=4 z-9YM*0~{wDYhjV>a;F`tFIHEUrl+n>p*Fo-Usx;ZWYH$i;r+(e^>5gX_12c=$-&pz zpExNsYPV{Hynl7^m&IaAE_Ca6dxTv2T(q$-o&5~OnS600U+={Hk=+iYYq4Hd+aWJQ zl~%>e?{8Oe&9&u)+U=P;{_y-CPo9@vk5Bx#Tz*3*Hs0-(X7}oQWBa8mhlhkr++6x} d^zrBezCxB4R~8QDA3S4R6a=*FJ6MU^`Ckiyt+W6D delta 46 zcmcc5eVLh$g@J+L4-o$V;!g|{`RpglFj`JlVN#nc$*eP3ghidXwzhUMAL~T`5eEzX diff --git a/test/expected/packthis.asar b/test/expected/packthis.asar index 3638e1d3dfbe48c5a3d77f3ada669ee55b734c0c..d5c630913176fe5e058755e9dd57b8225c061ae6 100644 GIT binary patch literal 1774 zcmbtVziSjh6yC@s#z-tIBuI*58xg`WKi=%jq`Q!i3u*{Sz=#OW&)i{hcjOi{hBRU$ z2sVnADN_XmLF@t+b~b6PMgIgr#J3^DJ5rqCirJZY`*z=a-}m0UQj#R45q#kIHJl_n zX>4qSPCC7lR_YD!L$lad@+Wy=>NK8)beig^v=w6~pe-foRN83fVSTHSZx=phW4(pz z`X;_ysa2WC(4%fTb=-R(#A_oVW->?y2BhXxaxZzHK)Cgpl^irD8D|kf2!a@HgrPy8 zufjjsjh1_~a(oV;8aROy8=m?8-~k@&?oOq?u|*3gr&JUR+Dk$=2UMSj1Ym@M_B<-V zlm=^LBOTxebkc1N&*clpFrQKZb$_2%` zl7STp{dYDv2ZULvGw(FB%+knB=7|B}WCL`Z=wb|gi5NJ* zaW)j<^Iq|JH)eXS5-?eYsVp=jz*y8CD8f>REqVvhDnq~^5iv#@4Us`>6;u{wl!i$Y zcwf*12RKnbY_{|5v&Bz(pclvaZ@mMz!~rm^qN5ILz(gCtDXp_;HG={l&;S`|t2sCW zT4+tJ#6q!3h`!JV4sfbB>D8O|3VtrsK#i8l2Bl(D(g?|k!ouc2Nrt;%qc(DasGGtms`zHd9k!OKR$MS4Bf^TYO~8Y zj}~L}0zPkT-u#BsP-l65COP;z`4cy#_3E8!lDvO)`M1Hol3Z%n?sbyn%I9JXeQE7` zG*0AmYxznm=8w%*NZJNdsnQI287id0Uw(hPgg!SGW~-|ccirK|KQ4J*dObYy<7)X0 z8CiR`SDM_b?G5dht{om?1G_!{>FDFp2mB;im|L7Zn0fe&GG5>zW!J(M)y)3_m0|n< delta 50 zcmaFIJBfvlg@J)VgOPzj1&HMtC-Rw27GgA;EY4&#S%Fz?vM7trWC2!n=GxlY$=qy@ E0ob()EC2ui diff --git a/test/util/compareFiles.js b/test/util/compareFiles.js index 3862079..485ba2b 100644 --- a/test/util/compareFiles.js +++ b/test/util/compareFiles.js @@ -4,6 +4,9 @@ const assert = require('assert') const fs = require('../../lib/wrapped-fs') module.exports = async function (actualFilePath, expectedFilePath) { + if (process.env.ELECTRON_ASAR_SPEC_UPDATE) { + await fs.writeFile(expectedFilePath, await fs.readFile(actualFilePath)) + } const [actual, expected] = await Promise.all([fs.readFile(actualFilePath, 'utf8'), fs.readFile(expectedFilePath, 'utf8')]) assert.strictEqual(actual, expected) }