From b028a5698641b8a203bb9eadd7001cee02101a64 Mon Sep 17 00:00:00 2001 From: Armin Samii Date: Fri, 23 Feb 2024 13:40:48 -0500 Subject: [PATCH] fix line endings for reproducible SHAs --- .gitattributes | 1 + .github/actions/Sha-Of-Directory.ps1 | 31 ---------------------------- .github/actions/sha-of-zip.bat | 27 +++++++++++------------- .github/actions/sha-of-zip.sh | 7 +------ .github/workflows/release.yml | 2 -- 5 files changed, 14 insertions(+), 54 deletions(-) create mode 100644 .gitattributes delete mode 100644 .github/actions/Sha-Of-Directory.ps1 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..fcadb2cf9 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text eol=lf diff --git a/.github/actions/Sha-Of-Directory.ps1 b/.github/actions/Sha-Of-Directory.ps1 deleted file mode 100644 index 8681e4bf5..000000000 --- a/.github/actions/Sha-Of-Directory.ps1 +++ /dev/null @@ -1,31 +0,0 @@ -## Creates a SHA of all files in the directory but ignores their metadata -## This is useful when you want to ignore all timestamps of a zip, for example: -## You can extract the contents to a directory, then run this. -## Usage: powershell Sha-Of-Directory.ps1 - -$Directory=$args[0] -$ShaA=$args[1] - -# First, create a hash algorithm object using SHA256. -$Algorithm = [System.Security.Cryptography.HashAlgorithm]::Create("SHA" + $ShaA) - -# Next, create a cryptographic stream of data using the SHA256 hash algorithm. -$CryptoStream = [System.Security.Cryptography.CryptoStream]::new( - ([System.IO.Stream]::Null), - $Algorithm, - "Write" -) - -# Retrieve each file and copy the data into the cryptographic stream. -echo $Directory -foreach ($File in Get-ChildItem -Recurse -Path $Directory -File) { - # Write-Host $File - $FileStream = [io.file]::OpenRead($File.FullName) - $FileStream.CopyTo($CryptoStream) -} - -# Close all files and close out the cryptographic stream. -$CryptoStream.FlushFinalBlock() - -# Combine all of the hashes as hexadecimal formats "X2" and join the values. -($Algorithm.Hash | ForEach-Object {$_.ToString("X2")}) -join '' diff --git a/.github/actions/sha-of-zip.bat b/.github/actions/sha-of-zip.bat index a02625096..5ccc7262d 100644 --- a/.github/actions/sha-of-zip.bat +++ b/.github/actions/sha-of-zip.bat @@ -11,8 +11,6 @@ setlocal EnableExtensions EnableDelayedExpansion :: All paths relative to pwd -- where this script is called from set "EXTRACTIONDIR=.\rcv\zip_extracted" -set "MODULESFILE=.\rcv\zip_extracted\rcv\lib\modules" -set "MODULESDIR=.\rcv\zip_extracted\rcv\lib\modules_extracted" set "HASHFILE_UNSORTED=all_hashes_unsorted.txt" set "HASHFILE_PATH_STRIPPED=all_hashes_path_stripped.txt" set "HASHFILE_SORTED=all_hashes_sorted.txt" @@ -21,24 +19,19 @@ if exist %HASHFILE_UNSORTED% ( del %HASHFILE_UNSORTED% ) -if exist %EXTRACTIONDIR% ( - rmdir /s /q %EXTRACTIONDIR% +if exist %HASHFILE_PATH_STRIPPED% ( + del %HASHFILE_PATH_STRIPPED% ) -if exist %MODULESDIR% ( - rmdir /s /q %EXTRACTIONDIR% +if exist %HASHFILE_SORTED% ( + del %HASHFILE_SORTED% ) - -powershell -command Expand-Archive -Path %ZIP_FILEPATH% -Destination %EXTRACTIONDIR% - -:: Extract modules, get the SHA-512 of it, and add it to the top of the hashfile -jimage extract --dir %MODULESDIR% %MODULESFILE% -powershell .github\actions\Sha-Of-Directory.ps1 %MODULESDIR% 512 > %HASHFILE_UNSORTED% +if exist %EXTRACTIONDIR% ( + rmdir /s /q %EXTRACTIONDIR% +) -:: Delete both modules and the extracted dir -del %MODULESFILE% -rmdir /s /q %MODULESDIR% +powershell -command Expand-Archive -Path %ZIP_FILEPATH% -Destination %EXTRACTIONDIR% :: Calculate the hash for every file here and in all subdirectories, appending to the file (format "(filename) = (hash)") ( @@ -59,6 +52,10 @@ for /f "delims=" %%A in ('type "%HASHFILE_UNSORTED%"') do ( sort "%HASHFILE_PATH_STRIPPED%" > "%HASHFILE_SORTED%" +:: dos2unix on the file to ensure consistent SHAs +powershell -Command "& {[IO.File]::WriteAllText(\"%HASHFILE_SORTED%\", $([IO.File]::ReadAllText(\"%HASHFILE_SORTED%\") -replace \"`r`n", "`n\"))}" + +:: echo the final hash C:\Windows\System32\certutil.exe -hashfile %HASHFILE_SORTED% SHA%SHA_A% | findstr /v ":" :: For debugging, enable printing the file-by-file hash diff --git a/.github/actions/sha-of-zip.sh b/.github/actions/sha-of-zip.sh index d4bfa9f42..38e5d11f6 100755 --- a/.github/actions/sha-of-zip.sh +++ b/.github/actions/sha-of-zip.sh @@ -11,7 +11,6 @@ os=$2 sha_a=$3 parentPath=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P ) -modulesPath="./rcv/lib/modules" # Make a temporary directory to extract zip, and a temporary file to hold SHAs tempDirectory=$(mktemp -d) @@ -21,12 +20,8 @@ touch $tempAllChecksumsFile # Extract the zip unzip -q $zipFilepath -d $tempDirectory 2>/dev/null -# Go into the extracted directory and delete the modules file -# That file is consistent on a single machine, but differs across machines -cd $tempDirectory -rm $modulesPath - # Get a checksum for each file in the zip +cd $tempDirectory for filename in $(find * -type f | sort); do checksum=$($parentPath/../workflows/sha.sh $filename $os $sha_a) echo $checksum >> $tempAllChecksumsFile diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index abe14b239..cacbefc1a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -169,7 +169,6 @@ jobs: name: Package if-no-files-found: error path: | - ${{ github.workspace }}/all_hashes_sorted.txt ${{ github.workspace }}/${{ steps.zipfn.outputs.FILEPATH }} ${{ github.workspace }}/${{ steps.zipfn.outputs.FILEPATH }}.sha512 ${{ github.workspace }}/${{ steps.zipfn.outputs.FILEPATH }}.golden.sha512 @@ -177,7 +176,6 @@ jobs: ${{ github.workspace }}/${{ steps.exefn.outputs.FILEPATH }}.sha512 ${{ github.workspace }}/${{ steps.cachefn.outputs.FILEPATH }} ${{ github.workspace }}/${{ steps.cachefn.outputs.FILEPATH }}.sha512 - ${{ github.workspace }}/${{ steps.cachefn.outputs.FILEPATH }}.golden.sha512 ${{ github.workspace }}/${{ steps.checksumsfn.outputs.FILEPATH }} ${{ github.workspace }}/${{ steps.checksumsfn.outputs.FILEPATH }}.sha512 retention-days: 1