Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

skip functional tests for models that are not downloaded. #48

Merged
merged 6 commits into from
Mar 9, 2024
Merged
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 53 additions & 0 deletions Tests/WhisperKitTests/UnitTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -827,6 +827,24 @@ final class UnitTests: XCTestCase {
XCTAssertEqual(mergedAlignmentTiming[i].probability, expectedWordTimings[i].probability, "Probability at index \(i) does not match")
}
}

func testGitLFSPointerFile() {
// Assumption:
// 1 - the openai_whisper-tiny is downloaded locally. This means that the proxyFile is an actual data file.
// 2 - the openai_whisper-large-v3_turbo is not downloaded locally. This means that the proxyFile is pointer file.
let proxyFile = "AudioEncoder.mlmodelc/coremldata.bin"

// First, we check that a data file is not considered a git lfs pointer file.
var filePath = URL(filePath: tinyModelPath()).appending(path: proxyFile)
var isPointerFile = isGitLFSPointerFile(url: filePath)
XCTAssertEqual(isPointerFile, false, "Assuming whisper-tiny was downloaded, \(proxyFile) should not be a git-lfs pointer file.")

// Second, we check that a pointer file is considered so.
let modelDir = largev3TurboModelPath()
filePath = URL(filePath: modelDir).appending(path: proxyFile)
isPointerFile = isGitLFSPointerFile(url: filePath)
XCTAssertEqual(isPointerFile, true, "Assuming whisper-large-v3_turbo was not downloaded, \(proxyFile) should be a git-lfs pointer file.")
}
}

// MARK: Helpers
Expand Down Expand Up @@ -904,6 +922,15 @@ extension XCTestCase {
return modelPath
}

func largev3TurboModelPath() -> String {
let modelDir = "whisperkit-coreml/openai_whisper-large-v3_turbo"
guard let modelPath = Bundle.module.urls(forResourcesWithExtension: "mlmodelc", subdirectory: modelDir)?.first?.deletingLastPathComponent().path else {
print("Failed to load model, ensure \"Models/\(modelDir)\" exists via Makefile command: `make download-models`")
return ""
}
return modelPath
}

func allModelPaths() -> [String] {
let fileManager = FileManager.default
var modelPaths: [String] = []
Expand All @@ -921,6 +948,13 @@ extension XCTestCase {
for folderURL in directoryContents {
let resourceValues = try folderURL.resourceValues(forKeys: Set(resourceKeys))
if resourceValues.isDirectory == true {
// Check if the directory contains actual data files, or if it contains pointer files.
// As a proxy, use the MelSpectrogramc.mlmodel/coredata.bin file.
let proxyFileToCheck = folderURL.appendingPathComponent("MelSpectrogram.mlmodelc/coremldata.bin")
if isGitLFSPointerFile(url: proxyFileToCheck) {
continue
}

// Check if the directory name contains the quantization pattern
// Only test large quantized models
let dirName = folderURL.lastPathComponent
Expand All @@ -935,6 +969,25 @@ extension XCTestCase {

return modelPaths
}

// Function to check if the beginning of the file matches a Git LFS pointer pattern
func isGitLFSPointerFile(url: URL) -> Bool {
do {
let fileHandle = try FileHandle(forReadingFrom: url)
// Read the first few bytes of the file to get enough for the Git LFS pointer signature
let data = fileHandle.readData(ofLength: 512) // Read first 512 bytes
fileHandle.closeFile()

if let string = String(data: data, encoding: .utf8),
string.starts(with: "version https://git-lfs.github.com/") {
return true
}
} catch {
print("Failed to read file: \(error)")
}

return false
}

func trackForMemoryLeaks(on instance: AnyObject, file: StaticString = #filePath, line: UInt = #line) {
addTeardownBlock { [weak instance] in
Expand Down