Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Check images before creating maia annotations #171

Merged
merged 7 commits into from
Jun 19, 2024
Merged

Check images before creating maia annotations #171

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
2d882f6
Fix error with removed images
lehecht May 23, 2024
5669f68
Add test
lehecht Jun 4, 2024
fb7ed27
Remove only maia annotations of deleted images
lehecht Jun 7, 2024
987fc71
Revert "Remove only maia annotations of deleted images"
lehecht Jun 19, 2024
5429834
Use single query to check annotations
lehecht Jun 19, 2024
14832eb
Simplify test
lehecht Jun 19, 2024
7acd0d0
Simplify test some more
mzur Jun 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions src/Jobs/DetectionJob.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Queue;
use Biigle\Image;
use Illuminate\Support\Arr;

abstract class DetectionJob implements ShouldQueue
{
Expand Down Expand Up @@ -217,6 +219,26 @@ protected function createMaiaAnnotations(array $annotations)
return $this->createMaiaAnnotation($annotation);
}, $annotations);

// Remove maia annotations of deleted images
$deletedImgId = [];
$maiaAnnotations = Arr::where($maiaAnnotations, function ($a) use ($deletedImgId) {
$imgId = $a['image_id'];
if(Arr::has($deletedImgId, $imgId)) {
if($deletedImgId[$imgId]) {
return false;
}
} else {
$isDeleted = Image::where('id', '=', $imgId)->doesntExist();
if($isDeleted) {
$deletedImgId[$imgId] = true;
return false;
} else {
$deletedImgId[$imgId] = false;
}
}
return true;
});

mzur marked this conversation as resolved.
Show resolved Hide resolved
// Chunk the insert because PDO's maximum number of query parameters is
// 65535. Each annotation has 7 parameters so we can store roughly 9000
// annotations in one call.
Expand Down
117 changes: 117 additions & 0 deletions tests/Jobs/ObjectDetectionTest.php
View file
Open in desktop
mzur marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
use Illuminate\Support\Facades\Queue;
use Str;
use TestCase;
use Biigle\Image;

class ObjectDetectionTest extends TestCase
{
Expand Down Expand Up @@ -128,6 +129,112 @@ public function testHandle()
}
}

public function testDeletedImage()
{
FileCache::fake();
config([
'maia.mmdet_train_batch_size' => 12,
'maia.max_workers' => 2,
]);

$params = [
'batch_size' => 12,
'max_workers' => 2,
];

$job = MaiaJobTest::create(['params' => $params]);
$image = ImageTest::create(['volume_id' => $job->volume_id]);
$image2 = ImageTest::create(['volume_id' => $job->volume_id, 'filename' => 'a']);
$trainingProposal = TrainingProposalTest::create([
'job_id' => $job->id,
'image_id' => $image->id,
'points' => [10.5, 20.4, 30],
'selected' => true,
]);
// Not selected and should not be included.
TrainingProposalTest::create([
'job_id' => $job->id,
'image_id' => $image->id,
'selected' => false,
]);
config(['maia.tmp_dir' => '/tmp']);
$tmpDir = "/tmp/maia-{$job->id}-object-detection";
$datasetInputJsonPath = "{$tmpDir}/input-dataset.json";
$datasetOutputJsonPath = "{$tmpDir}/output-dataset.json";
$trainingInputJsonPath = "{$tmpDir}/input-training.json";
$trainingOutputJsonPath = "{$tmpDir}/output-training.json";
$inferenceInputJsonPath = "{$tmpDir}/input-inference.json";

$expectDatasetJson = [
'max_workers' => 2,
'tmp_dir' => $tmpDir,
'training_proposals' => [$image->id => [[11, 20, 30]]],
'output_path' => "{$tmpDir}/output-dataset.json",
];

$expectTrainingJson = [
'max_workers' => 2,
'batch_size' => 12,
'tmp_dir' => $tmpDir,
'output_path' => "{$tmpDir}/output-training.json",
'base_config' => config('maia.mmdet_base_config'),
'backbone_model_path' => config('maia.backbone_model_path'),
'model_path' => config('maia.model_path'),
];

$expectInferenceJson = [
'max_workers' => 2,
'tmp_dir' => $tmpDir,
];

try {
$request = new OdJobStub($job);
$request->deleteImage = true;
$request->handle();
$this->assertTrue(File::isDirectory($tmpDir));

$this->assertTrue(File::exists($datasetInputJsonPath));
$inputJson = json_decode(File::get($datasetInputJsonPath), true);
$this->assertArrayHasKey('images', $inputJson);
$this->assertArrayHasKey($image->id, $inputJson['images']);
$this->assertArrayNotHasKey($image2->id, $inputJson['images']);
unset($inputJson['images']);
$this->assertEquals($expectDatasetJson, $inputJson);
$this->assertStringContainsString("DatasetGenerator.py {$datasetInputJsonPath}", $request->commands[0]);

$this->assertTrue(File::exists($trainingInputJsonPath));
$inputJson = json_decode(File::get($trainingInputJsonPath), true);
$this->assertEquals($expectTrainingJson, $inputJson);
$this->assertStringContainsString("TrainingRunner.py {$trainingInputJsonPath} {$datasetOutputJsonPath}", $request->commands[1]);

$this->assertTrue(File::exists($inferenceInputJsonPath));
$inputJson = json_decode(File::get($inferenceInputJsonPath), true);
$this->assertArrayHasKey('images', $inputJson);
$this->assertArrayHasKey($image->id, $inputJson['images']);
$this->assertArrayHasKey($image2->id, $inputJson['images']);
unset($inputJson['images']);
$this->assertEquals($expectInferenceJson, $inputJson);
$this->assertStringContainsString("InferenceRunner.py {$inferenceInputJsonPath} {$datasetOutputJsonPath} {$trainingOutputJsonPath}", $request->commands[2]);

$this->assertEquals(State::annotationCandidatesId(), $job->fresh()->state_id);

$annotations = $job->annotationCandidates()->get();
// One annotation for each image.
$this->assertEquals(1, $annotations->count());
$this->assertEquals([10, 20, 30], $annotations[0]->points);
$this->assertEquals(123, $annotations[0]->score);
$this->assertNull($annotations[0]->label_id);
$this->assertNull($annotations[0]->annotation_id);
$this->assertEquals($image2->id, $annotations[0]->image_id);
$this->assertEquals(Shape::circleId(), $annotations[0]->shape_id);

$this->assertTrue($request->cleanup);

} finally {
File::deleteDirectory($tmpDir);
}
}

public function testHandleKnowledgeTransfer()
{
FileCache::fake();
Expand Down Expand Up @@ -383,6 +490,7 @@ class OdJobStub extends ObjectDetection
public $commands = [];
public $cleanup = false;
public $crash = false;
public $deleteImage = false;

protected function maybeDownloadWeights($from, $to)
{
Expand Down Expand Up @@ -418,4 +526,13 @@ protected function updateJobState()

return parent::updateJobState();
}

public function createMaiaAnnotations($annotations)
{
// Simulate image removal after saving its annotations
if ($this->deleteImage && !empty($annotations)) {
Image::where('id', '=', $annotations[0][0])->delete();
}
parent::createMaiaAnnotations($annotations);
}
}