-
Notifications
You must be signed in to change notification settings - Fork 24
Yolo output #1320
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Yolo output #1320
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -37,6 +37,21 @@ public function rules() | |
| { | ||
| return array_merge(parent::rules(), [ | ||
| 'type_id' => 'required|integer|exists:report_types,id', | ||
| 'yolo_image_path' => 'nullable|string', | ||
| 'yolo_split_ratio' => 'nullable|string|regex:/^\d+(\.\d+)? \d+(\.\d+)? \d+(\.\d+)?$/', | ||
| ]); | ||
| } | ||
|
|
||
| /** | ||
| * Get the options for the new report. | ||
| * | ||
| * @return array | ||
| */ | ||
| public function getOptions() | ||
| { | ||
| return array_merge(parent::getOptions(), [ | ||
| 'yoloImagePath' => $this->input('yolo_image_path'), | ||
| 'yoloSplitRatio' => $this->input('yolo_split_ratio'), | ||
|
Comment on lines
+40
to
+54
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Add this to the |
||
| ]); | ||
| } | ||
|
|
||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -42,6 +42,7 @@ public function rules() | |
| ReportType::imageAnnotationsCsvId(), | ||
| ReportType::imageAnnotationsExtendedId(), | ||
| ReportType::imageAnnotationsCocoId(), | ||
| ReportType::imageAnnotationsYoloId(), | ||
| ReportType::imageAnnotationsFullId(), | ||
| ReportType::imageAnnotationsAbundanceId(), | ||
| ReportType::imageAnnotationsImageLocationId(), | ||
|
|
@@ -62,6 +63,8 @@ public function rules() | |
| return array_merge(parent::rules(), [ | ||
| 'type_id' => ['required', Rule::in($types)], | ||
| 'annotation_session_id' => "nullable|integer|exists:annotation_sessions,id,volume_id,{$this->volume->id}", | ||
| 'yolo_image_path' => 'nullable|string', | ||
| 'yolo_split_ratio' => 'nullable|string|regex:/^\d+(\.\d+)? \d+(\.\d+)? \d+(\.\d+)?$/', | ||
|
Comment on lines
+66
to
+67
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. See above. |
||
| ]); | ||
| } | ||
|
|
||
|
|
@@ -119,8 +122,13 @@ public function withValidator($validator) | |
| */ | ||
| public function getOptions() | ||
| { | ||
| return array_merge(parent::getOptions(), [ | ||
| \Log::info('StoreVolumeReport input:', $this->all()); | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Please don't use |
||
| $options = array_merge(parent::getOptions(), [ | ||
| 'annotationSession' => $this->input('annotation_session_id'), | ||
| 'yoloImagePath' => $this->input('yolo_image_path'), | ||
| 'yoloSplitRatio' => $this->input('yolo_split_ratio'), | ||
|
Comment on lines
+128
to
+129
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. See above. |
||
| ]); | ||
| \Log::info('StoreVolumeReport options:', $options); | ||
| return $options; | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
| @@ -0,0 +1,29 @@ | ||||||
| <?php | ||||||
|
|
||||||
| namespace Biigle\Services\Reports\Projects\ImageAnnotations; | ||||||
|
|
||||||
| use Biigle\Services\Reports\Volumes\ImageAnnotations\YoloReportGenerator as ReportGenerator; | ||||||
|
|
||||||
| class YoloReportGenerator extends AnnotationReportGenerator | ||||||
| { | ||||||
| /** | ||||||
| * The class of the volume report to use for this project report. | ||||||
| * | ||||||
| * @var string | ||||||
| */ | ||||||
| protected $reportClass = ReportGenerator::class; | ||||||
|
|
||||||
| /** | ||||||
| * Name of the report for use in text. | ||||||
| * | ||||||
| * @var string | ||||||
| */ | ||||||
| public $name = 'Yolo image annotation report'; | ||||||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||
|
|
||||||
| /** | ||||||
| * Name of the report for use as (part of) a filename. | ||||||
| * | ||||||
| * @var string | ||||||
| */ | ||||||
| public $filename = 'yolo_image_annotation_report'; | ||||||
| } | ||||||
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
| @@ -0,0 +1,219 @@ | ||||||
| <?php | ||||||
|
|
||||||
| namespace Biigle\Services\Reports\Volumes\ImageAnnotations; | ||||||
|
|
||||||
| use Biigle\LabelTree; | ||||||
| use Biigle\Services\Reports\CsvFile; | ||||||
| use Biigle\Services\Reports\MakesZipArchives; | ||||||
| use Biigle\User; | ||||||
| use DB; | ||||||
|
|
||||||
| class YoloReportGenerator extends AnnotationReportGenerator | ||||||
| { | ||||||
| use MakesZipArchives; | ||||||
| /** | ||||||
| * Name of the report for use in text. | ||||||
| * | ||||||
| * @var string | ||||||
| */ | ||||||
| public $name = 'yolo image annotation report'; | ||||||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||
|
|
||||||
| /** | ||||||
| * Name of the report for use as (part of) a filename. | ||||||
| * | ||||||
| * @var string | ||||||
| */ | ||||||
| public $filename = 'yolo_image_annotation_report'; | ||||||
|
|
||||||
| /** | ||||||
| * File extension of the report file. | ||||||
| * | ||||||
| * @var string | ||||||
| */ | ||||||
| public $extension = 'zip'; | ||||||
|
|
||||||
| /** | ||||||
| * Generate the report. | ||||||
| * | ||||||
| * @param string $path Path to the report file that should be generated | ||||||
| */ | ||||||
| public function generateReport($path) | ||||||
| { | ||||||
| $rows = $this->query()->get(); | ||||||
|
|
||||||
| // Always create a single unified dataset for YOLO | ||||||
| $csv = $this->createCsv($rows); | ||||||
| $this->tmpFiles[] = $csv; | ||||||
|
|
||||||
| $this->executeScript('to_yolo', $path); | ||||||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Please reuse the existing |
||||||
|
|
||||||
| // Python script creates output directory based on the first CSV file path | ||||||
| // Collect all files from that directory for zipping | ||||||
| $firstCsvPath = $this->tmpFiles[0]->getPath(); | ||||||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Since you push |
||||||
| $outputDir = pathinfo($firstCsvPath, PATHINFO_DIRNAME) . '/' . pathinfo($firstCsvPath, PATHINFO_FILENAME) . '_yolo_output'; | ||||||
|
|
||||||
|
|
||||||
| // Clear the toZip array and rebuild it with files from output directory | ||||||
| $toZip = []; | ||||||
| if (is_dir($outputDir)) { | ||||||
| $this->addDirectoryToZip($outputDir, $toZip); | ||||||
| } else { | ||||||
| \Log::warning("YOLO: Output directory not found", ['outputDir' => $outputDir]); | ||||||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Shouldn't this be an exception? |
||||||
| } | ||||||
|
|
||||||
| $this->makeZip($toZip, $path); | ||||||
| } | ||||||
|
|
||||||
| /** | ||||||
| * Override makeZip to handle symlinks properly | ||||||
| */ | ||||||
| protected function makeZip($files, $path) | ||||||
| { | ||||||
| $zip = \App::make(\ZipArchive::class); | ||||||
| $open = $zip->open($path, \ZipArchive::OVERWRITE); | ||||||
|
|
||||||
| if ($open !== true) { | ||||||
| throw new \Exception("Could not open ZIP file '{$path}'."); | ||||||
| } | ||||||
|
|
||||||
| try { | ||||||
| foreach ($files as $source => $target) { | ||||||
| // Check if file is a symlink | ||||||
| if (is_link($source)) { | ||||||
| // Add symlink to zip | ||||||
| $linkTarget = readlink($source); | ||||||
| $zip->addFromString($target, $linkTarget); | ||||||
| // Set external attributes to mark as symlink | ||||||
| $zip->setExternalAttributesName($target, \ZipArchive::OPSYS_UNIX, 0120777 << 16); | ||||||
|
Comment on lines
+84
to
+87
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This looks like it reads the contents of the symlink target and puts it to the ZIP before it does some arcane magic? This doesn't look right, as we should not have the actual target of the symlink, right? Is the arcane magic documented somewhere? |
||||||
| } else { | ||||||
| // Regular file | ||||||
| $zip->addFile($source, $target); | ||||||
| } | ||||||
| } | ||||||
| } finally { | ||||||
| $zip->close(); | ||||||
| } | ||||||
| } | ||||||
|
|
||||||
| /** | ||||||
| * Override executeScript to pass YOLO-specific arguments before CSV files | ||||||
| */ | ||||||
| protected function executeScript($scriptName, $path) | ||||||
| { | ||||||
| $imagePath = $this->options->get('yoloImagePath', ''); | ||||||
| $splitRatio = $this->options->get('yoloSplitRatio', '0.7 0.2 0.1'); | ||||||
|
|
||||||
| // Call parent's pythonScriptRunner directly with custom arguments | ||||||
| // Command format: python script.py volumeName path imagePath splitRatio csv1 csv2 ... | ||||||
| $python = config('reports.python'); | ||||||
| $script = config("reports.scripts.{$scriptName}"); | ||||||
| $csvs = implode(' ', array_map(fn ($csv) => $csv->getPath(), $this->tmpFiles)); | ||||||
|
|
||||||
| $command = sprintf( | ||||||
| "%s %s \"%s\" %s %s %s %s 2>&1", | ||||||
| $python, | ||||||
| $script, | ||||||
| $this->source->name, | ||||||
| $path, | ||||||
| escapeshellarg($imagePath), | ||||||
| escapeshellarg($splitRatio), | ||||||
| $csvs | ||||||
| ); | ||||||
|
|
||||||
| exec($command, $lines, $code); | ||||||
|
|
||||||
| if ($code !== 0) { | ||||||
| throw new \Exception("The report script '{$scriptName}' failed with exit code {$code}:\n".implode("\n", $lines)); | ||||||
| } | ||||||
| } | ||||||
|
|
||||||
| /** | ||||||
| * Recursively add all files from a directory to the zip array | ||||||
| */ | ||||||
| protected function addDirectoryToZip($dir, &$toZip, $basePath = '') | ||||||
| { | ||||||
| $items = new \DirectoryIterator($dir); | ||||||
| foreach ($items as $item) { | ||||||
| if ($item->isDot()) continue; | ||||||
|
|
||||||
| $relativePath = $basePath ? $basePath . '/' . $item->getFilename() : $item->getFilename(); | ||||||
|
|
||||||
| if ($item->isDir()) { | ||||||
| $this->addDirectoryToZip($item->getPathname(), $toZip, $relativePath); | ||||||
| } else { | ||||||
| $toZip[$item->getPathname()] = $relativePath; | ||||||
| } | ||||||
| } | ||||||
| } | ||||||
|
|
||||||
| /** | ||||||
| * Assemble a new DB query for the volume of this report. | ||||||
| * | ||||||
| * @return \Illuminate\Database\Query\Builder | ||||||
| */ | ||||||
| protected function query() | ||||||
| { | ||||||
| $query = $this | ||||||
| ->initQuery([ | ||||||
| 'image_annotation_labels.id as annotation_label_id', | ||||||
| 'image_annotation_labels.label_id', | ||||||
| 'labels.name as label_name', | ||||||
| 'users.id as user_id', | ||||||
| 'images.id as image_id', | ||||||
| 'images.filename', | ||||||
| 'images.lng as longitude', | ||||||
| 'images.lat as latitude', | ||||||
| 'shapes.name as shape_name', | ||||||
| 'image_annotations.points', | ||||||
| 'images.attrs', | ||||||
| ]) | ||||||
| ->join('shapes', 'image_annotations.shape_id', '=', 'shapes.id') | ||||||
| ->leftJoin('users', 'image_annotation_labels.user_id', '=', 'users.id') | ||||||
| ->orderBy('image_annotation_labels.id'); | ||||||
|
|
||||||
| return $query; | ||||||
| } | ||||||
|
|
||||||
| /** | ||||||
| * Create a CSV file for this report. | ||||||
| * | ||||||
| * @param \Illuminate\Support\Collection $rows The rows for the CSV | ||||||
| * @return CsvFile | ||||||
| */ | ||||||
| protected function createCsv($rows) | ||||||
| { | ||||||
| $csv = CsvFile::makeTmp(); | ||||||
| // column headers | ||||||
| $csv->putCsv([ | ||||||
| 'annotation_label_id', | ||||||
| 'label_id', | ||||||
| 'label_name', | ||||||
| 'image_id', | ||||||
| 'filename', | ||||||
| 'image_longitude', | ||||||
| 'image_latitude', | ||||||
| 'shape_name', | ||||||
| 'points', | ||||||
| 'attributes', | ||||||
| ]); | ||||||
|
|
||||||
| foreach ($rows as $row) { | ||||||
| $csv->putCsv([ | ||||||
| $row->annotation_label_id, | ||||||
| $row->label_id, | ||||||
| $row->label_name, | ||||||
| $row->image_id, | ||||||
| $row->filename, | ||||||
| $row->longitude, | ||||||
| $row->latitude, | ||||||
| $row->shape_name, | ||||||
| $row->points, | ||||||
| $row->attrs, | ||||||
| ]); | ||||||
| } | ||||||
|
|
||||||
| $csv->close(); | ||||||
|
|
||||||
| return $csv; | ||||||
| } | ||||||
| } | ||||||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| <?php | ||
|
|
||
| use Illuminate\Database\Migrations\Migration; | ||
| use Illuminate\Support\Facades\DB; | ||
|
|
||
| class AddYoloReportType extends Migration | ||
| { | ||
| /** | ||
| * Run the migrations. | ||
| * | ||
| * @return void | ||
| */ | ||
| public function up() | ||
| { | ||
| DB::table('report_types')->insert([ | ||
| 'name' => 'ImageAnnotations\Yolo', | ||
| ]); | ||
| } | ||
|
|
||
| /** | ||
| * Reverse the migrations. | ||
| * | ||
| * @return void | ||
| */ | ||
| public function down() | ||
| { | ||
| DB::table('report_types')->where('name', 'ImageAnnotations\Yolo')->delete(); | ||
| } | ||
| } |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Why not store the split ratio in 3 different variables? No joining/splitting and complicated validation needed.