Eval tab fix (#880)

* averaging method is now correctly selected for binary classification and thus not inflating the recall/precision values

* added warning when prediction files have no matching annotations

* ruff fixed

* .

* limited perch version for tests

* .

Author: max-mauermann

.github/workflows/lint.yml

@@ -21,7 +21,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install ruff
+          pip install .[dev]
       - name: Lint with Ruff
         run: |
           ruff check

birdnet_analyzer/evaluation/assessment/performance_assessor.py

@@ -123,10 +123,7 @@ def calculate_metrics(
             raise ValueError(f"The number of columns in predictions ({predictions.shape[1]}) " + f"must match num_classes ({self.num_classes}).")
 
         # Determine the averaging method for metrics
-        if per_class_metrics and self.num_classes == 1:
-            averaging_method = "macro"
-        else:
-            averaging_method = None if per_class_metrics else "macro"
+        averaging_method = None if per_class_metrics or self.num_classes == 1 else "macro"
 
         # Dictionary to store the results of each metric
         metrics_results = {}

birdnet_analyzer/evaluation/preprocessing/data_processor.py

@@ -100,6 +100,9 @@ def __init__(
         self.predictions_df: pd.DataFrame = pd.DataFrame()
         self.annotations_df: pd.DataFrame = pd.DataFrame()
 
+        # To track prediction files without matching annotation files
+        self.umatched_prediction_files: set[str] = set()
+
         # Placeholder for unique classes across predictions and annotations
         self.classes: tuple[str, ...] = ()
 
@@ -173,6 +176,9 @@ def load_data(self) -> None:
         Raises:
             ValueError: If file reading fails or data preparation encounters issues.
         """
+
+        self.umatched_prediction_files = set()  # Reset unmatched prediction files
+
         if self.prediction_file_name is None or self.annotation_file_name is None:
             # Case: No specific files provided; load all files in directories.
             self.predictions_df = read_and_concatenate_files_in_directory(self.prediction_directory_path)
@@ -193,6 +199,16 @@ def load_data(self) -> None:
             if self.class_mapping:
                 class_col_pred = self.get_column_name("Class", prediction=True)
                 self.predictions_df[class_col_pred] = self.predictions_df[class_col_pred].apply(lambda x: self.class_mapping.get(x, x))
+
+            prediction_filenames = self.predictions_df["recording_filename"].unique()
+            annotation_filenames = self.annotations_df["recording_filename"].unique()
+            self.umatched_prediction_files = set(prediction_filenames) - set(annotation_filenames)
+            if self.umatched_prediction_files:
+                warnings.warn(
+                    f"Prediction files without matching annotation files: {', '.join(self.umatched_prediction_files)}. "
+                    "These recordings will not be processed.",
+                    stacklevel=2,
+                )
         else:
             # Case: Specific files are provided for predictions and annotations.
             # Ensure filenames correspond to the same recording (heuristic check).

birdnet_analyzer/gui/evaluation.py

@@ -240,6 +240,13 @@ def initialize_processor(
             avail_classes = list(proc.classes)  # Ensure it's a list
             avail_recordings = proc.samples_df["filename"].unique().tolist()
 
+            if len(proc.umatched_prediction_files) > 0:
+                gr.Warning(
+                    f"{loc.localize('eval-tab-warning-unmatched-predictions')}: "
+                    f"{', '.join(proc.umatched_prediction_files)}. "
+                    f"{loc.localize('eval-tab-warning-unmatched-predictions-info')}"
+                )
+
             return avail_classes, avail_recordings, proc, annotation_dir, prediction_dir
         except KeyError as e:
             print(f"Column missing in files: {e}")

birdnet_analyzer/lang/de.json

@@ -117,6 +117,8 @@
     "eval-tab-title": "Auswertung",
     "eval-tab-upload-mapping-file-label": "Zuordnungsdatei hochladen",
     "eval-tab-warning-error-reading-file": "Fehler beim Lesen der Datei",
+    "eval-tab-warning-unmatched-predictions": "Folgende Vorhersagedateien haben keine entsprechende Annotationsdatei",
+    "eval-tab-warning-unmatched-predictions-info": "für diese Dateien werden die Annotationen als leer angenommen",
     "footer-help": "Dokumentation und Support finden Sie unter",
     "footer-support": "Unterstützen Sie unsere Arbeit",
     "inference-settings-accordion-label": "Inferenzeinstellungen",

birdnet_analyzer/lang/en.json

@@ -117,6 +117,8 @@
     "eval-tab-title": "Evaluation",
     "eval-tab-upload-mapping-file-label": "Upload mapping file",
     "eval-tab-warning-error-reading-file": "Error reading file",
+    "eval-tab-warning-unmatched-predictions": "Following prediction files do not have a corresponding annotation file",
+    "eval-tab-warning-unmatched-predictions-info": "for these files, the annotations will assumed to be empty",
     "footer-help": "For docs and support visit",
     "footer-support": "Support our work",
     "inference-settings-accordion-label": "Inference settings",

birdnet_analyzer/lang/fi.json

@@ -117,6 +117,8 @@
     "eval-tab-title": "Arviointi",
     "eval-tab-upload-mapping-file-label": "Lataa kartoitustiedosto",
     "eval-tab-warning-error-reading-file": "Virhe luettaessa tiedostoa",
+    "eval-tab-warning-unmatched-predictions": "Seuraavilla ennustetiedostoilla ei ole vastaavaa annotaatiotiedostoa",
+    "eval-tab-warning-unmatched-predictions-info": "näiden tiedostojen annotaatioiden oletetaan olevan tyhjiä",
     "footer-help": "Dokumentaatio ja tuki osoitteessa",
     "footer-support": "Tue työtämme",
     "inference-settings-accordion-label": "Päättelyasetukset",

birdnet_analyzer/lang/fr.json

@@ -117,6 +117,8 @@
     "eval-tab-title": "Évaluation",
     "eval-tab-upload-mapping-file-label": "Téléverser le fichier de correspondance",
     "eval-tab-warning-error-reading-file": "Erreur lors de la lecture du fichier",
+    "eval-tab-warning-unmatched-predictions": "Les fichiers de prédiction suivants n'ont pas de fichier d'annotation correspondant",
+    "eval-tab-warning-unmatched-predictions-info": "pour ces fichiers, les annotations seront supposées vides",
     "footer-help": "Pour de la documentation, visiter",
     "footer-support": "Soutenez notre travail",
     "inference-settings-accordion-label": "Paramètres d'inférence",

birdnet_analyzer/lang/id.json

@@ -117,6 +117,8 @@
     "eval-tab-title": "Evaluasi",
     "eval-tab-upload-mapping-file-label": "Unggah file pemetaan",
     "eval-tab-warning-error-reading-file": "Error saat membaca file",
+    "eval-tab-warning-unmatched-predictions": "File prediksi berikut tidak memiliki file anotasi yang sesuai",
+    "eval-tab-warning-unmatched-predictions-info": "untuk file-file ini, anotasi akan dianggap kosong",
     "footer-help": "Untuk dokumen dan dukungan kunjungi",
     "footer-support": "Dukung Pekerjaan Kami",
     "inference-settings-accordion-label": "Pengaturan inferensi",

birdnet_analyzer/lang/pt-br.json

@@ -117,6 +117,8 @@
     "eval-tab-title": "Avaliação",
     "eval-tab-upload-mapping-file-label": "Carregar arquivo de mapeamento",
     "eval-tab-warning-error-reading-file": "Erro ao ler arquivo",
+    "eval-tab-warning-unmatched-predictions": "Os seguintes arquivos de previsão não possuem arquivo de anotação correspondente",
+    "eval-tab-warning-unmatched-predictions-info": "para esses arquivos, as anotações serão consideradas vazias",
     "footer-help": "Para documentos e suporte visite",
     "footer-support": "Apoie nosso trabalho",
     "inference-settings-accordion-label": "Configurações de inferência",