Training results: result_2.3.0

.github/workflows/train.yml

@@ -26,7 +26,7 @@ jobs:
         with:
           mode: start
           github-token: ${{ secrets.GH_PAT }}
-          ec2-image-id: ami-06c62c213d8fdc88f
+          ec2-image-id: ami-078da566f22c68a26
           ec2-instance-type: g4dn.xlarge
           subnet-id: subnet-0ed91f5fc60959086
           security-group-id: sg-05cd9ff586699e8db
@@ -74,7 +74,7 @@ jobs:
       - name: Download test dataset
         run: |
           uv run dvc get https://github.com/pyronear/pyro-dataset \
-            data/processed/yolo_test --rev v2.1.0 \
+            data/processed/yolo_test --rev v2.2.0 \
             --out ./data/test/yolo_test
 
       - name: Run evaluation
@@ -90,7 +90,7 @@ jobs:
       - name: Download sequential test dataset
         run: |
           uv run dvc get https://github.com/pyronear/pyro-dataset \
-            data/processed/sequential_test --rev v2.1.0 \
+            data/processed/sequential_test --rev v2.2.0 \
             --out ./data/test/sequential_test
 
       - name: Run sequential predictions on test
@@ -238,7 +238,7 @@ jobs:
           {opt_table}
 
           {baseline_note}
-          **Branch:** `{os.environ.get('RESULT_BRANCH', '')}` | **Test dataset:** pyronear/pyro-dataset @ v2.1.0""")
+          **Branch:** `{os.environ.get('RESULT_BRANCH', '')}` | **Test dataset:** pyronear/pyro-dataset @ v2.2.0""")
           PYEOF
           )
 

README.md

@@ -18,10 +18,10 @@ The Data pipeline is organized with a [dvc.yaml](./dvc.yaml) file.
 This section list and describes all the DVC stages that are defined in the
 [dvc.yaml](./dvc.yaml) file:
 
-- __fetch_model_input__: Download the `yolo_train_val` dataset from [pyro-dataset](https://github.com/pyronear/pyro-dataset) at `v2.1.0`.
+- __fetch_model_input__: Download the `yolo_train_val` dataset from [pyro-dataset](https://github.com/pyronear/pyro-dataset) at `v2.2.0`.
 - __train_yolo_best__: Train the best YOLO model on the full dataset.
 - __build_manifest_yolo_best__: Build the manifest.yaml file to attach with the model.
-- __fetch_sequential_val__: Download the sequential val dataset from [pyro-dataset](https://github.com/pyronear/pyro-dataset) at `v2.1.0`.
+- __fetch_sequential_val__: Download the sequential val dataset from [pyro-dataset](https://github.com/pyronear/pyro-dataset) at `v2.2.0`.
 - __predict_sequential_val__: Run per-frame YOLO predictions on the sequential val set and save label files.
 - __optimize_sequential_val__: Grid-search engine parameters (nb_consecutive_frames × conf_thresh) on val predictions and save top-20 results.
 - __export_yolo_best__: Export the best YOLO model to ONNX and NCNN formats.
@@ -346,7 +346,7 @@ The test set is not part of the CI pipeline, so predictions must be run locally.
 ```sh
 SSL_CERT_FILE=$(uv run python -c "import certifi; print(certifi.where())") \
 uv run dvc get https://github.com/pyronear/pyro-dataset \
-  data/processed/sequential_test --rev v2.1.0 \
+  data/processed/sequential_test --rev v2.2.0 \
   --out ./data/test/sequential_test
 ```
 

dvc.lock

@@ -61,8 +61,8 @@ stages:
     deps:
     - path: ./data/01_model_input/yolo_train_val
       hash: md5
-      md5: fcd56c8728d160e957bf0acd8049368d.dir
-      size: 1780235401
+      md5: 75db0a70ab0148ab356e958476f1a7f4.dir
+      size: 1780231374
       nfiles: 30387
     - path: ./scripts/data/model_input/build.py
       hash: md5
@@ -79,8 +79,8 @@ stages:
     outs:
     - path: ./data/02_models/yolo/best
       hash: md5
-      md5: be5d61086d3a2b2e68e8a5dd61b14901.dir
-      size: 44003643
+      md5: c70f19067a7a6724990411caed3dc1dd.dir
+      size: 43845162
       nfiles: 24
   build_manifest_yolo_best:
     cmd:
@@ -90,8 +90,8 @@ stages:
     deps:
     - path: ./data/02_models/yolo/best/
       hash: md5
-      md5: be5d61086d3a2b2e68e8a5dd61b14901.dir
-      size: 44003643
+      md5: c70f19067a7a6724990411caed3dc1dd.dir
+      size: 43845162
       nfiles: 24
     - path: ./scripts/model/yolo/build_manifest.py
       hash: md5
@@ -100,8 +100,8 @@ stages:
     outs:
     - path: ./data/03_reporting/yolo/best/
       hash: md5
-      md5: d16e4d5660095fd5a38280ebea098ddc.dir
-      size: 13490
+      md5: 3dc26cb39d456a5e9434b251dcbdfd36.dir
+      size: 13489
       nfiles: 1
   export_yolo_best@onnx-cpu:
     cmd:
@@ -111,8 +111,8 @@ stages:
     deps:
     - path: ./data/02_models/yolo/best/
       hash: md5
-      md5: be5d61086d3a2b2e68e8a5dd61b14901.dir
-      size: 44003643
+      md5: c70f19067a7a6724990411caed3dc1dd.dir
+      size: 43845162
       nfiles: 24
     - path: ./scripts/model/yolo/export.py
       hash: md5
@@ -121,8 +121,8 @@ stages:
     outs:
     - path: ./data/02_models/yolo-export/best/onnx/cpu
       hash: md5
-      md5: 0b4f67d18810a40cb4d783312a4bdca8.dir
-      size: 38512704
+      md5: 18dc03ee9ec79eade6bc10efd6cd01ad.dir
+      size: 38209690
       nfiles: 1
   export_yolo_best@onnx-mps:
     cmd:
@@ -153,8 +153,8 @@ stages:
     deps:
     - path: ./data/02_models/yolo/best/
       hash: md5
-      md5: be5d61086d3a2b2e68e8a5dd61b14901.dir
-      size: 44003643
+      md5: c70f19067a7a6724990411caed3dc1dd.dir
+      size: 43845162
       nfiles: 24
     - path: ./scripts/model/yolo/export.py
       hash: md5
@@ -163,8 +163,8 @@ stages:
     outs:
     - path: ./data/02_models/yolo-export/best/ncnn/cpu
       hash: md5
-      md5: e1493ed19c061ad3ec6b00caead444bd.dir
-      size: 38001888
+      md5: 04f3c5ad010a8d18798138ad9acda38d.dir
+      size: 38001906
       nfiles: 5
   export_yolo_best@ncnn-mps:
     cmd:
@@ -190,23 +190,23 @@ stages:
   fetch_model_input:
     cmd:
     - uv run dvc get https://github.com/pyronear/pyro-dataset 
-      data/processed/yolo_train_val --rev v2.1.0 --out 
+      data/processed/yolo_train_val --rev v2.2.0 --out 
       ./data/01_model_input/yolo_train_val || uv run dvc get 
       git@github.com:pyronear/pyro-dataset.git data/processed/yolo_train_val 
-      --rev v2.1.0 --out ./data/01_model_input/yolo_train_val
+      --rev v2.2.0 --out ./data/01_model_input/yolo_train_val
     outs:
     - path: ./data/01_model_input/yolo_train_val
       hash: md5
-      md5: fcd56c8728d160e957bf0acd8049368d.dir
-      size: 1780235401
+      md5: 75db0a70ab0148ab356e958476f1a7f4.dir
+      size: 1780231374
       nfiles: 30387
   fetch_sequential_val:
     cmd:
     - uv run dvc get https://github.com/pyronear/pyro-dataset 
-      data/processed/sequential_train_val/val --rev v2.1.0 --out 
+      data/processed/sequential_train_val/val --rev v2.2.0 --out 
       ./data/01_model_input/sequential_train_val/val || uv run dvc get 
       git@github.com:pyronear/pyro-dataset.git 
-      data/processed/sequential_train_val/val --rev v2.1.0 --out 
+      data/processed/sequential_train_val/val --rev v2.2.0 --out 
       ./data/01_model_input/sequential_train_val/val
     outs:
     - path: ./data/01_model_input/sequential_train_val/val
@@ -256,7 +256,7 @@ stages:
       nfiles: 12448
     - path: ./data/02_models/yolo/best/weights/best.pt
       hash: md5
-      md5: fea4ddc8d904fcc5ff11ad900e040837
+      md5: 682e150c2b2f4c1c8e7b4a51a28ff669
       size: 19225626
     - path: predict_sequential.py
       hash: md5
@@ -265,8 +265,8 @@ stages:
     outs:
     - path: ./data/03_reporting/sequential/predictions_labels_val
       hash: md5
-      md5: cbe0e8e4d58627ca6856371246d400c2.dir
-      size: 104904
+      md5: 63dd8fce1409654b5dce0115bd5e6cd8.dir
+      size: 104058
       nfiles: 3438
   optimize_sequential_val:
     cmd:
@@ -277,8 +277,8 @@ stages:
     deps:
     - path: ./data/03_reporting/sequential/predictions_labels_val
       hash: md5
-      md5: cbe0e8e4d58627ca6856371246d400c2.dir
-      size: 104904
+      md5: 63dd8fce1409654b5dce0115bd5e6cd8.dir
+      size: 104058
       nfiles: 3438
     - path: optimize_sequential.py
       hash: md5
@@ -287,9 +287,9 @@ stages:
     outs:
     - path: ./data/03_reporting/sequential/grid_search_val.tsv
       hash: md5
-      md5: 8bae210202dc4210162712779346d572
-      size: 1896
+      md5: 7796d6510c82cb1ba9d4f911369ce20b
+      size: 1905
     - path: ./data/03_reporting/sequential/grid_search_val_top20.tsv
       hash: md5
-      md5: 42f461234b5a48ca27bb28a46afb2cd0
-      size: 975
+      md5: c103f084d2b3e897068bb9ff55b749c5
+      size: 986

dvc.yaml

@@ -3,11 +3,11 @@ stages:
     cmd:
     - >-
       uv run dvc get https://github.com/pyronear/pyro-dataset
-      data/processed/yolo_train_val --rev v2.1.0
+      data/processed/yolo_train_val --rev v2.2.0
       --out ./data/01_model_input/yolo_train_val
       ||
       uv run dvc get git@github.com:pyronear/pyro-dataset.git
-      data/processed/yolo_train_val --rev v2.1.0
+      data/processed/yolo_train_val --rev v2.2.0
       --out ./data/01_model_input/yolo_train_val
     outs:
     - ./data/01_model_input/yolo_train_val
@@ -53,11 +53,11 @@ stages:
     cmd:
     - >-
       uv run dvc get https://github.com/pyronear/pyro-dataset
-      data/processed/sequential_train_val/val --rev v2.1.0
+      data/processed/sequential_train_val/val --rev v2.2.0
       --out ./data/01_model_input/sequential_train_val/val
       ||
       uv run dvc get git@github.com:pyronear/pyro-dataset.git
-      data/processed/sequential_train_val/val --rev v2.1.0
+      data/processed/sequential_train_val/val --rev v2.2.0
       --out ./data/01_model_input/sequential_train_val/val
     outs:
     - ./data/01_model_input/sequential_train_val/val

fetch_data.sh

@@ -2,26 +2,26 @@
 SSL_CERT_FILE=/Users/mateo/pyronear/vision/train/pyro-train/.venv/lib/python3.12/site-packages/certifi/cacert.pem \
 uv run dvc get https://github.com/pyronear/pyro-dataset \
   data/processed/sequential_train_val/train \
-  --rev v2.1.0 \
+  --rev v2.2.0 \
   --out ./data/01_model_input/sequential_train_val/train \
 
 SSL_CERT_FILE=/Users/mateo/pyronear/vision/train/pyro-train/.venv/lib/python3.12/site-packages/certifi/cacert.pem \
 uv run dvc get https://github.com/pyronear/pyro-dataset \
   data/processed/sequential_train_val/val \
-  --rev v2.1.0 \
+  --rev v2.2.0 \
   --out ./data/01_model_input/sequential_train_val/val \
 
 
 SSL_CERT_FILE=/Users/mateo/pyronear/vision/train/pyro-train/.venv/lib/python3.12/site-packages/certifi/cacert.pem \
 uv run dvc get https://github.com/pyronear/pyro-dataset \
   data/processed/sequential_test \
-  --rev v2.1.0 \
+  --rev v2.2.0 \
   --out ./data/test/sequential_test \
 
 
 SSL_CERT_FILE=/Users/mateo/pyronear/vision/train/pyro-train/.venv/lib/python3.12/site-packages/certifi/cacert.pem \
 uv run dvc get https://github.com/pyronear/pyro-dataset \
   data/processed/yolo_train_val \
-  --rev v2.1.0 \
+  --rev v2.2.0 \
   --out ./data/01_model_input/yolo_train_val \
 

results/eval_results.json

@@ -1,8 +1,8 @@
 {
-  "precision": 0.788629074749298,
-  "recall": 0.7360742705570292,
-  "map50": 0.7555552278085748,
-  "map50_95": 0.4801768125555643,
+  "precision": 0.797326566811577,
+  "recall": 0.7164400821996553,
+  "map50": 0.7465604659279885,
+  "map50_95": 0.47813585423595273,
   "model_dir": "data/02_models/yolo/best",
   "data": "data/test/yolo_test/data.yaml",
   "split": "test"

results/eval_sequential_results.json

@@ -1,12 +1,12 @@
 {
-  "nb_consecutive_frames": 4,
-  "conf_thresh": 0.2,
-  "tp": 143,
-  "fn": 6,
-  "fp": 21,
-  "tn": 128,
-  "recall": 0.9597,
-  "fpr": 0.1409,
-  "precision": 0.872,
-  "f1": 0.9137
+  "nb_consecutive_frames": 8,
+  "conf_thresh": 0.25,
+  "tp": 114,
+  "fn": 3,
+  "fp": 8,
+  "tn": 115,
+  "recall": 0.9744,
+  "fpr": 0.065,
+  "precision": 0.9344,
+  "f1": 0.954
 }