From e6abcaa0bc3bdf2174c5d41b2f12adc06c7b2b0e Mon Sep 17 00:00:00 2001 From: Till Beemelmanns Date: Fri, 12 Jul 2024 11:11:21 +0000 Subject: [PATCH] add IS-Fusion --- README.md | 12 +- evaluation/evaluations_nds.csv | 2 +- evaluation/ra_results.csv | 1 + evaluation/ra_results_beamsreducing.csv | 8 +- evaluation/ra_results_brightness.csv | 8 +- evaluation/ra_results_dark.csv | 8 +- evaluation/ra_results_fog.csv | 8 +- evaluation/ra_results_missingcamera.csv | 8 +- evaluation/ra_results_motionblur.csv | 8 +- evaluation/ra_results_pointsreducing.csv | 8 +- evaluation/ra_results_snow.csv | 8 +- evaluation/ra_results_spatialmisalignment.csv | 8 +- .../ra_results_temporalmisalignment.csv | 8 +- evaluation/rra_nds_results.csv | 24 +- helper/calc_metrics.ipynb | 306 +++++++++++------- helper/calc_metrics_and_export.ipynb | 89 ++++- 16 files changed, 340 insertions(+), 174 deletions(-) diff --git a/README.md b/README.md index f1c0f73..ea91f6e 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,7 @@ Preprint: [https://arxiv.org/abs/2402.11677](https://arxiv.org/abs/2402.11677) *Note: Right click and click on `Open Image in new tab` to enlarge an animation* ## News +- [12.07.2024] **v0.0.7** [IS-Fusion](https://github.com/yinjunbo/IS-Fusion) was added to the benchmark - [30.03.2024] **MultiCorrupt** has been accepted to [IEEE Intelligent Vehicles Symposium (IV)](https://ieee-iv.org/2024/) - [28.03.2024] **v0.0.3** Changed severity configuration for *Brightness*, reevaluated all models and metrics - [17.02.2024] **v0.0.2** Changed severity configuration for *Pointsreducing*, reevaluated all models and metrics @@ -124,18 +125,19 @@ Preprint: [https://arxiv.org/abs/2402.11677](https://arxiv.org/abs/2402.11677) | Model | Clean | Beams Red. | Brightness | Darkness | Fog | Missing Cam. | Motion Blur | Points Red. | Snow | Spatial Mis. | Temporal Mis. | mRA | |:----------------------------------------------------------------|:----------|:-------------|:-------------|:-----------|:----------|:---------------|:--------------|:--------------|:----------|:---------------|:----------------|:----------| | [CMT](https://github.com/junjie18/CMT) | 0.729 | **0.786** | 0.937 | 0.948 | **0.806** | 0.974 | 0.841 | **0.925** | **0.833** | **0.809** | **0.788** | **0.865** | -| [Sparsefusion](https://github.com/yichen928/SparseFusion) | **0.732** | 0.689 | 0.975 | 0.963 | 0.767 | 0.954 | 0.848 | 0.879 | 0.770 | 0.714 | 0.777 | 0.834 | -| [BEVfusion](https://github.com/mit-han-lab/bevfusion) | 0.714 | 0.676 | 0.967 | 0.969 | 0.752 | 0.974 | **0.866** | 0.872 | 0.774 | 0.705 | 0.742 | 0.830 | +| [Sparsefusion](https://github.com/yichen928/SparseFusion) | 0.732 | 0.689 | 0.975 | 0.963 | 0.767 | 0.954 | 0.848 | 0.879 | 0.770 | 0.714 | 0.777 | 0.834 | +| [BEVfusion](https://github.com/mit-han-lab/bevfusion) | 0.714 | 0.676 | 0.967 | 0.969 | 0.752 | 0.974 | 0.866 | 0.872 | 0.774 | 0.705 | 0.742 | 0.830 | | [TransFusion](https://github.com/XuyangBai/TransFusion) | 0.708 | 0.633 | **0.993** | **0.988** | 0.754 | **0.985** | 0.826 | 0.851 | 0.748 | 0.685 | 0.777 | 0.824 | +| [IS-Fusion](https://github.com/yinjunbo/IS-Fusion) | **0.740** | 0.677 | 0.955 | 0.947 | 0.754 | 0.949 | **0.869** | 0.856 | 0.730 | 0.711 | 0.767 | 0.822 | | [DeepInteraction](https://github.com/fudan-zvg/DeepInteraction) | 0.691 | 0.655 | 0.969 | 0.929 | 0.583 | 0.842 | 0.832 | 0.882 | 0.759 | 0.731 | 0.768 | 0.795 | - ### Relative Resistance Ability (RRA) computed with NDS metric and baseline BEVfusion | Model | Clean | Beams Red. | Brightness | Darkness | Fog | Missing Cam. | Motion Blur | Points Red. | Snow | Spatial Mis. | Temporal Mis. | mRRA | |:----------------------------------------------------------------|:----------|:-------------|:-------------|:-----------|:----------|:---------------|:--------------|:--------------|:----------|:---------------|:----------------|:----------| -| [BEVfusion](https://github.com/mit-han-lab/bevfusion) | 0.714 | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| [BEVfusion](https://github.com/mit-han-lab/bevfusion) | 0.714 | 0.000 | 0.000 | 0.000 | 0.000 | 0.000 | 0.000 | 0.000 | 0.000 | 0.000 | 0.000 | 0.000 | | [CMT](https://github.com/junjie18/CMT) | 0.729 | **18.642** | -1.138 | -0.096 | **9.398** | **2.041** | -0.841 | **8.213** | **9.887** | **17.053** | **8.448** | **7.161** | -| [Sparsefusion](https://github.com/yichen928/SparseFusion) | **0.732** | 4.264 | **3.179** | **1.821** | 4.429 | 0.297 | **0.280** | 3.242 | 1.887 | 3.699 | 7.228 | 3.033 | +| [Sparsefusion](https://github.com/yichen928/SparseFusion) | 0.732 | 4.264 | **3.179** | **1.821** | 4.429 | 0.297 | 0.280 | 3.242 | 1.887 | 3.699 | 7.228 | 3.033 | +| [IS-Fusion](https://github.com/yinjunbo/IS-Fusion) | **0.740** | 3.684 | 2.291 | 1.267 | 3.890 | 0.920 | **3.994** | 1.691 | -2.351 | 4.513 | 7.177 | 2.708 | | [TransFusion](https://github.com/XuyangBai/TransFusion) | 0.708 | -7.210 | 1.799 | 1.146 | -0.552 | 0.340 | -5.412 | -3.296 | -4.220 | -3.626 | 3.850 | -1.718 | | [DeepInteraction](https://github.com/fudan-zvg/DeepInteraction) | 0.691 | -6.361 | -3.150 | -7.215 | -25.037 | -16.386 | -7.077 | -2.188 | -5.149 | 0.212 | 0.145 | -7.221 | diff --git a/evaluation/evaluations_nds.csv b/evaluation/evaluations_nds.csv index ad41807..a42affa 100644 --- a/evaluation/evaluations_nds.csv +++ b/evaluation/evaluations_nds.csv @@ -4,4 +4,4 @@ DeepInteraction,0.6909,0.6139,0.5237,0.4544,0.5863,0.5014,0.4267,0.6522,0.6242,0 TransFusion,0.7084,0.6306,0.5437,0.4766,0.5708,0.4761,0.4095,0.6554,0.619,0.5331,0.6727,0.5878,0.4944,0.7017,0.698,0.6946,0.6506,0.5996,0.3529,0.7025,0.6996,0.6978,0.7056,0.7041,0.7007,0.6222,0.4591,0.2636,0.5447,0.5353,0.5087 Sparsefusion,0.7315,0.6523,0.5622,0.4901,0.6061,0.5149,0.4461,0.6881,0.6578,0.5838,0.6997,0.6234,0.5374,0.7117,0.6981,0.6836,0.6762,0.6261,0.3811,0.7122,0.7039,0.6978,0.7205,0.7146,0.7039,0.6593,0.5139,0.338,0.5859,0.5704,0.5337 BEVfusion,0.7144,0.6341,0.5471,0.4085,0.5903,0.4943,0.4266,0.6684,0.6379,0.5628,0.6861,0.6217,0.5475,0.7038,0.6961,0.6873,0.6555,0.5981,0.3584,0.699,0.691,0.6861,0.6967,0.6913,0.6851,0.6396,0.4899,0.3199,0.5782,0.5572,0.5233 -Our,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +IS-Fusion,0.74,0.6537,0.5609,0.4892,0.6221,0.5214,0.4359,0.6827,0.6489,0.5691,0.7046,0.6445,0.5803,0.7164,0.7023,0.6877,0.6697,0.6107,0.3943,0.7101,0.6998,0.6925,0.7198,0.7091,0.6917,0.6694,0.5068,0.3266,0.5619,0.5463,0.5115 \ No newline at end of file diff --git a/evaluation/ra_results.csv b/evaluation/ra_results.csv index 23cd3b8..f3e23fe 100644 --- a/evaluation/ra_results.csv +++ b/evaluation/ra_results.csv @@ -4,3 +4,4 @@ DeepInteraction,0.8918801563178463,0.9820523954262557,0.9544072948328268,0.83774 TransFusion,0.8783173348390739,0.9960474308300394,0.9916713721061547,0.9184076792772444,0.9905420666290231,0.9496047430830039,0.9251835121400338,0.7689158667419536,0.8057594579333709,0.8901750423489554,0.6480801806888763,0.9939299830604177,0.9875776397515528,0.8464144551101073,0.9853190287972896,0.8297571993224166,0.8738001129305477,0.7556465273856577,0.672077922077922,0.7675042348955392,0.37210615471485037,0.9891304347826086,0.9850367024280067,0.4981648785996612,0.9805194805194805,0.6979107848673066,0.752540937323546,0.7180971202710333,0.5780632411067194,0.6727837380011293,0.8239695087521175 Sparsefusion,0.9012987012987013,0.9849624060150376,0.9736158578263842,0.9244019138755981,0.9729323308270676,0.9565276828434722,0.9406698564593301,0.800956937799043,0.8285714285714285,0.8917293233082706,0.7025290498974709,0.9768967874231032,0.9622693096377306,0.8559125085440874,0.9543403964456596,0.8522214627477784,0.8992481203007519,0.7797676008202323,0.7038961038961039,0.7685577580314422,0.46206425153793573,0.9622693096377306,0.9539302802460696,0.5209842788790157,0.9345181134654819,0.7346548188653451,0.7980861244019138,0.7295967190704031,0.6098427887901572,0.6699931647300068,0.8335748462064251 BEVfusion,0.8952967525195967,0.9752239641657334,0.9784434490481522,0.9175531914893617,0.9851623740201567,0.9603863381858903,0.9356103023516237,0.8093505039193729,0.8262877939529676,0.8875979843225084,0.6857502799552071,0.9676651735722285,0.967245240761478,0.8372060470324747,0.9743840985442329,0.8702407614781635,0.8929171332586786,0.7799552071668533,0.691909294512878,0.7658174692049272,0.44778835386338184,0.958986562150056,0.9603863381858903,0.5016797312430011,0.9620660694288914,0.7663773796192609,0.7877939529675251,0.7325027995520716,0.597144456886898,0.5718085106382979,0.8296845837999254 +IS-Fusion,0.9045945945945946,0.9727027027027028,0.9595945945945945,0.9049999999999999,0.9681081081081082,0.9521621621621622,0.9225675675675675,0.7593243243243243,0.8406756756756757,0.8833783783783783,0.6848648648648649,0.9582432432432432,0.9456756756756757,0.8252702702702703,0.9490540540540541,0.8709459459459459,0.876891891891892,0.7382432432432433,0.7045945945945946,0.757972972972973,0.44135135135135134,0.9347297297297297,0.9358108108108109,0.5328378378378378,0.9293243243243243,0.7841891891891892,0.7690540540540541,0.6912162162162162,0.5890540540540541,0.6610810810810811,0.821617117117117 diff --git a/evaluation/ra_results_beamsreducing.csv b/evaluation/ra_results_beamsreducing.csv index 65cfda2..8467ba6 100644 --- a/evaluation/ra_results_beamsreducing.csv +++ b/evaluation/ra_results_beamsreducing.csv @@ -1,4 +1,4 @@ -CMT,DeepInteraction,TransFusion,Sparsefusion,BEVfusion -0.9363511659807956,0.8918801563178463,0.8783173348390739,0.9012987012987013,0.8952967525195967 -0.8056241426611798,0.6616008105369808,0.6480801806888763,0.7025290498974709,0.6857502799552071 -0.6168724279835391,0.41091330149080907,0.37210615471485037,0.46206425153793573,0.44778835386338184 +CMT,DeepInteraction,TransFusion,Sparsefusion,BEVfusion,IS-Fusion +0.9363511659807956,0.8918801563178463,0.8783173348390739,0.9012987012987013,0.8952967525195967,0.9045945945945946 +0.8056241426611798,0.6616008105369808,0.6480801806888763,0.7025290498974709,0.6857502799552071,0.6848648648648649 +0.6168724279835391,0.41091330149080907,0.37210615471485037,0.46206425153793573,0.44778835386338184,0.44135135135135134 diff --git a/evaluation/ra_results_brightness.csv b/evaluation/ra_results_brightness.csv index ecac108..fb0ea58 100644 --- a/evaluation/ra_results_brightness.csv +++ b/evaluation/ra_results_brightness.csv @@ -1,4 +1,4 @@ -CMT,DeepInteraction,TransFusion,Sparsefusion,BEVfusion -0.9654320987654321,0.9820523954262557,0.9960474308300394,0.9849624060150376,0.9752239641657334 -0.9399176954732511,0.9724996381531338,0.9939299830604177,0.9768967874231032,0.9676651735722285 -0.9060356652949245,0.9515125199015777,0.9891304347826086,0.9622693096377306,0.958986562150056 +CMT,DeepInteraction,TransFusion,Sparsefusion,BEVfusion,IS-Fusion +0.9654320987654321,0.9820523954262557,0.9960474308300394,0.9849624060150376,0.9752239641657334,0.9727027027027028 +0.9399176954732511,0.9724996381531338,0.9939299830604177,0.9768967874231032,0.9676651735722285,0.9582432432432432 +0.9060356652949245,0.9515125199015777,0.9891304347826086,0.9622693096377306,0.958986562150056,0.9347297297297297 diff --git a/evaluation/ra_results_dark.csv b/evaluation/ra_results_dark.csv index 79cf192..4468e1b 100644 --- a/evaluation/ra_results_dark.csv +++ b/evaluation/ra_results_dark.csv @@ -1,4 +1,4 @@ -CMT,DeepInteraction,TransFusion,Sparsefusion,BEVfusion -0.9672153635116597,0.9544072948328268,0.9916713721061547,0.9736158578263842,0.9784434490481522 -0.9470507544581619,0.9284990591981473,0.9875776397515528,0.9622693096377306,0.967245240761478 -0.9308641975308642,0.9051961210015921,0.9850367024280067,0.9539302802460696,0.9603863381858903 +CMT,DeepInteraction,TransFusion,Sparsefusion,BEVfusion,IS-Fusion +0.9672153635116597,0.9544072948328268,0.9916713721061547,0.9736158578263842,0.9784434490481522,0.9595945945945945 +0.9470507544581619,0.9284990591981473,0.9875776397515528,0.9622693096377306,0.967245240761478,0.9456756756756757 +0.9308641975308642,0.9051961210015921,0.9850367024280067,0.9539302802460696,0.9603863381858903,0.9358108108108109 diff --git a/evaluation/ra_results_fog.csv b/evaluation/ra_results_fog.csv index e9742c1..0e8d8ef 100644 --- a/evaluation/ra_results_fog.csv +++ b/evaluation/ra_results_fog.csv @@ -1,4 +1,4 @@ -CMT,DeepInteraction,TransFusion,Sparsefusion,BEVfusion -0.9566529492455419,0.8377478651034882,0.9184076792772444,0.9244019138755981,0.9175531914893617 -0.8953360768175582,0.701114488348531,0.8464144551101073,0.8559125085440874,0.8372060470324747 -0.5670781893004115,0.21016066000868433,0.4981648785996612,0.5209842788790157,0.5016797312430011 +CMT,DeepInteraction,TransFusion,Sparsefusion,BEVfusion,IS-Fusion +0.9566529492455419,0.8377478651034882,0.9184076792772444,0.9244019138755981,0.9175531914893617,0.9049999999999999 +0.8953360768175582,0.701114488348531,0.8464144551101073,0.8559125085440874,0.8372060470324747,0.8252702702702703 +0.5670781893004115,0.21016066000868433,0.4981648785996612,0.5209842788790157,0.5016797312430011,0.5328378378378378 diff --git a/evaluation/ra_results_missingcamera.csv b/evaluation/ra_results_missingcamera.csv index 2cbd485..5f70ab6 100644 --- a/evaluation/ra_results_missingcamera.csv +++ b/evaluation/ra_results_missingcamera.csv @@ -1,4 +1,4 @@ -CMT,DeepInteraction,TransFusion,Sparsefusion,BEVfusion -0.9849108367626886,0.9050513822550297,0.9905420666290231,0.9729323308270676,0.9851623740201567 -0.9751714677640604,0.841800550007237,0.9853190287972896,0.9543403964456596,0.9743840985442329 -0.9614540466392318,0.7791286727456941,0.9805194805194805,0.9345181134654819,0.9620660694288914 +CMT,DeepInteraction,TransFusion,Sparsefusion,BEVfusion,IS-Fusion +0.9849108367626886,0.9050513822550297,0.9905420666290231,0.9729323308270676,0.9851623740201567,0.9681081081081082 +0.9751714677640604,0.841800550007237,0.9853190287972896,0.9543403964456596,0.9743840985442329,0.9490540540540541 +0.9614540466392318,0.7791286727456941,0.9805194805194805,0.9345181134654819,0.9620660694288914,0.9293243243243243 diff --git a/evaluation/ra_results_motionblur.csv b/evaluation/ra_results_motionblur.csv index 6bacfb2..5f9c4dd 100644 --- a/evaluation/ra_results_motionblur.csv +++ b/evaluation/ra_results_motionblur.csv @@ -1,4 +1,4 @@ -CMT,DeepInteraction,TransFusion,Sparsefusion,BEVfusion -0.9445816186556928,0.9480387899840789,0.9496047430830039,0.9565276828434722,0.9603863381858903 -0.8441700960219478,0.8323925314806774,0.8297571993224166,0.8522214627477784,0.8702407614781635 -0.7348422496570645,0.7148646692719641,0.6979107848673066,0.7346548188653451,0.7663773796192609 +CMT,DeepInteraction,TransFusion,Sparsefusion,BEVfusion,IS-Fusion +0.9445816186556928,0.9480387899840789,0.9496047430830039,0.9565276828434722,0.9603863381858903,0.9521621621621622 +0.8441700960219478,0.8323925314806774,0.8297571993224166,0.8522214627477784,0.8702407614781635,0.8709459459459459 +0.7348422496570645,0.7148646692719641,0.6979107848673066,0.7346548188653451,0.7663773796192609,0.7841891891891892 diff --git a/evaluation/ra_results_pointsreducing.csv b/evaluation/ra_results_pointsreducing.csv index 7ca0cfa..395541b 100644 --- a/evaluation/ra_results_pointsreducing.csv +++ b/evaluation/ra_results_pointsreducing.csv @@ -1,4 +1,4 @@ -CMT,DeepInteraction,TransFusion,Sparsefusion,BEVfusion -0.9618655692729767,0.9439861050803301,0.9251835121400338,0.9406698564593301,0.9356103023516237 -0.9374485596707819,0.9034592560428427,0.8738001129305477,0.8992481203007519,0.8929171332586786 -0.8751714677640604,0.7986684035316254,0.752540937323546,0.7980861244019138,0.7877939529675251 +CMT,DeepInteraction,TransFusion,Sparsefusion,BEVfusion,IS-Fusion +0.9618655692729767,0.9439861050803301,0.9251835121400338,0.9406698564593301,0.9356103023516237,0.9225675675675675 +0.9374485596707819,0.9034592560428427,0.8738001129305477,0.8992481203007519,0.8929171332586786,0.876891891891892 +0.8751714677640604,0.7986684035316254,0.752540937323546,0.7980861244019138,0.7877939529675251,0.7690540540540541 diff --git a/evaluation/ra_results_snow.csv b/evaluation/ra_results_snow.csv index 428aa03..3c79599 100644 --- a/evaluation/ra_results_snow.csv +++ b/evaluation/ra_results_snow.csv @@ -1,4 +1,4 @@ -CMT,DeepInteraction,TransFusion,Sparsefusion,BEVfusion -0.8893004115226337,0.799826313504125,0.7689158667419536,0.800956937799043,0.8093505039193729 -0.8491083676268861,0.7687074829931974,0.7556465273856577,0.7797676008202323,0.7799552071668533 -0.7618655692729767,0.7086409031697786,0.7180971202710333,0.7295967190704031,0.7325027995520716 +CMT,DeepInteraction,TransFusion,Sparsefusion,BEVfusion,IS-Fusion +0.8893004115226337,0.799826313504125,0.7689158667419536,0.800956937799043,0.8093505039193729,0.7593243243243243 +0.8491083676268861,0.7687074829931974,0.7556465273856577,0.7797676008202323,0.7799552071668533,0.7382432432432433 +0.7618655692729767,0.7086409031697786,0.7180971202710333,0.7295967190704031,0.7325027995520716,0.6912162162162162 diff --git a/evaluation/ra_results_spatialmisalignment.csv b/evaluation/ra_results_spatialmisalignment.csv index b6975d4..d76d1ac 100644 --- a/evaluation/ra_results_spatialmisalignment.csv +++ b/evaluation/ra_results_spatialmisalignment.csv @@ -1,4 +1,4 @@ -CMT,DeepInteraction,TransFusion,Sparsefusion,BEVfusion -0.8864197530864197,0.8486032710956725,0.8057594579333709,0.8285714285714285,0.8262877939529676 -0.803840877914952,0.7257200752641482,0.672077922077922,0.7038961038961039,0.691909294512878 -0.7362139917695473,0.6176002315819946,0.5780632411067194,0.6098427887901572,0.597144456886898 +CMT,DeepInteraction,TransFusion,Sparsefusion,BEVfusion,IS-Fusion +0.8864197530864197,0.8486032710956725,0.8057594579333709,0.8285714285714285,0.8262877939529676,0.8406756756756757 +0.803840877914952,0.7257200752641482,0.672077922077922,0.7038961038961039,0.691909294512878,0.7045945945945946 +0.7362139917695473,0.6176002315819946,0.5780632411067194,0.6098427887901572,0.597144456886898,0.5890540540540541 diff --git a/evaluation/ra_results_temporalmisalignment.csv b/evaluation/ra_results_temporalmisalignment.csv index fd23926..6582d1d 100644 --- a/evaluation/ra_results_temporalmisalignment.csv +++ b/evaluation/ra_results_temporalmisalignment.csv @@ -1,4 +1,4 @@ -CMT,DeepInteraction,TransFusion,Sparsefusion,BEVfusion -0.9015089163237312,0.8885511651469099,0.8901750423489554,0.8917293233082706,0.8875979843225084 -0.7791495198902606,0.7579968157475757,0.7675042348955392,0.7685577580314422,0.7658174692049272 -0.6842249657064472,0.6576928643797946,0.6727837380011293,0.6699931647300068,0.5718085106382979 +CMT,DeepInteraction,TransFusion,Sparsefusion,BEVfusion,IS-Fusion +0.9015089163237312,0.8885511651469099,0.8901750423489554,0.8917293233082706,0.8875979843225084,0.8833783783783783 +0.7791495198902606,0.7579968157475757,0.7675042348955392,0.7685577580314422,0.7658174692049272,0.757972972972973 +0.6842249657064472,0.6576928643797946,0.6727837380011293,0.6699931647300068,0.5718085106382979,0.6610810810810811 diff --git a/evaluation/rra_nds_results.csv b/evaluation/rra_nds_results.csv index bf97a5e..d914f08 100644 --- a/evaluation/rra_nds_results.csv +++ b/evaluation/rra_nds_results.csv @@ -1,12 +1,12 @@ -model,CMT,DeepInteraction,TransFusion,Sparsefusion,BEVfusion -clean,0.729,0.6909,0.7084,0.7315,0.7144 -RRA_beamsreducing,18.64219677107768,-6.3612529322478295,-7.209879950324272,4.263833310335308,0.0 -RRA_brightness,-1.1383917804254517,-3.1498721721094025,1.7992378563503664,3.1788143360185206,0.0 -RRA_dark,-0.09633447329125966,-7.215452049515914,1.1463802321660888,1.8207215452049619,0.0 -RRA_fog,9.398263027295272,-25.03722084367247,-0.5521091811414469,4.429280397022328,0.0 -RRA_missingcamera,2.041011881947097,-16.38558834802607,0.3401686469911791,0.2970486776542547,0.0 -RRA_motionblur,-0.8408343664097706,-7.0770225839487,-5.411523742790935,0.2802781221365791,0.0 -RRA_pointsreducing,8.212508694023878,-2.1882189288962683,-3.295703814670159,3.2422021293670777,0.0 -RRA_snow,9.887261108096702,-5.148610357508899,-4.2201724241876075,1.8870199553867373,0.0 -RRA_spatialmisalignment,17.05267337215457,0.21175224986764718,-3.6262572789836023,3.6990471148755866,0.0 -RRA_temporalmisalignment,8.44813486821414,0.1446813864251162,3.8497829779203574,7.2277788261936315,0.0 +model,CMT,DeepInteraction,TransFusion,Sparsefusion,BEVfusion,IS-Fusion +clean,0.729,0.6909,0.7084,0.7315,0.7144,0.74 +RRA_beamsreducing,18.64219677107768,-6.3612529322478295,-7.209879950324272,4.263833310335308,0.0,3.6842831516489705 +RRA_brightness,-1.1383917804254517,-3.1498721721094025,1.7992378563503664,3.1788143360185206,0.0,2.2912546428054625 +RRA_dark,-0.09633447329125966,-7.215452049515914,1.1463802321660888,1.8207215452049619,0.0,1.2667983237801606 +RRA_fog,9.398263027295272,-25.03722084367247,-0.5521091811414469,4.429280397022328,0.0,3.889578163771712 +RRA_missingcamera,2.041011881947097,-16.38558834802607,0.3401686469911791,0.2970486776542547,0.0,0.9198926791874174 +RRA_motionblur,-0.8408343664097706,-7.0770225839487,-5.411523742790935,0.2802781221365791,0.0,3.9939632404462744 +RRA_pointsreducing,8.212508694023878,-2.1882189288962683,-3.295703814670159,3.2422021293670777,0.0,1.6906532555775566 +RRA_snow,9.887261108096702,-5.148610357508899,-4.2201724241876075,1.8870199553867373,0.0,-2.351238922047394 +RRA_spatialmisalignment,17.05267337215457,0.21175224986764718,-3.6262572789836023,3.6990471148755866,0.0,4.51296982530438 +RRA_temporalmisalignment,8.44813486821414,0.1446813864251162,3.8497829779203574,7.2277788261936315,0.0,7.177454865697919 diff --git a/helper/calc_metrics.ipynb b/helper/calc_metrics.ipynb index fd09806..3834d4e 100644 --- a/helper/calc_metrics.ipynb +++ b/helper/calc_metrics.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 32, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -11,7 +11,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 16, "metadata": {}, "outputs": [], "source": [ @@ -30,13 +30,14 @@ " \"spatialmisalignment\": \"Spatial Mis.\",\n", " \"temporalmisalignment\": \"Temporal Mis.\"\n", " }\n", - "MODELS = [\"CMT\", \"DeepInteraction\", \"TransFusion\", \"Sparsefusion\", \"BEVfusion\"]\n", + "MODELS = [\"CMT\", \"DeepInteraction\", \"TransFusion\", \"Sparsefusion\", \"BEVfusion\", \"IS-Fusion\"]\n", "MODELS_TO_LINKS = {\n", " \"CMT\": \"https://github.com/junjie18/CMT\",\n", " \"DeepInteraction\": \"https://github.com/fudan-zvg/DeepInteraction\",\n", " \"TransFusion\": \"https://github.com/XuyangBai/TransFusion\",\n", " \"Sparsefusion\": \"https://github.com/yichen928/SparseFusion\",\n", - " \"BEVfusion\": \"https://github.com/mit-han-lab/bevfusion\"\n", + " \"BEVfusion\": \"https://github.com/mit-han-lab/bevfusion\",\n", + " \"IS-Fusion\": \"https://github.com/yinjunbo/IS-Fusion\",\n", "}\n", "MODELS_TO_LINKS_MD = {k: f\"[{k}]({v})\" for k, v in MODELS_TO_LINKS.items()}\n", "SEVERITY_LEVELS = [\"1\", \"2\", \"3\"]" @@ -44,7 +45,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ @@ -53,7 +54,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ @@ -62,7 +63,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 19, "metadata": {}, "outputs": [ { @@ -230,9 +231,33 @@ " 0.5572\n", " 0.5233\n", " \n", + " \n", + " 5\n", + " IS-Fusion\n", + " 0.7400\n", + " 0.6537\n", + " 0.5609\n", + " 0.4892\n", + " 0.6221\n", + " 0.5214\n", + " 0.4359\n", + " 0.6827\n", + " 0.6489\n", + " ...\n", + " 0.6925\n", + " 0.7198\n", + " 0.7091\n", + " 0.6917\n", + " 0.6694\n", + " 0.5068\n", + " 0.3266\n", + " 0.5619\n", + " 0.5463\n", + " 0.5115\n", + " \n", " \n", "\n", - "

5 rows × 32 columns

\n", + "

6 rows × 32 columns

\n", "" ], "text/plain": [ @@ -242,6 +267,7 @@ "2 TransFusion 0.7084 0.6306 0.5437 \n", "3 Sparsefusion 0.7315 0.6523 0.5622 \n", "4 BEVfusion 0.7144 0.6341 0.5471 \n", + "5 IS-Fusion 0.7400 0.6537 0.5609 \n", "\n", " temporalmisalignment_s3 spatialmisalignment_s1 spatialmisalignment_s2 \\\n", "0 0.4988 0.6462 0.5860 \n", @@ -249,6 +275,7 @@ "2 0.4766 0.5708 0.4761 \n", "3 0.4901 0.6061 0.5149 \n", "4 0.4085 0.5903 0.4943 \n", + "5 0.4892 0.6221 0.5214 \n", "\n", " spatialmisalignment_s3 pointsreducing_s1 pointsreducing_s2 ... dark_s3 \\\n", "0 0.5367 0.7012 0.6834 ... 0.6786 \n", @@ -256,6 +283,7 @@ "2 0.4095 0.6554 0.6190 ... 0.6978 \n", "3 0.4461 0.6881 0.6578 ... 0.6978 \n", "4 0.4266 0.6684 0.6379 ... 0.6861 \n", + "5 0.4359 0.6827 0.6489 ... 0.6925 \n", "\n", " brightness_s1 brightness_s2 brightness_s3 beamsreducing_s1 \\\n", "0 0.7038 0.6852 0.6605 0.6826 \n", @@ -263,6 +291,7 @@ "2 0.7056 0.7041 0.7007 0.6222 \n", "3 0.7205 0.7146 0.7039 0.6593 \n", "4 0.6967 0.6913 0.6851 0.6396 \n", + "5 0.7198 0.7091 0.6917 0.6694 \n", "\n", " beamsreducing_s2 beamsreducing_s3 snow_s1 snow_s2 snow_s3 \n", "0 0.5873 0.4497 0.6483 0.6190 0.5554 \n", @@ -270,11 +299,12 @@ "2 0.4591 0.2636 0.5447 0.5353 0.5087 \n", "3 0.5139 0.3380 0.5859 0.5704 0.5337 \n", "4 0.4899 0.3199 0.5782 0.5572 0.5233 \n", + "5 0.5068 0.3266 0.5619 0.5463 0.5115 \n", "\n", - "[5 rows x 32 columns]" + "[6 rows x 32 columns]" ] }, - "execution_count": 36, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } @@ -292,7 +322,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 20, "metadata": {}, "outputs": [], "source": [ @@ -307,7 +337,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 21, "metadata": {}, "outputs": [ { @@ -421,6 +451,21 @@ " 0.741741\n", " 0.829685\n", " \n", + " \n", + " 5\n", + " IS-Fusion\n", + " 0.676937\n", + " 0.955225\n", + " 0.947027\n", + " 0.754369\n", + " 0.948829\n", + " 0.869099\n", + " 0.856171\n", + " 0.729595\n", + " 0.711441\n", + " 0.767477\n", + " 0.821617\n", + " \n", " \n", "\n", "" @@ -432,6 +477,7 @@ "2 TransFusion 0.632835 0.993036 0.988095 0.754329 \n", "3 Sparsefusion 0.688631 0.974710 0.963272 0.767100 \n", "4 BEVfusion 0.676278 0.967292 0.968692 0.752146 \n", + "5 IS-Fusion 0.676937 0.955225 0.947027 0.754369 \n", "\n", " RA_missingcamera RA_motionblur RA_pointsreducing RA_snow \\\n", "0 0.973845 0.841198 0.924829 0.833425 \n", @@ -439,16 +485,18 @@ "2 0.985460 0.825758 0.850508 0.747553 \n", "3 0.953930 0.847801 0.879335 0.770107 \n", "4 0.973871 0.865668 0.872107 0.773936 \n", + "5 0.948829 0.869099 0.856171 0.729595 \n", "\n", " RA_spatialmisalignment RA_temporalmisalignment mRA \n", "0 0.808825 0.788294 0.864856 \n", "1 0.730641 0.768080 0.794944 \n", "2 0.685300 0.776821 0.823970 \n", "3 0.714103 0.776760 0.833575 \n", - "4 0.705114 0.741741 0.829685 " + "4 0.705114 0.741741 0.829685 \n", + "5 0.711441 0.767477 0.821617 " ] }, - "execution_count": 38, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } @@ -466,7 +514,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 22, "metadata": {}, "outputs": [ { @@ -481,7 +529,8 @@ "DeepInteraction & 0.655 & 0.969 & 0.929 & 0.583 & 0.842 & 0.832 & 0.882 & 0.759 & 0.731 & 0.768 & 0.795 \\\\\n", "\\textbf{TransFusion} & 0.633 & \\textbf{0.993} & \\textbf{0.988} & 0.754 & \\textbf{0.985} & 0.826 & 0.851 & 0.748 & 0.685 & 0.777 & 0.824 \\\\\n", "Sparsefusion & 0.689 & 0.975 & 0.963 & 0.767 & 0.954 & 0.848 & 0.879 & 0.770 & 0.714 & 0.777 & 0.834 \\\\\n", - "BEVfusion & 0.676 & 0.967 & 0.969 & 0.752 & 0.974 & \\textbf{0.866} & 0.872 & 0.774 & 0.705 & 0.742 & 0.830 \\\\\n", + "BEVfusion & 0.676 & 0.967 & 0.969 & 0.752 & 0.974 & 0.866 & 0.872 & 0.774 & 0.705 & 0.742 & 0.830 \\\\\n", + "IS-Fusion & 0.677 & 0.955 & 0.947 & 0.754 & 0.949 & \\textbf{0.869} & 0.856 & 0.730 & 0.711 & 0.767 & 0.822 \\\\\n", "\\bottomrule\n", "\\end{tabular}\n", "\n" @@ -505,7 +554,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 23, "metadata": {}, "outputs": [ { @@ -515,9 +564,10 @@ "| Model | Clean | Beams Red. | Brightness | Darkness | Fog | Missing Cam. | Motion Blur | Points Red. | Snow | Spatial Mis. | Temporal Mis. | mRA |\n", "|:----------------------------------------------------------------|:----------|:-------------|:-------------|:-----------|:----------|:---------------|:--------------|:--------------|:----------|:---------------|:----------------|:----------|\n", "| [CMT](https://github.com/junjie18/CMT) | 0.729 | **0.786** | 0.937 | 0.948 | **0.806** | 0.974 | 0.841 | **0.925** | **0.833** | **0.809** | **0.788** | **0.865** |\n", - "| [Sparsefusion](https://github.com/yichen928/SparseFusion) | **0.732** | 0.689 | 0.975 | 0.963 | 0.767 | 0.954 | 0.848 | 0.879 | 0.770 | 0.714 | 0.777 | 0.834 |\n", - "| [BEVfusion](https://github.com/mit-han-lab/bevfusion) | 0.714 | 0.676 | 0.967 | 0.969 | 0.752 | 0.974 | **0.866** | 0.872 | 0.774 | 0.705 | 0.742 | 0.830 |\n", + "| [Sparsefusion](https://github.com/yichen928/SparseFusion) | 0.732 | 0.689 | 0.975 | 0.963 | 0.767 | 0.954 | 0.848 | 0.879 | 0.770 | 0.714 | 0.777 | 0.834 |\n", + "| [BEVfusion](https://github.com/mit-han-lab/bevfusion) | 0.714 | 0.676 | 0.967 | 0.969 | 0.752 | 0.974 | 0.866 | 0.872 | 0.774 | 0.705 | 0.742 | 0.830 |\n", "| [TransFusion](https://github.com/XuyangBai/TransFusion) | 0.708 | 0.633 | **0.993** | **0.988** | 0.754 | **0.985** | 0.826 | 0.851 | 0.748 | 0.685 | 0.777 | 0.824 |\n", + "| [IS-Fusion](https://github.com/yinjunbo/IS-Fusion) | **0.740** | 0.677 | 0.955 | 0.947 | 0.754 | 0.949 | **0.869** | 0.856 | 0.730 | 0.711 | 0.767 | 0.822 |\n", "| [DeepInteraction](https://github.com/fudan-zvg/DeepInteraction) | 0.691 | 0.655 | 0.969 | 0.929 | 0.583 | 0.842 | 0.832 | 0.882 | 0.759 | 0.731 | 0.768 | 0.795 |\n" ] } @@ -561,120 +611,135 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 24, "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", - "\n", + "
\n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", "
ModelCleanBeams Red.BrightnessDarknessFogMissing Cam.Motion BlurPoints Red.SnowSpatial Mis.Temporal Mis.mRAModelCleanBeams Red.BrightnessDarknessFogMissing Cam.Motion BlurPoints Red.SnowSpatial Mis.Temporal Mis.mRA
CMT0.7290.7860.9370.9480.8060.9740.8410.9250.8330.8090.7880.865CMT0.7290.7860.9370.9480.8060.9740.8410.9250.8330.8090.7880.865
Sparsefusion0.7320.6890.9750.9630.7670.9540.8480.8790.7700.7140.7770.834Sparsefusion0.7320.6890.9750.9630.7670.9540.8480.8790.7700.7140.7770.834
BEVfusion0.7140.6760.9670.9690.7520.9740.8660.8720.7740.7050.7420.830BEVfusion0.7140.6760.9670.9690.7520.9740.8660.8720.7740.7050.7420.830
TransFusion0.7080.6330.9930.9880.7540.9850.8260.8510.7480.6850.7770.824TransFusion0.7080.6330.9930.9880.7540.9850.8260.8510.7480.6850.7770.824
DeepInteraction0.6910.6550.9690.9290.5830.8420.8320.8820.7590.7310.7680.795IS-Fusion0.7400.6770.9550.9470.7540.9490.8690.8560.7300.7110.7670.822
DeepInteraction0.6910.6550.9690.9290.5830.8420.8320.8820.7590.7310.7680.795
\n" ], "text/plain": [ - "" + "" ] }, - "execution_count": 41, + "execution_count": 24, "metadata": {}, "output_type": "execute_result" } @@ -701,7 +766,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 25, "metadata": {}, "outputs": [], "source": [ @@ -716,7 +781,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 26, "metadata": {}, "outputs": [ { @@ -836,6 +901,22 @@ " 0.000000\n", " 0.000000\n", " \n", + " \n", + " 5\n", + " IS-Fusion\n", + " 0.7400\n", + " 3.684283\n", + " 2.291255\n", + " 1.266798\n", + " 3.889578\n", + " 0.919893\n", + " 3.993963\n", + " 1.690653\n", + " -2.351239\n", + " 4.512970\n", + " 7.177455\n", + " 2.707561\n", + " \n", " \n", "\n", "" @@ -847,6 +928,7 @@ "2 TransFusion 0.7084 -7.209880 1.799238 1.146380 \n", "3 Sparsefusion 0.7315 4.263833 3.178814 1.820722 \n", "4 BEVfusion 0.7144 0.000000 0.000000 0.000000 \n", + "5 IS-Fusion 0.7400 3.684283 2.291255 1.266798 \n", "\n", " RRA_fog RRA_missingcamera RRA_motionblur RRA_pointsreducing RRA_snow \\\n", "0 9.398263 2.041012 -0.840834 8.212509 9.887261 \n", @@ -854,16 +936,18 @@ "2 -0.552109 0.340169 -5.411524 -3.295704 -4.220172 \n", "3 4.429280 0.297049 0.280278 3.242202 1.887020 \n", "4 0.000000 0.000000 0.000000 0.000000 0.000000 \n", + "5 3.889578 0.919893 3.993963 1.690653 -2.351239 \n", "\n", " RRA_spatialmisalignment RRA_temporalmisalignment mRRA \n", "0 17.052673 8.448135 7.160649 \n", "1 0.211752 0.144681 -7.220680 \n", "2 -3.626257 3.849783 -1.718008 \n", "3 3.699047 7.227779 3.032602 \n", - "4 0.000000 0.000000 0.000000 " + "4 0.000000 0.000000 0.000000 \n", + "5 4.512970 7.177455 2.707561 " ] }, - "execution_count": 43, + "execution_count": 26, "metadata": {}, "output_type": "execute_result" } @@ -881,7 +965,7 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 27, "metadata": {}, "outputs": [ { @@ -895,8 +979,9 @@ "CMT & \\textbf{18.642} & -1.138 & -0.096 & \\textbf{9.398} & \\textbf{2.041} & -0.841 & \\textbf{8.213} & \\textbf{9.887} & \\textbf{17.053} & \\textbf{8.448} & \\textbf{7.161} \\\\\n", "DeepInteraction & -6.361 & -3.150 & -7.215 & -25.037 & -16.386 & -7.077 & -2.188 & -5.149 & 0.212 & 0.145 & -7.221 \\\\\n", "\\textbf{TransFusion} & -7.210 & 1.799 & 1.146 & -0.552 & 0.340 & -5.412 & -3.296 & -4.220 & -3.626 & 3.850 & -1.718 \\\\\n", - "Sparsefusion & 4.264 & \\textbf{3.179} & \\textbf{1.821} & 4.429 & 0.297 & \\textbf{0.280} & 3.242 & 1.887 & 3.699 & 7.228 & 3.033 \\\\\n", + "Sparsefusion & 4.264 & \\textbf{3.179} & \\textbf{1.821} & 4.429 & 0.297 & 0.280 & 3.242 & 1.887 & 3.699 & 7.228 & 3.033 \\\\\n", "BEVfusion & 0.000 & 0.000 & 0.000 & 0.000 & 0.000 & 0.000 & 0.000 & 0.000 & 0.000 & 0.000 & 0.000 \\\\\n", + "IS-Fusion & 3.684 & 2.291 & 1.267 & 3.890 & 0.920 & \\textbf{3.994} & 1.691 & -2.351 & 4.513 & 7.177 & 2.708 \\\\\n", "\\bottomrule\n", "\\end{tabular}\n", "\n" @@ -921,7 +1006,7 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 28, "metadata": {}, "outputs": [ { @@ -932,7 +1017,8 @@ "|:----------------------------------------------------------------|:----------|:-------------|:-------------|:-----------|:----------|:---------------|:--------------|:--------------|:----------|:---------------|:----------------|:----------|\n", "| [BEVfusion](https://github.com/mit-han-lab/bevfusion) | 0.714 | 0.000 | 0.000 | 0.000 | 0.000 | 0.000 | 0.000 | 0.000 | 0.000 | 0.000 | 0.000 | 0.000 |\n", "| [CMT](https://github.com/junjie18/CMT) | 0.729 | **18.642** | -1.138 | -0.096 | **9.398** | **2.041** | -0.841 | **8.213** | **9.887** | **17.053** | **8.448** | **7.161** |\n", - "| [Sparsefusion](https://github.com/yichen928/SparseFusion) | **0.732** | 4.264 | **3.179** | **1.821** | 4.429 | 0.297 | **0.280** | 3.242 | 1.887 | 3.699 | 7.228 | 3.033 |\n", + "| [Sparsefusion](https://github.com/yichen928/SparseFusion) | 0.732 | 4.264 | **3.179** | **1.821** | 4.429 | 0.297 | 0.280 | 3.242 | 1.887 | 3.699 | 7.228 | 3.033 |\n", + "| [IS-Fusion](https://github.com/yinjunbo/IS-Fusion) | **0.740** | 3.684 | 2.291 | 1.267 | 3.890 | 0.920 | **3.994** | 1.691 | -2.351 | 4.513 | 7.177 | 2.708 |\n", "| [TransFusion](https://github.com/XuyangBai/TransFusion) | 0.708 | -7.210 | 1.799 | 1.146 | -0.552 | 0.340 | -5.412 | -3.296 | -4.220 | -3.626 | 3.850 | -1.718 |\n", "| [DeepInteraction](https://github.com/fudan-zvg/DeepInteraction) | 0.691 | -6.361 | -3.150 | -7.215 | -25.037 | -16.386 | -7.077 | -2.188 | -5.149 | 0.212 | 0.145 | -7.221 |\n" ] diff --git a/helper/calc_metrics_and_export.ipynb b/helper/calc_metrics_and_export.ipynb index b9ef779..4b6ff7f 100644 --- a/helper/calc_metrics_and_export.ipynb +++ b/helper/calc_metrics_and_export.ipynb @@ -17,7 +17,7 @@ "source": [ "MODEL_COLUMN = \"model\"\n", "BASELINE=\"BEVfusion\"\n", - "MODELS = [\"CMT\", \"DeepInteraction\", \"TransFusion\", \"Sparsefusion\", \"BEVfusion\"]\n", + "MODELS = [\"CMT\", \"DeepInteraction\", \"TransFusion\", \"Sparsefusion\", \"BEVfusion\", \"IS-Fusion\"]\n", "CORRUPTIONS=[\"beamsreducing\", \"brightness\", \"dark\", \"fog\", \"missingcamera\", \"motionblur\", \"pointsreducing\", \"snow\", \"spatialmisalignment\", \"temporalmisalignment\"]\n", "NICE_NAMES = {\"beamsreducing\": \"Beams Red.\",\n", " \"brightness\": \"Brightness\",\n", @@ -222,9 +222,33 @@ " 0.5572\n", " 0.5233\n", " \n", + " \n", + " 5\n", + " IS-Fusion\n", + " 0.7400\n", + " 0.6537\n", + " 0.5609\n", + " 0.4892\n", + " 0.6221\n", + " 0.5214\n", + " 0.4359\n", + " 0.6827\n", + " 0.6489\n", + " ...\n", + " 0.6925\n", + " 0.7198\n", + " 0.7091\n", + " 0.6917\n", + " 0.6694\n", + " 0.5068\n", + " 0.3266\n", + " 0.5619\n", + " 0.5463\n", + " 0.5115\n", + " \n", " \n", "\n", - "

5 rows × 32 columns

\n", + "

6 rows × 32 columns

\n", "" ], "text/plain": [ @@ -234,6 +258,7 @@ "2 TransFusion 0.7084 0.6306 0.5437 \n", "3 Sparsefusion 0.7315 0.6523 0.5622 \n", "4 BEVfusion 0.7144 0.6341 0.5471 \n", + "5 IS-Fusion 0.7400 0.6537 0.5609 \n", "\n", " temporalmisalignment_s3 spatialmisalignment_s1 spatialmisalignment_s2 \\\n", "0 0.4988 0.6462 0.5860 \n", @@ -241,6 +266,7 @@ "2 0.4766 0.5708 0.4761 \n", "3 0.4901 0.6061 0.5149 \n", "4 0.4085 0.5903 0.4943 \n", + "5 0.4892 0.6221 0.5214 \n", "\n", " spatialmisalignment_s3 pointsreducing_s1 pointsreducing_s2 ... dark_s3 \\\n", "0 0.5367 0.7012 0.6834 ... 0.6786 \n", @@ -248,6 +274,7 @@ "2 0.4095 0.6554 0.6190 ... 0.6978 \n", "3 0.4461 0.6881 0.6578 ... 0.6978 \n", "4 0.4266 0.6684 0.6379 ... 0.6861 \n", + "5 0.4359 0.6827 0.6489 ... 0.6925 \n", "\n", " brightness_s1 brightness_s2 brightness_s3 beamsreducing_s1 \\\n", "0 0.7038 0.6852 0.6605 0.6826 \n", @@ -255,6 +282,7 @@ "2 0.7056 0.7041 0.7007 0.6222 \n", "3 0.7205 0.7146 0.7039 0.6593 \n", "4 0.6967 0.6913 0.6851 0.6396 \n", + "5 0.7198 0.7091 0.6917 0.6694 \n", "\n", " beamsreducing_s2 beamsreducing_s3 snow_s1 snow_s2 snow_s3 \n", "0 0.5873 0.4497 0.6483 0.6190 0.5554 \n", @@ -262,8 +290,9 @@ "2 0.4591 0.2636 0.5447 0.5353 0.5087 \n", "3 0.5139 0.3380 0.5859 0.5704 0.5337 \n", "4 0.4899 0.3199 0.5782 0.5572 0.5233 \n", + "5 0.5068 0.3266 0.5619 0.5463 0.5115 \n", "\n", - "[5 rows x 32 columns]" + "[6 rows x 32 columns]" ] }, "execution_count": 17, @@ -467,9 +496,33 @@ " 0.571809\n", " 0.829685\n", " \n", + " \n", + " 5\n", + " IS-Fusion\n", + " 0.904595\n", + " 0.972703\n", + " 0.959595\n", + " 0.905000\n", + " 0.968108\n", + " 0.952162\n", + " 0.922568\n", + " 0.759324\n", + " 0.840676\n", + " ...\n", + " 0.934730\n", + " 0.935811\n", + " 0.532838\n", + " 0.929324\n", + " 0.784189\n", + " 0.769054\n", + " 0.691216\n", + " 0.589054\n", + " 0.661081\n", + " 0.821617\n", + " \n", " \n", "\n", - "

5 rows × 32 columns

\n", + "

6 rows × 32 columns

\n", "" ], "text/plain": [ @@ -479,6 +532,7 @@ "2 TransFusion 0.878317 0.996047 0.991671 \n", "3 Sparsefusion 0.901299 0.984962 0.973616 \n", "4 BEVfusion 0.895297 0.975224 0.978443 \n", + "5 IS-Fusion 0.904595 0.972703 0.959595 \n", "\n", " RA_fog_s1 RA_missingcamera_s1 RA_motionblur_s1 RA_pointsreducing_s1 \\\n", "0 0.956653 0.984911 0.944582 0.961866 \n", @@ -486,6 +540,7 @@ "2 0.918408 0.990542 0.949605 0.925184 \n", "3 0.924402 0.972932 0.956528 0.940670 \n", "4 0.917553 0.985162 0.960386 0.935610 \n", + "5 0.905000 0.968108 0.952162 0.922568 \n", "\n", " RA_snow_s1 RA_spatialmisalignment_s1 ... RA_brightness_s3 RA_dark_s3 \\\n", "0 0.889300 0.886420 ... 0.906036 0.930864 \n", @@ -493,6 +548,7 @@ "2 0.768916 0.805759 ... 0.989130 0.985037 \n", "3 0.800957 0.828571 ... 0.962269 0.953930 \n", "4 0.809351 0.826288 ... 0.958987 0.960386 \n", + "5 0.759324 0.840676 ... 0.934730 0.935811 \n", "\n", " RA_fog_s3 RA_missingcamera_s3 RA_motionblur_s3 RA_pointsreducing_s3 \\\n", "0 0.567078 0.961454 0.734842 0.875171 \n", @@ -500,6 +556,7 @@ "2 0.498165 0.980519 0.697911 0.752541 \n", "3 0.520984 0.934518 0.734655 0.798086 \n", "4 0.501680 0.962066 0.766377 0.787794 \n", + "5 0.532838 0.929324 0.784189 0.769054 \n", "\n", " RA_snow_s3 RA_spatialmisalignment_s3 RA_temporalmisalignment_s3 mRA \n", "0 0.761866 0.736214 0.684225 0.864856 \n", @@ -507,8 +564,9 @@ "2 0.718097 0.578063 0.672784 0.823970 \n", "3 0.729597 0.609843 0.669993 0.833575 \n", "4 0.732503 0.597144 0.571809 0.829685 \n", + "5 0.691216 0.589054 0.661081 0.821617 \n", "\n", - "[5 rows x 32 columns]" + "[6 rows x 32 columns]" ] }, "execution_count": 19, @@ -683,6 +741,22 @@ " 0.000000\n", " 0.000000\n", " \n", + " \n", + " 5\n", + " IS-Fusion\n", + " 0.7400\n", + " 3.684283\n", + " 2.291255\n", + " 1.266798\n", + " 3.889578\n", + " 0.919893\n", + " 3.993963\n", + " 1.690653\n", + " -2.351239\n", + " 4.512970\n", + " 7.177455\n", + " 2.707561\n", + " \n", " \n", "\n", "" @@ -694,6 +768,7 @@ "2 TransFusion 0.7084 -7.209880 1.799238 1.146380 \n", "3 Sparsefusion 0.7315 4.263833 3.178814 1.820722 \n", "4 BEVfusion 0.7144 0.000000 0.000000 0.000000 \n", + "5 IS-Fusion 0.7400 3.684283 2.291255 1.266798 \n", "\n", " RRA_fog RRA_missingcamera RRA_motionblur RRA_pointsreducing RRA_snow \\\n", "0 9.398263 2.041012 -0.840834 8.212509 9.887261 \n", @@ -701,13 +776,15 @@ "2 -0.552109 0.340169 -5.411524 -3.295704 -4.220172 \n", "3 4.429280 0.297049 0.280278 3.242202 1.887020 \n", "4 0.000000 0.000000 0.000000 0.000000 0.000000 \n", + "5 3.889578 0.919893 3.993963 1.690653 -2.351239 \n", "\n", " RRA_spatialmisalignment RRA_temporalmisalignment mRRA \n", "0 17.052673 8.448135 7.160649 \n", "1 0.211752 0.144681 -7.220680 \n", "2 -3.626257 3.849783 -1.718008 \n", "3 3.699047 7.227779 3.032602 \n", - "4 0.000000 0.000000 0.000000 " + "4 0.000000 0.000000 0.000000 \n", + "5 4.512970 7.177455 2.707561 " ] }, "execution_count": 23,