Fix minor problems when aggragating the results (#106)

desktop_env/evaluators/metrics/gimp.py

@@ -345,7 +345,10 @@ def check_structure_sim(src_path, tgt_path):
     img_src = Image.open(src_path)
     img_tgt = Image.open(tgt_path)
     structure_same = structure_check_by_ssim(img_src, img_tgt)
-    return structure_same
+    if structure_same:
+        return 1.
+    else:
+        return 0.
 
 
 def check_structure_sim_resized(src_path, tgt_path):
@@ -568,51 +571,3 @@ def check_image_file_size(src_path, rule):
         return 1.0
     else:
         return 0.0
-
-
-if __name__ == "__main__":
-    actual_config_path = "../../../cache/sessionrc_test"
-    rule = {
-        "key": "hide-docks",
-        "value": "no"
-    }
-    print(check_config_status(actual_config_path, rule))
-
-    actual_config_path = "../../../cache/action-history_test"
-    rule = {
-        "key": ["history-item", "\"filters-vignette\""],
-        "value": "1"
-    }
-    print(check_config_status(actual_config_path, rule))
-
-    actual_config_path = "../../../cache/gimprc_test"
-    rule = {
-        "key": "undo-levels",
-        "value": "100"
-    }
-    print(check_config_status(actual_config_path, rule))
-
-    src_path = "../../../cache/734d6579-c07d-47a8-9ae2-13339795476b/green_background_with_object.png"
-    tgt_path = "../../../cache/734d6579-c07d-47a8-9ae2-13339795476b/white_background_with_object.png"
-    print(check_green_background(src_path, tgt_path))
-
-    tgt_path = "../../../cache/f4aec372-4fb0-4df5-a52b-79e0e2a5d6ce/Triangle_In_The_Middle.png"
-    print(check_triangle_position(tgt_path))
-
-    src_path = "../../../cache/bb7db4c2-30b5-4be7-8dd7-b8c4ec7d3108/anmi_sharper.png"
-    tgt_path = "../../../cache/bb7db4c2-30b5-4be7-8dd7-b8c4ec7d3108/anmi.png"
-    print(check_sharper(src_path, tgt_path))
-
-    src_path = "../../../cache/3c8f201a-009d-4bbe-8b65-a6f8b35bb57f/compressed.jpeg"
-    rule = {
-        "max_size": 500000
-    }
-    print(check_image_file_size(src_path, rule))
-
-    src_path = "../../../cache/d16c99dc-2a1e-46f2-b350-d97c86c85c15/resized.png"
-    tgt_path = "../../../cache/d16c99dc-2a1e-46f2-b350-d97c86c85c15/dog_with_background.png"
-    rule = {
-        "height": 512
-    }
-    print(check_image_size(src_path, rule))
-    print(check_structure_sim_resized(src_path, tgt_path))

desktop_env/evaluators/metrics/vlc.py

@@ -186,10 +186,10 @@ def compare_videos(video_path1, video_path2, max_frames_to_check=100, threshold=
             mismatch_count += 1
             # If there's a significant difference, the frames are not the same
             if mismatch_count > threshold:
-                return False
+                return 0.
 
     # If we reach here, the content appears to be the same
-    return True
+    return 1.
 
 
 def are_audio_files_similar(mp3_file_path, mp4_file_path):
@@ -203,7 +203,7 @@ def are_audio_files_similar(mp3_file_path, mp4_file_path):
                         mp4_audio_path], check=True)
     except subprocess.CalledProcessError as e:
         print(f"An error occurred during audio extraction from MP4: {e}")
-        return False
+        return 0.
 
     # Extract audio fingerprint from the extracted audio
     mp4_fingerprint, mp4_duration = acoustid.fingerprint_file(mp4_audio_path)
@@ -213,9 +213,9 @@ def are_audio_files_similar(mp3_file_path, mp4_file_path):
 
     # Compare fingerprints (rudimentary comparison)
     if mp3_duration >= mp4_duration and mp3_fingerprint == mp4_fingerprint:
-        return True
+        return 1.
 
-    return False
+    return 0.
 
 
 def check_qt_bgcone(actual_config_path, rule):

show_result.py

@@ -25,7 +25,7 @@ def get_result(action_space, use_model, observation_type, result_dir):
                         try:
                             domain_result[domain].append(float(result))
                         except:
-                            domain_result[domain].append(float(bool(result)))
+                            domain_result[domain].append(float(eval(result)))
 
                         if domain not in all_result_for_analysis:
                             all_result_for_analysis[domain] = {}