Accomplish the exp scripts v1; Add video recording and trajectory recording of desktop agent; Fix minor bugs

desktop_env/server/README.md

@@ -78,3 +78,8 @@ Activating the window manager control requires the installation of `wmctrl`:
 ```bash
 sudo apt install wmctrl
 ```
+
+To enable recording in the virtual machine, you need to install `ffmpeg`:
+```bash
+sudo apt install ffmpeg
+```

desktop_env/server/main.py

@@ -1,6 +1,7 @@
 import ctypes
 import os
 import platform
+import shlex
 import subprocess
 from pathlib import Path
 from typing import Any, Optional
@@ -13,7 +14,7 @@ import pyautogui
 import requests
 from PIL import Image
 from Xlib import display, X
-from flask import Flask, request, jsonify, send_file, abort
+from flask import Flask, request, jsonify, send_file, abort, send_from_directory
 from lxml.etree import _Element
 from pyatspi import Accessible, StateType
 from pyatspi import Action as ATAction
@@ -29,7 +30,8 @@ pyautogui.PAUSE = 0
 pyautogui.DARWIN_CATCH_UP_TIME = 0
 
 logger = app.logger
-
+recording_process = None  # fixme: this is a temporary solution for recording, need to be changed to support multiple-process
+recording_path = "/tmp/recording.mp4"
 
 @app.route('/setup/execute', methods=['POST'])
 @app.route('/execute', methods=['POST'])
@@ -39,6 +41,9 @@ def execute_command():
     shell = data.get('shell', False)
     command = data.get('command', "" if shell else [])
 
+    if isinstance(command, str):
+        command = shlex.split(command)
+
     # Execute the command without any safety checks.
     try:
         result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=shell, text=True)
@@ -60,6 +65,9 @@ def launch_app():
     data = request.json
     command: List[str] = data.get("command", [])
 
+    if isinstance(command, str):
+        command = shlex.split(command)
+
     try:
         subprocess.Popen(command)
         return "{:} launched successfully".format(" ".join(command))
@@ -604,5 +612,42 @@ def activate_window():
     return "File opened successfully", 200
 
 
+@app.route('/start_recording', methods=['POST'])
+def start_recording():
+    global recording_process
+    if recording_process:
+        return jsonify({'status': 'error', 'message': 'Recording is already in progress.'}), 400
+
+    d = display.Display()
+    screen_width = d.screen().width_in_pixels
+    screen_height = d.screen().height_in_pixels
+
+    start_command = f"ffmpeg -y -f x11grab -draw_mouse 1 -s {screen_width}x{screen_height} -i :0.0 -c:v libx264 -r 30 {recording_path}"
+
+    recording_process = subprocess.Popen(shlex.split(start_command), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+    return jsonify({'status': 'success', 'message': 'Started recording.'})
+
+
+@app.route('/end_recording', methods=['POST'])
+def end_recording():
+    global recording_process
+
+    if not recording_process:
+        return jsonify({'status': 'error', 'message': 'No recording in progress to stop.'}), 400
+
+    recording_process.terminate()
+    recording_process.wait()
+    return_code = recording_process.returncode
+    output, error = recording_process.communicate()
+    recording_process = None
+
+    # return recording video file
+    if os.path.exists(recording_path):
+        return send_file(recording_path, as_attachment=True)
+    else:
+        return abort(404, description="Recording failed")
+
+
 if __name__ == '__main__':
     app.run(debug=True, host="0.0.0.0")