Clean code
This commit is contained in:
@@ -183,8 +183,8 @@ class DesktopEnv(gym.Env):
|
|||||||
self.path_to_vm = os.path.abspath(os.path.expandvars(os.path.expanduser(path_to_vm))) \
|
self.path_to_vm = os.path.abspath(os.path.expandvars(os.path.expanduser(path_to_vm))) \
|
||||||
if provider_name in {"vmware", "virtualbox"} else path_to_vm
|
if provider_name in {"vmware", "virtualbox"} else path_to_vm
|
||||||
else:
|
else:
|
||||||
|
|
||||||
self.path_to_vm = self.manager.get_vm_path(os_type=self.os_type, region=region, screen_size=(self.screen_width, self.screen_height))
|
self.path_to_vm = self.manager.get_vm_path(os_type=self.os_type, region=region, screen_size=(self.screen_width, self.screen_height))
|
||||||
|
|
||||||
try:
|
try:
|
||||||
self.snapshot_name = snapshot_name
|
self.snapshot_name = snapshot_name
|
||||||
self.cache_dir_base: str = cache_dir
|
self.cache_dir_base: str = cache_dir
|
||||||
|
|||||||
@@ -21,16 +21,16 @@ class AWSProviderWithProxy(Provider):
|
|||||||
super().__init__(region)
|
super().__init__(region)
|
||||||
self.current_proxy: Optional[ProxyInfo] = None
|
self.current_proxy: Optional[ProxyInfo] = None
|
||||||
|
|
||||||
# 初始化代理池
|
# Initialize proxy pool
|
||||||
if proxy_config_file:
|
if proxy_config_file:
|
||||||
init_proxy_pool(proxy_config_file)
|
init_proxy_pool(proxy_config_file)
|
||||||
logger.info(f"Initialized proxy pool from {proxy_config_file}")
|
logger.info(f"Initialized proxy pool from {proxy_config_file}")
|
||||||
|
|
||||||
# 获取下一个可用代理
|
# Get next available proxy
|
||||||
self._rotate_proxy()
|
self._rotate_proxy()
|
||||||
|
|
||||||
def _rotate_proxy(self):
|
def _rotate_proxy(self):
|
||||||
"""轮换到下一个可用代理"""
|
"""Rotate to next available proxy"""
|
||||||
proxy_pool = get_global_proxy_pool()
|
proxy_pool = get_global_proxy_pool()
|
||||||
self.current_proxy = proxy_pool.get_next_proxy()
|
self.current_proxy = proxy_pool.get_next_proxy()
|
||||||
|
|
||||||
@@ -40,7 +40,7 @@ class AWSProviderWithProxy(Provider):
|
|||||||
logger.warning("No proxy available, using direct connection")
|
logger.warning("No proxy available, using direct connection")
|
||||||
|
|
||||||
def _generate_proxy_user_data(self) -> str:
|
def _generate_proxy_user_data(self) -> str:
|
||||||
"""生成包含代理配置的user data脚本"""
|
"""Generate user data script with proxy configuration"""
|
||||||
if not self.current_proxy:
|
if not self.current_proxy:
|
||||||
return ""
|
return ""
|
||||||
|
|
||||||
@@ -107,7 +107,7 @@ echo "$(date): Configured proxy {self.current_proxy.host}:{self.current_proxy.po
|
|||||||
return base64.b64encode(user_data_script.encode()).decode()
|
return base64.b64encode(user_data_script.encode()).decode()
|
||||||
|
|
||||||
def _format_proxy_url(self, proxy: ProxyInfo) -> str:
|
def _format_proxy_url(self, proxy: ProxyInfo) -> str:
|
||||||
"""格式化代理URL"""
|
"""Format proxy URL"""
|
||||||
if proxy.username and proxy.password:
|
if proxy.username and proxy.password:
|
||||||
return f"{proxy.protocol}://{proxy.username}:{proxy.password}@{proxy.host}:{proxy.port}"
|
return f"{proxy.protocol}://{proxy.username}:{proxy.password}@{proxy.host}:{proxy.port}"
|
||||||
else:
|
else:
|
||||||
@@ -118,7 +118,7 @@ echo "$(date): Configured proxy {self.current_proxy.host}:{self.current_proxy.po
|
|||||||
ec2_client = boto3.client('ec2', region_name=self.region)
|
ec2_client = boto3.client('ec2', region_name=self.region)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# 如果实例已经存在,直接启动
|
# If instance already exists, start it directly
|
||||||
ec2_client.start_instances(InstanceIds=[path_to_vm])
|
ec2_client.start_instances(InstanceIds=[path_to_vm])
|
||||||
logger.info(f"Instance {path_to_vm} is starting...")
|
logger.info(f"Instance {path_to_vm} is starting...")
|
||||||
|
|
||||||
@@ -133,7 +133,7 @@ echo "$(date): Configured proxy {self.current_proxy.host}:{self.current_proxy.po
|
|||||||
|
|
||||||
def create_instance_with_proxy(self, image_id: str, instance_type: str,
|
def create_instance_with_proxy(self, image_id: str, instance_type: str,
|
||||||
security_groups: list, subnet_id: str) -> str:
|
security_groups: list, subnet_id: str) -> str:
|
||||||
"""创建带有代理配置的新实例"""
|
"""Create new instance with proxy configuration"""
|
||||||
ec2_client = boto3.client('ec2', region_name=self.region)
|
ec2_client = boto3.client('ec2', region_name=self.region)
|
||||||
|
|
||||||
user_data = self._generate_proxy_user_data()
|
user_data = self._generate_proxy_user_data()
|
||||||
@@ -291,7 +291,7 @@ echo "$(date): Configured proxy {self.current_proxy.host}:{self.current_proxy.po
|
|||||||
raise
|
raise
|
||||||
|
|
||||||
def get_current_proxy_info(self) -> Optional[dict]:
|
def get_current_proxy_info(self) -> Optional[dict]:
|
||||||
"""获取当前代理信息"""
|
"""Get current proxy information"""
|
||||||
if self.current_proxy:
|
if self.current_proxy:
|
||||||
return {
|
return {
|
||||||
'host': self.current_proxy.host,
|
'host': self.current_proxy.host,
|
||||||
@@ -302,7 +302,7 @@ echo "$(date): Configured proxy {self.current_proxy.host}:{self.current_proxy.po
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
def force_rotate_proxy(self):
|
def force_rotate_proxy(self):
|
||||||
"""强制轮换代理"""
|
"""Force rotate proxy"""
|
||||||
logger.info("Force rotating proxy...")
|
logger.info("Force rotating proxy...")
|
||||||
if self.current_proxy:
|
if self.current_proxy:
|
||||||
proxy_pool = get_global_proxy_pool()
|
proxy_pool = get_global_proxy_pool()
|
||||||
@@ -310,6 +310,6 @@ echo "$(date): Configured proxy {self.current_proxy.host}:{self.current_proxy.po
|
|||||||
self._rotate_proxy()
|
self._rotate_proxy()
|
||||||
|
|
||||||
def get_proxy_stats(self) -> dict:
|
def get_proxy_stats(self) -> dict:
|
||||||
"""获取代理池统计信息"""
|
"""Get proxy pool statistics"""
|
||||||
proxy_pool = get_global_proxy_pool()
|
proxy_pool = get_global_proxy_pool()
|
||||||
return proxy_pool.get_stats()
|
return proxy_pool.get_stats()
|
||||||
@@ -33,7 +33,7 @@ class ProxyPool:
|
|||||||
self.load_proxies_from_file(config_file)
|
self.load_proxies_from_file(config_file)
|
||||||
|
|
||||||
def load_proxies_from_file(self, config_file: str):
|
def load_proxies_from_file(self, config_file: str):
|
||||||
"""从配置文件加载代理列表"""
|
"""Load proxy list from config file"""
|
||||||
try:
|
try:
|
||||||
with open(config_file, 'r') as f:
|
with open(config_file, 'r') as f:
|
||||||
proxy_configs = json.load(f)
|
proxy_configs = json.load(f)
|
||||||
@@ -54,7 +54,7 @@ class ProxyPool:
|
|||||||
|
|
||||||
def add_proxy(self, host: str, port: int, username: str = None,
|
def add_proxy(self, host: str, port: int, username: str = None,
|
||||||
password: str = None, protocol: str = "http"):
|
password: str = None, protocol: str = "http"):
|
||||||
"""添加代理到池中"""
|
"""Add proxy to pool"""
|
||||||
proxy = ProxyInfo(host=host, port=port, username=username,
|
proxy = ProxyInfo(host=host, port=port, username=username,
|
||||||
password=password, protocol=protocol)
|
password=password, protocol=protocol)
|
||||||
with self.lock:
|
with self.lock:
|
||||||
@@ -62,19 +62,19 @@ class ProxyPool:
|
|||||||
logger.info(f"Added proxy {host}:{port}")
|
logger.info(f"Added proxy {host}:{port}")
|
||||||
|
|
||||||
def get_next_proxy(self) -> Optional[ProxyInfo]:
|
def get_next_proxy(self) -> Optional[ProxyInfo]:
|
||||||
"""获取下一个可用的代理"""
|
"""Get next available proxy"""
|
||||||
with self.lock:
|
with self.lock:
|
||||||
if not self.proxies:
|
if not self.proxies:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
# 过滤掉失败次数过多的代理
|
# Filter out proxies with too many failures
|
||||||
active_proxies = [p for p in self.proxies if self._is_proxy_available(p)]
|
active_proxies = [p for p in self.proxies if self._is_proxy_available(p)]
|
||||||
|
|
||||||
if not active_proxies:
|
if not active_proxies:
|
||||||
logger.warning("No active proxies available")
|
logger.warning("No active proxies available")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
# 轮询选择代理
|
# Round-robin selection of proxy
|
||||||
proxy = active_proxies[self.current_index % len(active_proxies)]
|
proxy = active_proxies[self.current_index % len(active_proxies)]
|
||||||
self.current_index += 1
|
self.current_index += 1
|
||||||
proxy.last_used = time.time()
|
proxy.last_used = time.time()
|
||||||
@@ -82,22 +82,22 @@ class ProxyPool:
|
|||||||
return proxy
|
return proxy
|
||||||
|
|
||||||
def _is_proxy_available(self, proxy: ProxyInfo) -> bool:
|
def _is_proxy_available(self, proxy: ProxyInfo) -> bool:
|
||||||
"""检查代理是否可用"""
|
"""Check if proxy is available"""
|
||||||
if not proxy.is_active:
|
if not proxy.is_active:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
if proxy.failed_count >= self.max_failures:
|
if proxy.failed_count >= self.max_failures:
|
||||||
# 检查是否过了冷却时间
|
# Check if cooldown time has passed
|
||||||
if time.time() - proxy.last_used < self.cooldown_time:
|
if time.time() - proxy.last_used < self.cooldown_time:
|
||||||
return False
|
return False
|
||||||
else:
|
else:
|
||||||
# 重置失败计数
|
# Reset failure count
|
||||||
proxy.failed_count = 0
|
proxy.failed_count = 0
|
||||||
|
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def mark_proxy_failed(self, proxy: ProxyInfo):
|
def mark_proxy_failed(self, proxy: ProxyInfo):
|
||||||
"""标记代理失败"""
|
"""Mark proxy as failed"""
|
||||||
with self.lock:
|
with self.lock:
|
||||||
proxy.failed_count += 1
|
proxy.failed_count += 1
|
||||||
if proxy.failed_count >= self.max_failures:
|
if proxy.failed_count >= self.max_failures:
|
||||||
@@ -105,13 +105,13 @@ class ProxyPool:
|
|||||||
f"(failures: {proxy.failed_count})")
|
f"(failures: {proxy.failed_count})")
|
||||||
|
|
||||||
def mark_proxy_success(self, proxy: ProxyInfo):
|
def mark_proxy_success(self, proxy: ProxyInfo):
|
||||||
"""标记代理成功"""
|
"""Mark proxy as successful"""
|
||||||
with self.lock:
|
with self.lock:
|
||||||
proxy.failed_count = 0
|
proxy.failed_count = 0
|
||||||
|
|
||||||
def test_proxy(self, proxy: ProxyInfo, test_url: str = "http://httpbin.org/ip",
|
def test_proxy(self, proxy: ProxyInfo, test_url: str = "http://httpbin.org/ip",
|
||||||
timeout: int = 10) -> bool:
|
timeout: int = 10) -> bool:
|
||||||
"""测试代理是否正常工作"""
|
"""Test if proxy is working"""
|
||||||
try:
|
try:
|
||||||
proxy_url = self._format_proxy_url(proxy)
|
proxy_url = self._format_proxy_url(proxy)
|
||||||
proxies = {
|
proxies = {
|
||||||
@@ -133,14 +133,14 @@ class ProxyPool:
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
def _format_proxy_url(self, proxy: ProxyInfo) -> str:
|
def _format_proxy_url(self, proxy: ProxyInfo) -> str:
|
||||||
"""格式化代理URL"""
|
"""Format proxy URL"""
|
||||||
if proxy.username and proxy.password:
|
if proxy.username and proxy.password:
|
||||||
return f"{proxy.protocol}://{proxy.username}:{proxy.password}@{proxy.host}:{proxy.port}"
|
return f"{proxy.protocol}://{proxy.username}:{proxy.password}@{proxy.host}:{proxy.port}"
|
||||||
else:
|
else:
|
||||||
return f"{proxy.protocol}://{proxy.host}:{proxy.port}"
|
return f"{proxy.protocol}://{proxy.host}:{proxy.port}"
|
||||||
|
|
||||||
def get_proxy_dict(self, proxy: ProxyInfo) -> Dict[str, str]:
|
def get_proxy_dict(self, proxy: ProxyInfo) -> Dict[str, str]:
|
||||||
"""获取requests库使用的代理字典"""
|
"""Get proxy dictionary for requests library"""
|
||||||
proxy_url = self._format_proxy_url(proxy)
|
proxy_url = self._format_proxy_url(proxy)
|
||||||
return {
|
return {
|
||||||
'http': proxy_url,
|
'http': proxy_url,
|
||||||
@@ -148,7 +148,7 @@ class ProxyPool:
|
|||||||
}
|
}
|
||||||
|
|
||||||
def test_all_proxies(self, test_url: str = "http://httpbin.org/ip"):
|
def test_all_proxies(self, test_url: str = "http://httpbin.org/ip"):
|
||||||
"""测试所有代理"""
|
"""Test all proxies"""
|
||||||
logger.info("Testing all proxies...")
|
logger.info("Testing all proxies...")
|
||||||
working_count = 0
|
working_count = 0
|
||||||
|
|
||||||
@@ -163,7 +163,7 @@ class ProxyPool:
|
|||||||
return working_count
|
return working_count
|
||||||
|
|
||||||
def get_stats(self) -> Dict:
|
def get_stats(self) -> Dict:
|
||||||
"""获取代理池统计信息"""
|
"""Get proxy pool statistics"""
|
||||||
with self.lock:
|
with self.lock:
|
||||||
total = len(self.proxies)
|
total = len(self.proxies)
|
||||||
active = len([p for p in self.proxies if self._is_proxy_available(p)])
|
active = len([p for p in self.proxies if self._is_proxy_available(p)])
|
||||||
@@ -176,18 +176,18 @@ class ProxyPool:
|
|||||||
'success_rate': active / total if total > 0 else 0
|
'success_rate': active / total if total > 0 else 0
|
||||||
}
|
}
|
||||||
|
|
||||||
# 全局代理池实例
|
# Global proxy pool instance
|
||||||
_proxy_pool = None
|
_proxy_pool = None
|
||||||
|
|
||||||
def get_global_proxy_pool() -> ProxyPool:
|
def get_global_proxy_pool() -> ProxyPool:
|
||||||
"""获取全局代理池实例"""
|
"""Get global proxy pool instance"""
|
||||||
global _proxy_pool
|
global _proxy_pool
|
||||||
if _proxy_pool is None:
|
if _proxy_pool is None:
|
||||||
_proxy_pool = ProxyPool()
|
_proxy_pool = ProxyPool()
|
||||||
return _proxy_pool
|
return _proxy_pool
|
||||||
|
|
||||||
def init_proxy_pool(config_file: str = None):
|
def init_proxy_pool(config_file: str = None):
|
||||||
"""初始化全局代理池"""
|
"""Initialize global proxy pool"""
|
||||||
global _proxy_pool
|
global _proxy_pool
|
||||||
_proxy_pool = ProxyPool(config_file)
|
_proxy_pool = ProxyPool(config_file)
|
||||||
return _proxy_pool
|
return _proxy_pool
|
||||||
Reference in New Issue
Block a user