From b0607c4f7909bb3e446dca94a7613ff708a3b502 Mon Sep 17 00:00:00 2001 From: Timothyxxx <384084775@qq.com> Date: Sat, 9 Mar 2024 19:32:05 +0800 Subject: [PATCH 01/12] Fix bugs imported by Xiaochuan xs --- desktop_env/evaluators/getters/file.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/desktop_env/evaluators/getters/file.py b/desktop_env/evaluators/getters/file.py index 39394d0..4b5428a 100644 --- a/desktop_env/evaluators/getters/file.py +++ b/desktop_env/evaluators/getters/file.py @@ -87,15 +87,14 @@ def get_vm_file(env, config: Dict[str, Any]) -> Union[Optional[str], List[Option paths: List[str] = [config["path"]] dests: List[str] = [config["dest"]] print(config) - if "time_suffix" in config.keys() and config["time_suffix"]: - if "time_format" in config.keys(): - time_format = config["time_format"] - # Insert time before . in file type suffix - paths = [p.split(".")[0] + datetime.now().strftime(time_format) + "." + p.split(".")[1] if "." in p else p for p in paths] - dests = [d.split(".")[0] + datetime.now().strftime(time_format) + "." + d.split(".")[1] if "." in d else d for d in dests] - else: - paths: List[str] = config["path"] - dests: List[str] = config["dest"] + + if "time_suffix" in config.keys() and config["time_suffix"]: + if "time_format" in config.keys(): + time_format = config["time_format"] + # Insert time before . in file type suffix + paths = [p.split(".")[0] + datetime.now().strftime(time_format) + "." + p.split(".")[1] if "." in p else p for p in paths] + dests = [d.split(".")[0] + datetime.now().strftime(time_format) + "." + d.split(".")[1] if "." in d else d for d in dests] + print(paths) print(dests) From 8d1264b627a111943dc0601b64d5315963941ae4 Mon Sep 17 00:00:00 2001 From: Timothyxxx <384084775@qq.com> Date: Sat, 9 Mar 2024 19:58:28 +0800 Subject: [PATCH 02/12] Update README --- README.md | 4 ++-- desktop_env/assets/icon.jpg | Bin 0 -> 8943 bytes 2 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 desktop_env/assets/icon.jpg diff --git a/README.md b/README.md index 71b5302..3e476fc 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# DesktopEnv: An Environment towards Human-like Computer Task Mastery +# OSWorld: Real Computer Environments with Open-Ended Tasks for Building Generalist Agents

SLOGAN

@@ -8,7 +8,7 @@ Paper

-![Overview](media/overview.png) +![Overview](desktop_env/assets/icon.jpg) ## Updates - 2024-03-01: We released our [paper](), [environment code](), [dataset](), and [project page](). Check it out! diff --git a/desktop_env/assets/icon.jpg b/desktop_env/assets/icon.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1879c95d566b22cef110aeaaff68fc292464e6c8 GIT binary patch literal 8943 zcmds+Wl)^Yx90~ZxWiyUh9HAG!GjJ2cTHfh!C`^~0wfR|1`8V8b&%i|Ab4R76%rN<>mZLETbYLD@t_LQ>aF z&*X)zqm!eYme*@fJAX@iN9aF;V31Q#&{5L~(9;V*pGiK0{+I1f4}csW1Aqa*!e9en zl4D?zWBlm{JO%(TaQ=GxkHi1NFfg&Oad7eQ9}y7#?a)L9z{J49!ok(?3BXw%(zS2xw^Ofb<+sIJvlaM8(7< zB&D7zDXXZesY5gj42_IUOwG)pcJ>aAPB3TqD<5A!|JMOw@53V^qoQMyQ$D1orDtSj z6&4kjl$MoOR5pBSY-(<4ZENrQ(myabG(0joGdnlGu(-60+T7Yk@9ge<-#<7#`*nVC zd3Akr`w!PYod47QJJ^5XBLB;UiH(hgjrR{12BzO%VUc6wunOT)DC*(acu}$mzr&|` zmQ+yR`-ojc|0lJr_cQ?wNOY6q^dGc;Bl|xC3;n+$`**Pa#f1V8VPX88JS=j6Jm8#w ztMKjrcj5+yRyeS5p;J(8u^sVB*6j#capUGXi2Q4wddOouJmxu!mU3r3QzBZ{+TYhqnnbJ=t zX}x^>vdg1C$^4)Lioox$q9RV^^khWC=VqYdj<01D7u5yiSR@y&%kF9YVa zlq~g`1^uzNv2K1@A`2?z@D18F3K43*D(1h(8@g4;`lZ7UOxRFsgi?D>59G6chcoT6 z3}2GWU&GhP#rhC*6;^E%fp_ydkDMaFP(Po%M(G{TkoVz|`?*c?Ms9^1BlSECRSBO| zKD!@##cgE<`KNigyPN(5Tp*!_^Jg{8KOTuJ*6)>x@x6z_^ZognKKddK0J`M8(8-;c zkrvHbB5mB=JS}Kw{!69R&)#a*JYebkx@R9JwsPR+U~l>b^(VRqqYA^5PqH0tMl(GR zn<#A3<1Iq%ctg`dY@BV zj=deDL=#h1Hsn(`8PMzK6%o(C)yheC|8#y7z*QLYFUsVtAkyKjrY56tkOfcyZo$kR zWT;O6S+8=PKLD9RPkiRi`7RN?1iiQI@yUZL98uXBE3Kj!DbW!M8ZF+!Fa^nQTctIJXk43(g>9D$OM*nu;s1|*vMB@M<06-_72Z7_J z8c!X!O&Zd4WR$BV$m34)4vnU)v}2WL^QX>l_)i{FHb4I*By&H+__)USr_XzvS~1+{ z1sJD~iEI3HPO;a;iMbp%%XbMq($8!*YPD>iL=(sqY4WHPKV|prhLYtS^-1vZg!MBW>wE=C_L8Tdj&h4u;))j?&JNb#I2w7W&9b+3M8JL!Qs ze3LLxbumj;(`l*gz50sxW`xk;M$CFc#w3l7Qqzi)IAR%6_IJP(p$ls`@Lx2nwr?`45xQYH_+{OTSa*@@yvIjhA z4)1yZb}5bXqg+^{AOpD;=YT( zGK;V++wgPVu`q1O8!$M~i=;W~aoE0AKyLKY6dpeRK-mWtV(v7_h}ubmRewwH5~h$E z(#Ytxn=Vb7Tj~DX+S(8KHS$B4_Ry?uzvWc=sVnH>@PWF`(W$vS*zN3bC)(TjqUuLF zgZJYd`FI$;GgqI_O6v6F3M~Dq0pH%vbkZ337zqv@Qf{98MxpVA%a4Xv=o)SOL6hFz zgfMr#DRWQm6Bn5f4ZgP>)06!VEZ<_pm!dTc){@sJMSrBFDMgb>eTW-0NK|0&g27PF zz08YWl7hQWI-NBcoSLhhec#g&;L%Z+1N7eYP-mC8ksEFXnN3*xYmBFyjrB*-y$UwU<7km>)BTljd5C;_IVlULOvluA{=mDv!=>bD>@SOWfPdrc)z0M zorRm@RsNzp{E?Boh@DrW*W$i)yo;lUqkZ$1*z@M6jx9}I+T5B-%_5yXQVAHb(HO!n2%#QqMT8_clzH;+$#eR6%M9k3h?+fLCuyNFcp?aR!Y+%zXU(_{q@387PoB zWBj@OE0`Wv6%%o`_FExveMHc~nwq62+kGmJrnq_tHY%9ouR zMm>HYF%WBan(OtOJKbb#VkOv!z07if?_!}hO_C|0HY)v@V+I8v z_T^SX6P-hhQI@aPubt6a9GwED$~hAqtl0e;qIwB(ifh!D)@@X|wPn}Zm$5&9e#lUX zo}Mvi_WUqjHH2r)JET*w=ia5}(@&N&wG|F#_t$563G>b;kq&jUuQDv(L+n&+MZS8C zRLx$$E6B5{Ps|^*TYD=TXd+rEQ9n!X+E%=;3-j&cQsl$}ooUOH2KZJ;q`C$${#4wH zqmLJJLMwdFG??rDgA3?FdO zu`%9IGM`xp(%$j#ShVFH)AJVpc7@v<#^UxCsc=`AKj#n_B!1xklUdSgk-Zf==GXOw z5OwREzOI-Q{#eF{6yxbklg2J_d2Wh|<9Cn1CrMb&3b75O8TeS&2ubFp8iM4|7_NNG ze-}?~EyLGhYc)Ct-|)Wh18ou^1)|OcURUH+Q(K8 zv{mA6^S2DVcrqqZV(50?JdV_X#wjgc<;SAgb?i)cCMIZ(lee1@7UA#A*F<Jv#&-JcIfw`;s-T6#1oO|>A7?pR%$IhZszQROvPx2m z>%bt22)7MEPfxFy?>@3dDbEex8A;fvt^+KYQwN+s6}b@0sEx(@X4h(mrb(A}rAe|! zUw*=Pf*Wkmd+T#(dBM&tOERiW9=}{s6ntgmGGZ2)XeJbQJS?-@?!O0tJj#_-fez63 zQVh`bmG@A4GbAzM2>QXp%)OTxKjn!coosOXccLA{G!A^Q z_D>cnRQs*bei9b`07|pV;r6t`<{5W##uNLaK{YYPBeD6R6f&tntNO&M)vVC7EiS(C z_yQbKN=Ye-L~s1_f=}xG35K0_Ru>Jj3FKl8Wi%3XV0%D(^o2Sv`2XB+0Rn(@3%DuNJt z;&AF5GduvAgp`SgkJPXZF^ro8?E%`ZO0<^SE%G->o8f+hvFZ*GRsAB#c^Viqz4j?b z%;n>yL^CQUYt=$BnRXZ*j7{eJUJf{GGl#l?+K#8}Fw*s2U{Sx@{7q>*UF5Q~!=U~| zQ$O<6TVLk)m$!=$4pL{ATfkKh&B8_ zocB^E6axd19a<#^(IBuM5iy1HkDnDZN~1vXsDiwsQ{yHx|BQZ@c)gnyd)vb6+|h(wX9=j=|p z!^*&|>8`1>f0Lgb=pc*P(j=lQajO)RW@ONj>f@R$>kWnu1d}^yTJI;N#btLAJYt_q z@NnL%ZmL>gRBf=KjqgyqHjd7eOi^}QR3WYdMjS0$cb=vgH)`U(Wyq8>_Ig`XIvYy- zR62*u?Pa9|4U?zrHPvt+{VNIu$wO1P^P>0< z8h^X^s9~afJ@H~@na1)ZG(Z)WG&;`hV5_Yyw20f_aU#C$H1fbKmkMl8wj)SXmw06> zc{^wDoX@aQP()(gW|AG1sllFwMlFll)O)epHC!yIJ^usH7|&0Ja8^`#!OY)JS0|ZG zeE-6l`Up3Fo-K#k8SJQ;F)*fG(TPihKRn6SMF~lsT2`aZUy72SPaCn;G1pq+PY|lf zVICZX5~T~vT_)l@$E=1=omxF<`<=wwr@8cIS7oJDU3ERc(}fiB^Rkj(LPmAj>#EqA zAvZMkisp{N@2Nv>+K(AU3a&vjb|B*d&P1%2pBICvc?Y|o)>q(CHyV*zBM|8$;lS;O zS@B)a|2LRcr>lEPV3=hvB-$+a=j7*^*h>zu`-}FUyDd-~a1v$Vc2s6&`z2k?IHwh2 zly0-F@s0-v*so&t5nz|_#;0`EzrHk7%57+Q5zqJjaf$v5A}g1-)?cNXY{e@Z+*iAl z9A!yDG;(uqf`Vo(3{OF};}1QA!+PzZ_}g*D)DdAJTfnW(xph6~(G{JK>5RF8@ON1? z@ce@;t9vm0&8fw3EyLTY)}PuKQZ*efXftLEyTjj?dri=K%;~uZyz*OI7v;N`xwTVl zT)7+czfpJic%^(xhNqEn9l{8q5l+7l+hfah3L$_Dsgcif;TjDwaoFQZOtfxv=Zqsp9VP(!0B_)z9u0k z<7!rq8&{-%Cg3}s!&S|Z+^HBLo%@!tVX7ZsO@)Tuk9S&yDuPN+1wcKpc5VF!pb3Q+ zT59pm$e&R9>!jCO#x1^pODrF)d+lx_TxFryXZq6>^Vb-q98_u~wQ{S9O!J zL<+YKs&z?rru@cEV-1ZD*~zn|^Ala{)V$J86QJny>o&Tm9H5FM7na-3K5g|yIu0ml zq`ZkK-O#mk;QBzfk*n>skhxi?F2!{rOwN0A`m@v-ct;`z@MqH$vV@ zeC#&apo$I_lfYd2GMHb$$9D64+b~XE9M>=T{tJZ||7@)kt=B(R!*7YaPFwuX%m*$6b#bqBE)S<3F(tP~^^~f&ji1%x2!eD;#!&Joz z4B+Z@*M8WkCUe`J!YfX5!ouN)W0}1tTP5}e!_$y5h>o)*&HuOa%~2Zx_YZgl+{gP( zhd*6*(9(x&(!N@yO*j2rP@YhNp!=74;_FkB7bzQ^uZ;_uSAr@%^|$&Hdw?zgeH|N>aXEi2y%+PUOczmLMp;oMEo8j!nu!QP>x2)^}cv3t#4LkdJOp4V$#&LP)I#yal$C)jUg#JQWTn(O_Qz z+WzJOdyKN)n=b&7l^FNSE9K-D17etdn*L!>PQ>n9QQaakq#;>3J$k8f*teAfBKOo#Jfi> z0iMn6YAydE!_q?D*gpG}V`2-#ESe$E=iY_ApBWenF{^#qscghr)WQSlW%1!$Z)^;q zo|H{dyo7PKsYt)cmjnPv1)t^Lgc>_A)xYH_P1pi|Wn9wQ zX|+6Z7?J!&0zWd@jQImFMELomW?bWdDYcsh-S0|RAsj7hfbKTNm3-$ay zE6oCTM@jXP`c*yW6WplhsrL09et~7C!*P79#MxCKC3fnZ*ISR66)(L~*>k>K+#lw; zgs)bHjX{@RSDgRmIn;06<=l?XVA-lrVVOS`KE5bcMZngE@i3w#x4ItpZJhBf{Q)e> zA6g!N+yr7*TZ`Z^isY<5_$-;Oe6*0v=WER$^opAsNyj!VjNu_9cqWWLA-nICPvUJw zX_R*Q;ZOqN{2GUWNVY1N4%!l>LcqgzZbZj;=BzQg$cyCXvovpGxX#4bRbhJf19r@Z z2pNQH_yh)q2;NvL`aD1V>??9nDMvj$Q3Chj-NOL!jOTq66WfbyqE`Rnp0h~Gz{UAh z_T84rq+gRwipE1#|&6r2r0Ph)gyv4i&qeLI3MDfXwGJ~R$f6Eb%rviedF#7 z1!^F?y?ks$)e1yC7f9`G)wx%L@oZ`xben}b9Y>IS7Zc;e%17NQDeyr|G(rTG&K6Lk z*xV=$boCGzi+;_5;EXFcvshgE?LzUZ=alb5=)xJ4qbH)i}&N1|Atg+nm#^hoc7@BDGctO23I>7#mF)b~#Vz~!V7B0E`${Yl-R%CTUQ=}YdcrOu3-k7G6|s~1QEi`nhzT9AubkrN@D4Ac-n zlviW<+OB_AeZS^j?6N<`%Id2QDWF(^eB^m1OR{|3imH`Hr0E>>@PX_h zsphFoGKL#BspqQd4OU7I-2h*IKzF^6G1Q3#gqtj=oOyo#**vWz$P7}GEIiQX4>u58 znmEQ_O;b8c7M8-IE|HHa?M&n-cDq|;V0xy8QZ{=in0-4yrzO4j8=4W(G*ga9NnR8s=N4^W`X(;bdX5}Y> zcmRPePPPx)F3)~8j(@`){BhzzH*r8iU?` z-(lC7z7kiSD!lQo^$|wU9x28}z4GsuEeC%2YBK4%!ol5*iAwgHzeQD;m0#vmYMcFC zgYYjn>q=%X{7>wJAVkfdE=*Hns`JmJabCD}_bTYo&7R-Z-e z9=^&OeTIzCjVF2Hv1QJ>LwwPuR6-n^3KD_z+r^=QI-L^AB)E7W->LwIDzPFUU({FP zR+}tsCR+llfsTf*nF|cD-eo!tcF$#y-FBJAe&Y#wRCShg#F{W^Vl(7T%G}(oLlJJ3 zTZ2}itD8N@?sNoeCO^FoWl`^E<{h$MP~oiuVO^=@LALo-TSJJ$sBM5CF!hk@Lt#Zo zpLF?)7a?VzKzPzQc0~P`Pud?hn!}8fdJyE~+CAeVLyI-GYu5?mO6bASjXn#y(ze|G z@A1SEDzlBnoP^THs|^}48hKCzH~$+I55@#$&}cRV;17Tv&Cfp!8vfz^7bz3{A*5*V z2vN-M+(xUr>$=cA@6OM`-yo{d24H1p-^guB;MgS{#$djHiE6I2Da2JH)D%cUQeV-? z5xypWBH#Yfo93C~_6oI0U9uFJLr0=&&Yfe-+SD5fjcFwK%;?c?AW1qm0?yob%yxewh4W)4k}y06-tDN+-Ka?{k;EbVk!??8E3s zD3rf>M^C{~)&C*T|JDfg>8%RdEGldu?uO{+qY|V#qsZxhs~j?VMEgL_Pbg98QJOAI zf}rQ&(rxxf9ZUzuRbpu+FH=w(7UtG3T{CwGPUFyVVM*sTkI9_=09K6kR&I7NlOJKd zUz;L7kafxPz@VjbGaLQ`K+6A4mBp!;h0$l5cZFK@Raf?hGL*B|Sg{DQ#D&e#hmKCH z_bs~}LCO0&!>Mf($E?(pTU<0FA}D5(K44u3g?rIQgu@-Wot1w&#tDA=4W4smsB$_! zLIDk_ZEhax8J5^G1~MDKd_*bUAWxs;3O~YLXz-6YwNYP&ra0YAChN@YHa3OeL0C+h zgMK`12CzSlhvzLPhlTluoFqJS!HPuul>N-ZG`{J8rI;{4+hW+7va7AJTwztvm_yM# z9CnLG?!_zIuC-rtrTvguD{7x2u;cR)s#xjZkz#a=o+Ox*WEB$3o=Mn(y`zp2nY%Ob zgB6%sC{$cJWxl=n=q^Fu%@%JbjIE1)*(}ADr@P}l=V}r$x`4Mq-08G5XyoZUKxO#h epOeLZNo%pcr*90(|K*D7U$49V-?mBrEdMum!iG}- literal 0 HcmV?d00001 From bec82726bc46e53b913b15c09c7de9d6aa556c2a Mon Sep 17 00:00:00 2001 From: Timothyxxx <384084775@qq.com> Date: Sat, 9 Mar 2024 20:23:55 +0800 Subject: [PATCH 03/12] Update README --- README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3e476fc..8eb867f 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ -# OSWorld: Real Computer Environments with Open-Ended Tasks for Building Generalist Agents +# OSWorld: Open-Ended Tasks in Real Computer Environments +

+ Logo +
SLOGAN

@@ -8,7 +11,7 @@ Paper

-![Overview](desktop_env/assets/icon.jpg) +![Overview]() ## Updates - 2024-03-01: We released our [paper](), [environment code](), [dataset](), and [project page](). Check it out! From 447c886b0a8c084ef3f0c24bfad2e72a1ae268f8 Mon Sep 17 00:00:00 2001 From: Timothyxxx <384084775@qq.com> Date: Sat, 9 Mar 2024 20:54:52 +0800 Subject: [PATCH 04/12] Fix multiple apps 5990457f-2adb-467b-a4af-5c857c92d762 --- desktop_env/evaluators/getters/chrome.py | 18 ++++++------------ desktop_env/evaluators/getters/file.py | 9 --------- .../5990457f-2adb-467b-a4af-5c857c92d762.json | 11 +---------- 3 files changed, 7 insertions(+), 31 deletions(-) diff --git a/desktop_env/evaluators/getters/chrome.py b/desktop_env/evaluators/getters/chrome.py index 0304827..13a5385 100644 --- a/desktop_env/evaluators/getters/chrome.py +++ b/desktop_env/evaluators/getters/chrome.py @@ -82,34 +82,28 @@ def get_info_from_website(env, config: Dict[Any, Any]) -> Any: page.wait_for_load_state('load') action = info_dict.get('action', 'inner_text') if action == "inner_text": - ele = page.locator(info_dict['selector']) - expect(ele).to_be_visible() + ele = page.wait_for_selector(info_dict['selector'], state='attached', timeout=10000) infos.append(ele.inner_text()) elif action == "attribute": - ele = page.locator(info_dict['selector']) - expect(ele).to_be_visible() + ele = page.wait_for_selector(info_dict['selector'], state='attached', timeout=10000) infos.append(ele.get_attribute(info_dict['attribute'])) elif action == 'click_and_inner_text': for idx, sel in enumerate(info_dict['selector']): if idx != len(info_dict['selector']) - 1: - link = page.locator(sel) - expect(link).to_be_visible() + link = page.wait_for_selector(sel, state='attached', timeout=10000) link.click() page.wait_for_load_state('load') else: - ele = page.locator(sel) - expect(ele).to_be_visible() + ele = page.wait_for_selector(sel, state='attached', timeout=10000) infos.append(ele.inner_text()) elif action == 'click_and_attribute': for idx, sel in enumerate(info_dict['selector']): if idx != len(info_dict['selector']) - 1: - link = page.locator(sel) - expect(link).to_be_visible() + link = page.wait_for_selector(sel, state='attached', timeout=10000) link.click() page.wait_for_load_state('load') else: - ele = page.locator(sel) - expect(ele).to_be_visible() + ele = page.wait_for_selector(sel, state='attached') infos.append(ele.get_attribute(info_dict['attribute'])) else: raise NotImplementedError(f'The action {action} is not supported yet.') diff --git a/desktop_env/evaluators/getters/file.py b/desktop_env/evaluators/getters/file.py index 4b5428a..d6e2087 100644 --- a/desktop_env/evaluators/getters/file.py +++ b/desktop_env/evaluators/getters/file.py @@ -95,19 +95,12 @@ def get_vm_file(env, config: Dict[str, Any]) -> Union[Optional[str], List[Option paths = [p.split(".")[0] + datetime.now().strftime(time_format) + "." + p.split(".")[1] if "." in p else p for p in paths] dests = [d.split(".")[0] + datetime.now().strftime(time_format) + "." + d.split(".")[1] if "." in d else d for d in dests] - print(paths) - print(dests) - cache_paths: List[str] = [] gives: Set[int] = set(config.get("gives", [0])) for i, (p, d) in enumerate(zip(paths, dests)): - print("env cache_dir: ") - print(env.cache_dir) _path = os.path.join(env.cache_dir, d) - print("_path: ") - print(_path) file = env.controller.get_file(p) if file is None: #return None @@ -121,8 +114,6 @@ def get_vm_file(env, config: Dict[str, Any]) -> Union[Optional[str], List[Option with open(_path, "wb") as f: f.write(file) # debug - print("cache_paths") - print(cache_paths) return cache_paths[0] if len(cache_paths)==1 else cache_paths diff --git a/evaluation_examples/examples/multi_apps/5990457f-2adb-467b-a4af-5c857c92d762.json b/evaluation_examples/examples/multi_apps/5990457f-2adb-467b-a4af-5c857c92d762.json index 9d754dd..eb77ab5 100644 --- a/evaluation_examples/examples/multi_apps/5990457f-2adb-467b-a4af-5c857c92d762.json +++ b/evaluation_examples/examples/multi_apps/5990457f-2adb-467b-a4af-5c857c92d762.json @@ -23,15 +23,6 @@ ] } }, - { - "type": "chrome_open_tabs", - "parameters": { - "urls_to_open": [ - "https://arxiv.org/abs/2005.14165", - "https://wallhaven.cc/" - ] - } - }, { "type": "download", "parameters": { @@ -105,7 +96,7 @@ "147", "372", "Deep learning", - "https://creativecoding.soe.ucsc.edu/courses/cs523/slides/week3/DeepLearning_LeCun.pdf" + "https://hal.science/hal-04206682/document" ] }, "options": { From b0854e519cf1108192a268048a23dbe5fe2c2ecd Mon Sep 17 00:00:00 2001 From: Timothyxxx <384084775@qq.com> Date: Sat, 9 Mar 2024 21:20:44 +0800 Subject: [PATCH 05/12] Minor fix on instruction of 81c425f5-78f3-4771-afd6-3d2973825947 --- .../multi_apps/81c425f5-78f3-4771-afd6-3d2973825947.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evaluation_examples/examples/multi_apps/81c425f5-78f3-4771-afd6-3d2973825947.json b/evaluation_examples/examples/multi_apps/81c425f5-78f3-4771-afd6-3d2973825947.json index 762fcc3..e564020 100644 --- a/evaluation_examples/examples/multi_apps/81c425f5-78f3-4771-afd6-3d2973825947.json +++ b/evaluation_examples/examples/multi_apps/81c425f5-78f3-4771-afd6-3d2973825947.json @@ -1,7 +1,7 @@ { "id": "81c425f5-78f3-4771-afd6-3d2973825947", "snapshot": "libreoffice_calc", - "instruction": "Can you assist me in transferring the data from LibreOffice Calc in this file to a LibreOffice Writer table while preserving the original format as in calc file? Save the document as \"price.docx\" on the desktop.", + "instruction": "Can you assist me in transferring the data from LibreOffice Calc in the current sheet to a LibreOffice Writer table while preserving the original format as in calc file? Save the document as \"price.docx\" on the desktop.", "source": "authors", "config": [ { From e481afcf5cd0b9af35ebb88c1f04f8054158023d Mon Sep 17 00:00:00 2001 From: Timothyxxx <384084775@qq.com> Date: Sat, 9 Mar 2024 23:01:22 +0800 Subject: [PATCH 06/12] Fix multiple examples --- desktop_env/evaluators/metrics/calc.py | 38 +++++++++++++------ desktop_env/evaluators/metrics/chrome.py | 11 +++++- .../6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a.json | 4 +- .../873cafdd-a581-47f6-8b33-b9696ddb7b05.json | 2 +- 4 files changed, 39 insertions(+), 16 deletions(-) diff --git a/desktop_env/evaluators/metrics/calc.py b/desktop_env/evaluators/metrics/calc.py index 701be27..0ff0744 100644 --- a/desktop_env/evaluators/metrics/calc.py +++ b/desktop_env/evaluators/metrics/calc.py @@ -1,8 +1,13 @@ +import logging +from typing import List + import openpyxl -def compare_conference_city_in_order( actual_city_list_path, expected_city): +logger = logging.getLogger("desktopenv.metrics.calc") + + +def compare_conference_city_in_order(actual_city_list_path, expected_city): expected_city_list = expected_city["expected"] - print(f"Reading csv file from {actual_city_list_path}") wb = openpyxl.load_workbook(actual_city_list_path) sheet = wb.active actual_city_list = [] @@ -12,16 +17,25 @@ def compare_conference_city_in_order( actual_city_list_path, expected_city): # expected_city is the city that we want to compare with the actual city list # must in order index # debug - print("expected_city_list:") - print(expected_city_list) - print("actual_city_list_path:") - print(actual_city_list) - wrong_list = [] try: for i in range(len(actual_city_list)): - if expected_city_list[i] not in actual_city_list[i]: - wrong_list.append(i) - print(f"Expected city {expected_city_list[i]}; Actual city {actual_city_list[i]}") + if isinstance(expected_city_list[i], str): + if expected_city_list[i] not in actual_city_list[i]: + logger.debug(f"Expected city {expected_city_list[i]}; Actual city {actual_city_list[i]}") + print(f"Expected city {expected_city_list[i]}; Actual city {actual_city_list[i]}") + return 0. + + + elif isinstance(expected_city_list[i], List): + if not any(possible_str in actual_city_list[i] for possible_str in expected_city_list[i]): + logger.debug(f"Expected city {expected_city_list[i]}; Actual city {actual_city_list[i]}") + print(f"Expected city {expected_city_list[i]}; Actual city {actual_city_list[i]}") + return 0. + + else: + raise TypeError("Expected city should be a string or a list of strings") + except: - return False - return True if len(wrong_list) == 0 else False \ No newline at end of file + return 0. + + return 1. diff --git a/desktop_env/evaluators/metrics/chrome.py b/desktop_env/evaluators/metrics/chrome.py index 5dc2a48..c67ec09 100644 --- a/desktop_env/evaluators/metrics/chrome.py +++ b/desktop_env/evaluators/metrics/chrome.py @@ -65,7 +65,16 @@ def is_expected_installed_extensions(installed_extensions, expected) -> float: print("installed_extensions: ") print(installed_extensions) expected_extensions = expected["expected"] - return 1 if expected_extensions == installed_extensions else 0. # must equal, no additional extensions allowed + + # whether the expected extensions are installed + set_expected_extensions = set(expected_extensions) + set_installed_extensions = set(installed_extensions) + + if set_expected_extensions.issubset(set_installed_extensions): + return 1. + else: + return 0. + def is_expected_tabs(open_tabs: List[Dict[str, str]], rule: Dict[str, Any]) -> float: """ diff --git a/evaluation_examples/examples/multi_apps/6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a.json b/evaluation_examples/examples/multi_apps/6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a.json index 4792da5..414aab4 100644 --- a/evaluation_examples/examples/multi_apps/6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a.json +++ b/evaluation_examples/examples/multi_apps/6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a.json @@ -51,7 +51,7 @@ "command": [ "python", "-c", - "import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");" + "import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5);" ] } } @@ -60,7 +60,7 @@ "expected": { "type": "rule", "rules":{ - "expected": ["Scottsdale","Atlanta","Lake Tahoe","Banff","Beijing","Montreal","San Diego","Lille","Montreal","San Juan","New York","Barcelona","Toulon","Sydney","Long Beach","Vancouver","Stockholm","Montréal","New Orleans","Long Beach","Vancouver"] + "expected": ["Scottsdale","Atlanta","Lake Tahoe","Banff","Beijing",["Montreal", "Montréal"],"San Diego","Lille",["Montreal", "Montréal"],"San Juan",["New York", "New York City", "NYC"],"Barcelona","Toulon","Sydney","Long Beach","Vancouver","Stockholm",["Montreal", "Montréal"],"New Orleans","Long Beach","Vancouver"] } }, "result": { diff --git a/evaluation_examples/examples/multi_apps/873cafdd-a581-47f6-8b33-b9696ddb7b05.json b/evaluation_examples/examples/multi_apps/873cafdd-a581-47f6-8b33-b9696ddb7b05.json index 2d9f175..f1cb660 100644 --- a/evaluation_examples/examples/multi_apps/873cafdd-a581-47f6-8b33-b9696ddb7b05.json +++ b/evaluation_examples/examples/multi_apps/873cafdd-a581-47f6-8b33-b9696ddb7b05.json @@ -56,7 +56,7 @@ "expected": { "type": "rule", "rules":{ - "expected": ["Zoom Chrome Extension", "Speechify Text to Speech Voice Reader", "React Developer Tools", "Momentum", "Google Translate", "Web Store", "Chromium PDF Viewer", "Google Hangouts"] + "expected": ["Zoom Chrome Extension", "Speechify Text to Speech Voice Reader", "React Developer Tools", "Momentum", "Google Translate"] } }, "result": { From a12dfacbd72b5f2818d56d46780fa221563bb31c Mon Sep 17 00:00:00 2001 From: Timothyxxx <384084775@qq.com> Date: Sun, 10 Mar 2024 12:10:05 +0800 Subject: [PATCH 07/12] Add more time for waiting impress in 47f7c0ce-a5fb-4100-a5e6-65cd0e7429e5 --- .../multi_apps/47f7c0ce-a5fb-4100-a5e6-65cd0e7429e5.json | 6 ++++++ .../multi_apps/9f3bb592-209d-43bc-bb47-d77d9df56504.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/evaluation_examples/examples/multi_apps/47f7c0ce-a5fb-4100-a5e6-65cd0e7429e5.json b/evaluation_examples/examples/multi_apps/47f7c0ce-a5fb-4100-a5e6-65cd0e7429e5.json index b778489..2cc3cdb 100644 --- a/evaluation_examples/examples/multi_apps/47f7c0ce-a5fb-4100-a5e6-65cd0e7429e5.json +++ b/evaluation_examples/examples/multi_apps/47f7c0ce-a5fb-4100-a5e6-65cd0e7429e5.json @@ -25,6 +25,12 @@ "path": "/home/user/Desktop/Robotic_Workshop_Infographics.pptx" } }, + { + "type": "sleep", + "parameters": { + "seconds": 3 + } + }, { "type": "launch", "parameters": { diff --git a/evaluation_examples/examples/multi_apps/9f3bb592-209d-43bc-bb47-d77d9df56504.json b/evaluation_examples/examples/multi_apps/9f3bb592-209d-43bc-bb47-d77d9df56504.json index 58c7ca7..89f5a21 100644 --- a/evaluation_examples/examples/multi_apps/9f3bb592-209d-43bc-bb47-d77d9df56504.json +++ b/evaluation_examples/examples/multi_apps/9f3bb592-209d-43bc-bb47-d77d9df56504.json @@ -30,7 +30,7 @@ ], "trajectory": "trajectories/", "related_apps": [ - "os", "vlc" + "os", "vlc", "ubuntu_media_player" ], "evaluator": { "postconfig":[ From 775cef744f5d65e3aeadfc319350ec96ab3eb845 Mon Sep 17 00:00:00 2001 From: Jason Lee Date: Sun, 10 Mar 2024 14:48:56 +0800 Subject: [PATCH 08/12] xiaochuan correct his bugs in multiapp examples, you can try it again now --- desktop_env/evaluators/getters/file.py | 6 - desktop_env/evaluators/metrics/__init__.py | 2 +- desktop_env/evaluators/metrics/general.py | 108 +++++++++++------- .../2373b66a-092d-44cb-bfd7-82e86e7a3b4d.json | 12 +- .../7ff48d5b-2df2-49da-b500-a5150ffc7f18.json | 18 +++ .../ce2b64a2-ddc1-4f91-8c7d-a88be7121aac.json | 8 +- .../da922383-bfa4-4cd3-bbad-6bebab3d7742.json | 2 +- 7 files changed, 97 insertions(+), 59 deletions(-) diff --git a/desktop_env/evaluators/getters/file.py b/desktop_env/evaluators/getters/file.py index 39394d0..6b283a9 100644 --- a/desktop_env/evaluators/getters/file.py +++ b/desktop_env/evaluators/getters/file.py @@ -86,7 +86,6 @@ def get_vm_file(env, config: Dict[str, Any]) -> Union[Optional[str], List[Option if not config.get("multi", False): paths: List[str] = [config["path"]] dests: List[str] = [config["dest"]] - print(config) if "time_suffix" in config.keys() and config["time_suffix"]: if "time_format" in config.keys(): time_format = config["time_format"] @@ -96,8 +95,6 @@ def get_vm_file(env, config: Dict[str, Any]) -> Union[Optional[str], List[Option else: paths: List[str] = config["path"] dests: List[str] = config["dest"] - print(paths) - print(dests) cache_paths: List[str] = [] @@ -121,9 +118,6 @@ def get_vm_file(env, config: Dict[str, Any]) -> Union[Optional[str], List[Option cache_paths.append(_path) with open(_path, "wb") as f: f.write(file) - # debug - print("cache_paths") - print(cache_paths) return cache_paths[0] if len(cache_paths)==1 else cache_paths diff --git a/desktop_env/evaluators/metrics/__init__.py b/desktop_env/evaluators/metrics/__init__.py index 6fdcf8d..c3f7f85 100644 --- a/desktop_env/evaluators/metrics/__init__.py +++ b/desktop_env/evaluators/metrics/__init__.py @@ -68,7 +68,7 @@ from .general import ( compare_time_in_speedtest_results, is_included_all_json_objects, is_gold_text_included_in_pdf, - check_csv_line_number, + check_line_number, file_contains, compare_terminal_and_txt, fuzzy_place_math, diff --git a/desktop_env/evaluators/metrics/general.py b/desktop_env/evaluators/metrics/general.py index 7ff7bf2..a764810 100644 --- a/desktop_env/evaluators/metrics/general.py +++ b/desktop_env/evaluators/metrics/general.py @@ -1,4 +1,5 @@ import csv +import os import datetime import difflib import functools @@ -91,23 +92,24 @@ def fuzzy_match(result, rules) -> float: def fuzzy_place_math(result_file_path, rules) -> float: + if result_file_path is None: + return 0. expect = rules["expected"] # a list of possible answers # read list.docx, and get all texts out, overlook blank lines, remove blanks before and after each line doc = Document(result_file_path) words_list = [] for para in doc.paragraphs: words_list.extend(para.text.split()) - # Print out the list of extracted words - print("Your Answers: ") - print(words_list) fuzzy_score_list = [] for word in words_list: max_score = 0 for ans in expect: - score = fuzz.ratio(word, ans) + score = fuzz.ratio(word, ans)/100 max_score = max(max_score, score) fuzzy_score_list.append(max_score) - return sum(fuzzy_score_list) / len(fuzzy_score_list) + if len(fuzzy_score_list) != 3: + return 0. + return sum(fuzzy_score_list) / 3 def check_csv(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float: """ @@ -304,29 +306,40 @@ def check_direct_json_object(result, rules) -> float: result = result.replace("'", '"') # load json object result = json.loads(result) - print("result: ") - print(result) - print("expected: ") - print(rules["expected"]) if result is None: return 0. + try: + expect_in_result = rules.get("expect_in_result", False) + if not expect_in_result: + expected_json = rules["expected"] + for key in expected_json.keys(): + expected_value = expected_json.get(key) + if expected_value != result.get(key): + return 0. + return 1.0 + else: + expected_json = rules["expected"] - expect_in_result = rules.get("expect_in_result", False) - if not expect_in_result: - expected_json = rules["expected"] - for key in expected_json.keys(): - expected_value = expected_json.get(key) - if expected_value != result.get(key): - return 0. - return 1.0 - else: - expected_json = rules["expected"] - for key in expected_json.keys(): - expected_value = expected_json.get(key) - if expected_value not in result.get(key): - return 0. - return 1.0 - + for key in expected_json.keys(): + if isinstance(expected_json.get(key), List): + flag = 0 + expected_value_list = expected_json.get(key) + for each_expected_value in expected_value_list: + if each_expected_value in result.get(key): + flag = 1 + break + if flag == 0: + return 0. + elif isinstance(expected_json.get(key), str): + if expected_json.get(key) not in result.get(key): + return 0. + else: + logger.debug("check_direct_json_object: expected value type not supported") + return 0. + return 1.0 + except: + logger.debug("check_direct_json_object: result is not a valid json object") + return 0. def compare_time_in_speedtest_results(speedtest_result_path, time_diff): if not speedtest_result_path: @@ -397,24 +410,37 @@ def is_gold_text_included_in_pdf(pdf_file_path, gold_text_path): def file_contains(file_path, config): # file_path ends with .txt if not file_path: - return 1 - with open(file_path, 'r') as f: - file_text = f.read() - for text in config["expected"]: - if text not in file_text: - return 0 - return 1 + return 0. + try: + with open(file_path, 'r') as f: + file_text = f.read() + for text in config["expected"]: + if text not in file_text: + logger.debug(f"file_contains: {text} not found in {file_path}") + return 0. + except: + logger.debug("file_contains: file not found or not readable") + return 0. + return 1. -def check_csv_line_number(file_path, line_number): - # check file_path suffix - if not file_path.endswith('.csv'): - return 0 - # check line number - with open(file_path, 'r') as f: - reader = csv.reader(f) - line_count = sum(1 for row in reader) - return 1 if line_count == int(line_number["expected"]) else 0 +def check_line_number(file_path, line_number): + # check if file_path exists + if file_path is None or not os.path.isfile(file_path): + return 0. + timeRegex = "([01]\d|2[0-3]):[0-5]\d:[0-5]\d" + # check if the string that matches the timeRegex in this txt file equals to line_number["expected"] + try: + with open(file_path, 'r') as f: + line_count = 0 + for line in f: + if re.search(timeRegex, line): + line_count += 1 + # if line_count equals to line_number["expected"], return 1, else return 0 + return 1 if line_count == int(line_number["expected"]) else 0 + except: + logger.debug("check_line_number: file not found or not readable") + return 0. def compare_terminal_and_txt(txt_file_path, terminal_output): diff --git a/evaluation_examples/examples/multi_apps/2373b66a-092d-44cb-bfd7-82e86e7a3b4d.json b/evaluation_examples/examples/multi_apps/2373b66a-092d-44cb-bfd7-82e86e7a3b4d.json index 8cf2e38..4e76c1d 100644 --- a/evaluation_examples/examples/multi_apps/2373b66a-092d-44cb-bfd7-82e86e7a3b4d.json +++ b/evaluation_examples/examples/multi_apps/2373b66a-092d-44cb-bfd7-82e86e7a3b4d.json @@ -1,7 +1,7 @@ { "id": "2373b66a-092d-44cb-bfd7-82e86e7a3b4d", "snapshot": "multiapps", - "instruction": "I want to understand the resource usage of my Ubuntu system under normal workloads. Please use the `sar` command in the `sysstat` toolkit to monitor the system CPU usage, evaluate the status once every second for 30 seconds, output the results to \"System_Resources_Report.txt\" under Desktop, and convert the txt to csv file with the same name.", + "instruction": "I want to understand the resource usage of my Ubuntu system under normal workloads. Please use the `sar` command in the `sysstat` toolkit to monitor system activity, evaluate the status once every second for 30 seconds, output the results to \"System_Resources_Report.txt\" under Desktop.", "source": "author", "config": [ { @@ -17,7 +17,7 @@ "os", "calc" ], "evaluator": { - "func": ["file_contains", "check_csv_line_number"], + "func": ["file_contains", "check_line_number"], "result": [ { @@ -27,8 +27,8 @@ }, { "type": "vm_file", - "path": "/home/user/Desktop/System_Resources_Report.csv", - "dest": "System_Resources_Report.csv" + "path": "/home/user/Desktop/System_Resources_Report.txt", + "dest": "System_Resources_Report.txt" } ], "expected": @@ -36,13 +36,13 @@ { "type": "rule", "rules" :{ - "expected": ["tps", "rtps", "wtps", "dtps", "bread/s", "bwrtn/s", "bdscd/s", "Average", "Linux"] + "expected": ["CPU", "%user","%nice","%system", "%iowait", "%steal", "%idle"] } }, { "type": "rule", "rules": { - "expected": "33" + "expected": "31" } }] } diff --git a/evaluation_examples/examples/multi_apps/7ff48d5b-2df2-49da-b500-a5150ffc7f18.json b/evaluation_examples/examples/multi_apps/7ff48d5b-2df2-49da-b500-a5150ffc7f18.json index 07d4c0d..4ce5836 100644 --- a/evaluation_examples/examples/multi_apps/7ff48d5b-2df2-49da-b500-a5150ffc7f18.json +++ b/evaluation_examples/examples/multi_apps/7ff48d5b-2df2-49da-b500-a5150ffc7f18.json @@ -33,6 +33,24 @@ "chrome", "os", "writer" ], "evaluator": { + "postconfig":[ + { + "type": "activate_window", + "parameters": { + "window_name": "AllLocations.docx - LibreOffice Writer" + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");" + ] + } + } + ], "func": "fuzzy_place_math", "result": { "type": "vm_file", diff --git a/evaluation_examples/examples/multi_apps/ce2b64a2-ddc1-4f91-8c7d-a88be7121aac.json b/evaluation_examples/examples/multi_apps/ce2b64a2-ddc1-4f91-8c7d-a88be7121aac.json index 9628868..95a068e 100644 --- a/evaluation_examples/examples/multi_apps/ce2b64a2-ddc1-4f91-8c7d-a88be7121aac.json +++ b/evaluation_examples/examples/multi_apps/ce2b64a2-ddc1-4f91-8c7d-a88be7121aac.json @@ -20,7 +20,7 @@ "parameters": { "files": [ { - "url": "https://drive.usercontent.google.com/download?id=1iquZNI4ktx2915srQ1MMlhcwCuXnmq5T&export=download&authuser=0&confirm=t&uuid=e0bfd86f-a0f1-4a9e-8344-0f645eb379dd&at=APZUnTWQnCd7bNtf8ZN4KfkQUaJ9:1709972645590", + "url": "https://drive.usercontent.google.com/download?id=1XLBhnopqQrTw3bxRWOyfTCslfawFyOaZ&export=download&authuser=0&confirm=t&uuid=f41a09c1-8319-4a50-b8c3-4fc58ba51c00&at=APZUnTV3uiv7wKiSvoCekJG_aVo3:1710051227101", "path": "/home/user/Pictures/picture2.jpg" } @@ -68,9 +68,9 @@ "type": "rule", "rules":{ "expected":{ - "ec076282f61ba74642e94b5a6a1250c6988204d59d9b02936606b6b8ef1e4433": "Kilimanjaro", - "999957c8de835bfa420d198270e7a6b079ee20ff53a3f214491e8134768a7c0b": "Himalayas", - "79f45d40d8413d4e81f1b9734ea39e58622cafd79e12bab32959643fc245147c": "Hua" + "ec076282f61ba74642e94b5a6a1250c6988204d59d9b02936606b6b8ef1e4433": ["Kilimanjaro"], + "6ed4239ecc2be3ec15ad65a78c5c823b9004d640b8cc83a6a7af5930f354de91": ["Himalayas", "Everest", "Sagarmatha"], + "79f45d40d8413d4e81f1b9734ea39e58622cafd79e12bab32959643fc245147c": ["Hua", "hua"] }, "expect_in_result": true } diff --git a/evaluation_examples/examples/multi_apps/da922383-bfa4-4cd3-bbad-6bebab3d7742.json b/evaluation_examples/examples/multi_apps/da922383-bfa4-4cd3-bbad-6bebab3d7742.json index c751d7b..2ec27c1 100644 --- a/evaluation_examples/examples/multi_apps/da922383-bfa4-4cd3-bbad-6bebab3d7742.json +++ b/evaluation_examples/examples/multi_apps/da922383-bfa4-4cd3-bbad-6bebab3d7742.json @@ -1,7 +1,7 @@ { "id": "da922383-bfa4-4cd3-bbad-6bebab3d7742", "snapshot": "multiapps", - "instruction": "I browsed a lot of interesting blog articles today. I hope to store these articles in my local designated folder just like zotero stores papers. Please download the blogs you are opening now in pdf format to /home/user/Documents/Blogs.", + "instruction": "I browsed a lot of interesting blog articles today. I hope to store these articles in my local designated folder just like zotero stores papers. Please download the blogs opening now in pdf format and save them in their tile to /home/user/Documents/Blogs.", "source": "authors", "config": [ { From 8fd27f155ecbe0fae548e238ed62590264c5cb60 Mon Sep 17 00:00:00 2001 From: Timothyxxx <384084775@qq.com> Date: Sun, 10 Mar 2024 15:16:28 +0800 Subject: [PATCH 09/12] Fix bugs in multiple apps example 0e53 --- .../multi_apps/0e5303d4-8820-42f6-b18d-daf7e633de21.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/evaluation_examples/examples/multi_apps/0e5303d4-8820-42f6-b18d-daf7e633de21.json b/evaluation_examples/examples/multi_apps/0e5303d4-8820-42f6-b18d-daf7e633de21.json index abc1284..f8ecc50 100644 --- a/evaluation_examples/examples/multi_apps/0e5303d4-8820-42f6-b18d-daf7e633de21.json +++ b/evaluation_examples/examples/multi_apps/0e5303d4-8820-42f6-b18d-daf7e633de21.json @@ -90,6 +90,10 @@ "type": "cloud_file", "path": "https://drive.usercontent.google.com/download?id=1Ej2iHG8p-QJe7FZQpPIIS82BHOlFAUQM&export=download&authuser=0&confirm=t", "dest": "gold_lecture_slides.zip" + }, + "options": { + "file_path": "lecture_slides", + "file_type": "pdf" } } } \ No newline at end of file From e51d0e8cc985fcb14cb7670258cec8ab0bb23b6b Mon Sep 17 00:00:00 2001 From: Timothyxxx <384084775@qq.com> Date: Sun, 10 Mar 2024 15:18:14 +0800 Subject: [PATCH 10/12] Fix bugs in multiple apps example 0e53 --- desktop_env/evaluators/metrics/chrome.py | 31 +++++++++++++++++++----- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/desktop_env/evaluators/metrics/chrome.py b/desktop_env/evaluators/metrics/chrome.py index c67ec09..3054bee 100644 --- a/desktop_env/evaluators/metrics/chrome.py +++ b/desktop_env/evaluators/metrics/chrome.py @@ -219,7 +219,10 @@ def compare_archive(pred_path: str, gold_path: str, **kwargs) -> float: """ Compare two archives. Note that the files in the archives should be of the same type. """ - if not pred_path: return 0. + file_path = kwargs.pop('file_path', '') + + if not pred_path: + return 0. pred_folder = os.path.splitext(pred_path)[0] + '_pred' gold_folder = os.path.splitext(gold_path)[0] + '_gold' @@ -227,13 +230,16 @@ def compare_archive(pred_path: str, gold_path: str, **kwargs) -> float: shutil.rmtree(pred_folder, ignore_errors=True) os.makedirs(pred_folder) shutil.unpack_archive(pred_path, pred_folder) + if not os.path.exists(gold_folder): # use cache if exists os.makedirs(gold_folder) shutil.unpack_archive(gold_path, gold_folder) - pred_files = sorted(os.listdir(pred_folder)) - gold_files = sorted(os.listdir(gold_folder)) - if pred_files != gold_files: return 0. + pred_files = sorted(os.listdir(os.path.join(pred_folder, file_path))) + gold_files = sorted(os.listdir(os.path.join(gold_folder, file_path))) + + if pred_files != gold_files: + return 0. def get_compare_function(): file_type = kwargs.pop('file_type', 'text') @@ -269,8 +275,8 @@ def compare_archive(pred_path: str, gold_path: str, **kwargs) -> float: score = 0 compare_function = get_compare_function() for f1, f2 in zip(pred_files, gold_files): - fp1 = os.path.join(pred_folder, f1) - fp2 = os.path.join(gold_folder, f2) + fp1 = os.path.join(pred_folder, file_path, f1) + fp2 = os.path.join(gold_folder, file_path, f2) score += compare_function(fp1, fp2, **kwargs) return score / len(pred_files) @@ -390,3 +396,16 @@ def is_added_to_steam_cart(active_tab_info, rule): return 0. return 1. + + +if __name__ == '__main__': + result = compare_archive( + r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\0e5303d4-8820-42f6-b18d-daf7e633de21\lecture_slides.zip", + r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\0e5303d4-8820-42f6-b18d-daf7e633de21\gold_lecture_slides.zip", + + **{ + "file_path": "lecture_slides", + "file_type": "pdf" + }) + + print(result) From 3f19cc511764369b0cc4910cec8a834a8b5a803c Mon Sep 17 00:00:00 2001 From: Timothyxxx <384084775@qq.com> Date: Sun, 10 Mar 2024 17:06:39 +0800 Subject: [PATCH 11/12] Fix bugs in chrome example --- .../chrome/9656a811-9b5b-4ddf-99c7-5117bcef0626.json | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/evaluation_examples/examples/chrome/9656a811-9b5b-4ddf-99c7-5117bcef0626.json b/evaluation_examples/examples/chrome/9656a811-9b5b-4ddf-99c7-5117bcef0626.json index a911041..70b5732 100644 --- a/evaluation_examples/examples/chrome/9656a811-9b5b-4ddf-99c7-5117bcef0626.json +++ b/evaluation_examples/examples/chrome/9656a811-9b5b-4ddf-99c7-5117bcef0626.json @@ -29,6 +29,15 @@ "chrome" ], "evaluator": { + "postconfig": [ + { + "type": "execute", + "parameters": { + "command": "pkill chrome", + "shell": "true" + } + } + ], "func": "exact_match", "result": { "type": "enable_enhanced_safety_browsing" From f08fa4912cf8fd6a09a6f5cf018da64f757f5c19 Mon Sep 17 00:00:00 2001 From: David Chang Date: Sun, 10 Mar 2024 18:03:02 +0800 Subject: [PATCH 12/12] ver Mar10th changed AT element filtering --- .gitignore | 1 + at_processing/.gitignore | 4 + at_processing/filter.anal | 249 ++++++++++++++++++ at_processing/ubuntu_node_statistics.py | 22 ++ desktop_env/server/main.py | 12 +- .../415ef462-bed3-493a-ac36-ca8c6d23bf1b.json | 2 +- .../heuristic_retrieve.py | 69 ++--- 7 files changed, 322 insertions(+), 37 deletions(-) create mode 100644 at_processing/.gitignore create mode 100644 at_processing/filter.anal create mode 100644 at_processing/ubuntu_node_statistics.py diff --git a/.gitignore b/.gitignore index fc9dc79..a3543d8 100644 --- a/.gitignore +++ b/.gitignore @@ -171,6 +171,7 @@ tags-opts snapshots *.syncthing.*.tmp cache +version.folder test.xlsx test2.xlsx diff --git a/at_processing/.gitignore b/at_processing/.gitignore new file mode 100644 index 0000000..1719b90 --- /dev/null +++ b/at_processing/.gitignore @@ -0,0 +1,4 @@ +/*.json +/*.xml +/*.list +/*.png diff --git a/at_processing/filter.anal b/at_processing/filter.anal new file mode 100644 index 0000000..6c2e203 --- /dev/null +++ b/at_processing/filter.anal @@ -0,0 +1,249 @@ +vimc: hi AnalSecTitle term=bold cterm=bold: +vimc: syn match AnalSecTitle /^|\d\+\.\ .*$/: +vimc: syn match Comment /^#.*$/: +vimc: hi AnalMetaInfo term=italic ctermfg=red: +vimc: syn match AnalMetaInfo /^[A-Z]\:\ /: +vimc: syn match Todo /\\|^I\:/: +vimc: hi VIMC ctermfg=grey: +vimc: syn match VIMC /^vimc\:\ .*\:$/: +vimc: hi AnalComplete term=reverse,bold ctermfg=green: +vimc: syn match AnalComplete /✓$/: +vimc: hi AnalFail term=reverse,bold ctermfg=red: +vimc: syn match AnalFail /✗$/: + +vimc: hi AnalInfoElm term=reverse,bold ctermfg=cyan: +vimc: syn match AnalInfoElm /\0): + coordinates: Tuple[int, int] = eval(node.get("{{{:}}}screencoord".format("uri:deskat:component.at-spi.gnome.org"))) + sizes: Tuple[int, int] = eval(node.get("{{{:}}}size".format("uri:deskat:component.at-spi.gnome.org"))) + if coordinates[0]>0 and coordinates[1]>0 and sizes[0]>0 and sizes[1]>0: + filtered_nodes.append(node) return filtered_nodes @@ -134,12 +141,14 @@ def print_nodes_with_indent(nodes, indent=0): if __name__ == '__main__': - with open('chrome_desktop_example_1.xml', 'r', encoding='utf-8') as f: - xml_file_str = f.read() + import json + with open('2.json', 'r', encoding='utf-8') as f: + xml_file_str = json.load(f)["AT"] filtered_nodes = filter_nodes(find_leaf_nodes(xml_file_str)) print(len(filtered_nodes)) - masks = draw_bounding_boxes(filtered_nodes, 'screenshot.png', - 'chrome_desktop_example_1_tagged_remove.png', ) + masks = draw_bounding_boxes( filtered_nodes, '2.png' + , '2.a.png' + ) # print(masks) print(len(masks))