Upload folder using huggingface_hub

run.py

@@ -84,7 +84,122 @@ def format_msg(it, init_msg, pdf_obs, warn_obs, web_img_b64, web_text):
             ]
         }
 
-# ... (rest of the helper functions remain the same)
+def call_gpt4v_api(args, openai_client, messages):
+    retry_times = 0
+    while True:
+        try:
+            if not args.text_only:
+                logging.info('Calling gpt4v API...')
+                openai_response = openai_client.chat.completions.create(
+                    model=args.api_model, messages=messages, max_tokens=1000, seed=args.seed
+                )
+            else:
+                logging.info('Calling gpt4 API...')
+                openai_response = openai_client.chat.completions.create(
+                    model=args.api_model, messages=messages, max_tokens=1000, seed=args.seed, timeout=30
+                )
+
+            prompt_tokens = openai_response.usage.prompt_tokens
+            completion_tokens = openai_response.usage.completion_tokens
+
+            logging.info(f'Prompt Tokens: {prompt_tokens}; Completion Tokens: {completion_tokens}')
+
+            gpt_call_error = False
+            return prompt_tokens, completion_tokens, gpt_call_error, openai_response
+
+        except Exception as e:
+            logging.info(f'Error occurred, retrying. Error type: {type(e).__name__}')
+
+            if type(e).__name__ == 'RateLimitError':
+                time.sleep(10)
+
+            elif type(e).__name__ == 'APIError':
+                time.sleep(15)
+
+            elif type(e).__name__ == 'InvalidRequestError':
+                gpt_call_error = True
+                return None, None, gpt_call_error, None
+
+            else:
+                gpt_call_error = True
+                return None, None, gpt_call_error, None
+
+        retry_times += 1
+        if retry_times == 10:
+            logging.info('Retrying too many times')
+            return None, None, True, None
+
+
+def exec_action_click(info, web_ele, driver_task):
+    driver_task.execute_script("arguments[0].setAttribute('target', '_self')", web_ele)
+    web_ele.click()
+    time.sleep(3)
+
+
+def exec_action_type(info, web_ele, driver_task):
+    warn_obs = ""
+    type_content = info['content']
+
+    ele_tag_name = web_ele.tag_name.lower()
+    ele_type = web_ele.get_attribute("type")
+    # outer_html = web_ele.get_attribute("outerHTML")
+    if (ele_tag_name != 'input' and ele_tag_name != 'textarea') or (ele_tag_name == 'input' and ele_type not in ['text', 'search', 'password', 'email', 'tel']):
+        warn_obs = f"note: The web element you're trying to type may not be a textbox, and its tag name is <{web_ele.tag_name}>, type is {ele_type}."
+    try:
+        # Not always work to delete
+        web_ele.clear()
+        # Another way to delete
+        if platform.system() == 'Darwin':
+            web_ele.send_keys(Keys.COMMAND + "a")
+        else:
+            web_ele.send_keys(Keys.CONTROL + "a")
+        web_ele.send_keys(" ")
+        web_ele.send_keys(Keys.BACKSPACE)
+    except:
+        pass
+
+    actions = ActionChains(driver_task)
+    actions.click(web_ele).perform()
+    actions.pause(1)
+
+    try:
+        driver_task.execute_script("""window.onkeydown = function(e) {if(e.keyCode == 32 && e.target.type != 'text' && e.target.type != 'textarea' && e.target.type != 'search') {e.preventDefault();}};""")
+    except:
+        pass
+
+    actions.send_keys(type_content)
+    actions.pause(2)
+
+    actions.send_keys(Keys.ENTER)
+    actions.perform()
+    time.sleep(10)
+    return warn_obs
+
+
+def exec_action_scroll(info, web_eles, driver_task, args, obs_info):
+    scroll_ele_number = info['number']
+    scroll_content = info['content']
+    if scroll_ele_number == "WINDOW":
+        if scroll_content == 'down':
+            driver_task.execute_script(f"window.scrollBy(0, {args.window_height*2//3});")
+        else:
+            driver_task.execute_script(f"window.scrollBy(0, {-args.window_height*2//3});")
+    else:
+        if not args.text_only:
+            scroll_ele_number = int(scroll_ele_number)
+            web_ele = web_eles[scroll_ele_number]
+        else:
+            element_box = obs_info[scroll_ele_number]['union_bound']
+            element_box_center = (element_box[0] + element_box[2] // 2, element_box[1] + element_box[3] // 2)
+            web_ele = driver_task.execute_script("return document.elementFromPoint(arguments[0], arguments[1]);", element_box_center[0], element_box_center[1])
+        actions = ActionChains(driver_task)
+        driver_task.execute_script("arguments[0].focus();", web_ele)
+        if scroll_content == 'down':
+            actions.key_down(Keys.ALT).send_keys(Keys.ARROW_DOWN).key_up(Keys.ALT).perform()
+        else:
+            actions.key_down(Keys.ALT).send_keys(Keys.ARROW_UP).key_up(Keys.ALT).perform()
+    time.sleep(3)
+
 
 def webvoyager_run(args, task, task_dir):
     """
@@ -133,7 +248,8 @@ def webvoyager_run(args, task, task_dir):
 
         action_key, info = extract_information(gpt_4v_res.split('Action:')[1].strip())
         
-        # ... (action execution logic remains the same)
+        # ... (action execution logic)
+        
         if action_key == 'answer':
             break