misc fixes

* add more files to known filenames and default ignores
* gracefully ignore file creation/removal OS errors
* handle user inputs larger than 64k
* show nice error message for TokenLimitError
* fix incorrect llm response stored in case of TokenLimitError
* gracefully handle edge case when saving template files

pilot/const/common.py

@@ -29,6 +29,7 @@
     '.next',
     '.DS_Store',
     '__pycache__',
+    "site-packages",
     'node_modules',
     'package-lock.json',
     'venv',

pilot/helpers/AgentConvo.py

@@ -68,7 +68,7 @@ def send_message(self, prompt_path=None, prompt_data=None, function_calls: Funct
                                                   function_calls=function_calls, prompt_data=prompt_data,
                                                   temperature=self.temperature)
         except TokenLimitError as e:
-            save_development_step(self.agent.project, prompt_path, prompt_data, self.messages, '', str(e))
+            save_development_step(self.agent.project, prompt_path, prompt_data, self.messages, {"text": ""}, str(e))
             raise e
 
         # TODO: move this code to Developer agent - https://github.com/Pythagora-io/gpt-pilot/issues/91#issuecomment-1751964079

pilot/helpers/Project.py

@@ -475,7 +475,7 @@ def normalize_path(path: str) -> Tuple[str, str]:
             # - /pilot -> /pilot/
             # - \pilot\server.js -> \pilot\server.js
             # - \pilot -> \pilot\
-            KNOWN_FILES = ["makefile", "dockerfile", "procfile", "readme", "license"]  # known exceptions that break the heuristic
+            KNOWN_FILES = ["makefile", "dockerfile", "procfile", "readme", "license", "podfile"]  # known exceptions that break the heuristic
             KNOWN_DIRS = []  # known exceptions that break the heuristic
             base = os.path.basename(path)
             if (
@@ -560,7 +560,10 @@ def restore_files(self, development_step_id):
 
         clear_directory(self.root_path, ignore=self.files)
         for file_snapshot in file_snapshots:
-            update_file(file_snapshot.file.full_path, file_snapshot.content, project=self)
+            try:
+                update_file(file_snapshot.file.full_path, file_snapshot.content, project=self)
+            except (PermissionError, NotADirectoryError) as err:  # noqa
+                print(f"Error restoring file {file_snapshot.file.full_path}: {err}")
             if file_snapshot.file.full_path not in self.files:
                 self.files.append(file_snapshot.file.full_path)
 

pilot/helpers/agents/Developer.py

@@ -866,7 +866,13 @@ def set_up_environment(self):
                 dep_text = dependency['name']
 
             logger.info('Checking %s', dependency)
-            llm_response = self.check_system_dependency(dependency)
+            try:
+                llm_response = self.check_system_dependency(dependency)
+            except Exception as err:
+                # This catches weird errors like people removing or renaming the workspace
+                # folder while we're trying to run system commands. Since these commands don't
+                # care about folders they run in, we don't want to crash just because of that.
+                llm_response = str(err)
 
             if llm_response == 'DONE':
                 print(color_green_bold(f"✅ {dep_text} is available."))

pilot/helpers/files.py

@@ -145,10 +145,18 @@ def clear_directory(directory: str, ignore: Optional[list[str]] = None):
             if matcher.ignore(full_path):
                 continue
 
-            os.remove(full_path)
+            try:
+                os.remove(full_path)
+            except:  # noqa
+                # Gracefully handle some weird edge cases instead of crashing
+                pass
 
         # Delete empty subdirectories not in ignore list
         for d in dirs:
-            subdir_path = os.path.join(dpath, d)
-            if not os.listdir(subdir_path):
-                os.rmdir(subdir_path)
+            try:
+                subdir_path = os.path.join(dpath, d)
+                if not os.listdir(subdir_path):
+                    os.rmdir(subdir_path)
+            except:  # noqa
+                # Gracefully handle some weird edge cases instead of crashing
+                pass

pilot/helpers/ipc.py

@@ -28,8 +28,17 @@ def listen(self):
             return
 
         while True:
-            data = self.client.recv(65536)
-            message = json.loads(data)
+
+            data = b''
+            while True:
+                data = data + self.client.recv(512 * 1024)
+                try:
+                    message = json.loads(data)
+                    break
+                except json.JSONDecodeError:
+                    # This means we still got an incomplete message, so
+                    # we should continue to receive more data.
+                    continue
 
             if message['type'] == 'response':
                 # self.client.close()

pilot/main.py

@@ -111,6 +111,14 @@ def init():
         telemetry.record_crash(err, end_result="failure:api-error")
         telemetry.send()
         run_exit_fn = False
+        if isinstance(err, TokenLimitError):
+            print('', type='verbose', category='error')
+            print(color_red(
+                "We sent too large request to the LLM, resulting in an error. "
+                "This is usually caused by including framework files in an LLM request. "
+                "Here's how you can get GPT Pilot to ignore those extra files: "
+                "https://bit.ly/faq-token-limit-error"
+            ))
         print('Exit', type='exit')
 
     except KeyboardInterrupt:

pilot/templates/__init__.py

@@ -75,7 +75,9 @@ def apply_project_template(
     print(color_green_bold(f"Applying project template {template['description']}...\n"))
     logger.info(f"Applying project template {template_name}...")
 
-    project.save_files_snapshot(project.checkpoints['last_development_step']['id'])
+    last_development_step = project.checkpoints.get('last_development_step')
+    if last_development_step:
+        project.save_files_snapshot(last_development_step['id'])
 
     try:
         if install_hook: