support existing projects (#976)

Experimental, limited to 10kloc

core/agents/importer.py

@@ -0,0 +1,86 @@
+from uuid import uuid4
+
+from core.agents.base import BaseAgent
+from core.agents.convo import AgentConvo
+from core.agents.response import AgentResponse, ResponseType
+from core.db.models import Complexity
+from core.llm.parser import JSONParser
+from core.log import get_logger
+from core.templates.example_project import EXAMPLE_PROJECT_DESCRIPTION
+
+log = get_logger(__name__)
+
+MAX_PROJECT_LINES = 10000
+
+
+class Importer(BaseAgent):
+    agent_type = "importer"
+    display_name = "Project Analyist"
+
+    async def run(self) -> AgentResponse:
+        if self.prev_response and self.prev_response.type == ResponseType.IMPORT_PROJECT:
+            # Called by SpecWriter to start the import process
+            await self.start_import_process()
+            return AgentResponse.describe_files(self)
+
+        await self.analyze_project()
+        return AgentResponse.done(self)
+
+    async def start_import_process(self):
+        # TODO: Send a signal to the UI to copy the project files to workspace
+        project_root = self.state_manager.get_full_project_root()
+        await self.ui.import_project(project_root)
+        await self.send_message(
+            f"This is experimental feature and is currently limited to projects with size up to {MAX_PROJECT_LINES} lines of code."
+        )
+
+        await self.ask_question(
+            f"Please copy your project files to {project_root} and press Continue",
+            allow_empty=False,
+            buttons={
+                "continue": "Continue",
+            },
+            buttons_only=True,
+            default="continue",
+        )
+
+        imported_files, _ = await self.state_manager.import_files()
+        imported_lines = sum(len(f.content.content.splitlines()) for f in imported_files)
+        if imported_lines > MAX_PROJECT_LINES:
+            await self.send_message(
+                "WARNING: Your project ({imported_lines} LOC) is larger than supported and may cause issues in Pythagora."
+            )
+        await self.state_manager.commit()
+
+    async def analyze_project(self):
+        llm = self.get_llm()
+
+        self.send_message("Inspecting most important project files ...")
+
+        convo = AgentConvo(self).template("get_entrypoints")
+        llm_response = await llm(convo, parser=JSONParser())
+        relevant_files = [f for f in self.current_state.files if f.path in llm_response]
+
+        self.send_message("Analyzing project ...")
+
+        convo = AgentConvo(self).template(
+            "analyze_project", relevant_files=relevant_files, example_spec=EXAMPLE_PROJECT_DESCRIPTION
+        )
+        llm_response = await llm(convo)
+
+        spec = self.current_state.specification.clone()
+        spec.description = llm_response
+        self.next_state.specification = spec
+        self.next_state.epics = [
+            {
+                "id": uuid4().hex,
+                "name": "Import project",
+                "description": "Import an existing project into Pythagora",
+                "tasks": [],
+                "completed": True,
+                "test_instructions": None,
+                "source": "app",
+                "summary": None,
+                "complexity": Complexity.HARD if len(self.current_state.files) > 5 else Complexity.SIMPLE,
+            }
+        ]

core/agents/orchestrator.py

@@ -8,6 +8,7 @@
 from core.agents.error_handler import ErrorHandler
 from core.agents.executor import Executor
 from core.agents.human_input import HumanInput
+from core.agents.importer import Importer
 from core.agents.problem_solver import ProblemSolver
 from core.agents.response import AgentResponse, ResponseType
 from core.agents.spec_writer import SpecWriter
@@ -167,10 +168,16 @@ def create_agent(self, prev_response: Optional[AgentResponse]) -> BaseAgent:
                 return HumanInput(self.state_manager, self.ui, prev_response=prev_response)
             if prev_response.type == ResponseType.TASK_REVIEW_FEEDBACK:
                 return Developer(self.state_manager, self.ui, prev_response=prev_response)
+            if prev_response.type == ResponseType.IMPORT_PROJECT:
+                return Importer(self.state_manager, self.ui, prev_response=prev_response)
 
         if not state.specification.description:
-            # Ask the Spec Writer to refine and save the project specification
-            return SpecWriter(self.state_manager, self.ui)
+            if state.files:
+                # The project has been imported, but not analyzed yet
+                return Importer(self.state_manager, self.ui)
+            else:
+                # New project: ask the Spec Writer to refine and save the project specification
+                return SpecWriter(self.state_manager, self.ui)
         elif not state.specification.architecture:
             # Ask the Architect to design the project architecture and determine dependencies
             return Architect(self.state_manager, self.ui, process_manager=self.process_manager)

core/agents/response.py

@@ -39,6 +39,9 @@ class ResponseType(str, Enum):
     TASK_REVIEW_FEEDBACK = "task-review-feedback"
     """Agent is providing feedback on the entire task."""
 
+    IMPORT_PROJECT = "import-project"
+    """User wants to import an existing project."""
+
 
 class AgentResponse:
     type: ResponseType = ResponseType.DONE
@@ -130,3 +133,7 @@ def task_review_feedback(agent: "BaseAgent", feedback: str) -> "AgentResponse":
                 "feedback": feedback,
             },
         )
+
+    @staticmethod
+    def import_project(agent: "BaseAgent") -> "AgentResponse":
+        return AgentResponse(type=ResponseType.IMPORT_PROJECT, agent=agent)

core/agents/spec_writer.py

@@ -31,11 +31,15 @@ async def run(self) -> AgentResponse:
                 # FIXME: must be lowercase becase VSCode doesn't recognize it otherwise. Needs a fix in the extension
                 "continue": "continue",
                 "example": "Start an example project",
+                "import": "Import an existing project",
             },
         )
         if response.cancelled:
             return AgentResponse.error(self, "No project description")
 
+        if response.button == "import":
+            return AgentResponse.import_project(self)
+
         if response.button == "example":
             await self.send_message("Starting example project with description:")
             await self.send_message(EXAMPLE_PROJECT_DESCRIPTION)

core/config/__init__.py

@@ -18,6 +18,7 @@
     "node_modules",
     "package-lock.json",
     "venv",
+    ".venv",
     "dist",
     "build",
     "target",

core/prompts/importer/analyze_project.prompt

@@ -0,0 +1,28 @@
+You're given an existing project you need to analyze and continue developing. To do this, you'll need to determine the project architecture, technologies used (platform, libraries, etc) and reverse-engineer the technical and functional spec.
+
+Here is the list of all the files in the project:
+
+{% for file in state.files %}
+* `{{ file.path }}` - {{ file.meta.get("description")}}
+{% endfor %}
+
+Here's the full content of interesting files that may help you to determine the specification:
+
+{% for file in state.files %}
+**`{{ file.path }}`**:
+```
+{{ file.content.content }}
+```
+
+{% endfor %}
+
+Based on this information, please provide detailed specification for the project. Here is an example specification format:
+
+---START_OF_EXAMPLE_SPEC---
+{{ example_spec }}
+---END_OF_EXAMPLE_SPEC---
+
+**IMPORTANT**: In the specification, you must include the following sections:
+* **Project Description**: A detailed description of what the project is about.
+* **Features**: A list of features that the project has implemented. Each feature should be described in detail.
+* **Technical Specification**: Detailed description of how the project works, including any important technical details.

core/prompts/importer/get_entrypoints.prompt

@@ -0,0 +1,21 @@
+You're given an existing project you need to analyze and continue developing. To do this, you'll need to determine the project architecture, technologies used (platform, libraries, etc) and reverse-engineer the technical and functional spec.
+
+As a first step, you have to identify which of the listed files to examine so you can determine this. After you identify the files, you'll be given full access to their contents so you can determine the project information.
+
+Here is the list of all the files in the project:
+
+{% for file in state.files %}
+* `{{ file.path }}` - {{ file.meta.get("description")}}
+{% endfor %}
+
+Based on this information, list the files (full path, as shown in the list) you would examine to determine the project architecture, technologies and specification. Output the list in JSON format like in the following example:
+
+```json
+{
+  "files": [
+    "README.md",
+    "pyproject.toml",
+    "settings/settings.py"
+  ]
+}
+```

core/ui/base.py

@@ -261,6 +261,17 @@ async def send_features_list(self, features: list[str]):
         """
         raise NotImplementedError()
 
+    async def import_project(self, project_dir: str):
+        """
+        Ask the UI to import files from the project directory.
+
+        The UI should provide a way for the user to select the directory with
+        existing project, and recursively copy the files over.
+
+        :param project_dir: Project directory.
+        """
+        raise NotImplementedError()
+
 
 pythagora_source = UISource("Pythagora", "pythagora")
 success_source = UISource("Congratulations", "success")

core/ui/console.py

@@ -122,5 +122,8 @@ async def send_project_description(self, description: str):
     async def send_features_list(self, features: list[str]):
         pass
 
+    async def import_project(self, project_dir: str):
+        pass
+
 
 __all__ = ["PlainConsoleUI"]

core/ui/ipc_client.py

@@ -39,6 +39,7 @@ class MessageType(str, Enum):
     LOADING_FINISHED = "loadingFinished"
     PROJECT_DESCRIPTION = "projectDescription"
     FEATURES_LIST = "featuresList"
+    IMPORT_PROJECT = "importProject"
 
 
 class Message(BaseModel):
@@ -332,5 +333,8 @@ async def send_project_description(self, description: str):
     async def send_features_list(self, features: list[str]):
         await self._send(MessageType.FEATURES_LIST, content={"featuresList": features})
 
+    async def import_project(self, project_dir: str):
+        await self._send(MessageType.IMPORT_PROJECT, content={"project_dir": project_dir})
+
 
 __all__ = ["IPCClientUI"]