"""Git repository ingester for comprehensive code indexing. This module provides functionality to ingest entire git repositories including: - Full git history and commit metadata - File contents at different points in time - AST analysis via tree-sitter - Symbol extraction and cross-references - Blame information for line-level attribution """ import logging import subprocess from typing import Any, Dict, List, Optional from pathlib import Path from datetime import datetime from dataclasses import dataclass from .ast_analyzer import ASTAnalyzer from .infinity_store import InfinityVectorStore logger = logging.getLogger(__name__) @dataclass class GitCommit: """Represents a git commit.""" hash: str author: str author_email: str timestamp: int message: str files: List[Dict[str, str]] # [{'status': 'M', 'filename': 'main.py'}] parent_hashes: List[str] @dataclass class GitFileHistory: """History of a single file.""" file_path: str commits: List[GitCommit] current_content: Optional[str] line_blame: Dict[int, Dict[str, Any]] # line_number -> blame info class GitIngester: """Ingests git repositories into vector store.""" def __init__(self, vector_store: InfinityVectorStore): """Initialize the git ingester. Args: vector_store: The vector store to ingest into """ self.vector_store = vector_store self.ast_analyzer = ASTAnalyzer() self._commit_cache: Dict[str, GitCommit] = {} def ingest_repository( self, repo_path: str, branch: str = "HEAD", include_history: bool = True, include_diffs: bool = True, include_blame: bool = True, file_patterns: Optional[List[str]] = None, ) -> Dict[str, Any]: """Ingest an entire git repository. Args: repo_path: Path to the git repository branch: Branch to ingest (default: HEAD) include_history: Whether to include commit history include_diffs: Whether to include diff information include_blame: Whether to include blame information file_patterns: List of file patterns to include (e.g., ["*.py", "*.js"]) Returns: Summary of ingestion results """ repo_path = Path(repo_path) if not (repo_path / ".git").exists(): raise ValueError(f"Not a git repository: {repo_path}") logger.info(f"Starting ingestion of repository: {repo_path}") results = { "repository": str(repo_path), "branch": branch, "commits_processed": 0, "commits_indexed": 0, "files_indexed": 0, "symbols_extracted": 0, "diffs_indexed": 0, "blame_entries": 0, "errors": [], } try: # Get current branch/commit current_commit = self._get_current_commit(repo_path) results["current_commit"] = current_commit # Get list of files to process files = self._get_repository_files(repo_path, file_patterns) logger.info(f"Found {len(files)} files to process") # Process each file for file_path in files: try: self._process_file( repo_path, file_path, include_history=include_history, include_blame=include_blame, results=results, ) except Exception as e: logger.error(f"Error processing {file_path}: {e}") results["errors"].append(f"{file_path}: {str(e)}") # Process commit history if requested if include_history: commits = self._get_commit_history(repo_path, branch) results["commits_processed"] = len(commits) for commit in commits: self._index_commit(commit, include_diffs=include_diffs) results["commits_indexed"] = results.get("commits_indexed", 0) + 1 if include_diffs: results["diffs_indexed"] += len(commit.files) # Create repository metadata document self._index_repository_metadata(repo_path, results) except Exception as e: logger.error(f"Repository ingestion failed: {e}") results["errors"].append(f"Fatal error: {str(e)}") logger.info(f"Ingestion complete: {results}") return results def _get_current_commit(self, repo_path: Path) -> str: """Get the current commit hash.""" result = subprocess.run( ["git", "rev-parse", "HEAD"], cwd=repo_path, capture_output=True, text=True, check=True, ) return result.stdout.strip() def _get_repository_files(self, repo_path: Path, patterns: Optional[List[str]] = None) -> List[Path]: """Get list of files in repository matching patterns.""" # Use git ls-files to respect .gitignore cmd = ["git", "ls-files"] result = subprocess.run(cmd, cwd=repo_path, capture_output=True, text=True, check=True) files = [] for line in result.stdout.strip().split("\n"): if line: file_path = repo_path / line if file_path.exists(): # Apply pattern filtering if specified if patterns: if any(file_path.match(pattern) for pattern in patterns): files.append(file_path) else: files.append(file_path) return files def _get_commit_history(self, repo_path: Path, branch: str = "HEAD", max_commits: int = 1000) -> List[GitCommit]: """Get commit history for the repository.""" # Get commit list with basic info result = subprocess.run( [ "git", "log", branch, f"--max-count={max_commits}", "--pretty=format:%H|%P|%an|%ae|%at|%s", ], cwd=repo_path, capture_output=True, text=True, check=True, ) commits = [] for line in result.stdout.strip().split("\n"): if line: parts = line.split("|", 5) if len(parts) >= 6: commit_hash = parts[0] parent_hashes = parts[1].split() if parts[1] else [] # Get file changes for this commit files = self._get_commit_files(repo_path, commit_hash) commit = GitCommit( hash=commit_hash, parent_hashes=parent_hashes, author=parts[2], author_email=parts[3], timestamp=int(parts[4]), message=parts[5], files=files, ) commits.append(commit) self._commit_cache[commit_hash] = commit return commits def _get_commit_files(self, repo_path: Path, commit_hash: str) -> List[Dict[str, str]]: """Get list of files changed in a commit.""" result = subprocess.run( ["git", "show", "--name-status", "--format=", commit_hash], cwd=repo_path, capture_output=True, text=True, check=True, ) files = [] for line in result.stdout.strip().split("\n"): if line and "\t" in line: parts = line.split("\t", 1) if len(parts) == 2: files.append({"status": parts[0], "filename": parts[1]}) return files def _process_file( self, repo_path: Path, file_path: Path, include_history: bool, include_blame: bool, results: Dict[str, Any], ): """Process a single file.""" relative_path = file_path.relative_to(repo_path) # Read current content try: content = file_path.read_text(encoding="utf-8") except UnicodeDecodeError: content = file_path.read_text(encoding="latin-1") # Get file metadata metadata = { "repository": str(repo_path), "relative_path": str(relative_path), "file_type": file_path.suffix, "size": file_path.stat().st_size, } # Add git history metadata if requested if include_history: history = self._get_file_history(repo_path, relative_path) metadata["commit_count"] = len(history) if history: metadata["first_commit"] = history[-1]["hash"] metadata["last_commit"] = history[0]["hash"] metadata["last_modified"] = datetime.fromtimestamp(history[0]["timestamp"]).isoformat() # Add blame information if requested if include_blame: blame_data = self._get_file_blame(repo_path, relative_path) metadata["unique_authors"] = len(set(b["author"] for b in blame_data.values())) # Index the file content doc_ids = self.vector_store.add_file(str(file_path), chunk_size=1000, chunk_overlap=200, metadata=metadata) results["files_indexed"] += 1 # Perform AST analysis for supported languages if file_path.suffix in [".py", ".js", ".ts", ".java", ".cpp", ".c"]: try: file_ast = self.ast_analyzer.analyze_file(str(file_path)) if file_ast: # Store complete AST self.vector_store._store_file_ast(file_ast) # Store individual symbols self.vector_store._store_symbols(file_ast.symbols) results["symbols_extracted"] += len(file_ast.symbols) # Store cross-references self.vector_store._store_references(file_ast) except Exception as e: logger.warning(f"AST analysis failed for {file_path}: {e}") def _get_file_history(self, repo_path: Path, file_path: Path) -> List[Dict[str, Any]]: """Get commit history for a specific file.""" result = subprocess.run( [ "git", "log", "--follow", "--pretty=format:%H|%at|%an|%s", "--", str(file_path), ], cwd=repo_path, capture_output=True, text=True, ) if result.returncode != 0: return [] history = [] for line in result.stdout.strip().split("\n"): if line: parts = line.split("|", 3) if len(parts) >= 4: history.append( { "hash": parts[0], "timestamp": int(parts[1]), "author": parts[2], "message": parts[3], } ) return history def _get_file_blame(self, repo_path: Path, file_path: Path) -> Dict[int, Dict[str, Any]]: """Get blame information for a file.""" result = subprocess.run( ["git", "blame", "--line-porcelain", "--", str(file_path)], cwd=repo_path, capture_output=True, text=True, ) if result.returncode != 0: return {} blame_data = {} current_commit = None current_line = None author = None timestamp = None for line in result.stdout.strip().split("\n"): if line and not line.startswith("\t"): parts = line.split(" ") if len(parts) >= 3 and len(parts[0]) == 40: # SHA-1 hash current_commit = parts[0] current_line = int(parts[2]) elif line.startswith("author "): author = line[7:] elif line.startswith("author-time "): timestamp = int(line[12:]) # We have all the data for this line if current_line and author: blame_data[current_line] = { "commit": current_commit, "author": author, "timestamp": timestamp, } return blame_data def _index_commit(self, commit: GitCommit, include_diffs: bool = True): """Index a single commit.""" # Create commit document commit_doc = f"""Git Commit: {commit.hash} Author: {commit.author} <{commit.author_email}> Date: {datetime.fromtimestamp(commit.timestamp).isoformat()} Message: {commit.message} Files changed: {len(commit.files)} """ for file_info in commit.files: commit_doc += f"\n{file_info['status']}\t{file_info['filename']}" # Index commit metadata = { "type": "git_commit", "commit_hash": commit.hash, "author": commit.author, "timestamp": commit.timestamp, "file_count": len(commit.files), } self.vector_store.add_document(commit_doc, metadata) # Index diffs if requested if include_diffs: for file_info in commit.files: self._index_commit_diff(commit, file_info["filename"]) def _index_commit_diff(self, commit: GitCommit, filename: str): """Index the diff for a specific file in a commit.""" # This is a simplified version - in practice you'd want to # parse the actual diff and store meaningful chunks metadata = { "type": "git_diff", "commit_hash": commit.hash, "filename": filename, "author": commit.author, "timestamp": commit.timestamp, } # Create a document representing this change diff_doc = f"""File: {filename} Commit: {commit.hash} Author: {commit.author} Message: {commit.message} """ self.vector_store.add_document(diff_doc, metadata) def _index_repository_metadata(self, repo_path: Path, results: Dict[str, Any]): """Index overall repository metadata.""" # Get repository info remote_result = subprocess.run( ["git", "remote", "get-url", "origin"], cwd=repo_path, capture_output=True, text=True, ) remote_url = remote_result.stdout.strip() if remote_result.returncode == 0 else None # Create repository summary document repo_doc = f"""Repository: {repo_path.name} Path: {repo_path} Remote: {remote_url or "No remote"} Current Commit: {results.get("current_commit", "Unknown")} Statistics: - Files indexed: {results["files_indexed"]} - Commits processed: {results["commits_processed"]} - Symbols extracted: {results["symbols_extracted"]} - Diffs indexed: {results["diffs_indexed"]} """ metadata = { "type": "repository", "name": repo_path.name, "path": str(repo_path), "remote_url": remote_url, **results, } self.vector_store.add_document(repo_doc, metadata)