Create repo_utils.py

repo_utils.py

@@ -0,0 +1,51 @@
+import os
+import subprocess
+from file_utils import validate_file_types, get_file_summary, read_file_content, summarize_content
+
+def validate_url(url):
+    return url.startswith('https://')
+
+def clone_repo(url, repo_dir, hf_token, hf_user):
+    env = os.environ.copy()
+    env['GIT_LFS_SKIP_SMUDGE'] = '1'
+    token_url = url.replace('https://', f'https://{hf_user}:{hf_token}@')
+    result = subprocess.run(["git", "clone", token_url, repo_dir], env=env, capture_output=True, text=True)
+    if result.returncode != 0:
+        return False, result.stderr
+    return True, None
+
+def extract_repo_content(url, hf_token, hf_user):
+    if not validate_url(url):
+        return [{"header": {"name": "Error", "type": "error", "size": 0}, "content": "Invalid URL"}]
+    
+    repo_dir = "./temp_repo"
+    if os.path.exists(repo_dir):
+        subprocess.run(["rm", "-rf", repo_dir])
+    
+    success, error = clone_repo(url, repo_dir, hf_token, hf_user)
+    if not success:
+        return [{"header": {"name": "Error", "type": "error", "size": 0}, "content": f"Failed to clone repository: {error}"}]
+    
+    file_types = validate_file_types(repo_dir)
+    extracted_content = []
+    for file_path, file_type in file_types.items():
+        file_summary = get_file_summary(file_path, file_type)
+        content = {"header": file_summary}
+        
+        if file_type in SUPPORTED_FILE_TYPES and file_summary["size"] <= 32 * 1024:
+            try:
+                file_content = read_file_content(file_path)
+                content["content"] = file_content
+                content["summary"] = summarize_content(file_content)
+            except Exception as e:
+                content["content"] = f"Failed to read file content: {str(e)}"
+                content["summary"] = ""
+        else:
+            content["content"] = "File too large or binary, content not captured."
+            content["summary"] = ""
+        
+        extracted_content.append(content)
+    
+    subprocess.run(["rm", "-rf", repo_dir])
+    
+    return extracted_content