diff --git a/api/backup_test.py b/api/backup_test.py index b5b43eac8..c7a748ebd 100644 --- a/api/backup_test.py +++ b/api/backup_test.py @@ -47,12 +47,13 @@ class BackupTest(ApiHandler): backup_service = BackupService() matched_files = await backup_service.test_patterns(metadata, max_files=max_files) + truncated = max_files is not None and len(matched_files) >= max_files return { "success": True, "files": matched_files, "total_count": len(matched_files), - "truncated": len(matched_files) >= max_files + "truncated": truncated } except Exception as e: diff --git a/api/backup_test.py.dox.md b/api/backup_test.py.dox.md index c5f01a155..a8e06cc6b 100644 --- a/api/backup_test.py.dox.md +++ b/api/backup_test.py.dox.md @@ -25,6 +25,7 @@ - `BackupTest` defines `requires_auth(...)`. - `BackupTest` defines `requires_loopback(...)`. - Imported dependency areas include: `helpers.api`, `helpers.backup`. +- The `truncated` response flag is true only when a finite `max_files` limit is supplied and the result reaches that limit. ## Key Concepts diff --git a/helpers/backup.py b/helpers/backup.py index 1aa651893..47e41c5f8 100644 --- a/helpers/backup.py +++ b/helpers/backup.py @@ -240,8 +240,12 @@ class BackupService: return translated_patterns - async def test_patterns(self, metadata: Dict[str, Any], max_files: int = 1000) -> List[Dict[str, Any]]: - """Test backup patterns and return list of matched files""" + async def test_patterns(self, metadata: Dict[str, Any], max_files: Optional[int] = 1000) -> List[Dict[str, Any]]: + """Test backup patterns and return list of matched files. + + Pass max_files=None for internal flows that must process the complete + match set, such as backup creation and restore cleanup. + """ include_patterns = metadata.get("include_patterns", []) exclude_patterns = metadata.get("exclude_patterns", []) include_hidden = metadata.get("include_hidden", True) @@ -258,6 +262,7 @@ class BackupService: # Get explicit patterns for hidden file handling explicit_patterns = self._get_explicit_patterns(include_patterns) + has_limit = max_files is not None matched_files = [] processed_count = 0 @@ -285,7 +290,7 @@ class BackupService: dirs[:] = dirs_to_keep for file in files_list: - if processed_count >= max_files: + if has_limit and processed_count >= max_files: break file_path = os.path.join(root, file) @@ -317,10 +322,10 @@ class BackupService: # Skip files we can't access continue - if processed_count >= max_files: + if has_limit and processed_count >= max_files: break - if processed_count >= max_files: + if has_limit and processed_count >= max_files: break except Exception as e: @@ -344,8 +349,10 @@ class BackupService: "include_hidden": include_hidden } - # Get matched files - matched_files = await self.test_patterns(metadata, max_files=50000) + # Get the complete matched file set. Preview and dry-run callers may + # cap their scans for UI responsiveness, but the archive itself must be + # complete. + matched_files = await self.test_patterns(metadata, max_files=None) if not matched_files: raise Exception("No files matched the backup patterns") @@ -824,7 +831,7 @@ class BackupService: # Find existing files that match the translated user-edited patterns try: - existing_files = await self.test_patterns(metadata, max_files=10000) + existing_files = await self.test_patterns(metadata, max_files=None) # Convert to delete operations format files_to_delete = [] diff --git a/helpers/backup.py.dox.md b/helpers/backup.py.dox.md index a72c49377..e08f7a74e 100644 --- a/helpers/backup.py.dox.md +++ b/helpers/backup.py.dox.md @@ -13,7 +13,7 @@ - Classes: - `BackupService` (no explicit base class) - `get_default_backup_metadata(self) -> Dict[str, Any]` - - `async test_patterns(self, metadata: Dict[str, Any], max_files: int=...) -> List[Dict[str, Any]]` + - `async test_patterns(self, metadata: Dict[str, Any], max_files: Optional[int]=...) -> List[Dict[str, Any]]` - `async create_backup(self, include_patterns: List[str], exclude_patterns: List[str], include_hidden: bool=..., backup_name: str=...) -> str` - `async inspect_backup(self, backup_file) -> Dict[str, Any]` - `async preview_restore(self, backup_file, restore_include_patterns: Optional[List[str]]=..., restore_exclude_patterns: Optional[List[str]]=..., overwrite_policy: str=..., clean_before_restore: bool=..., user_edited_metadata: Optional[Dict[str, Any]]=...) -> Dict[str, Any]` @@ -25,6 +25,7 @@ - Update this file whenever public functions, classes, persistence behavior, path/security assumptions, side effects, or cross-module contracts change. - Observed side-effect areas: filesystem reads, filesystem writes, filesystem deletion, settings/state persistence, secret handling. - Imported dependency areas include: `datetime`, `helpers`, `helpers.localization`, `helpers.print_style`, `json`, `os`, `pathspec`, `platform`, `tempfile`, `typing`, `zipfile`. +- `test_patterns(..., max_files=None)` is the unlimited scan mode. UI preview and dry-run callers may pass bounded limits, but real backup creation and restore clean-before-restore must use unlimited matching so archives and cleanup are not silently truncated. ## Key Concepts diff --git a/tests/test_backup_large_archives.py b/tests/test_backup_large_archives.py new file mode 100644 index 000000000..7638fbbc5 --- /dev/null +++ b/tests/test_backup_large_archives.py @@ -0,0 +1,162 @@ +import json +import shutil +import zipfile +from pathlib import Path + +import pytest + +from helpers.backup import BackupService + + +class UploadedBackup: + def __init__(self, path: Path): + self.path = path + + def save(self, target: str) -> None: + shutil.copyfile(self.path, target) + + +@pytest.mark.asyncio +async def test_pattern_scan_can_run_without_file_limit(tmp_path): + root = tmp_path / "a0" + usr = root / "usr" + usr.mkdir(parents=True) + for index in range(3): + (usr / f"file-{index}.txt").write_text(f"{index}\n", encoding="utf-8") + + service = BackupService() + service.agent_zero_root = str(root) + service.base_paths = {str(root): str(root)} + metadata = { + "include_patterns": [f"{root}/usr/**"], + "exclude_patterns": [], + "include_hidden": True, + } + + capped_files = await service.test_patterns(metadata, max_files=2) + all_files = await service.test_patterns(metadata, max_files=None) + + assert len(capped_files) == 2 + assert len(all_files) == 3 + + +@pytest.mark.asyncio +async def test_create_backup_uses_unlimited_pattern_scan(tmp_path, monkeypatch): + source_file = tmp_path / "source.txt" + source_file.write_text("payload\n", encoding="utf-8") + captured = {} + + service = BackupService() + + async def fake_test_patterns(metadata, max_files=1000): + captured["max_files"] = max_files + return [ + { + "path": f"{service.agent_zero_root.rstrip('/')}/usr/file-{index}.txt", + "real_path": str(source_file), + "size": source_file.stat().st_size, + "modified": "2026-06-26T00:00:00+00:00", + "type": "file", + } + for index in range(3) + ] + + async def fake_info(): + return {} + + async def fake_author(): + return "test" + + monkeypatch.setattr(service, "test_patterns", fake_test_patterns) + monkeypatch.setattr(service, "_get_system_info", fake_info) + monkeypatch.setattr(service, "_get_environment_info", fake_info) + monkeypatch.setattr(service, "_get_backup_author", fake_author) + + zip_path = await service.create_backup( + include_patterns=[f"{service.agent_zero_root}/usr/**"], + exclude_patterns=[], + include_hidden=True, + backup_name="large-backup", + ) + + assert captured["max_files"] is None + with zipfile.ZipFile(zip_path) as archive: + metadata = json.loads(archive.read("metadata.json").decode("utf-8")) + assert metadata["total_files"] == 3 + assert ( + f"{service.agent_zero_root.rstrip('/').lstrip('/')}/usr/file-2.txt" + in archive.namelist() + ) + + +@pytest.mark.asyncio +async def test_restore_can_reach_files_after_50000_archive_entries(tmp_path): + old_root = "/old-a0" + archive_root = old_root.lstrip("/") + file_count = 50_001 + last_index = file_count - 1 + zip_path = tmp_path / "large-backup.zip" + + metadata = { + "environment_info": {"agent_zero_root": old_root}, + "include_patterns": [f"{old_root}/usr/large/**"], + "exclude_patterns": [], + "include_hidden": True, + } + with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_STORED) as archive: + archive.writestr("metadata.json", json.dumps(metadata)) + for index in range(file_count): + payload = "tail payload\n" if index == last_index else "" + archive.writestr( + f"{archive_root}/usr/large/file-{index:05d}.txt", + payload, + ) + + service = BackupService() + service.agent_zero_root = str(tmp_path / "restored-a0") + + result = await service.restore_backup( + backup_file=UploadedBackup(zip_path), + restore_include_patterns=[ + f"{old_root}/usr/large/file-{last_index:05d}.txt" + ], + restore_exclude_patterns=[], + overwrite_policy="overwrite", + ) + + restored_path = ( + Path(service.agent_zero_root) + / "usr" + / "large" + / f"file-{last_index:05d}.txt" + ) + assert len(result["restored_files"]) == 1 + assert len(result["skipped_files"]) == file_count - 1 + assert result["errors"] == [] + assert restored_path.read_text(encoding="utf-8") == "tail payload\n" + + +@pytest.mark.asyncio +async def test_restore_clean_before_restore_uses_unlimited_pattern_scan(monkeypatch): + service = BackupService() + captured = {} + + async def fake_test_patterns(metadata, max_files=1000): + captured["max_files"] = max_files + return [] + + monkeypatch.setattr(service, "test_patterns", fake_test_patterns) + + result = await service._find_files_to_clean_with_user_metadata( + user_metadata={ + "include_patterns": [f"{service.agent_zero_root}/usr/**"], + "exclude_patterns": [], + "include_hidden": True, + }, + original_metadata={ + "environment_info": {"agent_zero_root": service.agent_zero_root} + }, + ) + + assert result == [] + assert captured["max_files"] is None