From 71607476071ee4ba4d9f0c7036dc62e0a26318b3 Mon Sep 17 00:00:00 2001 From: Alessandro <155005371+3clyp50@users.noreply.github.com> Date: Fri, 26 Jun 2026 14:03:37 +0200 Subject: [PATCH] Remove backup file count cap Let backup creation and restore cleanup use unlimited pattern scans so archives for agents with more than 50,000 files are complete. Keep UI preview and dry-run paths bounded for responsiveness while making the dry-run truncated flag safe for optional limits. Add regression coverage for unlimited scans, backup creation, clean-before-restore, and restoring an archive entry past the old 50,000-file boundary. --- api/backup_test.py | 3 +- api/backup_test.py.dox.md | 1 + helpers/backup.py | 23 ++-- helpers/backup.py.dox.md | 3 +- tests/test_backup_large_archives.py | 162 ++++++++++++++++++++++++++++ 5 files changed, 182 insertions(+), 10 deletions(-) create mode 100644 tests/test_backup_large_archives.py diff --git a/api/backup_test.py b/api/backup_test.py index b5b43eac8..c7a748ebd 100644 --- a/api/backup_test.py +++ b/api/backup_test.py @@ -47,12 +47,13 @@ class BackupTest(ApiHandler): backup_service = BackupService() matched_files = await backup_service.test_patterns(metadata, max_files=max_files) + truncated = max_files is not None and len(matched_files) >= max_files return { "success": True, "files": matched_files, "total_count": len(matched_files), - "truncated": len(matched_files) >= max_files + "truncated": truncated } except Exception as e: diff --git a/api/backup_test.py.dox.md b/api/backup_test.py.dox.md index c5f01a155..a8e06cc6b 100644 --- a/api/backup_test.py.dox.md +++ b/api/backup_test.py.dox.md @@ -25,6 +25,7 @@ - `BackupTest` defines `requires_auth(...)`. - `BackupTest` defines `requires_loopback(...)`. - Imported dependency areas include: `helpers.api`, `helpers.backup`. +- The `truncated` response flag is true only when a finite `max_files` limit is supplied and the result reaches that limit. ## Key Concepts diff --git a/helpers/backup.py b/helpers/backup.py index 1aa651893..47e41c5f8 100644 --- a/helpers/backup.py +++ b/helpers/backup.py @@ -240,8 +240,12 @@ class BackupService: return translated_patterns - async def test_patterns(self, metadata: Dict[str, Any], max_files: int = 1000) -> List[Dict[str, Any]]: - """Test backup patterns and return list of matched files""" + async def test_patterns(self, metadata: Dict[str, Any], max_files: Optional[int] = 1000) -> List[Dict[str, Any]]: + """Test backup patterns and return list of matched files. + + Pass max_files=None for internal flows that must process the complete + match set, such as backup creation and restore cleanup. + """ include_patterns = metadata.get("include_patterns", []) exclude_patterns = metadata.get("exclude_patterns", []) include_hidden = metadata.get("include_hidden", True) @@ -258,6 +262,7 @@ class BackupService: # Get explicit patterns for hidden file handling explicit_patterns = self._get_explicit_patterns(include_patterns) + has_limit = max_files is not None matched_files = [] processed_count = 0 @@ -285,7 +290,7 @@ class BackupService: dirs[:] = dirs_to_keep for file in files_list: - if processed_count >= max_files: + if has_limit and processed_count >= max_files: break file_path = os.path.join(root, file) @@ -317,10 +322,10 @@ class BackupService: # Skip files we can't access continue - if processed_count >= max_files: + if has_limit and processed_count >= max_files: break - if processed_count >= max_files: + if has_limit and processed_count >= max_files: break except Exception as e: @@ -344,8 +349,10 @@ class BackupService: "include_hidden": include_hidden } - # Get matched files - matched_files = await self.test_patterns(metadata, max_files=50000) + # Get the complete matched file set. Preview and dry-run callers may + # cap their scans for UI responsiveness, but the archive itself must be + # complete. + matched_files = await self.test_patterns(metadata, max_files=None) if not matched_files: raise Exception("No files matched the backup patterns") @@ -824,7 +831,7 @@ class BackupService: # Find existing files that match the translated user-edited patterns try: - existing_files = await self.test_patterns(metadata, max_files=10000) + existing_files = await self.test_patterns(metadata, max_files=None) # Convert to delete operations format files_to_delete = [] diff --git a/helpers/backup.py.dox.md b/helpers/backup.py.dox.md index a72c49377..e08f7a74e 100644 --- a/helpers/backup.py.dox.md +++ b/helpers/backup.py.dox.md @@ -13,7 +13,7 @@ - Classes: - `BackupService` (no explicit base class) - `get_default_backup_metadata(self) -> Dict[str, Any]` - - `async test_patterns(self, metadata: Dict[str, Any], max_files: int=...) -> List[Dict[str, Any]]` + - `async test_patterns(self, metadata: Dict[str, Any], max_files: Optional[int]=...) -> List[Dict[str, Any]]` - `async create_backup(self, include_patterns: List[str], exclude_patterns: List[str], include_hidden: bool=..., backup_name: str=...) -> str` - `async inspect_backup(self, backup_file) -> Dict[str, Any]` - `async preview_restore(self, backup_file, restore_include_patterns: Optional[List[str]]=..., restore_exclude_patterns: Optional[List[str]]=..., overwrite_policy: str=..., clean_before_restore: bool=..., user_edited_metadata: Optional[Dict[str, Any]]=...) -> Dict[str, Any]` @@ -25,6 +25,7 @@ - Update this file whenever public functions, classes, persistence behavior, path/security assumptions, side effects, or cross-module contracts change. - Observed side-effect areas: filesystem reads, filesystem writes, filesystem deletion, settings/state persistence, secret handling. - Imported dependency areas include: `datetime`, `helpers`, `helpers.localization`, `helpers.print_style`, `json`, `os`, `pathspec`, `platform`, `tempfile`, `typing`, `zipfile`. +- `test_patterns(..., max_files=None)` is the unlimited scan mode. UI preview and dry-run callers may pass bounded limits, but real backup creation and restore clean-before-restore must use unlimited matching so archives and cleanup are not silently truncated. ## Key Concepts diff --git a/tests/test_backup_large_archives.py b/tests/test_backup_large_archives.py new file mode 100644 index 000000000..7638fbbc5 --- /dev/null +++ b/tests/test_backup_large_archives.py @@ -0,0 +1,162 @@ +import json +import shutil +import zipfile +from pathlib import Path + +import pytest + +from helpers.backup import BackupService + + +class UploadedBackup: + def __init__(self, path: Path): + self.path = path + + def save(self, target: str) -> None: + shutil.copyfile(self.path, target) + + +@pytest.mark.asyncio +async def test_pattern_scan_can_run_without_file_limit(tmp_path): + root = tmp_path / "a0" + usr = root / "usr" + usr.mkdir(parents=True) + for index in range(3): + (usr / f"file-{index}.txt").write_text(f"{index}\n", encoding="utf-8") + + service = BackupService() + service.agent_zero_root = str(root) + service.base_paths = {str(root): str(root)} + metadata = { + "include_patterns": [f"{root}/usr/**"], + "exclude_patterns": [], + "include_hidden": True, + } + + capped_files = await service.test_patterns(metadata, max_files=2) + all_files = await service.test_patterns(metadata, max_files=None) + + assert len(capped_files) == 2 + assert len(all_files) == 3 + + +@pytest.mark.asyncio +async def test_create_backup_uses_unlimited_pattern_scan(tmp_path, monkeypatch): + source_file = tmp_path / "source.txt" + source_file.write_text("payload\n", encoding="utf-8") + captured = {} + + service = BackupService() + + async def fake_test_patterns(metadata, max_files=1000): + captured["max_files"] = max_files + return [ + { + "path": f"{service.agent_zero_root.rstrip('/')}/usr/file-{index}.txt", + "real_path": str(source_file), + "size": source_file.stat().st_size, + "modified": "2026-06-26T00:00:00+00:00", + "type": "file", + } + for index in range(3) + ] + + async def fake_info(): + return {} + + async def fake_author(): + return "test" + + monkeypatch.setattr(service, "test_patterns", fake_test_patterns) + monkeypatch.setattr(service, "_get_system_info", fake_info) + monkeypatch.setattr(service, "_get_environment_info", fake_info) + monkeypatch.setattr(service, "_get_backup_author", fake_author) + + zip_path = await service.create_backup( + include_patterns=[f"{service.agent_zero_root}/usr/**"], + exclude_patterns=[], + include_hidden=True, + backup_name="large-backup", + ) + + assert captured["max_files"] is None + with zipfile.ZipFile(zip_path) as archive: + metadata = json.loads(archive.read("metadata.json").decode("utf-8")) + assert metadata["total_files"] == 3 + assert ( + f"{service.agent_zero_root.rstrip('/').lstrip('/')}/usr/file-2.txt" + in archive.namelist() + ) + + +@pytest.mark.asyncio +async def test_restore_can_reach_files_after_50000_archive_entries(tmp_path): + old_root = "/old-a0" + archive_root = old_root.lstrip("/") + file_count = 50_001 + last_index = file_count - 1 + zip_path = tmp_path / "large-backup.zip" + + metadata = { + "environment_info": {"agent_zero_root": old_root}, + "include_patterns": [f"{old_root}/usr/large/**"], + "exclude_patterns": [], + "include_hidden": True, + } + with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_STORED) as archive: + archive.writestr("metadata.json", json.dumps(metadata)) + for index in range(file_count): + payload = "tail payload\n" if index == last_index else "" + archive.writestr( + f"{archive_root}/usr/large/file-{index:05d}.txt", + payload, + ) + + service = BackupService() + service.agent_zero_root = str(tmp_path / "restored-a0") + + result = await service.restore_backup( + backup_file=UploadedBackup(zip_path), + restore_include_patterns=[ + f"{old_root}/usr/large/file-{last_index:05d}.txt" + ], + restore_exclude_patterns=[], + overwrite_policy="overwrite", + ) + + restored_path = ( + Path(service.agent_zero_root) + / "usr" + / "large" + / f"file-{last_index:05d}.txt" + ) + assert len(result["restored_files"]) == 1 + assert len(result["skipped_files"]) == file_count - 1 + assert result["errors"] == [] + assert restored_path.read_text(encoding="utf-8") == "tail payload\n" + + +@pytest.mark.asyncio +async def test_restore_clean_before_restore_uses_unlimited_pattern_scan(monkeypatch): + service = BackupService() + captured = {} + + async def fake_test_patterns(metadata, max_files=1000): + captured["max_files"] = max_files + return [] + + monkeypatch.setattr(service, "test_patterns", fake_test_patterns) + + result = await service._find_files_to_clean_with_user_metadata( + user_metadata={ + "include_patterns": [f"{service.agent_zero_root}/usr/**"], + "exclude_patterns": [], + "include_hidden": True, + }, + original_metadata={ + "environment_info": {"agent_zero_root": service.agent_zero_root} + }, + ) + + assert result == [] + assert captured["max_files"] is None