diff --git a/docs/OPENWRT_PACKAGES.md b/docs/OPENWRT_PACKAGES.md index ee66514..5c78141 100644 --- a/docs/OPENWRT_PACKAGES.md +++ b/docs/OPENWRT_PACKAGES.md @@ -162,6 +162,9 @@ x86 и x86_64. Сохраняются отчёты всех 30 сборок. с официальным индексом; повреждённый кэш не используется. Прогресс загрузки выводится в журнал сборки. Для локальной сборки каталог кэша можно задать переменной `OPENWRT_SDK_CACHE`. +При обрыве сети загрузка продолжается через HTTP Range, если сервер подтвердил +точное смещение. Если Range не поддерживается, скачивание начинается заново. +Повреждённый или незавершённый архив не заменяет рабочую копию. Кэш Actions ограничен веткой/тегом: повторный запуск использует свой кэш, а новый тег может использовать кэш основной ветки, но не другого тега. Ручная сборка на `main` сохраняет кэш основной ветки для следующих выпусков. diff --git a/scripts/openwrt_package.py b/scripts/openwrt_package.py index 2faaf7a..2cbae97 100644 --- a/scripts/openwrt_package.py +++ b/scripts/openwrt_package.py @@ -101,31 +101,68 @@ def download(url, path, expected): partial = path.with_name(path.name + ".partial") for attempt in range(3): try: - digest = hashlib.sha256() + if partial.is_file() and sha256(partial) == expected: + partial.replace(path) + print("Recovered verified download: " + path.name, flush=True) + return started = progress = time.monotonic() - print(f"Downloading {path.name}; attempt {attempt + 1}/3", flush=True) - request = urllib.request.Request(url, headers={"User-Agent": "OpenWrt-Remote-package-builder"}) - with urllib.request.urlopen(request, timeout=60) as response, partial.open("wb") as output: - size = 0 - while data := response.read(1024 * 1024): - size += len(data) - if size > 2 * 1024 ** 3: - raise ValueError("SDK download exceeded 2 GiB") - output.write(data) - digest.update(data) - now = time.monotonic() - if now - progress >= 15: - print(f"Download progress: {path.name}: {size / 1024 ** 2:.1f} MiB; elapsed={now - started:.0f}s", flush=True) - progress = now + offset = partial.stat().st_size if partial.is_file() else 0 + if offset > 2 * 1024 ** 3: + raise ValueError("SDK download exceeded 2 GiB") + print(f"Downloading {path.name}; attempt {attempt + 1}/3; resume_bytes={offset}", flush=True) + headers = {"User-Agent": "OpenWrt-Remote-package-builder"} + if offset: + headers["Range"] = f"bytes={offset}-" + request = urllib.request.Request(url, headers=headers) + with urllib.request.urlopen(request, timeout=60) as response: + response_headers = getattr(response, "headers", {}) + status = getattr(response, "status", 200) + if status == 206 and offset: + span = re.fullmatch(r"bytes ([0-9]+)-([0-9]+)/([0-9]+)", response_headers.get("Content-Range", "")) + if not span or int(span[1]) != offset or not offset <= int(span[2]) < int(span[3]) <= 2 * 1024 ** 3: + raise ValueError("invalid SDK download Content-Range") + total = int(span[3]) + elif status == 200: + # A server may ignore Range; replace the partial file. + offset = 0 + length = response_headers.get("Content-Length") + total = int(length) if length is not None else None + if total is not None and not 0 <= total <= 2 * 1024 ** 3: + raise ValueError("invalid SDK download Content-Length") + else: + raise ValueError("unexpected SDK download HTTP status: " + str(status)) + digest = hashlib.sha256() + if offset: + with partial.open("rb") as existing: + while data := existing.read(1024 * 1024): + digest.update(data) + size = offset + with partial.open("ab" if offset else "wb") as output: + while data := response.read(1024 * 1024): + size += len(data) + if size > 2 * 1024 ** 3 or (total is not None and size > total): + raise ValueError("SDK download exceeded declared size or 2 GiB") + output.write(data) + digest.update(data) + now = time.monotonic() + if now - progress >= 15: + print(f"Download progress: {path.name}: {size / 1024 ** 2:.1f} MiB; elapsed={now - started:.0f}s", flush=True) + progress = now + if total is not None and size != total: + raise OSError("incomplete SDK download") if digest.hexdigest() != expected: raise ValueError("SDK SHA256 mismatch: " + url) partial.replace(path) print(f"Download SHA256 verified: {path.name}; elapsed={time.monotonic() - started:.1f}s", flush=True) return - except Exception: - partial.unlink(missing_ok=True) + except Exception as error: + # Preserve bounded partial bytes only for transport failures. + # Invalid range, content, or digest always starts a fresh download. + if isinstance(error, ValueError) or getattr(error, "code", None) == 416 or attempt == 2: + partial.unlink(missing_ok=True) if attempt == 2: raise + print(f"Download retry: {path.name}: {type(error).__name__}: {error}", flush=True) time.sleep(2) diff --git a/tests/test_openwrt_package.py b/tests/test_openwrt_package.py index 7f70617..ad3ed03 100644 --- a/tests/test_openwrt_package.py +++ b/tests/test_openwrt_package.py @@ -183,6 +183,61 @@ class PackageTests(unittest.TestCase): self.assertEqual(destination.read_bytes(), expected.read_bytes()) self.assertFalse(destination.with_name(destination.name + ".partial").exists()) + def test_interrupted_sdk_download_resumes_only_with_matching_content_range(self): + class Interrupted(io.BytesIO): + status = 200 + headers = {"Content-Length": "12"} + + def read(self, size=-1): + data = super().read(size) + if not data: + raise TimeoutError("network interrupted") + return data + + class Remaining(io.BytesIO): + status = 206 + headers = {"Content-Range": "bytes 5-11/12"} + + destination = self.root / "sdk.tar.xz" + destination.write_bytes(b"working old archive") + expected = self.root / "expected" + expected.write_bytes(b"first second") + seen = [] + + def network(request, **kwargs): + self.assertEqual(destination.read_bytes(), b"working old archive") + seen.append(request.get_header("Range")) + return Interrupted(b"first") if len(seen) == 1 else Remaining(b" second") + + with mock.patch.object(p.urllib.request, "urlopen", side_effect=network), mock.patch.object(p.time, "sleep"): + p.download("https://downloads.openwrt.org/example", destination, p.sha256(expected)) + self.assertEqual(seen, [None, "bytes=5-"]) + self.assertEqual(destination.read_bytes(), expected.read_bytes()) + + def test_download_ignoring_range_restarts_without_appending(self): + destination = self.root / "sdk.tar.xz" + destination.with_name(destination.name + ".partial").write_bytes(b"first") + expected = self.root / "expected" + expected.write_bytes(b"first second") + with mock.patch.object(p.urllib.request, "urlopen", return_value=io.BytesIO(expected.read_bytes())) as network: + p.download("https://downloads.openwrt.org/example", destination, p.sha256(expected)) + self.assertEqual(network.call_args.args[0].get_header("Range"), "bytes=5-") + self.assertEqual(destination.read_bytes(), expected.read_bytes()) + + def test_download_rejects_mismatched_range_without_replacing_working_file(self): + class WrongRange(io.BytesIO): + status = 206 + headers = {"Content-Range": "bytes 4-11/12"} + + destination = self.root / "sdk.tar.xz" + destination.write_bytes(b"working old archive") + destination.with_name(destination.name + ".partial").write_bytes(b"first") + with mock.patch.object(p.urllib.request, "urlopen", side_effect=lambda *a, **k: WrongRange(b" second")), mock.patch.object(p.time, "sleep"): + with self.assertRaises(ValueError): + p.download("https://downloads.openwrt.org/example", destination, "a" * 64) + self.assertEqual(destination.read_bytes(), b"working old archive") + self.assertFalse(destination.with_name(destination.name + ".partial").exists()) + def test_openwrt_disk_image_accepts_fwtool_footer_and_rejects_truncation(self): source, destination = self.root / "image.gz", self.root / "image.raw" body = b"OpenWrt disk sectors" * 10000