Resume interrupted SDK downloads with verified HTTP ranges
This commit is contained in:
1 parent
47536ef269
commit
3e80b4d0e2
3 files changed
+103
-8
No files matched your search
@@ -162,6 +162,9 @@ x86 и x86_64. Сохраняются отчёты всех 30 сборок.
|
||||
с официальным индексом; повреждённый кэш не используется. Прогресс загрузки
|
||||
выводится в журнал сборки. Для локальной сборки каталог кэша можно задать
|
||||
переменной `OPENWRT_SDK_CACHE`.
|
||||
При обрыве сети загрузка продолжается через HTTP Range, если сервер подтвердил
|
||||
точное смещение. Если Range не поддерживается, скачивание начинается заново.
|
||||
Повреждённый или незавершённый архив не заменяет рабочую копию.
|
||||
Кэш Actions ограничен веткой/тегом: повторный запуск использует свой кэш,
|
||||
а новый тег может использовать кэш основной ветки, но не другого тега.
|
||||
Ручная сборка на `main` сохраняет кэш основной ветки для следующих выпусков.
|
||||
|
||||
@@ -101,31 +101,68 @@ def download(url, path, expected):
|
||||
partial = path.with_name(path.name + ".partial")
|
||||
for attempt in range(3):
|
||||
try:
|
||||
digest = hashlib.sha256()
|
||||
if partial.is_file() and sha256(partial) == expected:
|
||||
partial.replace(path)
|
||||
print("Recovered verified download: " + path.name, flush=True)
|
||||
return
|
||||
started = progress = time.monotonic()
|
||||
print(f"Downloading {path.name}; attempt {attempt + 1}/3", flush=True)
|
||||
request = urllib.request.Request(url, headers={"User-Agent": "OpenWrt-Remote-package-builder"})
|
||||
with urllib.request.urlopen(request, timeout=60) as response, partial.open("wb") as output:
|
||||
size = 0
|
||||
offset = partial.stat().st_size if partial.is_file() else 0
|
||||
if offset > 2 * 1024 ** 3:
|
||||
raise ValueError("SDK download exceeded 2 GiB")
|
||||
print(f"Downloading {path.name}; attempt {attempt + 1}/3; resume_bytes={offset}", flush=True)
|
||||
headers = {"User-Agent": "OpenWrt-Remote-package-builder"}
|
||||
if offset:
|
||||
headers["Range"] = f"bytes={offset}-"
|
||||
request = urllib.request.Request(url, headers=headers)
|
||||
with urllib.request.urlopen(request, timeout=60) as response:
|
||||
response_headers = getattr(response, "headers", {})
|
||||
status = getattr(response, "status", 200)
|
||||
if status == 206 and offset:
|
||||
span = re.fullmatch(r"bytes ([0-9]+)-([0-9]+)/([0-9]+)", response_headers.get("Content-Range", ""))
|
||||
if not span or int(span[1]) != offset or not offset <= int(span[2]) < int(span[3]) <= 2 * 1024 ** 3:
|
||||
raise ValueError("invalid SDK download Content-Range")
|
||||
total = int(span[3])
|
||||
elif status == 200:
|
||||
# A server may ignore Range; replace the partial file.
|
||||
offset = 0
|
||||
length = response_headers.get("Content-Length")
|
||||
total = int(length) if length is not None else None
|
||||
if total is not None and not 0 <= total <= 2 * 1024 ** 3:
|
||||
raise ValueError("invalid SDK download Content-Length")
|
||||
else:
|
||||
raise ValueError("unexpected SDK download HTTP status: " + str(status))
|
||||
digest = hashlib.sha256()
|
||||
if offset:
|
||||
with partial.open("rb") as existing:
|
||||
while data := existing.read(1024 * 1024):
|
||||
digest.update(data)
|
||||
size = offset
|
||||
with partial.open("ab" if offset else "wb") as output:
|
||||
while data := response.read(1024 * 1024):
|
||||
size += len(data)
|
||||
if size > 2 * 1024 ** 3:
|
||||
raise ValueError("SDK download exceeded 2 GiB")
|
||||
if size > 2 * 1024 ** 3 or (total is not None and size > total):
|
||||
raise ValueError("SDK download exceeded declared size or 2 GiB")
|
||||
output.write(data)
|
||||
digest.update(data)
|
||||
now = time.monotonic()
|
||||
if now - progress >= 15:
|
||||
print(f"Download progress: {path.name}: {size / 1024 ** 2:.1f} MiB; elapsed={now - started:.0f}s", flush=True)
|
||||
progress = now
|
||||
if total is not None and size != total:
|
||||
raise OSError("incomplete SDK download")
|
||||
if digest.hexdigest() != expected:
|
||||
raise ValueError("SDK SHA256 mismatch: " + url)
|
||||
partial.replace(path)
|
||||
print(f"Download SHA256 verified: {path.name}; elapsed={time.monotonic() - started:.1f}s", flush=True)
|
||||
return
|
||||
except Exception:
|
||||
except Exception as error:
|
||||
# Preserve bounded partial bytes only for transport failures.
|
||||
# Invalid range, content, or digest always starts a fresh download.
|
||||
if isinstance(error, ValueError) or getattr(error, "code", None) == 416 or attempt == 2:
|
||||
partial.unlink(missing_ok=True)
|
||||
if attempt == 2:
|
||||
raise
|
||||
print(f"Download retry: {path.name}: {type(error).__name__}: {error}", flush=True)
|
||||
time.sleep(2)
|
||||
|
||||
|
||||
|
||||
@@ -183,6 +183,61 @@ class PackageTests(unittest.TestCase):
|
||||
self.assertEqual(destination.read_bytes(), expected.read_bytes())
|
||||
self.assertFalse(destination.with_name(destination.name + ".partial").exists())
|
||||
|
||||
def test_interrupted_sdk_download_resumes_only_with_matching_content_range(self):
|
||||
class Interrupted(io.BytesIO):
|
||||
status = 200
|
||||
headers = {"Content-Length": "12"}
|
||||
|
||||
def read(self, size=-1):
|
||||
data = super().read(size)
|
||||
if not data:
|
||||
raise TimeoutError("network interrupted")
|
||||
return data
|
||||
|
||||
class Remaining(io.BytesIO):
|
||||
status = 206
|
||||
headers = {"Content-Range": "bytes 5-11/12"}
|
||||
|
||||
destination = self.root / "sdk.tar.xz"
|
||||
destination.write_bytes(b"working old archive")
|
||||
expected = self.root / "expected"
|
||||
expected.write_bytes(b"first second")
|
||||
seen = []
|
||||
|
||||
def network(request, **kwargs):
|
||||
self.assertEqual(destination.read_bytes(), b"working old archive")
|
||||
seen.append(request.get_header("Range"))
|
||||
return Interrupted(b"first") if len(seen) == 1 else Remaining(b" second")
|
||||
|
||||
with mock.patch.object(p.urllib.request, "urlopen", side_effect=network), mock.patch.object(p.time, "sleep"):
|
||||
p.download("https://downloads.openwrt.org/example", destination, p.sha256(expected))
|
||||
self.assertEqual(seen, [None, "bytes=5-"])
|
||||
self.assertEqual(destination.read_bytes(), expected.read_bytes())
|
||||
|
||||
def test_download_ignoring_range_restarts_without_appending(self):
|
||||
destination = self.root / "sdk.tar.xz"
|
||||
destination.with_name(destination.name + ".partial").write_bytes(b"first")
|
||||
expected = self.root / "expected"
|
||||
expected.write_bytes(b"first second")
|
||||
with mock.patch.object(p.urllib.request, "urlopen", return_value=io.BytesIO(expected.read_bytes())) as network:
|
||||
p.download("https://downloads.openwrt.org/example", destination, p.sha256(expected))
|
||||
self.assertEqual(network.call_args.args[0].get_header("Range"), "bytes=5-")
|
||||
self.assertEqual(destination.read_bytes(), expected.read_bytes())
|
||||
|
||||
def test_download_rejects_mismatched_range_without_replacing_working_file(self):
|
||||
class WrongRange(io.BytesIO):
|
||||
status = 206
|
||||
headers = {"Content-Range": "bytes 4-11/12"}
|
||||
|
||||
destination = self.root / "sdk.tar.xz"
|
||||
destination.write_bytes(b"working old archive")
|
||||
destination.with_name(destination.name + ".partial").write_bytes(b"first")
|
||||
with mock.patch.object(p.urllib.request, "urlopen", side_effect=lambda *a, **k: WrongRange(b" second")), mock.patch.object(p.time, "sleep"):
|
||||
with self.assertRaises(ValueError):
|
||||
p.download("https://downloads.openwrt.org/example", destination, "a" * 64)
|
||||
self.assertEqual(destination.read_bytes(), b"working old archive")
|
||||
self.assertFalse(destination.with_name(destination.name + ".partial").exists())
|
||||
|
||||
def test_openwrt_disk_image_accepts_fwtool_footer_and_rejects_truncation(self):
|
||||
source, destination = self.root / "image.gz", self.root / "image.raw"
|
||||
body = b"OpenWrt disk sectors" * 10000
|
||||
|
||||
Reference in new issue
Block a user