Cache verified OpenWrt SDK archives and report download progress
This commit is contained in:
1 parent
5630b64d81
commit
47536ef269
4 files changed
+44
-1
No files matched your search
@@ -92,8 +92,14 @@ jobs:
|
|||||||
run: |
|
run: |
|
||||||
sudo apt-get update -qq
|
sudo apt-get update -qq
|
||||||
sudo apt-get install -y build-essential clang flex bison g++ gawk gcc-multilib gettext git libncurses-dev libssl-dev python3-setuptools rsync swig unzip zlib1g-dev file wget zstd
|
sudo apt-get install -y build-essential clang flex bison g++ gawk gcc-multilib gettext git libncurses-dev libssl-dev python3-setuptools rsync swig unzip zlib1g-dev file wget zstd
|
||||||
|
- name: Restore official SDK archive cache
|
||||||
|
uses: actions/cache@v4
|
||||||
|
with:
|
||||||
|
path: ${{ runner.temp }}/openwrt-sdk-cache
|
||||||
|
key: openwrt-sdk-v1-${{ runner.os }}-${{ matrix.version }}-${{ matrix.target }}-${{ matrix.subtarget }}
|
||||||
- name: Build and verify the native package
|
- name: Build and verify the native package
|
||||||
env:
|
env:
|
||||||
|
OPENWRT_SDK_CACHE: ${{ runner.temp }}/openwrt-sdk-cache
|
||||||
OPENWRT_APK_SIGNING_KEY: ${{ secrets.OPENWRT_APK_SIGNING_KEY }}
|
OPENWRT_APK_SIGNING_KEY: ${{ secrets.OPENWRT_APK_SIGNING_KEY }}
|
||||||
run: |
|
run: |
|
||||||
extra=()
|
extra=()
|
||||||
|
|||||||
@@ -157,6 +157,15 @@ Workflow `.github/workflows/openwrt-packages.yml` запускается на т
|
|||||||
на каждую ветку: A53, A72, generic ARM64, ARM A7/A9, MIPS, MIPSEL, MIPS64,
|
на каждую ветку: A53, A72, generic ARM64, ARM A7/A9, MIPS, MIPSEL, MIPS64,
|
||||||
x86 и x86_64. Сохраняются отчёты всех 30 сборок.
|
x86 и x86_64. Сохраняются отчёты всех 30 сборок.
|
||||||
|
|
||||||
|
Сжатые архивы SDK сохраняются в GitHub Actions cache отдельно для каждой
|
||||||
|
версии/платформы. Перед повторным использованием SHA256 снова сверяется
|
||||||
|
с официальным индексом; повреждённый кэш не используется. Прогресс загрузки
|
||||||
|
выводится в журнал сборки. Для локальной сборки каталог кэша можно задать
|
||||||
|
переменной `OPENWRT_SDK_CACHE`.
|
||||||
|
Кэш Actions ограничен веткой/тегом: повторный запуск использует свой кэш,
|
||||||
|
а новый тег может использовать кэш основной ветки, но не другого тега.
|
||||||
|
Ручная сборка на `main` сохраняет кэш основной ветки для следующих выпусков.
|
||||||
|
|
||||||
После сборки универсальные пакеты устанавливаются в три изолированные x86_64
|
После сборки универсальные пакеты устанавливаются в три изолированные x86_64
|
||||||
OpenWrt VM. Проверяется первичная установка, настоящая смена пакетной ревизии,
|
OpenWrt VM. Проверяется первичная установка, настоящая смена пакетной ревизии,
|
||||||
работа CGI, BusyBox ash, службы, удаление, сохранность настроек/ключа/Xray и
|
работа CGI, BusyBox ash, службы, удаление, сохранность настроек/ключа/Xray и
|
||||||
|
|||||||
@@ -96,11 +96,14 @@ def sha256(path):
|
|||||||
def download(url, path, expected):
|
def download(url, path, expected):
|
||||||
path = Path(path)
|
path = Path(path)
|
||||||
if path.is_file() and sha256(path) == expected:
|
if path.is_file() and sha256(path) == expected:
|
||||||
|
print("Verified download cache: " + path.name, flush=True)
|
||||||
return
|
return
|
||||||
partial = path.with_name(path.name + ".partial")
|
partial = path.with_name(path.name + ".partial")
|
||||||
for attempt in range(3):
|
for attempt in range(3):
|
||||||
try:
|
try:
|
||||||
digest = hashlib.sha256()
|
digest = hashlib.sha256()
|
||||||
|
started = progress = time.monotonic()
|
||||||
|
print(f"Downloading {path.name}; attempt {attempt + 1}/3", flush=True)
|
||||||
request = urllib.request.Request(url, headers={"User-Agent": "OpenWrt-Remote-package-builder"})
|
request = urllib.request.Request(url, headers={"User-Agent": "OpenWrt-Remote-package-builder"})
|
||||||
with urllib.request.urlopen(request, timeout=60) as response, partial.open("wb") as output:
|
with urllib.request.urlopen(request, timeout=60) as response, partial.open("wb") as output:
|
||||||
size = 0
|
size = 0
|
||||||
@@ -110,9 +113,14 @@ def download(url, path, expected):
|
|||||||
raise ValueError("SDK download exceeded 2 GiB")
|
raise ValueError("SDK download exceeded 2 GiB")
|
||||||
output.write(data)
|
output.write(data)
|
||||||
digest.update(data)
|
digest.update(data)
|
||||||
|
now = time.monotonic()
|
||||||
|
if now - progress >= 15:
|
||||||
|
print(f"Download progress: {path.name}: {size / 1024 ** 2:.1f} MiB; elapsed={now - started:.0f}s", flush=True)
|
||||||
|
progress = now
|
||||||
if digest.hexdigest() != expected:
|
if digest.hexdigest() != expected:
|
||||||
raise ValueError("SDK SHA256 mismatch: " + url)
|
raise ValueError("SDK SHA256 mismatch: " + url)
|
||||||
partial.replace(path)
|
partial.replace(path)
|
||||||
|
print(f"Download SHA256 verified: {path.name}; elapsed={time.monotonic() - started:.1f}s", flush=True)
|
||||||
return
|
return
|
||||||
except Exception:
|
except Exception:
|
||||||
partial.unlink(missing_ok=True)
|
partial.unlink(missing_ok=True)
|
||||||
@@ -410,7 +418,9 @@ def build(args):
|
|||||||
output.mkdir(parents=True, exist_ok=True)
|
output.mkdir(parents=True, exist_ok=True)
|
||||||
with tempfile.TemporaryDirectory(prefix="owrt-package-") as folder:
|
with tempfile.TemporaryDirectory(prefix="owrt-package-") as folder:
|
||||||
work = Path(folder)
|
work = Path(folder)
|
||||||
archive = work / name
|
cache = os.environ.get("OPENWRT_SDK_CACHE")
|
||||||
|
archive = (Path(cache).expanduser() if cache else work) / name
|
||||||
|
archive.parent.mkdir(parents=True, exist_ok=True)
|
||||||
print("Downloading verified SDK: " + base + name, flush=True)
|
print("Downloading verified SDK: " + base + name, flush=True)
|
||||||
download(base + name, archive, expected)
|
download(base + name, archive, expected)
|
||||||
subprocess.run(["tar", "-xf", str(archive), "-C", str(work)], check=True)
|
subprocess.run(["tar", "-xf", str(archive), "-C", str(work)], check=True)
|
||||||
|
|||||||
@@ -165,6 +165,24 @@ class PackageTests(unittest.TestCase):
|
|||||||
self.assertEqual(destination.read_bytes(), b"previous download")
|
self.assertEqual(destination.read_bytes(), b"previous download")
|
||||||
self.assertFalse(destination.with_name(destination.name + ".partial").exists())
|
self.assertFalse(destination.with_name(destination.name + ".partial").exists())
|
||||||
|
|
||||||
|
def test_verified_sdk_cache_needs_no_network(self):
|
||||||
|
destination = self.root / "sdk.tar.xz"
|
||||||
|
destination.write_bytes(b"verified SDK archive")
|
||||||
|
with mock.patch.object(p.urllib.request, "urlopen") as network:
|
||||||
|
p.download("https://downloads.openwrt.org/example", destination, p.sha256(destination))
|
||||||
|
network.assert_not_called()
|
||||||
|
|
||||||
|
def test_invalid_sdk_cache_is_replaced_only_by_verified_download(self):
|
||||||
|
destination = self.root / "sdk.tar.xz"
|
||||||
|
destination.write_bytes(b"invalid cache")
|
||||||
|
expected = self.root / "expected.tar.xz"
|
||||||
|
expected.write_bytes(b"verified SDK archive")
|
||||||
|
with mock.patch.object(p.urllib.request, "urlopen", return_value=io.BytesIO(expected.read_bytes())) as network:
|
||||||
|
p.download("https://downloads.openwrt.org/example", destination, p.sha256(expected))
|
||||||
|
network.assert_called_once()
|
||||||
|
self.assertEqual(destination.read_bytes(), expected.read_bytes())
|
||||||
|
self.assertFalse(destination.with_name(destination.name + ".partial").exists())
|
||||||
|
|
||||||
def test_openwrt_disk_image_accepts_fwtool_footer_and_rejects_truncation(self):
|
def test_openwrt_disk_image_accepts_fwtool_footer_and_rejects_truncation(self):
|
||||||
source, destination = self.root / "image.gz", self.root / "image.raw"
|
source, destination = self.root / "image.gz", self.root / "image.raw"
|
||||||
body = b"OpenWrt disk sectors" * 10000
|
body = b"OpenWrt disk sectors" * 10000
|
||||||
|
|||||||
Reference in new issue
Block a user