Add native OpenWrt IPK/APK packages and release build workflow

This commit is contained in:
kzolotarev95 committed 2026-11-07 16:03:39 +04:00
1 parent edbc86697a
commit 76fa63e7a1
13 files changed
+1413

No files matched your search

+4
View File
@@ -2,6 +2,10 @@
*.md text eol=lf
*.txt text eol=lf
*.sh text eol=lf
*.py text eol=lf
*.yml text eol=lf
package/**/Makefile text eol=lf
.gitattributes text eol=lf
.gitignore text eol=lf
LICENSE text eol=lf
+158
View File
@@ -0,0 +1,158 @@
name: OpenWrt IPK and APK
on:
push:
tags: ['v*']
release:
types: [published]
pull_request:
paths: ['files/**', 'package/**', 'scripts/*openwrt*', 'tests/test_openwrt_package.py', '.github/workflows/openwrt-packages.yml']
workflow_dispatch:
permissions:
contents: read
concurrency:
group: openwrt-packages-${{ github.ref }}
cancel-in-progress: false
jobs:
matrix:
name: Validate source and resolve stable SDKs
runs-on: ubuntu-24.04
outputs:
builds: ${{ steps.targets.outputs.builds }}
smoke: ${{ steps.targets.outputs.smoke }}
steps:
- uses: actions/checkout@v4
- name: Test package tooling and shell syntax
run: |
python3 -m unittest discover -s tests -p test_openwrt_package.py -v
sh -n scripts/build-openwrt-package.sh
sh -n package/luci-app-owrt-remote/root/usr/lib/owrt-remote/package.sh
- name: Validate tag against package version
if: startsWith(github.ref, 'refs/tags/')
env:
RELEASE_TAG: ${{ github.ref_name }}
run: |
python3 - <<'PY'
import os, sys
sys.path.insert(0, 'scripts')
import openwrt_package as p
if os.environ['RELEASE_TAG'] != 'v' + p.versions()[0]:
raise SystemExit('Release tag must match PKG_VERSION in package/luci-app-owrt-remote/Makefile')
PY
- id: targets
name: Resolve latest maintenance versions for each supported series
run: |
python3 - <<'PY'
import json, os, sys
sys.path.insert(0, 'scripts')
import openwrt_package as p
builds = p.matrix()
smoke = {'include': [x for x in builds['include'] if x.get('publish')]}
with open(os.environ['GITHUB_OUTPUT'], 'a') as f:
f.write('builds=' + json.dumps(builds, separators=(',', ':')) + '\n')
f.write('smoke=' + json.dumps(smoke, separators=(',', ':')) + '\n')
PY
build:
name: SDK ${{ matrix.version }} / ${{ matrix.id }}
needs: matrix
runs-on: ubuntu-24.04
timeout-minutes: 60
strategy:
fail-fast: false
max-parallel: 6
matrix: ${{ fromJSON(needs.matrix.outputs.builds) }}
steps:
- uses: actions/checkout@v4
- name: Install official SDK prerequisites
run: |
sudo apt-get update -qq
sudo apt-get install -y build-essential clang flex bison g++ gawk gcc-multilib gettext git libncurses-dev libssl-dev python3-setuptools rsync swig unzip zlib1g-dev file wget zstd
- name: Build and verify the native package
env:
OPENWRT_APK_SIGNING_KEY: ${{ secrets.OPENWRT_APK_SIGNING_KEY }}
run: |
extra=()
if [[ '${{ matrix.publish }}' == 'true' ]]; then extra+=(--upgrade-fixture); fi
python3 scripts/openwrt_package.py build \
--release '${{ matrix.version }}' \
--target '${{ matrix.target }}' \
--subtarget '${{ matrix.subtarget }}' \
--arch '${{ matrix.id }}' \
--output artifacts "${extra[@]}"
- name: Keep SDK verification report
uses: actions/upload-artifact@v4
with:
name: sdk-report-${{ matrix.series }}-${{ matrix.id }}
path: artifacts/report-*.json
if-no-files-found: error
- name: Keep universal packages for LuCI and SSH
if: matrix.publish == true
uses: actions/upload-artifact@v4
with:
name: packages-${{ matrix.series }}
path: artifacts/*
if-no-files-found: error
install-test:
name: Install, upgrade, remove / OpenWrt ${{ matrix.version }}
needs: [matrix, build]
runs-on: ubuntu-24.04
timeout-minutes: 25
strategy:
fail-fast: false
matrix: ${{ fromJSON(needs.matrix.outputs.smoke) }}
steps:
- uses: actions/checkout@v4
- uses: actions/download-artifact@v4
with:
name: packages-${{ matrix.series }}
path: artifacts
- name: Install VM test tools
run: |
sudo apt-get update -qq
sudo apt-get install -y qemu-system-x86 openssh-client
- name: Test with the real OpenWrt package manager and BusyBox ash
run: |
python3 scripts/test-openwrt-package-vm.py \
--release '${{ matrix.version }}' --package-dir artifacts \
--output vm-test
- uses: actions/upload-artifact@v4
if: always()
with:
name: install-test-${{ matrix.series }}
path: vm-test/*
if-no-files-found: warn
publish:
name: Attach verified packages to GitHub Release
needs: [build, install-test]
if: startsWith(github.ref, 'refs/tags/v') && github.event_name != 'workflow_dispatch'
runs-on: ubuntu-24.04
permissions:
contents: write
steps:
- uses: actions/download-artifact@v4
with:
pattern: packages-*
path: release-assets
merge-multiple: true
- name: Write combined SHA256 checksums
run: |
cd release-assets
sha256sum *.ipk *.apk > SHA256SUMS
if [[ -f owrt-remote-packages.pem ]]; then sha256sum owrt-remote-packages.pem >> SHA256SUMS; fi
printf '%s\n' *.ipk *.apk SHA256SUMS report-*.json > ../release-files.txt
if [[ -f owrt-remote-packages.pem ]]; then echo owrt-remote-packages.pem >> ../release-files.txt; fi
mkdir ../publish-files
while read -r name; do cp "$name" ../publish-files/; done < ../release-files.txt
- uses: softprops/action-gh-release@v2
with:
tag_name: ${{ github.ref_name }}
generate_release_notes: true
fail_on_unmatched_files: true
files: |
publish-files/*
+2
View File
@@ -1,4 +1,6 @@
*.ipk
*.apk
artifacts/
*.tar.gz
*.tar.gz.sha256
.test-output/
+11
View File
@@ -182,6 +182,17 @@ owrt-remote doctor
owrt-remote status
```
<h3> Готовые пакеты для OpenWrt</h3>
Подготовлена сборка IPK для OpenWrt 23.05/24.10 и APK для 25.12. Пакеты
архитектурно независимые (`all`/`noarch`): содержат тот же агент и LuCI-файлы,
что и `install.sh`, сохраняют `/etc/config/owrtremote` и существующий Xray.
GitHub Actions проверяет SDK разных архитектур, установку/обновление/удаление
в OpenWrt VM и при выпуске тега `v*` прикладывает пакеты к GitHub Release.
Для APK в штатной LuCI нужен доверенный ключ подписи; неподписанный локальный
APK ставится через SSH. Инструкция, подпись и локальная сборка:
[docs/OPENWRT_PACKAGES.md](docs/OPENWRT_PACKAGES.md).
</div>
---
+189
View File
@@ -0,0 +1,189 @@
# Пакеты OpenWrt Remote
Исходники пакета `luci-app-owrt-remote` находятся в
`package/luci-app-owrt-remote/`. Пакет содержит агент, CGI-панель, LuCI,
Heartbeat, управление Xray, WAN direct и журнал. Файлы берутся непосредственно
из `files/`, которую использует `install.sh`; отдельная копия агента не ведётся.
Это пакеты проекта для OpenWrt, а не включение проекта в официальный feed OpenWrt.
Публикация в Releases произойдёт только после отправки workflow и выпуска тега.
## Какой файл выбрать
| OpenWrt | Файл | Архитектура пакета |
| --- | --- | --- |
| 23.05.x | `luci-app-owrt-remote_121-r1_openwrt-23.05_all.ipk` | `all` |
| 24.10.x | `luci-app-owrt-remote_121-r1_openwrt-24.10_all.ipk` | `all` |
| 25.12.x | `luci-app-owrt-remote_121-r1_openwrt-25.12_noarch.apk` | `noarch` |
В пакете нет ELF-бинарников. Один файл своей ветки подходит `aarch64_cortex-a53`,
`aarch64_cortex-a72`, `aarch64_generic`, ARM, MIPS, MIPSEL, MIPS64, x86, x86_64
и другим архитектурам с доступными зависимостями. Формат определяется менеджером
пакетов: `opkg` использует IPK, `apk` использует APK; они не взаимозаменяемы.
Xray загружается отдельно существующим агентом при применении конфигурации Hub.
Его версия, ABI, SHA256 и загрузочный кэш проверяются существующим механизмом.
Установка пакета не запускает вложенный `opkg`/`apk` и не загружает Xray,
пока менеджер пакетов удерживает свою блокировку.
## Установка и обновление через LuCI
1. Скачай пакет своей ветки из [GitHub Releases](https://github.com/kzolotarev95/luci-app-owrt-remote/releases).
2. Открой **Система → Программное обеспечение**, обнови списки пакетов.
3. Нажми **Загрузить пакет**, выбери файл и подтверди установку.
4. Открой **Службы → OpenWrt Remote** и примени OpenWrt config из своей VPS-панели.
Обновление выполняется загрузкой пакета новой версии тем же способом.
Перезагрузка роутера не требуется. Зависимости устанавливает менеджер пакетов:
`luci-base`, `rpcd-mod-file`, `uhttpd`, `jsonfilter`, `curl`, `ca-bundle`, `etherwake`.
Для их первичной загрузки нужны доступные репозитории своей прошивки.
На OpenWrt 25.12 штатная LuCI принимает сторонний APK только с доверенной
подписью. Для подписанных выпусков один раз установи опубликованный публичный
ключ проекта в `/etc/apk/keys/owrt-remote-packages.pem`, проверив его отпечаток
по доверенному каналу. Далее подписанные APK устанавливаются через LuCI.
Без ключа штатная LuCI отклоняет пакет; для локального неподписанного APK
используй SSH-команду ниже. Глобально отключать проверку подписей не нужно.
## Установка по SSH из PowerShell Windows 11
Пример для OpenWrt 24.10 и основного роутера `192.168.2.1`:
```powershell
$Repo = "C:\Users\k.zolotarev95\Documents\openwrt overlay"
$Package = Join-Path $Repo "artifacts\openwrt-packages\24.10\luci-app-owrt-remote_121-r1_openwrt-24.10_all.ipk"
ssh root@192.168.2.1 'sysupgrade -b /root/owrt-before-package.tar.gz'
if ($LASTEXITCODE -ne 0) { throw 'Резервная копия не создана' }
scp -O -- "$Package" 'root@192.168.2.1:/tmp/owrt-remote.ipk'
if ($LASTEXITCODE -ne 0) { throw 'Не удалось загрузить пакет' }
ssh root@192.168.2.1 'opkg update && opkg install /tmp/owrt-remote.ipk'
if ($LASTEXITCODE -ne 0) { throw 'Установка не завершена; смотри ошибку выше' }
```
На OpenWrt 23.05 выбери файл из каталога `23.05`. Для OpenWrt 25.12 загрузи
файл из `25.12` как `/tmp/owrt-remote.apk` и выполни:
```powershell
ssh root@192.168.2.1 'apk update && apk add --allow-untrusted /tmp/owrt-remote.apk'
```
`--allow-untrusted` нужен только неподписанному локальному APK. Для подписанного
APK с установленным доверенным ключом используется `apk add` без этого флага.
После установки проверь `owrt-remote status` и `owrt-remote doctor`.
На чистой установке Remote выключен до ввода своего Hub config — это штатно.
## Сохранение настроек и откат
`/etc/config/owrtremote` объявлен conffile. Обновление сохраняет изменённые UCI,
ключ `/etc/owrt-remote/web.key`, Xray, JSON, постоянный кэш и журнал. Ключ панели
создаётся только при его отсутствии, с правами `0600`, через атомарную замену.
Удаление пакета останавливает и отключает его procd-службы. Пакет не очищает
сгенерированные данные пользователя и не запускает `uninstall.sh`:
```powershell
# Для IPK:
ssh root@192.168.2.1 'opkg remove luci-app-owrt-remote'
# Для APK:
ssh root@192.168.2.1 'apk del luci-app-owrt-remote'
```
Для отката к прежнему пакетному выпуску сохрани его файл до обновления,
удали новый пакет и установи сохранённый IPK/APK. Изменённый UCI при удалении
сохраняется. Чтобы вернуть настройки из сделанной выше резервной копии,
извлеки только конфигурацию Remote, затем перезапусти службу:
```powershell
ssh root@192.168.2.1 'tar -xzf /root/owrt-before-package.tar.gz -C / etc/config/owrtremote && /etc/init.d/owrt-remote restart'
```
`sysupgrade -b` сохраняет настройки, а не старые программные файлы.
При переходе с `install.sh` сохрани исходный checkout/предыдущий установщик:
удаление пакета само по себе не восстанавливает старую ручную установку.
## Сборка
Локальный IPK без SDK можно собрать и проверить на Windows с Python 3.11+:
```powershell
Set-Location "C:\Users\k.zolotarev95\Documents\openwrt overlay"
python scripts/openwrt_package.py local-ipk --output artifacts/remote-local.ipk
python scripts/openwrt_package.py verify artifacts/remote-local.ipk
```
Локальный APK создаётся настоящим `apk-tools v3 mkpkg` из host tools SDK 25.12.
Целевой `apk` на роутере может не содержать команду `mkpkg`. На Linux:
```sh
python3 scripts/openwrt_package.py local-apk --apk /path/to/sdk/staging_dir/host/bin/apk --output artifacts/remote-local.apk
```
Релизная сборка выполняется официальным SDK на Linux x86_64. Скрипт находит
имя SDK в индексе OpenWrt, проверяет SHA256, использует feeds этого SDK,
собирает пакет и проверяет содержимое, права, владельца, зависимости и conffiles:
```sh
sh scripts/build-openwrt-package.sh --release 24.10.3 --target ramips --subtarget mt7621 --arch mipsel_24kc --output artifacts
```
SDK поддерживаются в форматах `.tar.xz` и `.tar.zst`. Для подготовки автономного
каталога пакета внутри своего SDK:
```sh
python3 scripts/openwrt_package.py prepare --output /path/to/sdk/package/luci-app-owrt-remote
```
## GitHub Actions и подпись APK
Workflow `.github/workflows/openwrt-packages.yml` запускается на тегах `v*`,
публикации Release, изменениях пакета в pull request и вручную. Вручную и в PR
он только собирает и тестирует; в Releases пишет только запуск по тегу.
Тег должен совпадать с `PKG_VERSION`: текущий пример — `v121`.
Следующий выпуск должен получить новую версию или пакетную ревизию в Makefile.
На каждом запуске выбирается последняя стабильная patch-версия веток
23.05, 24.10 и 25.12. Матрица `package/openwrt-targets.json` содержит 10 SDK
на каждую ветку: A53, A72, generic ARM64, ARM A7/A9, MIPS, MIPSEL, MIPS64,
x86 и x86_64. Сохраняются отчёты всех 30 сборок.
После сборки универсальные пакеты устанавливаются в три изолированные x86_64
OpenWrt VM. Проверяется первичная установка, настоящая смена пакетной ревизии,
работа CGI, BusyBox ash, службы, удаление, сохранность настроек/ключа/Xray и
переход с не принадлежащей пакетному менеджеру конфигурации. Тестовая ревизия
из `test-fixtures/` не публикуется. Только после успешного завершения
всей матрицы публикуются три пакета, SHA256SUMS и отчёты SDK.
Для подписанных APK добавь в GitHub Actions secret `OPENWRT_APK_SIGNING_KEY`
постоянный приватный RSA PEM-ключ сопровождающего. Пример создания на доверенной
машине (приватный файл не добавлять в Git):
```sh
openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:3072 -out /secure/owrt-remote-private.pem
openssl pkey -in /secure/owrt-remote-private.pem -pubout -out owrt-remote-packages.pem
```
Workflow подписывает APK, проверяет установку с публичным ключом и добавляет
`owrt-remote-packages.pem` в Release. Приватный ключ остаётся secret/временным
файлом runner и не попадает в artifacts. Если secret отсутствует, выпуск APK
явно помечается в отчёте как неподписанный и устанавливается по SSH с
`--allow-untrusted`. IPK устанавливается локальным файлом обычным способом.
Новый случайный ключ на каждом запуске не создаётся.
## Проверки перед публикацией
```sh
python3 -m unittest discover -s tests -p test_openwrt_package.py -v
python3 scripts/test-openwrt-package-vm.py --release 24.10.3 --package-dir artifacts --output vm-test
```
Для VM-теста нужны QEMU, SSH и один основной пакет плюс пакет следующей
ревизии в `test-fixtures/`. SDK-сборка создаёт такой набор с `--upgrade-fixture`.
VM использует только loopback и не подключается к реальным роутерам.
Проверка установки на x86_64 не доказывает работу конкретного Xray на каждом
физическом CPU. Xray, Heartbeat с реальным Hub, WAN direct и аппаратные
особенности требуют отдельной проверки на устройствах.
Основа реализации: [OpenWrt SDK](https://openwrt.org/docs/guide-developer/toolchain/using_the_sdk),
[пакетный Makefile OpenWrt](https://github.com/openwrt/openwrt/blob/openwrt-25.12/include/package-pack.mk),
[жизненный цикл пакетов](https://github.com/openwrt/openwrt/blob/openwrt-25.12/package/base-files/files/lib/functions.sh).
+48
View File
@@ -0,0 +1,48 @@
# Локальная проверка пакетной установки
Дата: 07.11.2026. Версия приложения: v121, ревизия пакета: 1.
Проверки выполнялись локально; push и публикация Release не выполнялись.
Реальные роутеры и VPS не изменялись.
| Среда | Менеджер | Результат |
| --- | --- | --- |
| OpenWrt 23.05.6 x86_64, QEMU TCG | opkg, IPK `all` | Успешно |
| OpenWrt 24.10.8 x86_64, QEMU TCG | opkg, IPK `all` | Успешно |
| OpenWrt 25.12.6 x86_64, QEMU TCG | apk, APK v3 `noarch` | Успешно |
В каждой VM проверены:
- Чистая установка и загрузка зависимостей из репозиториев соответствующей прошивки.
- Создание приватного ключа панели, доступность CGI и пунктов LuCI.
- Синтаксис всех shell-файлов пакета штатным BusyBox ash.
- Работа службы журналов и сохранение выключенного по умолчанию Remote.
- Обновление с ревизии 1 на отдельный пакет ревизии 2.
- Побайтовое сохранение изменённого UCI, ключа панели, Xray JSON и бинарника-заглушки.
- Удаление файлов пакета и сохранение изменённой конфигурации/сгенерированных данных.
- Повторная установка с конфигурацией, не принадлежащей менеджеру пакетов.
Дополнительно:
- 16 автоматических тестов сборочного инструмента прошли.
- `actionlint v1.7.12` не обнаружил ошибок workflow.
- Проверены все 30 URL SDK и их записи SHA256 в официальных индексах:
10 целевых архитектур для каждой из трёх веток.
- Подтверждён отказ штатного apk от неподписанного APK без `--allow-untrusted`.
- APK подписан отдельным тестовым RSA-ключом; после добавления публичного
ключа штатный apk принял его без `--allow-untrusted` в проверке `--simulate`.
Тестовый приватный ключ не входит в исходники или установочные артефакты.
- Установка не вызывала скачивание/замену рабочего Xray внутри пакетной транзакции.
Локальные IPK созданы переносимым сборщиком из тех же исходников; APK создан
настоящим `apk-tools 3.0.9 mkpkg` в изолированной VM. Полная компиляция через
30 официальных SDK здесь не запускалась: она подготовлена в GitHub Actions.
Доступность SDK не равнозначна успешной сборке; публикация workflow зависит
от успешного завершения SDK-матрицы и последующих VM-тестов.
Физические ARM/MIPS-устройства, действующий Heartbeat, WAN direct и туннель
Xray с реальным Hub в этой проверке не тестировались. Код этих механизмов
не изменён пакетной доработкой.
Локальные протоколы находятся в `.test-output/vm-23.05/`,
`.test-output/vm-24.10/`, `.test-output/vm-25.12/`. Готовые тестовые пакеты —
в `artifacts/openwrt-packages/`. Эти каталоги исключены из Git.
+67
View File
@@ -0,0 +1,67 @@
include $(TOPDIR)/rules.mk
PKG_NAME:=luci-app-owrt-remote
PKG_VERSION:=121
PKG_RELEASE:=1
PKG_LICENSE:=MIT
PKG_MAINTAINER:=kzolotarev95
PKGARCH:=all
include $(INCLUDE_DIR)/package.mk
define Package/luci-app-owrt-remote
SECTION:=luci
CATEGORY:=LuCI
SUBMENU:=3. Applications
TITLE:=OpenWrt Remote Hub router agent
URL:=https://github.com/kzolotarev95/luci-app-owrt-remote
DEPENDS:=+luci-base +rpcd-mod-file +uhttpd +jsonfilter +curl +ca-bundle +etherwake
endef
define Package/luci-app-owrt-remote/description
Router agent and LuCI panel for OpenWrt Remote Hub. Includes Heartbeat,
independent remote Xray control, WAN direct and bounded persistent logs.
Xray is selected and verified for the router by the existing agent when
applying the Hub configuration; it is not bundled into this noarch package.
endef
define Package/luci-app-owrt-remote/conffiles
/etc/config/owrtremote
endef
# scripts/openwrt_package.py prepare creates this self-contained SDK package
# from files/ and root/. There are no separately maintained runtime copies.
define Build/Prepare
mkdir -p $(PKG_BUILD_DIR)
endef
define Build/Configure
endef
define Build/Compile
endef
define Package/luci-app-owrt-remote/install
$(CP) ./root/. $(1)/
endef
define Package/luci-app-owrt-remote/postinst
#!/bin/sh
[ -n "$${IPKG_INSTROOT}" ] || /bin/sh /usr/lib/owrt-remote/package.sh configure
endef
define Package/luci-app-owrt-remote/postrm
#!/bin/sh
[ -n "$${IPKG_INSTROOT}" ] || { \
rm -f /tmp/luci-indexcache /tmp/luci-indexcache.*; \
rm -rf /tmp/luci-modulecache; \
[ ! -x /etc/init.d/rpcd ] || /etc/init.d/rpcd reload; \
}
exit 0
endef
# OpenWrt's generated postinst/prerm handle enable/start and stop/disable for
# both procd services, and PKG_UPGRADE keeps upgrade semantics on IPK and APK.
# Do not duplicate them or depend on opkg's "configure" argument: apk invokes
# post-install/post-upgrade with version arguments instead.
$(eval $(call BuildPackage,luci-app-owrt-remote))
@@ -0,0 +1,40 @@
#!/bin/sh
# Native package installation needs the same private panel key as install.sh.
# No network downloads or nested opkg/apk operations run under the package lock.
set -eu
configure() {
local dir key temp
dir=/etc/owrt-remote
umask 077
mkdir -p "$dir"
if [ ! -s "$dir/web.key" ]; then
temp="$(mktemp "$dir/.web.key.XXXXXX")"
trap 'rm -f "$temp"' EXIT
dd if=/dev/urandom of="$temp" bs=32 count=1 2>/dev/null
[ "$(wc -c <"$temp")" -eq 32 ] || { echo 'ERROR: insufficient random bytes for panel key' >&2; return 1; }
key="$(sha256sum "$temp")"
key="${key%% *}"
[ "${#key}" = 64 ] || { echo 'ERROR: could not generate panel access key' >&2; return 1; }
printf '%s\n' "$key" >"$temp"
chmod 0600 "$temp"
mv -f "$temp" "$dir/web.key"
trap - EXIT
fi
chmod 0600 "$dir/web.key"
# Native conffiles handling preserves existing UCI settings on upgrades.
[ ! -f /etc/config/owrtremote ] || chmod 0600 /etc/config/owrtremote
rm -f /tmp/luci-indexcache /tmp/luci-indexcache.*
rm -rf /tmp/luci-modulecache
if [ -x /etc/init.d/rpcd ]; then
/etc/init.d/rpcd reload || { echo 'ERROR: rpcd reload failed' >&2; return 1; }
fi
printf '%s\n' 'OpenWrt Remote: installed. Open LuCI > Services > OpenWrt Remote.'
printf '%s\n' 'Apply OpenWrt config from Hub to prepare Xray and start Remote.'
}
case "${1:-}" in
configure) configure ;;
*) echo 'Usage: package.sh configure' >&2; exit 2 ;;
esac
+19
View File
@@ -0,0 +1,19 @@
{
"releases": [
{"series": "23.05", "baseline": "23.05.5", "format": "ipk"},
{"series": "24.10", "baseline": "24.10.3", "format": "ipk"},
{"series": "25.12", "baseline": "25.12.2", "format": "apk"}
],
"targets": [
{"id": "aarch64_cortex-a53", "target": "mvebu", "subtarget": "cortexa53"},
{"id": "aarch64_cortex-a72", "target": "mvebu", "subtarget": "cortexa72"},
{"id": "aarch64_generic", "target": "armsr", "subtarget": "armv8"},
{"id": "arm_cortex-a7", "target": "bcm27xx", "subtarget": "bcm2709"},
{"id": "arm_cortex-a9", "target": "mvebu", "subtarget": "cortexa9"},
{"id": "mips_24kc", "target": "ath79", "subtarget": "generic"},
{"id": "mipsel_24kc", "target": "ramips", "subtarget": "mt7621"},
{"id": "mips64", "target": "octeon", "subtarget": "generic"},
{"id": "x86", "target": "x86", "subtarget": "generic"},
{"id": "x86_64", "target": "x86", "subtarget": "64", "publish": true}
]
}
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
SOURCE_DIR="$(CDPATH= cd "$(dirname "$0")/.." && pwd)"
exec python3 "$SOURCE_DIR/scripts/openwrt_package.py" build "$@"
+535
View File
@@ -0,0 +1,535 @@
#!/usr/bin/env python3
"""Stage one overlay, build it with official SDKs, verify native packages."""
from __future__ import annotations
import argparse
import gzip
import hashlib
from html.parser import HTMLParser
import io
import json
import os
from pathlib import Path, PurePosixPath
import re
import shlex
import shutil
import subprocess
import sys
import tarfile
import tempfile
import time
import urllib.parse
import urllib.request
import zlib
ROOT = Path(__file__).resolve().parents[1]
PACKAGE = "luci-app-owrt-remote"
RECIPE = ROOT / "package" / PACKAGE
DOWNLOADS = "https://downloads.openwrt.org/releases/"
EXECUTABLES = {
"usr/sbin/owrt-remote", "www/cgi-bin/owrt-remote",
"etc/init.d/owrt-remote", "etc/init.d/owrt-remote-logs",
"etc/hotplug.d/iface/99-owrt-remote",
}
class Links(HTMLParser):
def __init__(self, text):
super().__init__()
self.links = []
self.feed(text)
def handle_starttag(self, tag, attrs):
if tag == "a":
self.links.extend(value for key, value in attrs if key == "href" and value)
def get_text(url):
request = urllib.request.Request(url, headers={"User-Agent": "OpenWrt-Remote-package-builder"})
with urllib.request.urlopen(request, timeout=45) as response:
data = response.read(4 * 1024 * 1024 + 1)
if len(data) > 4 * 1024 * 1024:
raise ValueError("oversized SDK metadata: " + url)
return data.decode("utf-8")
def versions():
text = (RECIPE / "Makefile").read_text(encoding="utf-8")
version = re.search(r"^PKG_VERSION:=([0-9][0-9.]*)$", text, re.M).group(1)
release = re.search(r"^PKG_RELEASE:=([0-9]+)$", text, re.M).group(1)
return version, release
def latest_release(series, html):
candidates = [link.rstrip("/") for link in Links(html).links
if re.fullmatch(re.escape(series) + r"\.[0-9]+/", link)]
if not candidates:
raise ValueError("no stable OpenWrt release for " + series)
return max(candidates, key=lambda v: tuple(map(int, v.split("."))))
def sdk_name(html):
names = sorted({link for link in Links(html).links if re.fullmatch(
r"openwrt-sdk-[A-Za-z0-9_.+-]+Linux-x86_64\.tar\.(?:xz|zst)", link)})
if len(names) != 1:
raise ValueError("expected exactly one official Linux x86_64 SDK; found " + str(names))
return names[0]
def checksum(text, filename):
values = []
for line in text.splitlines():
fields = line.split()
if len(fields) == 2 and fields[1].lstrip("*") == filename:
if re.fullmatch(r"[a-fA-F0-9]{64}", fields[0]):
values.append(fields[0].lower())
if len(set(values)) != 1:
raise ValueError("missing or ambiguous SHA256 for " + filename)
return values[0]
def sha256(path):
with Path(path).open("rb") as stream:
return hashlib.file_digest(stream, "sha256").hexdigest()
def download(url, path, expected):
path = Path(path)
if path.is_file() and sha256(path) == expected:
return
partial = path.with_name(path.name + ".partial")
for attempt in range(3):
try:
digest = hashlib.sha256()
request = urllib.request.Request(url, headers={"User-Agent": "OpenWrt-Remote-package-builder"})
with urllib.request.urlopen(request, timeout=60) as response, partial.open("wb") as output:
size = 0
while data := response.read(1024 * 1024):
size += len(data)
if size > 2 * 1024 ** 3:
raise ValueError("SDK download exceeded 2 GiB")
output.write(data)
digest.update(data)
if digest.hexdigest() != expected:
raise ValueError("SDK SHA256 mismatch: " + url)
partial.replace(path)
return
except Exception:
partial.unlink(missing_ok=True)
if attempt == 2:
raise
time.sleep(2)
def unpack_image(source, destination):
# Some official OpenWrt .img.gz files have fwtool metadata appended after
# the gzip member. The full download SHA256 is checked before this step.
# Python gzip.open treats that signed/image metadata as a second header.
decoder = zlib.decompressobj(16 + zlib.MAX_WBITS)
total = 0
with Path(source).open("rb") as compressed, Path(destination).open("wb") as raw:
while not decoder.eof:
block = compressed.read(65536)
if not block:
raise ValueError("truncated OpenWrt disk image")
data = decoder.decompress(block)
total += len(data)
if total > 2 * 1024 ** 3:
raise ValueError("OpenWrt disk image exceeded 2 GiB")
raw.write(data)
def payload():
result = {}
for base in (ROOT / "files", RECIPE / "root"):
for path in sorted(base.rglob("*")):
if not path.is_file():
continue
if path.is_symlink():
raise ValueError("unexpected source symlink: " + str(path))
name = path.relative_to(base).as_posix()
if name in result:
raise ValueError("duplicate package payload: " + name)
data = path.read_bytes().replace(b"\r\n", b"\n")
mode = 0o755 if name in EXECUTABLES else 0o644
if name == "etc/config/owrtremote":
mode = 0o600
result[name] = (data, mode)
return result
def package_hook(name):
text = (RECIPE / "Makefile").read_text(encoding="utf-8")
match = re.search(r"^define Package/" + PACKAGE + "/" + name + r"\n(.*?)^endef$", text, re.M | re.S)
return match.group(1).replace("$$", "$") if match else "#!/bin/sh\nexit 0\n"
def dependencies():
text = (RECIPE / "Makefile").read_text(encoding="utf-8")
line = re.search(r"^\s+DEPENDS:=(.+)$", text, re.M).group(1)
return [item.removeprefix("+") for item in line.split()]
def native_wrappers():
common = '#!/bin/sh\n[ -s "${IPKG_INSTROOT}/lib/functions.sh" ] || exit 0\n. "${IPKG_INSTROOT}/lib/functions.sh"\n'
return {
"postinst": common + 'default_postinst "$0" "$@"\n',
"prerm": common + 'default_prerm "$0" "$@"\n',
"postinst-pkg": package_hook("postinst"),
"postrm": package_hook("postrm"),
}
def tar_bytes(files):
data = io.BytesIO()
with tarfile.open(fileobj=data, mode="w", format=tarfile.USTAR_FORMAT) as archive:
directories = {parent.as_posix() for name in files for parent in PurePosixPath(name).parents
if parent.as_posix() != "."}
for name in sorted(directories):
info = tarfile.TarInfo("./" + name + "/")
info.type = tarfile.DIRTYPE
info.mode = 0o755
info.uid = info.gid = 0
archive.addfile(info)
for name, (content, mode) in sorted(files.items()):
info = tarfile.TarInfo("./" + name)
info.size = len(content)
info.mode = mode
info.uid = info.gid = 0
archive.addfile(info, io.BytesIO(content))
return gzip.compress(data.getvalue(), mtime=0)
def pack_ipk(output, revision=None):
"""Portable noarch preview; release CI also builds with the official SDK."""
version, release = versions()
revision = str(revision or release)
files = payload()
control = (f"Package: {PACKAGE}\nVersion: {version}-{revision}\nArchitecture: all\n"
f"Depends: {', '.join(dependencies())}\nSection: luci\nLicense: MIT\n"
"Maintainer: kzolotarev95\nURL: https://github.com/kzolotarev95/luci-app-owrt-remote\n"
f"Installed-Size: {sum(len(data) for data, _ in files.values())}\n"
"Description: OpenWrt Remote Hub router agent and LuCI panel\n")
metadata = {"control": (control.encode(), 0o644), "conffiles": (b"/etc/config/owrtremote\n", 0o644)}
metadata.update({name: (script.encode(), 0o755) for name, script in native_wrappers().items()})
output = Path(output)
output.parent.mkdir(parents=True, exist_ok=True)
output.write_bytes(tar_bytes({"debian-binary": (b"2.0\n", 0o644),
"control.tar.gz": (tar_bytes(metadata), 0o644),
"data.tar.gz": (tar_bytes(files), 0o644)}))
return output
def prepare_apk(folder, revision=None):
"""Prepare portable input for the genuine apk-tools v3 mkpkg command."""
folder = Path(folder).resolve()
staged = prepare(folder / "package") / "root"
version, release = versions()
meta = staged / "lib/apk/packages"
meta.mkdir(parents=True, exist_ok=True)
config = staged / "etc/config/owrtremote"
(meta / f"{PACKAGE}.conffiles").write_text("/etc/config/owrtremote\n", newline="\n")
(meta / f"{PACKAGE}.conffiles_static").write_text(f"/etc/config/owrtremote {sha256(config)}\n", newline="\n")
(meta / f"{PACKAGE}.list").write_text("".join("/" + f.relative_to(staged).as_posix() + "\n" for f in sorted(staged.rglob("*")) if f.is_file()), newline="\n")
common = '#!/bin/sh\n[ -s "${IPKG_INSTROOT}/lib/functions.sh" ] || exit 0\n. "${IPKG_INSTROOT}/lib/functions.sh"\n' + f'export pkgname="{PACKAGE}"\n'
install = common + "default_postinst\n" + package_hook("postinst").removeprefix("#!/bin/sh\n")
scripts = {"post-install": install,
"post-upgrade": "#!/bin/sh\nexport PKG_UPGRADE=1\n" + install.removeprefix("#!/bin/sh\n"),
"pre-deinstall": common + "default_prerm\n",
"post-deinstall": package_hook("postrm")}
args = ["mkpkg", "--info", "name:" + PACKAGE,
"--info", f"version:{version}-r{revision or release}", "--info", "arch:noarch",
"--info", "description:OpenWrt Remote Hub router agent and LuCI panel",
"--info", "license:MIT", "--info", "depends:" + " ".join(dependencies()),
"--info", "url:https://github.com/kzolotarev95/luci-app-owrt-remote",
"--files", "package/root", "--output", PACKAGE + ".apk"]
for name, script in scripts.items():
(folder / name).write_text(script, newline="\n")
args.extend(["--script", name + ":" + name])
(folder / "build.sh").write_text('#!/bin/sh\nset -eu\ncd "$(dirname "$0")"\nexec "${APK_TOOL:-apk}" ' + shlex.join(args) + "\n", newline="\n")
return args
def pack_apk(output, apk, revision=None):
"""Use the real apk-tools v3 packer, never rename an IPK or tar as APK."""
output = Path(output).resolve()
output.parent.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(prefix="owrt-apk-") as folder:
args = prepare_apk(folder, revision)
subprocess.run([str(apk), *args], cwd=folder, check=True)
shutil.copyfile(Path(folder) / (PACKAGE + ".apk"), output)
return output
def prepare(output):
output = Path(output).resolve()
# Fail instead of recursively deleting a caller-supplied directory.
if output.exists() and any(output.iterdir()):
raise ValueError("package staging directory must be empty: " + str(output))
output.mkdir(parents=True, exist_ok=True)
shutil.copyfile(RECIPE / "Makefile", output / "Makefile")
for name, (data, mode) in payload().items():
target = output / "root" / name
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(data)
target.chmod(mode)
return output
def ar_members(data):
if not data.startswith(b"!<arch>\n"):
raise ValueError("invalid ar package")
offset = 8
result = {}
while offset < len(data):
header = data[offset:offset + 60]
if len(header) != 60 or header[58:] != b"`\n":
raise ValueError("invalid ar member")
size = int(header[48:58].strip())
name = header[:16].decode("ascii").strip().rstrip("/")
result[name] = data[offset + 60:offset + 60 + size]
if len(result[name]) != size:
raise ValueError("truncated ar member")
offset += 60 + size + size % 2
return result
def ipk_contents(path):
data = Path(path).read_bytes()
if data.startswith(b"!<arch>\n"):
members = ar_members(data)
else:
with tarfile.open(fileobj=io.BytesIO(data), mode="r:*") as archive:
members = {item.name.removeprefix("./"): archive.extractfile(item).read()
for item in archive if item.isfile()}
if members.get("debian-binary") != b"2.0\n":
raise ValueError("invalid IPK version marker")
result = {}
for kind in ("control", "data"):
compressed = members.get(kind + ".tar.gz")
if compressed is None:
raise ValueError("IPK lacks " + kind)
with tarfile.open(fileobj=io.BytesIO(compressed), mode="r:gz") as archive:
result[kind] = {}
for item in archive:
if not item.isfile():
continue
name = item.name.removeprefix("./")
if ".." in PurePosixPath(name).parts or name.startswith("/"):
raise ValueError("unsafe package path: " + name)
result[kind][name] = (archive.extractfile(item).read(), item.mode & 0o777, item.uid, item.gid)
return result
def validate_payload(actual):
for name, (expected, mode) in payload().items():
if name not in actual:
raise ValueError("package file missing: " + name)
data, found_mode, uid, gid = actual[name]
if data != expected or found_mode != mode or uid != 0 or gid != 0:
raise ValueError("package content/mode/ownership mismatch: " + name)
def verify_package(path, apk=None, revision=None):
path = Path(path).resolve()
version, release = versions()
release = str(revision or release)
allowed_versions = {f"{version}-{release}", f"{version}-r{release}"}
if path.suffix == ".ipk":
content = ipk_contents(path)
validate_payload(content["data"])
control = content["control"]["control"][0].decode("utf-8")
fields = dict(line.split(": ", 1) for line in control.splitlines() if ": " in line and not line.startswith(" "))
if fields.get("Package") != PACKAGE or fields.get("Architecture") != "all":
raise ValueError("invalid IPK package identity/architecture")
if fields.get("Version") not in allowed_versions:
raise ValueError("IPK version mismatch")
if b"/etc/config/owrtremote" not in content["control"]["conffiles"][0]:
raise ValueError("UCI config must be a conffile")
for script in ("postinst", "prerm"):
if b"/lib/functions.sh" not in content["control"][script][0]:
raise ValueError("missing native OpenWrt lifecycle wrapper")
declared_dependencies = {x.strip().split(" ", 1)[0] for x in fields.get("Depends", "").split(",")}
else:
if not apk:
raise ValueError("APK validation requires the SDK host apk tool")
manifest = json.loads(subprocess.check_output([str(apk), "adbdump", "--format", "json", str(path)], text=True))
info = manifest["info"]
if info["name"] != PACKAGE or info["arch"] != "noarch" or info["version"] not in allowed_versions:
raise ValueError("invalid APK identity/version/architecture")
owners = {}
for directory in manifest.get("paths", []):
for entry in directory.get("files", []):
name = str(PurePosixPath(directory.get("name", "")) / entry["name"])
acl = entry.get("acl", directory.get("acl", {}))
if acl.get("user") != "root" or acl.get("group") != "root":
raise ValueError("APK payload must be owned by root: " + name)
owners[name] = acl["mode"]
with tempfile.TemporaryDirectory(prefix="owrt-apk-verify-") as folder:
subprocess.run([str(apk), "extract", "--allow-untrusted", "--no-chown", "--destination", folder, str(path)], check=True)
actual = {}
for name in payload():
file = Path(folder) / name
if file.is_file():
stat = file.stat()
if owners.get(name) != stat.st_mode & 0o777:
raise ValueError("APK mode metadata mismatch: " + name)
actual[name] = (file.read_bytes(), stat.st_mode & 0o777, 0, 0)
validate_payload(actual)
conffiles = Path(folder) / "lib/apk/packages" / f"{PACKAGE}.conffiles"
if "/etc/config/owrtremote" not in conffiles.read_text():
raise ValueError("APK missing OpenWrt conffiles metadata")
declared_dependencies = {re.split(r"[<=>~]", x, maxsplit=1)[0] for x in info.get("depends", [])}
for dependency in dependencies():
if dependency not in declared_dependencies:
raise ValueError("package dependency missing: " + dependency)
return {"file": path.name, "sha256": sha256(path), "bytes": path.stat().st_size, "verified": True}
def sign_apk(path, apk, private_key, public_key):
"""Use a persistent maintainer key; never generate a new trust root per build."""
subprocess.run(["openssl", "pkey", "-in", str(private_key), "-pubout", "-out", str(public_key)], check=True)
subprocess.run([str(apk), "adbsign", "--allow-untrusted", "--sign-key", str(private_key), str(path)], check=True)
def build(args):
if sys.platform != "linux":
raise ValueError("OpenWrt SDKs require Linux. Use a Linux machine or the prepared GitHub Actions workflow; no files on the router are changed.")
if not re.fullmatch(r"(?:23\.05|24\.10|25\.12)\.[0-9]+", args.release):
raise ValueError("unsupported OpenWrt release: " + args.release)
for value in (args.target, args.subtarget, args.arch):
if not re.fullmatch(r"[A-Za-z0-9_-]+", value):
raise ValueError("invalid target/architecture")
series = ".".join(args.release.split(".")[:2])
base = DOWNLOADS + f"{args.release}/targets/{args.target}/{args.subtarget}/"
name = sdk_name(get_text(base))
expected = checksum(get_text(base + "sha256sums"), name)
output = Path(args.output).resolve()
output.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(prefix="owrt-package-") as folder:
work = Path(folder)
archive = work / name
print("Downloading verified SDK: " + base + name, flush=True)
download(base + name, archive, expected)
subprocess.run(["tar", "-xf", str(archive), "-C", str(work)], check=True)
directories = list(work.glob("openwrt-sdk-*/"))
if len(directories) != 1:
raise ValueError("SDK archive layout changed")
sdk = directories[0]
prepare(sdk / "package" / PACKAGE)
# Use feed revisions from THIS SDK, not today's unrelated feed heads.
subprocess.run(["./scripts/feeds", "update", "-a"], cwd=sdk, check=True)
subprocess.run(["./scripts/feeds", "install", "luci-base", "rpcd-mod-file", "uhttpd", "jsonfilter", "curl", "ca-bundle", "etherwake"], cwd=sdk, check=True)
with (sdk / ".config").open("a") as config:
config.write(f"\nCONFIG_PACKAGE_{PACKAGE}=m\nCONFIG_SIGNED_PACKAGES=n\n")
subprocess.run(["make", "defconfig"], cwd=sdk, check=True)
config = (sdk / ".config").read_text()
is_apk = "CONFIG_USE_APK=y" in config
if is_apk != (series == "25.12"):
raise ValueError("SDK package format disagrees with the selected OpenWrt series")
subprocess.run(["make", f"package/{PACKAGE}/compile", "V=s", f"-j{args.jobs}"], cwd=sdk, check=True)
extension = "apk" if is_apk else "ipk"
packages = list((sdk / "bin/packages").rglob(f"{PACKAGE}*.{extension}"))
if len(packages) != 1:
raise ValueError("expected one native package, found " + str(packages))
verified = verify_package(packages[0], sdk / "staging_dir/host/bin/apk")
version, release = versions()
dest = output / f"{PACKAGE}_{version}-r{release}_openwrt-{series}_{'noarch' if is_apk else 'all'}.{extension}"
shutil.copyfile(packages[0], dest)
signing_key = None
if is_apk:
if args.signing_key:
signing_key = Path(args.signing_key).resolve()
elif os.environ.get("OPENWRT_APK_SIGNING_KEY"):
signing_key = work / "maintainer-signing-key.pem"
signing_key.touch(mode=0o600)
signing_key.write_text(os.environ["OPENWRT_APK_SIGNING_KEY"])
if signing_key:
sign_apk(dest, sdk / "staging_dir/host/bin/apk", signing_key, output / "owrt-remote-packages.pem")
verified = verify_package(dest, sdk / "staging_dir/host/bin/apk")
else:
print("APK is unsigned: SSH needs --allow-untrusted; LuCI requires a trusted maintainer signing key.", flush=True)
verified["file"] = dest.name
report = {"package": PACKAGE, "version": version, "package_release": release,
"openwrt": args.release, "series": series, "target": f"{args.target}/{args.subtarget}",
"architecture_test": args.arch, "sdk": {"url": base + name, "sha256": expected},
"package_asset": verified, "signed": bool(signing_key)}
(output / f"report-{series}-{args.arch}.json").write_text(json.dumps(report, indent=2) + "\n")
(output / "SHA256SUMS").write_text(f"{verified['sha256']} {dest.name}\n")
if args.upgrade_fixture:
# A real native version upgrade is exercised in the VM. This
# fixture stays in a subdirectory and is never a release asset.
recipe = sdk / "package" / PACKAGE / "Makefile"
next_release = str(int(release) + 1)
recipe.write_text(recipe.read_text().replace("PKG_RELEASE:=" + release, "PKG_RELEASE:=" + next_release))
subprocess.run(["make", f"package/{PACKAGE}/clean", "V=s"], cwd=sdk, check=True)
subprocess.run(["make", f"package/{PACKAGE}/compile", "V=s"], cwd=sdk, check=True)
candidate = list((sdk / "bin/packages").rglob(f"{PACKAGE}*.{extension}"))
if len(candidate) != 1:
raise ValueError("upgrade fixture was not built")
verify_package(candidate[0], sdk / "staging_dir/host/bin/apk", revision=next_release)
fixture = output / "test-fixtures" / f"{PACKAGE}_{version}-r{next_release}_openwrt-{series}.{extension}"
fixture.parent.mkdir(exist_ok=True)
shutil.copyfile(candidate[0], fixture)
if signing_key:
sign_apk(fixture, sdk / "staging_dir/host/bin/apk", signing_key, output / "owrt-remote-packages.pem")
print("OPENWRT_PACKAGE_BUILD_OK: " + str(dest), flush=True)
def matrix(baseline=False):
config = json.loads((ROOT / "package/openwrt-targets.json").read_text())
index = "" if baseline else get_text(DOWNLOADS)
result = []
for release in config["releases"]:
version = release["baseline"] if baseline else latest_release(release["series"], index)
for target in config["targets"]:
result.append({**release, **target, "version": version})
return {"include": result}
def main():
parser = argparse.ArgumentParser(description=__doc__)
sub = parser.add_subparsers(dest="command", required=True)
stage = sub.add_parser("prepare")
stage.add_argument("--output", required=True)
apk_stage = sub.add_parser("prepare-apk")
apk_stage.add_argument("--output", required=True)
targets = sub.add_parser("matrix")
targets.add_argument("--baseline", action="store_true")
check = sub.add_parser("verify")
check.add_argument("package")
check.add_argument("--apk")
for fmt in ("ipk", "apk"):
local = sub.add_parser("local-" + fmt)
local.add_argument("--output", required=True)
if fmt == "apk":
local.add_argument("--apk", default="apk")
sdk = sub.add_parser("build")
for option in ("release", "target", "subtarget", "arch", "output"):
sdk.add_argument("--" + option, required=True)
sdk.add_argument("--jobs", type=int, default=2)
sdk.add_argument("--upgrade-fixture", action="store_true")
sdk.add_argument("--signing-key", help="persistent maintainer PEM private key for APK signing")
args = parser.parse_args()
try:
if args.command == "prepare":
print(prepare(args.output))
elif args.command == "prepare-apk":
prepare_apk(args.output)
print(Path(args.output).resolve() / "build.sh")
elif args.command == "matrix":
print(json.dumps(matrix(args.baseline), separators=(",", ":")))
elif args.command == "verify":
print(json.dumps(verify_package(args.package, args.apk)))
elif args.command == "local-ipk":
print(pack_ipk(args.output))
elif args.command == "local-apk":
print(pack_apk(args.output, args.apk))
else:
build(args)
except (ValueError, OSError, subprocess.CalledProcessError) as error:
parser.exit(1, "ERROR: " + str(error) + "\n")
if __name__ == "__main__":
main()
+159
View File
@@ -0,0 +1,159 @@
#!/usr/bin/env python3
"""Exercise package install/upgrade/removal in a disposable local OpenWrt VM."""
import argparse
import contextlib
import functools
import http.server
import os
from pathlib import Path
import re
import shlex
import socket
import subprocess
import tempfile
import threading
import time
import openwrt_package as package
def free_port():
with socket.socket() as stream:
stream.bind(("127.0.0.1", 0))
return stream.getsockname()[1]
class QuietHandler(http.server.SimpleHTTPRequestHandler):
def log_message(self, *args):
pass
def stop_vm(proc):
if proc is not None and proc.poll() is None:
proc.terminate()
try:
proc.wait(timeout=10)
except subprocess.TimeoutExpired:
proc.kill()
proc.wait(timeout=10)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--release", required=True)
parser.add_argument("--package-dir", required=True)
parser.add_argument("--output", required=True)
parser.add_argument("--qemu", default="qemu-system-x86_64")
args = parser.parse_args()
if not re.fullmatch(r"(?:23\.05|24\.10|25\.12)\.[0-9]+", args.release):
parser.error("unsupported test release")
source = Path(args.package_dir).resolve()
candidates = list(source.glob("*.apk")) + list(source.glob("*.ipk"))
fixtures = list((source / "test-fixtures").glob("*.apk")) + list((source / "test-fixtures").glob("*.ipk"))
if len(candidates) != 1 or len(fixtures) != 1:
parser.error("one main package and one real upgrade fixture are required")
candidate, fixture = candidates[0], fixtures[0]
apk = candidate.suffix == ".apk"
output = Path(args.output).resolve()
output.mkdir(parents=True, exist_ok=True)
port = free_port()
ssh = ["ssh", "-p", str(port), "-o", "BatchMode=yes", "-o", "ConnectTimeout=3",
"-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=" + str(output / "known_hosts"),
"root@127.0.0.1"]
handler = functools.partial(QuietHandler, directory=str(source))
server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler)
threading.Thread(target=server.serve_forever, daemon=True).start()
log = (output / "package-install.log").open("w", encoding="utf-8")
def remote(command, timeout=180, check=True):
# The only target is the newly created loopback VM. No user router or
# configured SSH profile can be selected by this test program.
result = subprocess.run(ssh + [command], capture_output=True, timeout=timeout)
text = (result.stdout + result.stderr).decode("utf-8", "replace")
log.write(text + "\n")
log.flush()
if check and result.returncode:
raise RuntimeError(f"VM command failed (rc={result.returncode}): {command}\n{text}")
return result
proc = None
try:
with contextlib.ExitStack() as stack:
directory = stack.enter_context(tempfile.TemporaryDirectory(prefix="owrt-native-vm-"))
work = Path(directory)
base = package.DOWNLOADS + args.release + "/targets/x86/64/"
name = "openwrt-" + args.release + "-x86-64-generic-ext4-combined.img.gz"
digest = package.checksum(package.get_text(base + "sha256sums"), name)
image = work / name
print("Downloading verified OpenWrt VM " + args.release, flush=True)
package.download(base + name, image, digest)
disk = work / "router.img"
package.unpack_image(image, disk)
flags = {"creationflags": subprocess.CREATE_NO_WINDOW} if os.name == "nt" else {}
qemu_log = stack.enter_context((output / "qemu.log").open("wb"))
proc = subprocess.Popen([args.qemu, "-display", "none", "-monitor", "none", "-serial", "file:" + str(output / "console.log"),
"-accel", "tcg", "-m", "256", "-smp", "1", "-no-reboot",
"-drive", "file=" + str(disk) + ",format=raw",
"-device", "virtio-net-pci,netdev=lan", "-netdev",
"user,id=lan,net=192.168.1.0/24,host=192.168.1.2,dhcpstart=192.168.1.100,hostfwd=tcp:127.0.0.1:" + str(port) + "-192.168.1.1:22"],
stdin=subprocess.DEVNULL, stdout=qemu_log, stderr=qemu_log, **flags)
# Stop QEMU before closing its log and deleting the disk, including
# on Windows where an open image cannot be unlinked.
stack.callback(stop_vm, proc)
print("Waiting for the local VM SSH agent", flush=True)
for attempt in range(40):
if proc.poll() is not None:
raise RuntimeError("QEMU exited before OpenWrt booted; see qemu.log")
try:
if remote("true", timeout=8, check=False).returncode == 0:
break
except subprocess.TimeoutExpired:
pass
time.sleep(2)
else:
raise RuntimeError("OpenWrt VM did not boot within the test deadline")
remote("set -e; uci set network.lan.gateway=192.168.1.2; uci set network.lan.dns=192.168.1.3; uci commit network; /etc/init.d/network reload; sleep 2")
remote("apk update" if apk else "opkg update", timeout=300)
origin = "http://192.168.1.2:" + str(server.server_port) + "/"
remote("set -e; wget -qO /tmp/remote-package" + candidate.suffix + " " + shlex.quote(origin + candidate.name) +
"; wget -qO /tmp/remote-upgrade" + candidate.suffix + " " + shlex.quote(origin + "test-fixtures/" + fixture.name))
public_key = source / "owrt-remote-packages.pem"
if apk and public_key.is_file():
remote("wget -qO /etc/apk/keys/owrt-remote-packages.pem " + shlex.quote(origin + public_key.name))
install = ("apk add" if public_key.is_file() else "apk add --allow-untrusted") if apk else "opkg install"
remove = "apk del" if apk else "opkg remove"
print("Testing first install and dependency resolution", flush=True)
remote(install + " /tmp/remote-package" + candidate.suffix, timeout=300)
remote("set -e; test -s /etc/owrt-remote/web.key; test $(wc -c </etc/owrt-remote/web.key) -eq 65; "
"test -x /usr/sbin/owrt-remote; test -x /www/cgi-bin/owrt-remote; "
"test -f /usr/share/luci/menu.d/luci-app-owrt-remote.json; "
"test $(uci -q get owrtremote.main.enabled) = 0; "
"/etc/init.d/owrt-remote-logs running; "
"for script in /usr/sbin/owrt-remote /usr/lib/owrt-remote/*.sh /etc/init.d/owrt-remote* /www/cgi-bin/owrt-remote; do /bin/ash -n $script; done; "
"curl -fsS \"http://127.0.0.1/cgi-bin/owrt-remote?key=$(cat /etc/owrt-remote/web.key)\" | grep -q 'OpenWrt Remote Hub'")
remote("set -e; uci set owrtremote.main.router_name='Package upgrade test'; uci set owrtremote.main.hub_token='local-test-only'; uci commit owrtremote; "
"mkdir -p /etc/xray /tmp/owrt-xray; printf 'preserve JSON\\n' >/etc/xray/owrt-remote-client.json; "
"printf 'preserve Xray binary\\n' >/tmp/owrt-xray/xray; chmod 755 /tmp/owrt-xray/xray; "
"sha256sum /etc/config/owrtremote /etc/owrt-remote/web.key /etc/xray/owrt-remote-client.json /tmp/owrt-xray/xray >/tmp/remote-before.sha256")
print("Testing a genuine package version upgrade", flush=True)
remote(install + " /tmp/remote-upgrade" + candidate.suffix, timeout=300)
remote("set -e; sha256sum -c /tmp/remote-before.sha256; /etc/init.d/owrt-remote-logs running")
print("Testing remove, preserved state, and migration from unowned overlay files", flush=True)
remote(remove + " " + package.PACKAGE, timeout=180)
remote("set -e; test ! -e /usr/sbin/owrt-remote; test ! -e /www/cgi-bin/owrt-remote; test ! -e /etc/init.d/owrt-remote; sha256sum -c /tmp/remote-before.sha256")
# Configuration left by install.sh is unowned by a package manager.
# A fresh package install must preserve it just like an upgrade.
remote(install + " /tmp/remote-package" + candidate.suffix, timeout=300)
remote("set -e; sha256sum -c /tmp/remote-before.sha256; test -x /usr/sbin/owrt-remote")
remote(remove + " " + package.PACKAGE, timeout=180)
(output / "result.txt").write_text("OPENWRT_NATIVE_PACKAGE_TEST_OK " + args.release + "\n")
print("OPENWRT_NATIVE_PACKAGE_TEST_OK " + args.release, flush=True)
finally:
stop_vm(proc)
server.shutdown()
server.server_close()
log.close()
if __name__ == "__main__":
main()
+177
View File
@@ -0,0 +1,177 @@
"""Native package contents, release discovery, and safe target setup."""
import importlib.util
import gzip
import io
import os
from pathlib import Path
import shutil
import subprocess
import sys
import tarfile
import tempfile
import unittest
from unittest import mock
ROOT = Path(__file__).resolve().parents[1]
spec = importlib.util.spec_from_file_location("openwrt_package", ROOT / "scripts/openwrt_package.py")
p = importlib.util.module_from_spec(spec)
spec.loader.exec_module(p)
SH = shutil.which("sh") or str(Path(os.environ.get("ProgramFiles", "C:/Program Files")) / "Git/usr/bin/sh.exe")
class PackageTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory(prefix="owrt-package-tests-")
self.addCleanup(self.temp.cleanup)
self.root = Path(self.temp.name)
self.env = dict(os.environ)
if os.name == "nt":
self.env["PATH"] = str(Path(SH).parent) + os.pathsep + self.env["PATH"]
def test_prepared_sdk_package_is_self_contained_and_same_overlay(self):
directory = p.prepare(self.root / "staging")
for name, (data, mode) in p.payload().items():
path = directory / "root" / name
self.assertEqual(path.read_bytes(), data, name)
self.assertNotIn(b"\r\n", data, name)
if os.name != "nt":
self.assertEqual(path.stat().st_mode & 0o777, mode, name)
self.assertEqual((directory / "Makefile").read_bytes(), (p.RECIPE / "Makefile").read_bytes())
self.assertFalse((directory / "root/etc/owrt-remote/web.key").exists())
def test_staging_never_deletes_nonempty_directory(self):
keep = self.root / "keep"
keep.write_text("do not delete")
with self.assertRaises(ValueError):
p.prepare(self.root)
self.assertEqual(keep.read_text(), "do not delete")
def test_local_ipk_is_native_and_contains_valid_payload_and_dependencies(self):
package = p.pack_ipk(self.root / "test.ipk")
result = p.verify_package(package)
self.assertTrue(result["verified"])
self.assertGreater(result["bytes"], 10000)
with tarfile.open(package, "r:gz") as outer:
inner = outer.extractfile("./data.tar.gz").read()
with tarfile.open(fileobj=io.BytesIO(inner), mode="r:gz") as archive:
# opkg does not create arbitrary missing parent directories when
# extracting a file; a package needs explicit directory entries.
self.assertTrue(archive.getmember("./usr/lib/owrt-remote").isdir())
content = p.ipk_contents(package)
self.assertEqual(content["data"]["etc/config/owrtremote"][1:], (0o600, 0, 0))
for name in p.EXECUTABLES:
self.assertEqual(content["data"][name][1], 0o755, name)
self.assertIn(b"package.sh configure", content["control"]["postinst-pkg"][0])
def test_package_bytes_are_reproducible(self):
self.assertEqual(p.pack_ipk(self.root / "one.ipk").read_bytes(), p.pack_ipk(self.root / "two.ipk").read_bytes())
def test_package_rejects_missing_files_or_changed_modes(self):
files = {name: (data, mode, 0, 0) for name, (data, mode) in p.payload().items()}
files.pop("usr/sbin/owrt-remote")
with self.assertRaisesRegex(ValueError, "missing"):
p.validate_payload(files)
files["usr/sbin/owrt-remote"] = (b"bad replacement", 0o644, 0, 0)
with self.assertRaisesRegex(ValueError, "mismatch"):
p.validate_payload(files)
def test_source_version_matches_router_titles(self):
version, _ = p.versions()
for name in ("www/cgi-bin/owrt-remote", "www/luci-static/resources/view/owrt_remote.js"):
text = (ROOT / "files" / name).read_text(encoding="utf-8")
self.assertIn("OpenWrt Remote Hub v" + version, text)
def test_package_scripts_defer_procd_lifecycle_to_openwrt_for_apk_and_ipk(self):
script = p.package_hook("postinst")
self.assertNotIn("configure ]", script)
self.assertIn("IPKG_INSTROOT", script)
self.assertNotIn("/etc/init.d/owrt-remote restart", script)
for script in p.native_wrappers().values():
result = subprocess.run([SH, "-n"], input=script, capture_output=True, text=True)
self.assertEqual(result.returncode, 0, result.stderr)
def target_setup(self, existing=False, rpcd_fails=False):
target = self.root / "target"
target.mkdir()
for folder in ("etc/config", "etc/owrt-remote", "etc/init.d", "tmp/luci-modulecache"):
(target / folder).mkdir(parents=True, exist_ok=True)
config = target / "etc/config/owrtremote"
config.write_text("config remote 'main'\n option router_name 'Keep name'\n option hub_token 'test-secret'\n")
key = target / "etc/owrt-remote/web.key"
if existing:
key.write_text("existing-private-key\n")
rpcd = target / "etc/init.d/rpcd"
rpcd.write_text("#!/bin/sh\nexit " + ("9" if rpcd_fails else "0") + "\n", newline="\n")
rpcd.chmod(0o755)
helper = (p.RECIPE / "root/usr/lib/owrt-remote/package.sh").read_text()
# Redirect target paths in the harness. The production hook never
# honors arbitrary ROOT/host variables supplied to a package manager.
helper = helper.replace("/etc/", target.as_posix() + "/etc/").replace("/tmp/", target.as_posix() + "/tmp/")
result = subprocess.run([SH, "-s", "--", "configure"], input=helper, capture_output=True, text=True, env=self.env)
return result, key, config
def test_target_setup_creates_private_key_and_preserves_uci(self):
result, key, config = self.target_setup()
self.assertEqual(result.returncode, 0, result.stderr)
self.assertRegex(key.read_text(), r"^[0-9a-f]{64}\n$")
self.assertIn("test-secret", config.read_text())
self.assertNotIn("test-secret", result.stdout + result.stderr)
self.assertNotIn(key.read_text().strip(), result.stdout + result.stderr)
def test_target_setup_keeps_existing_panel_key(self):
result, key, _ = self.target_setup(existing=True)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(key.read_text(), "existing-private-key\n")
def test_target_setup_reports_rpcd_failure(self):
result, _, _ = self.target_setup(rpcd_fails=True)
self.assertNotEqual(result.returncode, 0)
self.assertIn("rpcd reload failed", result.stderr)
def test_sdk_discovery_handles_xz_and_zstd_without_guessed_filename(self):
for ext in ("xz", "zst"):
name = f"openwrt-sdk-24.10.8-x86-64_gcc-13.3.0_musl.Linux-x86_64.tar.{ext}"
self.assertEqual(p.sdk_name(f'<a href="{name}">SDK</a>'), name)
with self.assertRaises(ValueError):
p.sdk_name('<a href="../../malicious.tar.zst">wrong</a>')
def test_checksum_requires_exact_sdk_and_valid_digest(self):
digest = "a" * 64
self.assertEqual(p.checksum(digest + " *sdk.tar.xz\n", "sdk.tar.xz"), digest)
for text in ("bad *sdk.tar.xz\n", digest + " *other-sdk.tar.xz\n", ""):
with self.assertRaises(ValueError):
p.checksum(text, "sdk.tar.xz")
def test_latest_release_is_numeric_and_ignores_prereleases(self):
html = ''.join(f'<a href="{v}/">link</a>' for v in ("24.10.9", "24.10.10", "24.10.11-rc1", "25.12.6"))
self.assertEqual(p.latest_release("24.10", html), "24.10.10")
def test_matrix_has_all_requested_architectures_and_one_publication_per_series(self):
builds = p.matrix(baseline=True)["include"]
self.assertEqual(len(builds), 30)
self.assertEqual(sum(bool(x.get("publish")) for x in builds), 3)
self.assertEqual({x["format"] for x in builds}, {"ipk", "apk"})
self.assertTrue({"mips64", "mipsel_24kc", "aarch64_generic", "x86_64"} <= {x["id"] for x in builds})
def test_corrupt_sdk_download_never_replaces_existing_cache(self):
destination = self.root / "sdk.tar.xz"
destination.write_bytes(b"previous download")
with mock.patch.object(p.urllib.request, "urlopen", side_effect=lambda *a, **k: io.BytesIO(b"corrupt")), mock.patch.object(p.time, "sleep"):
with self.assertRaisesRegex(ValueError, "SHA256 mismatch"):
p.download("https://downloads.openwrt.org/example", destination, "a" * 64)
self.assertEqual(destination.read_bytes(), b"previous download")
self.assertFalse(destination.with_name(destination.name + ".partial").exists())
def test_openwrt_disk_image_accepts_fwtool_footer_and_rejects_truncation(self):
source, destination = self.root / "image.gz", self.root / "image.raw"
body = b"OpenWrt disk sectors" * 10000
source.write_bytes(gzip.compress(body) + b'# fwtool image metadata')
p.unpack_image(source, destination)
self.assertEqual(destination.read_bytes(), body)
source.write_bytes(gzip.compress(body)[:-5])
with self.assertRaisesRegex(ValueError, "truncated"):
p.unpack_image(source, destination)
if __name__ == "__main__":
unittest.main()