diff --git a/.gitattributes b/.gitattributes index 6df6e89..00d0590 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2,6 +2,10 @@ *.md text eol=lf *.txt text eol=lf +*.sh text eol=lf +*.py text eol=lf +*.yml text eol=lf +package/**/Makefile text eol=lf .gitattributes text eol=lf .gitignore text eol=lf LICENSE text eol=lf diff --git a/.github/workflows/openwrt-packages.yml b/.github/workflows/openwrt-packages.yml new file mode 100644 index 0000000..cf9326d --- /dev/null +++ b/.github/workflows/openwrt-packages.yml @@ -0,0 +1,158 @@ +name: OpenWrt IPK and APK + +on: + push: + tags: ['v*'] + release: + types: [published] + pull_request: + paths: ['files/**', 'package/**', 'scripts/*openwrt*', 'tests/test_openwrt_package.py', '.github/workflows/openwrt-packages.yml'] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: openwrt-packages-${{ github.ref }} + cancel-in-progress: false + +jobs: + matrix: + name: Validate source and resolve stable SDKs + runs-on: ubuntu-24.04 + outputs: + builds: ${{ steps.targets.outputs.builds }} + smoke: ${{ steps.targets.outputs.smoke }} + steps: + - uses: actions/checkout@v4 + - name: Test package tooling and shell syntax + run: | + python3 -m unittest discover -s tests -p test_openwrt_package.py -v + sh -n scripts/build-openwrt-package.sh + sh -n package/luci-app-owrt-remote/root/usr/lib/owrt-remote/package.sh + - name: Validate tag against package version + if: startsWith(github.ref, 'refs/tags/') + env: + RELEASE_TAG: ${{ github.ref_name }} + run: | + python3 - <<'PY' + import os, sys + sys.path.insert(0, 'scripts') + import openwrt_package as p + if os.environ['RELEASE_TAG'] != 'v' + p.versions()[0]: + raise SystemExit('Release tag must match PKG_VERSION in package/luci-app-owrt-remote/Makefile') + PY + - id: targets + name: Resolve latest maintenance versions for each supported series + run: | + python3 - <<'PY' + import json, os, sys + sys.path.insert(0, 'scripts') + import openwrt_package as p + builds = p.matrix() + smoke = {'include': [x for x in builds['include'] if x.get('publish')]} + with open(os.environ['GITHUB_OUTPUT'], 'a') as f: + f.write('builds=' + json.dumps(builds, separators=(',', ':')) + '\n') + f.write('smoke=' + json.dumps(smoke, separators=(',', ':')) + '\n') + PY + + build: + name: SDK ${{ matrix.version }} / ${{ matrix.id }} + needs: matrix + runs-on: ubuntu-24.04 + timeout-minutes: 60 + strategy: + fail-fast: false + max-parallel: 6 + matrix: ${{ fromJSON(needs.matrix.outputs.builds) }} + steps: + - uses: actions/checkout@v4 + - name: Install official SDK prerequisites + run: | + sudo apt-get update -qq + sudo apt-get install -y build-essential clang flex bison g++ gawk gcc-multilib gettext git libncurses-dev libssl-dev python3-setuptools rsync swig unzip zlib1g-dev file wget zstd + - name: Build and verify the native package + env: + OPENWRT_APK_SIGNING_KEY: ${{ secrets.OPENWRT_APK_SIGNING_KEY }} + run: | + extra=() + if [[ '${{ matrix.publish }}' == 'true' ]]; then extra+=(--upgrade-fixture); fi + python3 scripts/openwrt_package.py build \ + --release '${{ matrix.version }}' \ + --target '${{ matrix.target }}' \ + --subtarget '${{ matrix.subtarget }}' \ + --arch '${{ matrix.id }}' \ + --output artifacts "${extra[@]}" + - name: Keep SDK verification report + uses: actions/upload-artifact@v4 + with: + name: sdk-report-${{ matrix.series }}-${{ matrix.id }} + path: artifacts/report-*.json + if-no-files-found: error + - name: Keep universal packages for LuCI and SSH + if: matrix.publish == true + uses: actions/upload-artifact@v4 + with: + name: packages-${{ matrix.series }} + path: artifacts/* + if-no-files-found: error + + install-test: + name: Install, upgrade, remove / OpenWrt ${{ matrix.version }} + needs: [matrix, build] + runs-on: ubuntu-24.04 + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix.outputs.smoke) }} + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: packages-${{ matrix.series }} + path: artifacts + - name: Install VM test tools + run: | + sudo apt-get update -qq + sudo apt-get install -y qemu-system-x86 openssh-client + - name: Test with the real OpenWrt package manager and BusyBox ash + run: | + python3 scripts/test-openwrt-package-vm.py \ + --release '${{ matrix.version }}' --package-dir artifacts \ + --output vm-test + - uses: actions/upload-artifact@v4 + if: always() + with: + name: install-test-${{ matrix.series }} + path: vm-test/* + if-no-files-found: warn + + publish: + name: Attach verified packages to GitHub Release + needs: [build, install-test] + if: startsWith(github.ref, 'refs/tags/v') && github.event_name != 'workflow_dispatch' + runs-on: ubuntu-24.04 + permissions: + contents: write + steps: + - uses: actions/download-artifact@v4 + with: + pattern: packages-* + path: release-assets + merge-multiple: true + - name: Write combined SHA256 checksums + run: | + cd release-assets + sha256sum *.ipk *.apk > SHA256SUMS + if [[ -f owrt-remote-packages.pem ]]; then sha256sum owrt-remote-packages.pem >> SHA256SUMS; fi + printf '%s\n' *.ipk *.apk SHA256SUMS report-*.json > ../release-files.txt + if [[ -f owrt-remote-packages.pem ]]; then echo owrt-remote-packages.pem >> ../release-files.txt; fi + mkdir ../publish-files + while read -r name; do cp "$name" ../publish-files/; done < ../release-files.txt + - uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ github.ref_name }} + generate_release_notes: true + fail_on_unmatched_files: true + files: | + publish-files/* diff --git a/.gitignore b/.gitignore index 004568b..39fb4f8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ *.ipk +*.apk +artifacts/ *.tar.gz *.tar.gz.sha256 .test-output/ diff --git a/README.md b/README.md index a2c0226..dbb29ff 100644 --- a/README.md +++ b/README.md @@ -182,6 +182,17 @@ owrt-remote doctor owrt-remote status ``` +

Готовые пакеты для OpenWrt

+ +Подготовлена сборка IPK для OpenWrt 23.05/24.10 и APK для 25.12. Пакеты +архитектурно независимые (`all`/`noarch`): содержат тот же агент и LuCI-файлы, +что и `install.sh`, сохраняют `/etc/config/owrtremote` и существующий Xray. +GitHub Actions проверяет SDK разных архитектур, установку/обновление/удаление +в OpenWrt VM и при выпуске тега `v*` прикладывает пакеты к GitHub Release. +Для APK в штатной LuCI нужен доверенный ключ подписи; неподписанный локальный +APK ставится через SSH. Инструкция, подпись и локальная сборка: +[docs/OPENWRT_PACKAGES.md](docs/OPENWRT_PACKAGES.md). + --- diff --git a/docs/OPENWRT_PACKAGES.md b/docs/OPENWRT_PACKAGES.md new file mode 100644 index 0000000..0031352 --- /dev/null +++ b/docs/OPENWRT_PACKAGES.md @@ -0,0 +1,189 @@ +# Пакеты OpenWrt Remote + +Исходники пакета `luci-app-owrt-remote` находятся в +`package/luci-app-owrt-remote/`. Пакет содержит агент, CGI-панель, LuCI, +Heartbeat, управление Xray, WAN direct и журнал. Файлы берутся непосредственно +из `files/`, которую использует `install.sh`; отдельная копия агента не ведётся. + +Это пакеты проекта для OpenWrt, а не включение проекта в официальный feed OpenWrt. +Публикация в Releases произойдёт только после отправки workflow и выпуска тега. + +## Какой файл выбрать + +| OpenWrt | Файл | Архитектура пакета | +| --- | --- | --- | +| 23.05.x | `luci-app-owrt-remote_121-r1_openwrt-23.05_all.ipk` | `all` | +| 24.10.x | `luci-app-owrt-remote_121-r1_openwrt-24.10_all.ipk` | `all` | +| 25.12.x | `luci-app-owrt-remote_121-r1_openwrt-25.12_noarch.apk` | `noarch` | + +В пакете нет ELF-бинарников. Один файл своей ветки подходит `aarch64_cortex-a53`, +`aarch64_cortex-a72`, `aarch64_generic`, ARM, MIPS, MIPSEL, MIPS64, x86, x86_64 +и другим архитектурам с доступными зависимостями. Формат определяется менеджером +пакетов: `opkg` использует IPK, `apk` использует APK; они не взаимозаменяемы. + +Xray загружается отдельно существующим агентом при применении конфигурации Hub. +Его версия, ABI, SHA256 и загрузочный кэш проверяются существующим механизмом. +Установка пакета не запускает вложенный `opkg`/`apk` и не загружает Xray, +пока менеджер пакетов удерживает свою блокировку. + +## Установка и обновление через LuCI + +1. Скачай пакет своей ветки из [GitHub Releases](https://github.com/kzolotarev95/luci-app-owrt-remote/releases). +2. Открой **Система → Программное обеспечение**, обнови списки пакетов. +3. Нажми **Загрузить пакет**, выбери файл и подтверди установку. +4. Открой **Службы → OpenWrt Remote** и примени OpenWrt config из своей VPS-панели. + +Обновление выполняется загрузкой пакета новой версии тем же способом. +Перезагрузка роутера не требуется. Зависимости устанавливает менеджер пакетов: +`luci-base`, `rpcd-mod-file`, `uhttpd`, `jsonfilter`, `curl`, `ca-bundle`, `etherwake`. +Для их первичной загрузки нужны доступные репозитории своей прошивки. + +На OpenWrt 25.12 штатная LuCI принимает сторонний APK только с доверенной +подписью. Для подписанных выпусков один раз установи опубликованный публичный +ключ проекта в `/etc/apk/keys/owrt-remote-packages.pem`, проверив его отпечаток +по доверенному каналу. Далее подписанные APK устанавливаются через LuCI. +Без ключа штатная LuCI отклоняет пакет; для локального неподписанного APK +используй SSH-команду ниже. Глобально отключать проверку подписей не нужно. + +## Установка по SSH из PowerShell Windows 11 + +Пример для OpenWrt 24.10 и основного роутера `192.168.2.1`: + +```powershell +$Repo = "C:\Users\k.zolotarev95\Documents\openwrt overlay" +$Package = Join-Path $Repo "artifacts\openwrt-packages\24.10\luci-app-owrt-remote_121-r1_openwrt-24.10_all.ipk" +ssh root@192.168.2.1 'sysupgrade -b /root/owrt-before-package.tar.gz' +if ($LASTEXITCODE -ne 0) { throw 'Резервная копия не создана' } +scp -O -- "$Package" 'root@192.168.2.1:/tmp/owrt-remote.ipk' +if ($LASTEXITCODE -ne 0) { throw 'Не удалось загрузить пакет' } +ssh root@192.168.2.1 'opkg update && opkg install /tmp/owrt-remote.ipk' +if ($LASTEXITCODE -ne 0) { throw 'Установка не завершена; смотри ошибку выше' } +``` + +На OpenWrt 23.05 выбери файл из каталога `23.05`. Для OpenWrt 25.12 загрузи +файл из `25.12` как `/tmp/owrt-remote.apk` и выполни: + +```powershell +ssh root@192.168.2.1 'apk update && apk add --allow-untrusted /tmp/owrt-remote.apk' +``` + +`--allow-untrusted` нужен только неподписанному локальному APK. Для подписанного +APK с установленным доверенным ключом используется `apk add` без этого флага. +После установки проверь `owrt-remote status` и `owrt-remote doctor`. +На чистой установке Remote выключен до ввода своего Hub config — это штатно. + +## Сохранение настроек и откат + +`/etc/config/owrtremote` объявлен conffile. Обновление сохраняет изменённые UCI, +ключ `/etc/owrt-remote/web.key`, Xray, JSON, постоянный кэш и журнал. Ключ панели +создаётся только при его отсутствии, с правами `0600`, через атомарную замену. + +Удаление пакета останавливает и отключает его procd-службы. Пакет не очищает +сгенерированные данные пользователя и не запускает `uninstall.sh`: + +```powershell +# Для IPK: +ssh root@192.168.2.1 'opkg remove luci-app-owrt-remote' +# Для APK: +ssh root@192.168.2.1 'apk del luci-app-owrt-remote' +``` + +Для отката к прежнему пакетному выпуску сохрани его файл до обновления, +удали новый пакет и установи сохранённый IPK/APK. Изменённый UCI при удалении +сохраняется. Чтобы вернуть настройки из сделанной выше резервной копии, +извлеки только конфигурацию Remote, затем перезапусти службу: + +```powershell +ssh root@192.168.2.1 'tar -xzf /root/owrt-before-package.tar.gz -C / etc/config/owrtremote && /etc/init.d/owrt-remote restart' +``` + +`sysupgrade -b` сохраняет настройки, а не старые программные файлы. +При переходе с `install.sh` сохрани исходный checkout/предыдущий установщик: +удаление пакета само по себе не восстанавливает старую ручную установку. + +## Сборка + +Локальный IPK без SDK можно собрать и проверить на Windows с Python 3.11+: + +```powershell +Set-Location "C:\Users\k.zolotarev95\Documents\openwrt overlay" +python scripts/openwrt_package.py local-ipk --output artifacts/remote-local.ipk +python scripts/openwrt_package.py verify artifacts/remote-local.ipk +``` + +Локальный APK создаётся настоящим `apk-tools v3 mkpkg` из host tools SDK 25.12. +Целевой `apk` на роутере может не содержать команду `mkpkg`. На Linux: + +```sh +python3 scripts/openwrt_package.py local-apk --apk /path/to/sdk/staging_dir/host/bin/apk --output artifacts/remote-local.apk +``` + +Релизная сборка выполняется официальным SDK на Linux x86_64. Скрипт находит +имя SDK в индексе OpenWrt, проверяет SHA256, использует feeds этого SDK, +собирает пакет и проверяет содержимое, права, владельца, зависимости и conffiles: + +```sh +sh scripts/build-openwrt-package.sh --release 24.10.3 --target ramips --subtarget mt7621 --arch mipsel_24kc --output artifacts +``` + +SDK поддерживаются в форматах `.tar.xz` и `.tar.zst`. Для подготовки автономного +каталога пакета внутри своего SDK: + +```sh +python3 scripts/openwrt_package.py prepare --output /path/to/sdk/package/luci-app-owrt-remote +``` + +## GitHub Actions и подпись APK + +Workflow `.github/workflows/openwrt-packages.yml` запускается на тегах `v*`, +публикации Release, изменениях пакета в pull request и вручную. Вручную и в PR +он только собирает и тестирует; в Releases пишет только запуск по тегу. +Тег должен совпадать с `PKG_VERSION`: текущий пример — `v121`. +Следующий выпуск должен получить новую версию или пакетную ревизию в Makefile. + +На каждом запуске выбирается последняя стабильная patch-версия веток +23.05, 24.10 и 25.12. Матрица `package/openwrt-targets.json` содержит 10 SDK +на каждую ветку: A53, A72, generic ARM64, ARM A7/A9, MIPS, MIPSEL, MIPS64, +x86 и x86_64. Сохраняются отчёты всех 30 сборок. + +После сборки универсальные пакеты устанавливаются в три изолированные x86_64 +OpenWrt VM. Проверяется первичная установка, настоящая смена пакетной ревизии, +работа CGI, BusyBox ash, службы, удаление, сохранность настроек/ключа/Xray и +переход с не принадлежащей пакетному менеджеру конфигурации. Тестовая ревизия +из `test-fixtures/` не публикуется. Только после успешного завершения +всей матрицы публикуются три пакета, SHA256SUMS и отчёты SDK. + +Для подписанных APK добавь в GitHub Actions secret `OPENWRT_APK_SIGNING_KEY` +постоянный приватный RSA PEM-ключ сопровождающего. Пример создания на доверенной +машине (приватный файл не добавлять в Git): + +```sh +openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:3072 -out /secure/owrt-remote-private.pem +openssl pkey -in /secure/owrt-remote-private.pem -pubout -out owrt-remote-packages.pem +``` + +Workflow подписывает APK, проверяет установку с публичным ключом и добавляет +`owrt-remote-packages.pem` в Release. Приватный ключ остаётся secret/временным +файлом runner и не попадает в artifacts. Если secret отсутствует, выпуск APK +явно помечается в отчёте как неподписанный и устанавливается по SSH с +`--allow-untrusted`. IPK устанавливается локальным файлом обычным способом. +Новый случайный ключ на каждом запуске не создаётся. + +## Проверки перед публикацией + +```sh +python3 -m unittest discover -s tests -p test_openwrt_package.py -v +python3 scripts/test-openwrt-package-vm.py --release 24.10.3 --package-dir artifacts --output vm-test +``` + +Для VM-теста нужны QEMU, SSH и один основной пакет плюс пакет следующей +ревизии в `test-fixtures/`. SDK-сборка создаёт такой набор с `--upgrade-fixture`. +VM использует только loopback и не подключается к реальным роутерам. + +Проверка установки на x86_64 не доказывает работу конкретного Xray на каждом +физическом CPU. Xray, Heartbeat с реальным Hub, WAN direct и аппаратные +особенности требуют отдельной проверки на устройствах. + +Основа реализации: [OpenWrt SDK](https://openwrt.org/docs/guide-developer/toolchain/using_the_sdk), +[пакетный Makefile OpenWrt](https://github.com/openwrt/openwrt/blob/openwrt-25.12/include/package-pack.mk), +[жизненный цикл пакетов](https://github.com/openwrt/openwrt/blob/openwrt-25.12/package/base-files/files/lib/functions.sh). diff --git a/docs/OPENWRT_PACKAGE_TESTS.md b/docs/OPENWRT_PACKAGE_TESTS.md new file mode 100644 index 0000000..c965f07 --- /dev/null +++ b/docs/OPENWRT_PACKAGE_TESTS.md @@ -0,0 +1,48 @@ +# Локальная проверка пакетной установки + +Дата: 07.11.2026. Версия приложения: v121, ревизия пакета: 1. +Проверки выполнялись локально; push и публикация Release не выполнялись. +Реальные роутеры и VPS не изменялись. + +| Среда | Менеджер | Результат | +| --- | --- | --- | +| OpenWrt 23.05.6 x86_64, QEMU TCG | opkg, IPK `all` | Успешно | +| OpenWrt 24.10.8 x86_64, QEMU TCG | opkg, IPK `all` | Успешно | +| OpenWrt 25.12.6 x86_64, QEMU TCG | apk, APK v3 `noarch` | Успешно | + +В каждой VM проверены: + +- Чистая установка и загрузка зависимостей из репозиториев соответствующей прошивки. +- Создание приватного ключа панели, доступность CGI и пунктов LuCI. +- Синтаксис всех shell-файлов пакета штатным BusyBox ash. +- Работа службы журналов и сохранение выключенного по умолчанию Remote. +- Обновление с ревизии 1 на отдельный пакет ревизии 2. +- Побайтовое сохранение изменённого UCI, ключа панели, Xray JSON и бинарника-заглушки. +- Удаление файлов пакета и сохранение изменённой конфигурации/сгенерированных данных. +- Повторная установка с конфигурацией, не принадлежащей менеджеру пакетов. + +Дополнительно: + +- 16 автоматических тестов сборочного инструмента прошли. +- `actionlint v1.7.12` не обнаружил ошибок workflow. +- Проверены все 30 URL SDK и их записи SHA256 в официальных индексах: + 10 целевых архитектур для каждой из трёх веток. +- Подтверждён отказ штатного apk от неподписанного APK без `--allow-untrusted`. +- APK подписан отдельным тестовым RSA-ключом; после добавления публичного + ключа штатный apk принял его без `--allow-untrusted` в проверке `--simulate`. + Тестовый приватный ключ не входит в исходники или установочные артефакты. +- Установка не вызывала скачивание/замену рабочего Xray внутри пакетной транзакции. + +Локальные IPK созданы переносимым сборщиком из тех же исходников; APK создан +настоящим `apk-tools 3.0.9 mkpkg` в изолированной VM. Полная компиляция через +30 официальных SDK здесь не запускалась: она подготовлена в GitHub Actions. +Доступность SDK не равнозначна успешной сборке; публикация workflow зависит +от успешного завершения SDK-матрицы и последующих VM-тестов. + +Физические ARM/MIPS-устройства, действующий Heartbeat, WAN direct и туннель +Xray с реальным Hub в этой проверке не тестировались. Код этих механизмов +не изменён пакетной доработкой. + +Локальные протоколы находятся в `.test-output/vm-23.05/`, +`.test-output/vm-24.10/`, `.test-output/vm-25.12/`. Готовые тестовые пакеты — +в `artifacts/openwrt-packages/`. Эти каталоги исключены из Git. diff --git a/package/luci-app-owrt-remote/Makefile b/package/luci-app-owrt-remote/Makefile new file mode 100644 index 0000000..1442e7c --- /dev/null +++ b/package/luci-app-owrt-remote/Makefile @@ -0,0 +1,67 @@ +include $(TOPDIR)/rules.mk + +PKG_NAME:=luci-app-owrt-remote +PKG_VERSION:=121 +PKG_RELEASE:=1 +PKG_LICENSE:=MIT +PKG_MAINTAINER:=kzolotarev95 +PKGARCH:=all + +include $(INCLUDE_DIR)/package.mk + +define Package/luci-app-owrt-remote + SECTION:=luci + CATEGORY:=LuCI + SUBMENU:=3. Applications + TITLE:=OpenWrt Remote Hub router agent + URL:=https://github.com/kzolotarev95/luci-app-owrt-remote + DEPENDS:=+luci-base +rpcd-mod-file +uhttpd +jsonfilter +curl +ca-bundle +etherwake +endef + +define Package/luci-app-owrt-remote/description + Router agent and LuCI panel for OpenWrt Remote Hub. Includes Heartbeat, + independent remote Xray control, WAN direct and bounded persistent logs. + Xray is selected and verified for the router by the existing agent when + applying the Hub configuration; it is not bundled into this noarch package. +endef + +define Package/luci-app-owrt-remote/conffiles +/etc/config/owrtremote +endef + +# scripts/openwrt_package.py prepare creates this self-contained SDK package +# from files/ and root/. There are no separately maintained runtime copies. +define Build/Prepare + mkdir -p $(PKG_BUILD_DIR) +endef + +define Build/Configure +endef + +define Build/Compile +endef + +define Package/luci-app-owrt-remote/install + $(CP) ./root/. $(1)/ +endef + +define Package/luci-app-owrt-remote/postinst +#!/bin/sh +[ -n "$${IPKG_INSTROOT}" ] || /bin/sh /usr/lib/owrt-remote/package.sh configure +endef + +define Package/luci-app-owrt-remote/postrm +#!/bin/sh +[ -n "$${IPKG_INSTROOT}" ] || { \ + rm -f /tmp/luci-indexcache /tmp/luci-indexcache.*; \ + rm -rf /tmp/luci-modulecache; \ + [ ! -x /etc/init.d/rpcd ] || /etc/init.d/rpcd reload; \ +} +exit 0 +endef + +# OpenWrt's generated postinst/prerm handle enable/start and stop/disable for +# both procd services, and PKG_UPGRADE keeps upgrade semantics on IPK and APK. +# Do not duplicate them or depend on opkg's "configure" argument: apk invokes +# post-install/post-upgrade with version arguments instead. +$(eval $(call BuildPackage,luci-app-owrt-remote)) diff --git a/package/luci-app-owrt-remote/root/usr/lib/owrt-remote/package.sh b/package/luci-app-owrt-remote/root/usr/lib/owrt-remote/package.sh new file mode 100644 index 0000000..4228db6 --- /dev/null +++ b/package/luci-app-owrt-remote/root/usr/lib/owrt-remote/package.sh @@ -0,0 +1,40 @@ +#!/bin/sh + +# Native package installation needs the same private panel key as install.sh. +# No network downloads or nested opkg/apk operations run under the package lock. +set -eu + +configure() { + local dir key temp + dir=/etc/owrt-remote + umask 077 + mkdir -p "$dir" + if [ ! -s "$dir/web.key" ]; then + temp="$(mktemp "$dir/.web.key.XXXXXX")" + trap 'rm -f "$temp"' EXIT + dd if=/dev/urandom of="$temp" bs=32 count=1 2>/dev/null + [ "$(wc -c <"$temp")" -eq 32 ] || { echo 'ERROR: insufficient random bytes for panel key' >&2; return 1; } + key="$(sha256sum "$temp")" + key="${key%% *}" + [ "${#key}" = 64 ] || { echo 'ERROR: could not generate panel access key' >&2; return 1; } + printf '%s\n' "$key" >"$temp" + chmod 0600 "$temp" + mv -f "$temp" "$dir/web.key" + trap - EXIT + fi + chmod 0600 "$dir/web.key" + # Native conffiles handling preserves existing UCI settings on upgrades. + [ ! -f /etc/config/owrtremote ] || chmod 0600 /etc/config/owrtremote + rm -f /tmp/luci-indexcache /tmp/luci-indexcache.* + rm -rf /tmp/luci-modulecache + if [ -x /etc/init.d/rpcd ]; then + /etc/init.d/rpcd reload || { echo 'ERROR: rpcd reload failed' >&2; return 1; } + fi + printf '%s\n' 'OpenWrt Remote: installed. Open LuCI > Services > OpenWrt Remote.' + printf '%s\n' 'Apply OpenWrt config from Hub to prepare Xray and start Remote.' +} + +case "${1:-}" in + configure) configure ;; + *) echo 'Usage: package.sh configure' >&2; exit 2 ;; +esac diff --git a/package/openwrt-targets.json b/package/openwrt-targets.json new file mode 100644 index 0000000..b4be4cb --- /dev/null +++ b/package/openwrt-targets.json @@ -0,0 +1,19 @@ +{ + "releases": [ + {"series": "23.05", "baseline": "23.05.5", "format": "ipk"}, + {"series": "24.10", "baseline": "24.10.3", "format": "ipk"}, + {"series": "25.12", "baseline": "25.12.2", "format": "apk"} + ], + "targets": [ + {"id": "aarch64_cortex-a53", "target": "mvebu", "subtarget": "cortexa53"}, + {"id": "aarch64_cortex-a72", "target": "mvebu", "subtarget": "cortexa72"}, + {"id": "aarch64_generic", "target": "armsr", "subtarget": "armv8"}, + {"id": "arm_cortex-a7", "target": "bcm27xx", "subtarget": "bcm2709"}, + {"id": "arm_cortex-a9", "target": "mvebu", "subtarget": "cortexa9"}, + {"id": "mips_24kc", "target": "ath79", "subtarget": "generic"}, + {"id": "mipsel_24kc", "target": "ramips", "subtarget": "mt7621"}, + {"id": "mips64", "target": "octeon", "subtarget": "generic"}, + {"id": "x86", "target": "x86", "subtarget": "generic"}, + {"id": "x86_64", "target": "x86", "subtarget": "64", "publish": true} + ] +} diff --git a/scripts/build-openwrt-package.sh b/scripts/build-openwrt-package.sh new file mode 100644 index 0000000..ea5f715 --- /dev/null +++ b/scripts/build-openwrt-package.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +SOURCE_DIR="$(CDPATH= cd "$(dirname "$0")/.." && pwd)" +exec python3 "$SOURCE_DIR/scripts/openwrt_package.py" build "$@" diff --git a/scripts/openwrt_package.py b/scripts/openwrt_package.py new file mode 100644 index 0000000..ea9369e --- /dev/null +++ b/scripts/openwrt_package.py @@ -0,0 +1,535 @@ +#!/usr/bin/env python3 +"""Stage one overlay, build it with official SDKs, verify native packages.""" +from __future__ import annotations + +import argparse +import gzip +import hashlib +from html.parser import HTMLParser +import io +import json +import os +from pathlib import Path, PurePosixPath +import re +import shlex +import shutil +import subprocess +import sys +import tarfile +import tempfile +import time +import urllib.parse +import urllib.request +import zlib + +ROOT = Path(__file__).resolve().parents[1] +PACKAGE = "luci-app-owrt-remote" +RECIPE = ROOT / "package" / PACKAGE +DOWNLOADS = "https://downloads.openwrt.org/releases/" +EXECUTABLES = { + "usr/sbin/owrt-remote", "www/cgi-bin/owrt-remote", + "etc/init.d/owrt-remote", "etc/init.d/owrt-remote-logs", + "etc/hotplug.d/iface/99-owrt-remote", +} + + +class Links(HTMLParser): + def __init__(self, text): + super().__init__() + self.links = [] + self.feed(text) + + def handle_starttag(self, tag, attrs): + if tag == "a": + self.links.extend(value for key, value in attrs if key == "href" and value) + + +def get_text(url): + request = urllib.request.Request(url, headers={"User-Agent": "OpenWrt-Remote-package-builder"}) + with urllib.request.urlopen(request, timeout=45) as response: + data = response.read(4 * 1024 * 1024 + 1) + if len(data) > 4 * 1024 * 1024: + raise ValueError("oversized SDK metadata: " + url) + return data.decode("utf-8") + + +def versions(): + text = (RECIPE / "Makefile").read_text(encoding="utf-8") + version = re.search(r"^PKG_VERSION:=([0-9][0-9.]*)$", text, re.M).group(1) + release = re.search(r"^PKG_RELEASE:=([0-9]+)$", text, re.M).group(1) + return version, release + + +def latest_release(series, html): + candidates = [link.rstrip("/") for link in Links(html).links + if re.fullmatch(re.escape(series) + r"\.[0-9]+/", link)] + if not candidates: + raise ValueError("no stable OpenWrt release for " + series) + return max(candidates, key=lambda v: tuple(map(int, v.split(".")))) + + +def sdk_name(html): + names = sorted({link for link in Links(html).links if re.fullmatch( + r"openwrt-sdk-[A-Za-z0-9_.+-]+Linux-x86_64\.tar\.(?:xz|zst)", link)}) + if len(names) != 1: + raise ValueError("expected exactly one official Linux x86_64 SDK; found " + str(names)) + return names[0] + + +def checksum(text, filename): + values = [] + for line in text.splitlines(): + fields = line.split() + if len(fields) == 2 and fields[1].lstrip("*") == filename: + if re.fullmatch(r"[a-fA-F0-9]{64}", fields[0]): + values.append(fields[0].lower()) + if len(set(values)) != 1: + raise ValueError("missing or ambiguous SHA256 for " + filename) + return values[0] + + +def sha256(path): + with Path(path).open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def download(url, path, expected): + path = Path(path) + if path.is_file() and sha256(path) == expected: + return + partial = path.with_name(path.name + ".partial") + for attempt in range(3): + try: + digest = hashlib.sha256() + request = urllib.request.Request(url, headers={"User-Agent": "OpenWrt-Remote-package-builder"}) + with urllib.request.urlopen(request, timeout=60) as response, partial.open("wb") as output: + size = 0 + while data := response.read(1024 * 1024): + size += len(data) + if size > 2 * 1024 ** 3: + raise ValueError("SDK download exceeded 2 GiB") + output.write(data) + digest.update(data) + if digest.hexdigest() != expected: + raise ValueError("SDK SHA256 mismatch: " + url) + partial.replace(path) + return + except Exception: + partial.unlink(missing_ok=True) + if attempt == 2: + raise + time.sleep(2) + + +def unpack_image(source, destination): + # Some official OpenWrt .img.gz files have fwtool metadata appended after + # the gzip member. The full download SHA256 is checked before this step. + # Python gzip.open treats that signed/image metadata as a second header. + decoder = zlib.decompressobj(16 + zlib.MAX_WBITS) + total = 0 + with Path(source).open("rb") as compressed, Path(destination).open("wb") as raw: + while not decoder.eof: + block = compressed.read(65536) + if not block: + raise ValueError("truncated OpenWrt disk image") + data = decoder.decompress(block) + total += len(data) + if total > 2 * 1024 ** 3: + raise ValueError("OpenWrt disk image exceeded 2 GiB") + raw.write(data) + + +def payload(): + result = {} + for base in (ROOT / "files", RECIPE / "root"): + for path in sorted(base.rglob("*")): + if not path.is_file(): + continue + if path.is_symlink(): + raise ValueError("unexpected source symlink: " + str(path)) + name = path.relative_to(base).as_posix() + if name in result: + raise ValueError("duplicate package payload: " + name) + data = path.read_bytes().replace(b"\r\n", b"\n") + mode = 0o755 if name in EXECUTABLES else 0o644 + if name == "etc/config/owrtremote": + mode = 0o600 + result[name] = (data, mode) + return result + + +def package_hook(name): + text = (RECIPE / "Makefile").read_text(encoding="utf-8") + match = re.search(r"^define Package/" + PACKAGE + "/" + name + r"\n(.*?)^endef$", text, re.M | re.S) + return match.group(1).replace("$$", "$") if match else "#!/bin/sh\nexit 0\n" + + +def dependencies(): + text = (RECIPE / "Makefile").read_text(encoding="utf-8") + line = re.search(r"^\s+DEPENDS:=(.+)$", text, re.M).group(1) + return [item.removeprefix("+") for item in line.split()] + + +def native_wrappers(): + common = '#!/bin/sh\n[ -s "${IPKG_INSTROOT}/lib/functions.sh" ] || exit 0\n. "${IPKG_INSTROOT}/lib/functions.sh"\n' + return { + "postinst": common + 'default_postinst "$0" "$@"\n', + "prerm": common + 'default_prerm "$0" "$@"\n', + "postinst-pkg": package_hook("postinst"), + "postrm": package_hook("postrm"), + } + + +def tar_bytes(files): + data = io.BytesIO() + with tarfile.open(fileobj=data, mode="w", format=tarfile.USTAR_FORMAT) as archive: + directories = {parent.as_posix() for name in files for parent in PurePosixPath(name).parents + if parent.as_posix() != "."} + for name in sorted(directories): + info = tarfile.TarInfo("./" + name + "/") + info.type = tarfile.DIRTYPE + info.mode = 0o755 + info.uid = info.gid = 0 + archive.addfile(info) + for name, (content, mode) in sorted(files.items()): + info = tarfile.TarInfo("./" + name) + info.size = len(content) + info.mode = mode + info.uid = info.gid = 0 + archive.addfile(info, io.BytesIO(content)) + return gzip.compress(data.getvalue(), mtime=0) + + +def pack_ipk(output, revision=None): + """Portable noarch preview; release CI also builds with the official SDK.""" + version, release = versions() + revision = str(revision or release) + files = payload() + control = (f"Package: {PACKAGE}\nVersion: {version}-{revision}\nArchitecture: all\n" + f"Depends: {', '.join(dependencies())}\nSection: luci\nLicense: MIT\n" + "Maintainer: kzolotarev95\nURL: https://github.com/kzolotarev95/luci-app-owrt-remote\n" + f"Installed-Size: {sum(len(data) for data, _ in files.values())}\n" + "Description: OpenWrt Remote Hub router agent and LuCI panel\n") + metadata = {"control": (control.encode(), 0o644), "conffiles": (b"/etc/config/owrtremote\n", 0o644)} + metadata.update({name: (script.encode(), 0o755) for name, script in native_wrappers().items()}) + output = Path(output) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_bytes(tar_bytes({"debian-binary": (b"2.0\n", 0o644), + "control.tar.gz": (tar_bytes(metadata), 0o644), + "data.tar.gz": (tar_bytes(files), 0o644)})) + return output + + +def prepare_apk(folder, revision=None): + """Prepare portable input for the genuine apk-tools v3 mkpkg command.""" + folder = Path(folder).resolve() + staged = prepare(folder / "package") / "root" + version, release = versions() + meta = staged / "lib/apk/packages" + meta.mkdir(parents=True, exist_ok=True) + config = staged / "etc/config/owrtremote" + (meta / f"{PACKAGE}.conffiles").write_text("/etc/config/owrtremote\n", newline="\n") + (meta / f"{PACKAGE}.conffiles_static").write_text(f"/etc/config/owrtremote {sha256(config)}\n", newline="\n") + (meta / f"{PACKAGE}.list").write_text("".join("/" + f.relative_to(staged).as_posix() + "\n" for f in sorted(staged.rglob("*")) if f.is_file()), newline="\n") + common = '#!/bin/sh\n[ -s "${IPKG_INSTROOT}/lib/functions.sh" ] || exit 0\n. "${IPKG_INSTROOT}/lib/functions.sh"\n' + f'export pkgname="{PACKAGE}"\n' + install = common + "default_postinst\n" + package_hook("postinst").removeprefix("#!/bin/sh\n") + scripts = {"post-install": install, + "post-upgrade": "#!/bin/sh\nexport PKG_UPGRADE=1\n" + install.removeprefix("#!/bin/sh\n"), + "pre-deinstall": common + "default_prerm\n", + "post-deinstall": package_hook("postrm")} + args = ["mkpkg", "--info", "name:" + PACKAGE, + "--info", f"version:{version}-r{revision or release}", "--info", "arch:noarch", + "--info", "description:OpenWrt Remote Hub router agent and LuCI panel", + "--info", "license:MIT", "--info", "depends:" + " ".join(dependencies()), + "--info", "url:https://github.com/kzolotarev95/luci-app-owrt-remote", + "--files", "package/root", "--output", PACKAGE + ".apk"] + for name, script in scripts.items(): + (folder / name).write_text(script, newline="\n") + args.extend(["--script", name + ":" + name]) + (folder / "build.sh").write_text('#!/bin/sh\nset -eu\ncd "$(dirname "$0")"\nexec "${APK_TOOL:-apk}" ' + shlex.join(args) + "\n", newline="\n") + return args + + +def pack_apk(output, apk, revision=None): + """Use the real apk-tools v3 packer, never rename an IPK or tar as APK.""" + output = Path(output).resolve() + output.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="owrt-apk-") as folder: + args = prepare_apk(folder, revision) + subprocess.run([str(apk), *args], cwd=folder, check=True) + shutil.copyfile(Path(folder) / (PACKAGE + ".apk"), output) + return output + + +def prepare(output): + output = Path(output).resolve() + # Fail instead of recursively deleting a caller-supplied directory. + if output.exists() and any(output.iterdir()): + raise ValueError("package staging directory must be empty: " + str(output)) + output.mkdir(parents=True, exist_ok=True) + shutil.copyfile(RECIPE / "Makefile", output / "Makefile") + for name, (data, mode) in payload().items(): + target = output / "root" / name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(data) + target.chmod(mode) + return output + + +def ar_members(data): + if not data.startswith(b"!\n"): + raise ValueError("invalid ar package") + offset = 8 + result = {} + while offset < len(data): + header = data[offset:offset + 60] + if len(header) != 60 or header[58:] != b"`\n": + raise ValueError("invalid ar member") + size = int(header[48:58].strip()) + name = header[:16].decode("ascii").strip().rstrip("/") + result[name] = data[offset + 60:offset + 60 + size] + if len(result[name]) != size: + raise ValueError("truncated ar member") + offset += 60 + size + size % 2 + return result + + +def ipk_contents(path): + data = Path(path).read_bytes() + if data.startswith(b"!\n"): + members = ar_members(data) + else: + with tarfile.open(fileobj=io.BytesIO(data), mode="r:*") as archive: + members = {item.name.removeprefix("./"): archive.extractfile(item).read() + for item in archive if item.isfile()} + if members.get("debian-binary") != b"2.0\n": + raise ValueError("invalid IPK version marker") + result = {} + for kind in ("control", "data"): + compressed = members.get(kind + ".tar.gz") + if compressed is None: + raise ValueError("IPK lacks " + kind) + with tarfile.open(fileobj=io.BytesIO(compressed), mode="r:gz") as archive: + result[kind] = {} + for item in archive: + if not item.isfile(): + continue + name = item.name.removeprefix("./") + if ".." in PurePosixPath(name).parts or name.startswith("/"): + raise ValueError("unsafe package path: " + name) + result[kind][name] = (archive.extractfile(item).read(), item.mode & 0o777, item.uid, item.gid) + return result + + +def validate_payload(actual): + for name, (expected, mode) in payload().items(): + if name not in actual: + raise ValueError("package file missing: " + name) + data, found_mode, uid, gid = actual[name] + if data != expected or found_mode != mode or uid != 0 or gid != 0: + raise ValueError("package content/mode/ownership mismatch: " + name) + + +def verify_package(path, apk=None, revision=None): + path = Path(path).resolve() + version, release = versions() + release = str(revision or release) + allowed_versions = {f"{version}-{release}", f"{version}-r{release}"} + if path.suffix == ".ipk": + content = ipk_contents(path) + validate_payload(content["data"]) + control = content["control"]["control"][0].decode("utf-8") + fields = dict(line.split(": ", 1) for line in control.splitlines() if ": " in line and not line.startswith(" ")) + if fields.get("Package") != PACKAGE or fields.get("Architecture") != "all": + raise ValueError("invalid IPK package identity/architecture") + if fields.get("Version") not in allowed_versions: + raise ValueError("IPK version mismatch") + if b"/etc/config/owrtremote" not in content["control"]["conffiles"][0]: + raise ValueError("UCI config must be a conffile") + for script in ("postinst", "prerm"): + if b"/lib/functions.sh" not in content["control"][script][0]: + raise ValueError("missing native OpenWrt lifecycle wrapper") + declared_dependencies = {x.strip().split(" ", 1)[0] for x in fields.get("Depends", "").split(",")} + else: + if not apk: + raise ValueError("APK validation requires the SDK host apk tool") + manifest = json.loads(subprocess.check_output([str(apk), "adbdump", "--format", "json", str(path)], text=True)) + info = manifest["info"] + if info["name"] != PACKAGE or info["arch"] != "noarch" or info["version"] not in allowed_versions: + raise ValueError("invalid APK identity/version/architecture") + owners = {} + for directory in manifest.get("paths", []): + for entry in directory.get("files", []): + name = str(PurePosixPath(directory.get("name", "")) / entry["name"]) + acl = entry.get("acl", directory.get("acl", {})) + if acl.get("user") != "root" or acl.get("group") != "root": + raise ValueError("APK payload must be owned by root: " + name) + owners[name] = acl["mode"] + with tempfile.TemporaryDirectory(prefix="owrt-apk-verify-") as folder: + subprocess.run([str(apk), "extract", "--allow-untrusted", "--no-chown", "--destination", folder, str(path)], check=True) + actual = {} + for name in payload(): + file = Path(folder) / name + if file.is_file(): + stat = file.stat() + if owners.get(name) != stat.st_mode & 0o777: + raise ValueError("APK mode metadata mismatch: " + name) + actual[name] = (file.read_bytes(), stat.st_mode & 0o777, 0, 0) + validate_payload(actual) + conffiles = Path(folder) / "lib/apk/packages" / f"{PACKAGE}.conffiles" + if "/etc/config/owrtremote" not in conffiles.read_text(): + raise ValueError("APK missing OpenWrt conffiles metadata") + declared_dependencies = {re.split(r"[<=>~]", x, maxsplit=1)[0] for x in info.get("depends", [])} + for dependency in dependencies(): + if dependency not in declared_dependencies: + raise ValueError("package dependency missing: " + dependency) + return {"file": path.name, "sha256": sha256(path), "bytes": path.stat().st_size, "verified": True} + + +def sign_apk(path, apk, private_key, public_key): + """Use a persistent maintainer key; never generate a new trust root per build.""" + subprocess.run(["openssl", "pkey", "-in", str(private_key), "-pubout", "-out", str(public_key)], check=True) + subprocess.run([str(apk), "adbsign", "--allow-untrusted", "--sign-key", str(private_key), str(path)], check=True) + + +def build(args): + if sys.platform != "linux": + raise ValueError("OpenWrt SDKs require Linux. Use a Linux machine or the prepared GitHub Actions workflow; no files on the router are changed.") + if not re.fullmatch(r"(?:23\.05|24\.10|25\.12)\.[0-9]+", args.release): + raise ValueError("unsupported OpenWrt release: " + args.release) + for value in (args.target, args.subtarget, args.arch): + if not re.fullmatch(r"[A-Za-z0-9_-]+", value): + raise ValueError("invalid target/architecture") + series = ".".join(args.release.split(".")[:2]) + base = DOWNLOADS + f"{args.release}/targets/{args.target}/{args.subtarget}/" + name = sdk_name(get_text(base)) + expected = checksum(get_text(base + "sha256sums"), name) + output = Path(args.output).resolve() + output.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="owrt-package-") as folder: + work = Path(folder) + archive = work / name + print("Downloading verified SDK: " + base + name, flush=True) + download(base + name, archive, expected) + subprocess.run(["tar", "-xf", str(archive), "-C", str(work)], check=True) + directories = list(work.glob("openwrt-sdk-*/")) + if len(directories) != 1: + raise ValueError("SDK archive layout changed") + sdk = directories[0] + prepare(sdk / "package" / PACKAGE) + # Use feed revisions from THIS SDK, not today's unrelated feed heads. + subprocess.run(["./scripts/feeds", "update", "-a"], cwd=sdk, check=True) + subprocess.run(["./scripts/feeds", "install", "luci-base", "rpcd-mod-file", "uhttpd", "jsonfilter", "curl", "ca-bundle", "etherwake"], cwd=sdk, check=True) + with (sdk / ".config").open("a") as config: + config.write(f"\nCONFIG_PACKAGE_{PACKAGE}=m\nCONFIG_SIGNED_PACKAGES=n\n") + subprocess.run(["make", "defconfig"], cwd=sdk, check=True) + config = (sdk / ".config").read_text() + is_apk = "CONFIG_USE_APK=y" in config + if is_apk != (series == "25.12"): + raise ValueError("SDK package format disagrees with the selected OpenWrt series") + subprocess.run(["make", f"package/{PACKAGE}/compile", "V=s", f"-j{args.jobs}"], cwd=sdk, check=True) + extension = "apk" if is_apk else "ipk" + packages = list((sdk / "bin/packages").rglob(f"{PACKAGE}*.{extension}")) + if len(packages) != 1: + raise ValueError("expected one native package, found " + str(packages)) + verified = verify_package(packages[0], sdk / "staging_dir/host/bin/apk") + version, release = versions() + dest = output / f"{PACKAGE}_{version}-r{release}_openwrt-{series}_{'noarch' if is_apk else 'all'}.{extension}" + shutil.copyfile(packages[0], dest) + signing_key = None + if is_apk: + if args.signing_key: + signing_key = Path(args.signing_key).resolve() + elif os.environ.get("OPENWRT_APK_SIGNING_KEY"): + signing_key = work / "maintainer-signing-key.pem" + signing_key.touch(mode=0o600) + signing_key.write_text(os.environ["OPENWRT_APK_SIGNING_KEY"]) + if signing_key: + sign_apk(dest, sdk / "staging_dir/host/bin/apk", signing_key, output / "owrt-remote-packages.pem") + verified = verify_package(dest, sdk / "staging_dir/host/bin/apk") + else: + print("APK is unsigned: SSH needs --allow-untrusted; LuCI requires a trusted maintainer signing key.", flush=True) + verified["file"] = dest.name + report = {"package": PACKAGE, "version": version, "package_release": release, + "openwrt": args.release, "series": series, "target": f"{args.target}/{args.subtarget}", + "architecture_test": args.arch, "sdk": {"url": base + name, "sha256": expected}, + "package_asset": verified, "signed": bool(signing_key)} + (output / f"report-{series}-{args.arch}.json").write_text(json.dumps(report, indent=2) + "\n") + (output / "SHA256SUMS").write_text(f"{verified['sha256']} {dest.name}\n") + if args.upgrade_fixture: + # A real native version upgrade is exercised in the VM. This + # fixture stays in a subdirectory and is never a release asset. + recipe = sdk / "package" / PACKAGE / "Makefile" + next_release = str(int(release) + 1) + recipe.write_text(recipe.read_text().replace("PKG_RELEASE:=" + release, "PKG_RELEASE:=" + next_release)) + subprocess.run(["make", f"package/{PACKAGE}/clean", "V=s"], cwd=sdk, check=True) + subprocess.run(["make", f"package/{PACKAGE}/compile", "V=s"], cwd=sdk, check=True) + candidate = list((sdk / "bin/packages").rglob(f"{PACKAGE}*.{extension}")) + if len(candidate) != 1: + raise ValueError("upgrade fixture was not built") + verify_package(candidate[0], sdk / "staging_dir/host/bin/apk", revision=next_release) + fixture = output / "test-fixtures" / f"{PACKAGE}_{version}-r{next_release}_openwrt-{series}.{extension}" + fixture.parent.mkdir(exist_ok=True) + shutil.copyfile(candidate[0], fixture) + if signing_key: + sign_apk(fixture, sdk / "staging_dir/host/bin/apk", signing_key, output / "owrt-remote-packages.pem") + print("OPENWRT_PACKAGE_BUILD_OK: " + str(dest), flush=True) + + +def matrix(baseline=False): + config = json.loads((ROOT / "package/openwrt-targets.json").read_text()) + index = "" if baseline else get_text(DOWNLOADS) + result = [] + for release in config["releases"]: + version = release["baseline"] if baseline else latest_release(release["series"], index) + for target in config["targets"]: + result.append({**release, **target, "version": version}) + return {"include": result} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + stage = sub.add_parser("prepare") + stage.add_argument("--output", required=True) + apk_stage = sub.add_parser("prepare-apk") + apk_stage.add_argument("--output", required=True) + targets = sub.add_parser("matrix") + targets.add_argument("--baseline", action="store_true") + check = sub.add_parser("verify") + check.add_argument("package") + check.add_argument("--apk") + for fmt in ("ipk", "apk"): + local = sub.add_parser("local-" + fmt) + local.add_argument("--output", required=True) + if fmt == "apk": + local.add_argument("--apk", default="apk") + sdk = sub.add_parser("build") + for option in ("release", "target", "subtarget", "arch", "output"): + sdk.add_argument("--" + option, required=True) + sdk.add_argument("--jobs", type=int, default=2) + sdk.add_argument("--upgrade-fixture", action="store_true") + sdk.add_argument("--signing-key", help="persistent maintainer PEM private key for APK signing") + args = parser.parse_args() + try: + if args.command == "prepare": + print(prepare(args.output)) + elif args.command == "prepare-apk": + prepare_apk(args.output) + print(Path(args.output).resolve() / "build.sh") + elif args.command == "matrix": + print(json.dumps(matrix(args.baseline), separators=(",", ":"))) + elif args.command == "verify": + print(json.dumps(verify_package(args.package, args.apk))) + elif args.command == "local-ipk": + print(pack_ipk(args.output)) + elif args.command == "local-apk": + print(pack_apk(args.output, args.apk)) + else: + build(args) + except (ValueError, OSError, subprocess.CalledProcessError) as error: + parser.exit(1, "ERROR: " + str(error) + "\n") + + +if __name__ == "__main__": + main() diff --git a/scripts/test-openwrt-package-vm.py b/scripts/test-openwrt-package-vm.py new file mode 100644 index 0000000..4155db1 --- /dev/null +++ b/scripts/test-openwrt-package-vm.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +"""Exercise package install/upgrade/removal in a disposable local OpenWrt VM.""" +import argparse +import contextlib +import functools +import http.server +import os +from pathlib import Path +import re +import shlex +import socket +import subprocess +import tempfile +import threading +import time + +import openwrt_package as package + + +def free_port(): + with socket.socket() as stream: + stream.bind(("127.0.0.1", 0)) + return stream.getsockname()[1] + + +class QuietHandler(http.server.SimpleHTTPRequestHandler): + def log_message(self, *args): + pass + + +def stop_vm(proc): + if proc is not None and proc.poll() is None: + proc.terminate() + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=10) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--release", required=True) + parser.add_argument("--package-dir", required=True) + parser.add_argument("--output", required=True) + parser.add_argument("--qemu", default="qemu-system-x86_64") + args = parser.parse_args() + if not re.fullmatch(r"(?:23\.05|24\.10|25\.12)\.[0-9]+", args.release): + parser.error("unsupported test release") + source = Path(args.package_dir).resolve() + candidates = list(source.glob("*.apk")) + list(source.glob("*.ipk")) + fixtures = list((source / "test-fixtures").glob("*.apk")) + list((source / "test-fixtures").glob("*.ipk")) + if len(candidates) != 1 or len(fixtures) != 1: + parser.error("one main package and one real upgrade fixture are required") + candidate, fixture = candidates[0], fixtures[0] + apk = candidate.suffix == ".apk" + output = Path(args.output).resolve() + output.mkdir(parents=True, exist_ok=True) + port = free_port() + ssh = ["ssh", "-p", str(port), "-o", "BatchMode=yes", "-o", "ConnectTimeout=3", + "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=" + str(output / "known_hosts"), + "root@127.0.0.1"] + handler = functools.partial(QuietHandler, directory=str(source)) + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + log = (output / "package-install.log").open("w", encoding="utf-8") + + def remote(command, timeout=180, check=True): + # The only target is the newly created loopback VM. No user router or + # configured SSH profile can be selected by this test program. + result = subprocess.run(ssh + [command], capture_output=True, timeout=timeout) + text = (result.stdout + result.stderr).decode("utf-8", "replace") + log.write(text + "\n") + log.flush() + if check and result.returncode: + raise RuntimeError(f"VM command failed (rc={result.returncode}): {command}\n{text}") + return result + + proc = None + try: + with contextlib.ExitStack() as stack: + directory = stack.enter_context(tempfile.TemporaryDirectory(prefix="owrt-native-vm-")) + work = Path(directory) + base = package.DOWNLOADS + args.release + "/targets/x86/64/" + name = "openwrt-" + args.release + "-x86-64-generic-ext4-combined.img.gz" + digest = package.checksum(package.get_text(base + "sha256sums"), name) + image = work / name + print("Downloading verified OpenWrt VM " + args.release, flush=True) + package.download(base + name, image, digest) + disk = work / "router.img" + package.unpack_image(image, disk) + flags = {"creationflags": subprocess.CREATE_NO_WINDOW} if os.name == "nt" else {} + qemu_log = stack.enter_context((output / "qemu.log").open("wb")) + proc = subprocess.Popen([args.qemu, "-display", "none", "-monitor", "none", "-serial", "file:" + str(output / "console.log"), + "-accel", "tcg", "-m", "256", "-smp", "1", "-no-reboot", + "-drive", "file=" + str(disk) + ",format=raw", + "-device", "virtio-net-pci,netdev=lan", "-netdev", + "user,id=lan,net=192.168.1.0/24,host=192.168.1.2,dhcpstart=192.168.1.100,hostfwd=tcp:127.0.0.1:" + str(port) + "-192.168.1.1:22"], + stdin=subprocess.DEVNULL, stdout=qemu_log, stderr=qemu_log, **flags) + # Stop QEMU before closing its log and deleting the disk, including + # on Windows where an open image cannot be unlinked. + stack.callback(stop_vm, proc) + print("Waiting for the local VM SSH agent", flush=True) + for attempt in range(40): + if proc.poll() is not None: + raise RuntimeError("QEMU exited before OpenWrt booted; see qemu.log") + try: + if remote("true", timeout=8, check=False).returncode == 0: + break + except subprocess.TimeoutExpired: + pass + time.sleep(2) + else: + raise RuntimeError("OpenWrt VM did not boot within the test deadline") + remote("set -e; uci set network.lan.gateway=192.168.1.2; uci set network.lan.dns=192.168.1.3; uci commit network; /etc/init.d/network reload; sleep 2") + remote("apk update" if apk else "opkg update", timeout=300) + origin = "http://192.168.1.2:" + str(server.server_port) + "/" + remote("set -e; wget -qO /tmp/remote-package" + candidate.suffix + " " + shlex.quote(origin + candidate.name) + + "; wget -qO /tmp/remote-upgrade" + candidate.suffix + " " + shlex.quote(origin + "test-fixtures/" + fixture.name)) + public_key = source / "owrt-remote-packages.pem" + if apk and public_key.is_file(): + remote("wget -qO /etc/apk/keys/owrt-remote-packages.pem " + shlex.quote(origin + public_key.name)) + install = ("apk add" if public_key.is_file() else "apk add --allow-untrusted") if apk else "opkg install" + remove = "apk del" if apk else "opkg remove" + print("Testing first install and dependency resolution", flush=True) + remote(install + " /tmp/remote-package" + candidate.suffix, timeout=300) + remote("set -e; test -s /etc/owrt-remote/web.key; test $(wc -c /etc/xray/owrt-remote-client.json; " + "printf 'preserve Xray binary\\n' >/tmp/owrt-xray/xray; chmod 755 /tmp/owrt-xray/xray; " + "sha256sum /etc/config/owrtremote /etc/owrt-remote/web.key /etc/xray/owrt-remote-client.json /tmp/owrt-xray/xray >/tmp/remote-before.sha256") + print("Testing a genuine package version upgrade", flush=True) + remote(install + " /tmp/remote-upgrade" + candidate.suffix, timeout=300) + remote("set -e; sha256sum -c /tmp/remote-before.sha256; /etc/init.d/owrt-remote-logs running") + print("Testing remove, preserved state, and migration from unowned overlay files", flush=True) + remote(remove + " " + package.PACKAGE, timeout=180) + remote("set -e; test ! -e /usr/sbin/owrt-remote; test ! -e /www/cgi-bin/owrt-remote; test ! -e /etc/init.d/owrt-remote; sha256sum -c /tmp/remote-before.sha256") + # Configuration left by install.sh is unowned by a package manager. + # A fresh package install must preserve it just like an upgrade. + remote(install + " /tmp/remote-package" + candidate.suffix, timeout=300) + remote("set -e; sha256sum -c /tmp/remote-before.sha256; test -x /usr/sbin/owrt-remote") + remote(remove + " " + package.PACKAGE, timeout=180) + (output / "result.txt").write_text("OPENWRT_NATIVE_PACKAGE_TEST_OK " + args.release + "\n") + print("OPENWRT_NATIVE_PACKAGE_TEST_OK " + args.release, flush=True) + finally: + stop_vm(proc) + server.shutdown() + server.server_close() + log.close() + + +if __name__ == "__main__": + main() diff --git a/tests/test_openwrt_package.py b/tests/test_openwrt_package.py new file mode 100644 index 0000000..d682705 --- /dev/null +++ b/tests/test_openwrt_package.py @@ -0,0 +1,177 @@ +"""Native package contents, release discovery, and safe target setup.""" +import importlib.util +import gzip +import io +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tarfile +import tempfile +import unittest +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location("openwrt_package", ROOT / "scripts/openwrt_package.py") +p = importlib.util.module_from_spec(spec) +spec.loader.exec_module(p) +SH = shutil.which("sh") or str(Path(os.environ.get("ProgramFiles", "C:/Program Files")) / "Git/usr/bin/sh.exe") + + +class PackageTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(prefix="owrt-package-tests-") + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.env = dict(os.environ) + if os.name == "nt": + self.env["PATH"] = str(Path(SH).parent) + os.pathsep + self.env["PATH"] + + def test_prepared_sdk_package_is_self_contained_and_same_overlay(self): + directory = p.prepare(self.root / "staging") + for name, (data, mode) in p.payload().items(): + path = directory / "root" / name + self.assertEqual(path.read_bytes(), data, name) + self.assertNotIn(b"\r\n", data, name) + if os.name != "nt": + self.assertEqual(path.stat().st_mode & 0o777, mode, name) + self.assertEqual((directory / "Makefile").read_bytes(), (p.RECIPE / "Makefile").read_bytes()) + self.assertFalse((directory / "root/etc/owrt-remote/web.key").exists()) + + def test_staging_never_deletes_nonempty_directory(self): + keep = self.root / "keep" + keep.write_text("do not delete") + with self.assertRaises(ValueError): + p.prepare(self.root) + self.assertEqual(keep.read_text(), "do not delete") + + def test_local_ipk_is_native_and_contains_valid_payload_and_dependencies(self): + package = p.pack_ipk(self.root / "test.ipk") + result = p.verify_package(package) + self.assertTrue(result["verified"]) + self.assertGreater(result["bytes"], 10000) + with tarfile.open(package, "r:gz") as outer: + inner = outer.extractfile("./data.tar.gz").read() + with tarfile.open(fileobj=io.BytesIO(inner), mode="r:gz") as archive: + # opkg does not create arbitrary missing parent directories when + # extracting a file; a package needs explicit directory entries. + self.assertTrue(archive.getmember("./usr/lib/owrt-remote").isdir()) + content = p.ipk_contents(package) + self.assertEqual(content["data"]["etc/config/owrtremote"][1:], (0o600, 0, 0)) + for name in p.EXECUTABLES: + self.assertEqual(content["data"][name][1], 0o755, name) + self.assertIn(b"package.sh configure", content["control"]["postinst-pkg"][0]) + + def test_package_bytes_are_reproducible(self): + self.assertEqual(p.pack_ipk(self.root / "one.ipk").read_bytes(), p.pack_ipk(self.root / "two.ipk").read_bytes()) + + def test_package_rejects_missing_files_or_changed_modes(self): + files = {name: (data, mode, 0, 0) for name, (data, mode) in p.payload().items()} + files.pop("usr/sbin/owrt-remote") + with self.assertRaisesRegex(ValueError, "missing"): + p.validate_payload(files) + files["usr/sbin/owrt-remote"] = (b"bad replacement", 0o644, 0, 0) + with self.assertRaisesRegex(ValueError, "mismatch"): + p.validate_payload(files) + + def test_source_version_matches_router_titles(self): + version, _ = p.versions() + for name in ("www/cgi-bin/owrt-remote", "www/luci-static/resources/view/owrt_remote.js"): + text = (ROOT / "files" / name).read_text(encoding="utf-8") + self.assertIn("OpenWrt Remote Hub v" + version, text) + + def test_package_scripts_defer_procd_lifecycle_to_openwrt_for_apk_and_ipk(self): + script = p.package_hook("postinst") + self.assertNotIn("configure ]", script) + self.assertIn("IPKG_INSTROOT", script) + self.assertNotIn("/etc/init.d/owrt-remote restart", script) + for script in p.native_wrappers().values(): + result = subprocess.run([SH, "-n"], input=script, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + + def target_setup(self, existing=False, rpcd_fails=False): + target = self.root / "target" + target.mkdir() + for folder in ("etc/config", "etc/owrt-remote", "etc/init.d", "tmp/luci-modulecache"): + (target / folder).mkdir(parents=True, exist_ok=True) + config = target / "etc/config/owrtremote" + config.write_text("config remote 'main'\n option router_name 'Keep name'\n option hub_token 'test-secret'\n") + key = target / "etc/owrt-remote/web.key" + if existing: + key.write_text("existing-private-key\n") + rpcd = target / "etc/init.d/rpcd" + rpcd.write_text("#!/bin/sh\nexit " + ("9" if rpcd_fails else "0") + "\n", newline="\n") + rpcd.chmod(0o755) + helper = (p.RECIPE / "root/usr/lib/owrt-remote/package.sh").read_text() + # Redirect target paths in the harness. The production hook never + # honors arbitrary ROOT/host variables supplied to a package manager. + helper = helper.replace("/etc/", target.as_posix() + "/etc/").replace("/tmp/", target.as_posix() + "/tmp/") + result = subprocess.run([SH, "-s", "--", "configure"], input=helper, capture_output=True, text=True, env=self.env) + return result, key, config + + def test_target_setup_creates_private_key_and_preserves_uci(self): + result, key, config = self.target_setup() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertRegex(key.read_text(), r"^[0-9a-f]{64}\n$") + self.assertIn("test-secret", config.read_text()) + self.assertNotIn("test-secret", result.stdout + result.stderr) + self.assertNotIn(key.read_text().strip(), result.stdout + result.stderr) + + def test_target_setup_keeps_existing_panel_key(self): + result, key, _ = self.target_setup(existing=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(key.read_text(), "existing-private-key\n") + + def test_target_setup_reports_rpcd_failure(self): + result, _, _ = self.target_setup(rpcd_fails=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn("rpcd reload failed", result.stderr) + + def test_sdk_discovery_handles_xz_and_zstd_without_guessed_filename(self): + for ext in ("xz", "zst"): + name = f"openwrt-sdk-24.10.8-x86-64_gcc-13.3.0_musl.Linux-x86_64.tar.{ext}" + self.assertEqual(p.sdk_name(f'SDK'), name) + with self.assertRaises(ValueError): + p.sdk_name('wrong') + + def test_checksum_requires_exact_sdk_and_valid_digest(self): + digest = "a" * 64 + self.assertEqual(p.checksum(digest + " *sdk.tar.xz\n", "sdk.tar.xz"), digest) + for text in ("bad *sdk.tar.xz\n", digest + " *other-sdk.tar.xz\n", ""): + with self.assertRaises(ValueError): + p.checksum(text, "sdk.tar.xz") + + def test_latest_release_is_numeric_and_ignores_prereleases(self): + html = ''.join(f'link' for v in ("24.10.9", "24.10.10", "24.10.11-rc1", "25.12.6")) + self.assertEqual(p.latest_release("24.10", html), "24.10.10") + + def test_matrix_has_all_requested_architectures_and_one_publication_per_series(self): + builds = p.matrix(baseline=True)["include"] + self.assertEqual(len(builds), 30) + self.assertEqual(sum(bool(x.get("publish")) for x in builds), 3) + self.assertEqual({x["format"] for x in builds}, {"ipk", "apk"}) + self.assertTrue({"mips64", "mipsel_24kc", "aarch64_generic", "x86_64"} <= {x["id"] for x in builds}) + + def test_corrupt_sdk_download_never_replaces_existing_cache(self): + destination = self.root / "sdk.tar.xz" + destination.write_bytes(b"previous download") + with mock.patch.object(p.urllib.request, "urlopen", side_effect=lambda *a, **k: io.BytesIO(b"corrupt")), mock.patch.object(p.time, "sleep"): + with self.assertRaisesRegex(ValueError, "SHA256 mismatch"): + p.download("https://downloads.openwrt.org/example", destination, "a" * 64) + self.assertEqual(destination.read_bytes(), b"previous download") + self.assertFalse(destination.with_name(destination.name + ".partial").exists()) + + def test_openwrt_disk_image_accepts_fwtool_footer_and_rejects_truncation(self): + source, destination = self.root / "image.gz", self.root / "image.raw" + body = b"OpenWrt disk sectors" * 10000 + source.write_bytes(gzip.compress(body) + b'# fwtool image metadata') + p.unpack_image(source, destination) + self.assertEqual(destination.read_bytes(), body) + source.write_bytes(gzip.compress(body)[:-5]) + with self.assertRaisesRegex(ValueError, "truncated"): + p.unpack_image(source, destination) + + +if __name__ == "__main__": + unittest.main()