Improve VPS terminal hub restart recovery

This commit is contained in:
kzolotarev95 committed 2026-07-28 22:27:03 +04:00
1 parent f9d1bfe658
commit a7e169dc3d
1 file changed
+20 -10
+20 -10
View File
@@ -1780,8 +1780,8 @@ def vps_terminal_commands(host):
return [
{
"title": "Обновить Hub",
"note": "Свежий hub.py из main + restart сервиса",
"command": f'v=$(date +%s); curl -fsSL -o /opt/owrt-remote/owrt-remote-hub.py "{RAW_REPO_BASE}/vps/owrt-remote-hub.py?v=$v" && chmod +x /opt/owrt-remote/owrt-remote-hub.py && systemctl restart owrt-remote && systemctl status owrt-remote --no-pager -l',
"note": "Свежий hub.py из main; обновление уходит в отдельный systemd-run",
"command": f'u=owrt-remote-selfupdate-$(date +%s); systemd-run --unit=\"$u\" --collect /bin/sh -lc \'v=$(date +%s); curl -fsSL -o /opt/owrt-remote/owrt-remote-hub.py \"{RAW_REPO_BASE}/vps/owrt-remote-hub.py?v=$v\" && chmod +x /opt/owrt-remote/owrt-remote-hub.py && systemctl restart owrt-remote\' && echo \"$u started; reopen VPS terminal in 3-5 sec\"',
},
{
"title": "Статус Hub",
@@ -4251,11 +4251,13 @@ const quickRunButtons = Array.from(document.querySelectorAll('.js-run-cmd'));
const isMobileTerminal = window.matchMedia('(max-width: 680px)').matches || /Android|iPhone|iPad|iPod|Mobile|Telegram/i.test(navigator.userAgent);
const encoder = new TextEncoder();
const decoder = new TextDecoder();
let ws, term, fitAddon, httpSid = '', httpPollTimer = 0, terminalMode = 'ws';
let ws, term, fitAddon, httpSid = '', httpPollTimer = 0, httpReconnectTimer = 0, terminalMode = 'ws';
let wsOpened = false, receivedTerminalData = false, diagnosticStarted = false, httpPollFailures = 0;
let httpPollInFlight = false, httpRecovering = false, httpRecoveryNoticeShown = false;
let inputQueue = '', inputFlushTimer = 0;
const SSH_PASSWORD_KEY = 'owrtRemote:sshPassword:' + ROUTER_ID;
let passwordPromptActive = false, passwordBuffer = '', autoPasswordUsed = false;
let pendingHubRestartAt = 0;
document.body.classList.toggle('mobile', isMobileTerminal);
function appendQuery(url, params){const sep=url.indexOf('?')===-1?'?':'&';return url+sep+new URLSearchParams(params).toString();}
function normalizePaste(text){return String(text||'').replace(/\r\n/g,'\r').replace(/\n/g,'\r');}
@@ -4268,8 +4270,13 @@ function trackPasswordInput(text){if(!passwordPromptActive)return;for(const ch o
function isEditableTarget(target){const tag=String((target&&target.tagName)||'').toLowerCase();return tag==='input'||tag==='textarea'||tag==='select'||(target&&target.isContentEditable);}
function terminalFocused(){const active=document.activeElement;return terminalEl.contains(active)||active===document.body;}
function fitTerminal(force=false){if(!term)return;if(isMobileTerminal&&document.activeElement===cmdInput&&!force)return;try{if(fitAddon)fitAddon.fit();}catch(e){}setTimeout(sendResize,40);}
function pendingHubRestartActive(){return !!pendingHubRestartAt&&Date.now()-pendingHubRestartAt<20000;}
function isHubRestartCommand(text){const value=String(text||'');if(!IS_VPS_TERMINAL)return false;return /\bsystemctl\s+restart\s+owrt-remote\b/i.test(value)||(/\bsystemd-run\b/i.test(value)&&/\bowrt-remote-selfupdate-\$\(date \+%s\)\b/i.test(value))||(/\bsystemd-run\b/i.test(value)&&/\bsystemctl\s+restart\s+owrt-remote\b/i.test(value));}
function trackRiskyTerminalCommand(text){if(!isHubRestartCommand(text))return;pendingHubRestartAt=Date.now();notice('Команда отправлена. Hub сейчас перезапустится, поэтому этот VPS terminal может оборваться штатно. Хвост после restart вроде `&& systemctl status ...` в этой же сессии уже может не успеть показаться.','33');notice('Если окно замолчит, это нормально: открой VPS terminal заново через 3-5 секунд и отдельно выполни `systemctl status owrt-remote --no-pager -l`.','36');}
function clearHttpTimers(){if(httpPollTimer){clearTimeout(httpPollTimer);httpPollTimer=0;}if(httpReconnectTimer){clearTimeout(httpReconnectTimer);httpReconnectTimer=0;}}
function scheduleHttpPoll(delay=650){if(!httpSid||httpRecovering)return;clearTimeout(httpPollTimer);httpPollTimer=setTimeout(()=>{httpPollTimer=0;pollHttpTerminal();},delay);}
function sendResize(){if(!term)return;const cols=term.cols||80,rows=term.rows||24;if(ws&&ws.readyState===WebSocket.OPEN)ws.send(JSON.stringify({type:'resize',cols,rows}));if(httpSid){fetch('/api/ssh-session/'+encodeURIComponent(httpSid)+'/resize',{method:'POST',headers:{'Content-Type':'application/x-www-form-urlencoded'},body:new URLSearchParams({cols,rows})}).catch(()=>{});}}
function sendDataNow(text){if(!text)return Promise.resolve(true);trackPasswordInput(text);if(ws&&ws.readyState===WebSocket.OPEN){ws.send(encoder.encode(text));return Promise.resolve(true);}if(terminalMode==='http'&&httpSid){return fetch('/api/ssh-session-write',{method:'POST',headers:{'Content-Type':'application/x-www-form-urlencoded'},body:new URLSearchParams({sid:httpSid,data:text})}).then(async(res)=>{let data={};try{data=await res.json();}catch(e){}if(!res.ok||!data.ok){notice('HTTP-terminal: ввод не принят: '+(data.error||res.status),'31');return false;}setTimeout(pollHttpTerminal,80);return true;}).catch(()=>{notice('Не смог отправить ввод в HTTP-terminal','31');return false;});}notice('Терминал еще подключается, повтори ввод через секунду','33');return Promise.resolve(false);}
function sendDataNow(text){if(!text)return Promise.resolve(true);trackPasswordInput(text);trackRiskyTerminalCommand(text);if(ws&&ws.readyState===WebSocket.OPEN){ws.send(encoder.encode(text));return Promise.resolve(true);}if(terminalMode==='http'&&httpSid){return fetch('/api/ssh-session-write',{method:'POST',headers:{'Content-Type':'application/x-www-form-urlencoded'},body:new URLSearchParams({sid:httpSid,data:text})}).then(async(res)=>{let data={};try{data=await res.json();}catch(e){}if(!res.ok||!data.ok){notice('HTTP-terminal: ввод не принят: '+(data.error||res.status),'31');return false;}scheduleHttpPoll(80);return true;}).catch(()=>{notice('Не смог отправить ввод в HTTP-terminal','31');return false;});}notice(httpRecovering?'HTTP-terminal переподключается после перезапуска Hub, подожди пару секунд':'Терминал еще подключается, повтори ввод через секунду','33');return Promise.resolve(false);}
function flushInputQueue(){const text=inputQueue;inputQueue='';if(inputFlushTimer){clearTimeout(inputFlushTimer);inputFlushTimer=0;}if(text)sendDataNow(text);}
function sendData(text,immediate=false){if(!text)return Promise.resolve(true);if(immediate){flushInputQueue();return sendDataNow(text);}inputQueue+=text;if(text.indexOf('\r')!==-1||text.indexOf('\n')!==-1||text.indexOf('\x03')!==-1||text.indexOf('\x04')!==-1){flushInputQueue();return Promise.resolve(true);}if(!inputFlushTimer)inputFlushTimer=setTimeout(flushInputQueue,isMobileTerminal?12:6);return Promise.resolve(true);}
async function copyText(text){const value=String(text||'');if(!value)return false;try{await navigator.clipboard.writeText(value);return true;}catch(e){const area=document.createElement('textarea');area.value=value;area.setAttribute('readonly','readonly');area.style.position='fixed';area.style.left='-1000px';document.body.appendChild(area);area.select();let ok=false;try{ok=document.execCommand('copy');}catch(err){}area.remove();return ok;}}
@@ -4277,17 +4284,20 @@ async function copySelection(){if(!term)return false;const text=term.getSelectio
function terminalBufferText(){if(!term||!term.buffer||!term.buffer.active)return '';const buffer=term.buffer.active;const lines=[];for(let i=0;i<buffer.length;i++){const line=buffer.getLine(i);if(line)lines.push(line.translateToString(true));}return lines.join('\n').replace(/\s+$/,'');}
async function copyTerminalAll(){return copyText(terminalBufferText());}
function handlePaste(ev){const text=(ev.clipboardData||window.clipboardData)?.getData('text')||'';if(!text)return;sendData(normalizePaste(text),true);ev.preventDefault();ev.stopPropagation();}
async function sendCommandInput(){const value=cmdInput.value;if(!value)return;await sendData(normalizePaste(value)+'\r',true);cmdInput.value='';cmdInput.focus();setTimeout(pollHttpTerminal,120);}
async function sendCommandInput(){const value=cmdInput.value;if(!value)return;await sendData(normalizePaste(value)+'\r',true);cmdInput.value='';cmdInput.focus();scheduleHttpPoll(120);}
async function pasteIntoInput(){cmdInput.focus();try{const text=await navigator.clipboard.readText();if(!text)return;const start=cmdInput.selectionStart??cmdInput.value.length,end=cmdInput.selectionEnd??cmdInput.value.length;cmdInput.value=cmdInput.value.slice(0,start)+text+cmdInput.value.slice(end);const pos=start+text.length;cmdInput.setSelectionRange(pos,pos);}catch(e){cmdInput.placeholder='Зажми поле и выбери Вставить';}}
async function pollHttpTerminal(){if(!httpSid)return;try{const res=await fetch('/api/ssh-session/'+encodeURIComponent(httpSid)+'/read',{cache:'no-store'});const data=await res.json();httpPollFailures=0;if(data.data){term.write(data.data);maybeAutoSendPassword(data.data);}if(data.alive)httpPollTimer=setTimeout(pollHttpTerminal,650);else httpSid='';}catch(e){httpPollFailures+=1;if(httpPollFailures===1)notice('HTTP-terminal: временно потеряна связь с Hub, пробую дальше...','33');if(httpSid)httpPollTimer=setTimeout(pollHttpTerminal,Math.min(5000,650*httpPollFailures));}}
async function startHttpTerminal(reason){if(terminalMode==='http'||httpSid)return;terminalMode='http';try{if(ws&&(ws.readyState===WebSocket.OPEN||ws.readyState===WebSocket.CONNECTING))ws.close();}catch(e){}if(reason==='direct')notice('Подключаю HTTP-terminal...','36');else notice(reason==='mobile'?'HTTP-terminal подключается...':reason+'. Включаю запасной HTTP-terminal...','36');try{const res=await fetch(appendQuery(SESSION_PATH,{cols:term.cols||80,rows:term.rows||24}),{cache:'no-store'});const data=await res.json();if(!res.ok||!data.ok){notice('HTTP-terminal не стартовал: '+(data.error||res.status),'31');return;}httpSid=data.sid;notice(READY_LABEL || (isMobileTerminal?'HTTP-terminal подключен. Вводи через поле снизу или клавиатуру.':'HTTP-terminal подключен. Кликни в терминал, Ctrl+V вставляет.'),'32');sendResize();pollHttpTerminal();}catch(e){notice('HTTP-terminal не стартовал: '+e,'31');}}
function httpRecoveryMessage(){if(pendingHubRestartActive())return 'HTTP-terminal: похоже, команда рестарта Hub ушла успешно. Текущая VPS terminal сессия оборвалась штатно; переподключаю автоматически...';return IS_VPS_TERMINAL?'HTTP-terminal: Hub перезапускается или временно недоступен. Переподключаю VPS terminal автоматически...':'HTTP-terminal: связь с Hub временно пропала. Пробую переподключить SSH автоматически...';}
function httpRecoveredMessage(){if(pendingHubRestartActive())return 'HTTP-terminal переподключен. Hub уже поднялся заново; команда, скорее всего, отработала. Проверь `systemctl status owrt-remote --no-pager -l` в новой сессии.';return IS_VPS_TERMINAL?'HTTP-terminal переподключен. Открыта новая VPS shell-сессия после перезапуска Hub.':'HTTP-terminal переподключен. Открыта новая SSH-сессия.';}
function scheduleHttpRecovery(reason){if(httpRecovering)return;httpRecovering=true;httpPollInFlight=false;clearHttpTimers();httpSid='';httpPollFailures+=1;if(!httpRecoveryNoticeShown){notice(httpRecoveryMessage(),'33');httpRecoveryNoticeShown=true;}const delay=Math.min(8000,Math.max(1200,900*httpPollFailures));httpReconnectTimer=setTimeout(()=>{httpReconnectTimer=0;startHttpTerminal('reconnect');},delay);}
async function pollHttpTerminal(){if(!httpSid||httpPollInFlight||httpRecovering)return;httpPollInFlight=true;try{const res=await fetch('/api/ssh-session/'+encodeURIComponent(httpSid)+'/read',{cache:'no-store'});let data={};try{data=await res.json();}catch(e){}if(!res.ok||!data.ok){scheduleHttpRecovery('read-failed');return;}httpPollFailures=0;httpRecoveryNoticeShown=false;if(data.data){term.write(data.data);maybeAutoSendPassword(data.data);}if(data.alive)scheduleHttpPoll(650);else{{httpSid='';notice(CLOSED_LABEL,'33');}}}catch(e){scheduleHttpRecovery('read-error');}finally{httpPollInFlight=false;}}
async function startHttpTerminal(reason){if(httpSid||httpPollInFlight)return;terminalMode='http';httpRecovering=reason==='reconnect'||reason==='read-failed'||reason==='read-error';clearHttpTimers();try{if(ws&&(ws.readyState===WebSocket.OPEN||ws.readyState===WebSocket.CONNECTING))ws.close();}catch(e){}if(reason==='direct')notice('Подключаю HTTP-terminal...','36');else if(reason==='mobile')notice('HTTP-terminal подключается...','36');else if(httpRecovering)notice(IS_VPS_TERMINAL?'Пробую поднять новую VPS terminal сессию...':'Пробую поднять новую SSH-сессию...','36');else notice(reason+'. Включаю запасной HTTP-terminal...','36');try{const res=await fetch(appendQuery(SESSION_PATH,{cols:term.cols||80,rows:term.rows||24}),{cache:'no-store'});let data={};try{data=await res.json();}catch(e){}if(!res.ok||!data.ok){httpPollFailures=Math.max(1,httpPollFailures+1);const delay=Math.min(8000,Math.max(1200,900*httpPollFailures));notice('HTTP-terminal пока не стартовал: '+(data.error||res.status||'hub недоступен')+'. Повторю автоматически...','33');httpReconnectTimer=setTimeout(()=>{httpReconnectTimer=0;startHttpTerminal('reconnect');},delay);return;}httpSid=data.sid;const wasRecovering=httpRecovering;httpRecovering=false;httpPollFailures=0;if(wasRecovering&&httpRecoveryNoticeShown)notice(httpRecoveredMessage(),'32');else notice(READY_LABEL || (isMobileTerminal?'HTTP-terminal подключен. Вводи через поле снизу или клавиатуру.':'HTTP-terminal подключен. Кликни в терминал, Ctrl+V вставляет.'),'32');httpRecoveryNoticeShown=false;if(pendingHubRestartActive())pendingHubRestartAt=0;sendResize();scheduleHttpPoll(80);}catch(e){httpPollFailures=Math.max(1,httpPollFailures+1);const delay=Math.min(8000,Math.max(1200,900*httpPollFailures));notice('HTTP-terminal не стартовал: '+e+'. Повторю автоматически...','31');httpReconnectTimer=setTimeout(()=>{httpReconnectTimer=0;startHttpTerminal('reconnect');},delay);}}
async function explainTerminalError(source){if(diagnosticStarted)return;diagnosticStarted=true;try{const res=await fetch(CHECK_PATH,{cache:'no-store'});const data=await res.json();if(data.tcp_ok)await startHttpTerminal(source);else{notice('SSH-туннель на VPS не отвечает: '+(data.error||'порт закрыт'),'31');notice('В Hub нажми: Обновить Xray CFG, потом Рестарт Xray VPS, и проверь heartbeat роутера.','33');}}catch(e){notice('Не смог проверить SSH-туннель. Проверь firewall VPS и доступ к Hub.','31');}}
function connect(){wsOpened=false;receivedTerminalData=false;diagnosticStarted=false;httpPollFailures=0;terminalMode='ws';httpSid='';inputQueue='';passwordPromptActive=false;passwordBuffer='';autoPasswordUsed=false;if(inputFlushTimer){clearTimeout(inputFlushTimer);inputFlushTimer=0;}if(term){term.reset();notice('Подключение к '+CONNECT_LABEL+'...','36');}if(FORCE_HTTP_ONLY||isMobileTerminal){startHttpTerminal(FORCE_HTTP_ONLY?'direct':'mobile');return;}const proto=location.protocol==='https:'?'wss://':'ws://';ws=new WebSocket(proto+location.host+WS_PATH);ws.binaryType='arraybuffer';ws.onopen=()=>{wsOpened=true;sendResize();};ws.onmessage=async(ev)=>{if(terminalMode==='http')return;let text='';if(typeof ev.data==='string')text=ev.data;else if(ev.data instanceof Blob)text=await ev.data.text();else text=decoder.decode(ev.data);if(!receivedTerminalData)term.clear();receivedTerminalData=true;term.write(text);maybeAutoSendPassword(text);};ws.onerror=()=>explainTerminalError('ошибка web-terminal');ws.onclose=()=>{if(terminalMode==='http')return;if(wsOpened)notice(CLOSED_LABEL,'33');else explainTerminalError(CLOSED_LABEL);};setTimeout(()=>{if(!receivedTerminalData&&!httpSid)startHttpTerminal(SILENT_LABEL);},3000);}
function connect(){wsOpened=false;receivedTerminalData=false;diagnosticStarted=false;httpPollFailures=0;httpPollInFlight=false;httpRecovering=false;httpRecoveryNoticeShown=false;terminalMode='ws';httpSid='';clearHttpTimers();inputQueue='';passwordPromptActive=false;passwordBuffer='';autoPasswordUsed=false;if(inputFlushTimer){clearTimeout(inputFlushTimer);inputFlushTimer=0;}if(term){term.reset();notice('Подключение к '+CONNECT_LABEL+'...','36');}if(FORCE_HTTP_ONLY||isMobileTerminal){startHttpTerminal(FORCE_HTTP_ONLY?'direct':'mobile');return;}const proto=location.protocol==='https:'?'wss://':'ws://';ws=new WebSocket(proto+location.host+WS_PATH);ws.binaryType='arraybuffer';ws.onopen=()=>{wsOpened=true;sendResize();};ws.onmessage=async(ev)=>{if(terminalMode==='http')return;let text='';if(typeof ev.data==='string')text=ev.data;else if(ev.data instanceof Blob)text=await ev.data.text();else text=decoder.decode(ev.data);if(!receivedTerminalData)term.clear();receivedTerminalData=true;term.write(text);maybeAutoSendPassword(text);};ws.onerror=()=>explainTerminalError('ошибка web-terminal');ws.onclose=()=>{if(terminalMode==='http')return;if(wsOpened)notice(CLOSED_LABEL,'33');else explainTerminalError(CLOSED_LABEL);};setTimeout(()=>{if(!receivedTerminalData&&!httpSid)startHttpTerminal(SILENT_LABEL);},3000);}
function initTerminal(){if(!window.Terminal){terminalEl.classList.remove('loading');terminalEl.textContent='xterm.js не загрузился. Проверь доступ браузера к cdn.jsdelivr.net.';return;}terminalEl.classList.remove('loading');terminalEl.textContent='';term=new Terminal({cursorBlink:!isMobileTerminal,convertEol:false,scrollback:isMobileTerminal?200:5000,scrollSensitivity:isMobileTerminal?8:1,fastScrollSensitivity:isMobileTerminal?14:5,smoothScrollDuration:0,fontFamily:'"Cascadia Mono","Consolas","Liberation Mono",monospace',fontSize:isMobileTerminal?12:14,lineHeight:1.14,theme:{background:'#0b0714',foreground:'#f7f2ff',cursor:'#fbbf24',selectionBackground:'#334155',black:'#0b0714',red:'#fb7185',green:'#86efac',yellow:'#fde68a',blue:'#93c5fd',magenta:'#c084fc',cyan:'#67e8f9',white:'#f7f2ff'}});if(window.FitAddon&&FitAddon.FitAddon){fitAddon=new FitAddon.FitAddon();term.loadAddon(fitAddon);}term.open(terminalEl);term.onData(sendData);term.onResize(sendResize);term.attachCustomKeyEventHandler((ev)=>{const key=String(ev.key||'').toLowerCase();if((ev.ctrlKey||ev.metaKey)&&key==='c'&&term.hasSelection&&term.hasSelection()){copySelection();return false;}return true;});terminalEl.addEventListener('click',()=>term.focus());fitTerminal(true);connect();setTimeout(()=>{fitTerminal(true);term.focus();},120);}
document.addEventListener('paste',(ev)=>{if(isEditableTarget(ev.target))return;if(!terminalFocused())return;handlePaste(ev);});
window.addEventListener('beforeunload',()=>{if(httpSid)navigator.sendBeacon('/api/ssh-session/'+encodeURIComponent(httpSid)+'/close');});
window.addEventListener('beforeunload',()=>{clearHttpTimers();if(httpSid)navigator.sendBeacon('/api/ssh-session/'+encodeURIComponent(httpSid)+'/close');});
let resizeTimer=0;window.addEventListener('resize',()=>{clearTimeout(resizeTimer);resizeTimer=setTimeout(()=>fitTerminal(false),160);});
copyBtn.addEventListener('click',copyTerminalAll);clearBtn.addEventListener('click',()=>term&&term.clear());reconnectBtn.addEventListener('click',()=>{try{if(ws)ws.close();}catch(e){}if(httpSid)navigator.sendBeacon('/api/ssh-session/'+encodeURIComponent(httpSid)+'/close');httpSid='';connect();});
copyBtn.addEventListener('click',copyTerminalAll);clearBtn.addEventListener('click',()=>term&&term.clear());reconnectBtn.addEventListener('click',()=>{clearHttpTimers();try{if(ws)ws.close();}catch(e){}if(httpSid)navigator.sendBeacon('/api/ssh-session/'+encodeURIComponent(httpSid)+'/close');httpSid='';connect();});
cmdSend.addEventListener('click',sendCommandInput);cmdInput.addEventListener('keydown',(ev)=>{if(ev.key==='Enter'&&(ev.ctrlKey||ev.metaKey)){sendCommandInput();ev.preventDefault();}});
quickCopyButtons.forEach((btn)=>btn.addEventListener('click',()=>copyQuickCommand(btn)));
quickRunButtons.forEach((btn)=>btn.addEventListener('click',()=>runQuickCommand(btn)));