#!/usr/bin/env bash
# OpenVTS Linux all-in-one installer
# Target: Ubuntu Server 22.04 / 24.04 / 26.04 (amd64 or arm64)

set -Eeuo pipefail
IFS=$'\n\t'
umask 027

INSTALLER_VERSION="2.4.4"
INSTALLER_BUILD_ID="openvts-linux-installer-2.4.4-20261004-report-contract"
DEFAULT_APP_DIR="/opt/openvts"
DEFAULT_NODE_VERSION="24.18.0"
DEFAULT_PG_MAJOR="18"
DEFAULT_PG_PORT="5432"
MIN_TIMESCALE_VERSION="2.18.0"
DEFAULT_SOURCE_URL="https://openvts.io/downloads/openvts-linux.zip"
# Application releases are plain deployment ZIPs; no external hash or manifest is required.
MIN_PHYSICAL_RAM_MB=3500
TARGET_INSTALL_MEMORY_MB=8192
BASE_REQUIRED_FREE_MB=8192
MANAGED_SWAP_FILE="/swapfile-openvts"
APT_LOCK_TIMEOUT=600

SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]:-$0}")" 2>/dev/null && pwd -P || pwd -P)"
COMMAND="install"
APP_DIR="$DEFAULT_APP_DIR"
NODE_VERSION="$DEFAULT_NODE_VERSION"
REQUESTED_PG_MAJOR="$DEFAULT_PG_MAJOR"
PG_PORT="$DEFAULT_PG_PORT"
REDIS_PORT=""
REDIS_PORT_WAS_SET=0
APP_DIR_WAS_SET=0
DOMAIN_WAS_SET=0
EMAIL_WAS_SET=0
NODE_VERSION_WAS_SET=0
PG_MAJOR_WAS_SET=0
SOURCE_DIR=""
SOURCE_ARCHIVE=""
SOURCE_URL="$DEFAULT_SOURCE_URL"
CHECK_BUNDLE_ONLY=0
SOURCE_URL_WAS_SET=0
DOMAIN="_"
EMAIL=""
HEALTH_TIMEOUT=900
ADOPT_EXISTING_DATABASES=0
SKIP_FIREWALL=0
AUTO_SWAP=1
WORK_DIR=""
LOG_FILE=""
PREVIOUS_APP=""
ACTIVE_RELEASE=""
UPGRADE_WRITERS_STOPPED=0
SSL_INFRA_PAUSED=0
LEGACY_SSL_MANAGER_WAS_ACTIVE=0
LEGACY_SSL_MANAGER_RETIRED=0
BUNDLE_MODE=""
NPM_CACHE_ROOT="/var/cache/openvts"
NPM_CACHE_DIR="${NPM_CACHE_ROOT}/npm"
RUNTIME_NPM_CACHE_DIR="${NPM_CACHE_ROOT}/runtime-npm"

usage() {
  cat <<EOF
OpenVTS Linux Installer ${INSTALLER_VERSION}

One-command installation (downloads the official deployment bundle):
  curl -fsSL https://openvts.io/downloads/openvts-install-linux.sh | sudo bash

Install with a domain and automatic HTTPS:
  curl -fsSL https://openvts.io/downloads/openvts-install-linux.sh | \\
    sudo bash -s -- --domain vts.example.com --email admin@example.com

Install from a local deployment ZIP (no companion files required):
  sudo bash openvts-install-linux.sh --source-archive ./openvts-linux.zip

Application releases: upload only openvts-linux.zip. No .sha256 or
release manifest is downloaded or checked. Compiled application files are required.

Daily commands after installation:
  sudo openvts status
  sudo openvts doctor
  sudo openvts start
  sudo openvts stop
  sudo openvts restart
  sudo openvts logs [frontend|backend|listener|redis|nginx|manager|ssl|all]

Options:
  --source-url URL        Deployment ZIP URL (default: official OpenVTS URL)
  --source-archive PATH   Local prebuilt OpenVTS deployment ZIP
  --source-dir PATH       Extracted prebuilt deployment directory
  --check-bundle         Validate/prepare only; no root, APT, database or service changes
  --app-dir PATH          Install root under /opt or /srv (default: /opt/openvts)
  --domain NAME           DNS name for Nginx
  --email ADDRESS         Certbot email; enables automatic HTTPS with --domain
  --node-version VERSION  Exact official Node.js version (default: 24.18.0)
  --postgres-major N      PostgreSQL 15, 16, 17, or 18 (default: 18)
  --redis-port PORT       Dedicated OpenVTS Redis port (default: automatic)
  --health-timeout SEC    First-start health timeout (default: 900)
  --adopt-existing-databases
                         Reuse existing OpenVTS databases only after backup/review
  --skip-firewall        Do not enable/manage Ubuntu UFW
  --no-auto-swap         Do not create managed swap on 4-6 GB servers
  -h, --help              Show this help

Supported platform: clean Ubuntu Server 22.04, 24.04, or 26.04 LTS with systemd,
on amd64 or arm64, on AWS, DigitalOcean, GCP, Azure, other clouds, or on-premises.
Minimum supported size: 4 GB RAM. Use a 20 GB or larger system disk; a 4 GB
server needs about 13 GB free because the installer creates managed swap.
The installer never performs a PostgreSQL major upgrade and never deletes data.
Cloud-provider firewalls/security groups remain outside the VM and need matching inbound rules.
EOF
}

log()  { printf '\n[%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*"; }
info() { printf '  • %s\n' "$*"; }
warn() { printf '  WARNING: %s\n' "$*" >&2; }
die()  { printf '\nERROR: %s\n' "$*" >&2; exit 1; }

on_error() {
  local line="$1"
  printf '\nInstallation failed (line %s). Log: %s\n' "$line" "${LOG_FILE:-not available yet}" >&2
}
trap 'on_error "$LINENO"' ERR

cleanup() {
  local exit_status=$?
  if (( SSL_INFRA_PAUSED == 1 )); then
    systemctl start certbot.timer openvts-service-manager.service >/dev/null 2>&1 || true
    if (( LEGACY_SSL_MANAGER_WAS_ACTIVE == 1 && LEGACY_SSL_MANAGER_RETIRED == 0 )); then
      systemctl start openvts-ssl-manager.service >/dev/null 2>&1 || true
    fi
  fi
  if (( exit_status != 0 && UPGRADE_WRITERS_STOPPED == 1 )) && [[ "${ACTIVE_RELEASE:-}" != "${RELEASE_APP:-}" ]]; then
    warn "Upgrade stopped before activation; attempting to restart the previous application stack."
    systemctl start openvts.target || true
  fi
  if [[ -n "${RELEASE_ROOT:-}" && -d "${RELEASE_ROOT:-}" && "${ACTIVE_RELEASE:-}" != "${RELEASE_APP:-}" ]]; then
    rm -rf -- "$RELEASE_ROOT"
  fi
  if [[ -n "$WORK_DIR" && "$WORK_DIR" == /tmp/openvts-installer.* && -d "$WORK_DIR" ]]; then
    rm -rf -- "$WORK_DIR"
  fi
}
trap cleanup EXIT

write_file() {
  local path="$1" mode="$2" owner="$3" group="$4" tmp
  tmp="$(mktemp)"
  cat >"$tmp"
  install -D -o "$owner" -g "$group" -m "$mode" "$tmp" "$path"
  rm -f -- "$tmp"
}

command_exists() { command -v "$1" >/dev/null 2>&1; }

version_ge() {
  [[ "$(printf '%s\n%s\n' "$2" "$1" | sort -V | head -n1)" == "$2" ]]
}

port_is_listening() {
  local port="$1"
  ss -H -ltn "sport = :$port" 2>/dev/null | grep -q .
}

validate_port() {
  local name="$1" value="$2"
  [[ "$value" =~ ^[0-9]+$ ]] || die "$name must be numeric."
  (( value >= 1 && value <= 65535 )) || die "$name must be between 1 and 65535."
}

curl_download() {
  local url="$1" destination="$2" max_time="${3:-600}"
  local -a cache_headers=()
  if [[ "${4:-}" == revalidate ]]; then
    cache_headers=(--header "Cache-Control: no-cache" --header "Pragma: no-cache")
  fi
  curl --proto =https --proto-redir =https --retry-max-time 1200 --fail --silent --show-error --location \
    --retry 5 --retry-all-errors --retry-delay 2 \
    --connect-timeout 20 --max-time "$max_time" \
    "${cache_headers[@]}" --url "$url" --output "$destination"
}

# This trusted packager is embedded to keep curl|bash installation self-contained.
# Never execute a helper/script supplied by the downloaded ZIP as root.
bundle_tool() {
  python3 - "$@" <<'OPENVTS_RELEASE_PY'
#!/usr/bin/env python3
"""Open VTS deployment packager/validator. Python 3.10+, standard library only.

Works on a PREBUILT tree or ZIP, never compiles application code, never executes
bundle code. The same implementation is embedded in the standalone installer.
"""
from __future__ import annotations
import argparse
import json
import os
from pathlib import Path, PurePosixPath
import re
import shutil
import stat
import sys
import tempfile
import zipfile

VERSION = "2.4.4"
LEGACY_METADATA = "openvts-release.json"  # Ignored; never used to accept/reject a release.
MAX_BYTES = 4 * 1024**3
MAX_FILE_BYTES = 512 * 1024**2
MAX_ENTRIES = 150000
PORT_RE = re.compile(r"^\s*(TCP_PROTOCOL_[A-Z0-9_]+_PORT)\s*=\s*([0-9]+)\s*(?:#.*)?$")
IGNORE_DIRS = {"node_modules", ".git", ".cache", "coverage", "__pycache__"}
MUTABLE_DIRS = {"data", "uploads", "logs", "winston_logs", "api_logs", "runtime-data", "generated"}

class ReleaseError(Exception):
    pass

def read_json(path: Path):
    try:
        return json.loads(path.read_text(encoding="utf-8-sig"))
    except (OSError, ValueError) as exc:
        raise ReleaseError(f"Invalid/missing JSON {path}: {exc}") from exc

def safe_extract(archive: Path, destination: Path) -> None:
    """Validate every member before writing any. No links, devices or traversal."""
    with zipfile.ZipFile(archive) as zf:
        members = zf.infolist()
        if len(members) > MAX_ENTRIES or sum(i.file_size for i in members) > MAX_BYTES:
            raise ReleaseError("Archive exceeds deployment size/entry limits")
        names = set()
        for item in members:
            name = item.filename
            parts = name.rstrip("/").split("/")
            mode = item.external_attr >> 16
            kind = stat.S_IFMT(mode)
            if (not name or "\\" in name or "\x00" in name or name.startswith("/")
                or any(p in ("", ".", "..") for p in parts)
                or re.match(r"^[A-Za-z]:", name)):
                raise ReleaseError(f"Unsafe ZIP member: {name!r}")
            normalized = "/".join(parts)
            if normalized in names:
                raise ReleaseError(f"Duplicate ZIP member: {name!r}")
            names.add(normalized)
            if kind not in (0, stat.S_IFREG, stat.S_IFDIR):
                raise ReleaseError(f"ZIP links/special files are not allowed: {name!r}")
            if item.flag_bits & 1 or item.file_size > MAX_FILE_BYTES:
                raise ReleaseError(f"Encrypted/oversized ZIP member: {name!r}")
        # zipfile verifies each file's CRC while extracting.
        zf.extractall(destination)

def find_root(base: Path) -> Path:
    roots = []
    for folder, dirs, files in os.walk(base, followlinks=False):
        dirs[:] = [d for d in dirs if d not in IGNORE_DIRS and not (Path(folder)/d).is_symlink()]
        p = Path(folder)
        if len(p.relative_to(base).parts) > 7:
            dirs[:] = []
            continue
        if ((p/"backend/package.json").is_file() and
            (p/"listener/package.json").is_file() and
            any((p/f/"package.json").is_file() for f in ("frontend", "Frontend"))):
            roots.append(p)
    if len(roots) != 1:
        raise ReleaseError(f"Expected one Open VTS root, found {len(roots)} (need backend, listener, frontend)")
    return roots[0]

def frontend_dir(root: Path) -> Path:
    choices = [root/n for n in ("frontend", "Frontend") if (root/n).is_dir()]
    if len(choices) != 1:
        raise ReleaseError("Use exactly one frontend directory; do not package both frontend and Frontend")
    return choices[0]

def protocol_ports(root: Path) -> dict[str, int]:
    template = next((root/"listener"/f for f in (".env.example", ".env") if (root/"listener"/f).is_file()), None)
    if template is None:
        raise ReleaseError("Missing listener/.env.example protocol port template")
    ports = {}
    for line in template.read_text(encoding="utf-8-sig").splitlines():
        m = PORT_RE.fullmatch(line)
        if m:
            key, number = m[1], int(m[2])
            if key in ports or number in ports.values() or not 1024 <= number <= 65535:
                raise ReleaseError(f"Duplicate/out-of-range listener protocol port: {key}")
            if number in (3000, 4000, 5055, 5432, 6379, 6380):
                raise ReleaseError(f"Listener protocol port conflicts with a service: {key}")
            ports[key] = number
        elif line.lstrip().startswith("TCP_PROTOCOL_"):
            raise ReleaseError(f"Malformed protocol assignment: {line.split('=', 1)[0]}")
    if not ports:
        raise ReleaseError("No listener TCP protocol ports configured")
    return ports

def port_ranges(ports) -> list[str]:
    values = sorted(set(ports))
    ranges = []
    if not values:
        return ranges
    start = last = values[0]
    for value in values[1:] + [None]:
        if value is not None and value == last + 1:
            last = value
        else:
            ranges.append(str(start) if start == last else f"{start}:{last}")
            start = last = value
    return ranges

def write_changed(path: Path, text: str, changes: list[str], description: str) -> None:
    if path.read_text(encoding="utf-8") == text:
        return
    # Do not leave a stale source map describing different generated code.
    text = re.sub(r"\n?//# sourceMappingURL=.*?(?:\n|$)", "\n", text)
    path.write_text(text, encoding="utf-8", newline="\n")
    Path(str(path) + ".map").unlink(missing_ok=True)
    changes.append(description)

def patch_http_bind(path: Path, env_key: str, changes: list[str]) -> None:
    text = path.read_text(encoding="utf-8")
    old = re.compile(r"(await\s+(?:app|appInstance)\.listen\(\s*port\s*,\s*)(['\"])0\.0\.0\.0\2(\s*\))")
    text, count = old.subn(lambda m: m[1]+f'process.env.{env_key} || "127.0.0.1"'+m[3], text)
    verified = re.findall(r"await\s+(?:app|appInstance)\.listen\(\s*port\s*,\s*process\.env\." + env_key + r"\s*\|\|\s*['\"]127\.0\.0\.1['\"]\s*\)", text)
    if not verified:
        raise ReleaseError(f"Unsupported HTTP bootstrap layout: {path.name}; source must honor {env_key}")
    if old.search(text):
        raise ReleaseError(f"Unpatched wildcard HTTP binding in {path}")
    if count:
        write_changed(path, text, changes, f"{path.parent.parent.name}/{path.name}: {count} HTTP binding(s) honor {env_key}")

def apply_compatibility(root: Path) -> list[str]:
    """Narrow, idempotent compatibility adapter; no broad JS reformat/minify."""
    changes = []
    patch_http_bind(root/"backend/dist/main.js", "BIND_HOST", changes)
    listener_main = root/"listener/dist/main.js"
    # September's compile-cache launcher delegates HTTP startup to runtime-main.
    entry = root/"listener/dist/runtime-main.js" if re.search(r"import\(['\"]\./runtime-main\.js['\"]\)", listener_main.read_text()) else listener_main
    if not entry.is_file():
        raise ReleaseError("Listener's delegated runtime-main.js is missing")
    patch_http_bind(entry, "API_HOST", changes)
    p = root/"backend/dist/database/database-initializer.service.js"
    text = p.read_text(encoding="utf-8")
    old_calls = re.compile(r"execFileAsync\(\s*(['\"])psql\1")
    text, n = old_calls.subn('execFileAsync(process.env.OPENVTS_PSQL_BIN || "psql"', text)
    if not re.search(r"execFileAsync\(\s*process\.env\.OPENVTS_PSQL_BIN\s*\|\|\s*['\"]psql['\"]", text):
        raise ReleaseError("Unrecognized PostgreSQL initializer: OPENVTS_PSQL_BIN adapter needs review")
    guard = re.compile(r"return\s+err\?\.code\s*===\s*(['\"])ENOENT\1\s*\|\|\s*message\.includes\(\s*(['\"])spawn psql ENOENT\2\s*\)\s*;")
    text, guards = guard.subn('return err?.code === "ENOENT" || err?.code === "EACCES" || message.includes("spawn psql ENOENT") || message.includes("spawn psql EACCES");', text)
    if not guards and not re.search(r"err\?\.code\s*===\s*['\"]EACCES['\"]", text):
        raise ReleaseError("Unrecognized psql unavailable-error guard; review source before publishing")
    if n or guards:
        write_changed(p, text, changes, f"backend/database-initializer: {n} explicit psql path(s), {guards} EACCES guard(s)")
    f = root/"frontend/server.js"
    text = f.read_text(encoding="utf-8")
    marker = "process.env.__NEXT_PRIVATE_STANDALONE_CONFIG = JSON.stringify(nextConfig)"
    addition = ("// Open VTS: relocate build-machine tracing roots for Linux deployment.\n"
                "nextConfig.outputFileTracingRoot = __dirname;\n"
                "if (nextConfig.turbopack) nextConfig.turbopack.root = __dirname;\n")
    if marker not in text:
        raise ReleaseError("Unsupported Next standalone launcher; expected serialized nextConfig")
    if addition not in text:
        write_changed(f, text.replace(marker, addition+marker, 1), changes, "frontend/server.js: runtime-relative tracing roots")
    f = root/"frontend/.next/required-server-files.json"
    required = read_json(f)
    normalized = [name.replace("\\", "/") for name in required.get("files", [])]
    if normalized != required.get("files", []):
        required["files"] = normalized
        f.write_text(json.dumps(required, ensure_ascii=False, separators=(",", ":")), encoding="utf-8")
        changes.append("frontend required-server-files: normalize Windows path separators")
    return changes

def validate(root: Path) -> dict:
    frontend = frontend_dir(root)
    mandatory = [frontend/"server.js", frontend/".next/BUILD_ID", frontend/".next/required-server-files.json",
                 root/"backend/dist/main.js", root/"listener/dist/main.js",
                 root/"backend/dist/database/database-initializer.service.js", root/"backend/prisma/seed.js",
                 root/"backend/dist/user/reports/linux-report-time-zone.js",
                 root/"backend/dist/user/reports/linux-report-preflight.js"]
    versions = {}
    for component in ("backend", "listener", "frontend"):
        p = frontend if component == "frontend" else root/component
        mandatory += [p/"package.json", p/"package-lock.json"]
        package, lock = read_json(p/"package.json"), read_json(p/"package-lock.json")
        if lock.get("lockfileVersion") not in (2, 3):
            raise ReleaseError(f"{component}: require npm package-lock v2/v3")
        locked = lock.get("packages", {}).get("")
        if locked is None:
            raise ReleaseError(f"{component}: missing root package-lock entry")
        for group in ("dependencies", "devDependencies", "optionalDependencies"):
            if package.get(group, {}) != locked.get(group, {}):
                raise ReleaseError(f"{component}: package.json / package-lock.json {group} mismatch; rebuild with npm ci")
        deps = package.get("dependencies", {})
        if component == "backend" and "prisma" not in deps:
            raise ReleaseError("Backend requires the Prisma CLI at runtime for schema bootstrap")
        if component in ("backend", "listener"):
            schemas = ["primary", "logs"] + (["address"] if component == "backend" else [])
            mandatory += [p/"prisma"/(s+".prisma") for s in schemas]
            versions[component+"Prisma"] = lock.get("packages", {}).get("node_modules/@prisma/client", {}).get("version")
        else:
            versions["next"] = lock.get("packages", {}).get("node_modules/next", {}).get("version")
    for p in mandatory:
        if not p.is_file() or not p.stat().st_size:
            raise ReleaseError(f"Required deployment artifact missing/empty: {p.relative_to(root)}")
    required = read_json(frontend/".next/required-server-files.json")
    for name in required.get("files", []):
        path = PurePosixPath(name.replace("\\", "/"))
        if path.is_absolute() or ".." in path.parts or not (frontend/str(path)).is_file():
            raise ReleaseError(f"Next required runtime file missing/unsafe: {name}")
    if not (frontend/"public").is_dir() or not any((frontend/".next/static").rglob("*.js")):
        raise ReleaseError("Frontend requires public assets and compiled .next/static JavaScript")
    ports = protocol_ports(root)
    # Validate actual deployment files, not release-specific metadata or file hashes.
    # Legacy openvts-release.json files may be stale and are deliberately ignored.
    return {"status": "PASS", "bundleFormat": 1, "nextBuildId": (frontend/".next/BUILD_ID").read_text().strip(),
            "versions": versions, "listenerPortCount": len(ports),
            "listenerPorts": ports, "firewallRules": port_ranges(ports.values()),
            "fileCount": sum(p.is_file() for p in root.rglob("*"))}

def copy_component(source: Path, dest: Path, frontend: bool = False) -> None:
    for folder, dirs, files in os.walk(source, followlinks=False):
        here = Path(folder)
        relative = here.relative_to(source)
        def allowed_dir(name):
            if name in IGNORE_DIRS:
                return False
            if (here/name).is_symlink():
                raise ReleaseError(f"Source directory symlinks are not supported: {here/name}")
            if relative == Path(".") and name in MUTABLE_DIRS | {"src", "test", "tests", "e2e", ".github"}:
                return False
            if frontend and relative == Path(".next") and name in {"standalone", "cache", "types", "diagnostics"}:
                return False
            return True
        dirs[:] = [d for d in dirs if allowed_dir(d)]
        for filename in files:
            p = here/filename
            if p.is_symlink():
                raise ReleaseError(f"Source symlinks are not supported: {p}")
            if (filename.startswith(".env") or filename in {LEGACY_METADATA, ".DS_Store"} or
                filename.endswith((".log", ".tsbuildinfo"))):
                continue
            if filename == ".npmrc":
                # Preserve only lock-affecting non-secret settings; never registry tokens.
                for line in p.read_text().splitlines():
                    if line.strip() and not line.lstrip().startswith(("#", ";")) and not re.fullmatch(r"(?:legacy-peer-deps|install-links|engine-strict)\s*=\s*(?:true|false)\s*", line):
                        raise ReleaseError(".npmrc contains unsupported/secret settings; supply a clean release .npmrc")
            target = dest/relative/filename
            target.parent.mkdir(parents=True, exist_ok=True)
            shutil.copyfile(p, target)
            target.chmod(0o644)

def prepare(source: Path, dest: Path) -> dict:
    if dest.exists():
        raise ReleaseError(f"Destination must not exist: {dest}")
    with tempfile.TemporaryDirectory(prefix="openvts-unpack-") as tmp:
        search = source
        if source.is_file():
            search = Path(tmp)
            safe_extract(source, search)
        if not search.is_dir():
            raise ReleaseError(f"Source not found: {source}")
        root = find_root(search)
        if root == dest or root in dest.parents:
            raise ReleaseError("Prepared output must be outside the source tree")
        ports = protocol_ports(root)
        frontend = frontend_dir(root)
        dest.mkdir(parents=True, exist_ok=False)
        try:
            for component in ("backend", "listener"):
                copy_component(root/component, dest/component)
            if (frontend/"server.js").exists():
                copy_component(frontend, dest/"frontend", frontend=True)
            else:
                # Source checkout after a completed next build (non-monorepo layout).
                standalone = frontend/".next/standalone"
                if not (standalone/"server.js").exists():
                    raise ReleaseError("No frontend standalone build: build with output='standalone' first")
                copy_component(standalone, dest/"frontend", frontend=True)
                for folder in ("public", ".next/static"):
                    if not (frontend/folder).is_dir():
                        raise ReleaseError(f"Missing frontend/{folder}")
                    copy_component(frontend/folder, dest/"frontend"/folder)
                for filename in ("package.json", "package-lock.json"):
                    shutil.copyfile(frontend/filename, dest/"frontend"/filename)
            template = "# Open VTS protocol ports only. Host secrets are generated by the installer.\n"
            template += "".join(f"{key}={value}\n" for key, value in ports.items())
            (dest/"listener/.env.example").write_text(template, encoding="utf-8")
            changes = apply_compatibility(dest)
            result = validate(dest)
            result["compatibilityChanges"] = changes
            return result
        except Exception:
            shutil.rmtree(dest, ignore_errors=True)
            raise

def pack(source: Path, output: Path) -> dict:
    output.parent.mkdir(parents=True, exist_ok=True)
    with tempfile.TemporaryDirectory(prefix="openvts-pack-") as temp:
        root = Path(temp)/"openvts-linux"
        result = prepare(source, root)
        staging = output.with_name(output.name+f".tmp.{os.getpid()}")
        try:
            with zipfile.ZipFile(staging, "w", zipfile.ZIP_DEFLATED, compresslevel=6) as zf:
                for p in sorted(root.rglob("*")):
                    if p.is_file():
                        zi = zipfile.ZipInfo("openvts-linux/"+p.relative_to(root).as_posix(), (2026,1,1,0,0,0))
                        zi.create_system = 3
                        zi.external_attr = (stat.S_IFREG | 0o644) << 16
                        zi.compress_type = zipfile.ZIP_DEFLATED
                        zf.writestr(zi, p.read_bytes())
            os.replace(staging, output)
        finally:
            staging.unlink(missing_ok=True)
    result.update({"zip": str(output)})
    return result

def main(argv=None) -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    sub = parser.add_subparsers(dest="command", required=True)
    for cmd in ("pack", "prepare"):
        p = sub.add_parser(cmd)
        p.add_argument("--source", required=True, type=Path, help="Prebuilt ZIP or directory; never unbuilt source")
        p.add_argument("--output", required=True, type=Path)
    p = sub.add_parser("verify")
    p.add_argument("--source", required=True, type=Path)
    p = sub.add_parser("ports")
    p.add_argument("--source", required=True, type=Path)
    args = parser.parse_args(argv)
    try:
        if args.command == "pack":
            result = pack(args.source.resolve(), args.output.resolve())
        elif args.command == "prepare":
            result = prepare(args.source.resolve(), args.output.resolve())
        elif args.command == "ports":
            print(",".join(port_ranges(protocol_ports(find_root(args.source.resolve())).values())))
            return 0
        else:
            with tempfile.TemporaryDirectory(prefix="openvts-verify-") as tmp:
                search = args.source
                if search.is_file():
                    search = Path(tmp)
                    safe_extract(args.source, search)
                result = validate(find_root(search))
        # The protocol template is the source of truth; keep the console summary compact.
        result = {k:v for k,v in result.items() if k != "listenerPorts"}
        print(json.dumps(result, indent=2))
        return 0
    except (ReleaseError, OSError, ValueError, zipfile.BadZipFile) as exc:
        print(f"ERROR: {exc}", file=sys.stderr)
        return 1

if __name__ == "__main__":
    raise SystemExit(main())
OPENVTS_RELEASE_PY
}

prepare_bundle_early() {
  command_exists python3 || die "Python 3 is required for preflight. Install python3 including its standard library first."
  [[ -z "$SOURCE_DIR" || ( -z "$SOURCE_ARCHIVE" && "$SOURCE_URL_WAS_SET" == 0 ) ]] || die "Choose only one bundle source."
  [[ -z "$SOURCE_ARCHIVE" || "$SOURCE_URL_WAS_SET" == 0 ]] || die "Choose only one bundle source."
  WORK_DIR="$(mktemp -d /tmp/openvts-installer.XXXXXX)"
  if [[ -n "$SOURCE_DIR" ]]; then
    [[ -d "$SOURCE_DIR" ]] || die "Source directory not found: $SOURCE_DIR"
    SOURCE_INPUT="$(readlink -f -- "$SOURCE_DIR")"
  elif [[ -n "$SOURCE_ARCHIVE" ]]; then
    [[ -s "$SOURCE_ARCHIVE" && -f "$SOURCE_ARCHIVE" ]] || die "Source ZIP not found or empty: $SOURCE_ARCHIVE"
    SOURCE_ARCHIVE="$(readlink -f -- "$SOURCE_ARCHIVE")"
    SOURCE_INPUT="$SOURCE_ARCHIVE"
  else
    command_exists curl || die "curl is required to download the deployment ZIP."
    [[ "$SOURCE_URL" == https://* ]] || die "--source-url must use HTTPS."
    SOURCE_ARCHIVE="$WORK_DIR/openvts-linux.zip"
    info "Downloading deployment ZIP: $SOURCE_URL"
    # Download one file only. Never request a sidecar or reuse a previous partial ZIP.
    # Request cache revalidation without changing the publisher's URL/query string.
    curl_download "$SOURCE_URL" "${SOURCE_ARCHIVE}.part" 1800 revalidate
    [[ -s "${SOURCE_ARCHIVE}.part" ]] || die "Downloaded ZIP is empty. No application or database changes were made."
    mv -- "${SOURCE_ARCHIVE}.part" "$SOURCE_ARCHIVE"
    SOURCE_INPUT="$SOURCE_ARCHIVE"
  fi
  info "Preparing deployment ZIP/directory; no application checksum or release manifest is required."
  SOURCE_ROOT="$WORK_DIR/prepared/openvts-linux"
  # Keep ZIP safety/CRC, required-file, package-lock and compatibility validation.
  # This helper never executes application code; it only prepares a temporary copy.
  bundle_tool prepare --source "$SOURCE_INPUT" --output "$SOURCE_ROOT" > "$WORK_DIR/bundle-preflight.json"
  cat "$WORK_DIR/bundle-preflight.json"
  FRONTEND_SOURCE="$SOURCE_ROOT/frontend"
  LISTENER_ENV_TEMPLATE="$SOURCE_ROOT/listener/.env.example"
  BUNDLE_MODE="prebuilt"
  GPS_TCP_RULES="$(bundle_tool ports --source "$SOURCE_ROOT")"
}

if (($# > 0)); then
  case "$1" in
    install|start|stop|restart|status|doctor|logs)
      COMMAND="$1"
      shift
      ;;
    -h|--help)
      usage
      exit 0
      ;;
  esac
fi

if [[ "$COMMAND" != "install" ]]; then
  if [[ -x /usr/local/bin/openvts ]]; then
    exec /usr/local/bin/openvts "$COMMAND" "$@"
  fi
  die "OpenVTS is not installed yet. Run the install command first."
fi

while (($# > 0)); do
  case "$1" in
    --source-url)
      (($# >= 2)) || die "--source-url requires a URL."
      SOURCE_URL="$2"; SOURCE_URL_WAS_SET=1; shift 2 ;;
    --source-archive)
      (($# >= 2)) || die "--source-archive requires a path."
      SOURCE_ARCHIVE="$2"; shift 2 ;;
    --source-dir)
      (($# >= 2)) || die "--source-dir requires a path."
      SOURCE_DIR="$2"; shift 2 ;;
    --check-bundle)
      CHECK_BUNDLE_ONLY=1; shift ;;
    --app-dir)
      (($# >= 2)) || die "--app-dir requires a path."
      APP_DIR="${2%/}"; APP_DIR_WAS_SET=1; shift 2 ;;
    --domain)
      (($# >= 2)) || die "--domain requires a DNS name."
      DOMAIN="${2,,}"; DOMAIN_WAS_SET=1; shift 2 ;;
    --email)
      (($# >= 2)) || die "--email requires an address."
      EMAIL="$2"; EMAIL_WAS_SET=1; shift 2 ;;
    --node-version)
      (($# >= 2)) || die "--node-version requires a version."
      NODE_VERSION="${2#v}"; NODE_VERSION_WAS_SET=1; shift 2 ;;
    --postgres-major)
      (($# >= 2)) || die "--postgres-major requires a version."
      REQUESTED_PG_MAJOR="$2"; PG_MAJOR_WAS_SET=1; shift 2 ;;
    --redis-port)
      (($# >= 2)) || die "--redis-port requires a port."
      REDIS_PORT="$2"; REDIS_PORT_WAS_SET=1; shift 2 ;;
    --health-timeout)
      (($# >= 2)) || die "--health-timeout requires a number of seconds."
      HEALTH_TIMEOUT="$2"; shift 2 ;;
    --adopt-existing-databases)
      ADOPT_EXISTING_DATABASES=1; shift ;;
    --skip-firewall)
      SKIP_FIREWALL=1; shift ;;
    --no-auto-swap)
      AUTO_SWAP=0; shift ;;
    -h|--help)
      usage; exit 0 ;;
    *)
      die "Unknown option: $1 (use --help for assistance)"
      ;;
  esac
done

if (( CHECK_BUNDLE_ONLY == 1 )); then
  prepare_bundle_early
  info "Bundle preflight PASSED. Application startup and Ubuntu provisioning have not been tested by this command."
  exit 0
fi

(( EUID == 0 )) || die "Run this with root privileges: sudo bash openvts-install-linux.sh ..."
[[ "$INSTALLER_BUILD_ID" == *"$INSTALLER_VERSION"* ]] || die "Installer version/build identity mismatch."

# Reuse previously selected deployment settings automatically during reruns/upgrades.
if [[ -f /etc/openvts/install.conf ]]; then
  [[ "$(stat -c '%U' /etc/openvts/install.conf)" == root ]] || die "/etc/openvts/install.conf is not root-owned."
  INSTALL_CONF_MODE="$(stat -c '%a' /etc/openvts/install.conf)"
  (( (8#$INSTALL_CONF_MODE & 8#022) == 0 )) || die "/etc/openvts/install.conf is group/world-writable."
  # shellcheck disable=SC1091
  source /etc/openvts/install.conf
  (( APP_DIR_WAS_SET == 1 )) || APP_DIR="${OPENVTS_APP_DIR:-$APP_DIR}"
  if (( DOMAIN_WAS_SET == 0 )); then
    DOMAIN="${OPENVTS_DOMAIN:-$DOMAIN}"
    (( EMAIL_WAS_SET == 1 )) || EMAIL="${OPENVTS_EMAIL:-$EMAIL}"
  fi
  (( NODE_VERSION_WAS_SET == 1 )) || NODE_VERSION="${OPENVTS_NODE_VERSION:-$NODE_VERSION}"
  (( PG_MAJOR_WAS_SET == 1 )) || REQUESTED_PG_MAJOR="${OPENVTS_PG_MAJOR:-$REQUESTED_PG_MAJOR}"
  PG_PORT="${OPENVTS_PG_PORT:-$PG_PORT}"
fi

[[ "$APP_DIR" =~ ^/(opt|srv)/[A-Za-z0-9._/-]+$ ]] ||
  die "--app-dir must be a simple absolute path under /opt or /srv."
[[ "$APP_DIR" != "/opt" && "$APP_DIR" != "/srv" && "$APP_DIR" != *"/../"* && "$APP_DIR" != *"/.." ]] ||
  die "Unsafe --app-dir: $APP_DIR"
if [[ -e "$APP_DIR/app" && ! -L "$APP_DIR/app" ]]; then
  die "$APP_DIR/app is already a real directory. It will not be overwritten or adopted automatically, to protect data."
fi
[[ "$NODE_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || die "Provide the Node.js version in x.y.z format."
[[ "$REQUESTED_PG_MAJOR" =~ ^(15|16|17|18)$ ]] || die "PostgreSQL major version must be 15, 16, 17, or 18."
validate_port "PostgreSQL port" "$PG_PORT"
validate_port "health timeout" "$HEALTH_TIMEOUT"
(( HEALTH_TIMEOUT >= 30 && HEALTH_TIMEOUT <= 3600 )) || die "Health timeout must be between 30 and 3600 seconds."

# Runtime command path is derived after APP_DIR is final. The dedicated bin
# directory prevents Ubuntu's pg_wrapper/current-directory behavior from
# affecting the application bootstrap process.
RUNTIME_BIN_DIR="$APP_DIR/runtime/openvts-bin"
RUNTIME_PATH="$APP_DIR/runtime/node-current/bin:$RUNTIME_BIN_DIR:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"

if [[ "$DOMAIN" != "_" ]]; then
  [[ "$DOMAIN" =~ ^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?(\.[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)+$ ]] ||
    die "Provide a valid fully qualified DNS name, such as vts.example.com"
fi
if [[ -n "$EMAIL" ]]; then
  [[ "$EMAIL" =~ ^[^[:space:]@]+@[^[:space:]@]+\.[^[:space:]@]+$ ]] || die "The email address is invalid."
  [[ "$DOMAIN" != "_" ]] || die "--domain is also required when using --email."
fi

if [[ -n "$SOURCE_DIR" && -n "$SOURCE_ARCHIVE" ]]; then
  die "Use only one of --source-dir or --source-archive."
fi
if [[ -n "$SOURCE_DIR" && $SOURCE_URL_WAS_SET -eq 1 ]]; then
  die "--source-dir cannot be combined with --source-url."
fi
if [[ -n "$SOURCE_ARCHIVE" && $SOURCE_URL_WAS_SET -eq 1 ]]; then
  die "--source-archive cannot be combined with --source-url."
fi

# The zero-option path downloads the deployment ZIP from DEFAULT_SOURCE_URL.
# No application checksum or release metadata is fetched or required.
# Local files are used only with --source-archive or --source-dir.

[[ -z "$SOURCE_DIR" || -d "$SOURCE_DIR" ]] || die "Prebuilt deployment directory not found: $SOURCE_DIR"
[[ -z "$SOURCE_ARCHIVE" || -f "$SOURCE_ARCHIVE" ]] || die "Prebuilt deployment ZIP not found: $SOURCE_ARCHIVE"
if [[ -z "$SOURCE_DIR" && -z "$SOURCE_ARCHIVE" ]]; then
  [[ "$SOURCE_URL" =~ ^https:// ]] || die "--source-url must use HTTPS."
fi

if [[ ! -r /etc/os-release ]]; then die "/etc/os-release was not found."; fi
# shellcheck disable=SC1091
source /etc/os-release
[[ "${ID:-}" == "ubuntu" ]] || die "This installer is for Ubuntu Server; detected: ${PRETTY_NAME:-unknown Linux}"
case "${VERSION_ID:-}" in
  22.04|24.04|26.04) ;;
  *) die "Supported Ubuntu versions: 22.04, 24.04, and 26.04; detected ${VERSION_ID:-unknown}" ;;
esac
CODENAME="${VERSION_CODENAME:-}"
[[ -n "$CODENAME" ]] || die "Ubuntu codename was not found."

case "$(uname -m)" in
  x86_64) NODE_ARCH="x64" ;;
  aarch64|arm64) NODE_ARCH="arm64" ;;
  *) die "Supported CPU architectures: amd64 or arm64; detected $(uname -m)" ;;
esac
DEB_ARCH="$(dpkg --print-architecture 2>/dev/null || true)"
[[ "$DEB_ARCH" =~ ^(amd64|arm64)$ ]] || die "Unsupported Debian architecture: ${DEB_ARCH:-unknown}"

[[ -d /run/systemd/system ]] || die "This installer must run on Ubuntu Server with systemd (containers and WSL without systemd are not supported)."
PHYSICAL_MEM_MB="$(awk '/MemTotal:/ {print int($2/1024)}' /proc/meminfo)"
[[ "$PHYSICAL_MEM_MB" =~ ^[0-9]+$ ]] || die "Unable to determine system RAM."
(( PHYSICAL_MEM_MB >= MIN_PHYSICAL_RAM_MB )) ||
  die "OpenVTS requires a server with at least 4 GB RAM; detected ${PHYSICAL_MEM_MB} MB. Increase the VM size."
CURRENT_SWAP_MB="$(awk '/SwapTotal:/ {print int($2/1024)}' /proc/meminfo)"
[[ "$CURRENT_SWAP_MB" =~ ^[0-9]+$ ]] || CURRENT_SWAP_MB=0
MANAGED_SWAP_ACTIVE_MB="$(awk -v p="$MANAGED_SWAP_FILE" '$1==p {print int($3/1024)}' /proc/swaps 2>/dev/null | head -n1)"
[[ "$MANAGED_SWAP_ACTIVE_MB" =~ ^[0-9]+$ ]] || MANAGED_SWAP_ACTIVE_MB=0
OTHER_SWAP_MB=$(( CURRENT_SWAP_MB - MANAGED_SWAP_ACTIVE_MB ))
(( OTHER_SWAP_MB < 0 )) && OTHER_SWAP_MB=0
MANAGED_SWAP_TARGET_MB=0
if (( AUTO_SWAP == 1 && PHYSICAL_MEM_MB + OTHER_SWAP_MB < TARGET_INSTALL_MEMORY_MB )); then
  MANAGED_SWAP_TARGET_MB=$(( TARGET_INSTALL_MEMORY_MB - PHYSICAL_MEM_MB - OTHER_SWAP_MB ))
  MANAGED_SWAP_TARGET_MB=$(( (MANAGED_SWAP_TARGET_MB + 255) / 256 * 256 ))
fi
if [[ "$APP_DIR" == /opt/* ]]; then APP_FS_ROOT=/opt; else APP_FS_ROOT=/srv; fi
APP_FREE_DISK_MB="$(df -Pm "$APP_FS_ROOT" | awk 'NR==2 {print $4}')"
ROOT_FREE_DISK_MB="$(df -Pm / | awk 'NR==2 {print $4}')"
APP_FS_DEVICE="$(stat -c '%d' "$APP_FS_ROOT")"
ROOT_FS_DEVICE="$(stat -c '%d' /)"
[[ "$APP_FREE_DISK_MB" =~ ^[0-9]+$ ]] || die "Unable to determine free space on the application filesystem."
[[ "$ROOT_FREE_DISK_MB" =~ ^[0-9]+$ ]] || die "Unable to determine free space on the root filesystem."
REQUIRED_FREE_MB=$(( BASE_REQUIRED_FREE_MB + MANAGED_SWAP_TARGET_MB ))
if [[ "$APP_FS_DEVICE" == "$ROOT_FS_DEVICE" ]]; then
  (( APP_FREE_DISK_MB >= REQUIRED_FREE_MB )) ||
    die "OpenVTS and managed swap require at least $(( (REQUIRED_FREE_MB + 1023) / 1024 )) GB free; currently $((APP_FREE_DISK_MB / 1024)) GB is available."
else
  (( APP_FREE_DISK_MB >= BASE_REQUIRED_FREE_MB )) ||
    die "The OpenVTS application filesystem requires at least $((BASE_REQUIRED_FREE_MB / 1024)) GB free; currently $((APP_FREE_DISK_MB / 1024)) GB is available."
  # Managed swap always lives on /. With a separate /opt or /srv mount, check
  # root independently and leave 1GB of root headroom after swap allocation.
  if (( MANAGED_SWAP_TARGET_MB > 0 )); then
    (( ROOT_FREE_DISK_MB >= MANAGED_SWAP_TARGET_MB + 1024 )) ||
      die "Managed swap requires at least $(( (MANAGED_SWAP_TARGET_MB + 2047) / 1024 )) GB free on the root filesystem; currently $((ROOT_FREE_DISK_MB / 1024)) GB is available."
  fi
fi

exec 9>/run/lock/openvts-install.lock
flock -n 9 || die "Another OpenVTS installer is already running."

install -d -o root -g root -m 0750 /var/log/openvts
LOG_FILE="/var/log/openvts/install-$(date -u '+%Y%m%dT%H%M%SZ').log"
touch "$LOG_FILE"
chmod 0640 "$LOG_FILE"
exec > >(tee -a "$LOG_FILE") 2>&1

log "Starting OpenVTS Linux installation"
info "Installer: $INSTALLER_VERSION"
info "Build: $INSTALLER_BUILD_ID"
info "OS: ${PRETTY_NAME} ($CODENAME), architecture: $NODE_ARCH"
info "Install root: $APP_DIR"
info "Node.js: $NODE_VERSION; requested PostgreSQL: $REQUESTED_PG_MAJOR"
info "Memory: ${PHYSICAL_MEM_MB}MB RAM, ${CURRENT_SWAP_MB}MB active swap; app free: $((APP_FREE_DISK_MB / 1024))GB; root free: $((ROOT_FREE_DISK_MB / 1024))GB"
if [[ "$DOMAIN" == "_" ]]; then
  info "Nginx: HTTP catch-all (a domain can be added later)"
else
  info "Nginx domain: $DOMAIN"
fi

log "Validating the release before installing packages or changing databases/services"
prepare_bundle_early

NGINX_PREEXISTING=0
NGINX_PACKAGE_PREEXISTING=0
REDIS_PACKAGE_PREEXISTING=0
NGINX_CAN_REPLACE_DEFAULTS=0
NGINX_REPO_MANAGED=0
command_exists nginx && NGINX_PREEXISTING=1
dpkg-query -W -f='${Status}' nginx 2>/dev/null | grep -q 'install ok installed' && NGINX_PACKAGE_PREEXISTING=1 || true
dpkg-query -W -f='${Status}' redis-server 2>/dev/null | grep -q 'install ok installed' && REDIS_PACKAGE_PREEXISTING=1 || true
if [[ -f /etc/apt/sources.list.d/nginx.list ]] && grep -Fq 'nginx.org/packages/ubuntu' /etc/apt/sources.list.d/nginx.list; then
  NGINX_REPO_MANAGED=1
fi

nginx_has_custom_config() {
  local path
  for path in /etc/nginx/conf.d/*.conf /etc/nginx/sites-enabled/*; do
    [[ -e "$path" ]] || continue
    case "$path" in
      /etc/nginx/conf.d/default.conf|/etc/nginx/sites-enabled/default|/etc/nginx/conf.d/openvts.conf) ;;
      *) return 0 ;;
    esac
  done
  return 1
}

if (( NGINX_PREEXISTING == 0 )); then
  if port_is_listening 80; then
    die "TCP port 80 is already in use, but Nginx is not installed. Stop the conflicting service."
  fi
  NGINX_CAN_REPLACE_DEFAULTS=1
elif [[ -f /etc/nginx/conf.d/openvts.conf ]] || ! nginx_has_custom_config; then
  # Safe rerun after an earlier OpenVTS attempt or a default-only Nginx install.
  NGINX_CAN_REPLACE_DEFAULTS=1
elif [[ "$DOMAIN" == "_" ]]; then
  die "Custom Nginx sites already exist on this server. To protect them, provide --domain vts.example.com."
fi

log "Installing base tools"
export DEBIAN_FRONTEND=noninteractive
export NEEDRESTART_MODE=a
APT_GET=(apt-get
  -o "Dpkg::Lock::Timeout=${APT_LOCK_TIMEOUT}"
  -o Acquire::Retries=5
  -o Acquire::http::Timeout=30
  -o Acquire::https::Timeout=30
)
apt_update() { "${APT_GET[@]}" update; }
apt_install() { "${APT_GET[@]}" install -y "$@"; }

# Fresh cloud images often continue cloud-init/automatic apt work after SSH is
# available. Wait instead of failing on a transient package-manager lock.
if command_exists cloud-init; then
  if ! timeout 900 cloud-init status --wait >/dev/null 2>&1; then
    warn "cloud-init did not finish within the wait period; waiting for the package-manager lock will continue."
  fi
fi
for _ in $(seq 1 120); do
  if ! pgrep -f '(^|/)(apt|apt-get|dpkg|unattended-upgrade)([[:space:]]|$)' >/dev/null 2>&1; then
    break
  fi
  sleep 5
done
if pgrep -f '(^|/)(apt|apt-get|dpkg|unattended-upgrade)([[:space:]]|$)' >/dev/null 2>&1; then
  warn "The package manager is still busy; the installer will wait safely until the APT lock timeout."
fi

dpkg --configure -a
"${APT_GET[@]}" -f install -y

# Remove only repositories created by earlier OpenVTS attempts. They are
# recreated below with fresh, repository-scoped keyrings.
rm -f /etc/apt/sources.list.d/pgdg.sources /etc/apt/sources.list.d/pgdg.list
rm -f /etc/apt/sources.list.d/timescale_timescaledb.list
if (( NGINX_REPO_MANAGED == 1 )); then rm -f /etc/apt/sources.list.d/nginx.list; fi
apt_update
apt_install --no-install-recommends \
  ca-certificates curl gnupg lsb-release ubuntu-keyring \
  xz-utils unzip rsync openssl python3-minimal iproute2 util-linux ufw procps tzdata


# Ubuntu 24.04+ separates backwards-compatible IANA names (e.g.
# Asia/Calcutta, Europe/Kiev, US/Eastern) into tzdata-legacy. Browser ICU can
# still emit these names; --no-install-recommends must not silently omit them.
# On 22.04 the same names are supplied by tzdata itself.
if apt-cache show tzdata-legacy >/dev/null 2>&1; then
  apt_install --no-install-recommends tzdata-legacy
fi

ensure_managed_swap() {
  local requested_mb="$1" active_mb fs_type
  (( requested_mb > 0 )) || return 0
  active_mb="$(awk -v p="$MANAGED_SWAP_FILE" '$1==p {print int($3/1024)}' /proc/swaps 2>/dev/null | head -n1)"
  [[ "$active_mb" =~ ^[0-9]+$ ]] || active_mb=0
  if (( active_mb >= requested_mb )); then
    info "Managed swap already active: ${active_mb}MB"
    return 0
  fi

  if (( active_mb > 0 )); then swapoff "$MANAGED_SWAP_FILE"; fi
  rm -f -- "$MANAGED_SWAP_FILE"
  fs_type="$(findmnt -no FSTYPE -T / 2>/dev/null || true)"
  if [[ "$fs_type" == btrfs ]]; then
    command_exists chattr || apt_install --no-install-recommends e2fsprogs
    touch "$MANAGED_SWAP_FILE"
    chattr +C "$MANAGED_SWAP_FILE"
    dd if=/dev/zero of="$MANAGED_SWAP_FILE" bs=1M count="$requested_mb" status=none
  elif ! fallocate -l "${requested_mb}M" "$MANAGED_SWAP_FILE" 2>/dev/null; then
    dd if=/dev/zero of="$MANAGED_SWAP_FILE" bs=1M count="$requested_mb" status=none
  fi
  chmod 0600 "$MANAGED_SWAP_FILE"
  mkswap -f "$MANAGED_SWAP_FILE" >/dev/null
  swapon "$MANAGED_SWAP_FILE"
  sed -i '\|^/swapfile-openvts[[:space:]]|d' /etc/fstab
  printf '%s none swap sw 0 0\n' "$MANAGED_SWAP_FILE" >> /etc/fstab
  awk -v p="$MANAGED_SWAP_FILE" '$1==p {found=1} END {exit !found}' /proc/swaps ||
    die "Managed swap did not become active: $MANAGED_SWAP_FILE"
  info "Managed swap active: ${requested_mb}MB ($MANAGED_SWAP_FILE)"
}

if (( MANAGED_SWAP_TARGET_MB > 0 )); then
  log "Configuring managed swap for the 4 GB/low-memory cloud profile"
  ensure_managed_swap "$MANAGED_SWAP_TARGET_MB"
elif (( AUTO_SWAP == 0 && PHYSICAL_MEM_MB + CURRENT_SWAP_MB < TARGET_INSTALL_MEMORY_MB )); then
  warn "Automatic swap is disabled; memory pressure may occur during npm installation."
fi
CURRENT_SWAP_MB="$(awk '/SwapTotal:/ {print int($2/1024)}' /proc/meminfo)"

# Conservative host settings for PostgreSQL/Redis on small cloud VMs.
write_file /etc/sysctl.d/99-openvts.conf 0644 root root <<'EOF'
vm.overcommit_memory = 1
vm.swappiness = 10
net.core.somaxconn = 1024
EOF
sysctl -q -p /etc/sysctl.d/99-openvts.conf

log "Configuring official signed package repositories"

install -d -o root -g root -m 0755 /etc/apt/keyrings /usr/share/keyrings
curl_download https://www.postgresql.org/media/keys/ACCC4CF8.asc "$WORK_DIR/pgdg.asc" 300
PGDG_FPRS="$(gpg --batch --with-colons --show-keys "$WORK_DIR/pgdg.asc" | awk -F: '$1=="fpr" {print $10}')"
grep -q 'B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8' <<<"$PGDG_FPRS" || die "PostgreSQL repository key fingerprint mismatch."
rm -f /etc/apt/keyrings/postgresql-pgdg.gpg
gpg --batch --yes --dearmor \
  --output /etc/apt/keyrings/postgresql-pgdg.gpg \
  "$WORK_DIR/pgdg.asc"
chmod 0644 /etc/apt/keyrings/postgresql-pgdg.gpg
write_file /etc/apt/sources.list.d/pgdg.sources 0644 root root <<EOF
Types: deb
URIs: https://apt.postgresql.org/pub/repos/apt
Suites: ${CODENAME}-pgdg
Architectures: ${DEB_ARCH}
Components: main
Signed-By: /etc/apt/keyrings/postgresql-pgdg.gpg
EOF

curl_download https://packagecloud.io/timescale/timescaledb/gpgkey "$WORK_DIR/timescale.asc" 300
TIMESCALE_FPRS="$(gpg --batch --with-colons --show-keys "$WORK_DIR/timescale.asc" | awk -F: '$1=="fpr" {print $10}')"
grep -Eq 'E7391C94080429FF$' <<<"$TIMESCALE_FPRS" || die "TimescaleDB repository key fingerprint mismatch."
gpg --batch --yes --dearmor \
  --output /etc/apt/keyrings/timescale_timescaledb-archive-keyring.gpg \
  "$WORK_DIR/timescale.asc"
chmod 0644 /etc/apt/keyrings/timescale_timescaledb-archive-keyring.gpg
write_file /etc/apt/sources.list.d/timescale_timescaledb.list 0644 root root <<EOF
deb [arch=${DEB_ARCH} signed-by=/etc/apt/keyrings/timescale_timescaledb-archive-keyring.gpg] https://packagecloud.io/timescale/timescaledb/ubuntu/ ${CODENAME} main
EOF

if (( NGINX_PREEXISTING == 0 || NGINX_REPO_MANAGED == 1 )); then
  curl_download https://nginx.org/keys/nginx_signing.key "$WORK_DIR/nginx-signing.key" 300
  NGINX_FPRS="$(gpg --batch --with-colons --show-keys "$WORK_DIR/nginx-signing.key" | awk -F: '$1=="fpr" {print $10}')"
  grep -q '573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62' <<<"$NGINX_FPRS" || die "Nginx repository key fingerprint mismatch."
  gpg --batch --yes --dearmor \
    --output /usr/share/keyrings/nginx-archive-keyring.gpg \
    "$WORK_DIR/nginx-signing.key"
  chmod 0644 /usr/share/keyrings/nginx-archive-keyring.gpg
  write_file /etc/apt/sources.list.d/nginx.list 0644 root root <<EOF
deb [arch=${DEB_ARCH} signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://nginx.org/packages/ubuntu ${CODENAME} nginx
EOF
  write_file /etc/apt/preferences.d/99nginx 0644 root root <<'EOF'
Package: *
Pin: origin nginx.org
Pin: release o=nginx
Pin-Priority: 900
EOF
fi

apt_update

PG_MAJOR="$REQUESTED_PG_MAJOR"
PG_CLUSTER=""
PG_CLUSTER_WAS_PRESENT=0
if command_exists pg_lsclusters; then
  EXISTING_CLUSTER="$(pg_lsclusters --no-header 2>/dev/null | awk -v p="$PG_PORT" '$3==p {print $1 " " $2; exit}')"
  if [[ -n "$EXISTING_CLUSTER" ]]; then
    IFS=' ' read -r PG_MAJOR PG_CLUSTER <<<"$EXISTING_CLUSTER"
    PG_CLUSTER_WAS_PRESENT=1
    [[ "$PG_MAJOR" =~ ^(15|16|17|18)$ ]] ||
      die "PostgreSQL $PG_MAJOR was detected on port $PG_PORT; current TimescaleDB packages require PostgreSQL 15 through 18."
    if [[ "$PG_MAJOR" != "$REQUESTED_PG_MAJOR" ]]; then
      warn "Existing PostgreSQL $PG_MAJOR will be reused safely; no major upgrade will be performed."
    fi
  fi
fi

if (( PG_CLUSTER_WAS_PRESENT == 0 )) && port_is_listening "$PG_PORT"; then
  die "Port $PG_PORT is already in use, but it was not recognized as a Debian/Ubuntu PostgreSQL cluster."
fi

apt-cache show "timescaledb-2-postgresql-${PG_MAJOR}" >/dev/null 2>&1 ||
  die "The TimescaleDB repository does not provide a PostgreSQL $PG_MAJOR package for $CODENAME."

log "Installing PostgreSQL, PostGIS, TimescaleDB, Redis, and Nginx"
PACKAGES=(
  "postgresql-${PG_MAJOR}"
  "postgresql-client-${PG_MAJOR}"
  "postgresql-${PG_MAJOR}-postgis-3"
  "postgresql-${PG_MAJOR}-postgis-3-scripts"
  "timescaledb-2-postgresql-${PG_MAJOR}"
  "timescaledb-tools"
  "redis-server"
)
if (( NGINX_PREEXISTING == 0 || NGINX_PACKAGE_PREEXISTING == 1 )); then PACKAGES+=(nginx); fi
PACKAGES+=(certbot python3-certbot-nginx) # UI SSL must work even when no --email was supplied during install.
apt_install --no-install-recommends -o Dpkg::Options::="--force-confold" "${PACKAGES[@]}"

if (( NGINX_PREEXISTING == 1 )); then
  command_exists nginx || die "The existing Nginx executable can no longer be found."
fi

if (( PG_CLUSTER_WAS_PRESENT == 0 )); then
  CLUSTER_AT_PORT="$(pg_lsclusters --no-header 2>/dev/null | awk -v p="$PG_PORT" '$3==p {print $1 " " $2; exit}')"
  if [[ -z "$CLUSTER_AT_PORT" ]]; then
    if pg_lsclusters --no-header 2>/dev/null | awk -v v="$PG_MAJOR" '$1==v && $2=="openvts" {found=1} END {exit !found}'; then
      die "PostgreSQL cluster ${PG_MAJOR}/openvts already exists on another port; manual review is required."
    fi
    pg_createcluster "$PG_MAJOR" openvts --port "$PG_PORT" --start
    PG_CLUSTER="openvts"
  else
    IFS=' ' read -r PG_MAJOR PG_CLUSTER <<<"$CLUSTER_AT_PORT"
  fi
fi
[[ -n "$PG_CLUSTER" ]] || die "Unable to resolve the PostgreSQL cluster."
[[ "$PG_CLUSTER" =~ ^[A-Za-z0-9_-]+$ ]] || die "Unsafe PostgreSQL cluster name: $PG_CLUSTER"

PG_UNIT="postgresql@${PG_MAJOR}-${PG_CLUSTER}.service"
PG_BIN="/usr/lib/postgresql/${PG_MAJOR}/bin"
PG_PSQL="$PG_BIN/psql"
PG_ISREADY="$PG_BIN/pg_isready"
[[ -x "$PG_PSQL" ]] || die "PostgreSQL client missing: $PG_PSQL"
[[ -x "$PG_ISREADY" ]] || die "PostgreSQL readiness tool missing: $PG_ISREADY"

pg_ctlcluster "$PG_MAJOR" "$PG_CLUSTER" start || true
systemctl enable postgresql.service >/dev/null

for _ in $(seq 1 60); do
  if "$PG_ISREADY" -h 127.0.0.1 -p "$PG_PORT" >/dev/null 2>&1; then break; fi
  sleep 1
done
"$PG_ISREADY" -h 127.0.0.1 -p "$PG_PORT" >/dev/null || die "PostgreSQL did not become ready."

pg_super() (
  cd /
  exec runuser -u postgres -- "$PG_PSQL" -X -v ON_ERROR_STOP=1 -p "$PG_PORT" "$@"
)

log "Preloading and tuning TimescaleDB in PostgreSQL"
PG_CONF="$(pg_super -d postgres -Atqc 'SHOW config_file')"
[[ -f "$PG_CONF" ]] || die "PostgreSQL configuration file was not found: $PG_CONF"

TOTAL_MEM_MB="$PHYSICAL_MEM_MB"
CPU_COUNT="$(nproc)"
DB_MEMORY_MB=$(( TOTAL_MEM_MB * 35 / 100 ))
(( DB_MEMORY_MB < 1024 )) && DB_MEMORY_MB=1024
(( DB_MEMORY_MB > 16384 )) && DB_MEMORY_MB=16384
DB_CPUS=$(( CPU_COUNT > 1 ? CPU_COUNT - 1 : 1 ))
if (( TOTAL_MEM_MB < 4096 )); then
  warn "The server has ${TOTAL_MEM_MB} MB RAM; OpenVTS production requires at least 4 GB, with 8 GB or more recommended."
fi

ensure_timescale_preload() {
  local current new_value
  current="$(pg_super -d postgres -Atqc 'SHOW shared_preload_libraries')"
  if tr ',' '\n' <<<"$current" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' | grep -qx timescaledb; then
    return 0
  fi
  if [[ -n "$current" ]]; then new_value="${current},timescaledb"; else new_value="timescaledb"; fi
  pg_super -d postgres -v libs="$new_value" >/dev/null <<'SQL'
SELECT format('ALTER SYSTEM SET shared_preload_libraries = %L', :'libs') \gexec
SQL
}

if (( PG_CLUSTER_WAS_PRESENT == 0 )); then
  install -d -o root -g root -m 0755 "$APP_DIR/start/postgresql"
  install -o root -g root -m 0600 "$PG_CONF" \
    "$APP_DIR/start/postgresql/postgresql.conf.before-openvts-$(date -u '+%Y%m%dT%H%M%SZ')"
  if ! timescaledb-tune --quiet --yes \
      --memory="${DB_MEMORY_MB}MB" \
      --cpus="$DB_CPUS" \
      --conf-path="$PG_CONF"; then
    warn "timescaledb-tune did not complete; applying the minimum preload configuration."
  fi
  ensure_timescale_preload
  pg_super -d postgres -c "ALTER SYSTEM SET listen_addresses TO '127.0.0.1';" >/dev/null
else
  info "An existing cluster was found; memory settings will not be changed, and only TimescaleDB preload will be ensured."
  ensure_timescale_preload
fi

pg_ctlcluster "$PG_MAJOR" "$PG_CLUSTER" restart
for _ in $(seq 1 60); do
  "$PG_ISREADY" -h 127.0.0.1 -p "$PG_PORT" >/dev/null 2>&1 && break
  sleep 1
done
"$PG_ISREADY" -h 127.0.0.1 -p "$PG_PORT" >/dev/null || die "PostgreSQL did not become ready after restart."
pg_super -d postgres -Atqc 'SHOW shared_preload_libraries' | tr ',' '\n' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' | grep -qx timescaledb ||
  die "TimescaleDB was not loaded in shared_preload_libraries."

log "Preparing service users, persistent directories, and secrets"
if ! id -u openvts >/dev/null 2>&1; then
  useradd --system --user-group --home-dir /var/lib/openvts --shell /usr/sbin/nologin openvts
fi
if ! id -u openvts-redis >/dev/null 2>&1; then
  useradd --system --user-group --home-dir /var/lib/openvts/redis --shell /usr/sbin/nologin openvts-redis
fi
usermod -a -G openvts openvts-redis

install -d -o root -g root -m 0755 "$APP_DIR" "$APP_DIR/releases" "$APP_DIR/runtime" "$APP_DIR/start"
install -d -o root -g openvts -m 0750 /etc/openvts
install -d -o openvts -g openvts -m 0750 \
  /var/lib/openvts /var/lib/openvts/backend /var/lib/openvts/backend/uploads \
  /var/lib/openvts/backend/data /var/lib/openvts/backend/runtime /var/lib/openvts/listener /var/lib/openvts/listener/data /var/lib/openvts/listener/runtime \
  /var/log/openvts/backend/winston /var/log/openvts/backend/api \
  /var/log/openvts/listener/winston
install -d -o openvts-redis -g openvts-redis -m 0750 /var/lib/openvts/redis

# npm cache is disposable installer state. Recreate it on every run so a prior
# interrupted/root-run npm process cannot poison the next installation with
# stale, corrupt, or root-owned cache entries.
install -d -o root -g root -m 0755 "$NPM_CACHE_ROOT"
rm -rf -- "$NPM_CACHE_DIR"
install -d -o openvts -g openvts -m 0750 "$NPM_CACHE_DIR" "$RUNTIME_NPM_CACHE_DIR"

# Repair ownership only in OpenVTS-managed mutable paths. Do not recursively
# chown /var/lib/openvts because the dedicated Redis subtree has another owner.
chown -R openvts:openvts \
  /var/lib/openvts/backend /var/lib/openvts/listener \
  /var/log/openvts/backend /var/log/openvts/listener \
  "$RUNTIME_NPM_CACHE_DIR"
chown root:openvts /var/log/openvts
chmod 0750 /var/log/openvts

SECRETS_FILE=/etc/openvts/openvts.secrets
SECRETS_CREATED=0
load_openvts_secrets() {
  local line key value
  while IFS= read -r line || [[ -n "$line" ]]; do
    [[ "$line" =~ ^([A-Z0-9_]+)=(.*)$ ]] || die "Invalid line in $SECRETS_FILE."
    key="${BASH_REMATCH[1]}"
    value="${BASH_REMATCH[2]}"
    case "$key" in
      DB_USER|DB_PASSWORD|REDIS_PASSWORD|JWT_SECRET|LISTENER_KEY|INTEGRATIONS_ENCRYPTION_KEY|OTP_HASH_SECRET|EMAIL_UNSUBSCRIBE_SECRET|SESSION_SECRET)
        printf -v "$key" '%s' "$value"
        ;;
      *) die "Unknown key in $SECRETS_FILE: $key" ;;
    esac
  done <"$SECRETS_FILE"
}

if [[ -f "$SECRETS_FILE" ]]; then
  [[ "$(stat -c '%U' "$SECRETS_FILE")" == root ]] || die "$SECRETS_FILE must be root-owned."
  SECRETS_MODE="$(stat -c '%a' "$SECRETS_FILE")"
  (( (8#$SECRETS_MODE & 8#077) == 0 )) || die "$SECRETS_FILE must not be accessible by group/others."
  load_openvts_secrets
else
  DB_USER="openvts_app"
  if [[ "$(pg_super -d postgres -Atqc "SELECT 1 FROM pg_roles WHERE rolname='openvts_app'")" == "1" ]]; then
    DB_USER="openvts_app_$(openssl rand -hex 3)"
    warn "An existing openvts_app role was found; a new dedicated role named $DB_USER will be created instead of modifying it."
  fi
  DB_PASSWORD="$(openssl rand -hex 32)"
  REDIS_PASSWORD="$(openssl rand -hex 32)"
  JWT_SECRET="$(openssl rand -hex 64)"
  LISTENER_KEY="$(openssl rand -hex 32)"
  INTEGRATIONS_ENCRYPTION_KEY="$(openssl rand -base64 32 | tr -d '\n')"
  OTP_HASH_SECRET="$(openssl rand -hex 32)"
  EMAIL_UNSUBSCRIBE_SECRET="$(openssl rand -hex 32)"
  SESSION_SECRET="$(openssl rand -hex 48)"
  write_file "$SECRETS_FILE" 0600 root root <<EOF
DB_USER=${DB_USER}
DB_PASSWORD=${DB_PASSWORD}
REDIS_PASSWORD=${REDIS_PASSWORD}
JWT_SECRET=${JWT_SECRET}
LISTENER_KEY=${LISTENER_KEY}
INTEGRATIONS_ENCRYPTION_KEY=${INTEGRATIONS_ENCRYPTION_KEY}
OTP_HASH_SECRET=${OTP_HASH_SECRET}
EMAIL_UNSUBSCRIBE_SECRET=${EMAIL_UNSUBSCRIBE_SECRET}
SESSION_SECRET=${SESSION_SECRET}
EOF
  SECRETS_CREATED=1
fi

[[ "${DB_USER:-}" =~ ^[a-z_][a-z0-9_]{0,62}$ ]] || die "DB_USER is invalid in the secrets file."
for secret_name in DB_PASSWORD REDIS_PASSWORD JWT_SECRET LISTENER_KEY INTEGRATIONS_ENCRYPTION_KEY OTP_HASH_SECRET EMAIL_UNSUBSCRIBE_SECRET SESSION_SECRET; do
  [[ -n "${!secret_name:-}" ]] || die "$secret_name is missing from the secrets file."
done

log "Creating the dedicated PostgreSQL role, databases, and extensions"
DATABASE_MARKER=/etc/openvts/database-owner.marker
EXISTING_OPENVTS_DATABASES="$(pg_super -d postgres -Atqc \
  "SELECT datname FROM pg_database WHERE datname IN ('OpenVTS_db','OpenVTS_logs','OpenVTS_Address') ORDER BY datname")"
if [[ -n "$EXISTING_OPENVTS_DATABASES" && ! -f "$DATABASE_MARKER" && $ADOPT_EXISTING_DATABASES -eq 0 ]]; then
  printf '%s\n' "$EXISTING_OPENVTS_DATABASES" >&2
  die "Existing OpenVTS database(s) were found. The installer did not modify the data. Take a verified backup and review the code's TimescaleDB migration before using --adopt-existing-databases."
fi
if [[ ! -f "$DATABASE_MARKER" ]]; then
  write_file "$DATABASE_MARKER" 0600 root root <<EOF
managed_by=open-vts-install.sh
postgres_cluster=${PG_MAJOR}/${PG_CLUSTER}
postgres_port=${PG_PORT}
databases=OpenVTS_db,OpenVTS_logs,OpenVTS_Address
created_or_adopted_utc=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
EOF
fi

ROLE_EXISTS="$(pg_super -d postgres -Atqc "SELECT 1 FROM pg_roles WHERE rolname='${DB_USER}'")"
if [[ "$ROLE_EXISTS" != "1" ]]; then
  pg_super -d postgres -v role="$DB_USER" -v pass="$DB_PASSWORD" <<'SQL'
SET password_encryption = 'scram-sha-256';
SELECT format('CREATE ROLE %I LOGIN PASSWORD %L', :'role', :'pass') \gexec
SQL
else
  pg_super -d postgres -v role="$DB_USER" -v pass="$DB_PASSWORD" <<'SQL'
SET password_encryption = 'scram-sha-256';
ALTER ROLE :"role" WITH LOGIN PASSWORD :'pass';
SQL
fi

DB_PRIMARY="OpenVTS_db"
DB_LOGS="OpenVTS_logs"
DB_ADDRESS="OpenVTS_Address"

ensure_database() {
  local db="$1" exists
  exists="$(pg_super -d postgres -v db="$db" -At <<'SQL'
SELECT 1 FROM pg_database WHERE datname = :'db';
SQL
)"
  if [[ "$exists" != "1" ]]; then
    pg_super -d postgres -v db="$db" -v owner="$DB_USER" <<'SQL'
SELECT format('CREATE DATABASE %I OWNER %I', :'db', :'owner') \gexec
SQL
  fi
  pg_super -d postgres -v db="$db" -v role="$DB_USER" <<'SQL'
GRANT CONNECT, TEMPORARY ON DATABASE :"db" TO :"role";
SQL
}

ensure_database "$DB_PRIMARY"
ensure_database "$DB_LOGS"
ensure_database "$DB_ADDRESS"

pg_super -d "$DB_PRIMARY" -c 'CREATE EXTENSION IF NOT EXISTS postgis;' >/dev/null
pg_super -d "$DB_LOGS" -c 'CREATE EXTENSION IF NOT EXISTS postgis;' >/dev/null
pg_super -d "$DB_LOGS" -c 'CREATE EXTENSION IF NOT EXISTS timescaledb;' >/dev/null

grant_database_schema() {
  local db="$1"
  pg_super -d "$db" -v role="$DB_USER" <<'SQL'
GRANT USAGE, CREATE ON SCHEMA public TO :"role";
GRANT SELECT, INSERT, UPDATE, DELETE, TRUNCATE, REFERENCES, TRIGGER ON ALL TABLES IN SCHEMA public TO :"role";
GRANT USAGE, SELECT, UPDATE ON ALL SEQUENCES IN SCHEMA public TO :"role";
GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO :"role";
ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE, TRUNCATE, REFERENCES, TRIGGER ON TABLES TO :"role";
ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT USAGE, SELECT, UPDATE ON SEQUENCES TO :"role";
ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT EXECUTE ON FUNCTIONS TO :"role";
SQL
}
grant_database_schema "$DB_PRIMARY"
grant_database_schema "$DB_LOGS"
grant_database_schema "$DB_ADDRESS"

TIMESCALE_VERSION="$(pg_super -d "$DB_LOGS" -Atqc "SELECT extversion FROM pg_extension WHERE extname='timescaledb'")"
[[ -n "$TIMESCALE_VERSION" ]] || die "The TimescaleDB extension was not found in OpenVTS_logs."
version_ge "$TIMESCALE_VERSION" "$MIN_TIMESCALE_VERSION" ||
  die "TimescaleDB >= $MIN_TIMESCALE_VERSION is required; detected $TIMESCALE_VERSION"

PGPASSWORD="$DB_PASSWORD" "$PG_PSQL" -X -v ON_ERROR_STOP=1 \
  -h 127.0.0.1 -p "$PG_PORT" -U "$DB_USER" -d "$DB_PRIMARY" -Atqc 'SELECT 1' | grep -qx 1 ||
  die "The dedicated PostgreSQL login test failed."

log "Configuring the dedicated Redis service"
REDIS_MAXMEMORY_MB=384
(( TOTAL_MEM_MB >= 6144 )) && REDIS_MAXMEMORY_MB=512
(( TOTAL_MEM_MB >= 8192 )) && REDIS_MAXMEMORY_MB=768
(( TOTAL_MEM_MB >= 16384 )) && REDIS_MAXMEMORY_MB=1536
if [[ -f /etc/openvts/install.conf && $REDIS_PORT_WAS_SET -eq 0 ]]; then
  STORED_REDIS_PORT="$(awk -F= '$1=="OPENVTS_REDIS_PORT" {print $2}' /etc/openvts/install.conf | tail -n1)"
  if [[ "$STORED_REDIS_PORT" =~ ^[0-9]+$ ]]; then REDIS_PORT="$STORED_REDIS_PORT"; fi
fi
if [[ -z "$REDIS_PORT" && -f /etc/openvts/redis.conf && $REDIS_PORT_WAS_SET -eq 0 ]]; then
  STORED_REDIS_PORT="$(awk '$1=="port" {print $2}' /etc/openvts/redis.conf | tail -n1)"
  if [[ "$STORED_REDIS_PORT" =~ ^[0-9]+$ ]]; then REDIS_PORT="$STORED_REDIS_PORT"; fi
fi

if (( REDIS_PACKAGE_PREEXISTING == 0 )); then
  systemctl disable --now redis-server.service >/dev/null 2>&1 || true
fi
systemctl stop openvts-redis.service >/dev/null 2>&1 || true

if [[ -n "$REDIS_PORT" ]]; then
  validate_port "Redis port" "$REDIS_PORT"
  port_is_listening "$REDIS_PORT" && die "Redis port $REDIS_PORT is already in use."
else
  if (( REDIS_PACKAGE_PREEXISTING == 1 )); then REDIS_PORT=6380; else REDIS_PORT=6379; fi
  if port_is_listening "$REDIS_PORT"; then
    for candidate in $(seq 6380 6399); do
      if ! port_is_listening "$candidate"; then REDIS_PORT="$candidate"; break; fi
    done
  fi
  port_is_listening "$REDIS_PORT" && die "No free port was found for the dedicated Redis service."
fi

write_file /etc/openvts/redis.conf 0640 root openvts-redis <<EOF
bind 127.0.0.1
protected-mode yes
port ${REDIS_PORT}
tcp-backlog 511
timeout 0
tcp-keepalive 300
daemonize no
supervised no
pidfile /run/openvts-redis/redis-server.pid
loglevel notice
logfile ""
databases 16
always-show-logo no
dir /var/lib/openvts/redis
dbfilename dump.rdb
save 900 1
save 300 10
save 60 10000
stop-writes-on-bgsave-error yes
rdbcompression yes
rdbchecksum yes
appendonly yes
appendfilename "appendonly.aof"
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
requirepass ${REDIS_PASSWORD}
maxmemory ${REDIS_MAXMEMORY_MB}mb
maxmemory-policy noeviction
EOF

write_file /etc/systemd/system/openvts-redis.service 0644 root root <<'EOF'
[Unit]
Description=OpenVTS dedicated Redis
After=network.target
PartOf=openvts.target

[Service]
Type=simple
User=openvts-redis
Group=openvts-redis
ExecStart=/usr/bin/redis-server /etc/openvts/redis.conf
ExecStop=/bin/kill -s TERM $MAINPID
Restart=always
RestartSec=3
RuntimeDirectory=openvts-redis
RuntimeDirectoryMode=0750
LimitNOFILE=65535
UMask=0027
NoNewPrivileges=true
PrivateTmp=true
PrivateDevices=true
ProtectHome=true
ProtectSystem=full
ProtectKernelTunables=true
ProtectKernelModules=true
ProtectControlGroups=true
RestrictSUIDSGID=true

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl enable --now openvts-redis.service >/dev/null
for _ in $(seq 1 30); do
  if REDISCLI_AUTH="$REDIS_PASSWORD" redis-cli -h 127.0.0.1 -p "$REDIS_PORT" ping 2>/dev/null | grep -qx PONG; then break; fi
  sleep 1
done
REDISCLI_AUTH="$REDIS_PASSWORD" redis-cli -h 127.0.0.1 -p "$REDIS_PORT" ping 2>/dev/null | grep -qx PONG ||
  die "The dedicated Redis service did not become ready."

log "Downloading the official Node.js binary and verifying its checksum"
NODE_DIST="node-v${NODE_VERSION}-linux-${NODE_ARCH}"
NODE_TARGET="$APP_DIR/runtime/$NODE_DIST"
if [[ ! -x "$NODE_TARGET/bin/node" ]]; then
  curl_download "https://nodejs.org/dist/v${NODE_VERSION}/SHASUMS256.txt" "$WORK_DIR/SHASUMS256.txt" 300
  curl_download "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_DIST}.tar.xz" "$WORK_DIR/${NODE_DIST}.tar.xz" 1200
  EXPECTED_NODE_SHA="$(awk -v f="${NODE_DIST}.tar.xz" '$2==f {print $1}' "$WORK_DIR/SHASUMS256.txt")"
  [[ "$EXPECTED_NODE_SHA" =~ ^[a-f0-9]{64}$ ]] || die "The Node.js checksum entry was not found."
  ACTUAL_NODE_SHA="$(sha256sum "$WORK_DIR/${NODE_DIST}.tar.xz" | awk '{print $1}')"
  [[ "$ACTUAL_NODE_SHA" == "$EXPECTED_NODE_SHA" ]] || die "Node.js SHA-256 mismatch."
  tar -xJf "$WORK_DIR/${NODE_DIST}.tar.xz" -C "$APP_DIR/runtime"
  chown -R root:root "$NODE_TARGET"
  chmod -R a-w "$NODE_TARGET"
fi
[[ "$($NODE_TARGET/bin/node --version)" == "v$NODE_VERSION" ]] || die "Node.js version verification failed."
ln -sfn "$NODE_TARGET" "$APP_DIR/runtime/node-current"
NODE_BIN="$APP_DIR/runtime/node-current/bin/node"
NPM_BIN="$APP_DIR/runtime/node-current/bin/npm"

log "Verifying application runtime command wrappers"
install -d -o root -g root -m 0755 "$RUNTIME_BIN_DIR"
write_file "$RUNTIME_BIN_DIR/psql" 0755 root root <<EOF
#!/usr/bin/env bash
set -Eeuo pipefail
exec "$PG_PSQL" "\$@"
EOF

# Run the exact PostgreSQL binary as the same unprivileged account used by the
# backend. This catches EACCES/working-directory/PATH problems before systemd
# starts the application and before a 15-minute health timeout can occur.
runuser -u openvts -- env \
  HOME=/var/lib/openvts \
  PATH="$RUNTIME_PATH" \
  OPENVTS_PSQL_BIN="$PG_PSQL" \
  "$PG_PSQL" --version >/dev/null || \
  die "openvts service user cannot execute PostgreSQL client: $PG_PSQL"

for runtime_db in postgres "$DB_PRIMARY" "$DB_LOGS" "$DB_ADDRESS"; do
  runuser -u openvts -- env \
    HOME=/var/lib/openvts \
    PATH="$RUNTIME_PATH" \
    OPENVTS_PSQL_BIN="$PG_PSQL" \
    PGPASSWORD="$DB_PASSWORD" \
    "$PG_PSQL" -X -v ON_ERROR_STOP=1 \
      -h 127.0.0.1 -p "$PG_PORT" -U "$DB_USER" -d "$runtime_db" \
      -Atqc 'SELECT 1' | grep -qx 1 || \
    die "openvts runtime PostgreSQL preflight failed for database: $runtime_db"
done

log "Staging the validated OpenVTS prebuilt deployment"
[[ -d "$SOURCE_ROOT/frontend" && -d "$SOURCE_ROOT/backend" && -d "$SOURCE_ROOT/listener" ]] || die "Prepared deployment directories are missing."

RELEASE_ID="$(date -u '+%Y%m%dT%H%M%SZ')-$$"
RELEASE_ROOT="$APP_DIR/releases/$RELEASE_ID"
RELEASE_APP="$RELEASE_ROOT/app"
install -d -o openvts -g openvts -m 0750 \
  "$RELEASE_APP/frontend" "$RELEASE_APP/backend" "$RELEASE_APP/listener"

COMMON_EXCLUDES=(
  --exclude=/.git --exclude=/node_modules --exclude=/coverage
  --exclude=/.env --exclude=/.env.local --exclude=/.env.production
  --exclude=/winston_logs --exclude=/api_logs --exclude=/logs --exclude=/data
)
# Deploy the existing compiled artifacts exactly; never exclude .next or dist.
rsync -a "${COMMON_EXCLUDES[@]}" "$FRONTEND_SOURCE/" "$RELEASE_APP/frontend/"
rsync -a "${COMMON_EXCLUDES[@]}" "$SOURCE_ROOT/backend/" "$RELEASE_APP/backend/"
rsync -a "${COMMON_EXCLUDES[@]}" "$SOURCE_ROOT/listener/" "$RELEASE_APP/listener/"
# Installer-generated diagnostic summary only; not a supplied release requirement.
install -m 0644 "$WORK_DIR/bundle-preflight.json" "$RELEASE_ROOT/bundle-preflight.json"
chown -R openvts:openvts "$RELEASE_ROOT"

# Compiled bootstrap compatibility was validated/prepared before any host changes.
# No string-replacement surgery is performed against an active deployment.

seed_and_link_dir() {
  local source_path="$1" persistent_path="$2" backup_name="$3"
  install -d -o openvts -g openvts -m 0750 "$persistent_path"
  if [[ -d "$source_path" ]]; then
    rsync -a --ignore-existing "$source_path/" "$persistent_path/"
    chown -R openvts:openvts "$persistent_path"
  fi
  if [[ -e "$source_path" && ! -L "$source_path" ]]; then
    mv -- "$source_path" "$RELEASE_ROOT/$backup_name"
  fi
  ln -s "$persistent_path" "$source_path"
}

seed_and_link_dir "$RELEASE_APP/backend/uploads" /var/lib/openvts/backend/uploads source-backend-uploads
seed_and_link_dir "$RELEASE_APP/backend/data" /var/lib/openvts/backend/data source-backend-data
seed_and_link_dir "$RELEASE_APP/backend/winston_logs" /var/log/openvts/backend/winston source-backend-winston
seed_and_link_dir "$RELEASE_APP/backend/api_logs" /var/log/openvts/backend/api source-backend-api-logs
seed_and_link_dir "$RELEASE_APP/listener/data" /var/lib/openvts/listener/data source-listener-data
seed_and_link_dir "$RELEASE_APP/listener/winston_logs" /var/log/openvts/listener/winston source-listener-winston
ln -s frontend "$RELEASE_APP/Frontend"

PUBLIC_URL=""
if [[ "$DOMAIN" != "_" ]]; then
  if [[ -n "$EMAIL" ]]; then PUBLIC_URL="https://$DOMAIN"; else PUBLIC_URL="http://$DOMAIN"; fi
fi

DATABASE_BASE="postgresql://${DB_USER}:${DB_PASSWORD}@127.0.0.1:${PG_PORT}"
write_file "$RELEASE_APP/backend/.env" 0600 openvts openvts <<EOF
NODE_ENV=production
PORT=4000
BIND_HOST=127.0.0.1
OPENVTS_RUNTIME_DIR=/var/lib/openvts/backend/runtime
DEPLOYMENT_MODE=shared
JWT_SECRET=${JWT_SECRET}
JWT_EXPIRES_IN=24h
SESSION_SECRET=${SESSION_SECRET}
PRIMARY_DB_HOST=127.0.0.1
PRIMARY_DB_USER=${DB_USER}
PRIMARY_DB_PORT=${PG_PORT}
PRIMARY_DB_PASSWORD=${DB_PASSWORD}
PRIMARY_DB_NAME=${DB_PRIMARY}
PRIMARY_DATABASE_URL=${DATABASE_BASE}/${DB_PRIMARY}?schema=public
LOGS_DB_HOST=127.0.0.1
LOGS_DB_USER=${DB_USER}
LOGS_DB_PORT=${PG_PORT}
LOGS_DB_PASSWORD=${DB_PASSWORD}
LOGS_DB_NAME=${DB_LOGS}
LOGS_DATABASE_URL=${DATABASE_BASE}/${DB_LOGS}?schema=public
ADDRESS_DB_HOST=127.0.0.1
ADDRESS_DB_USER=${DB_USER}
ADDRESS_DB_PORT=${PG_PORT}
ADDRESS_DB_PASSWORD=${DB_PASSWORD}
ADDRESS_DB_NAME=${DB_ADDRESS}
ADDRESS_DATABASE_URL=${DATABASE_BASE}/${DB_ADDRESS}?schema=public
DATABASE_URL=${DATABASE_BASE}/${DB_PRIMARY}?schema=public
REDIS_HOST=127.0.0.1
REDIS_PORT=${REDIS_PORT}
REDIS_PASSWORD=${REDIS_PASSWORD}
REDIS_DB=0
OPENVTS_SERVICE_MANAGER_SOCKET=/run/openvts-service-manager/control.sock
OPENVTS_FRONTEND_UNIT=openvts-frontend.service
OPENVTS_BACKEND_UNIT=openvts-backend.service
OPENVTS_LISTENER_UNIT=openvts-listener.service
OPENVTS_NGINX_UNIT=nginx.service
OPENVTS_REDIS_UNIT=openvts-redis.service
OPENVTS_POSTGRES_UNIT=${PG_UNIT}
OPENVTS_FRONTEND_PORT=3000
OPENVTS_BACKEND_PORT=4000
OPENVTS_LISTENER_PORT=5055
OPENVTS_PG_PORT=${PG_PORT}
LISTNER_KEY=${LISTENER_KEY}
INTEGRATIONS_ENCRYPTION_KEY=${INTEGRATIONS_ENCRYPTION_KEY}
OTP_HASH_SECRET=${OTP_HASH_SECRET}
EMAIL_UNSUBSCRIBE_SECRET=${EMAIL_UNSUBSCRIBE_SECRET}
EMAIL_PUBLIC_BASE_URL=${PUBLIC_URL}
NEXT_PUBLIC_APP_URL=${PUBLIC_URL}
FRONTEND_URL=${PUBLIC_URL}
PUBLIC_BASE_URL=${PUBLIC_URL}
BACKEND_PERFORMANCE_PROFILE=balanced
LOG_LEVEL=info
WINSTON_LOGGING_ENABLED=true
WINSTON_LOG_DIR=winston_logs
CSV_LOG_DIR=api_logs
UPLOAD_DIR=uploads
EOF

{
  cat <<EOF
NODE_ENV=production
DEPLOYMENT_MODE=shared
LISTENER_RUNTIME_MODE=auto
LISTENER_PERFORMANCE_PROFILE=balanced
PRIMARY_DATABASE_URL=${DATABASE_BASE}/${DB_PRIMARY}?schema=public
LOGS_DATABASE_URL=${DATABASE_BASE}/${DB_LOGS}?schema=public
REDIS_HOST=127.0.0.1
REDIS_PORT=${REDIS_PORT}
REDIS_PASSWORD=${REDIS_PASSWORD}
REDIS_DB=0
API_HOST=127.0.0.1
OPENVTS_RUNTIME_DIR=/var/lib/openvts/listener/runtime
API_PORT=5055
UNKNOWN_DEVICE_POLICY=quarantine
SECRET_KEY=${LISTENER_KEY}
DATA_FORWARD_URL=http://127.0.0.1:4000/handledata
FORWARD_TIMEOUT_MS=2000
FORWARD_RETRY_MAX=5
FORWARD_RETRY_BASE_MS=200
CON_TIME_OUT=60000
SOCKET_TIMEOUT=120000
KEEP_ALIVE_INITIAL_DELAY_MS=10000
TRACE_WIRE=false
LOG_ENABLED=true
LOG_LEVEL=info
LISTENER_LOG_DIR=winston_logs
EOF
  # Extract only the canonical NAME=PORT assignment. awk removes the CR
  # from CRLF input and discards any optional inline comment safely.
  awk '
    { sub(/\r$/, "") }
    match($0, /^TCP_PROTOCOL_[A-Z0-9_]+_PORT=[0-9]+/) {
      print substr($0, RSTART, RLENGTH)
    }
  ' "$LISTENER_ENV_TEMPLATE"
} | write_file "$RELEASE_APP/listener/.env" 0600 openvts openvts

PROTOCOL_COUNT="$(awk '/^TCP_PROTOCOL_[A-Z0-9_]+_PORT=[0-9]+$/ {count++} END {print count+0}' "$RELEASE_APP/listener/.env")"
(( PROTOCOL_COUNT > 0 )) || die "No listener protocol ports are configured."
DUPLICATE_PROTOCOL_PORTS="$(awk -F= '/^TCP_PROTOCOL_.*_PORT=/{count[$2]++} END{for(p in count) if(count[p]>1) print p}' "$RELEASE_APP/listener/.env")"
[[ -z "$DUPLICATE_PROTOCOL_PORTS" ]] || die "Duplicate listener protocol ports: $DUPLICATE_PROTOCOL_PORTS"
if ! systemctl is-active --quiet openvts-listener.service 2>/dev/null; then
  while IFS= read -r protocol_port; do
    if port_is_listening "$protocol_port"; then
      die "Listener TCP port $protocol_port is already in use by another process."
    fi
  done < <(awk -F= '/^TCP_PROTOCOL_.*_PORT=/{print $2}' "$RELEASE_APP/listener/.env")
fi

write_file "$RELEASE_APP/frontend/.env.production" 0600 openvts openvts <<EOF
NODE_ENV=production
NEXT_PUBLIC_API_URL=/api
NEXT_PUBLIC_APP_URL=${PUBLIC_URL}
NEXT_TELEMETRY_DISABLED=1
EOF

log "Preparing runtime dependencies for the prebuilt OpenVTS release"
info "Application compilation disabled: deploying existing frontend/.next, backend/dist, listener/dist"

for writable_path in "$NPM_CACHE_DIR" /var/lib/openvts "$RELEASE_APP/frontend" "$RELEASE_APP/backend" "$RELEASE_APP/listener"; do
  runuser -u openvts -- test -r "$writable_path" || die "openvts user cannot read: $writable_path"
  runuser -u openvts -- test -w "$writable_path" || die "openvts user cannot write: $writable_path"
done

npm_as_openvts() (
  local cwd="$1"; shift
  cd "$cwd"
  exec timeout --signal=TERM --kill-after=30 1200 runuser -u openvts -- env \
    HOME=/var/lib/openvts \
    USER=openvts \
    LOGNAME=openvts \
    PATH="$RUNTIME_PATH" \
    npm_config_cache="$NPM_CACHE_DIR" \
    npm_config_audit=false \
    npm_config_fund=false \
    npm_config_update_notifier=false \
    npm_config_progress=false \
    npm_config_maxsockets=8 \
    npm_config_fetch_retries=3 \
    npm_config_fetch_timeout=120000 \
    NEXT_TELEMETRY_DISABLED=1 \
    CI=true \
    "$NPM_BIN" "$@"
)

# The release ZIP intentionally contains compiled application output but not
# node_modules. Install runtime dependencies only; do not compile application code.
info "Frontend: installing production runtime dependencies; using supplied .next build"
npm_as_openvts "$RELEASE_APP/frontend" ci --omit=dev --include=optional

info "Backend: installing production runtime dependencies; using supplied dist build"
npm_as_openvts "$RELEASE_APP/backend" ci --omit=dev --include=optional
info "Backend: generating Linux Prisma runtime clients only"
npm_as_openvts "$RELEASE_APP/backend" run prisma:generate:all

# Listener declares the Prisma CLI as a devDependency. It is installed only for
# client generation, then removed. listener/dist remains untouched by npm build.
info "Listener: installing dependencies for Prisma client generation; using supplied dist build"
npm_as_openvts "$RELEASE_APP/listener" ci --include=dev --include=optional
info "Listener: generating Linux Prisma runtime clients only"
npm_as_openvts "$RELEASE_APP/listener" run prisma:generate:all
npm_as_openvts "$RELEASE_APP/listener" prune --omit=dev --include=optional

for artifact in \
  "$RELEASE_APP/frontend/server.js" \
  "$RELEASE_APP/frontend/.next/BUILD_ID" \
  "$RELEASE_APP/backend/dist/main.js" \
  "$RELEASE_APP/backend/dist/user/reports/linux-report-time-zone.js" \
  "$RELEASE_APP/backend/dist/user/reports/linux-report-preflight.js" \
  "$RELEASE_APP/backend/generated/prisma-primary/index.js" \
  "$RELEASE_APP/backend/generated/prisma-logs/index.js" \
  "$RELEASE_APP/backend/generated/prisma-address/index.js" \
  "$RELEASE_APP/listener/dist/main.js" \
  "$RELEASE_APP/listener/generated/prisma-primary/index.js" \
  "$RELEASE_APP/listener/generated/prisma-logs/index.js"; do
  [[ -f "$artifact" ]] || die "Deployment artifact missing: $artifact"
done

log "Checking native modules and generated Prisma clients as the service user"
write_file "$RELEASE_ROOT/runtime-preflight.cjs" 0644 root openvts <<'RUNTIME_PREFLIGHT_JS'
'use strict';
// Run only after npm ci and Linux Prisma generation, from the component directory.
const {createRequire} = require('node:module');
const path = require('node:path');
const fs = require('node:fs');
const req = createRequire(path.join(process.cwd(), 'package.json'));
async function main() {
  const component = process.argv[2];
  const packageJson = req('./package.json');
  if (component === 'frontend') {
    req('next'); req('react'); req('react-dom');
    const metadata = JSON.parse(fs.readFileSync('.next/required-server-files.json','utf8'));
    for (const file of metadata.files) {
      if (!fs.existsSync(file.replaceAll('\\','/'))) throw new Error(`Missing Next file: ${file}`);
    }
    console.log('PASS frontend runtime dependency imports and required build files');
    return;
  }
  req('dotenv').config({path: path.join(process.cwd(), '.env'), quiet: true});
  if (packageJson.dependencies.sharp) {
    await req('sharp')({create: {width: 1,height: 1,channels: 4,background: '#000000'}}).png().toBuffer();
    console.log('PASS sharp native image operation');
  }
  if (packageJson.dependencies.pdfkit) {
    const PDFDocument = req('pdfkit');
    const doc = new PDFDocument(); doc.resume(); doc.text('Open VTS runtime check'); doc.end();
    console.log('PASS PDF runtime and base fonts');
  }
  if (packageJson.dependencies['quickjs-emscripten']) {
    const QuickJS = await req('quickjs-emscripten').getQuickJS();
    const ctx = QuickJS.newContext();
    try {
      const result = ctx.evalCode('1 + 1');
      if (result.error) { result.error.dispose(); throw new Error('QuickJS evaluation failed'); }
      try { if (ctx.getNumber(result.value) !== 2) throw new Error('QuickJS result mismatch'); }
      finally { result.value.dispose(); }
    } finally { ctx.dispose(); }
    console.log('PASS QuickJS WebAssembly execution');
  }
  const schemas = component === 'backend' ? ['primary','logs','address'] : ['primary','logs'];
  for (const schema of schemas) {
    const {PrismaClient} = req(`./generated/prisma-${schema}`);
    const client = new PrismaClient();
    try { await client.$connect(); await client.$queryRawUnsafe('SELECT 1'); }
    finally { await client.$disconnect(); }
    console.log(`PASS ${component} Linux Prisma ${schema} engine + database connection`);
  }
}
main().catch(error => {
  console.error('Runtime preflight failed:', error.message);
  process.exitCode=1;
});
RUNTIME_PREFLIGHT_JS
for component in frontend backend listener; do
  (
    cd "$RELEASE_APP/$component"
    timeout --signal=TERM --kill-after=10 120 runuser -u openvts -- env \
      HOME=/var/lib/openvts PATH="$RUNTIME_PATH" \
      "$NODE_BIN" "$RELEASE_ROOT/runtime-preflight.cjs" "$component"
  ) || die "Runtime dependency preflight failed for $component; no new app release was activated."
done

# Component-specific ceilings keep a 4 GB VM stable while preserving larger
# production profiles. These are V8 old-space ceilings, not upfront allocation.
FRONTEND_NODE_HEAP_MB=384
BACKEND_NODE_HEAP_MB=768
LISTENER_NODE_HEAP_MB=768
if (( TOTAL_MEM_MB >= 6144 )); then
  FRONTEND_NODE_HEAP_MB=512
  BACKEND_NODE_HEAP_MB=1024
  LISTENER_NODE_HEAP_MB=1024
fi
if (( TOTAL_MEM_MB >= 8192 )); then
  FRONTEND_NODE_HEAP_MB=768
  BACKEND_NODE_HEAP_MB=1536
  LISTENER_NODE_HEAP_MB=1536
fi
if (( TOTAL_MEM_MB >= 16384 )); then
  FRONTEND_NODE_HEAP_MB=1024
  BACKEND_NODE_HEAP_MB=2048
  LISTENER_NODE_HEAP_MB=2048
fi

log "Installing systemd services and one-command control"
OPENVTS_CALLER_UID="$(id -u openvts)"
# Drain host-control operations before changing their executable or Nginx.
# This also supports upgrades from 2.4.2 without leaving two SSL managers active.
SSL_INFRA_PAUSED=1
systemctl stop certbot.timer
if [[ -f /etc/systemd/system/openvts-ssl-manager.service ]]; then
  grep -Fqx 'ExecStart=/usr/bin/python3 -I /usr/local/libexec/openvts-ssl-manager' /etc/systemd/system/openvts-ssl-manager.service ||
    die "An unrecognized SSL manager unit exists; review it before upgrading."
  if systemctl is-active --quiet openvts-ssl-manager.service; then
    LEGACY_SSL_MANAGER_WAS_ACTIVE=1
    # 2.4.2 used KillMode=control-group. Signal its Python parent first so an
    # in-flight Certbot process is not sent SIGTERM in the middle of an edit.
    systemctl kill --kill-who=main --signal=SIGTERM openvts-ssl-manager.service
    legacy_wait_started=$SECONDS
    while systemctl is-active --quiet openvts-ssl-manager.service; do
      (( SECONDS - legacy_wait_started < 780 )) || die "An existing SSL operation has not drained; retry after it finishes."
      sleep 2
    done
  fi
  systemctl stop openvts-ssl-manager.service
fi
if [[ "$(systemctl show -p LoadState --value openvts-service-manager.service 2>/dev/null)" == loaded ]]; then
  systemctl stop openvts-service-manager.service
fi
ssl_wait_started=$SECONDS
while :; do
  # Type=oneshot is 'activating' while renewal runs, not necessarily 'active'.
  case "$(systemctl show -p ActiveState --value certbot.service 2>/dev/null || true)" in
    active|activating|reloading|deactivating)
      (( SECONDS - ssl_wait_started < 780 )) || die "A Certbot renewal is still running; retry after it finishes."
      sleep 5 ;;
    *) break ;;
  esac
done
# These are host-owned Certbot/Nginx paths, not additions to the application ZIP.
install -d -o root -g root -m 0755 /usr/local/libexec /etc/nginx /var/log/nginx
for tls_state_dir in /etc/letsencrypt /var/lib/letsencrypt /var/log/letsencrypt; do
  # Preserve existing operator permissions; never recursively chmod TLS state.
  if [[ ! -d "$tls_state_dir" ]]; then
    install -d -o root -g root -m 0700 "$tls_state_dir"
  fi
done
install -d -o root -g root -m 0755 /etc/letsencrypt/renewal-hooks/deploy
write_file /usr/local/libexec/openvts-service-manager 0750 root openvts <<'OPENVTS_MANAGER_PY'
#!/usr/bin/env python3
"""Existing Open VTS local service manager: fixed systemd and Certbot operations.

Root-owned Unix transport checks SO_PEERCRED. No arbitrary commands, paths, unit
names, plugins or hooks are accepted. The web backend remains unprivileged.
Certbot's own systemd timer handles renewal; no separate SSL daemon is needed.
"""
from __future__ import annotations

import json
import os
from pathlib import Path
import pwd
import re
import selectors
import signal
import socket
import socketserver
import ssl
import stat
import struct
import subprocess
import threading
import time
from typing import Callable

SOCKET_PATH = os.environ.get("OPENVTS_SERVICE_MANAGER_SOCKET", "/run/openvts-service-manager/control.sock")
NGINX_CONFIG = Path("/etc/nginx/conf.d/openvts.conf")
# Recognize the marker already written by 2.4.2; do not discard existing HTTPS.
MANAGED_MARKER = "# OpenVTS SSL manager: preserve managed virtual hosts"
MAX_REQUEST = 4096
MAX_OUTPUT = 96 * 1024
CERTBOT_TIMEOUT = 600
EXEC_ENV = {"PATH": "/usr/sbin:/usr/bin:/sbin:/bin", "HOME": "/var/lib/letsencrypt", "LANG": "C.UTF-8", "LC_ALL": "C.UTF-8"}
_OPERATION_LOCK = threading.Lock()
_CLIENT_SLOTS = threading.BoundedSemaphore(8)
LABEL = r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?"
DOMAIN_RE = re.compile(rf"{LABEL}(?:\.{LABEL})+\Z", re.ASCII)
EMAIL_RE = re.compile(r"[A-Za-z0-9.!#$%&'*+/=?^_`{|}~-]{1,64}@[A-Za-z0-9.-]+\Z", re.ASCII)
ID_RE = re.compile(r"[a-zA-Z0-9_-]{1,64}\Z", re.ASCII)
UNIT_RE = re.compile(r"[A-Za-z0-9_.@:-]+\.service\Z", re.ASCII)
ALLOWED_ACTIONS = {
    "frontend": {"start", "restart"}, "backend": {"restart"},
    "listener": {"start", "restart"}, "nginx": {"start", "stop", "restart", "reload"},
    "redis": {"restart"}, "postgres": {"restart"},
}

def validate_request(value: object) -> dict:
    if not isinstance(value, dict) or set(value) != {"version", "requestId", "componentId", "action", "domain", "email"}:
        raise ValueError("Invalid SSL request fields.")
    if type(value["version"]) is not int or value["version"] != 1:
        raise ValueError("Unsupported SSL protocol version.")
    if not isinstance(value["requestId"], str) or not ID_RE.fullmatch(value["requestId"]):
        raise ValueError("Invalid SSL request identity.")
    if value["componentId"] != "ssl" or value["action"] not in ("install", "renew"):
        raise ValueError("Only install and renew are supported.")
    domain, email = value["domain"], value["email"]
    if not isinstance(domain, str) or len(domain) > 253 or not DOMAIN_RE.fullmatch(domain):
        raise ValueError("A lowercase DNS hostname is required (no URL, IP address, wildcard or path).")
    if not any(c.isalpha() for c in domain.rsplit(".", 1)[1]):
        raise ValueError("An IP address is not a DNS hostname.")
    if not isinstance(email, str) or len(email) > 254 or not EMAIL_RE.fullmatch(email):
        raise ValueError("A valid ASCII certificate contact email is required.")
    if not DOMAIN_RE.fullmatch(email.rsplit("@", 1)[1].lower()):
        raise ValueError("A valid certificate contact email domain is required.")
    return value


def require_trusted_file(path: Path) -> None:
    info = path.lstat()
    if not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
        raise RuntimeError(f"Expected a root-owned, non-writable-by-application regular file: {path}")


def preserve_nginx_configuration() -> None:
    # A failed ACME attempt can still leave a legitimate Certbot configuration
    # edit. Mark BEFORE issuance so subsequent application upgrades preserve it.
    require_trusted_file(NGINX_CONFIG)
    with NGINX_CONFIG.open("r+", encoding="utf-8") as stream:
        current = stream.read()
        if MANAGED_MARKER not in current:
            stream.write("\n" + MANAGED_MARKER + "\n")
            stream.flush()
            os.fsync(stream.fileno())


def run_command(argv: list[str], timeout: float, log: Callable[[str], None]) -> None:
    """Bound time, child group lifetime, and retained/output bytes independently."""
    if not argv or not argv[0].startswith("/"):
        raise RuntimeError("An absolute executable is required.")
    log(f"[system] Running {Path(argv[0]).name}.")
    child = subprocess.Popen(argv, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
                             stderr=subprocess.STDOUT, shell=False, start_new_session=True,
                             close_fds=True, cwd="/", env=EXEC_ENV)
    assert child.stdout is not None
    deadline = time.monotonic() + timeout
    pending = bytearray()
    emitted = 0
    selector = selectors.DefaultSelector()
    os.set_blocking(child.stdout.fileno(), False)
    selector.register(child.stdout, selectors.EVENT_READ)
    timed_out = False
    try:
        while selector.get_map():
            if time.monotonic() >= deadline:
                timed_out = True
                break
            for key, _ in selector.select(min(0.5, max(0, deadline - time.monotonic()))):
                chunk = os.read(key.fd, 8192)
                if not chunk:
                    selector.unregister(key.fileobj)
                    continue
                # Log at most MAX_OUTPUT bytes, but always drain the pipe so a
                # verbose/malfunctioning child cannot deadlock or fill memory.
                allowed = chunk[:max(0, MAX_OUTPUT - emitted)]
                emitted += len(allowed)
                pending.extend(allowed)
                while b"\n" in pending or len(pending) >= 2048:
                    end = pending.find(b"\n")
                    take = min(end if end >= 0 else 2048, 2048)
                    text = bytes(pending[:take]).decode("utf-8", "replace").rstrip("\r")
                    del pending[:take + (1 if end == take else 0)]
                    if text:
                        log(text)
        if timed_out:
            raise TimeoutError(f"{Path(argv[0]).name} exceeded its {int(timeout)}-second limit.")
        remaining = max(0.01, deadline - time.monotonic())
        code = child.wait(timeout=remaining)
        if pending:
            log(bytes(pending).decode("utf-8", "replace").rstrip())
        if emitted >= MAX_OUTPUT:
            log("[system] Further command output was truncated; consult the host Certbot log.")
        if code != 0:
            # stderr is already streamed; avoid returning raw paths/content in a
            # separate uncontrolled error string. No command argument reflection.
            raise RuntimeError(f"{Path(argv[0]).name} failed (exit {code}); review the job log above.")
    except BaseException:
        try:
            os.killpg(child.pid, signal.SIGKILL)
        except ProcessLookupError:
            pass
        child.wait(timeout=5)
        raise
    finally:
        selector.close()
        child.stdout.close()


def verify_local_https(domain: str) -> None:
    # Verify hostname, CA trust and expiry on the actual local Nginx listener,
    # rather than reporting success just because certificate files were written.
    context = ssl.create_default_context()
    with socket.create_connection(("127.0.0.1", 443), timeout=10) as connection:
        with context.wrap_socket(connection, server_hostname=domain) as tls:
            if not tls.getpeercert():
                raise RuntimeError("Nginx did not serve a certificate.")


def install_certificate(request: dict, log: Callable[[str], None]) -> None:
    domain = request["domain"]
    for executable in ("/usr/bin/certbot", "/usr/sbin/nginx", "/usr/bin/systemctl"):
        if not os.access(executable, os.X_OK):
            raise RuntimeError("Linux SSL prerequisites are missing. Upgrade the host using the Open VTS Linux installer.")
    run_command(["/usr/bin/systemctl", "is-enabled", "--quiet", "certbot.timer"], 15, log)
    run_command(["/usr/bin/systemctl", "is-active", "--quiet", "certbot.timer"], 15, log)
    run_command(["/usr/sbin/nginx", "-t"], 30, log)
    preserve_nginx_configuration()
    log("[system] Public DNS must point here and inbound TCP ports 80/443 must be reachable.")
    # 'run --nginx' obtains AND installs. No force-renewal, arbitrary hook,
    # alternate CA, proxy target or plugin path can be supplied by the client.
    run_command([
        "/usr/bin/certbot", "run", "--nginx", "--non-interactive", "--agree-tos",
        "--redirect", "--keep-until-expiring", "--email", request["email"],
        "--cert-name", domain, "--domains", domain,
    ], CERTBOT_TIMEOUT, log)
    certificate = Path("/etc/letsencrypt/live") / domain
    for filename in ("fullchain.pem", "privkey.pem"):
        if not (certificate / filename).is_file():
            raise RuntimeError("Certbot did not create the expected certificate files.")
    run_command(["/usr/sbin/nginx", "-t"], 30, log)
    run_command(["/usr/bin/systemctl", "reload", "nginx.service"], 30, log)
    verify_local_https(domain)
    log("[system] HTTPS is active on local Nginx and certbot.timer is enabled for automatic renewal.")


def validate_service_request(value: dict) -> dict:
    if set(value) != {"version", "requestId", "componentId", "action"}:
        raise ValueError("Unexpected service action fields.")
    component, action = value["componentId"], value["action"]
    if not isinstance(component, str) or not isinstance(action, str):
        raise ValueError("Invalid service component/action.")
    if component not in ALLOWED_ACTIONS or action not in ALLOWED_ACTIONS[component]:
        raise ValueError("Component/action is not allowed.")
    return value


def run_systemctl(units: dict[str, str], component: str, action: str) -> str:
    # Both configuration and request are validated before entering this method.
    unit = units[component]
    run_command(["/usr/bin/systemctl", action, "--", unit], 90, lambda line: print(line, flush=True))
    return f"{component} {action} completed through {unit}"


class Handler(socketserver.StreamRequestHandler):
    def handle(self) -> None:
        self.request.settimeout(5)
        request_id = "invalid"
        is_ssl = False
        connected = True

        def emit(**values: object) -> None:
            nonlocal connected
            if not connected:
                return
            frame = {"version": 1, "requestId": request_id, **values}
            try:
                self.wfile.write((json.dumps(frame, ensure_ascii=True) + "\n").encode("ascii"))
                self.wfile.flush()
            except (OSError, TimeoutError):
                # An operation already changing Nginx must finish safely even
                # when its initiating backend/browser disconnects.
                connected = False

        def done(ok: bool, message: str, accepted: bool = False) -> None:
            if is_ssl:
                emit(type="done", ok=ok, message=message[:2048])
            else:
                # Preserve the original Server-page protocol exactly.
                emit(ok=ok, accepted=accepted, message=message[:1000])

        try:
            _, uid, _ = struct.unpack("3i", self.request.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, 12))
            if uid not in (0, self.server.application_uid):
                raise PermissionError("The peer account is not authorized for Linux management.")
            line = self.rfile.readline(MAX_REQUEST + 1)
            if len(line) > MAX_REQUEST or not line.endswith(b"\n"):
                raise ValueError("Invalid or oversized manager request.")
            value = json.loads(line.decode("utf-8"))
            if not isinstance(value, dict):
                raise ValueError("Invalid request object.")
            if type(value.get("version")) is not int or value["version"] != 1:
                raise ValueError("Unsupported protocol version.")
            if not isinstance(value.get("requestId"), str) or not ID_RE.fullmatch(value["requestId"]):
                raise ValueError("Invalid request identity.")
            request_id = value["requestId"]
            if value.get("action") == "ping":
                if set(value) != {"version", "requestId", "action"}:
                    raise ValueError("Unexpected ping fields.")
                done(True, "Linux systemd manager is ready (Certbot support enabled).")
                return
            is_ssl = value.get("componentId") == "ssl"
            request = validate_request(value) if is_ssl else validate_service_request(value)
            if self.server.stopping.is_set():
                done(False, "Linux service manager is shutting down. Retry after the upgrade completes.")
                return
            # One lock prevents a Server-page Nginx restart from interrupting
            # certificate installation; pings remain available while it is busy.
            if not _OPERATION_LOCK.acquire(blocking=False):
                done(False, "Another Linux service or SSL action is running. Retry after it completes.")
                return
            try:
                if is_ssl:
                    emit(type="log", message="[system] Existing Linux service manager accepted the SSL request.")
                    self.server.install_operation(request, lambda message: emit(type="log", message=message[:4096]))
                    done(True, "HTTPS installation completed.")
                elif request["componentId"] == "backend":
                    done(True, "Backend restart accepted by systemd manager", accepted=True)
                    # Flush acceptance before restarting the initiating process.
                    # Retain this bounded handler so shutdown drains it safely.
                    time.sleep(1.5)
                    try:
                        self.server.service_operation("backend", "restart")
                    except Exception as error:
                        print(f"Deferred backend restart failed: {error}", flush=True)
                else:
                    message = self.server.service_operation(request["componentId"], request["action"])
                    done(True, message)
            finally:
                _OPERATION_LOCK.release()
        except Exception as error:
            print(f"Linux manager request failed: {type(error).__name__}: {error}", flush=True)
            done(False, str(error))


class Server(socketserver.ThreadingMixIn, socketserver.UnixStreamServer):
    daemon_threads = False
    block_on_close = True
    request_queue_size = 8
    application_uid: int
    install_operation = staticmethod(install_certificate)

    def __init__(self, address, handler):
        self.stopping = threading.Event()
        self.units: dict[str, str] = {}
        super().__init__(address, handler)

    def service_operation(self, component: str, action: str) -> str:
        return run_systemctl(self.units, component, action)

    def process_request(self, request: socket.socket, client_address: object) -> None:
        if not _CLIENT_SLOTS.acquire(blocking=False):
            request.close()
            return
        try:
            super().process_request(request, client_address)
        except BaseException:
            _CLIENT_SLOTS.release()
            raise

    def process_request_thread(self, request: socket.socket, client_address: object) -> None:
        try:
            super().process_request_thread(request, client_address)
        finally:
            _CLIENT_SLOTS.release()


def main() -> None:
    if os.geteuid() != 0:
        raise SystemExit("Run the Linux service manager through its root-owned systemd unit.")
    account = pwd.getpwnam("openvts")
    configured_uid = int(os.environ.get("OPENVTS_CALLER_UID", str(account.pw_uid)))
    if configured_uid != account.pw_uid:
        raise SystemExit("Configured caller UID does not match the openvts account.")
    units = {component: os.environ[f"OPENVTS_{component.upper()}_UNIT"] for component in ALLOWED_ACTIONS}
    if any(not UNIT_RE.fullmatch(unit) for unit in units.values()):
        raise SystemExit("Invalid configured systemd service unit.")
    if not os.path.isabs(SOCKET_PATH) or len(os.fsencode(SOCKET_PATH)) > 107:
        raise SystemExit("Manager socket must be an absolute Unix socket path.")
    parent = Path(SOCKET_PATH).parent
    info = parent.lstat()
    if not stat.S_ISDIR(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
        raise SystemExit("Insecure service manager runtime directory.")
    if os.path.lexists(SOCKET_PATH):
        previous = os.lstat(SOCKET_PATH)
        if not stat.S_ISSOCK(previous.st_mode) or previous.st_uid != 0:
            raise SystemExit("Refusing to replace an untrusted manager socket path.")
        os.unlink(SOCKET_PATH)
    os.umask(0o077)
    with Server(SOCKET_PATH, Handler) as server:
        server.application_uid = account.pw_uid
        server.units = units
        os.chown(SOCKET_PATH, 0, account.pw_gid)
        os.chmod(SOCKET_PATH, 0o660)

        def stop(_signal: int, _frame: object) -> None:
            if not server.stopping.is_set():
                server.stopping.set()
                # shutdown must run outside serve_forever. Non-daemon request
                # threads drain bounded ACME operations before systemd proceeds.
                threading.Thread(target=server.shutdown, daemon=True).start()

        signal.signal(signal.SIGTERM, stop)
        signal.signal(signal.SIGINT, stop)
        print("Open VTS Linux service manager ready (Certbot enabled).", flush=True)
        try:
            server.serve_forever(poll_interval=0.5)
        finally:
            server.stopping.set()


if __name__ == "__main__":
    main()
OPENVTS_MANAGER_PY

write_file /etc/letsencrypt/renewal-hooks/deploy/openvts-nginx-reload 0750 root root <<'OPENVTS_SSL_RENEW'
#!/bin/sh
# Certbot invokes this root-owned deploy hook only after successful renewal.
set -eu
/usr/sbin/nginx -t
/usr/bin/systemctl reload nginx.service
OPENVTS_SSL_RENEW

write_file /etc/systemd/system/openvts-service-manager.service 0644 root root <<EOF
[Unit]
Description=OpenVTS restricted local systemd and Certbot manager
After=systemd-user-sessions.service
Before=openvts-backend.service

[Service]
Type=simple
User=root
Group=openvts
Environment=OPENVTS_SERVICE_MANAGER_SOCKET=/run/openvts-service-manager/control.sock
Environment=OPENVTS_CALLER_UID=${OPENVTS_CALLER_UID}
Environment=OPENVTS_FRONTEND_UNIT=openvts-frontend.service
Environment=OPENVTS_BACKEND_UNIT=openvts-backend.service
Environment=OPENVTS_LISTENER_UNIT=openvts-listener.service
Environment=OPENVTS_NGINX_UNIT=nginx.service
Environment=OPENVTS_REDIS_UNIT=openvts-redis.service
Environment=OPENVTS_POSTGRES_UNIT=${PG_UNIT}
ExecStart=/usr/bin/python3 -I /usr/local/libexec/openvts-service-manager
Restart=always
RestartSec=2
RuntimeDirectory=openvts-service-manager
RuntimeDirectoryMode=0750
UMask=0077
# Drain an in-flight bounded certificate operation before replacing this process.
TimeoutStopSec=800
KillMode=mixed
LimitNOFILE=1024
TasksMax=64
NoNewPrivileges=true
PrivateTmp=true
PrivateDevices=true
ProtectHome=true
ProtectSystem=strict
ReadWritePaths=/etc/nginx /etc/letsencrypt /var/lib/letsencrypt /var/log/letsencrypt /var/log/nginx
ReadWritePaths=-/run/nginx.pid
ProtectKernelTunables=true
ProtectKernelModules=true
ProtectControlGroups=true
RestrictSUIDSGID=true
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK

[Install]
WantedBy=multi-user.target
EOF

write_file /etc/systemd/system/openvts-frontend.service 0644 root root <<EOF
[Unit]
Description=OpenVTS frontend
After=network-online.target openvts-backend.service
Wants=network-online.target
Requires=openvts-backend.service
PartOf=openvts.target

[Service]
Type=simple
User=openvts
Group=openvts
WorkingDirectory=${APP_DIR}/app/frontend
EnvironmentFile=${APP_DIR}/app/frontend/.env.production
EnvironmentFile=-/etc/openvts/frontend.env.override
Environment=HOME=/var/lib/openvts
Environment=PATH=${RUNTIME_PATH}
Environment=HOSTNAME=127.0.0.1
Environment=PORT=3000
Environment=NODE_OPTIONS=--max-old-space-size=${FRONTEND_NODE_HEAP_MB}
ExecStart=${NODE_BIN} server.js
Restart=always
RestartSec=5
TimeoutStopSec=45
KillMode=mixed
UMask=0027
LimitNOFILE=65535
NoNewPrivileges=true
PrivateTmp=true
PrivateDevices=true
ProtectHome=true
ProtectSystem=full
ProtectKernelTunables=true
ProtectKernelModules=true
ProtectControlGroups=true
RestrictSUIDSGID=true
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6

[Install]
WantedBy=openvts.target
EOF

write_file /etc/systemd/system/openvts-backend.service 0644 root root <<EOF
[Unit]
Description=OpenVTS backend
After=network-online.target ${PG_UNIT} openvts-redis.service openvts-service-manager.service
Wants=network-online.target openvts-service-manager.service
Requires=${PG_UNIT} openvts-redis.service
PartOf=openvts.target

[Service]
Type=simple
User=openvts
Group=openvts
WorkingDirectory=${APP_DIR}/app/backend
EnvironmentFile=${APP_DIR}/app/backend/.env
EnvironmentFile=-/etc/openvts/backend.env.override
Environment=HOME=/var/lib/openvts
Environment=NPM_CONFIG_CACHE=${RUNTIME_NPM_CACHE_DIR}
Environment=PATH=${RUNTIME_PATH}
Environment=OPENVTS_PSQL_BIN=${PG_PSQL}
Environment=NODE_OPTIONS=--max-old-space-size=${BACKEND_NODE_HEAP_MB}
ExecStartPre=${PG_PSQL} --version
ExecStart=${NODE_BIN} dist/main.js
Restart=always
RestartSec=5
TimeoutStopSec=90
KillMode=mixed
UMask=0027
LimitNOFILE=262144
NoNewPrivileges=true
PrivateTmp=true
PrivateDevices=true
ProtectHome=true
ProtectSystem=full
ProtectKernelTunables=true
ProtectKernelModules=true
ProtectControlGroups=true
RestrictSUIDSGID=true
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6

[Install]
WantedBy=openvts.target
EOF

write_file /etc/systemd/system/openvts-listener.service 0644 root root <<EOF
[Unit]
Description=OpenVTS GPS listener
After=network-online.target ${PG_UNIT} openvts-redis.service openvts-backend.service
Wants=network-online.target
Requires=${PG_UNIT} openvts-redis.service openvts-backend.service
PartOf=openvts.target

[Service]
Type=simple
User=openvts
Group=openvts
WorkingDirectory=${APP_DIR}/app/listener
EnvironmentFile=${APP_DIR}/app/listener/.env
EnvironmentFile=-/etc/openvts/listener.env.override
Environment=HOME=/var/lib/openvts
Environment=PATH=${RUNTIME_PATH}
Environment=NODE_OPTIONS=--max-old-space-size=${LISTENER_NODE_HEAP_MB}
ExecStart=${NODE_BIN} dist/main.js
Restart=always
RestartSec=5
TimeoutStopSec=90
KillMode=mixed
UMask=0027
LimitNOFILE=262144
NoNewPrivileges=true
PrivateTmp=true
PrivateDevices=true
ProtectHome=true
ProtectSystem=full
ProtectKernelTunables=true
ProtectKernelModules=true
ProtectControlGroups=true
RestrictSUIDSGID=true
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6

[Install]
WantedBy=openvts.target
EOF

write_file /etc/systemd/system/openvts.target 0644 root root <<EOF
[Unit]
Description=OpenVTS complete application stack
Requires=${PG_UNIT} openvts-redis.service
Wants=network-online.target nginx.service openvts-service-manager.service openvts-backend.service openvts-listener.service openvts-frontend.service
After=network-online.target ${PG_UNIT} openvts-redis.service openvts-service-manager.service

[Install]
WantedBy=multi-user.target
EOF

install -d -o root -g root -m 0755 "/etc/systemd/system/${PG_UNIT}.d" /etc/systemd/system/nginx.service.d
write_file "/etc/systemd/system/${PG_UNIT}.d/20-openvts-restart.conf" 0644 root root <<'EOF'
[Service]
Restart=on-failure
RestartSec=5
EOF
write_file /etc/systemd/system/nginx.service.d/20-openvts-restart.conf 0644 root root <<'EOF'
[Service]
Restart=on-failure
RestartSec=3
EOF

{
  printf 'OPENVTS_APP_DIR=%q\n' "$APP_DIR"
  printf 'OPENVTS_PG_MAJOR=%q\n' "$PG_MAJOR"
  printf 'OPENVTS_PG_CLUSTER=%q\n' "$PG_CLUSTER"
  printf 'OPENVTS_PG_PORT=%q\n' "$PG_PORT"
  printf 'OPENVTS_PG_UNIT=%q\n' "$PG_UNIT"
  printf 'OPENVTS_REDIS_PORT=%q\n' "$REDIS_PORT"
  printf 'OPENVTS_GPS_TCP_RULES=%q\n' "$GPS_TCP_RULES"
  printf 'OPENVTS_FIREWALL_MANAGED=%q\n' "$((1-SKIP_FIREWALL))"
  printf 'OPENVTS_PHYSICAL_RAM_MB=%q\n' "$PHYSICAL_MEM_MB"
  printf 'OPENVTS_SWAP_MB=%q\n' "$CURRENT_SWAP_MB"
  printf 'OPENVTS_DOMAIN=%q\n' "$DOMAIN"
  printf 'OPENVTS_EMAIL=%q\n' "$EMAIL"
  printf 'OPENVTS_NODE_VERSION=%q\n' "$NODE_VERSION"
  printf 'OPENVTS_INSTALLER_VERSION=%q\n' "$INSTALLER_VERSION"
  printf 'OPENVTS_INSTALLER_BUILD_ID=%q\n' "$INSTALLER_BUILD_ID"
  printf 'OPENVTS_BUNDLE_MODE=%q\n' "$BUNDLE_MODE"
} | write_file /etc/openvts/install.conf 0644 root root

write_file /usr/local/bin/openvts 0755 root root <<'CONTROL'
#!/usr/bin/env bash
set -Eeuo pipefail

if (( EUID != 0 )); then
  command -v sudo >/dev/null 2>&1 || { echo "sudo is not available; run this as root." >&2; exit 1; }
  exec sudo -- "$0" "$@"
fi

[[ -r /etc/openvts/install.conf ]] || { echo "OpenVTS installation metadata was not found." >&2; exit 1; }
# shellcheck disable=SC1091
source /etc/openvts/install.conf
PG_BIN="/usr/lib/postgresql/${OPENVTS_PG_MAJOR}/bin"
PG_PSQL="$PG_BIN/psql"
PG_ISREADY="$PG_BIN/pg_isready"
CMD="${1:-status}"
COMPONENT="${2:-all}"

app_units=(openvts-backend.service openvts-listener.service openvts-frontend.service)

case "$CMD" in
  start)
    systemctl start "$OPENVTS_PG_UNIT" openvts-redis.service nginx.service openvts-service-manager.service
    systemctl start openvts.target
    echo "OpenVTS start command completed."
    ;;
  stop)
    systemctl stop openvts.target
    echo "OpenVTS application services and the dedicated Redis service have stopped. PostgreSQL and Nginx were left running safely."
    ;;
  restart)
    systemctl start "$OPENVTS_PG_UNIT" openvts-redis.service nginx.service openvts-service-manager.service
    systemctl restart "${app_units[@]}"
    systemctl start openvts.target
    echo "OpenVTS restart command completed."
    ;;
  status)
    printf '%-32s %s\n' openvts.target "$(systemctl is-active openvts.target 2>/dev/null || true)"
    for unit in "$OPENVTS_PG_UNIT" openvts-redis.service nginx.service openvts-service-manager.service "${app_units[@]}"; do
      printf '%-32s %s\n' "$unit" "$(systemctl is-active "$unit" 2>/dev/null || true)"
    done
    ;;
  logs)
    case "$COMPONENT" in
      frontend) units=(-u openvts-frontend.service) ;;
      backend) units=(-u openvts-backend.service) ;;
      listener) units=(-u openvts-listener.service) ;;
      redis) units=(-u openvts-redis.service) ;;
      nginx) units=(-u nginx.service) ;;
      manager) units=(-u openvts-service-manager.service) ;;
      ssl) units=(-u openvts-service-manager.service -u certbot.service) ;;
      all) units=(-u openvts-frontend.service -u openvts-backend.service -u openvts-listener.service -u openvts-redis.service -u openvts-service-manager.service) ;;
      *) echo "Component: frontend|backend|listener|redis|nginx|manager|ssl|all" >&2; exit 2 ;;
    esac
    exec journalctl "${units[@]}" -n 200 -f
    ;;
  doctor)
    REDIS_PASSWORD="$(awk -F= '$1=="REDIS_PASSWORD" {sub(/^[^=]*=/, ""); print; exit}' /etc/openvts/openvts.secrets)"
    [[ -n "$REDIS_PASSWORD" ]] || { echo "REDIS_PASSWORD missing in secrets file." >&2; exit 1; }
    failures=0
    check() {
      local label="$1"; shift
      if "$@" >/dev/null 2>&1; then printf 'OK   %s\n' "$label"; else printf 'FAIL %s\n' "$label"; failures=$((failures+1)); fi
    }
    redis_ping() {
      [[ "$(REDISCLI_AUTH="$REDIS_PASSWORD" redis-cli -h 127.0.0.1 -p "$OPENVTS_REDIS_PORT" ping 2>/dev/null)" == PONG ]]
    }
    extension_exists() {
      local database="$1" extension="$2" result
      [[ -x "$PG_PSQL" ]] || return 1
      result="$(cd / && runuser -u postgres -- "$PG_PSQL" -X -p "$OPENVTS_PG_PORT" -d "$database" -Atqc \
        "SELECT 1 FROM pg_extension WHERE extname='${extension}'" 2>/dev/null)"
      [[ "$result" == 1 ]]
    }
    hourly_report_aggregate_registered() {
      local result
      [[ -x "$PG_PSQL" ]] || return 1
      result="$(cd / && runuser -u postgres -- "$PG_PSQL" -X -p "$OPENVTS_PG_PORT" -d OpenVTS_logs -Atqc \
        "SELECT CASE WHEN to_regclass('public.telemetry_hourly') IS NOT NULL AND EXISTS (SELECT 1 FROM timescaledb_information.continuous_aggregates WHERE view_schema='public' AND view_name='telemetry_hourly') THEN 1 ELSE 0 END" 2>/dev/null)"
      [[ "$result" == 1 ]]
    }
proxy_curl() {
  local host="${OPENVTS_DOMAIN:-_}" base="http://127.0.0.1"
  local -a resolve=()
  if [[ "$host" != "_" ]]; then
    resolve=(--resolve "${host}:80:127.0.0.1" --resolve "${host}:443:127.0.0.1")
    base="http://${host}"
    if [[ -f "/etc/letsencrypt/live/${host}/fullchain.pem" ]]; then base="https://${host}"; fi
  fi
  local path="$1"; shift
  curl "${resolve[@]}" --noproxy '*' "$@" "${base}${path}"
}

    nginx_proxy_active() {
      proxy_curl / -sSI --max-time 10 | tr -d '\r' | grep -qi '^X-OpenVTS-Proxy: active$'
    }
    ufw_active() {
      LC_ALL=C ufw status 2>/dev/null | grep -q '^Status: active$'
    }
    ufw_allows() {
      local rule="$1"
      LC_ALL=C ufw status 2>/dev/null | grep -Eq "^${rule//:/\\:}[[:space:]]+ALLOW"
    }
    check "PostgreSQL" "$PG_ISREADY" -h 127.0.0.1 -p "$OPENVTS_PG_PORT"
    check "Redis" redis_ping
    check "Nginx service" systemctl is-active --quiet nginx.service
    check "Local service manager" systemctl is-active --quiet openvts-service-manager.service
    check "Local service-manager socket" test -S /run/openvts-service-manager/control.sock
    check "Certificate renewal timer" systemctl is-active --quiet certbot.timer
    check "Nginx config" nginx -t
    check "Nginx OpenVTS proxy" nginx_proxy_active
    check "Nginx backend proxy" proxy_curl /api/health -fsS --max-time 10
    if [[ "${OPENVTS_FIREWALL_MANAGED:-1}" == 1 ]]; then
      check "Firewall active" ufw_active
      check "Firewall 80/tcp" ufw_allows 80/tcp
      check "Firewall 443/tcp" ufw_allows 443/tcp
      IFS=, read -r -a gps_rules <<<"${OPENVTS_GPS_TCP_RULES:-${OPENVTS_GPS_TCP_PORT_RANGE:-7500:7800}}"
      for rule in "${gps_rules[@]}"; do
        check "Firewall GPS ${rule}/tcp" ufw_allows "${rule}/tcp"
      done
    else
      printf 'SKIP Firewall (not managed by installer)
'
    fi
    check "PostGIS primary" extension_exists OpenVTS_db postgis
    check "TimescaleDB logs" extension_exists OpenVTS_logs timescaledb
    check "Hourly reports aggregate" hourly_report_aggregate_registered
    check "Backend ready" curl -fsS --max-time 10 http://127.0.0.1:4000/health/ready
    check "Listener ready" curl -fsS --max-time 10 http://127.0.0.1:5055/health/ready
    check "Frontend" curl -fsS --max-time 10 http://127.0.0.1:3000/
    if (( failures == 0 )); then echo "All primary checks passed."; else echo "$failures check(s) failed. Run openvts logs." >&2; exit 1; fi
    ;;
  -h|--help|help)
    echo "Usage: openvts {start|stop|restart|status|doctor|logs [component]}"
    ;;
  *)
    echo "Usage: openvts {start|stop|restart|status|doctor|logs [component]}" >&2
    exit 2
    ;;
esac
CONTROL

systemctl daemon-reload
systemctl enable "$PG_UNIT" openvts-redis.service nginx.service openvts-service-manager.service openvts.target >/dev/null

proxy_curl() {
  local host="${DOMAIN:-_}" base="http://127.0.0.1"
  local -a resolve=()
  if [[ "$host" != "_" ]]; then
    resolve=(--resolve "${host}:80:127.0.0.1" --resolve "${host}:443:127.0.0.1")
    base="http://${host}"
    if [[ -f "/etc/letsencrypt/live/${host}/fullchain.pem" ]]; then base="https://${host}"; fi
  fi
  local path="$1"; shift
  curl "${resolve[@]}" --noproxy '*' "$@" "${base}${path}"
}

log "Configuring the Nginx reverse proxy"
install -d -o root -g root -m 0755 /etc/nginx/openvts-disabled
NGINX_DEFAULT_CONF_BACKUP="$WORK_DIR/nginx-default.conf"
NGINX_DEFAULT_SITE_BACKUP="$WORK_DIR/nginx-sites-default"
if (( NGINX_CAN_REPLACE_DEFAULTS == 1 )); then
  if [[ -e /etc/nginx/conf.d/default.conf ]]; then
    cp -a /etc/nginx/conf.d/default.conf "$NGINX_DEFAULT_CONF_BACKUP"
    mv -- /etc/nginx/conf.d/default.conf /etc/nginx/openvts-disabled/default.conf
  fi
  if [[ -e /etc/nginx/sites-enabled/default ]]; then
    cp -aL /etc/nginx/sites-enabled/default "$NGINX_DEFAULT_SITE_BACKUP"
    mv -- /etc/nginx/sites-enabled/default /etc/nginx/openvts-disabled/sites-enabled-default
  fi
fi

NGINX_PREVIOUS="$WORK_DIR/openvts.nginx.previous"
[[ -f /etc/nginx/conf.d/openvts.conf ]] && cp -a /etc/nginx/conf.d/openvts.conf "$NGINX_PREVIOUS"
TLS_LISTEN_CONFIG=""
TLS_ACTIVE=0
HTTP_LISTEN_CONFIG="    listen 80;"
NGINX_IPV6_HTTP=""
if [[ "$DOMAIN" == "_" ]]; then
  # IP/no-domain installs must own the default HTTP server so the packaged
  # nginx welcome site can never win request routing.
  HTTP_LISTEN_CONFIG="    listen 80 default_server;"
  [[ -s /proc/net/if_inet6 ]] && NGINX_IPV6_HTTP="    listen [::]:80 default_server;"
else
  [[ -s /proc/net/if_inet6 ]] && NGINX_IPV6_HTTP="    listen [::]:80;"
fi
if [[ "$DOMAIN" != "_" && \
      -f "/etc/letsencrypt/live/${DOMAIN}/fullchain.pem" && \
      -f "/etc/letsencrypt/live/${DOMAIN}/privkey.pem" ]]; then
  TLS_ACTIVE=1
  TLS_LISTEN_CONFIG="    listen 443 ssl;"
  [[ -s /proc/net/if_inet6 ]] && TLS_LISTEN_CONFIG+="\n    listen [::]:443 ssl;"
  TLS_LISTEN_CONFIG+="\n    ssl_certificate /etc/letsencrypt/live/${DOMAIN}/fullchain.pem;\n    ssl_certificate_key /etc/letsencrypt/live/${DOMAIN}/privkey.pem;\n    ssl_protocols TLSv1.2 TLSv1.3;"
fi
# Certbot can add per-domain HTTPS vhosts to this file. Application upgrades
# must not replace those edits with a fresh HTTP-only template.
if [[ -f /etc/nginx/conf.d/openvts.conf ]] && grep -Eq 'OpenVTS SSL manager: preserve managed virtual hosts|managed by Certbot' /etc/nginx/conf.d/openvts.conf; then
  info "Preserving existing Certbot-managed Open VTS Nginx configuration and certificates."
else
write_file /etc/nginx/conf.d/openvts.conf 0644 root root <<EOF
map \$http_upgrade \$openvts_connection_upgrade {
    default upgrade;
    ''      close;
}

upstream openvts_frontend {
    server 127.0.0.1:3000;
    keepalive 32;
}

upstream openvts_backend {
    server 127.0.0.1:4000;
    keepalive 64;
}

server {
$(printf '%b' "$HTTP_LISTEN_CONFIG")
$(printf '%b' "$NGINX_IPV6_HTTP")
$(printf '%b' "$TLS_LISTEN_CONFIG")
    server_name ${DOMAIN};

    add_header X-OpenVTS-Proxy active always;
    client_max_body_size 50m;

    location = /api {
        return 308 /api/;
    }

    # Stable public health endpoint. The backend's native readiness endpoint is
    # /health/ready; expose /api/health for operators and load balancers.
    location = /api/health {
        proxy_pass http://openvts_backend/health/ready;
        proxy_http_version 1.1;
        proxy_set_header Host \$host;
        proxy_set_header X-Real-IP \$remote_addr;
        proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto \$scheme;
        proxy_read_timeout 30s;
        proxy_buffering off;
    }

    location /api/ {
        proxy_pass http://openvts_backend/;
        proxy_http_version 1.1;
        proxy_set_header Host \$host;
        proxy_set_header X-Real-IP \$remote_addr;
        proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto \$scheme;
        proxy_read_timeout 300s;
        # Buffer ordinary JSON/API responses so slow clients do not occupy a
        # backend connection. True SSE endpoints emit X-Accel-Buffering: no,
        # which Nginx honours per response without disabling buffering here.
        proxy_buffering on;
        proxy_buffer_size 16k;
        proxy_buffers 16 16k;
        proxy_busy_buffers_size 64k;
    }

    location /socket.io/ {
        proxy_pass http://openvts_backend;
        proxy_http_version 1.1;
        proxy_set_header Upgrade \$http_upgrade;
        proxy_set_header Connection \$openvts_connection_upgrade;
        proxy_set_header Host \$host;
        proxy_set_header X-Real-IP \$remote_addr;
        proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto \$scheme;
        proxy_read_timeout 3600s;
        proxy_send_timeout 3600s;
    }

    location /uploads/ {
        proxy_pass http://openvts_backend;
        proxy_http_version 1.1;
        proxy_set_header Host \$host;
        proxy_set_header X-Real-IP \$remote_addr;
        proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto \$scheme;
    }

    location / {
        proxy_pass http://openvts_frontend;
        proxy_http_version 1.1;
        proxy_set_header Upgrade \$http_upgrade;
        proxy_set_header Connection \$openvts_connection_upgrade;
        proxy_set_header Host \$host;
        proxy_set_header X-Real-IP \$remote_addr;
        proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto \$scheme;
        proxy_read_timeout 300s;
    }
}
EOF
fi

if ! nginx -t; then
  mv -- /etc/nginx/conf.d/openvts.conf "$APP_DIR/start/openvts.nginx.invalid.$RELEASE_ID"
  if [[ -f "$NGINX_PREVIOUS" ]]; then cp -a "$NGINX_PREVIOUS" /etc/nginx/conf.d/openvts.conf; fi
  if [[ -f "$NGINX_DEFAULT_CONF_BACKUP" ]]; then cp -a "$NGINX_DEFAULT_CONF_BACKUP" /etc/nginx/conf.d/default.conf; fi
  if [[ -f "$NGINX_DEFAULT_SITE_BACKUP" ]]; then
    install -d -m 0755 /etc/nginx/sites-enabled
    cp -a "$NGINX_DEFAULT_SITE_BACKUP" /etc/nginx/sites-enabled/default
  fi
  die "Nginx configuration test failed; the previous configuration was restored."
fi
systemctl enable --now nginx.service >/dev/null
# enable --now does not reload an already-running nginx process. Explicitly
# reload so a rerun immediately replaces the packaged welcome page.
systemctl reload nginx.service || systemctl restart nginx.service

NGINX_ACTIVE_CONFIG="$(nginx -T 2>&1)"
grep -Fq '# configuration file /etc/nginx/conf.d/openvts.conf:' <<<"$NGINX_ACTIVE_CONFIG" ||
  die "Nginx did not include /etc/nginx/conf.d/openvts.conf."
for _ in $(seq 1 20); do
  if proxy_curl / -sSI --max-time 5 | tr -d '\r' | grep -qi '^X-OpenVTS-Proxy: active$'; then
    break
  fi
  sleep 1
done
proxy_curl / -sSI --max-time 5 | tr -d '\r' | grep -qi '^X-OpenVTS-Proxy: active$' ||
  die "The OpenVTS reverse proxy did not become active in the running Nginx configuration."

if (( SKIP_FIREWALL == 0 )); then
  log "Configuring the Ubuntu firewall"
  # Allow all detected SSH ports before enabling UFW to avoid cloud lockout.
  SSH_PORTS=()
  if [[ "${SSH_CONNECTION:-}" =~ [[:space:]]([0-9]+)$ ]]; then
    SSH_PORTS+=("${BASH_REMATCH[1]}")
  fi
  while IFS= read -r endpoint; do
    ssh_port="${endpoint##*:}"
    [[ "$ssh_port" =~ ^[0-9]+$ ]] && SSH_PORTS+=("$ssh_port")
  done < <(ss -H -ltnp 2>/dev/null | awk '/sshd/ {print $4}')
  if [[ -x /usr/sbin/sshd ]]; then
    while IFS= read -r ssh_port; do
      [[ "$ssh_port" =~ ^[0-9]+$ ]] && SSH_PORTS+=("$ssh_port")
    done < <(/usr/sbin/sshd -T 2>/dev/null | awk '$1=="port" {print $2}')
  fi
  mapfile -t SSH_PORTS < <(printf '%s
' "${SSH_PORTS[@]:-22}" | awk '/^[0-9]+$/' | sort -nu)
  ((${#SSH_PORTS[@]} > 0)) || SSH_PORTS=(22)
  for ssh_port in "${SSH_PORTS[@]}"; do
    ufw allow "${ssh_port}/tcp" comment 'SSH' >/dev/null
  done
  ufw allow 80/tcp comment 'OpenVTS HTTP' >/dev/null
  ufw allow 443/tcp comment 'OpenVTS HTTPS' >/dev/null
  IFS=, read -r -a gps_rules <<<"$GPS_TCP_RULES"
  for rule in "${gps_rules[@]}"; do
    ufw allow "${rule}/tcp" comment 'OpenVTS GPS TCP' >/dev/null
  done
  ufw --force enable >/dev/null
  ufw reload >/dev/null
  LC_ALL=C ufw status | grep -q '^Status: active$' || die "UFW did not become active."
  LC_ALL=C ufw status | grep -Eq '^80/tcp[[:space:]]+ALLOW' || die "UFW does not contain an allow rule for 80/tcp."
  LC_ALL=C ufw status | grep -Eq '^443/tcp[[:space:]]+ALLOW' || die "UFW does not contain an allow rule for 443/tcp."
  for rule in "${gps_rules[@]}"; do
    LC_ALL=C ufw status | grep -Eq "^${rule}/tcp[[:space:]]+ALLOW" ||
      die "UFW does not contain an allow rule for ${rule}/tcp."
  done
else
  warn "Ubuntu UFW configuration skipped by --skip-firewall."
fi

if [[ -n "$EMAIL" ]]; then
  log "Certbot is attempting to obtain an HTTPS certificate"
  if certbot --nginx --non-interactive --agree-tos --redirect --keep-until-expiring \
      --email "$EMAIL" -d "$DOMAIN"; then
    TLS_ACTIVE=1
    systemctl enable --now certbot.timer >/dev/null 2>&1 || true
    info "The HTTPS certificate is active."
  else
    warn "The HTTPS certificate could not be created. Point $DOMAIN to this server in DNS, then run: sudo certbot --nginx -d $DOMAIN"
  fi
fi
nginx -t
systemctl reload nginx.service || systemctl restart nginx.service
systemctl enable --now certbot.timer openvts-service-manager.service >/dev/null
for ssl_unit in certbot.timer openvts-service-manager.service; do
  systemctl is-active --quiet "$ssl_unit" || die "Linux SSL prerequisite did not start: $ssl_unit"
done
# Retire only the exact auxiliary service introduced by 2.4.2. Never remove
# certificates, renewal state, private keys or the existing service manager.
if [[ -f /etc/systemd/system/openvts-ssl-manager.service ]]; then
  systemctl disable openvts-ssl-manager.service >/dev/null
  rm -f -- /etc/systemd/system/openvts-ssl-manager.service /usr/local/libexec/openvts-ssl-manager
  rm -f -- /run/openvts-ssl-manager/control.sock
  rmdir /run/openvts-ssl-manager 2>/dev/null || true
  rm -f -- "$APP_DIR/start/systemd/openvts-ssl-manager.service"
  systemctl daemon-reload
fi
LEGACY_SSL_MANAGER_RETIRED=1
SSL_INFRA_PAUSED=0

NGINX_VERSION="$(nginx -v 2>&1 | sed -E 's#.*nginx/##')"
NGINX_SNAPSHOT_DIR="$APP_DIR/start/nginx-${NGINX_VERSION}"
install -d -o root -g root -m 0755 "$NGINX_SNAPSHOT_DIR" "$APP_DIR/start/systemd" "$APP_DIR/start/certbot"
install -o root -g root -m 0644 /etc/nginx/conf.d/openvts.conf "$NGINX_SNAPSHOT_DIR/openvts.conf"
for unit_file in openvts.target openvts-frontend.service openvts-backend.service openvts-listener.service openvts-redis.service openvts-service-manager.service; do
  install -o root -g root -m 0644 "/etc/systemd/system/$unit_file" "$APP_DIR/start/systemd/$unit_file"
done
write_file "$APP_DIR/start/README-LINUX.txt" 0644 root root <<EOF
OpenVTS Linux deployment (installer ${INSTALLER_VERSION}, bundle mode ${BUNDLE_MODE})

Nginx binary: /usr/sbin/nginx (APT-managed version ${NGINX_VERSION})
Nginx live config: /etc/nginx/conf.d/openvts.conf
TLS client: Certbot (Linux replacement for Windows win-acme)
Systemd live units: /etc/systemd/system/openvts*
Control command: /usr/local/bin/openvts

Do not copy nginx.exe or win-acme into this Linux installation.
EOF
write_file "$APP_DIR/start/certbot/README.txt" 0644 root root <<EOF
On Linux, Certbot is used instead of win-acme.
Certificate status: sudo certbot certificates
Renewal test: sudo certbot renew --dry-run
EOF

if [[ -L "$APP_DIR/app" && -d "$APP_DIR/app/backend" ]]; then
  log "Backing up existing OpenVTS databases before new-release schema bootstrap"
  BACKUP_DIR="/var/backups/openvts/$RELEASE_ID"
  install -d -o root -g root -m 0700 "$BACKUP_DIR"
  # Stop application writers for the three-database backup. Shared PostgreSQL
  # and Nginx are not stopped. Any backup failure restarts the existing stack.
  systemctl stop openvts.target
  UPGRADE_WRITERS_STOPPED=1
  backup_ok=1
  for db in "$DB_PRIMARY" "$DB_LOGS" "$DB_ADDRESS"; do
    if ! (cd / && timeout --signal=TERM --kill-after=30 1800 runuser -u postgres -- \
        "$PG_BIN/pg_dump" -Fc -p "$PG_PORT" -d "$db") > "$BACKUP_DIR/$db.dump"; then
      backup_ok=0; break
    fi
    "$PG_BIN/pg_restore" --list "$BACKUP_DIR/$db.dump" >/dev/null || { backup_ok=0; break; }
  done
  if (( backup_ok == 0 )); then
    systemctl start openvts.target || true
    die "Database backup failed; activation cancelled. Review $BACKUP_DIR."
  fi
  cp -a /etc/openvts "$BACKUP_DIR/config"
  cp -a "$APP_DIR/app/backend/.env" "$BACKUP_DIR/backend.env"
  cp -a "$APP_DIR/app/listener/.env" "$BACKUP_DIR/listener.env"
  cp -a "$APP_DIR/app/frontend/.env.production" "$BACKUP_DIR/frontend.env.production"
  info "Upgrade backup: $BACKUP_DIR (logical dumps checked with pg_restore --list; restore rehearsal still required)"
fi

log "Activating the new release"
if [[ -e "$APP_DIR/app" && ! -L "$APP_DIR/app" ]]; then
  die "$APP_DIR/app is already a real directory. It was not overwritten automatically, to protect data."
fi
if [[ -L "$APP_DIR/app" ]]; then
  PREVIOUS_APP="$(readlink -f "$APP_DIR/app" || true)"
  [[ -z "$PREVIOUS_APP" || "$PREVIOUS_APP" == "$APP_DIR"/releases/*/app ]] ||
    die "The existing $APP_DIR/app symlink points outside the OpenVTS releases directory: $PREVIOUS_APP"
fi
TEMP_LINK="$APP_DIR/.app-link.$$"
ln -s "$RELEASE_APP" "$TEMP_LINK"
mv -Tf "$TEMP_LINK" "$APP_DIR/app"
ACTIVE_RELEASE="$RELEASE_APP"

systemctl daemon-reload
systemctl start "$PG_UNIT" openvts-redis.service nginx.service openvts-service-manager.service
systemctl restart openvts-backend.service openvts-listener.service openvts-frontend.service
systemctl start openvts.target
UPGRADE_WRITERS_STOPPED=0

wait_http() {
  local label="$1" url="$2" unit="${3:-}" elapsed=0 baseline_restarts=0 current_restarts=0 started=$SECONDS
  if [[ -n "$unit" ]]; then
    baseline_restarts="$(systemctl show "$unit" -p NRestarts --value 2>/dev/null || printf '0')"
    [[ "$baseline_restarts" =~ ^[0-9]+$ ]] || baseline_restarts=0
  fi
  while (( SECONDS - started < HEALTH_TIMEOUT )); do
    if curl --fail --silent --show-error --max-time 10 "$url" >/dev/null 2>&1; then
      info "$label is ready."
      return 0
    fi
    if [[ -n "$unit" ]]; then
      current_restarts="$(systemctl show "$unit" -p NRestarts --value 2>/dev/null || printf '0')"
      [[ "$current_restarts" =~ ^[0-9]+$ ]] || current_restarts=0
      if (( current_restarts >= baseline_restarts + 5 )); then
        warn "$label service crash-loop detected ($((current_restarts - baseline_restarts)) restarts): $unit"
        return 1
      fi
    fi
    sleep 5
    elapsed=$((SECONDS - started))
  done
  warn "$label health timeout (${HEALTH_TIMEOUT}s): $url"
  return 1
}

log "Running first-start database bootstrap and health checks"
HEALTH_OK=1
if wait_http "Backend" http://127.0.0.1:4000/health/ready openvts-backend.service; then
  # SELECT 1 readiness cannot prove report schemas, IANA names, or Timescale
  # aggregates work. Run the compiled release's read-only contract after schema
  # bootstrap, as the backend account and with the SAME systemd EnvironmentFiles.
  # A transient one-shot unit leaves no additional installed daemon/unit file.
  info "Checking Linux report database capabilities as the backend account"
  if ! timeout --signal=TERM --kill-after=10 100 systemd-run \
    --quiet --wait --pipe --collect --unit="openvts-report-preflight-$$" \
    --property=Type=exec --property=User=openvts --property=Group=openvts \
    --property="WorkingDirectory=$RELEASE_APP/backend" \
    --property="EnvironmentFile=$RELEASE_APP/backend/.env" \
    --property="EnvironmentFile=-/etc/openvts/backend.env.override" \
    --property=NoNewPrivileges=yes --property=ProtectSystem=strict \
    --property=PrivateTmp=yes --property=RuntimeMaxSec=90s \
    --property=TimeoutStopSec=5s \
    --setenv=HOME=/var/lib/openvts --setenv="PATH=$RUNTIME_PATH" \
    "$NODE_BIN" "$RELEASE_APP/backend/dist/user/reports/linux-report-preflight.js"; then
    warn "Linux report preflight failed; this release is not considered healthy."
    HEALTH_OK=0
  fi
  wait_http "Listener" http://127.0.0.1:5055/health/ready openvts-listener.service || HEALTH_OK=0
  wait_http "Frontend" http://127.0.0.1:3000/ openvts-frontend.service || HEALTH_OK=0
  if ! proxy_curl /api/health -fsS --max-time 15 >/dev/null; then
    warn "Nginx backend reverse-proxy health check failed: http://127.0.0.1/api/health"
    HEALTH_OK=0
  fi
  if ! proxy_curl / -sSI --max-time 10 | tr -d '\r' | grep -qi '^X-OpenVTS-Proxy: active$'; then
    warn "Nginx OpenVTS proxy marker header missing."
    HEALTH_OK=0
  fi
else
  HEALTH_OK=0
fi

if (( HEALTH_OK == 0 )); then
  journalctl -u openvts-backend.service -u openvts-listener.service -u openvts-frontend.service -n 120 --no-pager || true
  if [[ -n "$PREVIOUS_APP" && -d "$PREVIOUS_APP" ]]; then
    warn "Health check failed; restoring the previous release."
    TEMP_LINK="$APP_DIR/.app-rollback.$$"
    ln -s "$PREVIOUS_APP" "$TEMP_LINK"
    mv -Tf "$TEMP_LINK" "$APP_DIR/app"
    ACTIVE_RELEASE=""
    systemctl restart openvts-backend.service openvts-listener.service openvts-frontend.service || true
    die "The new release health check failed; the previous application files were restored. Database schema changes are NOT rolled back automatically. Backups: ${BACKUP_DIR:-not created}."
  fi
  systemctl stop openvts.target || true
  if [[ -L "$APP_DIR/app" && "$(readlink -f "$APP_DIR/app" || true)" == "$RELEASE_APP" ]]; then
    rm -f -- "$APP_DIR/app"
  fi
  ACTIVE_RELEASE=""
  die "The first-install health check failed. Review the journal output above and $LOG_FILE."
fi

# Keep the active release and one rollback release; remove older installer-managed
# releases so repeated upgrades do not silently consume the system disk.
while IFS= read -r old_release; do
  [[ "$old_release/app" == "$ACTIVE_RELEASE" ]] && continue
  [[ -n "$PREVIOUS_APP" && "$old_release/app" == "$PREVIOUS_APP" ]] && continue
  rm -rf -- "$old_release"
done < <(find "$APP_DIR/releases" -mindepth 1 -maxdepth 1 -type d -print)

log "Installation completed successfully"
info "Application: $APP_DIR/app -> $ACTIVE_RELEASE"
info "Nginx: $NGINX_VERSION"
info "PostgreSQL: $PG_MAJOR, PostGIS enabled, TimescaleDB: $TIMESCALE_VERSION"
info "Node.js: $NODE_VERSION; Redis port: $REDIS_PORT; Redis maxmemory: ${REDIS_MAXMEMORY_MB}MB"
info "Memory profile: ${PHYSICAL_MEM_MB}MB RAM, ${CURRENT_SWAP_MB}MB swap; Node heaps F/B/L=${FRONTEND_NODE_HEAP_MB}/${BACKEND_NODE_HEAP_MB}/${LISTENER_NODE_HEAP_MB}MB"
info "Control: sudo openvts {start|stop|restart|status|doctor|logs}"
info "Persistent data: /var/lib/openvts; logs: /var/log/openvts"
if [[ "$DOMAIN" == "_" ]]; then
  info "Browser: http://SERVER_IP/"
elif (( TLS_ACTIVE == 1 )); then
  info "Browser: https://${DOMAIN}/"
else
  info "Browser: http://${DOMAIN}/ (the HTTPS certificate is not active yet)"
fi
if (( SKIP_FIREWALL == 0 )); then
  info "Ubuntu UFW: SSH, 80/tcp, 443/tcp, and GPS TCP ${GPS_TCP_RULES} are allowed."
else
  info "Ubuntu UFW: installer management skipped."
fi
info "Also allow ports 80, 443, and GPS TCP ${GPS_TCP_RULES} in the AWS Security Group, GCP VPC firewall, or DigitalOcean Cloud Firewall."