kfj001/kmtnflower
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

incorporates and implements mobile build service functionality.

Browse Source
This commit is contained in:
kfj001
2026-07-08 15:02:22 -07:00
parent 7869574fe2
commit 2c66a802c0
12 changed files with 491 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/api/mobile_builds.py
+227 -13
View File
@@ -68,7 +68,20 @@ def _check_mobileprovision_for_carplay(path: Path) -> bool:
return "carplay" in text
def _run_sync_command(cmd: list[str]) -> tuple[int, str]:
result = subprocess.run(cmd, capture_output=True, text=True)
output = (result.stdout or "") + ("\n" + result.stderr if result.stderr else "")
return result.returncode, output.strip()
async def _run_command(cmd: list[str]) -> tuple[int, str]:
return await asyncio.to_thread(_run_sync_command, cmd)
# ── Build container SSH helpers (direct SSH over Docker network) ──
def _build_ssh_prefix(host: str) -> list[str]:
"""Build SSH prefix for connecting to the build container over the Docker network."""
if not host:
raise RuntimeError("Build host is not configured")
return [
@@ -85,6 +98,7 @@ def _build_ssh_prefix(host: str) -> list[str]:
def _build_scp_prefix(host: str) -> list[str]:
"""Build SCP prefix for connecting to the build container over the Docker network."""
if not host:
raise RuntimeError("Build host is not configured")
return [
@@ -99,14 +113,25 @@ def _build_scp_prefix(host: str) -> list[str]:
]
def _run_sync_command(cmd: list[str]) -> tuple[int, str]:
result = subprocess.run(cmd, capture_output=True, text=True)
output = (result.stdout or "") + ("\n" + result.stderr if result.stderr else "")
return result.returncode, output.strip()
async def _ssh_run(host: str, cmd: str) -> tuple[int, str]:
"""Run a command inside the build container via SSH."""
ssh_prefix = _build_ssh_prefix(host)
full_cmd = ssh_prefix + [cmd]
return await _run_command(full_cmd)
async def _run_command(cmd: list[str]) -> tuple[int, str]:
return await asyncio.to_thread(_run_sync_command, cmd)
async def _scp_to_container(host: str, local_path: str, remote_path: str) -> tuple[int, str]:
"""SCP a file from the backend into the build container."""
scp_prefix = _build_scp_prefix(host)
full_cmd = scp_prefix + [str(local_path), f"{settings.MOBILE_BUILD_SSH_USER}@{host}:{remote_path}"]
return await _run_command(full_cmd)
async def _scp_from_container(host: str, remote_path: str, local_path: str) -> tuple[int, str]:
"""SCP a file from the build container to the backend."""
scp_prefix = _build_scp_prefix(host)
full_cmd = scp_prefix + [f"{settings.MOBILE_BUILD_SSH_USER}@{host}:{remote_path}", str(local_path)]
return await _run_command(full_cmd)
async def _append_log(session: AsyncSession, request: MobileBuildRequest, line: str) -> None:
@@ -142,8 +167,8 @@ def _preflight_issues(request: MobileBuildRequest, files: list[MobileBuildUpload
if not request.build_number.strip():
issues.append("Build number is required.")
if request.platform_android and not settings.MOBILE_BUILD_ANDROID_HOST:
issues.append("Android build host is not configured.")
if request.platform_android and not settings.FLUTTER_BUILD_HOST:
issues.append("Android build container host is not configured.")
if request.platform_ios and not settings.MOBILE_BUILD_IOS_HOST:
issues.append("iOS build host is not configured.")
@@ -256,7 +281,8 @@ async def _copy_worker_artifact(
local_dir.mkdir(parents=True, exist_ok=True)
ssh_prefix = _build_ssh_prefix(host)
list_cmd = ssh_prefix + [f"ls -1t {shlex.quote(artifact_glob)} 2>/dev/null | head -n 1"]
# Don't shlex.quote the glob — the shell needs to expand the wildcard
list_cmd = ssh_prefix + [f"ls -1t {artifact_glob} 2>/dev/null | head -n 1"]
code, out = await _run_command(list_cmd)
if code != 0 or not out:
raise RuntimeError(f"No {platform} artifact found using glob {artifact_glob}")
@@ -362,6 +388,196 @@ async def _run_platform_build(
await _append_log(session, request, f"[{platform}] Artifact copied successfully")
# ── Container-based Android build flow (direct SSH to build service) ──
async def _copy_container_artifact(
session: AsyncSession,
request: MobileBuildRequest,
host: str,
project_dir: str,
) -> None:
"""Copy the APK from the Flutter build container back to the backend via SCP."""
_, artifact_dir = _ensure_dirs()
local_dir = artifact_dir / f"request_{request.id}" / "android"
local_dir.mkdir(parents=True, exist_ok=True)
# Find the APK inside the container
apk_glob = settings.FLUTTER_APK_GLOB
# Don't shlex.quote the glob — the shell needs to expand the wildcard
list_cmd = f"ls -1t {apk_glob} 2>/dev/null | head -n 1"
code, out = await _ssh_run(host, list_cmd)
if code != 0 or not out:
raise RuntimeError(f"No APK artifact found using glob {apk_glob}")
apk_path = out.strip()
file_name = Path(apk_path).name
# SCP directly from build container to backend
local_path = local_dir / file_name
code, out = await _scp_from_container(host, apk_path, local_path)
if code != 0:
raise RuntimeError(f"Failed to SCP APK from build container: {out}")
artifact = MobileBuildArtifact(
request_id=request.id,
platform="android",
artifact_type="apk",
file_name=file_name,
file_path=str(local_path),
sha256=_hash_file(local_path),
size=local_path.stat().st_size,
)
session.add(artifact)
await session.commit()
async def _run_flutter_container_build(
session: AsyncSession,
request: MobileBuildRequest,
host: str,
) -> None:
"""Execute an Android build inside the Flutter build container via direct SSH.
Flow:
1. Clone the Flutter project (if not already cloned)
2. Run flutter pub get
3. Set up Python venv + activate
4. Copy theme.json into the project directory
5. Copy signing files (keystore + descriptor) into the project
6. Run build_whitelabel.py
7. Run flutter build apk
8. Copy the APK artifact back to the backend
"""
project_dir = settings.FLUTTER_PROJECT_DIR # ~/kryz-go-flutter
repo_url = settings.FLUTTER_PROJECT_REPO_URL
# Step 1: Clone the Flutter project (skip if already exists)
await _append_log(session, request, "[android] Checking Flutter project directory...")
check_cmd = f"test -d {shlex.quote(project_dir)}/.git"
code, out = await _ssh_run(host, check_cmd)
directory_exists = (code == 0)
if not directory_exists:
await _append_log(session, request, f"[android] Cloning {repo_url} into {project_dir}")
clone_cmd = f"cd /home/vscode && git clone {shlex.quote(repo_url)} kryz-go-flutter"
code, out = await _ssh_run(host, clone_cmd)
await _append_log(session, request, f"[android] Clone output: {out}")
if code != 0:
raise RuntimeError(f"Git clone failed: {out}")
else:
await _append_log(session, request, "[android] Project directory exists, pulling latest...")
pull_cmd = f"cd {shlex.quote(project_dir)} && git pull"
code, out = await _ssh_run(host, pull_cmd)
await _append_log(session, request, f"[android] Pull output: {out}")
if code != 0:
# Directory may be stale/corrupt — remove and re-clone
await _append_log(session, request, "[android] Pull failed, re-cloning...")
rm_cmd = f"rm -rf {shlex.quote(project_dir)}"
await _ssh_run(host, rm_cmd)
clone_cmd = f"cd /home/vscode && git clone {shlex.quote(repo_url)} kryz-go-flutter"
code, out = await _ssh_run(host, clone_cmd)
await _append_log(session, request, f"[android] Re-clone output: {out}")
if code != 0:
raise RuntimeError(f"Git re-clone failed: {out}")
# Step 2: flutter pub get
await _append_log(session, request, "[android] Running flutter pub get...")
pub_cmd = f"cd {shlex.quote(project_dir)} && flutter pub get"
code, out = await _ssh_run(host, pub_cmd)
await _append_log(session, request, f"[android] Pub get output: {out}")
if code != 0:
raise RuntimeError(f"flutter pub get failed: {out}")
# Step 3: Python venv setup
await _append_log(session, request, "[android] Setting up Python venv...")
venv_cmd = (
f"cd {shlex.quote(project_dir)} && "
f"python3 -m venv .venv && "
f"source .venv/bin/activate && "
f"pip install --upgrade pip"
)
code, out = await _ssh_run(host, venv_cmd)
await _append_log(session, request, f"[android] Venv setup output: {out}")
if code != 0:
raise RuntimeError(f"Python venv setup failed: {out}")
# Step 4: Copy theme.json into the project directory
theme_path = Path(settings.THEME_JSON_PATH)
if theme_path.exists():
code, out = await _scp_to_container(host, theme_path, f"{project_dir}/theme.json")
if code != 0:
raise RuntimeError(f"Failed to copy theme.json to build container: {out}")
await _append_log(session, request, "[android] theme.json copied to build container")
else:
await _append_log(session, request, "[android] theme.json not found, skipping")
# Step 5: Copy signing files into the container
files = await _load_files(session, request.id)
android_files = [f for f in files if f.platform == "android"]
for row in android_files:
local_path = Path(row.stored_path)
if not local_path.exists():
raise RuntimeError(f"Signing file missing: {row.original_name}")
# SCP directly to build container
remote_path = f"{project_dir}/{row.original_name}"
code, out = await _scp_to_container(host, local_path, remote_path)
if code != 0:
raise RuntimeError(f"Failed to SCP {row.original_name} to build container: {out}")
await _append_log(session, request, f"[android] Signing file {row.original_name} copied to build container")
# Step 6: Run build_whitelabel.py
await _append_log(session, request, "[android] Running build_whitelabel.py...")
whitelabel_cmd = (
f"cd {shlex.quote(project_dir)} && "
f"source .venv/bin/activate && "
f"python3 build_whitelabel.py"
)
code, out = await _ssh_run(host, whitelabel_cmd)
await _append_log(session, request, f"[android] build_whitelabel.py output:\n{out}")
if code != 0:
raise RuntimeError(f"build_whitelabel.py failed: {out}")
# Step 7: flutter build apk
# Container-safe flags:
# - GRADLE_OPTS disables native VFS file watching (inotify EBADF in Docker)
# - KOTLIN_COMPILE_DAEMON_ENABLED=false avoids the Kotlin compile daemon
# crashing with the same inotify issue
# - Project-level gradle.properties ensures the Gradle wrapper always
# honors these settings (user-level ~/.gradle/ may not be read)
# - Memory is aggressively capped: 1g JVM heap, single worker, no
# parallel builds, in-process Kotlin compilation, SerialGC (avoids
# parallel GC overhead). Combined with the 4g container memory limit,
# this leaves ~3g for native compilation (clang, dll, etc.)
await _append_log(session, request, "[android] Running flutter build apk...")
android_props = f"{project_dir}/android/gradle.properties"
build_cmd = (
f"cd {shlex.quote(project_dir)}/android && "
f"echo 'org.gradle.daemon=false' >> gradle.properties && "
# f"echo 'org.gradle.jvmargs=-Xmx4g -XX:MaxMetaspaceSize=512m -XX:+UseSerialGC' >> gradle.properties && "
f"echo 'org.gradle.workers.max=1' >> gradle.properties && "
# f"echo 'org.gradle.parallel=false' >> gradle.properties && "
# f"echo 'kotlin.compiler.execution.strategy=in-process' >> gradle.properties && "
# f"echo 'kotlin.incremental=false' >> gradle.properties && "
# f"echo 'android.enableParallelPlugin=false' >> gradle.properties && "
f"echo 'org.gradle.vfs.watch=false' >> gradle.properties && "
f"cd {shlex.quote(project_dir)} && "
f"GRADLE_OPTS='-Dorg.gradle.vfs.watch=false' "
# f"KOTLIN_COMPILE_DAEMON_ENABLED=false "
f"flutter build apk --release"
)
code, out = await _ssh_run(host, build_cmd)
await _append_log(session, request, f"[android] Build output:\n{out}")
if code != 0:
raise RuntimeError(f"flutter build apk failed: {out}")
# Step 8: Copy APK artifact back
await _copy_container_artifact(session, request, host, project_dir)
await _append_log(session, request, "[android] APK artifact copied successfully")
async def _execute_request(request_id: int) -> None:
async with async_session() as session:
result = await session.execute(
@@ -379,12 +595,10 @@ async def _execute_request(request_id: int) -> None:
try:
if request.platform_android:
await _run_platform_build(
await _run_flutter_container_build(
session,
request,
"android",
settings.MOBILE_BUILD_ANDROID_HOST,
settings.MOBILE_BUILD_ANDROID_SCRIPT,
settings.FLUTTER_BUILD_HOST,
)
if request.platform_ios:
backend/app/config.py
+12 -6
View File
@@ -18,19 +18,25 @@ class Settings(BaseSettings):
LOGS_DIR: str = "/logs"
GEOLITE2_DB_PATH: str = "/app/geo/GeoLite2-City.mmdb"
# Mobile build orchestration
MOBILE_BUILD_SSH_USER: str = "buildbot"
MOBILE_BUILD_SSH_PASSWORD: str = "buildbot"
# Mobile build orchestration — matches the user created in build/Dockerfile
MOBILE_BUILD_SSH_USER: str = "vscode"
MOBILE_BUILD_SSH_PASSWORD: str = "123"
MOBILE_BUILD_SSH_PORT: int = 22
MOBILE_BUILD_ANDROID_HOST: str = ""
MOBILE_BUILD_IOS_HOST: str = ""
MOBILE_BUILD_REMOTE_APP_DIR: str = "~/mobile_app"
MOBILE_BUILD_REMOTE_APP_DIR: str = "/home/vscode/mobile_app"
MOBILE_BUILD_ANDROID_SCRIPT: str = "do_android_build.sh"
MOBILE_BUILD_IOS_SCRIPT: str = "do_ios_build.sh"
MOBILE_BUILD_UPLOAD_DIR: str = "/tmp/kmtn_mobile_build_uploads"
MOBILE_BUILD_ARTIFACT_DIR: str = "/tmp/kmtn_mobile_build_artifacts"
MOBILE_BUILD_ANDROID_ARTIFACT_GLOB: str = "~/mobile_app/build/app/outputs/bundle/release/*.aab"
MOBILE_BUILD_IOS_ARTIFACT_GLOB: str = "~/mobile_app/build/ios/ipa/*.ipa"
MOBILE_BUILD_ANDROID_ARTIFACT_GLOB: str = "/home/vscode/mobile_app/build/app/outputs/bundle/release/*.aab"
MOBILE_BUILD_IOS_ARTIFACT_GLOB: str = "/home/vscode/mobile_app/build/ios/ipa/*.ipa"
# Flutter container-based Android build (SSH directly to build service over Docker network)
FLUTTER_BUILD_HOST: str = "build" # docker-compose service name
FLUTTER_PROJECT_REPO_URL: str = "http://192.168.1.74:30008/kfj001/kryz-go-flutter.git"
FLUTTER_PROJECT_DIR: str = "/home/vscode/kryz-go-flutter"
FLUTTER_APK_GLOB: str = "/home/vscode/kryz-go-flutter/build/app/outputs/flutter-apk/app-*.apk"
# Theme export
WEBSITE_URL: str = "https://kmountainflower.org"