From 15c217cc0a726e22b48fcdd38ac8085543a2e3a2 Mon Sep 17 00:00:00 2001 From: Luca Haid Date: Mon, 29 Dec 2025 07:34:20 +0100 Subject: [PATCH] added new db downloader --- docker/db-downloader/Dockerfile | 25 ++ docker/db-downloader/app.py | 444 +++++++++++++++++++++ docker/db-downloader/requirements.txt | 5 + docker/db-downloader/static/app.js | 457 ++++++++++++++++++++++ docker/db-downloader/templates/index.html | 189 +++++++++ 5 files changed, 1120 insertions(+) create mode 100644 docker/db-downloader/Dockerfile create mode 100644 docker/db-downloader/app.py create mode 100644 docker/db-downloader/requirements.txt create mode 100644 docker/db-downloader/static/app.js create mode 100644 docker/db-downloader/templates/index.html diff --git a/docker/db-downloader/Dockerfile b/docker/db-downloader/Dockerfile new file mode 100644 index 000000000..760d57da8 --- /dev/null +++ b/docker/db-downloader/Dockerfile @@ -0,0 +1,25 @@ +FROM python:3.12-slim-bookworm + +RUN apt-get update && apt-get install -y \ + mariadb-client \ + openssh-client \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY app.py . +COPY templates/ ./templates/ +COPY static/ ./static/ + +RUN mkdir -p /app/downloads /app/ssh-keys + +ENV FLASK_APP=app:app +ENV FLASK_ENV=production +ENV PYTHONUNBUFFERED=1 + +EXPOSE 8082 + +CMD ["gunicorn", "--bind", "0.0.0.0:8082", "--workers", "2", "--threads", "4", "app:app"] diff --git a/docker/db-downloader/app.py b/docker/db-downloader/app.py new file mode 100644 index 000000000..d9988ee74 --- /dev/null +++ b/docker/db-downloader/app.py @@ -0,0 +1,444 @@ +import os +import stat +import uuid +import gzip +import struct +import subprocess +import threading +import time +from datetime import datetime +from flask import Flask, render_template, request, jsonify, session +import paramiko + + +# ============================================================================= +# Configuration +# ============================================================================= +class Config: + SCP_HOST = os.getenv('SCP_HOST', 'localhost') + SCP_PORT = int(os.getenv('SCP_PORT', 22)) + SCP_USERNAME = os.getenv('SCP_USERNAME', 'root') + SCP_DEFAULT_PATH = os.getenv('SCP_DEFAULT_PATH', '/backups') + + DB_HOST = os.getenv('DB_HOST', 'db') + DB_PORT = int(os.getenv('DB_PORT', 3306)) + DB_USER = os.getenv('DB_USER', 'root') + DB_PASSWORD = os.getenv('DB_PASSWORD', '') + DB_AVAILABLE = os.getenv('DB_AVAILABLE', 'thetool,addressdb').split(',') + + DOWNLOAD_PATH = '/app/downloads' + SSH_KEYS_PATH = '/app/ssh-keys' + SECRET_KEY = os.getenv('SECRET_KEY', os.urandom(24).hex()) + + +# ============================================================================= +# SFTP Client +# ============================================================================= +class SFTPClient: + def __init__(self, host, port, username): + self.host = host + self.port = port + self.username = username + self.client = None + self.sftp = None + + def connect_password(self, password): + self.client = paramiko.SSHClient() + self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + self.client.connect( + hostname=self.host, port=self.port, username=self.username, + password=password, look_for_keys=False, allow_agent=False + ) + self.sftp = self.client.open_sftp() + + def connect_key(self, key_path, passphrase=None): + self.client = paramiko.SSHClient() + self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + self.client.connect( + hostname=self.host, port=self.port, username=self.username, + key_filename=key_path, passphrase=passphrase, + look_for_keys=False, allow_agent=False + ) + self.sftp = self.client.open_sftp() + + def list_directory(self, path): + entries = [] + for entry in self.sftp.listdir_attr(path): + is_dir = stat.S_ISDIR(entry.st_mode) + entries.append({ + 'name': entry.filename, + 'size': entry.st_size, + 'size_human': self._human_size(entry.st_size), + 'mtime': entry.st_mtime, + 'mtime_human': datetime.fromtimestamp(entry.st_mtime).strftime('%Y-%m-%d %H:%M'), + 'is_dir': is_dir, + 'is_sql': entry.filename.endswith(('.sql', '.sql.gz')), + 'path': os.path.join(path, entry.filename) + }) + return sorted(entries, key=lambda x: (not x['is_dir'], -x['mtime'])) + + def get_file_info(self, path): + entry = self.sftp.stat(path) + return { + 'name': os.path.basename(path), + 'size': entry.st_size, + 'size_human': self._human_size(entry.st_size), + 'mtime': entry.st_mtime, + 'mtime_human': datetime.fromtimestamp(entry.st_mtime).strftime('%Y-%m-%d %H:%M'), + 'path': path + } + + def download_file(self, remote_path, local_path, callback=None): + self.sftp.get(remote_path, local_path, callback=callback) + + def close(self): + if self.sftp: + self.sftp.close() + if self.client: + self.client.close() + + @staticmethod + def _human_size(size): + for unit in ['B', 'KB', 'MB', 'GB', 'TB']: + if size < 1024: + return f"{size:.1f} {unit}" + size /= 1024 + return f"{size:.1f} PB" + + +# ============================================================================= +# Database Restore +# ============================================================================= +class DatabaseRestore: + def __init__(self): + self.host = Config.DB_HOST + self.port = Config.DB_PORT + self.user = Config.DB_USER + self.password = Config.DB_PASSWORD + self.available_dbs = Config.DB_AVAILABLE + self.cancelled = False + + def cancel(self): + self.cancelled = True + + @staticmethod + def get_gzip_uncompressed_size(filepath): + with open(filepath, 'rb') as f: + f.seek(-4, 2) + return struct.unpack(' 0 else 0, 'downloaded': transferred, 'total': total}) + + client.download_file(remote_file, local_file, callback=download_progress) + client.close() + + if jobs[job_id].get('cancelled'): + jobs[job_id].update({'status': 'cancelled', 'message': 'Restore cancelled by user'}) + if os.path.exists(local_file): + os.remove(local_file) + return + + jobs[job_id].update({'progress': 45, 'message': 'Download complete. Preparing restore...', 'status': 'restoring'}) + jobs[job_id]['progress'] = 50 + jobs[job_id]['message'] = f'Clearing database {target_db}...' + + restorer = DatabaseRestore() + restorers[job_id] = restorer + + uncompressed_size = restorer.get_gzip_uncompressed_size(local_file) if local_file.endswith('.gz') else os.path.getsize(local_file) + + def restore_progress(bytes_processed): + if jobs[job_id].get('cancelled'): + restorer.cancel() + pct = 50 + min(45, int((bytes_processed / uncompressed_size) * 45)) if uncompressed_size > 0 else 50 + jobs[job_id].update({'progress': pct, 'message': f'Restoring to {target_db}... ({bytes_processed // (1024*1024)} MB / {uncompressed_size // (1024*1024)} MB)'}) + + result = restorer.restore_from_file(local_file, target_db, progress_callback=restore_progress) + + if os.path.exists(local_file): + os.remove(local_file) + + jobs[job_id].update({ + 'status': 'completed', 'progress': 100, + 'message': f'Restore complete! Dropped {result["tables_dropped"]} tables and imported {result["file"]}', + 'completed_at': time.time(), 'duration': time.time() - jobs[job_id]['started_at'] + }) + + except Exception as e: + error_msg = str(e) + if 'cancelled' in error_msg.lower(): + jobs[job_id].update({'status': 'cancelled', 'message': 'Restore cancelled by user'}) + else: + jobs[job_id].update({'status': 'error', 'error': error_msg, 'message': f'Error: {error_msg}'}) + if os.path.exists(local_file): + os.remove(local_file) + finally: + restorers.pop(job_id, None) + + +@app.route('/api/status/') +def status(job_id): + if job_id not in jobs: + return jsonify({'success': False, 'error': 'Job not found'}), 404 + job = jobs[job_id].copy() + job['success'] = True + if 'started_at' in job: + elapsed = (job.get('completed_at') or time.time()) - job['started_at'] + job['elapsed'] = f'{int(elapsed // 60)}m {int(elapsed % 60)}s' + return jsonify(job) + + +@app.route('/api/jobs', methods=['GET']) +def list_jobs(): + return jsonify({'success': True, 'jobs': dict(jobs)}) + + +@app.route('/api/cancel/', methods=['POST']) +def cancel(job_id): + if job_id not in jobs: + return jsonify({'success': False, 'error': 'Job not found'}), 404 + if jobs[job_id]['status'] in ('completed', 'error', 'cancelled'): + return jsonify({'success': False, 'error': 'Job already finished'}), 400 + jobs[job_id]['cancelled'] = True + if job_id in restorers: + restorers[job_id].cancel() + return jsonify({'success': True, 'message': 'Cancel signal sent'}) + + +if __name__ == '__main__': + app.run(host='0.0.0.0', port=8082, debug=True) diff --git a/docker/db-downloader/requirements.txt b/docker/db-downloader/requirements.txt new file mode 100644 index 000000000..973a9f474 --- /dev/null +++ b/docker/db-downloader/requirements.txt @@ -0,0 +1,5 @@ +flask==3.0.0 +gunicorn==21.2.0 +paramiko==3.4.0 +mysql-connector-python==8.2.0 +python-dotenv==1.0.0 diff --git a/docker/db-downloader/static/app.js b/docker/db-downloader/static/app.js new file mode 100644 index 000000000..c949e1164 --- /dev/null +++ b/docker/db-downloader/static/app.js @@ -0,0 +1,457 @@ +// DB Restore Tool - Frontend JavaScript + +let currentPath = ''; +let selectedFile = null; +let isConnected = false; +let currentJobId = null; +let pollInterval = null; + +// Initialize +document.addEventListener('DOMContentLoaded', function() { + loadAvailableKeys(); + setupEventListeners(); +}); + +function setupEventListeners() { + // Auth type toggle + document.getElementById('auth-type').addEventListener('change', function() { + const passwordAuth = document.getElementById('password-auth'); + const keyAuth = document.getElementById('key-auth'); + if (this.value === 'password') { + passwordAuth.classList.remove('hidden'); + keyAuth.classList.add('hidden'); + } else { + passwordAuth.classList.add('hidden'); + keyAuth.classList.remove('hidden'); + } + }); + + // Connect form + document.getElementById('connect-form').addEventListener('submit', function(e) { + e.preventDefault(); + connect(); + }); + + // Disconnect button + document.getElementById('disconnect-btn').addEventListener('click', disconnect); + + // Restore button + document.getElementById('restore-btn').addEventListener('click', startRestore); + + // Cancel button + document.getElementById('cancel-btn').addEventListener('click', cancelRestore); +} + +async function loadAvailableKeys() { + try { + const response = await fetch('/api/keys'); + const data = await response.json(); + const select = document.getElementById('key-file'); + select.innerHTML = ''; + data.keys.forEach(key => { + const option = document.createElement('option'); + option.value = key; + option.textContent = key; + select.appendChild(option); + }); + } catch (error) { + console.error('Failed to load keys:', error); + } +} + +async function connect() { + const authType = document.getElementById('auth-type').value; + const password = document.getElementById('password').value; + const keyFile = document.getElementById('key-file').value; + const keyPassphrase = document.getElementById('key-passphrase').value; + + const btn = document.getElementById('connect-btn'); + btn.disabled = true; + btn.textContent = 'Connecting...'; + + try { + const response = await fetch('/api/connect', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + auth_type: authType, + password: password, + key_file: keyFile, + key_passphrase: keyPassphrase + }) + }); + + const data = await response.json(); + + if (data.success) { + isConnected = true; + currentPath = data.path; + showStatus('Connected to ' + data.host, 'success'); + renderFiles(data.files, data.path); + updateBreadcrumb(data.path); + + // Toggle buttons + btn.classList.add('hidden'); + document.getElementById('disconnect-btn').classList.remove('hidden'); + + // Clear password field for security + document.getElementById('password').value = ''; + } else { + showStatus('Connection failed: ' + data.error, 'error'); + } + } catch (error) { + showStatus('Connection error: ' + error.message, 'error'); + } finally { + btn.disabled = false; + btn.textContent = 'Connect'; + } +} + +async function disconnect() { + try { + await fetch('/api/disconnect', { method: 'POST' }); + } catch (e) {} + + isConnected = false; + selectedFile = null; + currentPath = ''; + + // Reset UI + document.getElementById('connect-btn').classList.remove('hidden'); + document.getElementById('disconnect-btn').classList.add('hidden'); + document.getElementById('file-browser').innerHTML = ` +
+ + + +

Connect to browse remote files

+
+ `; + document.getElementById('breadcrumb').innerHTML = 'Not connected'; + document.getElementById('selected-file-info').innerHTML = '

No file selected

'; + document.getElementById('restore-btn').disabled = true; + + hideStatus(); +} + +async function browse(path) { + if (!isConnected) return; + + try { + const response = await fetch('/api/browse', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ path: path }) + }); + + const data = await response.json(); + + if (data.success) { + currentPath = data.path; + renderFiles(data.files, data.path); + updateBreadcrumb(data.path); + } else { + showStatus('Browse failed: ' + data.error, 'error'); + } + } catch (error) { + showStatus('Browse error: ' + error.message, 'error'); + } +} + +function renderFiles(files, path) { + const container = document.getElementById('file-browser'); + + if (files.length === 0) { + container.innerHTML = '
Empty directory
'; + return; + } + + let html = '
'; + + // Parent directory link + if (path !== '/') { + const parentPath = path.split('/').slice(0, -1).join('/') || '/'; + html += ` +
+ + + + .. +
+ `; + } + + files.forEach(file => { + const isSelected = selectedFile && selectedFile.path === file.path; + const selectedClass = isSelected ? 'selected' : ''; + + if (file.is_dir) { + html += ` +
+ + + + ${file.name} +
+ `; + } else if (file.is_sql) { + html += ` +
+ + + + + ${file.name} + ${file.size_human} + ${file.mtime_human} +
+ `; + } else { + html += ` +
+ + + + ${file.name} + ${file.size_human} +
+ `; + } + }); + + html += '
'; + container.innerHTML = html; +} + +function selectFile(file) { + selectedFile = file; + document.getElementById('selected-file-info').innerHTML = ` +

${file.name}

+

${file.size_human} - ${file.mtime_human}

+ `; + document.getElementById('restore-btn').disabled = false; + + // Auto-detect target database from filename + const filename = file.name.toLowerCase(); + const targetDbSelect = document.getElementById('target-db'); + const availableDbs = Array.from(targetDbSelect.options).map(o => o.value); + + for (const db of availableDbs) { + if (filename.includes(db.toLowerCase())) { + targetDbSelect.value = db; + break; + } + } + + // Re-render to show selection + browse(currentPath); +} + +function updateBreadcrumb(path) { + const parts = path.split('/').filter(p => p); + let html = `/`; + + let currentPathBuild = ''; + parts.forEach((part, index) => { + currentPathBuild += '/' + part; + const isLast = index === parts.length - 1; + html += ` + / + ${part} + `; + }); + + document.getElementById('breadcrumb').innerHTML = html; +} + +async function startRestore() { + if (!selectedFile) return; + + const targetDb = document.getElementById('target-db').value; + + if (!confirm(`Are you sure you want to restore ${selectedFile.name} to database "${targetDb}"?\n\nThis will DROP ALL TABLES in the database!`)) { + return; + } + + const btn = document.getElementById('restore-btn'); + btn.disabled = true; + btn.textContent = 'Starting...'; + + try { + const response = await fetch('/api/restore', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + file: selectedFile.path, + database: targetDb + }) + }); + + const data = await response.json(); + + if (data.success) { + currentJobId = data.job_id; + showProgressPanel(); + startPolling(); + } else { + showStatus('Restore failed: ' + data.error, 'error'); + btn.disabled = false; + btn.textContent = 'Start Restore'; + } + } catch (error) { + showStatus('Restore error: ' + error.message, 'error'); + btn.disabled = false; + btn.textContent = 'Start Restore'; + } +} + +function showProgressPanel() { + document.getElementById('progress-panel').classList.remove('hidden'); + document.getElementById('progress-bar').style.width = '0%'; + document.getElementById('progress-bar').classList.remove('bg-green-600', 'bg-red-600', 'bg-yellow-600'); + document.getElementById('progress-bar').classList.add('bg-blue-600'); + document.getElementById('progress-percent').textContent = '0%'; + document.getElementById('progress-status').textContent = 'Starting...'; + document.getElementById('progress-message').textContent = 'Initializing...'; + document.getElementById('cancel-btn').classList.remove('hidden'); + document.getElementById('cancel-btn').disabled = false; + document.getElementById('cancel-btn').textContent = 'Cancel Restore'; +} + +function startPolling() { + if (pollInterval) clearInterval(pollInterval); + + pollInterval = setInterval(async () => { + try { + const response = await fetch(`/api/status/${currentJobId}`); + const data = await response.json(); + + if (data.success) { + updateProgress(data); + + if (data.status === 'completed' || data.status === 'error' || data.status === 'cancelled') { + stopPolling(); + document.getElementById('restore-btn').disabled = false; + document.getElementById('restore-btn').textContent = 'Start Restore'; + + if (data.status === 'completed') { + showStatus('Restore completed successfully!', 'success'); + } else if (data.status === 'cancelled') { + showStatus('Restore was cancelled', 'error'); + } else { + showStatus('Restore failed: ' + data.error, 'error'); + } + } + } + } catch (error) { + console.error('Polling error:', error); + } + }, 250); +} + +function stopPolling() { + if (pollInterval) { + clearInterval(pollInterval); + pollInterval = null; + } +} + +async function cancelRestore() { + if (!currentJobId) return; + + if (!confirm('Are you sure you want to cancel the restore?')) { + return; + } + + const btn = document.getElementById('cancel-btn'); + btn.disabled = true; + btn.textContent = 'Cancelling...'; + + try { + const response = await fetch(`/api/cancel/${currentJobId}`, { + method: 'POST' + }); + + const data = await response.json(); + + if (data.success) { + showStatus('Cancellation requested...', 'error'); + } else { + showStatus('Cancel failed: ' + data.error, 'error'); + btn.disabled = false; + btn.textContent = 'Cancel Restore'; + } + } catch (error) { + showStatus('Cancel error: ' + error.message, 'error'); + btn.disabled = false; + btn.textContent = 'Cancel Restore'; + } +} + +function updateProgress(data) { + const bar = document.getElementById('progress-bar'); + const percent = document.getElementById('progress-percent'); + const status = document.getElementById('progress-status'); + const message = document.getElementById('progress-message'); + const elapsed = document.getElementById('progress-elapsed'); + const cancelBtn = document.getElementById('cancel-btn'); + + bar.style.width = data.progress + '%'; + percent.textContent = data.progress + '%'; + + // Update status label + const statusLabels = { + 'starting': 'Starting', + 'downloading': 'Downloading', + 'restoring': 'Restoring', + 'completed': 'Completed', + 'error': 'Error', + 'cancelled': 'Cancelled' + }; + status.textContent = statusLabels[data.status] || data.status; + + // Update color based on status + bar.classList.remove('bg-green-600', 'bg-red-600', 'bg-yellow-600'); + if (data.status === 'completed') { + bar.classList.remove('bg-blue-600'); + bar.classList.add('bg-green-600'); + } else if (data.status === 'error') { + bar.classList.remove('bg-blue-600'); + bar.classList.add('bg-red-600'); + } else if (data.status === 'cancelled') { + bar.classList.remove('bg-blue-600'); + bar.classList.add('bg-yellow-600'); + } + + // Show/hide cancel button based on job status + if (data.status === 'completed' || data.status === 'error' || data.status === 'cancelled') { + cancelBtn.classList.add('hidden'); + } else { + cancelBtn.classList.remove('hidden'); + cancelBtn.disabled = false; + cancelBtn.textContent = 'Cancel Restore'; + } + + message.textContent = data.message || ''; + if (data.elapsed) { + elapsed.textContent = 'Elapsed: ' + data.elapsed; + } +} + +function showStatus(message, type) { + const status = document.getElementById('connection-status'); + status.classList.remove('hidden', 'bg-green-100', 'bg-red-100', 'text-green-800', 'text-red-800'); + + if (type === 'success') { + status.classList.add('bg-green-100', 'text-green-800'); + } else if (type === 'error') { + status.classList.add('bg-red-100', 'text-red-800'); + } + + status.textContent = message; +} + +function hideStatus() { + document.getElementById('connection-status').classList.add('hidden'); +} diff --git a/docker/db-downloader/templates/index.html b/docker/db-downloader/templates/index.html new file mode 100644 index 000000000..1e5909e3d --- /dev/null +++ b/docker/db-downloader/templates/index.html @@ -0,0 +1,189 @@ + + + + + + DB Restore Tool + + + + +
+ +
+

Database Restore Tool

+

Browse and restore database backups from remote server

+
+ +
+ +
+
+

+ + + + Connection +

+ + + +
+
+ + +
+ +
+ + +
+ +
+ + +
+ + +
+
+ + +
+
+ + + + + + + +
+
+ + +
+

+ + + + Restore +

+ +
+

No file selected

+
+ +
+ + +
+ +
+

+ Warning: This will DROP all tables in the selected database before restoring! +

+
+ + +
+
+ + +
+
+

+ + + + Remote Browser +

+ + + + + +
+
+ + + +

Connect to browse remote files

+
+
+
+ + + +
+
+
+ + + +