docs: correct performance description
This commit is contained in:
parent
367d366dec
commit
799a0bf6ad
27
README.md
27
README.md
|
|
@ -14,7 +14,8 @@ Designed specifically as a **Railway deployment template**, with built-in suppor
|
||||||
- 🔗 **Flexible Database URLs** — supports private and public PostgreSQL URLs
|
- 🔗 **Flexible Database URLs** — supports private and public PostgreSQL URLs
|
||||||
- ⚡ **Optimized Performance** — parallel pg_dump and multipart R2 uploads
|
- ⚡ **Optimized Performance** — parallel pg_dump and multipart R2 uploads
|
||||||
- 🐳 **Docker Ready** — portable, lightweight container
|
- 🐳 **Docker Ready** — portable, lightweight container
|
||||||
- 🚀 **Railway Template First** — no fork required for normal usage
|
- 🚀 **Railway Template First** — no fork required for normal usage
|
||||||
|
- ⚡ **Optimized Performance** — efficient custom-format dumps and multipart R2 uploads
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|
@ -39,7 +40,6 @@ USE_PUBLIC_URL=false # Set true to use DATABASE_PUBLIC_URL
|
||||||
DUMP_FORMAT=dump # sql | plain | dump | custom | tar
|
DUMP_FORMAT=dump # sql | plain | dump | custom | tar
|
||||||
FILENAME_PREFIX=backup # Backup filename prefix
|
FILENAME_PREFIX=backup # Backup filename prefix
|
||||||
MAX_BACKUPS=7 # Number of backups to retain
|
MAX_BACKUPS=7 # Number of backups to retain
|
||||||
PG_DUMP_JOBS=1 # Optional: parallel pg_dump jobs (use 2–4 for 1–2GB DBs)
|
|
||||||
|
|
||||||
R2_ACCESS_KEY= # Cloudflare R2 access key
|
R2_ACCESS_KEY= # Cloudflare R2 access key
|
||||||
R2_SECRET_KEY= # Cloudflare R2 secret key
|
R2_SECRET_KEY= # Cloudflare R2 secret key
|
||||||
|
|
@ -52,29 +52,6 @@ BACKUP_TIME=00:00 # Daily backup time (UTC, HH:MM)
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## ⚡ Performance Optimization (Optional)
|
|
||||||
|
|
||||||
For larger databases (≈1–2 GB), you can significantly speed up backups by enabling
|
|
||||||
parallel PostgreSQL dumps.
|
|
||||||
|
|
||||||
### Parallel pg_dump
|
|
||||||
|
|
||||||
Set the number of parallel jobs:
|
|
||||||
|
|
||||||
```env
|
|
||||||
PG_DUMP_JOBS=4
|
|
||||||
```
|
|
||||||
|
|
||||||
**Notes**
|
|
||||||
- Only applies to `dump`, `custom`, or `tar` formats
|
|
||||||
- Default is `1` (safe for all users)
|
|
||||||
- Recommended values: `2–4`
|
|
||||||
- Higher values may overload small databases
|
|
||||||
|
|
||||||
This feature is **fully optional** and disabled by default.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## ⏰ Railway Cron Jobs
|
## ⏰ Railway Cron Jobs
|
||||||
|
|
||||||
You can configure the backup schedule using **Railway Cron Jobs**:
|
You can configure the backup schedule using **Railway Cron Jobs**:
|
||||||
|
|
|
||||||
40
main.py
40
main.py
|
|
@ -12,7 +12,7 @@ import shutil
|
||||||
|
|
||||||
load_dotenv()
|
load_dotenv()
|
||||||
|
|
||||||
##Env
|
## ENV
|
||||||
|
|
||||||
DATABASE_URL = os.environ.get("DATABASE_URL")
|
DATABASE_URL = os.environ.get("DATABASE_URL")
|
||||||
DATABASE_PUBLIC_URL = os.environ.get("DATABASE_PUBLIC_URL")
|
DATABASE_PUBLIC_URL = os.environ.get("DATABASE_PUBLIC_URL")
|
||||||
|
|
@ -27,41 +27,35 @@ DUMP_FORMAT = os.environ.get("DUMP_FORMAT", "dump")
|
||||||
BACKUP_PASSWORD = os.environ.get("BACKUP_PASSWORD")
|
BACKUP_PASSWORD = os.environ.get("BACKUP_PASSWORD")
|
||||||
USE_PUBLIC_URL = os.environ.get("USE_PUBLIC_URL", "false").lower() == "true"
|
USE_PUBLIC_URL = os.environ.get("USE_PUBLIC_URL", "false").lower() == "true"
|
||||||
BACKUP_TIME = os.environ.get("BACKUP_TIME", "00:00")
|
BACKUP_TIME = os.environ.get("BACKUP_TIME", "00:00")
|
||||||
PG_DUMP_JOBS = int(os.environ.get("PG_DUMP_JOBS", "1"))
|
|
||||||
|
|
||||||
def log(msg):
|
def log(msg):
|
||||||
print(msg, flush=True)
|
print(msg, flush=True)
|
||||||
|
|
||||||
|
## Validate BACKUP_TIME
|
||||||
try:
|
try:
|
||||||
hour, minute = BACKUP_TIME.split(":")
|
hour, minute = BACKUP_TIME.split(":")
|
||||||
if not (0 <= int(hour) <= 23 and 0 <= int(minute) <= 59):
|
if not (0 <= int(hour) <= 23 and 0 <= int(minute) <= 59):
|
||||||
log("[WARNING] Invalid BACKUP_TIME format. Using default: 00:00")
|
raise ValueError
|
||||||
BACKUP_TIME = "00:00"
|
|
||||||
except ValueError:
|
except ValueError:
|
||||||
log("[WARNING] Invalid BACKUP_TIME format. Using default: 00:00")
|
log("[WARNING] Invalid BACKUP_TIME format. Using default: 00:00")
|
||||||
BACKUP_TIME = "00:00"
|
BACKUP_TIME = "00:00"
|
||||||
|
|
||||||
def get_database_url():
|
def get_database_url():
|
||||||
"""Get the appropriate database URL based on configuration"""
|
|
||||||
if USE_PUBLIC_URL:
|
if USE_PUBLIC_URL:
|
||||||
if not DATABASE_PUBLIC_URL:
|
if not DATABASE_PUBLIC_URL:
|
||||||
raise ValueError("[ERROR] DATABASE_PUBLIC_URL not set but USE_PUBLIC_URL=true!")
|
raise ValueError("[ERROR] DATABASE_PUBLIC_URL not set but USE_PUBLIC_URL=true!")
|
||||||
return DATABASE_PUBLIC_URL
|
return DATABASE_PUBLIC_URL
|
||||||
|
|
||||||
if not DATABASE_URL:
|
if not DATABASE_URL:
|
||||||
raise ValueError("[ERROR] DATABASE_URL not set!")
|
raise ValueError("[ERROR] DATABASE_URL not set!")
|
||||||
return DATABASE_URL
|
return DATABASE_URL
|
||||||
|
|
||||||
def run_backup():
|
def run_backup():
|
||||||
"""Main backup function that handles the entire backup process"""
|
|
||||||
if shutil.which("pg_dump") is None:
|
if shutil.which("pg_dump") is None:
|
||||||
log("[ERROR] pg_dump not found. Install postgresql-client.")
|
log("[ERROR] pg_dump not found. Install postgresql-client.")
|
||||||
return
|
return
|
||||||
|
|
||||||
database_url = get_database_url()
|
database_url = get_database_url()
|
||||||
url = urlparse(database_url)
|
|
||||||
db_name = url.path[1:]
|
|
||||||
|
|
||||||
log(f"[INFO] Using {'public' if USE_PUBLIC_URL else 'private'} database URL")
|
log(f"[INFO] Using {'public' if USE_PUBLIC_URL else 'private'} database URL")
|
||||||
|
|
||||||
format_map = {
|
format_map = {
|
||||||
|
|
@ -73,7 +67,7 @@ def run_backup():
|
||||||
}
|
}
|
||||||
pg_format, ext = format_map.get(DUMP_FORMAT.lower(), ("c", "dump"))
|
pg_format, ext = format_map.get(DUMP_FORMAT.lower(), ("c", "dump"))
|
||||||
|
|
||||||
timestamp = datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')
|
timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
|
||||||
backup_file = f"{FILENAME_PREFIX}_{timestamp}.{ext}"
|
backup_file = f"{FILENAME_PREFIX}_{timestamp}.{ext}"
|
||||||
|
|
||||||
compressed_file = (
|
compressed_file = (
|
||||||
|
|
@ -82,9 +76,7 @@ def run_backup():
|
||||||
|
|
||||||
compressed_file_r2 = f"{BACKUP_PREFIX}{compressed_file}"
|
compressed_file_r2 = f"{BACKUP_PREFIX}{compressed_file}"
|
||||||
|
|
||||||
# --------------------------
|
## Create backup
|
||||||
# Create backup
|
|
||||||
# --------------------------
|
|
||||||
try:
|
try:
|
||||||
log(f"[INFO] Creating backup {backup_file}")
|
log(f"[INFO] Creating backup {backup_file}")
|
||||||
|
|
||||||
|
|
@ -97,17 +89,11 @@ def run_backup():
|
||||||
"-f", backup_file
|
"-f", backup_file
|
||||||
]
|
]
|
||||||
|
|
||||||
if pg_format in ("c", "t") and PG_DUMP_JOBS > 1:
|
|
||||||
dump_cmd.insert(-2, f"--jobs={PG_DUMP_JOBS}")
|
|
||||||
log(f"[INFO] Using parallel pg_dump with {PG_DUMP_JOBS} jobs")
|
|
||||||
|
|
||||||
subprocess.run(dump_cmd, check=True)
|
subprocess.run(dump_cmd, check=True)
|
||||||
|
|
||||||
if BACKUP_PASSWORD:
|
if BACKUP_PASSWORD:
|
||||||
log("[INFO] Encrypting backup with 7z...")
|
log("[INFO] Encrypting backup with 7z...")
|
||||||
with py7zr.SevenZipFile(
|
with py7zr.SevenZipFile(compressed_file, "w", password=BACKUP_PASSWORD) as archive:
|
||||||
compressed_file, "w", password=BACKUP_PASSWORD
|
|
||||||
) as archive:
|
|
||||||
archive.write(backup_file)
|
archive.write(backup_file)
|
||||||
log("[SUCCESS] Backup encrypted successfully")
|
log("[SUCCESS] Backup encrypted successfully")
|
||||||
else:
|
else:
|
||||||
|
|
@ -118,9 +104,6 @@ def run_backup():
|
||||||
except subprocess.CalledProcessError as e:
|
except subprocess.CalledProcessError as e:
|
||||||
log(f"[ERROR] Backup creation failed: {e}")
|
log(f"[ERROR] Backup creation failed: {e}")
|
||||||
return
|
return
|
||||||
except Exception as e:
|
|
||||||
log(f"[ERROR] Compression/encryption failed: {e}")
|
|
||||||
return
|
|
||||||
finally:
|
finally:
|
||||||
if os.path.exists(backup_file):
|
if os.path.exists(backup_file):
|
||||||
os.remove(backup_file)
|
os.remove(backup_file)
|
||||||
|
|
@ -175,7 +158,6 @@ def run_backup():
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log(f"[ERROR] R2 operation failed: {e}")
|
log(f"[ERROR] R2 operation failed: {e}")
|
||||||
return
|
|
||||||
finally:
|
finally:
|
||||||
if os.path.exists(compressed_file):
|
if os.path.exists(compressed_file):
|
||||||
os.remove(compressed_file)
|
os.remove(compressed_file)
|
||||||
|
|
@ -183,11 +165,11 @@ def run_backup():
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
log("[INFO] Starting backup scheduler...")
|
log("[INFO] Starting backup scheduler...")
|
||||||
log(f"[INFO] Scheduled backup time: {BACKUP_TIME} UTC")
|
log(f"[INFO] Scheduled backup time: {BACKUP_TIME} UTC")
|
||||||
|
|
||||||
schedule.every().day.at(BACKUP_TIME).do(run_backup)
|
schedule.every().day.at(BACKUP_TIME).do(run_backup)
|
||||||
|
|
||||||
run_backup()
|
run_backup()
|
||||||
|
|
||||||
while True:
|
while True:
|
||||||
schedule.run_pending()
|
schedule.run_pending()
|
||||||
time.sleep(60)
|
time.sleep(60)
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue