Initial commit
This commit is contained in:
Kaki Filem Team
commit
10213621f2
|
|
@ -0,0 +1,19 @@
|
|||
# Database
|
||||
DATABASE_URL=
|
||||
DATABASE_PUBLIC_URL=
|
||||
USE_PUBLIC_URL=false
|
||||
|
||||
# S3 / R2 (S3-compatible storage)
|
||||
R2_ENDPOINT=
|
||||
R2_BUCKET_NAME=
|
||||
R2_ACCESS_KEY=
|
||||
R2_SECRET_KEY=
|
||||
S3_REGION=us-east-1
|
||||
|
||||
# Backup settings
|
||||
MAX_BACKUPS=7
|
||||
BACKUP_PREFIX=
|
||||
FILENAME_PREFIX=backup
|
||||
DUMP_FORMAT=dump
|
||||
BACKUP_PASSWORD=
|
||||
BACKUP_TIME=00:00
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
*.so
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
|
||||
# Virtual Environment
|
||||
venv/
|
||||
env/
|
||||
ENV/
|
||||
|
||||
# IDE
|
||||
.idea/
|
||||
.vscode/
|
||||
*.swp
|
||||
*.swo
|
||||
|
||||
# Environment variables
|
||||
.env
|
||||
.env.local
|
||||
|
||||
# Backup files
|
||||
*.gz
|
||||
*.7z
|
||||
*.dump
|
||||
*.sql
|
||||
*.tar
|
||||
|
||||
# Logs
|
||||
*.log
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
# Contributing to Postgres-to-R2 Backup
|
||||
|
||||
Thanks for your interest in contributing 🎉
|
||||
All contributions are welcome — bug reports, documentation improvements, and code changes.
|
||||
|
||||
---
|
||||
|
||||
## Getting Started
|
||||
|
||||
1. Fork the repository
|
||||
2. Clone your fork locally
|
||||
3. Create a feature branch from `main`
|
||||
4. Submit a pull request against `main`
|
||||
|
||||
---
|
||||
|
||||
## Development
|
||||
|
||||
### Requirements
|
||||
- Python 3.9+
|
||||
- PostgreSQL client tools (`pg_dump`)
|
||||
- pip
|
||||
- (Optional) Docker
|
||||
|
||||
### Local Setup
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
python main.py
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
FROM python:3.12-slim AS builder
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y gcc libpq-dev && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN pip install --upgrade pip setuptools wheel
|
||||
|
||||
COPY requirements.txt /app/requirements.txt
|
||||
RUN pip install --prefix=/install -r /app/requirements.txt
|
||||
|
||||
FROM python:3.12-slim
|
||||
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends postgresql-client gzip && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY --from=builder /install /usr/local
|
||||
|
||||
COPY main.py /app/main.py
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
CMD ["python", "main.py"]
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
MIT License
|
||||
|
||||
Copyright (c) 2025 Aman
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
|
|
@ -0,0 +1,214 @@
|
|||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# Postgres-to-R2 Backup (S3-Compatible)
|
||||
|
||||
A lightweight automation service that creates scheduled PostgreSQL backups and securely uploads them to **S3-compatible object storage**
|
||||
such as **Cloudflare R2, AWS S3, Wasabi, Backblaze B2, or MinIO**.
|
||||
Designed specifically as a **Railway deployment template**, with built-in support for Docker and cron scheduling.
|
||||
|
||||
---
|
||||
|
||||
## ✨ Features
|
||||
|
||||
- 📦 **Automated Backups** — scheduled daily or hourly PostgreSQL backups
|
||||
- 🔐 **Optional Encryption** — gzip compression or 7z encryption with password
|
||||
- ☁️ **Cloudflare R2 Integration** — seamless S3-compatible storage support
|
||||
- 🧹 **Retention Policy** — automatically delete old backups
|
||||
- 🔗 **Flexible Database URLs** — supports private and public PostgreSQL URLs
|
||||
- ⚡ **Optimized Performance** — parallel pg_dump and multipart S3 uploads
|
||||
- 🐳 **Docker Ready** — portable, lightweight container
|
||||
- 🚀 **Railway Template First** — no fork required for normal usage
|
||||
- 🪣 **S3-Compatible Storage** — works with R2, AWS S3, Wasabi, B2, MinIO
|
||||
|
||||
---
|
||||
|
||||
## 🚀 Deployment on Railway
|
||||
|
||||
1. Click the **Deploy on Railway** button below
|
||||
2. Railway will create a new project using the latest version of this repository
|
||||
3. Add the required environment variables in the Railway dashboard
|
||||
4. (Optional) Configure a cron job for your desired backup schedule
|
||||
|
||||
[](https://railway.com/deploy/postgres-to-r2-backup?referralCode=nIQTyp&utm_medium=integration&utm_source=template&utm_campaign=generic)
|
||||
|
||||
---
|
||||
|
||||
## 🔧 Environment Variables (S3-Compatible)
|
||||
|
||||
```env
|
||||
DATABASE_URL= # PostgreSQL database URL (private)
|
||||
DATABASE_PUBLIC_URL= # Public PostgreSQL URL (optional)
|
||||
USE_PUBLIC_URL=false # Set true to use DATABASE_PUBLIC_URL
|
||||
|
||||
DUMP_FORMAT=dump # sql | plain | dump | custom | tar
|
||||
FILENAME_PREFIX=backup # Backup filename prefix
|
||||
MAX_BACKUPS=7 # Number of backups to retain
|
||||
|
||||
R2_ENDPOINT= # S3 endpoint URL
|
||||
R2_BUCKET_NAME= # Bucket name
|
||||
R2_ACCESS_KEY= # Access key
|
||||
R2_SECRET_KEY= # Secret key
|
||||
S3_REGION=us-east-1 # Required for AWS S3 (ignored by R2/MinIO)
|
||||
|
||||
BACKUP_PASSWORD= # Optional: enables 7z encryption
|
||||
BACKUP_TIME=00:00 # Daily backup time (UTC, HH:MM)
|
||||
```
|
||||
|
||||
> Variable names use `R2_*` for historical reasons, but **any S3-compatible provider** can be used by changing the endpoint and credentials.
|
||||
> For AWS S3 users: ensure `S3_REGION` matches your bucket’s region.
|
||||
|
||||
---
|
||||
|
||||
## ☁️ Supported S3-Compatible Providers
|
||||
|
||||
This project uses the **standard AWS S3 API via boto3**, and works with:
|
||||
|
||||
- Cloudflare R2 (recommended)
|
||||
- AWS S3
|
||||
- Wasabi
|
||||
- Backblaze B2 (S3 API)
|
||||
- MinIO (self-hosted)
|
||||
|
||||
### Example Endpoints
|
||||
|
||||
| Provider | Endpoint Example |
|
||||
|--------|------------------|
|
||||
| Cloudflare R2 | `https://<accountid>.r2.cloudflarestorage.com` |
|
||||
| AWS S3 | `https://s3.amazonaws.com` |
|
||||
| Wasabi | `https://s3.wasabisys.com` |
|
||||
| Backblaze B2 | `https://s3.us-west-004.backblazeb2.com` |
|
||||
| MinIO | `http://localhost:9000` |
|
||||
|
||||
---
|
||||
|
||||
## ⏰ Railway Cron Jobs
|
||||
|
||||
You can configure the backup schedule using **Railway Cron Jobs**:
|
||||
|
||||
1. Open your Railway project
|
||||
2. Go to **Deployments → Cron**
|
||||
3. Add a cron job targeting this service
|
||||
|
||||
### Common Cron Expressions
|
||||
|
||||
| Schedule | Cron Expression | Description |
|
||||
|--------|----------------|------------|
|
||||
| Hourly | `0 * * * *` | Every hour |
|
||||
| Daily | `0 0 * * *` | Once per day (UTC midnight) |
|
||||
| Twice Daily | `0 */12 * * *` | Every 12 hours |
|
||||
| Weekly | `0 0 * * 0` | Every Sunday |
|
||||
| Monthly | `0 0 1 * *` | First day of the month |
|
||||
|
||||
**Tips**
|
||||
- All cron times are **UTC**
|
||||
- Use https://crontab.guru to validate expressions
|
||||
- Adjust `MAX_BACKUPS` to match your schedule
|
||||
|
||||
---
|
||||
|
||||
## 🖥️ Running Locally or on Other Platforms
|
||||
|
||||
It can run on **any platform** that supports:
|
||||
- Python 3.9+
|
||||
- `pg_dump` (PostgreSQL client tools)
|
||||
- Environment variables
|
||||
- Long-running background processes or cron
|
||||
|
||||
> Docker images use **Python 3.12** by default.
|
||||
> Local execution supports **Python 3.9+**.
|
||||
|
||||
### Supported Environments
|
||||
|
||||
- Local machine (Linux / macOS / Windows*)
|
||||
- VPS (Netcup, Hetzner, DigitalOcean, etc.)
|
||||
- Docker containers
|
||||
- Other PaaS providers (Heroku, Fly.io, Render, etc.)
|
||||
|
||||
> *Windows is supported when `pg_dump` is installed and available in PATH.*
|
||||
|
||||
### Local Requirements
|
||||
|
||||
- Python 3.9+
|
||||
- PostgreSQL client tools (`pg_dump`)
|
||||
- pip
|
||||
|
||||
### Run Manually (Local)
|
||||
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
python main.py
|
||||
```
|
||||
|
||||
### Run with Docker (Optional)
|
||||
|
||||
Build and run the image locally:
|
||||
|
||||
```bash
|
||||
docker build -t postgres-to-r2-backup .
|
||||
docker run --env-file .env postgres-to-r2-backup
|
||||
```
|
||||
|
||||
> Ensure the container is allowed to run continuously when not using an external cron scheduler.
|
||||
|
||||
> All scheduling uses **UTC** by default (e.g. Malaysia UTC+8 → set `BACKUP_TIME=16:00` for midnight).
|
||||
|
||||
### Run from Prebuilt Docker Image
|
||||
|
||||
If you downloaded a prebuilt Docker image archive (`.tar` or `.tar.gz`), you can run it without building locally:
|
||||
|
||||
```bash
|
||||
# Extract the archive (if compressed)
|
||||
tar -xzf postgres-to-r2-backup_v1.0.0.tar.gz
|
||||
|
||||
# Load the image into Docker
|
||||
docker load -i postgres-to-r2-backup_v1.0.0.tar
|
||||
|
||||
# Run the container
|
||||
docker run --env-file .env postgres-to-r2-backup:v1.0.0
|
||||
```
|
||||
|
||||
> Prebuilt images are architecture-specific (amd64 / arm64).
|
||||
|
||||
---
|
||||
|
||||
## 🔐 Security
|
||||
|
||||
- **Do not expose PostgreSQL directly to the public internet.**
|
||||
If your database is not on a private network, use a secure tunnel instead.
|
||||
|
||||
- **Recommended: Cloudflare Tunnel**
|
||||
When using a public database URL, it is strongly recommended to connect via a secure tunnel such as **Cloudflare Tunnel** rather than opening database ports.
|
||||
|
||||
- **Protect credentials**
|
||||
Store all secrets (database URLs, R2 keys, encryption passwords) using environment variables.
|
||||
Never commit `.env` files to version control.
|
||||
|
||||
- **Encrypted backups (optional)**
|
||||
Set `BACKUP_PASSWORD` to enable encrypted backups using 7z before uploading to S3-compatible storage.
|
||||
|
||||
- **Least privilege access**
|
||||
Use a PostgreSQL user with read-only access where possible, and restrict R2 credentials to the required bucket only.
|
||||
|
||||
---
|
||||
|
||||
## 🛠 Development & Contributions
|
||||
|
||||
Fork this repository **only if you plan to**:
|
||||
|
||||
- Modify the backup logic
|
||||
- Add features or integrations
|
||||
- Submit pull requests
|
||||
- Run locally for development
|
||||
|
||||
---
|
||||
|
||||
## 📜 License
|
||||
|
||||
This project is open source under the **MIT License**.
|
||||
|
||||
You are free to use, modify, and distribute it with attribution.
|
||||
|
|
@ -0,0 +1,180 @@
|
|||
import os
|
||||
import subprocess
|
||||
import boto3
|
||||
from boto3.session import Config
|
||||
from datetime import datetime, timezone
|
||||
from boto3.s3.transfer import TransferConfig
|
||||
from dotenv import load_dotenv
|
||||
import time
|
||||
import schedule
|
||||
import py7zr
|
||||
import shutil
|
||||
|
||||
load_dotenv()
|
||||
|
||||
## ENV
|
||||
|
||||
DATABASE_URL = os.environ.get("DATABASE_URL")
|
||||
DATABASE_PUBLIC_URL = os.environ.get("DATABASE_PUBLIC_URL")
|
||||
R2_ACCESS_KEY = os.environ.get("R2_ACCESS_KEY")
|
||||
R2_SECRET_KEY = os.environ.get("R2_SECRET_KEY")
|
||||
R2_BUCKET_NAME = os.environ.get("R2_BUCKET_NAME")
|
||||
R2_ENDPOINT = os.environ.get("R2_ENDPOINT")
|
||||
MAX_BACKUPS = int(os.environ.get("MAX_BACKUPS", 7))
|
||||
BACKUP_PREFIX = os.environ.get("BACKUP_PREFIX", "")
|
||||
FILENAME_PREFIX = os.environ.get("FILENAME_PREFIX", "backup")
|
||||
DUMP_FORMAT = os.environ.get("DUMP_FORMAT", "dump")
|
||||
BACKUP_PASSWORD = os.environ.get("BACKUP_PASSWORD")
|
||||
USE_PUBLIC_URL = os.environ.get("USE_PUBLIC_URL", "false").lower() == "true"
|
||||
BACKUP_TIME = os.environ.get("BACKUP_TIME", "00:00")
|
||||
S3_REGION = os.environ.get("S3_REGION", "us-east-1")
|
||||
|
||||
def log(msg):
|
||||
print(msg, flush=True)
|
||||
|
||||
## Validate BACKUP_TIME
|
||||
try:
|
||||
hour, minute = BACKUP_TIME.split(":")
|
||||
if not (0 <= int(hour) <= 23 and 0 <= int(minute) <= 59):
|
||||
raise ValueError
|
||||
except ValueError:
|
||||
log("[WARNING] Invalid BACKUP_TIME format. Using default: 00:00")
|
||||
BACKUP_TIME = "00:00"
|
||||
|
||||
def get_database_url():
|
||||
if USE_PUBLIC_URL:
|
||||
if not DATABASE_PUBLIC_URL:
|
||||
raise ValueError("[ERROR] DATABASE_PUBLIC_URL not set but USE_PUBLIC_URL=true!")
|
||||
return DATABASE_PUBLIC_URL
|
||||
|
||||
if not DATABASE_URL:
|
||||
raise ValueError("[ERROR] DATABASE_URL not set!")
|
||||
return DATABASE_URL
|
||||
|
||||
def run_backup():
|
||||
if shutil.which("pg_dump") is None:
|
||||
log("[ERROR] pg_dump not found. Install postgresql-client.")
|
||||
return
|
||||
|
||||
database_url = get_database_url()
|
||||
log(f"[INFO] Using {'public' if USE_PUBLIC_URL else 'private'} database URL")
|
||||
|
||||
format_map = {
|
||||
"sql": ("p", "sql"),
|
||||
"plain": ("p", "sql"),
|
||||
"dump": ("c", "dump"),
|
||||
"custom": ("c", "dump"),
|
||||
"tar": ("t", "tar")
|
||||
}
|
||||
pg_format, ext = format_map.get(DUMP_FORMAT.lower(), ("c", "dump"))
|
||||
|
||||
timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
|
||||
backup_file = f"{FILENAME_PREFIX}_{timestamp}.{ext}"
|
||||
|
||||
compressed_file = (
|
||||
f"{backup_file}.7z" if BACKUP_PASSWORD else f"{backup_file}.gz"
|
||||
)
|
||||
|
||||
compressed_file_r2 = f"{BACKUP_PREFIX}{compressed_file}"
|
||||
|
||||
## Create backup
|
||||
try:
|
||||
log(f"[INFO] Creating backup {backup_file}")
|
||||
|
||||
dump_cmd = [
|
||||
"pg_dump",
|
||||
f"--dbname={database_url}",
|
||||
"-F", pg_format,
|
||||
"--no-owner",
|
||||
"--no-acl",
|
||||
"-f", backup_file
|
||||
]
|
||||
|
||||
subprocess.run(dump_cmd, check=True)
|
||||
|
||||
if BACKUP_PASSWORD:
|
||||
log("[INFO] Encrypting backup with 7z...")
|
||||
with py7zr.SevenZipFile(compressed_file, "w", password=BACKUP_PASSWORD) as archive:
|
||||
archive.write(backup_file)
|
||||
log("[SUCCESS] Backup encrypted successfully")
|
||||
else:
|
||||
log("[INFO] Compressing backup with gzip...")
|
||||
subprocess.run(["gzip", "-f", backup_file], check=True)
|
||||
log("[SUCCESS] Backup compressed successfully")
|
||||
|
||||
except subprocess.CalledProcessError as e:
|
||||
log(f"[ERROR] Backup creation failed: {e}")
|
||||
return
|
||||
finally:
|
||||
if os.path.exists(backup_file):
|
||||
os.remove(backup_file)
|
||||
|
||||
## Upload to R2
|
||||
if os.path.exists(compressed_file):
|
||||
size = os.path.getsize(compressed_file)
|
||||
log(f"[INFO] Final backup size: {size / 1024 / 1024:.2f} MB")
|
||||
|
||||
try:
|
||||
client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=R2_ENDPOINT,
|
||||
aws_access_key_id=R2_ACCESS_KEY,
|
||||
aws_secret_access_key=R2_SECRET_KEY,
|
||||
region_name=S3_REGION,
|
||||
config=Config(
|
||||
s3={"addressing_style": "path"}
|
||||
)
|
||||
)
|
||||
|
||||
config = TransferConfig(
|
||||
multipart_threshold=8 * 1024 * 1024,
|
||||
multipart_chunksize=8 * 1024 * 1024,
|
||||
max_concurrency=4,
|
||||
use_threads=True
|
||||
)
|
||||
|
||||
client.upload_file(
|
||||
compressed_file,
|
||||
R2_BUCKET_NAME,
|
||||
compressed_file_r2,
|
||||
Config=config
|
||||
)
|
||||
|
||||
log(f"[SUCCESS] Backup uploaded: {compressed_file_r2}")
|
||||
|
||||
objects = client.list_objects_v2(
|
||||
Bucket=R2_BUCKET_NAME,
|
||||
Prefix=BACKUP_PREFIX
|
||||
)
|
||||
|
||||
if "Contents" in objects:
|
||||
backups = sorted(
|
||||
objects["Contents"],
|
||||
key=lambda x: x["LastModified"],
|
||||
reverse=True
|
||||
)
|
||||
|
||||
for obj in backups[MAX_BACKUPS:]:
|
||||
client.delete_object(
|
||||
Bucket=R2_BUCKET_NAME,
|
||||
Key=obj["Key"]
|
||||
)
|
||||
log(f"[INFO] Deleted old backup: {obj['Key']}")
|
||||
|
||||
except Exception as e:
|
||||
log(f"[ERROR] R2 operation failed: {e}")
|
||||
finally:
|
||||
if os.path.exists(compressed_file):
|
||||
os.remove(compressed_file)
|
||||
|
||||
if __name__ == "__main__":
|
||||
log("[INFO] Starting backup scheduler...")
|
||||
log(f"[INFO] Scheduled backup time: {BACKUP_TIME} UTC")
|
||||
|
||||
schedule.every().day.at(BACKUP_TIME).do(run_backup)
|
||||
|
||||
run_backup()
|
||||
|
||||
while True:
|
||||
schedule.run_pending()
|
||||
time.sleep(60)
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
boto3==1.42.26
|
||||
psycopg2-binary==2.9.10
|
||||
python-dotenv==1.2.1
|
||||
py7zr==1.1.0
|
||||
schedule==1.2.2
|
||||
Loading…
Reference in New Issue