commit 9d8cd7b762cd71977eb5e3384eb57e710ad7d134 Author: Kakifilem Team Date: Sun Feb 15 20:35:49 2026 +0800 Improve S3 uploads with queueing, multipart tuning, and better UX diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..16c92cf --- /dev/null +++ b/.env.example @@ -0,0 +1,30 @@ +# Telegram +API_ID=123456 +API_HASH=your_api_hash_here +BOT_TOKEN=your_bot_token_here + +# Allowed Telegram user IDs (comma-separated) +ALLOWED_USERS=123456789 + +# S3 / S3-Compatible +S3_ENDPOINT= +S3_REGION=us-east-1 +S3_ACCESS_KEY=your_access_key +S3_SECRET_KEY=your_secret_key +S3_BUCKET=your_bucket_name + +# Return presigned download link after upload +ENABLE_PRESIGNED_URL=true + +# Presigned URL expiration (seconds) +PRESIGNED_EXPIRE_SECONDS=3600 + +# S3 upload tuning (per-file multipart) +S3_MAX_CONCURRENCY=4 + +# Optional advanced multipart tuning +S3_MULTIPART_THRESHOLD_MB=100 +S3_MULTIPART_CHUNK_MB=50 + +# Upload queue limits (global) +MAX_PARALLEL_UPLOADS=3 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7495464 --- /dev/null +++ b/.gitignore @@ -0,0 +1,15 @@ +.env + +__pycache__/ +*.pyc +*.pyo +*.pyd + +.venv/ +venv/ + +downloads/* +logs/ + +.DS_Store +Thumbs.db diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..c3eccce --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,53 @@ +# Contributing + +Contributions are welcome πŸ™‚ + +If you have an idea, improvement, or bug fix, feel free to jump in. + +--- + +## How to contribute + +1. Fork the repository +2. Create a branch for your change +3. Make your changes +4. Open a pull request + +That’s all. + +--- + +## A few guidelines + +- Keep changes small and focused +- Follow the existing code style where possible +- Don’t commit secrets or credentials +- Test your changes before opening a PR +- Keep features within the scope of **private file upload and backup** + +--- + +## Project scope + +This project is meant for: +- Personal backups +- Private file storage +- Archiving Telegram content you own or manage + +PRs that go beyond this project’s scope may be declined to keep things simple and focused. + +--- + +## Ideas to work on + +These are possible future improvements, not a fixed roadmap: + +- Channel backup support +- Streaming uploads (no local disk) +- File deduplication +- Simple admin commands +- Better logging or error handling + +--- + +Thanks for taking the time to contribute! diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ab8adac --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.13-slim + +WORKDIR /app + +ENV PYTHONUNBUFFERED=1 + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +CMD ["python", "bot.py"] diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..11618b8 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Aman + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..3d4e013 --- /dev/null +++ b/README.md @@ -0,0 +1,241 @@ +# Telegram β†’ S3 Uploader + +![License](https://img.shields.io/badge/license-MIT-blue.svg) +![Platform](https://img.shields.io/badge/platform-Telegram-blue) +![Python](https://img.shields.io/badge/python-3.13%2B-blue) +![Framework](https://img.shields.io/badge/framework-Pyrofork%20(async)-green) +![Deploy](https://img.shields.io/badge/deploy-Railway-purple) +![Docker](https://img.shields.io/badge/docker-supported-blue) + +A simple, private Telegram bot that uploads files to **any S3-compatible storage** +(AWS S3, Cloudflare R2, MinIO, etc.) and returns a **temporary presigned download link**. + +This project is designed to be: +- Simple +- Safe by default +- Easy to deploy +- Easy to extend + +--- + +## ✨ Features + +- Upload files from Telegram to S3-compatible storage +- Supports documents, videos, audio, and photos +- Private bucket (no public access required) +- Temporary **presigned download links** +- User allowlist (private bot) +- Works on local machine, VPS, Docker, and Railway +- Clean, minimal codebase + +--- + +## πŸ“¦ Supported Storage + +This bot works with any S3-compatible provider, including: + +- AWS S3 +- Cloudflare R2 +- MinIO (local NAS) +- Wasabi +- DigitalOcean Spaces +- Backblaze B2 (S3 API) + +--- + +## πŸ” Security Model + +- Files are uploaded to a **private bucket** +- Access is granted via **temporary presigned URLs** +- Links expire automatically +- Storage credentials are never exposed to users +- Bot usage can be restricted with an allowlist +- Local files are stored temporarily and cleaned up automatically + +This bot is intended for **private file upload and backup use cases**. + +--- + +## πŸš€ Deployment + +### β–Ά Deploy on Railway + +You can deploy this bot directly to Railway: + +[![Deploy on Railway](https://railway.com/button.svg)](https://railway.com/deploy/telegram-s3-uploader?referralCode=nIQTyp&utm_medium=integration&utm_source=template&utm_campaign=generic) + +After deployment, set the required environment variables in the Railway dashboard. + +> When deployed on Railway, this template attaches a Railway Bucket by default. +> You can remove it and configure any other S3-compatible storage via environment variables. + +--- + +### β–Ά Run locally + +> Python 3.13 is supported. Python 3.11+ is recommended for widest compatibility. + +#### 1. Clone the repository +```bash +git clone https://github.com/BigDaddyAman/telegram-s3-uploader +cd telegram-s3-uploader +``` + +#### 2. Create virtual environment +```bash +python3.13 -m venv venv +source venv/bin/activate # On Windows: venv\Scripts\activate +``` + +#### 3. Install dependencies +```bash +pip install --upgrade pip +pip install -r requirements.txt +``` + +#### 4. Configure environment + +Copy .env.example to .env and fill in your values: + +```bash +cp .env.example .env +``` + +#### 5. Run the bot +```bash +python bot.py +``` + +--- + +### β–Ά Run with Docker (build locally) + +```bash +docker build -t telegram-s3-uploader . +docker run --env-file .env telegram-s3-uploader +``` + +--- + +## βš™οΈ Environment Variables + +| Variable | Description | +|----------|-------------| +| API_ID | Telegram API ID | +| API_HASH | Telegram API hash | +| BOT_TOKEN | Telegram bot token | +| ALLOWED_USERS | Allowed Telegram user IDs (comma-separated) | +| S3_ENDPOINT | Custom S3 endpoint (leave empty for AWS) | +| S3_REGION | S3 region (required for AWS) | +| S3_ACCESS_KEY | S3 access key | +| S3_SECRET_KEY | S3 secret key | +| S3_BUCKET | Bucket name | +| PRESIGNED_EXPIRE_SECONDS | Presigned link expiration time (seconds) | +| ENABLE_PRESIGNED_URL | Enable or disable presigned download links (`true` / `false`) | +| S3_MAX_CONCURRENCY | Parallel multipart uploads per file (default: 4–6) | +| S3_MULTIPART_THRESHOLD_MB | File size threshold to enable multipart uploads (default: 100) | +| S3_MULTIPART_CHUNK_MB | Multipart chunk size in MB (default: 50) | +| MAX_PARALLEL_UPLOADS | Maximum number of files uploading at the same time (default: 3) | + +--- + +## 🚦 Upload Limits & Queueing + +To keep the bot stable and prevent overload, uploads are limited using two mechanisms: + +### Global upload limit +Only a fixed number of files can upload at the same time. +Additional uploads are automatically queued. + +Controlled by: MAX_PARALLEL_UPLOADS=3 + +### Per-user limit +Each user can upload **only one file at a time**. +This prevents a single user from occupying all upload slots. + +These limits ensure: +- Stable CPU and memory usage +- Fair access for all users +- Safe operation on Railway and small VPS instances + +--- + + +## ⚑ Performance Tuning + +For large files (2–4 GB), upload performance can be tuned using environment variables. +The default values are safe and work well for most deployments, but advanced users can +adjust them to balance speed, CPU usage, and memory consumption. + +### Multipart upload settings + +These settings control how a **single file** is uploaded to S3-compatible storage. + +```env +S3_MAX_CONCURRENCY=4 +S3_MULTIPART_THRESHOLD_MB=100 +S3_MULTIPART_CHUNK_MB=50 +``` + +- **`S3_MAX_CONCURRENCY`** + Number of multipart chunks uploaded in parallel for a single file. + Higher values increase upload speed but also increase CPU usage. + +- **`S3_MULTIPART_THRESHOLD_MB`** + File size (in MB) at which multipart uploads are enabled. + Files smaller than this value are uploaded using a single request. + +- **`S3_MULTIPART_CHUNK_MB`** + Size of each multipart chunk (in MB). + Larger chunks reduce overhead but increase memory usage. + +--- + +## πŸ“ File Storage Layout + +Uploaded files are stored using a simple structure: + +> Files are downloaded to a temporary local directory and removed automatically after upload. + +``` +users/ +└── / + └── YYYYMMDD_HHMMSS_filename.ext +``` + +This makes the storage easy to browse and suitable for backups. + +--- + +## 🧭 Project Scope + +This project is intentionally kept small and focused. + +It is meant for: + +- Personal backups +- Private file storage +- Archiving Telegram content you own or manage + +Features related to public distribution, streaming, or content sharing are out of scope. + +--- + +## 🀝 Contributing + +Contributions are welcome! + +Please see [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines and project scope. + +--- + +## πŸ” Security + +If you discover a security issue, please follow the instructions in [SECURITY.md](SECURITY.md). + +--- + +## πŸ“„ License + +This project is licensed under the MIT License. +See [LICENSE](LICENSE) for details. \ No newline at end of file diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..09d87ac --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,41 @@ +# Security Policy + +## Supported versions + +Only the latest version of this project is supported. + +If you’re running an older version, please update before reporting issues. + +--- + +## Reporting a security issue + +If you discover a security issue, please **do not open a public issue**. + +Instead: +- Open a GitHub Security Advisory + or +- Contact the maintainer privately (if contact info is available) + +This helps prevent accidental exposure while the issue is being fixed. + +--- + +## Security notes + +- This bot uploads files to **private S3-compatible storage** +- Files are accessed using **temporary presigned URLs** +- Storage credentials are never shared with Telegram users +- Bot access can be restricted using an allowlist +- No public access is enabled by default + +--- + +## Responsibility + +You are responsible for: +- Protecting your credentials +- Controlling who can use your bot +- Running the bot in an environment you trust + +This project is provided as-is and is intended for **private file upload and backup use cases**. diff --git a/bot.py b/bot.py new file mode 100644 index 0000000..eecd97d --- /dev/null +++ b/bot.py @@ -0,0 +1,18 @@ +from pyrogram import Client +from config import API_ID, API_HASH, BOT_TOKEN +from handlers.private_upload import handle_private_upload +from handlers.start import handle_start + +app = Client( + "telegram-s3-uploader", + api_id=API_ID, + api_hash=API_HASH, + bot_token=BOT_TOKEN, +) + +handle_start(app) +handle_private_upload(app) +handle_private_upload(app) + +print("πŸš€ Bot started") +app.run() diff --git a/config.py b/config.py new file mode 100644 index 0000000..767ef9e --- /dev/null +++ b/config.py @@ -0,0 +1,49 @@ +import os +from dotenv import load_dotenv + +load_dotenv() + +API_ID = int(os.getenv("API_ID")) +API_HASH = os.getenv("API_HASH") +BOT_TOKEN = os.getenv("BOT_TOKEN") + +ALLOWED_USERS = os.getenv("ALLOWED_USERS", "") +if ALLOWED_USERS: + ALLOWED_USERS = {int(x.strip()) for x in ALLOWED_USERS.split(",")} +else: + ALLOWED_USERS = set() + +S3_ENDPOINT = os.getenv("S3_ENDPOINT") +S3_REGION = os.getenv("S3_REGION") +S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY") +S3_SECRET_KEY = os.getenv("S3_SECRET_KEY") +S3_BUCKET = os.getenv("S3_BUCKET") + +DOWNLOAD_DIR = "downloads" + +ENABLE_PRESIGNED_URL = os.getenv( + "ENABLE_PRESIGNED_URL", "true" +).lower() == "true" + +PRESIGNED_EXPIRE_SECONDS = int( + os.getenv("PRESIGNED_EXPIRE_SECONDS", "3600") +) + +S3_MAX_CONCURRENCY = int(os.getenv("S3_MAX_CONCURRENCY", "6")) +S3_MULTIPART_THRESHOLD_MB = int(os.getenv("S3_MULTIPART_THRESHOLD_MB", "100")) +S3_MULTIPART_CHUNK_MB = int(os.getenv("S3_MULTIPART_CHUNK_MB", "50")) + +MAX_PARALLEL_UPLOADS = int(os.getenv("MAX_PARALLEL_UPLOADS", "3")) + +required = [ + "API_ID", + "API_HASH", + "BOT_TOKEN", + "S3_ACCESS_KEY", + "S3_SECRET_KEY", + "S3_BUCKET", +] + +for var in required: + if not globals().get(var): + raise RuntimeError(f"Missing env var: {var}") diff --git a/handlers/__init__.py b/handlers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/handlers/private_upload.py b/handlers/private_upload.py new file mode 100644 index 0000000..26b1d38 --- /dev/null +++ b/handlers/private_upload.py @@ -0,0 +1,131 @@ +import os +import time +import shutil +from pyrogram import filters +import asyncio + +from utils.filenames import sanitize_filename +from utils.helpers import ensure_dir +from s3.client import ( + S3_TRANSFER_CONFIG, + get_s3_client, + generate_presigned_url, +) +from config import ( + DOWNLOAD_DIR, + S3_BUCKET, + ALLOWED_USERS, + ENABLE_PRESIGNED_URL, + PRESIGNED_EXPIRE_SECONDS, + MAX_PARALLEL_UPLOADS, +) + +s3 = get_s3_client() + +upload_semaphore = asyncio.Semaphore(MAX_PARALLEL_UPLOADS) +user_locks: dict[int, asyncio.Lock] = {} + +async def s3_upload_async(s3, file_path, bucket, key): + loop = asyncio.get_running_loop() + await loop.run_in_executor( + None, + lambda: s3.upload_file( + file_path, + bucket, + key, + Config=S3_TRANSFER_CONFIG + ) + ) + + +def handle_private_upload(app): + + @app.on_message( + filters.private + & (filters.document | filters.video | filters.audio | filters.photo) + ) + async def upload_handler(client, message): + user_id = message.from_user.id + + if ALLOWED_USERS and user_id not in ALLOWED_USERS: + await message.reply("β›” You are not allowed to use this bot.") + return + + status = await message.reply("⏳ Waiting for slot...") + + ensure_dir(DOWNLOAD_DIR) + temp_dir = os.path.join(DOWNLOAD_DIR, f"{user_id}_{message.id}") + ensure_dir(temp_dir) + + try: + file = ( + message.document + or message.video + or message.audio + or message.photo + ) + + size_bytes = getattr(file, "file_size", None) + size_text = ( + f"{size_bytes / (1024 * 1024):.2f} MB" + if size_bytes else "Unknown" + ) + + if file and getattr(file, "file_name", None): + ts = time.strftime("%Y%m%d_%H%M%S") + filename = f"{ts}_{sanitize_filename(file.file_name)}" + else: + if message.photo: + filename = sanitize_filename(None, fallback_ext=".jpg") + else: + filename = sanitize_filename(None) + + local_path = os.path.join(temp_dir, filename) + s3_key = f"users/{user_id}/{filename}" + + lock = user_locks.setdefault(user_id, asyncio.Lock()) + + async with lock: + async with upload_semaphore: + await status.edit("πŸ“₯ Downloading...") + file_path = await message.download(file_name=local_path) + + await status.edit("☁️ Uploading to storage...") + await s3_upload_async( + s3, + file_path, + S3_BUCKET, + s3_key + ) + + if ENABLE_PRESIGNED_URL: + presigned_url = generate_presigned_url( + s3=s3, + bucket=S3_BUCKET, + key=s3_key, + expires=PRESIGNED_EXPIRE_SECONDS, + ) + + minutes = PRESIGNED_EXPIRE_SECONDS // 60 + await status.edit( + f"βœ… Uploaded!\n\n" + f"πŸ“„ File:\n" + f"{filename}\n\n" + f"πŸ“¦ Size: {size_text}\n\n" + f"πŸ”— Download link (expires in {minutes} min):\n" + f"{presigned_url}" + ) + else: + await status.edit( + f"βœ… Uploaded!\n\n" + f"πŸ“„ File:\n" + f"{filename}\n" + f"πŸ“¦ Size: {size_text}\n\n" + f"πŸ—‚ Stored at:\n" + f"`{s3_key}`" + ) + + finally: + if os.path.exists(temp_dir): + shutil.rmtree(temp_dir, ignore_errors=True) + diff --git a/handlers/start.py b/handlers/start.py new file mode 100644 index 0000000..3b1a613 --- /dev/null +++ b/handlers/start.py @@ -0,0 +1,17 @@ +from pyrogram import filters +from config import ALLOWED_USERS + +def handle_start(app): + + @app.on_message(filters.command("start") & filters.private) + async def start_handler(client, message): + user_id = message.from_user.id + + if user_id not in ALLOWED_USERS: + await message.reply("β›” You are not allowed to use this bot.") + return + + await message.reply( + "βœ… Bot is alive!\n\n" + "πŸ“€ Send me any file and I will upload it to storage." + ) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..7ef91c6 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +pyrofork==2.3.69 +TgCrypto-pyrofork==1.2.8 +boto3==1.42.49 +python-dotenv==1.2.1 diff --git a/s3/client.py b/s3/client.py new file mode 100644 index 0000000..540cfd9 --- /dev/null +++ b/s3/client.py @@ -0,0 +1,45 @@ +import boto3 +from botocore.client import Config +from boto3.s3.transfer import TransferConfig + +from config import ( + S3_ENDPOINT, + S3_REGION, + S3_ACCESS_KEY, + S3_SECRET_KEY, + S3_MAX_CONCURRENCY, + S3_MULTIPART_THRESHOLD_MB, + S3_MULTIPART_CHUNK_MB, +) + +S3_TRANSFER_CONFIG = TransferConfig( + multipart_threshold=S3_MULTIPART_THRESHOLD_MB * 1024 * 1024, + multipart_chunksize=S3_MULTIPART_CHUNK_MB * 1024 * 1024, + max_concurrency=S3_MAX_CONCURRENCY, + use_threads=True, +) + +def get_s3_client(): + kwargs = { + "aws_access_key_id": S3_ACCESS_KEY, + "aws_secret_access_key": S3_SECRET_KEY, + "config": Config(signature_version="s3v4"), + } + + if S3_ENDPOINT: + kwargs["endpoint_url"] = S3_ENDPOINT + kwargs["region_name"] = S3_REGION or "us-east-1" + else: + if not S3_REGION: + raise RuntimeError("AWS S3 requires S3_REGION") + kwargs["region_name"] = S3_REGION + + return boto3.client("s3", **kwargs) + + +def generate_presigned_url(s3, bucket: str, key: str, expires: int): + return s3.generate_presigned_url( + ClientMethod="get_object", + Params={"Bucket": bucket, "Key": key}, + ExpiresIn=expires, + ) diff --git a/utils/__init__.py b/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/utils/filenames.py b/utils/filenames.py new file mode 100644 index 0000000..f756b29 --- /dev/null +++ b/utils/filenames.py @@ -0,0 +1,12 @@ +import os +import re +import time + +def sanitize_filename(name: str | None, fallback_ext: str = "") -> str: + if not name: + ts = time.strftime("%Y%m%d_%H%M%S") + return f"file_{ts}{fallback_ext}" + + name = os.path.basename(name) + name = re.sub(r"[^\w\-. ]", "_", name) + return name.strip() diff --git a/utils/helpers.py b/utils/helpers.py new file mode 100644 index 0000000..70dc93a --- /dev/null +++ b/utils/helpers.py @@ -0,0 +1,4 @@ +import os + +def ensure_dir(path: str): + os.makedirs(path, exist_ok=True)