pretty big rework

- thrown out the multi-domain support, instead you can now specify the full base url in the config
- improved config docs
- changed authorization to follow the standard Bearer Token pattern
- link response is now simple json
- updated and modernized nginx config example
- reworked config and argument parsing
- removed unnecessary extra file write buffer layer
- removed no-cache response header. (why did I do this?)
- various minor improvements
- clarified hash length limit to be 32 (16 bytes, one aes block)
This commit is contained in:
BluBb_mADe 2023-12-12 13:20:44 +01:00
parent ef3d829dba
commit 6faac9742b
Signed by: Gardener
GPG Key ID: 1FAEB4540A5B4D1D
3 changed files with 147 additions and 96 deletions

View File

@ -1,28 +1,29 @@
# if this is false, everyone can upload stuff. # Enable upload authorization
auth: True auth: True
# if auth is true, only people that provide one of the tokens in the http headers are allowed to upload. # When auth is true, only clients providing one of the valid Bearer token in the HTTP Authorization header are permitted to upload.
# The token must be included in the HTTP headers using the standard format:
# "Authorization: Bearer <token>"
tokens: tokens:
- 'example token' - 'example_token'
# everyone who has it can delete files based on the download link. # To make deletion links work without a token while not having to maintain a database of deletion codes
# the script encrypts deletion links from file links using this key.
del_crypt_key: 'secret delete link key' del_crypt_key: 'secret delete link key'
# in bytes # Maximum file size in bytes (100MB in this example)
max_filesize: 1024 ** 2 * 100 max_filesize: 1024 ** 2 * 100
# uploaded files will be stored here # Directory where uploaded files will be stored
data_path: 'data' data_path: 'data'
# this can only be a multiple of 2 # URL hash length must be a multiple of 2 and can not exceed 32
# Uploads will start failing if the script can not find a free url hash with 100 random generated ones
url_hash_len: 6 url_hash_len: 6
# just affects the printed links (e.g. for reverse proxies) # Base URL for routing to this service. The trailing slash is optional.
protocol: 'https' base_url: 'https://your-domain.net/f/'
# uri that routes to this # Should file extensions be appended to the generated links.
prefix: '/f' # Links always work with or without file extensions (like puush.me).
show_ext: True
# whether or not extensions are appended in the generated links.
# links without file extensions still work. (just like push)
show_ext: True

View File

@ -1,15 +1,22 @@
upstream filehoster { upstream filehoster {
# the hostname/ip and port on which the sharex_server script is reachable
server file_hoster:80; server file_hoster:80;
} }
server { server {
listen 443 ssl http2; listen 443 ssl;
listen [::]:443 ssl;
http2 on;
ssl_certificate /path/to/fullchain.pem;
ssl_certificate_key /path/to/privkey.pem;
server_name your-domain.net; server_name your-domain.net;
# ShareX_Storage prefix is /f # Include the location in your ShareX_Storage base_url
location /f/ { location /f/ {
# Should match ShareX_Storage max_filesize
client_max_body_size 100M; client_max_body_size 100M;
proxy_pass http://filehoster$uri/$server_name; proxy_pass http://filehoster;
} }
} }

View File

@ -1,78 +1,114 @@
import os import os
import io
import sys
import yaml import yaml
import string import string
import hashlib import hashlib
import argparse
from collections import defaultdict from collections import defaultdict
from urllib.parse import urlparse
from Cryptodome.Cipher import AES from Cryptodome.Cipher import AES
from Cryptodome.Util import Padding from Cryptodome.Util import Padding
from aiohttp import web, hdrs from aiohttp import web, hdrs
class AttrDict(dict): class AppConfig:
def __init__(self, *args, **kwargs): _instance = None
super().__init__(*args, **kwargs)
self.__dict__ = self
def update(self, *d, **kwargs): def __new__(cls, *args, **kwargs):
for key, val in (d[0] if d else kwargs).items(): if not cls._instance:
setattr(self, key, val) cls._instance = super(AppConfig, cls).__new__(cls)
return cls._instance
def __getattr__(self, item): def __init__(self, config_file):
return self.setdefault(item, AttrDict()) if not hasattr(self, 'is_loaded'):
self.load_config(config_file)
self.is_loaded = True
def load_config(self, config_file):
with open(config_file) as f:
data = yaml.safe_load(f)
self.url_hash_len = data.get('url_hash_len', 8)
self.data_path = data.get('data_path', '/data')
self.auth = data.get('auth', True)
self.tokens = set(data.get('tokens', []))
self.del_crypt_key = data.get('del_crypt_key', 'default_key')
self.show_ext = data.get('show_ext', True)
self.max_filesize = data.get('max_filesize', '1024 ** 2 * 100') # Default 100 MB
self.base_url = data.get('base_url', 'http://localhost/f/')
self.validate_config()
def validate_config(self):
if self.url_hash_len > 32:
raise ValueError('url_hash_len cannot be greater than 32')
if self.url_hash_len % 2 != 0:
raise ValueError('url_hash_len must be a multiple of 2')
self.max_filesize = self.evaluate_filesize(self.max_filesize)
self.del_crypt_key = hashlib.md5(self.del_crypt_key.encode()).digest()[:16]
if not os.path.isdir(self.data_path):
os.mkdir(self.data_path)
self.base_url = f"{self.base_url.strip("/ \t\r\n")}"
@staticmethod @staticmethod
def from_dict_recur(d): def evaluate_filesize(size_str):
if not isinstance(d, AttrDict): valid_chars = set(string.hexdigits + '* ')
d = AttrDict(d) if not set(size_str).issubset(valid_chars):
for k, v in dict(d.items()).items(): raise ValueError('Invalid characters in max_filesize')
if " " in k:
del d[k] try:
d[k.replace(" ", "_")] = v return eval(size_str)
if isinstance(v, dict): except Exception:
d[k] = AttrDict.from_dict_recur(v) raise ValueError('Invalid format for max_filesize')
return d
def sizeof_fmt(num, suffix='B'): def sizeof_fmt(num, suffix='B'):
for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']: for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']:
if abs(num) < 1024.0: if abs(num) < 1024.:
return f"{num:3.1f}{unit}{suffix}" return f"{num:3.1f}{unit}{suffix}"
num /= 1024.0 num /= 1024.
return f"{num:.1f}Yi{suffix}" return f"{num:.1f}Yi{suffix}"
async def prepare(_, handler): async def handle_upload(req):
async def prepare_handler(req): if conf.auth:
if 'acc' not in req.match_info: auth_header = req.headers.get(hdrs.AUTHORIZATION, None)
return web.Response(text='bad request', status=400)
return await handler(req, req.match_info["acc"], file_db[req.match_info["acc"]])
return prepare_handler
if auth_header is None:
return web.Response(text='Authentication required', status=401)
try:
scheme, token = auth_header.split(' ')
if scheme.lower() != 'bearer':
raise ValueError
except ValueError:
return web.Response(text='Invalid authentication scheme', status=401)
if token not in conf.tokens:
return web.Response(text='Access denied', status=403)
async def handle_upload(req, acc, acc_db):
if conf.auth and req.headers.get('auth') not in conf.auth_tokens:
return web.Response(text='access denied', status=403)
reader = await req.multipart() reader = await req.multipart()
file = await reader.next() file = await reader.next()
filename = os.path.basename(file.filename) filename = os.path.basename(file.filename)
if not os.path.isdir(f'{conf.data_path}/{acc}'): if not os.path.isdir(f'{conf.data_path}'):
os.mkdir(f'{conf.data_path}/{acc}') os.mkdir(f'{conf.data_path}')
for _ in range(100): for _ in range(100):
hb = os.urandom(conf.url_hash_len//2) hb = os.urandom(conf.url_hash_len//2)
h = hb.hex() h = hb.hex()
if h not in acc_db: if h not in file_db:
break break
else: else:
return web.Response(text='url key-space full', status=500) return web.Response(text='url key-space full', status=500)
acc_db[h] = filename file_db[h] = filename
local_fname = f'{conf.data_path}/{acc}/{h}_{filename}' local_fname = f'{conf.data_path}/{h}_{filename}'
ext = os.path.splitext(filename)[1] if conf.show_ext else '' ext = os.path.splitext(filename)[1] if conf.show_ext else ''
os.fdopen(os.open(local_fname, os.O_WRONLY | os.O_CREAT, 0o600)).close() os.fdopen(os.open(local_fname, os.O_WRONLY | os.O_CREAT, 0o600)).close()
try: try:
@ -83,16 +119,17 @@ async def handle_upload(req, acc, acc_db):
c = AES.new(conf.del_crypt_key, AES.MODE_CBC) c = AES.new(conf.del_crypt_key, AES.MODE_CBC)
hb = Padding.pad(hb, AES.block_size) hb = Padding.pad(hb, AES.block_size)
del_h = (c.encrypt(hb) + c.iv).hex() del_h = (c.encrypt(hb) + c.iv).hex()
return web.Response(text=f'{conf.protocol}://{acc}{conf.prefix}/{h[:conf.url_hash_len]}{ext}\n' return web.Response(text=f'{{"file_link":"{conf.base_url}/{h}{ext}",'
f'{conf.protocol}://{acc}{conf.prefix}/del/{del_h}') f'"delete_link":"{conf.base_url}/del/{del_h}"}}', status=200)
os.unlink(local_fname) os.unlink(local_fname)
del acc_db[h] del file_db[h]
return web.Response(text=f'file is bigger than {sizeof_fmt(conf.max_filesize)}', status=413) return web.Response(text=f'file is bigger than {sizeof_fmt(conf.max_filesize)}', status=413)
async def recv_file(file, local_fname): async def recv_file(file, local_fname):
size = 0 size = 0
with io.BufferedWriter(open(local_fname, 'wb')) as f: with open(local_fname, 'wb') as f:
while True: while True:
chunk = await file.read_chunk() chunk = await file.read_chunk()
if not chunk: if not chunk:
@ -103,9 +140,9 @@ async def recv_file(file, local_fname):
f.write(chunk) f.write(chunk)
async def handle_delete(req, acc, acc_db): async def handle_delete(req):
chashiv = req.match_info.get('hash', 'x') chashiv = req.match_info.get('hash', 'x')
if not set(chashiv).issubset(valid_hash_chars) or len(chashiv) != 64: if len(chashiv) != 64 or not set(chashiv).issubset(hexdigits_set):
return web.Response(text='invalid delete link', status=400) return web.Response(text='invalid delete link', status=400)
chashiv = bytes.fromhex(chashiv) chashiv = bytes.fromhex(chashiv)
@ -115,58 +152,64 @@ async def handle_delete(req, acc, acc_db):
fhash = Padding.unpad(fhash, AES.block_size).hex() fhash = Padding.unpad(fhash, AES.block_size).hex()
except ValueError: except ValueError:
pass pass
if fhash not in acc_db or len(fhash) == 32: if fhash not in file_db:
return web.Response(text='this file doesn\'t exist on the server', status=404) return web.Response(text='this file doesn\'t exist on the server', status=404)
os.unlink(f"{conf.data_path}/{acc}/{fhash}_{acc_db[fhash]}") os.unlink(f"{conf.data_path}/{fhash}_{file_db[fhash]}")
del acc_db[fhash] del file_db[fhash]
return web.Response(text='file deleted') return web.Response(text='file deleted')
async def handle_download(req, acc, acc_db): async def handle_download(req):
fhash = req.match_info.get('hash', '').split('.', 1)[0] fhash = req.match_info.get('hash', '').split('.', 1)[0]
if fhash not in acc_db: if fhash not in file_db:
return web.Response(text='file not found', status=404) return web.Response(text='file not found', status=404)
return web.FileResponse(f"{conf.data_path}/{acc}/{fhash}_{acc_db[fhash]}", headers={ return web.FileResponse(f"{conf.data_path}/{fhash}_{file_db[fhash]}", headers={
hdrs.CACHE_CONTROL: "no-cache", hdrs.CONTENT_DISPOSITION: f'inline;filename="{file_db[fhash]}"'
hdrs.CONTENT_DISPOSITION: f'inline;filename="{acc_db[fhash]}"'
}) })
def main(): def main():
if conf.url_hash_len > 31: for file in os.listdir(f"{conf.data_path}"):
raise ValueError('url_hash_len can\'t be bigger than 31') try:
if not set(conf.max_filesize.replace(' ', ''))\ fhash, fname = file.split('_', 1)
.issubset(valid_hash_chars | {'*'}): except ValueError:
raise ValueError('max_filesize only can contain numbers and *') print(f"file \"{file}\" has an invalid file name format, skipping...")
conf.max_filesize = eval(conf.max_filesize)
conf.auth_tokens = set(conf.tokens)
conf.prefix = conf.prefix.strip("/")
if conf.prefix:
conf.prefix = f'/{conf.prefix}'
conf.del_crypt_key = hashlib.md5(conf.del_crypt_key.encode()).digest()[:16]
if not os.path.isdir(conf.data_path):
os.mkdir(conf.data_path)
for acc in os.listdir(conf.data_path):
if not os.path.isdir(f'{conf.data_path}/{acc}'):
continue continue
for file in os.listdir(f"{conf.data_path}/{acc}"): file_db[fhash] = fname
if "_" in file:
fhash, fname = file.split('_', 1)
file_db[acc][fhash] = fname
app = web.Application(middlewares=[prepare]) parsed_url = urlparse(conf.base_url)
app.router.add_post(conf.prefix + '/post/{acc}', handle_upload) base_path = parsed_url.path
app.router.add_get(conf.prefix + '/del/{hash}/{acc}', handle_delete)
app.router.add_get(conf.prefix + '/{hash}/{acc}', handle_download) app = web.Application()
app.router.add_post(base_path + '/post', handle_upload)
app.router.add_get(base_path + '/del/{hash}', handle_delete)
app.router.add_get(base_path + '/{hash}', handle_download)
web.run_app(app, port=80) web.run_app(app, port=80)
def parse_args():
parser = argparse.ArgumentParser(description="File serving and uploading server intended for use as a ShareX host.")
parser.add_argument('-c', '--config', default=None,
help='Path to the configuration file.')
parser.add_argument('config_file', nargs='?', default='config.yaml',
help='Path to the configuration file (positional argument).')
args = parser.parse_args()
if args.config and args.config_file != 'config.yaml':
print("Warning: Both positional and optional config arguments provided. Using the -c argument.")
return args.config
return args.config or args.config_file
if __name__ == '__main__': if __name__ == '__main__':
valid_hash_chars = set(string.hexdigits) hexdigits_set = set(string.hexdigits)
file_db = defaultdict(dict) file_db = defaultdict(dict)
confname = sys.argv[1] if sys.argv[1:] and os.path.isfile(sys.argv[1]) else 'config.yaml' conf_name = parse_args()
with open(confname) as cf: print("Loading config file", conf_name)
conf = AttrDict.from_dict_recur(yaml.safe_load(cf)) conf = AppConfig(conf_name)
main() main()