#!/usr/bin/python3 #/* vim:set ts=2 set noexpandtab */ from flask import Flask, render_template, url_for, request, send_file, redirect from flask_babel import Babel, gettext as _, lazy_gettext from werkzeug import secure_filename from hurry.filesize import size from hashlib import sha1 import threading import binascii import bencoder import requests import hashlib import sqlite3 import base64 import urllib import json import uuid import time app = Flask(__name__) babel = Babel(app) LANGUAGES = ['en', 'de'] settings = None class Categories(): def __init__(self): self.categories = settings["categories"] for c in self.categories: c["label"] = str(lazy_gettext(c["label"])) for s in c["subcategories"]: s["label"] = str(lazy_gettext(s["label"])) def find(self, category, subcategory = None): cat_name = "" sub_name = "" for cat in self.categories: if cat["id"] == category: cat_name = cat["label"] if subcategory != None: for sub in cat["subcategories"]: if sub["id"] == subcategory: sub_name = sub["label"] return (cat_name, sub_name) class ScrapeState(): stats = {} def __init__(self): pass def update(self): self._statedump() self._tpbs() self._fullscrape() def _statedump(self): url = settings["scrape_url"] statedump_url = url + "/stats" params = { "mode" : "statedump" } req = requests.get(statedump_url, params=params) dump = req.text.strip() dump = dump.split("\n") for entry in dump: entry = entry.split(":") key = entry[0].lower() if not key in self.stats.keys(): self.stats.update({ key : {}}) self.stats.get(key).update({ "base" : entry[1], "unsure_downloaded" : entry[2] }) def _tpbs(self): url = settings["scrape_url"] tpbs_url = url + "/stats" params = { "mode" : "tpbs", "format" : "ben" } req = requests.get(tpbs_url, params=params) decoded = bencoder.decode(req.content) for torrent in decoded[b"files"]: info_hash = binascii.b2a_hex(torrent) stats = decoded[b"files"][torrent] key = info_hash.decode("utf-8").lower() self.stats.get(key).update({ "seeds" : stats[b"complete"], "peers" : stats[b"incomplete"], "complete" : stats[b"downloaded"] }) def _fullscrape(self): connection = sqlite3.connect("torrentdb.sqlite") c = connection.cursor() c.execute("SELECT fileid FROM torrents") all_hashes = c.fetchall() for info_hash in all_hashes: info_hash = info_hash[0] url_param = binascii.a2b_hex(info_hash.encode()) url = settings["scrape_url"] req = requests.get(url + "/scrape", params={"info_hash" : url_param}) decoded = bencoder.decode(req.content) info = decoded[b"files"] try: ugly_hash, stats = info.popitem() key = info_hash.lower() self.stats.get(key).update({ "seeds" : stats[b"complete"], "peers" : stats[b"incomplete"], "complete" : stats[b"downloaded"] }) except KeyError: print("No stats found for {}".format(info_hash)) @app.route("/") def index(): return render_template("search.html", categories=categories.categories) @app.route("/categories") def categorys(): return render_template("categories.html", categories=categories.categories) @app.route("/create", methods=['GET','POST']) def create(): if request.method == "GET": return render_template("create.html", categories=categories.categories, errors=None) elif request.method == "POST": newTorrent = createNewTorrent(request) if len(newTorrent.errors) == 0: message = _("Successfully created torrent {}").format(newTorrent.fileid, newTorrent.fileid[:-20]) return render_template("create.html", categories=categories.categories, messages=[message]) else: return render_template("create.html", categories=categories.categories, errors=newTorrent.errors) @app.route("/download/") def download(filename): connection = sqlite3.connect("torrentdb.sqlite") c = connection.cursor() c.execute("SELECT name FROM torrents WHERE fileid = :fileid", { 'fileid' : filename}) name = c.fetchone()[0] return send_file("torrentFiles/" + filename, as_attachment=True, attachment_filename=name + ".torrent", conditional=True) @app.route("/search", methods=['GET']) def search(): connection = sqlite3.connect("torrentdb.sqlite") c = connection.cursor() search_params = [] search = "" fields = list(request.args.keys()) for field in fields: query_list = request.args.getlist(field) for query in query_list: if len(search) > 0: search += " AND " if field is "q": names = query.split(" ") search_params += list(map(lambda x: "%" + x + "%", names)) search += " AND ".join(["torrents.name LIKE (?)"] * len(query.split(" "))) elif field is "c": search_params += query.split(" ") search += " AND ".join(["torrents.category LIKE (?)"] * len(query.split(" "))) elif field is "s": search_params += query.split(" ") search += " AND ".join(["torrents.subcategory LIKE (?)"] * len(query.split(" "))) elif field is "h": hashes = query.split(" ") search_params += list(map(lambda x: x + "%", hashes)) search += " AND ".join(["torrents.fileid LIKE (?)"] * len(query.split(" "))) print(search) results = list() for row in c.execute("SELECT torrents.fileid, torrents.name, metadata.torrentsize FROM torrents LEFT JOIN metadata on metadata.fileid = torrents.fileid WHERE " + search, search_params): r = row[0:2] + (size(float(row[2])) , ) + row[3:] results.append(r) return render_template("result.html", results=results, categories=categories.categories, stats=scrapeState.stats) @app.route("/details", methods=['GET']) def details(): info_hash = request.args["h"] tf = TorrentFile(fileid=info_hash) tf.fromDb() return render_template("details.html", categories=categories.categories, torrent=tf) def init(): global settings with open("settings.json") as settingsJson: settings = json.load(settingsJson) initDb() global categories categories = Categories() global scrapeState scrapeState = ScrapeState() scrape = threading.Thread(target=scraper) scrape.start() def scraper(): while True: print("Start scraping") scrapeState.update() print("Scraping done") time.sleep(60) def initDb(): connection = sqlite3.connect("torrentdb.sqlite") c = connection.cursor() c.execute('CREATE TABLE IF NOT EXISTS torrents (fileid TEXT PRIMARY KEY NOT NULL, name TEXT NOT NULL, category TEXT NOT NULL, subcategory TEXT NOT NULL, description TEXT NOT NULL, audioquality_description TEXT NOT NULL, videoquality_description TEXT NOT NULL);') c.execute('CREATE TABLE IF NOT EXISTS metadata (fileid TEXT PRIMARY KEY NOT NULL, created_by TEXT, creation_date TEXT, announce_url TEXT NOT NULL, source TEXT, torrentsize TEXT NOT NULL, name TEXT NOT NULL, private TEXT NOT NULL)') connection.commit() connection.close() def createNewTorrent(reuqest): uploadfile = request.files["torrentFile"] filename = secure_filename(uploadfile.filename) content = request.files["torrentFile"].stream.read() bcoded = bencoder.decode(content) info_hash = sha1(bencoder.encode(bcoded[b'info'])).hexdigest() #TODO: Validate the input serverside before writing it to the database name = request.form["name"] category = request.form["category"] subcategory = request.form["subcategory"] description = request.form["description"] audioquality_description = request.form["audioquality_description"] videoquality_description = request.form["videoquality_description"] newTFile = TorrentFile(info_hash, name, category, subcategory, description, audioquality_description, videoquality_description) try: announce = bcoded[b'announce-list'] except KeyError: announce = (bcoded[b'announce'], ) is_ours = False for a in announce: a = a.decode("utf-8", "ignore") if a in settings["valid_tracker"]: is_ours = True break if not is_ours: newTFile.errors = ["Rejecting torrent {}, as it does not use our tracker".format(info_hash, info_hash[:-20])] return newTFile with open("torrentFiles/" + info_hash, "wb") as torrent_file: torrent_file.write(content) bcoded = bencoder.decode(content) size = ((len(bcoded[b'info'][b'pieces']) / 20) * bcoded[b'info'][b'piece length']) / 1024 / 1024 print("=== CREATE NEW TORRENT FILE ===") print( "Name: " + request.form["name"] ) print( "Torrent file: " + info_hash ) print( "Category: " + request.form["category"] ) print( "Subcategory: " + request.form["subcategory"] ) print( "Description: " + request.form["description"] ) #TODO: Validate the input serverside before writing it to the database name = request.form["name"] category = request.form["category"] subcategory = request.form["subcategory"] description = request.form["description"] audioquality_description = request.form["audioquality_description"] videoquality_description = request.form["videoquality_description"] newTFile = TorrentFile(info_hash, name, category, subcategory, description, audioquality_description, videoquality_description) try: connection = sqlite3.connect("torrentdb.sqlite") newTFile.writeToDb(connection.cursor()) newTFile.metadata.writeToDb(connection.cursor()) connection.commit() connection.close() except sqlite3.IntegrityError as e: print(e) newTFile.errors = [_("Torrent {} already exists").format(info_hash, info_hash[:-20])] except Exception as e: newTFile.errors = [_("Unknown error in creation")] return newTFile class Metadata(): def __init__(self, fileid): try: with open("torrentFiles/" + fileid, "rb") as f: torrent = f.read() except FileNotFoundError: return self.fileid = fileid self.bcoded = bencoder.decode(torrent) self.created_by = self.bcoded.get(b'created by', b"").decode("utf-8", "ignore") self.creation_date = self.bcoded.get(b'creation date', 0) self.announce_url = self.bcoded.get(b'info', dict()).get(b'', "") self.source = self.bcoded.get(b'info', dict()).get(b'source', b"") self.torrentsize = ((len(self.bcoded.get(b'info', dict()).get(b'pieces', "")) / 20) * self.bcoded.get(b'info', dict()).get(b'piece length')) self.torrentsize_human = size(self.torrentsize) self.name = self.bcoded.get(b'info', dict()).get(b'name', b"").decode("utf-8", "ignore") self.private = self.bcoded.get(b'info', dict()).get(b'private', b"") def writeToDb(self, cursor): c = cursor b64created_by = base64.b64encode(self.created_by) if self.created_by else "" b64announce_url = base64.b64encode(self.announce_url.decode()) if self.announce_url else "" b64source = base64.b64encode(self.source) if self.source else "" b64name = base64.b64encode(self.name) c.execute("INSERT INTO metadata(fileid, created_by, creation_date, announce_url, source, torrentsize, name, private) VALUES(:fileid, :created_by, :creation_date, :announce_url, :source, :torrentsize, :name, :private)", { 'fileid' : self.fileid, 'created_by' : b64created_by, 'creation_date' : self.creation_date, 'announce_url' : b64announce_url, 'source' : b64source , 'torrentsize' : self.torrentsize, 'name' : b64name, 'private' : self.private}) class TorrentFile(): errors = [] fileid = None name = None category = None subcategory = None description = None audioquality_description = None videoquality_description = None def __init__(self, fileid=fileid, name=name, category=category, subcategory=subcategory, description=description, audioquality_description=audioquality_description, videoquality_description=videoquality_description): self.fileid = fileid self.name = name self.category = category self.subcategory = subcategory self.description = description self.audioquality_description = audioquality_description self.videoquality_description = videoquality_description if self.fileid: self.metadata = Metadata(fileid) def writeToDb(self, cursor): c = cursor b64description = base64.b64encode(self.description.encode()) b64audioquality_description = base64.b64encode(self.audioquality_description.encode()) b64videoquality_description = base64.b64encode(self.videoquality_description.encode()) c.execute("INSERT INTO torrents(fileid, name, category, subcategory, description, audioquality_description, videoquality_description) VALUES(:fileid, :name, :category, :subcategory, :description, :audioquality_description, :videoquality_description)", { 'fileid' : self.fileid, 'name' : self.name, 'category' : self.category, 'subcategory' : self.subcategory, 'description' : b64description , 'audioquality_description' : b64audioquality_description, 'videoquality_description' : b64videoquality_description}) def fromDb(self): def dict_factory(cursor, row): d = {} for idx, col in enumerate(cursor.description): d[col[0]] = row[idx] return d con = sqlite3.connect("torrentdb.sqlite") con.row_factory = dict_factory c = con.cursor() res = c.execute("SELECT torrents.*, metadata.* FROM torrents LEFT JOIN metadata on metadata.fileid = torrents.fileid WHERE torrents.fileid LIKE :fileid", { "fileid" : self.fileid }) res = res.fetchone() self.fileid = res["fileid"] self.name = (base64.b64decode(res["name"])).decode() self.category = res["category"] self.subcategory = res["subcategory"] self.category_string, self.subcategory_string = categories.find(int(self.category), int(self.subcategory)) self.description = (base64.b64decode(res["description"])).decode() self.audioquality_description = (base64.b64decode(res["audioquality_description"])).decode() self.videoquality_description = (base64.b64decode(res["videoquality_description"])).decode() self.metadata = Metadata(self.fileid) @babel.localeselector def get_locale(): return request.accept_languages.best_match(LANGUAGES) if __name__ == "__main__": init() app.jinja_env.globals.update(json=json) app.jinja_env.globals.update(sorted=sorted) app.run(debug=False, host="127.0.0.1") # vim: set ts=2 sts=2 sw=2 noexpandtab: