From b86e6cb7cb21de5a9640cd65a4201e2bd54cf43d Mon Sep 17 00:00:00 2001 From: sqozz Date: Thu, 28 Dec 2017 08:36:44 +0100 Subject: [PATCH] Add metadata to database and UI --- indexer.py | 60 +++++++++++++++++++++++++++++++++++---- static/css/categories.css | 10 +++++++ static/js/create.js | 13 --------- static/js/main.js | 14 +++++++++ templates/categories.html | 33 ++++++++++----------- templates/result.html | 11 +++---- 6 files changed, 101 insertions(+), 40 deletions(-) create mode 100644 static/css/categories.css diff --git a/indexer.py b/indexer.py index 2d17a21..214cd65 100644 --- a/indexer.py +++ b/indexer.py @@ -1,7 +1,9 @@ #!/usr/bin/python3 #/* vim:set ts=2 set noexpandtab */ import json, uuid, hashlib, sqlite3, base64 -from flask import Flask, render_template, url_for, request +import bencoder +import requests +from flask import Flask, render_template, url_for, request, send_file from werkzeug import secure_filename app = Flask(__name__) strings = None @@ -27,6 +29,14 @@ def create(): else: return render_template("create.html", language="english", categories=settings["categories"], strings=strings, errors=errors) +@app.route("/download/") +def download(filename): + connection = sqlite3.connect("torrentdb.sqlite") + c = connection.cursor() + c.execute("SELECT name FROM torrents WHERE fileid = :fileid", { 'fileid' : filename}) + name = c.fetchone()[0] + return send_file("torrentFiles/" + filename, as_attachment=True, attachment_filename=name + ".torrent", conditional=True) + @app.route("/search", methods=['GET']) def search(): global strings @@ -46,19 +56,24 @@ def search(): if field is "q": names = query.split(" ") search_params += list(map(lambda x: "%" + x + "%", names)) - search += " AND ".join(["name LIKE (?)"] * len(query.split(" "))) + search += " AND ".join(["torrents.name LIKE (?)"] * len(query.split(" "))) elif field is "c": search_params += query.split(" ") - search += " AND ".join(["category LIKE (?)"] * len(query.split(" "))) + search += " AND ".join(["torrents.category LIKE (?)"] * len(query.split(" "))) elif field is "s": search_params += query.split(" ") - search += " AND ".join(["subcategory LIKE (?)"] * len(query.split(" "))) + search += " AND ".join(["torrents.subcategory LIKE (?)"] * len(query.split(" "))) print(search) - c.execute("SELECT fileid, name FROM torrents WHERE " + search, search_params) + c.execute("SELECT torrents.fileid, torrents.name, metadata.torrentsize FROM torrents LEFT JOIN metadata on metadata.fileid = torrents.fileid WHERE " + search, search_params) results = c.fetchall() return render_template("result.html", results=results, strings=strings, language="english", categories=settings["categories"]) +def scrapeAll(): + TRACKER_URL = "http://tracker.lootbox.cf:6969/" + statedump = requests.get(TRACKER_URL + "stats" + "?mode=statedump") + return + def init(): global strings global settings @@ -68,11 +83,13 @@ def init(): with open("settings.json") as settingsJson: settings = json.load(settingsJson) initDb() + scrapeAll() def initDb(): connection = sqlite3.connect("torrentdb.sqlite") c = connection.cursor() c.execute('CREATE TABLE IF NOT EXISTS torrents (fileid TEXT PRIMARY KEY NOT NULL, name TEXT NOT NULL, category TEXT NOT NULL, subcategory TEXT NOT NULL, description TEXT NOT NULL, audioquality_description TEXT NOT NULL, videoquality_description TEXT NOT NULL);') + c.execute('CREATE TABLE IF NOT EXISTS metadata (fileid TEXT PRIMARY KEY NOT NULL, created_by TEXT, creation_date TEXT, announce_url TEXT NOT NULL, source TEXT, torrentsize TEXT NOT NULL, name TEXT NOT NULL, private TEXT NOT NULL)') connection.commit() connection.close() @@ -94,6 +111,13 @@ def createNewTorrent(reuqest): h.update((str(uuid.uuid4()) + filename).encode()) safeFilename = h.hexdigest() uploadfile.save("torrentFiles/" + safeFilename) + import bencoder + with open("torrentFiles/" + safeFilename, "rb") as content: + content = content.read() + + bcoded = bencoder.decode(content) + size = ((len(bcoded[b'info'][b'pieces']) / 20) * bcoded[b'info'][b'piece length']) / 1024 / 1024 + print("=== CREATE NEW TORRENT FILE ===") print( "Name: " + request.form["name"] ) print( "Torrent file: " + safeFilename ) @@ -111,10 +135,33 @@ def createNewTorrent(reuqest): newTFile = TorrentFile(safeFilename, name, category, subcategory, description, audioquality_description, videoquality_description) connection = sqlite3.connect("torrentdb.sqlite") newTFile.writeToDb(connection.cursor()) + newTFile.metadata.writeToDb(connection.cursor()) connection.commit() connection.close() return ["Error1"] +class Metadata(): + def __init__(self, fileid): + with open("torrentFiles/" + fileid, "rb") as f: + torrent = f.read() + self.fileid = fileid + self.bcoded = bencoder.decode(torrent) + self.created_by = self.bcoded.get(b'created by', "") + self.creation_date = self.bcoded.get(b'creation date', "") + self.announce_url = self.bcoded.get(b'info', dict()).get(b'', "") + self.source = self.bcoded.get(b'info', dict()).get(b'source', "") + self.torrentsize = ((len(self.bcoded.get(b'info', dict()).get(b'pieces', "")) / 20) * self.bcoded.get(b'info', dict()).get(b'piece length')) + self.name = self.bcoded.get(b'info', dict()).get(b'name', "") + self.private = self.bcoded.get(b'info', dict()).get(b'private', "") + + def writeToDb(self, cursor): + c = cursor + b64created_by = base64.b64encode(self.created_by) + b64announce_url = base64.b64encode(self.announce_url.encode()) if self.announce_url else "" + b64source = base64.b64encode(self.source.encode()) if self.source else "" + b64name = base64.b64encode(self.name) + c.execute("INSERT INTO metadata(fileid, created_by, creation_date, announce_url, source, torrentsize, name, private) VALUES(:fileid, :created_by, :creation_date, :announce_url, :source, :torrentsize, :name, :private)", { 'fileid' : self.fileid, 'created_by' : b64created_by, 'creation_date' : self.creation_date, 'announce_url' : b64announce_url, 'source' : b64source , 'torrentsize' : self.torrentsize, 'name' : b64name, 'private' : self.private}) + class TorrentFile(): fileid = None name = None @@ -131,6 +178,7 @@ class TorrentFile(): self.description = description self.audioquality_description = audioquality_description self.videoquality_description = videoquality_description + self.metadata = Metadata(fileid) def writeToDb(self, cursor): c = cursor @@ -144,4 +192,4 @@ if __name__ == "__main__": app.jinja_env.globals.update(getLocalString=getLocalString) app.jinja_env.globals.update(json=json) app.jinja_env.globals.update(sorted=sorted) - app.run(debug=True, host="127.0.0.1") + app.run(debug=True, host="0.0.0.0") diff --git a/static/css/categories.css b/static/css/categories.css new file mode 100644 index 0000000..3e935c9 --- /dev/null +++ b/static/css/categories.css @@ -0,0 +1,10 @@ +.inner.cover { + display: flex; + align-items: baseline; + flex-wrap: wrap; +} + +.inner.cover > span { + width: 33%; + margin-bottom: 10px; +} diff --git a/static/js/create.js b/static/js/create.js index aed1968..9482624 100644 --- a/static/js/create.js +++ b/static/js/create.js @@ -80,19 +80,6 @@ function fillDetectedFilelist(file) { reader.readAsArrayBuffer(file) } -function getNextUnit(bSize) { - units = ["B", "kB", "MB", "GB", "TB", "PB", "To damn high"] - iters = 0 - size = bSize - do { - size /= 1024 - iters++ - } while(size >= 1) - size *= 1024 - if(iters > units.length) { iters = units.length } - return [size, units[iters - 1]] -} - // Fill a defined dropdown with values. // These values will be generated out of the categories.json function fillDropdownByValue(value, dropdownToFill) { diff --git a/static/js/main.js b/static/js/main.js index 0998134..6359536 100644 --- a/static/js/main.js +++ b/static/js/main.js @@ -14,3 +14,17 @@ function getDescriptorByLocalString(language, localString){ } return localString } + +function getNextUnit(bSize) { + units = ["B", "kB", "MB", "GB", "TB", "PB", "To damn high"] + iters = 0 + size = bSize + do { + size /= 1024 + iters++ + } while(size >= 1) + size *= 1024 + if(iters > units.length) { iters = units.length } + return [size, units[iters - 1]] +} + diff --git a/templates/categories.html b/templates/categories.html index 951f1fa..6cc8ef3 100644 --- a/templates/categories.html +++ b/templates/categories.html @@ -5,20 +5,21 @@ vim: ts=2 noexpandtab {% block title %}{{ super() }} - Categories{% endblock%} {% set active_page = "categories" %} {% block content %} -

-

-

+ + {% for category in categories %} + + + + {% endfor %} {% endblock content%} diff --git a/templates/result.html b/templates/result.html index e542c5d..a70fd06 100644 --- a/templates/result.html +++ b/templates/result.html @@ -6,6 +6,7 @@ vim: ts=2 noexpandtab {% set active_page = "search" %} {% block content %} + @@ -16,11 +17,11 @@ vim: ts=2 noexpandtab {% for result in results %} - - - - - + + + + + {% endfor %}
Name
{{ result[1] }}testN/AN/AN/A{{ result[1] }}{{ result[2]|int / 1000 / 1000 }}N/AN/AN/A