struppi/TorrentIndexer
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add torrent statistics

Browse Source
This commit is contained in:
sqozz
2018-04-01 18:14:22 +02:00
parent 51c9a55968
commit 0e417ee30a
3 changed files with 102 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

110
indexer.py
View File

@@ -1,18 +1,25 @@
#!/usr/bin/python3
#/* vim:set ts=2 set noexpandtab */
import json, uuid, hashlib, sqlite3, base64
from hashlib import sha1
import bencoder
import requests
from flask import Flask, render_template, url_for, request, send_file, redirect
from flask_babel import Babel, gettext as _, lazy_gettext
from werkzeug import secure_filename
from hurry.filesize import size
from hashlib import sha1
import threading
import binascii
import bencoder
import requests
import hashlib
import sqlite3
import base64
import urllib
import json
import uuid
import time
app = Flask(__name__)
babel = Babel(app)
LANGUAGES = ['en', 'de']
settings = None
@@ -38,14 +45,74 @@ class Categories():
return (cat_name, sub_name)
class ScrapeState():
stats = {}
def __init__(self):
pass
def update(self):
self._statedump()
self._tpbs()
self._fullscrape()
def _statedump(self):
url = settings["scrape_url"]
statedump_url = url + "/stats"
params = { "mode" : "statedump" }
req = requests.get(statedump_url, params=params)
dump = req.text.strip()
dump = dump.split("\n")
for entry in dump:
entry = entry.split(":")
key = entry[0].lower()
if not key in self.stats.keys():
self.stats.update({ key : {}})
self.stats.get(key).update({ "base" : entry[1], "unsure_downloaded" : entry[2] })
def _tpbs(self):
url = settings["scrape_url"]
tpbs_url = url + "/stats"
params = { "mode" : "tpbs", "format" : "ben" }
req = requests.get(tpbs_url, params=params)
decoded = bencoder.decode(req.content)
for torrent in decoded[b"files"]:
info_hash = binascii.b2a_hex(torrent)
stats = decoded[b"files"][torrent]
key = info_hash.decode("utf-8").lower()
self.stats.get(key).update({ "seeds" : stats[b"complete"], "peers" : stats[b"incomplete"], "complete" : stats[b"downloaded"] })
def _fullscrape(self):
connection = sqlite3.connect("torrentdb.sqlite")
c = connection.cursor()
c.execute("SELECT fileid FROM torrents")
all_hashes = c.fetchall()
for info_hash in all_hashes:
info_hash = info_hash[0]
url_param = binascii.a2b_hex(info_hash.encode())
url = settings["scrape_url"]
req = requests.get(url + "/scrape", params={"info_hash" : url_param})
decoded = bencoder.decode(req.content)
info = decoded[b"files"]
try:
ugly_hash, stats = info.popitem()
key = info_hash.lower()
self.stats.get(key).update({ "seeds" : stats[b"complete"], "peers" : stats[b"incomplete"], "complete" : stats[b"downloaded"] })
except KeyError:
print("No stats found for {}".format(info_hash))
@app.route("/")
def index():
return render_template("search.html", categories=categories.categories)
@app.route("/categories")
def categorys():
return render_template("categories.html", categories=categories.categories)
@app.route("/create", methods=['GET','POST'])
def create():
if request.method == "GET":
@@ -58,6 +125,7 @@ def create():
else:
return render_template("create.html", categories=categories.categories, errors=newTorrent.errors)
@app.route("/download/<filename>")
def download(filename):
connection = sqlite3.connect("torrentdb.sqlite")
@@ -66,6 +134,7 @@ def download(filename):
name = c.fetchone()[0]
return send_file("torrentFiles/" + filename, as_attachment=True, attachment_filename=name + ".torrent", conditional=True)
@app.route("/search", methods=['GET'])
def search():
connection = sqlite3.connect("torrentdb.sqlite")
@@ -101,7 +170,8 @@ def search():
r = row[0:2] + (size(float(row[2])) , ) + row[3:]
results.append(r)
return render_template("result.html", results=results, categories=categories.categories)
return render_template("result.html", results=results, categories=categories.categories, stats=scrapeState.stats)
@app.route("/details", methods=['GET'])
def details():
@@ -110,11 +180,6 @@ def details():
tf.fromDb()
return render_template("details.html", categories=categories.categories, torrent=tf)
def scrapeAll():
TRACKER_URL = ""
statedump = requests.get(TRACKER_URL + "stats" + "?mode=statedump")
return
def init():
global settings
@@ -123,7 +188,19 @@ def init():
initDb()
global categories
categories = Categories()
#scrapeAll()
global scrapeState
scrapeState = ScrapeState()
scrape = threading.Thread(target=scraper)
scrape.start()
def scraper():
while True:
print("Start scraping")
scrapeState.update()
print("Scraping done")
time.sleep(60)
def initDb():
connection = sqlite3.connect("torrentdb.sqlite")
@@ -133,6 +210,7 @@ def initDb():
connection.commit()
connection.close()
def createNewTorrent(reuqest):
uploadfile = request.files["torrentFile"]
filename = secure_filename(uploadfile.filename)
@@ -201,6 +279,7 @@ def createNewTorrent(reuqest):
return newTFile
class Metadata():
def __init__(self, fileid):
try:
@@ -211,7 +290,7 @@ class Metadata():
self.fileid = fileid
self.bcoded = bencoder.decode(torrent)
self.created_by = self.bcoded.get(b'created by', b"").decode("utf-8", "ignore")
self.creation_date = self.bcoded.get(b'creation date', b"").decode("utf-8", "ignore")
self.creation_date = self.bcoded.get(b'creation date', 0)
self.announce_url = self.bcoded.get(b'info', dict()).get(b'', "")
self.source = self.bcoded.get(b'info', dict()).get(b'source', b"")
self.torrentsize = ((len(self.bcoded.get(b'info', dict()).get(b'pieces', "")) / 20) * self.bcoded.get(b'info', dict()).get(b'piece length'))
@@ -227,6 +306,7 @@ class Metadata():
b64name = base64.b64encode(self.name)
c.execute("INSERT INTO metadata(fileid, created_by, creation_date, announce_url, source, torrentsize, name, private) VALUES(:fileid, :created_by, :creation_date, :announce_url, :source, :torrentsize, :name, :private)", { 'fileid' : self.fileid, 'created_by' : b64created_by, 'creation_date' : self.creation_date, 'announce_url' : b64announce_url, 'source' : b64source , 'torrentsize' : self.torrentsize, 'name' : b64name, 'private' : self.private})
class TorrentFile():
errors = []
fileid = None
@@ -277,10 +357,12 @@ class TorrentFile():
self.videoquality_description = (base64.b64decode(res["videoquality_description"])).decode()
self.metadata = Metadata(self.fileid)
@babel.localeselector
def get_locale():
return request.accept_languages.best_match(LANGUAGES)
if __name__ == "__main__":
init()
app.jinja_env.globals.update(json=json)

4
settings.json
View File

@@ -189,6 +189,8 @@
"valid_tracker" : [
"udp://tracker.lootbox.cf:6969/announce",
"udp://tracker.lootbox.cf:6969/announce/"
]
],
"scrape_url" : "http://tracker.lootbox.gq:6969"
}

6
templates/result.html
View File

@@ -19,9 +19,9 @@ vim: ts=2 noexpandtab
<tr>
<td><a href="/details?h={{ result[0] }}">{{ result[1] }}</a></td>
<td>{{ result[2] }}</td>
<td>{{ _("N/A") }}</td>
<td>{{ _("N/A") }}</td>
<td>{{ _("N/A") }}</td>
<td>{{ stats.get(result[0], {}).get("complete", _("N/A")) }}</td>
<td>{{ stats.get(result[0], {}).get("seeds", _("N/A")) }}</td>
<td>{{ stats.get(result[0], {}).get("peers", _("N/A")) }}</td>
</tr>
{% endfor %}
</table>